diff options
| author | agri <agri@yandex-team.ru> | 2022-02-10 16:48:12 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:12 +0300 |
| commit | d3530b2692e400bd4d29bd4f07cafaee139164e7 (patch) | |
| tree | b7ae636a74490e649a2ed0fdd5361f1bec83b9f9 | |
| parent | 0f4c5d1e8c0672bf0a1f2f2d8acac5ba24772435 (diff) | |
| download | ydb-d3530b2692e400bd4d29bd4f07cafaee139164e7.tar.gz | |
Restoring authorship annotation for <agri@yandex-team.ru>. Commit 1 of 2.
180 files changed, 8465 insertions, 8465 deletions
diff --git a/contrib/libs/cxxsupp/libcxxrt/exception.cc b/contrib/libs/cxxsupp/libcxxrt/exception.cc index 6baf428ead..f61242b1ad 100644 --- a/contrib/libs/cxxsupp/libcxxrt/exception.cc +++ b/contrib/libs/cxxsupp/libcxxrt/exception.cc @@ -837,10 +837,10 @@ static void throw_exception(__cxa_exception *ex) report_failure(err, ex); } -typedef void (*cxa_throw_hook_t)(void*, std::type_info*, void(*)(void*)) noexcept; - -__attribute__((weak)) cxa_throw_hook_t cxa_throw_hook = nullptr; +typedef void (*cxa_throw_hook_t)(void*, std::type_info*, void(*)(void*)) noexcept; +__attribute__((weak)) cxa_throw_hook_t cxa_throw_hook = nullptr; + /** * ABI function for throwing an exception. Takes the object to be thrown (the * pointer returned by __cxa_allocate_exception()), the type info for the @@ -850,11 +850,11 @@ extern "C" void __cxa_throw(void *thrown_exception, std::type_info *tinfo, void(*dest)(void*)) { - if (cxa_throw_hook) - { - cxa_throw_hook(thrown_exception, tinfo, dest); - } - + if (cxa_throw_hook) + { + cxa_throw_hook(thrown_exception, tinfo, dest); + } + __cxa_exception *ex = reinterpret_cast<__cxa_exception*>(thrown_exception) - 1; ex->referenceCount = 1; diff --git a/library/cpp/actors/core/actor_bootstrapped.h b/library/cpp/actors/core/actor_bootstrapped.h index a37887c939..e15bb86ce6 100644 --- a/library/cpp/actors/core/actor_bootstrapped.h +++ b/library/cpp/actors/core/actor_bootstrapped.h @@ -28,8 +28,8 @@ namespace NActors { } else { static_assert(dependent_false<TDerived>::value, "No correct Bootstrap() signature"); } - } - + } + TActorBootstrapped() : TActor<TDerived>(&TDerived::StateBootstrap) {} diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h index 40499d7586..58d360edcc 100644 --- a/library/cpp/actors/core/actorsystem.h +++ b/library/cpp/actors/core/actorsystem.h @@ -129,7 +129,7 @@ namespace NActors { virtual void SetRealTimeMode() const {} }; - + // could be proxy to in-pool schedulers (for NUMA-aware executors) class ISchedulerThread : TNonCopyable { public: @@ -352,7 +352,7 @@ namespace NActors { NLog::TSettings* LoggerSettings() const { return LoggerSettings0.Get(); } - + void GetPoolStats(ui32 poolId, TExecutorPoolStats& poolStats, TVector<TExecutorThreadStats>& statsCopy) const; void DeferPreStop(std::function<void()> fn) { @@ -360,8 +360,8 @@ namespace NActors { } /* This is the base for memory profiling tags. - System sets memory profiling tag for debug version of lfalloc. - The tag is set as "base_tag + actor_activity_type". */ + System sets memory profiling tag for debug version of lfalloc. + The tag is set as "base_tag + actor_activity_type". */ static ui32 MemProfActivityBase; }; } diff --git a/library/cpp/actors/core/event.cpp b/library/cpp/actors/core/event.cpp index 33f8ce2aaf..1c05ffc3fe 100644 --- a/library/cpp/actors/core/event.cpp +++ b/library/cpp/actors/core/event.cpp @@ -1,7 +1,7 @@ #include "event.h" -#include "event_pb.h" - -namespace NActors { +#include "event_pb.h" + +namespace NActors { const TScopeId TScopeId::LocallyGenerated{ Max<ui64>(), Max<ui64>() @@ -22,8 +22,8 @@ namespace NActors { return chainBuf; } return new TEventSerializedData; - } - + } + TIntrusivePtr<TEventSerializedData> IEventHandle::GetChainBuffer() { if (Buffer) return Buffer; @@ -34,5 +34,5 @@ namespace NActors { return Buffer; } return new TEventSerializedData; - } -} + } +} diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h index 6ff02aaf94..081549071d 100644 --- a/library/cpp/actors/core/event.h +++ b/library/cpp/actors/core/event.h @@ -3,7 +3,7 @@ #include "defs.h" #include "actorid.h" #include "callstack.h" -#include "event_load.h" +#include "event_load.h" #include <library/cpp/actors/wilson/wilson_trace.h> @@ -17,13 +17,13 @@ namespace NActors { public: virtual bool SerializeToArcadiaStream(TChunkSerializer*) const = 0; }; - + class IEventBase : TNonCopyable, public ISerializerToStream { public: // actual typing is performed by IEventHandle - + virtual ~IEventBase() { } @@ -87,7 +87,7 @@ namespace NActors { Buffer.Reset(); return x; } - + enum EFlags { FlagTrackDelivery = 1 << 0, FlagForwardOnNondelivery = 1 << 1, @@ -236,7 +236,7 @@ namespace NActors { , RewriteType(Type) { } - + TIntrusivePtr<TEventSerializedData> GetChainBuffer(); TIntrusivePtr<TEventSerializedData> ReleaseChainBuffer(); @@ -248,15 +248,15 @@ namespace NActors { } else { return 0; } - } + } bool HasBuffer() const { return bool(Buffer); - } + } bool HasEvent() const { return bool(Event); - } + } IEventBase* GetBase() { if (!Event) { @@ -326,7 +326,7 @@ namespace NActors { } \ bool IsSerializable() const override { \ return false; \ - } + } #define DEFINE_SIMPLE_NONLOCAL_EVENT(eventType, header) \ TString ToStringHeader() const override { \ @@ -340,5 +340,5 @@ namespace NActors { } \ bool IsSerializable() const override { \ return true; \ - } + } } diff --git a/library/cpp/actors/core/event_load.h b/library/cpp/actors/core/event_load.h index 0dab1dd374..da2adc28ea 100644 --- a/library/cpp/actors/core/event_load.h +++ b/library/cpp/actors/core/event_load.h @@ -1,24 +1,24 @@ -#pragma once +#pragma once #include <util/stream/walk.h> -#include <util/system/types.h> +#include <util/system/types.h> #include <util/generic/string.h> #include <library/cpp/actors/util/rope.h> #include <library/cpp/actors/wilson/wilson_trace.h> - -namespace NActors { + +namespace NActors { class IEventHandle; - + struct TConstIoVec { const void* Data; size_t Size; }; - + struct TIoVec { void* Data; size_t Size; }; - + class TEventSerializedData : public TThrRefBase { @@ -70,7 +70,7 @@ namespace NActors { } return result; } - + TRope EraseBack(size_t count) { Y_VERIFY(count <= Rope.GetSize()); TRope::TIterator iter = Rope.End(); @@ -81,25 +81,25 @@ namespace NActors { void Append(TRope&& from) { Rope.Insert(Rope.End(), std::move(from)); } - + void Append(TString buffer) { if (buffer) { Rope.Insert(Rope.End(), TRope(std::move(buffer))); } } }; -} - +} + class TChainBufWalk : public IWalkInput { TIntrusivePtr<NActors::TEventSerializedData> Buffer; TRope::TConstIterator Iter; - + public: TChainBufWalk(TIntrusivePtr<NActors::TEventSerializedData> buffer) : Buffer(std::move(buffer)) , Iter(Buffer->GetBeginIter()) {} - + private: size_t DoUnboundedNext(const void **ptr) override { const size_t size = Iter.ContiguousSize(); @@ -108,5 +108,5 @@ private: Iter.AdvanceToNextContiguousBlock(); } return size; - } + } }; diff --git a/library/cpp/actors/core/event_local.h b/library/cpp/actors/core/event_local.h index 2845aa94dd..2a4ff9fa55 100644 --- a/library/cpp/actors/core/event_local.h +++ b/library/cpp/actors/core/event_local.h @@ -2,7 +2,7 @@ #include "event.h" #include "scheduler_cookie.h" -#include "event_load.h" +#include "event_load.h" #include <util/system/type_name.h> namespace NActors { diff --git a/library/cpp/actors/core/event_pb.cpp b/library/cpp/actors/core/event_pb.cpp index 018ff9ac34..bae0a0a64b 100644 --- a/library/cpp/actors/core/event_pb.cpp +++ b/library/cpp/actors/core/event_pb.cpp @@ -1,6 +1,6 @@ -#include "event_pb.h" - -namespace NActors { +#include "event_pb.h" + +namespace NActors { bool TRopeStream::Next(const void** data, int* size) { *data = Iter.ContiguousData(); *size = Iter.ContiguousSize(); @@ -13,13 +13,13 @@ namespace NActors { TotalByteCount += *size; return *size != 0; } - + void TRopeStream::BackUp(int count) { Y_VERIFY(count <= TotalByteCount); Iter -= count; TotalByteCount -= count; } - + bool TRopeStream::Skip(int count) { if (static_cast<size_t>(TotalByteCount + count) > Size) { count = Size - TotalByteCount; @@ -27,20 +27,20 @@ namespace NActors { Iter += count; TotalByteCount += count; return static_cast<size_t>(TotalByteCount) != Size; - } - + } + TCoroutineChunkSerializer::TCoroutineChunkSerializer() : TotalSerializedDataSize(0) , Stack(64 * 1024) , SelfClosure{this, TArrayRef(Stack.Begin(), Stack.End())} , InnerContext(SelfClosure) {} - + TCoroutineChunkSerializer::~TCoroutineChunkSerializer() { CancelFlag = true; Resume(); Y_VERIFY(Finished); - } + } bool TCoroutineChunkSerializer::AllowsAliasing() const { return true; @@ -85,10 +85,10 @@ namespace NActors { } else { InnerContext.SwitchTo(BufFeedContext); } - } + } return true; - } - + } + bool TCoroutineChunkSerializer::Next(void** data, int* size) { if (CancelFlag || AbortFlag) { return false; @@ -122,15 +122,15 @@ namespace NActors { BufferPtr -= count; SizeRemain += count; TotalSerializedDataSize -= count; - } - + } + void TCoroutineChunkSerializer::Resume() { TContMachineContext feedContext; BufFeedContext = &feedContext; feedContext.SwitchTo(&InnerContext); BufFeedContext = nullptr; - } - + } + bool TCoroutineChunkSerializer::WriteRope(const TRope *rope) { for (auto iter = rope->Begin(); iter.Valid(); iter.AdvanceToNextContiguousBlock()) { if (!WriteAliasedRaw(iter.ContiguousData(), iter.ContiguousSize())) { @@ -156,14 +156,14 @@ namespace NActors { return {Chunks, Chunks + NumChunks}; } - + void TCoroutineChunkSerializer::SetSerializingEvent(const IEventBase *event) { Y_VERIFY(Event == nullptr); Event = event; TotalSerializedDataSize = 0; AbortFlag = false; } - + void TCoroutineChunkSerializer::Abort() { Y_VERIFY(Event); AbortFlag = true; @@ -181,8 +181,8 @@ namespace NActors { } Finished = true; InnerContext.SwitchTo(BufFeedContext); - } - + } + bool TAllocChunkSerializer::Next(void** pdata, int* psize) { if (Backup) { // we have some data in backup rope -- move the first chunk from the backup rope to the buffer and return @@ -200,12 +200,12 @@ namespace NActors { Buffers->Append(TRope(std::move(item))); } return true; - } - + } + void TAllocChunkSerializer::BackUp(int count) { Backup.Insert(Backup.Begin(), Buffers->EraseBack(count)); } - + bool TAllocChunkSerializer::WriteAliasedRaw(const void*, int) { Y_VERIFY(false); return false; diff --git a/library/cpp/actors/core/event_pb.h b/library/cpp/actors/core/event_pb.h index d7546b901a..1c69d7e9bf 100644 --- a/library/cpp/actors/core/event_pb.h +++ b/library/cpp/actors/core/event_pb.h @@ -1,15 +1,15 @@ #pragma once #include "event.h" -#include "event_load.h" - +#include "event_load.h" + #include <google/protobuf/io/zero_copy_stream.h> #include <google/protobuf/arena.h> #include <library/cpp/actors/protos/actors.pb.h> -#include <util/generic/deque.h> -#include <util/system/context.h> -#include <util/system/filemap.h> -#include <array> +#include <util/generic/deque.h> +#include <util/system/context.h> +#include <util/system/filemap.h> +#include <array> namespace NActors { @@ -29,11 +29,11 @@ namespace NActors { int64_t ByteCount() const override { return TotalByteCount; } - + private: int64_t TotalByteCount = 0; }; - + class TChunkSerializer : public NProtoBuf::io::ZeroCopyOutputStream { public: TChunkSerializer() = default; @@ -42,7 +42,7 @@ namespace NActors { virtual bool WriteRope(const TRope *rope) = 0; virtual bool WriteString(const TString *s) = 0; }; - + class TAllocChunkSerializer final : public TChunkSerializer { public: bool Next(void** data, int* size) override; @@ -51,7 +51,7 @@ namespace NActors { return Buffers->GetSize(); } bool WriteAliasedRaw(const void* data, int size) override; - + // WARNING: these methods require owner to retain ownership and immutability of passed objects bool WriteRope(const TRope *rope) override; bool WriteString(const TString *s) override; @@ -62,19 +62,19 @@ namespace NActors { } return std::move(Buffers); } - + protected: TIntrusivePtr<TEventSerializedData> Buffers = new TEventSerializedData; TRope Backup; }; - + class TCoroutineChunkSerializer final : public TChunkSerializer, protected ITrampoLine { public: using TChunk = std::pair<const char*, size_t>; TCoroutineChunkSerializer(); ~TCoroutineChunkSerializer(); - + void SetSerializingEvent(const IEventBase *event); void Abort(); std::pair<TChunk*, TChunk*> FeedBuf(void* data, size_t size); @@ -87,7 +87,7 @@ namespace NActors { const IEventBase *GetCurrentEvent() const { return Event; } - + bool Next(void** data, int* size) override; void BackUp(int count) override; int64_t ByteCount() const override { @@ -95,7 +95,7 @@ namespace NActors { } bool WriteAliasedRaw(const void* data, int size) override; bool AllowsAliasing() const override; - + bool WriteRope(const TRope *rope) override; bool WriteString(const TString *s) override; @@ -103,7 +103,7 @@ namespace NActors { void DoRun() override; void Resume(); bool Produce(const void *data, size_t size); - + i64 TotalSerializedDataSize; TMappedAllocation Stack; TContClosure SelfClosure; @@ -120,7 +120,7 @@ namespace NActors { bool SerializationSuccess; bool Finished = false; }; - + #ifdef ACTORLIB_HUGE_PB_SIZE static const size_t EventMaxByteSize = 140 << 20; // (140MB) #else @@ -137,9 +137,9 @@ namespace NActors { public: using ProtoRecordType = TRecord; - + TEventPBBase() = default; - + explicit TEventPBBase(const TRecord& rec) { Record = rec; @@ -153,7 +153,7 @@ namespace NActors { TString ToStringHeader() const override { return Record.GetTypeName(); } - + TString ToString() const override { return Record.ShortDebugString(); } @@ -274,7 +274,7 @@ namespace NActors { ev->CachedByteSize = input->GetSize(); return ev.Release(); } - + size_t GetCachedByteSize() const { if (CachedByteSize == 0) { CachedByteSize = CalculateSerializedSize(); diff --git a/library/cpp/actors/core/events.h b/library/cpp/actors/core/events.h index 702cf50fad..88103e888c 100644 --- a/library/cpp/actors/core/events.h +++ b/library/cpp/actors/core/events.h @@ -1,11 +1,11 @@ #pragma once #include "event.h" -#include "event_pb.h" +#include "event_pb.h" #include <library/cpp/actors/protos/actors.pb.h> #include <util/system/unaligned_mem.h> - + namespace NActors { struct TEvents { enum EEventSpace { @@ -213,7 +213,7 @@ namespace NActors { using TEvPoisonPill = TEvPoison; // Legacy name, deprecated using TEvActorDied = TEvGone; - }; + }; } template <> diff --git a/library/cpp/actors/core/executelater.h b/library/cpp/actors/core/executelater.h index e7a13c1005..53da592373 100644 --- a/library/cpp/actors/core/executelater.h +++ b/library/cpp/actors/core/executelater.h @@ -1,10 +1,10 @@ -#pragma once - -#include "actor_bootstrapped.h" - -#include <utility> - -namespace NActors { +#pragma once + +#include "actor_bootstrapped.h" + +#include <utility> + +namespace NActors { template <typename TCallback> class TExecuteLater: public TActorBootstrapped<TExecuteLater<TCallback>> { public: @@ -13,10 +13,10 @@ namespace NActors { } TExecuteLater( - TCallback&& callback, - IActor::EActivityType activityType, - ui32 channel = 0, - ui64 cookie = 0, + TCallback&& callback, + IActor::EActivityType activityType, + ui32 channel = 0, + ui64 cookie = 0, const TActorId& reportCompletionTo = TActorId(), const TActorId& reportExceptionTo = TActorId()) noexcept : Callback(std::move(callback)) @@ -27,16 +27,16 @@ namespace NActors { { this->SetActivityType(activityType); } - + void Bootstrap(const TActorContext& ctx) noexcept { try { { /* RAII, Callback should be destroyed right before sending - TEvCallbackCompletion */ - + TEvCallbackCompletion */ + auto local = std::move(Callback); using T = decltype(local); - + if constexpr (std::is_invocable_v<T, const TActorContext&>) { local(ctx); } else { @@ -56,11 +56,11 @@ namespace NActors { new TEvents::TEvCallbackException(ctx.SelfID, msg), Channel, Cookie); } - } - + } + this->Die(ctx); - } - + } + private: TCallback Callback; const ui32 Channel; @@ -68,13 +68,13 @@ namespace NActors { const TActorId ReportCompletionTo; const TActorId ReportExceptionTo; }; - + template <typename T> IActor* CreateExecuteLaterActor( - T&& func, - IActor::EActivityType activityType, - ui32 channel = 0, - ui64 cookie = 0, + T&& func, + IActor::EActivityType activityType, + ui32 channel = 0, + ui64 cookie = 0, const TActorId& reportCompletionTo = TActorId(), const TActorId& reportExceptionTo = TActorId()) noexcept { return new TExecuteLater<T>(std::forward<T>(func), @@ -84,4 +84,4 @@ namespace NActors { reportCompletionTo, reportExceptionTo); } -} +} diff --git a/library/cpp/actors/core/executor_pool_basic.cpp b/library/cpp/actors/core/executor_pool_basic.cpp index 4dce16939a..3123e9b1a6 100644 --- a/library/cpp/actors/core/executor_pool_basic.cpp +++ b/library/cpp/actors/core/executor_pool_basic.cpp @@ -4,23 +4,23 @@ #include <library/cpp/actors/util/affinity.h> #include <library/cpp/actors/util/datetime.h> -#ifdef _linux_ +#ifdef _linux_ #include <pthread.h> -#endif - +#endif + namespace NActors { LWTRACE_USING(ACTORLIB_PROVIDER); constexpr TDuration TBasicExecutorPool::DEFAULT_TIME_PER_MAILBOX; TBasicExecutorPool::TBasicExecutorPool( - ui32 poolId, - ui32 threads, - ui64 spinThreshold, + ui32 poolId, + ui32 threads, + ui64 spinThreshold, const TString& poolName, TAffinity* affinity, - TDuration timePerMailbox, - ui32 eventsPerMailbox, + TDuration timePerMailbox, + ui32 eventsPerMailbox, int realtimePriority, ui32 maxActivityType) : TExecutorPoolBase(poolId, threads, affinity, maxActivityType) @@ -330,10 +330,10 @@ namespace NActors { if (pthread_setschedparam(threadSelf, SCHED_FIFO, ¶m)) { Y_FAIL("Cannot set realtime priority"); } - } -#else + } +#else Y_UNUSED(RealtimePriority); -#endif +#endif } ui32 TBasicExecutorPool::GetThreadCount() const { diff --git a/library/cpp/actors/core/executor_pool_basic.h b/library/cpp/actors/core/executor_pool_basic.h index 023190f7fe..65ceed2669 100644 --- a/library/cpp/actors/core/executor_pool_basic.h +++ b/library/cpp/actors/core/executor_pool_basic.h @@ -62,7 +62,7 @@ namespace NActors { TAtomic ThreadUtilization; TAtomic MaxUtilizationCounter; TAtomic MaxUtilizationAccumulator; - + TAtomic ThreadCount; TMutex ChangeThreadsLock; @@ -81,7 +81,7 @@ namespace NActors { ui32 maxActivityType = 1); explicit TBasicExecutorPool(const TBasicExecutorPoolConfig& cfg); ~TBasicExecutorPool(); - + ui32 GetReadyActivation(TWorkerContext& wctx, ui64 revolvingReadCounter) override; void Schedule(TInstant deadline, TAutoPtr<IEventHandle> ev, ISchedulerCookie* cookie, TWorkerId workerId) override; diff --git a/library/cpp/actors/core/executor_pool_united.cpp b/library/cpp/actors/core/executor_pool_united.cpp index dac6245635..e5968609e7 100644 --- a/library/cpp/actors/core/executor_pool_united.cpp +++ b/library/cpp/actors/core/executor_pool_united.cpp @@ -14,7 +14,7 @@ #include <util/system/datetime.h> #include <util/system/hp_timer.h> - + #include <algorithm> namespace NActors { @@ -1315,7 +1315,7 @@ namespace NActors { if (Y_UNLIKELY(result == CpuStopped) || TryAcquireToken(result)) { break; // token acquired (or stop) } - } + } wctx.AddElapsedCycles(IActor::ACTOR_SYSTEM, timeTracker.Elapsed()); return result; diff --git a/library/cpp/actors/core/executor_pool_united.h b/library/cpp/actors/core/executor_pool_united.h index a090ba2466..01be95b778 100644 --- a/library/cpp/actors/core/executor_pool_united.h +++ b/library/cpp/actors/core/executor_pool_united.h @@ -63,7 +63,7 @@ namespace NActors { // Sets executor for specified pool void SetupPool(TPoolId pool, IExecutorPool* executorPool, TMailboxTable* mailboxTable); - + // Add activation of newly scheduled mailbox and wake cpu to execute it if required void PushActivation(TPoolId pool, ui32 activation, ui64 revolvingCounter); @@ -72,7 +72,7 @@ namespace NActors { // Try to wake idle cpu waiting for tokens on specified pool void TryWake(TPoolId pool); - + // Get activation from pool; requires pool's token void BeginExecution(TPoolId pool, ui32& activation, ui64 revolvingCounter); diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp index 446b651efd..ac97689f31 100644 --- a/library/cpp/actors/core/executor_thread.cpp +++ b/library/cpp/actors/core/executor_thread.cpp @@ -303,7 +303,7 @@ namespace NActors { ExecutorPool->SetRealTimeMode(); TAffinityGuard affinity(ExecutorPool->Affinity()); - + NHPTimer::STime hpnow = GetCycleCountFast(); NHPTimer::STime hpprev = hpnow; ui64 execCount = 0; diff --git a/library/cpp/actors/core/executor_thread.h b/library/cpp/actors/core/executor_thread.h index 9d3c573f0d..66b97bd351 100644 --- a/library/cpp/actors/core/executor_thread.h +++ b/library/cpp/actors/core/executor_thread.h @@ -45,7 +45,7 @@ namespace NActors { void UnregisterActor(TMailboxHeader* mailbox, ui64 localActorId); void DropUnregistered(); const std::vector<THolder<IActor>>& GetUnregistered() const { return DyingActors; } - + void Schedule(TInstant deadline, TAutoPtr<IEventHandle> ev, ISchedulerCookie* cookie = nullptr); void Schedule(TMonotonic deadline, TAutoPtr<IEventHandle> ev, ISchedulerCookie* cookie = nullptr); void Schedule(TDuration delta, TAutoPtr<IEventHandle> ev, ISchedulerCookie* cookie = nullptr); diff --git a/library/cpp/actors/core/log.cpp b/library/cpp/actors/core/log.cpp index 5f63b5af58..bfac7d30e4 100644 --- a/library/cpp/actors/core/log.cpp +++ b/library/cpp/actors/core/log.cpp @@ -195,7 +195,7 @@ namespace NActors { , Metrics(std::make_unique<TLoggerMetrics>(metrics)) { } - + TLoggerActor::TLoggerActor(TIntrusivePtr<NLog::TSettings> settings, std::shared_ptr<TLogBackend> logBackend, std::shared_ptr<NMonitoring::TMetricRegistry> metrics) @@ -260,8 +260,8 @@ namespace NActors { break; default: break; - } - + } + } void TLoggerActor::HandleLogEvent(NLog::TEvLog::TPtr& ev, const NActors::TActorContext& ctx) { diff --git a/library/cpp/actors/core/log.h b/library/cpp/actors/core/log.h index c11a7cf3c1..514ff51c14 100644 --- a/library/cpp/actors/core/log.h +++ b/library/cpp/actors/core/log.h @@ -42,7 +42,7 @@ actorCtxOrSystem, priority, component, __VA_ARGS__); \ } \ } while (0) /**/ - + #define LOG_LOG_S_SAMPLED_BY(actorCtxOrSystem, priority, component, sampleBy, stream) \ LOG_LOG_SAMPLED_BY(actorCtxOrSystem, priority, component, sampleBy, "%s", [&]() { \ TStringBuilder logStringBuilder; \ @@ -304,7 +304,7 @@ namespace NActors { ///////////////////////////////////////////////////////////////////// // Logging adaptors for memory log and logging into filesystem ///////////////////////////////////////////////////////////////////// - + namespace NDetail { inline void Y_PRINTF_FORMAT(2, 3) PrintfV(TString& dst, const char* format, ...) { va_list params; @@ -318,7 +318,7 @@ namespace NActors { } } // namespace NDetail - template <typename TCtx> + template <typename TCtx> inline void DeliverLogMessage(TCtx& ctx, NLog::EPriority mPriority, NLog::EComponent mComponent, TString &&str) { const NLog::TSettings *mSettings = ctx.LoggerSettings(); @@ -327,14 +327,14 @@ namespace NActors { } template <typename TCtx, typename... TArgs> - inline void MemLogAdapter( + inline void MemLogAdapter( TCtx& actorCtxOrSystem, NLog::EPriority mPriority, NLog::EComponent mComponent, const char* format, TArgs&&... params) { TString Formatted; - - + + if constexpr (sizeof... (params) > 0) { NDetail::PrintfV(Formatted, format, std::forward<TArgs>(params)...); } else { @@ -343,9 +343,9 @@ namespace NActors { MemLogWrite(Formatted.data(), Formatted.size(), true); DeliverLogMessage(actorCtxOrSystem, mPriority, mComponent, std::move(Formatted)); - } - - template <typename TCtx> + } + + template <typename TCtx> Y_WRAPPER inline void MemLogAdapter( TCtx& actorCtxOrSystem, NLog::EPriority mPriority, @@ -355,7 +355,7 @@ namespace NActors { MemLogWrite(str.data(), str.size(), true); DeliverLogMessage(actorCtxOrSystem, mPriority, mComponent, TString(str)); } - + template <typename TCtx> Y_WRAPPER inline void MemLogAdapter( TCtx& actorCtxOrSystem, @@ -365,5 +365,5 @@ namespace NActors { MemLogWrite(str.data(), str.size(), true); DeliverLogMessage(actorCtxOrSystem, mPriority, mComponent, std::move(str)); - } + } } diff --git a/library/cpp/actors/core/mailbox.cpp b/library/cpp/actors/core/mailbox.cpp index d84b4f9e46..ac598eff86 100644 --- a/library/cpp/actors/core/mailbox.cpp +++ b/library/cpp/actors/core/mailbox.cpp @@ -214,49 +214,49 @@ namespace NActors { return true; case TMailboxType::HTSwap: { THTSwapMailbox* const mailbox = THTSwapMailbox::Get(lineHint, x); -#if (!defined(_tsan_enabled_)) +#if (!defined(_tsan_enabled_)) Y_VERIFY_DEBUG(mailbox->Type == (ui32)x->MailboxType); -#endif +#endif mailbox->Queue.Push(ev.Release()); if (mailbox->MarkForSchedule()) { RelaxedStore<NHPTimer::STime>(&mailbox->ScheduleMoment, GetCycleCountFast()); executorPool->ScheduleActivation(hint); } - } + } return true; case TMailboxType::ReadAsFilled: { if (lineHint > TReadAsFilledMailbox::MaxMailboxesInLine()) return false; - + TReadAsFilledMailbox* const mailbox = TReadAsFilledMailbox::Get(lineHint, x); -#if (!defined(_tsan_enabled_)) +#if (!defined(_tsan_enabled_)) Y_VERIFY_DEBUG(mailbox->Type == (ui32)x->MailboxType); -#endif +#endif mailbox->Queue.Push(ev.Release()); if (mailbox->MarkForSchedule()) { RelaxedStore<NHPTimer::STime>(&mailbox->ScheduleMoment, GetCycleCountFast()); executorPool->ScheduleActivation(hint); } - } + } return true; case TMailboxType::TinyReadAsFilled: { if (lineHint > TTinyReadAsFilledMailbox::MaxMailboxesInLine()) return false; - + TTinyReadAsFilledMailbox* const mailbox = TTinyReadAsFilledMailbox::Get(lineHint, x); -#if (!defined(_tsan_enabled_)) +#if (!defined(_tsan_enabled_)) Y_VERIFY_DEBUG(mailbox->Type == (ui32)x->MailboxType); -#endif +#endif mailbox->Queue.Push(ev.Release()); if (mailbox->MarkForSchedule()) { RelaxedStore<NHPTimer::STime>(&mailbox->ScheduleMoment, GetCycleCountFast()); executorPool->ScheduleActivation(hint); } - } + } return true; default: Y_FAIL("unknown mailbox type"); - } + } } return false; diff --git a/library/cpp/actors/core/mailbox.h b/library/cpp/actors/core/mailbox.h index 0bd9c4d314..8a2c0d0608 100644 --- a/library/cpp/actors/core/mailbox.h +++ b/library/cpp/actors/core/mailbox.h @@ -10,7 +10,7 @@ #include <library/cpp/threading/queue/mpsc_read_as_filled.h> #include <util/generic/hash.h> #include <util/system/hp_timer.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> // TODO: clean all broken arcadia atomic stuff and replace with intrinsics namespace NActors { @@ -389,52 +389,52 @@ namespace NActors { constexpr static ui32 AlignedSize() { return ((sizeof(TRevolvingMailbox) + 63) / 64) * 64; } - + std::pair<ui32, ui32> CountRevolvingMailboxEvents(ui64 localActorId, ui32 maxTraverse); bool CleanupEvents(); }; - + static_assert(sizeof(TRevolvingMailbox) == 128, "expect sizeof(TRevolvingMailbox) == 128"); - + struct THTSwapMailbox: public TMailboxHeader { using TQueueType = NThreading::THTSwapQueue<IEventHandle*>; - + TQueueType Queue; NHPTimer::STime ScheduleMoment; char Padding_[16]; - + THTSwapMailbox() : TMailboxHeader(TMailboxType::HTSwap) , ScheduleMoment(0) { } - + ~THTSwapMailbox() { CleanupEvents(); } - + IEventHandle* Pop() { return Queue.Pop(); } - + IEventHandle* Head() { return Queue.Peek(); } - + static THTSwapMailbox* Get(ui32 hint, void* line) { return (THTSwapMailbox*)((ui8*)line + 64 + (hint - 1) * 64); } - + constexpr static ui64 MaxMailboxesInLine() { return (LineSize - 64) / AlignedSize(); } - + static const TMailboxType::EType MailboxType = TMailboxType::HTSwap; - + constexpr static ui32 AlignedSize() { return ((sizeof(THTSwapMailbox) + 63) / 64) * 64; } - + bool CleanupEvents() { const bool done = (Queue.Peek() == nullptr); while (IEventHandle* ev = Queue.Pop()) @@ -442,50 +442,50 @@ namespace NActors { return done; } }; - + static_assert(sizeof(THTSwapMailbox) == 64, "expect sizeof(THTSwapMailbox) == 64"); - + struct TReadAsFilledMailbox: public TMailboxHeader { using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>; - + TQueueType Queue; NHPTimer::STime ScheduleMoment; char Padding_[8]; - + TReadAsFilledMailbox() : TMailboxHeader(TMailboxType::ReadAsFilled) , ScheduleMoment(0) { } - + ~TReadAsFilledMailbox() { CleanupEvents(); } - + IEventHandle* Pop() { return Queue.Pop(); } - + IEventHandle* Head() { return Queue.Peek(); } - + static TReadAsFilledMailbox* Get(ui32 hint, void* line) { return (TReadAsFilledMailbox*)((ui8*)line + 64 + (hint - 1) * 192); } - + constexpr static ui64 MaxMailboxesInLine() { return (LineSize - 64) / AlignedSize(); } - + static const TMailboxType::EType MailboxType = TMailboxType::ReadAsFilled; - + constexpr static ui32 AlignedSize() { return ((sizeof(TReadAsFilledMailbox) + 63) / 64) * 64; } - + bool CleanupEvents() { const bool done = (Queue.Peek() == nullptr); while (IEventHandle* ev = Queue.Pop()) @@ -493,52 +493,52 @@ namespace NActors { return done; } }; - + static_assert(sizeof(TReadAsFilledMailbox) == 192, "expect sizeof(TReadAsFilledMailbox) == 192"); - + struct TTinyReadAsFilledMailbox: public TMailboxHeader { using TQueueType = NThreading::TReadAsFilledQueue< IEventHandle, NThreading::TRaFQueueBunchSize<4>>; - + TQueueType Queue; NHPTimer::STime ScheduleMoment; char Padding_[8]; - + TTinyReadAsFilledMailbox() : TMailboxHeader(TMailboxType::TinyReadAsFilled) , ScheduleMoment(0) { } - + ~TTinyReadAsFilledMailbox() { CleanupEvents(); } - + IEventHandle* Pop() { return Queue.Pop(); } - + IEventHandle* Head() { return Queue.Peek(); } - + static TTinyReadAsFilledMailbox* Get(ui32 hint, void* line) { return (TTinyReadAsFilledMailbox*)((ui8*)line + 64 + (hint - 1) * 192); } - + constexpr static ui64 MaxMailboxesInLine() { return (LineSize - 64) / AlignedSize(); } - + static const TMailboxType::EType MailboxType = TMailboxType::TinyReadAsFilled; - + constexpr static ui32 AlignedSize() { return ((sizeof(TTinyReadAsFilledMailbox) + 63) / 64) * 64; } - + bool CleanupEvents() { const bool done = (Queue.Peek() == nullptr); while (IEventHandle* ev = Queue.Pop()) @@ -546,8 +546,8 @@ namespace NActors { return done; } }; - + static_assert(sizeof(TTinyReadAsFilledMailbox) == 192, "expect sizeof(TTinyReadAsFilledMailbox) == 192"); - }; + }; } diff --git a/library/cpp/actors/core/mon.h b/library/cpp/actors/core/mon.h index c450f2338e..3ebf6a0bed 100644 --- a/library/cpp/actors/core/mon.h +++ b/library/cpp/actors/core/mon.h @@ -123,7 +123,7 @@ namespace NActors { return true; } - static IEventBase* Load(TEventSerializedData* bufs) { + static IEventBase* Load(TEventSerializedData* bufs) { return new TEvRemoteHttpInfo(bufs->GetString()); } @@ -160,7 +160,7 @@ namespace NActors { return true; } - static IEventBase* Load(TEventSerializedData* bufs) { + static IEventBase* Load(TEventSerializedData* bufs) { return new TEvRemoteHttpInfoRes(bufs->GetString()); } }; @@ -192,7 +192,7 @@ namespace NActors { return true; } - static IEventBase* Load(TEventSerializedData* bufs) { + static IEventBase* Load(TEventSerializedData* bufs) { return new TEvRemoteJsonInfoRes(bufs->GetString()); } }; diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index d55552af0c..f1d66664b6 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -13,17 +13,17 @@ namespace NActors { inline void Add(ui64 val, ui64 inc = 1) { size_t ind = 0; -#if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 +#if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 asm volatile("" :: : "memory"); -#endif +#endif if (val > 1) { ind = GetValueBitCount(val - 1); } -#if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 +#if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 asm volatile("" :: : "memory"); -#endif +#endif RelaxedStore(&TotalSamples, RelaxedLoad(&TotalSamples) + inc); RelaxedStore(&Buckets[ind], RelaxedLoad(&Buckets[ind]) + inc); } diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make index 880a9d00db..22155dbeec 100644 --- a/library/cpp/actors/core/ya.make +++ b/library/cpp/actors/core/ya.make @@ -32,8 +32,8 @@ SRCS( ask.h balancer.h balancer.cpp - buffer.cpp - buffer.h + buffer.cpp + buffer.h callstack.cpp callstack.h config.h @@ -45,7 +45,7 @@ SRCS( event.h event_load.h event_local.h - event_pb.cpp + event_pb.cpp event_pb.h events.h events_undelivered.cpp diff --git a/library/cpp/actors/dnscachelib/dnscache.cpp b/library/cpp/actors/dnscachelib/dnscache.cpp index 649339ddb2..580956c92e 100644 --- a/library/cpp/actors/dnscachelib/dnscache.cpp +++ b/library/cpp/actors/dnscachelib/dnscache.cpp @@ -155,19 +155,19 @@ void TDnsCache::GetStats(ui64& a_cache_hits, ui64& a_cache_misses, } bool TDnsCache::THost::IsStale(int family, const TDnsCache* ctx) const noexcept { - time_t resolved = family == AF_INET ? ResolvedV4 : ResolvedV6; - time_t notfound = family == AF_INET ? NotFoundV4 : NotFoundV6; - - if (TTimeKeeper::GetTime() - resolved < ctx->EntryLifetime) - return false; - - if (TTimeKeeper::GetTime() - notfound < ctx->NegativeLifetime) - return false; - - return true; -} - -const TDnsCache::THost& + time_t resolved = family == AF_INET ? ResolvedV4 : ResolvedV6; + time_t notfound = family == AF_INET ? NotFoundV4 : NotFoundV6; + + if (TTimeKeeper::GetTime() - resolved < ctx->EntryLifetime) + return false; + + if (TTimeKeeper::GetTime() - notfound < ctx->NegativeLifetime) + return false; + + return true; +} + +const TDnsCache::THost& TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { if (!ValidateHName(hostname)) { LWPROBE(ResolveNullHost, hostname, family); @@ -182,7 +182,7 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { TGuard<TMutex> lock(CacheMtx); p = HostCache.find(hostname); if (p != HostCache.end()) { - if (!p->second.IsStale(family, this)) { + if (!p->second.IsStale(family, this)) { /* Recently resolved, just return cached value */ ACacheHits += 1; THost& host = p->second; @@ -199,9 +199,9 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { ACacheMisses += 1; } - if (cacheOnly) - return NullHost; - + if (cacheOnly) + return NullHost; + TAtomic& inprogress = (family == AF_INET ? p->second.InProgressV4 : p->second.InProgressV6); { @@ -219,7 +219,7 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { ctx->Hostname = hostname; ctx->Family = family; - AtomicSet(inprogress, 1); + AtomicSet(inprogress, 1); ares_gethostbyname(chan, hostname.c_str(), family, &TDnsCache::GHBNCallback, ctx); } @@ -269,7 +269,7 @@ const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) ctx->Owner = this; ctx->Addr = addr; - AtomicSet(p->second.InProgress, 1); + AtomicSet(p->second.InProgress, 1); ares_gethostbyaddr(chan, &addr, family == AF_INET ? sizeof(in_addr) : sizeof(in6_addr), family, &TDnsCache::GHBACallback, ctx); @@ -284,7 +284,7 @@ const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) void TDnsCache::WaitTask(TAtomic& flag) { const TInstant start = TInstant(TTimeKeeper::GetTimeval()); - while (AtomicGet(flag)) { + while (AtomicGet(flag)) { ares_channel chan = static_cast<ares_channel>(Channel); struct pollfd pfd[ARES_GETSOCK_MAXNUM]; @@ -380,7 +380,7 @@ void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { */ p->second.ResolvedV4 = TTimeKeeper::GetTime(); p->second.ResolvedV4 = 0; - AtomicSet(p->second.InProgressV4, 0); + AtomicSet(p->second.InProgressV4, 0); } else if (info->h_addrtype == AF_INET6) { p->second.AddrsV6.clear(); for (int i = 0; info->h_addr_list[i] != nullptr; i++) { @@ -395,7 +395,7 @@ void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { notfound = TTimeKeeper::GetTime(); resolved = 0; } - AtomicSet(inprogress, 0); + AtomicSet(inprogress, 0); } void TDnsCache::GHBACallback(void* arg, int status, int, struct hostent* info) { @@ -413,7 +413,7 @@ void TDnsCache::GHBACallback(void* arg, int status, int, struct hostent* info) { p->second.NotFound = TTimeKeeper::GetTime(); p->second.Resolved = 0; } - AtomicSet(p->second.InProgress, 0); + AtomicSet(p->second.InProgress, 0); } TString TDnsCache::THost::AddrsV4ToString() const { @@ -441,5 +441,5 @@ TString TDnsCache::THost::AddrsV6ToString() const { } return ss.Str(); } - -TDnsCache::TAresLibInit TDnsCache::InitAresLib; + +TDnsCache::TAresLibInit TDnsCache::InitAresLib; diff --git a/library/cpp/actors/dnscachelib/dnscache.h b/library/cpp/actors/dnscachelib/dnscache.h index 3313a251a1..586957b9a0 100644 --- a/library/cpp/actors/dnscachelib/dnscache.h +++ b/library/cpp/actors/dnscachelib/dnscache.h @@ -1,6 +1,6 @@ #pragma once -#include <contrib/libs/c-ares/ares.h> +#include <contrib/libs/c-ares/ares.h> #include <util/generic/map.h> #include <util/generic/vector.h> #include <util/network/address.h> @@ -28,9 +28,9 @@ public: /* use with AF_INET, AF_INET6 or AF_UNSPEC */ NAddr::IRemoteAddrPtr GetAddr(const TString& host, - int family, - TIpPort port = 0, - bool cacheOnly = false); + int family, + TIpPort port = 0, + bool cacheOnly = false); void GetAllAddresses(const TString& host, TVector<NAddr::IRemoteAddrPtr>&); @@ -68,8 +68,8 @@ private: TString AddrsV4ToString() const; TString AddrsV6ToString() const; - - bool IsStale(int family, const TDnsCache* ctx) const noexcept; + + bool IsStale(int family, const TDnsCache* ctx) const noexcept; }; typedef TMap<TString, THost> THostCache; @@ -99,9 +99,9 @@ private: typedef TMap<in6_addr, TAddr, TAddrCmp> TAddrCache; const THost& Resolve(const TString&, int family, bool cacheOnly = false); - + const TAddr& ResolveAddr(const in6_addr&, int family); - + void WaitTask(TAtomic&); static void GHBNCallback(void* arg, int status, int timeouts, @@ -128,21 +128,21 @@ private: TMutex AresMtx; void* Channel; - - struct TAresLibInit { - TAresLibInit() { + + struct TAresLibInit { + TAresLibInit() { #ifdef _win_ - const auto res = ares_library_init(ARES_LIB_INIT_ALL); - Y_VERIFY(res == 0); + const auto res = ares_library_init(ARES_LIB_INIT_ALL); + Y_VERIFY(res == 0); #endif - } - - ~TAresLibInit() { + } + + ~TAresLibInit() { #ifdef _win_ - ares_library_cleanup(); + ares_library_cleanup(); #endif - } - }; - - static TAresLibInit InitAresLib; + } + }; + + static TAresLibInit InitAresLib; }; diff --git a/library/cpp/actors/memory_log/memlog.cpp b/library/cpp/actors/memory_log/memlog.cpp index 8e6b46727d..f20162db70 100644 --- a/library/cpp/actors/memory_log/memlog.cpp +++ b/library/cpp/actors/memory_log/memlog.cpp @@ -1,28 +1,28 @@ -#include "memlog.h" - +#include "memlog.h" + #include <library/cpp/actors/util/datetime.h> -#include <util/system/info.h> -#include <util/system/atomic.h> -#include <util/system/align.h> - -#include <contrib/libs/linuxvdso/interface.h> - -#if (defined(_i386_) || defined(_x86_64_)) && defined(_linux_) -#define HAVE_VDSO_GETCPU 1 -#include <contrib/libs/linuxvdso/interface.h> -static int (*FastGetCpu)(unsigned* cpu, unsigned* node, void* unused); -#endif - -#if defined(_unix_) +#include <util/system/info.h> +#include <util/system/atomic.h> +#include <util/system/align.h> + +#include <contrib/libs/linuxvdso/interface.h> + +#if (defined(_i386_) || defined(_x86_64_)) && defined(_linux_) +#define HAVE_VDSO_GETCPU 1 +#include <contrib/libs/linuxvdso/interface.h> +static int (*FastGetCpu)(unsigned* cpu, unsigned* node, void* unused); +#endif + +#if defined(_unix_) #include <sched.h> -#elif defined(_win_) +#elif defined(_win_) #include <WinBase.h> -#else +#else #error NO IMPLEMENTATION FOR THE PLATFORM -#endif - -const char TMemoryLog::DEFAULT_LAST_MARK[16] = { +#endif + +const char TMemoryLog::DEFAULT_LAST_MARK[16] = { 'c', 'b', '7', @@ -39,9 +39,9 @@ const char TMemoryLog::DEFAULT_LAST_MARK[16] = { '4', '5', '\n', -}; - -const char TMemoryLog::CLEAR_MARK[16] = { +}; + +const char TMemoryLog::CLEAR_MARK[16] = { ' ', ' ', ' ', @@ -58,146 +58,146 @@ const char TMemoryLog::CLEAR_MARK[16] = { ' ', ' ', '\n', -}; - -unsigned TMemoryLog::GetSelfCpu() noexcept { -#if defined(_unix_) +}; + +unsigned TMemoryLog::GetSelfCpu() noexcept { +#if defined(_unix_) #if HAVE_VDSO_GETCPU - unsigned cpu; - if (Y_LIKELY(FastGetCpu != nullptr)) { - auto result = FastGetCpu(&cpu, nullptr, nullptr); - Y_VERIFY(result == 0); + unsigned cpu; + if (Y_LIKELY(FastGetCpu != nullptr)) { + auto result = FastGetCpu(&cpu, nullptr, nullptr); + Y_VERIFY(result == 0); return cpu; - } else { - return 0; - } - + } else { + return 0; + } + #elif defined(_x86_64_) || defined(_i386_) - + #define CPUID(func, eax, ebx, ecx, edx) \ __asm__ __volatile__( \ "cpuid" \ : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) \ : "a"(func)); - - int a = 0, b = 0, c = 0, d = 0; - CPUID(0x1, a, b, c, d); - int acpiID = (b >> 24); - return acpiID; - + + int a = 0, b = 0, c = 0, d = 0; + CPUID(0x1, a, b, c, d); + int acpiID = (b >> 24); + return acpiID; + #elif defined(__CNUC__) - return sched_getcpu(); + return sched_getcpu(); #else - return 0; + return 0; #endif - -#elif defined(_win_) - return GetCurrentProcessorNumber(); -#else - return 0; -#endif -} - -TMemoryLog* TMemoryLog::MemLogBuffer = nullptr; + +#elif defined(_win_) + return GetCurrentProcessorNumber(); +#else + return 0; +#endif +} + +TMemoryLog* TMemoryLog::MemLogBuffer = nullptr; Y_POD_THREAD(TThread::TId) TMemoryLog::LogThreadId; -char* TMemoryLog::LastMarkIsHere = nullptr; - -std::atomic<bool> TMemoryLog::PrintLastMark(true); - +char* TMemoryLog::LastMarkIsHere = nullptr; + +std::atomic<bool> TMemoryLog::PrintLastMark(true); + TMemoryLog::TMemoryLog(size_t totalSize, size_t grainSize) : GrainSize(grainSize) , FreeGrains(DEFAULT_TOTAL_SIZE / DEFAULT_GRAIN_SIZE * 2) , Buf(totalSize) -{ - Y_VERIFY(DEFAULT_TOTAL_SIZE % DEFAULT_GRAIN_SIZE == 0); - NumberOfGrains = DEFAULT_TOTAL_SIZE / DEFAULT_GRAIN_SIZE; - - for (size_t i = 0; i < NumberOfGrains; ++i) { - new (GetGrain(i)) TGrain; - } - - NumberOfCpus = NSystemInfo::NumberOfCpus(); - Y_VERIFY(NumberOfGrains > NumberOfCpus); - ActiveGrains.Reset(new TGrain*[NumberOfCpus]); - for (size_t i = 0; i < NumberOfCpus; ++i) { - ActiveGrains[i] = GetGrain(i); - } - - for (size_t i = NumberOfCpus; i < NumberOfGrains; ++i) { - FreeGrains.StubbornPush(GetGrain(i)); - } - -#if HAVE_VDSO_GETCPU - auto vdsoFunc = (decltype(FastGetCpu)) - NVdso::Function("__vdso_getcpu", "LINUX_2.6"); - AtomicSet(FastGetCpu, vdsoFunc); -#endif -} - -void* TMemoryLog::GetWriteBuffer(size_t amount) noexcept { - // alignment required by NoCacheMemcpy - amount = AlignUp<size_t>(amount, MemcpyAlignment); - - for (ui16 tries = MAX_GET_BUFFER_TRIES; tries-- > 0;) { - auto myCpu = GetSelfCpu(); - - TGrain* grain = AtomicGet(ActiveGrains[myCpu]); - - if (grain != nullptr) { - auto mine = AtomicGetAndAdd(grain->WritePointer, amount); - if (mine + amount <= GrainSize - sizeof(TGrain)) { - return &grain->Data[mine]; - } - - if (!AtomicCas(&ActiveGrains[myCpu], 0, grain)) { - continue; - } - - FreeGrains.StubbornPush(grain); - } - - grain = (TGrain*)FreeGrains.Pop(); - - if (grain == nullptr) { - return nullptr; - } - - grain->WritePointer = 0; - - if (!AtomicCas(&ActiveGrains[myCpu], grain, 0)) { - FreeGrains.StubbornPush(grain); - continue; - } - } - - return nullptr; -} - -void ClearAlignedTail(char* tail) noexcept { - auto aligned = AlignUp(tail, TMemoryLog::MemcpyAlignment); - if (aligned > tail) { - memset(tail, 0, aligned - tail); - } -} - -#if defined(_x86_64_) || defined(_i386_) -#include <xmmintrin.h> -// the main motivation is not poluting CPU cache -NO_SANITIZE_THREAD -void NoCacheMemcpy(char* dst, const char* src, size_t size) noexcept { - while (size >= sizeof(__m128) * 2) { - __m128 a = _mm_load_ps((float*)(src + 0 * sizeof(__m128))); - __m128 b = _mm_load_ps((float*)(src + 1 * sizeof(__m128))); - _mm_stream_ps((float*)(dst + 0 * sizeof(__m128)), a); - _mm_stream_ps((float*)(dst + 1 * sizeof(__m128)), b); - - size -= sizeof(__m128) * 2; - src += sizeof(__m128) * 2; - dst += sizeof(__m128) * 2; - } - memcpy(dst, src, size); -} +{ + Y_VERIFY(DEFAULT_TOTAL_SIZE % DEFAULT_GRAIN_SIZE == 0); + NumberOfGrains = DEFAULT_TOTAL_SIZE / DEFAULT_GRAIN_SIZE; + + for (size_t i = 0; i < NumberOfGrains; ++i) { + new (GetGrain(i)) TGrain; + } + + NumberOfCpus = NSystemInfo::NumberOfCpus(); + Y_VERIFY(NumberOfGrains > NumberOfCpus); + ActiveGrains.Reset(new TGrain*[NumberOfCpus]); + for (size_t i = 0; i < NumberOfCpus; ++i) { + ActiveGrains[i] = GetGrain(i); + } + + for (size_t i = NumberOfCpus; i < NumberOfGrains; ++i) { + FreeGrains.StubbornPush(GetGrain(i)); + } + +#if HAVE_VDSO_GETCPU + auto vdsoFunc = (decltype(FastGetCpu)) + NVdso::Function("__vdso_getcpu", "LINUX_2.6"); + AtomicSet(FastGetCpu, vdsoFunc); +#endif +} + +void* TMemoryLog::GetWriteBuffer(size_t amount) noexcept { + // alignment required by NoCacheMemcpy + amount = AlignUp<size_t>(amount, MemcpyAlignment); + + for (ui16 tries = MAX_GET_BUFFER_TRIES; tries-- > 0;) { + auto myCpu = GetSelfCpu(); + + TGrain* grain = AtomicGet(ActiveGrains[myCpu]); + + if (grain != nullptr) { + auto mine = AtomicGetAndAdd(grain->WritePointer, amount); + if (mine + amount <= GrainSize - sizeof(TGrain)) { + return &grain->Data[mine]; + } + + if (!AtomicCas(&ActiveGrains[myCpu], 0, grain)) { + continue; + } + + FreeGrains.StubbornPush(grain); + } + + grain = (TGrain*)FreeGrains.Pop(); + + if (grain == nullptr) { + return nullptr; + } + + grain->WritePointer = 0; + + if (!AtomicCas(&ActiveGrains[myCpu], grain, 0)) { + FreeGrains.StubbornPush(grain); + continue; + } + } + + return nullptr; +} + +void ClearAlignedTail(char* tail) noexcept { + auto aligned = AlignUp(tail, TMemoryLog::MemcpyAlignment); + if (aligned > tail) { + memset(tail, 0, aligned - tail); + } +} + +#if defined(_x86_64_) || defined(_i386_) +#include <xmmintrin.h> +// the main motivation is not poluting CPU cache +NO_SANITIZE_THREAD +void NoCacheMemcpy(char* dst, const char* src, size_t size) noexcept { + while (size >= sizeof(__m128) * 2) { + __m128 a = _mm_load_ps((float*)(src + 0 * sizeof(__m128))); + __m128 b = _mm_load_ps((float*)(src + 1 * sizeof(__m128))); + _mm_stream_ps((float*)(dst + 0 * sizeof(__m128)), a); + _mm_stream_ps((float*)(dst + 1 * sizeof(__m128)), b); + + size -= sizeof(__m128) * 2; + src += sizeof(__m128) * 2; + dst += sizeof(__m128) * 2; + } + memcpy(dst, src, size); +} NO_SANITIZE_THREAD void NoWCacheMemcpy(char* dst, const char* src, size_t size) noexcept { @@ -224,144 +224,144 @@ void NoWCacheMemcpy(char* dst, const char* src, size_t size) noexcept { } } -#endif - -NO_SANITIZE_THREAD -char* BareMemLogWrite(const char* begin, size_t msgSize, bool isLast) noexcept { - bool lastMark = - isLast && TMemoryLog::PrintLastMark.load(std::memory_order_acquire); - size_t amount = lastMark ? msgSize + TMemoryLog::LAST_MARK_SIZE : msgSize; - - char* buffer = (char*)TMemoryLog::GetWriteBufferStatic(amount); - if (buffer == nullptr) { - return nullptr; - } - -#if defined(_x86_64_) || defined(_i386_) - if (AlignDown(begin, TMemoryLog::MemcpyAlignment) == begin) { - NoCacheMemcpy(buffer, begin, msgSize); +#endif + +NO_SANITIZE_THREAD +char* BareMemLogWrite(const char* begin, size_t msgSize, bool isLast) noexcept { + bool lastMark = + isLast && TMemoryLog::PrintLastMark.load(std::memory_order_acquire); + size_t amount = lastMark ? msgSize + TMemoryLog::LAST_MARK_SIZE : msgSize; + + char* buffer = (char*)TMemoryLog::GetWriteBufferStatic(amount); + if (buffer == nullptr) { + return nullptr; + } + +#if defined(_x86_64_) || defined(_i386_) + if (AlignDown(begin, TMemoryLog::MemcpyAlignment) == begin) { + NoCacheMemcpy(buffer, begin, msgSize); } else { NoWCacheMemcpy(buffer, begin, msgSize); } #else memcpy(buffer, begin, msgSize); #endif - - if (lastMark) { - TMemoryLog::ChangeLastMark(buffer + msgSize); - } - - ClearAlignedTail(buffer + amount); - return buffer; -} - -NO_SANITIZE_THREAD -bool MemLogWrite(const char* begin, size_t msgSize, bool addLF) noexcept { - bool lastMark = TMemoryLog::PrintLastMark.load(std::memory_order_acquire); - size_t amount = lastMark ? msgSize + TMemoryLog::LAST_MARK_SIZE : msgSize; - - // Let's construct prolog with timestamp and thread id - auto threadId = TMemoryLog::GetTheadId(); - - // alignment required by NoCacheMemcpy - // check for format for snprintf - constexpr size_t prologSize = 48; + + if (lastMark) { + TMemoryLog::ChangeLastMark(buffer + msgSize); + } + + ClearAlignedTail(buffer + amount); + return buffer; +} + +NO_SANITIZE_THREAD +bool MemLogWrite(const char* begin, size_t msgSize, bool addLF) noexcept { + bool lastMark = TMemoryLog::PrintLastMark.load(std::memory_order_acquire); + size_t amount = lastMark ? msgSize + TMemoryLog::LAST_MARK_SIZE : msgSize; + + // Let's construct prolog with timestamp and thread id + auto threadId = TMemoryLog::GetTheadId(); + + // alignment required by NoCacheMemcpy + // check for format for snprintf + constexpr size_t prologSize = 48; alignas(TMemoryLog::MemcpyAlignment) char prolog[prologSize + 1]; Y_VERIFY(AlignDown(&prolog, TMemoryLog::MemcpyAlignment) == &prolog); - - int snprintfResult = snprintf(prolog, prologSize + 1, + + int snprintfResult = snprintf(prolog, prologSize + 1, "TS %020" PRIu64 " TI %020" PRIu64 " ", GetCycleCountFast(), threadId); - - if (snprintfResult < 0) { - return false; - } - Y_VERIFY(snprintfResult == prologSize); - - amount += prologSize; - if (addLF) { - ++amount; // add 1 byte for \n at the end of the message - } - - char* buffer = (char*)TMemoryLog::GetWriteBufferStatic(amount); - if (buffer == nullptr) { - return false; - } - -#if defined(_x86_64_) || defined(_i386_) + + if (snprintfResult < 0) { + return false; + } + Y_VERIFY(snprintfResult == prologSize); + + amount += prologSize; + if (addLF) { + ++amount; // add 1 byte for \n at the end of the message + } + + char* buffer = (char*)TMemoryLog::GetWriteBufferStatic(amount); + if (buffer == nullptr) { + return false; + } + +#if defined(_x86_64_) || defined(_i386_) // warning: copy prolog first to avoid corruption of the message // by prolog tail NoCacheMemcpy(buffer, prolog, prologSize); if (AlignDown(begin + prologSize, TMemoryLog::MemcpyAlignment) == begin + prologSize) { NoCacheMemcpy(buffer + prologSize, begin, msgSize); - } else { + } else { NoWCacheMemcpy(buffer + prologSize, begin, msgSize); } #else memcpy(buffer, prolog, prologSize); memcpy(buffer + prologSize, begin, msgSize); #endif - - if (addLF) { - buffer[prologSize + msgSize] = '\n'; - } - - if (lastMark) { - TMemoryLog::ChangeLastMark(buffer + prologSize + msgSize + (int)addLF); - } - - ClearAlignedTail(buffer + amount); - return true; -} - -NO_SANITIZE_THREAD -void TMemoryLog::ChangeLastMark(char* buffer) noexcept { - memcpy(buffer, DEFAULT_LAST_MARK, LAST_MARK_SIZE); - auto oldMark = AtomicSwap(&LastMarkIsHere, buffer); - if (Y_LIKELY(oldMark != nullptr)) { - memcpy(oldMark, CLEAR_MARK, LAST_MARK_SIZE); - } - if (AtomicGet(LastMarkIsHere) != buffer) { - memcpy(buffer, CLEAR_MARK, LAST_MARK_SIZE); - AtomicBarrier(); - } -} - -bool MemLogVPrintF(const char* format, va_list params) noexcept { - auto logger = TMemoryLog::GetMemoryLogger(); - if (logger == nullptr) { - return false; - } - - auto threadId = TMemoryLog::GetTheadId(); - - // alignment required by NoCacheMemcpy + + if (addLF) { + buffer[prologSize + msgSize] = '\n'; + } + + if (lastMark) { + TMemoryLog::ChangeLastMark(buffer + prologSize + msgSize + (int)addLF); + } + + ClearAlignedTail(buffer + amount); + return true; +} + +NO_SANITIZE_THREAD +void TMemoryLog::ChangeLastMark(char* buffer) noexcept { + memcpy(buffer, DEFAULT_LAST_MARK, LAST_MARK_SIZE); + auto oldMark = AtomicSwap(&LastMarkIsHere, buffer); + if (Y_LIKELY(oldMark != nullptr)) { + memcpy(oldMark, CLEAR_MARK, LAST_MARK_SIZE); + } + if (AtomicGet(LastMarkIsHere) != buffer) { + memcpy(buffer, CLEAR_MARK, LAST_MARK_SIZE); + AtomicBarrier(); + } +} + +bool MemLogVPrintF(const char* format, va_list params) noexcept { + auto logger = TMemoryLog::GetMemoryLogger(); + if (logger == nullptr) { + return false; + } + + auto threadId = TMemoryLog::GetTheadId(); + + // alignment required by NoCacheMemcpy alignas(TMemoryLog::MemcpyAlignment) char buf[TMemoryLog::MAX_MESSAGE_SIZE]; Y_VERIFY(AlignDown(&buf, TMemoryLog::MemcpyAlignment) == &buf); - + int prologSize = snprintf(buf, TMemoryLog::MAX_MESSAGE_SIZE - 2, "TS %020" PRIu64 " TI %020" PRIu64 " ", GetCycleCountFast(), threadId); - - if (Y_UNLIKELY(prologSize < 0)) { - return false; - } - Y_VERIFY((ui32)prologSize <= TMemoryLog::MAX_MESSAGE_SIZE); - - int add = vsnprintf( + + if (Y_UNLIKELY(prologSize < 0)) { + return false; + } + Y_VERIFY((ui32)prologSize <= TMemoryLog::MAX_MESSAGE_SIZE); + + int add = vsnprintf( &buf[prologSize], - TMemoryLog::MAX_MESSAGE_SIZE - prologSize - 2, - format, params); - - if (Y_UNLIKELY(add < 0)) { - return false; - } - Y_VERIFY(add >= 0); - auto totalSize = prologSize + add; - + TMemoryLog::MAX_MESSAGE_SIZE - prologSize - 2, + format, params); + + if (Y_UNLIKELY(add < 0)) { + return false; + } + Y_VERIFY(add >= 0); + auto totalSize = prologSize + add; + buf[totalSize++] = '\n'; - Y_VERIFY((ui32)totalSize <= TMemoryLog::MAX_MESSAGE_SIZE); - + Y_VERIFY((ui32)totalSize <= TMemoryLog::MAX_MESSAGE_SIZE); + return BareMemLogWrite(buf, totalSize) != nullptr; -} +} diff --git a/library/cpp/actors/memory_log/memlog.h b/library/cpp/actors/memory_log/memlog.h index 2aa27272a6..fe66efc4fb 100644 --- a/library/cpp/actors/memory_log/memlog.h +++ b/library/cpp/actors/memory_log/memlog.h @@ -1,211 +1,211 @@ -#pragma once - +#pragma once + #include <library/cpp/threading/queue/mpmc_unordered_ring.h> #include <util/generic/string.h> -#include <util/string/printf.h> -#include <util/system/datetime.h> -#include <util/system/thread.h> -#include <util/system/types.h> -#include <util/system/atomic.h> -#include <util/system/align.h> -#include <util/system/tls.h> - -#include <atomic> -#include <cstdio> - -#ifdef _win_ -#include <util/system/winint.h> -#endif - -#ifndef NO_SANITIZE_THREAD +#include <util/string/printf.h> +#include <util/system/datetime.h> +#include <util/system/thread.h> +#include <util/system/types.h> +#include <util/system/atomic.h> +#include <util/system/align.h> +#include <util/system/tls.h> + +#include <atomic> +#include <cstdio> + +#ifdef _win_ +#include <util/system/winint.h> +#endif + +#ifndef NO_SANITIZE_THREAD #define NO_SANITIZE_THREAD #if defined(__has_feature) #if __has_feature(thread_sanitizer) #undef NO_SANITIZE_THREAD #define NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) +#endif #endif #endif -#endif - -class TMemoryLog { -public: - static constexpr size_t DEFAULT_TOTAL_SIZE = 10 * 1024 * 1024; - static constexpr size_t DEFAULT_GRAIN_SIZE = 1024 * 64; - static constexpr size_t MAX_MESSAGE_SIZE = 1024; - static constexpr ui16 MAX_GET_BUFFER_TRIES = 4; - static constexpr ui16 MemcpyAlignment = 16; - - // search for cb7B68a8A561645 - static const char DEFAULT_LAST_MARK[16]; - static const char CLEAR_MARK[16]; - - static constexpr size_t LAST_MARK_SIZE = sizeof(DEFAULT_LAST_MARK); - - inline static TMemoryLog* GetMemoryLogger() noexcept { - return AtomicGet(MemLogBuffer); - } - + +class TMemoryLog { +public: + static constexpr size_t DEFAULT_TOTAL_SIZE = 10 * 1024 * 1024; + static constexpr size_t DEFAULT_GRAIN_SIZE = 1024 * 64; + static constexpr size_t MAX_MESSAGE_SIZE = 1024; + static constexpr ui16 MAX_GET_BUFFER_TRIES = 4; + static constexpr ui16 MemcpyAlignment = 16; + + // search for cb7B68a8A561645 + static const char DEFAULT_LAST_MARK[16]; + static const char CLEAR_MARK[16]; + + static constexpr size_t LAST_MARK_SIZE = sizeof(DEFAULT_LAST_MARK); + + inline static TMemoryLog* GetMemoryLogger() noexcept { + return AtomicGet(MemLogBuffer); + } + void* GetWriteBuffer(size_t amount) noexcept; - - inline static void* GetWriteBufferStatic(size_t amount) noexcept { - auto logger = GetMemoryLogger(); - if (logger == nullptr) { - return nullptr; - } - return logger->GetWriteBuffer(amount); - } - - size_t GetGlobalBufferSize() const noexcept { - return Buf.GetSize(); - } - - inline static void CreateMemoryLogBuffer( + + inline static void* GetWriteBufferStatic(size_t amount) noexcept { + auto logger = GetMemoryLogger(); + if (logger == nullptr) { + return nullptr; + } + return logger->GetWriteBuffer(amount); + } + + size_t GetGlobalBufferSize() const noexcept { + return Buf.GetSize(); + } + + inline static void CreateMemoryLogBuffer( size_t totalSize = DEFAULT_TOTAL_SIZE, size_t grainSize = DEFAULT_GRAIN_SIZE) Y_COLD { - if (AtomicGet(MemLogBuffer) != nullptr) { - return; - } - - AtomicSet(MemLogBuffer, new TMemoryLog(totalSize, grainSize)); - } - - static std::atomic<bool> PrintLastMark; - - // buffer must be at least 16 bytes + if (AtomicGet(MemLogBuffer) != nullptr) { + return; + } + + AtomicSet(MemLogBuffer, new TMemoryLog(totalSize, grainSize)); + } + + static std::atomic<bool> PrintLastMark; + + // buffer must be at least 16 bytes static void ChangeLastMark(char* buffer) noexcept; - - inline static TThread::TId GetTheadId() noexcept { - if (LogThreadId == 0) { - LogThreadId = TThread::CurrentThreadId(); - } - return LogThreadId; - } - -private: + + inline static TThread::TId GetTheadId() noexcept { + if (LogThreadId == 0) { + LogThreadId = TThread::CurrentThreadId(); + } + return LogThreadId; + } + +private: TMemoryLog(size_t totalSize, size_t grainSize) Y_COLD; - - struct TGrain { - TAtomic WritePointer = 0; - char Padding[MemcpyAlignment - sizeof(TAtomic)]; - char Data[]; - }; - - size_t NumberOfCpus; - size_t GrainSize; - size_t NumberOfGrains; - TArrayPtr<TGrain*> ActiveGrains; - NThreading::TMPMCUnorderedRing FreeGrains; - - TGrain* GetGrain(size_t grainIndex) const noexcept { - return (TGrain*)((char*)GetGlobalBuffer() + GrainSize * grainIndex); - } - - class TMMapArea { - public: - TMMapArea(size_t amount) Y_COLD { - MMap(amount); - } - - TMMapArea(const TMMapArea&) = delete; - TMMapArea& operator=(const TMMapArea& copy) = delete; - - TMMapArea(TMMapArea&& move) Y_COLD { - BufPtr = move.BufPtr; - Size = move.Size; - - move.BufPtr = nullptr; - move.Size = 0; - } - - TMMapArea& operator=(TMMapArea&& move) Y_COLD { - BufPtr = move.BufPtr; - Size = move.Size; - - move.BufPtr = nullptr; - move.Size = 0; - return *this; - } - - void Reset(size_t amount) Y_COLD { - MUnmap(); - MMap(amount); - } - - ~TMMapArea() noexcept Y_COLD { - MUnmap(); - } - - size_t GetSize() const noexcept { - return Size; - } - - void* GetPtr() const noexcept { - return BufPtr; - } - - private: - void* BufPtr; - size_t Size; -#ifdef _win_ - HANDLE Mapping; -#endif - - void MMap(size_t amount); - void MUnmap(); - }; - - TMMapArea Buf; - - void* GetGlobalBuffer() const noexcept { - return Buf.GetPtr(); - } - - static unsigned GetSelfCpu() noexcept; - - static TMemoryLog* MemLogBuffer; - static Y_POD_THREAD(TThread::TId) LogThreadId; - static char* LastMarkIsHere; -}; - -// it's no use of sanitizing this function -NO_SANITIZE_THREAD + + struct TGrain { + TAtomic WritePointer = 0; + char Padding[MemcpyAlignment - sizeof(TAtomic)]; + char Data[]; + }; + + size_t NumberOfCpus; + size_t GrainSize; + size_t NumberOfGrains; + TArrayPtr<TGrain*> ActiveGrains; + NThreading::TMPMCUnorderedRing FreeGrains; + + TGrain* GetGrain(size_t grainIndex) const noexcept { + return (TGrain*)((char*)GetGlobalBuffer() + GrainSize * grainIndex); + } + + class TMMapArea { + public: + TMMapArea(size_t amount) Y_COLD { + MMap(amount); + } + + TMMapArea(const TMMapArea&) = delete; + TMMapArea& operator=(const TMMapArea& copy) = delete; + + TMMapArea(TMMapArea&& move) Y_COLD { + BufPtr = move.BufPtr; + Size = move.Size; + + move.BufPtr = nullptr; + move.Size = 0; + } + + TMMapArea& operator=(TMMapArea&& move) Y_COLD { + BufPtr = move.BufPtr; + Size = move.Size; + + move.BufPtr = nullptr; + move.Size = 0; + return *this; + } + + void Reset(size_t amount) Y_COLD { + MUnmap(); + MMap(amount); + } + + ~TMMapArea() noexcept Y_COLD { + MUnmap(); + } + + size_t GetSize() const noexcept { + return Size; + } + + void* GetPtr() const noexcept { + return BufPtr; + } + + private: + void* BufPtr; + size_t Size; +#ifdef _win_ + HANDLE Mapping; +#endif + + void MMap(size_t amount); + void MUnmap(); + }; + + TMMapArea Buf; + + void* GetGlobalBuffer() const noexcept { + return Buf.GetPtr(); + } + + static unsigned GetSelfCpu() noexcept; + + static TMemoryLog* MemLogBuffer; + static Y_POD_THREAD(TThread::TId) LogThreadId; + static char* LastMarkIsHere; +}; + +// it's no use of sanitizing this function +NO_SANITIZE_THREAD char* BareMemLogWrite( - const char* begin, size_t msgSize, bool isLast = true) noexcept; - -// it's no use of sanitizing this function -NO_SANITIZE_THREAD + const char* begin, size_t msgSize, bool isLast = true) noexcept; + +// it's no use of sanitizing this function +NO_SANITIZE_THREAD bool MemLogWrite( - const char* begin, size_t msgSize, bool addLF = false) noexcept; - -Y_WRAPPER inline bool MemLogWrite(const char* begin, const char* end) noexcept { - if (end <= begin) { - return false; - } - - size_t msgSize = end - begin; - return MemLogWrite(begin, msgSize); -} - -template <typename TObj> -bool MemLogWriteStruct(const TObj* obj) noexcept { - auto begin = (const char*)(const void*)obj; - return MemLogWrite(begin, begin + sizeof(TObj)); -} - + const char* begin, size_t msgSize, bool addLF = false) noexcept; + +Y_WRAPPER inline bool MemLogWrite(const char* begin, const char* end) noexcept { + if (end <= begin) { + return false; + } + + size_t msgSize = end - begin; + return MemLogWrite(begin, msgSize); +} + +template <typename TObj> +bool MemLogWriteStruct(const TObj* obj) noexcept { + auto begin = (const char*)(const void*)obj; + return MemLogWrite(begin, begin + sizeof(TObj)); +} + Y_PRINTF_FORMAT(1, 0) -bool MemLogVPrintF(const char* format, va_list params) noexcept; - +bool MemLogVPrintF(const char* format, va_list params) noexcept; + Y_PRINTF_FORMAT(1, 2) Y_WRAPPER -inline bool MemLogPrintF(const char* format, ...) noexcept { - va_list params; - va_start(params, format); - auto result = MemLogVPrintF(format, params); - va_end(params); - return result; -} - -Y_WRAPPER inline bool MemLogWriteNullTerm(const char* str) noexcept { - return MemLogWrite(str, strlen(str)); -} +inline bool MemLogPrintF(const char* format, ...) noexcept { + va_list params; + va_start(params, format); + auto result = MemLogVPrintF(format, params); + va_end(params); + return result; +} + +Y_WRAPPER inline bool MemLogWriteNullTerm(const char* str) noexcept { + return MemLogWrite(str, strlen(str)); +} diff --git a/library/cpp/actors/memory_log/mmap.cpp b/library/cpp/actors/memory_log/mmap.cpp index 201998d343..b72feb1112 100644 --- a/library/cpp/actors/memory_log/mmap.cpp +++ b/library/cpp/actors/memory_log/mmap.cpp @@ -1,63 +1,63 @@ -#include "memlog.h" - +#include "memlog.h" + #if defined(_unix_) #include <sys/mman.h> #elif defined(_win_) #include <util/system/winint.h> -#else +#else #error NO IMPLEMENTATION FOR THE PLATFORM -#endif - -void TMemoryLog::TMMapArea::MMap(size_t amount) { - Y_VERIFY(amount > 0); - +#endif + +void TMemoryLog::TMMapArea::MMap(size_t amount) { + Y_VERIFY(amount > 0); + +#if defined(_unix_) + constexpr int mmapProt = PROT_READ | PROT_WRITE; +#if defined(_linux_) + constexpr int mmapFlags = MAP_PRIVATE | MAP_ANON | MAP_POPULATE; +#else + constexpr int mmapFlags = MAP_PRIVATE | MAP_ANON; +#endif + + BufPtr = ::mmap(nullptr, amount, mmapProt, mmapFlags, -1, 0); + if (BufPtr == MAP_FAILED) { + throw std::bad_alloc(); + } + +#elif defined(_win_) + Mapping = ::CreateFileMapping( + (HANDLE)-1, nullptr, PAGE_READWRITE, 0, amount, nullptr); + if (Mapping == NULL) { + throw std::bad_alloc(); + } + BufPtr = ::MapViewOfFile(Mapping, FILE_MAP_WRITE, 0, 0, amount); + if (BufPtr == NULL) { + throw std::bad_alloc(); + } +#endif + + Size = amount; +} + +void TMemoryLog::TMMapArea::MUnmap() { + if (BufPtr == nullptr) { + return; + } + #if defined(_unix_) - constexpr int mmapProt = PROT_READ | PROT_WRITE; -#if defined(_linux_) - constexpr int mmapFlags = MAP_PRIVATE | MAP_ANON | MAP_POPULATE; -#else - constexpr int mmapFlags = MAP_PRIVATE | MAP_ANON; -#endif - - BufPtr = ::mmap(nullptr, amount, mmapProt, mmapFlags, -1, 0); - if (BufPtr == MAP_FAILED) { - throw std::bad_alloc(); - } - -#elif defined(_win_) - Mapping = ::CreateFileMapping( - (HANDLE)-1, nullptr, PAGE_READWRITE, 0, amount, nullptr); - if (Mapping == NULL) { - throw std::bad_alloc(); - } - BufPtr = ::MapViewOfFile(Mapping, FILE_MAP_WRITE, 0, 0, amount); - if (BufPtr == NULL) { - throw std::bad_alloc(); - } -#endif - - Size = amount; -} - -void TMemoryLog::TMMapArea::MUnmap() { - if (BufPtr == nullptr) { - return; - } - -#if defined(_unix_) - int result = ::munmap(BufPtr, Size); - Y_VERIFY(result == 0); - -#elif defined(_win_) - BOOL result = ::UnmapViewOfFile(BufPtr); - Y_VERIFY(result != 0); - - result = ::CloseHandle(Mapping); - Y_VERIFY(result != 0); - - Mapping = 0; -#endif - - BufPtr = nullptr; - Size = 0; -} + int result = ::munmap(BufPtr, Size); + Y_VERIFY(result == 0); + +#elif defined(_win_) + BOOL result = ::UnmapViewOfFile(BufPtr); + Y_VERIFY(result != 0); + + result = ::CloseHandle(Mapping); + Y_VERIFY(result != 0); + + Mapping = 0; +#endif + + BufPtr = nullptr; + Size = 0; +} diff --git a/library/cpp/actors/memory_log/ya.make b/library/cpp/actors/memory_log/ya.make index d89d5db4d7..441b51b3c7 100644 --- a/library/cpp/actors/memory_log/ya.make +++ b/library/cpp/actors/memory_log/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER( agri g:kikimr ) - -SRCS( - memlog.cpp - memlog.h - mmap.cpp -) - -PEERDIR( + +SRCS( + memlog.cpp + memlog.h + mmap.cpp +) + +PEERDIR( library/cpp/threading/queue - contrib/libs/linuxvdso -) - -END() + contrib/libs/linuxvdso +) + +END() diff --git a/library/cpp/actors/prof/tag.cpp b/library/cpp/actors/prof/tag.cpp index 9ccf03e1a9..46b53d804f 100644 --- a/library/cpp/actors/prof/tag.cpp +++ b/library/cpp/actors/prof/tag.cpp @@ -1,6 +1,6 @@ -#include "tag.h" +#include "tag.h" #include "tcmalloc.h" - + #include <library/cpp/charset/ci_string.h> #include <library/cpp/containers/atomizer/atomizer.h> #include <library/cpp/malloc/api/malloc.h> @@ -13,9 +13,9 @@ #include <util/generic/singleton.h> #include <util/generic/string.h> #include <util/generic/vector.h> -#include <util/system/mutex.h> - -namespace NProfiling { +#include <util/system/mutex.h> + +namespace NProfiling { class TStringAtoms { private: TMutex Mutex; @@ -59,19 +59,19 @@ namespace NProfiling { } } }; - + ui32 MakeTag(const char* s) { return TStringAtoms::Instance().MakeTag(s); } - + ui32 MakeTags(const TVector<const char*>& ss) { return TStringAtoms::Instance().MakeTags(ss); } - + const char* GetTag(ui32 tag) { return TStringAtoms::Instance().GetTag(tag); - } - + } + size_t GetTagsCount() { return TStringAtoms::Instance().GetTagsCount(); } diff --git a/library/cpp/actors/prof/tag.h b/library/cpp/actors/prof/tag.h index 357e264a22..ec4bed5b08 100644 --- a/library/cpp/actors/prof/tag.h +++ b/library/cpp/actors/prof/tag.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <util/generic/fwd.h> - -/* - Common registry for tagging memory profiler. - Register a new tag with MakeTag using a unique string. + +/* + Common registry for tagging memory profiler. + Register a new tag with MakeTag using a unique string. Use registered tags with SetThreadAllocTag function in allocator API. -*/ - -namespace NProfiling { +*/ + +namespace NProfiling { ui32 MakeTag(const char* s); - + // Make only unique tags. Y_VERIFY inside. ui32 MakeTags(const TVector<const char*>& ss); - + const char* GetTag(ui32 tag); size_t GetTagsCount(); - + using TSetThreadAllocTag = ui32(ui32 tag); extern TSetThreadAllocTag* SetThreadAllocTag; @@ -31,32 +31,32 @@ namespace NProfiling { ui32 newTag = MakeTag(tagName); RestoreTag = SetThreadAllocTag(newTag); } - + TMemoryTagScope(TMemoryTagScope&& move) : RestoreTag(move.RestoreTag) , Released(move.Released) { move.Released = true; } - + TMemoryTagScope& operator=(TMemoryTagScope&& move) { RestoreTag = move.RestoreTag; Released = move.Released; move.Released = true; return *this; } - + static void Reset(ui32 tag) { SetThreadAllocTag(tag); - } - + } + void Release() { if (!Released) { SetThreadAllocTag(RestoreTag); Released = true; } } - + ~TMemoryTagScope() { if (!Released) { SetThreadAllocTag(RestoreTag); diff --git a/library/cpp/actors/prof/ut/tag_ut.cpp b/library/cpp/actors/prof/ut/tag_ut.cpp index accf3921ab..43c56ecddc 100644 --- a/library/cpp/actors/prof/ut/tag_ut.cpp +++ b/library/cpp/actors/prof/ut/tag_ut.cpp @@ -1,68 +1,68 @@ -#include "tag.h" - +#include "tag.h" + #include <library/cpp/testing/unittest/registar.h> + +using namespace NProfiling; + +class TAtomTagsTest: public TTestBase { +private: + UNIT_TEST_SUITE(TAtomTagsTest); + UNIT_TEST(Test_MakeTag); + UNIT_TEST(Test_Make2Tags); + UNIT_TEST(Test_MakeTagTwice); + + UNIT_TEST(Test_MakeAndGetTag); + + UNIT_TEST(Test_MakeVector); + UNIT_TEST_SUITE_END(); -using namespace NProfiling; - -class TAtomTagsTest: public TTestBase { -private: - UNIT_TEST_SUITE(TAtomTagsTest); - UNIT_TEST(Test_MakeTag); - UNIT_TEST(Test_Make2Tags); - UNIT_TEST(Test_MakeTagTwice); - - UNIT_TEST(Test_MakeAndGetTag); - - UNIT_TEST(Test_MakeVector); - UNIT_TEST_SUITE_END(); - -public: - void Test_MakeTag(); - void Test_Make2Tags(); - void Test_MakeTagTwice(); - void Test_MakeAndGetTag(); - void Test_MakeVector(); -}; - -UNIT_TEST_SUITE_REGISTRATION(TAtomTagsTest); - -void TAtomTagsTest::Test_MakeTag() { - ui32 tag = MakeTag("a tag"); - UNIT_ASSERT(tag != 0); -} - -void TAtomTagsTest::Test_Make2Tags() { - ui32 tag1 = MakeTag("a tag 1"); - ui32 tag2 = MakeTag("a tag 2"); - UNIT_ASSERT(tag1 != 0); - UNIT_ASSERT(tag2 != 0); - UNIT_ASSERT(tag1 != tag2); -} - -void TAtomTagsTest::Test_MakeTagTwice() { - ui32 tag1 = MakeTag("a tag twice"); - ui32 tag2 = MakeTag("a tag twice"); - UNIT_ASSERT(tag1 != 0); - UNIT_ASSERT(tag1 == tag2); -} - -void TAtomTagsTest::Test_MakeAndGetTag() { - const char* makeStr = "tag to get"; - ui32 tag = MakeTag(makeStr); - const char* tagStr = GetTag(tag); - UNIT_ASSERT_STRINGS_EQUAL(makeStr, tagStr); -} - -void TAtomTagsTest::Test_MakeVector() { +public: + void Test_MakeTag(); + void Test_Make2Tags(); + void Test_MakeTagTwice(); + void Test_MakeAndGetTag(); + void Test_MakeVector(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TAtomTagsTest); + +void TAtomTagsTest::Test_MakeTag() { + ui32 tag = MakeTag("a tag"); + UNIT_ASSERT(tag != 0); +} + +void TAtomTagsTest::Test_Make2Tags() { + ui32 tag1 = MakeTag("a tag 1"); + ui32 tag2 = MakeTag("a tag 2"); + UNIT_ASSERT(tag1 != 0); + UNIT_ASSERT(tag2 != 0); + UNIT_ASSERT(tag1 != tag2); +} + +void TAtomTagsTest::Test_MakeTagTwice() { + ui32 tag1 = MakeTag("a tag twice"); + ui32 tag2 = MakeTag("a tag twice"); + UNIT_ASSERT(tag1 != 0); + UNIT_ASSERT(tag1 == tag2); +} + +void TAtomTagsTest::Test_MakeAndGetTag() { + const char* makeStr = "tag to get"; + ui32 tag = MakeTag(makeStr); + const char* tagStr = GetTag(tag); + UNIT_ASSERT_STRINGS_EQUAL(makeStr, tagStr); +} + +void TAtomTagsTest::Test_MakeVector() { TVector<const char*> strs = { - "vector tag 0", - "vector tag 1", - "vector tag 3", + "vector tag 0", + "vector tag 1", + "vector tag 3", "vector tag 4"}; - ui32 baseTag = MakeTags(strs); - UNIT_ASSERT(baseTag != 0); - for (ui32 i = 0; i < strs.size(); ++i) { - const char* str = GetTag(baseTag + i); - UNIT_ASSERT_STRINGS_EQUAL(str, strs[i]); - } -} + ui32 baseTag = MakeTags(strs); + UNIT_ASSERT(baseTag != 0); + for (ui32 i = 0; i < strs.size(); ++i) { + const char* str = GetTag(baseTag + i); + UNIT_ASSERT_STRINGS_EQUAL(str, strs[i]); + } +} diff --git a/library/cpp/actors/prof/ut/ya.make b/library/cpp/actors/prof/ut/ya.make index 47c58a8fb7..d177fbdd22 100644 --- a/library/cpp/actors/prof/ut/ya.make +++ b/library/cpp/actors/prof/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/actors/prof) - + OWNER( agri g:kikimr ) - -SRCS( - tag_ut.cpp -) - -END() + +SRCS( + tag_ut.cpp +) + +END() diff --git a/library/cpp/actors/prof/ya.make b/library/cpp/actors/prof/ya.make index b5e2497563..cdd3e57d1f 100644 --- a/library/cpp/actors/prof/ya.make +++ b/library/cpp/actors/prof/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER( agri g:kikimr ) - -SRCS( - tag.cpp -) - -PEERDIR( + +SRCS( + tag.cpp +) + +PEERDIR( library/cpp/charset library/cpp/containers/atomizer -) - +) + IF (PROFILE_MEMORY_ALLOCATIONS) CFLAGS(-DPROFILE_MEMORY_ALLOCATIONS) PEERDIR( @@ -30,4 +30,4 @@ ELSE() SRCS(tcmalloc_null.cpp) ENDIF() -END() +END() diff --git a/library/cpp/actors/protos/actors.proto b/library/cpp/actors/protos/actors.proto index 5fbd6d44ee..5e40cbf6c2 100644 --- a/library/cpp/actors/protos/actors.proto +++ b/library/cpp/actors/protos/actors.proto @@ -6,8 +6,8 @@ message TActorId { required fixed64 RawX1 = 1; required fixed64 RawX2 = 2; } - -message TCallbackException { + +message TCallbackException { required TActorId ActorId = 1; - required string ExceptionMessage = 2; -} + required string ExceptionMessage = 2; +} diff --git a/library/cpp/actors/protos/interconnect.proto b/library/cpp/actors/protos/interconnect.proto index 2e3b0d0d15..30a5c1bb74 100644 --- a/library/cpp/actors/protos/interconnect.proto +++ b/library/cpp/actors/protos/interconnect.proto @@ -14,7 +14,7 @@ message TEvNodeInfo { optional string Address = 2; optional uint32 Port = 3; } - + extend google.protobuf.FieldOptions { optional string PrintName = 50376; } @@ -43,19 +43,19 @@ message TScopeId { optional fixed64 X2 = 2; } -message THandshakeRequest { - required uint64 Protocol = 1; - - required uint64 ProgramPID = 2; - required uint64 ProgramStartTime = 3; - required uint64 Serial = 4; - - required uint32 ReceiverNodeId = 5; +message THandshakeRequest { + required uint64 Protocol = 1; + + required uint64 ProgramPID = 2; + required uint64 ProgramStartTime = 3; + required uint64 Serial = 4; + + required uint32 ReceiverNodeId = 5; required string SenderActorId = 6; - - optional string SenderHostName = 7; - optional string ReceiverHostName = 8; - optional string UUID = 9; + + optional string SenderHostName = 7; + optional string ReceiverHostName = 8; + optional string UUID = 9; optional TClusterUUIDs ClusterUUIDs = 13; optional bytes Ballast = 10; @@ -72,15 +72,15 @@ message THandshakeRequest { optional bool RequestModernFrame = 18; optional bool RequestAuthOnly = 19; -} - -message THandshakeSuccess { - required uint64 Protocol = 1; - - required uint64 ProgramPID = 2; - required uint64 ProgramStartTime = 3; - required uint64 Serial = 4; - +} + +message THandshakeSuccess { + required uint64 Protocol = 1; + + required uint64 ProgramPID = 2; + required uint64 ProgramStartTime = 3; + required uint64 Serial = 4; + required string SenderActorId = 5; optional string VersionTag = 6; @@ -94,13 +94,13 @@ message THandshakeSuccess { optional bool UseModernFrame = 11; optional bool AuthOnly = 12; -} - -message THandshakeReply { - optional THandshakeSuccess Success = 1; - optional string ErrorExplaination = 2; +} + +message THandshakeReply { + optional THandshakeSuccess Success = 1; + optional string ErrorExplaination = 2; optional bool CookieCheckResult = 3; -} +} message TEvLoadMessage { message THop { diff --git a/library/cpp/actors/protos/services_common.proto b/library/cpp/actors/protos/services_common.proto index afa0ec0073..99347ad37e 100644 --- a/library/cpp/actors/protos/services_common.proto +++ b/library/cpp/actors/protos/services_common.proto @@ -7,8 +7,8 @@ enum EServiceCommon { GLOBAL = 0; INTERCONNECT = 1; - TEST = 2; - PROTOCOLS = 3; + TEST = 2; + PROTOCOLS = 3; INTERCONNECT_SPEED_TEST = 4; INTERCONNECT_STATUS = 5; INTERCONNECT_NETWORK = 6; diff --git a/library/cpp/actors/protos/unittests.proto b/library/cpp/actors/protos/unittests.proto index a856b0942a..68b662b9b3 100644 --- a/library/cpp/actors/protos/unittests.proto +++ b/library/cpp/actors/protos/unittests.proto @@ -1,17 +1,17 @@ option cc_enable_arenas = true; -message TSimple { - required string Str1 = 1; - optional string Str2 = 2; - optional uint64 Number1 = 3; -} - -message TBigMessage { - repeated TSimple Simples = 1; - repeated string ManyStr = 2; - optional string OneMoreStr = 3; - optional uint64 YANumber = 4; -} +message TSimple { + required string Str1 = 1; + optional string Str2 = 2; + optional uint64 Number1 = 3; +} + +message TBigMessage { + repeated TSimple Simples = 1; + repeated string ManyStr = 2; + optional string OneMoreStr = 3; + optional uint64 YANumber = 4; +} message TMessageWithPayload { optional string Meta = 1; diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 6fa25b9965..0459f76386 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -74,7 +74,7 @@ namespace NActors { ActorSystem->Stop(); ActorSystem.Destroy(); - Poller.Reset(); + Poller.Reset(); } TTestActorRuntimeBase::TNodeDataBase::~TNodeDataBase() { @@ -909,17 +909,17 @@ namespace NActors { case TMailboxType::Revolving: UnlockFromExecution((TMailboxTable::TRevolvingMailbox *)mailbox, node->ExecutorPools[0], false, hint, MaxWorkers, ++revolvingCounter); break; - case TMailboxType::HTSwap: + case TMailboxType::HTSwap: UnlockFromExecution((TMailboxTable::THTSwapMailbox *)mailbox, node->ExecutorPools[0], false, hint, MaxWorkers, ++revolvingCounter); - break; - case TMailboxType::ReadAsFilled: + break; + case TMailboxType::ReadAsFilled: UnlockFromExecution((TMailboxTable::TReadAsFilledMailbox *)mailbox, node->ExecutorPools[0], false, hint, MaxWorkers, ++revolvingCounter); - break; - case TMailboxType::TinyReadAsFilled: + break; + case TMailboxType::TinyReadAsFilled: UnlockFromExecution((TMailboxTable::TTinyReadAsFilledMailbox *)mailbox, node->ExecutorPools[0], false, hint, MaxWorkers, ++revolvingCounter); - break; + break; default: - Y_FAIL("Unsupported mailbox type"); + Y_FAIL("Unsupported mailbox type"); } return actorId; @@ -1645,13 +1645,13 @@ namespace NActors { setup->LocalServices = node->LocalServices; setup->Interconnect.ProxyActors.resize(FirstNodeId + NodeCount); const TActorId nameserviceId = GetNameserviceActorId(); - - TIntrusivePtr<TInterconnectProxyCommon> common; - common.Reset(new TInterconnectProxyCommon); - common->NameserviceId = nameserviceId; - common->MonCounters = interconnectCounters; + + TIntrusivePtr<TInterconnectProxyCommon> common; + common.Reset(new TInterconnectProxyCommon); + common->NameserviceId = nameserviceId; + common->MonCounters = interconnectCounters; common->TechnicalSelfHostName = "::1"; - + if (!UseRealThreads) { common->Settings.DeadPeer = TDuration::Max(); common->Settings.CloseOnIdle = TDuration::Max(); @@ -1668,7 +1668,7 @@ namespace NActors { continue; const ui32 peerNodeId = FirstNodeId + proxyNodeIndex; - + IActor *proxyActor = UseRealInterconnect ? new TInterconnectProxyTCP(peerNodeId, common) : InterconnectMock.CreateProxyMock(setup->NodeId, peerNodeId, common); diff --git a/library/cpp/actors/testlib/test_runtime.h b/library/cpp/actors/testlib/test_runtime.h index 26e3b45c98..cca5876645 100644 --- a/library/cpp/actors/testlib/test_runtime.h +++ b/library/cpp/actors/testlib/test_runtime.h @@ -556,7 +556,7 @@ namespace NActors { TIntrusivePtr<NMonitoring::TDynamicCounters> DynamicCounters; TIntrusivePtr<NActors::NLog::TSettings> LogSettings; - TIntrusivePtr<NInterconnect::TPollerThreads> Poller; + TIntrusivePtr<NInterconnect::TPollerThreads> Poller; volatile ui64* ActorSystemTimestamp; volatile ui64* ActorSystemMonotonic; TVector<std::pair<TActorId, TActorSetupCmd> > LocalServices; diff --git a/library/cpp/actors/util/funnel_queue.h b/library/cpp/actors/util/funnel_queue.h index 0e21e2617c..d760252054 100644 --- a/library/cpp/actors/util/funnel_queue.h +++ b/library/cpp/actors/util/funnel_queue.h @@ -91,62 +91,62 @@ protected: delete entry; return next; } - -protected: - struct TEntryIter { - TEntry* ptr; - - ElementType& operator*() { - return ptr->Data; - } - - ElementType* operator->() { - return &ptr->Data; - } - - TEntryIter& operator++() { - ptr = AtomicGet(ptr->Next); - return *this; - } - - bool operator!=(const TEntryIter& other) const { - return ptr != other.ptr; - } - - bool operator==(const TEntryIter& other) const { - return ptr == other.ptr; - } - }; - - struct TConstEntryIter { - const TEntry* ptr; - - const ElementType& operator*() { - return ptr->Data; - } - - const ElementType* operator->() { - return &ptr->Data; - } - - TEntryIter& operator++() { - ptr = AtomicGet(ptr->Next); - return *this; - } - - bool operator!=(const TConstEntryIter& other) const { - return ptr != other.ptr; - } - - bool operator==(const TConstEntryIter& other) const { - return ptr == other.ptr; - } - }; - -public: - using const_iterator = TConstEntryIter; - using iterator = TEntryIter; - + +protected: + struct TEntryIter { + TEntry* ptr; + + ElementType& operator*() { + return ptr->Data; + } + + ElementType* operator->() { + return &ptr->Data; + } + + TEntryIter& operator++() { + ptr = AtomicGet(ptr->Next); + return *this; + } + + bool operator!=(const TEntryIter& other) const { + return ptr != other.ptr; + } + + bool operator==(const TEntryIter& other) const { + return ptr == other.ptr; + } + }; + + struct TConstEntryIter { + const TEntry* ptr; + + const ElementType& operator*() { + return ptr->Data; + } + + const ElementType* operator->() { + return &ptr->Data; + } + + TEntryIter& operator++() { + ptr = AtomicGet(ptr->Next); + return *this; + } + + bool operator!=(const TConstEntryIter& other) const { + return ptr != other.ptr; + } + + bool operator==(const TConstEntryIter& other) const { + return ptr == other.ptr; + } + }; + +public: + using const_iterator = TConstEntryIter; + using iterator = TEntryIter; + iterator begin() { return {AtomicGet(Front)}; } @@ -156,7 +156,7 @@ public: const_iterator begin() const { return {AtomicGet(Front)}; } - + iterator end() { return {nullptr}; } diff --git a/library/cpp/actors/util/recentwnd.h b/library/cpp/actors/util/recentwnd.h index ba1ede6f29..29425301e4 100644 --- a/library/cpp/actors/util/recentwnd.h +++ b/library/cpp/actors/util/recentwnd.h @@ -1,28 +1,28 @@ -#pragma once +#pragma once -#include <util/generic/deque.h> - -template <typename TElem, +#include <util/generic/deque.h> + +template <typename TElem, template <typename, typename...> class TContainer = TDeque> -class TRecentWnd { -public: +class TRecentWnd { +public: TRecentWnd(ui32 wndSize) : MaxWndSize_(wndSize) { } - - void Push(const TElem& elem) { - if (Window_.size() == MaxWndSize_) - Window_.erase(Window_.begin()); - Window_.emplace_back(elem); - } - - void Push(TElem&& elem) { - if (Window_.size() == MaxWndSize_) - Window_.erase(Window_.begin()); - Window_.emplace_back(std::move(elem)); - } - + + void Push(const TElem& elem) { + if (Window_.size() == MaxWndSize_) + Window_.erase(Window_.begin()); + Window_.emplace_back(elem); + } + + void Push(TElem&& elem) { + if (Window_.size() == MaxWndSize_) + Window_.erase(Window_.begin()); + Window_.emplace_back(std::move(elem)); + } + TElem& Last() { return Window_.back(); } @@ -35,33 +35,33 @@ public: ui64 Size() const { return Window_.size(); } - - using const_iterator = typename TContainer<TElem>::const_iterator; - + + using const_iterator = typename TContainer<TElem>::const_iterator; + const_iterator begin() { return Window_.begin(); } const_iterator end() { return Window_.end(); } + + void Reset(ui32 wndSize = 0) { + Window_.clear(); + if (wndSize != 0) { + MaxWndSize_ = wndSize; + } + } + + void ResetWnd(ui32 wndSize) { + Y_VERIFY(wndSize != 0); + MaxWndSize_ = wndSize; + if (Window_.size() > MaxWndSize_) { + Window_.erase(Window_.begin(), + Window_.begin() + Window_.size() - MaxWndSize_); + } + } - void Reset(ui32 wndSize = 0) { - Window_.clear(); - if (wndSize != 0) { - MaxWndSize_ = wndSize; - } - } - - void ResetWnd(ui32 wndSize) { - Y_VERIFY(wndSize != 0); - MaxWndSize_ = wndSize; - if (Window_.size() > MaxWndSize_) { - Window_.erase(Window_.begin(), - Window_.begin() + Window_.size() - MaxWndSize_); - } - } - -private: - TContainer<TElem> Window_; +private: + TContainer<TElem> Window_; ui32 MaxWndSize_; -}; +}; diff --git a/library/cpp/actors/util/thread.h b/library/cpp/actors/util/thread.h index d742c8c585..d90ab745fe 100644 --- a/library/cpp/actors/util/thread.h +++ b/library/cpp/actors/util/thread.h @@ -10,17 +10,17 @@ inline void SetCurrentThreadName(const TString& name, const ui32 maxCharsFromProcessName = 8) { #if defined(_linux_) - // linux limits threadname by 15 + \0 - - TStringBuf procName(GetExecPath()); - procName = procName.RNextTok('/'); - procName = procName.SubStr(0, maxCharsFromProcessName); - + // linux limits threadname by 15 + \0 + + TStringBuf procName(GetExecPath()); + procName = procName.RNextTok('/'); + procName = procName.SubStr(0, maxCharsFromProcessName); + TStringStream linuxName; - linuxName << procName << "." << name; + linuxName << procName << "." << name; TThread::SetCurrentThreadName(linuxName.Str().data()); #else - Y_UNUSED(maxCharsFromProcessName); + Y_UNUSED(maxCharsFromProcessName); TThread::SetCurrentThreadName(name.data()); #endif } diff --git a/library/cpp/balloc/balloc.cpp b/library/cpp/balloc/balloc.cpp index fab489db4c..ea37357c13 100644 --- a/library/cpp/balloc/balloc.cpp +++ b/library/cpp/balloc/balloc.cpp @@ -32,21 +32,21 @@ namespace NBalloc { static void Y_FORCE_INLINE Free(void* ptr) { if (ptr == nullptr) { return; - } - TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; - size_t size = allocHeader->AllocSize; + } + TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; + size_t size = allocHeader->AllocSize; const size_t signature = size & SIGNATURE_MASK; if (Y_LIKELY(signature == ALIVE_SIGNATURE)) { - allocHeader->AllocSize = 0; // abort later on double free + allocHeader->AllocSize = 0; // abort later on double free #ifdef DBG_FILL_MEMORY memset(ptr, 0xde, size - signature); #endif - FreeRaw(allocHeader->Block); + FreeRaw(allocHeader->Block); if (NAllocStats::IsEnabled()) { NAllocStats::DecThreadAllocStats(size - signature); } } else if (signature == DISABLED_SIGNATURE) { - LibcFree(allocHeader->Block); + LibcFree(allocHeader->Block); } else { NMalloc::AbortFromCorruptedAllocator(); } diff --git a/library/cpp/http/io/stream.cpp b/library/cpp/http/io/stream.cpp index 6689be684f..083a531b31 100644 --- a/library/cpp/http/io/stream.cpp +++ b/library/cpp/http/io/stream.cpp @@ -145,7 +145,7 @@ public: , HasContentLength_(false) , ContentLength_(0) , ContentEncoded_(false) - , Expect100Continue_(false) + , Expect100Continue_(false) { BuildInputChain(); Y_ASSERT(Input_); @@ -204,10 +204,10 @@ public: return HasContentLength_ || ChunkedInput_; } - inline bool HasExpect100Continue() const noexcept { - return Expect100Continue_; - } - + inline bool HasExpect100Continue() const noexcept { + return Expect100Continue_; + } + private: template <class Operation> inline size_t Perform(size_t len, const Operation& operation) { @@ -324,14 +324,14 @@ private: } } [[fallthrough]]; - HEADERCMP(header, "expect") { - auto findContinue = [&](const TStringBuf& s) { + HEADERCMP(header, "expect") { + auto findContinue = [&](const TStringBuf& s) { if (strnicmp(s.data(), "100-continue", 13) == 0) { - Expect100Continue_ = true; - } - }; - ForEach(header.Value(), findContinue); - } + Expect100Continue_ = true; + } + }; + ForEach(header.Value(), findContinue); + } break; } } @@ -386,7 +386,7 @@ private: ui64 ContentLength_; bool ContentEncoded_; - bool Expect100Continue_; + bool Expect100Continue_; }; THttpInput::THttpInput(IInputStream* slave) @@ -452,10 +452,10 @@ bool THttpInput::HasContent() const noexcept { return Impl_->HasContent(); } -bool THttpInput::HasExpect100Continue() const noexcept { - return Impl_->HasExpect100Continue(); -} - +bool THttpInput::HasExpect100Continue() const noexcept { + return Impl_->HasExpect100Continue(); +} + class THttpOutput::TImpl { class TSizeCalculator: public IOutputStream { public: @@ -512,11 +512,11 @@ public: inline ~TImpl() { } - inline void SendContinue() { - Output_->Write("HTTP/1.1 100 Continue\r\n\r\n"); - Output_->Flush(); - } - + inline void SendContinue() { + Output_->Write("HTTP/1.1 100 Continue\r\n\r\n"); + Output_->Flush(); + } + inline void Write(const void* buf, size_t len) { if (Finished_) { ythrow THttpException() << "can not write to finished stream"; @@ -954,10 +954,10 @@ bool THttpOutput::CanBeKeepAlive() const noexcept { return Impl_->CanBeKeepAlive(); } -void THttpOutput::SendContinue() { - Impl_->SendContinue(); -} - +void THttpOutput::SendContinue() { + Impl_->SendContinue(); +} + const TString& THttpOutput::FirstLine() const noexcept { return Impl_->FirstLine(); } diff --git a/library/cpp/http/io/stream.h b/library/cpp/http/io/stream.h index 78ca4fc814..e0846ef107 100644 --- a/library/cpp/http/io/stream.h +++ b/library/cpp/http/io/stream.h @@ -84,8 +84,8 @@ public: /// Returns true if Content-Length or Transfer-Encoding header received bool HasContent() const noexcept; - bool HasExpect100Continue() const noexcept; - + bool HasExpect100Continue() const noexcept; + private: size_t DoRead(void* buf, size_t len) override; size_t DoSkip(size_t len) override; @@ -145,8 +145,8 @@ public: /// не завершается после окончания транзакции. bool CanBeKeepAlive() const noexcept; - void SendContinue(); - + void SendContinue(); + /* * first line - response or request */ diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 128583bdd7..a1b70f10e1 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -67,7 +67,7 @@ public: THttpServer::TImpl* HttpServ_ = nullptr; bool Reject_ = false; TInstant LastUsed; - TInstant AcceptMoment; + TInstant AcceptMoment; size_t ReceivedRequests = 0; }; @@ -300,7 +300,7 @@ public: ~TListenSocket() override { } - void OnPollEvent(TInstant) override { + void OnPollEvent(TInstant) override { SOCKET s = ::accept(S_, nullptr, nullptr); if (s == INVALID_SOCKET) { @@ -589,7 +589,7 @@ void TClientConnection::OnPollEvent(TInstant now) { } THolder<TClientRequest> obj(HttpServ_->CreateRequest(this_)); - AcceptMoment = now; + AcceptMoment = now; HttpServ_->AddRequest(obj, Reject_); } @@ -776,10 +776,10 @@ NAddr::IRemoteAddrRef TClientRequest::GetListenerSockAddrRef() const noexcept { return Conn_->ListenerSockAddrRef_; } -TInstant TClientRequest::AcceptMoment() const noexcept { - return Conn_->AcceptMoment; -} - +TInstant TClientRequest::AcceptMoment() const noexcept { + return Conn_->AcceptMoment; +} + /* * TRequestReplier */ diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h index b292d38f27..0b1607bfbb 100644 --- a/library/cpp/http/server/http.h +++ b/library/cpp/http/server/http.h @@ -8,7 +8,7 @@ #include <util/memory/blob.h> #include <util/generic/ptr.h> #include <util/generic/vector.h> -#include <util/system/atomic.h> +#include <util/system/atomic.h> class IThreadFactory; class TClientRequest; @@ -90,8 +90,8 @@ public: const IThreadPool& GetRequestQueue() const; const IThreadPool& GetFailQueue() const; - static TAtomicBase AcceptReturnsInvalidSocketCounter(); - + static TAtomicBase AcceptReturnsInvalidSocketCounter(); + private: bool MaxRequestsReached() const; @@ -120,7 +120,7 @@ public: THttpServer* HttpServ() const noexcept; const TSocket& Socket() const noexcept; NAddr::IRemoteAddrRef GetListenerSockAddrRef() const noexcept; - TInstant AcceptMoment() const noexcept; + TInstant AcceptMoment() const noexcept; bool IsLocal() const; bool CheckLoopback(); diff --git a/library/cpp/http/server/http_ex.cpp b/library/cpp/http/server/http_ex.cpp index e07db22bfc..7fb6378482 100644 --- a/library/cpp/http/server/http_ex.cpp +++ b/library/cpp/http/server/http_ex.cpp @@ -27,45 +27,45 @@ bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBl char* s = RequestString.begin(); - enum EMethod { - NotImplemented, - Get, - Post, - Put, + enum EMethod { + NotImplemented, + Get, + Post, + Put, Patch, Delete, - }; - - enum EMethod foundMethod; - char* urlStart; - + }; + + enum EMethod foundMethod; + char* urlStart; + if (strnicmp(s, "GET ", 4) == 0) { - foundMethod = Get; - urlStart = s + 4; - } else if (strnicmp(s, "POST ", 5) == 0) { - foundMethod = Post; - urlStart = s + 5; - } else if (strnicmp(s, "PUT ", 4) == 0) { - foundMethod = Put; - urlStart = s + 4; + foundMethod = Get; + urlStart = s + 4; + } else if (strnicmp(s, "POST ", 5) == 0) { + foundMethod = Post; + urlStart = s + 5; + } else if (strnicmp(s, "PUT ", 4) == 0) { + foundMethod = Put; + urlStart = s + 4; } else if (strnicmp(s, "PATCH ", 6) == 0) { foundMethod = Patch; urlStart = s + 6; } else if (strnicmp(s, "DELETE ", 7) == 0) { foundMethod = Delete; urlStart = s + 7; - } else { - foundMethod = NotImplemented; - } - - switch (foundMethod) { + } else { + foundMethod = NotImplemented; + } + + switch (foundMethod) { case Get: case Delete: if (!Parse(urlStart, rd)) { return false; } break; - + case Post: case Put: case Patch: @@ -91,8 +91,8 @@ bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBl } catch (...) { Output() << "HTTP/1.1 400 Bad request\r\n\r\n"; return false; - } - + } + if (!Parse(urlStart, rd)) { return false; } diff --git a/library/cpp/http/server/http_ut.cpp b/library/cpp/http/server/http_ut.cpp index cc62bb988e..4e0e6bd69d 100644 --- a/library/cpp/http/server/http_ut.cpp +++ b/library/cpp/http/server/http_ut.cpp @@ -137,7 +137,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { }; static const TString CrLf = "\r\n"; - + struct TTestRequest { TTestRequest(ui16 port, TString content = TString()) : Port(port) @@ -145,23 +145,23 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { { } - void CheckContinue(TSocketInput& si) { - if (Expect100Continue) { - TStringStream ss; + void CheckContinue(TSocketInput& si) { + if (Expect100Continue) { + TStringStream ss; TString firstLine; - si.ReadLine(firstLine); - for (;;) { + si.ReadLine(firstLine); + for (;;) { TString buf; - si.ReadLine(buf); + si.ReadLine(buf); if (buf.size() == 0) { - break; - } - ss << buf << CrLf; - } - UNIT_ASSERT_EQUAL(firstLine, "HTTP/1.1 100 Continue"); - } - } - + break; + } + ss << buf << CrLf; + } + UNIT_ASSERT_EQUAL(firstLine, "HTTP/1.1 100 Continue"); + } + } + TString Execute() { TSocket* s = nullptr; THolder<TSocket> singleReqSocket; @@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { s = singleReqSocket.Get(); } bool isPost = Type == "POST"; - TSocketInput si(*s); + TSocketInput si(*s); if (UseHttpOutput) { TSocketOutput so(*s); @@ -194,21 +194,21 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { } else { r << "Transfer-Encoding: chunked" << CrLf; } - if (Expect100Continue) { - r << "Expect: 100-continue" << CrLf; - } + if (Expect100Continue) { + r << "Expect: 100-continue" << CrLf; + } } r << CrLf; if (isPost) { - output.Write(r.Str()); - output.Flush(); - CheckContinue(si); - output.Write(Content); - output.Finish(); - } else { - output.Write(r.Str()); - output.Finish(); + output.Write(r.Str()); + output.Flush(); + CheckContinue(si); + output.Write(Content); + output.Finish(); + } else { + output.Write(r.Str()); + output.Finish(); } } else { TStringStream r; @@ -222,9 +222,9 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { if (EnableResponseEncoding) { r << "Accept-Encoding: gzip, deflate, x-gzip, x-deflate, y-lzo, y-lzf, y-lzq, y-bzip2, y-lzma" << CrLf; } - if (isPost && Expect100Continue) { - r << "Expect: 100-continue" << CrLf; - } + if (isPost && Expect100Continue) { + r << "Expect: 100-continue" << CrLf; + } if (isPost && ContentEncoding.size() && Content.size()) { r << "Content-Encoding: " << ContentEncoding << CrLf; TStringStream compressedContent; @@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { r << "Content-Length: " << compressedContent.Size() << CrLf; r << CrLf; s->Send(r.Data(), r.Size()); - CheckContinue(si); + CheckContinue(si); Hdr = r.Str(); TString tosend = compressedContent.Str(); s->Send(tosend.data(), tosend.size()); @@ -246,7 +246,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { r << "Content-Length: " << Content.size() << CrLf; r << CrLf; s->Send(r.Data(), r.Size()); - CheckContinue(si); + CheckContinue(si); Hdr = r.Str(); s->Send(Content.data(), Content.size()); } else { @@ -286,7 +286,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { THolder<TSocket> KeepAlivedSocket; bool EnableResponseEncoding = false; TString Hdr; - bool Expect100Continue = false; + bool Expect100Continue = false; }; class TFailingMtpQueue: public TSimpleThreadPool { @@ -354,10 +354,10 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { r.ContentEncoding = encoder; for (bool expect100Continue : trueFalse) { - r.Expect100Continue = expect100Continue; + r.Expect100Continue = expect100Continue; TString resp = r.Execute(); - UNIT_ASSERT_C(resp == res, "diff echo response for request:\n" + r.GetDescription()); - } + UNIT_ASSERT_C(resp == res, "diff echo response for request:\n" + r.GetDescription()); + } } } } diff --git a/library/cpp/lfalloc/lf_allocX64.h b/library/cpp/lfalloc/lf_allocX64.h index fd2a906d6f..850e3a0b4f 100644 --- a/library/cpp/lfalloc/lf_allocX64.h +++ b/library/cpp/lfalloc/lf_allocX64.h @@ -666,7 +666,7 @@ class TLFAllocFreeList { TNode* volatile Head; TNode* volatile Pending; - TAtomic PendingToFreeListCounter; + TAtomic PendingToFreeListCounter; TAtomic AllocCount; void* Padding; @@ -682,10 +682,10 @@ class TLFAllocFreeList { TNode* res; for (res = Head; res; res = Head) { TNode* keepNext = res->Next; - if (DoCas(&Head, keepNext, res) == res) { - //Y_VERIFY(keepNext == res->Next); + if (DoCas(&Head, keepNext, res) == res) { + //Y_VERIFY(keepNext == res->Next); break; - } + } } return res; } @@ -712,19 +712,19 @@ public: Enqueue(&Pending, newFree); } Y_FORCE_INLINE void* Alloc() { - TAtomic keepCounter = AtomicAdd(PendingToFreeListCounter, 0); + TAtomic keepCounter = AtomicAdd(PendingToFreeListCounter, 0); TNode* fl = Pending; if (AtomicAdd(AllocCount, 1) == 1) { - // No other allocs in progress. - // If (keepCounter == PendingToFreeListCounter) then Pending was not freed by other threads. - // Hence Pending is not used in any concurrent DoAlloc() atm and can be safely moved to FreeList + // No other allocs in progress. + // If (keepCounter == PendingToFreeListCounter) then Pending was not freed by other threads. + // Hence Pending is not used in any concurrent DoAlloc() atm and can be safely moved to FreeList if (fl && keepCounter == AtomicAdd(PendingToFreeListCounter, 0) && DoCas(&Pending, (TNode*)nullptr, fl) == fl) { // pick first element from Pending and return it void* res = fl; fl = fl->Next; // if there are other elements in Pending list, add them to main free list FreeList(fl); - AtomicAdd(PendingToFreeListCounter, 1); + AtomicAdd(PendingToFreeListCounter, 1); AtomicAdd(AllocCount, -1); return res; } @@ -1308,7 +1308,7 @@ static void AllocThreadInfo() { struct TAllocHeader { uint64_t Size; - int Tag; + int Tag; int Cookie; }; @@ -1331,7 +1331,7 @@ static inline TAllocHeader* GetAllocHeader(void* p) { PERTHREAD int AllocationTag; extern "C" int SetThreadAllocTag(int tag) { int prevTag = AllocationTag; - if (tag < DBG_ALLOC_MAX_TAG && tag >= 0) { + if (tag < DBG_ALLOC_MAX_TAG && tag >= 0) { AllocationTag = tag; } return prevTag; @@ -1417,7 +1417,7 @@ static inline void SampleDeallocation(TAllocHeader* p, int sizeIdx) { } static inline void TrackPerTagAllocation(TAllocHeader* p, int sizeIdx) { - if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { + if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { Y_ASSERT_NOBT(sizeIdx < DBG_ALLOC_NUM_SIZES); auto& global = GlobalPerTagAllocCounters[p->Tag][sizeIdx]; @@ -1432,7 +1432,7 @@ static inline void TrackPerTagAllocation(TAllocHeader* p, int sizeIdx) { } static inline void TrackPerTagDeallocation(TAllocHeader* p, int sizeIdx) { - if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { + if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { Y_ASSERT_NOBT(sizeIdx < DBG_ALLOC_NUM_SIZES); auto& global = GlobalPerTagAllocCounters[p->Tag][sizeIdx]; @@ -1609,10 +1609,10 @@ static Y_FORCE_INLINE void LFFree(void* p) { return; } -#if defined(LFALLOC_DBG) - TrackDeallocation(p, nSizeIdx); -#endif - +#if defined(LFALLOC_DBG) + TrackDeallocation(p, nSizeIdx); +#endif + #ifdef DBG_FILL_MEMORY memset(p, 0xfe, nSizeIdxToSize[nSizeIdx]); #endif diff --git a/library/cpp/sse/sse.h b/library/cpp/sse/sse.h index 19bac17de0..918a942803 100644 --- a/library/cpp/sse/sse.h +++ b/library/cpp/sse/sse.h @@ -1,18 +1,18 @@ -#pragma once - -/* - The header chooses appropriate SSE support. - On Intel: SSE intrinsics - On ARM64: translation to NEON intrinsics or software emulation +#pragma once + +/* + The header chooses appropriate SSE support. + On Intel: SSE intrinsics + On ARM64: translation to NEON intrinsics or software emulation On PowerPc: translation to Altivec intrinsics or software emulation -*/ +*/ /* Author: Vitaliy Manushkin <agri@yandex-team.ru>, Danila Kutenin <danlark@yandex-team.ru> */ - -#include <util/system/platform.h> + +#include <util/system/platform.h> #if (defined(_i386_) || defined(_x86_64_)) && defined(_sse_) -#include <xmmintrin.h> -#include <emmintrin.h> +#include <xmmintrin.h> +#include <emmintrin.h> #include <pmmintrin.h> #define ARCADIA_SSE #if defined(_ssse3_) @@ -24,10 +24,10 @@ #if defined(_sse4_2_) #include <nmmintrin.h> #endif -#elif defined(_arm64_) -#include "sse2neon.h" +#elif defined(_arm64_) +#include "sse2neon.h" #define ARCADIA_SSE #elif defined(_ppc64_) #include "powerpc.h" #define ARCADIA_SSE -#endif +#endif diff --git a/library/cpp/sse/sse2neon.h b/library/cpp/sse/sse2neon.h index 695dbd3041..af7f3ed242 100644 --- a/library/cpp/sse/sse2neon.h +++ b/library/cpp/sse/sse2neon.h @@ -1,60 +1,60 @@ -#pragma once - -/* - The header contains inlining code - which translates SSE intrinsics to NEON intrinsics or software emulation. - You are encouraged for commitments. - Add missing intrinsics, add unittests, purify the implementation, - merge and simplify templates. - Warning: The code is made in deep nights, so it surely contains bugs, - imperfections, flaws and all other kinds of errors and mistakes. -*/ -/* Author: Vitaliy Manushkin <agri@yandex-team.ru> */ - -#include <util/system/platform.h> +#pragma once + +/* + The header contains inlining code + which translates SSE intrinsics to NEON intrinsics or software emulation. + You are encouraged for commitments. + Add missing intrinsics, add unittests, purify the implementation, + merge and simplify templates. + Warning: The code is made in deep nights, so it surely contains bugs, + imperfections, flaws and all other kinds of errors and mistakes. +*/ +/* Author: Vitaliy Manushkin <agri@yandex-team.ru> */ + +#include <util/system/platform.h> #include <util/system/compiler.h> -#include <util/system/types.h> - -#if !defined(_arm64_) -#error "This header is for ARM64 (aarch64) platform only. " \ +#include <util/system/types.h> + +#if !defined(_arm64_) +#error "This header is for ARM64 (aarch64) platform only. " \ "Include sse.h instead of including this header directly." -#endif - -#include <arm_neon.h> - -union __m128i { - uint64x2_t AsUi64x2; - int64x2_t AsSi64x2; +#endif + +#include <arm_neon.h> + +union __m128i { + uint64x2_t AsUi64x2; + int64x2_t AsSi64x2; + + uint32x4_t AsUi32x4; + int32x4_t AsSi32x4; + + uint16x8_t AsUi16x8; + int16x8_t AsSi16x8; + + uint8x16_t AsUi8x16; + int8x16_t AsSi8x16; + + float32x4_t AsFloat32x4; + float64x2_t AsFloat64x2; +}; + +union __m128 { + float32x4_t AsFloat32x4; + float64x2_t AsFloat64x2; uint32x4_t AsUi32x4; int32x4_t AsSi32x4; - uint16x8_t AsUi16x8; - int16x8_t AsSi16x8; + uint64x2_t AsUi64x2; + int64x2_t AsSi64x2; - uint8x16_t AsUi8x16; - int8x16_t AsSi8x16; - - float32x4_t AsFloat32x4; - float64x2_t AsFloat64x2; -}; - -union __m128 { - float32x4_t AsFloat32x4; - float64x2_t AsFloat64x2; - - uint32x4_t AsUi32x4; - int32x4_t AsSi32x4; - - uint64x2_t AsUi64x2; - int64x2_t AsSi64x2; - - uint8x16_t AsUi8x16; + uint8x16_t AsUi8x16; int8x16_t AsSi8x16; __m128i As128i; -}; - +}; + typedef float64x2_t __m128d; enum _mm_hint @@ -72,128 +72,128 @@ Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) { __builtin_prefetch(p); } -template <typename TType> -struct TQType; - -template <> -struct TQType<uint8x16_t> { - static inline uint8x16_t& As(__m128i& value) { - return value.AsUi8x16; - } - static inline const uint8x16_t& As(const __m128i& value) { - return value.AsUi8x16; - } -}; - -template <> -struct TQType<int8x16_t> { - static inline int8x16_t& As(__m128i& value) { - return value.AsSi8x16; - } - static inline const int8x16_t& As(const __m128i& value) { - return value.AsSi8x16; - } -}; - -template <> -struct TQType<uint16x8_t> { - static inline uint16x8_t& As(__m128i& value) { - return value.AsUi16x8; - } - static inline const uint16x8_t& As(const __m128i& value) { - return value.AsUi16x8; - } -}; - -template <> -struct TQType<int16x8_t> { - static inline int16x8_t& As(__m128i& value) { - return value.AsSi16x8; - } - static inline const int16x8_t& As(const __m128i& value) { - return value.AsSi16x8; - } -}; - -template <> -struct TQType<uint32x4_t> { - static inline uint32x4_t& As(__m128i& value) { - return value.AsUi32x4; - } - static inline const uint32x4_t& As(const __m128i& value) { - return value.AsUi32x4; - } -}; - -template <> -struct TQType<int32x4_t> { - static inline int32x4_t& As(__m128i& value) { - return value.AsSi32x4; - } - static inline const int32x4_t& As(const __m128i& value) { - return value.AsSi32x4; - } -}; - -template <> -struct TQType<uint64x2_t> { - static inline uint64x2_t& As(__m128i& value) { - return value.AsUi64x2; - } - static inline const uint64x2_t& As(const __m128i& value) { - return value.AsUi64x2; - } - static inline uint64x2_t& As(__m128& value) { - return value.AsUi64x2; - } - static inline const uint64x2_t& As(const __m128& value) { - return value.AsUi64x2; - } -}; - -template <> -struct TQType<int64x2_t> { - static inline int64x2_t& As(__m128i& value) { - return value.AsSi64x2; - } - static inline const int64x2_t& As(const __m128i& value) { - return value.AsSi64x2; - } -}; - -template <typename TValue> -struct TBaseWrapper { - TValue Value; - +template <typename TType> +struct TQType; + +template <> +struct TQType<uint8x16_t> { + static inline uint8x16_t& As(__m128i& value) { + return value.AsUi8x16; + } + static inline const uint8x16_t& As(const __m128i& value) { + return value.AsUi8x16; + } +}; + +template <> +struct TQType<int8x16_t> { + static inline int8x16_t& As(__m128i& value) { + return value.AsSi8x16; + } + static inline const int8x16_t& As(const __m128i& value) { + return value.AsSi8x16; + } +}; + +template <> +struct TQType<uint16x8_t> { + static inline uint16x8_t& As(__m128i& value) { + return value.AsUi16x8; + } + static inline const uint16x8_t& As(const __m128i& value) { + return value.AsUi16x8; + } +}; + +template <> +struct TQType<int16x8_t> { + static inline int16x8_t& As(__m128i& value) { + return value.AsSi16x8; + } + static inline const int16x8_t& As(const __m128i& value) { + return value.AsSi16x8; + } +}; + +template <> +struct TQType<uint32x4_t> { + static inline uint32x4_t& As(__m128i& value) { + return value.AsUi32x4; + } + static inline const uint32x4_t& As(const __m128i& value) { + return value.AsUi32x4; + } +}; + +template <> +struct TQType<int32x4_t> { + static inline int32x4_t& As(__m128i& value) { + return value.AsSi32x4; + } + static inline const int32x4_t& As(const __m128i& value) { + return value.AsSi32x4; + } +}; + +template <> +struct TQType<uint64x2_t> { + static inline uint64x2_t& As(__m128i& value) { + return value.AsUi64x2; + } + static inline const uint64x2_t& As(const __m128i& value) { + return value.AsUi64x2; + } + static inline uint64x2_t& As(__m128& value) { + return value.AsUi64x2; + } + static inline const uint64x2_t& As(const __m128& value) { + return value.AsUi64x2; + } +}; + +template <> +struct TQType<int64x2_t> { + static inline int64x2_t& As(__m128i& value) { + return value.AsSi64x2; + } + static inline const int64x2_t& As(const __m128i& value) { + return value.AsSi64x2; + } +}; + +template <typename TValue> +struct TBaseWrapper { + TValue Value; + Y_FORCE_INLINE - operator TValue&() { - return Value; - } - + operator TValue&() { + return Value; + } + Y_FORCE_INLINE - operator const TValue&() const { - return Value; - } -}; - -template <typename TOp, typename TFunc, TFunc* func, - typename TDup, TDup* dupfunc> -struct TWrapperSingleDup: public TBaseWrapper<__m128i> { + operator const TValue&() const { + return Value; + } +}; + +template <typename TOp, typename TFunc, TFunc* func, + typename TDup, TDup* dupfunc> +struct TWrapperSingleDup: public TBaseWrapper<__m128i> { Y_FORCE_INLINE - TWrapperSingleDup(const __m128i& op, const int shift) { - TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(shift)); - } -}; - -template <typename TOp, typename TFunc, TFunc* func, - typename TDup, TDup* dupfunc> -struct TWrapperSingleNegDup: public TBaseWrapper<__m128i> { + TWrapperSingleDup(const __m128i& op, const int shift) { + TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(shift)); + } +}; + +template <typename TOp, typename TFunc, TFunc* func, + typename TDup, TDup* dupfunc> +struct TWrapperSingleNegDup: public TBaseWrapper<__m128i> { Y_FORCE_INLINE - TWrapperSingleNegDup(const __m128i& op, const int shift) { - TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(-shift)); - } -}; - + TWrapperSingleNegDup(const __m128i& op, const int shift) { + TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(-shift)); + } +}; + inline __m128i _mm_srl_epi16(__m128i a, __m128i count) { __m128i res; res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(-count.AsUi16x8[0])); @@ -225,16 +225,16 @@ inline __m128i _mm_srai_epi32(__m128i a, int count) { return res; } -using _mm_srli_epi16 = - TWrapperSingleNegDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, - decltype(vdupq_n_s16), vdupq_n_s16>; -using _mm_srli_epi32 = - TWrapperSingleNegDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32, - decltype(vdupq_n_s32), vdupq_n_s32>; -using _mm_srli_epi64 = - TWrapperSingleNegDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64, - decltype(vdupq_n_s64), vdupq_n_s64>; - +using _mm_srli_epi16 = + TWrapperSingleNegDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, + decltype(vdupq_n_s16), vdupq_n_s16>; +using _mm_srli_epi32 = + TWrapperSingleNegDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32, + decltype(vdupq_n_s32), vdupq_n_s32>; +using _mm_srli_epi64 = + TWrapperSingleNegDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64, + decltype(vdupq_n_s64), vdupq_n_s64>; + inline __m128i _mm_sll_epi16(__m128i a, __m128i count) { __m128i res; @@ -255,57 +255,57 @@ inline __m128i _mm_sll_epi64(__m128i a, __m128i count) { return res; } -using _mm_slli_epi16 = - TWrapperSingleDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, - decltype(vdupq_n_s16), vdupq_n_s16>; -using _mm_slli_epi32 = - TWrapperSingleDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32, - decltype(vdupq_n_s32), vdupq_n_s32>; -using _mm_slli_epi64 = - TWrapperSingleDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64, - decltype(vdupq_n_s64), vdupq_n_s64>; - -template <typename TOp, typename TFunc, TFunc* func, typename... TParams> -struct TWrapperDual : TBaseWrapper<__m128i> { +using _mm_slli_epi16 = + TWrapperSingleDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, + decltype(vdupq_n_s16), vdupq_n_s16>; +using _mm_slli_epi32 = + TWrapperSingleDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32, + decltype(vdupq_n_s32), vdupq_n_s32>; +using _mm_slli_epi64 = + TWrapperSingleDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64, + decltype(vdupq_n_s64), vdupq_n_s64>; + +template <typename TOp, typename TFunc, TFunc* func, typename... TParams> +struct TWrapperDual : TBaseWrapper<__m128i> { Y_FORCE_INLINE - TWrapperDual(const __m128i& op1, const __m128i& op2, TParams... params) { - TQType<TOp>::As(Value) = (TOp) - func(TQType<TOp>::As(op1), - TQType<TOp>::As(op2), - params...); - } -}; - -template <typename TOp, typename TFunc, TFunc* func, typename... TParams> -struct TWrapperDualSwap : TBaseWrapper<__m128i> { + TWrapperDual(const __m128i& op1, const __m128i& op2, TParams... params) { + TQType<TOp>::As(Value) = (TOp) + func(TQType<TOp>::As(op1), + TQType<TOp>::As(op2), + params...); + } +}; + +template <typename TOp, typename TFunc, TFunc* func, typename... TParams> +struct TWrapperDualSwap : TBaseWrapper<__m128i> { Y_FORCE_INLINE - TWrapperDualSwap(const __m128i& op1, const __m128i& op2, TParams... params) { - TQType<TOp>::As(Value) = - func(TQType<TOp>::As(op2), - TQType<TOp>::As(op1), - params...); - } -}; - + TWrapperDualSwap(const __m128i& op1, const __m128i& op2, TParams... params) { + TQType<TOp>::As(Value) = + func(TQType<TOp>::As(op2), + TQType<TOp>::As(op1), + params...); + } +}; + template <typename TOp, typename TFunc, TFunc* func, typename TArgument = __m128> struct TWrapperDualF : TBaseWrapper<TArgument> { Y_FORCE_INLINE TWrapperDualF(const TArgument& op1, const TArgument& op2) { TQType<TOp>::As(TBaseWrapper<TArgument>::Value) = (TOp) func(TQType<TOp>::As(op1), TQType<TOp>::As(op2)); - } -}; - -using _mm_or_si128 = TWrapperDual<uint64x2_t, decltype(vorrq_u64), vorrq_u64>; -using _mm_and_si128 = TWrapperDual<uint64x2_t, decltype(vandq_u64), vandq_u64>; -using _mm_andnot_si128 = - TWrapperDualSwap<uint64x2_t, decltype(vbicq_u64), vbicq_u64>; + } +}; + +using _mm_or_si128 = TWrapperDual<uint64x2_t, decltype(vorrq_u64), vorrq_u64>; +using _mm_and_si128 = TWrapperDual<uint64x2_t, decltype(vandq_u64), vandq_u64>; +using _mm_andnot_si128 = + TWrapperDualSwap<uint64x2_t, decltype(vbicq_u64), vbicq_u64>; using _mm_xor_si128 = TWrapperDual<uint64x2_t, decltype(veorq_u64), veorq_u64>; - + using _mm_add_epi8 = TWrapperDual<uint8x16_t, decltype(vaddq_u8), vaddq_u8>; -using _mm_add_epi16 = TWrapperDual<uint16x8_t, decltype(vaddq_u16), vaddq_u16>; -using _mm_add_epi32 = TWrapperDual<uint32x4_t, decltype(vaddq_u32), vaddq_u32>; -using _mm_add_epi64 = TWrapperDual<uint64x2_t, decltype(vaddq_u64), vaddq_u64>; - +using _mm_add_epi16 = TWrapperDual<uint16x8_t, decltype(vaddq_u16), vaddq_u16>; +using _mm_add_epi32 = TWrapperDual<uint32x4_t, decltype(vaddq_u32), vaddq_u32>; +using _mm_add_epi64 = TWrapperDual<uint64x2_t, decltype(vaddq_u64), vaddq_u64>; + inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { int32x4_t aLow; int32x4_t aHigh; @@ -343,118 +343,118 @@ inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { } using _mm_sub_epi8 = TWrapperDual<uint8x16_t, decltype(vsubq_u8), vsubq_u8>; -using _mm_sub_epi16 = TWrapperDual<uint16x8_t, decltype(vsubq_u16), vsubq_u16>; -using _mm_sub_epi32 = TWrapperDual<uint32x4_t, decltype(vsubq_u32), vsubq_u32>; -using _mm_sub_epi64 = TWrapperDual<uint64x2_t, decltype(vsubq_u64), vsubq_u64>; - -using _mm_unpacklo_epi8 = - TWrapperDual<uint8x16_t, decltype(vzip1q_u8), vzip1q_u8>; -using _mm_unpackhi_epi8 = - TWrapperDual<uint8x16_t, decltype(vzip2q_u8), vzip2q_u8>; -using _mm_unpacklo_epi16 = - TWrapperDual<uint16x8_t, decltype(vzip1q_u16), vzip1q_u16>; -using _mm_unpackhi_epi16 = - TWrapperDual<uint16x8_t, decltype(vzip2q_u16), vzip2q_u16>; -using _mm_unpacklo_epi32 = - TWrapperDual<uint32x4_t, decltype(vzip1q_u32), vzip1q_u32>; -using _mm_unpackhi_epi32 = - TWrapperDual<uint32x4_t, decltype(vzip2q_u32), vzip2q_u32>; -using _mm_unpacklo_epi64 = - TWrapperDual<uint64x2_t, decltype(vzip1q_u64), vzip1q_u64>; -using _mm_unpackhi_epi64 = - TWrapperDual<uint64x2_t, decltype(vzip2q_u64), vzip2q_u64>; - -using _mm_cmpeq_epi8 = - TWrapperDual<uint8x16_t, decltype(vceqq_u8), vceqq_u8>; -using _mm_cmpeq_epi16 = - TWrapperDual<uint16x8_t, decltype(vceqq_u16), vceqq_u16>; -using _mm_cmpeq_epi32 = - TWrapperDual<uint32x4_t, decltype(vceqq_u32), vceqq_u32>; - -using _mm_cmpgt_epi8 = - TWrapperDual<int8x16_t, decltype(vcgtq_s8), vcgtq_s8>; -using _mm_cmpgt_epi16 = - TWrapperDual<int16x8_t, decltype(vcgtq_s16), vcgtq_s16>; -using _mm_cmpgt_epi32 = - TWrapperDual<int32x4_t, decltype(vcgtq_s32), vcgtq_s32>; - -using _mm_cmplt_epi8 = - TWrapperDual<int8x16_t, decltype(vcltq_s8), vcltq_s8>; -using _mm_cmplt_epi16 = - TWrapperDual<int16x8_t, decltype(vcltq_s16), vcltq_s16>; -using _mm_cmplt_epi32 = - TWrapperDual<int32x4_t, decltype(vcltq_s32), vcltq_s32>; - +using _mm_sub_epi16 = TWrapperDual<uint16x8_t, decltype(vsubq_u16), vsubq_u16>; +using _mm_sub_epi32 = TWrapperDual<uint32x4_t, decltype(vsubq_u32), vsubq_u32>; +using _mm_sub_epi64 = TWrapperDual<uint64x2_t, decltype(vsubq_u64), vsubq_u64>; + +using _mm_unpacklo_epi8 = + TWrapperDual<uint8x16_t, decltype(vzip1q_u8), vzip1q_u8>; +using _mm_unpackhi_epi8 = + TWrapperDual<uint8x16_t, decltype(vzip2q_u8), vzip2q_u8>; +using _mm_unpacklo_epi16 = + TWrapperDual<uint16x8_t, decltype(vzip1q_u16), vzip1q_u16>; +using _mm_unpackhi_epi16 = + TWrapperDual<uint16x8_t, decltype(vzip2q_u16), vzip2q_u16>; +using _mm_unpacklo_epi32 = + TWrapperDual<uint32x4_t, decltype(vzip1q_u32), vzip1q_u32>; +using _mm_unpackhi_epi32 = + TWrapperDual<uint32x4_t, decltype(vzip2q_u32), vzip2q_u32>; +using _mm_unpacklo_epi64 = + TWrapperDual<uint64x2_t, decltype(vzip1q_u64), vzip1q_u64>; +using _mm_unpackhi_epi64 = + TWrapperDual<uint64x2_t, decltype(vzip2q_u64), vzip2q_u64>; + +using _mm_cmpeq_epi8 = + TWrapperDual<uint8x16_t, decltype(vceqq_u8), vceqq_u8>; +using _mm_cmpeq_epi16 = + TWrapperDual<uint16x8_t, decltype(vceqq_u16), vceqq_u16>; +using _mm_cmpeq_epi32 = + TWrapperDual<uint32x4_t, decltype(vceqq_u32), vceqq_u32>; + +using _mm_cmpgt_epi8 = + TWrapperDual<int8x16_t, decltype(vcgtq_s8), vcgtq_s8>; +using _mm_cmpgt_epi16 = + TWrapperDual<int16x8_t, decltype(vcgtq_s16), vcgtq_s16>; +using _mm_cmpgt_epi32 = + TWrapperDual<int32x4_t, decltype(vcgtq_s32), vcgtq_s32>; + +using _mm_cmplt_epi8 = + TWrapperDual<int8x16_t, decltype(vcltq_s8), vcltq_s8>; +using _mm_cmplt_epi16 = + TWrapperDual<int16x8_t, decltype(vcltq_s16), vcltq_s16>; +using _mm_cmplt_epi32 = + TWrapperDual<int32x4_t, decltype(vcltq_s32), vcltq_s32>; + Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* ptr) { - __m128i result; + __m128i result; result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); - return result; -} - + return result; +} + Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* ptr) { - __m128i result; + __m128i result; result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); - return result; -} - + return result; +} + Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* ptr) { return _mm_loadu_si128(ptr); } Y_FORCE_INLINE void _mm_storeu_si128(__m128i* ptr, const __m128i& op) { vst1q_u64((uint64_t*)ptr, op.AsUi64x2); -} - +} + Y_FORCE_INLINE void -_mm_store_si128(__m128i* ptr, const __m128i& op) { +_mm_store_si128(__m128i* ptr, const __m128i& op) { vst1q_u64((uint64_t*)ptr, op.AsUi64x2); -} - -template <typename TOp, typename TFunc, TFunc* func, typename... TParams> -struct TWrapperSimple : TBaseWrapper<__m128i> { +} + +template <typename TOp, typename TFunc, TFunc* func, typename... TParams> +struct TWrapperSimple : TBaseWrapper<__m128i> { Y_FORCE_INLINE - TWrapperSimple(TParams... params) { - TQType<TOp>::As(Value) = func(params...); - } -}; - -template <typename TOp, typename TFunc, TFunc* func, typename... TParams> -struct TWrapperSimpleF : TBaseWrapper<__m128> { + TWrapperSimple(TParams... params) { + TQType<TOp>::As(Value) = func(params...); + } +}; + +template <typename TOp, typename TFunc, TFunc* func, typename... TParams> +struct TWrapperSimpleF : TBaseWrapper<__m128> { Y_FORCE_INLINE - TWrapperSimpleF(TParams... params) { - TQType<TOp>::As(Value) = func(params...); - } -}; - -using _mm_set1_epi8 = - TWrapperSimple<int8x16_t, decltype(vdupq_n_s8), vdupq_n_s8, const char>; -using _mm_set1_epi16 = - TWrapperSimple<int16x8_t, decltype(vdupq_n_s16), vdupq_n_s16, const ui16>; -using _mm_set1_epi32 = - TWrapperSimple<int32x4_t, decltype(vdupq_n_s32), vdupq_n_s32, const ui32>; - -struct _mm_setzero_si128 : TBaseWrapper<__m128i> { + TWrapperSimpleF(TParams... params) { + TQType<TOp>::As(Value) = func(params...); + } +}; + +using _mm_set1_epi8 = + TWrapperSimple<int8x16_t, decltype(vdupq_n_s8), vdupq_n_s8, const char>; +using _mm_set1_epi16 = + TWrapperSimple<int16x8_t, decltype(vdupq_n_s16), vdupq_n_s16, const ui16>; +using _mm_set1_epi32 = + TWrapperSimple<int32x4_t, decltype(vdupq_n_s32), vdupq_n_s32, const ui32>; + +struct _mm_setzero_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_setzero_si128() { - TQType<uint64x2_t>::As(Value) = vdupq_n_u64(0); - } -}; - -struct _mm_loadl_epi64 : TBaseWrapper<__m128i> { + _mm_setzero_si128() { + TQType<uint64x2_t>::As(Value) = vdupq_n_u64(0); + } +}; + +struct _mm_loadl_epi64 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_loadl_epi64(const __m128i* p) { + _mm_loadl_epi64(const __m128i* p) { uint64x1_t im = vld1_u64((const uint64_t*)p); - TQType<uint64x2_t>::As(Value) = vcombine_u64(im, vdup_n_u64(0)); - } -}; - -struct _mm_storel_epi64 : TBaseWrapper<__m128i> { + TQType<uint64x2_t>::As(Value) = vcombine_u64(im, vdup_n_u64(0)); + } +}; + +struct _mm_storel_epi64 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_storel_epi64(__m128i* a, __m128i op) { + _mm_storel_epi64(__m128i* a, __m128i op) { vst1_u64((uint64_t*)a, vget_low_u64(op.AsUi64x2)); - } -}; - + } +}; + struct ShuffleStruct4 { ui8 x[4]; }; @@ -470,45 +470,45 @@ _MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) { } Y_FORCE_INLINE __m128i -_mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { - __m128i result; +_mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { + __m128i result; const ui8 xi[4] = { ui8(op2.x[0] * 4), ui8(op2.x[1] * 4), ui8(op2.x[2] * 4), ui8(op2.x[3] * 4) }; const uint8x16_t transform = { - ui8(xi[0]), ui8(xi[0] + 1), ui8(xi[0] + 2), ui8(xi[0] + 3), - ui8(xi[1]), ui8(xi[1] + 1), ui8(xi[1] + 2), ui8(xi[1] + 3), - ui8(xi[2]), ui8(xi[2] + 1), ui8(xi[2] + 2), ui8(xi[2] + 3), + ui8(xi[0]), ui8(xi[0] + 1), ui8(xi[0] + 2), ui8(xi[0] + 3), + ui8(xi[1]), ui8(xi[1] + 1), ui8(xi[1] + 2), ui8(xi[1] + 3), + ui8(xi[2]), ui8(xi[2] + 1), ui8(xi[2] + 2), ui8(xi[2] + 3), ui8(xi[3]), ui8(xi[3] + 1), ui8(xi[3] + 2), ui8(xi[3] + 3) }; - result.AsUi8x16 = vqtbl1q_u8(op1.AsUi8x16, transform); - return result; -} - + result.AsUi8x16 = vqtbl1q_u8(op1.AsUi8x16, transform); + return result; +} + Y_FORCE_INLINE int -_mm_movemask_epi8(const __m128i& op) { - uint8x16_t mask = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; - uint8x16_t opmasked = vandq_u8(op.AsUi8x16, mask); - int8x16_t byteshifter = { - 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7}; - uint8x16_t opshifted = vshlq_u8(opmasked, byteshifter); - int16x8_t wordshifter = {-7, -5, -3, -1, 1, 3, 5, 7}; - uint16x8_t wordshifted = - vshlq_u16(vreinterpretq_u16_u8(opshifted), wordshifter); - return vaddvq_u16(wordshifted); -} - -template <int imm> -struct THelper_mm_srli_si128 : TBaseWrapper<__m128i> { +_mm_movemask_epi8(const __m128i& op) { + uint8x16_t mask = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + uint8x16_t opmasked = vandq_u8(op.AsUi8x16, mask); + int8x16_t byteshifter = { + 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7}; + uint8x16_t opshifted = vshlq_u8(opmasked, byteshifter); + int16x8_t wordshifter = {-7, -5, -3, -1, 1, 3, 5, 7}; + uint16x8_t wordshifted = + vshlq_u16(vreinterpretq_u16_u8(opshifted), wordshifter); + return vaddvq_u16(wordshifted); +} + +template <int imm> +struct THelper_mm_srli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE THelper_mm_srli_si128(const __m128i a) { const auto zero = vdupq_n_u8(0); - TQType<uint8x16_t>::As(Value) = vextq_u8(a.AsUi8x16, zero, imm); - } -}; - + TQType<uint8x16_t>::As(Value) = vextq_u8(a.AsUi8x16, zero, imm); + } +}; + template <> struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> { Y_FORCE_INLINE @@ -518,8 +518,8 @@ struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> { } }; -#define _mm_srli_si128(a, imm) THelper_mm_srli_si128<imm>(a) - +#define _mm_srli_si128(a, imm) THelper_mm_srli_si128<imm>(a) + template<int imm> inline uint8x16_t vextq_u8_function(uint8x16_t a, uint8x16_t b) { return vextq_u8(a, b, imm); @@ -531,33 +531,33 @@ inline uint8x16_t vextq_u8_function<16>(uint8x16_t /* a */, uint8x16_t b) { } -template <int imm> -struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> { +template <int imm> +struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE THelper_mm_slli_si128(const __m128i a) { - auto zero = vdupq_n_u8(0); + auto zero = vdupq_n_u8(0); TQType<uint8x16_t>::As(Value) = vextq_u8_function<16 - imm>(zero, a.AsUi8x16); - } -}; - -#define _mm_slli_si128(a, imm) THelper_mm_slli_si128<imm>(a) - + } +}; + +#define _mm_slli_si128(a, imm) THelper_mm_slli_si128<imm>(a) + Y_FORCE_INLINE int _mm_cvtsi128_si32(const __m128i& op) { - return vgetq_lane_s32(op.AsSi32x4, 0); -} - -struct _mm_set_epi16 : TBaseWrapper<__m128i> { + return vgetq_lane_s32(op.AsSi32x4, 0); +} + +struct _mm_set_epi16 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_set_epi16(const short w7, const short w6, - const short w5, const short w4, - const short w3, const short w2, - const short w1, const short w0) { - int16x4_t d0 = {w0, w1, w2, w3}; - int16x4_t d1 = {w4, w5, w6, w7}; - TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1); - } -}; - + _mm_set_epi16(const short w7, const short w6, + const short w5, const short w4, + const short w3, const short w2, + const short w1, const short w0) { + int16x4_t d0 = {w0, w1, w2, w3}; + int16x4_t d1 = {w4, w5, w6, w7}; + TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1); + } +}; + struct _mm_setr_epi16 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_setr_epi16(const short w7, const short w6, @@ -570,16 +570,16 @@ struct _mm_setr_epi16 : TBaseWrapper<__m128i> { } }; -struct _mm_set_epi32 : TBaseWrapper<__m128i> { +struct _mm_set_epi32 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_set_epi32(const int x3, const int x2, - const int x1, const int x0) { - int32x2_t d0 = {x0, x1}; - int32x2_t d1 = {x2, x3}; - TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1); - } -}; - + _mm_set_epi32(const int x3, const int x2, + const int x1, const int x0) { + int32x2_t d0 = {x0, x1}; + int32x2_t d1 = {x2, x3}; + TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1); + } +}; + struct _mm_setr_epi32 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_setr_epi32(const int x3, const int x2, @@ -590,14 +590,14 @@ struct _mm_setr_epi32 : TBaseWrapper<__m128i> { } }; -struct _mm_cvtsi32_si128 : TBaseWrapper<__m128i> { +struct _mm_cvtsi32_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - _mm_cvtsi32_si128(int op) { - auto zero = vdupq_n_s32(0); - TQType<int32x4_t>::As(Value) = vsetq_lane_s32(op, zero, 0); - } -}; - + _mm_cvtsi32_si128(int op) { + auto zero = vdupq_n_s32(0); + TQType<int32x4_t>::As(Value) = vsetq_lane_s32(op, zero, 0); + } +}; + struct _mm_cvtsi64_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_cvtsi64_si128(i64 op) { @@ -606,41 +606,41 @@ struct _mm_cvtsi64_si128 : TBaseWrapper<__m128i> { } }; -template <typename TOpOut, typename TOpIn, - typename TFunc, TFunc* func, - typename TCombine, TCombine* combine> -struct TCombineWrapper : TBaseWrapper<__m128i> { +template <typename TOpOut, typename TOpIn, + typename TFunc, TFunc* func, + typename TCombine, TCombine* combine> +struct TCombineWrapper : TBaseWrapper<__m128i> { Y_FORCE_INLINE - TCombineWrapper(const __m128i op1, const __m128i op2) { - TQType<TOpOut>::As(Value) = - combine(func(TQType<TOpIn>::As(op1)), - func(TQType<TOpIn>::As(op2))); - } -}; - -using _mm_packs_epi16 = - TCombineWrapper<int8x16_t, int16x8_t, - decltype(vqmovn_s16), vqmovn_s16, - decltype(vcombine_s8), vcombine_s8>; -using _mm_packs_epi32 = - TCombineWrapper<int16x8_t, int32x4_t, - decltype(vqmovn_s32), vqmovn_s32, - decltype(vcombine_s16), vcombine_s16>; -using _mm_packus_epi16 = - TCombineWrapper<uint8x16_t, int16x8_t, - decltype(vqmovun_s16), vqmovun_s16, - decltype(vcombine_u8), vcombine_u8>; - -template <typename TOpOut, typename TOpIn, - typename TFunc, TFunc* func, typename... TParams> -struct TScalarOutWrapper : TBaseWrapper<TOpOut> { + TCombineWrapper(const __m128i op1, const __m128i op2) { + TQType<TOpOut>::As(Value) = + combine(func(TQType<TOpIn>::As(op1)), + func(TQType<TOpIn>::As(op2))); + } +}; + +using _mm_packs_epi16 = + TCombineWrapper<int8x16_t, int16x8_t, + decltype(vqmovn_s16), vqmovn_s16, + decltype(vcombine_s8), vcombine_s8>; +using _mm_packs_epi32 = + TCombineWrapper<int16x8_t, int32x4_t, + decltype(vqmovn_s32), vqmovn_s32, + decltype(vcombine_s16), vcombine_s16>; +using _mm_packus_epi16 = + TCombineWrapper<uint8x16_t, int16x8_t, + decltype(vqmovun_s16), vqmovun_s16, + decltype(vcombine_u8), vcombine_u8>; + +template <typename TOpOut, typename TOpIn, + typename TFunc, TFunc* func, typename... TParams> +struct TScalarOutWrapper : TBaseWrapper<TOpOut> { Y_FORCE_INLINE - TScalarOutWrapper(const __m128i op, TParams... params) { - TBaseWrapper<TOpOut>::Value = - func(TQType<TOpIn>::As(op), params...); - } -}; - + TScalarOutWrapper(const __m128i op, TParams... params) { + TBaseWrapper<TOpOut>::Value = + func(TQType<TOpIn>::As(op), params...); + } +}; + template<int imm> int extract_epi8_arm(__m128i arg) { return vgetq_lane_u8(arg.AsUi8x16, imm); @@ -649,13 +649,13 @@ int extract_epi8_arm(__m128i arg) { template<int imm> int extract_epi16_arm(__m128i arg) { return vgetq_lane_u16(arg.AsUi16x8, imm); -} - +} + template<int imm> int extract_epi32_arm(__m128i arg) { return vgetq_lane_s32(arg.AsSi32x4, imm); } - + template<int imm> long long extract_epi64_arm(__m128i arg) { return vgetq_lane_s64(arg.AsSi64x2, imm); @@ -669,49 +669,49 @@ long long extract_epi64_arm(__m128i arg) { static Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i op1, __m128i op2) { - __m128i result; - uint32x4_t r1 = vuzp1q_u32(op1.AsUi32x4, op2.AsUi32x4); - uint32x4_t r2 = vuzp1q_u32(op2.AsUi32x4, op1.AsUi32x4); - result.AsUi64x2 = vmull_u32(vget_low_u32(r1), vget_low_u32(r2)); - return result; -} - -template <> -struct TQType<float32x4_t> { - static inline float32x4_t& As(__m128& value) { - return value.AsFloat32x4; - } - - static inline const float32x4_t& As(const __m128& value) { - return value.AsFloat32x4; - } - - static inline float32x4_t& As(__m128i& value) { - return value.AsFloat32x4; - } - - static inline const float32x4_t& As(const __m128i& value) { - return value.AsFloat32x4; - } -}; - -template <> -struct TQType<float64x2_t> { - static inline float64x2_t& As(__m128& value) { - return value.AsFloat64x2; - } - - static inline const float64x2_t& As(const __m128& value) { - return value.AsFloat64x2; - } - - static inline float64x2_t& As(__m128i& value) { - return value.AsFloat64x2; - } - - static inline const float64x2_t& As(const __m128i& value) { - return value.AsFloat64x2; - } + __m128i result; + uint32x4_t r1 = vuzp1q_u32(op1.AsUi32x4, op2.AsUi32x4); + uint32x4_t r2 = vuzp1q_u32(op2.AsUi32x4, op1.AsUi32x4); + result.AsUi64x2 = vmull_u32(vget_low_u32(r1), vget_low_u32(r2)); + return result; +} + +template <> +struct TQType<float32x4_t> { + static inline float32x4_t& As(__m128& value) { + return value.AsFloat32x4; + } + + static inline const float32x4_t& As(const __m128& value) { + return value.AsFloat32x4; + } + + static inline float32x4_t& As(__m128i& value) { + return value.AsFloat32x4; + } + + static inline const float32x4_t& As(const __m128i& value) { + return value.AsFloat32x4; + } +}; + +template <> +struct TQType<float64x2_t> { + static inline float64x2_t& As(__m128& value) { + return value.AsFloat64x2; + } + + static inline const float64x2_t& As(const __m128& value) { + return value.AsFloat64x2; + } + + static inline float64x2_t& As(__m128i& value) { + return value.AsFloat64x2; + } + + static inline const float64x2_t& As(const __m128i& value) { + return value.AsFloat64x2; + } static inline float64x2_t& As(__m128d& value) { return value; @@ -720,30 +720,30 @@ struct TQType<float64x2_t> { static inline const float64x2_t& As(const __m128d& value) { return value; } -}; - -using _mm_set1_ps = TWrapperSimpleF<float32x4_t, - decltype(vdupq_n_f32), vdupq_n_f32, const float>; -using _mm_set_ps1 = TWrapperSimpleF<float32x4_t, - decltype(vdupq_n_f32), vdupq_n_f32, const float>; - -struct _mm_setzero_ps : TBaseWrapper<__m128> { +}; + +using _mm_set1_ps = TWrapperSimpleF<float32x4_t, + decltype(vdupq_n_f32), vdupq_n_f32, const float>; +using _mm_set_ps1 = TWrapperSimpleF<float32x4_t, + decltype(vdupq_n_f32), vdupq_n_f32, const float>; + +struct _mm_setzero_ps : TBaseWrapper<__m128> { Y_FORCE_INLINE - _mm_setzero_ps() { - TQType<float32x4_t>::As(Value) = vdupq_n_f32(0.); - } -}; - + _mm_setzero_ps() { + TQType<float32x4_t>::As(Value) = vdupq_n_f32(0.); + } +}; + Y_FORCE_INLINE __m128d _mm_setzero_pd() { return vdupq_n_f64(0.); } Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) { - __m128 result; - result.AsFloat32x4 = vld1q_f32(ptr); - return result; -} - + __m128 result; + result.AsFloat32x4 = vld1q_f32(ptr); + return result; +} + Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) { __m128 result; result.AsFloat32x4 = vld1q_f32(ptr); @@ -751,23 +751,23 @@ Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) { } Y_FORCE_INLINE void _mm_storeu_ps(float* ptr, const __m128& op) { - vst1q_f32(ptr, op.AsFloat32x4); -} - + vst1q_f32(ptr, op.AsFloat32x4); +} + Y_FORCE_INLINE void _mm_store_ps(float* ptr, const __m128& op) { vst1q_f32(ptr, op.AsFloat32x4); } -struct _mm_set_ps : TBaseWrapper<__m128> { +struct _mm_set_ps : TBaseWrapper<__m128> { Y_FORCE_INLINE - _mm_set_ps(const float x3, const float x2, - const float x1, const float x0) { - float32x2_t d0 = {x0, x1}; - float32x2_t d1 = {x2, x3}; - TQType<float32x4_t>::As(Value) = vcombine_f32(d0, d1); - } -}; - + _mm_set_ps(const float x3, const float x2, + const float x1, const float x0) { + float32x2_t d0 = {x0, x1}; + float32x2_t d1 = {x2, x3}; + TQType<float32x4_t>::As(Value) = vcombine_f32(d0, d1); + } +}; + Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { const float64x1_t p0 = {d0}; const float64x1_t p1 = {d1}; @@ -788,81 +788,81 @@ Y_FORCE_INLINE void _mm_store_pd(double* res, __m128d a) { vst1q_f64(res, a); } -using _mm_add_ps = TWrapperDualF<float32x4_t, decltype(vaddq_f32), vaddq_f32>; -using _mm_sub_ps = TWrapperDualF<float32x4_t, decltype(vsubq_f32), vsubq_f32>; -using _mm_mul_ps = TWrapperDualF<float32x4_t, decltype(vmulq_f32), vmulq_f32>; -using _mm_div_ps = TWrapperDualF<float32x4_t, decltype(vdivq_f32), vdivq_f32>; -using _mm_cmpeq_ps = TWrapperDualF<float32x4_t, decltype(vceqq_f32), vceqq_f32>; -using _mm_cmpgt_ps = TWrapperDualF<float32x4_t, decltype(vcgtq_f32), vcgtq_f32>; -using _mm_max_ps = TWrapperDualF<float32x4_t, decltype(vmaxq_f32), vmaxq_f32>; -using _mm_min_ps = TWrapperDualF<float32x4_t, decltype(vminq_f32), vminq_f32>; - +using _mm_add_ps = TWrapperDualF<float32x4_t, decltype(vaddq_f32), vaddq_f32>; +using _mm_sub_ps = TWrapperDualF<float32x4_t, decltype(vsubq_f32), vsubq_f32>; +using _mm_mul_ps = TWrapperDualF<float32x4_t, decltype(vmulq_f32), vmulq_f32>; +using _mm_div_ps = TWrapperDualF<float32x4_t, decltype(vdivq_f32), vdivq_f32>; +using _mm_cmpeq_ps = TWrapperDualF<float32x4_t, decltype(vceqq_f32), vceqq_f32>; +using _mm_cmpgt_ps = TWrapperDualF<float32x4_t, decltype(vcgtq_f32), vcgtq_f32>; +using _mm_max_ps = TWrapperDualF<float32x4_t, decltype(vmaxq_f32), vmaxq_f32>; +using _mm_min_ps = TWrapperDualF<float32x4_t, decltype(vminq_f32), vminq_f32>; + using _mm_add_pd = TWrapperDualF<float64x2_t, decltype(vaddq_f64), vaddq_f64, __m128d>; using _mm_sub_pd = TWrapperDualF<float64x2_t, decltype(vsubq_f64), vsubq_f64, __m128d>; using _mm_mul_pd = TWrapperDualF<float64x2_t, decltype(vmulq_f64), vmulq_f64, __m128d>; using _mm_div_pd = TWrapperDualF<float64x2_t, decltype(vdivq_f64), vdivq_f64, __m128d>; -struct _mm_and_ps : TBaseWrapper<__m128> { +struct _mm_and_ps : TBaseWrapper<__m128> { Y_FORCE_INLINE - _mm_and_ps(const __m128& op1, const __m128& op2) { - TQType<uint64x2_t>::As(Value) = - vandq_u64(TQType<uint64x2_t>::As(op1), - TQType<uint64x2_t>::As(op2)); - } -}; - + _mm_and_ps(const __m128& op1, const __m128& op2) { + TQType<uint64x2_t>::As(Value) = + vandq_u64(TQType<uint64x2_t>::As(op1), + TQType<uint64x2_t>::As(op2)); + } +}; + Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { return vandq_u64(a, b); } Y_FORCE_INLINE void _MM_TRANSPOSE4_PS(__m128& op0, __m128& op1, __m128& op2, __m128& op3) { - float64x2_t im0 = - (float64x2_t)vtrn1q_f32(op0.AsFloat32x4, op1.AsFloat32x4); - float64x2_t im1 = - (float64x2_t)vtrn2q_f32(op0.AsFloat32x4, op1.AsFloat32x4); - float64x2_t im2 = - (float64x2_t)vtrn1q_f32(op2.AsFloat32x4, op3.AsFloat32x4); - float64x2_t im3 = - (float64x2_t)vtrn2q_f32(op2.AsFloat32x4, op3.AsFloat32x4); - - TQType<float64x2_t>::As(op0) = vtrn1q_f64(im0, im2); - TQType<float64x2_t>::As(op1) = vtrn1q_f64(im1, im3); - TQType<float64x2_t>::As(op2) = vtrn2q_f64(im0, im2); - TQType<float64x2_t>::As(op3) = vtrn2q_f64(im1, im3); -}; - + float64x2_t im0 = + (float64x2_t)vtrn1q_f32(op0.AsFloat32x4, op1.AsFloat32x4); + float64x2_t im1 = + (float64x2_t)vtrn2q_f32(op0.AsFloat32x4, op1.AsFloat32x4); + float64x2_t im2 = + (float64x2_t)vtrn1q_f32(op2.AsFloat32x4, op3.AsFloat32x4); + float64x2_t im3 = + (float64x2_t)vtrn2q_f32(op2.AsFloat32x4, op3.AsFloat32x4); + + TQType<float64x2_t>::As(op0) = vtrn1q_f64(im0, im2); + TQType<float64x2_t>::As(op1) = vtrn1q_f64(im1, im3); + TQType<float64x2_t>::As(op2) = vtrn2q_f64(im0, im2); + TQType<float64x2_t>::As(op3) = vtrn2q_f64(im1, im3); +}; + Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i op) { - return reinterpret_cast<__m128&>(op); -} - + return reinterpret_cast<__m128&>(op); +} + Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 op) { - return reinterpret_cast<__m128i&>(op); -} - -template <typename TOpOut, typename TOpIn, - typename TFunc, TFunc* func, typename... TParams> -struct TCvtS2FWrapperSingle : TBaseWrapper<__m128> { + return reinterpret_cast<__m128i&>(op); +} + +template <typename TOpOut, typename TOpIn, + typename TFunc, TFunc* func, typename... TParams> +struct TCvtS2FWrapperSingle : TBaseWrapper<__m128> { Y_FORCE_INLINE - TCvtS2FWrapperSingle(const __m128i& op, TParams... params) { - TQType<TOpOut>::As(Value) = - func(TQType<TOpIn>::As(op), params...); - } -}; - -using _mm_cvtepi32_ps = - TCvtS2FWrapperSingle<float32x4_t, int32x4_t, - decltype(vcvtq_f32_s32), vcvtq_f32_s32>; - -template <typename TOpOut, typename TOpIn, - typename TFunc, TFunc* func, typename... TParams> -struct TCvtF2SWrapperSingle : TBaseWrapper<__m128i> { + TCvtS2FWrapperSingle(const __m128i& op, TParams... params) { + TQType<TOpOut>::As(Value) = + func(TQType<TOpIn>::As(op), params...); + } +}; + +using _mm_cvtepi32_ps = + TCvtS2FWrapperSingle<float32x4_t, int32x4_t, + decltype(vcvtq_f32_s32), vcvtq_f32_s32>; + +template <typename TOpOut, typename TOpIn, + typename TFunc, TFunc* func, typename... TParams> +struct TCvtF2SWrapperSingle : TBaseWrapper<__m128i> { Y_FORCE_INLINE - TCvtF2SWrapperSingle(const __m128& op, TParams... params) { - TQType<TOpOut>::As(Value) = - func(TQType<TOpIn>::As(op), params...); - } -}; - + TCvtF2SWrapperSingle(const __m128& op, TParams... params) { + TQType<TOpOut>::As(Value) = + func(TQType<TOpIn>::As(op), params...); + } +}; + inline __m128i _mm_cvtps_epi32(__m128 a) { /// vcvtq_s32_f32 rounds to zero, but we need to round to the nearest. static const float32x4_t half = vdupq_n_f32(0.5f); @@ -874,26 +874,26 @@ inline __m128i _mm_cvtps_epi32(__m128 a) { return res; } -using _mm_cvttps_epi32 = - TCvtF2SWrapperSingle<int32x4_t, float32x4_t, - decltype(vcvtq_s32_f32), vcvtq_s32_f32>; - +using _mm_cvttps_epi32 = + TCvtF2SWrapperSingle<int32x4_t, float32x4_t, + decltype(vcvtq_s32_f32), vcvtq_s32_f32>; + Y_FORCE_INLINE int -_mm_movemask_ps(const __m128& op) { - uint32x4_t mask = {0x80000000, 0x80000000, 0x80000000, 0x80000000}; - uint32x4_t bits = vandq_u32(op.AsUi32x4, mask); - int32x4_t shifts = {-31, -30, -29, -28}; - bits = vshlq_u32(bits, shifts); - return vaddvq_u32(bits); -} +_mm_movemask_ps(const __m128& op) { + uint32x4_t mask = {0x80000000, 0x80000000, 0x80000000, 0x80000000}; + uint32x4_t bits = vandq_u32(op.AsUi32x4, mask); + int32x4_t shifts = {-31, -30, -29, -28}; + bits = vshlq_u32(bits, shifts); + return vaddvq_u32(bits); +} Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { return vgetq_lane_s64(a.AsSi64x2, 0); } - -static inline void _mm_pause() { + +static inline void _mm_pause() { __asm__ ("YIELD"); -} +} static inline __m128 _mm_rsqrt_ps(__m128 a) { __m128 res; diff --git a/library/cpp/sse/ut/test.cpp b/library/cpp/sse/ut/test.cpp index 33c999d284..42a82a8cfa 100644 --- a/library/cpp/sse/ut/test.cpp +++ b/library/cpp/sse/ut/test.cpp @@ -1,10 +1,10 @@ -/* - Unittests for all SSE instrinsics translated to NEON instrinsics or - software implementation. - Should be tested both on Intel and ARM64. - */ -/* Author: Vitaliy Manushkin <agri@yandex-team.ru */ - +/* + Unittests for all SSE instrinsics translated to NEON instrinsics or + software implementation. + Should be tested both on Intel and ARM64. + */ +/* Author: Vitaliy Manushkin <agri@yandex-team.ru */ + #include <library/cpp/testing/unittest/registar.h> #include <util/generic/typetraits.h> @@ -13,35 +13,35 @@ #include <util/stream/output.h> #include <algorithm> -#include <array> -#include <limits> +#include <array> +#include <limits> #include <memory> #include <type_traits> #include <utility> - -template <typename TResult, typename TFunc, TFunc* func> -struct T_mm_CallWrapper { - TResult Value; - - template <typename... TParams> - T_mm_CallWrapper(TParams&&... params) { - Value = func(std::forward<TParams>(params)...); - } - - operator TResult&() { - return Value; - } - - operator const TResult&() const { - return Value; - } -}; - -#if defined(_arm64_) + +template <typename TResult, typename TFunc, TFunc* func> +struct T_mm_CallWrapper { + TResult Value; + + template <typename... TParams> + T_mm_CallWrapper(TParams&&... params) { + Value = func(std::forward<TParams>(params)...); + } + + operator TResult&() { + return Value; + } + + operator const TResult&() const { + return Value; + } +}; + +#if defined(_arm64_) #include "library/cpp/sse/sse2neon.h" #elif defined(_i386_) || defined(_x86_64_) -#include <xmmintrin.h> -#include <emmintrin.h> +#include <xmmintrin.h> +#include <emmintrin.h> #include <smmintrin.h> #elif defined(_ppc64_) #include "library/cpp/sse/powerpc.h" @@ -54,10 +54,10 @@ struct T_mm_CallWrapper { #define WrapF(T_mm_func) T_mm_func #define WrapD(T_mm_func) T_mm_func #elif defined(_ppc64_) || defined(_i386_) || defined(_x86_64_) -#define Wrap(_mm_func) \ - T_mm_CallWrapper<__m128i, decltype(_mm_func), _mm_func> -#define WrapF(_mm_func) \ - T_mm_CallWrapper<__m128, decltype(_mm_func), _mm_func> +#define Wrap(_mm_func) \ + T_mm_CallWrapper<__m128i, decltype(_mm_func), _mm_func> +#define WrapF(_mm_func) \ + T_mm_CallWrapper<__m128, decltype(_mm_func), _mm_func> #define WrapD(_mm_func) \ T_mm_CallWrapper<__m128d, decltype(_mm_func), _mm_func> using int8x16_t = std::array<i8, 16>; @@ -70,69 +70,69 @@ using uint32x4_t = std::array<ui32, 4>; using uint64x2_t = std::array<ui64, 2>; using float32x4_t = std::array<float, 4>; using float64x2_t = std::array<double, 2>; - + template <typename TVectorType> -struct TQType { +struct TQType { static TVectorType As(__m128i param) { TVectorType value; - _mm_storeu_si128((__m128i*)&value, param); - return value; - } + _mm_storeu_si128((__m128i*)&value, param); + return value; + } static TVectorType As(__m128 param) { TVectorType value; - _mm_storeu_ps((float*)&value, param); - return value; - } + _mm_storeu_ps((float*)&value, param); + return value; + } static TVectorType As(__m128d param) { TVectorType value; _mm_storeu_pd((double*)&value, param); return value; } -}; -#endif - +}; +#endif + template <typename TVectorType> -struct TFuncLoad; +struct TFuncLoad; template <typename TVectorType> -struct TFuncStore; - -template <> -struct TFuncLoad<__m128i> { - __m128i Value; - - template <typename TPointer> - TFuncLoad(TPointer* ptr) { - Value = _mm_loadu_si128((__m128i*)ptr); - } - - operator __m128i&() { - return Value; - } - - operator const __m128i&() const { - return Value; - } -}; - -template <> -struct TFuncLoad<__m128> { - __m128 Value; - - template <typename TPointer> - TFuncLoad(TPointer* ptr) { - Value = _mm_loadu_ps((float*)ptr); - } - - operator __m128&() { - return Value; - } - - operator const __m128&() const { - return Value; - } -}; - -template <> +struct TFuncStore; + +template <> +struct TFuncLoad<__m128i> { + __m128i Value; + + template <typename TPointer> + TFuncLoad(TPointer* ptr) { + Value = _mm_loadu_si128((__m128i*)ptr); + } + + operator __m128i&() { + return Value; + } + + operator const __m128i&() const { + return Value; + } +}; + +template <> +struct TFuncLoad<__m128> { + __m128 Value; + + template <typename TPointer> + TFuncLoad(TPointer* ptr) { + Value = _mm_loadu_ps((float*)ptr); + } + + operator __m128&() { + return Value; + } + + operator const __m128&() const { + return Value; + } +}; + +template <> struct TFuncLoad<__m128d> { __m128d Value; @@ -151,153 +151,153 @@ struct TFuncLoad<__m128d> { }; template <> -struct TFuncStore<__m128i> { - template <typename TPointer> - TFuncStore(TPointer* ptr, __m128i Value) { - _mm_storeu_si128((__m128i*)ptr, Value); - } -}; - -template <> -struct TFuncStore<__m128> { - template <typename TPointer> - TFuncStore(TPointer* ptr, __m128 Value) { - _mm_storeu_ps((float*)ptr, Value); - } -}; - -class TSSEEmulTest: public TTestBase { -private: - UNIT_TEST_SUITE(TSSEEmulTest); - UNIT_TEST(Test_mm_load_si128); - UNIT_TEST(Test_mm_loadu_si128); +struct TFuncStore<__m128i> { + template <typename TPointer> + TFuncStore(TPointer* ptr, __m128i Value) { + _mm_storeu_si128((__m128i*)ptr, Value); + } +}; + +template <> +struct TFuncStore<__m128> { + template <typename TPointer> + TFuncStore(TPointer* ptr, __m128 Value) { + _mm_storeu_ps((float*)ptr, Value); + } +}; + +class TSSEEmulTest: public TTestBase { +private: + UNIT_TEST_SUITE(TSSEEmulTest); + UNIT_TEST(Test_mm_load_si128); + UNIT_TEST(Test_mm_loadu_si128); UNIT_TEST(Test_mm_storeu_si128); UNIT_TEST(Test_mm_loadu_si128_2); UNIT_TEST(Test_mm_loadu_ps); UNIT_TEST(Test_mm_storeu_ps); - + UNIT_TEST(Test_mm_slli_epi16); UNIT_TEST(Test_mm_slli_epi32); UNIT_TEST(Test_mm_slli_epi64); UNIT_TEST(Test_mm_slli_si128); - UNIT_TEST(Test_mm_srli_epi16); - UNIT_TEST(Test_mm_srli_epi32); - UNIT_TEST(Test_mm_srli_epi64); + UNIT_TEST(Test_mm_srli_epi16); + UNIT_TEST(Test_mm_srli_epi32); + UNIT_TEST(Test_mm_srli_epi64); UNIT_TEST(Test_mm_srli_si128); - + UNIT_TEST(Test_mm_srai_epi16); UNIT_TEST(Test_mm_srai_epi32); UNIT_TEST(Test_mm_sll_epi16); UNIT_TEST(Test_mm_sll_epi32); UNIT_TEST(Test_mm_sll_epi64); - + UNIT_TEST(Test_mm_srl_epi16); UNIT_TEST(Test_mm_srl_epi32); UNIT_TEST(Test_mm_srl_epi64); - UNIT_TEST(Test_mm_add_epi16); - UNIT_TEST(Test_mm_add_epi32); - UNIT_TEST(Test_mm_add_epi64); - UNIT_TEST(Test_mm_add_ps); + UNIT_TEST(Test_mm_add_epi16); + UNIT_TEST(Test_mm_add_epi32); + UNIT_TEST(Test_mm_add_epi64); + UNIT_TEST(Test_mm_add_ps); UNIT_TEST(Test_mm_add_pd); - + UNIT_TEST(Test_mm_madd_epi16); - UNIT_TEST(Test_mm_sub_epi16); - UNIT_TEST(Test_mm_sub_epi32); - UNIT_TEST(Test_mm_sub_epi64); - UNIT_TEST(Test_mm_sub_ps); + UNIT_TEST(Test_mm_sub_epi16); + UNIT_TEST(Test_mm_sub_epi32); + UNIT_TEST(Test_mm_sub_epi64); + UNIT_TEST(Test_mm_sub_ps); UNIT_TEST(Test_mm_sub_pd); - - UNIT_TEST(Test_mm_mul_ps); + + UNIT_TEST(Test_mm_mul_ps); UNIT_TEST(Test_mm_mul_pd); - UNIT_TEST(Test_mm_div_ps); + UNIT_TEST(Test_mm_div_ps); UNIT_TEST(Test_mm_div_pd); - UNIT_TEST(Test_mm_max_ps); - UNIT_TEST(Test_mm_min_ps); - UNIT_TEST(Test_mm_and_ps); - - UNIT_TEST(Test_mm_unpacklo_epi8); - UNIT_TEST(Test_mm_unpackhi_epi8); - UNIT_TEST(Test_mm_unpacklo_epi16); - UNIT_TEST(Test_mm_unpackhi_epi16); - UNIT_TEST(Test_mm_unpacklo_epi32); - UNIT_TEST(Test_mm_unpackhi_epi32); - UNIT_TEST(Test_mm_unpacklo_epi64); - UNIT_TEST(Test_mm_unpackhi_epi64); - - UNIT_TEST(Test_mm_or_si128); - UNIT_TEST(Test_mm_and_si128); - UNIT_TEST(Test_mm_andnot_si128); - - UNIT_TEST(Test_mm_cmpeq_epi8); - UNIT_TEST(Test_mm_cmpeq_epi16); - UNIT_TEST(Test_mm_cmpeq_epi32); - UNIT_TEST(Test_mm_cmpeq_ps); - - UNIT_TEST(Test_mm_cmpgt_epi8); - UNIT_TEST(Test_mm_cmpgt_epi16); - UNIT_TEST(Test_mm_cmpgt_epi32); - UNIT_TEST(Test_mm_cmpgt_ps); - - UNIT_TEST(Test_mm_cmplt_epi8); - UNIT_TEST(Test_mm_cmplt_epi16); - UNIT_TEST(Test_mm_cmplt_epi32); - - UNIT_TEST(Test_mm_set1_epi8); - UNIT_TEST(Test_mm_set1_epi16); - UNIT_TEST(Test_mm_set1_epi32); - UNIT_TEST(Test_mm_set1_ps); + UNIT_TEST(Test_mm_max_ps); + UNIT_TEST(Test_mm_min_ps); + UNIT_TEST(Test_mm_and_ps); + + UNIT_TEST(Test_mm_unpacklo_epi8); + UNIT_TEST(Test_mm_unpackhi_epi8); + UNIT_TEST(Test_mm_unpacklo_epi16); + UNIT_TEST(Test_mm_unpackhi_epi16); + UNIT_TEST(Test_mm_unpacklo_epi32); + UNIT_TEST(Test_mm_unpackhi_epi32); + UNIT_TEST(Test_mm_unpacklo_epi64); + UNIT_TEST(Test_mm_unpackhi_epi64); + + UNIT_TEST(Test_mm_or_si128); + UNIT_TEST(Test_mm_and_si128); + UNIT_TEST(Test_mm_andnot_si128); + + UNIT_TEST(Test_mm_cmpeq_epi8); + UNIT_TEST(Test_mm_cmpeq_epi16); + UNIT_TEST(Test_mm_cmpeq_epi32); + UNIT_TEST(Test_mm_cmpeq_ps); + + UNIT_TEST(Test_mm_cmpgt_epi8); + UNIT_TEST(Test_mm_cmpgt_epi16); + UNIT_TEST(Test_mm_cmpgt_epi32); + UNIT_TEST(Test_mm_cmpgt_ps); + + UNIT_TEST(Test_mm_cmplt_epi8); + UNIT_TEST(Test_mm_cmplt_epi16); + UNIT_TEST(Test_mm_cmplt_epi32); + + UNIT_TEST(Test_mm_set1_epi8); + UNIT_TEST(Test_mm_set1_epi16); + UNIT_TEST(Test_mm_set1_epi32); + UNIT_TEST(Test_mm_set1_ps); UNIT_TEST(Test_mm_set_ps1); - - UNIT_TEST(Test_mm_setzero_si128); - UNIT_TEST(Test_mm_setzero_ps); + + UNIT_TEST(Test_mm_setzero_si128); + UNIT_TEST(Test_mm_setzero_ps); UNIT_TEST(Test_mm_setzero_pd); - - UNIT_TEST(Test_mm_storel_epi64); - UNIT_TEST(Test_mm_loadl_epi64); - + + UNIT_TEST(Test_mm_storel_epi64); + UNIT_TEST(Test_mm_loadl_epi64); + UNIT_TEST(Test_mm_loadl_pd); UNIT_TEST(Test_mm_loadh_pd); UNIT_TEST(Test_mm_cvtsd_f64); - UNIT_TEST(Test_mm_shuffle_epi32); - UNIT_TEST(Test_mm_movemask_epi8); - UNIT_TEST(Test_mm_cvtsi128_si32); + UNIT_TEST(Test_mm_shuffle_epi32); + UNIT_TEST(Test_mm_movemask_epi8); + UNIT_TEST(Test_mm_cvtsi128_si32); UNIT_TEST(Test_mm_cvtsi128_si64); - - UNIT_TEST(Test_mm_set_epi16); - UNIT_TEST(Test_mm_set_epi32); - UNIT_TEST(Test_mm_set_ps); + + UNIT_TEST(Test_mm_set_epi16); + UNIT_TEST(Test_mm_set_epi32); + UNIT_TEST(Test_mm_set_ps); UNIT_TEST(Test_mm_set_pd); - - UNIT_TEST(Test_mm_cvtsi32_si128); + + UNIT_TEST(Test_mm_cvtsi32_si128); UNIT_TEST(Test_mm_cvtsi64_si128); - - UNIT_TEST(Test_mm_packs_epi16); - UNIT_TEST(Test_mm_packs_epi32); - UNIT_TEST(Test_mm_packus_epi16); - - UNIT_TEST(Test_mm_extract_epi16); + + UNIT_TEST(Test_mm_packs_epi16); + UNIT_TEST(Test_mm_packs_epi32); + UNIT_TEST(Test_mm_packus_epi16); + + UNIT_TEST(Test_mm_extract_epi16); UNIT_TEST(Test_mm_extract_epi8); UNIT_TEST(Test_mm_extract_epi32); UNIT_TEST(Test_mm_extract_epi64); - - UNIT_TEST(Test_MM_TRANSPOSE4_PS); - UNIT_TEST(Test_mm_movemask_ps); + + UNIT_TEST(Test_MM_TRANSPOSE4_PS); + UNIT_TEST(Test_mm_movemask_ps); UNIT_TEST(Test_mm_movemask_ps_2); - - UNIT_TEST(Test_mm_cvtepi32_ps); - UNIT_TEST(Test_mm_cvtps_epi32); - UNIT_TEST(Test_mm_cvttps_epi32); - - UNIT_TEST(Test_mm_castsi128_ps); - UNIT_TEST(Test_mm_castps_si128); - - UNIT_TEST(Test_mm_mul_epu32); - + + UNIT_TEST(Test_mm_cvtepi32_ps); + UNIT_TEST(Test_mm_cvtps_epi32); + UNIT_TEST(Test_mm_cvttps_epi32); + + UNIT_TEST(Test_mm_castsi128_ps); + UNIT_TEST(Test_mm_castps_si128); + + UNIT_TEST(Test_mm_mul_epu32); + UNIT_TEST(Test_mm_cmpunord_ps); UNIT_TEST(Test_mm_andnot_ps); UNIT_TEST(Test_mm_shuffle_ps); @@ -310,36 +310,36 @@ private: UNIT_TEST(Test_mm_rsqrt_ps); UNIT_TEST(Test_matrixnet_powerpc); - UNIT_TEST_SUITE_END(); - -public: - void Test_mm_load_si128(); - void Test_mm_loadu_si128(); + UNIT_TEST_SUITE_END(); + +public: + void Test_mm_load_si128(); + void Test_mm_loadu_si128(); void Test_mm_storeu_si128(); void Test_mm_loadu_si128_2(); void Test_mm_loadu_ps(); void Test_mm_storeu_ps(); - - template <typename TElem, int bits, int elemCount, + + template <typename TElem, int bits, int elemCount, typename TFunc, typename TShifter, typename TOp, typename TElemFunc> - void Test_mm_shifter_epiXX(); - + void Test_mm_shifter_epiXX(); + enum class EDirection { Left, Right }; - + struct TShiftRes { __m128i Value[17]; }; void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo); - void Test_mm_slli_epi16(); - void Test_mm_slli_epi32(); - void Test_mm_slli_epi64(); + void Test_mm_slli_epi16(); + void Test_mm_slli_epi32(); + void Test_mm_slli_epi64(); void Test_mm_slli_si128(); - + void Test_mm_srli_epi16(); void Test_mm_srli_epi32(); void Test_mm_srli_epi64(); @@ -356,134 +356,134 @@ public: void Test_mm_srl_epi32(); void Test_mm_srl_epi64(); - void Test_mm_add_epi8(); - void Test_mm_add_epi16(); - void Test_mm_add_epi32(); - void Test_mm_add_epi64(); - void Test_mm_add_ps(); + void Test_mm_add_epi8(); + void Test_mm_add_epi16(); + void Test_mm_add_epi32(); + void Test_mm_add_epi64(); + void Test_mm_add_ps(); void Test_mm_add_pd(); - + void Test_mm_madd_epi16(); - void Test_mm_sub_epi8(); - void Test_mm_sub_epi16(); - void Test_mm_sub_epi32(); - void Test_mm_sub_epi64(); - void Test_mm_sub_ps(); + void Test_mm_sub_epi8(); + void Test_mm_sub_epi16(); + void Test_mm_sub_epi32(); + void Test_mm_sub_epi64(); + void Test_mm_sub_ps(); void Test_mm_sub_pd(); - - void Test_mm_mul_ps(); + + void Test_mm_mul_ps(); void Test_mm_mul_pd(); - void Test_mm_div_ps(); + void Test_mm_div_ps(); void Test_mm_div_pd(); - void Test_mm_max_ps(); - void Test_mm_min_ps(); - void Test_mm_and_ps(); - - template <typename TElem, int bits, int elemCount, int shift, - typename TFunc, typename TOp> - void Test_mm_unpack_epiXX(); - void Test_mm_unpacklo_epi8(); - void Test_mm_unpackhi_epi8(); - void Test_mm_unpacklo_epi16(); - void Test_mm_unpackhi_epi16(); - void Test_mm_unpacklo_epi32(); - void Test_mm_unpackhi_epi32(); - void Test_mm_unpacklo_epi64(); - void Test_mm_unpackhi_epi64(); - - template <typename TElem, unsigned elemCount, - typename TFunc, typename TElemFunc, + void Test_mm_max_ps(); + void Test_mm_min_ps(); + void Test_mm_and_ps(); + + template <typename TElem, int bits, int elemCount, int shift, + typename TFunc, typename TOp> + void Test_mm_unpack_epiXX(); + void Test_mm_unpacklo_epi8(); + void Test_mm_unpackhi_epi8(); + void Test_mm_unpacklo_epi16(); + void Test_mm_unpackhi_epi16(); + void Test_mm_unpacklo_epi32(); + void Test_mm_unpackhi_epi32(); + void Test_mm_unpacklo_epi64(); + void Test_mm_unpackhi_epi64(); + + template <typename TElem, unsigned elemCount, + typename TFunc, typename TElemFunc, typename TOp, typename TVectorType = __m128i> - void Test_mm_dualop(); - - template <typename TElem, unsigned elemCount, - typename TFunc, typename TElemFunc, + void Test_mm_dualop(); + + template <typename TElem, unsigned elemCount, + typename TFunc, typename TElemFunc, typename TOp, typename TVectorType = __m128i> - void Test_mm_dualcmp(); - - void Test_mm_or_si128(); - void Test_mm_and_si128(); - void Test_mm_andnot_si128(); - - void Test_mm_cmpeq_epi8(); - void Test_mm_cmpeq_epi16(); - void Test_mm_cmpeq_epi32(); - void Test_mm_cmpeq_ps(); - - void Test_mm_cmpgt_epi8(); - void Test_mm_cmpgt_epi16(); - void Test_mm_cmpgt_epi32(); - void Test_mm_cmpgt_ps(); - - void Test_mm_cmplt_epi8(); - void Test_mm_cmplt_epi16(); - void Test_mm_cmplt_epi32(); - - template <typename TElem, int elemCount, + void Test_mm_dualcmp(); + + void Test_mm_or_si128(); + void Test_mm_and_si128(); + void Test_mm_andnot_si128(); + + void Test_mm_cmpeq_epi8(); + void Test_mm_cmpeq_epi16(); + void Test_mm_cmpeq_epi32(); + void Test_mm_cmpeq_ps(); + + void Test_mm_cmpgt_epi8(); + void Test_mm_cmpgt_epi16(); + void Test_mm_cmpgt_epi32(); + void Test_mm_cmpgt_ps(); + + void Test_mm_cmplt_epi8(); + void Test_mm_cmplt_epi16(); + void Test_mm_cmplt_epi32(); + + template <typename TElem, int elemCount, typename TFunc, typename TOp, typename TVectorType> - void Test_mm_setter_epiXX(); - void Test_mm_set1_epi8(); - void Test_mm_set1_epi16(); - void Test_mm_set1_epi32(); - void Test_mm_set1_ps(); + void Test_mm_setter_epiXX(); + void Test_mm_set1_epi8(); + void Test_mm_set1_epi16(); + void Test_mm_set1_epi32(); + void Test_mm_set1_ps(); void Test_mm_set_ps1(); - - void Test_mm_setzero_si128(); - void Test_mm_setzero_ps(); + + void Test_mm_setzero_si128(); + void Test_mm_setzero_ps(); void Test_mm_setzero_pd(); - - void Test_mm_loadl_epi64(); - void Test_mm_storel_epi64(); - + + void Test_mm_loadl_epi64(); + void Test_mm_storel_epi64(); + void Test_mm_loadl_pd(); void Test_mm_loadh_pd(); void Test_mm_cvtsd_f64(); - void Test_mm_shuffle_epi32(); - void Test_mm_movemask_epi8(); - void Test_mm_cvtsi128_si32(); + void Test_mm_shuffle_epi32(); + void Test_mm_movemask_epi8(); + void Test_mm_cvtsi128_si32(); void Test_mm_cvtsi128_si64(); - - void Test_mm_set_epi16(); - void Test_mm_set_epi32(); - void Test_mm_set_ps(); + + void Test_mm_set_epi16(); + void Test_mm_set_epi32(); + void Test_mm_set_ps(); void Test_mm_set_pd(); - - void Test_mm_cvtsi32_si128(); + + void Test_mm_cvtsi32_si128(); void Test_mm_cvtsi64_si128(); - - template <typename TElem, typename TNarrow, unsigned elemCount, - typename TFunc> - void Test_mm_packs_epiXX(); - void Test_mm_packs_epi16(); - void Test_mm_packs_epi32(); - void Test_mm_packus_epi16(); - - void Test_mm_extract_epi16(); + + template <typename TElem, typename TNarrow, unsigned elemCount, + typename TFunc> + void Test_mm_packs_epiXX(); + void Test_mm_packs_epi16(); + void Test_mm_packs_epi32(); + void Test_mm_packus_epi16(); + + void Test_mm_extract_epi16(); void Test_mm_extract_epi8(); void Test_mm_extract_epi32(); void Test_mm_extract_epi64(); - - void Test_MM_TRANSPOSE4_PS(); - void Test_mm_movemask_ps(); + + void Test_MM_TRANSPOSE4_PS(); + void Test_mm_movemask_ps(); void Test_mm_movemask_ps_2(); - - template <typename TFrom, typename TTo, unsigned elemCount, - typename TLoadVector, typename TResultVector, - typename TElemFunc, typename TFunc, typename TOp> - void Test_mm_convertop(); - void Test_mm_cvtepi32_ps(); - void Test_mm_cvtps_epi32(); - void Test_mm_cvttps_epi32(); - - template <typename TLoadVector, typename TCastVector, - typename TFunc, TFunc* func> - void Test_mm_castXX(); - void Test_mm_castsi128_ps(); - void Test_mm_castps_si128(); - - void Test_mm_mul_epu32(); + + template <typename TFrom, typename TTo, unsigned elemCount, + typename TLoadVector, typename TResultVector, + typename TElemFunc, typename TFunc, typename TOp> + void Test_mm_convertop(); + void Test_mm_cvtepi32_ps(); + void Test_mm_cvtps_epi32(); + void Test_mm_cvttps_epi32(); + + template <typename TLoadVector, typename TCastVector, + typename TFunc, TFunc* func> + void Test_mm_castXX(); + void Test_mm_castsi128_ps(); + void Test_mm_castps_si128(); + + void Test_mm_mul_epu32(); void Test_mm_cmpunord_ps(); void Test_mm_store_ss(); @@ -497,30 +497,30 @@ public: void Test_mm_rsqrt_ps(); void Test_mm_rsqrt_ss(); void Test_matrixnet_powerpc(); -}; - -UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest); - -void TSSEEmulTest::Test_mm_load_si128() { +}; + +UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest); + +void TSSEEmulTest::Test_mm_load_si128() { alignas(16) char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - __m128i value = _mm_load_si128((__m128i*)&data); - UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[0], 0xAABB2211CCFF00AAUL); - UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[1], 0x1C66775588449933UL); -} - -void TSSEEmulTest::Test_mm_loadu_si128() { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + __m128i value = _mm_load_si128((__m128i*)&data); + UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[0], 0xAABB2211CCFF00AAUL); + UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[1], 0x1C66775588449933UL); +} + +void TSSEEmulTest::Test_mm_loadu_si128() { alignas(16) char data[17] = { - '\x66', - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - UNIT_ASSERT((ui64(&data[1]) & 0x1) == 0x1); - __m128i value = _mm_loadu_si128((__m128i*)&data[1]); - UNIT_ASSERT(TQType<uint64x2_t>::As(value)[0] == 0xAABB2211CCFF00AAUL); - UNIT_ASSERT(TQType<uint64x2_t>::As(value)[1] == 0x1C66775588449933UL); -} - + '\x66', + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + UNIT_ASSERT((ui64(&data[1]) & 0x1) == 0x1); + __m128i value = _mm_loadu_si128((__m128i*)&data[1]); + UNIT_ASSERT(TQType<uint64x2_t>::As(value)[0] == 0xAABB2211CCFF00AAUL); + UNIT_ASSERT(TQType<uint64x2_t>::As(value)[1] == 0x1C66775588449933UL); +} + void TSSEEmulTest::Test_mm_storeu_si128() { alignas(16) unsigned char stub[32] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, @@ -640,32 +640,32 @@ unsigned MakeNumber<unsigned>(unsigned number) { return number; } -template <typename TElem, int bits, int elemCount, +template <typename TElem, int bits, int elemCount, typename TFunc, typename TShifter, typename TOp, typename TElemFunc> -void TSSEEmulTest::Test_mm_shifter_epiXX() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - TElem* dataw = reinterpret_cast<TElem*>(&data); - - __m128i value = _mm_loadu_si128((__m128i*)&data); - +void TSSEEmulTest::Test_mm_shifter_epiXX() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + TElem* dataw = reinterpret_cast<TElem*>(&data); + + __m128i value = _mm_loadu_si128((__m128i*)&data); + for (unsigned shifter = 0; shifter <= bits; ++shifter) { - TElem shiftedData[elemCount]; + TElem shiftedData[elemCount]; for (unsigned i = 0; i < elemCount; ++i) { - shiftedData[i] = TElemFunc::Call(dataw[i], shifter); + shiftedData[i] = TElemFunc::Call(dataw[i], shifter); } - + const TShifter adhoc_shifter = MakeNumber<TShifter>(shifter); __m128i result = TFunc(value, adhoc_shifter); for (unsigned i = 0; i < elemCount; ++i) { - UNIT_ASSERT_EQUAL(shiftedData[i], TQType<TOp>::As(result)[i]); + UNIT_ASSERT_EQUAL(shiftedData[i], TQType<TOp>::As(result)[i]); } - } -} - + } +} + void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) { const char data[48] = { @@ -713,52 +713,52 @@ struct THelperASHR { } }; -template <typename TElem> -struct THelperSHR { - static TElem Call(const TElem op, const int shift) { +template <typename TElem> +struct THelperSHR { + static TElem Call(const TElem op, const int shift) { constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; return shift < nBitsInOp ? op >> shift : 0; - } -}; - -void TSSEEmulTest::Test_mm_srli_epi16() { + } +}; + +void TSSEEmulTest::Test_mm_srli_epi16() { Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_srli_epi16), unsigned, uint16x8_t, - THelperSHR<ui16>>(); -} - -void TSSEEmulTest::Test_mm_srli_epi32() { + THelperSHR<ui16>>(); +} + +void TSSEEmulTest::Test_mm_srli_epi32() { Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_srli_epi32), unsigned, uint32x4_t, - THelperSHR<ui32>>(); -} - -void TSSEEmulTest::Test_mm_srli_epi64() { + THelperSHR<ui32>>(); +} + +void TSSEEmulTest::Test_mm_srli_epi64() { Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_srli_epi64), unsigned, uint64x2_t, - THelperSHR<ui64>>(); -} - -template <typename TElem> -struct THelperSHL { - static TElem Call(const TElem op, const int shift) { + THelperSHR<ui64>>(); +} + +template <typename TElem> +struct THelperSHL { + static TElem Call(const TElem op, const int shift) { constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; return shift < nBitsInOp ? op << shift : 0; - } -}; - -void TSSEEmulTest::Test_mm_slli_epi16() { + } +}; + +void TSSEEmulTest::Test_mm_slli_epi16() { Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_slli_epi16), unsigned, uint16x8_t, - THelperSHL<ui16>>(); -} - -void TSSEEmulTest::Test_mm_slli_epi32() { + THelperSHL<ui16>>(); +} + +void TSSEEmulTest::Test_mm_slli_epi32() { Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_slli_epi32), unsigned, uint32x4_t, - THelperSHL<ui32>>(); -} - -void TSSEEmulTest::Test_mm_slli_epi64() { + THelperSHL<ui32>>(); +} + +void TSSEEmulTest::Test_mm_slli_epi64() { Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_slli_epi64), unsigned, uint64x2_t, - THelperSHL<ui64>>(); -} - + THelperSHL<ui64>>(); +} + void TSSEEmulTest::Test_mm_slli_si128() { Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes { TShiftRes res; @@ -849,30 +849,30 @@ void TSSEEmulTest::Test_mm_sll_epi64() { THelperSHL<ui64>>(); } -template <typename TElem> -struct THelperAdd { - static TElem Call(const TElem op1, const TElem op2) { - return op1 + op2; - } -}; - -void TSSEEmulTest::Test_mm_add_epi16() { - Test_mm_dualop<ui16, 8, Wrap(_mm_add_epi16), THelperAdd<ui16>, uint16x8_t>(); -} - -void TSSEEmulTest::Test_mm_add_epi32() { - Test_mm_dualop<ui32, 4, Wrap(_mm_add_epi32), THelperAdd<ui32>, uint32x4_t>(); -} - -void TSSEEmulTest::Test_mm_add_epi64() { - Test_mm_dualop<ui64, 2, Wrap(_mm_add_epi64), THelperAdd<ui64>, uint64x2_t>(); -} - -void TSSEEmulTest::Test_mm_add_ps() { - Test_mm_dualop<float, 2, WrapF(_mm_add_ps), - THelperAdd<float>, float32x4_t, __m128>(); -} - +template <typename TElem> +struct THelperAdd { + static TElem Call(const TElem op1, const TElem op2) { + return op1 + op2; + } +}; + +void TSSEEmulTest::Test_mm_add_epi16() { + Test_mm_dualop<ui16, 8, Wrap(_mm_add_epi16), THelperAdd<ui16>, uint16x8_t>(); +} + +void TSSEEmulTest::Test_mm_add_epi32() { + Test_mm_dualop<ui32, 4, Wrap(_mm_add_epi32), THelperAdd<ui32>, uint32x4_t>(); +} + +void TSSEEmulTest::Test_mm_add_epi64() { + Test_mm_dualop<ui64, 2, Wrap(_mm_add_epi64), THelperAdd<ui64>, uint64x2_t>(); +} + +void TSSEEmulTest::Test_mm_add_ps() { + Test_mm_dualop<float, 2, WrapF(_mm_add_ps), + THelperAdd<float>, float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_add_pd() { Test_mm_dualop<double, 2, WrapD(_mm_add_pd), THelperAdd<double>, float64x2_t, __m128d>(); @@ -904,44 +904,44 @@ void TSSEEmulTest::Test_mm_madd_epi16() { } -template <typename TElem> -struct THelperSub { - static TElem Call(const TElem op1, const TElem op2) { - return op1 - op2; - } -}; - -void TSSEEmulTest::Test_mm_sub_epi16() { - Test_mm_dualop<ui16, 8, Wrap(_mm_sub_epi16), THelperSub<ui16>, uint16x8_t>(); -} - -void TSSEEmulTest::Test_mm_sub_epi32() { - Test_mm_dualop<ui32, 4, Wrap(_mm_sub_epi32), THelperSub<ui32>, uint32x4_t>(); -} - -void TSSEEmulTest::Test_mm_sub_epi64() { - Test_mm_dualop<ui64, 2, Wrap(_mm_sub_epi64), THelperSub<ui64>, uint64x2_t>(); -} - -void TSSEEmulTest::Test_mm_sub_ps() { - Test_mm_dualop<float, 4, WrapF(_mm_sub_ps), THelperSub<float>, - float32x4_t, __m128>(); -} - +template <typename TElem> +struct THelperSub { + static TElem Call(const TElem op1, const TElem op2) { + return op1 - op2; + } +}; + +void TSSEEmulTest::Test_mm_sub_epi16() { + Test_mm_dualop<ui16, 8, Wrap(_mm_sub_epi16), THelperSub<ui16>, uint16x8_t>(); +} + +void TSSEEmulTest::Test_mm_sub_epi32() { + Test_mm_dualop<ui32, 4, Wrap(_mm_sub_epi32), THelperSub<ui32>, uint32x4_t>(); +} + +void TSSEEmulTest::Test_mm_sub_epi64() { + Test_mm_dualop<ui64, 2, Wrap(_mm_sub_epi64), THelperSub<ui64>, uint64x2_t>(); +} + +void TSSEEmulTest::Test_mm_sub_ps() { + Test_mm_dualop<float, 4, WrapF(_mm_sub_ps), THelperSub<float>, + float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_sub_pd() { Test_mm_dualop<double, 2, WrapD(_mm_sub_pd), THelperSub<double>, float64x2_t, __m128d>(); } -void TSSEEmulTest::Test_mm_mul_ps() { - struct THelper { - static float Call(const float op1, const float op2) { - return op1 * op2; - } - }; - Test_mm_dualop<float, 4, WrapF(_mm_mul_ps), THelper, float32x4_t, __m128>(); -} - +void TSSEEmulTest::Test_mm_mul_ps() { + struct THelper { + static float Call(const float op1, const float op2) { + return op1 * op2; + } + }; + Test_mm_dualop<float, 4, WrapF(_mm_mul_ps), THelper, float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_mul_pd() { struct THelper { static double Call(const double op1, const double op2) { @@ -951,15 +951,15 @@ void TSSEEmulTest::Test_mm_mul_pd() { Test_mm_dualop<double, 2, WrapD(_mm_mul_pd), THelper, float64x2_t, __m128d>(); } -void TSSEEmulTest::Test_mm_div_ps() { - struct THelper { - static float Call(const float op1, const float op2) { - return op1 / op2; - } - }; - Test_mm_dualop<float, 4, WrapF(_mm_div_ps), THelper, float32x4_t, __m128>(); -} - +void TSSEEmulTest::Test_mm_div_ps() { + struct THelper { + static float Call(const float op1, const float op2) { + return op1 / op2; + } + }; + Test_mm_dualop<float, 4, WrapF(_mm_div_ps), THelper, float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_div_pd() { struct THelper { static double Call(const double op1, const double op2) { @@ -969,441 +969,441 @@ void TSSEEmulTest::Test_mm_div_pd() { Test_mm_dualop<double, 2, WrapD(_mm_div_pd), THelper, float64x2_t, __m128d>(); } -void TSSEEmulTest::Test_mm_max_ps() { - struct THelper { - static float Call(const float op1, const float op2) { - return std::max(op1, op2); - } - }; - Test_mm_dualop<float, 4, WrapF(_mm_max_ps), THelper, float32x4_t, __m128>(); -} - -void TSSEEmulTest::Test_mm_min_ps() { - struct THelper { - static float Call(const float op1, const float op2) { - return std::min(op1, op2); - } - }; - Test_mm_dualop<float, 4, WrapF(_mm_min_ps), THelper, float32x4_t, __m128>(); -} - -void TSSEEmulTest::Test_mm_and_ps() { - struct THelper { - static float Call(const float op1, const float op2) { - union Cast { - unsigned int AsUInt; - float AsFloat; - }; - Cast v1, v2, result; - v1.AsFloat = op1; - v2.AsFloat = op2; - result.AsUInt = v1.AsUInt & v2.AsUInt; - return result.AsFloat; - } - }; - Test_mm_dualcmp<float, 4, WrapF(_mm_and_ps), - THelper, float32x4_t, __m128>(); -} - -template <typename TElem, int bits, int elemCount, int shift, - typename TFunc, typename TOp> -void TSSEEmulTest::Test_mm_unpack_epiXX() { - char data1[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - char data2[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; - TElem* dataw1 = reinterpret_cast<TElem*>(&data1); - TElem* dataw2 = reinterpret_cast<TElem*>(&data2); - - __m128i value1 = _mm_loadu_si128((__m128i*)&data1); - __m128i value2 = _mm_loadu_si128((__m128i*)&data2); - - TElem zippedData[elemCount]; - for (unsigned i = 0; i < elemCount / 2; ++i) { - zippedData[i * 2] = dataw1[i + shift]; - zippedData[i * 2 + 1] = dataw2[i + shift]; - } - __m128i result = TFunc(value1, value2); - - for (unsigned i = 0; i < elemCount / 2; ++i) { - UNIT_ASSERT_EQUAL(zippedData[i * 2], TQType<TOp>::As(result)[i * 2]); - UNIT_ASSERT_EQUAL(zippedData[i * 2 + 1], - TQType<TOp>::As(result)[i * 2 + 1]); - } -} - -void TSSEEmulTest::Test_mm_unpacklo_epi8() { - Test_mm_unpack_epiXX<ui8, 8, 16, 0, Wrap(_mm_unpacklo_epi8), uint8x16_t>(); -} - -void TSSEEmulTest::Test_mm_unpackhi_epi8() { - Test_mm_unpack_epiXX<ui8, 8, 16, 8, Wrap(_mm_unpackhi_epi8), uint8x16_t>(); -} - -void TSSEEmulTest::Test_mm_unpacklo_epi16() { - Test_mm_unpack_epiXX<ui16, 16, 8, 0, Wrap(_mm_unpacklo_epi16), uint16x8_t>(); -} - -void TSSEEmulTest::Test_mm_unpackhi_epi16() { - Test_mm_unpack_epiXX<ui16, 16, 8, 4, Wrap(_mm_unpackhi_epi16), uint16x8_t>(); -} - -void TSSEEmulTest::Test_mm_unpacklo_epi32() { - Test_mm_unpack_epiXX<ui32, 32, 4, 0, Wrap(_mm_unpacklo_epi32), uint32x4_t>(); -} - -void TSSEEmulTest::Test_mm_unpackhi_epi32() { - Test_mm_unpack_epiXX<ui32, 32, 4, 2, Wrap(_mm_unpackhi_epi32), uint32x4_t>(); -} - -void TSSEEmulTest::Test_mm_unpacklo_epi64() { - Test_mm_unpack_epiXX<ui64, 64, 2, 0, Wrap(_mm_unpacklo_epi64), uint64x2_t>(); -} - -void TSSEEmulTest::Test_mm_unpackhi_epi64() { - Test_mm_unpack_epiXX<ui64, 64, 2, 1, Wrap(_mm_unpackhi_epi64), uint64x2_t>(); -} - -template <typename TElem, unsigned elemCount, - typename TFunc, typename TElemFunc, +void TSSEEmulTest::Test_mm_max_ps() { + struct THelper { + static float Call(const float op1, const float op2) { + return std::max(op1, op2); + } + }; + Test_mm_dualop<float, 4, WrapF(_mm_max_ps), THelper, float32x4_t, __m128>(); +} + +void TSSEEmulTest::Test_mm_min_ps() { + struct THelper { + static float Call(const float op1, const float op2) { + return std::min(op1, op2); + } + }; + Test_mm_dualop<float, 4, WrapF(_mm_min_ps), THelper, float32x4_t, __m128>(); +} + +void TSSEEmulTest::Test_mm_and_ps() { + struct THelper { + static float Call(const float op1, const float op2) { + union Cast { + unsigned int AsUInt; + float AsFloat; + }; + Cast v1, v2, result; + v1.AsFloat = op1; + v2.AsFloat = op2; + result.AsUInt = v1.AsUInt & v2.AsUInt; + return result.AsFloat; + } + }; + Test_mm_dualcmp<float, 4, WrapF(_mm_and_ps), + THelper, float32x4_t, __m128>(); +} + +template <typename TElem, int bits, int elemCount, int shift, + typename TFunc, typename TOp> +void TSSEEmulTest::Test_mm_unpack_epiXX() { + char data1[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + char data2[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; + TElem* dataw1 = reinterpret_cast<TElem*>(&data1); + TElem* dataw2 = reinterpret_cast<TElem*>(&data2); + + __m128i value1 = _mm_loadu_si128((__m128i*)&data1); + __m128i value2 = _mm_loadu_si128((__m128i*)&data2); + + TElem zippedData[elemCount]; + for (unsigned i = 0; i < elemCount / 2; ++i) { + zippedData[i * 2] = dataw1[i + shift]; + zippedData[i * 2 + 1] = dataw2[i + shift]; + } + __m128i result = TFunc(value1, value2); + + for (unsigned i = 0; i < elemCount / 2; ++i) { + UNIT_ASSERT_EQUAL(zippedData[i * 2], TQType<TOp>::As(result)[i * 2]); + UNIT_ASSERT_EQUAL(zippedData[i * 2 + 1], + TQType<TOp>::As(result)[i * 2 + 1]); + } +} + +void TSSEEmulTest::Test_mm_unpacklo_epi8() { + Test_mm_unpack_epiXX<ui8, 8, 16, 0, Wrap(_mm_unpacklo_epi8), uint8x16_t>(); +} + +void TSSEEmulTest::Test_mm_unpackhi_epi8() { + Test_mm_unpack_epiXX<ui8, 8, 16, 8, Wrap(_mm_unpackhi_epi8), uint8x16_t>(); +} + +void TSSEEmulTest::Test_mm_unpacklo_epi16() { + Test_mm_unpack_epiXX<ui16, 16, 8, 0, Wrap(_mm_unpacklo_epi16), uint16x8_t>(); +} + +void TSSEEmulTest::Test_mm_unpackhi_epi16() { + Test_mm_unpack_epiXX<ui16, 16, 8, 4, Wrap(_mm_unpackhi_epi16), uint16x8_t>(); +} + +void TSSEEmulTest::Test_mm_unpacklo_epi32() { + Test_mm_unpack_epiXX<ui32, 32, 4, 0, Wrap(_mm_unpacklo_epi32), uint32x4_t>(); +} + +void TSSEEmulTest::Test_mm_unpackhi_epi32() { + Test_mm_unpack_epiXX<ui32, 32, 4, 2, Wrap(_mm_unpackhi_epi32), uint32x4_t>(); +} + +void TSSEEmulTest::Test_mm_unpacklo_epi64() { + Test_mm_unpack_epiXX<ui64, 64, 2, 0, Wrap(_mm_unpacklo_epi64), uint64x2_t>(); +} + +void TSSEEmulTest::Test_mm_unpackhi_epi64() { + Test_mm_unpack_epiXX<ui64, 64, 2, 1, Wrap(_mm_unpackhi_epi64), uint64x2_t>(); +} + +template <typename TElem, unsigned elemCount, + typename TFunc, typename TElemFunc, typename TOp, typename TVectorType> -void TSSEEmulTest::Test_mm_dualop() { - char data1[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - char data2[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; - TElem* dataw1 = reinterpret_cast<TElem*>(&data1); - TElem* dataw2 = reinterpret_cast<TElem*>(&data2); - +void TSSEEmulTest::Test_mm_dualop() { + char data1[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + char data2[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; + TElem* dataw1 = reinterpret_cast<TElem*>(&data1); + TElem* dataw2 = reinterpret_cast<TElem*>(&data2); + TVectorType value1 = TFuncLoad<TVectorType>(&data1); TVectorType value2 = TFuncLoad<TVectorType>(&data2); - - TElem procData[elemCount]; - for (unsigned i = 0; i < elemCount; ++i) { - procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]); - } + + TElem procData[elemCount]; + for (unsigned i = 0; i < elemCount; ++i) { + procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]); + } TVectorType result = TFunc(value1, value2); - - for (unsigned i = 0; i < elemCount; ++i) { - UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]); - } -} - -/* This is almost the same as Test_mm_dualop, - but different data1 and data2 */ -template <typename TElem, unsigned elemCount, - typename TFunc, typename TElemFunc, + + for (unsigned i = 0; i < elemCount; ++i) { + UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]); + } +} + +/* This is almost the same as Test_mm_dualop, + but different data1 and data2 */ +template <typename TElem, unsigned elemCount, + typename TFunc, typename TElemFunc, typename TOp, typename TVectorType> -void TSSEEmulTest::Test_mm_dualcmp() { - char data1[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x66', '\x77', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C'}; - char data2[16] = { - '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; - TElem* dataw1 = reinterpret_cast<TElem*>(&data1); - TElem* dataw2 = reinterpret_cast<TElem*>(&data2); - +void TSSEEmulTest::Test_mm_dualcmp() { + char data1[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x66', '\x77', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C'}; + char data2[16] = { + '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; + TElem* dataw1 = reinterpret_cast<TElem*>(&data1); + TElem* dataw2 = reinterpret_cast<TElem*>(&data2); + TVectorType value1 = TFuncLoad<TVectorType>(&data1); TVectorType value2 = TFuncLoad<TVectorType>(&data2); - - TElem procData[elemCount]; - for (unsigned i = 0; i < elemCount; ++i) { - procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]); - } + + TElem procData[elemCount]; + for (unsigned i = 0; i < elemCount; ++i) { + procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]); + } TVectorType result = TFunc(value1, value2); - - for (unsigned i = 0; i < elemCount; ++i) { - /* memcmp is for compare to invalid floats in results */ + + for (unsigned i = 0; i < elemCount; ++i) { + /* memcmp is for compare to invalid floats in results */ const TElem value = TQType<TOp>::As(result)[i]; UNIT_ASSERT(memcmp(&(procData[i]), &value, sizeof(TElem)) == 0); - } -} - -void TSSEEmulTest::Test_mm_or_si128() { - struct THelper { - static ui64 Call(const ui64 op1, const ui64 op2) { - return op1 | op2; - } - }; - - Test_mm_dualop<ui64, 2, Wrap(_mm_or_si128), THelper, uint64x2_t>(); -} - -void TSSEEmulTest::Test_mm_and_si128() { - struct THelper { - static ui64 Call(const ui64 op1, const ui64 op2) { - return op1 & op2; - } - }; - - Test_mm_dualop<ui64, 2, Wrap(_mm_and_si128), THelper, uint64x2_t>(); -} - -void TSSEEmulTest::Test_mm_andnot_si128() { - struct THelper { - static ui64 Call(const ui64 op1, const ui64 op2) { - return (~op1) & op2; - } - }; - - Test_mm_dualop<ui64, 2, Wrap(_mm_andnot_si128), THelper, uint64x2_t>(); -} - -template <typename TElem> -struct THelperCMPEQ { - static TElem Call(const TElem op1, const TElem op2) { - return op1 == op2 ? ~TElem(0) : TElem(0); - } -}; - -void TSSEEmulTest::Test_mm_cmpeq_epi8() { - Test_mm_dualcmp<ui8, 16, Wrap(_mm_cmpeq_epi8), - THelperCMPEQ<ui8>, uint8x16_t>(); -} - -void TSSEEmulTest::Test_mm_cmpeq_epi16() { - Test_mm_dualcmp<ui16, 8, Wrap(_mm_cmpeq_epi16), - THelperCMPEQ<ui16>, uint16x8_t>(); -} - -void TSSEEmulTest::Test_mm_cmpeq_epi32() { - Test_mm_dualcmp<ui32, 4, Wrap(_mm_cmpeq_epi32), - THelperCMPEQ<ui32>, uint32x4_t>(); -} - -void TSSEEmulTest::Test_mm_cmpeq_ps() { - struct THelperFloat { - static float Call(const float op1, const float op2) { - union Cast { - unsigned int AsUInt; - float AsFloat; - }; - Cast value; - value.AsUInt = op1 == op2 ? 0xFFFFFFFF : 0; - return value.AsFloat; - } - }; - - Test_mm_dualcmp<float, 4, WrapF(_mm_cmpeq_ps), - THelperFloat, float32x4_t, __m128>(); -} - -template <typename TElem> -struct THelperCMPGT { - static TElem Call(const TElem op1, const TElem op2) { - return op1 > op2 ? ~TElem(0) : TElem(0); - } -}; - -void TSSEEmulTest::Test_mm_cmpgt_epi8() { - Test_mm_dualcmp<i8, 16, Wrap(_mm_cmpgt_epi8), - THelperCMPGT<i8>, int8x16_t>(); -} - -void TSSEEmulTest::Test_mm_cmpgt_epi16() { - Test_mm_dualcmp<i16, 8, Wrap(_mm_cmpgt_epi16), - THelperCMPGT<i16>, int16x8_t>(); -} - -void TSSEEmulTest::Test_mm_cmpgt_epi32() { - Test_mm_dualcmp<i32, 4, Wrap(_mm_cmpgt_epi32), - THelperCMPGT<i32>, int32x4_t>(); -} - -void TSSEEmulTest::Test_mm_cmpgt_ps() { - struct THelperFloat { - static float Call(const float op1, const float op2) { - union Cast { - unsigned int AsUInt; - float AsFloat; - }; - Cast value; - value.AsUInt = op1 > op2 ? 0xFFFFFFFF : 0; - return value.AsFloat; - } - }; - - Test_mm_dualcmp<float, 4, WrapF(_mm_cmpgt_ps), - THelperFloat, float32x4_t, __m128>(); -} - -template <typename TElem> -struct THelperCMPLT { - static TElem Call(const TElem op1, const TElem op2) { - return op1 < op2 ? ~TElem(0) : TElem(0); - } -}; - -void TSSEEmulTest::Test_mm_cmplt_epi8() { - Test_mm_dualcmp<i8, 16, Wrap(_mm_cmplt_epi8), - THelperCMPLT<i8>, int8x16_t>(); -} - -void TSSEEmulTest::Test_mm_cmplt_epi16() { - Test_mm_dualcmp<i16, 8, Wrap(_mm_cmplt_epi16), - THelperCMPLT<i16>, int16x8_t>(); -} - -void TSSEEmulTest::Test_mm_cmplt_epi32() { - Test_mm_dualcmp<i32, 4, Wrap(_mm_cmplt_epi32), - THelperCMPLT<i32>, int32x4_t>(); -} - -template <typename TElem, int elemCount, + } +} + +void TSSEEmulTest::Test_mm_or_si128() { + struct THelper { + static ui64 Call(const ui64 op1, const ui64 op2) { + return op1 | op2; + } + }; + + Test_mm_dualop<ui64, 2, Wrap(_mm_or_si128), THelper, uint64x2_t>(); +} + +void TSSEEmulTest::Test_mm_and_si128() { + struct THelper { + static ui64 Call(const ui64 op1, const ui64 op2) { + return op1 & op2; + } + }; + + Test_mm_dualop<ui64, 2, Wrap(_mm_and_si128), THelper, uint64x2_t>(); +} + +void TSSEEmulTest::Test_mm_andnot_si128() { + struct THelper { + static ui64 Call(const ui64 op1, const ui64 op2) { + return (~op1) & op2; + } + }; + + Test_mm_dualop<ui64, 2, Wrap(_mm_andnot_si128), THelper, uint64x2_t>(); +} + +template <typename TElem> +struct THelperCMPEQ { + static TElem Call(const TElem op1, const TElem op2) { + return op1 == op2 ? ~TElem(0) : TElem(0); + } +}; + +void TSSEEmulTest::Test_mm_cmpeq_epi8() { + Test_mm_dualcmp<ui8, 16, Wrap(_mm_cmpeq_epi8), + THelperCMPEQ<ui8>, uint8x16_t>(); +} + +void TSSEEmulTest::Test_mm_cmpeq_epi16() { + Test_mm_dualcmp<ui16, 8, Wrap(_mm_cmpeq_epi16), + THelperCMPEQ<ui16>, uint16x8_t>(); +} + +void TSSEEmulTest::Test_mm_cmpeq_epi32() { + Test_mm_dualcmp<ui32, 4, Wrap(_mm_cmpeq_epi32), + THelperCMPEQ<ui32>, uint32x4_t>(); +} + +void TSSEEmulTest::Test_mm_cmpeq_ps() { + struct THelperFloat { + static float Call(const float op1, const float op2) { + union Cast { + unsigned int AsUInt; + float AsFloat; + }; + Cast value; + value.AsUInt = op1 == op2 ? 0xFFFFFFFF : 0; + return value.AsFloat; + } + }; + + Test_mm_dualcmp<float, 4, WrapF(_mm_cmpeq_ps), + THelperFloat, float32x4_t, __m128>(); +} + +template <typename TElem> +struct THelperCMPGT { + static TElem Call(const TElem op1, const TElem op2) { + return op1 > op2 ? ~TElem(0) : TElem(0); + } +}; + +void TSSEEmulTest::Test_mm_cmpgt_epi8() { + Test_mm_dualcmp<i8, 16, Wrap(_mm_cmpgt_epi8), + THelperCMPGT<i8>, int8x16_t>(); +} + +void TSSEEmulTest::Test_mm_cmpgt_epi16() { + Test_mm_dualcmp<i16, 8, Wrap(_mm_cmpgt_epi16), + THelperCMPGT<i16>, int16x8_t>(); +} + +void TSSEEmulTest::Test_mm_cmpgt_epi32() { + Test_mm_dualcmp<i32, 4, Wrap(_mm_cmpgt_epi32), + THelperCMPGT<i32>, int32x4_t>(); +} + +void TSSEEmulTest::Test_mm_cmpgt_ps() { + struct THelperFloat { + static float Call(const float op1, const float op2) { + union Cast { + unsigned int AsUInt; + float AsFloat; + }; + Cast value; + value.AsUInt = op1 > op2 ? 0xFFFFFFFF : 0; + return value.AsFloat; + } + }; + + Test_mm_dualcmp<float, 4, WrapF(_mm_cmpgt_ps), + THelperFloat, float32x4_t, __m128>(); +} + +template <typename TElem> +struct THelperCMPLT { + static TElem Call(const TElem op1, const TElem op2) { + return op1 < op2 ? ~TElem(0) : TElem(0); + } +}; + +void TSSEEmulTest::Test_mm_cmplt_epi8() { + Test_mm_dualcmp<i8, 16, Wrap(_mm_cmplt_epi8), + THelperCMPLT<i8>, int8x16_t>(); +} + +void TSSEEmulTest::Test_mm_cmplt_epi16() { + Test_mm_dualcmp<i16, 8, Wrap(_mm_cmplt_epi16), + THelperCMPLT<i16>, int16x8_t>(); +} + +void TSSEEmulTest::Test_mm_cmplt_epi32() { + Test_mm_dualcmp<i32, 4, Wrap(_mm_cmplt_epi32), + THelperCMPLT<i32>, int32x4_t>(); +} + +template <typename TElem, int elemCount, typename TFunc, typename TOp, typename TVectorType> -void TSSEEmulTest::Test_mm_setter_epiXX() { - char data[64] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', - '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; - TElem* dataw = reinterpret_cast<TElem*>(&data); - - for (unsigned dataItem = 0; dataItem < elemCount * 4; ++dataItem) { +void TSSEEmulTest::Test_mm_setter_epiXX() { + char data[64] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', + '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; + TElem* dataw = reinterpret_cast<TElem*>(&data); + + for (unsigned dataItem = 0; dataItem < elemCount * 4; ++dataItem) { TVectorType value = TFunc(dataw[dataItem]); - - for (unsigned i = 0; i < elemCount; ++i) - UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<TOp>::As(value)[i]); - } -} - -void TSSEEmulTest::Test_mm_set1_epi8() { - Test_mm_setter_epiXX<i8, 16, Wrap(_mm_set1_epi8), int8x16_t, __m128i>(); -} -void TSSEEmulTest::Test_mm_set1_epi16() { - Test_mm_setter_epiXX<i16, 8, Wrap(_mm_set1_epi16), int16x8_t, __m128i>(); -} -void TSSEEmulTest::Test_mm_set1_epi32() { - Test_mm_setter_epiXX<i32, 4, Wrap(_mm_set1_epi32), int32x4_t, __m128i>(); -} -void TSSEEmulTest::Test_mm_set1_ps() { - Test_mm_setter_epiXX<float, 4, WrapF(_mm_set1_ps), float32x4_t, __m128>(); -} - + + for (unsigned i = 0; i < elemCount; ++i) + UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<TOp>::As(value)[i]); + } +} + +void TSSEEmulTest::Test_mm_set1_epi8() { + Test_mm_setter_epiXX<i8, 16, Wrap(_mm_set1_epi8), int8x16_t, __m128i>(); +} +void TSSEEmulTest::Test_mm_set1_epi16() { + Test_mm_setter_epiXX<i16, 8, Wrap(_mm_set1_epi16), int16x8_t, __m128i>(); +} +void TSSEEmulTest::Test_mm_set1_epi32() { + Test_mm_setter_epiXX<i32, 4, Wrap(_mm_set1_epi32), int32x4_t, __m128i>(); +} +void TSSEEmulTest::Test_mm_set1_ps() { + Test_mm_setter_epiXX<float, 4, WrapF(_mm_set1_ps), float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_set_ps1() { Test_mm_setter_epiXX<float, 4, WrapF(_mm_set_ps1), float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_setzero_si128() { - __m128i value = _mm_setzero_si128(); - for (unsigned i = 0; i < 4; ++i) - UNIT_ASSERT_EQUAL(0, TQType<uint32x4_t>::As(value)[i]); -} - -void TSSEEmulTest::Test_mm_setzero_ps() { - __m128 value = _mm_setzero_ps(); - for (unsigned i = 0; i < 4; ++i) - UNIT_ASSERT_EQUAL(0.0, TQType<float32x4_t>::As(value)[i]); -} - +void TSSEEmulTest::Test_mm_setzero_si128() { + __m128i value = _mm_setzero_si128(); + for (unsigned i = 0; i < 4; ++i) + UNIT_ASSERT_EQUAL(0, TQType<uint32x4_t>::As(value)[i]); +} + +void TSSEEmulTest::Test_mm_setzero_ps() { + __m128 value = _mm_setzero_ps(); + for (unsigned i = 0; i < 4; ++i) + UNIT_ASSERT_EQUAL(0.0, TQType<float32x4_t>::As(value)[i]); +} + void TSSEEmulTest::Test_mm_setzero_pd() { __m128d value = _mm_setzero_pd(); for (unsigned i = 0; i < 2; ++i) UNIT_ASSERT_EQUAL(0.0, TQType<float64x2_t>::As(value)[i]); } -void TSSEEmulTest::Test_mm_loadl_epi64() { - char data[64] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', - '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; - ui64* dataw = reinterpret_cast<ui64*>(&data); - - for (unsigned dataItem = 0; dataItem < 8; ++dataItem) { - __m128i value = _mm_loadl_epi64((__m128i const*)&dataw[dataItem]); - - UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<uint64x2_t>::As(value)[0]); - UNIT_ASSERT_EQUAL(0, TQType<uint64x2_t>::As(value)[1]); - } -} - -void TSSEEmulTest::Test_mm_storel_epi64() { - char data[64] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', - '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', - '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; - ui64* dataw = reinterpret_cast<ui64*>(&data); - - for (unsigned dataItem = 0; dataItem < 4; ++dataItem) { - __m128i value = _mm_loadu_si128((__m128i*)&dataw[dataItem * 2]); - - ui64 buf[2] = {55, 81}; - _mm_storel_epi64((__m128i*)&buf, value); - - UNIT_ASSERT_EQUAL(dataw[dataItem * 2], buf[0]); - UNIT_ASSERT_EQUAL(81, buf[1]); - } -} - -void TSSEEmulTest::Test_mm_shuffle_epi32() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - ui32* dataw = reinterpret_cast<ui32*>(&data); - __m128i value = _mm_loadu_si128((__m128i*)&data); - - int coding[4] = {1, 3, 0, 2}; - __m128i result = _mm_shuffle_epi32(value, _MM_SHUFFLE(2, 0, 3, 1)); - - for (unsigned i = 0; i < 4; ++i) - UNIT_ASSERT_EQUAL(dataw[coding[i]], - TQType<uint32x4_t>::As(result)[i]); -} - -static int GetHighBitAt(char data, int at) { - ui8 udata = data & 0x80; - return int(udata >> 7) << at; -} - -void TSSEEmulTest::Test_mm_movemask_epi8() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - __m128i value = _mm_loadu_si128((__m128i*)&data); - - int result = _mm_movemask_epi8(value); - int verify = 0; - for (unsigned i = 0; i < 16; ++i) { - verify |= GetHighBitAt(data[i], i); - } - - UNIT_ASSERT_EQUAL(result, verify); -} - -void TSSEEmulTest::Test_mm_movemask_ps() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - __m128 value = _mm_loadu_ps((float*)&data); - - int result = _mm_movemask_ps(value); - int verify = 0; - for (unsigned i = 0; i < 4; ++i) { - verify |= GetHighBitAt(data[i * 4 + 3], i); - } - - UNIT_ASSERT_EQUAL(result, verify); -} - +void TSSEEmulTest::Test_mm_loadl_epi64() { + char data[64] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', + '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; + ui64* dataw = reinterpret_cast<ui64*>(&data); + + for (unsigned dataItem = 0; dataItem < 8; ++dataItem) { + __m128i value = _mm_loadl_epi64((__m128i const*)&dataw[dataItem]); + + UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<uint64x2_t>::As(value)[0]); + UNIT_ASSERT_EQUAL(0, TQType<uint64x2_t>::As(value)[1]); + } +} + +void TSSEEmulTest::Test_mm_storel_epi64() { + char data[64] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C', + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF', + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C', + '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44', + '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'}; + ui64* dataw = reinterpret_cast<ui64*>(&data); + + for (unsigned dataItem = 0; dataItem < 4; ++dataItem) { + __m128i value = _mm_loadu_si128((__m128i*)&dataw[dataItem * 2]); + + ui64 buf[2] = {55, 81}; + _mm_storel_epi64((__m128i*)&buf, value); + + UNIT_ASSERT_EQUAL(dataw[dataItem * 2], buf[0]); + UNIT_ASSERT_EQUAL(81, buf[1]); + } +} + +void TSSEEmulTest::Test_mm_shuffle_epi32() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + ui32* dataw = reinterpret_cast<ui32*>(&data); + __m128i value = _mm_loadu_si128((__m128i*)&data); + + int coding[4] = {1, 3, 0, 2}; + __m128i result = _mm_shuffle_epi32(value, _MM_SHUFFLE(2, 0, 3, 1)); + + for (unsigned i = 0; i < 4; ++i) + UNIT_ASSERT_EQUAL(dataw[coding[i]], + TQType<uint32x4_t>::As(result)[i]); +} + +static int GetHighBitAt(char data, int at) { + ui8 udata = data & 0x80; + return int(udata >> 7) << at; +} + +void TSSEEmulTest::Test_mm_movemask_epi8() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + __m128i value = _mm_loadu_si128((__m128i*)&data); + + int result = _mm_movemask_epi8(value); + int verify = 0; + for (unsigned i = 0; i < 16; ++i) { + verify |= GetHighBitAt(data[i], i); + } + + UNIT_ASSERT_EQUAL(result, verify); +} + +void TSSEEmulTest::Test_mm_movemask_ps() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + __m128 value = _mm_loadu_ps((float*)&data); + + int result = _mm_movemask_ps(value); + int verify = 0; + for (unsigned i = 0; i < 4; ++i) { + verify |= GetHighBitAt(data[i * 4 + 3], i); + } + + UNIT_ASSERT_EQUAL(result, verify); +} + void TSSEEmulTest::Test_mm_movemask_ps_2() { char data[16] = { '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', @@ -1414,19 +1414,19 @@ void TSSEEmulTest::Test_mm_movemask_ps_2() { UNIT_ASSERT_EQUAL(result, 0xf); } -void TSSEEmulTest::Test_mm_cvtsi128_si32() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - __m128i value = _mm_loadu_si128((__m128i*)&data); - - int result = _mm_cvtsi128_si32(value); - i32* datap = reinterpret_cast<i32*>(&data); - int verify = datap[0]; - - UNIT_ASSERT_EQUAL(result, verify); -} - +void TSSEEmulTest::Test_mm_cvtsi128_si32() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + __m128i value = _mm_loadu_si128((__m128i*)&data); + + int result = _mm_cvtsi128_si32(value); + i32* datap = reinterpret_cast<i32*>(&data); + int verify = datap[0]; + + UNIT_ASSERT_EQUAL(result, verify); +} + void TSSEEmulTest::Test_mm_cvtsi128_si64() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1440,52 +1440,52 @@ void TSSEEmulTest::Test_mm_cvtsi128_si64() { UNIT_ASSERT_EQUAL(result, verify); } -void TSSEEmulTest::Test_mm_set_epi16() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - i16* dataw = reinterpret_cast<i16*>(&data); - ui64* dataq = reinterpret_cast<ui64*>(&data); - - __m128i result = _mm_set_epi16(dataw[7], dataw[6], dataw[5], dataw[4], - dataw[3], dataw[2], dataw[1], dataw[0]); - ui64 buf[2] = {53, 81}; - _mm_storeu_si128((__m128i*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataq[0]); - UNIT_ASSERT_EQUAL(buf[1], dataq[1]); -} - -void TSSEEmulTest::Test_mm_set_epi32() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - i32* dataw = reinterpret_cast<i32*>(&data); - ui64* dataq = reinterpret_cast<ui64*>(&data); - - __m128i result = _mm_set_epi32(dataw[3], dataw[2], dataw[1], dataw[0]); - ui64 buf[2] = {53, 81}; - _mm_storeu_si128((__m128i*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataq[0]); - UNIT_ASSERT_EQUAL(buf[1], dataq[1]); -} - -void TSSEEmulTest::Test_mm_set_ps() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - float* dataw = reinterpret_cast<float*>(&data); - ui64* dataq = reinterpret_cast<ui64*>(&data); - - __m128 result = _mm_set_ps(dataw[3], dataw[2], dataw[1], dataw[0]); - ui64 buf[2] = {53, 81}; - _mm_storeu_ps((float*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataq[0]); - UNIT_ASSERT_EQUAL(buf[1], dataq[1]); -} - +void TSSEEmulTest::Test_mm_set_epi16() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + i16* dataw = reinterpret_cast<i16*>(&data); + ui64* dataq = reinterpret_cast<ui64*>(&data); + + __m128i result = _mm_set_epi16(dataw[7], dataw[6], dataw[5], dataw[4], + dataw[3], dataw[2], dataw[1], dataw[0]); + ui64 buf[2] = {53, 81}; + _mm_storeu_si128((__m128i*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataq[0]); + UNIT_ASSERT_EQUAL(buf[1], dataq[1]); +} + +void TSSEEmulTest::Test_mm_set_epi32() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + i32* dataw = reinterpret_cast<i32*>(&data); + ui64* dataq = reinterpret_cast<ui64*>(&data); + + __m128i result = _mm_set_epi32(dataw[3], dataw[2], dataw[1], dataw[0]); + ui64 buf[2] = {53, 81}; + _mm_storeu_si128((__m128i*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataq[0]); + UNIT_ASSERT_EQUAL(buf[1], dataq[1]); +} + +void TSSEEmulTest::Test_mm_set_ps() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + float* dataw = reinterpret_cast<float*>(&data); + ui64* dataq = reinterpret_cast<ui64*>(&data); + + __m128 result = _mm_set_ps(dataw[3], dataw[2], dataw[1], dataw[0]); + ui64 buf[2] = {53, 81}; + _mm_storeu_ps((float*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataq[0]); + UNIT_ASSERT_EQUAL(buf[1], dataq[1]); +} + void TSSEEmulTest::Test_mm_set_pd() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1501,22 +1501,22 @@ void TSSEEmulTest::Test_mm_set_pd() { UNIT_ASSERT_EQUAL(buf[1], dataq[1]); } -void TSSEEmulTest::Test_mm_cvtsi32_si128() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - i32* dataw = reinterpret_cast<i32*>(&data); - - __m128i result = _mm_cvtsi32_si128(dataw[0]); - i32 buf[4] = {53, 81, -43, 2132}; - _mm_storeu_si128((__m128i*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataw[0]); - UNIT_ASSERT_EQUAL(buf[1], 0); - UNIT_ASSERT_EQUAL(buf[2], 0); - UNIT_ASSERT_EQUAL(buf[3], 0); -} - +void TSSEEmulTest::Test_mm_cvtsi32_si128() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + i32* dataw = reinterpret_cast<i32*>(&data); + + __m128i result = _mm_cvtsi32_si128(dataw[0]); + i32 buf[4] = {53, 81, -43, 2132}; + _mm_storeu_si128((__m128i*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataw[0]); + UNIT_ASSERT_EQUAL(buf[1], 0); + UNIT_ASSERT_EQUAL(buf[2], 0); + UNIT_ASSERT_EQUAL(buf[3], 0); +} + void TSSEEmulTest::Test_mm_cvtsi64_si128() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1531,44 +1531,44 @@ void TSSEEmulTest::Test_mm_cvtsi64_si128() { UNIT_ASSERT_EQUAL(buf[1], 0); } -template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc> -void TSSEEmulTest::Test_mm_packs_epiXX() { - char data[32] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x00', '\x66', '\x1C', - '\x99', '\x33', '\x1C', '\x55', '\x00', '\x00', '\x00', '\x00', - '\x00', '\xAA', '\x00', '\x00', '\xCC', '\xBB', '\x22', '\xFF'}; - __m128i value0 = _mm_loadu_si128((__m128i*)&data); - __m128i value1 = _mm_loadu_si128(((__m128i*)&data) + 1); - TElem* dataw = reinterpret_cast<TElem*>(&data); - - __m128i result = TFunc(value0, value1); - - TNarrow verify[elemCount]; - for (unsigned i = 0; i < elemCount; ++i) { - TElem sum = dataw[i]; - if (sum > std::numeric_limits<TNarrow>::max()) - sum = std::numeric_limits<TNarrow>::max(); - if (sum < std::numeric_limits<TNarrow>::min()) - sum = std::numeric_limits<TNarrow>::min(); - verify[i] = TNarrow(sum); - } - - ui64* verifyp = (ui64*)&verify; - UNIT_ASSERT_EQUAL(verifyp[0], TQType<uint64x2_t>::As(result)[0]); - UNIT_ASSERT_EQUAL(verifyp[1], TQType<uint64x2_t>::As(result)[1]); -} - -void TSSEEmulTest::Test_mm_packs_epi16() { - Test_mm_packs_epiXX<i16, i8, 16, Wrap(_mm_packs_epi16)>(); -} -void TSSEEmulTest::Test_mm_packs_epi32() { - Test_mm_packs_epiXX<i32, i16, 8, Wrap(_mm_packs_epi32)>(); -} -void TSSEEmulTest::Test_mm_packus_epi16() { - Test_mm_packs_epiXX<i16, ui8, 16, Wrap(_mm_packus_epi16)>(); -} - +template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc> +void TSSEEmulTest::Test_mm_packs_epiXX() { + char data[32] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x00', '\x66', '\x1C', + '\x99', '\x33', '\x1C', '\x55', '\x00', '\x00', '\x00', '\x00', + '\x00', '\xAA', '\x00', '\x00', '\xCC', '\xBB', '\x22', '\xFF'}; + __m128i value0 = _mm_loadu_si128((__m128i*)&data); + __m128i value1 = _mm_loadu_si128(((__m128i*)&data) + 1); + TElem* dataw = reinterpret_cast<TElem*>(&data); + + __m128i result = TFunc(value0, value1); + + TNarrow verify[elemCount]; + for (unsigned i = 0; i < elemCount; ++i) { + TElem sum = dataw[i]; + if (sum > std::numeric_limits<TNarrow>::max()) + sum = std::numeric_limits<TNarrow>::max(); + if (sum < std::numeric_limits<TNarrow>::min()) + sum = std::numeric_limits<TNarrow>::min(); + verify[i] = TNarrow(sum); + } + + ui64* verifyp = (ui64*)&verify; + UNIT_ASSERT_EQUAL(verifyp[0], TQType<uint64x2_t>::As(result)[0]); + UNIT_ASSERT_EQUAL(verifyp[1], TQType<uint64x2_t>::As(result)[1]); +} + +void TSSEEmulTest::Test_mm_packs_epi16() { + Test_mm_packs_epiXX<i16, i8, 16, Wrap(_mm_packs_epi16)>(); +} +void TSSEEmulTest::Test_mm_packs_epi32() { + Test_mm_packs_epiXX<i32, i16, 8, Wrap(_mm_packs_epi32)>(); +} +void TSSEEmulTest::Test_mm_packus_epi16() { + Test_mm_packs_epiXX<i16, ui8, 16, Wrap(_mm_packus_epi16)>(); +} + void TSSEEmulTest::Test_mm_extract_epi8() { alignas(16) char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1594,23 +1594,23 @@ void TSSEEmulTest::Test_mm_extract_epi8() { UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 15)), int(dataw[15])); } -void TSSEEmulTest::Test_mm_extract_epi16() { +void TSSEEmulTest::Test_mm_extract_epi16() { alignas(16) char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; const ui16* dataw = reinterpret_cast<const ui16*>(&data); const __m128i value = _mm_loadu_si128((__m128i*)&data); - - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 1)), int(dataw[1])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 2)), int(dataw[2])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 3)), int(dataw[3])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 4)), int(dataw[4])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 5)), int(dataw[5])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 6)), int(dataw[6])); - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 7)), int(dataw[7])); -} - + + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 1)), int(dataw[1])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 2)), int(dataw[2])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 3)), int(dataw[3])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 4)), int(dataw[4])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 5)), int(dataw[5])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 6)), int(dataw[6])); + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 7)), int(dataw[7])); +} + void TSSEEmulTest::Test_mm_extract_epi64() { alignas(16) char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1635,160 +1635,160 @@ void TSSEEmulTest::Test_mm_extract_epi32() { UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 3)), int(dataw[3])); } -void TSSEEmulTest::Test_MM_TRANSPOSE4_PS() { - char data0[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - char data1[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; - char data2[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - char data3[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; - - __m128 value0 = _mm_loadu_ps((float*)&data0); - __m128 value1 = _mm_loadu_ps((float*)&data1); - __m128 value2 = _mm_loadu_ps((float*)&data2); - __m128 value3 = _mm_loadu_ps((float*)&data3); - - _MM_TRANSPOSE4_PS(value0, value1, value2, value3); - - ui64 tbuf0[2] = {0, 0}; - ui64 tbuf1[2] = {0, 0}; - ui64 tbuf2[2] = {0, 0}; - ui64 tbuf3[2] = {0, 0}; - - _mm_storeu_ps((float*)&tbuf0, value0); - _mm_storeu_ps((float*)&tbuf1, value1); - _mm_storeu_ps((float*)&tbuf2, value2); - _mm_storeu_ps((float*)&tbuf3, value3); - - char tdata0[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55', - '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55'}; - char tdata1[16] = { - '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44', - '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44'}; - char tdata2[16] = { - '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11', - '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11'}; - char tdata3[16] = { - '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF', - '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF'}; - - UNIT_ASSERT(memcmp(tbuf0, tdata0, 16) == 0); - UNIT_ASSERT(memcmp(tbuf1, tdata1, 16) == 0); - UNIT_ASSERT(memcmp(tbuf2, tdata2, 16) == 0); - UNIT_ASSERT(memcmp(tbuf3, tdata3, 16) == 0); -} - -template <typename TFrom, typename TTo, unsigned elemCount, - typename TLoadVector, typename TResultVector, - typename TElemFunc, typename TFunc, typename TOp> -void TSSEEmulTest::Test_mm_convertop() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - TFrom* datap = reinterpret_cast<TFrom*>(&data); - - TLoadVector value = TFuncLoad<TLoadVector>(&data); - - TTo procData[elemCount]; - for (unsigned i = 0; i < elemCount; ++i) { - procData[i] = TElemFunc::Call(datap[i]); - } - - TResultVector result = TFunc(value); - - for (unsigned i = 0; i < elemCount; ++i) { - UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]); - } -} - -void TSSEEmulTest::Test_mm_cvtepi32_ps() { - struct THelper { - static float Call(const i32 op) { - return float(op); - } - }; - Test_mm_convertop<i32, float, 4, __m128i, __m128, - THelper, WrapF(_mm_cvtepi32_ps), float32x4_t>(); -}; - -void TSSEEmulTest::Test_mm_cvtps_epi32() { - struct THelper { - static i32 Call(const float op) { - return i32(op); - } - }; - Test_mm_convertop<float, i32, 4, __m128, __m128i, +void TSSEEmulTest::Test_MM_TRANSPOSE4_PS() { + char data0[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + char data1[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; + char data2[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + char data3[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; + + __m128 value0 = _mm_loadu_ps((float*)&data0); + __m128 value1 = _mm_loadu_ps((float*)&data1); + __m128 value2 = _mm_loadu_ps((float*)&data2); + __m128 value3 = _mm_loadu_ps((float*)&data3); + + _MM_TRANSPOSE4_PS(value0, value1, value2, value3); + + ui64 tbuf0[2] = {0, 0}; + ui64 tbuf1[2] = {0, 0}; + ui64 tbuf2[2] = {0, 0}; + ui64 tbuf3[2] = {0, 0}; + + _mm_storeu_ps((float*)&tbuf0, value0); + _mm_storeu_ps((float*)&tbuf1, value1); + _mm_storeu_ps((float*)&tbuf2, value2); + _mm_storeu_ps((float*)&tbuf3, value3); + + char tdata0[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55', + '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55'}; + char tdata1[16] = { + '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44', + '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44'}; + char tdata2[16] = { + '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11', + '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11'}; + char tdata3[16] = { + '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF', + '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF'}; + + UNIT_ASSERT(memcmp(tbuf0, tdata0, 16) == 0); + UNIT_ASSERT(memcmp(tbuf1, tdata1, 16) == 0); + UNIT_ASSERT(memcmp(tbuf2, tdata2, 16) == 0); + UNIT_ASSERT(memcmp(tbuf3, tdata3, 16) == 0); +} + +template <typename TFrom, typename TTo, unsigned elemCount, + typename TLoadVector, typename TResultVector, + typename TElemFunc, typename TFunc, typename TOp> +void TSSEEmulTest::Test_mm_convertop() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + TFrom* datap = reinterpret_cast<TFrom*>(&data); + + TLoadVector value = TFuncLoad<TLoadVector>(&data); + + TTo procData[elemCount]; + for (unsigned i = 0; i < elemCount; ++i) { + procData[i] = TElemFunc::Call(datap[i]); + } + + TResultVector result = TFunc(value); + + for (unsigned i = 0; i < elemCount; ++i) { + UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]); + } +} + +void TSSEEmulTest::Test_mm_cvtepi32_ps() { + struct THelper { + static float Call(const i32 op) { + return float(op); + } + }; + Test_mm_convertop<i32, float, 4, __m128i, __m128, + THelper, WrapF(_mm_cvtepi32_ps), float32x4_t>(); +}; + +void TSSEEmulTest::Test_mm_cvtps_epi32() { + struct THelper { + static i32 Call(const float op) { + return i32(op); + } + }; + Test_mm_convertop<float, i32, 4, __m128, __m128i, THelper, T_mm_CallWrapper<__m128i, decltype(_mm_cvtps_epi32), _mm_cvtps_epi32>, int32x4_t>(); -}; - -void TSSEEmulTest::Test_mm_cvttps_epi32() { - struct THelper { - static i32 Call(const float op) { - return i32(op); - } - }; - Test_mm_convertop<float, i32, 4, __m128, __m128i, - THelper, Wrap(_mm_cvttps_epi32), int32x4_t>(); -}; - -template <typename TLoadVector, typename TCastVector, - typename TFunc, TFunc* func> -void TSSEEmulTest::Test_mm_castXX() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - - TLoadVector value = TFuncLoad<TLoadVector>(&data); - const TLoadVector constvalue = TFuncLoad<TLoadVector>(&data); - TCastVector casted = func(value); - const TCastVector constcasted = func(constvalue); - char verify[16]; - char constverify[16]; - TFuncStore<TCastVector>(&verify, casted); - TFuncStore<TCastVector>(&constverify, constcasted); - - UNIT_ASSERT(memcmp(&data, &verify, 16) == 0); - UNIT_ASSERT(memcmp(&data, &constverify, 16) == 0); -}; - -void TSSEEmulTest::Test_mm_castsi128_ps() { - Test_mm_castXX<__m128i, __m128, - decltype(_mm_castsi128_ps), _mm_castsi128_ps>(); -} - -void TSSEEmulTest::Test_mm_castps_si128() { - Test_mm_castXX<__m128, __m128i, - decltype(_mm_castps_si128), _mm_castps_si128>(); -} - -void TSSEEmulTest::Test_mm_mul_epu32() { - char data0[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - char data1[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; - ui32* dataw0 = reinterpret_cast<ui32*>(&data0); - ui32* dataw1 = reinterpret_cast<ui32*>(&data1); - - __m128i value0 = _mm_loadu_si128((__m128i*)&data0); - __m128i value1 = _mm_loadu_si128((__m128i*)&data1); - +}; + +void TSSEEmulTest::Test_mm_cvttps_epi32() { + struct THelper { + static i32 Call(const float op) { + return i32(op); + } + }; + Test_mm_convertop<float, i32, 4, __m128, __m128i, + THelper, Wrap(_mm_cvttps_epi32), int32x4_t>(); +}; + +template <typename TLoadVector, typename TCastVector, + typename TFunc, TFunc* func> +void TSSEEmulTest::Test_mm_castXX() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + + TLoadVector value = TFuncLoad<TLoadVector>(&data); + const TLoadVector constvalue = TFuncLoad<TLoadVector>(&data); + TCastVector casted = func(value); + const TCastVector constcasted = func(constvalue); + char verify[16]; + char constverify[16]; + TFuncStore<TCastVector>(&verify, casted); + TFuncStore<TCastVector>(&constverify, constcasted); + + UNIT_ASSERT(memcmp(&data, &verify, 16) == 0); + UNIT_ASSERT(memcmp(&data, &constverify, 16) == 0); +}; + +void TSSEEmulTest::Test_mm_castsi128_ps() { + Test_mm_castXX<__m128i, __m128, + decltype(_mm_castsi128_ps), _mm_castsi128_ps>(); +} + +void TSSEEmulTest::Test_mm_castps_si128() { + Test_mm_castXX<__m128, __m128i, + decltype(_mm_castps_si128), _mm_castps_si128>(); +} + +void TSSEEmulTest::Test_mm_mul_epu32() { + char data0[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + char data1[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'}; + ui32* dataw0 = reinterpret_cast<ui32*>(&data0); + ui32* dataw1 = reinterpret_cast<ui32*>(&data1); + + __m128i value0 = _mm_loadu_si128((__m128i*)&data0); + __m128i value1 = _mm_loadu_si128((__m128i*)&data1); + ui64 mul0 = (ui64) dataw0[0] * (ui64) dataw1[0]; ui64 mul1 = (ui64) dataw0[2] * (ui64) dataw1[2]; - - __m128i result = _mm_mul_epu32(value0, value1); - - UNIT_ASSERT_EQUAL(mul0, TQType<uint64x2_t>::As(result)[0]); - UNIT_ASSERT_EQUAL(mul1, TQType<uint64x2_t>::As(result)[1]); -} + + __m128i result = _mm_mul_epu32(value0, value1); + + UNIT_ASSERT_EQUAL(mul0, TQType<uint64x2_t>::As(result)[0]); + UNIT_ASSERT_EQUAL(mul1, TQType<uint64x2_t>::As(result)[1]); +} void TSSEEmulTest::Test_mm_cmpunord_ps() { alignas(16) float valuesBits[4] = {1.f, 2.f, 3.f, 4.f}; diff --git a/library/cpp/sse/ut/ya.make b/library/cpp/sse/ut/ya.make index 45e104971e..14cac6727a 100644 --- a/library/cpp/sse/ut/ya.make +++ b/library/cpp/sse/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST_FOR(library/cpp/sse) - + OWNER(danlark) - -SRCS( + +SRCS( test.cpp -) - +) + IF (ARCH_X86_64) CFLAGS(-msse4.1 -msse4.2) ENDIF() -END() +END() diff --git a/library/cpp/testing/unittest/registar.h b/library/cpp/testing/unittest/registar.h index 44517a0092..28256b53f2 100644 --- a/library/cpp/testing/unittest/registar.h +++ b/library/cpp/testing/unittest/registar.h @@ -279,8 +279,8 @@ private: \ } \ \ virtual void Execute() override { \ - this->AtStart(); - + this->AtStart(); + #ifndef UT_SKIP_EXCEPTIONS #define CATCH_REACTION(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), context) #define CATCH_REACTION_BT(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), (e.BackTrace() ? e.BackTrace()->PrintToString() : TString()), context) diff --git a/library/cpp/testing/unittest/utmain.cpp b/library/cpp/testing/unittest/utmain.cpp index 305bc6b40f..cec11773ed 100644 --- a/library/cpp/testing/unittest/utmain.cpp +++ b/library/cpp/testing/unittest/utmain.cpp @@ -207,7 +207,7 @@ public: } else { TString suite = TString(name).substr(0, colon); EnabledSuites_.insert(suite); - EnabledSuites_.insert(name); + EnabledSuites_.insert(name); EnabledTests_.insert(name); EnabledTests_.insert(TString() + name + "::*"); } diff --git a/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp b/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp index c3027ea544..5e217c25ad 100644 --- a/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp +++ b/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp @@ -1,188 +1,188 @@ #include <library/cpp/threading/light_rw_lock/lightrwlock.h> -#include <util/random/random.h> - -#ifdef _linux_ -// Light rw lock is implemented only for linux - -using namespace NS_LightRWLock; - -#include <pthread.h> -#include <stdlib.h> -#include <stdio.h> - -#define LIGHT - -#ifdef RWSPINLOCK +#include <util/random/random.h> + +#ifdef _linux_ +// Light rw lock is implemented only for linux + +using namespace NS_LightRWLock; + +#include <pthread.h> +#include <stdlib.h> +#include <stdio.h> + +#define LIGHT + +#ifdef RWSPINLOCK #include <library/cpp/lwtrace/rwspinlock.h> -#endif - -#define CHECK_LOGIC 1 -#define LOOPCOUNT 1000000 -#define RANRCOUNT 100 -#define THREADCOUNT 40 -#define WRITELOCKS 100 - -#if defined(_MSC_VER) -static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { - return _InterlockedExchangeAdd((&item, value); -} -#elif defined(__GNUC__) -#else +#endif + +#define CHECK_LOGIC 1 +#define LOOPCOUNT 1000000 +#define RANRCOUNT 100 +#define THREADCOUNT 40 +#define WRITELOCKS 100 + +#if defined(_MSC_VER) +static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { + return _InterlockedExchangeAdd((&item, value); +} +#elif defined(__GNUC__) +#else #error unsupported platform -#endif - -class TPosixRWLock { -public: - TPosixRWLock() { - } - - ~TPosixRWLock() { - pthread_rwlock_destroy(&rwlock); - } - - TPosixRWLock(const TPosixRWLock&) = delete; - void operator=(const TPosixRWLock&) = delete; - -private: - pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER; - friend class TPosixRWShareLocker; - friend class TPosixRWExclusiveLocker; -}; - -#if defined(LIGHT) -TLightRWLock __attribute__((aligned(64))) rwlock; -#elif defined(POSIX) -TPosixRWLock rwlock; -#elif defined(RWSPINLOCK) -TRWSpinLock __attribute__((aligned(64))) rwlock; -#else -#error "define lock type" -#endif - -volatile __attribute__((aligned(64))) int checkIt = 0; -volatile int checkExcl = 0; - -class TPosixRWShareLocker { -public: - TPosixRWShareLocker(TPosixRWLock& lock) - : LockP_(&lock) - { - pthread_rwlock_rdlock(&LockP_->rwlock); - } - - ~TPosixRWShareLocker() { - pthread_rwlock_unlock(&LockP_->rwlock); - } - - TPosixRWShareLocker(const TPosixRWShareLocker&) = delete; - void operator=(const TPosixRWShareLocker&) = delete; - -private: - TPosixRWLock* LockP_; -}; - -class TPosixRWExclusiveLocker { -public: - TPosixRWExclusiveLocker(TPosixRWLock& lock) - : LockP_(&lock) - { - pthread_rwlock_wrlock(&LockP_->rwlock); - } - - ~TPosixRWExclusiveLocker() { - pthread_rwlock_unlock(&LockP_->rwlock); - } - TPosixRWExclusiveLocker(const TPosixRWExclusiveLocker&) = delete; - void operator=(const TPosixRWExclusiveLocker&) = delete; - -private: - TPosixRWLock* LockP_; -}; - -template <typename TLocker, bool excl> -static Y_FORCE_INLINE void Run() { - TLocker lockIt(rwlock); - -#if defined(CHECK_LOGIC) && CHECK_LOGIC - if (!excl && checkExcl == 1) { - printf("there is a bug\n"); - } - - int result = AtomicFetchAdd(checkIt, 1); - if (excl) - checkExcl = 1; - - if (excl && result > 1) - printf("there is a bug\n"); -#endif - - for (unsigned w = 0; w < RANRCOUNT; ++w) - RandomNumber<ui32>(); - -#if defined(CHECK_LOGIC) && CHECK_LOGIC - if (excl) - checkExcl = 0; - - AtomicFetchAdd(checkIt, -1); -#endif -} - -#ifdef LIGHT -static void* fast_thread_start(__attribute__((unused)) void* arg) { - for (unsigned q = 0; q < LOOPCOUNT; ++q) { - char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; - if (excl) - Run<TLightWriteGuard, 1>(); - else - Run<TLightReadGuard, 0>(); - } - return NULL; -} -#endif - -#ifdef POSIX -static void* fast_thread_start(__attribute__((unused)) void* arg) { - for (unsigned q = 0; q < LOOPCOUNT; ++q) { - char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; - if (excl) - Run<TPosixRWExclusiveLocker, 1>(); - else - Run<TPosixRWShareLocker, 0>(); - } - return NULL; -} -#endif - -#ifdef RWSPINLOCK -static void* fast_thread_start(__attribute__((unused)) void* arg) { - for (unsigned q = 0; q < LOOPCOUNT; ++q) { - char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; - if (excl) - Run<TWriteSpinLockGuard, 1>(); - else - Run<TReadSpinLockGuard, 0>(); - } - return NULL; -} -#endif - -int main() { - pthread_t threads[THREADCOUNT]; - - for (unsigned q = 0; q < THREADCOUNT; ++q) { - pthread_create(&(threads[q]), NULL, &fast_thread_start, NULL); - } - - for (unsigned q = 0; q < THREADCOUNT; ++q) - pthread_join(threads[q], NULL); - - return 0; -} - -#else // !_linux_ - -int main() { - return 0; -} - -#endif +#endif + +class TPosixRWLock { +public: + TPosixRWLock() { + } + + ~TPosixRWLock() { + pthread_rwlock_destroy(&rwlock); + } + + TPosixRWLock(const TPosixRWLock&) = delete; + void operator=(const TPosixRWLock&) = delete; + +private: + pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER; + friend class TPosixRWShareLocker; + friend class TPosixRWExclusiveLocker; +}; + +#if defined(LIGHT) +TLightRWLock __attribute__((aligned(64))) rwlock; +#elif defined(POSIX) +TPosixRWLock rwlock; +#elif defined(RWSPINLOCK) +TRWSpinLock __attribute__((aligned(64))) rwlock; +#else +#error "define lock type" +#endif + +volatile __attribute__((aligned(64))) int checkIt = 0; +volatile int checkExcl = 0; + +class TPosixRWShareLocker { +public: + TPosixRWShareLocker(TPosixRWLock& lock) + : LockP_(&lock) + { + pthread_rwlock_rdlock(&LockP_->rwlock); + } + + ~TPosixRWShareLocker() { + pthread_rwlock_unlock(&LockP_->rwlock); + } + + TPosixRWShareLocker(const TPosixRWShareLocker&) = delete; + void operator=(const TPosixRWShareLocker&) = delete; + +private: + TPosixRWLock* LockP_; +}; + +class TPosixRWExclusiveLocker { +public: + TPosixRWExclusiveLocker(TPosixRWLock& lock) + : LockP_(&lock) + { + pthread_rwlock_wrlock(&LockP_->rwlock); + } + + ~TPosixRWExclusiveLocker() { + pthread_rwlock_unlock(&LockP_->rwlock); + } + TPosixRWExclusiveLocker(const TPosixRWExclusiveLocker&) = delete; + void operator=(const TPosixRWExclusiveLocker&) = delete; + +private: + TPosixRWLock* LockP_; +}; + +template <typename TLocker, bool excl> +static Y_FORCE_INLINE void Run() { + TLocker lockIt(rwlock); + +#if defined(CHECK_LOGIC) && CHECK_LOGIC + if (!excl && checkExcl == 1) { + printf("there is a bug\n"); + } + + int result = AtomicFetchAdd(checkIt, 1); + if (excl) + checkExcl = 1; + + if (excl && result > 1) + printf("there is a bug\n"); +#endif + + for (unsigned w = 0; w < RANRCOUNT; ++w) + RandomNumber<ui32>(); + +#if defined(CHECK_LOGIC) && CHECK_LOGIC + if (excl) + checkExcl = 0; + + AtomicFetchAdd(checkIt, -1); +#endif +} + +#ifdef LIGHT +static void* fast_thread_start(__attribute__((unused)) void* arg) { + for (unsigned q = 0; q < LOOPCOUNT; ++q) { + char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; + if (excl) + Run<TLightWriteGuard, 1>(); + else + Run<TLightReadGuard, 0>(); + } + return NULL; +} +#endif + +#ifdef POSIX +static void* fast_thread_start(__attribute__((unused)) void* arg) { + for (unsigned q = 0; q < LOOPCOUNT; ++q) { + char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; + if (excl) + Run<TPosixRWExclusiveLocker, 1>(); + else + Run<TPosixRWShareLocker, 0>(); + } + return NULL; +} +#endif + +#ifdef RWSPINLOCK +static void* fast_thread_start(__attribute__((unused)) void* arg) { + for (unsigned q = 0; q < LOOPCOUNT; ++q) { + char excl = (RandomNumber<ui32>() % WRITELOCKS) == 0; + if (excl) + Run<TWriteSpinLockGuard, 1>(); + else + Run<TReadSpinLockGuard, 0>(); + } + return NULL; +} +#endif + +int main() { + pthread_t threads[THREADCOUNT]; + + for (unsigned q = 0; q < THREADCOUNT; ++q) { + pthread_create(&(threads[q]), NULL, &fast_thread_start, NULL); + } + + for (unsigned q = 0; q < THREADCOUNT; ++q) + pthread_join(threads[q], NULL); + + return 0; +} + +#else // !_linux_ + +int main() { + return 0; +} + +#endif diff --git a/library/cpp/threading/light_rw_lock/bench/ya.make b/library/cpp/threading/light_rw_lock/bench/ya.make index 7969b52a50..ed89e3a9b0 100644 --- a/library/cpp/threading/light_rw_lock/bench/ya.make +++ b/library/cpp/threading/light_rw_lock/bench/ya.make @@ -1,13 +1,13 @@ -PROGRAM(lightrwlock_test) - -OWNER(agri) - -SRCS( - lightrwlock_test.cpp -) - -PEERDIR( +PROGRAM(lightrwlock_test) + +OWNER(agri) + +SRCS( + lightrwlock_test.cpp +) + +PEERDIR( library/cpp/threading/light_rw_lock -) - -END() +) + +END() diff --git a/library/cpp/threading/light_rw_lock/lightrwlock.cpp b/library/cpp/threading/light_rw_lock/lightrwlock.cpp index fbb63fd47f..58456907d2 100644 --- a/library/cpp/threading/light_rw_lock/lightrwlock.cpp +++ b/library/cpp/threading/light_rw_lock/lightrwlock.cpp @@ -1,113 +1,113 @@ -#include "lightrwlock.h" -#include <util/system/spinlock.h> - -#if defined(_linux_) - -using namespace NS_LightRWLock; - -void TLightRWLock::WaitForUntrappedShared() { - for (;;) { - for (ui32 i = 0; i < SpinCount_; ++i) { - SpinLockPause(); - - if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) - return; - } - - SequenceStore(UnshareFutex_, 1); - if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { - AtomicStore(UnshareFutex_, 0); - return; - } - FutexWait(UnshareFutex_, 1); - } -} - -void TLightRWLock::WaitForExclusiveAndUntrappedShared() { - for (;;) { - for (ui32 i = 0; i < SpinCount_; ++i) { - SpinLockPause(); - - if (AtomicLoad(Counter_) >= 0) - goto try_to_get_lock; - if (AtomicLoad(TrappedFutex_) == 1) - goto skip_store_trapped; - } - - SequenceStore(TrappedFutex_, 1); - skip_store_trapped: - - if (AtomicLoad(Counter_) < 0) { - FutexWait(TrappedFutex_, 1); - } - - try_to_get_lock: - if (!AtomicSetBit(Counter_, 31)) - break; - } - - for (ui32 j = 0;; ++j) { - for (ui32 i = 0; i < SpinCount_; ++i) { - if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) - return; - - SpinLockPause(); - } - - SequenceStore(UnshareFutex_, 1); - - if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { - AtomicStore(UnshareFutex_, 0); - return; - } - - FutexWait(UnshareFutex_, 1); - } -} - -void TLightRWLock::WaitForUntrappedAndAcquireRead() { - if (AtomicFetchAdd(Counter_, -1) < 0) - goto skip_lock_try; - - for (;;) { - again: - if (Y_UNLIKELY(AtomicFetchAdd(Counter_, 1) >= 0)) { - return; - } else { - if (AtomicFetchAdd(Counter_, -1) >= 0) - goto again; - } - - skip_lock_try: +#include "lightrwlock.h" +#include <util/system/spinlock.h> + +#if defined(_linux_) + +using namespace NS_LightRWLock; + +void TLightRWLock::WaitForUntrappedShared() { + for (;;) { + for (ui32 i = 0; i < SpinCount_; ++i) { + SpinLockPause(); + + if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) + return; + } + + SequenceStore(UnshareFutex_, 1); + if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { + AtomicStore(UnshareFutex_, 0); + return; + } + FutexWait(UnshareFutex_, 1); + } +} + +void TLightRWLock::WaitForExclusiveAndUntrappedShared() { + for (;;) { + for (ui32 i = 0; i < SpinCount_; ++i) { + SpinLockPause(); + + if (AtomicLoad(Counter_) >= 0) + goto try_to_get_lock; + if (AtomicLoad(TrappedFutex_) == 1) + goto skip_store_trapped; + } + + SequenceStore(TrappedFutex_, 1); + skip_store_trapped: + + if (AtomicLoad(Counter_) < 0) { + FutexWait(TrappedFutex_, 1); + } + + try_to_get_lock: + if (!AtomicSetBit(Counter_, 31)) + break; + } + + for (ui32 j = 0;; ++j) { + for (ui32 i = 0; i < SpinCount_; ++i) { + if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) + return; + + SpinLockPause(); + } + + SequenceStore(UnshareFutex_, 1); + + if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { + AtomicStore(UnshareFutex_, 0); + return; + } + + FutexWait(UnshareFutex_, 1); + } +} + +void TLightRWLock::WaitForUntrappedAndAcquireRead() { + if (AtomicFetchAdd(Counter_, -1) < 0) + goto skip_lock_try; + + for (;;) { + again: + if (Y_UNLIKELY(AtomicFetchAdd(Counter_, 1) >= 0)) { + return; + } else { + if (AtomicFetchAdd(Counter_, -1) >= 0) + goto again; + } + + skip_lock_try: if (AtomicLoad(UnshareFutex_) && (AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { - SequenceStore(UnshareFutex_, 0); - FutexWake(UnshareFutex_, 1); - } - - for (;;) { - for (ui32 i = 0; i < SpinCount_; ++i) { - SpinLockPause(); - - if (AtomicLoad(Counter_) >= 0) - goto again; - if (AtomicLoad(TrappedFutex_) == 1) - goto skip_store_trapped; - } - - SequenceStore(TrappedFutex_, 1); - skip_store_trapped: - - if (AtomicLoad(Counter_) < 0) { - FutexWait(TrappedFutex_, 1); - if (AtomicLoad(Counter_) < 0) - goto again; - } else if (AtomicLoad(TrappedFutex_)) { - SequenceStore(TrappedFutex_, 0); - FutexWake(TrappedFutex_, 0x7fffffff); - } - break; - } - } -} - -#endif // _linux_ + SequenceStore(UnshareFutex_, 0); + FutexWake(UnshareFutex_, 1); + } + + for (;;) { + for (ui32 i = 0; i < SpinCount_; ++i) { + SpinLockPause(); + + if (AtomicLoad(Counter_) >= 0) + goto again; + if (AtomicLoad(TrappedFutex_) == 1) + goto skip_store_trapped; + } + + SequenceStore(TrappedFutex_, 1); + skip_store_trapped: + + if (AtomicLoad(Counter_) < 0) { + FutexWait(TrappedFutex_, 1); + if (AtomicLoad(Counter_) < 0) + goto again; + } else if (AtomicLoad(TrappedFutex_)) { + SequenceStore(TrappedFutex_, 0); + FutexWake(TrappedFutex_, 0x7fffffff); + } + break; + } + } +} + +#endif // _linux_ diff --git a/library/cpp/threading/light_rw_lock/lightrwlock.h b/library/cpp/threading/light_rw_lock/lightrwlock.h index 931a1817bc..4411787169 100644 --- a/library/cpp/threading/light_rw_lock/lightrwlock.h +++ b/library/cpp/threading/light_rw_lock/lightrwlock.h @@ -1,45 +1,45 @@ -#pragma once +#pragma once -#include <util/system/rwlock.h> +#include <util/system/rwlock.h> #include <util/system/sanitizers.h> - -#if defined(_linux_) -/* TLightRWLock is optimized for read lock and very fast lock/unlock switching. - Read lock increments counter. - Write lock sets highest bit of counter (makes counter negative). - - Whenever a thread tries to acquire read lock that thread increments - the counter. If the thread gets negative value of the counter right just - after the increment that means write lock was acquired in another thread. - In that case the thread decrements the counter back, wakes one thread on - UnshareFutex, waits on the TrappedFutex and then tries acquire read lock - from the beginning. - If the thread gets positive value of the counter after the increment - then read lock was successfully acquired and - the thread can proceed execution. - - Whenever a thread tries to acquire write lock that thread set the highest bit - of the counter. If the thread determine that the bit was set previously then - write lock was acquired in another thread. In that case the thread waits on - the TrappedFutex and then tries again from the beginning. - If the highest bit was successfully set then thread check if any read lock - exists at the moment. If so the thread waits on UnshareFutex. If there is - no more read locks then write lock was successfully acquired and the thread - can proceed execution. -*/ - -#include <linux/futex.h> + +#if defined(_linux_) +/* TLightRWLock is optimized for read lock and very fast lock/unlock switching. + Read lock increments counter. + Write lock sets highest bit of counter (makes counter negative). + + Whenever a thread tries to acquire read lock that thread increments + the counter. If the thread gets negative value of the counter right just + after the increment that means write lock was acquired in another thread. + In that case the thread decrements the counter back, wakes one thread on + UnshareFutex, waits on the TrappedFutex and then tries acquire read lock + from the beginning. + If the thread gets positive value of the counter after the increment + then read lock was successfully acquired and + the thread can proceed execution. + + Whenever a thread tries to acquire write lock that thread set the highest bit + of the counter. If the thread determine that the bit was set previously then + write lock was acquired in another thread. In that case the thread waits on + the TrappedFutex and then tries again from the beginning. + If the highest bit was successfully set then thread check if any read lock + exists at the moment. If so the thread waits on UnshareFutex. If there is + no more read locks then write lock was successfully acquired and the thread + can proceed execution. +*/ + +#include <linux/futex.h> #include <unistd.h> -#include <sys/syscall.h> -#include <errno.h> - -namespace NS_LightRWLock { +#include <sys/syscall.h> +#include <errno.h> + +namespace NS_LightRWLock { static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { return __atomic_fetch_add(&item, value, __ATOMIC_SEQ_CST); } - -#if defined(_x86_64_) || defined(_i386_) - + +#if defined(_x86_64_) || defined(_i386_) + static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { char ret; __asm__ __volatile__( @@ -54,7 +54,7 @@ namespace NS_LightRWLock { return ret; } - + static char Y_FORCE_INLINE AtomicClearBit(volatile int& item, unsigned bit) { char ret; __asm__ __volatile__( @@ -69,22 +69,22 @@ namespace NS_LightRWLock { return ret; } - - -#else - + + +#else + static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { int prev = __atomic_fetch_or(&item, 1 << bit, __ATOMIC_SEQ_CST); return (prev & (1 << bit)) != 0 ? 1 : 0; } - + static char Y_FORCE_INLINE AtomicClearBit(volatile int& item, unsigned bit) { int prev = __atomic_fetch_and(&item, ~(1 << bit), __ATOMIC_SEQ_CST); return (prev & (1 << bit)) != 0 ? 1 : 0; } #endif - + #if defined(_x86_64_) || defined(_i386_) || defined (__aarch64__) || defined (__powerpc64__) static bool AtomicLockHighByte(volatile int& item) { union TA { @@ -98,23 +98,23 @@ namespace NS_LightRWLock { __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); } -#endif - +#endif + template <typename TInt> static void Y_FORCE_INLINE AtomicStore(volatile TInt& var, TInt value) { __atomic_store_n(&var, value, __ATOMIC_RELEASE); } - + template <typename TInt> static void Y_FORCE_INLINE SequenceStore(volatile TInt& var, TInt value) { __atomic_store_n(&var, value, __ATOMIC_SEQ_CST); } - + template <typename TInt> static TInt Y_FORCE_INLINE AtomicLoad(const volatile TInt& var) { return __atomic_load_n(&var, __ATOMIC_ACQUIRE); } - + static void Y_FORCE_INLINE FutexWait(volatile int& fvar, int value) { for (;;) { int result = @@ -126,9 +126,9 @@ namespace NS_LightRWLock { continue; Y_FAIL("futex error"); } - } - } - + } + } + static void Y_FORCE_INLINE FutexWake(volatile int& fvar, int amount) { const int result = syscall(SYS_futex, &fvar, FUTEX_WAKE_PRIVATE, amount, NULL, NULL, 0); @@ -136,85 +136,85 @@ namespace NS_LightRWLock { Y_FAIL("futex error"); } -} - -class alignas(64) TLightRWLock { -public: - TLightRWLock(ui32 spinCount = 10) - : Counter_(0) - , TrappedFutex_(0) - , UnshareFutex_(0) - , SpinCount_(spinCount) +} + +class alignas(64) TLightRWLock { +public: + TLightRWLock(ui32 spinCount = 10) + : Counter_(0) + , TrappedFutex_(0) + , UnshareFutex_(0) + , SpinCount_(spinCount) { } - - TLightRWLock(const TLightRWLock&) = delete; - void operator=(const TLightRWLock&) = delete; - - Y_FORCE_INLINE void AcquireWrite() { - using namespace NS_LightRWLock; - - if (AtomicLockHighByte(Counter_)) { - if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) - return; - return WaitForUntrappedShared(); - } - WaitForExclusiveAndUntrappedShared(); - } - - Y_FORCE_INLINE void AcquireRead() { - using namespace NS_LightRWLock; - - if (Y_LIKELY(AtomicFetchAdd(Counter_, 1) >= 0)) - return; - WaitForUntrappedAndAcquireRead(); - } - - Y_FORCE_INLINE void ReleaseWrite() { - using namespace NS_LightRWLock; - - AtomicClearBit(Counter_, 31); - if (AtomicLoad(TrappedFutex_)) { - SequenceStore(TrappedFutex_, 0); - FutexWake(TrappedFutex_, 0x7fffffff); - } - } - - Y_FORCE_INLINE void ReleaseRead() { - using namespace NS_LightRWLock; - - if (Y_LIKELY(AtomicFetchAdd(Counter_, -1) >= 0)) - return; - if (!AtomicLoad(UnshareFutex_)) - return; - if ((AtomicLoad(Counter_) & 0x7fffffff) == 0) { - SequenceStore(UnshareFutex_, 0); - FutexWake(UnshareFutex_, 1); - } - } - -private: - volatile int Counter_; - volatile int TrappedFutex_; - volatile int UnshareFutex_; - const ui32 SpinCount_; - - void WaitForUntrappedShared(); - void WaitForExclusiveAndUntrappedShared(); - void WaitForUntrappedAndAcquireRead(); -}; - -#else - -class TLightRWLock: public TRWMutex { -public: + + TLightRWLock(const TLightRWLock&) = delete; + void operator=(const TLightRWLock&) = delete; + + Y_FORCE_INLINE void AcquireWrite() { + using namespace NS_LightRWLock; + + if (AtomicLockHighByte(Counter_)) { + if ((AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) + return; + return WaitForUntrappedShared(); + } + WaitForExclusiveAndUntrappedShared(); + } + + Y_FORCE_INLINE void AcquireRead() { + using namespace NS_LightRWLock; + + if (Y_LIKELY(AtomicFetchAdd(Counter_, 1) >= 0)) + return; + WaitForUntrappedAndAcquireRead(); + } + + Y_FORCE_INLINE void ReleaseWrite() { + using namespace NS_LightRWLock; + + AtomicClearBit(Counter_, 31); + if (AtomicLoad(TrappedFutex_)) { + SequenceStore(TrappedFutex_, 0); + FutexWake(TrappedFutex_, 0x7fffffff); + } + } + + Y_FORCE_INLINE void ReleaseRead() { + using namespace NS_LightRWLock; + + if (Y_LIKELY(AtomicFetchAdd(Counter_, -1) >= 0)) + return; + if (!AtomicLoad(UnshareFutex_)) + return; + if ((AtomicLoad(Counter_) & 0x7fffffff) == 0) { + SequenceStore(UnshareFutex_, 0); + FutexWake(UnshareFutex_, 1); + } + } + +private: + volatile int Counter_; + volatile int TrappedFutex_; + volatile int UnshareFutex_; + const ui32 SpinCount_; + + void WaitForUntrappedShared(); + void WaitForExclusiveAndUntrappedShared(); + void WaitForUntrappedAndAcquireRead(); +}; + +#else + +class TLightRWLock: public TRWMutex { +public: TLightRWLock() { } TLightRWLock(ui32) { } -}; - -#endif - -using TLightReadGuard = TReadGuardBase<TLightRWLock>; -using TLightWriteGuard = TWriteGuardBase<TLightRWLock>; +}; + +#endif + +using TLightReadGuard = TReadGuardBase<TLightRWLock>; +using TLightWriteGuard = TWriteGuardBase<TLightRWLock>; diff --git a/library/cpp/threading/light_rw_lock/ut/rwlock_ut.cpp b/library/cpp/threading/light_rw_lock/ut/rwlock_ut.cpp index e82063d959..3ed4bf68fa 100644 --- a/library/cpp/threading/light_rw_lock/ut/rwlock_ut.cpp +++ b/library/cpp/threading/light_rw_lock/ut/rwlock_ut.cpp @@ -1,122 +1,122 @@ #include <library/cpp/threading/light_rw_lock/lightrwlock.h> #include <library/cpp/testing/unittest/registar.h> -#include <util/random/random.h> -#include <util/system/atomic.h> +#include <util/random/random.h> +#include <util/system/atomic.h> #include <util/thread/pool.h> - -class TRWMutexTest: public TTestBase { - UNIT_TEST_SUITE(TRWMutexTest); - UNIT_TEST(TestReaders) - UNIT_TEST(TestReadersWriters) - UNIT_TEST_SUITE_END(); - - struct TSharedData { - TSharedData() - : writersIn(0) - , readersIn(0) - , failed(false) - { - } - - TAtomic writersIn; - TAtomic readersIn; - - bool failed; - - TLightRWLock mutex; - }; - - class TThreadTask: public IObjectInQueue { - public: - using PFunc = void (TThreadTask::*)(void); - - TThreadTask(PFunc func, TSharedData& data, size_t id, size_t total) - : Func_(func) - , Data_(data) - , Id_(id) - , Total_(total) - { - } - - void Process(void*) override { - THolder<TThreadTask> This(this); - - (this->*Func_)(); - } - -#define FAIL_ASSERT(cond) \ - if (!(cond)) { \ - Data_.failed = true; \ - } - void RunReaders() { - Data_.mutex.AcquireRead(); - - AtomicIncrement(Data_.readersIn); - usleep(100); - FAIL_ASSERT(Data_.readersIn == long(Total_)); - usleep(100); - AtomicDecrement(Data_.readersIn); - - Data_.mutex.ReleaseRead(); - } - - void RunReadersWriters() { - if (Id_ % 2 == 0) { - for (size_t i = 0; i < 10; ++i) { - Data_.mutex.AcquireRead(); - - AtomicIncrement(Data_.readersIn); - FAIL_ASSERT(Data_.writersIn == 0); - usleep(RandomNumber<ui32>() % 5); - AtomicDecrement(Data_.readersIn); - - Data_.mutex.ReleaseRead(); - } - } else { - for (size_t i = 0; i < 10; ++i) { - Data_.mutex.AcquireWrite(); - - AtomicIncrement(Data_.writersIn); - FAIL_ASSERT(Data_.readersIn == 0 && Data_.writersIn == 1); - usleep(RandomNumber<ui32>() % 5); - AtomicDecrement(Data_.writersIn); - - Data_.mutex.ReleaseWrite(); - } - } - } -#undef FAIL_ASSERT - - private: - PFunc Func_; - TSharedData& Data_; - size_t Id_; - size_t Total_; - }; - -private: -#define RUN_CYCLE(what, count) \ - Q_.Start(count); \ - for (size_t i = 0; i < count; ++i) { \ - UNIT_ASSERT(Q_.Add(new TThreadTask(&TThreadTask::what, Data_, i, count))); \ - } \ - Q_.Stop(); \ - bool b = Data_.failed; \ - Data_.failed = false; \ - UNIT_ASSERT(!b); - - void TestReaders() { - RUN_CYCLE(RunReaders, 1); - } - - void TestReadersWriters() { - RUN_CYCLE(RunReadersWriters, 1); - } - -#undef RUN_CYCLE -private: - TSharedData Data_; + +class TRWMutexTest: public TTestBase { + UNIT_TEST_SUITE(TRWMutexTest); + UNIT_TEST(TestReaders) + UNIT_TEST(TestReadersWriters) + UNIT_TEST_SUITE_END(); + + struct TSharedData { + TSharedData() + : writersIn(0) + , readersIn(0) + , failed(false) + { + } + + TAtomic writersIn; + TAtomic readersIn; + + bool failed; + + TLightRWLock mutex; + }; + + class TThreadTask: public IObjectInQueue { + public: + using PFunc = void (TThreadTask::*)(void); + + TThreadTask(PFunc func, TSharedData& data, size_t id, size_t total) + : Func_(func) + , Data_(data) + , Id_(id) + , Total_(total) + { + } + + void Process(void*) override { + THolder<TThreadTask> This(this); + + (this->*Func_)(); + } + +#define FAIL_ASSERT(cond) \ + if (!(cond)) { \ + Data_.failed = true; \ + } + void RunReaders() { + Data_.mutex.AcquireRead(); + + AtomicIncrement(Data_.readersIn); + usleep(100); + FAIL_ASSERT(Data_.readersIn == long(Total_)); + usleep(100); + AtomicDecrement(Data_.readersIn); + + Data_.mutex.ReleaseRead(); + } + + void RunReadersWriters() { + if (Id_ % 2 == 0) { + for (size_t i = 0; i < 10; ++i) { + Data_.mutex.AcquireRead(); + + AtomicIncrement(Data_.readersIn); + FAIL_ASSERT(Data_.writersIn == 0); + usleep(RandomNumber<ui32>() % 5); + AtomicDecrement(Data_.readersIn); + + Data_.mutex.ReleaseRead(); + } + } else { + for (size_t i = 0; i < 10; ++i) { + Data_.mutex.AcquireWrite(); + + AtomicIncrement(Data_.writersIn); + FAIL_ASSERT(Data_.readersIn == 0 && Data_.writersIn == 1); + usleep(RandomNumber<ui32>() % 5); + AtomicDecrement(Data_.writersIn); + + Data_.mutex.ReleaseWrite(); + } + } + } +#undef FAIL_ASSERT + + private: + PFunc Func_; + TSharedData& Data_; + size_t Id_; + size_t Total_; + }; + +private: +#define RUN_CYCLE(what, count) \ + Q_.Start(count); \ + for (size_t i = 0; i < count; ++i) { \ + UNIT_ASSERT(Q_.Add(new TThreadTask(&TThreadTask::what, Data_, i, count))); \ + } \ + Q_.Stop(); \ + bool b = Data_.failed; \ + Data_.failed = false; \ + UNIT_ASSERT(!b); + + void TestReaders() { + RUN_CYCLE(RunReaders, 1); + } + + void TestReadersWriters() { + RUN_CYCLE(RunReadersWriters, 1); + } + +#undef RUN_CYCLE +private: + TSharedData Data_; TThreadPool Q_; -}; - -UNIT_TEST_SUITE_REGISTRATION(TRWMutexTest) +}; + +UNIT_TEST_SUITE_REGISTRATION(TRWMutexTest) diff --git a/library/cpp/threading/light_rw_lock/ut/ya.make b/library/cpp/threading/light_rw_lock/ut/ya.make index 92928b837c..9b1a54d7ec 100644 --- a/library/cpp/threading/light_rw_lock/ut/ya.make +++ b/library/cpp/threading/light_rw_lock/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/threading/light_rw_lock) - + OWNER(agri) - -SRCS( - rwlock_ut.cpp -) - -END() + +SRCS( + rwlock_ut.cpp +) + +END() diff --git a/library/cpp/threading/light_rw_lock/ya.make b/library/cpp/threading/light_rw_lock/ya.make index a196fb8588..e86fd42223 100644 --- a/library/cpp/threading/light_rw_lock/ya.make +++ b/library/cpp/threading/light_rw_lock/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(agri) - -SRCS( - lightrwlock.cpp - lightrwlock.h -) - -END() +LIBRARY() + +OWNER(agri) + +SRCS( + lightrwlock.cpp + lightrwlock.h +) + +END() diff --git a/library/cpp/threading/queue/basic_ut.cpp b/library/cpp/threading/queue/basic_ut.cpp index 5f56f8583e..2db5d6e8e8 100644 --- a/library/cpp/threading/queue/basic_ut.cpp +++ b/library/cpp/threading/queue/basic_ut.cpp @@ -1,92 +1,92 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/vector.h> -#include <util/system/thread.h> - -#include "ut_helpers.h" - +#include <util/generic/vector.h> +#include <util/system/thread.h> + +#include "ut_helpers.h" + template <typename TQueueType> -class TQueueTestsInSingleThread: public TTestBase { -private: +class TQueueTestsInSingleThread: public TTestBase { +private: using TSelf = TQueueTestsInSingleThread<TQueueType>; - using TLink = TIntrusiveLink; - - UNIT_TEST_SUITE_DEMANGLE(TSelf); - UNIT_TEST(OnePushOnePop) - UNIT_TEST(OnePushOnePop_Repeat1M) - UNIT_TEST(Threads8_Repeat1M_Push1Pop1) - UNIT_TEST_SUITE_END(); - -public: - void OnePushOnePop() { + using TLink = TIntrusiveLink; + + UNIT_TEST_SUITE_DEMANGLE(TSelf); + UNIT_TEST(OnePushOnePop) + UNIT_TEST(OnePushOnePop_Repeat1M) + UNIT_TEST(Threads8_Repeat1M_Push1Pop1) + UNIT_TEST_SUITE_END(); + +public: + void OnePushOnePop() { TQueueType queue; - - auto popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - - TLink msg; - queue.Push(&msg); - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(&msg, popped); - - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - }; - - void OnePushOnePop_Repeat1M() { + + auto popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + + TLink msg; + queue.Push(&msg); + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(&msg, popped); + + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + }; + + void OnePushOnePop_Repeat1M() { TQueueType queue; - TLink msg; - - auto popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - - for (int i = 0; i < 1000000; ++i) { - queue.Push(&msg); - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(&msg, popped); - - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - } - } - - template <size_t NUMBER_OF_THREADS> - void RepeatPush1Pop1_InManyThreads() { + TLink msg; + + auto popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + + for (int i = 0; i < 1000000; ++i) { + queue.Push(&msg); + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(&msg, popped); + + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + } + } + + template <size_t NUMBER_OF_THREADS> + void RepeatPush1Pop1_InManyThreads() { class TCycleThread: public ISimpleThread { - public: - void* ThreadProc() override { + public: + void* ThreadProc() override { TQueueType queue; - TLink msg; - auto popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - - for (size_t i = 0; i < 1000000; ++i) { - queue.Push(&msg); - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, &msg); - - popped = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); - } - return nullptr; - } - }; - + TLink msg; + auto popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + + for (size_t i = 0; i < 1000000; ++i) { + queue.Push(&msg); + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, &msg); + + popped = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(popped, nullptr); + } + return nullptr; + } + }; + TVector<TAutoPtr<TCycleThread>> cyclers; - - for (size_t i = 0; i < NUMBER_OF_THREADS; ++i) { - cyclers.emplace_back(new TCycleThread); - cyclers.back()->Start(); - } - - for (size_t i = 0; i < NUMBER_OF_THREADS; ++i) { - cyclers[i]->Join(); - } - } - - void Threads8_Repeat1M_Push1Pop1() { - RepeatPush1Pop1_InManyThreads<8>(); - } -}; - -REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TQueueTestsInSingleThread); -REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TQueueTestsInSingleThread) + + for (size_t i = 0; i < NUMBER_OF_THREADS; ++i) { + cyclers.emplace_back(new TCycleThread); + cyclers.back()->Start(); + } + + for (size_t i = 0; i < NUMBER_OF_THREADS; ++i) { + cyclers[i]->Join(); + } + } + + void Threads8_Repeat1M_Push1Pop1() { + RepeatPush1Pop1_InManyThreads<8>(); + } +}; + +REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TQueueTestsInSingleThread); +REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TQueueTestsInSingleThread) diff --git a/library/cpp/threading/queue/mpmc_unordered_ring.cpp b/library/cpp/threading/queue/mpmc_unordered_ring.cpp index 160547f594..df48182210 100644 --- a/library/cpp/threading/queue/mpmc_unordered_ring.cpp +++ b/library/cpp/threading/queue/mpmc_unordered_ring.cpp @@ -1,74 +1,74 @@ -#include "mpmc_unordered_ring.h" - -namespace NThreading { - TMPMCUnorderedRing::TMPMCUnorderedRing(size_t size) { - Y_VERIFY(size > 0); - RingSize = size; - RingBuffer.Reset(new void*[size]); - memset(&RingBuffer[0], 0, sizeof(void*) * size); - } - - bool TMPMCUnorderedRing::Push(void* msg, ui16 retryCount) noexcept { - if (retryCount == 0) { - StubbornPush(msg); - return true; - } - for (ui16 itry = retryCount; itry-- > 0;) { - if (WeakPush(msg)) { - return true; - } - } - return false; - } - - bool TMPMCUnorderedRing::WeakPush(void* msg) noexcept { - auto pawl = AtomicIncrement(WritePawl); - if (pawl - AtomicGet(ReadFront) >= RingSize) { - // Queue is full - AtomicDecrement(WritePawl); - return false; - } - - auto writeSlot = AtomicGetAndIncrement(WriteFront); - if (AtomicCas(&RingBuffer[writeSlot % RingSize], msg, nullptr)) { - return true; - } - // slot is occupied for some reason, retry - return false; - } - - void* TMPMCUnorderedRing::Pop() noexcept { - ui64 readSlot; - - for (ui16 itry = MAX_POP_TRIES; itry-- > 0;) { - auto pawl = AtomicIncrement(ReadPawl); - if (pawl > AtomicGet(WriteFront)) { - // Queue is empty - AtomicDecrement(ReadPawl); - return nullptr; - } - - readSlot = AtomicGetAndIncrement(ReadFront); - - auto msg = AtomicSwap(&RingBuffer[readSlot % RingSize], nullptr); - if (msg != nullptr) { - return msg; - } - } - - /* got no message in the slot, let's try to rollback readfront */ - AtomicCas(&ReadFront, readSlot - 1, readSlot); - return nullptr; - } - - void* TMPMCUnorderedRing::UnsafeScanningPop(ui64* last) noexcept { - for (; *last < RingSize;) { - auto msg = AtomicSwap(&RingBuffer[*last], nullptr); - ++*last; - if (msg != nullptr) { - return msg; - } - } - return nullptr; - } -} +#include "mpmc_unordered_ring.h" + +namespace NThreading { + TMPMCUnorderedRing::TMPMCUnorderedRing(size_t size) { + Y_VERIFY(size > 0); + RingSize = size; + RingBuffer.Reset(new void*[size]); + memset(&RingBuffer[0], 0, sizeof(void*) * size); + } + + bool TMPMCUnorderedRing::Push(void* msg, ui16 retryCount) noexcept { + if (retryCount == 0) { + StubbornPush(msg); + return true; + } + for (ui16 itry = retryCount; itry-- > 0;) { + if (WeakPush(msg)) { + return true; + } + } + return false; + } + + bool TMPMCUnorderedRing::WeakPush(void* msg) noexcept { + auto pawl = AtomicIncrement(WritePawl); + if (pawl - AtomicGet(ReadFront) >= RingSize) { + // Queue is full + AtomicDecrement(WritePawl); + return false; + } + + auto writeSlot = AtomicGetAndIncrement(WriteFront); + if (AtomicCas(&RingBuffer[writeSlot % RingSize], msg, nullptr)) { + return true; + } + // slot is occupied for some reason, retry + return false; + } + + void* TMPMCUnorderedRing::Pop() noexcept { + ui64 readSlot; + + for (ui16 itry = MAX_POP_TRIES; itry-- > 0;) { + auto pawl = AtomicIncrement(ReadPawl); + if (pawl > AtomicGet(WriteFront)) { + // Queue is empty + AtomicDecrement(ReadPawl); + return nullptr; + } + + readSlot = AtomicGetAndIncrement(ReadFront); + + auto msg = AtomicSwap(&RingBuffer[readSlot % RingSize], nullptr); + if (msg != nullptr) { + return msg; + } + } + + /* got no message in the slot, let's try to rollback readfront */ + AtomicCas(&ReadFront, readSlot - 1, readSlot); + return nullptr; + } + + void* TMPMCUnorderedRing::UnsafeScanningPop(ui64* last) noexcept { + for (; *last < RingSize;) { + auto msg = AtomicSwap(&RingBuffer[*last], nullptr); + ++*last; + if (msg != nullptr) { + return msg; + } + } + return nullptr; + } +} diff --git a/library/cpp/threading/queue/mpmc_unordered_ring.h b/library/cpp/threading/queue/mpmc_unordered_ring.h index 5042f7528e..59758d2c35 100644 --- a/library/cpp/threading/queue/mpmc_unordered_ring.h +++ b/library/cpp/threading/queue/mpmc_unordered_ring.h @@ -1,42 +1,42 @@ -#pragma once - -/* - It's not a general purpose queue. - No order guarantee, but it mostly ordered. - Items may stuck in almost empty queue. - Use UnsafeScanningPop to pop all stuck items. - Almost wait-free for producers and consumers. - */ - -#include <util/system/atomic.h> -#include <util/generic/ptr.h> - -namespace NThreading { - struct TMPMCUnorderedRing { - public: - static constexpr ui16 MAX_PUSH_TRIES = 4; - static constexpr ui16 MAX_POP_TRIES = 4; - - TMPMCUnorderedRing(size_t size); - - bool Push(void* msg, ui16 retryCount = MAX_PUSH_TRIES) noexcept; - void StubbornPush(void* msg) { - while (!WeakPush(msg)) { - } - } - - void* Pop() noexcept; - - void* UnsafeScanningPop(ui64* last) noexcept; - - private: - bool WeakPush(void* msg) noexcept; - - size_t RingSize; - TArrayPtr<void*> RingBuffer; - ui64 WritePawl = 0; - ui64 WriteFront = 0; - ui64 ReadPawl = 0; - ui64 ReadFront = 0; - }; -} +#pragma once + +/* + It's not a general purpose queue. + No order guarantee, but it mostly ordered. + Items may stuck in almost empty queue. + Use UnsafeScanningPop to pop all stuck items. + Almost wait-free for producers and consumers. + */ + +#include <util/system/atomic.h> +#include <util/generic/ptr.h> + +namespace NThreading { + struct TMPMCUnorderedRing { + public: + static constexpr ui16 MAX_PUSH_TRIES = 4; + static constexpr ui16 MAX_POP_TRIES = 4; + + TMPMCUnorderedRing(size_t size); + + bool Push(void* msg, ui16 retryCount = MAX_PUSH_TRIES) noexcept; + void StubbornPush(void* msg) { + while (!WeakPush(msg)) { + } + } + + void* Pop() noexcept; + + void* UnsafeScanningPop(ui64* last) noexcept; + + private: + bool WeakPush(void* msg) noexcept; + + size_t RingSize; + TArrayPtr<void*> RingBuffer; + ui64 WritePawl = 0; + ui64 WriteFront = 0; + ui64 ReadPawl = 0; + ui64 ReadFront = 0; + }; +} diff --git a/library/cpp/threading/queue/mpsc_htswap.cpp b/library/cpp/threading/queue/mpsc_htswap.cpp index 610c8f67f1..d8ab0d4f48 100644 --- a/library/cpp/threading/queue/mpsc_htswap.cpp +++ b/library/cpp/threading/queue/mpsc_htswap.cpp @@ -1 +1 @@ -#include "mpsc_htswap.h" +#include "mpsc_htswap.h" diff --git a/library/cpp/threading/queue/mpsc_htswap.h b/library/cpp/threading/queue/mpsc_htswap.h index c42caa7ac0..2d0bfd1146 100644 --- a/library/cpp/threading/queue/mpsc_htswap.h +++ b/library/cpp/threading/queue/mpsc_htswap.h @@ -1,132 +1,132 @@ -#pragma once - -/* - http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue - - Simple semi-wait-free queue. Many producers - one consumer. - Tracking of allocated memory is not required. - No CAS. Only atomic swap (exchange) operations. - - WARNING: a sleeping producer can stop progress for consumer. - - WARNING: there is no wait¬ify mechanic for consumer, - consumer receives nullptr if queue was empty. - - WARNING: the algorithm itself is lock-free - but producers and consumer could be blocked by memory allocator - - Reference design: rtmapreduce/libs/threading/lfqueue.h - */ - -#include <util/generic/noncopyable.h> -#include <util/system/types.h> -#include <util/system/atomic.h> - -#include "tune.h" - -namespace NThreading { - namespace NHTSwapPrivate { - template <typename T, typename TTuneup> - struct TNode +#pragma once + +/* + http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue + + Simple semi-wait-free queue. Many producers - one consumer. + Tracking of allocated memory is not required. + No CAS. Only atomic swap (exchange) operations. + + WARNING: a sleeping producer can stop progress for consumer. + + WARNING: there is no wait¬ify mechanic for consumer, + consumer receives nullptr if queue was empty. + + WARNING: the algorithm itself is lock-free + but producers and consumer could be blocked by memory allocator + + Reference design: rtmapreduce/libs/threading/lfqueue.h + */ + +#include <util/generic/noncopyable.h> +#include <util/system/types.h> +#include <util/system/atomic.h> + +#include "tune.h" + +namespace NThreading { + namespace NHTSwapPrivate { + template <typename T, typename TTuneup> + struct TNode : public TTuneup::TNodeBase, public TTuneup::template TNodeLayout<TNode<T, TTuneup>, T> { - TNode(const T& item) { - this->Next = nullptr; - this->Item = item; - } - - TNode(T&& item) { - this->Next = nullptr; - this->Item = std::move(item); - } - }; - - struct TDefaultTuneup { - struct TNodeBase: private TNonCopyable { - }; - - template <typename TNode, typename T> - struct TNodeLayout { - TNode* Next; - T Item; - }; - - template <typename TNode> - struct TQueueLayout { - TNode* Head; - TNode* Tail; - }; - }; - - template <typename T, typename TTuneup> - class THTSwapQueueImpl + TNode(const T& item) { + this->Next = nullptr; + this->Item = item; + } + + TNode(T&& item) { + this->Next = nullptr; + this->Item = std::move(item); + } + }; + + struct TDefaultTuneup { + struct TNodeBase: private TNonCopyable { + }; + + template <typename TNode, typename T> + struct TNodeLayout { + TNode* Next; + T Item; + }; + + template <typename TNode> + struct TQueueLayout { + TNode* Head; + TNode* Tail; + }; + }; + + template <typename T, typename TTuneup> + class THTSwapQueueImpl : protected TTuneup::template TQueueLayout<TNode<T, TTuneup>> { - protected: - using TTunedNode = TNode<T, TTuneup>; - - public: - using TItem = T; - - THTSwapQueueImpl() { - this->Head = new TTunedNode(T()); - this->Tail = this->Head; - } - - ~THTSwapQueueImpl() { - TTunedNode* node = this->Head; - while (node != nullptr) { - TTunedNode* next = node->Next; - delete node; - node = next; - } - } - - template <typename TT> - void Push(TT&& item) { - Enqueue(new TTunedNode(std::forward<TT>(item))); - } - - T Peek() { - TTunedNode* next = AtomicGet(this->Head->Next); - if (next == nullptr) { - return T(); - } - return next->Item; - } - - void Enqueue(TTunedNode* node) { - // our goal is to avoid expensive CAS here, - // but now consumer will be blocked until new tail linked. - // fortunately 'window of inconsistency' is extremely small. - TTunedNode* prev = AtomicSwap(&this->Tail, node); - AtomicSet(prev->Next, node); - } - - T Pop() { - TTunedNode* next = AtomicGet(this->Head->Next); - if (next == nullptr) { - return nullptr; - } - auto item = std::move(next->Item); - std::swap(this->Head, next); // no need atomic here - delete next; - return item; - } - - bool IsEmpty() const { - TTunedNode* next = AtomicGet(this->Head->Next); - return (next == nullptr); - } - }; - } - - DeclareTuneTypeParam(THTSwapNodeBase, TNodeBase); - DeclareTuneTypeParam(THTSwapNodeLayout, TNodeLayout); - DeclareTuneTypeParam(THTSwapQueueLayout, TQueueLayout); - + protected: + using TTunedNode = TNode<T, TTuneup>; + + public: + using TItem = T; + + THTSwapQueueImpl() { + this->Head = new TTunedNode(T()); + this->Tail = this->Head; + } + + ~THTSwapQueueImpl() { + TTunedNode* node = this->Head; + while (node != nullptr) { + TTunedNode* next = node->Next; + delete node; + node = next; + } + } + + template <typename TT> + void Push(TT&& item) { + Enqueue(new TTunedNode(std::forward<TT>(item))); + } + + T Peek() { + TTunedNode* next = AtomicGet(this->Head->Next); + if (next == nullptr) { + return T(); + } + return next->Item; + } + + void Enqueue(TTunedNode* node) { + // our goal is to avoid expensive CAS here, + // but now consumer will be blocked until new tail linked. + // fortunately 'window of inconsistency' is extremely small. + TTunedNode* prev = AtomicSwap(&this->Tail, node); + AtomicSet(prev->Next, node); + } + + T Pop() { + TTunedNode* next = AtomicGet(this->Head->Next); + if (next == nullptr) { + return nullptr; + } + auto item = std::move(next->Item); + std::swap(this->Head, next); // no need atomic here + delete next; + return item; + } + + bool IsEmpty() const { + TTunedNode* next = AtomicGet(this->Head->Next); + return (next == nullptr); + } + }; + } + + DeclareTuneTypeParam(THTSwapNodeBase, TNodeBase); + DeclareTuneTypeParam(THTSwapNodeLayout, TNodeLayout); + DeclareTuneTypeParam(THTSwapQueueLayout, TQueueLayout); + template <typename T = void*, typename... TParams> - class THTSwapQueue + class THTSwapQueue : public NHTSwapPrivate::THTSwapQueueImpl<T, TTune<NHTSwapPrivate::TDefaultTuneup, TParams...>> { - }; -} + }; +} diff --git a/library/cpp/threading/queue/mpsc_intrusive_unordered.cpp b/library/cpp/threading/queue/mpsc_intrusive_unordered.cpp index 3bb1a04f7e..a6a2fcef39 100644 --- a/library/cpp/threading/queue/mpsc_intrusive_unordered.cpp +++ b/library/cpp/threading/queue/mpsc_intrusive_unordered.cpp @@ -1,79 +1,79 @@ -#include "mpsc_intrusive_unordered.h" -#include <util/system/atomic.h> - -namespace NThreading { - void TMPSCIntrusiveUnordered::Push(TIntrusiveNode* node) noexcept { - auto head = AtomicGet(HeadForCaS); - for (ui32 i = NUMBER_OF_TRIES_FOR_CAS; i-- > 0;) { - // no ABA here, because Next is exactly head - // it does not matter how many travels head was made/ - node->Next = head; - auto prev = AtomicGetAndCas(&HeadForCaS, node, head); - if (head == prev) { - return; - } - head = prev; - } - // boring of trying to do cas, let's just swap - - // no need for atomic here, because the next is atomic swap - node->Next = 0; - - head = AtomicSwap(&HeadForSwap, node); - if (head != nullptr) { - AtomicSet(node->Next, head); - } else { - // consumer must know if no other thread may access the memory, - // setting Next to node is a way to notify consumer - AtomicSet(node->Next, node); - } - } - - TIntrusiveNode* TMPSCIntrusiveUnordered::PopMany() noexcept { - if (NotReadyChain == nullptr) { - auto head = AtomicSwap(&HeadForSwap, nullptr); - NotReadyChain = head; - } - - if (NotReadyChain != nullptr) { - auto next = AtomicGet(NotReadyChain->Next); - if (next != nullptr) { - auto ready = NotReadyChain; - TIntrusiveNode* cut; - do { - cut = NotReadyChain; - NotReadyChain = next; - next = AtomicGet(NotReadyChain->Next); - if (next == NotReadyChain) { - cut = NotReadyChain; - NotReadyChain = nullptr; - break; - } - } while (next != nullptr); - cut->Next = nullptr; - return ready; - } - } - - if (AtomicGet(HeadForCaS) != nullptr) { - return AtomicSwap(&HeadForCaS, nullptr); - } - return nullptr; - } - - TIntrusiveNode* TMPSCIntrusiveUnordered::Pop() noexcept { - if (PopOneQueue != nullptr) { - auto head = PopOneQueue; - PopOneQueue = PopOneQueue->Next; - return head; - } - - PopOneQueue = PopMany(); - if (PopOneQueue != nullptr) { - auto head = PopOneQueue; - PopOneQueue = PopOneQueue->Next; - return head; - } - return nullptr; - } -} +#include "mpsc_intrusive_unordered.h" +#include <util/system/atomic.h> + +namespace NThreading { + void TMPSCIntrusiveUnordered::Push(TIntrusiveNode* node) noexcept { + auto head = AtomicGet(HeadForCaS); + for (ui32 i = NUMBER_OF_TRIES_FOR_CAS; i-- > 0;) { + // no ABA here, because Next is exactly head + // it does not matter how many travels head was made/ + node->Next = head; + auto prev = AtomicGetAndCas(&HeadForCaS, node, head); + if (head == prev) { + return; + } + head = prev; + } + // boring of trying to do cas, let's just swap + + // no need for atomic here, because the next is atomic swap + node->Next = 0; + + head = AtomicSwap(&HeadForSwap, node); + if (head != nullptr) { + AtomicSet(node->Next, head); + } else { + // consumer must know if no other thread may access the memory, + // setting Next to node is a way to notify consumer + AtomicSet(node->Next, node); + } + } + + TIntrusiveNode* TMPSCIntrusiveUnordered::PopMany() noexcept { + if (NotReadyChain == nullptr) { + auto head = AtomicSwap(&HeadForSwap, nullptr); + NotReadyChain = head; + } + + if (NotReadyChain != nullptr) { + auto next = AtomicGet(NotReadyChain->Next); + if (next != nullptr) { + auto ready = NotReadyChain; + TIntrusiveNode* cut; + do { + cut = NotReadyChain; + NotReadyChain = next; + next = AtomicGet(NotReadyChain->Next); + if (next == NotReadyChain) { + cut = NotReadyChain; + NotReadyChain = nullptr; + break; + } + } while (next != nullptr); + cut->Next = nullptr; + return ready; + } + } + + if (AtomicGet(HeadForCaS) != nullptr) { + return AtomicSwap(&HeadForCaS, nullptr); + } + return nullptr; + } + + TIntrusiveNode* TMPSCIntrusiveUnordered::Pop() noexcept { + if (PopOneQueue != nullptr) { + auto head = PopOneQueue; + PopOneQueue = PopOneQueue->Next; + return head; + } + + PopOneQueue = PopMany(); + if (PopOneQueue != nullptr) { + auto head = PopOneQueue; + PopOneQueue = PopOneQueue->Next; + return head; + } + return nullptr; + } +} diff --git a/library/cpp/threading/queue/mpsc_intrusive_unordered.h b/library/cpp/threading/queue/mpsc_intrusive_unordered.h index 6ac7537ae9..c07cf761f6 100644 --- a/library/cpp/threading/queue/mpsc_intrusive_unordered.h +++ b/library/cpp/threading/queue/mpsc_intrusive_unordered.h @@ -1,35 +1,35 @@ -#pragma once - -/* - Simple almost-wait-free unordered queue for low contention operations. - - It's wait-free for producers. - Hanging producer can hide some items from consumer. - */ - -#include <util/system/types.h> - -namespace NThreading { - struct TIntrusiveNode { - TIntrusiveNode* Next; - }; - - class TMPSCIntrusiveUnordered { - public: - static constexpr ui32 NUMBER_OF_TRIES_FOR_CAS = 3; - - void Push(TIntrusiveNode* node) noexcept; - TIntrusiveNode* PopMany() noexcept; - TIntrusiveNode* Pop() noexcept; - - void Push(void* node) noexcept { - Push(reinterpret_cast<TIntrusiveNode*>(node)); - } - - private: - TIntrusiveNode* HeadForCaS = nullptr; - TIntrusiveNode* HeadForSwap = nullptr; - TIntrusiveNode* NotReadyChain = nullptr; - TIntrusiveNode* PopOneQueue = nullptr; - }; -} +#pragma once + +/* + Simple almost-wait-free unordered queue for low contention operations. + + It's wait-free for producers. + Hanging producer can hide some items from consumer. + */ + +#include <util/system/types.h> + +namespace NThreading { + struct TIntrusiveNode { + TIntrusiveNode* Next; + }; + + class TMPSCIntrusiveUnordered { + public: + static constexpr ui32 NUMBER_OF_TRIES_FOR_CAS = 3; + + void Push(TIntrusiveNode* node) noexcept; + TIntrusiveNode* PopMany() noexcept; + TIntrusiveNode* Pop() noexcept; + + void Push(void* node) noexcept { + Push(reinterpret_cast<TIntrusiveNode*>(node)); + } + + private: + TIntrusiveNode* HeadForCaS = nullptr; + TIntrusiveNode* HeadForSwap = nullptr; + TIntrusiveNode* NotReadyChain = nullptr; + TIntrusiveNode* PopOneQueue = nullptr; + }; +} diff --git a/library/cpp/threading/queue/mpsc_read_as_filled.cpp b/library/cpp/threading/queue/mpsc_read_as_filled.cpp index 8b4664a6f3..3b89fb1df6 100644 --- a/library/cpp/threading/queue/mpsc_read_as_filled.cpp +++ b/library/cpp/threading/queue/mpsc_read_as_filled.cpp @@ -1 +1 @@ -#include "mpsc_read_as_filled.h" +#include "mpsc_read_as_filled.h" diff --git a/library/cpp/threading/queue/mpsc_read_as_filled.h b/library/cpp/threading/queue/mpsc_read_as_filled.h index be33ba5a58..4dfdb1fbbf 100644 --- a/library/cpp/threading/queue/mpsc_read_as_filled.h +++ b/library/cpp/threading/queue/mpsc_read_as_filled.h @@ -1,611 +1,611 @@ -#pragma once - -/* - Completely wait-free queue, multiple producers - one consumer. Strict order. - The queue algorithm is using concept of virtual infinite array. - +#pragma once + +/* + Completely wait-free queue, multiple producers - one consumer. Strict order. + The queue algorithm is using concept of virtual infinite array. + A producer takes a number from a counter and atomically increments the counter. - The number taken is a number of a slot for the producer to put a new message - into infinite array. - - Then producer constructs a virtual infinite array by bidirectional linked list - of blocks. Each block contains several slots. - + The number taken is a number of a slot for the producer to put a new message + into infinite array. + + Then producer constructs a virtual infinite array by bidirectional linked list + of blocks. Each block contains several slots. + There is a hint pointer which optimistically points to the last block - of the list and never goes backward. - - Consumer exploits the property of the hint pointer always going forward - to free old blocks eventually. Consumer periodically read the hint pointer - and the counter and thus deduce producers which potentially holds the pointer - to a block. Consumer can free the block if all that producers filled their - slots and left the queue. - - No producer can stop the progress for other producers. - - Consumer can't stop the progress for producers. - Consumer can skip not-yet-filled slots and read them later. - Thus no producer can stop the progress for consumer. + of the list and never goes backward. + + Consumer exploits the property of the hint pointer always going forward + to free old blocks eventually. Consumer periodically read the hint pointer + and the counter and thus deduce producers which potentially holds the pointer + to a block. Consumer can free the block if all that producers filled their + slots and left the queue. + + No producer can stop the progress for other producers. + + Consumer can't stop the progress for producers. + Consumer can skip not-yet-filled slots and read them later. + Thus no producer can stop the progress for consumer. The algorithm is virtually strictly ordered because it skips slots only - if it is really does not matter in which order the slots were produced and - consumed. - - WARNING: there is no wait¬ify mechanic for consumer, - consumer receives nullptr if queue was empty. - - WARNING: though the algorithm itself is completely wait-free - but producers and consumer could be blocked by memory allocator - + if it is really does not matter in which order the slots were produced and + consumed. + + WARNING: there is no wait¬ify mechanic for consumer, + consumer receives nullptr if queue was empty. + + WARNING: though the algorithm itself is completely wait-free + but producers and consumer could be blocked by memory allocator + WARNING: copy constructors of the queue are not thread-safe - */ - -#include <util/generic/deque.h> -#include <util/generic/ptr.h> -#include <util/system/atomic.h> -#include <util/system/spinlock.h> - -#include "tune.h" - -namespace NThreading { - namespace NReadAsFilledPrivate { - typedef void* TMsgLink; - - static constexpr ui32 DEFAULT_BUNCH_SIZE = 251; - - struct TEmpty { - }; - - struct TEmptyAux { - TEmptyAux Retrieve() const { - return TEmptyAux(); - } - - void Store(TEmptyAux&) { - } - - static constexpr TEmptyAux Zero() { - return TEmptyAux(); - } - }; - - template <typename TAux> - struct TSlot { - TMsgLink volatile Msg; - TAux AuxiliaryData; - - inline void Store(TAux& aux) { - AuxiliaryData.Store(aux); - } - - inline TAux Retrieve() const { - return AuxiliaryData.Retrieve(); - } - - static TSlot<TAux> NullElem() { - return {nullptr, TAux::Zero()}; - } - - static TSlot<TAux> Pair(TMsgLink msg, TAux aux) { - return {msg, std::move(aux)}; - } - }; - - template <> - struct TSlot<TEmptyAux> { - TMsgLink volatile Msg; - - inline void Store(TEmptyAux&) { - } - - inline TEmptyAux Retrieve() const { - return TEmptyAux(); - } - - static TSlot<TEmptyAux> NullElem() { - return {nullptr}; - } - - static TSlot<TEmptyAux> Pair(TMsgLink msg, TEmptyAux) { - return {msg}; - } - }; - - enum TPushResult { - PUSH_RESULT_OK, - PUSH_RESULT_BACKWARD, - PUSH_RESULT_FORWARD, - }; - - template <ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE, - typename TBase = TEmpty, - typename TAux = TEmptyAux> - struct TMsgBunch: public TBase { - static constexpr size_t RELEASE_SIZE = BUNCH_SIZE * 2; - - ui64 FirstSlot; - - TSlot<TAux> LinkArray[BUNCH_SIZE]; - - TMsgBunch* volatile NextBunch; - TMsgBunch* volatile BackLink; - - ui64 volatile Token; - TMsgBunch* volatile NextToken; - - /* this push can return PUSH_RESULT_BLOCKED */ + */ + +#include <util/generic/deque.h> +#include <util/generic/ptr.h> +#include <util/system/atomic.h> +#include <util/system/spinlock.h> + +#include "tune.h" + +namespace NThreading { + namespace NReadAsFilledPrivate { + typedef void* TMsgLink; + + static constexpr ui32 DEFAULT_BUNCH_SIZE = 251; + + struct TEmpty { + }; + + struct TEmptyAux { + TEmptyAux Retrieve() const { + return TEmptyAux(); + } + + void Store(TEmptyAux&) { + } + + static constexpr TEmptyAux Zero() { + return TEmptyAux(); + } + }; + + template <typename TAux> + struct TSlot { + TMsgLink volatile Msg; + TAux AuxiliaryData; + + inline void Store(TAux& aux) { + AuxiliaryData.Store(aux); + } + + inline TAux Retrieve() const { + return AuxiliaryData.Retrieve(); + } + + static TSlot<TAux> NullElem() { + return {nullptr, TAux::Zero()}; + } + + static TSlot<TAux> Pair(TMsgLink msg, TAux aux) { + return {msg, std::move(aux)}; + } + }; + + template <> + struct TSlot<TEmptyAux> { + TMsgLink volatile Msg; + + inline void Store(TEmptyAux&) { + } + + inline TEmptyAux Retrieve() const { + return TEmptyAux(); + } + + static TSlot<TEmptyAux> NullElem() { + return {nullptr}; + } + + static TSlot<TEmptyAux> Pair(TMsgLink msg, TEmptyAux) { + return {msg}; + } + }; + + enum TPushResult { + PUSH_RESULT_OK, + PUSH_RESULT_BACKWARD, + PUSH_RESULT_FORWARD, + }; + + template <ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE, + typename TBase = TEmpty, + typename TAux = TEmptyAux> + struct TMsgBunch: public TBase { + static constexpr size_t RELEASE_SIZE = BUNCH_SIZE * 2; + + ui64 FirstSlot; + + TSlot<TAux> LinkArray[BUNCH_SIZE]; + + TMsgBunch* volatile NextBunch; + TMsgBunch* volatile BackLink; + + ui64 volatile Token; + TMsgBunch* volatile NextToken; + + /* this push can return PUSH_RESULT_BLOCKED */ inline TPushResult Push(TMsgLink msg, ui64 slot, TAux auxiliary) { - if (Y_UNLIKELY(slot < FirstSlot)) { - return PUSH_RESULT_BACKWARD; - } - - if (Y_UNLIKELY(slot >= FirstSlot + BUNCH_SIZE)) { - return PUSH_RESULT_FORWARD; - } - - LinkArray[slot - FirstSlot].Store(auxiliary); - - AtomicSet(LinkArray[slot - FirstSlot].Msg, msg); - return PUSH_RESULT_OK; - } - - inline bool IsSlotHere(ui64 slot) { - return slot < FirstSlot + BUNCH_SIZE; - } - - inline TMsgLink GetSlot(ui64 slot) const { - return AtomicGet(LinkArray[slot - FirstSlot].Msg); - } - - inline TSlot<TAux> GetSlotAux(ui64 slot) const { - auto msg = GetSlot(slot); - auto aux = LinkArray[slot - FirstSlot].Retrieve(); - return TSlot<TAux>::Pair(msg, aux); - } - - inline TMsgBunch* GetNextBunch() const { - return AtomicGet(NextBunch); - } - - inline bool SetNextBunch(TMsgBunch* ptr) { - return AtomicCas(&NextBunch, ptr, nullptr); - } - - inline TMsgBunch* GetBackLink() const { - return AtomicGet(BackLink); - } - - inline TMsgBunch* GetToken(ui64 slot) { - return reinterpret_cast<TMsgBunch*>( - LinkArray[slot - FirstSlot].Msg); - } - - inline void IncrementToken() { - AtomicIncrement(Token); - } - - // the object could be destroyed after this method - inline void DecrementToken() { - if (Y_UNLIKELY(AtomicDecrement(Token) == RELEASE_SIZE)) { - Release(this); - AtomicGet(NextToken)->DecrementToken(); - // this could be invalid here - } - } - - // the object could be destroyed after this method - inline void SetNextToken(TMsgBunch* next) { - AtomicSet(NextToken, next); + if (Y_UNLIKELY(slot < FirstSlot)) { + return PUSH_RESULT_BACKWARD; + } + + if (Y_UNLIKELY(slot >= FirstSlot + BUNCH_SIZE)) { + return PUSH_RESULT_FORWARD; + } + + LinkArray[slot - FirstSlot].Store(auxiliary); + + AtomicSet(LinkArray[slot - FirstSlot].Msg, msg); + return PUSH_RESULT_OK; + } + + inline bool IsSlotHere(ui64 slot) { + return slot < FirstSlot + BUNCH_SIZE; + } + + inline TMsgLink GetSlot(ui64 slot) const { + return AtomicGet(LinkArray[slot - FirstSlot].Msg); + } + + inline TSlot<TAux> GetSlotAux(ui64 slot) const { + auto msg = GetSlot(slot); + auto aux = LinkArray[slot - FirstSlot].Retrieve(); + return TSlot<TAux>::Pair(msg, aux); + } + + inline TMsgBunch* GetNextBunch() const { + return AtomicGet(NextBunch); + } + + inline bool SetNextBunch(TMsgBunch* ptr) { + return AtomicCas(&NextBunch, ptr, nullptr); + } + + inline TMsgBunch* GetBackLink() const { + return AtomicGet(BackLink); + } + + inline TMsgBunch* GetToken(ui64 slot) { + return reinterpret_cast<TMsgBunch*>( + LinkArray[slot - FirstSlot].Msg); + } + + inline void IncrementToken() { + AtomicIncrement(Token); + } + + // the object could be destroyed after this method + inline void DecrementToken() { + if (Y_UNLIKELY(AtomicDecrement(Token) == RELEASE_SIZE)) { + Release(this); + AtomicGet(NextToken)->DecrementToken(); + // this could be invalid here + } + } + + // the object could be destroyed after this method + inline void SetNextToken(TMsgBunch* next) { + AtomicSet(NextToken, next); if (Y_UNLIKELY(AtomicAdd(Token, RELEASE_SIZE) == RELEASE_SIZE)) { - Release(this); - next->DecrementToken(); - } - // this could be invalid here - } - - TMsgBunch(ui64 start, TMsgBunch* backLink) { - AtomicSet(FirstSlot, start); - memset(&LinkArray, 0, sizeof(LinkArray)); - AtomicSet(NextBunch, nullptr); - AtomicSet(BackLink, backLink); - - AtomicSet(Token, 1); - AtomicSet(NextToken, nullptr); - } - - static void Release(TMsgBunch* block) { - auto backLink = AtomicGet(block->BackLink); - if (backLink == nullptr) { - return; - } - AtomicSet(block->BackLink, nullptr); - - do { - auto bbackLink = backLink->BackLink; - delete backLink; - backLink = bbackLink; - } while (backLink != nullptr); - } - - void Destroy() { - for (auto tail = BackLink; tail != nullptr;) { - auto next = tail->BackLink; - delete tail; - tail = next; - } - - for (auto next = this; next != nullptr;) { - auto nnext = next->NextBunch; - delete next; - next = nnext; - } - } - }; - - template <ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE, - typename TBunchBase = NReadAsFilledPrivate::TEmpty, - typename TAux = TEmptyAux> - class TWriteBucket { - public: - using TUsingAux = TAux; // for TReadBucket binding - using TBunch = TMsgBunch<BUNCH_SIZE, TBunchBase, TAux>; - - TWriteBucket(TBunch* bunch = new TBunch(0, nullptr)) { - AtomicSet(LastBunch, bunch); - AtomicSet(SlotCounter, 0); - } - - TWriteBucket(TWriteBucket&& move) - : LastBunch(move.LastBunch) - , SlotCounter(move.SlotCounter) - { - move.LastBunch = nullptr; - } - - ~TWriteBucket() { - if (LastBunch != nullptr) { - LastBunch->Destroy(); - } - } - - inline void Push(TMsgLink msg, TAux aux) { - ui64 pushSlot = AtomicGetAndIncrement(SlotCounter); - TBunch* hintBunch = GetLastBunch(); - - for (;;) { - auto hint = hintBunch->Push(msg, pushSlot, aux); - if (Y_LIKELY(hint == PUSH_RESULT_OK)) { - return; - } - HandleHint(hintBunch, hint); - } - } - - protected: - template <typename, template <typename, typename...> class> - friend class TReadBucket; - - TBunch* volatile LastBunch; // Hint - volatile ui64 SlotCounter; - - inline TBunch* GetLastBunch() const { - return AtomicGet(LastBunch); - } - - void HandleHint(TBunch*& hintBunch, TPushResult hint) { - if (Y_UNLIKELY(hint == PUSH_RESULT_BACKWARD)) { - hintBunch = hintBunch->GetBackLink(); - return; - } - - // PUSH_RESULT_FORWARD - auto nextBunch = hintBunch->GetNextBunch(); - - if (nextBunch == nullptr) { - auto first = hintBunch->FirstSlot + BUNCH_SIZE; - nextBunch = new TBunch(first, hintBunch); - if (Y_UNLIKELY(!hintBunch->SetNextBunch(nextBunch))) { - delete nextBunch; - nextBunch = hintBunch->GetNextBunch(); - } - } - - // hintBunch could not be freed here so it cannot be reused - // it's alright if this CAS was not succeeded, - // it means that other thread did that recently - AtomicCas(&LastBunch, nextBunch, hintBunch); - - hintBunch = nextBunch; - } - }; - + Release(this); + next->DecrementToken(); + } + // this could be invalid here + } + + TMsgBunch(ui64 start, TMsgBunch* backLink) { + AtomicSet(FirstSlot, start); + memset(&LinkArray, 0, sizeof(LinkArray)); + AtomicSet(NextBunch, nullptr); + AtomicSet(BackLink, backLink); + + AtomicSet(Token, 1); + AtomicSet(NextToken, nullptr); + } + + static void Release(TMsgBunch* block) { + auto backLink = AtomicGet(block->BackLink); + if (backLink == nullptr) { + return; + } + AtomicSet(block->BackLink, nullptr); + + do { + auto bbackLink = backLink->BackLink; + delete backLink; + backLink = bbackLink; + } while (backLink != nullptr); + } + + void Destroy() { + for (auto tail = BackLink; tail != nullptr;) { + auto next = tail->BackLink; + delete tail; + tail = next; + } + + for (auto next = this; next != nullptr;) { + auto nnext = next->NextBunch; + delete next; + next = nnext; + } + } + }; + + template <ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE, + typename TBunchBase = NReadAsFilledPrivate::TEmpty, + typename TAux = TEmptyAux> + class TWriteBucket { + public: + using TUsingAux = TAux; // for TReadBucket binding + using TBunch = TMsgBunch<BUNCH_SIZE, TBunchBase, TAux>; + + TWriteBucket(TBunch* bunch = new TBunch(0, nullptr)) { + AtomicSet(LastBunch, bunch); + AtomicSet(SlotCounter, 0); + } + + TWriteBucket(TWriteBucket&& move) + : LastBunch(move.LastBunch) + , SlotCounter(move.SlotCounter) + { + move.LastBunch = nullptr; + } + + ~TWriteBucket() { + if (LastBunch != nullptr) { + LastBunch->Destroy(); + } + } + + inline void Push(TMsgLink msg, TAux aux) { + ui64 pushSlot = AtomicGetAndIncrement(SlotCounter); + TBunch* hintBunch = GetLastBunch(); + + for (;;) { + auto hint = hintBunch->Push(msg, pushSlot, aux); + if (Y_LIKELY(hint == PUSH_RESULT_OK)) { + return; + } + HandleHint(hintBunch, hint); + } + } + + protected: + template <typename, template <typename, typename...> class> + friend class TReadBucket; + + TBunch* volatile LastBunch; // Hint + volatile ui64 SlotCounter; + + inline TBunch* GetLastBunch() const { + return AtomicGet(LastBunch); + } + + void HandleHint(TBunch*& hintBunch, TPushResult hint) { + if (Y_UNLIKELY(hint == PUSH_RESULT_BACKWARD)) { + hintBunch = hintBunch->GetBackLink(); + return; + } + + // PUSH_RESULT_FORWARD + auto nextBunch = hintBunch->GetNextBunch(); + + if (nextBunch == nullptr) { + auto first = hintBunch->FirstSlot + BUNCH_SIZE; + nextBunch = new TBunch(first, hintBunch); + if (Y_UNLIKELY(!hintBunch->SetNextBunch(nextBunch))) { + delete nextBunch; + nextBunch = hintBunch->GetNextBunch(); + } + } + + // hintBunch could not be freed here so it cannot be reused + // it's alright if this CAS was not succeeded, + // it means that other thread did that recently + AtomicCas(&LastBunch, nextBunch, hintBunch); + + hintBunch = nextBunch; + } + }; + template <typename TWBucket = TWriteBucket<>, template <typename, typename...> class TContainer = TDeque> - class TReadBucket { - public: - using TAux = typename TWBucket::TUsingAux; - using TBunch = typename TWBucket::TBunch; - - static constexpr int MAX_NUMBER_OF_TRIES_TO_READ = 5; - - TReadBucket(TWBucket* writer) - : Writer(writer) - , ReadBunch(writer->GetLastBunch()) - , LastKnownPushBunch(writer->GetLastBunch()) - { - ReadBunch->DecrementToken(); // no previous token - } - - TReadBucket(TReadBucket toCopy, TWBucket* writer) - : TReadBucket(std::move(toCopy)) - { - Writer = writer; - } - - ui64 ReadyCount() const { - return AtomicGet(Writer->SlotCounter) - ReadSlot; - } - - TMsgLink Pop() { - return PopAux().Msg; - } - - TMsgLink Peek() { - return PeekAux().Msg; - } - - TSlot<TAux> PopAux() { - for (;;) { - if (Y_UNLIKELY(ReadNow.size() != 0)) { - auto result = PopSkipped(); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - } - - if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { - if (Y_LIKELY(!RereadPushSlot())) { - return TSlot<TAux>::NullElem(); - } - continue; - } - - if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { - if (Y_UNLIKELY(!SwitchToNextBunch())) { - return TSlot<TAux>::NullElem(); - } - } - - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - ++ReadSlot; - return result; - } - - result = StubbornPop(); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - } - } - - TSlot<TAux> PeekAux() { - for (;;) { - if (Y_UNLIKELY(ReadNow.size() != 0)) { - auto result = PeekSkipped(); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - } - - if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { - if (Y_LIKELY(!RereadPushSlot())) { - return TSlot<TAux>::NullElem(); - } - continue; - } - - if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { - if (Y_UNLIKELY(!SwitchToNextBunch())) { - return TSlot<TAux>::NullElem(); - } - } - - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - - result = StubbornPeek(); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - } - } - - private: - TWBucket* Writer; - TBunch* ReadBunch; - ui64 ReadSlot = 0; - TBunch* LastKnownPushBunch; - ui64 LastKnownPushSlot = 0; - - struct TSkipItem { - TBunch* Bunch; - ui64 Slot; - TBunch* Token; - }; - - TContainer<TSkipItem> ReadNow; - TContainer<TSkipItem> ReadLater; - - void AddToReadLater() { - ReadLater.push_back({ReadBunch, ReadSlot, LastKnownPushBunch}); - LastKnownPushBunch->IncrementToken(); - ++ReadSlot; - } - - // MUST BE: ReadSlot == LastKnownPushSlot - bool RereadPushSlot() { - ReadNow = std::move(ReadLater); - ReadLater.clear(); - - auto oldSlot = LastKnownPushSlot; - - auto currentPushBunch = Writer->GetLastBunch(); - auto currentPushSlot = AtomicGet(Writer->SlotCounter); - - if (currentPushBunch != LastKnownPushBunch) { - // LastKnownPushBunch could be invalid after this line - LastKnownPushBunch->SetNextToken(currentPushBunch); - } - - LastKnownPushBunch = currentPushBunch; - LastKnownPushSlot = currentPushSlot; - - return oldSlot != LastKnownPushSlot; - } - - bool SwitchToNextBunch() { - for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { - auto next = ReadBunch->GetNextBunch(); - if (next != nullptr) { - ReadBunch = next; - return true; - } - SpinLockPause(); - } - return false; - } - - TSlot<TAux> StubbornPop() { - for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - ++ReadSlot; - return result; - } - SpinLockPause(); - } - - AddToReadLater(); - return TSlot<TAux>::NullElem(); - } - - TSlot<TAux> StubbornPeek() { - for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - SpinLockPause(); - } - - AddToReadLater(); - return TSlot<TAux>::NullElem(); - } - - TSlot<TAux> PopSkipped() { - do { - auto elem = ReadNow.front(); - ReadNow.pop_front(); - - auto result = elem.Bunch->GetSlotAux(elem.Slot); - if (Y_LIKELY(result.Msg != nullptr)) { - elem.Token->DecrementToken(); - return result; - } - - ReadLater.emplace_back(elem); - - } while (ReadNow.size() > 0); - - return TSlot<TAux>::NullElem(); - } - - TSlot<TAux> PeekSkipped() { - do { - auto elem = ReadNow.front(); - - auto result = elem.Bunch->GetSlotAux(elem.Slot); - if (Y_LIKELY(result.Msg != nullptr)) { - return result; - } - - ReadNow.pop_front(); - ReadLater.emplace_back(elem); - - } while (ReadNow.size() > 0); - - return TSlot<TAux>::NullElem(); - } - }; - - struct TDefaultParams { - static constexpr ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE; - using TBunchBase = TEmpty; - + class TReadBucket { + public: + using TAux = typename TWBucket::TUsingAux; + using TBunch = typename TWBucket::TBunch; + + static constexpr int MAX_NUMBER_OF_TRIES_TO_READ = 5; + + TReadBucket(TWBucket* writer) + : Writer(writer) + , ReadBunch(writer->GetLastBunch()) + , LastKnownPushBunch(writer->GetLastBunch()) + { + ReadBunch->DecrementToken(); // no previous token + } + + TReadBucket(TReadBucket toCopy, TWBucket* writer) + : TReadBucket(std::move(toCopy)) + { + Writer = writer; + } + + ui64 ReadyCount() const { + return AtomicGet(Writer->SlotCounter) - ReadSlot; + } + + TMsgLink Pop() { + return PopAux().Msg; + } + + TMsgLink Peek() { + return PeekAux().Msg; + } + + TSlot<TAux> PopAux() { + for (;;) { + if (Y_UNLIKELY(ReadNow.size() != 0)) { + auto result = PopSkipped(); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + } + + if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { + if (Y_LIKELY(!RereadPushSlot())) { + return TSlot<TAux>::NullElem(); + } + continue; + } + + if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { + if (Y_UNLIKELY(!SwitchToNextBunch())) { + return TSlot<TAux>::NullElem(); + } + } + + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + ++ReadSlot; + return result; + } + + result = StubbornPop(); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + } + } + + TSlot<TAux> PeekAux() { + for (;;) { + if (Y_UNLIKELY(ReadNow.size() != 0)) { + auto result = PeekSkipped(); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + } + + if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { + if (Y_LIKELY(!RereadPushSlot())) { + return TSlot<TAux>::NullElem(); + } + continue; + } + + if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { + if (Y_UNLIKELY(!SwitchToNextBunch())) { + return TSlot<TAux>::NullElem(); + } + } + + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + + result = StubbornPeek(); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + } + } + + private: + TWBucket* Writer; + TBunch* ReadBunch; + ui64 ReadSlot = 0; + TBunch* LastKnownPushBunch; + ui64 LastKnownPushSlot = 0; + + struct TSkipItem { + TBunch* Bunch; + ui64 Slot; + TBunch* Token; + }; + + TContainer<TSkipItem> ReadNow; + TContainer<TSkipItem> ReadLater; + + void AddToReadLater() { + ReadLater.push_back({ReadBunch, ReadSlot, LastKnownPushBunch}); + LastKnownPushBunch->IncrementToken(); + ++ReadSlot; + } + + // MUST BE: ReadSlot == LastKnownPushSlot + bool RereadPushSlot() { + ReadNow = std::move(ReadLater); + ReadLater.clear(); + + auto oldSlot = LastKnownPushSlot; + + auto currentPushBunch = Writer->GetLastBunch(); + auto currentPushSlot = AtomicGet(Writer->SlotCounter); + + if (currentPushBunch != LastKnownPushBunch) { + // LastKnownPushBunch could be invalid after this line + LastKnownPushBunch->SetNextToken(currentPushBunch); + } + + LastKnownPushBunch = currentPushBunch; + LastKnownPushSlot = currentPushSlot; + + return oldSlot != LastKnownPushSlot; + } + + bool SwitchToNextBunch() { + for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { + auto next = ReadBunch->GetNextBunch(); + if (next != nullptr) { + ReadBunch = next; + return true; + } + SpinLockPause(); + } + return false; + } + + TSlot<TAux> StubbornPop() { + for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + ++ReadSlot; + return result; + } + SpinLockPause(); + } + + AddToReadLater(); + return TSlot<TAux>::NullElem(); + } + + TSlot<TAux> StubbornPeek() { + for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + SpinLockPause(); + } + + AddToReadLater(); + return TSlot<TAux>::NullElem(); + } + + TSlot<TAux> PopSkipped() { + do { + auto elem = ReadNow.front(); + ReadNow.pop_front(); + + auto result = elem.Bunch->GetSlotAux(elem.Slot); + if (Y_LIKELY(result.Msg != nullptr)) { + elem.Token->DecrementToken(); + return result; + } + + ReadLater.emplace_back(elem); + + } while (ReadNow.size() > 0); + + return TSlot<TAux>::NullElem(); + } + + TSlot<TAux> PeekSkipped() { + do { + auto elem = ReadNow.front(); + + auto result = elem.Bunch->GetSlotAux(elem.Slot); + if (Y_LIKELY(result.Msg != nullptr)) { + return result; + } + + ReadNow.pop_front(); + ReadLater.emplace_back(elem); + + } while (ReadNow.size() > 0); + + return TSlot<TAux>::NullElem(); + } + }; + + struct TDefaultParams { + static constexpr ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE; + using TBunchBase = TEmpty; + template <typename TElem, typename... TRest> using TContainer = TDeque<TElem, TRest...>; - - static constexpr bool DeleteItems = true; - }; - - } //namespace NReadAsFilledPrivate - - DeclareTuneValueParam(TRaFQueueBunchSize, ui32, BUNCH_SIZE); - DeclareTuneTypeParam(TRaFQueueBunchBase, TBunchBase); - DeclareTuneContainer(TRaFQueueSkipContainer, TContainer); - DeclareTuneValueParam(TRaFQueueDeleteItems, bool, DeleteItems); - + + static constexpr bool DeleteItems = true; + }; + + } //namespace NReadAsFilledPrivate + + DeclareTuneValueParam(TRaFQueueBunchSize, ui32, BUNCH_SIZE); + DeclareTuneTypeParam(TRaFQueueBunchBase, TBunchBase); + DeclareTuneContainer(TRaFQueueSkipContainer, TContainer); + DeclareTuneValueParam(TRaFQueueDeleteItems, bool, DeleteItems); + template <typename TItem = void, typename... TParams> - class TReadAsFilledQueue { - private: - using TTuned = TTune<NReadAsFilledPrivate::TDefaultParams, TParams...>; - - static constexpr ui32 BUNCH_SIZE = TTuned::BUNCH_SIZE; - - using TBunchBase = typename TTuned::TBunchBase; - + class TReadAsFilledQueue { + private: + using TTuned = TTune<NReadAsFilledPrivate::TDefaultParams, TParams...>; + + static constexpr ui32 BUNCH_SIZE = TTuned::BUNCH_SIZE; + + using TBunchBase = typename TTuned::TBunchBase; + template <typename TElem, typename... TRest> - using TContainer = - typename TTuned::template TContainer<TElem, TRest...>; - - using TWriteBucket = - NReadAsFilledPrivate::TWriteBucket<BUNCH_SIZE, TBunchBase>; - using TReadBucket = - NReadAsFilledPrivate::TReadBucket<TWriteBucket, TContainer>; - - public: - TReadAsFilledQueue() - : RBucket(&WBucket) - { - } - - ~TReadAsFilledQueue() { - if (TTuned::DeleteItems) { - for (;;) { - auto msg = Pop(); - if (msg == nullptr) { - break; - } - TDelete::Destroy(msg); - } - } - } - - void Push(TItem* msg) { - WBucket.Push((void*)msg, NReadAsFilledPrivate::TEmptyAux()); - } - - TItem* Pop() { - return (TItem*)RBucket.Pop(); - } - - TItem* Peek() { - return (TItem*)RBucket.Peek(); - } - - protected: - TWriteBucket WBucket; - TReadBucket RBucket; - }; -} + using TContainer = + typename TTuned::template TContainer<TElem, TRest...>; + + using TWriteBucket = + NReadAsFilledPrivate::TWriteBucket<BUNCH_SIZE, TBunchBase>; + using TReadBucket = + NReadAsFilledPrivate::TReadBucket<TWriteBucket, TContainer>; + + public: + TReadAsFilledQueue() + : RBucket(&WBucket) + { + } + + ~TReadAsFilledQueue() { + if (TTuned::DeleteItems) { + for (;;) { + auto msg = Pop(); + if (msg == nullptr) { + break; + } + TDelete::Destroy(msg); + } + } + } + + void Push(TItem* msg) { + WBucket.Push((void*)msg, NReadAsFilledPrivate::TEmptyAux()); + } + + TItem* Pop() { + return (TItem*)RBucket.Pop(); + } + + TItem* Peek() { + return (TItem*)RBucket.Peek(); + } + + protected: + TWriteBucket WBucket; + TReadBucket RBucket; + }; +} diff --git a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.cpp b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.cpp index 2bd0c29821..00dbfeaa64 100644 --- a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.cpp +++ b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.cpp @@ -1 +1 @@ -#include "mpsc_vinfarr_obstructive.h" +#include "mpsc_vinfarr_obstructive.h" diff --git a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h index 5f91f1b5a8..3e1ae92342 100644 --- a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h +++ b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h @@ -1,528 +1,528 @@ -#pragma once - -/* - Semi-wait-free queue, multiple producers - one consumer. Strict order. - The queue algorithm is using concept of virtual infinite array. - - A producer takes a number from a counter and atomicaly increments the counter. - The number taken is a number of a slot for the producer to put a new message - into infinite array. - - Then producer constructs a virtual infinite array by bidirectional linked list - of blocks. Each block contains several slots. - - There is a hint pointer which optimisticly points to the last block - of the list and never goes backward. - - Consumer exploits the property of the hint pointer always going forward - to free old blocks eventually. Consumer periodically read the hint pointer - and the counter and thus deduce producers which potentially holds the pointer - to a block. Consumer can free the block if all that producers filled their - slots and left the queue. - - No producer can stop the progress for other producers. - - Consumer can obstruct a slot of a delayed producer by putting special mark. - Thus no producer can stop the progress for consumer. - But a slow producer may be forced to retry unlimited number of times. - Though it's very unlikely for a non-preempted producer to be obstructed. - That's why the algorithm is semi-wait-free. - - WARNING: there is no wait¬ify mechanic for consumer, - consumer receives nullptr if queue was empty. - - WARNING: though the algorithm itself is lock-free - but producers and consumer could be blocked by memory allocator - - WARNING: copy constructers of the queue are not thread-safe - */ - -#include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> -#include <util/system/atomic.h> -#include <util/system/spinlock.h> - -#include "tune.h" - -namespace NThreading { - namespace NObstructiveQueuePrivate { - typedef void* TMsgLink; - - struct TEmpty { - }; - - struct TEmptyAux { - TEmptyAux Retrieve() const { - return TEmptyAux(); - } - void Store(TEmptyAux&) { - } - static constexpr TEmptyAux Zero() { - return TEmptyAux(); - } - }; - - template <typename TAux> - struct TSlot { - TMsgLink volatile Msg; - TAux AuxiliaryData; - - inline void Store(TAux& aux) { - AuxiliaryData.Store(aux); - } - - inline TAux Retrieve() const { - return AuxiliaryData.Retrieve(); - } - - static TSlot<TAux> NullElem() { - return {nullptr, TAux::Zero()}; - } - - static TSlot<TAux> Pair(TMsgLink msg, TAux aux) { - return {msg, std::move(aux)}; - } - }; - - template <> - struct TSlot<TEmptyAux> { - TMsgLink volatile Msg; - inline void Store(TEmptyAux&) { - } - inline TEmptyAux Retrieve() const { - return TEmptyAux(); - } - - static TSlot<TEmptyAux> NullElem() { - return {nullptr}; - } - - static TSlot<TEmptyAux> Pair(TMsgLink msg, TEmptyAux) { - return {msg}; - } - }; - - enum TPushResult { - PUSH_RESULT_OK, - PUSH_RESULT_BACKWARD, - PUSH_RESULT_FORWARD, - PUSH_RESULT_BLOCKED, - }; - - template <typename TAux, ui32 BUNCH_SIZE, typename TBase = TEmpty> - struct TMsgBunch: public TBase { - ui64 FirstSlot; - - TSlot<TAux> LinkArray[BUNCH_SIZE]; - - TMsgBunch* volatile NextBunch; - TMsgBunch* volatile BackLink; - - ui64 volatile Token; - TMsgBunch* volatile NextToken; - - /* this push can return PUSH_RESULT_BLOCKED */ - inline TPushResult Push(TMsgLink msg, ui64 slot, TAux auxiliary) { - if (Y_UNLIKELY(slot < FirstSlot)) { - return PUSH_RESULT_BACKWARD; - } - - if (Y_UNLIKELY(slot >= FirstSlot + BUNCH_SIZE)) { - return PUSH_RESULT_FORWARD; - } - - LinkArray[slot - FirstSlot].Store(auxiliary); - - auto oldValue = AtomicSwap(&LinkArray[slot - FirstSlot].Msg, msg); - - if (Y_LIKELY(oldValue == nullptr)) { - return PUSH_RESULT_OK; - } else { - LeaveBlocked(oldValue); - return PUSH_RESULT_BLOCKED; - } - } - - inline bool IsSlotHere(ui64 slot) { - return slot < FirstSlot + BUNCH_SIZE; - } - - inline TMsgLink GetSlot(ui64 slot) const { - return AtomicGet(LinkArray[slot - FirstSlot].Msg); - } - - inline TSlot<TAux> GetSlotAux(ui64 slot) const { - auto msg = GetSlot(slot); - auto aux = LinkArray[slot - FirstSlot].Retrieve(); - return TSlot<TAux>::Pair(msg, aux); - } - - void LeaveBlocked(ui64 slot) { - auto token = GetToken(slot); - token->DecrementToken(); - } - - void LeaveBlocked(TMsgLink msg) { - auto token = reinterpret_cast<TMsgBunch*>(msg); - token->DecrementToken(); - } - - TSlot<TAux> BlockSlotAux(ui64 slot, TMsgBunch* token) { - auto old = - AtomicSwap(&LinkArray[slot - FirstSlot].Msg, (TMsgLink)token); - if (old == nullptr) { - // It's valid to increment after AtomicCas - // because token will release data only after SetNextToken - token->IncrementToken(); - return TSlot<TAux>::NullElem(); - } - return TSlot<TAux>::Pair(old, LinkArray[slot - FirstSlot].Retrieve()); - } - - inline TMsgBunch* GetNextBunch() const { - return AtomicGet(NextBunch); - } - - inline bool SetNextBunch(TMsgBunch* ptr) { - return AtomicCas(&NextBunch, ptr, nullptr); - } - - inline TMsgBunch* GetBackLink() const { - return AtomicGet(BackLink); - } - - inline TMsgBunch* GetToken(ui64 slot) { - return reinterpret_cast<TMsgBunch*>(LinkArray[slot - FirstSlot].Msg); - } - - inline void IncrementToken() { - AtomicIncrement(Token); - } - - // the object could be destroyed after this method - inline void DecrementToken() { - if (Y_UNLIKELY(AtomicDecrement(Token) == BUNCH_SIZE)) { - Release(this); - AtomicGet(NextToken)->DecrementToken(); - // this could be invalid here - } - } - - // the object could be destroyed after this method - inline void SetNextToken(TMsgBunch* next) { - AtomicSet(NextToken, next); - if (Y_UNLIKELY(AtomicAdd(Token, BUNCH_SIZE) == BUNCH_SIZE)) { - Release(this); - next->DecrementToken(); - } - // this could be invalid here - } - - TMsgBunch(ui64 start, TMsgBunch* backLink) { - AtomicSet(FirstSlot, start); - memset(&LinkArray, 0, sizeof(LinkArray)); - AtomicSet(NextBunch, nullptr); - AtomicSet(BackLink, backLink); - - AtomicSet(Token, 1); - AtomicSet(NextToken, nullptr); - } - - static void Release(TMsgBunch* bunch) { - auto backLink = AtomicGet(bunch->BackLink); - if (backLink == nullptr) { - return; - } - AtomicSet(bunch->BackLink, nullptr); - - do { - auto bbackLink = backLink->BackLink; - delete backLink; - backLink = bbackLink; - } while (backLink != nullptr); - } - - void Destroy() { - for (auto tail = BackLink; tail != nullptr;) { - auto next = tail->BackLink; - delete tail; - tail = next; - } - - for (auto next = this; next != nullptr;) { - auto nnext = next->NextBunch; - delete next; - next = nnext; - } - } - }; - - template <typename TAux, ui32 BUNCH_SIZE, typename TBunchBase = TEmpty> - class TWriteBucket { - public: - static const ui64 GROSS_SIZE; - - using TBunch = TMsgBunch<TAux, BUNCH_SIZE, TBunchBase>; - - TWriteBucket(TBunch* bunch = new TBunch(0, nullptr)) - : LastBunch(bunch) - , SlotCounter(0) - { - } - - TWriteBucket(TWriteBucket&& move) - : LastBunch(move.LastBunch) - , SlotCounter(move.SlotCounter) - { - move.LastBunch = nullptr; - } - - ~TWriteBucket() { - if (LastBunch != nullptr) { - LastBunch->Destroy(); - } - } - - inline bool Push(TMsgLink msg, TAux aux) { - ui64 pushSlot = AtomicGetAndIncrement(SlotCounter); - TBunch* hintBunch = GetLastBunch(); - - for (;;) { - auto hint = hintBunch->Push(msg, pushSlot, aux); - if (Y_LIKELY(hint == PUSH_RESULT_OK)) { - return true; - } - bool hhResult = HandleHint(hintBunch, hint); - if (Y_UNLIKELY(!hhResult)) { - return false; - } - } - } - - protected: - template <typename, ui32, typename> - friend class TReadBucket; - - TBunch* volatile LastBunch; // Hint - volatile ui64 SlotCounter; - - inline TBunch* GetLastBunch() const { - return AtomicGet(LastBunch); - } - - bool HandleHint(TBunch*& hintBunch, TPushResult hint) { - if (Y_UNLIKELY(hint == PUSH_RESULT_BLOCKED)) { - return false; - } - - if (Y_UNLIKELY(hint == PUSH_RESULT_BACKWARD)) { - hintBunch = hintBunch->GetBackLink(); - return true; - } - - // PUSH_RESULT_FORWARD - auto nextBunch = hintBunch->GetNextBunch(); - - if (nextBunch == nullptr) { - auto first = hintBunch->FirstSlot + BUNCH_SIZE; - nextBunch = new TBunch(first, hintBunch); - if (Y_UNLIKELY(!hintBunch->SetNextBunch(nextBunch))) { - delete nextBunch; - nextBunch = hintBunch->GetNextBunch(); - } - } - - // hintBunch could not be freed here so it cannot be reused - // it's alright if this CAS was not succeeded, - // it means that other thread did that recently - AtomicCas(&LastBunch, nextBunch, hintBunch); - - hintBunch = nextBunch; - return true; - } - }; - - template <typename TAux, ui32 BUNCH_SIZE, typename TBunchBase> - class TReadBucket { - public: - static constexpr int MAX_NUMBER_OF_TRIES_TO_READ = 20; - - using TWBucket = TWriteBucket<TAux, BUNCH_SIZE, TBunchBase>; - using TBunch = TMsgBunch<TAux, BUNCH_SIZE, TBunchBase>; - - TReadBucket(TWBucket* writer) - : Writer(writer) - , ReadBunch(writer->GetLastBunch()) - , LastKnownPushBunch(writer->GetLastBunch()) - { - ReadBunch->DecrementToken(); // no previous token - } - - TReadBucket(TReadBucket toCopy, TWBucket* writer) - : TReadBucket(std::move(toCopy)) - { - Writer = writer; - } - - ui64 ReadyCount() const { - return AtomicGet(Writer->SlotCounter) - ReadSlot; - } - - inline TMsgLink Pop() { - return PopAux().Msg; - } - - inline TSlot<TAux> PopAux() { - for (;;) { - if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { - if (Y_LIKELY(!RereadPushSlot())) { - return TSlot<TAux>::NullElem(); - } - } - - if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { - if (Y_UNLIKELY(!SwitchToNextBunch())) { - return TSlot<TAux>::NullElem(); - } - } - - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - ++ReadSlot; - return result; - } - - if (ReadSlot + 1 == AtomicGet(Writer->SlotCounter)) { - return TSlot<TAux>::NullElem(); - } - - result = StubbornPopAux(); - - if (result.Msg != nullptr) { - return result; - } - } - } - - private: - TWBucket* Writer; - TBunch* ReadBunch; - ui64 ReadSlot = 0; - TBunch* LastKnownPushBunch; - ui64 LastKnownPushSlot = 0; - - // MUST BE: ReadSlot == LastKnownPushSlot - bool RereadPushSlot() { - auto oldSlot = LastKnownPushSlot; - - auto currentPushBunch = Writer->GetLastBunch(); - auto currentPushSlot = AtomicGet(Writer->SlotCounter); - - if (currentPushBunch != LastKnownPushBunch) { - // LastKnownPushBunch could be invalid after this line - LastKnownPushBunch->SetNextToken(currentPushBunch); - } - - LastKnownPushBunch = currentPushBunch; - LastKnownPushSlot = currentPushSlot; - - return oldSlot != LastKnownPushSlot; - } - - bool SwitchToNextBunch() { - for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { - auto next = ReadBunch->GetNextBunch(); - if (next != nullptr) { - ReadBunch = next; - return true; - } - SpinLockPause(); - } - return false; - } - - TSlot<TAux> StubbornPopAux() { - for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { - auto result = ReadBunch->GetSlotAux(ReadSlot); - if (Y_LIKELY(result.Msg != nullptr)) { - ++ReadSlot; - return result; - } - SpinLockPause(); - } - - return ReadBunch->BlockSlotAux(ReadSlot++, LastKnownPushBunch); - } - }; - - struct TDefaultParams { - static constexpr bool DeleteItems = true; - using TAux = NObstructiveQueuePrivate::TEmptyAux; - using TBunchBase = NObstructiveQueuePrivate::TEmpty; - static constexpr ui32 BUNCH_SIZE = 251; - }; - - } //namespace NObstructiveQueuePrivate - - DeclareTuneValueParam(TObstructiveQueueBunchSize, ui32, BUNCH_SIZE); - DeclareTuneValueParam(TObstructiveQueueDeleteItems, bool, DeleteItems); - DeclareTuneTypeParam(TObstructiveQueueBunchBase, TBunchBase); - DeclareTuneTypeParam(TObstructiveQueueAux, TAux); - +#pragma once + +/* + Semi-wait-free queue, multiple producers - one consumer. Strict order. + The queue algorithm is using concept of virtual infinite array. + + A producer takes a number from a counter and atomicaly increments the counter. + The number taken is a number of a slot for the producer to put a new message + into infinite array. + + Then producer constructs a virtual infinite array by bidirectional linked list + of blocks. Each block contains several slots. + + There is a hint pointer which optimisticly points to the last block + of the list and never goes backward. + + Consumer exploits the property of the hint pointer always going forward + to free old blocks eventually. Consumer periodically read the hint pointer + and the counter and thus deduce producers which potentially holds the pointer + to a block. Consumer can free the block if all that producers filled their + slots and left the queue. + + No producer can stop the progress for other producers. + + Consumer can obstruct a slot of a delayed producer by putting special mark. + Thus no producer can stop the progress for consumer. + But a slow producer may be forced to retry unlimited number of times. + Though it's very unlikely for a non-preempted producer to be obstructed. + That's why the algorithm is semi-wait-free. + + WARNING: there is no wait¬ify mechanic for consumer, + consumer receives nullptr if queue was empty. + + WARNING: though the algorithm itself is lock-free + but producers and consumer could be blocked by memory allocator + + WARNING: copy constructers of the queue are not thread-safe + */ + +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> +#include <util/system/atomic.h> +#include <util/system/spinlock.h> + +#include "tune.h" + +namespace NThreading { + namespace NObstructiveQueuePrivate { + typedef void* TMsgLink; + + struct TEmpty { + }; + + struct TEmptyAux { + TEmptyAux Retrieve() const { + return TEmptyAux(); + } + void Store(TEmptyAux&) { + } + static constexpr TEmptyAux Zero() { + return TEmptyAux(); + } + }; + + template <typename TAux> + struct TSlot { + TMsgLink volatile Msg; + TAux AuxiliaryData; + + inline void Store(TAux& aux) { + AuxiliaryData.Store(aux); + } + + inline TAux Retrieve() const { + return AuxiliaryData.Retrieve(); + } + + static TSlot<TAux> NullElem() { + return {nullptr, TAux::Zero()}; + } + + static TSlot<TAux> Pair(TMsgLink msg, TAux aux) { + return {msg, std::move(aux)}; + } + }; + + template <> + struct TSlot<TEmptyAux> { + TMsgLink volatile Msg; + inline void Store(TEmptyAux&) { + } + inline TEmptyAux Retrieve() const { + return TEmptyAux(); + } + + static TSlot<TEmptyAux> NullElem() { + return {nullptr}; + } + + static TSlot<TEmptyAux> Pair(TMsgLink msg, TEmptyAux) { + return {msg}; + } + }; + + enum TPushResult { + PUSH_RESULT_OK, + PUSH_RESULT_BACKWARD, + PUSH_RESULT_FORWARD, + PUSH_RESULT_BLOCKED, + }; + + template <typename TAux, ui32 BUNCH_SIZE, typename TBase = TEmpty> + struct TMsgBunch: public TBase { + ui64 FirstSlot; + + TSlot<TAux> LinkArray[BUNCH_SIZE]; + + TMsgBunch* volatile NextBunch; + TMsgBunch* volatile BackLink; + + ui64 volatile Token; + TMsgBunch* volatile NextToken; + + /* this push can return PUSH_RESULT_BLOCKED */ + inline TPushResult Push(TMsgLink msg, ui64 slot, TAux auxiliary) { + if (Y_UNLIKELY(slot < FirstSlot)) { + return PUSH_RESULT_BACKWARD; + } + + if (Y_UNLIKELY(slot >= FirstSlot + BUNCH_SIZE)) { + return PUSH_RESULT_FORWARD; + } + + LinkArray[slot - FirstSlot].Store(auxiliary); + + auto oldValue = AtomicSwap(&LinkArray[slot - FirstSlot].Msg, msg); + + if (Y_LIKELY(oldValue == nullptr)) { + return PUSH_RESULT_OK; + } else { + LeaveBlocked(oldValue); + return PUSH_RESULT_BLOCKED; + } + } + + inline bool IsSlotHere(ui64 slot) { + return slot < FirstSlot + BUNCH_SIZE; + } + + inline TMsgLink GetSlot(ui64 slot) const { + return AtomicGet(LinkArray[slot - FirstSlot].Msg); + } + + inline TSlot<TAux> GetSlotAux(ui64 slot) const { + auto msg = GetSlot(slot); + auto aux = LinkArray[slot - FirstSlot].Retrieve(); + return TSlot<TAux>::Pair(msg, aux); + } + + void LeaveBlocked(ui64 slot) { + auto token = GetToken(slot); + token->DecrementToken(); + } + + void LeaveBlocked(TMsgLink msg) { + auto token = reinterpret_cast<TMsgBunch*>(msg); + token->DecrementToken(); + } + + TSlot<TAux> BlockSlotAux(ui64 slot, TMsgBunch* token) { + auto old = + AtomicSwap(&LinkArray[slot - FirstSlot].Msg, (TMsgLink)token); + if (old == nullptr) { + // It's valid to increment after AtomicCas + // because token will release data only after SetNextToken + token->IncrementToken(); + return TSlot<TAux>::NullElem(); + } + return TSlot<TAux>::Pair(old, LinkArray[slot - FirstSlot].Retrieve()); + } + + inline TMsgBunch* GetNextBunch() const { + return AtomicGet(NextBunch); + } + + inline bool SetNextBunch(TMsgBunch* ptr) { + return AtomicCas(&NextBunch, ptr, nullptr); + } + + inline TMsgBunch* GetBackLink() const { + return AtomicGet(BackLink); + } + + inline TMsgBunch* GetToken(ui64 slot) { + return reinterpret_cast<TMsgBunch*>(LinkArray[slot - FirstSlot].Msg); + } + + inline void IncrementToken() { + AtomicIncrement(Token); + } + + // the object could be destroyed after this method + inline void DecrementToken() { + if (Y_UNLIKELY(AtomicDecrement(Token) == BUNCH_SIZE)) { + Release(this); + AtomicGet(NextToken)->DecrementToken(); + // this could be invalid here + } + } + + // the object could be destroyed after this method + inline void SetNextToken(TMsgBunch* next) { + AtomicSet(NextToken, next); + if (Y_UNLIKELY(AtomicAdd(Token, BUNCH_SIZE) == BUNCH_SIZE)) { + Release(this); + next->DecrementToken(); + } + // this could be invalid here + } + + TMsgBunch(ui64 start, TMsgBunch* backLink) { + AtomicSet(FirstSlot, start); + memset(&LinkArray, 0, sizeof(LinkArray)); + AtomicSet(NextBunch, nullptr); + AtomicSet(BackLink, backLink); + + AtomicSet(Token, 1); + AtomicSet(NextToken, nullptr); + } + + static void Release(TMsgBunch* bunch) { + auto backLink = AtomicGet(bunch->BackLink); + if (backLink == nullptr) { + return; + } + AtomicSet(bunch->BackLink, nullptr); + + do { + auto bbackLink = backLink->BackLink; + delete backLink; + backLink = bbackLink; + } while (backLink != nullptr); + } + + void Destroy() { + for (auto tail = BackLink; tail != nullptr;) { + auto next = tail->BackLink; + delete tail; + tail = next; + } + + for (auto next = this; next != nullptr;) { + auto nnext = next->NextBunch; + delete next; + next = nnext; + } + } + }; + + template <typename TAux, ui32 BUNCH_SIZE, typename TBunchBase = TEmpty> + class TWriteBucket { + public: + static const ui64 GROSS_SIZE; + + using TBunch = TMsgBunch<TAux, BUNCH_SIZE, TBunchBase>; + + TWriteBucket(TBunch* bunch = new TBunch(0, nullptr)) + : LastBunch(bunch) + , SlotCounter(0) + { + } + + TWriteBucket(TWriteBucket&& move) + : LastBunch(move.LastBunch) + , SlotCounter(move.SlotCounter) + { + move.LastBunch = nullptr; + } + + ~TWriteBucket() { + if (LastBunch != nullptr) { + LastBunch->Destroy(); + } + } + + inline bool Push(TMsgLink msg, TAux aux) { + ui64 pushSlot = AtomicGetAndIncrement(SlotCounter); + TBunch* hintBunch = GetLastBunch(); + + for (;;) { + auto hint = hintBunch->Push(msg, pushSlot, aux); + if (Y_LIKELY(hint == PUSH_RESULT_OK)) { + return true; + } + bool hhResult = HandleHint(hintBunch, hint); + if (Y_UNLIKELY(!hhResult)) { + return false; + } + } + } + + protected: + template <typename, ui32, typename> + friend class TReadBucket; + + TBunch* volatile LastBunch; // Hint + volatile ui64 SlotCounter; + + inline TBunch* GetLastBunch() const { + return AtomicGet(LastBunch); + } + + bool HandleHint(TBunch*& hintBunch, TPushResult hint) { + if (Y_UNLIKELY(hint == PUSH_RESULT_BLOCKED)) { + return false; + } + + if (Y_UNLIKELY(hint == PUSH_RESULT_BACKWARD)) { + hintBunch = hintBunch->GetBackLink(); + return true; + } + + // PUSH_RESULT_FORWARD + auto nextBunch = hintBunch->GetNextBunch(); + + if (nextBunch == nullptr) { + auto first = hintBunch->FirstSlot + BUNCH_SIZE; + nextBunch = new TBunch(first, hintBunch); + if (Y_UNLIKELY(!hintBunch->SetNextBunch(nextBunch))) { + delete nextBunch; + nextBunch = hintBunch->GetNextBunch(); + } + } + + // hintBunch could not be freed here so it cannot be reused + // it's alright if this CAS was not succeeded, + // it means that other thread did that recently + AtomicCas(&LastBunch, nextBunch, hintBunch); + + hintBunch = nextBunch; + return true; + } + }; + + template <typename TAux, ui32 BUNCH_SIZE, typename TBunchBase> + class TReadBucket { + public: + static constexpr int MAX_NUMBER_OF_TRIES_TO_READ = 20; + + using TWBucket = TWriteBucket<TAux, BUNCH_SIZE, TBunchBase>; + using TBunch = TMsgBunch<TAux, BUNCH_SIZE, TBunchBase>; + + TReadBucket(TWBucket* writer) + : Writer(writer) + , ReadBunch(writer->GetLastBunch()) + , LastKnownPushBunch(writer->GetLastBunch()) + { + ReadBunch->DecrementToken(); // no previous token + } + + TReadBucket(TReadBucket toCopy, TWBucket* writer) + : TReadBucket(std::move(toCopy)) + { + Writer = writer; + } + + ui64 ReadyCount() const { + return AtomicGet(Writer->SlotCounter) - ReadSlot; + } + + inline TMsgLink Pop() { + return PopAux().Msg; + } + + inline TSlot<TAux> PopAux() { + for (;;) { + if (Y_UNLIKELY(ReadSlot == LastKnownPushSlot)) { + if (Y_LIKELY(!RereadPushSlot())) { + return TSlot<TAux>::NullElem(); + } + } + + if (Y_UNLIKELY(!ReadBunch->IsSlotHere(ReadSlot))) { + if (Y_UNLIKELY(!SwitchToNextBunch())) { + return TSlot<TAux>::NullElem(); + } + } + + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + ++ReadSlot; + return result; + } + + if (ReadSlot + 1 == AtomicGet(Writer->SlotCounter)) { + return TSlot<TAux>::NullElem(); + } + + result = StubbornPopAux(); + + if (result.Msg != nullptr) { + return result; + } + } + } + + private: + TWBucket* Writer; + TBunch* ReadBunch; + ui64 ReadSlot = 0; + TBunch* LastKnownPushBunch; + ui64 LastKnownPushSlot = 0; + + // MUST BE: ReadSlot == LastKnownPushSlot + bool RereadPushSlot() { + auto oldSlot = LastKnownPushSlot; + + auto currentPushBunch = Writer->GetLastBunch(); + auto currentPushSlot = AtomicGet(Writer->SlotCounter); + + if (currentPushBunch != LastKnownPushBunch) { + // LastKnownPushBunch could be invalid after this line + LastKnownPushBunch->SetNextToken(currentPushBunch); + } + + LastKnownPushBunch = currentPushBunch; + LastKnownPushSlot = currentPushSlot; + + return oldSlot != LastKnownPushSlot; + } + + bool SwitchToNextBunch() { + for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { + auto next = ReadBunch->GetNextBunch(); + if (next != nullptr) { + ReadBunch = next; + return true; + } + SpinLockPause(); + } + return false; + } + + TSlot<TAux> StubbornPopAux() { + for (int q = 0; q < MAX_NUMBER_OF_TRIES_TO_READ; ++q) { + auto result = ReadBunch->GetSlotAux(ReadSlot); + if (Y_LIKELY(result.Msg != nullptr)) { + ++ReadSlot; + return result; + } + SpinLockPause(); + } + + return ReadBunch->BlockSlotAux(ReadSlot++, LastKnownPushBunch); + } + }; + + struct TDefaultParams { + static constexpr bool DeleteItems = true; + using TAux = NObstructiveQueuePrivate::TEmptyAux; + using TBunchBase = NObstructiveQueuePrivate::TEmpty; + static constexpr ui32 BUNCH_SIZE = 251; + }; + + } //namespace NObstructiveQueuePrivate + + DeclareTuneValueParam(TObstructiveQueueBunchSize, ui32, BUNCH_SIZE); + DeclareTuneValueParam(TObstructiveQueueDeleteItems, bool, DeleteItems); + DeclareTuneTypeParam(TObstructiveQueueBunchBase, TBunchBase); + DeclareTuneTypeParam(TObstructiveQueueAux, TAux); + template <typename TItem = void, typename... TParams> - class TObstructiveConsumerAuxQueue { - private: - using TTuned = - TTune<NObstructiveQueuePrivate::TDefaultParams, TParams...>; - - using TAux = typename TTuned::TAux; - using TSlot = NObstructiveQueuePrivate::TSlot<TAux>; - using TMsgLink = NObstructiveQueuePrivate::TMsgLink; - using TBunchBase = typename TTuned::TBunchBase; - static constexpr bool DeleteItems = TTuned::DeleteItems; - static constexpr ui32 BUNCH_SIZE = TTuned::BUNCH_SIZE; - - public: - TObstructiveConsumerAuxQueue() - : RBuckets(&WBucket) - { - } - - ~TObstructiveConsumerAuxQueue() { - if (DeleteItems) { - for (;;) { - auto msg = Pop(); - if (msg == nullptr) { - break; - } - TDelete::Destroy(msg); - } - } - } - - void Push(TItem* msg) { - while (!WBucket.Push(reinterpret_cast<TMsgLink>(msg), TAux())) { - } - } - - TItem* Pop() { - return reinterpret_cast<TItem*>(RBuckets.Pop()); - } - - TSlot PopAux() { - return RBuckets.PopAux(); - } - - private: - NObstructiveQueuePrivate::TWriteBucket<TAux, BUNCH_SIZE, TBunchBase> - WBucket; - NObstructiveQueuePrivate::TReadBucket<TAux, BUNCH_SIZE, TBunchBase> - RBuckets; - }; - - template <typename TItem = void, bool DeleteItems = true> - class TObstructiveConsumerQueue + class TObstructiveConsumerAuxQueue { + private: + using TTuned = + TTune<NObstructiveQueuePrivate::TDefaultParams, TParams...>; + + using TAux = typename TTuned::TAux; + using TSlot = NObstructiveQueuePrivate::TSlot<TAux>; + using TMsgLink = NObstructiveQueuePrivate::TMsgLink; + using TBunchBase = typename TTuned::TBunchBase; + static constexpr bool DeleteItems = TTuned::DeleteItems; + static constexpr ui32 BUNCH_SIZE = TTuned::BUNCH_SIZE; + + public: + TObstructiveConsumerAuxQueue() + : RBuckets(&WBucket) + { + } + + ~TObstructiveConsumerAuxQueue() { + if (DeleteItems) { + for (;;) { + auto msg = Pop(); + if (msg == nullptr) { + break; + } + TDelete::Destroy(msg); + } + } + } + + void Push(TItem* msg) { + while (!WBucket.Push(reinterpret_cast<TMsgLink>(msg), TAux())) { + } + } + + TItem* Pop() { + return reinterpret_cast<TItem*>(RBuckets.Pop()); + } + + TSlot PopAux() { + return RBuckets.PopAux(); + } + + private: + NObstructiveQueuePrivate::TWriteBucket<TAux, BUNCH_SIZE, TBunchBase> + WBucket; + NObstructiveQueuePrivate::TReadBucket<TAux, BUNCH_SIZE, TBunchBase> + RBuckets; + }; + + template <typename TItem = void, bool DeleteItems = true> + class TObstructiveConsumerQueue : public TObstructiveConsumerAuxQueue<TItem, TObstructiveQueueDeleteItems<DeleteItems>> { - }; + }; } diff --git a/library/cpp/threading/queue/queue_ut.cpp b/library/cpp/threading/queue/queue_ut.cpp index 80eca147da..8b36437034 100644 --- a/library/cpp/threading/queue/queue_ut.cpp +++ b/library/cpp/threading/queue/queue_ut.cpp @@ -1,242 +1,242 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/system/thread.h> - -#include "ut_helpers.h" - -typedef void* TMsgLink; - +#include <util/system/thread.h> + +#include "ut_helpers.h" + +typedef void* TMsgLink; + template <typename TQueueType> -class TQueueTestProcs: public TTestBase { -private: +class TQueueTestProcs: public TTestBase { +private: UNIT_TEST_SUITE_DEMANGLE(TQueueTestProcs<TQueueType>); - UNIT_TEST(Threads2_Push1M_Threads1_Pop2M) - UNIT_TEST(Threads4_Push1M_Threads1_Pop4M) - UNIT_TEST(Threads8_RndPush100K_Threads8_Queues) + UNIT_TEST(Threads2_Push1M_Threads1_Pop2M) + UNIT_TEST(Threads4_Push1M_Threads1_Pop4M) + UNIT_TEST(Threads8_RndPush100K_Threads8_Queues) /* - UNIT_TEST(Threads24_RndPush100K_Threads24_Queues) - UNIT_TEST(Threads24_RndPush100K_Threads8_Queues) - UNIT_TEST(Threads24_RndPush100K_Threads4_Queues) -*/ - UNIT_TEST_SUITE_END(); - -public: - void Push1M_Pop1M() { + UNIT_TEST(Threads24_RndPush100K_Threads24_Queues) + UNIT_TEST(Threads24_RndPush100K_Threads8_Queues) + UNIT_TEST(Threads24_RndPush100K_Threads4_Queues) +*/ + UNIT_TEST_SUITE_END(); + +public: + void Push1M_Pop1M() { TQueueType queue; - TMsgLink msg = &queue; - - auto pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - - for (int i = 0; i < 1000000; ++i) { - queue.Push((char*)msg + i); - } - - for (int i = 0; i < 1000000; ++i) { - auto popped = queue.Pop(); - UNIT_ASSERT_EQUAL((char*)msg + i, popped); - } - - pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - } - - void Threads2_Push1M_Threads1_Pop2M() { + TMsgLink msg = &queue; + + auto pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + + for (int i = 0; i < 1000000; ++i) { + queue.Push((char*)msg + i); + } + + for (int i = 0; i < 1000000; ++i) { + auto popped = queue.Pop(); + UNIT_ASSERT_EQUAL((char*)msg + i, popped); + } + + pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + } + + void Threads2_Push1M_Threads1_Pop2M() { TQueueType queue; - + class TPusherThread: public ISimpleThread { - public: + public: TPusherThread(TQueueType& theQueue, char* start) : Queue(theQueue) - , Arg(start) - { - } - + , Arg(start) + { + } + TQueueType& Queue; - char* Arg; - - void* ThreadProc() override { - for (int i = 0; i < 1000000; ++i) { - Queue.Push(Arg + i); - } - return nullptr; - } - }; - - TPusherThread pusher1(queue, (char*)&queue); - TPusherThread pusher2(queue, (char*)&queue + 2000000); - - pusher1.Start(); - pusher2.Start(); - - for (int i = 0; i < 2000000; ++i) { - while (queue.Pop() == nullptr) { - SpinLockPause(); - } - } - - auto pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - } - - void Threads4_Push1M_Threads1_Pop4M() { + char* Arg; + + void* ThreadProc() override { + for (int i = 0; i < 1000000; ++i) { + Queue.Push(Arg + i); + } + return nullptr; + } + }; + + TPusherThread pusher1(queue, (char*)&queue); + TPusherThread pusher2(queue, (char*)&queue + 2000000); + + pusher1.Start(); + pusher2.Start(); + + for (int i = 0; i < 2000000; ++i) { + while (queue.Pop() == nullptr) { + SpinLockPause(); + } + } + + auto pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + } + + void Threads4_Push1M_Threads1_Pop4M() { TQueueType queue; - + class TPusherThread: public ISimpleThread { - public: + public: TPusherThread(TQueueType& theQueue, char* start) : Queue(theQueue) - , Arg(start) - { - } - + , Arg(start) + { + } + TQueueType& Queue; - char* Arg; - - void* ThreadProc() override { - for (int i = 0; i < 1000000; ++i) { - Queue.Push(Arg + i); - } - return nullptr; - } - }; - - TPusherThread pusher1(queue, (char*)&queue); - TPusherThread pusher2(queue, (char*)&queue + 2000000); - TPusherThread pusher3(queue, (char*)&queue + 4000000); - TPusherThread pusher4(queue, (char*)&queue + 6000000); - - pusher1.Start(); - pusher2.Start(); - pusher3.Start(); - pusher4.Start(); - - for (int i = 0; i < 4000000; ++i) { - while (queue.Pop() == nullptr) { - SpinLockPause(); - } - } - - auto pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - } - - template <size_t NUMBER_OF_PUSHERS, size_t NUMBER_OF_QUEUES> - void ManyRndPush100K_ManyQueues() { + char* Arg; + + void* ThreadProc() override { + for (int i = 0; i < 1000000; ++i) { + Queue.Push(Arg + i); + } + return nullptr; + } + }; + + TPusherThread pusher1(queue, (char*)&queue); + TPusherThread pusher2(queue, (char*)&queue + 2000000); + TPusherThread pusher3(queue, (char*)&queue + 4000000); + TPusherThread pusher4(queue, (char*)&queue + 6000000); + + pusher1.Start(); + pusher2.Start(); + pusher3.Start(); + pusher4.Start(); + + for (int i = 0; i < 4000000; ++i) { + while (queue.Pop() == nullptr) { + SpinLockPause(); + } + } + + auto pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + } + + template <size_t NUMBER_OF_PUSHERS, size_t NUMBER_OF_QUEUES> + void ManyRndPush100K_ManyQueues() { TQueueType queue[NUMBER_OF_QUEUES]; - + class TPusherThread: public ISimpleThread { - public: + public: TPusherThread(TQueueType* queues, char* start) - : Queues(queues) - , Arg(start) - { - } - + : Queues(queues) + , Arg(start) + { + } + TQueueType* Queues; - char* Arg; - - void* ThreadProc() override { - ui64 counters[NUMBER_OF_QUEUES]; - for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { - counters[i] = 0; - } - - for (int i = 0; i < 100000; ++i) { - size_t rnd = GetCycleCount() % NUMBER_OF_QUEUES; - int cookie = counters[rnd]++; - Queues[rnd].Push(Arg + cookie); - } - - for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { - Queues[i].Push((void*)2ULL); - } - - return nullptr; - } - }; - + char* Arg; + + void* ThreadProc() override { + ui64 counters[NUMBER_OF_QUEUES]; + for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { + counters[i] = 0; + } + + for (int i = 0; i < 100000; ++i) { + size_t rnd = GetCycleCount() % NUMBER_OF_QUEUES; + int cookie = counters[rnd]++; + Queues[rnd].Push(Arg + cookie); + } + + for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { + Queues[i].Push((void*)2ULL); + } + + return nullptr; + } + }; + class TPopperThread: public ISimpleThread { - public: + public: TPopperThread(TQueueType* theQueue, char* base) : Queue(theQueue) - , Base(base) - { - } - + , Base(base) + { + } + TQueueType* Queue; - char* Base; - - void* ThreadProc() override { - ui64 counters[NUMBER_OF_PUSHERS]; - for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { - counters[i] = 0; - } - - for (size_t fin = 0; fin < NUMBER_OF_PUSHERS;) { - auto msg = Queue->Pop(); - if (msg == nullptr) { - SpinLockPause(); - continue; - } - if (msg == (void*)2ULL) { - ++fin; - continue; - } - ui64 shift = (char*)msg - Base; - auto pusherNum = shift / 200000000ULL; - auto msgNum = shift % 200000000ULL; - - UNIT_ASSERT_EQUAL(counters[pusherNum], msgNum); - ++counters[pusherNum]; - } - - auto pmsg = Queue->Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - - return nullptr; - } - }; - + char* Base; + + void* ThreadProc() override { + ui64 counters[NUMBER_OF_PUSHERS]; + for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { + counters[i] = 0; + } + + for (size_t fin = 0; fin < NUMBER_OF_PUSHERS;) { + auto msg = Queue->Pop(); + if (msg == nullptr) { + SpinLockPause(); + continue; + } + if (msg == (void*)2ULL) { + ++fin; + continue; + } + ui64 shift = (char*)msg - Base; + auto pusherNum = shift / 200000000ULL; + auto msgNum = shift % 200000000ULL; + + UNIT_ASSERT_EQUAL(counters[pusherNum], msgNum); + ++counters[pusherNum]; + } + + auto pmsg = Queue->Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + + return nullptr; + } + }; + TVector<TAutoPtr<TPopperThread>> poppers; TVector<TAutoPtr<TPusherThread>> pushers; - - for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { - poppers.emplace_back(new TPopperThread(&queue[i], (char*)&queue)); - poppers.back()->Start(); - } - - for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { - pushers.emplace_back( - new TPusherThread(queue, (char*)&queue + 200000000ULL * i)); - pushers.back()->Start(); - } - - for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { - poppers[i]->Join(); - } - - for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { - pushers[i]->Join(); - } - } - - void Threads8_RndPush100K_Threads8_Queues() { - ManyRndPush100K_ManyQueues<8, 8>(); - } - - /* - void Threads24_RndPush100K_Threads24_Queues() { - ManyRndPush100K_ManyQueues<24, 24>(); - } - - void Threads24_RndPush100K_Threads8_Queues() { - ManyRndPush100K_ManyQueues<24, 8>(); - } - - void Threads24_RndPush100K_Threads4_Queues() { - ManyRndPush100K_ManyQueues<24, 4>(); - } - */ -}; - -REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TQueueTestProcs); + + for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { + poppers.emplace_back(new TPopperThread(&queue[i], (char*)&queue)); + poppers.back()->Start(); + } + + for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { + pushers.emplace_back( + new TPusherThread(queue, (char*)&queue + 200000000ULL * i)); + pushers.back()->Start(); + } + + for (size_t i = 0; i < NUMBER_OF_QUEUES; ++i) { + poppers[i]->Join(); + } + + for (size_t i = 0; i < NUMBER_OF_PUSHERS; ++i) { + pushers[i]->Join(); + } + } + + void Threads8_RndPush100K_Threads8_Queues() { + ManyRndPush100K_ManyQueues<8, 8>(); + } + + /* + void Threads24_RndPush100K_Threads24_Queues() { + ManyRndPush100K_ManyQueues<24, 24>(); + } + + void Threads24_RndPush100K_Threads8_Queues() { + ManyRndPush100K_ManyQueues<24, 8>(); + } + + void Threads24_RndPush100K_Threads4_Queues() { + ManyRndPush100K_ManyQueues<24, 4>(); + } + */ +}; + +REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TQueueTestProcs); diff --git a/library/cpp/threading/queue/tune.h b/library/cpp/threading/queue/tune.h index 50fc3dc17c..43ad5efe3e 100644 --- a/library/cpp/threading/queue/tune.h +++ b/library/cpp/threading/queue/tune.h @@ -1,101 +1,101 @@ -#pragma once - -/* - Motivation: consider you have a template class with many parameters - with default associations - - template <typename A = TDefA, - typename B = TDefB, - typename C = TDefC, - typename D = TDefD> - class TExample { - }; - - consider you would like to provide easy to use interface to tune all - these parameters in position independed manner, - In that case TTune would be helpful for you. - - How to use: - First step: declare a struct with all default associations - - struct TDefaultTune { - using TStructA = TDefA; - using TStructB = TDefB; - using TStructC = TDefC; - using TStructD = TDefD; - }; - - Second step: declare helper names visible to a user - - DeclareTuneTypeParam(TTuneParamA, TStructA); - DeclareTuneTypeParam(TTuneParamB, TStructB); - DeclareTuneTypeParam(TTuneParamC, TStructC); - DeclareTuneTypeParam(TTuneParamD, TStructD); - - Third step: declare TExample this way: - - template <typename...TParams> - class TExample { - using TMyParams = TTune<TDefaultTune, TParams...>; - - using TActualA = TMyParams::TStructA; - using TActualB = TMyParams::TStructB; - ... - }; - - TTune<TDefaultTune, TParams...> is a struct with the default parameteres - taken from TDefaultTune and overridden from "TParams...". - - for example: "TTune<TDefaultTune, TTuneParamC<TUserClass>>" - will be virtually the same as: - - struct TTunedClass { - using TStructA = TDefA; - using TStructB = TDefB; - using TStructC = TUserClass; - using TStructD = TDefD; - }; - - From now on you can tune your TExample in the following manner: - - using TCustomClass = - TExample <TTuneParamA<TUserStruct1>, TTuneParamD<TUserStruct2>>; - - You can also tweak constant expressions in your TDefaultTune. - Consider you have: - - struct TDefaultTune { - static constexpr ui32 MySize = 42; - }; - - declare an interface to modify the parameter this way: - - DeclareTuneValueParam(TStructSize, ui32, MySize); - - and tweak your class: - - using TTwiceBigger = TExample<TStructSize<84>>; - - */ - -#define DeclareTuneTypeParam(TParamName, InternalName) \ - template <typename TNewType> \ - struct TParamName { \ - template <typename TBase> \ - struct TApply: public TBase { \ - using InternalName = TNewType; \ - }; \ - } - -#define DeclareTuneValueParam(TParamName, TValueType, InternalName) \ - template <TValueType NewValue> \ - struct TParamName { \ - template <typename TBase> \ - struct TApply: public TBase { \ - static constexpr TValueType InternalName = NewValue; \ - }; \ - } - +#pragma once + +/* + Motivation: consider you have a template class with many parameters + with default associations + + template <typename A = TDefA, + typename B = TDefB, + typename C = TDefC, + typename D = TDefD> + class TExample { + }; + + consider you would like to provide easy to use interface to tune all + these parameters in position independed manner, + In that case TTune would be helpful for you. + + How to use: + First step: declare a struct with all default associations + + struct TDefaultTune { + using TStructA = TDefA; + using TStructB = TDefB; + using TStructC = TDefC; + using TStructD = TDefD; + }; + + Second step: declare helper names visible to a user + + DeclareTuneTypeParam(TTuneParamA, TStructA); + DeclareTuneTypeParam(TTuneParamB, TStructB); + DeclareTuneTypeParam(TTuneParamC, TStructC); + DeclareTuneTypeParam(TTuneParamD, TStructD); + + Third step: declare TExample this way: + + template <typename...TParams> + class TExample { + using TMyParams = TTune<TDefaultTune, TParams...>; + + using TActualA = TMyParams::TStructA; + using TActualB = TMyParams::TStructB; + ... + }; + + TTune<TDefaultTune, TParams...> is a struct with the default parameteres + taken from TDefaultTune and overridden from "TParams...". + + for example: "TTune<TDefaultTune, TTuneParamC<TUserClass>>" + will be virtually the same as: + + struct TTunedClass { + using TStructA = TDefA; + using TStructB = TDefB; + using TStructC = TUserClass; + using TStructD = TDefD; + }; + + From now on you can tune your TExample in the following manner: + + using TCustomClass = + TExample <TTuneParamA<TUserStruct1>, TTuneParamD<TUserStruct2>>; + + You can also tweak constant expressions in your TDefaultTune. + Consider you have: + + struct TDefaultTune { + static constexpr ui32 MySize = 42; + }; + + declare an interface to modify the parameter this way: + + DeclareTuneValueParam(TStructSize, ui32, MySize); + + and tweak your class: + + using TTwiceBigger = TExample<TStructSize<84>>; + + */ + +#define DeclareTuneTypeParam(TParamName, InternalName) \ + template <typename TNewType> \ + struct TParamName { \ + template <typename TBase> \ + struct TApply: public TBase { \ + using InternalName = TNewType; \ + }; \ + } + +#define DeclareTuneValueParam(TParamName, TValueType, InternalName) \ + template <TValueType NewValue> \ + struct TParamName { \ + template <typename TBase> \ + struct TApply: public TBase { \ + static constexpr TValueType InternalName = NewValue; \ + }; \ + } + #define DeclareTuneContainer(TParamName, InternalName) \ template <template <typename, typename...> class TNewContainer> \ struct TParamName { \ @@ -104,22 +104,22 @@ template <typename TElem, typename... TRest> \ using InternalName = TNewContainer<TElem, TRest...>; \ }; \ - } - -namespace NTunePrivate { - template <typename TBase, typename... TParams> - struct TFold; - - template <typename TBase> - struct TFold<TBase>: public TBase { - }; - - template <typename TBase, typename TFirstArg, typename... TRest> - struct TFold<TBase, TFirstArg, TRest...> - : public TFold<typename TFirstArg::template TApply<TBase>, TRest...> { - }; -} - -template <typename TDefault, typename... TParams> -struct TTune: public NTunePrivate::TFold<TDefault, TParams...> { -}; + } + +namespace NTunePrivate { + template <typename TBase, typename... TParams> + struct TFold; + + template <typename TBase> + struct TFold<TBase>: public TBase { + }; + + template <typename TBase, typename TFirstArg, typename... TRest> + struct TFold<TBase, TFirstArg, TRest...> + : public TFold<typename TFirstArg::template TApply<TBase>, TRest...> { + }; +} + +template <typename TDefault, typename... TParams> +struct TTune: public NTunePrivate::TFold<TDefault, TParams...> { +}; diff --git a/library/cpp/threading/queue/tune_ut.cpp b/library/cpp/threading/queue/tune_ut.cpp index 7e980d3e27..64bc8fd427 100644 --- a/library/cpp/threading/queue/tune_ut.cpp +++ b/library/cpp/threading/queue/tune_ut.cpp @@ -1,118 +1,118 @@ #include <library/cpp/testing/unittest/registar.h> -#include "tune.h" - -struct TDefaultStructA { -}; - -struct TDefaultStructB { -}; - -struct TDefaults { - using TStructA = TDefaultStructA; - using TStructB = TDefaultStructB; - static constexpr ui32 Param1 = 42; - static constexpr ui32 Param2 = 42; -}; - -DeclareTuneTypeParam(TweakStructA, TStructA); -DeclareTuneTypeParam(TweakStructB, TStructB); -DeclareTuneValueParam(TweakParam1, ui32, Param1); -DeclareTuneValueParam(TweakParam2, ui32, Param2); - +#include "tune.h" + +struct TDefaultStructA { +}; + +struct TDefaultStructB { +}; + +struct TDefaults { + using TStructA = TDefaultStructA; + using TStructB = TDefaultStructB; + static constexpr ui32 Param1 = 42; + static constexpr ui32 Param2 = 42; +}; + +DeclareTuneTypeParam(TweakStructA, TStructA); +DeclareTuneTypeParam(TweakStructB, TStructB); +DeclareTuneValueParam(TweakParam1, ui32, Param1); +DeclareTuneValueParam(TweakParam2, ui32, Param2); + Y_UNIT_TEST_SUITE(TestTuning) { Y_UNIT_TEST(Defaults) { - using TTuned = TTune<TDefaults>; - using TunedA = TTuned::TStructA; - using TunedB = TTuned::TStructB; - auto sameA = std::is_same<TDefaultStructA, TunedA>::value; - auto sameB = std::is_same<TDefaultStructB, TunedB>::value; - auto param1 = TTuned::Param1; - auto param2 = TTuned::Param2; - - UNIT_ASSERT(sameA); - UNIT_ASSERT(sameB); - UNIT_ASSERT_EQUAL(param1, 42); - UNIT_ASSERT_EQUAL(param2, 42); - } - + using TTuned = TTune<TDefaults>; + using TunedA = TTuned::TStructA; + using TunedB = TTuned::TStructB; + auto sameA = std::is_same<TDefaultStructA, TunedA>::value; + auto sameB = std::is_same<TDefaultStructB, TunedB>::value; + auto param1 = TTuned::Param1; + auto param2 = TTuned::Param2; + + UNIT_ASSERT(sameA); + UNIT_ASSERT(sameB); + UNIT_ASSERT_EQUAL(param1, 42); + UNIT_ASSERT_EQUAL(param2, 42); + } + Y_UNIT_TEST(TuneStructA) { - struct TMyStruct { - }; - - using TTuned = TTune<TDefaults, TweakStructA<TMyStruct>>; - - using TunedA = TTuned::TStructA; - using TunedB = TTuned::TStructB; - //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; - auto sameB = std::is_same<TDefaultStructB, TunedB>::value; - auto param1 = TTuned::Param1; - auto param2 = TTuned::Param2; - - auto sameA = std::is_same<TMyStruct, TunedA>::value; - - UNIT_ASSERT(sameA); - UNIT_ASSERT(sameB); - UNIT_ASSERT_EQUAL(param1, 42); - UNIT_ASSERT_EQUAL(param2, 42); - } - + struct TMyStruct { + }; + + using TTuned = TTune<TDefaults, TweakStructA<TMyStruct>>; + + using TunedA = TTuned::TStructA; + using TunedB = TTuned::TStructB; + //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; + auto sameB = std::is_same<TDefaultStructB, TunedB>::value; + auto param1 = TTuned::Param1; + auto param2 = TTuned::Param2; + + auto sameA = std::is_same<TMyStruct, TunedA>::value; + + UNIT_ASSERT(sameA); + UNIT_ASSERT(sameB); + UNIT_ASSERT_EQUAL(param1, 42); + UNIT_ASSERT_EQUAL(param2, 42); + } + Y_UNIT_TEST(TuneParam1) { - using TTuned = TTune<TDefaults, TweakParam1<24>>; - - using TunedA = TTuned::TStructA; - using TunedB = TTuned::TStructB; - auto sameA = std::is_same<TDefaultStructA, TunedA>::value; - auto sameB = std::is_same<TDefaultStructB, TunedB>::value; - auto param1 = TTuned::Param1; - auto param2 = TTuned::Param2; - - UNIT_ASSERT(sameA); - UNIT_ASSERT(sameB); - UNIT_ASSERT_EQUAL(param1, 24); - UNIT_ASSERT_EQUAL(param2, 42); - } - + using TTuned = TTune<TDefaults, TweakParam1<24>>; + + using TunedA = TTuned::TStructA; + using TunedB = TTuned::TStructB; + auto sameA = std::is_same<TDefaultStructA, TunedA>::value; + auto sameB = std::is_same<TDefaultStructB, TunedB>::value; + auto param1 = TTuned::Param1; + auto param2 = TTuned::Param2; + + UNIT_ASSERT(sameA); + UNIT_ASSERT(sameB); + UNIT_ASSERT_EQUAL(param1, 24); + UNIT_ASSERT_EQUAL(param2, 42); + } + Y_UNIT_TEST(TuneStructAAndParam1) { - struct TMyStruct { - }; - - using TTuned = - TTune<TDefaults, TweakStructA<TMyStruct>, TweakParam1<24>>; - - using TunedA = TTuned::TStructA; - using TunedB = TTuned::TStructB; - //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; - auto sameB = std::is_same<TDefaultStructB, TunedB>::value; - auto param1 = TTuned::Param1; - auto param2 = TTuned::Param2; - - auto sameA = std::is_same<TMyStruct, TunedA>::value; - - UNIT_ASSERT(sameA); - UNIT_ASSERT(sameB); - UNIT_ASSERT_EQUAL(param1, 24); - UNIT_ASSERT_EQUAL(param2, 42); - } - + struct TMyStruct { + }; + + using TTuned = + TTune<TDefaults, TweakStructA<TMyStruct>, TweakParam1<24>>; + + using TunedA = TTuned::TStructA; + using TunedB = TTuned::TStructB; + //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; + auto sameB = std::is_same<TDefaultStructB, TunedB>::value; + auto param1 = TTuned::Param1; + auto param2 = TTuned::Param2; + + auto sameA = std::is_same<TMyStruct, TunedA>::value; + + UNIT_ASSERT(sameA); + UNIT_ASSERT(sameB); + UNIT_ASSERT_EQUAL(param1, 24); + UNIT_ASSERT_EQUAL(param2, 42); + } + Y_UNIT_TEST(TuneParam1AndStructA) { - struct TMyStruct { - }; - - using TTuned = - TTune<TDefaults, TweakParam1<24>, TweakStructA<TMyStruct>>; - - using TunedA = TTuned::TStructA; - using TunedB = TTuned::TStructB; - //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; - auto sameB = std::is_same<TDefaultStructB, TunedB>::value; - auto param1 = TTuned::Param1; - auto param2 = TTuned::Param2; - - auto sameA = std::is_same<TMyStruct, TunedA>::value; - - UNIT_ASSERT(sameA); - UNIT_ASSERT(sameB); - UNIT_ASSERT_EQUAL(param1, 24); - UNIT_ASSERT_EQUAL(param2, 42); - } -} + struct TMyStruct { + }; + + using TTuned = + TTune<TDefaults, TweakParam1<24>, TweakStructA<TMyStruct>>; + + using TunedA = TTuned::TStructA; + using TunedB = TTuned::TStructB; + //auto sameA = std::is_same<TDefaultStructA, TunedA>::value; + auto sameB = std::is_same<TDefaultStructB, TunedB>::value; + auto param1 = TTuned::Param1; + auto param2 = TTuned::Param2; + + auto sameA = std::is_same<TMyStruct, TunedA>::value; + + UNIT_ASSERT(sameA); + UNIT_ASSERT(sameB); + UNIT_ASSERT_EQUAL(param1, 24); + UNIT_ASSERT_EQUAL(param2, 42); + } +} diff --git a/library/cpp/threading/queue/unordered_ut.cpp b/library/cpp/threading/queue/unordered_ut.cpp index a43b7f520e..2018538bf7 100644 --- a/library/cpp/threading/queue/unordered_ut.cpp +++ b/library/cpp/threading/queue/unordered_ut.cpp @@ -1,154 +1,154 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/system/thread.h> -#include <algorithm> -#include <util/generic/vector.h> -#include <util/random/fast.h> - -#include "ut_helpers.h" - +#include <util/system/thread.h> +#include <algorithm> +#include <util/generic/vector.h> +#include <util/random/fast.h> + +#include "ut_helpers.h" + template <typename TQueueType> -class TTestUnorderedQueue: public TTestBase { -private: - using TLink = TIntrusiveLink; - +class TTestUnorderedQueue: public TTestBase { +private: + using TLink = TIntrusiveLink; + UNIT_TEST_SUITE_DEMANGLE(TTestUnorderedQueue<TQueueType>); - UNIT_TEST(Push1M_Pop1M_Unordered) - UNIT_TEST_SUITE_END(); - -public: - void Push1M_Pop1M_Unordered() { - constexpr int REPEAT = 1000000; + UNIT_TEST(Push1M_Pop1M_Unordered) + UNIT_TEST_SUITE_END(); + +public: + void Push1M_Pop1M_Unordered() { + constexpr int REPEAT = 1000000; TQueueType queue; - TLink msg[REPEAT]; - - auto pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - - for (int i = 0; i < REPEAT; ++i) { - queue.Push(&msg[i]); - } - + TLink msg[REPEAT]; + + auto pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + + for (int i = 0; i < REPEAT; ++i) { + queue.Push(&msg[i]); + } + TVector<TLink*> popped; - popped.reserve(REPEAT); - for (int i = 0; i < REPEAT; ++i) { - popped.push_back((TLink*)queue.Pop()); - } - - pmsg = queue.Pop(); - UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); - - std::sort(popped.begin(), popped.end()); - for (int i = 0; i < REPEAT; ++i) { - UNIT_ASSERT_VALUES_EQUAL(&msg[i], popped[i]); - } - } -}; - + popped.reserve(REPEAT); + for (int i = 0; i < REPEAT; ++i) { + popped.push_back((TLink*)queue.Pop()); + } + + pmsg = queue.Pop(); + UNIT_ASSERT_VALUES_EQUAL(pmsg, nullptr); + + std::sort(popped.begin(), popped.end()); + for (int i = 0; i < REPEAT; ++i) { + UNIT_ASSERT_VALUES_EQUAL(&msg[i], popped[i]); + } + } +}; + template <typename TQueueType> -class TTestWeakQueue: public TTestBase { -private: +class TTestWeakQueue: public TTestBase { +private: UNIT_TEST_SUITE_DEMANGLE(TTestWeakQueue<TQueueType>); - UNIT_TEST(Threads8_Rnd_Exchange) - UNIT_TEST_SUITE_END(); - -public: - template <ui16 COUNT = 48, ui32 MSG_COUNT = 10000> - void ManyThreadsRndExchange() { + UNIT_TEST(Threads8_Rnd_Exchange) + UNIT_TEST_SUITE_END(); + +public: + template <ui16 COUNT = 48, ui32 MSG_COUNT = 10000> + void ManyThreadsRndExchange() { TQueueType queues[COUNT]; - + class TWorker: public ISimpleThread { - public: - TWorker( + public: + TWorker( TQueueType* queues_, ui16 mine, TAtomic* pushDone) - : Queues(queues_) - , MineQueue(mine) - , PushDone(pushDone) - { - } - + : Queues(queues_) + , MineQueue(mine) + , PushDone(pushDone) + { + } + TQueueType* Queues; - ui16 MineQueue; + ui16 MineQueue; TVector<uintptr_t> Received; - TAtomic* PushDone; - - void* ThreadProc() override { - TReallyFastRng32 rng(GetCycleCount()); - Received.reserve(MSG_COUNT * 2); - - for (ui32 loop = 1; loop <= MSG_COUNT; ++loop) { - for (;;) { - auto msg = Queues[MineQueue].Pop(); - if (msg == nullptr) { - break; - } - - Received.push_back((uintptr_t)msg); - } - - ui16 rnd = rng.GenRand64() % COUNT; - ui64 msg = ((ui64)MineQueue << 32) + loop; - while (!Queues[rnd].Push((void*)msg)) { - } - } - - AtomicIncrement(*PushDone); - - for (;;) { - bool isItLast = AtomicGet(*PushDone) == COUNT; - auto msg = Queues[MineQueue].Pop(); - if (msg != nullptr) { - Received.push_back((uintptr_t)msg); - } else { - if (isItLast) { - break; - } - SpinLockPause(); - } - } - - for (ui64 last = 0;;) { - auto msg = Queues[MineQueue].UnsafeScanningPop(&last); - if (msg == nullptr) { - break; - } - Received.push_back((uintptr_t)msg); - } - - return nullptr; - } - }; - + TAtomic* PushDone; + + void* ThreadProc() override { + TReallyFastRng32 rng(GetCycleCount()); + Received.reserve(MSG_COUNT * 2); + + for (ui32 loop = 1; loop <= MSG_COUNT; ++loop) { + for (;;) { + auto msg = Queues[MineQueue].Pop(); + if (msg == nullptr) { + break; + } + + Received.push_back((uintptr_t)msg); + } + + ui16 rnd = rng.GenRand64() % COUNT; + ui64 msg = ((ui64)MineQueue << 32) + loop; + while (!Queues[rnd].Push((void*)msg)) { + } + } + + AtomicIncrement(*PushDone); + + for (;;) { + bool isItLast = AtomicGet(*PushDone) == COUNT; + auto msg = Queues[MineQueue].Pop(); + if (msg != nullptr) { + Received.push_back((uintptr_t)msg); + } else { + if (isItLast) { + break; + } + SpinLockPause(); + } + } + + for (ui64 last = 0;;) { + auto msg = Queues[MineQueue].UnsafeScanningPop(&last); + if (msg == nullptr) { + break; + } + Received.push_back((uintptr_t)msg); + } + + return nullptr; + } + }; + TVector<TAutoPtr<TWorker>> workers; - TAtomic pushDone = 0; - - for (ui32 i = 0; i < COUNT; ++i) { - workers.emplace_back(new TWorker(&queues[0], i, &pushDone)); - workers.back()->Start(); - } - + TAtomic pushDone = 0; + + for (ui32 i = 0; i < COUNT; ++i) { + workers.emplace_back(new TWorker(&queues[0], i, &pushDone)); + workers.back()->Start(); + } + TVector<uintptr_t> all; - for (ui32 i = 0; i < COUNT; ++i) { - workers[i]->Join(); - all.insert(all.begin(), + for (ui32 i = 0; i < COUNT; ++i) { + workers[i]->Join(); + all.insert(all.begin(), workers[i]->Received.begin(), workers[i]->Received.end()); - } - - std::sort(all.begin(), all.end()); - auto iter = all.begin(); - for (ui32 i = 0; i < COUNT; ++i) { - for (ui32 k = 1; k <= MSG_COUNT; ++k) { - UNIT_ASSERT_VALUES_EQUAL(((ui64)i << 32) + k, *iter); - ++iter; - } - } - } - - void Threads8_Rnd_Exchange() { - ManyThreadsRndExchange<8>(); - } -}; - -REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TTestUnorderedQueue); -UNIT_TEST_SUITE_REGISTRATION(TTestWeakQueue<TMPMCUnorderedRing>); + } + + std::sort(all.begin(), all.end()); + auto iter = all.begin(); + for (ui32 i = 0; i < COUNT; ++i) { + for (ui32 k = 1; k <= MSG_COUNT; ++k) { + UNIT_ASSERT_VALUES_EQUAL(((ui64)i << 32) + k, *iter); + ++iter; + } + } + } + + void Threads8_Rnd_Exchange() { + ManyThreadsRndExchange<8>(); + } +}; + +REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TTestUnorderedQueue); +UNIT_TEST_SUITE_REGISTRATION(TTestWeakQueue<TMPMCUnorderedRing>); diff --git a/library/cpp/threading/queue/ut/ya.make b/library/cpp/threading/queue/ut/ya.make index 8883d9bf69..dda204155e 100644 --- a/library/cpp/threading/queue/ut/ya.make +++ b/library/cpp/threading/queue/ut/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(library/cpp/threading/queue) - + OWNER(agri) - -ALLOCATOR(B) - -SRCS( - basic_ut.cpp - queue_ut.cpp - tune_ut.cpp - unordered_ut.cpp - ut_helpers.cpp - ut_helpers.h -) - -END() + +ALLOCATOR(B) + +SRCS( + basic_ut.cpp + queue_ut.cpp + tune_ut.cpp + unordered_ut.cpp + ut_helpers.cpp + ut_helpers.h +) + +END() diff --git a/library/cpp/threading/queue/ut_helpers.cpp b/library/cpp/threading/queue/ut_helpers.cpp index aa3a831441..342aa125a0 100644 --- a/library/cpp/threading/queue/ut_helpers.cpp +++ b/library/cpp/threading/queue/ut_helpers.cpp @@ -1 +1 @@ -#include "ut_helpers.h" +#include "ut_helpers.h" diff --git a/library/cpp/threading/queue/ut_helpers.h b/library/cpp/threading/queue/ut_helpers.h index 2756b52601..c720366593 100644 --- a/library/cpp/threading/queue/ut_helpers.h +++ b/library/cpp/threading/queue/ut_helpers.h @@ -1,40 +1,40 @@ -#pragma once - -#include "mpsc_read_as_filled.h" -#include "mpsc_htswap.h" -#include "mpsc_vinfarr_obstructive.h" -#include "mpsc_intrusive_unordered.h" -#include "mpmc_unordered_ring.h" - -struct TBasicHTSwap: public NThreading::THTSwapQueue<> { -}; - -struct TBasicReadAsFilled: public NThreading::TReadAsFilledQueue<> { -}; - -struct TBasicObstructiveConsumer +#pragma once + +#include "mpsc_read_as_filled.h" +#include "mpsc_htswap.h" +#include "mpsc_vinfarr_obstructive.h" +#include "mpsc_intrusive_unordered.h" +#include "mpmc_unordered_ring.h" + +struct TBasicHTSwap: public NThreading::THTSwapQueue<> { +}; + +struct TBasicReadAsFilled: public NThreading::TReadAsFilledQueue<> { +}; + +struct TBasicObstructiveConsumer : public NThreading::TObstructiveConsumerQueue<> { -}; - -struct TBasicMPSCIntrusiveUnordered +}; + +struct TBasicMPSCIntrusiveUnordered : public NThreading::TMPSCIntrusiveUnordered { -}; - -struct TIntrusiveLink: public NThreading::TIntrusiveNode { -}; - -struct TMPMCUnorderedRing: public NThreading::TMPMCUnorderedRing { - TMPMCUnorderedRing() - : NThreading::TMPMCUnorderedRing(10000000) - { - } -}; - +}; + +struct TIntrusiveLink: public NThreading::TIntrusiveNode { +}; + +struct TMPMCUnorderedRing: public NThreading::TMPMCUnorderedRing { + TMPMCUnorderedRing() + : NThreading::TMPMCUnorderedRing(10000000) + { + } +}; + #define REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TestTemplate) \ UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicHTSwap>); \ UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicReadAsFilled>); \ - UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicObstructiveConsumer>) - + UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicObstructiveConsumer>) + #define REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TestTemplate) \ - UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicMPSCIntrusiveUnordered>); \ - UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TMPMCUnorderedRing>); + UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicMPSCIntrusiveUnordered>); \ + UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TMPMCUnorderedRing>); diff --git a/library/cpp/threading/queue/ya.make b/library/cpp/threading/queue/ya.make index 6570b38ce5..3a11eb2d92 100644 --- a/library/cpp/threading/queue/ya.make +++ b/library/cpp/threading/queue/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER(agri) - -SRCS( - mpmc_unordered_ring.cpp - mpmc_unordered_ring.h - mpsc_htswap.cpp - mpsc_htswap.h - mpsc_intrusive_unordered.cpp - mpsc_intrusive_unordered.h - mpsc_read_as_filled.cpp - mpsc_read_as_filled.h - mpsc_vinfarr_obstructive.cpp - mpsc_vinfarr_obstructive.h -) - -END() +LIBRARY() + +OWNER(agri) + +SRCS( + mpmc_unordered_ring.cpp + mpmc_unordered_ring.h + mpsc_htswap.cpp + mpsc_htswap.h + mpsc_intrusive_unordered.cpp + mpsc_intrusive_unordered.h + mpsc_read_as_filled.cpp + mpsc_read_as_filled.h + mpsc_vinfarr_obstructive.cpp + mpsc_vinfarr_obstructive.h +) + +END() diff --git a/util/generic/ptr.h b/util/generic/ptr.h index 19db0e3ec5..fd1cd162b1 100644 --- a/util/generic/ptr.h +++ b/util/generic/ptr.h @@ -366,19 +366,19 @@ public: inline ~TRefCounted() = default; inline void Ref(TAtomicBase d) noexcept { - auto resultCount = Counter_.Add(d); - Y_ASSERT(resultCount >= d); - (void)resultCount; + auto resultCount = Counter_.Add(d); + Y_ASSERT(resultCount >= d); + (void)resultCount; } inline void Ref() noexcept { - auto resultCount = Counter_.Inc(); - Y_ASSERT(resultCount != 0); - (void)resultCount; + auto resultCount = Counter_.Inc(); + Y_ASSERT(resultCount != 0); + (void)resultCount; } inline void UnRef(TAtomicBase d) noexcept { - auto resultCount = Counter_.Sub(d); + auto resultCount = Counter_.Sub(d); Y_ASSERT(resultCount >= 0); if (resultCount == 0) { D::Destroy(static_cast<T*>(this)); @@ -389,14 +389,14 @@ public: UnRef(1); } - inline TAtomicBase RefCount() const noexcept { + inline TAtomicBase RefCount() const noexcept { return Counter_.Val(); } inline void DecRef() noexcept { - auto resultCount = Counter_.Dec(); - Y_ASSERT(resultCount >= 0); - (void)resultCount; + auto resultCount = Counter_.Dec(); + Y_ASSERT(resultCount >= 0); + (void)resultCount; } TRefCounted(const TRefCounted&) @@ -543,11 +543,11 @@ public: return *this; } - // Effectively replace both: - // Reset(const TIntrusivePtr&) - // Reset(TIntrusivePtr&&) - inline void Reset(TIntrusivePtr t) noexcept { - Swap(t); + // Effectively replace both: + // Reset(const TIntrusivePtr&) + // Reset(TIntrusivePtr&&) + inline void Reset(TIntrusivePtr t) noexcept { + Swap(t); } inline void Reset() noexcept { @@ -663,11 +663,11 @@ public: return *this; } - // Effectively replace both: - // Reset(const TIntrusiveConstPtr&) - // Reset(TIntrusiveConstPtr&&) - inline void Reset(TIntrusiveConstPtr t) noexcept { - Swap(t); + // Effectively replace both: + // Reset(const TIntrusiveConstPtr&) + // Reset(TIntrusiveConstPtr&&) + inline void Reset(TIntrusiveConstPtr t) noexcept { + Swap(t); } inline void Reset() noexcept { @@ -783,11 +783,11 @@ template <typename T, class Ops = TDefaultIntrusivePtrOps<T>, typename... Args> return new T{std::forward<Args>(args)...}; } -template <typename T, class Ops = TDefaultIntrusivePtrOps<T>, typename... Args> +template <typename T, class Ops = TDefaultIntrusivePtrOps<T>, typename... Args> [[nodiscard]] TIntrusiveConstPtr<T, Ops> MakeIntrusiveConst(Args&&... args) { - return new T{std::forward<Args>(args)...}; -} - + return new T{std::forward<Args>(args)...}; +} + template <class T, class C, class D> class TSharedPtr: public TPointerBase<TSharedPtr<T, C, D>, T> { template <class TT, class CC, class DD> @@ -862,11 +862,11 @@ public: return *this; } - // Effectively replace both: - // Reset(const TSharedPtr& t) - // Reset(TSharedPtr&& t) - inline void Reset(TSharedPtr t) noexcept { - Swap(t); + // Effectively replace both: + // Reset(const TSharedPtr& t) + // Reset(TSharedPtr&& t) + inline void Reset(TSharedPtr t) noexcept { + Swap(t); } inline void Reset() noexcept { diff --git a/util/generic/refcount.h b/util/generic/refcount.h index 966e853b77..bcee5a7860 100644 --- a/util/generic/refcount.h +++ b/util/generic/refcount.h @@ -19,21 +19,21 @@ public: Check(); } - inline TAtomicBase Add(TAtomicBase d) noexcept { + inline TAtomicBase Add(TAtomicBase d) noexcept { Check(); - return Counter_ += d; + return Counter_ += d; } - inline TAtomicBase Inc() noexcept { - return Add(1); + inline TAtomicBase Inc() noexcept { + return Add(1); } - inline TAtomicBase Sub(TAtomicBase d) noexcept { + inline TAtomicBase Sub(TAtomicBase d) noexcept { Check(); return Counter_ -= d; } - inline TAtomicBase Dec() noexcept { + inline TAtomicBase Dec() noexcept { return Sub(1); } @@ -43,17 +43,17 @@ public: } Inc(); - Y_ASSERT(Counter_ != 0); - + Y_ASSERT(Counter_ != 0); + return true; } - inline TAtomicBase Val() const noexcept { + inline TAtomicBase Val() const noexcept { return Counter_; } private: - TAtomicBase Counter_; + TAtomicBase Counter_; }; class TNoCheckPolicy { @@ -109,12 +109,12 @@ public: inline ~TAtomicCounter() = default; - inline TAtomicBase Add(TAtomicBase d) noexcept { - return AtomicAdd(Counter_, d); + inline TAtomicBase Add(TAtomicBase d) noexcept { + return AtomicAdd(Counter_, d); } - inline TAtomicBase Inc() noexcept { - return Add(1); + inline TAtomicBase Inc() noexcept { + return Add(1); } inline TAtomicBase Sub(TAtomicBase d) noexcept { @@ -138,7 +138,7 @@ public: } intptr_t newValue = curValue + 1; - Y_ASSERT(newValue != 0); + Y_ASSERT(newValue != 0); if (AtomicCas(&Counter_, newValue, curValue)) { return true; diff --git a/util/stream/input.cpp b/util/stream/input.cpp index 6e8170f2f9..494ece8c25 100644 --- a/util/stream/input.cpp +++ b/util/stream/input.cpp @@ -128,21 +128,21 @@ size_t IInputStream::DoSkip(size_t sz) { return Load(alloca(sz), sz); } - TTempBuf buf; - size_t total = 0; + TTempBuf buf; + size_t total = 0; while (sz) { const size_t lresult = Read(buf.Data(), Min<size_t>(sz, buf.Size())); if (lresult == 0) { - return total; + return total; } - total += lresult; - sz -= lresult; + total += lresult; + sz -= lresult; } - - return total; + + return total; } TString IInputStream::ReadAll() { diff --git a/util/system/align.h b/util/system/align.h index ea0bbc5b46..e2e2dbef79 100644 --- a/util/system/align.h +++ b/util/system/align.h @@ -2,19 +2,19 @@ #include "yassert.h" #include "defaults.h" -#include <util/generic/bitops.h> +#include <util/generic/bitops.h> template <class T> static inline T AlignDown(T len, T align) noexcept { - Y_ASSERT(IsPowerOf2(align)); // align should be power of 2 + Y_ASSERT(IsPowerOf2(align)); // align should be power of 2 return len & ~(align - 1); } template <class T> static inline T AlignUp(T len, T align) noexcept { - const T alignedResult = AlignDown(len + (align - 1), align); - Y_ASSERT(alignedResult >= len); // check for overflow - return alignedResult; + const T alignedResult = AlignDown(len + (align - 1), align); + Y_ASSERT(alignedResult >= len); // check for overflow + return alignedResult; } template <class T> @@ -25,12 +25,12 @@ static inline T AlignUpSpace(T len, T align) noexcept { template <class T> static inline T* AlignUp(T* ptr, size_t align) noexcept { - return (T*)AlignUp((uintptr_t)ptr, align); + return (T*)AlignUp((uintptr_t)ptr, align); } template <class T> static inline T* AlignDown(T* ptr, size_t align) noexcept { - return (T*)AlignDown((uintptr_t)ptr, align); + return (T*)AlignDown((uintptr_t)ptr, align); } template <class T> diff --git a/util/system/atomic.h b/util/system/atomic.h index 80265babfd..680754d6b3 100644 --- a/util/system/atomic.h +++ b/util/system/atomic.h @@ -21,10 +21,10 @@ static inline TAtomicBase AtomicSub(TAtomic& a, TAtomicBase v) { return AtomicAdd(a, -v); } -static inline TAtomicBase AtomicGetAndSub(TAtomic& a, TAtomicBase v) { - return AtomicGetAndAdd(a, -v); -} - +static inline TAtomicBase AtomicGetAndSub(TAtomic& a, TAtomicBase v) { + return AtomicGetAndAdd(a, -v); +} + #if defined(USE_GENERIC_SETGET) static inline TAtomicBase AtomicGet(const TAtomic& a) { return a; diff --git a/util/system/atomic_gcc.h b/util/system/atomic_gcc.h index ed8dc2bdc5..88c1943575 100644 --- a/util/system/atomic_gcc.h +++ b/util/system/atomic_gcc.h @@ -35,26 +35,26 @@ static inline intptr_t AtomicIncrement(TAtomic& p) { return __atomic_add_fetch(&p, 1, __ATOMIC_SEQ_CST); } -static inline intptr_t AtomicGetAndIncrement(TAtomic& p) { - return __atomic_fetch_add(&p, 1, __ATOMIC_SEQ_CST); -} - +static inline intptr_t AtomicGetAndIncrement(TAtomic& p) { + return __atomic_fetch_add(&p, 1, __ATOMIC_SEQ_CST); +} + static inline intptr_t AtomicDecrement(TAtomic& p) { return __atomic_sub_fetch(&p, 1, __ATOMIC_SEQ_CST); } -static inline intptr_t AtomicGetAndDecrement(TAtomic& p) { - return __atomic_fetch_sub(&p, 1, __ATOMIC_SEQ_CST); -} - +static inline intptr_t AtomicGetAndDecrement(TAtomic& p) { + return __atomic_fetch_sub(&p, 1, __ATOMIC_SEQ_CST); +} + static inline intptr_t AtomicAdd(TAtomic& p, intptr_t v) { return __atomic_add_fetch(&p, v, __ATOMIC_SEQ_CST); } -static inline intptr_t AtomicGetAndAdd(TAtomic& p, intptr_t v) { - return __atomic_fetch_add(&p, v, __ATOMIC_SEQ_CST); -} - +static inline intptr_t AtomicGetAndAdd(TAtomic& p, intptr_t v) { + return __atomic_fetch_add(&p, v, __ATOMIC_SEQ_CST); +} + static inline intptr_t AtomicSwap(TAtomic* p, intptr_t v) { (void)p; // disable strange 'parameter set but not used' warning on gcc intptr_t ret; @@ -67,12 +67,12 @@ static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) { return __atomic_compare_exchange(a, &compare, &exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } -static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) { - (void)a; // disable strange 'parameter set but not used' warning on gcc - __atomic_compare_exchange(a, &compare, &exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); - return compare; -} - +static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) { + (void)a; // disable strange 'parameter set but not used' warning on gcc + __atomic_compare_exchange(a, &compare, &exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + return compare; +} + static inline intptr_t AtomicOr(TAtomic& a, intptr_t b) { return __atomic_or_fetch(&a, b, __ATOMIC_SEQ_CST); } diff --git a/util/system/atomic_ops.h b/util/system/atomic_ops.h index 76766b4a0a..532e5b5bcb 100644 --- a/util/system/atomic_ops.h +++ b/util/system/atomic_ops.h @@ -40,41 +40,41 @@ inline TEnableIfCastable<T, T> AtomicIncrement(T volatile& target) { } template <typename T> -inline TEnableIfCastable<T, T> AtomicGetAndIncrement(T volatile& target) { - return static_cast<T>(AtomicGetAndIncrement(*AsAtomicPtr(&target))); -} - -template <typename T> +inline TEnableIfCastable<T, T> AtomicGetAndIncrement(T volatile& target) { + return static_cast<T>(AtomicGetAndIncrement(*AsAtomicPtr(&target))); +} + +template <typename T> inline TEnableIfCastable<T, T> AtomicDecrement(T volatile& target) { return static_cast<T>(AtomicDecrement(*AsAtomicPtr(&target))); } template <typename T> -inline TEnableIfCastable<T, T> AtomicGetAndDecrement(T volatile& target) { - return static_cast<T>(AtomicGetAndDecrement(*AsAtomicPtr(&target))); -} - -template <typename T> +inline TEnableIfCastable<T, T> AtomicGetAndDecrement(T volatile& target) { + return static_cast<T>(AtomicGetAndDecrement(*AsAtomicPtr(&target))); +} + +template <typename T> inline TEnableIfCastable<T, T> AtomicAdd(T volatile& target, TAtomicBase value) { return static_cast<T>(AtomicAdd(*AsAtomicPtr(&target), value)); } template <typename T> -inline TEnableIfCastable<T, T> AtomicGetAndAdd(T volatile& target, TAtomicBase value) { - return static_cast<T>(AtomicGetAndAdd(*AsAtomicPtr(&target), value)); -} - -template <typename T> +inline TEnableIfCastable<T, T> AtomicGetAndAdd(T volatile& target, TAtomicBase value) { + return static_cast<T>(AtomicGetAndAdd(*AsAtomicPtr(&target), value)); +} + +template <typename T> inline TEnableIfCastable<T, T> AtomicSub(T volatile& target, TAtomicBase value) { return static_cast<T>(AtomicSub(*AsAtomicPtr(&target), value)); } template <typename T> -inline TEnableIfCastable<T, T> AtomicGetAndSub(T volatile& target, TAtomicBase value) { - return static_cast<T>(AtomicGetAndSub(*AsAtomicPtr(&target), value)); -} - -template <typename T> +inline TEnableIfCastable<T, T> AtomicGetAndSub(T volatile& target, TAtomicBase value) { + return static_cast<T>(AtomicGetAndSub(*AsAtomicPtr(&target), value)); +} + +template <typename T> inline TEnableIfCastable<T, T> AtomicSwap(T volatile* target, TAtomicBase exchange) { return static_cast<T>(AtomicSwap(AsAtomicPtr(target), exchange)); } @@ -85,11 +85,11 @@ inline TEnableIfCastable<T, bool> AtomicCas(T volatile* target, TAtomicBase exch } template <typename T> -inline TEnableIfCastable<T, T> AtomicGetAndCas(T volatile* target, TAtomicBase exchange, TAtomicBase compare) { - return static_cast<T>(AtomicGetAndCas(AsAtomicPtr(target), exchange, compare)); -} - -template <typename T> +inline TEnableIfCastable<T, T> AtomicGetAndCas(T volatile* target, TAtomicBase exchange, TAtomicBase compare) { + return static_cast<T>(AtomicGetAndCas(AsAtomicPtr(target), exchange, compare)); +} + +template <typename T> inline TEnableIfCastable<T, bool> AtomicTryLock(T volatile* target) { return AtomicTryLock(AsAtomicPtr(target)); } @@ -154,36 +154,36 @@ inline bool AtomicCas(T* volatile* target, T* exchange, T* compare) { } template <typename T> -inline T* AtomicGetAndCas(T* volatile* target, T* exchange, T* compare) { - return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), reinterpret_cast<TAtomicBase>(compare))); -} - -template <typename T> +inline T* AtomicGetAndCas(T* volatile* target, T* exchange, T* compare) { + return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), reinterpret_cast<TAtomicBase>(compare))); +} + +template <typename T> inline bool AtomicCas(T* volatile* target, T* exchange, TNullPtr) { return AtomicCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), 0); } template <typename T> -inline T* AtomicGetAndCas(T* volatile* target, T* exchange, TNullPtr) { - return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), 0)); -} - -template <typename T> +inline T* AtomicGetAndCas(T* volatile* target, T* exchange, TNullPtr) { + return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), 0)); +} + +template <typename T> inline bool AtomicCas(T* volatile* target, TNullPtr, T* compare) { return AtomicCas(AsAtomicPtr(target), 0, reinterpret_cast<TAtomicBase>(compare)); } template <typename T> -inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, T* compare) { - return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, reinterpret_cast<TAtomicBase>(compare))); -} - -template <typename T> +inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, T* compare) { + return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, reinterpret_cast<TAtomicBase>(compare))); +} + +template <typename T> inline bool AtomicCas(T* volatile* target, TNullPtr, TNullPtr) { return AtomicCas(AsAtomicPtr(target), 0, 0); } - -template <typename T> -inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, TNullPtr) { - return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, 0)); -} + +template <typename T> +inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, TNullPtr) { + return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, 0)); +} diff --git a/util/system/atomic_ut.cpp b/util/system/atomic_ut.cpp index 07211ffba7..746853228f 100644 --- a/util/system/atomic_ut.cpp +++ b/util/system/atomic_ut.cpp @@ -10,19 +10,19 @@ class TAtomicTest UNIT_TEST_SUITE(TAtomicTest); UNIT_TEST(TestAtomicInc1) UNIT_TEST(TestAtomicInc2) - UNIT_TEST(TestAtomicGetAndInc) + UNIT_TEST(TestAtomicGetAndInc) UNIT_TEST(TestAtomicDec) - UNIT_TEST(TestAtomicGetAndDec) + UNIT_TEST(TestAtomicGetAndDec) UNIT_TEST(TestAtomicAdd) - UNIT_TEST(TestAtomicGetAndAdd) + UNIT_TEST(TestAtomicGetAndAdd) UNIT_TEST(TestAtomicSub) - UNIT_TEST(TestAtomicGetAndSub) + UNIT_TEST(TestAtomicGetAndSub) UNIT_TEST(TestAtomicSwap) UNIT_TEST(TestAtomicOr) UNIT_TEST(TestAtomicAnd) UNIT_TEST(TestAtomicXor) UNIT_TEST(TestCAS) - UNIT_TEST(TestGetAndCAS) + UNIT_TEST(TestGetAndCAS) UNIT_TEST(TestLockUnlock) UNIT_TEST_SUITE_END(); @@ -49,18 +49,18 @@ private: UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>()); } - inline void TestGetAndCAS() { - TAtomic v = 0; - - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 1, 0), 0); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 2, 0), 1); - UNIT_ASSERT_VALUES_EQUAL(v, 1); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 0, 1), 1); - UNIT_ASSERT_VALUES_EQUAL(v, 0); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, Max<intptr_t>(), 0), 0); - UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>()); - } - + inline void TestGetAndCAS() { + TAtomic v = 0; + + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 1, 0), 0); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 2, 0), 1); + UNIT_ASSERT_VALUES_EQUAL(v, 1); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 0, 1), 1); + UNIT_ASSERT_VALUES_EQUAL(v, 0); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, Max<intptr_t>(), 0), 0); + UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>()); + } + inline void TestAtomicInc1() { TAtomic v = 0; @@ -79,15 +79,15 @@ private: UNIT_ASSERT_VALUES_EQUAL(v, 2); } - inline void TestAtomicGetAndInc() { - TAtomic v = 0; - - UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 0); - UNIT_ASSERT_VALUES_EQUAL(v, 1); - UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 1); - UNIT_ASSERT_VALUES_EQUAL(v, 2); - } - + inline void TestAtomicGetAndInc() { + TAtomic v = 0; + + UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 0); + UNIT_ASSERT_VALUES_EQUAL(v, 1); + UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 1); + UNIT_ASSERT_VALUES_EQUAL(v, 2); + } + inline void TestAtomicDec() { TAtomic v = 2; @@ -97,15 +97,15 @@ private: UNIT_ASSERT_VALUES_EQUAL(v, 0); } - inline void TestAtomicGetAndDec() { - TAtomic v = 2; - - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 2); - UNIT_ASSERT_VALUES_EQUAL(v, 1); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 1); - UNIT_ASSERT_VALUES_EQUAL(v, 0); - } - + inline void TestAtomicGetAndDec() { + TAtomic v = 2; + + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 2); + UNIT_ASSERT_VALUES_EQUAL(v, 1); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 1); + UNIT_ASSERT_VALUES_EQUAL(v, 0); + } + inline void TestAtomicAdd() { TAtomic v = 0; @@ -115,15 +115,15 @@ private: UNIT_ASSERT_VALUES_EQUAL(v, -1); } - inline void TestAtomicGetAndAdd() { - TAtomic v = 0; - - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 1), 0); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 2), 1); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, -4), 3); - UNIT_ASSERT_VALUES_EQUAL(v, -1); - } - + inline void TestAtomicGetAndAdd() { + TAtomic v = 0; + + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 1), 0); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 2), 1); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, -4), 3); + UNIT_ASSERT_VALUES_EQUAL(v, -1); + } + inline void TestAtomicSub() { TAtomic v = 4; @@ -133,15 +133,15 @@ private: UNIT_ASSERT_VALUES_EQUAL(v, -2); } - inline void TestAtomicGetAndSub() { - TAtomic v = 4; - - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 1), 4); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 2), 3); - UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 3), 1); - UNIT_ASSERT_VALUES_EQUAL(v, -2); - } - + inline void TestAtomicGetAndSub() { + TAtomic v = 4; + + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 1), 4); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 2), 3); + UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 3), 1); + UNIT_ASSERT_VALUES_EQUAL(v, -2); + } + inline void TestAtomicSwap() { TAtomic v = 0; diff --git a/util/system/atomic_win.h b/util/system/atomic_win.h index 65c290e6cc..40bcdad6b4 100644 --- a/util/system/atomic_win.h +++ b/util/system/atomic_win.h @@ -16,26 +16,26 @@ static inline intptr_t AtomicIncrement(TAtomic& a) { return _InterlockedIncrement((volatile long*)&a); } -static inline intptr_t AtomicGetAndIncrement(TAtomic& a) { - return _InterlockedIncrement((volatile long*)&a) - 1; -} - +static inline intptr_t AtomicGetAndIncrement(TAtomic& a) { + return _InterlockedIncrement((volatile long*)&a) - 1; +} + static inline intptr_t AtomicDecrement(TAtomic& a) { return _InterlockedDecrement((volatile long*)&a); } -static inline intptr_t AtomicGetAndDecrement(TAtomic& a) { - return _InterlockedDecrement((volatile long*)&a) + 1; -} - +static inline intptr_t AtomicGetAndDecrement(TAtomic& a) { + return _InterlockedDecrement((volatile long*)&a) + 1; +} + static inline intptr_t AtomicAdd(TAtomic& a, intptr_t b) { return _InterlockedExchangeAdd((volatile long*)&a, b) + b; } -static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) { - return _InterlockedExchangeAdd((volatile long*)&a, b); -} - +static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) { + return _InterlockedExchangeAdd((volatile long*)&a, b); +} + static inline intptr_t AtomicSwap(TAtomic* a, intptr_t b) { return _InterlockedExchange((volatile long*)a, b); } @@ -45,9 +45,9 @@ static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) { } static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) { - return _InterlockedCompareExchange((volatile long*)a, exchange, compare); -} - + return _InterlockedCompareExchange((volatile long*)a, exchange, compare); +} + #else // _x86_64_ #pragma intrinsic(_InterlockedIncrement64) @@ -60,26 +60,26 @@ static inline intptr_t AtomicIncrement(TAtomic& a) { return _InterlockedIncrement64((volatile __int64*)&a); } -static inline intptr_t AtomicGetAndIncrement(TAtomic& a) { - return _InterlockedIncrement64((volatile __int64*)&a) - 1; -} - +static inline intptr_t AtomicGetAndIncrement(TAtomic& a) { + return _InterlockedIncrement64((volatile __int64*)&a) - 1; +} + static inline intptr_t AtomicDecrement(TAtomic& a) { return _InterlockedDecrement64((volatile __int64*)&a); } -static inline intptr_t AtomicGetAndDecrement(TAtomic& a) { - return _InterlockedDecrement64((volatile __int64*)&a) + 1; -} - +static inline intptr_t AtomicGetAndDecrement(TAtomic& a) { + return _InterlockedDecrement64((volatile __int64*)&a) + 1; +} + static inline intptr_t AtomicAdd(TAtomic& a, intptr_t b) { return _InterlockedExchangeAdd64((volatile __int64*)&a, b) + b; } -static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) { - return _InterlockedExchangeAdd64((volatile __int64*)&a, b); -} - +static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) { + return _InterlockedExchangeAdd64((volatile __int64*)&a, b); +} + static inline intptr_t AtomicSwap(TAtomic* a, intptr_t b) { return _InterlockedExchange64((volatile __int64*)a, b); } @@ -88,10 +88,10 @@ static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) { return _InterlockedCompareExchange64((volatile __int64*)a, exchange, compare) == compare; } -static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) { - return _InterlockedCompareExchange64((volatile __int64*)a, exchange, compare); -} - +static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) { + return _InterlockedCompareExchange64((volatile __int64*)a, exchange, compare); +} + static inline intptr_t AtomicOr(TAtomic& a, intptr_t b) { return _InterlockedOr64(&a, b) | b; } diff --git a/util/system/compiler.h b/util/system/compiler.h index b373edcc46..cec05dd024 100644 --- a/util/system/compiler.h +++ b/util/system/compiler.h @@ -319,7 +319,7 @@ _YandexAbort(); #else #define Y_CUDA_AT_LEAST(x, y) 0 #endif - + // NVidia CUDA C++ Compiler did not know about noexcept keyword until version 9.0 #if !Y_CUDA_AT_LEAST(9, 0) #if defined(__CUDACC__) && !defined(noexcept) @@ -327,15 +327,15 @@ _YandexAbort(); #endif #endif -#if defined(__GNUC__) +#if defined(__GNUC__) #define Y_COLD __attribute__((cold)) #define Y_LEAF __attribute__((leaf)) #define Y_WRAPPER __attribute__((artificial)) -#else +#else #define Y_COLD #define Y_LEAF #define Y_WRAPPER -#endif +#endif /** * @def Y_PRAGMA diff --git a/util/system/datetime.h b/util/system/datetime.h index aa009974e0..9cc7d7fd60 100644 --- a/util/system/datetime.h +++ b/util/system/datetime.h @@ -36,14 +36,14 @@ void NanoSleep(ui64 ns) noexcept; // GetCycleCount guarantees to return synchronous values on different cores // and provide constant rate only on modern Intel and AMD processors -// NOTE: rdtscp is used to prevent out of order execution -// rdtsc can be reordered, while rdtscp cannot be reordered -// with preceding instructions -// PERFORMANCE: rdtsc - 15 cycles per call , rdtscp - 19 cycles per call -// WARNING: following instruction can be executed out-of-order +// NOTE: rdtscp is used to prevent out of order execution +// rdtsc can be reordered, while rdtscp cannot be reordered +// with preceding instructions +// PERFORMANCE: rdtsc - 15 cycles per call , rdtscp - 19 cycles per call +// WARNING: following instruction can be executed out-of-order Y_FORCE_INLINE ui64 GetCycleCount() noexcept { #if defined(_MSC_VER) - // Generates the rdtscp instruction, which returns the processor time stamp. + // Generates the rdtscp instruction, which returns the processor time stamp. // The processor time stamp records the number of clock cycles since the last reset. extern const bool HaveRdtscp; @@ -82,7 +82,7 @@ Y_FORCE_INLINE ui64 GetCycleCount() noexcept { #elif defined(_darwin_) return mach_absolute_time(); #elif defined(__clang__) && !defined(_arm_) - return __builtin_readcyclecounter(); + return __builtin_readcyclecounter(); #elif defined(_arm32_) return MicroSeconds(); #elif defined(_arm64_) diff --git a/util/system/file.cpp b/util/system/file.cpp index 4a261d020c..ce62a3ccab 100644 --- a/util/system/file.cpp +++ b/util/system/file.cpp @@ -599,12 +599,12 @@ int TFileHandle::Duplicate2Posix(int dstHandle) const noexcept { bool TFileHandle::LinkTo(const TFileHandle& fh) const noexcept { #if defined(_unix_) - while (dup2(fh.Fd_, Fd_) == -1) { - if (errno != EINTR) { - return false; - } - } - return true; + while (dup2(fh.Fd_, Fd_) == -1) { + if (errno != EINTR) { + return false; + } + } + return true; #elif defined(_win_) TFileHandle nh(fh.Duplicate()); diff --git a/util/system/shellcommand.cpp b/util/system/shellcommand.cpp index b1989b5c8c..31538ac9aa 100644 --- a/util/system/shellcommand.cpp +++ b/util/system/shellcommand.cpp @@ -738,10 +738,10 @@ void TShellCommand::TImpl::OnFork(TPipes& pipes, sigset_t oldmask, char* const* afterFork(); } - if (envp == nullptr) { - execvp(argv[0], argv); + if (envp == nullptr) { + execvp(argv[0], argv); } else { - execve(argv[0], argv, envp); + execve(argv[0], argv, envp); } Cerr << "Process was not created: " << LastSystemErrorText() << Endl; } catch (const std::exception& error) { @@ -782,52 +782,52 @@ void TShellCommand::TImpl::Run() { ythrow TSystemError() << "Cannot block all signals in parent"; } - /* arguments holders */ + /* arguments holders */ TString shellArg; TVector<char*> qargv; - /* - Following "const_cast"s are safe: - http://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html - */ - if (UseShell) { - shellArg = GetQuotedCommand(); - qargv.reserve(4); - qargv.push_back(const_cast<char*>("/bin/sh")); - qargv.push_back(const_cast<char*>("-c")); - // two args for 'sh -c -- ', - // one for program name, and one for NULL at the end + /* + Following "const_cast"s are safe: + http://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html + */ + if (UseShell) { + shellArg = GetQuotedCommand(); + qargv.reserve(4); + qargv.push_back(const_cast<char*>("/bin/sh")); + qargv.push_back(const_cast<char*>("-c")); + // two args for 'sh -c -- ', + // one for program name, and one for NULL at the end qargv.push_back(const_cast<char*>(shellArg.data())); - } else { - qargv.reserve(Arguments.size() + 2); + } else { + qargv.reserve(Arguments.size() + 2); qargv.push_back(const_cast<char*>(Command.data())); for (auto& i : Arguments) { qargv.push_back(const_cast<char*>(i.data())); - } - } - - qargv.push_back(nullptr); - + } + } + + qargv.push_back(nullptr); + TVector<TString> envHolder; TVector<char*> envp; - if (!Environment.empty()) { + if (!Environment.empty()) { for (auto& env : Environment) { - envHolder.emplace_back(env.first + '=' + env.second); + envHolder.emplace_back(env.first + '=' + env.second); envp.push_back(const_cast<char*>(envHolder.back().data())); - } - envp.push_back(nullptr); - } - + } + envp.push_back(nullptr); + } + pid_t pid = fork(); if (pid == -1) { AtomicSet(ExecutionStatus, SHELL_ERROR); /// @todo check if pipes are still open ythrow TSystemError() << "Cannot fork"; } else if (pid == 0) { // child - if (envp.size() != 0) { + if (envp.size() != 0) { OnFork(pipes, oldmask, qargv.data(), envp.data(), FuncAfterFork); - } else { + } else { OnFork(pipes, oldmask, qargv.data(), nullptr, FuncAfterFork); - } + } } else { // parent // restore signal mask if (SigProcMask(SIG_SETMASK, &oldmask, nullptr) != 0) { diff --git a/util/thread/lfqueue.h b/util/thread/lfqueue.h index ab523631e4..dcb1c0cd0a 100644 --- a/util/thread/lfqueue.h +++ b/util/thread/lfqueue.h @@ -76,16 +76,16 @@ class TLockFreeQueue: public TNonCopyable { alignas(64) TRootNode* volatile FreePtr; void TryToFreeAsyncMemory() { - TAtomic keepCounter = AtomicAdd(FreeingTaskCounter, 0); + TAtomic keepCounter = AtomicAdd(FreeingTaskCounter, 0); TRootNode* current = AtomicGet(FreePtr); if (current == nullptr) return; if (AtomicAdd(FreememCounter, 0) == 1) { // we are the last thread, try to cleanup - // check if another thread have cleaned up - if (keepCounter != AtomicAdd(FreeingTaskCounter, 0)) { - return; - } + // check if another thread have cleaned up + if (keepCounter != AtomicAdd(FreeingTaskCounter, 0)) { + return; + } if (AtomicCas(&FreePtr, (TRootNode*)nullptr, current)) { // free list while (current) { @@ -94,7 +94,7 @@ class TLockFreeQueue: public TNonCopyable { delete current; current = p; } - AtomicAdd(FreeingTaskCounter, 1); + AtomicAdd(FreeingTaskCounter, 1); } } } @@ -228,7 +228,7 @@ public: TLockFreeQueue() : JobQueue(new TRootNode) , FreememCounter(0) - , FreeingTaskCounter(0) + , FreeingTaskCounter(0) , FreePtr(nullptr) { } diff --git a/ydb/core/actorlib_impl/async_destroyer.h b/ydb/core/actorlib_impl/async_destroyer.h index 9d0d6f4c4c..c47e0caebf 100644 --- a/ydb/core/actorlib_impl/async_destroyer.h +++ b/ydb/core/actorlib_impl/async_destroyer.h @@ -29,7 +29,7 @@ private: template <class TVictim> void AsyncDestroy(TAutoPtr<TVictim> victim, const NActors::TActorContext &ctx, ui32 poolId = Max<ui32>()) { if (victim) - ctx.Register(new TAsyncDestroyer<TVictim>(victim), NActors::TMailboxType::HTSwap, poolId); + ctx.Register(new TAsyncDestroyer<TVictim>(victim), NActors::TMailboxType::HTSwap, poolId); } } diff --git a/ydb/core/actorlib_impl/connect_socket_protocol.cpp b/ydb/core/actorlib_impl/connect_socket_protocol.cpp index c4c6db9f13..01114d17e6 100644 --- a/ydb/core/actorlib_impl/connect_socket_protocol.cpp +++ b/ydb/core/actorlib_impl/connect_socket_protocol.cpp @@ -1,163 +1,163 @@ -#include "connect_socket_protocol.h" - +#include "connect_socket_protocol.h" + #include <ydb/core/base/appdata.h> - -namespace NActors { - -void TConnectSocketProtocol::ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept -{ - switch (ev->GetTypeRewrite()) { - - case TEvConnectWakeup::EventType: - TryAgain(ctx); - break; - - case TEvSocketConnect::EventType: - CatchConnectedSocket(ctx, std::move(Socket)); - break; - - case TEvSocketError::EventType: - { - auto errorMsg = ev->Get<TEvSocketError>(); - if (Socket.Get() == errorMsg->Socket.Get()) - CheckConnectResult(ctx, errorMsg->Error); - } - break; - - default: - Y_FAIL("Unknown message type dispatched"); - } -} - - -void TConnectSocketProtocol::CheckRetry( - const TActorContext& ctx, + +namespace NActors { + +void TConnectSocketProtocol::ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept +{ + switch (ev->GetTypeRewrite()) { + + case TEvConnectWakeup::EventType: + TryAgain(ctx); + break; + + case TEvSocketConnect::EventType: + CatchConnectedSocket(ctx, std::move(Socket)); + break; + + case TEvSocketError::EventType: + { + auto errorMsg = ev->Get<TEvSocketError>(); + if (Socket.Get() == errorMsg->Socket.Get()) + CheckConnectResult(ctx, errorMsg->Error); + } + break; + + default: + Y_FAIL("Unknown message type dispatched"); + } +} + + +void TConnectSocketProtocol::CheckRetry( + const TActorContext& ctx, TString explain) noexcept -{ - if (++Retries > 5) { - CatchConnectError(ctx, std::move(explain)); - } else { - ctx.Schedule(TDuration::Seconds(1), new TEvConnectWakeup); - } -} - - -bool TConnectSocketProtocol::CheckConnectResult( - const TActorContext& ctx, - int result) noexcept -{ - switch (result) { - case EINPROGRESS: - return false; - - case 0: - CatchConnectedSocket(ctx, std::move(Socket)); - break; - - case EBADF: - Y_FAIL("bad descriptor"); - case EFAULT: - Y_FAIL("socket structure address is outside the user's address space"); - case EAFNOSUPPORT: - Y_FAIL("EAFNOSUPPORT"); - case EISCONN: - Y_FAIL("socket is already connected"); - case ENOTSOCK: - Y_FAIL("descriptor is not a socket"); - - case EACCES: - CatchConnectError(ctx, "permission denied"); - break; - - case EADDRINUSE: - CheckRetry(ctx, "EADDRINUSE"); - break; - - case EAGAIN: - CheckRetry(ctx, "No more free local ports"); - break; - - case ECONNREFUSED: - CheckRetry(ctx, "Connection refused"); - break; - - case ENETUNREACH: - CheckRetry(ctx, "Network is unreachable"); - break; - - default: - CheckRetry(ctx, "unexpected error from 'socket'"); - break; - } - return true; -} - - -static TDelegate CheckConnectionRoutine( - const TIntrusivePtr<TSharedDescriptor>& FDPtr, - const TActorContext& ctx) noexcept -{ - NInterconnect::TStreamSocket* sock = - (NInterconnect::TStreamSocket*)FDPtr.Get(); - - int errCode; - if (GetSockOpt(*sock, SOL_SOCKET, SO_ERROR, errCode) == -1) { - switch (errno) { - case EBADF: - Y_FAIL("Bad descriptor"); - - case EFAULT: - Y_FAIL("Invalid optval in getsockopt"); - - case EINVAL: - Y_FAIL("Invalid optlne in getsockopt"); - - case ENOPROTOOPT: - Y_FAIL("Unknown option getsockopt"); - - case ENOTSOCK: - Y_FAIL("Not a socket in getsockopt"); - - default: - Y_FAIL("Unexpected error from getsockopt"); - } - } - - switch (errCode) { - case EINPROGRESS: - return TDelegate(); - - case 0: - return [=]() { ctx.Send(ctx.SelfID, new TEvSocketConnect); }; - - default: - { - TIntrusivePtr<NInterconnect::TStreamSocket> sockPtr(sock); - return [=]() { - ctx.Send(ctx.SelfID, new TEvSocketError(errCode, sockPtr)); - }; - } - } -} - - -void TConnectSocketProtocol::TryAgain(const TActorContext& ctx) noexcept { - Socket = NInterconnect::TStreamSocket::Make(Address->Addr()->sa_family); - if (*Socket == -1) { - CheckRetry(ctx, "could not create socket"); - return; - } - - const auto result = Socket->Connect(Address.Get()); - if (CheckConnectResult(ctx, -result)) - return; - - IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); +{ + if (++Retries > 5) { + CatchConnectError(ctx, std::move(explain)); + } else { + ctx.Schedule(TDuration::Seconds(1), new TEvConnectWakeup); + } +} + + +bool TConnectSocketProtocol::CheckConnectResult( + const TActorContext& ctx, + int result) noexcept +{ + switch (result) { + case EINPROGRESS: + return false; + + case 0: + CatchConnectedSocket(ctx, std::move(Socket)); + break; + + case EBADF: + Y_FAIL("bad descriptor"); + case EFAULT: + Y_FAIL("socket structure address is outside the user's address space"); + case EAFNOSUPPORT: + Y_FAIL("EAFNOSUPPORT"); + case EISCONN: + Y_FAIL("socket is already connected"); + case ENOTSOCK: + Y_FAIL("descriptor is not a socket"); + + case EACCES: + CatchConnectError(ctx, "permission denied"); + break; + + case EADDRINUSE: + CheckRetry(ctx, "EADDRINUSE"); + break; + + case EAGAIN: + CheckRetry(ctx, "No more free local ports"); + break; + + case ECONNREFUSED: + CheckRetry(ctx, "Connection refused"); + break; + + case ENETUNREACH: + CheckRetry(ctx, "Network is unreachable"); + break; + + default: + CheckRetry(ctx, "unexpected error from 'socket'"); + break; + } + return true; +} + + +static TDelegate CheckConnectionRoutine( + const TIntrusivePtr<TSharedDescriptor>& FDPtr, + const TActorContext& ctx) noexcept +{ + NInterconnect::TStreamSocket* sock = + (NInterconnect::TStreamSocket*)FDPtr.Get(); + + int errCode; + if (GetSockOpt(*sock, SOL_SOCKET, SO_ERROR, errCode) == -1) { + switch (errno) { + case EBADF: + Y_FAIL("Bad descriptor"); + + case EFAULT: + Y_FAIL("Invalid optval in getsockopt"); + + case EINVAL: + Y_FAIL("Invalid optlne in getsockopt"); + + case ENOPROTOOPT: + Y_FAIL("Unknown option getsockopt"); + + case ENOTSOCK: + Y_FAIL("Not a socket in getsockopt"); + + default: + Y_FAIL("Unexpected error from getsockopt"); + } + } + + switch (errCode) { + case EINPROGRESS: + return TDelegate(); + + case 0: + return [=]() { ctx.Send(ctx.SelfID, new TEvSocketConnect); }; + + default: + { + TIntrusivePtr<NInterconnect::TStreamSocket> sockPtr(sock); + return [=]() { + ctx.Send(ctx.SelfID, new TEvSocketError(errCode, sockPtr)); + }; + } + } +} + + +void TConnectSocketProtocol::TryAgain(const TActorContext& ctx) noexcept { + Socket = NInterconnect::TStreamSocket::Make(Address->Addr()->sa_family); + if (*Socket == -1) { + CheckRetry(ctx, "could not create socket"); + return; + } + + const auto result = Socket->Connect(Address.Get()); + if (CheckConnectResult(ctx, -result)) + return; + + IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); poller->StartWrite(Socket, - std::bind(CheckConnectionRoutine, std::placeholders::_1, ctx)); -} - - -} + std::bind(CheckConnectionRoutine, std::placeholders::_1, ctx)); +} + + +} diff --git a/ydb/core/actorlib_impl/connect_socket_protocol.h b/ydb/core/actorlib_impl/connect_socket_protocol.h index df6e08276a..b13dce67fd 100644 --- a/ydb/core/actorlib_impl/connect_socket_protocol.h +++ b/ydb/core/actorlib_impl/connect_socket_protocol.h @@ -1,65 +1,65 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/interconnect/events_local.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/interconnect_stream.h> - -namespace NActors { - -class TConnectSocketProtocol { -public: - template <typename TOrigActor> - void ConnectSocket( - TOrigActor* orig, - const TActorContext& ctx, - NAddr::IRemoteAddrRef address) noexcept - { - Y_VERIFY(address.Get() != nullptr); - Y_VERIFY(orig != nullptr); - - orig->template AddMsgProtocol<TEvConnectWakeup>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TConnectSocketProtocol, - &TConnectSocketProtocol::ProtocolFunc>); - - orig->template AddMsgProtocol<TEvSocketConnect>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TConnectSocketProtocol, - &TConnectSocketProtocol::ProtocolFunc>); - - orig->template AddMsgProtocol<TEvSocketError>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TConnectSocketProtocol, - &TConnectSocketProtocol::ProtocolFunc>); - - Address = std::move(address); - TryAgain(ctx); - } - - virtual void CatchConnectError( + +namespace NActors { + +class TConnectSocketProtocol { +public: + template <typename TOrigActor> + void ConnectSocket( + TOrigActor* orig, + const TActorContext& ctx, + NAddr::IRemoteAddrRef address) noexcept + { + Y_VERIFY(address.Get() != nullptr); + Y_VERIFY(orig != nullptr); + + orig->template AddMsgProtocol<TEvConnectWakeup>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TConnectSocketProtocol, + &TConnectSocketProtocol::ProtocolFunc>); + + orig->template AddMsgProtocol<TEvSocketConnect>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TConnectSocketProtocol, + &TConnectSocketProtocol::ProtocolFunc>); + + orig->template AddMsgProtocol<TEvSocketError>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TConnectSocketProtocol, + &TConnectSocketProtocol::ProtocolFunc>); + + Address = std::move(address); + TryAgain(ctx); + } + + virtual void CatchConnectError( const TActorContext& ctx, TString error) noexcept = 0; - - virtual void CatchConnectedSocket( - const TActorContext& ctx, - TIntrusivePtr<NInterconnect::TStreamSocket> socket) noexcept = 0; - - virtual ~TConnectSocketProtocol() = default; - -private: - void ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept; - - void TryAgain(const TActorContext& ctx) noexcept; - void CheckRetry(const TActorContext& ctx, TString explain) noexcept; - bool CheckConnectResult(const TActorContext& ctx, int result) noexcept; - - NAddr::IRemoteAddrRef Address; - ui16 Retries = 0; - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; -}; - -} + + virtual void CatchConnectedSocket( + const TActorContext& ctx, + TIntrusivePtr<NInterconnect::TStreamSocket> socket) noexcept = 0; + + virtual ~TConnectSocketProtocol() = default; + +private: + void ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept; + + void TryAgain(const TActorContext& ctx) noexcept; + void CheckRetry(const TActorContext& ctx, TString explain) noexcept; + bool CheckConnectResult(const TActorContext& ctx, int result) noexcept; + + NAddr::IRemoteAddrRef Address; + ui16 Retries = 0; + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; +}; + +} diff --git a/ydb/core/actorlib_impl/destruct_actor.h b/ydb/core/actorlib_impl/destruct_actor.h index adc236911d..6edfaa4fdf 100644 --- a/ydb/core/actorlib_impl/destruct_actor.h +++ b/ydb/core/actorlib_impl/destruct_actor.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/protos/services_common.pb.h> #include <ydb/core/protos/services.pb.h> #include <library/cpp/actors/core/events.h> - -namespace NActors { - -class TDestructActor: public TActor<TDestructActor> { -public: + +namespace NActors { + +class TDestructActor: public TActor<TDestructActor> { +public: static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::INTERCONNECT_DESTRUCT_ACTOR; - } - - TDestructActor() noexcept - : TActor(&TThis::WorkingState) - {} - - -private: + } + + TDestructActor() noexcept + : TActor(&TThis::WorkingState) + {} + + +private: STATEFN(WorkingState) { - /* Destroy event and eventhandle */ - ev.Reset(); - } -}; - + /* Destroy event and eventhandle */ + ev.Reset(); + } +}; + TActorId GetDestructActorID() noexcept { return TActorId(0, "destructor"); -} - -} +} + +} diff --git a/ydb/core/actorlib_impl/http_request_protocol.h b/ydb/core/actorlib_impl/http_request_protocol.h index d5a5b4a0bc..56f33c7314 100644 --- a/ydb/core/actorlib_impl/http_request_protocol.h +++ b/ydb/core/actorlib_impl/http_request_protocol.h @@ -1,273 +1,273 @@ -#pragma once - -#include "name_service_client_protocol.h" -#include "connect_socket_protocol.h" -#include "send_data_protocol.h" -#include "read_http_reply_protocol.h" - -namespace NActors { - -template <typename TOrigActor> -class THTTPRequestProtocol - : public TResolveClientProtocol - , public TConnectSocketProtocol - , public TSendDataProtocol - , public TReadHTTPReplyProtocol - , public TReadDataProtocol<THTTPRequestProtocol<TOrigActor>> -{ - friend class TResolveClientProtocol; - friend class TConnectSocketProtocol; - friend class TSendDataProtocol; - friend class TReadHTTPReplyProtocol; - friend class TReadDataProtocol<THTTPRequestProtocol<TOrigActor>>; -public: - - void HTTPRequest( - TOrigActor* orig, - const TActorContext& ctx, +#pragma once + +#include "name_service_client_protocol.h" +#include "connect_socket_protocol.h" +#include "send_data_protocol.h" +#include "read_http_reply_protocol.h" + +namespace NActors { + +template <typename TOrigActor> +class THTTPRequestProtocol + : public TResolveClientProtocol + , public TConnectSocketProtocol + , public TSendDataProtocol + , public TReadHTTPReplyProtocol + , public TReadDataProtocol<THTTPRequestProtocol<TOrigActor>> +{ + friend class TResolveClientProtocol; + friend class TConnectSocketProtocol; + friend class TSendDataProtocol; + friend class TReadHTTPReplyProtocol; + friend class TReadDataProtocol<THTTPRequestProtocol<TOrigActor>>; +public: + + void HTTPRequest( + TOrigActor* orig, + const TActorContext& ctx, TString host, TString url, TString headers = TString(), - ui16 port = 80) noexcept - { - OriginalActor = orig; - - orig->template AddMsgProtocol<TEvHTTPProtocolRetry>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - THTTPRequestProtocol<TOrigActor>, - &THTTPRequestProtocol<TOrigActor>::ProtocolFunc>); - - Host = std::move(host); - Url = std::move(url); - Port = port; - HttpRequestMessage = Sprintf( - "%s HTTP/1.1\r\n" - "Host: %s\r\n" - "%s" - "Connection: close\r\n" - "\r\n", + ui16 port = 80) noexcept + { + OriginalActor = orig; + + orig->template AddMsgProtocol<TEvHTTPProtocolRetry>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + THTTPRequestProtocol<TOrigActor>, + &THTTPRequestProtocol<TOrigActor>::ProtocolFunc>); + + Host = std::move(host); + Url = std::move(url); + Port = port; + HttpRequestMessage = Sprintf( + "%s HTTP/1.1\r\n" + "Host: %s\r\n" + "%s" + "Connection: close\r\n" + "\r\n", Url.data(), Host.data(), headers.data()); - + MemLogPrintF("HTTPRequest %s", HttpRequestMessage.data()); - - WriteTask = EWriteTask::COMPLETE; - NumberOfTriesLeft = 4; - RetryCall = [=](const TActorContext& ctx) { - SendResolveMessage<TOrigActor>(orig, ctx, Host, port); - }; - SendResolveMessage<TOrigActor>(orig, ctx, Host, port); - } - - void HTTPReadContent( - TOrigActor* orig, - const TActorContext& ctx, - ui64 amount) noexcept - { - OriginalActor = orig; - ReadChunks = false; - ReadBuffer.resize(amount); - Filled = 0; - TReadDataProtocol<THTTPRequestProtocol<TOrigActor>>::ReadData( - this, OriginalActor, ctx, - Socket.Get(), ReadBuffer.data(), amount); - } - - void HTTPWriteContent( - TOrigActor* orig, - const TActorContext& ctx, - const char* data, - size_t len, - bool complete) noexcept - { - OriginalActor = orig; - WriteTask = complete ? EWriteTask::COMPLETE : EWriteTask::CONTINUE; - SendData<TOrigActor>(OriginalActor, ctx, Socket.Get(), data, len); - } - - void HTTPExpectReply( - TOrigActor* orig, - const TActorContext& ctx, + + WriteTask = EWriteTask::COMPLETE; + NumberOfTriesLeft = 4; + RetryCall = [=](const TActorContext& ctx) { + SendResolveMessage<TOrigActor>(orig, ctx, Host, port); + }; + SendResolveMessage<TOrigActor>(orig, ctx, Host, port); + } + + void HTTPReadContent( + TOrigActor* orig, + const TActorContext& ctx, + ui64 amount) noexcept + { + OriginalActor = orig; + ReadChunks = false; + ReadBuffer.resize(amount); + Filled = 0; + TReadDataProtocol<THTTPRequestProtocol<TOrigActor>>::ReadData( + this, OriginalActor, ctx, + Socket.Get(), ReadBuffer.data(), amount); + } + + void HTTPWriteContent( + TOrigActor* orig, + const TActorContext& ctx, + const char* data, + size_t len, + bool complete) noexcept + { + OriginalActor = orig; + WriteTask = complete ? EWriteTask::COMPLETE : EWriteTask::CONTINUE; + SendData<TOrigActor>(OriginalActor, ctx, Socket.Get(), data, len); + } + + void HTTPExpectReply( + TOrigActor* orig, + const TActorContext& ctx, TVector<char> buf = TVector<char>()) noexcept - { - ReadHTTPReply<TOrigActor>(orig, ctx, Socket, buf); - } - + { + ReadHTTPReply<TOrigActor>(orig, ctx, Socket, buf); + } + virtual void CatchHTTPRequestError(TString error) noexcept = 0; - - virtual void CatchHTTPContent( + + virtual void CatchHTTPContent( const TActorContext& ctx, TVector<char> buf) noexcept = 0; - - virtual void CatchHTTPWriteComplete(const TActorContext&) noexcept {} - + + virtual void CatchHTTPWriteComplete(const TActorContext&) noexcept {} + virtual void CatchHTTPConnectionClosed() noexcept = 0; - -public: - void CatchHostAddress( - const TActorContext& ctx, - NAddr::IRemoteAddrPtr address) noexcept override - { - Y_VERIFY(address.Get() != nullptr); - - NAddr::IRemoteAddrRef addr(address.Release()); - - Y_VERIFY(addr.Get() != nullptr); - - MemLogPrintF("%s" - ", actorId #%s" - ", address %s", - __func__, + +public: + void CatchHostAddress( + const TActorContext& ctx, + NAddr::IRemoteAddrPtr address) noexcept override + { + Y_VERIFY(address.Get() != nullptr); + + NAddr::IRemoteAddrRef addr(address.Release()); + + Y_VERIFY(addr.Get() != nullptr); + + MemLogPrintF("%s" + ", actorId #%s" + ", address %s", + __func__, ctx.SelfID.ToString().data(), PrintHostAndPort(*addr).data()); - - NumberOfTriesLeft = 4; - RetryCall = [=](const TActorContext& ctx) { - ConnectSocket<TOrigActor>(OriginalActor, ctx, addr); - }; - - Y_VERIFY(addr.Get() != nullptr); - - ConnectSocket<TOrigActor>(OriginalActor, ctx, addr); - } - - - void CatchResolveError( + + NumberOfTriesLeft = 4; + RetryCall = [=](const TActorContext& ctx) { + ConnectSocket<TOrigActor>(OriginalActor, ctx, addr); + }; + + Y_VERIFY(addr.Get() != nullptr); + + ConnectSocket<TOrigActor>(OriginalActor, ctx, addr); + } + + + void CatchResolveError( const TActorContext& ctx, TString error) noexcept override - { - if (NumberOfTriesLeft && --NumberOfTriesLeft > 0) { - MemLogPrintF("%s" - ", schedule retry" - ", actorId #%s" - ", error %s", - __func__, + { + if (NumberOfTriesLeft && --NumberOfTriesLeft > 0) { + MemLogPrintF("%s" + ", schedule retry" + ", actorId #%s" + ", error %s", + __func__, ctx.SelfID.ToString().data(), error.data()); - - ctx.Schedule(TDuration::Seconds(1), new TEvHTTPProtocolRetry); - return; - } else { - RetryCall = std::function<void(const TActorContext&)>(); - } - - MemLogPrintF("%s" - ", actorId #%s" - ", error %s", - __func__, + + ctx.Schedule(TDuration::Seconds(1), new TEvHTTPProtocolRetry); + return; + } else { + RetryCall = std::function<void(const TActorContext&)>(); + } + + MemLogPrintF("%s" + ", actorId #%s" + ", error %s", + __func__, ctx.SelfID.ToString().data(), error.data()); - + CatchHTTPRequestError(std::move(error)); - } - - - void CatchConnectError( + } + + + void CatchConnectError( const TActorContext& ctx, TString error) noexcept override - { - if (--NumberOfTriesLeft > 0) { - ctx.Schedule(TDuration::Seconds(1), new TEvHTTPProtocolRetry); - return; - } else { - RetryCall = std::function<void(const TActorContext& ctx)>(); - } - + { + if (--NumberOfTriesLeft > 0) { + ctx.Schedule(TDuration::Seconds(1), new TEvHTTPProtocolRetry); + return; + } else { + RetryCall = std::function<void(const TActorContext& ctx)>(); + } + CatchHTTPRequestError(std::move(error)); - } - - - void CatchConnectedSocket( - const TActorContext& ctx, - TIntrusivePtr<NInterconnect::TStreamSocket> socket - ) noexcept override - { - Socket = std::move(socket); - SendData<TOrigActor>(OriginalActor, ctx, Socket.Get(), + } + + + void CatchConnectedSocket( + const TActorContext& ctx, + TIntrusivePtr<NInterconnect::TStreamSocket> socket + ) noexcept override + { + Socket = std::move(socket); + SendData<TOrigActor>(OriginalActor, ctx, Socket.Get(), HttpRequestMessage.data(), HttpRequestMessage.size()); - ReadHTTPReply<TOrigActor>(OriginalActor, ctx, Socket); - } - + ReadHTTPReply<TOrigActor>(OriginalActor, ctx, Socket); + } + void CatchSendDataError(TString error) noexcept override - { + { CatchHTTPRequestError(std::move(error)); - } - + } + void CatchReadDataError(TString error) noexcept override - { + { CatchHTTPRequestError(std::move(error)); - } - - void CatchSendDataComplete(const TActorContext& ctx) noexcept override { - switch (WriteTask) { - case EWriteTask::COMPLETE: - MemLogPrintF("CatchSendDataComplete, EWriteTask::COMPLETE"); - break; - - case EWriteTask::CONTINUE: - MemLogPrintF("CatchSendDataComplete, EWriteTask::COMPLETE"); - - CatchHTTPWriteComplete(ctx); - break; - } - } - - bool CatchReadDataComplete( - const TActorContext& ctx, size_t amount) noexcept - { - if (ReadChunks == false) { - Filled += amount; - if (Filled == ReadBuffer.size()) { - CatchHTTPContent(ctx, std::move(ReadBuffer)); - return true; - } else { - return false; - } - } - return true; - } - + } + + void CatchSendDataComplete(const TActorContext& ctx) noexcept override { + switch (WriteTask) { + case EWriteTask::COMPLETE: + MemLogPrintF("CatchSendDataComplete, EWriteTask::COMPLETE"); + break; + + case EWriteTask::CONTINUE: + MemLogPrintF("CatchSendDataComplete, EWriteTask::COMPLETE"); + + CatchHTTPWriteComplete(ctx); + break; + } + } + + bool CatchReadDataComplete( + const TActorContext& ctx, size_t amount) noexcept + { + if (ReadChunks == false) { + Filled += amount; + if (Filled == ReadBuffer.size()) { + CatchHTTPContent(ctx, std::move(ReadBuffer)); + return true; + } else { + return false; + } + } + return true; + } + void CatchReadDataClosed() noexcept { CatchHTTPConnectionClosed(); - } - -private: - void ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept - { - switch (ev->GetTypeRewrite()) { - CFunc(TEvHTTPProtocolRetry::EventType, Retry); - default: - Y_FAIL("Unknown message type dispatched"); - } - } - - void Retry(const TActorContext& ctx) noexcept { - Y_VERIFY(RetryCall); - - RetryCall(ctx); - } - - TOrigActor* OriginalActor; - IActor::TReceiveFunc DefaultStateFunc; + } + +private: + void ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept + { + switch (ev->GetTypeRewrite()) { + CFunc(TEvHTTPProtocolRetry::EventType, Retry); + default: + Y_FAIL("Unknown message type dispatched"); + } + } + + void Retry(const TActorContext& ctx) noexcept { + Y_VERIFY(RetryCall); + + RetryCall(ctx); + } + + TOrigActor* OriginalActor; + IActor::TReceiveFunc DefaultStateFunc; TString Host; TString Url; - ui16 Port; + ui16 Port; TString HttpRequestMessage; - + TVector<char> ReadBuffer; - size_t Filled; - bool ReadChunks; - - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; - - enum class EWriteTask { - COMPLETE, - CONTINUE, - }; - - EWriteTask WriteTask; - - std::function<void(const TActorContext&)> RetryCall; - ui32 NumberOfTriesLeft; -}; - -} + size_t Filled; + bool ReadChunks; + + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + + enum class EWriteTask { + COMPLETE, + CONTINUE, + }; + + EWriteTask WriteTask; + + std::function<void(const TActorContext&)> RetryCall; + ui32 NumberOfTriesLeft; +}; + +} diff --git a/ydb/core/actorlib_impl/load_network.cpp b/ydb/core/actorlib_impl/load_network.cpp index ae6f1b248a..87ee3e4ed0 100644 --- a/ydb/core/actorlib_impl/load_network.cpp +++ b/ydb/core/actorlib_impl/load_network.cpp @@ -1,100 +1,100 @@ -#include "load_network.h" - +#include "load_network.h" + #include <library/cpp/actors/core/actorid.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/events.h> - -namespace { - + +namespace { + NActors::TActorId GetLoadNetworkActorID(ui32 myNodeId) noexcept { return NActors::TActorId(myNodeId, "loadnetwork"); -} - -using NActors::IEventHandle; -using NActors::TActorContext; -using NActors::TEvents; - -class TLoadNetwork: public NActors::TActorBootstrapped<TLoadNetwork> { -public: +} + +using NActors::IEventHandle; +using NActors::TActorContext; +using NActors::TEvents; + +class TLoadNetwork: public NActors::TActorBootstrapped<TLoadNetwork> { +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TEST_ACTOR_RUNTIME; } - TLoadNetwork(ui32 selfNodeId, ui32 totalNodesCount) - : SelfNodeId(selfNodeId) - , TotalNodesCount(totalNodesCount) - { - - } - - void Bootstrap(const TActorContext &ctx) { - Become(&TThis::DelayMe); - ctx.Schedule(TDuration::Seconds(30), new NActors::TEvents::TEvWakeup); - } - -private: - ui32 SelfNodeId; - ui32 TotalNodesCount; - + TLoadNetwork(ui32 selfNodeId, ui32 totalNodesCount) + : SelfNodeId(selfNodeId) + , TotalNodesCount(totalNodesCount) + { + + } + + void Bootstrap(const TActorContext &ctx) { + Become(&TThis::DelayMe); + ctx.Schedule(TDuration::Seconds(30), new NActors::TEvents::TEvWakeup); + } + +private: + ui32 SelfNodeId; + ui32 TotalNodesCount; + static const TString DataToSend; - - static constexpr ui32 PER_NODE_INFLIGHT_COUNT = 20240; - - STFUNC(DelayMe) { - Y_UNUSED(ev); - - Become(&TThis::Working); - - for (ui32 i = 1; i <= TotalNodesCount; ++i) { - if (i == SelfNodeId) - continue; - for (ui32 j = 0; j < PER_NODE_INFLIGHT_COUNT; j++) { - ctx.Send(GetLoadNetworkActorID(i), - new TEvents::TEvBlob(DataToSend), - IEventHandle::MakeFlags(2, - IEventHandle::FlagTrackDelivery)); - } - } - } - - STFUNC(Working) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvBlob, SendOneMoreBlob); - HFunc(TEvents::TEvUndelivered, SendOneMoreBlob); - } - } - - void SendOneMoreBlob(TEvents::TEvBlob::TPtr &ev, - const TActorContext &ctx) noexcept - { - ctx.Send(ev->Sender, - new TEvents::TEvBlob(DataToSend), - IEventHandle::MakeFlags(2, 0)); - } - - void SendOneMoreBlob(TEvents::TEvUndelivered::TPtr &ev, - const TActorContext &ctx) noexcept - { - ctx.Send(ev->Sender, - new TEvents::TEvBlob(DataToSend), - IEventHandle::MakeFlags(2, 0)); - } -}; - + + static constexpr ui32 PER_NODE_INFLIGHT_COUNT = 20240; + + STFUNC(DelayMe) { + Y_UNUSED(ev); + + Become(&TThis::Working); + + for (ui32 i = 1; i <= TotalNodesCount; ++i) { + if (i == SelfNodeId) + continue; + for (ui32 j = 0; j < PER_NODE_INFLIGHT_COUNT; j++) { + ctx.Send(GetLoadNetworkActorID(i), + new TEvents::TEvBlob(DataToSend), + IEventHandle::MakeFlags(2, + IEventHandle::FlagTrackDelivery)); + } + } + } + + STFUNC(Working) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvBlob, SendOneMoreBlob); + HFunc(TEvents::TEvUndelivered, SendOneMoreBlob); + } + } + + void SendOneMoreBlob(TEvents::TEvBlob::TPtr &ev, + const TActorContext &ctx) noexcept + { + ctx.Send(ev->Sender, + new TEvents::TEvBlob(DataToSend), + IEventHandle::MakeFlags(2, 0)); + } + + void SendOneMoreBlob(TEvents::TEvUndelivered::TPtr &ev, + const TActorContext &ctx) noexcept + { + ctx.Send(ev->Sender, + new TEvents::TEvBlob(DataToSend), + IEventHandle::MakeFlags(2, 0)); + } +}; + const TString TLoadNetwork::DataToSend = TString(3*1024, '!'); - -} - -namespace IC_Load { - void InitializeService(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData, - ui32 totalNodesCount) - { - auto actor = new TLoadNetwork(setup->NodeId, totalNodesCount); - setup->LocalServices.emplace_back( - GetLoadNetworkActorID(setup->NodeId), - NActors::TActorSetupCmd(actor, - NActors::TMailboxType::Simple, - appData->UserPoolId)); - } -} + +} + +namespace IC_Load { + void InitializeService(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData, + ui32 totalNodesCount) + { + auto actor = new TLoadNetwork(setup->NodeId, totalNodesCount); + setup->LocalServices.emplace_back( + GetLoadNetworkActorID(setup->NodeId), + NActors::TActorSetupCmd(actor, + NActors::TMailboxType::Simple, + appData->UserPoolId)); + } +} diff --git a/ydb/core/actorlib_impl/load_network.h b/ydb/core/actorlib_impl/load_network.h index 767ab5c98d..4b0510a0cf 100644 --- a/ydb/core/actorlib_impl/load_network.h +++ b/ydb/core/actorlib_impl/load_network.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorsystem.h> #include <ydb/core/base/appdata.h> - -namespace IC_Load { - void InitializeService(NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData, - ui32 totalNodesCount); -} + +namespace IC_Load { + void InitializeService(NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData, + ui32 totalNodesCount); +} diff --git a/ydb/core/actorlib_impl/name_service_client_protocol.cpp b/ydb/core/actorlib_impl/name_service_client_protocol.cpp index 682bc3b44e..4d06f37aec 100644 --- a/ydb/core/actorlib_impl/name_service_client_protocol.cpp +++ b/ydb/core/actorlib_impl/name_service_client_protocol.cpp @@ -1,29 +1,29 @@ -#include "name_service_client_protocol.h" - -namespace NActors { - -void TResolveClientProtocol::ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept -{ - switch (ev->GetTypeRewrite()) { - - case TEvAddressInfo::EventType: - Y_VERIFY(ev->Get<TEvAddressInfo>() != nullptr); - Y_VERIFY(ev->Get<TEvAddressInfo>()->Address.Get() != nullptr); - - MemLogPrintF("TResolveClientProtocol received address info"); - - CatchHostAddress(ctx, std::move(ev->Get<TEvAddressInfo>()->Address)); - break; - - case TEvResolveError::EventType: - CatchResolveError(ctx, std::move(ev->Get<TEvResolveError>()->Explain)); - break; - - default: - Y_FAIL("Unknown message type dispatched"); - } -} - -} +#include "name_service_client_protocol.h" + +namespace NActors { + +void TResolveClientProtocol::ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept +{ + switch (ev->GetTypeRewrite()) { + + case TEvAddressInfo::EventType: + Y_VERIFY(ev->Get<TEvAddressInfo>() != nullptr); + Y_VERIFY(ev->Get<TEvAddressInfo>()->Address.Get() != nullptr); + + MemLogPrintF("TResolveClientProtocol received address info"); + + CatchHostAddress(ctx, std::move(ev->Get<TEvAddressInfo>()->Address)); + break; + + case TEvResolveError::EventType: + CatchResolveError(ctx, std::move(ev->Get<TEvResolveError>()->Explain)); + break; + + default: + Y_FAIL("Unknown message type dispatched"); + } +} + +} diff --git a/ydb/core/actorlib_impl/name_service_client_protocol.h b/ydb/core/actorlib_impl/name_service_client_protocol.h index c46ffe9b03..3179155162 100644 --- a/ydb/core/actorlib_impl/name_service_client_protocol.h +++ b/ydb/core/actorlib_impl/name_service_client_protocol.h @@ -1,54 +1,54 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/memory_log/memlog.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/events_local.h> - -namespace NActors { - -class TResolveClientProtocol { -public: - template <typename TOrigActor> - void SendResolveMessage( - TOrigActor* orig, - const TActorContext& ctx, + +namespace NActors { + +class TResolveClientProtocol { +public: + template <typename TOrigActor> + void SendResolveMessage( + TOrigActor* orig, + const TActorContext& ctx, TString address, - ui16 port) noexcept - { - auto msg = new TEvResolveAddress; - msg->Address = std::move(address); - msg->Port = port; - - MemLogPrintF("TResolveClientProtocol send name request: %s, %u", + ui16 port) noexcept + { + auto msg = new TEvResolveAddress; + msg->Address = std::move(address); + msg->Port = port; + + MemLogPrintF("TResolveClientProtocol send name request: %s, %u", msg->Address.data(), (int)msg->Port); - + ctx.Send(GetNameserviceActorId(), msg); - - orig->template AddMsgProtocol<TEvAddressInfo>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TResolveClientProtocol, - &TResolveClientProtocol::ProtocolFunc>); - - orig->template AddMsgProtocol<TEvResolveError>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TResolveClientProtocol, - &TResolveClientProtocol::ProtocolFunc>); - } - - virtual void CatchHostAddress( - const TActorContext& ctx, NAddr::IRemoteAddrPtr address) noexcept = 0; - - virtual void CatchResolveError( + + orig->template AddMsgProtocol<TEvAddressInfo>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TResolveClientProtocol, + &TResolveClientProtocol::ProtocolFunc>); + + orig->template AddMsgProtocol<TEvResolveError>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TResolveClientProtocol, + &TResolveClientProtocol::ProtocolFunc>); + } + + virtual void CatchHostAddress( + const TActorContext& ctx, NAddr::IRemoteAddrPtr address) noexcept = 0; + + virtual void CatchResolveError( const TActorContext& ctx, TString error) noexcept = 0; - - virtual ~TResolveClientProtocol() = default; - -private: - void ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept; -}; - -} + + virtual ~TResolveClientProtocol() = default; + +private: + void ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept; +}; + +} diff --git a/ydb/core/actorlib_impl/proto_ready_actor.h b/ydb/core/actorlib_impl/proto_ready_actor.h index f0b29ec65f..c97657d839 100644 --- a/ydb/core/actorlib_impl/proto_ready_actor.h +++ b/ydb/core/actorlib_impl/proto_ready_actor.h @@ -1,94 +1,94 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actor.h> #include <util/system/type_name.h> - -template <typename TType, TType val> -struct TName4Ptr2Member { -}; - -namespace NActors { - -class TProtocolNest { -public: - template <typename T> - inline void AddProtocol(T stateFunc, ui32 msgType) noexcept { - AddProtocol(static_cast<IActor::TReceiveFunc>(stateFunc), msgType); - } - - template <typename TMsg, typename TStateFunc> - inline void AddMsgProtocol(TStateFunc stateFunc) noexcept { - AddProtocol(static_cast<IActor::TReceiveFunc>(stateFunc), - TMsg::EventType); - } - - void AddProtocol(IActor::TReceiveFunc stateFunc, ui32 msgType) noexcept { - ProtocolFunctions.erase(msgType); - ProtocolFunctions.emplace(msgType, stateFunc); - } - - void RemoveProtocol(ui32 msgType) noexcept { - ProtocolFunctions.erase(msgType); - } - -protected: + +template <typename TType, TType val> +struct TName4Ptr2Member { +}; + +namespace NActors { + +class TProtocolNest { +public: + template <typename T> + inline void AddProtocol(T stateFunc, ui32 msgType) noexcept { + AddProtocol(static_cast<IActor::TReceiveFunc>(stateFunc), msgType); + } + + template <typename TMsg, typename TStateFunc> + inline void AddMsgProtocol(TStateFunc stateFunc) noexcept { + AddProtocol(static_cast<IActor::TReceiveFunc>(stateFunc), + TMsg::EventType); + } + + void AddProtocol(IActor::TReceiveFunc stateFunc, ui32 msgType) noexcept { + ProtocolFunctions.erase(msgType); + ProtocolFunctions.emplace(msgType, stateFunc); + } + + void RemoveProtocol(ui32 msgType) noexcept { + ProtocolFunctions.erase(msgType); + } + +protected: TMap<ui32, IActor::TReceiveFunc> ProtocolFunctions; -}; - - -template <template <typename ...> class TBaseActor, typename TDerived> -class TProtoReadyActor: public TProtocolNest, public TBaseActor<TDerived> { -public: - - template < - typename TOrigActor, - typename TProtocol, - void (TProtocol::*Member)( - TAutoPtr<IEventHandle>&, const TActorContext&)> - void CallProtocolStateFunc( - TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) - { - TOrigActor* orig = static_cast<TOrigActor*>(this); - TProtocol* protoThis = static_cast<TProtocol*>(orig); - (protoThis->*Member)(ev, ctx); - } - - TProtoReadyActor() { - DerivedActorFunc = this->CurrentStateFunc(); - this->TBaseActor<TDerived>::Become( - &TProtoReadyActor::ProtocolDispatcher); - } - - template <typename ... TArgs> - TProtoReadyActor(TArgs&&...args) - : TBaseActor<TDerived>(std::move(args)...) - { - DerivedActorFunc = this->CurrentStateFunc(); - this->TBaseActor<TDerived>::Become( - &TProtoReadyActor::ProtocolDispatcher); - } - - template <typename T> - void Become(T stateFunc, const char* hint = "undef") noexcept { +}; + + +template <template <typename ...> class TBaseActor, typename TDerived> +class TProtoReadyActor: public TProtocolNest, public TBaseActor<TDerived> { +public: + + template < + typename TOrigActor, + typename TProtocol, + void (TProtocol::*Member)( + TAutoPtr<IEventHandle>&, const TActorContext&)> + void CallProtocolStateFunc( + TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) + { + TOrigActor* orig = static_cast<TOrigActor*>(this); + TProtocol* protoThis = static_cast<TProtocol*>(orig); + (protoThis->*Member)(ev, ctx); + } + + TProtoReadyActor() { + DerivedActorFunc = this->CurrentStateFunc(); + this->TBaseActor<TDerived>::Become( + &TProtoReadyActor::ProtocolDispatcher); + } + + template <typename ... TArgs> + TProtoReadyActor(TArgs&&...args) + : TBaseActor<TDerived>(std::move(args)...) + { + DerivedActorFunc = this->CurrentStateFunc(); + this->TBaseActor<TDerived>::Become( + &TProtoReadyActor::ProtocolDispatcher); + } + + template <typename T> + void Become(T stateFunc, const char* hint = "undef") noexcept { Y_UNUSED(hint); - DerivedActorFunc = static_cast<IActor::TReceiveFunc>(stateFunc); - } - -private: - IActor::TReceiveFunc DerivedActorFunc; - + DerivedActorFunc = static_cast<IActor::TReceiveFunc>(stateFunc); + } + +private: + IActor::TReceiveFunc DerivedActorFunc; + STFUNC(ProtocolDispatcher) { - Y_VERIFY(ev.Get() != nullptr); - - auto funcIter = ProtocolFunctions.find(ev->Type); - - if (funcIter == ProtocolFunctions.end()) { - return (this->*DerivedActorFunc)(ev, ctx); - } - - auto func = funcIter->second; - return (this->*func)(ev, ctx); - } -}; - -} + Y_VERIFY(ev.Get() != nullptr); + + auto funcIter = ProtocolFunctions.find(ev->Type); + + if (funcIter == ProtocolFunctions.end()) { + return (this->*DerivedActorFunc)(ev, ctx); + } + + auto func = funcIter->second; + return (this->*func)(ev, ctx); + } +}; + +} diff --git a/ydb/core/actorlib_impl/read_data_protocol.cpp b/ydb/core/actorlib_impl/read_data_protocol.cpp index 62045309f8..92978c4903 100644 --- a/ydb/core/actorlib_impl/read_data_protocol.cpp +++ b/ydb/core/actorlib_impl/read_data_protocol.cpp @@ -1,74 +1,74 @@ -#include "read_data_protocol.h" - +#include "read_data_protocol.h" + #include <ydb/core/base/appdata.h> -#include <util/system/error.h> -#include <util/system/yassert.h> - -namespace NActors { - -void TReadDataProtocolImpl::ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept -{ - if (Cancelled) { - return; - } - - switch (ev->GetTypeRewrite()) { - case TEvSocketReadyRead::EventType: - TryAgain(ctx); - break; - - default: - Y_FAIL("Unknown message type dispatched"); - } -} - - -static TDelegate NotifyReadyRead( - const TIntrusivePtr<TSharedDescriptor>& FDPtr, - const TActorContext& ctx) -{ - Y_UNUSED(FDPtr); - return [=]() { ctx.Send(ctx.SelfID, new TEvSocketReadyRead); }; -} - - -void TReadDataProtocolImpl::TryAgain(const TActorContext& ctx) noexcept { - int recvResult; - for (;;) { - recvResult = Socket->Recv(Data, Len); - - if (recvResult > 0) { - Y_VERIFY(Len >= (size_t)recvResult); - Data += recvResult; - Len -= recvResult; - Filled += recvResult; - if (Len == 0) { - /* ResetReadBuf may change Data, Len and Filled here */ - if (Catcher->CatchReadDataComplete(ctx, Filled)) { - return; - } - - if (Cancelled) { - return; - } - - continue; - } - } else { +#include <util/system/error.h> +#include <util/system/yassert.h> + +namespace NActors { + +void TReadDataProtocolImpl::ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept +{ + if (Cancelled) { + return; + } + + switch (ev->GetTypeRewrite()) { + case TEvSocketReadyRead::EventType: + TryAgain(ctx); + break; + + default: + Y_FAIL("Unknown message type dispatched"); + } +} + + +static TDelegate NotifyReadyRead( + const TIntrusivePtr<TSharedDescriptor>& FDPtr, + const TActorContext& ctx) +{ + Y_UNUSED(FDPtr); + return [=]() { ctx.Send(ctx.SelfID, new TEvSocketReadyRead); }; +} + + +void TReadDataProtocolImpl::TryAgain(const TActorContext& ctx) noexcept { + int recvResult; + for (;;) { + recvResult = Socket->Recv(Data, Len); + + if (recvResult > 0) { + Y_VERIFY(Len >= (size_t)recvResult); + Data += recvResult; + Len -= recvResult; + Filled += recvResult; + if (Len == 0) { + /* ResetReadBuf may change Data, Len and Filled here */ + if (Catcher->CatchReadDataComplete(ctx, Filled)) { + return; + } + + if (Cancelled) { + return; + } + + continue; + } + } else { if (-recvResult == EAGAIN || -recvResult == EWOULDBLOCK) { - if (Filled > 0) { - if (Catcher->CatchReadDataComplete(ctx, Filled)) { - return; - } - - if (Cancelled) { - return; - } - - break; - } + if (Filled > 0) { + if (Catcher->CatchReadDataComplete(ctx, Filled)) { + return; + } + + if (Cancelled) { + return; + } + + break; + } } else if (-recvResult == EINTR) { continue; } else if (!recvResult) { @@ -77,11 +77,11 @@ void TReadDataProtocolImpl::TryAgain(const TActorContext& ctx) noexcept { } Catcher->CatchReadDataClosed(); return; - } - break; - } - } - + } + break; + } + } + if (-recvResult == EAGAIN || -recvResult == EWOULDBLOCK) { IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); poller->StartRead(Socket, @@ -89,39 +89,39 @@ void TReadDataProtocolImpl::TryAgain(const TActorContext& ctx) noexcept { return; } - switch (-recvResult) { - case ECONNRESET: + switch (-recvResult) { + case ECONNRESET: Catcher->CatchReadDataError("Connection reset by peer"); - return; - - case EPIPE: + return; + + case EPIPE: Catcher->CatchReadDataError("Connection is closed"); - return; - - default: - { - char buf[1024]; - LastSystemErrorText(buf, 1024, -recvResult); + return; + + default: + { + char buf[1024]; + LastSystemErrorText(buf, 1024, -recvResult); Catcher->CatchReadDataError(TString("Socker error: ") + buf); - return; - } - - case EBADF: - case EFAULT: - case EINVAL: - case ENOTCONN: - case ENOTSOCK: - case EOPNOTSUPP: - { - Y_FAIL("Very bad socket error"); - } - } -} - + return; + } + + case EBADF: + case EFAULT: + case EINVAL: + case ENOTCONN: + case ENOTSOCK: + case EOPNOTSUPP: + { + Y_FAIL("Very bad socket error"); + } + } +} + void TReadDataProtocolImpl::CancelReadData(const TActorContext& /*ctx*/) noexcept { - Cancelled = true; + Cancelled = true; // IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); // poller->CancelRead(Socket); -} - -} +} + +} diff --git a/ydb/core/actorlib_impl/read_data_protocol.h b/ydb/core/actorlib_impl/read_data_protocol.h index 7edefab299..80fdde1e7c 100644 --- a/ydb/core/actorlib_impl/read_data_protocol.h +++ b/ydb/core/actorlib_impl/read_data_protocol.h @@ -1,144 +1,144 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/interconnect/events_local.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/interconnect_stream.h> - -namespace NActors { - -struct IReadDataCatch { + +namespace NActors { + +struct IReadDataCatch { virtual void CatchReadDataError(TString error) noexcept = 0; - /* returns false if the protocol should continue to read data */ - virtual bool CatchReadDataComplete( - const TActorContext& ctx, size_t amount) noexcept = 0; - + /* returns false if the protocol should continue to read data */ + virtual bool CatchReadDataComplete( + const TActorContext& ctx, size_t amount) noexcept = 0; + virtual void CatchReadDataClosed() noexcept = 0; -}; - -class TReadDataProtocolImpl { -public: - template <typename TOrigActor> - void ReadData( - IReadDataCatch* catcher, - TOrigActor* orig, - const TActorContext& ctx, - NInterconnect::TStreamSocket* socket, - char* data, - size_t len) noexcept - { - Catcher = catcher; - Socket = socket; - Data = data; - Len = len; - Filled = 0; - Cancelled = false; - - orig->template AddMsgProtocol<TEvSocketReadyRead>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TReadDataProtocolImpl, - &TReadDataProtocolImpl::ProtocolFunc>); - - TryAgain(ctx); - } - - void ResetReadBuf(char* data, size_t len) { - Data = data; - Len = len; - Filled = 0; - } - - void CancelReadData(const TActorContext& ctx) noexcept; - -private: - void ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept; - - void TryAgain(const TActorContext& ctx) noexcept; - - IReadDataCatch* Catcher; - NInterconnect::TStreamSocket* Socket; - char* Data; - size_t Len; - size_t Filled; - bool Cancelled; -}; - -template <typename TDerived = void> -class TReadDataProtocol: virtual public TReadDataProtocolImpl { -public: - template <typename TOrigActor> - void ReadData( - TOrigActor* orig, - const TActorContext& ctx, - NInterconnect::TStreamSocket* socket, - char* data, - size_t len) noexcept - { - CatchRelay.Catcher = static_cast<TDerived*>(orig); - TReadDataProtocolImpl::ReadData( - &CatchRelay, orig, ctx, socket, data, len); - } - - template <typename TOrigActor> - void ReadData( - TDerived* catcher, - TOrigActor* orig, - const TActorContext& ctx, - NInterconnect::TStreamSocket* socket, - char* data, - size_t len) noexcept - { - CatchRelay.Catcher = catcher; - TReadDataProtocolImpl::ReadData( - &CatchRelay, orig, ctx, socket, data, len); - } - - using TReadDataProtocolImpl::ResetReadBuf; - -private: - struct TCatch: public IReadDataCatch { - TDerived* Catcher; - +}; + +class TReadDataProtocolImpl { +public: + template <typename TOrigActor> + void ReadData( + IReadDataCatch* catcher, + TOrigActor* orig, + const TActorContext& ctx, + NInterconnect::TStreamSocket* socket, + char* data, + size_t len) noexcept + { + Catcher = catcher; + Socket = socket; + Data = data; + Len = len; + Filled = 0; + Cancelled = false; + + orig->template AddMsgProtocol<TEvSocketReadyRead>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TReadDataProtocolImpl, + &TReadDataProtocolImpl::ProtocolFunc>); + + TryAgain(ctx); + } + + void ResetReadBuf(char* data, size_t len) { + Data = data; + Len = len; + Filled = 0; + } + + void CancelReadData(const TActorContext& ctx) noexcept; + +private: + void ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept; + + void TryAgain(const TActorContext& ctx) noexcept; + + IReadDataCatch* Catcher; + NInterconnect::TStreamSocket* Socket; + char* Data; + size_t Len; + size_t Filled; + bool Cancelled; +}; + +template <typename TDerived = void> +class TReadDataProtocol: virtual public TReadDataProtocolImpl { +public: + template <typename TOrigActor> + void ReadData( + TOrigActor* orig, + const TActorContext& ctx, + NInterconnect::TStreamSocket* socket, + char* data, + size_t len) noexcept + { + CatchRelay.Catcher = static_cast<TDerived*>(orig); + TReadDataProtocolImpl::ReadData( + &CatchRelay, orig, ctx, socket, data, len); + } + + template <typename TOrigActor> + void ReadData( + TDerived* catcher, + TOrigActor* orig, + const TActorContext& ctx, + NInterconnect::TStreamSocket* socket, + char* data, + size_t len) noexcept + { + CatchRelay.Catcher = catcher; + TReadDataProtocolImpl::ReadData( + &CatchRelay, orig, ctx, socket, data, len); + } + + using TReadDataProtocolImpl::ResetReadBuf; + +private: + struct TCatch: public IReadDataCatch { + TDerived* Catcher; + virtual void CatchReadDataError(TString error) noexcept override - { + { Catcher->CatchReadDataError(error); - } - - /* returns false if the protocol should continue to read data */ - virtual bool CatchReadDataComplete( - const TActorContext& ctx, size_t amount) noexcept override - { - return Catcher->CatchReadDataComplete(ctx, amount); - } - + } + + /* returns false if the protocol should continue to read data */ + virtual bool CatchReadDataComplete( + const TActorContext& ctx, size_t amount) noexcept override + { + return Catcher->CatchReadDataComplete(ctx, amount); + } + virtual void CatchReadDataClosed() noexcept override - { + { Catcher->CatchReadDataClosed(); - } - }; - - TCatch CatchRelay; -}; - - -template<> -class TReadDataProtocol<void>: virtual public TReadDataProtocolImpl { -public: - template <typename TOrigActor> - void ReadData( - TOrigActor* orig, - const TActorContext& ctx, - NInterconnect::TStreamSocket* socket, - char* data, - size_t len) noexcept - { - ReadData(orig, orig, ctx, socket, data, len); - } - - void ResetReadBuf(char* data, size_t len); -}; - - -} + } + }; + + TCatch CatchRelay; +}; + + +template<> +class TReadDataProtocol<void>: virtual public TReadDataProtocolImpl { +public: + template <typename TOrigActor> + void ReadData( + TOrigActor* orig, + const TActorContext& ctx, + NInterconnect::TStreamSocket* socket, + char* data, + size_t len) noexcept + { + ReadData(orig, orig, ctx, socket, data, len); + } + + void ResetReadBuf(char* data, size_t len); +}; + + +} diff --git a/ydb/core/actorlib_impl/read_http_reply_protocol.cpp b/ydb/core/actorlib_impl/read_http_reply_protocol.cpp index 63ad2050c1..53bf302596 100644 --- a/ydb/core/actorlib_impl/read_http_reply_protocol.cpp +++ b/ydb/core/actorlib_impl/read_http_reply_protocol.cpp @@ -1,67 +1,67 @@ -#include "read_http_reply_protocol.h" - -#include <cstring> - -namespace NActors { - -bool TReadHTTPReplyProtocol::CatchReadDataComplete( - const TActorContext& ctx, size_t amount) noexcept -{ - Filled += amount; - Y_VERIFY(Buf.size() >= Filled); - - if (Filled < MINIMUM_HTTP_REPLY_SIZE) { - if (Buf.size() - Filled < FREE_SPACE_LOW_WATER_MARK) - { - Buf.resize(Filled + FREE_SPACE_HIGH_WATER_MARK); - } +#include "read_http_reply_protocol.h" + +#include <cstring> + +namespace NActors { + +bool TReadHTTPReplyProtocol::CatchReadDataComplete( + const TActorContext& ctx, size_t amount) noexcept +{ + Filled += amount; + Y_VERIFY(Buf.size() >= Filled); + + if (Filled < MINIMUM_HTTP_REPLY_SIZE) { + if (Buf.size() - Filled < FREE_SPACE_LOW_WATER_MARK) + { + Buf.resize(Filled + FREE_SPACE_HIGH_WATER_MARK); + } ResetReadBuf(&Buf[Filled], Buf.size() - Filled); - return false; - } - - if (Filled == Buf.size()) { - Buf.push_back(0); - } else { - Buf[Filled] = 0; - } - - /* null char ensured, it's safe to use strstr */ - auto result = strstr(Buf.data(), "\r\n\r\n"); - - if (result == nullptr) { - /* Check if there no null char in data read from a socket */ - if (strlen(Buf.data()) != Filled) { - /* got garbage from the socket */ + return false; + } + + if (Filled == Buf.size()) { + Buf.push_back(0); + } else { + Buf[Filled] = 0; + } + + /* null char ensured, it's safe to use strstr */ + auto result = strstr(Buf.data(), "\r\n\r\n"); + + if (result == nullptr) { + /* Check if there no null char in data read from a socket */ + if (strlen(Buf.data()) != Filled) { + /* got garbage from the socket */ CatchReadDataError("null char in HTTP reply"); - return true; - } - - /* Check maximum size limit */ - if (Filled >= HTTPReplySizeLimit) { + return true; + } + + /* Check maximum size limit */ + if (Filled >= HTTPReplySizeLimit) { CatchReadDataError("HTTP reply is too large"); - return true; - } - - if (Buf.size() - Filled < FREE_SPACE_LOW_WATER_MARK) - { - Buf.resize(Filled + FREE_SPACE_HIGH_WATER_MARK); - } - + return true; + } + + if (Buf.size() - Filled < FREE_SPACE_LOW_WATER_MARK) + { + Buf.resize(Filled + FREE_SPACE_HIGH_WATER_MARK); + } + ResetReadBuf(&Buf[Filled], Buf.size() - Filled); - return false; - } - - const size_t messageSize = result - Buf.data() + 4; - Buf.resize(Filled + 1); - CatchHTTPReply(ctx, std::move(Buf), messageSize); - return true; -} - - + return false; + } + + const size_t messageSize = result - Buf.data() + 4; + Buf.resize(Filled + 1); + CatchHTTPReply(ctx, std::move(Buf), messageSize); + return true; +} + + void TReadHTTPReplyProtocol::CatchReadDataClosed() noexcept -{ +{ CatchReadDataError("Socket closed prematurely"); -} - -} +} + +} diff --git a/ydb/core/actorlib_impl/read_http_reply_protocol.h b/ydb/core/actorlib_impl/read_http_reply_protocol.h index 6050a3edfe..c878d24950 100644 --- a/ydb/core/actorlib_impl/read_http_reply_protocol.h +++ b/ydb/core/actorlib_impl/read_http_reply_protocol.h @@ -1,67 +1,67 @@ -#pragma once - -#include "read_data_protocol.h" - -namespace NActors { - -class TReadHTTPReplyProtocol - : public TReadDataProtocol<TReadHTTPReplyProtocol> -{ -public: - template <typename TOrigActor> - void ReadHTTPReply( - TOrigActor* orig, - const TActorContext& ctx, - TIntrusivePtr<NInterconnect::TStreamSocket> socket, +#pragma once + +#include "read_data_protocol.h" + +namespace NActors { + +class TReadHTTPReplyProtocol + : public TReadDataProtocol<TReadHTTPReplyProtocol> +{ +public: + template <typename TOrigActor> + void ReadHTTPReply( + TOrigActor* orig, + const TActorContext& ctx, + TIntrusivePtr<NInterconnect::TStreamSocket> socket, TVector<char> readyBuf = TVector<char>() - ) noexcept - { - Socket = socket; - if (readyBuf.size() == 0) { - Filled = 0; - Buf.resize(FREE_SPACE_HIGH_WATER_MARK); - } else { - Buf = std::move(readyBuf); - Filled = Buf.size(); - Buf.resize(FREE_SPACE_HIGH_WATER_MARK + Buf.size()); - if (CatchReadDataComplete(ctx, 0)) { - return; - } - } - ReadData<TOrigActor>(this, orig, ctx, Socket.Get(), - Buf.data(), Buf.size() - Filled); - } - + ) noexcept + { + Socket = socket; + if (readyBuf.size() == 0) { + Filled = 0; + Buf.resize(FREE_SPACE_HIGH_WATER_MARK); + } else { + Buf = std::move(readyBuf); + Filled = Buf.size(); + Buf.resize(FREE_SPACE_HIGH_WATER_MARK + Buf.size()); + if (CatchReadDataComplete(ctx, 0)) { + return; + } + } + ReadData<TOrigActor>(this, orig, ctx, Socket.Get(), + Buf.data(), Buf.size() - Filled); + } + virtual void CatchReadDataError(TString error) noexcept = 0; - - virtual void CatchHTTPReply( - const TActorContext& ctx, + + virtual void CatchHTTPReply( + const TActorContext& ctx, TVector<char> buf, - size_t httpMessageSize) noexcept = 0; - - size_t HTTPReplySizeLimit = DEFAULT_MAXIMUM_HTTP_REPLY_SIZE; - - virtual ~TReadHTTPReplyProtocol() = default; - -private: - friend class TReadDataProtocol<TReadHTTPReplyProtocol>; - - bool CatchReadDataComplete( - const TActorContext& ctx, size_t amount) noexcept; - + size_t httpMessageSize) noexcept = 0; + + size_t HTTPReplySizeLimit = DEFAULT_MAXIMUM_HTTP_REPLY_SIZE; + + virtual ~TReadHTTPReplyProtocol() = default; + +private: + friend class TReadDataProtocol<TReadHTTPReplyProtocol>; + + bool CatchReadDataComplete( + const TActorContext& ctx, size_t amount) noexcept; + void CatchReadDataClosed() noexcept; - - - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + + + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; TVector<char> Buf; - size_t Filled = 0; - - static constexpr size_t FREE_SPACE_LOW_WATER_MARK = 1024; - static constexpr size_t FREE_SPACE_HIGH_WATER_MARK = 2048; - - /* HTTP/X.X XXX X\r\n\r\n */ - static constexpr size_t MINIMUM_HTTP_REPLY_SIZE = 18; - static constexpr size_t DEFAULT_MAXIMUM_HTTP_REPLY_SIZE = 65536; -}; - -} + size_t Filled = 0; + + static constexpr size_t FREE_SPACE_LOW_WATER_MARK = 1024; + static constexpr size_t FREE_SPACE_HIGH_WATER_MARK = 2048; + + /* HTTP/X.X XXX X\r\n\r\n */ + static constexpr size_t MINIMUM_HTTP_REPLY_SIZE = 18; + static constexpr size_t DEFAULT_MAXIMUM_HTTP_REPLY_SIZE = 65536; +}; + +} diff --git a/ydb/core/actorlib_impl/send_data_protocol.cpp b/ydb/core/actorlib_impl/send_data_protocol.cpp index 7d79c00039..4241c19617 100644 --- a/ydb/core/actorlib_impl/send_data_protocol.cpp +++ b/ydb/core/actorlib_impl/send_data_protocol.cpp @@ -1,63 +1,63 @@ -#include "send_data_protocol.h" - +#include "send_data_protocol.h" + #include <ydb/core/base/appdata.h> -#include <util/system/error.h> -#include <util/system/yassert.h> - -namespace NActors { - -void TSendDataProtocol::ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept -{ - if (Cancelled) { - return; - } - - switch (ev->GetTypeRewrite()) { - case TEvSocketReadyWrite::EventType: - TryAgain(ctx); - break; - - default: - Y_FAIL("Unknown message type dispatched"); - } -} - - -static TDelegate NotifyReadyWrite( - const TIntrusivePtr<TSharedDescriptor>& FDPtr, - const TActorContext& ctx) -{ - Y_UNUSED(FDPtr); - return [=]() { ctx.Send(ctx.SelfID, new TEvSocketReadyWrite); }; -} - - -void TSendDataProtocol::TryAgain(const TActorContext& ctx) noexcept { - int sendResult; - for (;;) { - sendResult = Socket->Send(Data, Len); - - if (sendResult > 0) { - Y_VERIFY(Len >= (size_t)sendResult); - MemLogPrintF("TSendDataProtocol::TryAgain, sent %d bytes", - sendResult); - - Data += sendResult; - Len -= sendResult; - if (Len == 0) { - CatchSendDataComplete(ctx); - return; - } - } else { - if (sendResult < 0 && sendResult == -EINTR) - continue; - else - break; - } - } - +#include <util/system/error.h> +#include <util/system/yassert.h> + +namespace NActors { + +void TSendDataProtocol::ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept +{ + if (Cancelled) { + return; + } + + switch (ev->GetTypeRewrite()) { + case TEvSocketReadyWrite::EventType: + TryAgain(ctx); + break; + + default: + Y_FAIL("Unknown message type dispatched"); + } +} + + +static TDelegate NotifyReadyWrite( + const TIntrusivePtr<TSharedDescriptor>& FDPtr, + const TActorContext& ctx) +{ + Y_UNUSED(FDPtr); + return [=]() { ctx.Send(ctx.SelfID, new TEvSocketReadyWrite); }; +} + + +void TSendDataProtocol::TryAgain(const TActorContext& ctx) noexcept { + int sendResult; + for (;;) { + sendResult = Socket->Send(Data, Len); + + if (sendResult > 0) { + Y_VERIFY(Len >= (size_t)sendResult); + MemLogPrintF("TSendDataProtocol::TryAgain, sent %d bytes", + sendResult); + + Data += sendResult; + Len -= sendResult; + if (Len == 0) { + CatchSendDataComplete(ctx); + return; + } + } else { + if (sendResult < 0 && sendResult == -EINTR) + continue; + else + break; + } + } + if (-sendResult == EAGAIN || -sendResult == EWOULDBLOCK) { IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); poller->StartWrite(Socket, @@ -65,44 +65,44 @@ void TSendDataProtocol::TryAgain(const TActorContext& ctx) noexcept { return; } - switch (-sendResult) { - case ECONNRESET: + switch (-sendResult) { + case ECONNRESET: CatchSendDataError("Connection reset by peer"); - return; - - case EPIPE: + return; + + case EPIPE: CatchSendDataError("Connection is closed"); - return; - - case 0: - /* Not realy sure what to do with 0 result, assume socket is closed */ + return; + + case 0: + /* Not realy sure what to do with 0 result, assume socket is closed */ CatchSendDataError("Connection is closed"); - return; - - default: - { - char buf[1024]; - LastSystemErrorText(buf, 1024, -sendResult); + return; + + default: + { + char buf[1024]; + LastSystemErrorText(buf, 1024, -sendResult); CatchSendDataError(TString("Socker error: ") + buf); - return; - } - - case EBADF: - case EFAULT: - case EINVAL: - case ENOTCONN: - case ENOTSOCK: - case EOPNOTSUPP: - { - Y_FAIL("Very bad socket error"); - } - } -} - + return; + } + + case EBADF: + case EFAULT: + case EINVAL: + case ENOTCONN: + case ENOTSOCK: + case EOPNOTSUPP: + { + Y_FAIL("Very bad socket error"); + } + } +} + void TSendDataProtocol::CancelSendData(const TActorContext& /*ctx*/) noexcept { - Cancelled = true; + Cancelled = true; // IPoller* poller = NKikimr::AppData(ctx)->PollerThreads.Get(); // poller->CancelWrite(Socket); -} - -} +} + +} diff --git a/ydb/core/actorlib_impl/send_data_protocol.h b/ydb/core/actorlib_impl/send_data_protocol.h index 2c22936034..8c851cc0d0 100644 --- a/ydb/core/actorlib_impl/send_data_protocol.h +++ b/ydb/core/actorlib_impl/send_data_protocol.h @@ -1,55 +1,55 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/interconnect/events_local.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/interconnect_stream.h> - -namespace NActors { - -class TSendDataProtocol { -public: - template <typename TOrigActor> - void SendData( - TOrigActor* orig, - const TActorContext& ctx, - NInterconnect::TStreamSocket* socket, - const char* data, - size_t len) noexcept - { - Socket = socket; - Data = data; - Len = len; - Cancelled = false; - - orig->template AddMsgProtocol<TEvSocketReadyWrite>( - &TOrigActor::template CallProtocolStateFunc< - TOrigActor, - TSendDataProtocol, - &TSendDataProtocol::ProtocolFunc>); - - TryAgain(ctx); - } - + +namespace NActors { + +class TSendDataProtocol { +public: + template <typename TOrigActor> + void SendData( + TOrigActor* orig, + const TActorContext& ctx, + NInterconnect::TStreamSocket* socket, + const char* data, + size_t len) noexcept + { + Socket = socket; + Data = data; + Len = len; + Cancelled = false; + + orig->template AddMsgProtocol<TEvSocketReadyWrite>( + &TOrigActor::template CallProtocolStateFunc< + TOrigActor, + TSendDataProtocol, + &TSendDataProtocol::ProtocolFunc>); + + TryAgain(ctx); + } + virtual void CatchSendDataError(TString error) noexcept = 0; - - virtual void CatchSendDataComplete( - const TActorContext& ctx) noexcept = 0; - - void CancelSendData(const TActorContext& ctx) noexcept; - - virtual ~TSendDataProtocol() = default; - -private: - void ProtocolFunc( - TAutoPtr<NActors::IEventHandle>& ev, - const TActorContext& ctx) noexcept; - - void TryAgain(const TActorContext& ctx) noexcept; - - NInterconnect::TStreamSocket* Socket; - const char* Data; - size_t Len; - bool Cancelled; -}; - -} + + virtual void CatchSendDataComplete( + const TActorContext& ctx) noexcept = 0; + + void CancelSendData(const TActorContext& ctx) noexcept; + + virtual ~TSendDataProtocol() = default; + +private: + void ProtocolFunc( + TAutoPtr<NActors::IEventHandle>& ev, + const TActorContext& ctx) noexcept; + + void TryAgain(const TActorContext& ctx) noexcept; + + NInterconnect::TStreamSocket* Socket; + const char* Data; + size_t Len; + bool Cancelled; +}; + +} diff --git a/ydb/core/actorlib_impl/test_protocols_ut.cpp b/ydb/core/actorlib_impl/test_protocols_ut.cpp index 2e622406a0..b4a76ccb8a 100644 --- a/ydb/core/actorlib_impl/test_protocols_ut.cpp +++ b/ydb/core/actorlib_impl/test_protocols_ut.cpp @@ -7,371 +7,371 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/actors/core/actor_bootstrapped.h> -#include <util/stream/null.h> +#include <util/stream/null.h> #include <util/system/event.h> - -#include "proto_ready_actor.h" -#include "name_service_client_protocol.h" -#include "connect_socket_protocol.h" -#include "send_data_protocol.h" -#include "read_http_reply_protocol.h" -#include "http_request_protocol.h" - -using namespace NActors; - -auto& Ctest = Cnull; -//auto& Ctest = Cerr; - -namespace { + +#include "proto_ready_actor.h" +#include "name_service_client_protocol.h" +#include "connect_socket_protocol.h" +#include "send_data_protocol.h" +#include "read_http_reply_protocol.h" +#include "http_request_protocol.h" + +using namespace NActors; + +auto& Ctest = Cnull; +//auto& Ctest = Cerr; + +namespace { class TFailingMtpQueue: public TSimpleThreadPool { - private: - bool FailOnAdd_ = false; - public: - void SetFailOnAdd(bool fail = true) { - FailOnAdd_ = fail; - } + private: + bool FailOnAdd_ = false; + public: + void SetFailOnAdd(bool fail = true) { + FailOnAdd_ = fail; + } [[nodiscard]] bool Add(IObjectInQueue* pObj) override { - if (FailOnAdd_) { - return false; - } - + if (FailOnAdd_) { + return false; + } + return TSimpleThreadPool::Add(pObj); - } - TFailingMtpQueue() = default; + } + TFailingMtpQueue() = default; TFailingMtpQueue(IThreadFactory* pool) : TSimpleThreadPool(pool) - { - } - }; - - using TFailingServerMtpQueue = + { + } + }; + + using TFailingServerMtpQueue = TThreadPoolBinder<TFailingMtpQueue, THttpServer::ICallBack>; - - class THTTP200OkServer: public THttpServer::ICallBack { - class TRequest: public THttpClientRequestEx { - public: - inline TRequest(THTTP200OkServer* parent) - : Parent_(parent) - { - } - - bool Reply(void* /*tsr*/) override { - if (!ProcessHeaders()) { + + class THTTP200OkServer: public THttpServer::ICallBack { + class TRequest: public THttpClientRequestEx { + public: + inline TRequest(THTTP200OkServer* parent) + : Parent_(parent) + { + } + + bool Reply(void* /*tsr*/) override { + if (!ProcessHeaders()) { return true; - } - + } + if (strncmp(RequestString.data(), "GET /hosts HTTP/1.", 18) == 0) { TString list = Sprintf("[\"localhost\"]"); - Output() << "HTTP/1.1 200 Ok\r\n"; - Output() << "Connection: close\r\n"; + Output() << "HTTP/1.1 200 Ok\r\n"; + Output() << "Connection: close\r\n"; Output() << "Content-Length: " << list.size() << "\r\n"; - Output() << "\r\n"; - Output() << list; - return true; - } - - Output() << "HTTP/1.1 200 Ok\r\n"; + Output() << "\r\n"; + Output() << list; + return true; + } + + Output() << "HTTP/1.1 200 Ok\r\n"; if (Buf.Size()) { Output() << "Content-Length: " << Buf.Size() << "\r\n\r\n"; Output().Write(Buf.AsCharPtr(), Buf.Size()); - } else { + } else { Output() << "Content-Length: " << (Parent_->Res_).size() - << "\r\n\r\n"; - Output() << Parent_->Res_; - } - Output().Finish(); - - return true; - } - - private: - THTTP200OkServer* Parent_ = nullptr; - }; - - public: + << "\r\n\r\n"; + Output() << Parent_->Res_; + } + Output().Finish(); + + return true; + } + + private: + THTTP200OkServer* Parent_ = nullptr; + }; + + public: inline THTTP200OkServer(TString res) - : Res_(std::move(res)) - { - } - - TClientRequest* CreateClient() override { - return new TRequest(this); - } - - private: + : Res_(std::move(res)) + { + } + + TClientRequest* CreateClient() override { + return new TRequest(this); + } + + private: TString Res_; - }; -} - + }; +} + Y_UNIT_TEST_SUITE(TestProtocols) { - class TResolveTester - : public TProtoReadyActor<TActorBootstrapped, TResolveTester> - , public TResolveClientProtocol - { - public: - void Bootstrap(const TActorContext& ctx) { - Become(&TThis::DefaultFunc); - SendResolveMessage(this, ctx, "localhost", 8080); - } - - STFUNC(DefaultFunc) { - Y_UNUSED(ctx); - Y_UNUSED(ev); - } - - void CatchHostAddress( - const TActorContext& ctx, - NAddr::IRemoteAddrPtr address) noexcept override - { - Address = std::move(address); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - void CatchResolveError( + class TResolveTester + : public TProtoReadyActor<TActorBootstrapped, TResolveTester> + , public TResolveClientProtocol + { + public: + void Bootstrap(const TActorContext& ctx) { + Become(&TThis::DefaultFunc); + SendResolveMessage(this, ctx, "localhost", 8080); + } + + STFUNC(DefaultFunc) { + Y_UNUSED(ctx); + Y_UNUSED(ev); + } + + void CatchHostAddress( + const TActorContext& ctx, + NAddr::IRemoteAddrPtr address) noexcept override + { + Address = std::move(address); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + void CatchResolveError( const TActorContext& ctx, TString error) noexcept override - { - Error = std::move(error); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - NAddr::IRemoteAddrPtr Address; + { + Error = std::move(error); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + NAddr::IRemoteAddrPtr Address; TString Error; TActorId Edge; - }; - - + }; + + Y_UNIT_TEST(TestResolveProtocol) { - for (size_t i = 0; i < 10; ++i) { + for (size_t i = 0; i < 10; ++i) { TTestBasicRuntime runtime(2); runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); - - auto tester = new TResolveTester; - tester->Edge = runtime.AllocateEdgeActor(0); - runtime.Register(tester, 0); - - TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); - - if (NAddr::PrintHostAndPort(*tester->Address) == - "[::1]:8080") { - return; - } - } - UNIT_FAIL("Could not get ip address of yandex.ru in 10 tries"); - } - - - class TConnectTester - : public TProtoReadyActor<TActorBootstrapped, TConnectTester> - , public TResolveClientProtocol - , public TConnectSocketProtocol - { - public: - void Bootstrap(const TActorContext& ctx) { - Become(&TThis::DefaultFunc); - SendResolveMessage(this, ctx, "localhost", Port); - } - - STFUNC(DefaultFunc) { - Y_UNUSED(ctx); - Y_UNUSED(ev); - } - - void CatchHostAddress( - const TActorContext& ctx, - NAddr::IRemoteAddrPtr address) noexcept override - { + + auto tester = new TResolveTester; + tester->Edge = runtime.AllocateEdgeActor(0); + runtime.Register(tester, 0); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); + + if (NAddr::PrintHostAndPort(*tester->Address) == + "[::1]:8080") { + return; + } + } + UNIT_FAIL("Could not get ip address of yandex.ru in 10 tries"); + } + + + class TConnectTester + : public TProtoReadyActor<TActorBootstrapped, TConnectTester> + , public TResolveClientProtocol + , public TConnectSocketProtocol + { + public: + void Bootstrap(const TActorContext& ctx) { + Become(&TThis::DefaultFunc); + SendResolveMessage(this, ctx, "localhost", Port); + } + + STFUNC(DefaultFunc) { + Y_UNUSED(ctx); + Y_UNUSED(ev); + } + + void CatchHostAddress( + const TActorContext& ctx, + NAddr::IRemoteAddrPtr address) noexcept override + { Address = std::move(address); - ConnectSocket(this, ctx, Address); - } - - void CatchResolveError( + ConnectSocket(this, ctx, Address); + } + + void CatchResolveError( const TActorContext& ctx, TString error) noexcept override - { - Error = std::move(error); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - void CatchConnectError( + { + Error = std::move(error); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + void CatchConnectError( const TActorContext& ctx, TString error) noexcept override - { - Error = std::move(error); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - void CatchConnectedSocket( - const TActorContext& ctx, - TIntrusivePtr<NInterconnect::TStreamSocket> socket - ) noexcept override - { - Socket = std::move(socket); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - - NAddr::IRemoteAddrRef Address; + { + Error = std::move(error); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + void CatchConnectedSocket( + const TActorContext& ctx, + TIntrusivePtr<NInterconnect::TStreamSocket> socket + ) noexcept override + { + Socket = std::move(socket); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + + NAddr::IRemoteAddrRef Address; TString Error; TActorId Edge; - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; - ui64 Port; - }; - - + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + ui64 Port; + }; + + Y_UNIT_TEST(TestConnectProtocol) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - THTTP200OkServer serverImpl(""); - - THttpServer::TOptions options(port); - THttpServer::TMtpQueueRef mainWorkers = + TPortManager portManager; + const ui16 port = portManager.GetPort(); + THTTP200OkServer serverImpl(""); + + THttpServer::TOptions options(port); + THttpServer::TMtpQueueRef mainWorkers = new TFailingServerMtpQueue(&serverImpl, SystemThreadFactory()); - THttpServer::TMtpQueueRef failWorkers = + THttpServer::TMtpQueueRef failWorkers = new TThreadPool(SystemThreadFactory()); - - THttpServer server(&serverImpl, mainWorkers, failWorkers, options); - UNIT_ASSERT(server.Start()); - + + THttpServer server(&serverImpl, mainWorkers, failWorkers, options); + UNIT_ASSERT(server.Start()); + TTestBasicRuntime runtime(2); runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); - - auto tester = new TConnectTester; - tester->Port = port; - tester->Edge = runtime.AllocateEdgeActor(0); - runtime.Register(tester, 0); - - TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); - - UNIT_ASSERT_EQUAL(NAddr::PrintHostAndPort(*tester->Address), - Sprintf("[::1]:%u", port)); - UNIT_ASSERT(tester->Socket.Get() != nullptr); - UNIT_ASSERT(*tester->Socket != -1); - } - - - class THTTPTester - : public TProtoReadyActor<TActorBootstrapped, THTTPTester> - , public TResolveClientProtocol - , public TConnectSocketProtocol - , public TSendDataProtocol - , public TReadHTTPReplyProtocol - { - private: - friend class TActorBootstrapped<THTTPTester>; - friend class TResolveClientProtocol; - friend class TConnectSocketProtocol; - friend class TSendDataProtocol; - friend class TReadHTTPReplyProtocol; - - void Bootstrap(const TActorContext& ctx) { - Become(&TThis::DefaultFunc); - SendResolveMessage(this, ctx, "localhost", Port); - } - - STFUNC(DefaultFunc) { - Y_UNUSED(ctx); - Y_UNUSED(ev); - } - - void CatchHostAddress( - const TActorContext& ctx, - NAddr::IRemoteAddrPtr address) noexcept override - { + + auto tester = new TConnectTester; + tester->Port = port; + tester->Edge = runtime.AllocateEdgeActor(0); + runtime.Register(tester, 0); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); + + UNIT_ASSERT_EQUAL(NAddr::PrintHostAndPort(*tester->Address), + Sprintf("[::1]:%u", port)); + UNIT_ASSERT(tester->Socket.Get() != nullptr); + UNIT_ASSERT(*tester->Socket != -1); + } + + + class THTTPTester + : public TProtoReadyActor<TActorBootstrapped, THTTPTester> + , public TResolveClientProtocol + , public TConnectSocketProtocol + , public TSendDataProtocol + , public TReadHTTPReplyProtocol + { + private: + friend class TActorBootstrapped<THTTPTester>; + friend class TResolveClientProtocol; + friend class TConnectSocketProtocol; + friend class TSendDataProtocol; + friend class TReadHTTPReplyProtocol; + + void Bootstrap(const TActorContext& ctx) { + Become(&TThis::DefaultFunc); + SendResolveMessage(this, ctx, "localhost", Port); + } + + STFUNC(DefaultFunc) { + Y_UNUSED(ctx); + Y_UNUSED(ev); + } + + void CatchHostAddress( + const TActorContext& ctx, + NAddr::IRemoteAddrPtr address) noexcept override + { ConnectSocket(this, ctx, std::move(address)); - } - - void CatchResolveError( + } + + void CatchResolveError( const TActorContext& ctx, TString error) noexcept override - { - Error = std::move(error); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - void CatchConnectError( + { + Error = std::move(error); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + void CatchConnectError( const TActorContext& ctx, TString error) noexcept override - { - Error = std::move(error); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - void CatchConnectedSocket( - const TActorContext& ctx, - TIntrusivePtr<NInterconnect::TStreamSocket> socket - ) noexcept override - { - Socket = std::move(socket); - const char* request = - "GET / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Connection: closer\r\n" - "\r\n"; - - SendData(this, ctx, Socket.Get(), request, strlen(request)); - } - + { + Error = std::move(error); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + void CatchConnectedSocket( + const TActorContext& ctx, + TIntrusivePtr<NInterconnect::TStreamSocket> socket + ) noexcept override + { + Socket = std::move(socket); + const char* request = + "GET / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Connection: closer\r\n" + "\r\n"; + + SendData(this, ctx, Socket.Get(), request, strlen(request)); + } + void CatchSendDataError(TString error) noexcept override - { - Error = std::move(error); + { + Error = std::move(error); Send(Edge, new TEvents::TEvWakeup); - } - + } + void CatchReadDataError(TString error) noexcept override - { - Error = std::move(error); + { + Error = std::move(error); Send(Edge, new TEvents::TEvWakeup); - } - - void CatchSendDataComplete(const TActorContext& ctx) noexcept override { - ReadHTTPReply<THTTPTester>(this, ctx, Socket); - } - - void CatchHTTPReply( - const TActorContext& ctx, + } + + void CatchSendDataComplete(const TActorContext& ctx) noexcept override { + ReadHTTPReply<THTTPTester>(this, ctx, Socket); + } + + void CatchHTTPReply( + const TActorContext& ctx, TVector<char> buf, - size_t httpMessageSize) noexcept override - { - Y_UNUSED(httpMessageSize); - Data.assign(buf.begin(), buf.end()); - ctx.Send(Edge, new TEvents::TEvWakeup); - } - - public: + size_t httpMessageSize) noexcept override + { + Y_UNUSED(httpMessageSize); + Data.assign(buf.begin(), buf.end()); + ctx.Send(Edge, new TEvents::TEvWakeup); + } + + public: TString Error; TString Data; TActorId Edge; - ui16 Port; - - private: - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; - }; - - + ui16 Port; + + private: + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + }; + + Y_UNIT_TEST(TestHTTPCollected) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - THTTP200OkServer serverImpl(""); - - THttpServer::TOptions options(port); - THttpServer::TMtpQueueRef mainWorkers = + TPortManager portManager; + const ui16 port = portManager.GetPort(); + THTTP200OkServer serverImpl(""); + + THttpServer::TOptions options(port); + THttpServer::TMtpQueueRef mainWorkers = new TFailingServerMtpQueue(&serverImpl, SystemThreadFactory()); - THttpServer::TMtpQueueRef failWorkers = + THttpServer::TMtpQueueRef failWorkers = new TThreadPool(SystemThreadFactory()); - - THttpServer server(&serverImpl, mainWorkers, failWorkers, options); - UNIT_ASSERT(server.Start()); - + + THttpServer server(&serverImpl, mainWorkers, failWorkers, options); + UNIT_ASSERT(server.Start()); + TTestBasicRuntime runtime(2); runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); - - auto tester = new THTTPTester; - tester->Edge = runtime.AllocateEdgeActor(0); - tester->Port = port; - runtime.Register(tester, 0); - - TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); - } - + + auto tester = new THTTPTester; + tester->Edge = runtime.AllocateEdgeActor(0); + tester->Port = port; + runtime.Register(tester, 0); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); + } + /** * Server that replies to a single request very very slowly @@ -447,289 +447,289 @@ Y_UNIT_TEST_SUITE(TestProtocols) { } - class THTTPRequestTester - : public TProtoReadyActor<TActorBootstrapped, THTTPRequestTester> - , public THTTPRequestProtocol<THTTPRequestTester> - , public TReadDataProtocol<THTTPRequestTester> - { - friend class TReadDataProtocol<THTTPRequestTester>; - friend class TActorBootstrapped<THTTPRequestTester>; - friend class THTTPRequestProtocol<THTTPRequestTester>; - - enum class EState { - READ_PROXY_LIST, - EXPECT_CONTINUE, - WRITE_DATA_COMPLETE, - }; - - EState State = EState::READ_PROXY_LIST; - - private: - void Bootstrap(const TActorContext& ctx) { - Become(&TThis::DefaultFunc); - HTTPRequest(this, ctx, - "localhost", - "GET /hosts", - "Accept: application/json\r\n", - Port); - } - - STFUNC(DefaultFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvHTTPSendContent, YetMoreContent); - } - } - - void YetMoreContent(TEvHTTPSendContent::TPtr& ev, - const TActorContext& ctx) noexcept - { - if (State != EState::WRITE_DATA_COMPLETE) - Y_FAIL("Stream is not ready to write data"); - - TEvHTTPSendContent const* msg = ev->Get(); - HTTPWriteContent(this, ctx, msg->Data, msg->Len,msg->Last); - } - + class THTTPRequestTester + : public TProtoReadyActor<TActorBootstrapped, THTTPRequestTester> + , public THTTPRequestProtocol<THTTPRequestTester> + , public TReadDataProtocol<THTTPRequestTester> + { + friend class TReadDataProtocol<THTTPRequestTester>; + friend class TActorBootstrapped<THTTPRequestTester>; + friend class THTTPRequestProtocol<THTTPRequestTester>; + + enum class EState { + READ_PROXY_LIST, + EXPECT_CONTINUE, + WRITE_DATA_COMPLETE, + }; + + EState State = EState::READ_PROXY_LIST; + + private: + void Bootstrap(const TActorContext& ctx) { + Become(&TThis::DefaultFunc); + HTTPRequest(this, ctx, + "localhost", + "GET /hosts", + "Accept: application/json\r\n", + Port); + } + + STFUNC(DefaultFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvHTTPSendContent, YetMoreContent); + } + } + + void YetMoreContent(TEvHTTPSendContent::TPtr& ev, + const TActorContext& ctx) noexcept + { + if (State != EState::WRITE_DATA_COMPLETE) + Y_FAIL("Stream is not ready to write data"); + + TEvHTTPSendContent const* msg = ev->Get(); + HTTPWriteContent(this, ctx, msg->Data, msg->Len,msg->Last); + } + void CatchHTTPRequestError(TString error) noexcept override - { - Error = std::move(error); - exit(1); - } - - - void CatchHTTPReply( - const TActorContext& ctx, + { + Error = std::move(error); + exit(1); + } + + + void CatchHTTPReply( + const TActorContext& ctx, TVector<char> buf, - size_t httpMessageSize) noexcept override - { - Ctest << "HTTP message size: " << httpMessageSize << Endl; - Ctest << "Total read size: " << buf.size() - 1 << Endl; - TMemoryInput bufStream(buf.data(), httpMessageSize); - THttpInput httpMsg(&bufStream); - httpMsg.Headers().OutTo(&Ctest); - - switch (State) { - case EState::READ_PROXY_LIST: - CatchProxyList(ctx, std::move(buf), httpMessageSize, httpMsg); - break; - case EState::EXPECT_CONTINUE: - CatchWriteData(ctx, httpMsg); - break; - case EState::WRITE_DATA_COMPLETE: - HTTPWriteComplete(ctx, httpMsg); - break; - } - } - - void CatchProxyList( - const TActorContext& ctx, + size_t httpMessageSize) noexcept override + { + Ctest << "HTTP message size: " << httpMessageSize << Endl; + Ctest << "Total read size: " << buf.size() - 1 << Endl; + TMemoryInput bufStream(buf.data(), httpMessageSize); + THttpInput httpMsg(&bufStream); + httpMsg.Headers().OutTo(&Ctest); + + switch (State) { + case EState::READ_PROXY_LIST: + CatchProxyList(ctx, std::move(buf), httpMessageSize, httpMsg); + break; + case EState::EXPECT_CONTINUE: + CatchWriteData(ctx, httpMsg); + break; + case EState::WRITE_DATA_COMPLETE: + HTTPWriteComplete(ctx, httpMsg); + break; + } + } + + void CatchProxyList( + const TActorContext& ctx, TVector<char> buf, - size_t httpMessageSize, - THttpInput& httpMsg) noexcept - { - do { - auto status = ParseHttpRetCode(httpMsg.FirstLine()); - - if (status < 200 || status >= 300) { - Error = "bad http reply status"; - break; - } - - if (!httpMsg.HasContent()) { - Error = "reply has no content"; - break; - } - - ui64 value; - if (httpMsg.GetContentLength(value)) { - Ctest << "Content length is: " << value << Endl; - value -= buf.size() - 1 - httpMessageSize; - Ctest << "Yet to read: " << value << Endl; - Buf = std::move(buf); - Buf.erase(Buf.begin(), Buf.begin() + httpMessageSize); - Buf.pop_back(); - if (value == 0) { + size_t httpMessageSize, + THttpInput& httpMsg) noexcept + { + do { + auto status = ParseHttpRetCode(httpMsg.FirstLine()); + + if (status < 200 || status >= 300) { + Error = "bad http reply status"; + break; + } + + if (!httpMsg.HasContent()) { + Error = "reply has no content"; + break; + } + + ui64 value; + if (httpMsg.GetContentLength(value)) { + Ctest << "Content length is: " << value << Endl; + value -= buf.size() - 1 - httpMessageSize; + Ctest << "Yet to read: " << value << Endl; + Buf = std::move(buf); + Buf.erase(Buf.begin(), Buf.begin() + httpMessageSize); + Buf.pop_back(); + if (value == 0) { CatchHTTPContent(ctx, TVector<char>()); - return; - } - HTTPReadContent(this, ctx, value); - return; - } else { - } - } while (0); - Ctest << "Got an error: " << Error << Endl; - exit(1); - } - - - void CatchWriteData( - const TActorContext& ctx, - THttpInput& httpMsg) noexcept - { - do { - auto status = ParseHttpRetCode(httpMsg.FirstLine()); - - if (status < 100 || status >= 200) { - Error = "bad http reply status"; - break; - } - - if (httpMsg.HasContent()) { - Error = "reply must not have content"; - break; - } - - Ctest << "Ready to send PUT content" << Endl; - - auto msg = new TEvHTTPStreamStatus; - msg->Status = TEvHTTPStreamStatus::READY; - ctx.Send(Edge, msg); - State = EState::WRITE_DATA_COMPLETE; - HTTPExpectReply(this, ctx); - return; - } while (0); - Ctest << "Got an error: " << Error << Endl; - exit(1); - } - - void CatchHTTPContent( + return; + } + HTTPReadContent(this, ctx, value); + return; + } else { + } + } while (0); + Ctest << "Got an error: " << Error << Endl; + exit(1); + } + + + void CatchWriteData( + const TActorContext& ctx, + THttpInput& httpMsg) noexcept + { + do { + auto status = ParseHttpRetCode(httpMsg.FirstLine()); + + if (status < 100 || status >= 200) { + Error = "bad http reply status"; + break; + } + + if (httpMsg.HasContent()) { + Error = "reply must not have content"; + break; + } + + Ctest << "Ready to send PUT content" << Endl; + + auto msg = new TEvHTTPStreamStatus; + msg->Status = TEvHTTPStreamStatus::READY; + ctx.Send(Edge, msg); + State = EState::WRITE_DATA_COMPLETE; + HTTPExpectReply(this, ctx); + return; + } while (0); + Ctest << "Got an error: " << Error << Endl; + exit(1); + } + + void CatchHTTPContent( const TActorContext& ctx, TVector<char> buf) noexcept override - { - Buf.insert(Buf.end(), buf.begin(), buf.end()); - buf.clear(); - Ctest << "Content complete" << Endl; + { + Buf.insert(Buf.end(), buf.begin(), buf.end()); + buf.clear(); + Ctest << "Content complete" << Endl; Ctest << TString(Buf.data(), Buf.size()) << Endl; - - do { - TMemoryInput input(Buf.data(), Buf.size()); - NJson::TJsonValue jsonValue; - - if (!NJson::ReadJsonTree(&input, &jsonValue)) - break; - - if (!jsonValue.IsArray()) - break; - - auto result = jsonValue.GetArray(); - - Ctest << "First proxy is: " << result.front() << Endl; - + + do { + TMemoryInput input(Buf.data(), Buf.size()); + NJson::TJsonValue jsonValue; + + if (!NJson::ReadJsonTree(&input, &jsonValue)) + break; + + if (!jsonValue.IsArray()) + break; + + auto result = jsonValue.GetArray(); + + Ctest << "First proxy is: " << result.front() << Endl; + TString headers = - "Accept: application/json\r\n" - "Content-Type: text/tab-separated-values\r\n" - "Transfer-Encoding: chunked\r\n" - "Expect: 100-continue\r\n" - "Authorization:" - " OAuth AQAD-qJSJgO0AAAA-xdaPY4N0ECerT77peyOg4s\r\n"; - - HTTPRequest(this, ctx, - result.front().GetString(), - "PUT /api/v3/write_table?path=//tmp/agri-ws/test", - headers, - Port); - - State = EState::EXPECT_CONTINUE; - return; - } while (0); - Ctest << "Damn! Something is wrong" << Endl; - } - - void HTTPWriteComplete( - const TActorContext& ctx, - THttpInput& httpMsg) noexcept - { - do { - auto status = ParseHttpRetCode(httpMsg.FirstLine()); - - if (status < 200 || status >= 300) { - Error = Sprintf("bad http reply status: %u", status); - break; - } - - if (!httpMsg.HasContent()) { - Error = "reply must have content"; - break; - } - - Ctest << "Complete" << Endl; - ctx.Send(Edge, new TEvents::TEvWakeup); - return; - } while (0); - Ctest << "Got an error: " << Error << Endl; - exit(1); - } - - void CatchHTTPWriteComplete(const TActorContext&) noexcept override { - auto msg = new TEvHTTPStreamStatus; - msg->Status = TEvHTTPStreamStatus::READY; - State = EState::WRITE_DATA_COMPLETE; - Send(Edge, msg); - } - + "Accept: application/json\r\n" + "Content-Type: text/tab-separated-values\r\n" + "Transfer-Encoding: chunked\r\n" + "Expect: 100-continue\r\n" + "Authorization:" + " OAuth AQAD-qJSJgO0AAAA-xdaPY4N0ECerT77peyOg4s\r\n"; + + HTTPRequest(this, ctx, + result.front().GetString(), + "PUT /api/v3/write_table?path=//tmp/agri-ws/test", + headers, + Port); + + State = EState::EXPECT_CONTINUE; + return; + } while (0); + Ctest << "Damn! Something is wrong" << Endl; + } + + void HTTPWriteComplete( + const TActorContext& ctx, + THttpInput& httpMsg) noexcept + { + do { + auto status = ParseHttpRetCode(httpMsg.FirstLine()); + + if (status < 200 || status >= 300) { + Error = Sprintf("bad http reply status: %u", status); + break; + } + + if (!httpMsg.HasContent()) { + Error = "reply must have content"; + break; + } + + Ctest << "Complete" << Endl; + ctx.Send(Edge, new TEvents::TEvWakeup); + return; + } while (0); + Ctest << "Got an error: " << Error << Endl; + exit(1); + } + + void CatchHTTPWriteComplete(const TActorContext&) noexcept override { + auto msg = new TEvHTTPStreamStatus; + msg->Status = TEvHTTPStreamStatus::READY; + State = EState::WRITE_DATA_COMPLETE; + Send(Edge, msg); + } + void CatchHTTPConnectionClosed() noexcept override { - Ctest << "Connection closed prematurely" << Endl; - exit(1); - } - - public: + Ctest << "Connection closed prematurely" << Endl; + exit(1); + } + + public: TString Error; TActorId Edge; - ui16 Port; - - private: + ui16 Port; + + private: TVector<char> Buf; - }; - - + }; + + Y_UNIT_TEST(TestHTTPRequest) { - TPortManager portManager; - const ui16 port = portManager.GetPort(); - THTTP200OkServer serverImpl(""); - - THttpServer::TOptions options(port); - THttpServer::TMtpQueueRef mainWorkers = + TPortManager portManager; + const ui16 port = portManager.GetPort(); + THTTP200OkServer serverImpl(""); + + THttpServer::TOptions options(port); + THttpServer::TMtpQueueRef mainWorkers = new TFailingServerMtpQueue(&serverImpl, SystemThreadFactory()); - THttpServer::TMtpQueueRef failWorkers = + THttpServer::TMtpQueueRef failWorkers = new TThreadPool(SystemThreadFactory()); - - THttpServer server(&serverImpl, mainWorkers, failWorkers, options); - UNIT_ASSERT(server.Start()); - + + THttpServer server(&serverImpl, mainWorkers, failWorkers, options); + UNIT_ASSERT(server.Start()); + TTestBasicRuntime runtime(2); runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); - - auto tester = new THTTPRequestTester; - tester->Edge = runtime.AllocateEdgeActor(0); - tester->Port = port; - auto testerId = runtime.Register(tester, 0); - - TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEvent<TEvHTTPStreamStatus>(handle); - Ctest << "Grabed TEvHTTPStreamStatus" << Endl; - - const char* writeData = - "11\r\n" - "love=me\tleave=me\n\r\n"; - - auto msg = new TEvHTTPSendContent; - msg->Data = writeData; - msg->Len = strlen(writeData); - msg->Last = false; - runtime.Send(new IEventHandle(testerId, tester->Edge, msg), 0); - - runtime.GrabEdgeEvent<TEvHTTPStreamStatus>(handle); - - const char* moreData = - "15\r\n" - "trust=is a\tnew=glory\n\r\n" - "0\r\n\r\n"; - - msg = new TEvHTTPSendContent; - msg->Data = moreData; - msg->Len = strlen(moreData); - msg->Last = true; - runtime.Send(new IEventHandle(testerId, tester->Edge, msg), 0); - - runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); - } -} + + auto tester = new THTTPRequestTester; + tester->Edge = runtime.AllocateEdgeActor(0); + tester->Port = port; + auto testerId = runtime.Register(tester, 0); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEvent<TEvHTTPStreamStatus>(handle); + Ctest << "Grabed TEvHTTPStreamStatus" << Endl; + + const char* writeData = + "11\r\n" + "love=me\tleave=me\n\r\n"; + + auto msg = new TEvHTTPSendContent; + msg->Data = writeData; + msg->Len = strlen(writeData); + msg->Last = false; + runtime.Send(new IEventHandle(testerId, tester->Edge, msg), 0); + + runtime.GrabEdgeEvent<TEvHTTPStreamStatus>(handle); + + const char* moreData = + "15\r\n" + "trust=is a\tnew=glory\n\r\n" + "0\r\n\r\n"; + + msg = new TEvHTTPSendContent; + msg->Data = moreData; + msg->Len = strlen(moreData); + msg->Last = true; + runtime.Send(new IEventHandle(testerId, tester->Edge, msg), 0); + + runtime.GrabEdgeEvent<TEvents::TEvWakeup>(handle); + } +} diff --git a/ydb/core/actorlib_impl/ut/ya.make b/ydb/core/actorlib_impl/ut/ya.make index f76d57807a..710437b81d 100644 --- a/ydb/core/actorlib_impl/ut/ya.make +++ b/ydb/core/actorlib_impl/ut/ya.make @@ -32,7 +32,7 @@ SRCS( actor_bootstrapped_ut.cpp actor_tracker_ut.cpp test_interconnect_ut.cpp - test_protocols_ut.cpp + test_protocols_ut.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/actorlib_impl/ya.make b/ydb/core/actorlib_impl/ya.make index ba63f39617..f84dd74004 100644 --- a/ydb/core/actorlib_impl/ya.make +++ b/ydb/core/actorlib_impl/ya.make @@ -10,30 +10,30 @@ SRCS( actor_tracker.cpp actor_tracker.h async_destroyer.h - connect_socket_protocol.cpp - connect_socket_protocol.h + connect_socket_protocol.cpp + connect_socket_protocol.h defs.h destruct_actor.h - http_request_protocol.h + http_request_protocol.h load_network.cpp - load_network.h + load_network.h long_timer.cpp long_timer.h mad_squirrel.cpp mad_squirrel.h melancholic_gopher.cpp - name_service_client_protocol.cpp - name_service_client_protocol.h + name_service_client_protocol.cpp + name_service_client_protocol.h node_identifier.cpp node_identifier.h - proto_ready_actor.h - read_data_protocol.cpp - read_data_protocol.h - read_http_reply_protocol.cpp - read_http_reply_protocol.h + proto_ready_actor.h + read_data_protocol.cpp + read_data_protocol.h + read_http_reply_protocol.cpp + read_http_reply_protocol.h router_rr.h - send_data_protocol.cpp - send_data_protocol.h + send_data_protocol.cpp + send_data_protocol.h ) PEERDIR( diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp index f9e517fc42..cc3fa9c120 100644 --- a/ydb/core/base/appdata.cpp +++ b/ydb/core/base/appdata.cpp @@ -15,7 +15,7 @@ TAppData::TAppData( , UserPoolId(userPoolId) , IOPoolId(ioPoolId) , BatchPoolId(batchPoolId) - , ServicePools(servicePools) + , ServicePools(servicePools) , TypeRegistry(typeRegistry) , FunctionRegistry(functionRegistry) , FormatFactory(formatFactory) diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index c666f7468c..830b5c9767 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -127,8 +127,8 @@ struct TAppData { TIntrusivePtr<NKikimr::TControlBoard> Icb; TIntrusivePtr<NGRpcService::TInFlightLimiterRegistry> InFlightLimiterRegistry; - TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; - + TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; + THolder<NKikimrBlobStorage::TNodeWardenServiceSet> StaticBlobStorageConfig; THolder<NKikimrCms::TCmsConfig> DefaultCmsConfig; diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index f5fedfe19b..1aacc83856 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -11,11 +11,11 @@ namespace NKikimr { struct TKikimrEvents : TEvents { enum EEventSpaceKikimr { - /* WARNING: - Please mind that you should never change the order - for the following keywords, you should consider - issues about "rolling update". - */ + /* WARNING: + Please mind that you should never change the order + for the following keywords, you should consider + issues about "rolling update". + */ ES_KIKIMR_ES_BEGIN = ES_USERSPACE, //4096 ES_STATESTORAGE, //4097 ES_DEPRECATED_4098, //4098 diff --git a/ydb/core/blobstorage/crypto/ya.make b/ydb/core/blobstorage/crypto/ya.make index 4540b3e837..e555e7eb25 100644 --- a/ydb/core/blobstorage/crypto/ya.make +++ b/ydb/core/blobstorage/crypto/ya.make @@ -22,11 +22,11 @@ ELSE() ) ENDIF() -PEERDIR( +PEERDIR( contrib/libs/t1ha library/cpp/sse -) - +) + END() RECURSE_FOR_TESTS( diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp index b73da3df8d..4d2b8dfe23 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp @@ -1744,7 +1744,7 @@ void PrintDebug(NKikimr::TEvBlobStorage::TEvVGetResult::TPtr &ev, const NActors: ingressStr = ingress.ToString(&info->GetTopology(), vdisk, id); ingressRaw = ingress.Raw(); } - LOG_NOTICE(ctx, NActorsServices::TEST, " @@@@@@@@@@ Status=%s LogoBlob=%s Data='%s' Ingress='%s' Raw=0x%lx", + LOG_NOTICE(ctx, NActorsServices::TEST, " @@@@@@@@@@ Status=%s LogoBlob=%s Data='%s' Ingress='%s' Raw=0x%lx", NKikimrProto::EReplyStatus_Name(q.GetStatus()).data(), id.ToString().data(), LimitData(q.GetBuffer()).data(), ingressStr.data(), ingressRaw); } diff --git a/ydb/core/client/flat_ut.cpp b/ydb/core/client/flat_ut.cpp index 362c052aa1..74de6e7a8c 100644 --- a/ydb/core/client/flat_ut.cpp +++ b/ydb/core/client/flat_ut.cpp @@ -24,91 +24,91 @@ namespace NFlatTests { using namespace Tests; using NClient::TValue; -namespace { +namespace { class TFailingMtpQueue: public TSimpleThreadPool { - private: - bool FailOnAdd_ = false; - public: - void SetFailOnAdd(bool fail = true) { - FailOnAdd_ = fail; - } + private: + bool FailOnAdd_ = false; + public: + void SetFailOnAdd(bool fail = true) { + FailOnAdd_ = fail; + } [[nodiscard]] bool Add(IObjectInQueue* pObj) override { - if (FailOnAdd_) { - return false; - } - + if (FailOnAdd_) { + return false; + } + return TSimpleThreadPool::Add(pObj); - } - TFailingMtpQueue() = default; + } + TFailingMtpQueue() = default; TFailingMtpQueue(IThreadFactory* pool) : TSimpleThreadPool(pool) - { - } - }; - - using TFailingServerMtpQueue = + { + } + }; + + using TFailingServerMtpQueue = TThreadPoolBinder<TFailingMtpQueue, THttpServer::ICallBack>; - - class THTTP200OkServer: public THttpServer::ICallBack { - class TRequest: public THttpClientRequestEx { - public: - inline TRequest(THTTP200OkServer* parent) - : Parent_(parent) - { - } - - bool Reply(void* /*tsr*/) override { - if (!ProcessHeaders()) { + + class THTTP200OkServer: public THttpServer::ICallBack { + class TRequest: public THttpClientRequestEx { + public: + inline TRequest(THTTP200OkServer* parent) + : Parent_(parent) + { + } + + bool Reply(void* /*tsr*/) override { + if (!ProcessHeaders()) { return true; - } - + } + if (strncmp(RequestString.data(), "GET /hosts HTTP/1.", 18) == 0) { TString list = Sprintf("[\"localhost\"]"); - Output() << "HTTP/1.1 200 Ok\r\n"; - Output() << "Connection: close\r\n"; - Output() << "X-Server: unit test server\r\n"; + Output() << "HTTP/1.1 200 Ok\r\n"; + Output() << "Connection: close\r\n"; + Output() << "X-Server: unit test server\r\n"; Output() << "Content-Length: " << list.size() << "\r\n"; - Output() << "\r\n"; - Output() << list; - return true; - } - - Output() << "HTTP/1.1 200 Ok\r\n"; + Output() << "\r\n"; + Output() << list; + return true; + } + + Output() << "HTTP/1.1 200 Ok\r\n"; if (Buf.Size()) { - Output() << "X-Server: unit test server\r\n"; + Output() << "X-Server: unit test server\r\n"; Output() << "Content-Length: " << Buf.Size() << "\r\n\r\n"; Output().Write(Buf.AsCharPtr(), Buf.Size()); - } else { - Output() << "X-Server: unit test server\r\n"; + } else { + Output() << "X-Server: unit test server\r\n"; Output() << "Content-Length: " << (Parent_->Res_).size() - << "\r\n\r\n"; - Output() << Parent_->Res_; - } - Output().Finish(); - - return true; - } - - private: - THTTP200OkServer* Parent_ = nullptr; - }; - - public: + << "\r\n\r\n"; + Output() << Parent_->Res_; + } + Output().Finish(); + + return true; + } + + private: + THTTP200OkServer* Parent_ = nullptr; + }; + + public: inline THTTP200OkServer(TString res) - : Res_(std::move(res)) - { - } - - TClientRequest* CreateClient() override { - return new TRequest(this); - } - - private: + : Res_(std::move(res)) + { + } + + TClientRequest* CreateClient() override { + return new TRequest(this); + } + + private: TString Res_; - }; -} - - + }; +} + + Y_UNIT_TEST_SUITE(TFlatTest) { Y_UNIT_TEST(Init) { @@ -2882,7 +2882,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) { annoyingClient.DeleteTable("/dc-1/Dir", "TableOld"); annoyingClient.Ls("/dc-1/Dir/TableOld"); } - + Y_UNIT_TEST(AutoSplitBySize) { TPortManager pm; ui16 port = pm.GetPort(2134); diff --git a/ydb/core/client/minikql_compile/mkql_compile_service.cpp b/ydb/core/client/minikql_compile/mkql_compile_service.cpp index e01d61a22c..baee93cf4f 100644 --- a/ydb/core/client/minikql_compile/mkql_compile_service.cpp +++ b/ydb/core/client/minikql_compile/mkql_compile_service.cpp @@ -149,7 +149,7 @@ private: std::move(next->CompileResolveCookies), next->ForceRefresh); auto *appData = AppData(ctx); - auto actId = ctx.ExecutorThread.RegisterActor(act, TMailboxType::HTSwap, appData->UserPoolId); + auto actId = ctx.ExecutorThread.RegisterActor(act, TMailboxType::HTSwap, appData->UserPoolId); Compiling.insert(TCompilingMap::value_type(actId, next)); CompileQueue.pop(); } diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 7f3146d21a..c76238f60a 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -91,10 +91,10 @@ public: record->SetSchemeshardId(Request->Record.GetSchemeshardId()); record->SetPathId(Request->Record.GetPathId()); } - if (Request->Record.HasOptions()) { - auto options = record->MutableOptions(); - options->CopyFrom(Request->Record.GetOptions()); - } + if (Request->Record.HasOptions()) { + auto options = record->MutableOptions(); + options->CopyFrom(Request->Record.GetOptions()); + } req->Record.SetUserToken(TBase::GetSerializedToken()); ctx.Send(MakeTxProxyID(), req.Release()); } diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 450d4135c6..c1310bb0c7 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -1329,7 +1329,7 @@ public: } }; - + class TClientCommandS3Listing: public TClientCommand { public: TClientCommandS3Listing() diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index faf1797413..002b01ad9d 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -41,7 +41,7 @@ union TBasicKikimrServicesMask { bool EnableFailureInjectionService:1; bool EnablePersQueueL2Cache:1; bool EnableKqp:1; - bool EnableMemoryLog:1; + bool EnableMemoryLog:1; bool EnableGRpcService:1; bool EnableNodeIdentifier:1; bool EnableCms:1; diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index bb68656291..261e5b85ad 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -58,7 +58,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("drivemodel-file", "drive model config file").OptionalArgument("PATH"); opts.AddLongOption("kqp-file", "Kikimr Query Processor config file").OptionalArgument("PATH"); opts.AddLongOption("incrhuge-file", "incremental huge blob keeper config file").OptionalArgument("PATH"); - opts.AddLongOption("memorylog-file", "set buffer size for memory log").OptionalArgument("PATH"); + opts.AddLongOption("memorylog-file", "set buffer size for memory log").OptionalArgument("PATH"); opts.AddLongOption("grpc-file", "gRPC config file").OptionalArgument("PATH"); opts.AddLongOption("grpc-port", "enable gRPC server on port").RequiredArgument("PORT"); opts.AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT"); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 819c1478d1..d0f127a0f9 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -128,7 +128,7 @@ #include <ydb/core/util/sig.h> #include <ydb/core/viewer/viewer.h> - + #include <ydb/public/lib/deprecated/client/msgbus_client.h> #include <ydb/core/ymq/actor/serviceid.h> @@ -225,9 +225,9 @@ void AddExecutorPool( ui32 poolId, ui32 maxActivityType, ui32& unitedThreads) -{ - switch (poolConfig.GetType()) { - case NKikimrConfig::TActorSystemConfig::TExecutor::BASIC: { +{ + switch (poolConfig.GetType()) { + case NKikimrConfig::TActorSystemConfig::TExecutor::BASIC: { TBasicExecutorPoolConfig basic; basic.PoolId = poolId; basic.PoolName = poolConfig.GetName(); @@ -236,20 +236,20 @@ void AddExecutorPool( basic.Affinity = ParseAffinity(poolConfig.GetAffinity()); basic.RealtimePriority = poolConfig.GetRealtimePriority(); basic.MaxActivityType = maxActivityType; - if (poolConfig.HasTimePerMailboxMicroSecs()) { + if (poolConfig.HasTimePerMailboxMicroSecs()) { basic.TimePerMailbox = TDuration::MicroSeconds(poolConfig.GetTimePerMailboxMicroSecs()); - } else if (systemConfig.HasTimePerMailboxMicroSecs()) { + } else if (systemConfig.HasTimePerMailboxMicroSecs()) { basic.TimePerMailbox = TDuration::MicroSeconds(systemConfig.GetTimePerMailboxMicroSecs()); - } - if (poolConfig.HasEventsPerMailbox()) { + } + if (poolConfig.HasEventsPerMailbox()) { basic.EventsPerMailbox = poolConfig.GetEventsPerMailbox(); - } else if (systemConfig.HasEventsPerMailbox()) { + } else if (systemConfig.HasEventsPerMailbox()) { basic.EventsPerMailbox = systemConfig.GetEventsPerMailbox(); - } + } Y_VERIFY(basic.EventsPerMailbox != 0); cpuManager.Basic.emplace_back(std::move(basic)); break; - } + } case NKikimrConfig::TActorSystemConfig::TExecutor::IO: { TIOExecutorPoolConfig io; io.PoolId = poolId; @@ -357,50 +357,50 @@ TBasicServicesInitializer::TBasicServicesInitializer(const TKikimrRunConfig& run { } -static ui32 GetInterconnectThreadPoolId(const NKikimr::TAppData* appData) { - Y_VERIFY_DEBUG(appData != nullptr); - auto item = appData->ServicePools.find("Interconnect"); - if (item != appData->ServicePools.end()) - return item->second; - else - return appData->SystemPoolId; -} - +static ui32 GetInterconnectThreadPoolId(const NKikimr::TAppData* appData) { + Y_VERIFY_DEBUG(appData != nullptr); + auto item = appData->ServicePools.find("Interconnect"); + if (item != appData->ServicePools.end()) + return item->second; + else + return appData->SystemPoolId; +} + static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInterconnectConfig& config, ui32 numNodes, ui32 numDataCenters) { TInterconnectSettings result; if (config.HasSelfKickDelayDuration() || config.HasSelfKickDelay()) { Cerr << "SelfKickDelayDuration/SelfKickDelay option is deprecated" << Endl; } - + if (config.HasHandshakeTimeoutDuration()) { result.Handshake = DurationFromProto(config.GetHandshakeTimeoutDuration()); } else if (config.HasHandshakeTimeout()) { result.Handshake = TDuration::MilliSeconds(config.GetHandshakeTimeout()); } - + if (config.HasHeartbeatIntervalDuration() || config.HasHeartbeatInterval()) { Cerr << "HeartbeatIntervalDuration/HeartbeatInterval option is deprecated" << Endl; } - + if (config.HasDeadPeerTimeoutDuration()) { result.DeadPeer = DurationFromProto(config.GetDeadPeerTimeoutDuration()); } else if (config.HasDeadPeerTimeout()) { result.DeadPeer = TDuration::MilliSeconds(config.GetDeadPeerTimeout()); } - + if (config.HasSendBufferDieLimitInMB()) { - result.SendBufferDieLimitInMB = config.GetSendBufferDieLimitInMB(); + result.SendBufferDieLimitInMB = config.GetSendBufferDieLimitInMB(); } else { - result.SendBufferDieLimitInMB = 512; + result.SendBufferDieLimitInMB = 512; } - + if (config.HasCloseOnIdleTimeoutDuration()) { result.CloseOnIdle = DurationFromProto(config.GetCloseOnIdleTimeoutDuration()); } else if (config.HasCloseOnIdleTimeout()) { - result.CloseOnIdle = TDuration::Seconds(config.GetCloseOnIdleTimeout()); + result.CloseOnIdle = TDuration::Seconds(config.GetCloseOnIdleTimeout()); } - + auto mode = config.GetCounterMergeMode(); if (config.HasMergePerPeerCounters() && !config.HasCounterMergeMode()) { mode = !config.GetMergePerPeerCounters() @@ -427,7 +427,7 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter case NKikimrConfig::TInterconnectConfig::NO_MERGE: break; } - + switch (config.GetEncryptionMode()) { case NKikimrConfig::TInterconnectConfig::DISABLED: result.EncryptionMode = EEncryptionMode::DISABLED; @@ -441,13 +441,13 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter } result.TlsAuthOnly = config.GetTlsAuthOnly(); - if (config.HasTCPSocketBufferSize()) - result.TCPSocketBufferSize = config.GetTCPSocketBufferSize(); - - if (config.HasMaxTimePerEventInMks()) { + if (config.HasTCPSocketBufferSize()) + result.TCPSocketBufferSize = config.GetTCPSocketBufferSize(); + + if (config.HasMaxTimePerEventInMks()) { Cerr << "MaxTimePerEventInMks option is deprecated" << Endl; - } - + } + if (config.HasTotalInflightAmountOfData()) { result.TotalInflightAmountOfData = config.GetTotalInflightAmountOfData(); } @@ -501,17 +501,17 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter result.MessagePendingSize = config.GetMessagePendingSize(); } - return result; -} - + return result; +} + void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { Y_VERIFY(Config.HasActorSystemConfig()); - auto& systemConfig = Config.GetActorSystemConfig(); - Y_VERIFY(systemConfig.HasScheduler()); - Y_VERIFY(systemConfig.ExecutorSize()); - + auto& systemConfig = Config.GetActorSystemConfig(); + Y_VERIFY(systemConfig.HasScheduler()); + Y_VERIFY(systemConfig.ExecutorSize()); + const ui32 systemPoolId = appData->SystemPoolId; const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters = appData->Counters; @@ -519,7 +519,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->MaxActivityType = GetActivityTypeCount(); setup->CpuManager = CreateCpuManagerConfig(systemConfig, setup->MaxActivityType); for (ui32 poolId = 0; poolId != setup->GetExecutorsCount(); ++poolId) { - const auto &execConfig = systemConfig.GetExecutor(poolId); + const auto &execConfig = systemConfig.GetExecutor(poolId); if (execConfig.HasInjectMadSquirrels()) { for (ui32 i = execConfig.GetInjectMadSquirrels(); i > 0; --i) { setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(TActorId(), TActorSetupCmd(CreateMadSquirrel(), TMailboxType::HTSwap, poolId))); @@ -536,7 +536,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s NLwTraceMonPage::DashboardRegistry().Register(NActors::LWTraceDashboards(setup)); if (Config.HasNameserviceConfig()) { - const auto& nsConfig = Config.GetNameserviceConfig(); + const auto& nsConfig = Config.GetNameserviceConfig(); const TActorId resolverId = NDnsResolver::MakeDnsResolverActorId(); const TActorId nameserviceId = GetNameserviceActorId(); @@ -544,14 +544,14 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s TSet<TString> dataCenters; TIntrusivePtr<TTableNameserverSetup> table(new TTableNameserverSetup()); - for (const auto &node : nsConfig.GetNode()) { + for (const auto &node : nsConfig.GetNode()) { const ui32 nodeId = node.GetNodeId(); const TString host = node.HasHost() ? node.GetHost() : TString(); const ui32 port = node.GetPort(); const TString resolveHost = node.HasInterconnectHost() ? - node.GetInterconnectHost() : host; - + node.GetInterconnectHost() : host; + // Use ip address only when dns host not specified const TString addr = resolveHost ? TString() : node.GetAddress(); @@ -596,16 +596,16 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s break; } - setup->LocalServices.emplace_back( - nameserviceId, + setup->LocalServices.emplace_back( + nameserviceId, TActorSetupCmd(nameservice, TMailboxType::HTSwap, systemPoolId)); if (Config.HasInterconnectConfig() && Config.GetInterconnectConfig().GetStartTcp()) { - const auto& icConfig = Config.GetInterconnectConfig(); - + const auto& icConfig = Config.GetInterconnectConfig(); + TChannelsConfig channels; auto settings = GetInterconnectSettings(icConfig, numNodes, dataCenters.size()); - ui32 interconnectPoolId = GetInterconnectThreadPoolId(appData); + ui32 interconnectPoolId = GetInterconnectThreadPoolId(appData); for (const auto& channel : icConfig.GetChannel()) { const auto index = channel.GetIndex(); @@ -628,13 +628,13 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s auto destructorQueueSize = std::make_shared<std::atomic<TAtomicBase>>(0); - TIntrusivePtr<TInterconnectProxyCommon> icCommon; - icCommon.Reset(new TInterconnectProxyCommon); - icCommon->NameserviceId = nameserviceId; + TIntrusivePtr<TInterconnectProxyCommon> icCommon; + icCommon.Reset(new TInterconnectProxyCommon); + icCommon->NameserviceId = nameserviceId; icCommon->MonCounters = GetServiceCounters(counters, "interconnect"); - icCommon->ChannelsConfig = channels; + icCommon->ChannelsConfig = channels; icCommon->Settings = settings; - icCommon->DestructorId = GetDestructActorID(); + icCommon->DestructorId = GetDestructActorID(); icCommon->DestructorQueueSize = destructorQueueSize; icCommon->HandshakeBallastSize = icConfig.GetHandshakeBallastSize(); icCommon->LocalScopeId = ScopeId.GetInterconnectScopeId(); @@ -683,14 +683,14 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s NInterconnect::CreateInterconnectMonActor(icCommon), TMailboxType::ReadAsFilled, systemPoolId)); } - if (nsConfig.HasClusterUUID()) { - icCommon->ClusterUUID = nsConfig.GetClusterUUID(); - } - - for (const auto& item: nsConfig.GetAcceptUUID()) { - icCommon->AcceptUUID.emplace_back(item); - } - + if (nsConfig.HasClusterUUID()) { + icCommon->ClusterUUID = nsConfig.GetClusterUUID(); + } + + for (const auto& item: nsConfig.GetAcceptUUID()) { + icCommon->AcceptUUID.emplace_back(item); + } + if (!nsConfig.GetSuppressVersionCheck()) { icCommon->VersionInfo = VERSION; CheckVersionTag(); @@ -698,7 +698,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->LocalServices.emplace_back(GetDestructActorID(), TActorSetupCmd(new TDestructActor, TMailboxType::ReadAsFilled, interconnectPoolId)); - + Y_VERIFY(!table->StaticNodeTable.empty()); ui32 maxNode = 0; for (const auto &node : table->StaticNodeTable) { @@ -717,11 +717,11 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s TMailboxType::ReadAsFilled, interconnectPoolId); } else { TYandexQueryInitializer::SetIcPort(node.second.second); - icCommon->TechnicalSelfHostName = node.second.Host; + icCommon->TechnicalSelfHostName = node.second.Host; TString address = "::"; //bind ipv6 interfaces by default if (node.second.first) address = node.second.first; - auto listener = new TInterconnectListenerTCP( + auto listener = new TInterconnectListenerTCP( address, node.second.second, icCommon); if (int err = listener->Bind()) { Cerr << "Failed to set up IC listener on port " << node.second.second @@ -732,7 +732,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s TMailboxType::ReadAsFilled, interconnectPoolId)); } } - + // Prepare listener for dynamic node. if (Config.GetDynamicNodeConfig().HasNodeInfo()) { auto &info = Config.GetDynamicNodeConfig().GetNodeInfo(); @@ -757,7 +757,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->LocalServices.emplace_back(NInterconnect::MakeLoadResponderActorId(NodeId), TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, systemPoolId)); - //IC_Load::InitializeService(setup, appData, maxNode); + //IC_Load::InitializeService(setup, appData, maxNode); } } } @@ -866,14 +866,14 @@ void TStateStorageServiceInitializer::InitializeServices(NActors::TActorSystemSe setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeStateStorageProxyID(ssid), TActorSetupCmd(CreateStateStorageProxy(ssrInfo.Get(), ssbInfo.Get(), sbrInfo.Get()), - TMailboxType::ReadAsFilled, + TMailboxType::ReadAsFilled, appData->SystemPoolId))); } for (ui32 ssid = 0; ssid <= maxssid; ++ssid) { if (!knownss[ssid]) setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeStateStorageProxyID(ssid), TActorSetupCmd(CreateStateStorageProxyStub(), - TMailboxType::HTSwap, + TMailboxType::HTSwap, appData->SystemPoolId))); } @@ -901,13 +901,13 @@ void TLocalServiceInitializer::InitializeServices( // setup local TLocalConfig::TPtr localConfig(new TLocalConfig()); localConfig->TabletClassInfo[appData->DefaultTabletTypes.SchemeShard] = TLocalConfig::TTabletClassInfo( - new TTabletSetupInfo(&CreateFlatTxSchemeShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); + new TTabletSetupInfo(&CreateFlatTxSchemeShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.DataShard] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreateDataShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.KeyValue] = TLocalConfig::TTabletClassInfo( - new TTabletSetupInfo(&CreateKeyValueFlat, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); + new TTabletSetupInfo(&CreateKeyValueFlat, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.PersQueue] = TLocalConfig::TTabletClassInfo( - new TTabletSetupInfo(&CreatePersQueue, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); + new TTabletSetupInfo(&CreatePersQueue, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.PersQueueReadBalancer] = TLocalConfig::TTabletClassInfo( new TTabletSetupInfo(&CreatePersQueueReadBalancer, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId)); localConfig->TabletClassInfo[appData->DefaultTabletTypes.Coordinator] = TLocalConfig::TTabletClassInfo( @@ -1032,7 +1032,7 @@ void TLoggerInitializer::InitializeServices( // log settings must be initialized before calling this method NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, LogBackend, utilsCounters); - NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::HTSwap, appData->IOPoolId); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::HTSwap, appData->IOPoolId); std::pair<NActors::TActorId, NActors::TActorSetupCmd> loggerActorPair(LogSettings->LoggerActorId, loggerActorCmd); setup->LocalServices.push_back(loggerActorPair); @@ -1367,7 +1367,7 @@ void TTabletsInitializer::InitializeServices( tabletInfo.Get(), new TBootstrapperInfo(tabletSetup.Get()), tabletConfig.GetStandBy()), - TMailboxType::ReadAsFilled, + TMailboxType::ReadAsFilled, appData->SystemPoolId))); } } @@ -1385,7 +1385,7 @@ void TMediatorTimeCastProxyInitializer::InitializeServices( const NKikimr::TAppData* appData) { setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( MakeMediatorTimecastProxyID(), - TActorSetupCmd(CreateMediatorTimecastProxy(), TMailboxType::ReadAsFilled, appData->SystemPoolId))); + TActorSetupCmd(CreateMediatorTimecastProxy(), TMailboxType::ReadAsFilled, appData->SystemPoolId))); } // TMiniKQLCompileServiceInitializer @@ -1692,7 +1692,7 @@ void TSelfPingInitializer::InitializeServices( IActor* selfPingActor = CreateSelfPingActor(selfPingInterval, counter, cpuTimeCounter); setup->LocalServices.push_back(std::make_pair(TActorId(), TActorSetupCmd(selfPingActor, - TMailboxType::HTSwap, + TMailboxType::HTSwap, poolId))); } } @@ -1708,7 +1708,7 @@ void TWhiteBoardServiceInitializer::InitializeServices(NActors::TActorSystemSetu IActor* tabletStateService = NNodeWhiteboard::CreateNodeWhiteboardService(); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NNodeWhiteboard::MakeNodeWhiteboardServiceId(NodeId), TActorSetupCmd(tabletStateService, - TMailboxType::HTSwap, + TMailboxType::HTSwap, appData->SystemPoolId))); } @@ -1743,7 +1743,7 @@ void TTabletMonitorInitializer::InitializeServices(NActors::TActorSystemSetup* s IActor* nodeTabletMonitor = NNodeTabletMonitor::CreateNodeTabletMonitor(TabletStateClassifier, TabletListRenderer); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NNodeTabletMonitor::MakeNodeTabletMonitorID(NodeId), TActorSetupCmd(nodeTabletMonitor, - TMailboxType::HTSwap, + TMailboxType::HTSwap, appData->UserPoolId))); } @@ -1762,7 +1762,7 @@ void TViewerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, SetupDBVirtualHandlers(dynamic_cast<IViewer*>(viewer)); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeViewerID(NodeId), TActorSetupCmd(viewer, - TMailboxType::HTSwap, + TMailboxType::HTSwap, appData->BatchPoolId))); } @@ -1774,7 +1774,7 @@ TLoadInitializer::TLoadInitializer(const TKikimrRunConfig& runConfig) void TLoadInitializer::InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) { IActor *actor = CreateTestLoadActor(appData->Counters); - setup->LocalServices.emplace_back(MakeBlobStorageLoadID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); + setup->LocalServices.emplace_back(MakeBlobStorageLoadID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); // FIXME: correct service id } @@ -1787,7 +1787,7 @@ TFailureInjectionInitializer::TFailureInjectionInitializer(const TKikimrRunConfi void TFailureInjectionInitializer::InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) { IActor *actor = CreateFailureInjectionActor(); setup->LocalServices.emplace_back(MakeBlobStorageFailureInjectionID(NodeId), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); // FIXME: correct service id } @@ -1830,7 +1830,7 @@ void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup IActor* actor = NPQ::CreateNodePersQueueL2Cache(params, pqCacheGroup); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( NPQ::MakePersQueueL2CacheID(), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); } // TNetClassifierInitializer @@ -1882,10 +1882,10 @@ TMemProfMonitorInitializer::TMemProfMonitorInitializer(const TKikimrRunConfig& r : IKikimrServicesInitializer(runConfig) {} -void TMemProfMonitorInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) -{ +void TMemProfMonitorInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) +{ IActor* monitorActor = CreateMemProfMonitor( 1, // seconds appData->Counters); @@ -1896,7 +1896,7 @@ void TMemProfMonitorInitializer::InitializeServices( monitorActor, TMailboxType::HTSwap, appData->UserPoolId)); -} +} // TMemoryTrackerInitializer @@ -1968,40 +1968,40 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu } } -TMemoryLogInitializer::TMemoryLogInitializer( - const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{ -} - -void TMemoryLogInitializer::InitializeServices( - NActors::TActorSystemSetup*, - const NKikimr::TAppData*) -{ - if (!Config.HasMemoryLogConfig()) { - return; - } - - if (!Config.GetMemoryLogConfig().HasLogBufferSize()) { - return; - } - if (Config.GetMemoryLogConfig().GetLogBufferSize() == 0ULL) { - return; - } - - LogBufferSize = Config.GetMemoryLogConfig().GetLogBufferSize(); - - if (Config.GetMemoryLogConfig().HasLogGrainSize()) { - LogGrainSize = Config.GetMemoryLogConfig().GetLogGrainSize(); - } - - if (LogGrainSize != 0) { - TMemoryLog::CreateMemoryLogBuffer(LogBufferSize, LogGrainSize); - } else { - TMemoryLog::CreateMemoryLogBuffer(LogBufferSize); - } -} - +TMemoryLogInitializer::TMemoryLogInitializer( + const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TMemoryLogInitializer::InitializeServices( + NActors::TActorSystemSetup*, + const NKikimr::TAppData*) +{ + if (!Config.HasMemoryLogConfig()) { + return; + } + + if (!Config.GetMemoryLogConfig().HasLogBufferSize()) { + return; + } + if (Config.GetMemoryLogConfig().GetLogBufferSize() == 0ULL) { + return; + } + + LogBufferSize = Config.GetMemoryLogConfig().GetLogBufferSize(); + + if (Config.GetMemoryLogConfig().HasLogGrainSize()) { + LogGrainSize = Config.GetMemoryLogConfig().GetLogGrainSize(); + } + + if (LogGrainSize != 0) { + TMemoryLog::CreateMemoryLogBuffer(LogBufferSize, LogGrainSize); + } else { + TMemoryLog::CreateMemoryLogBuffer(LogBufferSize); + } +} + TCmsServiceInitializer::TCmsServiceInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 407ce1bb7b..5d227757ab 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -42,9 +42,9 @@ public: // base, nameservice, interconnect class TBasicServicesInitializer : public IKikimrServicesInitializer { - static IExecutorPool* + static IExecutorPool* CreateExecutorPool(const NKikimrConfig::TActorSystemConfig::TExecutor& poolConfig, - const NKikimrConfig::TActorSystemConfig& systemConfig, + const NKikimrConfig::TActorSystemConfig& systemConfig, ui32 poolId, ui32 maxActivityType); static ISchedulerThread* CreateScheduler(const NKikimrConfig::TActorSystemConfig::TScheduler &config); @@ -357,13 +357,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TMemProfMonitorInitializer : public IKikimrServicesInitializer { -public: +class TMemProfMonitorInitializer : public IKikimrServicesInitializer { +public: TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +}; + class TMemoryTrackerInitializer : public IKikimrServicesInitializer { public: TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig); @@ -391,12 +391,12 @@ class TMemoryLogInitializer : public IKikimrServicesInitializer { size_t LogBufferSize = 0; size_t LogGrainSize = 0; -public: - TMemoryLogInitializer(const TKikimrRunConfig& runConfig); - +public: + TMemoryLogInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +}; + class TCmsServiceInitializer : public IKikimrServicesInitializer { public: TCmsServiceInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index a4f74aa4e0..229c868f17 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -117,7 +117,7 @@ #include <library/cpp/actors/core/memory_track.h> #include <library/cpp/actors/prof/tag.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> - + #include <util/folder/dirut.h> #include <util/system/file.h> #include <util/system/getpid.h> @@ -837,15 +837,15 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) const ui32 ioPoolId = cfg.GetActorSystemConfig().HasIoExecutor() ? cfg.GetActorSystemConfig().GetIoExecutor() : 0; const ui32 batchPoolId = cfg.GetActorSystemConfig().HasBatchExecutor() ? cfg.GetActorSystemConfig().GetBatchExecutor() : 0; TMap<TString, ui32> servicePools; - for (ui32 i = 0; i < cfg.GetActorSystemConfig().ServiceExecutorSize(); ++i) { - auto item = cfg.GetActorSystemConfig().GetServiceExecutor(i); + for (ui32 i = 0; i < cfg.GetActorSystemConfig().ServiceExecutorSize(); ++i) { + auto item = cfg.GetActorSystemConfig().GetServiceExecutor(i); const TString service = item.GetServiceName(); - const ui32 pool = item.GetExecutorId(); + const ui32 pool = item.GetExecutorId(); servicePools.insert(std::pair<TString, ui32>(service, pool)); - } + } AppData.Reset(new TAppData(sysPoolId, userPoolId, ioPoolId, batchPoolId, - servicePools, + servicePools, TypeRegistry.Get(), FunctionRegistry.Get(), FormatFactory.Get(), @@ -866,7 +866,7 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->Counters = Counters; AppData->Mon = Monitoring.Get(); AppData->BusMonPage = BusMonPage.Get(); - AppData->PollerThreads = PollerThreads; + AppData->PollerThreads = PollerThreads; AppData->LocalScopeId = runConfig.ScopeId; // setup streaming config @@ -1134,10 +1134,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers using namespace NKikimrServicesInitializers; TIntrusivePtr<TServiceInitializersList> sil(new TServiceInitializersList); - if (serviceMask.EnableMemoryLog) { - sil->AddServiceInitializer(new TMemoryLogInitializer(runConfig)); - } - + if (serviceMask.EnableMemoryLog) { + sil->AddServiceInitializer(new TMemoryLogInitializer(runConfig)); + } + if (serviceMask.EnableBasicServices) { sil->AddServiceInitializer(new TBasicServicesInitializer(runConfig)); } @@ -1262,8 +1262,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig)); - sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig)); - + sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig)); + #if defined(ENABLE_MEMORY_TRACKING) sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig)); #endif @@ -1328,25 +1328,25 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers return sil; } -void RegisterBaseTagForMemoryProfiling(TActorSystem* as) { - Y_VERIFY(as != nullptr); - if (as->MemProfActivityBase != 0) - return; +void RegisterBaseTagForMemoryProfiling(TActorSystem* as) { + Y_VERIFY(as != nullptr); + if (as->MemProfActivityBase != 0) + return; TVector<TString> holders; TVector<const char*> activityNames; for (ui32 i = 0; i < NKikimrServices::TActivity::EType_ARRAYSIZE; ++i) { auto current = (NKikimrServices::TActivity::EType)i; const char* currName = NKikimrServices::TActivity::EType_Name(current).c_str(); - if (currName[0] == '\0') { - holders.push_back("EActivityType_" + ToString<ui32>(i)); + if (currName[0] == '\0') { + holders.push_back("EActivityType_" + ToString<ui32>(i)); currName = holders.back().c_str(); - } - activityNames.push_back(currName); - } - as->MemProfActivityBase = NProfiling::MakeTags(activityNames); - Y_VERIFY(as->MemProfActivityBase != 0); -} - + } + activityNames.push_back(currName); + } + as->MemProfActivityBase = NProfiling::MakeTags(activityNames); + Y_VERIFY(as->MemProfActivityBase != 0); +} + void TKikimrRunner::KikimrStart() { if (!!Monitoring) { @@ -1359,7 +1359,7 @@ void TKikimrRunner::KikimrStart() { ThreadSigmask(SIG_BLOCK); if (ActorSystem) { - RegisterBaseTagForMemoryProfiling(ActorSystem.Get()); + RegisterBaseTagForMemoryProfiling(ActorSystem.Get()); ActorSystem->Start(); } diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index dfd93a24e7..4624dedf51 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -47,7 +47,7 @@ protected: THolder<NKqp::TKqpShutdownController> KqpShutdownController; - TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; + TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; TAutoPtr<TAppData> AppData; NBus::TBusQueueConfig ProxyBusQueueConfig; diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index f410ef42f2..633236d8a8 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -5,12 +5,12 @@ OWNER( g:kikimr ) -IF (PROFILE_MEMORY_ALLOCATIONS) +IF (PROFILE_MEMORY_ALLOCATIONS) CFLAGS( -DPROFILE_MEMORY_ALLOCATIONS ) -ENDIF() - +ENDIF() + IF (KIKIMR_UDF_DYNAMIC_LINK) CFLAGS( -DKIKIMR_UDF_DYNAMIC_LINK diff --git a/ydb/core/mon_alloc/monitor.h b/ydb/core/mon_alloc/monitor.h index f6a8404b65..b543d35440 100644 --- a/ydb/core/mon_alloc/monitor.h +++ b/ydb/core/mon_alloc/monitor.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <ydb/core/control/immediate_control_board_impl.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/actor.h> #include <library/cpp/monlib/dynamic_counters/counters.h> - + namespace NActors { class TMon; } -namespace NKikimr { +namespace NKikimr { inline NActors::TActorId MakeMemProfMonitorID(ui32 node = 0) { char x[12] = {'m', 'e', 'm', 'p', 'r', 'o', 'f', 'm', 'o', 'n', 'i', 't'}; return NActors::TActorId(node, TStringBuf(x, 12)); @@ -47,4 +47,4 @@ namespace NKikimr { NActors::IActor* CreateMemProfMonitor( ui32 intervalSec, TIntrusivePtr<NMonitoring::TDynamicCounters> counters); -} +} diff --git a/ydb/core/node_whiteboard/node_whiteboard.h b/ydb/core/node_whiteboard/node_whiteboard.h index cdf7601807..e06da69c51 100644 --- a/ydb/core/node_whiteboard/node_whiteboard.h +++ b/ydb/core/node_whiteboard/node_whiteboard.h @@ -346,7 +346,7 @@ struct TEvWhiteboard{ struct TEvSystemStateRequest : public TEventPB<TEvSystemStateRequest, NKikimrWhiteboard::TEvSystemStateRequest, EvSystemStateRequest> {}; struct TEvSystemStateResponse : public TEventPB<TEvSystemStateResponse, NKikimrWhiteboard::TEvSystemStateResponse, EvSystemStateResponse> {}; - + struct TEvNodeStateUpdate : TEventPB<TEvNodeStateUpdate, NKikimrWhiteboard::TNodeStateInfo, EvNodeStateUpdate> { TEvNodeStateUpdate() = default; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d64169d4fc..0f8dc62eb0 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -52,9 +52,9 @@ message TActorSystemConfig { optional TAffinity Affinity = 4; optional uint32 InjectMadSquirrels = 5; optional string Name = 6; - optional uint32 TimePerMailboxMicroSecs = 7; - optional uint32 EventsPerMailbox = 8; - optional uint32 RealtimePriority = 9; + optional uint32 TimePerMailboxMicroSecs = 7; + optional uint32 EventsPerMailbox = 8; + optional uint32 RealtimePriority = 9; // Actorsystem 2.0: cpu sharing by different pools with preemption optional uint32 Concurrency = 10; // Limits simultaneously running mailboxes of UNITED pool @@ -82,15 +82,15 @@ message TActorSystemConfig { optional uint32 UserExecutor = 4; optional uint32 IoExecutor = 5; optional uint32 BatchExecutor = 6; - - message TServiceExecutor { - required string ServiceName = 1; - required uint32 ExecutorId = 2; - } - - repeated TServiceExecutor ServiceExecutor = 7; - optional uint32 TimePerMailboxMicroSecs = 8; - optional uint32 EventsPerMailbox = 9; + + message TServiceExecutor { + required string ServiceName = 1; + required uint32 ExecutorId = 2; + } + + repeated TServiceExecutor ServiceExecutor = 7; + optional uint32 TimePerMailboxMicroSecs = 8; + optional uint32 EventsPerMailbox = 9; optional uint32 SelfPingInterval = 10; // in microseconds message TUnitedWorkers { @@ -141,9 +141,9 @@ message TStaticNameserviceConfig { } repeated TNode Node = 1; - - optional string ClusterUUID = 2; - repeated string AcceptUUID = 3; + + optional string ClusterUUID = 2; + repeated string AcceptUUID = 3; optional bool SuppressVersionCheck = 4; optional ENameserviceType Type = 5; } @@ -375,16 +375,16 @@ message TInterconnectConfig { optional bool FirstTryBeforePoll = 2; // DEPRECATED optional bool StartTcp = 3 [default = false]; optional uint32 SelfKickDelay = 4; // DEPRECATED - optional uint32 HandshakeTimeout = 5; - optional uint32 HeartbeatInterval = 6; - optional uint32 DeadPeerTimeout = 7; - optional uint32 SendBufferDieLimitInMB = 8; - optional uint32 CloseOnIdleTimeout = 9; - optional uint32 MaxInflightAmountOfDataInKB = 10; - optional bool MergePerPeerCounters = 11; + optional uint32 HandshakeTimeout = 5; + optional uint32 HeartbeatInterval = 6; + optional uint32 DeadPeerTimeout = 7; + optional uint32 SendBufferDieLimitInMB = 8; + optional uint32 CloseOnIdleTimeout = 9; + optional uint32 MaxInflightAmountOfDataInKB = 10; + optional bool MergePerPeerCounters = 11; optional EMergeMode CounterMergeMode = 15 [default = AUTO]; - optional uint32 TCPSocketBufferSize = 12; - optional uint32 MaxTimePerEventInMks = 13; + optional uint32 TCPSocketBufferSize = 12; + optional uint32 MaxTimePerEventInMks = 13; optional bool BindOnAllAddresses = 16 [default = true]; optional EEncryptionMode EncryptionMode = 17 [default = DISABLED]; optional bool TlsAuthOnly = 38; // do not encrypt traffic @@ -552,11 +552,11 @@ message TKQPConfig { repeated NKikimrKqp.TKqpSetting Settings = 10; } -message TMemoryLogConfig { - optional uint64 LogBufferSize = 1; - optional uint64 LogGrainSize = 2; -} - +message TMemoryLogConfig { + optional uint64 LogBufferSize = 1; + optional uint64 LogGrainSize = 2; +} + message TGRpcConfig { optional bool StartGRpcProxy = 1 [default = true]; optional string Host = 2 [default = "[::]"]; @@ -1349,7 +1349,7 @@ message TAppConfig { optional string UDFsDir = 15; repeated string UDFsPaths = 16; optional TKQPConfig KQPConfig = 17; - optional TMemoryLogConfig MemoryLogConfig = 19; + optional TMemoryLogConfig MemoryLogConfig = 19; optional TGRpcConfig GRpcConfig = 20; optional TDynamicNameserviceConfig DynamicNameserviceConfig = 22; //optional TLocalConfig LocalConfig = 23; DEPRECATED diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 33e598c1c2..e82b9bfb22 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -808,8 +808,8 @@ message TCopyTableConfig { //TTableDescription implemets copying a table in orig optional bool OmitIndexes = 3 [default = false]; optional bool OmitFollowers = 4 [default = false]; optional bool IsBackup = 5 [default = false]; -} - +} + message TConsistentTableCopyingConfig { repeated TCopyTableConfig CopyTableDescriptions = 1; } @@ -817,7 +817,7 @@ message TConsistentTableCopyingConfig { message TYTSettings { optional string Host = 1; optional string TablePattern = 2; - optional string Token = 3; + optional string Token = 3; optional uint32 Port = 4 [default = 80]; reserved 5; // YSONasText @@ -825,8 +825,8 @@ message TYTSettings { reserved 7; // CreateYTTable optional bool UseTypeV3 = 8; -}; - +}; + message TS3Settings { enum EScheme { HTTP = 0; @@ -849,11 +849,11 @@ message TS3Settings { optional TLimits Limits = 100; }; -message TBackupTask { - optional string TableName = 1; +message TBackupTask { + optional string TableName = 1; optional uint64 TableId = 2; - optional uint32 ShardNum = 3; - optional uint32 NumberOfRetries = 5; + optional uint32 ShardNum = 3; + optional uint32 NumberOfRetries = 5; reserved 6; // IfFailedBeforeTxId reserved 7; // UserProvidedId @@ -873,8 +873,8 @@ message TBackupTask { optional TScanSettings ScanSettings = 11; optional bool NeedToBill = 12; -} - +} + message TRestoreTask { optional string TableName = 1; optional uint64 TableId = 2; @@ -1077,7 +1077,7 @@ enum EOperationType { ESchemeOpModifyACL = 8; ESchemeOpRmDir = 9; ESchemeOpSplitMergeTablePartitions = 10; - ESchemeOpBackup = 11; + ESchemeOpBackup = 11; ESchemeOpCreateSubDomain = 12; ESchemeOpDropSubDomain = 13; ESchemeOpCreateRtmrVolume = 14; @@ -1233,7 +1233,7 @@ message TModifyScheme { optional TModifyACL ModifyACL = 8; optional TTableDescription AlterTable = 9; optional TSplitMergeTablePartitions SplitMergeTablePartitions = 10; - optional TBackupTask Backup = 11; + optional TBackupTask Backup = 11; optional NKikimrSubDomains.TSubDomainSettings SubDomain = 12; optional TRtmrVolumeDescription CreateRtmrVolume = 13; optional TBlockStoreVolumeDescription CreateBlockStoreVolume = 14; @@ -1280,17 +1280,17 @@ message TModifyScript { repeated TModifyScheme ModifyScheme = 1; } -message TDescribeOptions { - optional bool ReturnPartitioningInfo = 1 [default = true]; - optional bool ReturnPartitionConfig = 2 [default = true]; - optional bool BackupInfo = 3 [default = false]; +message TDescribeOptions { + optional bool ReturnPartitioningInfo = 1 [default = true]; + optional bool ReturnPartitionConfig = 2 [default = true]; + optional bool BackupInfo = 3 [default = false]; optional bool ReturnPartitionStats = 4 [default = false]; optional bool ReturnChildren = 5 [default = true]; optional bool ReturnBoundaries = 6 [default = false]; optional bool ShowPrivateTable = 7 [default = false]; optional bool ReturnChannelsBinding = 8 [default = false]; -} - +} + // Request to read scheme for a specific path // Path can be specified in two ways: // 1. full path @@ -1299,12 +1299,12 @@ message TDescribePath { optional string Path = 1; optional uint64 PathId = 2; optional uint64 SchemeshardId = 3; - - optional bool ReturnPartitioningInfo = 4 [default = true]; // deprecated - optional bool ReturnPartitionConfig = 5 [default = true]; // deprecated - optional bool BackupInfo = 6 [default = false]; // deprecated - - optional TDescribeOptions Options = 7; + + optional bool ReturnPartitioningInfo = 4 [default = true]; // deprecated + optional bool ReturnPartitionConfig = 5 [default = true]; // deprecated + optional bool BackupInfo = 6 [default = false]; // deprecated + + optional TDescribeOptions Options = 7; } // Must be sync with Ydb::EntryType @@ -1342,7 +1342,7 @@ enum EPathState { EPathStateAlter = 4; EPathStateDrop = 5; EPathStateCopying = 6; - EPathStateBackup = 7; + EPathStateBackup = 7; EPathStateUpgrade = 8; EPathStateMigrated = 9; EPathStateRestore = 10; @@ -1407,32 +1407,32 @@ message TTablePartition { optional uint64 DatashardId = 4; } -message TShardError { - optional uint64 ShardId = 1; - optional string Explain = 2; -}; - +message TShardError { + optional uint64 ShardId = 1; + optional string Explain = 2; +}; + message TBackupProgress { optional uint32 NotCompleteYet = 1; optional uint32 Total = 2; - optional uint32 ErrorCount = 3; - optional uint64 StartTime = 4; - repeated TShardError Errors = 5; + optional uint32 ErrorCount = 3; + optional uint64 StartTime = 4; + repeated TShardError Errors = 5; optional TYTSettings YTSettings = 6; - optional uint64 DataTotalSize = 7; + optional uint64 DataTotalSize = 7; optional uint64 TxId = 8; } -message TLastBackupResult { - optional uint32 ErrorCount = 1; - optional uint64 CompleteTimeStamp = 2; +message TLastBackupResult { + optional uint32 ErrorCount = 1; + optional uint64 CompleteTimeStamp = 2; optional uint64 StartTimeStamp = 3; - repeated TShardError Errors = 4; + repeated TShardError Errors = 4; optional TYTSettings YTSettings = 5; - optional uint64 DataTotalSize = 6; + optional uint64 DataTotalSize = 6; optional uint64 TxId = 7; -}; - +}; + // Result for TDescribePath request message TPathDescription { optional TDirEntry Self = 1; // info about the path itself @@ -1441,7 +1441,7 @@ message TPathDescription { repeated TTablePartition TablePartitions = 4; // for table optional TPersQueueGroupDescription PersQueueGroup = 5; // for pq group optional TBackupProgress BackupProgress = 6; - repeated TLastBackupResult LastBackupResult = 7; + repeated TLastBackupResult LastBackupResult = 7; optional NKikimrTableStats.TTableStats TableStats = 8; optional NKikimrTabletBase.TMetrics TabletMetrics = 9; optional NKikimrSubDomains.TDomainDescription DomainDescription = 10; diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index dc06523e5c..cf8d89e2da 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -79,18 +79,18 @@ message TEvDescribeSchemeResult { message TEvCancelTx { optional uint64 TargetTxId = 1; - optional string Path = 2; - optional string SecurityToken = 3; - optional uint64 TxId = 4; -} - + optional string Path = 2; + optional string SecurityToken = 3; + optional uint64 TxId = 4; +} + message TEvCancelTxResult { - optional EStatus Status = 1; - optional string Result = 2; + optional EStatus Status = 1; + optional string Result = 2; optional uint64 TargetTxId = 3; optional uint64 TxId = 4; -} - +} + message TEvUpdateConfig { optional NActorsProto.TActorId Source = 1; optional TConfig Config = 2; diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index df7cda5980..d1abb64ab4 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -434,7 +434,7 @@ message TSchemeOperation { optional string SecurityToken = 5; }; - + message TSchemeDescribe { optional string Path = 1; optional uint64 PathId = 2; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index c17c8a7dc3..2c4b655199 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -189,10 +189,10 @@ enum EServiceKikimr { // HEALTH section HEALTH = 500; - + MEMORY_PROFILER = 510; - DATASHARD_BACKUP = 522; + DATASHARD_BACKUP = 522; CMS = 523; diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 89863b2540..a95c71937f 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -53,12 +53,12 @@ message TEvGetShardStateResult { } message TShardOpResult { - optional bool Success = 1; - optional string Explain = 2; + optional bool Success = 1; + optional string Explain = 2; optional uint64 BytesProcessed = 3; optional uint64 RowsProcessed = 4; -} - +} + message TEvSchemaChanged { optional NActorsProto.TActorId Source = 1; optional uint64 Origin = 2; @@ -66,7 +66,7 @@ message TEvSchemaChanged { optional uint64 TxId = 4; optional uint64 Step = 5; optional uint32 Generation = 6; - + optional TShardOpResult OpResult = 7; } @@ -706,11 +706,11 @@ message TEvSplitPartitioningChangedAck { optional uint64 OperationCookie = 1; optional uint64 TabletId = 2; } - -message TEvCancelBackup { - optional uint64 BackupTxId = 1; + +message TEvCancelBackup { + optional uint64 BackupTxId = 1; optional uint64 TableId = 2; -} +} message TEvCancelRestore { optional uint64 RestoreTxId = 1; diff --git a/ydb/core/protos/tx_proxy.proto b/ydb/core/protos/tx_proxy.proto index dedd2f9cbb..5b391819a5 100644 --- a/ydb/core/protos/tx_proxy.proto +++ b/ydb/core/protos/tx_proxy.proto @@ -274,7 +274,7 @@ message TEvInvalidateTable { optional uint64 SchemeShardId = 1; optional uint64 TableId = 2; } - + message TEvExportRequest { optional string DatabaseName = 1; optional string UserToken = 2; @@ -285,10 +285,10 @@ message TEvExportRequest { NKikimrExport.TForgetExportRequest ForgetExport = 6; NKikimrExport.TListExportsRequest ListExports = 7; } -} - +} + message TEvExportResponse { - optional uint32 Status = 1; + optional uint32 Status = 1; optional string ErrorDescription = 7; oneof Response { NKikimrExport.TCreateExportResponse CreateExport = 2; @@ -297,4 +297,4 @@ message TEvExportResponse { NKikimrExport.TForgetExportResponse ForgetExport = 5; NKikimrExport.TListExportsResponse ListExports = 6; } -} +} diff --git a/ydb/core/scheme/scheme_tablecell.cpp b/ydb/core/scheme/scheme_tablecell.cpp index c9ffaccc8e..f6d57d06e7 100644 --- a/ydb/core/scheme/scheme_tablecell.cpp +++ b/ydb/core/scheme/scheme_tablecell.cpp @@ -76,7 +76,7 @@ TString DbgPrintCell(const TCell& r, NScheme::TTypeId typeId, const NScheme::TTy void DbgPrintValue(TString &res, const TCell &r, ui32 type) { if (r.IsNull()) { - res += "NULL"; + res += "NULL"; } else { switch (type) { case NScheme::NTypeIds::Bool: diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h index bccfdca97d..18c2007d4e 100644 --- a/ydb/core/scheme/scheme_tablecell.h +++ b/ydb/core/scheme/scheme_tablecell.h @@ -74,8 +74,8 @@ public: TArrayRef<const char> AsRef() const noexcept { return { Data(), Size() }; - } - + } + TStringBuf AsBuf() const noexcept { return { Data(), Size() }; diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp index 783758e99d..64d3ba7f0e 100644 --- a/ydb/core/tablet/node_whiteboard.cpp +++ b/ydb/core/tablet/node_whiteboard.cpp @@ -346,9 +346,9 @@ protected: switch (ev->GetTypeRewrite()) { HFunc(TEvWhiteboard::TEvTabletStateUpdate, Handle); HFunc(TEvWhiteboard::TEvTabletStateRequest, Handle); - HFunc(TEvWhiteboard::TEvNodeStateUpdate, Handle); - HFunc(TEvWhiteboard::TEvNodeStateDelete, Handle); - HFunc(TEvWhiteboard::TEvNodeStateRequest, Handle); + HFunc(TEvWhiteboard::TEvNodeStateUpdate, Handle); + HFunc(TEvWhiteboard::TEvNodeStateDelete, Handle); + HFunc(TEvWhiteboard::TEvNodeStateRequest, Handle); HFunc(TEvWhiteboard::TEvPDiskStateUpdate, Handle); HFunc(TEvWhiteboard::TEvPDiskStateRequest, Handle); HFunc(TEvWhiteboard::TEvPDiskStateDelete, Handle); @@ -384,14 +384,14 @@ protected: } } - void Handle(TEvWhiteboard::TEvNodeStateUpdate::TPtr &ev, const TActorContext &ctx) { + void Handle(TEvWhiteboard::TEvNodeStateUpdate::TPtr &ev, const TActorContext &ctx) { auto& nodeStateInfo = NodeStateInfo[ev->Get()->Record.GetPeerName()]; if (CheckedMerge(nodeStateInfo, ev->Get()->Record) >= 100) { nodeStateInfo.SetChangeTime(ctx.Now().MilliSeconds()); } } - void Handle(TEvWhiteboard::TEvNodeStateDelete::TPtr &ev, const TActorContext &ctx) { + void Handle(TEvWhiteboard::TEvNodeStateDelete::TPtr &ev, const TActorContext &ctx) { auto& nodeStateInfo = NodeStateInfo[ev->Get()->Record.GetPeerName()]; if (nodeStateInfo.HasConnected()) { nodeStateInfo.ClearConnected(); @@ -600,10 +600,10 @@ protected: ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); } - void Handle(TEvWhiteboard::TEvNodeStateRequest::TPtr &ev, const TActorContext &ctx) { + void Handle(TEvWhiteboard::TEvNodeStateRequest::TPtr &ev, const TActorContext &ctx) { const auto& request = ev->Get()->Record; ui64 changedSince = request.HasChangedSince() ? request.GetChangedSince() : 0; - TAutoPtr<TEvWhiteboard::TEvNodeStateResponse> response = new TEvWhiteboard::TEvNodeStateResponse(); + TAutoPtr<TEvWhiteboard::TEvNodeStateResponse> response = new TEvWhiteboard::TEvNodeStateResponse(); auto& record = response->Record; for (const auto& pr : NodeStateInfo) { if (pr.second.GetChangeTime() >= changedSince) { @@ -615,8 +615,8 @@ protected: ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); } -// void Handle(TEvWhiteboard::TEvNodeStateRequest::TPtr &ev, const TActorContext &ctx) { -// TAutoPtr<TEvWhiteboard::TEvNodeStateResponse> response = new TEvWhiteboard::TEvNodeStateResponse(); +// void Handle(TEvWhiteboard::TEvNodeStateRequest::TPtr &ev, const TActorContext &ctx) { +// TAutoPtr<TEvWhiteboard::TEvNodeStateResponse> response = new TEvWhiteboard::TEvNodeStateResponse(); // auto& record = response->Record; // const TIntrusivePtr<NMonitoring::TDynamicCounters> &counters = AppData(ctx)->Counters; // TIntrusivePtr<NMonitoring::TDynamicCounters> interconnectCounters = GetServiceCounters(counters, "interconnect"); diff --git a/ydb/core/tablet/tablet_pipe_server.cpp b/ydb/core/tablet/tablet_pipe_server.cpp index b7c669e72c..99e4462cb0 100644 --- a/ydb/core/tablet/tablet_pipe_server.cpp +++ b/ydb/core/tablet/tablet_pipe_server.cpp @@ -136,10 +136,10 @@ namespace NTabletPipe { if (ev->HasEvent()) { result = new IEventHandle(ctx.SelfID, originalSender, ev->ReleaseBase().Release(), 0, ev->Cookie); } else { - result = new IEventHandle(ev->Type, 0, ctx.SelfID, - originalSender, - ev->ReleaseChainBuffer(), - ev->Cookie); + result = new IEventHandle(ev->Type, 0, ctx.SelfID, + originalSender, + ev->ReleaseChainBuffer(), + ev->Cookie); } result->Rewrite(ev->Type, RecipientId); diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 27a260bcb2..afd1f35e78 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -225,7 +225,7 @@ template <> struct NSchemeTypeMapper<NScheme::NTypeIds::String4k> : NSchemeTypeM template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Utf8> { typedef TString Type; }; template <> struct NSchemeTypeMapper<NScheme::NTypeIds::ActorId> { typedef TActorId Type; }; template <> struct NSchemeTypeMapper<NScheme::NTypeIds::PairUi64Ui64> { typedef std::pair<ui64, ui64> Type; }; -template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Double> { typedef double Type; }; +template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Double> { typedef double Type; }; template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Decimal> { typedef std::pair<ui64, i64> Type; }; template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Date> { typedef ui16 Type; }; template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Datetime> { typedef ui32 Type; }; diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 59afc46215..3fd11737c4 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -121,7 +121,7 @@ void TExecutor::PassAway() { if (Broker || Scans || Memory) { Send(NResourceBroker::MakeResourceBrokerID(), new NResourceBroker::TEvResourceBroker::TEvNotifyActorDied); } - + Scans->Drop(); Owner = nullptr; @@ -2741,7 +2741,7 @@ THolder<TScanSnapshot> TExecutor::PrepareScanSnapshot(ui32 table, const NTable:: TIntrusivePtr<TBarrier> barrier = new TBarrier(commit->Step); CommitManager->Commit(commit); - + TAutoPtr<NTable::TSubset> subset; if (params) { @@ -2907,7 +2907,7 @@ void TExecutor::Handle(NOps::TEvScanStat::TPtr &ev, const TActorContext &ctx) { void TExecutor::Handle(NOps::TEvResult::TPtr &ev) { auto *msg = ev->Get(); - + const auto outcome = Scans->Release(msg->Serial, msg->Status, msg->Result); if (outcome.System) { /* System scans are used for compactions and specially handled */ @@ -3452,7 +3452,7 @@ void TExecutor::DropScanSnapshot(ui64 snap) } ui64 TExecutor::QueueScan(ui32 tableId, TAutoPtr<NTable::IScan> scan, ui64 cookie, const TScanOptions& options) -{ +{ THolder<TScanSnapshot> snapshot; if (const auto* byId = std::get_if<TScanOptions::TSnapshotById>(&options.Snapshot)) { @@ -3473,9 +3473,9 @@ ui64 TExecutor::QueueScan(ui32 tableId, TAutoPtr<NTable::IScan> scan, ui64 cooki } snapshot = PrepareScanSnapshot(tableId, nullptr, rowVersion); } - + ui64 serial = Scans->Queue(tableId, scan, cookie, options, std::move(snapshot)); - + if (options.IsResourceBrokerDisabled()) { StartScan(serial, tableId); } @@ -3492,8 +3492,8 @@ bool TExecutor::CancelScan(ui32, ui64 serial) { } return false; -} - +} + TFinishedCompactionInfo TExecutor::GetFinishedCompactionInfo(ui32 tableId) const { if (CompactionLogic) { return CompactionLogic->GetFinishedCompactionInfo(tableId); diff --git a/ydb/core/tablet_flat/flat_executor.h b/ydb/core/tablet_flat/flat_executor.h index df08b3a338..896258a22e 100644 --- a/ydb/core/tablet_flat/flat_executor.h +++ b/ydb/core/tablet_flat/flat_executor.h @@ -516,7 +516,7 @@ class TExecutor void Handle(NBlockIO::TEvStat::TPtr &ev, const TActorContext &ctx); void Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled); void Handle(TEvBlobStorage::TEvGetResult::TPtr&, const TActorContext&); - + void UpdateUsedTabletMemory(); void UpdateCounters(const TActorContext &ctx); void UpdateYellow(); @@ -587,7 +587,7 @@ public: ui64 CompactMemTable(ui32 tableId) override; ui64 CompactTable(ui32 tableId) override; bool CompactTables() override; - + void FollowerAttached() override; void FollowerSyncComplete() override; void FollowerGcApplied(ui32 step, TDuration followerSyncDelay) override; diff --git a/ydb/core/tablet_flat/flat_executor_compaction_logic.cpp b/ydb/core/tablet_flat/flat_executor_compaction_logic.cpp index 3d699e8f87..a313f00b34 100644 --- a/ydb/core/tablet_flat/flat_executor_compaction_logic.cpp +++ b/ydb/core/tablet_flat/flat_executor_compaction_logic.cpp @@ -114,14 +114,14 @@ void TCompactionLogic::PrepareTableSnapshot(ui32 table, NTable::TSnapEdge edge, tableInfo->SnapRequests.emplace_back(TCompactionLogicState::TSnapRequest(edge, snapContext)); switch (inMem.State) { - case ECompactionState::Free: + case ECompactionState::Free: SubmitCompactionTask(table, 0, tableInfo->Policy->SnapshotResourceBrokerTask, tableInfo->Policy->DefaultTaskPriority, inMem.CompactionTask); inMem.State = ECompactionState::SnapshotPending; break; - case ECompactionState::Pending: + case ECompactionState::Pending: inMem.State = ECompactionState::SnapshotPending; break; case ECompactionState::PendingBackground: @@ -465,16 +465,16 @@ bool TCompactionLogic::BeginMemTableCompaction(ui64 taskId, ui32 tableId) "Unexpected BeginMemTableCompaction(%" PRIu64 ", %" PRIu32 ") for a dropped table", taskId, tableId); - TCompactionLogicState::TInMem &inMem = tableInfo->InMem; + TCompactionLogicState::TInMem &inMem = tableInfo->InMem; Y_VERIFY(taskId == inMem.CompactionTask.TaskId); NTable::TSnapEdge edge; - switch (inMem.State) { - case ECompactionState::Pending: + switch (inMem.State) { + case ECompactionState::Pending: case ECompactionState::PendingBackground: inMem.CompactingSteps = inMem.Steps; - inMem.State = ECompactionState::Compaction; + inMem.State = ECompactionState::Compaction; edge.Head = NTable::TEpoch::Max(); break; @@ -483,10 +483,10 @@ bool TCompactionLogic::BeginMemTableCompaction(ui64 taskId, ui32 tableId) inMem.State = ECompactionState::SnapshotCompaction; edge = tableInfo->SnapRequests.front().Edge; break; - - default: - Y_FAIL("Invalid inMem.State"); - } + + default: + Y_FAIL("Invalid inMem.State"); + } ui64 forcedCompactionId = 0; if (edge.Head == NTable::TEpoch::Max() && @@ -511,14 +511,14 @@ bool TCompactionLogic::BeginMemTableCompaction(ui64 taskId, ui32 tableId) inMem.CompactionTask.CompactionId = tableInfo->Strategy->BeginMemCompaction(taskId, edge, forcedCompactionId); return true; -} - +} + TCompactionLogicState::TTableInfo* TCompactionLogic::HandleCompaction( ui64 compactionId, const NTable::TCompactionParams* params, TTableCompactionResult* ret) -{ +{ const ui32 tableId = params->Table; const auto edge = params->Edge; @@ -530,9 +530,9 @@ TCompactionLogic::HandleCompaction( Y_VERIFY(params->TaskId == inMem.CompactionTask.TaskId); switch (inMem.State) { - case ECompactionState::Compaction: + case ECompactionState::Compaction: inMem.Steps -= std::exchange(inMem.CompactingSteps, 0); - inMem.State = ECompactionState::Free; + inMem.State = ECompactionState::Free; inMem.CompactionTask.TaskId = 0; inMem.CompactionTask.CompactionId = 0; break; @@ -544,7 +544,7 @@ TCompactionLogic::HandleCompaction( tableInfo->SnapRequests.pop_front(); } inMem.Steps -= std::exchange(inMem.CompactingSteps, 0); - inMem.State = ECompactionState::Free; + inMem.State = ECompactionState::Free; inMem.CompactionTask.TaskId = 0; inMem.CompactionTask.CompactionId = 0; break; @@ -604,9 +604,9 @@ TCompactionLogic::CompleteCompaction( ret.Changes = tableInfo->Strategy->CompactionFinished(compactionId, std::move(params), std::move(result)); ret.Strategy = tableInfo->StrategyType; - return ret; -} - + return ret; +} + void TCompactionLogic::CancelledCompaction( ui64 compactionId, diff --git a/ydb/core/tablet_flat/flat_executor_compaction_logic.h b/ydb/core/tablet_flat/flat_executor_compaction_logic.h index b85a13a19e..6a92c8b1c0 100644 --- a/ydb/core/tablet_flat/flat_executor_compaction_logic.h +++ b/ydb/core/tablet_flat/flat_executor_compaction_logic.h @@ -17,7 +17,7 @@ namespace NTable{ namespace NTabletFlatExecutor { -class TTableSnapshotContext; +class TTableSnapshotContext; using TCompactionPolicy = NLocalDb::TCompactionPolicy; @@ -27,16 +27,16 @@ enum class EForceCompaction { Full, }; -enum class ECompactionState { - Unknown, - Free, - Pending, +enum class ECompactionState { + Unknown, + Free, + Pending, PendingBackground, - Compaction, + Compaction, SnapshotPending, SnapshotCompaction, -}; - +}; + enum class EForcedCompactionState { None, PendingMem, @@ -78,7 +78,7 @@ struct TCompactionLogicState { ui32 TableId = Max<ui32>(); TInMem InMem; - + // This identifies currently active strategy type // The default value is used as a marker for uninitialized strategies NKikimrSchemeOp::ECompactionStrategy StrategyType = NKikimrSchemeOp::CompactionStrategyUnset; @@ -127,8 +127,8 @@ struct TCompactionLogicState { TMap<ui32, TTableInfo> Tables; THashMap<ui32, TSnapshotState> Snapshots; -}; - +}; + class TFlatTableScan; struct TTableCompactionResult { @@ -230,7 +230,7 @@ public: ui64 compactionId, THolder<NTable::TCompactionParams> params, THolder<NTable::TCompactionResult> result); - + void CancelledCompaction( ui64 compactionId, THolder<NTable::TCompactionParams> params); diff --git a/ydb/core/tablet_flat/flat_part_iface.h b/ydb/core/tablet_flat/flat_part_iface.h index 9be72fc735..f65378d1d2 100644 --- a/ydb/core/tablet_flat/flat_part_iface.h +++ b/ydb/core/tablet_flat/flat_part_iface.h @@ -8,8 +8,8 @@ #include <ydb/core/base/shared_data.h> #include <util/generic/string.h> -#include <util/system/types.h> - +#include <util/system/types.h> + namespace NKikimr { namespace NTable { diff --git a/ydb/core/tablet_flat/tablet_flat_executor.cpp b/ydb/core/tablet_flat/tablet_flat_executor.cpp index f69991ece9..a1d201f235 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.cpp +++ b/ydb/core/tablet_flat/tablet_flat_executor.cpp @@ -25,13 +25,13 @@ namespace NFlatExecutorSetup { } void ITablet::ScanComplete(NTable::EAbort status, TAutoPtr<IDestructable> prod, ui64 cookie, const TActorContext &ctx) - { + { Y_UNUSED(status); Y_UNUSED(prod); Y_UNUSED(cookie); - Y_UNUSED(ctx); - } - + Y_UNUSED(ctx); + } + bool ITablet::ReassignChannelsEnabled() const { // By default channels are reassigned automatically return true; diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h index 075ebce354..ad1f0ebf68 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.h +++ b/ydb/core/tablet_flat/tablet_flat_executor.h @@ -24,7 +24,7 @@ namespace NTabletFlatExecutor { class TTransactionContext; class TExecutor; struct TPageCollectionTxEnv; - + class TTableSnapshotContext : public TThrRefBase, TNonCopyable { friend class TExecutor; friend struct TPageCollectionTxEnv; @@ -444,7 +444,7 @@ namespace NFlatExecutorSetup { virtual void CompactionComplete(ui32 tableId, const TActorContext &ctx); // would be no-op in default implementation virtual void ScanComplete(NTable::EAbort status, TAutoPtr<IDestructable> prod, ui64 cookie, const TActorContext &ctx); - + virtual bool ReassignChannelsEnabled() const; // memory usage excluding transactions and executor cache. diff --git a/ydb/core/testlib/fake_coordinator.cpp b/ydb/core/testlib/fake_coordinator.cpp index 0be35b3bcf..31e3945816 100644 --- a/ydb/core/testlib/fake_coordinator.cpp +++ b/ydb/core/testlib/fake_coordinator.cpp @@ -1,5 +1,5 @@ -#include "fake_coordinator.h" -#include "tablet_helpers.h" +#include "fake_coordinator.h" +#include "tablet_helpers.h" namespace NKikimr { @@ -15,4 +15,4 @@ namespace NKikimr { runtime.DispatchEvents(options); } } -} +} diff --git a/ydb/core/testlib/fake_coordinator.h b/ydb/core/testlib/fake_coordinator.h index b3098d60a5..09bf857810 100644 --- a/ydb/core/testlib/fake_coordinator.h +++ b/ydb/core/testlib/fake_coordinator.h @@ -6,7 +6,7 @@ #include <ydb/core/engine/minikql/flat_local_tx_factory.h> namespace NKikimr { - + class TFakeCoordinator : public TActor<TFakeCoordinator>, public NTabletFlatExecutor::TTabletExecutedFlat { public: struct TState : TThrRefBase { @@ -229,4 +229,4 @@ namespace NKikimr { }; void BootFakeCoordinator(TTestActorRuntime& runtime, ui64 tabletId, TFakeCoordinator::TState::TPtr state); -} +} diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index d4907c26f1..f6baa14f6b 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -1532,17 +1532,17 @@ namespace Tests { NMsgBusProxy::EResponseStatus TClient::StoreTableBackup(const TString& parent, const NKikimrSchemeOp::TBackupTask& task) { TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); - auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); + auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); op->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpBackup); - op->SetWorkingDir(parent); - op->MutableBackup()->CopyFrom(task); - TAutoPtr<NBus::TBusMessage> reply; - NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply); - UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); + op->SetWorkingDir(parent); + op->MutableBackup()->CopyFrom(task); + TAutoPtr<NBus::TBusMessage> reply; + NBus::EMessageStatus status = SendAndWaitCompletion(request.Release(), reply); + UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); const NKikimrClient::TResponse &response = dynamic_cast<NMsgBusProxy::TBusResponse *>(reply.Get())->Record; - return (NMsgBusProxy::EResponseStatus)response.GetStatus(); - } - + return (NMsgBusProxy::EResponseStatus)response.GetStatus(); + } + NMsgBusProxy::EResponseStatus TClient::DeleteTable(const TString& parent, const TString& name) { TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation()); auto *op = request->Record.MutableTransaction()->MutableModifyScheme(); diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 137fa829f3..5cb7484785 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -10,8 +10,8 @@ OWNER( SRCS( actor_helpers.cpp actor_helpers.h - fake_coordinator.cpp - fake_coordinator.h + fake_coordinator.cpp + fake_coordinator.h fake_scheme_shard.h minikql_compile.h mock_pq_metacache.h diff --git a/ydb/core/tx/datashard/datashard.h b/ydb/core/tx/datashard/datashard.h index 5a93c03502..b8aa084824 100644 --- a/ydb/core/tx/datashard/datashard.h +++ b/ydb/core/tx/datashard/datashard.h @@ -197,7 +197,7 @@ struct TEvDataShard { EvUpdateConfig, EvSchemaChanged, EvStateChanged, - EvCancelBackup, + EvCancelBackup, EvMigrateSchemeShardRequest, EvMigrateSchemeShardResponse, EvCancelRestore, @@ -606,22 +606,22 @@ struct TEvDataShard { return std::make_pair(Record.GetStep(), Record.GetOrderId()); } - TString GetError() const { - if (Record.ErrorSize() > 0) { - TString result; - for (ui32 i = 0; i < Record.ErrorSize(); ++i) { - if (Record.GetError(i).HasReason()) { - result += Record.GetError(i).GetReason() + "|"; - } else { - result += "no reason|"; - } - } - return result; - } else { - return TString(); - } - } - + TString GetError() const { + if (Record.ErrorSize() > 0) { + TString result; + for (ui32 i = 0; i < Record.ErrorSize(); ++i) { + if (Record.GetError(i).HasReason()) { + result += Record.GetError(i).GetReason() + "|"; + } else { + result += "no reason|"; + } + } + return result; + } else { + return TString(); + } + } + void AddError(NKikimrTxDataShard::TError::EKind kind, const TStringBuf& reason, const TStringBuf& keyBuffer = TStringBuf()) { auto error = Record.MutableError()->Add(); error->SetKind(kind); @@ -761,20 +761,20 @@ struct TEvDataShard { Record.SetTabletId(tabletId); } }; - - struct TEvCancelBackup - : public TEventPB<TEvCancelBackup, - NKikimrTxDataShard::TEvCancelBackup, - TEvDataShard::EvCancelBackup> - { - TEvCancelBackup() = default; - + + struct TEvCancelBackup + : public TEventPB<TEvCancelBackup, + NKikimrTxDataShard::TEvCancelBackup, + TEvDataShard::EvCancelBackup> + { + TEvCancelBackup() = default; + explicit TEvCancelBackup(ui64 txid, ui64 tableId) { - Record.SetBackupTxId(txid); - Record.SetTableId(tableId); - } + Record.SetBackupTxId(txid); + Record.SetTableId(tableId); + } }; - + struct TEvCancelRestore : public TEventPB<TEvCancelRestore, NKikimrTxDataShard::TEvCancelRestore, @@ -799,7 +799,7 @@ struct TEvDataShard { Record.SetCollectKeySample(collectKeySample); } }; - + struct TEvGetTableStatsResult : public TEventPB<TEvGetTableStatsResult, NKikimrTxDataShard::TEvGetTableStatsResult, TEvDataShard::EvGetTableStatsResult> { @@ -809,7 +809,7 @@ struct TEvDataShard { Record.SetTableOwnerId(tableOwnerId); Record.SetTableLocalId(tableLocalId); } - }; + }; struct TEvPeriodicTableStats : public TEventPB<TEvPeriodicTableStats, NKikimrTxDataShard::TEvPeriodicTableStats, diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 11313474f8..92ede5842d 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -1253,7 +1253,7 @@ public: void SnapshotComplete(TIntrusivePtr<NTabletFlatExecutor::TTableSnapshotContext> snapContext, const TActorContext &ctx) override; void CompactionComplete(ui32 tableId, const TActorContext &ctx) override; void CompletedLoansChanged(const TActorContext &ctx) override; - + void ReplyCompactionWaiters(ui32 tableId, ui64 edge, const TActorContext &ctx); TUserTable::TSpecialUpdate SpecialUpdates(const NTable::TDatabase& db, const TTableId& tableId) const; @@ -1271,7 +1271,7 @@ public: void ScanComplete(NTable::EAbort status, TAutoPtr<IDestructable> prod, ui64 cookie, const TActorContext &ctx) override; bool ReassignChannelsEnabled() const override; ui64 GetMemoryUsage() const override; - + bool HasSharedBlobs() const; void CheckInitiateBorrowedPartsReturn(const TActorContext& ctx); void CheckStateChange(const TActorContext& ctx); @@ -2137,7 +2137,7 @@ protected: HFuncTraced(TEvDataShard::TEvStateChangedResult, Handle); HFuncTraced(TEvDataShard::TEvProposeTransaction, Handle); HFuncTraced(TEvDataShard::TEvProposeTransactionAttach, Handle); - HFuncTraced(TEvDataShard::TEvCancelBackup, Handle); + HFuncTraced(TEvDataShard::TEvCancelBackup, Handle); HFuncTraced(TEvDataShard::TEvCancelRestore, Handle); HFuncTraced(TEvDataShard::TEvGetS3Upload, Handle); HFuncTraced(TEvDataShard::TEvStoreS3UploadId, Handle); diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h index 67c35260b2..c0245a5eaa 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.h +++ b/ydb/core/tx/datashard/datashard_pipeline.h @@ -154,7 +154,7 @@ public: bool HasCreate() const { return SchemaTx && SchemaTx->IsCreate(); } bool HasAlter() const { return SchemaTx && SchemaTx->IsAlter(); } bool HasDrop() const { return SchemaTx && SchemaTx->IsDrop(); } - bool HasBackup() const { return SchemaTx && SchemaTx->IsBackup(); } + bool HasBackup() const { return SchemaTx && SchemaTx->IsBackup(); } bool HasRestore() const { return SchemaTx && SchemaTx->IsRestore(); } bool HasCopy() const { return SchemaTx && SchemaTx->IsCopy(); } bool HasCreatePersistentSnapshot() const { return SchemaTx && SchemaTx->IsCreatePersistentSnapshot(); } diff --git a/ydb/core/tx/mediator/execute_queue.cpp b/ydb/core/tx/mediator/execute_queue.cpp index 07740e9f82..5edc8d1d5b 100644 --- a/ydb/core/tx/mediator/execute_queue.cpp +++ b/ydb/core/tx/mediator/execute_queue.cpp @@ -168,7 +168,7 @@ namespace NTxMediator { void Bootstrap(const TActorContext &ctx) { Buckets.resize(BucketSelector.Buckets()); for (ui32 bucketIdx = 0; bucketIdx < Buckets.size(); ++bucketIdx) - Buckets[bucketIdx].ActiveActor = ctx.ExecutorThread.RegisterActor(CreateTxMediatorTabletQueue(ctx.SelfID, MediatorId, 1, bucketIdx), TMailboxType::ReadAsFilled); + Buckets[bucketIdx].ActiveActor = ctx.ExecutorThread.RegisterActor(CreateTxMediatorTabletQueue(ctx.SelfID, MediatorId, 1, bucketIdx), TMailboxType::ReadAsFilled); } public: diff --git a/ydb/core/tx/mediator/tablet_queue.cpp b/ydb/core/tx/mediator/tablet_queue.cpp index b28179517c..371be3a925 100644 --- a/ydb/core/tx/mediator/tablet_queue.cpp +++ b/ydb/core/tx/mediator/tablet_queue.cpp @@ -89,7 +89,7 @@ class TTxMediatorTabletQueue : public TActor<TTxMediatorTabletQueue> { x->SetModerator(tx.Moderator); ActorIdToProto(tx.AckTo, x->MutableAckTo()); LOG_DEBUG(ctx, NKikimrServices::TX_MEDIATOR_PRIVATE, "Send from %" PRIu64 " to tablet %" PRIu64 ", step# %" - PRIu64 ", txid# %" PRIu64 ", marker M5" PRIu64, Mediator, tablet, tabletStep->StepRef->Step, tx.TxId); + PRIu64 ", txid# %" PRIu64 ", marker M5" PRIu64, Mediator, tablet, tabletStep->StepRef->Step, tx.TxId); } LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TABLETQUEUE, "Actor# " << ctx.SelfID.ToString() << " Mediator# " << Mediator << " SEND to# " << tablet << " " << evx->ToString()); diff --git a/ydb/core/tx/schemeshard/schemeshard.h b/ydb/core/tx/schemeshard/schemeshard.h index d33173508c..4733f2270f 100644 --- a/ydb/core/tx/schemeshard/schemeshard.h +++ b/ydb/core/tx/schemeshard/schemeshard.h @@ -56,11 +56,11 @@ struct TEvSchemeShard { EvLoginResult, - EvBackupDatashard = EvModifySchemeTransaction + 6 * 512, - EvBackupDatashardResult, + EvBackupDatashard = EvModifySchemeTransaction + 6 * 512, + EvBackupDatashardResult, EvCancelTx, EvCancelTxResult, - + EvEnd }; @@ -96,21 +96,21 @@ struct TEvSchemeShard { : public TEventPB<TEvCancelTx, NKikimrScheme::TEvCancelTx, EvCancelTx> - { - }; - + { + }; + struct TEvCancelTxResult: public TEventPB<TEvCancelTxResult, NKikimrScheme::TEvCancelTxResult, EvCancelTxResult> - { + { TEvCancelTxResult() = default; TEvCancelTxResult(ui64 targetTxId, ui64 txId) { Record.SetTargetTxId(targetTxId); Record.SetTxId(txId); - } - }; - + } + }; + using EStatus = NKikimrScheme::EStatus; struct TEvModifySchemeTransactionResult : public TEventPB<TEvModifySchemeTransactionResult, diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 650506549e..7ff304ffdf 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -3191,21 +3191,21 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto byShardBackupStatus = db.Table<Schema::ShardBackupStatus>().Range(operationId.GetTxId()).Select(); auto byMigratedShardBackupStatus = db.Table<Schema::MigratedShardBackupStatus>().Range(operationId.GetTxId()).Select(); auto byTxShardStatus = db.Table<Schema::TxShardStatus>().Range(operationId.GetTxId()).Select(); - + TShardBackupStatusRows statuses; if (!LoadBackupStatusesImpl(statuses, byShardBackupStatus, byMigratedShardBackupStatus, byTxShardStatus)) { - return false; - } - + return false; + } + for (auto& rec : statuses) { auto shardIdx = std::get<1>(rec); auto success = std::get<2>(rec); auto error = std::get<3>(rec); auto bytes = std::get<4>(rec); auto rows = std::get<5>(rec); - + txState.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); - } + } } else if (txState.TxType == TTxState::TxForceDropSubDomain || txState.TxType == TTxState::TxForceDropExtSubDomain) { forceDropOpIds.push_back(operationId); } else if (txState.TxType == TTxState::TxAlterUserAttributes) { @@ -3249,7 +3249,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { << ", txId: " << operationId.GetTxId() << ", TxType: " << TTxState::TypeName(txState.TxType) << ", LastTxId: " << path->LastTxId); - + if (!Self->Operations.contains(operationId.GetTxId())) { Self->Operations[operationId.GetTxId()] = new TOperation(operationId.GetTxId()); } @@ -3419,12 +3419,12 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } } - // Read backup settings - { + // Read backup settings + { TBackupSettingsRows backupSettings; if (!LoadBackupSettings(db, backupSettings)) { - return false; - } + return false; + } LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for BackupSettings" @@ -3442,14 +3442,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 nRetries = std::get<7>(rec); Y_VERIFY(tableName.size() > 0); - + TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); - Y_VERIFY(tableInfo.Get() != nullptr); + Y_VERIFY(tableInfo.Get() != nullptr); - tableInfo->BackupSettings.SetTableName(tableName); + tableInfo->BackupSettings.SetTableName(tableName); tableInfo->BackupSettings.SetNeedToBill(needToBill); tableInfo->BackupSettings.SetNumberOfRetries(nRetries); - + if (ytSerializedSettings) { auto settings = tableInfo->BackupSettings.MutableYTSettings(); Y_VERIFY(ParseFromStringNoSizeLimit(*settings, ytSerializedSettings)); @@ -3473,11 +3473,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Loaded backup settings" << ", pathId: " << pathId << ", tablename: " << tableName.data()); - } - } - + } + } + // Read restore tasks - { + { auto rowSet = db.Table<Schema::RestoreTasks>().Range().Select(); if (!rowSet.IsReady()) { return false; @@ -3565,25 +3565,25 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { { TShardBackupStatusRows backupStatuses; if (!LoadBackupStatuses(db, backupStatuses)) { - return false; - } - + return false; + } + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for ShardBackupStatus" << ", readed records: " << backupStatuses.size() << ", at schemeshard: " << Self->TabletID()); - + THashMap<TTxId, TShardBackupStatusRows> statusesByTxId; for (auto& rec: backupStatuses) { TTxId txId = std::get<0>(rec); statusesByTxId[txId].push_back(rec); } - + TCompletedBackupRestoreRows history; if (!LoadBackupRestoreHistory(db, history)) { return false; } - + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxInit for CompletedBackup" << ", readed records: " << history.size() @@ -3606,17 +3606,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { info.SuccessShardCount = successShardsCount; info.StartDateTime = startTime; info.DataTotalSize = dataSize; - - if (!Self->Tables.FindPtr(pathId)) { + + if (!Self->Tables.FindPtr(pathId)) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Skip record in CompletedBackups" << ", pathId: " << pathId << ", txid: " << txId); - continue; - } + continue; + } TTableInfo::TPtr tableInfo = Self->Tables.at(pathId); - + if (statusesByTxId.contains(txId)) { for (auto& recByTxId: statusesByTxId.at(txId)) { auto shardIdx = std::get<1>(recByTxId); @@ -3624,11 +3624,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto error = std::get<3>(recByTxId); auto bytes = std::get<4>(recByTxId); auto rows = std::get<5>(recByTxId); - + info.ShardStatuses[shardIdx] = TTxState::TShardStatus(success, error, bytes, rows); - } - } - + } + } + switch (kind) { case TTableInfo::TBackupRestoreResult::EKind::Backup: tableInfo->BackupHistory[txId] = std::move(info); @@ -3637,14 +3637,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->RestoreHistory[txId] = std::move(info); break; } - + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Loaded completed backup status" << ", pathId: " << pathId << ", txid: " << txId); - } - } - + } + } + // Other persistent params for (const auto& si : Self->ShardInfos) { auto shardIdx = si.first; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 0715543a22..ab46cdf23d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -2616,10 +2616,10 @@ void TSchemeShard::PersistTxShardStatus(NIceDb::TNiceDb& db, TOperationId opId, } void TSchemeShard::PersistBackupSettings( - NIceDb::TNiceDb& db, - TPathId pathId, + NIceDb::TNiceDb& db, + TPathId pathId, const NKikimrSchemeOp::TBackupTask& settings) -{ +{ #define PERSIST_BACKUP_SETTINGS(Kind) \ if (settings.Has##Kind()) { \ if (IsLocalId(pathId)) { \ @@ -2640,13 +2640,13 @@ void TSchemeShard::PersistBackupSettings( NIceDb::TUpdate<Schema::MigratedBackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \ } \ } - + PERSIST_BACKUP_SETTINGS(YTSettings) PERSIST_BACKUP_SETTINGS(S3Settings) #undef PERSIST_BACKUP_SETTINGS -} - +} + void TSchemeShard::PersistCompletedBackupRestore(NIceDb::TNiceDb& db, TTxId txId, const TTxState& txState, const TTableInfo::TBackupRestoreResult& info, TTableInfo::TBackupRestoreResult::EKind kind) { TPathId pathId = txState.TargetPathId; @@ -2684,8 +2684,8 @@ void TSchemeShard::PersistBackupDone(NIceDb::TNiceDb& db, TPathId pathId) { } db.Table<Schema::MigratedBackupSettings>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); -} - +} + void TSchemeShard::PersistBlockStorePartition(NIceDb::TNiceDb& db, TPathId pathId, ui32 partitionId, TShardIdx shardIdx, ui64 version) { Y_VERIFY(IsLocalId(pathId)); @@ -2693,7 +2693,7 @@ void TSchemeShard::PersistBlockStorePartition(NIceDb::TNiceDb& db, TPathId pathI NIceDb::TUpdate<Schema::BlockStorePartitions::ShardIdx>(shardIdx.GetLocalId()), NIceDb::TUpdate<Schema::BlockStorePartitions::AlterVersion>(version)); } - + void TSchemeShard::PersistBlockStoreVolume(NIceDb::TNiceDb& db, TPathId pathId, const TBlockStoreVolumeInfo::TPtr volume) { Y_VERIFY(IsLocalId(pathId)); @@ -5268,17 +5268,17 @@ void TSchemeShard::Handle(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext } void TSchemeShard::Handle(TEvSchemeShard::TEvCancelTx::TPtr& ev, const TActorContext& ctx) { - if (IsReadOnlyMode) { + if (IsReadOnlyMode) { LOG_ERROR_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Ignoring messgage TEvSchemeShard::TEvCancelTx" << " reason# schemeshard in readonly" << " schemeshard# " << TabletID()); - return; - } - + return; + } + Execute(CreateTxOperationPropose(ev), ctx); -} - +} + void TSchemeShard::Handle(TEvSchemeShard::TEvPublishTenantAsReadOnly::TPtr &ev, const TActorContext &ctx) { Execute(CreateTxPublishTenantAsReadOnly(ev), ctx); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index e1a1d08b9a..8ced3e2053 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -753,7 +753,7 @@ public: void Handle(TEvSchemeShard::TEvNotifyTxCompletion::TPtr &ev, const TActorContext &ctx); void Handle(TEvSchemeShard::TEvCancelTx::TPtr& ev, const TActorContext& ctx); - + void Handle(TEvPrivate::TEvProgressOperation::TPtr &ev, const TActorContext &ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index a76dcf9d5b..8af5912a6b 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -1,15 +1,15 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard_utils.h> - + #include <ydb/core/base/compile_time_flags.h> #include <util/generic/size_literals.h> #include <util/string/cast.h> -using namespace NKikimr; +using namespace NKikimr; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - +using namespace NSchemeShardUT_Private; + Y_UNIT_TEST_SUITE(TSchemeShardTest) { Y_UNIT_TEST(Boot) { TTestBasicRuntime runtime; @@ -3781,7 +3781,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::CheckColumns("Table", cols, dropCols, keyCol)}); - Cdbg << "AlterTable: add column" << Endl; + Cdbg << "AlterTable: add column" << Endl; cols.insert("add_1"); cols.insert("add_2"); cols.insert("add_50"); @@ -3796,7 +3796,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::CheckColumns("Table", cols, dropCols, keyCol)}); - Cdbg << "AlterTable: add existed column" << Endl; + Cdbg << "AlterTable: add existed column" << Endl; TestAlterTable(runtime, ++txId, "/MyRoot", R"(Name: "Table" Columns { Name: "value" Type: "Utf8"})", {NKikimrScheme::StatusSchemeError, NKikimrScheme::StatusInvalidParameter}); @@ -3830,7 +3830,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::CheckColumns("Table", cols, dropCols, keyCol)}); - Cdbg << "AlterTable: drop column" << Endl; + Cdbg << "AlterTable: drop column" << Endl; cols.erase("value"); cols.erase("add_2"); dropCols.insert("value"); @@ -3851,12 +3851,12 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::CheckColumns("Table", cols, dropCols, keyCol)}); - Cdbg << "AlterTable: drop dropped column" << Endl; + Cdbg << "AlterTable: drop dropped column" << Endl; TestAlterTable(runtime, ++txId, "/MyRoot", R"(Name: "Table" DropColumns { Name: "add_2"})", {NKikimrScheme::StatusSchemeError, NKikimrScheme::StatusInvalidParameter}); - Cdbg << "AlterTable: drop key column" << Endl; + Cdbg << "AlterTable: drop key column" << Endl; TestAlterTable(runtime, ++txId, "/MyRoot", R"(Name: "Table" DropColumns { Name: "key1" })", {NKikimrScheme::StatusSchemeError, NKikimrScheme::StatusInvalidParameter}); @@ -3866,7 +3866,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { R"(Name: "Table" DropColumns { Id: 3 })", {NKikimrScheme::StatusSchemeError, NKikimrScheme::StatusInvalidParameter}); - Cdbg << "AlterTable: add + drop different column" << Endl; + Cdbg << "AlterTable: add + drop different column" << Endl; cols.insert("add_3"); dropCols.insert("add_1"); TestAlterTable(runtime, ++txId, "/MyRoot", @@ -3879,14 +3879,14 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::CheckColumns("Table", cols, dropCols, keyCol)}); - Cdbg << "AlterTable: add + drop same column (exist)" << Endl; + Cdbg << "AlterTable: add + drop same column (exist)" << Endl; TestAlterTable(runtime, ++txId, "/MyRoot", R"(Name: "Table" Columns { Name: "add_3" Type: "Uint32"} DropColumns { Name: "add_3" } )", {NKikimrScheme::StatusSchemeError, NKikimrScheme::StatusInvalidParameter}); - Cdbg << "AlterTable: add + drop same column (not exist)" << Endl; + Cdbg << "AlterTable: add + drop same column (not exist)" << Endl; TestAlterTable(runtime, ++txId, "/MyRoot", R"(Name: "Table" Columns { Name: "add_4" Type: "Uint32"} @@ -6258,7 +6258,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropTable(runtime, ++txId, "/MyRoot/Ops", "Table", {NKikimrScheme::StatusPathDoesNotExist}); - Cdbg << "Create, Drop (simple table)" << Endl; + Cdbg << "Create, Drop (simple table)" << Endl; TestCreateTable(runtime, ++txId, "/MyRoot/Ops", tcfg1); env.TestWaitNotification(runtime, txId); @@ -6278,7 +6278,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { env.TestWaitTabletDeletion(runtime, TTestTxConfig::FakeHiveTablets); - Cdbg << "Create, Drop (partitioned table)" << Endl; + Cdbg << "Create, Drop (partitioned table)" << Endl; TestCreateTable(runtime, ++txId, "/MyRoot/Ops", tcfg2); env.TestWaitNotification(runtime, txId); @@ -6417,7 +6417,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { // - Cdbg << "Create + Drop + Drop" << Endl; + Cdbg << "Create + Drop + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot/Ops", pqGroupConfig); auto pVer = TestDescribeResult(DescribePath(runtime, "/MyRoot/Ops/DropMeBaby")); AsyncDropPQGroup(runtime, ++txId, "/MyRoot/Ops", "DropMeBaby"); @@ -6463,7 +6463,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TString pqGroupAlter = "Name: \"Isolda\"" "TotalGroupCount: 100 "; - Cdbg << "Create + Drop + Create" << Endl; + Cdbg << "Create + Drop + Create" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); @@ -6475,7 +6475,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Create + Drop" << Endl; + Cdbg << "Create + Create + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); @@ -6487,7 +6487,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Alter + Drop" << Endl; + Cdbg << "Create + Alter + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncAlterPQGroup(runtime, ++txId, "/MyRoot", pqGroupAlter); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); @@ -6499,7 +6499,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Drop + Alter" << Endl; + Cdbg << "Create + Drop + Alter" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); AsyncAlterPQGroup(runtime, ++txId, "/MyRoot", pqGroupAlter); @@ -6511,7 +6511,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Drop + Drop" << Endl; + Cdbg << "Create + Drop + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); AsyncDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); @@ -6523,7 +6523,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Alter + Create + Drop" << Endl; + Cdbg << "Create + Alter + Create + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncAlterPQGroup(runtime, ++txId, "/MyRoot", pqGroupAlter); AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); @@ -6537,7 +6537,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { TestDropPQGroup(runtime, ++txId, "/MyRoot", "Isolda"); env.TestWaitNotification(runtime, txId); - Cdbg << "Create + Alter + Alter + Drop" << Endl; + Cdbg << "Create + Alter + Alter + Drop" << Endl; AsyncCreatePQGroup(runtime, ++txId, "/MyRoot", pqGroupConfig); AsyncAlterPQGroup(runtime, ++txId, "/MyRoot", pqGroupAlter); AsyncAlterPQGroup(runtime, ++txId, "/MyRoot", pqGroupAlter); @@ -7024,7 +7024,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { if (!a.IsNull()) { TString aVal(a.Data(), a.Size()); TString bVal(b.Data(), b.Size()); -// Cdbg << aVal << Endl; +// Cdbg << aVal << Endl; UNIT_ASSERT_VALUES_EQUAL_C(aVal, bVal, "data mismatch"); } } diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 6f999b268b..762b34dedc 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -1,5 +1,5 @@ #include "helpers.h" - + #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/tx/schemeshard/schemeshard.h> diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index 6b12c25534..6876e593d9 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include "ls_checks.h" #include "test_env.h" @@ -16,18 +16,18 @@ #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> -#include <util/stream/null.h> +#include <util/stream/null.h> #include <functional> -#undef Cdbg +#undef Cdbg #if 1 #define Cdbg Cerr #else #define Cdbg Cnull #endif - -namespace NSchemeShardUT_Private { + +namespace NSchemeShardUT_Private { using namespace NKikimr; using TEvTx = TEvSchemeShard::TEvModifySchemeTransaction; @@ -36,7 +36,7 @@ namespace NSchemeShardUT_Private { NKikimrProto::EReplyStatus LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query, NKikimrMiniKQL::TResult& result, TString& err); NKikimrMiniKQL::TResult LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query); NKikimrProto::EReplyStatus LocalSchemeTx(TTestActorRuntime& runtime, ui64 tabletId, const TString& schemeChangesStr, bool dryRun, NTabletFlatScheme::TSchemeChanges& scheme, TString& err); - + bool CheckLocalRowExists(TTestActorRuntime& runtime, ui64 tabletId, const TString& tableName, const TString& keyColumn, ui64 keyValue); ////////// describe options @@ -62,7 +62,7 @@ namespace NSchemeShardUT_Private { TString TestLs(TTestActorRuntime& runtime, const TString& path, bool returnPartitioningInfo = false, NLs::TCheckFunc check = nullptr); TString TestLs(TTestActorRuntime& runtime, const TString& path, const NKikimrSchemeOp::TDescribeOptions& opts, NLs::TCheckFunc check = nullptr); TString TestLsPathId(TTestActorRuntime& runtime, ui64 pathId, NLs::TCheckFunc check = nullptr); - + ////////// modification results void CheckExpected(const TVector<TEvSchemeShard::EStatus>& expected, TEvSchemeShard::EStatus result, const TString& reason); void CheckExpected(const TVector<Ydb::StatusIds::StatusCode>& expected, Ydb::StatusIds::StatusCode result, const TString& reason); @@ -465,4 +465,4 @@ namespace NSchemeShardUT_Private { TTestActorRuntimeBase::TEventObserver SetSuppressObserver(TTestActorRuntime& runtime, TVector<THolder<IEventHandle>>& suppressed, ui32 type); void WaitForSuppressed(TTestActorRuntime& runtime, TVector<THolder<IEventHandle>>& suppressed, ui32 count, TTestActorRuntime::TEventObserver prevObserver); -} //NSchemeShardUT_Private +} //NSchemeShardUT_Private diff --git a/ydb/core/tx/schemeshard/ut_split_merge.cpp b/ydb/core/tx/schemeshard/ut_split_merge.cpp index 7d91e7ae31..55fefc4f2b 100644 --- a/ydb/core/tx/schemeshard/ut_split_merge.cpp +++ b/ydb/core/tx/schemeshard/ut_split_merge.cpp @@ -1,12 +1,12 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> - + #include <ydb/library/yql/minikql/mkql_node.h> -using namespace NKikimr; +using namespace NKikimr; using namespace NKikimr::NMiniKQL; using namespace NSchemeShard; -using namespace NSchemeShardUT_Private; - +using namespace NSchemeShardUT_Private; + Y_UNIT_TEST_SUITE(TSchemeShardSplitTest) { Y_UNIT_TEST(Test) { } diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index deb38ca2b7..ade2d45c29 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -238,7 +238,7 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct auto entry = result->Record.MutablePathDescription()->AddChildren(); FillRootDescr(entry, domain.second->Name, domain.second->SchemeRoot); } - + ctx.Send(Source, result.Release(), 0, SourceCookie); return Die(ctx); } @@ -247,7 +247,7 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct if (!record.GetUserToken().empty()) { UserToken = new NACLib::TUserToken(record.GetUserToken()); } - + if (UserToken == nullptr && record.GetDescribePath().HasPathId()) { TAutoPtr<NSchemeShard::TEvSchemeShard::TEvDescribeScheme> req = new NSchemeShard::TEvSchemeShard::TEvDescribeScheme( @@ -255,11 +255,11 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct record.GetDescribePath().GetPathId()); const ui64 shardToRequest = record.GetDescribePath().GetSchemeshardId(); - if (record.GetDescribePath().HasOptions()) { - auto options = req->Record.MutableOptions(); - options->CopyFrom(record.GetDescribePath().GetOptions()); - } - + if (record.GetDescribePath().HasOptions()) { + auto options = req->Record.MutableOptions(); + options->CopyFrom(record.GetDescribePath().GetOptions()); + } + LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, "Actor# " << ctx.SelfID.ToString() << " SEND to# " << shardToRequest << " shardToRequest " << req->ToString()); diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h index 86d4f1ac79..3d5e0f723f 100644 --- a/ydb/core/tx/tx_proxy/proxy.h +++ b/ydb/core/tx/tx_proxy/proxy.h @@ -44,7 +44,7 @@ struct TEvTxUserProxy { EvCancelBackupRequestDeprecated, EvCancelBackupResultDeprecated, - + EvProposeKqpTransaction, EvAllocateTxId, @@ -170,7 +170,7 @@ struct TEvTxUserProxy { }; struct TEvInvalidateTableResult : public TEventSimple<TEvInvalidateTableResult, EvInvalidateTableResult> {}; - + struct TEvProposeKqpTransaction : public TEventLocal<TEvProposeKqpTransaction, EvProposeKqpTransaction> { TActorId ExecuterId; diff --git a/ydb/core/viewer/content/viewer.js b/ydb/core/viewer/content/viewer.js index 1108f42278..e3e397648a 100644 --- a/ydb/core/viewer/content/viewer.js +++ b/ydb/core/viewer/content/viewer.js @@ -3076,7 +3076,7 @@ function onTreeNodeComplete(result, obj) { name.innerHTML = "Life Time"; value.innerHTML = partitionConfig.LifetimeSeconds + " seconds"; } - + if ((result.PathDescription.Self.PathType === 10 || result.PathDescription.Self.PathType === 4) && result.PathDescription.DomainDescription !== undefined) { if (result.PathDescription.DomainDescription.ProcessingParams !== undefined) { if (result.PathDescription.DomainDescription.ProcessingParams.Version !== undefined) { @@ -3123,39 +3123,39 @@ function onTreeNodeComplete(result, obj) { } if (result.PathDescription.BackupProgress !== undefined) { - row = tab.insertRow(); - name = row.insertCell(-1); - value = row.insertCell(-1); - name.innerHTML = "Backup progress"; - - var total = result.PathDescription.BackupProgress.Total; - var complete = total - - result.PathDescription.BackupProgress.NotCompleteYet; - var procent = Math.round(complete * 100 / total); - value.innerHTML = complete + "/" + total + " (" + procent + "%)"; - } - - var backupResultList = result.PathDescription.LastBackupResult; + row = tab.insertRow(); + name = row.insertCell(-1); + value = row.insertCell(-1); + name.innerHTML = "Backup progress"; + + var total = result.PathDescription.BackupProgress.Total; + var complete = total - + result.PathDescription.BackupProgress.NotCompleteYet; + var procent = Math.round(complete * 100 / total); + value.innerHTML = complete + "/" + total + " (" + procent + "%)"; + } + + var backupResultList = result.PathDescription.LastBackupResult; if (backupResultList !== undefined && backupResultList.length > 0) { - var backupResult = backupResultList[backupResultList.length - 1]; + var backupResult = backupResultList[backupResultList.length - 1]; if (backupResult.CompleteTimeStamp !== undefined) { - row = tab.insertRow(); - name = row.insertCell(-1); - value = row.insertCell(-1); - name.innerHTML = "Last backup time"; - var timestampMS = backupResult.CompleteTimeStamp * 1000; - var dateObj = new Date(timestampMS); - value.innerHTML = dateObj.toLocaleString(); - } - + row = tab.insertRow(); + name = row.insertCell(-1); + value = row.insertCell(-1); + name.innerHTML = "Last backup time"; + var timestampMS = backupResult.CompleteTimeStamp * 1000; + var dateObj = new Date(timestampMS); + value.innerHTML = dateObj.toLocaleString(); + } + if (backupResult.ErrorCount !== undefined) { - row = tab.insertRow(); - name = row.insertCell(-1); - value = row.insertCell(-1); - name.innerHTML = "Last backup error count"; - value.innerHTML = backupResult.ErrorCount; - } - } + row = tab.insertRow(); + name = row.insertCell(-1); + value = row.insertCell(-1); + name.innerHTML = "Last backup error count"; + value.innerHTML = backupResult.ErrorCount; + } + } } function onTreeNodeSelected(e, data) { diff --git a/ydb/core/viewer/json_describe.h b/ydb/core/viewer/json_describe.h index 05ef441f8f..fd23dc7ab6 100644 --- a/ydb/core/viewer/json_describe.h +++ b/ydb/core/viewer/json_describe.h @@ -57,7 +57,7 @@ public: JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); InitConfig(params); - + if (params.Has("schemeshard_id")) { THolder<TEvSchemeShard::TEvDescribeScheme> request = MakeHolder<TEvSchemeShard::TEvDescribeScheme>(); FillParams(&request->Record, params); diff --git a/ydb/core/viewer/json_nodeinfo.h b/ydb/core/viewer/json_nodeinfo.h index 60245326fa..0bc598c0dd 100644 --- a/ydb/core/viewer/json_nodeinfo.h +++ b/ydb/core/viewer/json_nodeinfo.h @@ -11,8 +11,8 @@ namespace NKikimr { namespace NViewer { template <> -struct TWhiteboardInfo<TEvWhiteboard::TEvNodeStateResponse> { - using TResponseType = TEvWhiteboard::TEvNodeStateResponse; +struct TWhiteboardInfo<TEvWhiteboard::TEvNodeStateResponse> { + using TResponseType = TEvWhiteboard::TEvNodeStateResponse; using TElementType = NKikimrWhiteboard::TNodeStateInfo; using TElementKeyType = TString; @@ -42,7 +42,7 @@ struct TWhiteboardInfo<TEvWhiteboard::TEvNodeStateResponse> { } }; -using TJsonNodeInfo = TJsonWhiteboardRequest<TEvWhiteboard::TEvNodeStateRequest, TEvWhiteboard::TEvNodeStateResponse>; +using TJsonNodeInfo = TJsonWhiteboardRequest<TEvWhiteboard::TEvNodeStateRequest, TEvWhiteboard::TEvNodeStateResponse>; template <> struct TJsonRequestSummary<TJsonNodeInfo> { diff --git a/ydb/core/viewer/json_wb_req.h b/ydb/core/viewer/json_wb_req.h index f04c85688f..25fcfaea28 100644 --- a/ydb/core/viewer/json_wb_req.h +++ b/ydb/core/viewer/json_wb_req.h @@ -202,7 +202,7 @@ public: } } NodesRequestedTime = AppData()->TimeProvider->Now(); - const TEvWhiteboard::TEvNodeStateResponse* nodesInfo = ev->Get(); + const TEvWhiteboard::TEvNodeStateResponse* nodesInfo = ev->Get(); for (const auto& ni : nodesInfo->Record.GetNodeStateInfo()) { if (ni.GetConnected()) { TNodeId nodeId = GetNodeIdFromPeerName(ni.GetPeerName()); diff --git a/ydb/public/lib/base/msgbus.cpp b/ydb/public/lib/base/msgbus.cpp index 3de1861098..30587e2dbd 100644 --- a/ydb/public/lib/base/msgbus.cpp +++ b/ydb/public/lib/base/msgbus.cpp @@ -40,24 +40,24 @@ void ExplainExecutionEngineStatus(ui32 status, TString& name, TString& descripti } } -void ExplainResponseStatus(ui32 status, TString& name, TString& description) { - name = ToString(status); - description = "Unknown status"; - - auto field = NKikimrClient::TResponse::descriptor()->FindFieldByNumber( - NKikimrClient::TResponse::kStatusFieldNumber); - - auto extension = - field->options().GetExtension(NKikimrClient::EnumValueHint); - - for (auto item : extension.GetHints()) { - if (status == item.GetValue()) { - name = item.GetName(); - description = item.GetMan(); - break; - } - } -} - +void ExplainResponseStatus(ui32 status, TString& name, TString& description) { + name = ToString(status); + description = "Unknown status"; + + auto field = NKikimrClient::TResponse::descriptor()->FindFieldByNumber( + NKikimrClient::TResponse::kStatusFieldNumber); + + auto extension = + field->options().GetExtension(NKikimrClient::EnumValueHint); + + for (auto item : extension.GetHints()) { + if (status == item.GetValue()) { + name = item.GetName(); + description = item.GetMan(); + break; + } + } +} + } // namespace NKikimr } // namespace NMsgBusProxy diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h index f11bd9771f..755c82a869 100644 --- a/ydb/public/lib/base/msgbus.h +++ b/ydb/public/lib/base/msgbus.h @@ -66,7 +66,7 @@ enum { MTYPE_CLIENT_GET_RESPONSE = 10455, MTYPE_CLIENT_DB_QUERY = 10456, MTYPE_CLIENT_TABLET_COUNTERS_REQUEST = 10457, - MTYPE_CLIENT_CANCEL_BACKUP = 10458, + MTYPE_CLIENT_CANCEL_BACKUP = 10458, MTYPE_CLIENT_BLOB_STORAGE_CONFIG_REQUEST = 10459, MTYPE_CLIENT_DRAIN_NODE = 10460, MTYPE_CLIENT_FILL_NODE = 10461, diff --git a/ydb/public/lib/base/msgbus_status.h b/ydb/public/lib/base/msgbus_status.h index 15876b170d..1941d47564 100644 --- a/ydb/public/lib/base/msgbus_status.h +++ b/ydb/public/lib/base/msgbus_status.h @@ -25,6 +25,6 @@ ENUM_TO_STRING(EResponseStatus, RESPONSE_STATUS_MAP) void ExplainProposeTransactionStatus(ui32 status, TString& name, TString& description); void ExplainExecutionEngineStatus(ui32 status, TString& name, TString& description); -void ExplainResponseStatus(ui32 status, TString& name, TString& description); +void ExplainResponseStatus(ui32 status, TString& name, TString& description); }} |