diff options
| author | kruall <[email protected]> | 2022-02-10 16:50:43 +0300 |
|---|---|---|
| committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:50:43 +0300 |
| commit | 08510f0e20c4cccf75a4a7577b1471638c521f08 (patch) | |
| tree | 9fd60922961d950d6761fcb0c694bcfca6594c9a | |
| parent | ce384ae51a2d2c465e4e0fe4fe0346e382459bfe (diff) | |
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
185 files changed, 18847 insertions, 18847 deletions
diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index ed29bd14b9e..dad2bc59c13 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -207,8 +207,8 @@ namespace NActors { virtual TActorId RegisterWithSameMailbox(IActor*) const noexcept = 0; }; - class TDecorator; - + class TDecorator; + class IActor : protected IActorOps { public: typedef void (IActor::*TReceiveFunc)(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); @@ -220,7 +220,7 @@ namespace NActors { ui64 HandledEvents; friend void DoActorInit(TActorSystem*, IActor*, const TActorId&, const TActorId&); - friend class TDecorator; + friend class TDecorator; public: /// @sa services.proto NKikimrServices::TActivity::EType @@ -376,11 +376,11 @@ namespace NActors { TActorId RegisterWithSameMailbox(IActor* actor) const noexcept final; std::pair<ui32, ui32> CountMailboxEvents(ui32 maxTraverse = Max<ui32>()) const; - - private: - void ChangeSelfId(TActorId actorId) { - SelfActorId = actorId; - } + + private: + void ChangeSelfId(TActorId actorId) { + SelfActorId = actorId; + } }; struct TActorActivityTag {}; @@ -458,63 +458,63 @@ namespace NActors { auto& tls = *TlsActivationContext; return TActorContext(tls.Mailbox, tls.ExecutorThread, tls.EventStart, id); } - - class TDecorator : public IActor { - protected: - THolder<IActor> Actor; - - public: - TDecorator(THolder<IActor>&& actor) - : IActor(static_cast<TReceiveFunc>(&TDecorator::State), actor->GetActivityType()) - , Actor(std::move(actor)) - { - } - - void Registered(TActorSystem* sys, const TActorId& owner) override { - Actor->ChangeSelfId(SelfId()); - Actor->Registered(sys, owner); - } - - virtual bool DoBeforeReceiving(TAutoPtr<IEventHandle>& /*ev*/, const TActorContext& /*ctx*/) { - return true; - } - - virtual void DoAfterReceiving(const TActorContext& /*ctx*/) - { - } - - STFUNC(State) { - if (DoBeforeReceiving(ev, ctx)) { - Actor->Receive(ev, ctx); - DoAfterReceiving(ctx); - } - } - }; - - // TTestDecorator doesn't work with the real actor system - struct TTestDecorator : public TDecorator { - TTestDecorator(THolder<IActor>&& actor) - : TDecorator(std::move(actor)) - { - } - - virtual ~TTestDecorator() = default; - - // This method must be called in the test actor system - bool BeforeSending(TAutoPtr<IEventHandle>& ev) - { - bool send = true; - TTestDecorator *decorator = dynamic_cast<TTestDecorator*>(Actor.Get()); - if (decorator) { - send = decorator->BeforeSending(ev); - } - return send && ev && DoBeforeSending(ev); - } - - virtual bool DoBeforeSending(TAutoPtr<IEventHandle>& /*ev*/) { - return true; - } - }; + + class TDecorator : public IActor { + protected: + THolder<IActor> Actor; + + public: + TDecorator(THolder<IActor>&& actor) + : IActor(static_cast<TReceiveFunc>(&TDecorator::State), actor->GetActivityType()) + , Actor(std::move(actor)) + { + } + + void Registered(TActorSystem* sys, const TActorId& owner) override { + Actor->ChangeSelfId(SelfId()); + Actor->Registered(sys, owner); + } + + virtual bool DoBeforeReceiving(TAutoPtr<IEventHandle>& /*ev*/, const TActorContext& /*ctx*/) { + return true; + } + + virtual void DoAfterReceiving(const TActorContext& /*ctx*/) + { + } + + STFUNC(State) { + if (DoBeforeReceiving(ev, ctx)) { + Actor->Receive(ev, ctx); + DoAfterReceiving(ctx); + } + } + }; + + // TTestDecorator doesn't work with the real actor system + struct TTestDecorator : public TDecorator { + TTestDecorator(THolder<IActor>&& actor) + : TDecorator(std::move(actor)) + { + } + + virtual ~TTestDecorator() = default; + + // This method must be called in the test actor system + bool BeforeSending(TAutoPtr<IEventHandle>& ev) + { + bool send = true; + TTestDecorator *decorator = dynamic_cast<TTestDecorator*>(Actor.Get()); + if (decorator) { + send = decorator->BeforeSending(ev); + } + return send && ev && DoBeforeSending(ev); + } + + virtual bool DoBeforeSending(TAutoPtr<IEventHandle>& /*ev*/) { + return true; + } + }; } template <> diff --git a/library/cpp/actors/core/actor_ut.cpp b/library/cpp/actors/core/actor_ut.cpp index e1b765ec72a..a0a703e297b 100644 --- a/library/cpp/actors/core/actor_ut.cpp +++ b/library/cpp/actors/core/actor_ut.cpp @@ -1,24 +1,24 @@ -#include "actor.cpp" -#include "events.h" -#include "actorsystem.h" -#include "executor_pool_basic.h" -#include "scheduler_basic.h" -#include "actor_bootstrapped.h" - +#include "actor.cpp" +#include "events.h" +#include "actorsystem.h" +#include "executor_pool_basic.h" +#include "scheduler_basic.h" +#include "actor_bootstrapped.h" + #include <library/cpp/actors/util/threadparkpad.h> -#include <library/cpp/testing/unittest/registar.h> - +#include <library/cpp/testing/unittest/registar.h> + #include <util/generic/algorithm.h> #include <util/system/atomic.h> #include <util/system/rwlock.h> #include <util/system/hp_timer.h> - -using namespace NActors; - + +using namespace NActors; + struct TTestEndDecorator : TDecorator { TThreadParkPad* Pad; TAtomic* ActorsAlive; - + TTestEndDecorator(THolder<IActor>&& actor, TThreadParkPad* pad, TAtomic* actorsAlive) : TDecorator(std::move(actor)) , Pad(pad) @@ -57,7 +57,7 @@ Y_UNIT_TEST_SUITE(ActorBenchmark) { public: static constexpr auto ActorActivityType() { return ACTORLIB_COMMON; - } + } TSendReceiveActor(double* elapsedTime, TActorId receiver, bool allocation, ERole role, ui32 neighbours = 0) : EventsCounter(TotalEventsAmount) @@ -108,8 +108,8 @@ Y_UNIT_TEST_SUITE(ActorBenchmark) { bool AllocatesMemory; ERole Role; ui32 MailboxNeighboursCount; - }; - + }; + void AddBasicPool(THolder<TActorSystemSetup>& setup, ui32 threads, bool activateEveryEvent) { TBasicExecutorPoolConfig basic; basic.PoolId = setup->GetExecutorsCount(); @@ -482,90 +482,90 @@ Y_UNIT_TEST_SUITE(ActorBenchmark) { } Y_UNIT_TEST_SUITE(TestDecorator) { - struct TPingDecorator : TDecorator { - TAutoPtr<IEventHandle> SavedEvent = nullptr; + struct TPingDecorator : TDecorator { + TAutoPtr<IEventHandle> SavedEvent = nullptr; ui64* Counter; - + TPingDecorator(THolder<IActor>&& actor, ui64* counter) - : TDecorator(std::move(actor)) - , Counter(counter) - { - } - + : TDecorator(std::move(actor)) + , Counter(counter) + { + } + bool DoBeforeReceiving(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) override { - *Counter += 1; - if (ev->Type != TEvents::THelloWorld::Pong) { - TAutoPtr<IEventHandle> pingEv = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPing()); - SavedEvent = ev; - Actor->Receive(pingEv, ctx); - } else { - Actor->Receive(SavedEvent, ctx); - } - return false; - } - }; - - struct TPongDecorator : TDecorator { + *Counter += 1; + if (ev->Type != TEvents::THelloWorld::Pong) { + TAutoPtr<IEventHandle> pingEv = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPing()); + SavedEvent = ev; + Actor->Receive(pingEv, ctx); + } else { + Actor->Receive(SavedEvent, ctx); + } + return false; + } + }; + + struct TPongDecorator : TDecorator { ui64* Counter; - + TPongDecorator(THolder<IActor>&& actor, ui64* counter) - : TDecorator(std::move(actor)) - , Counter(counter) - { - } + : TDecorator(std::move(actor)) + , Counter(counter) + { + } bool DoBeforeReceiving(TAutoPtr<IEventHandle>& ev, const TActorContext&) override { - *Counter += 1; - if (ev->Type == TEvents::THelloWorld::Ping) { - TAutoPtr<IEventHandle> pongEv = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPong()); - Send(SelfId(), new TEvents::TEvPong()); - return false; - } - return true; - } - }; - + *Counter += 1; + if (ev->Type == TEvents::THelloWorld::Ping) { + TAutoPtr<IEventHandle> pongEv = new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPong()); + Send(SelfId(), new TEvents::TEvPong()); + return false; + } + return true; + } + }; + struct TTestActor : TActorBootstrapped<TTestActor> { static constexpr char ActorName[] = "TestActor"; - void Bootstrap() - { + void Bootstrap() + { const auto& activityTypeIndex = GetActivityType(); Y_ENSURE(activityTypeIndex < GetActivityTypeCount()); Y_ENSURE(GetActivityTypeName(activityTypeIndex) == "TestActor"); - PassAway(); - } - }; - - Y_UNIT_TEST(Basic) { - THolder<TActorSystemSetup> setup = MakeHolder<TActorSystemSetup>(); - setup->NodeId = 0; - setup->ExecutorsCount = 1; - setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); - for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { - setup->Executors[i] = new TBasicExecutorPool(i, 1, 10, "basic"); - } - setup->Scheduler = new TBasicSchedulerThread; - - TActorSystem actorSystem(setup); - actorSystem.Start(); - + PassAway(); + } + }; + + Y_UNIT_TEST(Basic) { + THolder<TActorSystemSetup> setup = MakeHolder<TActorSystemSetup>(); + setup->NodeId = 0; + setup->ExecutorsCount = 1; + setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); + for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { + setup->Executors[i] = new TBasicExecutorPool(i, 1, 10, "basic"); + } + setup->Scheduler = new TBasicSchedulerThread; + + TActorSystem actorSystem(setup); + actorSystem.Start(); + THolder<IActor> innerActor = MakeHolder<TTestActor>(); - ui64 pongCounter = 0; + ui64 pongCounter = 0; THolder<IActor> pongActor = MakeHolder<TPongDecorator>(std::move(innerActor), &pongCounter); - ui64 pingCounter = 0; + ui64 pingCounter = 0; THolder<IActor> pingActor = MakeHolder<TPingDecorator>(std::move(pongActor), &pingCounter); TThreadParkPad pad; TAtomic actorsAlive = 0; - + THolder<IActor> endActor = MakeHolder<TTestEndDecorator>(std::move(pingActor), &pad, &actorsAlive); actorSystem.Register(endActor.Release(), TMailboxType::HTSwap); pad.Park(); - actorSystem.Stop(); - UNIT_ASSERT(pongCounter == 2 && pingCounter == 2); - } + actorSystem.Stop(); + UNIT_ASSERT(pongCounter == 2 && pingCounter == 2); + } Y_UNIT_TEST(LocalProcessKey) { static constexpr char ActorName[] = "TestActor"; @@ -575,4 +575,4 @@ Y_UNIT_TEST_SUITE(TestDecorator) { UNIT_ASSERT((TLocalProcessKey<TActorActivityTag, ActorName>::GetName() == ActorName)); UNIT_ASSERT((TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetIndex(IActor::INTERCONNECT_PROXY_TCP) == IActor::INTERCONNECT_PROXY_TCP)); } -} +} diff --git a/library/cpp/actors/core/executor_pool_basic.cpp b/library/cpp/actors/core/executor_pool_basic.cpp index 4dce16939ae..f7a9418a825 100644 --- a/library/cpp/actors/core/executor_pool_basic.cpp +++ b/library/cpp/actors/core/executor_pool_basic.cpp @@ -34,7 +34,7 @@ namespace NActors { , ThreadUtilization(0) , MaxUtilizationCounter(0) , MaxUtilizationAccumulator(0) - , ThreadCount(threads) + , ThreadCount(threads) { } @@ -62,26 +62,26 @@ namespace NActors { NHPTimer::STime elapsed = 0; NHPTimer::STime parked = 0; - NHPTimer::STime blocked = 0; + NHPTimer::STime blocked = 0; NHPTimer::STime hpstart = GetCycleCountFast(); NHPTimer::STime hpnow; TThreadCtx& threadCtx = Threads[workerId]; - AtomicSet(threadCtx.WaitingFlag, TThreadCtx::WS_NONE); - - if (Y_UNLIKELY(AtomicGet(threadCtx.BlockedFlag) != TThreadCtx::BS_NONE)) { - do { - if (AtomicCas(&threadCtx.BlockedFlag, TThreadCtx::BS_BLOCKED, TThreadCtx::BS_BLOCKING)) { + AtomicSet(threadCtx.WaitingFlag, TThreadCtx::WS_NONE); + + if (Y_UNLIKELY(AtomicGet(threadCtx.BlockedFlag) != TThreadCtx::BS_NONE)) { + do { + if (AtomicCas(&threadCtx.BlockedFlag, TThreadCtx::BS_BLOCKED, TThreadCtx::BS_BLOCKING)) { hpnow = GetCycleCountFast(); - elapsed += hpnow - hpstart; - if (threadCtx.BlockedPad.Park()) // interrupted - return 0; + elapsed += hpnow - hpstart; + if (threadCtx.BlockedPad.Park()) // interrupted + return 0; hpstart = GetCycleCountFast(); - blocked += hpstart - hpnow; - } - } while (AtomicGet(threadCtx.BlockedFlag) != TThreadCtx::BS_NONE && !AtomicLoad(&StopFlag)); - } - + blocked += hpstart - hpnow; + } + } while (AtomicGet(threadCtx.BlockedFlag) != TThreadCtx::BS_NONE && !AtomicLoad(&StopFlag)); + } + const TAtomic x = AtomicDecrement(Semaphore); if (x < 0) { @@ -101,7 +101,7 @@ namespace NActors { } #endif - Y_VERIFY(AtomicLoad(&threadCtx.WaitingFlag) == TThreadCtx::WS_NONE); + Y_VERIFY(AtomicLoad(&threadCtx.WaitingFlag) == TThreadCtx::WS_NONE); if (SpinThreshold > 0) { // spin configured period @@ -155,7 +155,7 @@ namespace NActors { } while (AtomicLoad(&threadCtx.WaitingFlag) == TThreadCtx::WS_BLOCKED); } - Y_VERIFY_DEBUG(AtomicLoad(&StopFlag) || AtomicLoad(&threadCtx.WaitingFlag) == TThreadCtx::WS_RUNNING); + Y_VERIFY_DEBUG(AtomicLoad(&StopFlag) || AtomicLoad(&threadCtx.WaitingFlag) == TThreadCtx::WS_RUNNING); #if defined ACTORSLIB_COLLECT_EXEC_STATS if (AtomicDecrement(ThreadUtilization) == 0) { @@ -176,8 +176,8 @@ namespace NActors { AtomicStore(&MaxUtilizationAccumulator, x); } #endif - } else { - AtomicSet(threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING); + } else { + AtomicSet(threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING); } // ok, has work suggested, must dequeue @@ -189,9 +189,9 @@ namespace NActors { if (parked > 0) { wctx.AddParkedCycles(parked); } - if (blocked > 0) { + if (blocked > 0) { wctx.AddBlockedCycles(blocked); - } + } return activation; } SpinLockPause(); @@ -202,30 +202,30 @@ namespace NActors { } inline void TBasicExecutorPool::WakeUpLoop() { - for (ui32 i = 0;;) { - TThreadCtx& threadCtx = Threads[i % PoolThreads]; - switch (AtomicLoad(&threadCtx.WaitingFlag)) { - case TThreadCtx::WS_NONE: - case TThreadCtx::WS_RUNNING: - ++i; - break; - case TThreadCtx::WS_ACTIVE: // in active spin-lock, just set flag - if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_ACTIVE)) { - return; - } - break; - case TThreadCtx::WS_BLOCKED: - if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_BLOCKED)) { - threadCtx.Pad.Unpark(); - return; - } - break; - default: - Y_FAIL(); - } - } - } - + for (ui32 i = 0;;) { + TThreadCtx& threadCtx = Threads[i % PoolThreads]; + switch (AtomicLoad(&threadCtx.WaitingFlag)) { + case TThreadCtx::WS_NONE: + case TThreadCtx::WS_RUNNING: + ++i; + break; + case TThreadCtx::WS_ACTIVE: // in active spin-lock, just set flag + if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_ACTIVE)) { + return; + } + break; + case TThreadCtx::WS_BLOCKED: + if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_BLOCKED)) { + threadCtx.Pad.Unpark(); + return; + } + break; + default: + Y_FAIL(); + } + } + } + void TBasicExecutorPool::ScheduleActivationEx(ui32 activation, ui64 revolvingCounter) { Activations.Push(activation, revolvingCounter); const TAtomic x = AtomicIncrement(Semaphore); @@ -286,10 +286,10 @@ namespace NActors { void TBasicExecutorPool::PrepareStop() { AtomicStore(&StopFlag, true); - for (ui32 i = 0; i != PoolThreads; ++i) { + for (ui32 i = 0; i != PoolThreads; ++i) { Threads[i].Pad.Interrupt(); - Threads[i].BlockedPad.Interrupt(); - } + Threads[i].BlockedPad.Interrupt(); + } } void TBasicExecutorPool::Shutdown() { @@ -335,97 +335,97 @@ namespace NActors { Y_UNUSED(RealtimePriority); #endif } - - ui32 TBasicExecutorPool::GetThreadCount() const { - return AtomicGet(ThreadCount); - } - - void TBasicExecutorPool::SetThreadCount(ui32 threads) { - threads = Max(1u, Min(PoolThreads, threads)); - with_lock (ChangeThreadsLock) { - size_t prevCount = GetThreadCount(); - AtomicSet(ThreadCount, threads); - if (prevCount < threads) { - for (size_t i = prevCount; i < threads; ++i) { - bool repeat = true; - while (repeat) { - switch (AtomicGet(Threads[i].BlockedFlag)) { - case TThreadCtx::BS_BLOCKING: - if (AtomicCas(&Threads[i].BlockedFlag, TThreadCtx::BS_NONE, TThreadCtx::BS_BLOCKING)) { - // thread not entry to blocked loop - repeat = false; - } - break; - case TThreadCtx::BS_BLOCKED: - // thread entry to blocked loop and we wake it - AtomicSet(Threads[i].BlockedFlag, TThreadCtx::BS_NONE); - Threads[i].BlockedPad.Unpark(); - repeat = false; - break; - default: - // thread mustn't has TThreadCtx::BS_NONE because last time it was started to block - Y_FAIL("BlockedFlag is not TThreadCtx::BS_BLOCKING and TThreadCtx::BS_BLOCKED when thread was waked up"); - } - } - } - } else if (prevCount > threads) { - // at first, start to block - for (size_t i = threads; i < prevCount; ++i) { - Y_VERIFY(AtomicGet(Threads[i].BlockedFlag) == TThreadCtx::BS_NONE); - AtomicSet(Threads[i].BlockedFlag, TThreadCtx::BS_BLOCKING); - } - // after check need to wake up threads - for (size_t idx = threads; idx < prevCount; ++idx) { - TThreadCtx& threadCtx = Threads[idx]; - auto waitingFlag = AtomicGet(threadCtx.WaitingFlag); - auto blockedFlag = AtomicGet(threadCtx.BlockedFlag); - // while thread has this states (WS_NONE and BS_BLOCKING) we can't guess which way thread will go. - // Either go to sleep and it will have to wake up, - // or go to execute task and after completion will be blocked. - while (waitingFlag == TThreadCtx::WS_NONE && blockedFlag == TThreadCtx::BS_BLOCKING) { - waitingFlag = AtomicGet(threadCtx.WaitingFlag); - blockedFlag = AtomicGet(threadCtx.BlockedFlag); - } - // next states: - // 1) WS_ACTIVE BS_BLOCKING - waiting and start spinig | need wake up to block - // 2) WS_BLOCKED BS_BLOCKING - waiting and start sleep | need wake up to block - // 3) WS_RUNNING BS_BLOCKING - start execute | not need wake up, will block after executing - // 4) WS_NONE BS_BLOCKED - blocked | not need wake up, already blocked - - if (waitingFlag == TThreadCtx::WS_ACTIVE || waitingFlag == TThreadCtx::WS_BLOCKED) { - // need wake up - Y_VERIFY(blockedFlag == TThreadCtx::BS_BLOCKING); - - // creaty empty mailBoxHint, where LineIndex == 1 and LineHint == 0, and activations will be ignored - constexpr auto emptyMailBoxHint = TMailboxTable::LineIndexMask & -TMailboxTable::LineIndexMask; - ui64 revolvingCounter = AtomicGet(ActivationsRevolvingCounter); - - Activations.Push(emptyMailBoxHint, revolvingCounter); - - auto x = AtomicIncrement(Semaphore); - if (x <= 0) { - // try wake up. if success then go to next thread - switch (waitingFlag){ - case TThreadCtx::WS_ACTIVE: // in active spin-lock, just set flag - if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_ACTIVE)) { - continue; - } - break; - case TThreadCtx::WS_BLOCKED: - if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_BLOCKED)) { - threadCtx.Pad.Unpark(); - continue; - } - break; - default: - ; // other thread woke this sleeping thread - } - // if thread has already been awakened then we must awaken the other + + ui32 TBasicExecutorPool::GetThreadCount() const { + return AtomicGet(ThreadCount); + } + + void TBasicExecutorPool::SetThreadCount(ui32 threads) { + threads = Max(1u, Min(PoolThreads, threads)); + with_lock (ChangeThreadsLock) { + size_t prevCount = GetThreadCount(); + AtomicSet(ThreadCount, threads); + if (prevCount < threads) { + for (size_t i = prevCount; i < threads; ++i) { + bool repeat = true; + while (repeat) { + switch (AtomicGet(Threads[i].BlockedFlag)) { + case TThreadCtx::BS_BLOCKING: + if (AtomicCas(&Threads[i].BlockedFlag, TThreadCtx::BS_NONE, TThreadCtx::BS_BLOCKING)) { + // thread not entry to blocked loop + repeat = false; + } + break; + case TThreadCtx::BS_BLOCKED: + // thread entry to blocked loop and we wake it + AtomicSet(Threads[i].BlockedFlag, TThreadCtx::BS_NONE); + Threads[i].BlockedPad.Unpark(); + repeat = false; + break; + default: + // thread mustn't has TThreadCtx::BS_NONE because last time it was started to block + Y_FAIL("BlockedFlag is not TThreadCtx::BS_BLOCKING and TThreadCtx::BS_BLOCKED when thread was waked up"); + } + } + } + } else if (prevCount > threads) { + // at first, start to block + for (size_t i = threads; i < prevCount; ++i) { + Y_VERIFY(AtomicGet(Threads[i].BlockedFlag) == TThreadCtx::BS_NONE); + AtomicSet(Threads[i].BlockedFlag, TThreadCtx::BS_BLOCKING); + } + // after check need to wake up threads + for (size_t idx = threads; idx < prevCount; ++idx) { + TThreadCtx& threadCtx = Threads[idx]; + auto waitingFlag = AtomicGet(threadCtx.WaitingFlag); + auto blockedFlag = AtomicGet(threadCtx.BlockedFlag); + // while thread has this states (WS_NONE and BS_BLOCKING) we can't guess which way thread will go. + // Either go to sleep and it will have to wake up, + // or go to execute task and after completion will be blocked. + while (waitingFlag == TThreadCtx::WS_NONE && blockedFlag == TThreadCtx::BS_BLOCKING) { + waitingFlag = AtomicGet(threadCtx.WaitingFlag); + blockedFlag = AtomicGet(threadCtx.BlockedFlag); + } + // next states: + // 1) WS_ACTIVE BS_BLOCKING - waiting and start spinig | need wake up to block + // 2) WS_BLOCKED BS_BLOCKING - waiting and start sleep | need wake up to block + // 3) WS_RUNNING BS_BLOCKING - start execute | not need wake up, will block after executing + // 4) WS_NONE BS_BLOCKED - blocked | not need wake up, already blocked + + if (waitingFlag == TThreadCtx::WS_ACTIVE || waitingFlag == TThreadCtx::WS_BLOCKED) { + // need wake up + Y_VERIFY(blockedFlag == TThreadCtx::BS_BLOCKING); + + // creaty empty mailBoxHint, where LineIndex == 1 and LineHint == 0, and activations will be ignored + constexpr auto emptyMailBoxHint = TMailboxTable::LineIndexMask & -TMailboxTable::LineIndexMask; + ui64 revolvingCounter = AtomicGet(ActivationsRevolvingCounter); + + Activations.Push(emptyMailBoxHint, revolvingCounter); + + auto x = AtomicIncrement(Semaphore); + if (x <= 0) { + // try wake up. if success then go to next thread + switch (waitingFlag){ + case TThreadCtx::WS_ACTIVE: // in active spin-lock, just set flag + if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_ACTIVE)) { + continue; + } + break; + case TThreadCtx::WS_BLOCKED: + if (AtomicCas(&threadCtx.WaitingFlag, TThreadCtx::WS_RUNNING, TThreadCtx::WS_BLOCKED)) { + threadCtx.Pad.Unpark(); + continue; + } + break; + default: + ; // other thread woke this sleeping thread + } + // if thread has already been awakened then we must awaken the other WakeUpLoop(); - } - } - } - } - } - } + } + } + } + } + } + } } diff --git a/library/cpp/actors/core/executor_pool_basic.h b/library/cpp/actors/core/executor_pool_basic.h index 023190f7fe3..0169fe4fd8e 100644 --- a/library/cpp/actors/core/executor_pool_basic.h +++ b/library/cpp/actors/core/executor_pool_basic.h @@ -8,40 +8,40 @@ #include <library/cpp/actors/util/threadparkpad.h> #include <library/cpp/monlib/dynamic_counters/counters.h> -#include <util/system/mutex.h> - +#include <util/system/mutex.h> + namespace NActors { class TBasicExecutorPool: public TExecutorPoolBase { struct TThreadCtx { TAutoPtr<TExecutorThread> Thread; TThreadParkPad Pad; - TThreadParkPad BlockedPad; + TThreadParkPad BlockedPad; TAtomic WaitingFlag; - TAtomic BlockedFlag; + TAtomic BlockedFlag; // different threads must spin/block on different cache-lines. // we add some padding bytes to enforce this rule - static const size_t SizeWithoutPadding = sizeof(TAutoPtr<TExecutorThread>) + 2 * sizeof(TThreadParkPad) + 2 * sizeof(TAtomic); - ui8 Padding[64 - SizeWithoutPadding]; - static_assert(64 >= SizeWithoutPadding); + static const size_t SizeWithoutPadding = sizeof(TAutoPtr<TExecutorThread>) + 2 * sizeof(TThreadParkPad) + 2 * sizeof(TAtomic); + ui8 Padding[64 - SizeWithoutPadding]; + static_assert(64 >= SizeWithoutPadding); enum EWaitState { WS_NONE, WS_ACTIVE, - WS_BLOCKED, - WS_RUNNING - }; - - enum EBlockedState { - BS_NONE, - BS_BLOCKING, - BS_BLOCKED + WS_BLOCKED, + WS_RUNNING }; - TThreadCtx() - : WaitingFlag(WS_NONE) - , BlockedFlag(BS_NONE) - { + enum EBlockedState { + BS_NONE, + BS_BLOCKING, + BS_BLOCKED + }; + + TThreadCtx() + : WaitingFlag(WS_NONE) + , BlockedFlag(BS_NONE) + { } }; @@ -63,9 +63,9 @@ namespace NActors { TAtomic MaxUtilizationCounter; TAtomic MaxUtilizationAccumulator; - TAtomic ThreadCount; - TMutex ChangeThreadsLock; - + TAtomic ThreadCount; + TMutex ChangeThreadsLock; + public: static constexpr TDuration DEFAULT_TIME_PER_MAILBOX = TBasicExecutorPoolConfig::DEFAULT_TIME_PER_MAILBOX; static constexpr ui32 DEFAULT_EVENTS_PER_MAILBOX = TBasicExecutorPoolConfig::DEFAULT_EVENTS_PER_MAILBOX; @@ -101,11 +101,11 @@ namespace NActors { } void SetRealTimeMode() const override; - - ui32 GetThreadCount() const; - void SetThreadCount(ui32 threads); - - private: + + ui32 GetThreadCount() const; + void SetThreadCount(ui32 threads); + + private: void WakeUpLoop(); }; } diff --git a/library/cpp/actors/core/executor_pool_basic_ut.cpp b/library/cpp/actors/core/executor_pool_basic_ut.cpp index 76dff693af5..3a6ec8f9db9 100644 --- a/library/cpp/actors/core/executor_pool_basic_ut.cpp +++ b/library/cpp/actors/core/executor_pool_basic_ut.cpp @@ -1,382 +1,382 @@ -#include "actorsystem.h" -#include "executor_pool_basic.h" -#include "hfunc.h" -#include "scheduler_basic.h" - +#include "actorsystem.h" +#include "executor_pool_basic.h" +#include "hfunc.h" +#include "scheduler_basic.h" + #include <library/cpp/actors/util/should_continue.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/actors/protos/unittests.pb.h> - -using namespace NActors; - -//////////////////////////////////////////////////////////////////////////////// - -struct TEvMsg : public NActors::TEventBase<TEvMsg, 10347> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvMsg, "ExecutorPoolTest: Msg"); -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TTestSenderActor : public IActor { -private: - using EActivityType = IActor::EActivityType ; - using EActorActivity = IActor::EActorActivity; - -private: - TAtomic Counter; + +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +struct TEvMsg : public NActors::TEventBase<TEvMsg, 10347> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvMsg, "ExecutorPoolTest: Msg"); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TTestSenderActor : public IActor { +private: + using EActivityType = IActor::EActivityType ; + using EActorActivity = IActor::EActorActivity; + +private: + TAtomic Counter; TActorId Receiver; - - std::function<void(void)> Action; - -public: - TTestSenderActor(std::function<void(void)> action = [](){}, - EActivityType activityType = EActorActivity::OTHER) - : IActor(static_cast<TReceiveFunc>(&TTestSenderActor::Execute), activityType) - , Action(action) - {} - + + std::function<void(void)> Action; + +public: + TTestSenderActor(std::function<void(void)> action = [](){}, + EActivityType activityType = EActorActivity::OTHER) + : IActor(static_cast<TReceiveFunc>(&TTestSenderActor::Execute), activityType) + , Action(action) + {} + void Start(TActorId receiver, size_t count) - { - AtomicSet(Counter, count); - Receiver = receiver; - } - - void Stop() { - while (true) { - if (GetCounter() == 0) { - break; - } + { + AtomicSet(Counter, count); + Receiver = receiver; + } + + void Stop() { + while (true) { + if (GetCounter() == 0) { + break; + } Sleep(TDuration::MilliSeconds(1)); - } - } - - size_t GetCounter() const { - return AtomicGet(Counter); - } - -private: - STFUNC(Execute) - { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - hFunc(TEvMsg, Handle); - } - } - - void Handle(TEvMsg::TPtr &ev) - { - Y_UNUSED(ev); - Action(); + } + } + + size_t GetCounter() const { + return AtomicGet(Counter); + } + +private: + STFUNC(Execute) + { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + hFunc(TEvMsg, Handle); + } + } + + void Handle(TEvMsg::TPtr &ev) + { + Y_UNUSED(ev); + Action(); TAtomicBase count = AtomicDecrement(Counter); Y_VERIFY(count != Max<TAtomicBase>()); - if (count) { - Send(Receiver, new TEvMsg()); - } - } -}; - -THolder<TActorSystemSetup> GetActorSystemSetup(TBasicExecutorPool* pool) -{ - auto setup = MakeHolder<NActors::TActorSystemSetup>(); - setup->NodeId = 1; - setup->ExecutorsCount = 1; + if (count) { + Send(Receiver, new TEvMsg()); + } + } +}; + +THolder<TActorSystemSetup> GetActorSystemSetup(TBasicExecutorPool* pool) +{ + auto setup = MakeHolder<NActors::TActorSystemSetup>(); + setup->NodeId = 1; + setup->ExecutorsCount = 1; setup->Executors.Reset(new TAutoPtr<NActors::IExecutorPool>[1]); - setup->Executors[0] = pool; - setup->Scheduler = new TBasicSchedulerThread(NActors::TSchedulerConfig(512, 0)); - return setup; -} - -Y_UNIT_TEST_SUITE(BasicExecutorPool) { - - Y_UNIT_TEST(DecreaseIncreaseThreadsCount) { - const size_t msgCount = 1e4; - const size_t size = 4; - const size_t halfSize = size / 2; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - executorPool->SetThreadCount(halfSize); - TTestSenderActor* actors[size]; + setup->Executors[0] = pool; + setup->Scheduler = new TBasicSchedulerThread(NActors::TSchedulerConfig(512, 0)); + return setup; +} + +Y_UNIT_TEST_SUITE(BasicExecutorPool) { + + Y_UNIT_TEST(DecreaseIncreaseThreadsCount) { + const size_t msgCount = 1e4; + const size_t size = 4; + const size_t halfSize = size / 2; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + executorPool->SetThreadCount(halfSize); + TTestSenderActor* actors[size]; TActorId actorIds[size]; - for (size_t i = 0; i < size; ++i) { - actors[i] = new TTestSenderActor(); - actorIds[i] = actorSystem.Register(actors[i]); - } - - const int testCount = 2; - - TExecutorPoolStats poolStats[testCount]; - TVector<TExecutorThreadStats> statsCopy[testCount]; - - for (size_t testIdx = 0; testIdx < testCount; ++testIdx) { - for (size_t i = 0; i < size; ++i) { - actors[i]->Start(actors[i]->SelfId(), msgCount); + for (size_t i = 0; i < size; ++i) { + actors[i] = new TTestSenderActor(); + actorIds[i] = actorSystem.Register(actors[i]); + } + + const int testCount = 2; + + TExecutorPoolStats poolStats[testCount]; + TVector<TExecutorThreadStats> statsCopy[testCount]; + + for (size_t testIdx = 0; testIdx < testCount; ++testIdx) { + for (size_t i = 0; i < size; ++i) { + actors[i]->Start(actors[i]->SelfId(), msgCount); } for (size_t i = 0; i < size; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + Sleep(TDuration::MilliSeconds(100)); + + for (size_t i = 0; i < size; ++i) { + actors[i]->Stop(); + } + + executorPool->GetCurrentStats(poolStats[testIdx], statsCopy[testIdx]); + } + + for (size_t i = 1; i <= halfSize; ++i) { + UNIT_ASSERT_UNEQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); + } + + for (size_t i = halfSize + 1; i <= size; ++i) { + UNIT_ASSERT_EQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); + } + + executorPool->SetThreadCount(size); + + for (size_t testIdx = 0; testIdx < testCount; ++testIdx) { + for (size_t i = 0; i < size; ++i) { + actors[i]->Start(actors[i]->SelfId(), msgCount); } - - Sleep(TDuration::MilliSeconds(100)); - for (size_t i = 0; i < size; ++i) { - actors[i]->Stop(); - } - - executorPool->GetCurrentStats(poolStats[testIdx], statsCopy[testIdx]); - } - - for (size_t i = 1; i <= halfSize; ++i) { - UNIT_ASSERT_UNEQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); - } - - for (size_t i = halfSize + 1; i <= size; ++i) { - UNIT_ASSERT_EQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); - } - - executorPool->SetThreadCount(size); - - for (size_t testIdx = 0; testIdx < testCount; ++testIdx) { - for (size_t i = 0; i < size; ++i) { - actors[i]->Start(actors[i]->SelfId(), msgCount); - } - for (size_t i = 0; i < size; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); - } - - Sleep(TDuration::MilliSeconds(100)); - - for (size_t i = 0; i < size; ++i) { - actors[i]->Stop(); - } - - executorPool->GetCurrentStats(poolStats[testIdx], statsCopy[testIdx]); - } - - for (size_t i = 1; i <= size; ++i) { - UNIT_ASSERT_UNEQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); - } - } - - Y_UNIT_TEST(ChangeCount) { - const size_t msgCount = 1e3; - const size_t size = 4; - const size_t halfSize = size / 2; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto begin = TInstant::Now(); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - executorPool->SetThreadCount(halfSize); - - TTestSenderActor* actors[size]; + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + Sleep(TDuration::MilliSeconds(100)); + + for (size_t i = 0; i < size; ++i) { + actors[i]->Stop(); + } + + executorPool->GetCurrentStats(poolStats[testIdx], statsCopy[testIdx]); + } + + for (size_t i = 1; i <= size; ++i) { + UNIT_ASSERT_UNEQUAL(statsCopy[0][i].ReceivedEvents, statsCopy[1][i].ReceivedEvents); + } + } + + Y_UNIT_TEST(ChangeCount) { + const size_t msgCount = 1e3; + const size_t size = 4; + const size_t halfSize = size / 2; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto begin = TInstant::Now(); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + executorPool->SetThreadCount(halfSize); + + TTestSenderActor* actors[size]; TActorId actorIds[size]; - for (size_t i = 0; i < size; ++i) { - actors[i] = new TTestSenderActor(); - actorIds[i] = actorSystem.Register(actors[i]); - } - - for (size_t i = 0; i < size; ++i) { - actors[i]->Start(actorIds[i], msgCount); + for (size_t i = 0; i < size; ++i) { + actors[i] = new TTestSenderActor(); + actorIds[i] = actorSystem.Register(actors[i]); + } + + for (size_t i = 0; i < size; ++i) { + actors[i]->Start(actorIds[i], msgCount); } for (size_t i = 0; i < size; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); - } - - const i32 N = 6; - const i32 threadsCouns[N] = { 1, 3, 2, 3, 1, 4 }; - - ui64 counter = 0; - - TTestSenderActor* changerActor = new TTestSenderActor([&]{ - executorPool->SetThreadCount(threadsCouns[counter]); - counter++; - if (counter == N) { - counter = 0; - } - }); + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + const i32 N = 6; + const i32 threadsCouns[N] = { 1, 3, 2, 3, 1, 4 }; + + ui64 counter = 0; + + TTestSenderActor* changerActor = new TTestSenderActor([&]{ + executorPool->SetThreadCount(threadsCouns[counter]); + counter++; + if (counter == N) { + counter = 0; + } + }); TActorId changerActorId = actorSystem.Register(changerActor); - changerActor->Start(changerActorId, msgCount); - actorSystem.Send(changerActorId, new TEvMsg()); - - while (true) { + changerActor->Start(changerActorId, msgCount); + actorSystem.Send(changerActorId, new TEvMsg()); + + while (true) { size_t maxCounter = 0; - for (size_t i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { maxCounter = Max(maxCounter, actors[i]->GetCounter()); - } - + } + if (maxCounter == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Max counter is " << maxCounter); - + Sleep(TDuration::MilliSeconds(1)); - } - - changerActor->Stop(); - } - - Y_UNIT_TEST(CheckCompleteOne) { - const size_t size = 4; - const size_t msgCount = 1e4; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - - auto actor = new TTestSenderActor(); - auto actorId = actorSystem.Register(actor); - actor->Start(actor->SelfId(), msgCount); - actorSystem.Send(actorId, new TEvMsg()); - - while (actor->GetCounter()) { - auto now = TInstant::Now(); + } + + changerActor->Stop(); + } + + Y_UNIT_TEST(CheckCompleteOne) { + const size_t size = 4; + const size_t msgCount = 1e4; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + + auto actor = new TTestSenderActor(); + auto actorId = actorSystem.Register(actor); + actor->Start(actor->SelfId(), msgCount); + actorSystem.Send(actorId, new TEvMsg()); + + while (actor->GetCounter()) { + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Counter is " << actor->GetCounter()); Sleep(TDuration::MilliSeconds(1)); - } - } - - Y_UNIT_TEST(CheckCompleteAll) { - const size_t size = 4; - const size_t msgCount = 1e4; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - - TTestSenderActor* actors[size]; + } + } + + Y_UNIT_TEST(CheckCompleteAll) { + const size_t size = 4; + const size_t msgCount = 1e4; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + + TTestSenderActor* actors[size]; TActorId actorIds[size]; - - for (size_t i = 0; i < size; ++i) { - actors[i] = new TTestSenderActor(); - actorIds[i] = actorSystem.Register(actors[i]); + + for (size_t i = 0; i < size; ++i) { + actors[i] = new TTestSenderActor(); + actorIds[i] = actorSystem.Register(actors[i]); + } + for (size_t i = 0; i < size; ++i) { + actors[i]->Start(actors[i]->SelfId(), msgCount); } for (size_t i = 0; i < size; ++i) { - actors[i]->Start(actors[i]->SelfId(), msgCount); - } - for (size_t i = 0; i < size; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); - } - - - while (true) { + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + + while (true) { size_t maxCounter = 0; - for (size_t i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { maxCounter = Max(maxCounter, actors[i]->GetCounter()); - } - + } + if (maxCounter == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Max counter is " << maxCounter); - + Sleep(TDuration::MilliSeconds(1)); - } - } - - Y_UNIT_TEST(CheckCompleteOver) { - const size_t size = 4; - const size_t actorsCount = size * 2; - const size_t msgCount = 1e4; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - - TTestSenderActor* actors[actorsCount]; + } + } + + Y_UNIT_TEST(CheckCompleteOver) { + const size_t size = 4; + const size_t actorsCount = size * 2; + const size_t msgCount = 1e4; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + + TTestSenderActor* actors[actorsCount]; TActorId actorIds[actorsCount]; - - for (size_t i = 0; i < actorsCount; ++i) { - actors[i] = new TTestSenderActor(); - actorIds[i] = actorSystem.Register(actors[i]); - } - for (size_t i = 0; i < actorsCount; ++i) { - actors[i]->Start(actors[i]->SelfId(), msgCount); + + for (size_t i = 0; i < actorsCount; ++i) { + actors[i] = new TTestSenderActor(); + actorIds[i] = actorSystem.Register(actors[i]); + } + for (size_t i = 0; i < actorsCount; ++i) { + actors[i]->Start(actors[i]->SelfId(), msgCount); } for (size_t i = 0; i < actorsCount; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); - } - - - while (true) { + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + + while (true) { size_t maxCounter = 0; - for (size_t i = 0; i < actorsCount; ++i) { + for (size_t i = 0; i < actorsCount; ++i) { maxCounter = Max(maxCounter, actors[i]->GetCounter()); - } - + } + if (maxCounter == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Max counter is " << maxCounter); - + Sleep(TDuration::MilliSeconds(1)); - } - } - - Y_UNIT_TEST(CheckCompleteRoundRobinOver) { - const size_t size = 4; - const size_t actorsCount = size * 2; - const size_t msgCount = 1e2; - TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); - - auto setup = GetActorSystemSetup(executorPool); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - - TTestSenderActor* actors[actorsCount]; + } + } + + Y_UNIT_TEST(CheckCompleteRoundRobinOver) { + const size_t size = 4; + const size_t actorsCount = size * 2; + const size_t msgCount = 1e2; + TBasicExecutorPool* executorPool = new TBasicExecutorPool(0, size, 50); + + auto setup = GetActorSystemSetup(executorPool); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + + TTestSenderActor* actors[actorsCount]; TActorId actorIds[actorsCount]; - - for (size_t i = 0; i < actorsCount; ++i) { - actors[i] = new TTestSenderActor(); - actorIds[i] = actorSystem.Register(actors[i]); - } - for (size_t i = 0; i < actorsCount; ++i) { - actors[i]->Start(actorIds[(i + 1) % actorsCount], msgCount); + + for (size_t i = 0; i < actorsCount; ++i) { + actors[i] = new TTestSenderActor(); + actorIds[i] = actorSystem.Register(actors[i]); + } + for (size_t i = 0; i < actorsCount; ++i) { + actors[i]->Start(actorIds[(i + 1) % actorsCount], msgCount); } for (size_t i = 0; i < actorsCount; ++i) { - actorSystem.Send(actorIds[i], new TEvMsg()); - } - - while (true) { + actorSystem.Send(actorIds[i], new TEvMsg()); + } + + while (true) { size_t maxCounter = 0; - for (size_t i = 0; i < actorsCount; ++i) { + for (size_t i = 0; i < actorsCount; ++i) { maxCounter = Max(maxCounter, actors[i]->GetCounter()); - } - + } + if (maxCounter == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Max counter is " << maxCounter); - + Sleep(TDuration::MilliSeconds(1)); - } - } + } + } Y_UNIT_TEST(CheckStats) { const size_t size = 4; @@ -432,4 +432,4 @@ Y_UNIT_TEST_SUITE(BasicExecutorPool) { UNIT_ASSERT(stats[0].MailboxPushedOutByTime + stats[0].MailboxPushedOutByEventCount >= msgCount / TBasicExecutorPoolConfig::DEFAULT_EVENTS_PER_MAILBOX); UNIT_ASSERT_VALUES_EQUAL(stats[0].MailboxPushedOutBySoftPreemption, 0); } -} +} diff --git a/library/cpp/actors/core/executor_pool_united.cpp b/library/cpp/actors/core/executor_pool_united.cpp index dac6245635d..3c766433666 100644 --- a/library/cpp/actors/core/executor_pool_united.cpp +++ b/library/cpp/actors/core/executor_pool_united.cpp @@ -34,7 +34,7 @@ namespace NActors { TAtomic Active = 0; // Number of mailboxes ready for execution or currently executing TAtomic Tokens = 0; // Pending tokens (token is required for worker to start execution, guarantees concurrency limit and activation availability) volatile bool StopFlag = false; - + // Configuration TPoolId PoolId; TAtomicBase Concurrency; // Max concurrent workers running this pool @@ -78,7 +78,7 @@ namespace NActors { value = RelaxedLoad(tokens); } else { value = AtomicLoad(tokens); - } + } if (value > 0) { if (AtomicCas(tokens, value - 1, value)) { return true; // token acquired @@ -87,8 +87,8 @@ namespace NActors { return false; // no more tokens } } - } - + } + // Try acquire pending token. Must be done before execution bool TryAcquireToken() { return TryAcquireTokenImpl<false>(&Tokens); @@ -739,7 +739,7 @@ namespace NActors { #else NanoSleep(timeoutNs); // non-linux wake is not supported, cpu will go idle on slow -> fast switch #endif - } + } } } @@ -985,7 +985,7 @@ namespace NActors { if (pool->TryAcquireTokenRelaxed()) { result = WakeWithTokenAcquired(united, pool->PoolId); return true; // token acquired or stop - } + } } } else { if (assignedPool->TryAcquireTokenRelaxed()) { @@ -1009,12 +1009,12 @@ namespace NActors { } else { result = current; return true; // wakeup - } + } } - } + } return false; // spin threshold exceeded, no wakeups - } - + } + bool StartBlocking(TPoolId& result) { // Switch into blocked state if (State.StartBlocking()) { @@ -1210,7 +1210,7 @@ namespace NActors { AtomicStore(&StopFlag, true); for (TPoolId pool = 0; pool < PoolCount; pool++) { Pools[pool].Stop(); - } + } for (TCpuId cpuId = 0; cpuId < CpuCount; cpuId++) { Cpus[cpuId].Stop(); } @@ -1320,7 +1320,7 @@ namespace NActors { wctx.AddElapsedCycles(IActor::ACTOR_SYSTEM, timeTracker.Elapsed()); return result; } - + TPoolId TUnitedWorkers::WaitSequence(TCpu& cpu, TWorkerContext& wctx, TTimeTracker& timeTracker) { TPoolId result; if (cpu.ActiveWait(Us2Ts(Config.SpinThresholdUs), result)) { @@ -1338,8 +1338,8 @@ namespace NActors { wctx.AddParkedCycles(timeTracker.Elapsed()); } while (!wakeup); return result; - } - + } + void TUnitedWorkers::GetCurrentStats(TPoolId pool, TVector<TExecutorThreadStats>& statsCopy) const { size_t idx = 1; statsCopy.resize(idx + Pools[pool].WakeOrderCpus.size()); @@ -1349,7 +1349,7 @@ namespace NActors { s.Aggregate(cpu->PoolStats[pool]); } } - + TUnitedExecutorPool::TUnitedExecutorPool(const TUnitedExecutorPoolConfig& cfg, TUnitedWorkers* united) : TExecutorPoolBaseMailboxed(cfg.PoolId, cfg.MaxActivityType) , United(united) @@ -1357,10 +1357,10 @@ namespace NActors { { United->SetupPool(TPoolId(cfg.PoolId), this, MailboxTable.Get()); } - + void TUnitedExecutorPool::Prepare(TActorSystem* actorSystem, NSchedulerQueue::TReader** scheduleReaders, ui32* scheduleSz) { ActorSystem = actorSystem; - + // Schedule readers are initialized through TUnitedWorkers::Prepare *scheduleReaders = nullptr; *scheduleSz = 0; @@ -1406,9 +1406,9 @@ namespace NActors { const auto current = ActorSystem->Monotonic(); if (deadline < current) { deadline = current; - } + } United->GetScheduleWriter(workerId)->Push(deadline.MicroSeconds(), ev.Release(), cookie); - } + } void TUnitedExecutorPool::Schedule(TDuration delta, TAutoPtr<IEventHandle> ev, ISchedulerCookie* cookie, TWorkerId workerId) { Y_VERIFY_DEBUG(workerId < United->GetWorkerCount()); diff --git a/library/cpp/actors/core/executor_pool_united.h b/library/cpp/actors/core/executor_pool_united.h index a090ba24665..65ee3be2650 100644 --- a/library/cpp/actors/core/executor_pool_united.h +++ b/library/cpp/actors/core/executor_pool_united.h @@ -32,7 +32,7 @@ namespace NActors { TUnitedWorkersConfig Config; TCpuAllocationConfig Allocation; - + volatile bool StopFlag = false; public: @@ -66,7 +66,7 @@ namespace NActors { // Add activation of newly scheduled mailbox and wake cpu to execute it if required void PushActivation(TPoolId pool, ui32 activation, ui64 revolvingCounter); - + // Try acquire pending token. Must be done before execution bool TryAcquireToken(TPoolId pool); diff --git a/library/cpp/actors/core/executor_pool_united_ut.cpp b/library/cpp/actors/core/executor_pool_united_ut.cpp index d4df17f1b8c..1a301ff645d 100644 --- a/library/cpp/actors/core/executor_pool_united_ut.cpp +++ b/library/cpp/actors/core/executor_pool_united_ut.cpp @@ -1,23 +1,23 @@ -#include "actorsystem.h" -#include "executor_pool_basic.h" -#include "hfunc.h" -#include "scheduler_basic.h" - +#include "actorsystem.h" +#include "executor_pool_basic.h" +#include "hfunc.h" +#include "scheduler_basic.h" + #include <library/cpp/actors/util/should_continue.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/actors/protos/unittests.pb.h> - -using namespace NActors; - -//////////////////////////////////////////////////////////////////////////////// - -struct TEvMsg : public NActors::TEventBase<TEvMsg, 10347> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvMsg, "ExecutorPoolTest: Msg"); -}; - -//////////////////////////////////////////////////////////////////////////////// - + +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +struct TEvMsg : public NActors::TEventBase<TEvMsg, 10347> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvMsg, "ExecutorPoolTest: Msg"); +}; + +//////////////////////////////////////////////////////////////////////////////// + inline ui64 DoTimedWork(ui64 workUs) { ui64 startUs = ThreadCPUTime(); ui64 endUs = startUs + workUs; @@ -30,62 +30,62 @@ inline ui64 DoTimedWork(ui64 workUs) { return nowUs - startUs; } -class TTestSenderActor : public IActor { -private: - using EActivityType = IActor::EActivityType ; - using EActorActivity = IActor::EActorActivity; - -private: - TAtomic Counter; +class TTestSenderActor : public IActor { +private: + using EActivityType = IActor::EActivityType ; + using EActorActivity = IActor::EActorActivity; + +private: + TAtomic Counter; TActorId Receiver; - - std::function<void(void)> Action; - -public: - TTestSenderActor(std::function<void(void)> action = [](){}, - EActivityType activityType = EActorActivity::OTHER) - : IActor(static_cast<TReceiveFunc>(&TTestSenderActor::Execute), activityType) - , Action(action) - {} - + + std::function<void(void)> Action; + +public: + TTestSenderActor(std::function<void(void)> action = [](){}, + EActivityType activityType = EActorActivity::OTHER) + : IActor(static_cast<TReceiveFunc>(&TTestSenderActor::Execute), activityType) + , Action(action) + {} + void Start(TActorId receiver, size_t count) { - AtomicSet(Counter, count); - Receiver = receiver; - } - - void Stop() { - while (true) { - if (GetCounter() == 0) { - break; - } + AtomicSet(Counter, count); + Receiver = receiver; + } + + void Stop() { + while (true) { + if (GetCounter() == 0) { + break; + } Sleep(TDuration::MilliSeconds(1)); - } - } - - size_t GetCounter() const { - return AtomicGet(Counter); - } - -private: + } + } + + size_t GetCounter() const { + return AtomicGet(Counter); + } + +private: STFUNC(Execute) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { - hFunc(TEvMsg, Handle); - } - } - + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { + hFunc(TEvMsg, Handle); + } + } + void Handle(TEvMsg::TPtr &ev) { - Y_UNUSED(ev); - Action(); + Y_UNUSED(ev); + Action(); TAtomicBase count = AtomicDecrement(Counter); Y_VERIFY(count != Max<TAtomicBase>()); - if (count) { - Send(Receiver, new TEvMsg()); - } - } -}; - + if (count) { + Send(Receiver, new TEvMsg()); + } + } +}; + // Single cpu balancer that switches pool on every activation; not thread-safe struct TRoundRobinBalancer: public IBalancer { TCpuState* State; @@ -121,38 +121,38 @@ void AddUnitedPool(THolder<TActorSystemSetup>& setup, ui32 concurrency = 0) { } THolder<TActorSystemSetup> GetActorSystemSetup(ui32 cpuCount) { - auto setup = MakeHolder<NActors::TActorSystemSetup>(); - setup->NodeId = 1; + auto setup = MakeHolder<NActors::TActorSystemSetup>(); + setup->NodeId = 1; setup->CpuManager.UnitedWorkers.CpuCount = cpuCount; setup->CpuManager.UnitedWorkers.NoRealtime = true; // unavailable in test environment - setup->Scheduler = new TBasicSchedulerThread(NActors::TSchedulerConfig(512, 0)); - return setup; -} - + setup->Scheduler = new TBasicSchedulerThread(NActors::TSchedulerConfig(512, 0)); + return setup; +} + Y_UNIT_TEST_SUITE(UnitedExecutorPool) { - + #ifdef _linux_ - + Y_UNIT_TEST(OnePoolManyCpus) { - const size_t msgCount = 1e4; + const size_t msgCount = 1e4; auto setup = GetActorSystemSetup(4); AddUnitedPool(setup); - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - - auto actor = new TTestSenderActor(); - auto actorId = actorSystem.Register(actor); - actor->Start(actor->SelfId(), msgCount); - actorSystem.Send(actorId, new TEvMsg()); - - while (actor->GetCounter()) { - auto now = TInstant::Now(); + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + + auto actor = new TTestSenderActor(); + auto actorId = actorSystem.Register(actor); + actor->Start(actor->SelfId(), msgCount); + actorSystem.Send(actorId, new TEvMsg()); + + while (actor->GetCounter()) { + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "Counter is " << actor->GetCounter()); Sleep(TDuration::MilliSeconds(1)); - } + } TVector<TExecutorThreadStats> stats; TExecutorPoolStats poolStats; @@ -183,20 +183,20 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { UNIT_ASSERT_VALUES_EQUAL(stats[0].PoolDestroyedActors, 0); UNIT_ASSERT_VALUES_EQUAL(stats[0].PoolAllocatedMailboxes, 4095); // one line UNIT_ASSERT(stats[0].MailboxPushedOutByTime + stats[0].MailboxPushedOutByEventCount + stats[0].MailboxPushedOutBySoftPreemption >= msgCount / TBasicExecutorPoolConfig::DEFAULT_EVENTS_PER_MAILBOX); - } - + } + Y_UNIT_TEST(ManyPoolsOneSharedCpu) { - const size_t msgCount = 1e4; + const size_t msgCount = 1e4; const size_t pools = 4; auto setup = GetActorSystemSetup(1); for (size_t pool = 0; pool < pools; pool++) { AddUnitedPool(setup); } - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + TVector<TTestSenderActor*> actors; for (size_t pool = 0; pool < pools; pool++) { auto actor = new TTestSenderActor(); @@ -204,20 +204,20 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { actor->Start(actor->SelfId(), msgCount); actorSystem.Send(actorId, new TEvMsg()); actors.push_back(actor); - } - - while (true) { + } + + while (true) { size_t left = 0; for (auto actor : actors) { left += actor->GetCounter(); - } + } if (left == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(5), "left " << left); Sleep(TDuration::MilliSeconds(1)); - } + } for (size_t pool = 0; pool < pools; pool++) { TVector<TExecutorThreadStats> stats; @@ -231,8 +231,8 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { UNIT_ASSERT_VALUES_EQUAL(stats[0].ReceivedEvents, msgCount); UNIT_ASSERT_VALUES_EQUAL(stats[0].PoolActorRegistrations, 1); } - } - + } + Y_UNIT_TEST(ManyPoolsOneAssignedCpu) { const size_t msgCount = 1e4; const size_t pools = 4; @@ -289,11 +289,11 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { for (size_t pool = 0; pool < pools; pool++) { AddUnitedPool(setup); } - TActorSystem actorSystem(setup); - actorSystem.Start(); - - auto begin = TInstant::Now(); - + TActorSystem actorSystem(setup); + actorSystem.Start(); + + auto begin = TInstant::Now(); + TVector<TTestSenderActor*> actors; for (size_t pool = 0; pool < pools; pool++) { auto actor = new TTestSenderActor([]() { @@ -303,21 +303,21 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { actor->Start(actor->SelfId(), msgCount); actorSystem.Send(actorId, new TEvMsg()); actors.push_back(actor); - } - - while (true) { + } + + while (true) { size_t left = 0; for (auto actor : actors) { left += actor->GetCounter(); - } + } if (left == 0) { break; } - auto now = TInstant::Now(); + auto now = TInstant::Now(); UNIT_ASSERT_C(now - begin < TDuration::Seconds(15), "left " << left); Sleep(TDuration::MilliSeconds(1)); - } - + } + for (size_t pool = 0; pool < pools; pool++) { TVector<TExecutorThreadStats> stats; TExecutorPoolStats poolStats; @@ -326,13 +326,13 @@ Y_UNIT_TEST_SUITE(UnitedExecutorPool) { for (ui32 idx = 1; idx < stats.size(); ++idx) { stats[0].Aggregate(stats[idx]); } - + UNIT_ASSERT_VALUES_EQUAL(stats[0].ReceivedEvents, msgCount); UNIT_ASSERT_VALUES_EQUAL(stats[0].PreemptedEvents, msgCount); // every 100ms event should be preempted UNIT_ASSERT_VALUES_EQUAL(stats[0].PoolActorRegistrations, 1); - } + } } - + #endif - -} + +} diff --git a/library/cpp/actors/core/executor_thread.h b/library/cpp/actors/core/executor_thread.h index 9d3c573f0d6..fd9cde789f8 100644 --- a/library/cpp/actors/core/executor_thread.h +++ b/library/cpp/actors/core/executor_thread.h @@ -80,7 +80,7 @@ namespace NActors { IExecutorPool* const ExecutorPool; // Event-specific (currently executing) - TVector<THolder<IActor>> DyingActors; + TVector<THolder<IActor>> DyingActors; TActorId CurrentRecipient; ui64 CurrentActorScheduledEventsCounter = 0; diff --git a/library/cpp/actors/core/log.cpp b/library/cpp/actors/core/log.cpp index 5f63b5af580..888eb804fc4 100644 --- a/library/cpp/actors/core/log.cpp +++ b/library/cpp/actors/core/log.cpp @@ -228,7 +228,7 @@ namespace NActors { Sleep(settings.ThrottleDelay); } - void TLoggerActor::LogIgnoredCount(TInstant now) { + void TLoggerActor::LogIgnoredCount(TInstant now) { TString message = Sprintf("Ignored IgnoredCount# %" PRIu64 " log records due to logger overflow!", IgnoredCount); if (!OutputRecord(now, NActors::NLog::EPrio::Error, Settings->LoggerComponent, message)) { BecomeDefunct(); @@ -237,7 +237,7 @@ namespace NActors { void TLoggerActor::HandleIgnoredEvent(TLogIgnored::TPtr& ev, const NActors::TActorContext& ctx) { Y_UNUSED(ev); - LogIgnoredCount(ctx.Now()); + LogIgnoredCount(ctx.Now()); IgnoredCount = 0; PassedCount = 0; } diff --git a/library/cpp/actors/core/log.h b/library/cpp/actors/core/log.h index c11a7cf3c19..d380ed56e50 100644 --- a/library/cpp/actors/core/log.h +++ b/library/cpp/actors/core/log.h @@ -253,7 +253,7 @@ namespace NActors { void HandleWakeup(); [[nodiscard]] bool OutputRecord(TInstant time, NLog::EPrio priority, NLog::EComponent component, const TString& formatted) noexcept; void RenderComponentPriorities(IOutputStream& str); - void LogIgnoredCount(TInstant now); + void LogIgnoredCount(TInstant now); void WriteMessageStat(const NLog::TEvLog& ev); static const char* FormatLocalTimestamp(TInstant time, char* buf); }; diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index d55552af0cb..7fdc202aad7 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -113,7 +113,7 @@ namespace NActors { CpuNs += RelaxedLoad(&other.CpuNs); ElapsedTicks += RelaxedLoad(&other.ElapsedTicks); ParkedTicks += RelaxedLoad(&other.ParkedTicks); - BlockedTicks += RelaxedLoad(&other.BlockedTicks); + BlockedTicks += RelaxedLoad(&other.BlockedTicks); MailboxPushedOutBySoftPreemption += RelaxedLoad(&other.MailboxPushedOutBySoftPreemption); MailboxPushedOutByTime += RelaxedLoad(&other.MailboxPushedOutByTime); MailboxPushedOutByEventCount += RelaxedLoad(&other.MailboxPushedOutByEventCount); diff --git a/library/cpp/actors/core/ut/ya.make b/library/cpp/actors/core/ut/ya.make index 3ee28d58503..bfb4928f535 100644 --- a/library/cpp/actors/core/ut/ya.make +++ b/library/cpp/actors/core/ut/ya.make @@ -36,7 +36,7 @@ SRCS( balancer_ut.cpp event_pb_payload_ut.cpp event_pb_ut.cpp - executor_pool_basic_ut.cpp + executor_pool_basic_ut.cpp executor_pool_united_ut.cpp log_ut.cpp memory_tracker_ut.cpp diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make index 880a9d00dba..bbeb7d48990 100644 --- a/library/cpp/actors/core/ya.make +++ b/library/cpp/actors/core/ya.make @@ -117,7 +117,7 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) + +RECURSE_FOR_TESTS( + ut +) diff --git a/library/cpp/actors/helpers/selfping_actor.cpp b/library/cpp/actors/helpers/selfping_actor.cpp index f9bfaf8dc09..50df38e972e 100644 --- a/library/cpp/actors/helpers/selfping_actor.cpp +++ b/library/cpp/actors/helpers/selfping_actor.cpp @@ -67,8 +67,8 @@ private: NSlidingWindow::TSlidingWindow<NSlidingWindow::TMaxOperation<ui64>> SlidingWindow; NSlidingWindow::TSlidingWindow<TAvgOperation<ui64>> CalculationSlidingWindow; - THPTimer Timer; - + THPTimer Timer; + public: static constexpr auto ActorActivityType() { return SELF_PING_ACTOR; @@ -87,7 +87,7 @@ public: void Bootstrap(const TActorContext& ctx) { Become(&TSelfPingActor::RunningState); - SchedulePing(ctx, Timer.Passed()); + SchedulePing(ctx, Timer.Passed()); } STFUNC(RunningState) @@ -148,23 +148,23 @@ public: void HandlePing(TEvPing::TPtr &ev, const TActorContext &ctx) { - const auto now = ctx.Now(); - const double hpNow = Timer.Passed(); + const auto now = ctx.Now(); + const double hpNow = Timer.Passed(); const auto& e = *ev->Get(); - const double passedTime = hpNow - e.TimeStart; - const ui64 delayUs = passedTime > 0.0 ? static_cast<ui64>(passedTime * 1e6) : 0; + const double passedTime = hpNow - e.TimeStart; + const ui64 delayUs = passedTime > 0.0 ? static_cast<ui64>(passedTime * 1e6) : 0; - *Counter = SlidingWindow.Update(delayUs, now); + *Counter = SlidingWindow.Update(delayUs, now); ui64 d = MeasureTaskDurationNs(); - auto res = CalculationSlidingWindow.Update({1, d}, now); + auto res = CalculationSlidingWindow.Update({1, d}, now); *CalculationTimeCounter = double(res.Sum) / double(res.Count + 1); - SchedulePing(ctx, hpNow); + SchedulePing(ctx, hpNow); } private: - void SchedulePing(const TActorContext &ctx, double hpNow) const + void SchedulePing(const TActorContext &ctx, double hpNow) const { ctx.Schedule(SendInterval, new TEvPing(hpNow)); } diff --git a/library/cpp/actors/testlib/decorator_ut.cpp b/library/cpp/actors/testlib/decorator_ut.cpp index e9a2fa35604..cc937080daa 100644 --- a/library/cpp/actors/testlib/decorator_ut.cpp +++ b/library/cpp/actors/testlib/decorator_ut.cpp @@ -1,327 +1,327 @@ -#include "test_runtime.h" - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/testing/unittest/registar.h> - - -using namespace NActors; - - -Y_UNIT_TEST_SUITE(TesTTestDecorator) { - - bool IsVerbose = false; - void Write(TString msg) { - if (IsVerbose) { - Cerr << (TStringBuilder() << msg << Endl); - } - } - - struct TDyingChecker : TTestDecorator { - TActorId MasterId; - - TDyingChecker(THolder<IActor> &&actor, TActorId masterId) - : TTestDecorator(std::move(actor)) - , MasterId(masterId) - { - Write("TDyingChecker::Construct\n"); - } - - virtual ~TDyingChecker() { - Write("TDyingChecker::~TDyingChecker"); - TActivationContext::Send(new IEventHandle(MasterId, SelfId(), new TEvents::TEvPing())); - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { - Write("TDyingChecker::DoBeforeReceiving"); - return true; - } - - void DoAfterReceiving(const TActorContext &/*ctx*/) override { - Write("TDyingChecker::DoAfterReceiving"); - } - }; - - struct TTestMasterActor : TActorBootstrapped<TTestMasterActor> { - friend TActorBootstrapped<TTestMasterActor>; - - TSet<TActorId> ActorIds; - TVector<THolder<IActor>> Actors; - TActorId EdgeActor; - - TTestMasterActor(TVector<THolder<IActor>> &&actors, TActorId edgeActor) - : TActorBootstrapped() - , Actors(std::move(actors)) - , EdgeActor(edgeActor) - { - } - - void Bootstrap() - { - Write("Start master actor"); - for (auto &actor : Actors) { +#include "test_runtime.h" + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/testing/unittest/registar.h> + + +using namespace NActors; + + +Y_UNIT_TEST_SUITE(TesTTestDecorator) { + + bool IsVerbose = false; + void Write(TString msg) { + if (IsVerbose) { + Cerr << (TStringBuilder() << msg << Endl); + } + } + + struct TDyingChecker : TTestDecorator { + TActorId MasterId; + + TDyingChecker(THolder<IActor> &&actor, TActorId masterId) + : TTestDecorator(std::move(actor)) + , MasterId(masterId) + { + Write("TDyingChecker::Construct\n"); + } + + virtual ~TDyingChecker() { + Write("TDyingChecker::~TDyingChecker"); + TActivationContext::Send(new IEventHandle(MasterId, SelfId(), new TEvents::TEvPing())); + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { + Write("TDyingChecker::DoBeforeReceiving"); + return true; + } + + void DoAfterReceiving(const TActorContext &/*ctx*/) override { + Write("TDyingChecker::DoAfterReceiving"); + } + }; + + struct TTestMasterActor : TActorBootstrapped<TTestMasterActor> { + friend TActorBootstrapped<TTestMasterActor>; + + TSet<TActorId> ActorIds; + TVector<THolder<IActor>> Actors; + TActorId EdgeActor; + + TTestMasterActor(TVector<THolder<IActor>> &&actors, TActorId edgeActor) + : TActorBootstrapped() + , Actors(std::move(actors)) + , EdgeActor(edgeActor) + { + } + + void Bootstrap() + { + Write("Start master actor"); + for (auto &actor : Actors) { THolder<IActor> decaratedActor = MakeHolder<TDyingChecker>(std::move(actor), SelfId()); - TActorId id = Register(decaratedActor.Release()); - Write("Register test actor"); - UNIT_ASSERT(ActorIds.insert(id).second); - } - Become(&TTestMasterActor::State); - } - - STATEFN(State) { - auto it = ActorIds.find(ev->Sender); - UNIT_ASSERT(it != ActorIds.end()); - Write("End test actor"); - ActorIds.erase(it); - if (!ActorIds) { - Send(EdgeActor, new TEvents::TEvPing()); - PassAway(); - } - } - }; - - enum { - Begin = EventSpaceBegin(TEvents::ES_USERSPACE), - EvWords - }; - - struct TEvWords : TEventLocal<TEvWords, EvWords> { - TVector<TString> Words; - - TEvWords() - : TEventLocal() - { - } - }; - - struct TFizzBuzzToFooBar : TTestDecorator { - TFizzBuzzToFooBar(THolder<IActor> &&actor) - : TTestDecorator(std::move(actor)) - { - } - - bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { - if (ev->Type == TEvents::TSystem::Bootstrap) { - return true; - } - Write("TFizzBuzzToFooBar::DoBeforeSending"); - TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); - UNIT_ASSERT(handle); - TEvWords *event = handle->Get(); - TVector<TString> &words = event->Words; - TStringBuilder wordsMsg; - for (auto &word : words) { - wordsMsg << word << ';'; - } - Write(TStringBuilder() << "Send# " << wordsMsg); - if (words.size() == 2 && words[0] == "Fizz" && words[1] == "Buzz") { - words[0] = "Foo"; - words[1] = "Bar"; - } - return true; - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { - Write("TFizzBuzzToFooBar::DoBeforeReceiving"); - return true; - } - - void DoAfterReceiving(const TActorContext &/*ctx*/) override { - Write("TFizzBuzzToFooBar::DoAfterReceiving"); - } - }; - - struct TWordEraser : TTestDecorator { - TString ErasingWord; - - TWordEraser(THolder<IActor> &&actor, TString word) - : TTestDecorator(std::move(actor)) - , ErasingWord(word) - { - } - - bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { - if (ev->Type == TEvents::TSystem::Bootstrap) { - return true; - } - Write("TWordEraser::DoBeforeSending"); - TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); - UNIT_ASSERT(handle); - TEvWords *event = handle->Get(); - TVector<TString> &words = event->Words; - auto it = Find(words.begin(), words.end(), ErasingWord); - if (it != words.end()) { - words.erase(it); - } - return true; - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { - Write("TWordEraser::DoBeforeReceiving"); - return true; - } - - void DoAfterReceiving(const TActorContext &/*ctx*/) override { - Write("TWordEraser::DoAfterReceiving"); - } - }; - - struct TWithoutWordsDroper : TTestDecorator { - TWithoutWordsDroper(THolder<IActor> &&actor) - : TTestDecorator(std::move(actor)) - { - } - - bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { - if (ev->Type == TEvents::TSystem::Bootstrap) { - return true; - } - Write("TWithoutWordsDroper::DoBeforeSending"); - TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); - UNIT_ASSERT(handle); - TEvWords *event = handle->Get(); - return bool(event->Words); - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { - Write("TWithoutWordsDroper::DoBeforeReceiving"); - return true; - } - - void DoAfterReceiving(const TActorContext &/*ctx*/) override { - Write("TWithoutWordsDroper::DoAfterReceiving"); - } - }; - - struct TFooBarReceiver : TActorBootstrapped<TFooBarReceiver> { - TActorId MasterId; - ui64 Counter = 0; - - TFooBarReceiver(TActorId masterId) - : TActorBootstrapped() - , MasterId(masterId) - { - } - - void Bootstrap() - { - Become(&TFooBarReceiver::State); - } - - STATEFN(State) { - TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); - UNIT_ASSERT(handle); - UNIT_ASSERT(handle->Sender == MasterId); - TEvWords *event = handle->Get(); - TVector<TString> &words = event->Words; - UNIT_ASSERT(words.size() == 2 && words[0] == "Foo" && words[1] == "Bar"); - Write(TStringBuilder() << "Receive# " << Counter + 1 << '/' << 2); - if (++Counter == 2) { - PassAway(); - } - } - }; - - struct TFizzBuzzSender : TActorBootstrapped<TFizzBuzzSender> { - TActorId SlaveId; - - TFizzBuzzSender() - : TActorBootstrapped() - { - Write("TFizzBuzzSender::Construct"); - } - - void Bootstrap() { - Write("TFizzBuzzSender::Bootstrap"); + TActorId id = Register(decaratedActor.Release()); + Write("Register test actor"); + UNIT_ASSERT(ActorIds.insert(id).second); + } + Become(&TTestMasterActor::State); + } + + STATEFN(State) { + auto it = ActorIds.find(ev->Sender); + UNIT_ASSERT(it != ActorIds.end()); + Write("End test actor"); + ActorIds.erase(it); + if (!ActorIds) { + Send(EdgeActor, new TEvents::TEvPing()); + PassAway(); + } + } + }; + + enum { + Begin = EventSpaceBegin(TEvents::ES_USERSPACE), + EvWords + }; + + struct TEvWords : TEventLocal<TEvWords, EvWords> { + TVector<TString> Words; + + TEvWords() + : TEventLocal() + { + } + }; + + struct TFizzBuzzToFooBar : TTestDecorator { + TFizzBuzzToFooBar(THolder<IActor> &&actor) + : TTestDecorator(std::move(actor)) + { + } + + bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { + if (ev->Type == TEvents::TSystem::Bootstrap) { + return true; + } + Write("TFizzBuzzToFooBar::DoBeforeSending"); + TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); + UNIT_ASSERT(handle); + TEvWords *event = handle->Get(); + TVector<TString> &words = event->Words; + TStringBuilder wordsMsg; + for (auto &word : words) { + wordsMsg << word << ';'; + } + Write(TStringBuilder() << "Send# " << wordsMsg); + if (words.size() == 2 && words[0] == "Fizz" && words[1] == "Buzz") { + words[0] = "Foo"; + words[1] = "Bar"; + } + return true; + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { + Write("TFizzBuzzToFooBar::DoBeforeReceiving"); + return true; + } + + void DoAfterReceiving(const TActorContext &/*ctx*/) override { + Write("TFizzBuzzToFooBar::DoAfterReceiving"); + } + }; + + struct TWordEraser : TTestDecorator { + TString ErasingWord; + + TWordEraser(THolder<IActor> &&actor, TString word) + : TTestDecorator(std::move(actor)) + , ErasingWord(word) + { + } + + bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { + if (ev->Type == TEvents::TSystem::Bootstrap) { + return true; + } + Write("TWordEraser::DoBeforeSending"); + TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); + UNIT_ASSERT(handle); + TEvWords *event = handle->Get(); + TVector<TString> &words = event->Words; + auto it = Find(words.begin(), words.end(), ErasingWord); + if (it != words.end()) { + words.erase(it); + } + return true; + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { + Write("TWordEraser::DoBeforeReceiving"); + return true; + } + + void DoAfterReceiving(const TActorContext &/*ctx*/) override { + Write("TWordEraser::DoAfterReceiving"); + } + }; + + struct TWithoutWordsDroper : TTestDecorator { + TWithoutWordsDroper(THolder<IActor> &&actor) + : TTestDecorator(std::move(actor)) + { + } + + bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { + if (ev->Type == TEvents::TSystem::Bootstrap) { + return true; + } + Write("TWithoutWordsDroper::DoBeforeSending"); + TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); + UNIT_ASSERT(handle); + TEvWords *event = handle->Get(); + return bool(event->Words); + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { + Write("TWithoutWordsDroper::DoBeforeReceiving"); + return true; + } + + void DoAfterReceiving(const TActorContext &/*ctx*/) override { + Write("TWithoutWordsDroper::DoAfterReceiving"); + } + }; + + struct TFooBarReceiver : TActorBootstrapped<TFooBarReceiver> { + TActorId MasterId; + ui64 Counter = 0; + + TFooBarReceiver(TActorId masterId) + : TActorBootstrapped() + , MasterId(masterId) + { + } + + void Bootstrap() + { + Become(&TFooBarReceiver::State); + } + + STATEFN(State) { + TEventHandle<TEvWords> *handle = reinterpret_cast<TEventHandle<TEvWords>*>(ev.Get()); + UNIT_ASSERT(handle); + UNIT_ASSERT(handle->Sender == MasterId); + TEvWords *event = handle->Get(); + TVector<TString> &words = event->Words; + UNIT_ASSERT(words.size() == 2 && words[0] == "Foo" && words[1] == "Bar"); + Write(TStringBuilder() << "Receive# " << Counter + 1 << '/' << 2); + if (++Counter == 2) { + PassAway(); + } + } + }; + + struct TFizzBuzzSender : TActorBootstrapped<TFizzBuzzSender> { + TActorId SlaveId; + + TFizzBuzzSender() + : TActorBootstrapped() + { + Write("TFizzBuzzSender::Construct"); + } + + void Bootstrap() { + Write("TFizzBuzzSender::Bootstrap"); THolder<IActor> actor = MakeHolder<TFooBarReceiver>(SelfId()); THolder<IActor> decoratedActor = MakeHolder<TDyingChecker>(std::move(actor), SelfId()); - SlaveId = Register(decoratedActor.Release()); - for (ui64 idx = 1; idx <= 30; ++idx) { + SlaveId = Register(decoratedActor.Release()); + for (ui64 idx = 1; idx <= 30; ++idx) { THolder<TEvWords> ev = MakeHolder<TEvWords>(); - if (idx % 3 == 0) { - ev->Words.push_back("Fizz"); - } - if (idx % 5 == 0) { - ev->Words.push_back("Buzz"); - } - Send(SlaveId, ev.Release()); - Write("TFizzBuzzSender::Send words"); - } - Become(&TFizzBuzzSender::State); - } - - STATEFN(State) { - UNIT_ASSERT(ev->Sender == SlaveId); - PassAway(); - } - }; - - struct TCounters { - ui64 SendedCount = 0; - ui64 RecievedCount = 0; - }; - - struct TCountingDecorator : TTestDecorator { - TCounters *Counters; - - TCountingDecorator(THolder<IActor> &&actor, TCounters *counters) - : TTestDecorator(std::move(actor)) - , Counters(counters) - { - } - - bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { - if (ev->Type == TEvents::TSystem::Bootstrap) { - return true; - } - Write("TCountingDecorator::DoBeforeSending"); - Counters->SendedCount++; - return true; - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { - Write("TCountingDecorator::DoBeforeReceiving"); - Counters->RecievedCount++; - return true; - } - }; - - bool ScheduledFilterFunc(NActors::TTestActorRuntimeBase& runtime, TAutoPtr<NActors::IEventHandle>& event, - TDuration delay, TInstant& deadline) { - if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { - deadline = runtime.GetTimeProvider()->Now() + delay; - return false; - } - return true; - } - - THolder<IActor> CreateFizzBuzzSender() { + if (idx % 3 == 0) { + ev->Words.push_back("Fizz"); + } + if (idx % 5 == 0) { + ev->Words.push_back("Buzz"); + } + Send(SlaveId, ev.Release()); + Write("TFizzBuzzSender::Send words"); + } + Become(&TFizzBuzzSender::State); + } + + STATEFN(State) { + UNIT_ASSERT(ev->Sender == SlaveId); + PassAway(); + } + }; + + struct TCounters { + ui64 SendedCount = 0; + ui64 RecievedCount = 0; + }; + + struct TCountingDecorator : TTestDecorator { + TCounters *Counters; + + TCountingDecorator(THolder<IActor> &&actor, TCounters *counters) + : TTestDecorator(std::move(actor)) + , Counters(counters) + { + } + + bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { + if (ev->Type == TEvents::TSystem::Bootstrap) { + return true; + } + Write("TCountingDecorator::DoBeforeSending"); + Counters->SendedCount++; + return true; + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &/*ev*/, const TActorContext &/*ctx*/) override { + Write("TCountingDecorator::DoBeforeReceiving"); + Counters->RecievedCount++; + return true; + } + }; + + bool ScheduledFilterFunc(NActors::TTestActorRuntimeBase& runtime, TAutoPtr<NActors::IEventHandle>& event, + TDuration delay, TInstant& deadline) { + if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { + deadline = runtime.GetTimeProvider()->Now() + delay; + return false; + } + return true; + } + + THolder<IActor> CreateFizzBuzzSender() { THolder<IActor> actor = MakeHolder<TFizzBuzzSender>(); THolder<IActor> foobar = MakeHolder<TFizzBuzzToFooBar>(std::move(actor)); THolder<IActor> fizzEraser = MakeHolder<TWordEraser>(std::move(foobar), "Fizz"); THolder<IActor> buzzEraser = MakeHolder<TWordEraser>(std::move(fizzEraser), "Buzz"); return MakeHolder<TWithoutWordsDroper>(std::move(buzzEraser)); - } - - Y_UNIT_TEST(Basic) { - TTestActorRuntimeBase runtime(1, false); - - runtime.SetScheduledEventFilter(&ScheduledFilterFunc); - runtime.SetEventFilter([](NActors::TTestActorRuntimeBase&, TAutoPtr<NActors::IEventHandle>&) { - return false; - }); - runtime.Initialize(); - - TActorId edgeActor = runtime.AllocateEdgeActor(); - TVector<THolder<IActor>> actors(1); - actors[0] = CreateFizzBuzzSender(); - //actors[1] = CreateFizzBuzzSender(); + } + + Y_UNIT_TEST(Basic) { + TTestActorRuntimeBase runtime(1, false); + + runtime.SetScheduledEventFilter(&ScheduledFilterFunc); + runtime.SetEventFilter([](NActors::TTestActorRuntimeBase&, TAutoPtr<NActors::IEventHandle>&) { + return false; + }); + runtime.Initialize(); + + TActorId edgeActor = runtime.AllocateEdgeActor(); + TVector<THolder<IActor>> actors(1); + actors[0] = CreateFizzBuzzSender(); + //actors[1] = CreateFizzBuzzSender(); THolder<IActor> testActor = MakeHolder<TTestMasterActor>(std::move(actors), edgeActor); - Write("Start test"); - runtime.Register(testActor.Release()); - - TAutoPtr<IEventHandle> handle; - auto ev = runtime.GrabEdgeEventRethrow<TEvents::TEvPing>(handle); - UNIT_ASSERT(ev); - Write("Stop test"); - } -} + Write("Start test"); + runtime.Register(testActor.Release()); + + TAutoPtr<IEventHandle> handle; + auto ev = runtime.GrabEdgeEventRethrow<TEvents::TEvPing>(handle); + UNIT_ASSERT(ev); + Write("Stop test"); + } +} diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 6fa25b99656..1fc7b1e9ea9 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -358,12 +358,12 @@ namespace NActors { if (!Runtime->EventFilterFunc(*Runtime, ev)) { ui32 nodeId = ev->GetRecipientRewrite().NodeId(); Y_VERIFY(nodeId != 0); - TNodeDataBase* node = Runtime->Nodes[nodeId].Get(); - - if (!AllowSendFrom(node, ev)) { - return true; - } - + TNodeDataBase* node = Runtime->Nodes[nodeId].Get(); + + if (!AllowSendFrom(node, ev)) { + return true; + } + ui32 mailboxHint = ev->GetRecipientRewrite().Hint(); if (ev->GetTypeRewrite() == ui32(NActors::NLog::EEv::Log)) { const NActors::TActorId loggerActorId = NActors::TActorId(nodeId, "logger"); @@ -373,10 +373,10 @@ namespace NActors { IActor* recipientActor = mailbox->FindActor(ev->GetRecipientRewrite().LocalId()); if (recipientActor) { TActorContext ctx(*mailbox, *node->ExecutorThread, GetCycleCountFast(), ev->GetRecipientRewrite()); - TActivationContext *prevTlsActivationContext = TlsActivationContext; - TlsActivationContext = &ctx; + TActivationContext *prevTlsActivationContext = TlsActivationContext; + TlsActivationContext = &ctx; recipientActor->Receive(ev, ctx); - TlsActivationContext = prevTlsActivationContext; + TlsActivationContext = prevTlsActivationContext; // we expect the logger to never die in tests } } @@ -515,18 +515,18 @@ namespace NActors { node->ActorSystem->Start(); } - bool TTestActorRuntimeBase::AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev) { - ui64 senderLocalId = ev->Sender.LocalId(); - ui64 senderMailboxHint = ev->Sender.Hint(); - TMailboxHeader* senderMailbox = node->MailboxTable->Get(senderMailboxHint); - if (senderMailbox) { - IActor* senderActor = senderMailbox->FindActor(senderLocalId); - TTestDecorator *decorator = dynamic_cast<TTestDecorator*>(senderActor); - return !decorator || decorator->BeforeSending(ev); - } - return true; - } - + bool TTestActorRuntimeBase::AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev) { + ui64 senderLocalId = ev->Sender.LocalId(); + ui64 senderMailboxHint = ev->Sender.Hint(); + TMailboxHeader* senderMailbox = node->MailboxTable->Get(senderMailboxHint); + if (senderMailbox) { + IActor* senderActor = senderMailbox->FindActor(senderLocalId); + TTestDecorator *decorator = dynamic_cast<TTestDecorator*>(senderActor); + return !decorator || decorator->BeforeSending(ev); + } + return true; + } + TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount) : TTestActorRuntimeBase(nodeCount, dataCenterCount, false) { } @@ -1547,10 +1547,10 @@ namespace NActors { Y_VERIFY(!ev->GetRecipientRewrite().IsService() && (targetNodeIndex == nodeIndex)); TAutoPtr<IEventHandle> evHolder(ev); - if (!AllowSendFrom(node, evHolder)) { - return; - } - + if (!AllowSendFrom(node, evHolder)) { + return; + } + ui32 mailboxHint = ev->GetRecipientRewrite().Hint(); TEventMailBox& mbox = GetMailbox(nodeId, mailboxHint); if (!mbox.IsActive(TInstant::MicroSeconds(CurrentTimestamp))) { @@ -1774,15 +1774,15 @@ namespace NActors { } TStrandingActorDecorator(const TActorId& delegatee, bool isSync, const TVector<TActorId>& additionalActors, - TSimpleSharedPtr<TStrandingActorDecoratorContext> context, TTestActorRuntimeBase* runtime, - TReplyCheckerCreator createReplyChecker) + TSimpleSharedPtr<TStrandingActorDecoratorContext> context, TTestActorRuntimeBase* runtime, + TReplyCheckerCreator createReplyChecker) : Delegatee(delegatee) , IsSync(isSync) , AdditionalActors(additionalActors) , Context(context) , HasReply(false) , Runtime(runtime) - , ReplyChecker(createReplyChecker()) + , ReplyChecker(createReplyChecker()) { if (IsSync) { Y_VERIFY(!runtime->IsRealThreads()); @@ -1812,12 +1812,12 @@ namespace NActors { STFUNC(Reply) { Y_VERIFY(!HasReply); - IEventHandle *requestEv = Context->Queue->Head(); + IEventHandle *requestEv = Context->Queue->Head(); TActorId originalSender = requestEv->Sender; - HasReply = !ReplyChecker->IsWaitingForMoreResponses(ev.Get()); - if (HasReply) { - delete Context->Queue->Pop(); - } + HasReply = !ReplyChecker->IsWaitingForMoreResponses(ev.Get()); + if (HasReply) { + delete Context->Queue->Pop(); + } ctx.ExecutorThread.Send(ev->Forward(originalSender)); if (!IsSync && Context->Queue->Head()) { SendHead(ctx); @@ -1849,7 +1849,7 @@ namespace NActors { TAutoPtr<IEventHandle> GetForwardedEvent() { IEventHandle* ev = Context->Queue->Head(); - ReplyChecker->OnRequest(ev); + ReplyChecker->OnRequest(ev); TAutoPtr<IEventHandle> forwardedEv = ev->HasEvent() ? new IEventHandle(Delegatee, ReplyId, ev->ReleaseBase().Release(), ev->Flags, ev->Cookie) : new IEventHandle(ev->GetTypeRewrite(), ev->Flags, Delegatee, ReplyId, ev->ReleaseChainBuffer(), ev->Cookie); @@ -1865,7 +1865,7 @@ namespace NActors { bool HasReply; TDispatchOptions DelegateeOptions; TTestActorRuntimeBase* Runtime; - THolder<IReplyChecker> ReplyChecker; + THolder<IReplyChecker> ReplyChecker; }; void TStrandingActorDecorator::TReplyActor::StateFunc(STFUNC_SIG) { @@ -1874,28 +1874,28 @@ namespace NActors { class TStrandingDecoratorFactory : public IStrandingDecoratorFactory { public: - TStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, - TReplyCheckerCreator createReplyChecker) + TStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, + TReplyCheckerCreator createReplyChecker) : Context(new TStrandingActorDecoratorContext()) , Runtime(runtime) - , CreateReplyChecker(createReplyChecker) + , CreateReplyChecker(createReplyChecker) { } IActor* Wrap(const TActorId& delegatee, bool isSync, const TVector<TActorId>& additionalActors) override { - return new TStrandingActorDecorator(delegatee, isSync, additionalActors, Context, Runtime, - CreateReplyChecker); + return new TStrandingActorDecorator(delegatee, isSync, additionalActors, Context, Runtime, + CreateReplyChecker); } private: TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; TTestActorRuntimeBase* Runtime; - TReplyCheckerCreator CreateReplyChecker; + TReplyCheckerCreator CreateReplyChecker; }; - TAutoPtr<IStrandingDecoratorFactory> CreateStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, - TReplyCheckerCreator createReplyChecker) { - return TAutoPtr<IStrandingDecoratorFactory>(new TStrandingDecoratorFactory(runtime, createReplyChecker)); + TAutoPtr<IStrandingDecoratorFactory> CreateStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, + TReplyCheckerCreator createReplyChecker) { + return TAutoPtr<IStrandingDecoratorFactory>(new TStrandingDecoratorFactory(runtime, createReplyChecker)); } ui64 DefaultRandomSeed = 9999; diff --git a/library/cpp/actors/testlib/test_runtime.h b/library/cpp/actors/testlib/test_runtime.h index 26e3b45c984..95ac8b0aa4c 100644 --- a/library/cpp/actors/testlib/test_runtime.h +++ b/library/cpp/actors/testlib/test_runtime.h @@ -593,8 +593,8 @@ namespace NActors { void CleanupNodes(); virtual void InitNodeImpl(TNodeDataBase*, size_t); - static bool AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev); - + static bool AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev); + protected: THolder<INodeFactory> NodeFactory{new TDefaultNodeFactory}; @@ -689,28 +689,28 @@ namespace NActors { virtual IActor* Wrap(const TActorId& delegatee, bool isSync, const TVector<TActorId>& additionalActors) = 0; }; - struct IReplyChecker { - virtual ~IReplyChecker() {} - virtual void OnRequest(IEventHandle *request) = 0; - virtual bool IsWaitingForMoreResponses(IEventHandle *response) = 0; - }; - - struct TNoneReplyChecker : IReplyChecker { - void OnRequest(IEventHandle*) override { - } - - bool IsWaitingForMoreResponses(IEventHandle*) override { - return false; - } - }; - - using TReplyCheckerCreator = std::function<THolder<IReplyChecker>(void)>; - - inline THolder<IReplyChecker> CreateNoneReplyChecker() { + struct IReplyChecker { + virtual ~IReplyChecker() {} + virtual void OnRequest(IEventHandle *request) = 0; + virtual bool IsWaitingForMoreResponses(IEventHandle *response) = 0; + }; + + struct TNoneReplyChecker : IReplyChecker { + void OnRequest(IEventHandle*) override { + } + + bool IsWaitingForMoreResponses(IEventHandle*) override { + return false; + } + }; + + using TReplyCheckerCreator = std::function<THolder<IReplyChecker>(void)>; + + inline THolder<IReplyChecker> CreateNoneReplyChecker() { return MakeHolder<TNoneReplyChecker>(); - } - - TAutoPtr<IStrandingDecoratorFactory> CreateStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, - TReplyCheckerCreator createReplyChecker = CreateNoneReplyChecker); + } + + TAutoPtr<IStrandingDecoratorFactory> CreateStrandingDecoratorFactory(TTestActorRuntimeBase* runtime, + TReplyCheckerCreator createReplyChecker = CreateNoneReplyChecker); extern ui64 DefaultRandomSeed; } diff --git a/library/cpp/actors/testlib/ut/ya.make b/library/cpp/actors/testlib/ut/ya.make index 1d4aec06ffa..ef16812aed4 100644 --- a/library/cpp/actors/testlib/ut/ya.make +++ b/library/cpp/actors/testlib/ut/ya.make @@ -1,20 +1,20 @@ -UNITTEST_FOR(library/cpp/actors/testlib) - -OWNER( - kruall - g:kikimr -) - -FORK_SUBTESTS() -SIZE(SMALL) - - -PEERDIR( - library/cpp/actors/core -) - -SRCS( - decorator_ut.cpp -) - -END() +UNITTEST_FOR(library/cpp/actors/testlib) + +OWNER( + kruall + g:kikimr +) + +FORK_SUBTESTS() +SIZE(SMALL) + + +PEERDIR( + library/cpp/actors/core +) + +SRCS( + decorator_ut.cpp +) + +END() diff --git a/library/cpp/actors/testlib/ya.make b/library/cpp/actors/testlib/ya.make index 1afb3f60591..8818f10458c 100644 --- a/library/cpp/actors/testlib/ya.make +++ b/library/cpp/actors/testlib/ya.make @@ -21,7 +21,7 @@ IF (GCC) ENDIF() END() - -RECURSE_FOR_TESTS( - ut -) + +RECURSE_FOR_TESTS( + ut +) diff --git a/library/cpp/actors/util/threadparkpad.cpp b/library/cpp/actors/util/threadparkpad.cpp index 74069ff15bf..ece54844598 100644 --- a/library/cpp/actors/util/threadparkpad.cpp +++ b/library/cpp/actors/util/threadparkpad.cpp @@ -48,7 +48,7 @@ namespace NActors { namespace NActors { class TThreadParkPad::TImpl { - TAtomic Interrupted; + TAtomic Interrupted; HANDLE EvHandle; public: @@ -66,7 +66,7 @@ namespace NActors { bool Park() noexcept { ::WaitForSingleObject(EvHandle, INFINITE); - return AtomicGet(Interrupted); + return AtomicGet(Interrupted); } void Unpark() noexcept { @@ -74,12 +74,12 @@ namespace NActors { } void Interrupt() noexcept { - AtomicSet(Interrupted, true); + AtomicSet(Interrupted, true); Unpark(); } bool IsInterrupted() const noexcept { - return AtomicGet(Interrupted); + return AtomicGet(Interrupted); } }; @@ -89,7 +89,7 @@ namespace NActors { namespace NActors { class TThreadParkPad::TImpl { - TAtomic Interrupted; + TAtomic Interrupted; TSystemEvent Ev; public: @@ -103,7 +103,7 @@ namespace NActors { bool Park() noexcept { Ev.Wait(); - return AtomicGet(Interrupted); + return AtomicGet(Interrupted); } void Unpark() noexcept { @@ -111,12 +111,12 @@ namespace NActors { } void Interrupt() noexcept { - AtomicSet(Interrupted, true); + AtomicSet(Interrupted, true); Unpark(); } bool IsInterrupted() const noexcept { - return AtomicGet(Interrupted); + return AtomicGet(Interrupted); } }; #endif diff --git a/ydb/core/base/blobstorage.cpp b/ydb/core/base/blobstorage.cpp index 60bef7f4dff..b15cfb1c35a 100644 --- a/ydb/core/base/blobstorage.cpp +++ b/ydb/core/base/blobstorage.cpp @@ -73,20 +73,20 @@ std::unique_ptr<TEvBlobStorage::TEvBlockResult> TEvBlobStorage::TEvBlock::MakeEr return res; } -std::unique_ptr<TEvBlobStorage::TEvPatchResult> TEvBlobStorage::TEvPatch::MakeErrorResponse( - NKikimrProto::EReplyStatus status, const TString& errorReason, ui32 groupId) { - auto res = std::make_unique<TEvPatchResult>(status, PatchedId, TStorageStatusFlags(), groupId, 0.0f); - res->ErrorReason = errorReason; - return res; -} - -std::unique_ptr<TEvBlobStorage::TEvInplacePatchResult> TEvBlobStorage::TEvInplacePatch::MakeErrorResponse( - NKikimrProto::EReplyStatus status, const TString& errorReason) { - auto res = std::make_unique<TEvInplacePatchResult>(status, PatchedId, TStorageStatusFlags(), 0.0f); - res->ErrorReason = errorReason; - return res; -} - +std::unique_ptr<TEvBlobStorage::TEvPatchResult> TEvBlobStorage::TEvPatch::MakeErrorResponse( + NKikimrProto::EReplyStatus status, const TString& errorReason, ui32 groupId) { + auto res = std::make_unique<TEvPatchResult>(status, PatchedId, TStorageStatusFlags(), groupId, 0.0f); + res->ErrorReason = errorReason; + return res; +} + +std::unique_ptr<TEvBlobStorage::TEvInplacePatchResult> TEvBlobStorage::TEvInplacePatch::MakeErrorResponse( + NKikimrProto::EReplyStatus status, const TString& errorReason) { + auto res = std::make_unique<TEvInplacePatchResult>(status, PatchedId, TStorageStatusFlags(), 0.0f); + res->ErrorReason = errorReason; + return res; +} + std::unique_ptr<TEvBlobStorage::TEvDiscoverResult> TEvBlobStorage::TEvDiscover::MakeErrorResponse( NKikimrProto::EReplyStatus status, const TString& errorReason, ui32 /*groupId*/) { auto res = std::make_unique<TEvDiscoverResult>(status, MinGeneration, 0); diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index a2faee326e5..5375e56ca53 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -453,8 +453,8 @@ struct TEvBlobStorage { EvCollectGarbage, EvStatus, EvVBaldSyncLog, - EvPatch, - EvInplacePatch, + EvPatch, + EvInplacePatch, // EvPutResult = EvPut + 512, /// 268 632 576 @@ -466,8 +466,8 @@ struct TEvBlobStorage { EvCollectGarbageResult, EvStatusResult, EvVBaldSyncLogResult, - EvPatchResult, - EvInplacePatchResult, + EvPatchResult, + EvInplacePatchResult, // proxy <-> vdisk interface EvVPut = EvPut + 2 * 512, /// 268 633 088 @@ -481,13 +481,13 @@ struct TEvBlobStorage { EvVDbStat, EvVCheckReadiness, EvVCompact, /// 268 633 098 - EvVMultiPut, - EvVMovedPatch, - EvVPatchStart, - EvVPatchDiff, - EvVPatchXorDiff, + EvVMultiPut, + EvVMovedPatch, + EvVPatchStart, + EvVPatchDiff, + EvVPatchXorDiff, EvVDefrag, - EvVInplacePatch, + EvVInplacePatch, EvVPutResult = EvPut + 3 * 512, /// 268 633 600 EvVGetResult, @@ -500,13 +500,13 @@ struct TEvBlobStorage { EvVWindowChange, EvVCheckReadinessResult, EvVCompactResult, - EvVMultiPutResult, - EvVMovedPatchResult, - EvVPatchFoundParts, - EvVPatchXorDiffResult, - EvVPatchResult, + EvVMultiPutResult, + EvVMovedPatchResult, + EvVPatchFoundParts, + EvVPatchXorDiffResult, + EvVPatchResult, EvVDefragResult, - EvVInplacePatchResult, + EvVInplacePatchResult, // vdisk <-> vdisk interface EvVDisk = EvPut + 4 * 512, /// 268 634 112 @@ -644,8 +644,8 @@ struct TEvBlobStorage { EvMonStreamQuery, // 268 636 270 EvMonStreamActorDeathNote, EvPDiskErrorStateChange, - EvMultiLog, - EvVMultiPutItemResult, + EvMultiLog, + EvVMultiPutItemResult, EvEnrichNotYet, EvCommenceRepl, // for debugging purposes EvRecoverBlob, @@ -666,8 +666,8 @@ struct TEvBlobStorage { EvCompactVDisk, EvCompactVDiskResult, EvDefragRewritten, - EvVPatchDyingRequest, - EvVPatchDyingConfirm, + EvVPatchDyingRequest, + EvVPatchDyingConfirm, EvNonrestoredCorruptedBlobNotify, EvHugeLockChunks, EvHugeStat, @@ -739,9 +739,9 @@ struct TEvBlobStorage { EvGroupStatReport, EvAccelerateGet, EvAcceleratePut, - EvRequestProxyQueueState, - EvRequestProxySessionsState, - EvProxySessionsState, + EvRequestProxyQueueState, + EvRequestProxySessionsState, + EvProxySessionsState, EvBunchOfEvents, // blobstorage controller interface @@ -855,15 +855,15 @@ struct TEvBlobStorage { struct TEvRangeResult; struct TEvCollectGarbageResult; struct TEvStatusResult; - struct TEvPatchResult; - struct TEvInplacePatchResult; + struct TEvPatchResult; + struct TEvInplacePatchResult; struct TEvPut : public TEventLocal<TEvPut, EvPut> { enum ETactic { - TacticMaxThroughput = 0, + TacticMaxThroughput = 0, TacticMinLatency, - TacticDefault, // Default depends on the erasure type - TacticCount // This is not a tactic, but a number of tactics. Add new tactics before this line. + TacticDefault, // Default depends on the erasure type + TacticCount // This is not a tactic, but a number of tactics. Add new tactics before this line. }; static const char* TacticName(ETactic tactic) { switch (tactic) { @@ -873,8 +873,8 @@ struct TEvBlobStorage { return "MinLatency"; case TacticDefault: return "Default"; - default: - return "unknown"; + default: + return "unknown"; } }; @@ -1268,293 +1268,293 @@ struct TEvBlobStorage { } }; - struct TEvPatch : public TEventLocal<TEvPatch, EvPatch> { - private: - static constexpr ui32 BaseDomainsCount = 8; - static constexpr ui32 MaxStepsForFindingId = 128; - - public: - struct TDiff { - TString Buffer; - ui32 Offset; - - TDiff() - : Offset(0) - { - } - - void Set(const TString &buffer, ui32 offset) { - Buffer = buffer; - Offset = offset; - Y_VERIFY_S(buffer.Size(), "EvPatchDiff invalid: Diff size must be non-zero"); - } - - template <typename TOStream> - void Output(TOStream &os) const { - os << "TDiff {Offset# " << Offset << " Size# " << Buffer.Size() << '}'; - } - - TString ToString() const { - TStringBuilder str; - Output(str); - return str; - } - }; - - const ui32 OriginalGroupId; - const TLogoBlobID OriginalId; - const TLogoBlobID PatchedId; - const ui32 MaskForCookieBruteForcing = 0; - - TArrayHolder<TDiff> Diffs; - const ui64 DiffCount; - const TInstant Deadline; - mutable NLWTrace::TOrbit Orbit; - ui32 RestartCounter = 0; - - TEvPatch(ui32 originalGroupId, const TLogoBlobID &originalId, const TLogoBlobID &patchedId, - ui32 maskForCookieBruteForcing, TArrayHolder<TDiff> &&diffs, ui64 diffCount, TInstant deadline) - : OriginalGroupId(originalGroupId) - , OriginalId(originalId) - , PatchedId(patchedId) - , MaskForCookieBruteForcing(maskForCookieBruteForcing) - , Diffs(std::move(diffs)) - , DiffCount(diffCount) - , Deadline(deadline) - { - CheckContructorArgs(originalId, patchedId, Diffs, DiffCount); - } - + struct TEvPatch : public TEventLocal<TEvPatch, EvPatch> { + private: + static constexpr ui32 BaseDomainsCount = 8; + static constexpr ui32 MaxStepsForFindingId = 128; + + public: + struct TDiff { + TString Buffer; + ui32 Offset; + + TDiff() + : Offset(0) + { + } + + void Set(const TString &buffer, ui32 offset) { + Buffer = buffer; + Offset = offset; + Y_VERIFY_S(buffer.Size(), "EvPatchDiff invalid: Diff size must be non-zero"); + } + + template <typename TOStream> + void Output(TOStream &os) const { + os << "TDiff {Offset# " << Offset << " Size# " << Buffer.Size() << '}'; + } + + TString ToString() const { + TStringBuilder str; + Output(str); + return str; + } + }; + + const ui32 OriginalGroupId; + const TLogoBlobID OriginalId; + const TLogoBlobID PatchedId; + const ui32 MaskForCookieBruteForcing = 0; + + TArrayHolder<TDiff> Diffs; + const ui64 DiffCount; + const TInstant Deadline; + mutable NLWTrace::TOrbit Orbit; + ui32 RestartCounter = 0; + + TEvPatch(ui32 originalGroupId, const TLogoBlobID &originalId, const TLogoBlobID &patchedId, + ui32 maskForCookieBruteForcing, TArrayHolder<TDiff> &&diffs, ui64 diffCount, TInstant deadline) + : OriginalGroupId(originalGroupId) + , OriginalId(originalId) + , PatchedId(patchedId) + , MaskForCookieBruteForcing(maskForCookieBruteForcing) + , Diffs(std::move(diffs)) + , DiffCount(diffCount) + , Deadline(deadline) + { + CheckContructorArgs(originalId, patchedId, Diffs, DiffCount); + } + static void CheckContructorArgs(const TLogoBlobID &originalId, const TLogoBlobID &patchedId, - const TArrayHolder<TDiff> &diffs, ui64 diffCount) - { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&originalId, sizeof(originalId)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&patchedId, sizeof(patchedId)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(diffs.Get(), sizeof(*diffs.Get()) * diffCount); - - Y_VERIFY_S(originalId, "EvPatch invalid: LogoBlobId must have non-zero tablet field," - << " OriginalId# " << originalId); - Y_VERIFY_S(patchedId, "EvPatch invalid: LogoBlobId must have non-zero tablet field," - << " PatchedId# " << patchedId); - Y_VERIFY_S(originalId != patchedId, "EvPatch invalid: OriginalId and PatchedId mustn't be equal" - << " OriginalId# " << originalId - << " PatchedId# " << patchedId); - Y_VERIFY_S(originalId.BlobSize() == patchedId.BlobSize(), - "EvPatch invalid: LogoBlobId must have non-zero tablet field," - << " OriginalId# " << originalId - << " PatchedId# " << patchedId); - - for (ui32 idx = 0; idx < diffCount; ++idx) { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(diffs[idx].Buffer.Data(), diffs[idx].Buffer.size()); - - if (idx) { - Y_VERIFY_S(diffs[idx - 1].Offset + diffs[idx].Buffer.Size() <= diffs[idx].Offset, - "EvPatch invalid: Diffs mustn't be re-covered," - << " [" << idx - 1 << "].Offset# " << diffs[idx - 1].Offset - << " [" << idx - 1 << "].Size# " << diffs[idx - 1].Buffer.Size() - << " [" << idx << "].Offset# " << diffs[idx].Offset - << " [" << idx << "].Size# " << diffs[idx].Buffer.Size()); - } - Y_VERIFY_S(diffs[idx].Offset + diffs[idx].Buffer.Size() <= originalId.BlobSize(), - "EvPatch invalid: Blob size bound was overflow by diff," - << " [" << idx << "].Offset# " << diffs[idx].Offset - << " [" << idx << "].Size# " << diffs[idx].Buffer.Size() - << " [" << idx << "].EndIdx# " << diffs[idx].Offset + diffs[idx].Buffer.Size() - << " BlobSize# " << originalId.BlobSize()); - Y_VERIFY_S(diffs[idx].Buffer.Size(), - "EvPatch invalid: Diff size must be non-zero," - << " [" << idx << "].Size# " << diffs[idx].Buffer.Size()); - } - } - - static bool GetBlobIdWithSamePlacement(const TLogoBlobID &originalId, TLogoBlobID *patchedId, - ui32 bitsForBruteForce, ui32 originalGroupId, ui32 currentGroupId) - { - if (originalGroupId != currentGroupId) { - return false; - } - - ui32 expectedValue = originalId.Hash() % BaseDomainsCount; - Y_VERIFY(patchedId); - if (patchedId->Hash() % BaseDomainsCount == expectedValue) { - return true; - } - - Y_VERIFY(bitsForBruteForce <= TLogoBlobID::MaxCookie); - ui32 baseCookie = ~bitsForBruteForce & patchedId->Cookie(); - ui32 extraCookie = TLogoBlobID::MaxCookie + 1; - ui32 steps = 0; - do { - extraCookie = (extraCookie - 1) & bitsForBruteForce; - ui32 cookie = baseCookie | (extraCookie ^ bitsForBruteForce); - steps++; - - TLogoBlobID id(patchedId->TabletID(), patchedId->Generation(), patchedId->Step(), patchedId->Channel(), - patchedId->BlobSize(), cookie, 0, patchedId->CrcMode()); - if (id.Hash() % BaseDomainsCount == expectedValue) { - *patchedId = id; - return true; - } - - } while(extraCookie && steps < MaxStepsForFindingId); - - return false; - } - - TString Print(bool isFull) const { - Y_UNUSED(isFull); - TStringBuilder str; - str << "TEvPatch {OriginalGroupId# " << OriginalGroupId; - str << " OriginalId# " << OriginalId; - str << " PatchedId# " << PatchedId; - str << " Deadline# " << Deadline.MilliSeconds(); - str << " DiffCount# " << DiffCount; - for (ui32 idx = 0; idx < DiffCount; ++idx) { - str << ' '; - Diffs[idx].Output(str); - } - str << '}'; - return str; - } - - TString ToString() const { - return Print(false); - } - - ui32 CalculateSize() const { - return sizeof(*this) + sizeof(TDiff) * DiffCount; - } - - std::unique_ptr<TEvPatchResult> MakeErrorResponse(NKikimrProto::EReplyStatus status, - const TString& errorReason, ui32 groupId); - }; - - struct TEvPatchResult : public TEventLocal<TEvPatchResult, EvPatchResult> { - NKikimrProto::EReplyStatus Status; - const TLogoBlobID Id; - const TStorageStatusFlags StatusFlags; - const ui32 GroupId; - const float ApproximateFreeSpaceShare; // 0.f has special meaning 'data could not be obtained' - TString ErrorReason; - mutable NLWTrace::TOrbit Orbit; - - TEvPatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &id, TStorageStatusFlags statusFlags, - ui32 groupId, float approximateFreeSpaceShare) - : Status(status) - , Id(id) - , StatusFlags(statusFlags) - , GroupId(groupId) - , ApproximateFreeSpaceShare(approximateFreeSpaceShare) - { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&id, sizeof(id)); - } - - TString Print(bool isFull) const { - Y_UNUSED(isFull); - TStringBuilder str; - str << "TEvPatchResult {Id# " << Id; - str << " Status# " << NKikimrProto::EReplyStatus_Name(Status).data(); - str << " StatusFlags# " << StatusFlags; - if (ErrorReason.size()) { - str << " ErrorReason# \"" << ErrorReason << "\""; - } - str << " ApproximateFreeSpaceShare# " << ApproximateFreeSpaceShare; - str << "}"; - return str; - } - - TString ToString() const { - return Print(false); - } - }; - - struct TEvInplacePatch : public TEventLocal<TEvInplacePatch, EvInplacePatch> { - using TDiff = TEvPatch::TDiff; - - const TLogoBlobID OriginalId; - const TLogoBlobID PatchedId; - - TArrayHolder<TDiff> Diffs; - const ui64 DiffCount; - const TInstant Deadline; - mutable NLWTrace::TOrbit Orbit; - ui32 RestartCounter = 0; - - TEvInplacePatch(const TLogoBlobID &originalId, const TLogoBlobID &patchedId, TArrayHolder<TDiff> &&diffs, - ui64 diffCount, TInstant deadline) - : OriginalId(originalId) - , PatchedId(patchedId) - , Diffs(std::move(diffs)) - , DiffCount(diffCount) - , Deadline(deadline) - { - TEvPatch::CheckContructorArgs(originalId, patchedId, Diffs, DiffCount); - } - - TString Print(bool isFull) const { - Y_UNUSED(isFull); - TStringBuilder str; - str << "TEvInplacePatch {OriginalId# " << OriginalId; - str << " PatchedId# " << PatchedId; - str << " Deadline# " << Deadline.MilliSeconds(); - str << " DiffCount# " << DiffCount; - for (ui32 idx = 0; idx < DiffCount; ++idx) { - str << ' '; - Diffs[idx].Output(str); - } - str << '}'; - return str; - } - - TString ToString() const { - return Print(false); - } - - ui32 CalculateSize() const { - return sizeof(*this) + sizeof(TDiff) * DiffCount; - } - - std::unique_ptr<TEvInplacePatchResult> MakeErrorResponse(NKikimrProto::EReplyStatus status, - const TString& errorReason); - }; - - struct TEvInplacePatchResult : public TEventLocal<TEvInplacePatchResult, EvInplacePatchResult> { - NKikimrProto::EReplyStatus Status; - const TLogoBlobID Id; - const TStorageStatusFlags StatusFlags; - const float ApproximateFreeSpaceShare; // 0.f has special meaning 'data could not be obtained' - TString ErrorReason; - mutable NLWTrace::TOrbit Orbit; - - TEvInplacePatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &id, TStorageStatusFlags statusFlags, - float approximateFreeSpaceShare) - : Status(status) - , Id(id) - , StatusFlags(statusFlags) - , ApproximateFreeSpaceShare(approximateFreeSpaceShare) - { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&id, sizeof(id)); - } - - TString Print(bool isFull) const { - Y_UNUSED(isFull); - TStringBuilder str; - str << "TEvPatchResult {Id# " << Id; - str << " Status# " << NKikimrProto::EReplyStatus_Name(Status).data(); - str << " StatusFlags# " << StatusFlags; - if (ErrorReason.size()) { - str << " ErrorReason# \"" << ErrorReason << "\""; - } - str << " ApproximateFreeSpaceShare# " << ApproximateFreeSpaceShare; - str << "}"; - return str; - } - - TString ToString() const { - return Print(false); - } - }; - + const TArrayHolder<TDiff> &diffs, ui64 diffCount) + { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&originalId, sizeof(originalId)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&patchedId, sizeof(patchedId)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(diffs.Get(), sizeof(*diffs.Get()) * diffCount); + + Y_VERIFY_S(originalId, "EvPatch invalid: LogoBlobId must have non-zero tablet field," + << " OriginalId# " << originalId); + Y_VERIFY_S(patchedId, "EvPatch invalid: LogoBlobId must have non-zero tablet field," + << " PatchedId# " << patchedId); + Y_VERIFY_S(originalId != patchedId, "EvPatch invalid: OriginalId and PatchedId mustn't be equal" + << " OriginalId# " << originalId + << " PatchedId# " << patchedId); + Y_VERIFY_S(originalId.BlobSize() == patchedId.BlobSize(), + "EvPatch invalid: LogoBlobId must have non-zero tablet field," + << " OriginalId# " << originalId + << " PatchedId# " << patchedId); + + for (ui32 idx = 0; idx < diffCount; ++idx) { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(diffs[idx].Buffer.Data(), diffs[idx].Buffer.size()); + + if (idx) { + Y_VERIFY_S(diffs[idx - 1].Offset + diffs[idx].Buffer.Size() <= diffs[idx].Offset, + "EvPatch invalid: Diffs mustn't be re-covered," + << " [" << idx - 1 << "].Offset# " << diffs[idx - 1].Offset + << " [" << idx - 1 << "].Size# " << diffs[idx - 1].Buffer.Size() + << " [" << idx << "].Offset# " << diffs[idx].Offset + << " [" << idx << "].Size# " << diffs[idx].Buffer.Size()); + } + Y_VERIFY_S(diffs[idx].Offset + diffs[idx].Buffer.Size() <= originalId.BlobSize(), + "EvPatch invalid: Blob size bound was overflow by diff," + << " [" << idx << "].Offset# " << diffs[idx].Offset + << " [" << idx << "].Size# " << diffs[idx].Buffer.Size() + << " [" << idx << "].EndIdx# " << diffs[idx].Offset + diffs[idx].Buffer.Size() + << " BlobSize# " << originalId.BlobSize()); + Y_VERIFY_S(diffs[idx].Buffer.Size(), + "EvPatch invalid: Diff size must be non-zero," + << " [" << idx << "].Size# " << diffs[idx].Buffer.Size()); + } + } + + static bool GetBlobIdWithSamePlacement(const TLogoBlobID &originalId, TLogoBlobID *patchedId, + ui32 bitsForBruteForce, ui32 originalGroupId, ui32 currentGroupId) + { + if (originalGroupId != currentGroupId) { + return false; + } + + ui32 expectedValue = originalId.Hash() % BaseDomainsCount; + Y_VERIFY(patchedId); + if (patchedId->Hash() % BaseDomainsCount == expectedValue) { + return true; + } + + Y_VERIFY(bitsForBruteForce <= TLogoBlobID::MaxCookie); + ui32 baseCookie = ~bitsForBruteForce & patchedId->Cookie(); + ui32 extraCookie = TLogoBlobID::MaxCookie + 1; + ui32 steps = 0; + do { + extraCookie = (extraCookie - 1) & bitsForBruteForce; + ui32 cookie = baseCookie | (extraCookie ^ bitsForBruteForce); + steps++; + + TLogoBlobID id(patchedId->TabletID(), patchedId->Generation(), patchedId->Step(), patchedId->Channel(), + patchedId->BlobSize(), cookie, 0, patchedId->CrcMode()); + if (id.Hash() % BaseDomainsCount == expectedValue) { + *patchedId = id; + return true; + } + + } while(extraCookie && steps < MaxStepsForFindingId); + + return false; + } + + TString Print(bool isFull) const { + Y_UNUSED(isFull); + TStringBuilder str; + str << "TEvPatch {OriginalGroupId# " << OriginalGroupId; + str << " OriginalId# " << OriginalId; + str << " PatchedId# " << PatchedId; + str << " Deadline# " << Deadline.MilliSeconds(); + str << " DiffCount# " << DiffCount; + for (ui32 idx = 0; idx < DiffCount; ++idx) { + str << ' '; + Diffs[idx].Output(str); + } + str << '}'; + return str; + } + + TString ToString() const { + return Print(false); + } + + ui32 CalculateSize() const { + return sizeof(*this) + sizeof(TDiff) * DiffCount; + } + + std::unique_ptr<TEvPatchResult> MakeErrorResponse(NKikimrProto::EReplyStatus status, + const TString& errorReason, ui32 groupId); + }; + + struct TEvPatchResult : public TEventLocal<TEvPatchResult, EvPatchResult> { + NKikimrProto::EReplyStatus Status; + const TLogoBlobID Id; + const TStorageStatusFlags StatusFlags; + const ui32 GroupId; + const float ApproximateFreeSpaceShare; // 0.f has special meaning 'data could not be obtained' + TString ErrorReason; + mutable NLWTrace::TOrbit Orbit; + + TEvPatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &id, TStorageStatusFlags statusFlags, + ui32 groupId, float approximateFreeSpaceShare) + : Status(status) + , Id(id) + , StatusFlags(statusFlags) + , GroupId(groupId) + , ApproximateFreeSpaceShare(approximateFreeSpaceShare) + { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&id, sizeof(id)); + } + + TString Print(bool isFull) const { + Y_UNUSED(isFull); + TStringBuilder str; + str << "TEvPatchResult {Id# " << Id; + str << " Status# " << NKikimrProto::EReplyStatus_Name(Status).data(); + str << " StatusFlags# " << StatusFlags; + if (ErrorReason.size()) { + str << " ErrorReason# \"" << ErrorReason << "\""; + } + str << " ApproximateFreeSpaceShare# " << ApproximateFreeSpaceShare; + str << "}"; + return str; + } + + TString ToString() const { + return Print(false); + } + }; + + struct TEvInplacePatch : public TEventLocal<TEvInplacePatch, EvInplacePatch> { + using TDiff = TEvPatch::TDiff; + + const TLogoBlobID OriginalId; + const TLogoBlobID PatchedId; + + TArrayHolder<TDiff> Diffs; + const ui64 DiffCount; + const TInstant Deadline; + mutable NLWTrace::TOrbit Orbit; + ui32 RestartCounter = 0; + + TEvInplacePatch(const TLogoBlobID &originalId, const TLogoBlobID &patchedId, TArrayHolder<TDiff> &&diffs, + ui64 diffCount, TInstant deadline) + : OriginalId(originalId) + , PatchedId(patchedId) + , Diffs(std::move(diffs)) + , DiffCount(diffCount) + , Deadline(deadline) + { + TEvPatch::CheckContructorArgs(originalId, patchedId, Diffs, DiffCount); + } + + TString Print(bool isFull) const { + Y_UNUSED(isFull); + TStringBuilder str; + str << "TEvInplacePatch {OriginalId# " << OriginalId; + str << " PatchedId# " << PatchedId; + str << " Deadline# " << Deadline.MilliSeconds(); + str << " DiffCount# " << DiffCount; + for (ui32 idx = 0; idx < DiffCount; ++idx) { + str << ' '; + Diffs[idx].Output(str); + } + str << '}'; + return str; + } + + TString ToString() const { + return Print(false); + } + + ui32 CalculateSize() const { + return sizeof(*this) + sizeof(TDiff) * DiffCount; + } + + std::unique_ptr<TEvInplacePatchResult> MakeErrorResponse(NKikimrProto::EReplyStatus status, + const TString& errorReason); + }; + + struct TEvInplacePatchResult : public TEventLocal<TEvInplacePatchResult, EvInplacePatchResult> { + NKikimrProto::EReplyStatus Status; + const TLogoBlobID Id; + const TStorageStatusFlags StatusFlags; + const float ApproximateFreeSpaceShare; // 0.f has special meaning 'data could not be obtained' + TString ErrorReason; + mutable NLWTrace::TOrbit Orbit; + + TEvInplacePatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &id, TStorageStatusFlags statusFlags, + float approximateFreeSpaceShare) + : Status(status) + , Id(id) + , StatusFlags(statusFlags) + , ApproximateFreeSpaceShare(approximateFreeSpaceShare) + { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&id, sizeof(id)); + } + + TString Print(bool isFull) const { + Y_UNUSED(isFull); + TStringBuilder str; + str << "TEvPatchResult {Id# " << Id; + str << " Status# " << NKikimrProto::EReplyStatus_Name(Status).data(); + str << " StatusFlags# " << StatusFlags; + if (ErrorReason.size()) { + str << " ErrorReason# \"" << ErrorReason << "\""; + } + str << " ApproximateFreeSpaceShare# " << ApproximateFreeSpaceShare; + str << "}"; + return str; + } + + TString ToString() const { + return Print(false); + } + }; + // special kind of request, strictly used for tablet discovery // returns logoblobid of last known control-channel (zero) entry. struct TEvDiscover : public TEventLocal<TEvDiscover, EvDiscover> { @@ -1978,22 +1978,22 @@ struct TEvBlobStorage { struct TEvConfigureProxy; struct TEvUpdateGroupInfo; - struct TEvVMovedPatch; - struct TEvVMovedPatchResult; - struct TEvVInplacePatch; - struct TEvVInplacePatchResult; + struct TEvVMovedPatch; + struct TEvVMovedPatchResult; + struct TEvVInplacePatch; + struct TEvVInplacePatchResult; struct TEvVPut; struct TEvVPutResult; - struct TEvVMultiPut; - struct TEvVMultiPutResult; + struct TEvVMultiPut; + struct TEvVMultiPutResult; struct TEvVGet; struct TEvVGetResult; - struct TEvVPatchStart; - struct TEvVPatchFoundParts; - struct TEvVPatchDiff; - struct TEvVPatchResult; - struct TEvVPatchXorDiff; - struct TEvVPatchXorDiffResult; + struct TEvVPatchStart; + struct TEvVPatchFoundParts; + struct TEvVPatchDiff; + struct TEvVPatchResult; + struct TEvVPatchXorDiff; + struct TEvVPatchXorDiffResult; struct TEvVBlock; struct TEvVBlockResult; struct TEvVGetBlock; diff --git a/ydb/core/base/blobstorage_grouptype.cpp b/ydb/core/base/blobstorage_grouptype.cpp index 66c79ba42cb..c89afcbf17a 100644 --- a/ydb/core/base/blobstorage_grouptype.cpp +++ b/ydb/core/base/blobstorage_grouptype.cpp @@ -10,7 +10,7 @@ #if IS_VERBOSE # include <util/stream/str.h> # define VERBOSE_COUT(a) \ - Cerr << a + Cerr << a static TString DebugFormatBits(ui64 value) { TStringStream s; @@ -96,7 +96,7 @@ ui32 ReverseMask(ui32 mask) { } bool TBlobStorageGroupType::CorrectLayout(const TPartLayout &layout, TPartPlacement &outCorrection) const { - VERBOSE_COUT("Start CorrectLayout" << Endl); + VERBOSE_COUT("Start CorrectLayout" << Endl); // TODO: produce 'properly hosted part idx' for each VDisk available TReorderablePartLayout remaining; TStackVec<ui8, 8> missingParts; @@ -107,33 +107,33 @@ bool TBlobStorageGroupType::CorrectLayout(const TPartLayout &layout, TPartPlacem const ui32 lastBit = 0x80000000; ui32 vDiskMask = layout.VDiskMask; - ui32 slowVDiskMask = layout.SlowVDiskMask; - - VERBOSE_COUT("reverseVDiskMask# " << DebugFormatBits(vDiskMask) << Endl); + ui32 slowVDiskMask = layout.SlowVDiskMask; + VERBOSE_COUT("reverseVDiskMask# " << DebugFormatBits(vDiskMask) << Endl); + ui32 handoffDestinedPartMaskInv = 0; for (ui32 i = 0; i < totalPartCount; ++i) { - ui32 bit = (1 << i); - if (vDiskMask & bit) { - VERBOSE_COUT("layout.VDiskPartMask[" << i << "]# " << DebugFormatBits(layout.VDiskPartMask[i]) << Endl); + ui32 bit = (1 << i); + if (vDiskMask & bit) { + VERBOSE_COUT("layout.VDiskPartMask[" << i << "]# " << DebugFormatBits(layout.VDiskPartMask[i]) << Endl); if (!(layout.VDiskPartMask[i] & (1 << i))) { - if (slowVDiskMask & bit) { - missingParts.push_back(i); - handoffDestinedPartMaskInv |= (lastBit >> i); - } else { - outCorrection.Records.push_back(TPartPlacement::TVDiskPart(i, i)); - } + if (slowVDiskMask & bit) { + missingParts.push_back(i); + handoffDestinedPartMaskInv |= (lastBit >> i); + } else { + outCorrection.Records.push_back(TPartPlacement::TVDiskPart(i, i)); + } } } else { missingParts.push_back(i); handoffDestinedPartMaskInv |= (lastBit >> i); } } - VERBOSE_COUT("handoffDestinedPartMaskInv# " << DebugFormatBits(ReverseMask(handoffDestinedPartMaskInv)) << Endl); + VERBOSE_COUT("handoffDestinedPartMaskInv# " << DebugFormatBits(ReverseMask(handoffDestinedPartMaskInv)) << Endl); for (ui32 i = totalPartCount; i < blobSubgroupSize; ++i) { - if (vDiskMask & ~slowVDiskMask & (1 << i)) { - VERBOSE_COUT("layout.VDiskPartMask[" << i << "]# " << DebugFormatBits(layout.VDiskPartMask[i]) << Endl); + if (vDiskMask & ~slowVDiskMask & (1 << i)) { + VERBOSE_COUT("layout.VDiskPartMask[" << i << "]# " << DebugFormatBits(layout.VDiskPartMask[i]) << Endl); remaining.Records.push_back(TReorderablePartLayout::TVDiskParts(i, handoffDestinedPartMaskInv & ReverseMask(layout.VDiskPartMask[i]) )); } @@ -174,14 +174,14 @@ bool TBlobStorageGroupType::CorrectLayout(const TPartLayout &layout, TPartPlacem } } else { if (missingParts.size()) { - VERBOSE_COUT(missingParts.size() << " missing part " << (ui64)missingParts[0] << Endl); + VERBOSE_COUT(missingParts.size() << " missing part " << (ui64)missingParts[0] << Endl); outCorrection.Records.push_back( TPartPlacement::TVDiskPart(disk.VDiskIdx, missingParts[0])); missingParts.erase(missingParts.begin()); } } } - VERBOSE_COUT("End CorrectLayout" << Endl); + VERBOSE_COUT("End CorrectLayout" << Endl); return (missingParts.size() == 0); } diff --git a/ydb/core/base/blobstorage_grouptype.h b/ydb/core/base/blobstorage_grouptype.h index cd38dfcfa86..58af450732a 100644 --- a/ydb/core/base/blobstorage_grouptype.h +++ b/ydb/core/base/blobstorage_grouptype.h @@ -25,11 +25,11 @@ struct TBlobStorageGroupType : public TErasureType { struct TPartLayout { TStackVec<ui32, 32> VDiskPartMask; ui32 VDiskMask; - ui32 SlowVDiskMask; + ui32 SlowVDiskMask; TPartLayout() : VDiskMask(0) - , SlowVDiskMask(0) + , SlowVDiskMask(0) {} TString ToString() const { @@ -51,10 +51,10 @@ struct TBlobStorageGroupType : public TErasureType { bool isPresent = (VDiskMask & (1 << i)); if (isPresent) { str << (isFirst ? "" : ", ") << "VDiskPartMask[" << i << "]# {"; - bool isSlow = (SlowVDiskMask & (1 << i)); - if (isSlow) { - str << "SlowDisk "; - } + bool isSlow = (SlowVDiskMask & (1 << i)); + if (isSlow) { + str << "SlowDisk "; + } bool isFirstPart = true; for (ui32 partIdx = 0; partIdx < 32; ++partIdx) { bool isPartPresent = (VDiskPartMask[i] & (1 << partIdx)); diff --git a/ydb/core/base/statestorage.cpp b/ydb/core/base/statestorage.cpp index de39a327ae1..9920e511796 100644 --- a/ydb/core/base/statestorage.cpp +++ b/ydb/core/base/statestorage.cpp @@ -41,9 +41,9 @@ static const ui32 Primes[128] = { 106013, 106303, 106591, 106823, }; -constexpr ui64 MaxRingCount = 1024; -constexpr ui64 MaxNodeCount = 1024; - +constexpr ui64 MaxRingCount = 1024; +constexpr ui64 MaxNodeCount = 1024; + class TStateStorageRingWalker { const ui32 Sz; const ui32 Delta; @@ -195,7 +195,7 @@ static void CopyStateStorageRingInfo( if (hasRings) { // has explicitely defined rings, use them as info rings Y_VERIFY(!hasNodes); - Y_VERIFY(source.RingSize() < MaxRingCount); + Y_VERIFY(source.RingSize() < MaxRingCount); info->Rings.resize(source.RingSize()); for (ui32 iring = 0, ering = source.RingSize(); iring != ering; ++iring) { @@ -227,7 +227,7 @@ static void CopyStateStorageRingInfo( if (hasNodes) { // has explicitely defined replicas, use nodes as 1-node rings Y_VERIFY(!hasRings); - Y_VERIFY(source.NodeSize() < MaxNodeCount); + Y_VERIFY(source.NodeSize() < MaxNodeCount); info->Rings.resize(source.NodeSize()); for (ui32 inode = 0, enode = source.NodeSize(); inode != enode; ++inode) { diff --git a/ydb/core/blobstorage/backpressure/event.h b/ydb/core/blobstorage/backpressure/event.h index 7b8ea72d4c2..efd0e8adbeb 100644 --- a/ydb/core/blobstorage/backpressure/event.h +++ b/ydb/core/blobstorage/backpressure/event.h @@ -6,10 +6,10 @@ namespace NKikimr::NBsQueue { template<typename T> struct TMatchingResultType; -template<> struct TMatchingResultType<TEvBlobStorage::TEvVMovedPatch> { using Type = TEvBlobStorage::TEvVMovedPatchResult; }; -template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchStart> { using Type = TEvBlobStorage::TEvVPatchFoundParts; }; -template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchDiff> { using Type = TEvBlobStorage::TEvVPatchResult; }; -template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchXorDiff> { using Type = TEvBlobStorage::TEvVPatchXorDiffResult; }; +template<> struct TMatchingResultType<TEvBlobStorage::TEvVMovedPatch> { using Type = TEvBlobStorage::TEvVMovedPatchResult; }; +template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchStart> { using Type = TEvBlobStorage::TEvVPatchFoundParts; }; +template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchDiff> { using Type = TEvBlobStorage::TEvVPatchResult; }; +template<> struct TMatchingResultType<TEvBlobStorage::TEvVPatchXorDiff> { using Type = TEvBlobStorage::TEvVPatchXorDiffResult; }; template<> struct TMatchingResultType<TEvBlobStorage::TEvVPut> { using Type = TEvBlobStorage::TEvVPutResult; }; template<> struct TMatchingResultType<TEvBlobStorage::TEvVMultiPut> { using Type = TEvBlobStorage::TEvVMultiPutResult; }; template<> struct TMatchingResultType<TEvBlobStorage::TEvVGet> { using Type = TEvBlobStorage::TEvVGetResult; }; @@ -98,10 +98,10 @@ public: auto Apply(TCallable&& callable) { switch (Type) { #define CASE(T) case TEvBlobStorage::T::EventType: return callable(static_cast<TEvBlobStorage::T*>(LocalEvent.get())) - CASE(TEvVMovedPatch); - CASE(TEvVPatchStart); - CASE(TEvVPatchDiff); - CASE(TEvVPatchXorDiff); + CASE(TEvVMovedPatch); + CASE(TEvVPatchStart); + CASE(TEvVPatchDiff); + CASE(TEvVPatchXorDiff); CASE(TEvVPut); CASE(TEvVMultiPut); CASE(TEvVGet); diff --git a/ydb/core/blobstorage/backpressure/queue.cpp b/ydb/core/blobstorage/backpressure/queue.cpp index c573b7b16b4..9102d772967 100644 --- a/ydb/core/blobstorage/backpressure/queue.cpp +++ b/ydb/core/blobstorage/backpressure/queue.cpp @@ -4,14 +4,14 @@ namespace NKikimr::NBsQueue { TBlobStorageQueue::TBlobStorageQueue(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, TString& logPrefix, const TBSProxyContextPtr& bspctx, const NBackpressure::TQueueClientId& clientId, ui32 interconnectChannel, - const TBlobStorageGroupType& gType, NMonitoring::TCountableBase::EVisibility visibility) + const TBlobStorageGroupType& gType, NMonitoring::TCountableBase::EVisibility visibility) : Queues(bspctx) , WindowSize(0) , InFlightCost(0) , NextMsgId(0) , CurrentSequenceId(1) , LogPrefix(logPrefix) - , CostModel(2000, 100000000, 50000000, 540000, 540000, 500000, gType) // default cost model + , CostModel(2000, 100000000, 50000000, 540000, 540000, 500000, gType) // default cost model , BSProxyCtx(bspctx) , ClientId(clientId) , BytesWaiting(0) @@ -49,9 +49,9 @@ TBlobStorageQueue::~TBlobStorageQueue() { } } -void TBlobStorageQueue::UpdateCostModel(TInstant now, const NKikimrBlobStorage::TVDiskCostSettings& settings, - const TBlobStorageGroupType& type) { - TCostModel newCostModel(settings, type); +void TBlobStorageQueue::UpdateCostModel(TInstant now, const NKikimrBlobStorage::TVDiskCostSettings& settings, + const TBlobStorageGroupType& type) { + TCostModel newCostModel(settings, type); if (newCostModel != CostModel) { CostModel = std::move(newCostModel); InvalidateCosts(); @@ -146,7 +146,7 @@ void TBlobStorageQueue::SendToVDisk(const TActorContext& ctx, const TActorId& re auto getTypeName = [&]() -> TString { switch (item.Event.GetType()) { #define TYPE_CASE(X) case X::EventType: return #X; - TYPE_CASE(TEvBlobStorage::TEvVMovedPatch) + TYPE_CASE(TEvBlobStorage::TEvVMovedPatch) TYPE_CASE(TEvBlobStorage::TEvVPut) TYPE_CASE(TEvBlobStorage::TEvVMultiPut) TYPE_CASE(TEvBlobStorage::TEvVGet) diff --git a/ydb/core/blobstorage/backpressure/queue.h b/ydb/core/blobstorage/backpressure/queue.h index 209346cc2db..52e3c8c6b76 100644 --- a/ydb/core/blobstorage/backpressure/queue.h +++ b/ydb/core/blobstorage/backpressure/queue.h @@ -156,7 +156,7 @@ public: public: TBlobStorageQueue(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, TString& logPrefix, const TBSProxyContextPtr& bspctx, const NBackpressure::TQueueClientId& clientId, ui32 interconnectChannel, - const TBlobStorageGroupType &gType, + const TBlobStorageGroupType &gType, NMonitoring::TCountableBase::EVisibility visibility = NMonitoring::TCountableBase::EVisibility::Public); ~TBlobStorageQueue(); @@ -183,8 +183,8 @@ public: return Queues.InFlight.size(); } - void UpdateCostModel(TInstant now, const NKikimrBlobStorage::TVDiskCostSettings& settings, - const TBlobStorageGroupType& type); + void UpdateCostModel(TInstant now, const NKikimrBlobStorage::TVDiskCostSettings& settings, + const TBlobStorageGroupType& type); void InvalidateCosts(); bool SetMaxWindowSize(ui64 maxWindowSize); diff --git a/ydb/core/blobstorage/backpressure/queue_backpressure_client.cpp b/ydb/core/blobstorage/backpressure/queue_backpressure_client.cpp index 200b85f616d..11c1992c6d1 100644 --- a/ydb/core/blobstorage/backpressure/queue_backpressure_client.cpp +++ b/ydb/core/blobstorage/backpressure/queue_backpressure_client.cpp @@ -77,8 +77,8 @@ public: : BSProxyCtx(bspctx) , QueueName(queueName) , Counters(counters->GetSubgroup("queue", queueName)) - , Queue(Counters, LogPrefix, bspctx, clientId, interconnectChannel, - (info ? info->Type : TErasureType::ErasureNone), visibility) + , Queue(Counters, LogPrefix, bspctx, clientId, interconnectChannel, + (info ? info->Type : TErasureType::ErasureNone), visibility) , VDiskIdShort(vdiskId) , QueueId(queueId) , QueueWatchdogTimeout(watchdogTimeout) @@ -218,10 +218,10 @@ private: if (ev->InterconnectSession) { // analyze only messages coming through IC ui32 expected = -1; switch (const ui32 type = ev->GetTypeRewrite()) { - case TEvBlobStorage::EvVMovedPatchResult: - case TEvBlobStorage::EvVPatchFoundParts: - case TEvBlobStorage::EvVPatchXorDiffResult: - case TEvBlobStorage::EvVPatchResult: + case TEvBlobStorage::EvVMovedPatchResult: + case TEvBlobStorage::EvVPatchFoundParts: + case TEvBlobStorage::EvVPatchXorDiffResult: + case TEvBlobStorage::EvVPatchResult: case TEvBlobStorage::EvVPutResult: case TEvBlobStorage::EvVMultiPutResult: case TEvBlobStorage::EvVBlockResult: @@ -339,7 +339,7 @@ private: throw TExFatal() << "window Status# " << NKikimrBlobStorage::TWindowFeedback_EStatus_Name(ws) << " != expected Status# " << NKikimrBlobStorage::TWindowFeedback_EStatus_Name(expected); } - + //////////////////////////////////////////////////////////////////////////////////////////////////// // check that the failed message id is correct //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -431,7 +431,7 @@ private: } void Handle(TEvBlobStorage::TEvVWindowChange::TPtr &ev, const TActorContext &ctx) { - QLOG_DEBUG_S("BSQ09", "WindowChange# " << ev->Get()->ToString()); + QLOG_DEBUG_S("BSQ09", "WindowChange# " << ev->Get()->ToString()); auto record = ev->Get()->Record; if (record.GetDropConnection()) { ResetConnection(ctx, NKikimrProto::VDISK_ERROR_STATE, "VDisk disconnected due to error", TDuration::Seconds(10)); @@ -738,27 +738,27 @@ private: SessionId = {}; } - void Handle(TEvRequestProxyQueueState::TPtr &ev, const TActorContext &ctx) { - bool isConnected = State == EState::READY; - QLOG_DEBUG_S("BSQ35", "RequestProxyQueueState" - << " RemoteVDisk# " << RemoteVDisk - << " VDiskId# " << VDiskId - << " IsConnected# " << isConnected); - ctx.Send(ev->Sender, new TEvProxyQueueState(VDiskId, QueueId, isConnected)); - } - + void Handle(TEvRequestProxyQueueState::TPtr &ev, const TActorContext &ctx) { + bool isConnected = State == EState::READY; + QLOG_DEBUG_S("BSQ35", "RequestProxyQueueState" + << " RemoteVDisk# " << RemoteVDisk + << " VDiskId# " << VDiskId + << " IsConnected# " << isConnected); + ctx.Send(ev->Sender, new TEvProxyQueueState(VDiskId, QueueId, isConnected)); + } + #define QueueRequestHFunc(TEvType) \ case TEvType::EventType: { \ TEvType::TPtr *x = reinterpret_cast<TEvType::TPtr *>(&ev); \ HandleRequest<TEvType::TPtr, TEvType, TEvType##Result>(*x, ctx); \ break; \ } -#define QueueRequestSpecialHFunc(TEvType, TEvResult) \ - case TEvType::EventType: { \ - TEvType::TPtr *x = reinterpret_cast<TEvType::TPtr *>(&ev); \ - HandleRequest<TEvType::TPtr, TEvType, TEvResult>(*x, ctx); \ - break; \ - } +#define QueueRequestSpecialHFunc(TEvType, TEvResult) \ + case TEvType::EventType: { \ + TEvType::TPtr *x = reinterpret_cast<TEvType::TPtr *>(&ev); \ + HandleRequest<TEvType::TPtr, TEvType, TEvResult>(*x, ctx); \ + break; \ + } void ChangeGroupInfo(const TBlobStorageGroupInfo& info, const TActorContext& ctx) { if (info.GroupGeneration > VDiskId.GroupGeneration) { // ignore any possible races with old generations @@ -787,7 +787,7 @@ private: #if BSQUEUE_EVENT_COUNTERS #define DEFINE_EVENTS(XX) \ - XX(TEvBlobStorage::EvVMovedPatch, EvVMovedPatch) \ + XX(TEvBlobStorage::EvVMovedPatch, EvVMovedPatch) \ XX(TEvBlobStorage::EvVPut, EvVPut) \ XX(TEvBlobStorage::EvVMultiPut, EvVMultiPut) \ XX(TEvBlobStorage::EvVGet, EvVGet) \ @@ -795,7 +795,7 @@ private: XX(TEvBlobStorage::EvVGetBlock, EvVGetBlock) \ XX(TEvBlobStorage::EvVCollectGarbage, EvVCollectGarbage) \ XX(TEvBlobStorage::EvVGetBarrier, EvVGetBarrier) \ - XX(TEvBlobStorage::EvVMovedPatchResult, EvVMovedPatchResult) \ + XX(TEvBlobStorage::EvVMovedPatchResult, EvVMovedPatchResult) \ XX(TEvBlobStorage::EvVPutResult, EvVPutResult) \ XX(TEvBlobStorage::EvVMultiPutResult, EvVMultiPutResult) \ XX(TEvBlobStorage::EvVGetResult, EvVGetResult) \ @@ -850,10 +850,10 @@ private: } #endif switch (type) { - QueueRequestHFunc(TEvBlobStorage::TEvVMovedPatch) - QueueRequestSpecialHFunc(TEvBlobStorage::TEvVPatchStart, TEvBlobStorage::TEvVPatchFoundParts) - QueueRequestSpecialHFunc(TEvBlobStorage::TEvVPatchDiff, TEvBlobStorage::TEvVPatchResult) - QueueRequestHFunc(TEvBlobStorage::TEvVPatchXorDiff) + QueueRequestHFunc(TEvBlobStorage::TEvVMovedPatch) + QueueRequestSpecialHFunc(TEvBlobStorage::TEvVPatchStart, TEvBlobStorage::TEvVPatchFoundParts) + QueueRequestSpecialHFunc(TEvBlobStorage::TEvVPatchDiff, TEvBlobStorage::TEvVPatchResult) + QueueRequestHFunc(TEvBlobStorage::TEvVPatchXorDiff) QueueRequestHFunc(TEvBlobStorage::TEvVPut) QueueRequestHFunc(TEvBlobStorage::TEvVMultiPut) QueueRequestHFunc(TEvBlobStorage::TEvVGet) @@ -862,10 +862,10 @@ private: QueueRequestHFunc(TEvBlobStorage::TEvVCollectGarbage) QueueRequestHFunc(TEvBlobStorage::TEvVGetBarrier) - HFunc(TEvBlobStorage::TEvVMovedPatchResult, HandleResponse) - HFunc(TEvBlobStorage::TEvVPatchFoundParts, HandleResponse) - HFunc(TEvBlobStorage::TEvVPatchXorDiffResult, HandleResponse) - HFunc(TEvBlobStorage::TEvVPatchResult, HandleResponse) + HFunc(TEvBlobStorage::TEvVMovedPatchResult, HandleResponse) + HFunc(TEvBlobStorage::TEvVPatchFoundParts, HandleResponse) + HFunc(TEvBlobStorage::TEvVPatchXorDiffResult, HandleResponse) + HFunc(TEvBlobStorage::TEvVPatchResult, HandleResponse) HFunc(TEvBlobStorage::TEvVPutResult, HandleResponse) HFunc(TEvBlobStorage::TEvVMultiPutResult, HandleResponse) HFunc(TEvBlobStorage::TEvVGetResult, HandleResponse) diff --git a/ydb/core/blobstorage/backpressure/queue_backpressure_client.h b/ydb/core/blobstorage/backpressure/queue_backpressure_client.h index e6cc006a7a9..60fcdbfd0b0 100644 --- a/ydb/core/blobstorage/backpressure/queue_backpressure_client.h +++ b/ydb/core/blobstorage/backpressure/queue_backpressure_client.h @@ -33,10 +33,10 @@ namespace NKikimr { } }; - struct TEvRequestProxyQueueState - : public TEventLocal<TEvRequestProxyQueueState, TEvBlobStorage::EvRequestProxyQueueState> - {}; - + struct TEvRequestProxyQueueState + : public TEventLocal<TEvRequestProxyQueueState, TEvBlobStorage::EvRequestProxyQueueState> + {}; + IActor* CreateVDiskBackpressureClient(const TIntrusivePtr<TBlobStorageGroupInfo>& info, TVDiskIdShort vdiskId, NKikimrBlobStorage::EVDiskQueueId queueId,const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, const TBSProxyContextPtr& bspctx, const NBackpressure::TQueueClientId& clientId, const TString& queueName, diff --git a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h index 290c529ea56..5a06c040185 100644 --- a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h +++ b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h @@ -33,7 +33,7 @@ public: void Bootstrap() { NKikimrBlobStorage::TVDiskCostSettings settings; FillInCostSettings(&settings); - CostModel.emplace(settings, TErasureType::ErasureNone); + CostModel.emplace(settings, TErasureType::ErasureNone); Become(&TThis::StateFunc, TDuration::Seconds(10), new TEvents::TEvWakeup); } diff --git a/ydb/core/blobstorage/base/batched_vec.h b/ydb/core/blobstorage/base/batched_vec.h index f582e9d821c..670ba5e994f 100644 --- a/ydb/core/blobstorage/base/batched_vec.h +++ b/ydb/core/blobstorage/base/batched_vec.h @@ -1,40 +1,40 @@ -#pragma once - -#include "defs.h" -#include "utility.h" - -#include <library/cpp/containers/stack_vector/stack_vec.h> - -namespace NKikimr { - - constexpr ui64 MaxBatchedPutRequests = 16; - - template <typename T> - class TBatchedVec : public TStackVec<T, MaxBatchedPutRequests> { - public: - using TBase = TStackVec<T, MaxBatchedPutRequests>; - using TSelf = TBatchedVec<T>; - - using typename TBase::const_iterator; - using typename TBase::const_reverse_iterator; - using typename TBase::iterator; - using typename TBase::reverse_iterator; - using typename TBase::size_type; - using typename TBase::value_type; - - using TBase::TBase; - using TBase::operator=; - - static TString ToString(const TSelf &self) { - return FormatList(self); - } - - TString ToString() const { - return ToString(*this); - } - - void Output(IOutputStream &os) const { - FormatList(os, *this); - } - }; -} +#pragma once + +#include "defs.h" +#include "utility.h" + +#include <library/cpp/containers/stack_vector/stack_vec.h> + +namespace NKikimr { + + constexpr ui64 MaxBatchedPutRequests = 16; + + template <typename T> + class TBatchedVec : public TStackVec<T, MaxBatchedPutRequests> { + public: + using TBase = TStackVec<T, MaxBatchedPutRequests>; + using TSelf = TBatchedVec<T>; + + using typename TBase::const_iterator; + using typename TBase::const_reverse_iterator; + using typename TBase::iterator; + using typename TBase::reverse_iterator; + using typename TBase::size_type; + using typename TBase::value_type; + + using TBase::TBase; + using TBase::operator=; + + static TString ToString(const TSelf &self) { + return FormatList(self); + } + + TString ToString() const { + return ToString(*this); + } + + void Output(IOutputStream &os) const { + FormatList(os, *this); + } + }; +} diff --git a/ydb/core/blobstorage/base/batched_vec_ut.cpp b/ydb/core/blobstorage/base/batched_vec_ut.cpp index 2cbcd98a660..97c76230213 100644 --- a/ydb/core/blobstorage/base/batched_vec_ut.cpp +++ b/ydb/core/blobstorage/base/batched_vec_ut.cpp @@ -1,30 +1,30 @@ -#include "batched_vec.h" - +#include "batched_vec.h" + #include <library/cpp/testing/unittest/registar.h> - - -namespace NKikimr { - - Y_UNIT_TEST_SUITE(TBatchedVecTest) { - Y_UNIT_TEST(TestToStringInt) { - TBatchedVec<ui64> vec {0, 1, 2, 3}; - UNIT_ASSERT_C(vec.ToString() == "[0 1 2 3]", "given string: " << vec.ToString()); - } - - struct TOutputType { - char c; - - void Output(IOutputStream &os) const { - os << c; - } - }; - - Y_UNIT_TEST(TestOutputTOutputType) { - TBatchedVec<TOutputType> vec { {'a'}, {'b'}, {'c'}, {'d'} }; - TStringStream str; - vec.Output(str); - UNIT_ASSERT_C(str.Str() == "[a b c d]", "given string: " << str.Str()); - } - } - -} + + +namespace NKikimr { + + Y_UNIT_TEST_SUITE(TBatchedVecTest) { + Y_UNIT_TEST(TestToStringInt) { + TBatchedVec<ui64> vec {0, 1, 2, 3}; + UNIT_ASSERT_C(vec.ToString() == "[0 1 2 3]", "given string: " << vec.ToString()); + } + + struct TOutputType { + char c; + + void Output(IOutputStream &os) const { + os << c; + } + }; + + Y_UNIT_TEST(TestOutputTOutputType) { + TBatchedVec<TOutputType> vec { {'a'}, {'b'}, {'c'}, {'d'} }; + TStringStream str; + vec.Output(str); + UNIT_ASSERT_C(str.Str() == "[a b c d]", "given string: " << str.Str()); + } + } + +} diff --git a/ydb/core/blobstorage/base/ut/ya.make b/ydb/core/blobstorage/base/ut/ya.make index 3e223c0bfd8..4f80e3cb89e 100644 --- a/ydb/core/blobstorage/base/ut/ya.make +++ b/ydb/core/blobstorage/base/ut/ya.make @@ -21,7 +21,7 @@ PEERDIR( ) SRCS( - batched_vec_ut.cpp + batched_vec_ut.cpp bufferwithgaps_ut.cpp ptr_ut.cpp ) diff --git a/ydb/core/blobstorage/base/ya.make b/ydb/core/blobstorage/base/ya.make index a5345d772a4..c4e0999b948 100644 --- a/ydb/core/blobstorage/base/ya.make +++ b/ydb/core/blobstorage/base/ya.make @@ -14,7 +14,7 @@ PEERDIR( ) SRCS( - batched_vec.h + batched_vec.h blobstorage_events.h blobstorage_oos_defs.h blobstorage_vdiskid.cpp diff --git a/ydb/core/blobstorage/docs/blob_patching.drawio b/ydb/core/blobstorage/docs/blob_patching.drawio index 527f1c8708c..2e36f8bd607 100644 --- a/ydb/core/blobstorage/docs/blob_patching.drawio +++ b/ydb/core/blobstorage/docs/blob_patching.drawio @@ -1 +1 @@ -<mxfile host="drawio.yandex-team.ru" modified="2020-11-13T11:22:48.503Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 YaBrowser/20.9.0.1697 Yowser/2.5 Safari/537.36" etag="Vvx8akMHjgNF9RnE2p6e" version="12.7.0" type="device" pages="4"><diagram id="oeXZIZbNdJ13JvgRRL20" name="General">7VxZl6I4FP41PlYdIGw+lmVNL9NLzak+M/MaICgtEDrErX/9JCxKCKJUg1r21It6SUKS794vd8mpEXiMNu8ITOafsYfCkaZ4mxGYjjRNVW2TfXDJNpfYwMoFMxJ4RaO94CX4iQqhUkiXgYdSoSHFOKRBIgpdHMfIpYIMEoLXYjMfh+JbEzhDkuDFhaEs/Sfw6LxYhWbt5e9RMJuXb1bNcf4kgmXjYiXpHHp4XRGBpxF4JBjT/Fu0eUQh37xyX/J+fxx4upsYQTE9pcO3v80vyXLzl/KsaR5+eHi3/T69M+xicnRbrhh5bAOKn5jQOZ7hGIZPe+mE4GXsIT6swn7t23zCOGFClQm/I0q3BZpwSTETzWkUFk/9IAwfcYhJ9kbgQWT7LpOnlOAFqjwxXRs5/u5JiQDbu0k+bz7Zg/uh7naZqSfCEaJky5oQFEIarMR+sNCT2a7druszDtiImlLqtFIAWmh0iW85QoqXxEVFpyoetXFscRizNgyFZIaoNAz7UlnMXpSB3QX4Qk1XMFwWO/GCYo9v6Qpl0/QJjrjhpgnBm62kJhRtqIgqDINZzL67rD9iAE5WiNCAmdJD8SAKPC9XIJQGP6GTDcVVKOGrzNZtTEbGlI/FdCbN1Ufdgc3HQ5uucBcdxuJ2AyP/ud4btWoUTeYVg66jW9UPAZOuAJj6KyzPXZJVZnjqb2+GujoWzdB4nRnqdm2c0+zwgRC4rTQrVPjk6RpKp1mJzdmX/P29UoJpSJTwGTN8NMUJsZODSt88DxhAP0YE47PygC3t+kgzQ76nPs5UYr/Z5o8lLh/c5dvywBpoerLZP2TfZvxTLYdh08pHyuUy7YQh86Y4Hut5QNFLAl3+ZM0cuhqwaZK7WH6w4bzzCgLpAUDrgOFXAAQNAILBAJRPUkyCWcCY+MZMxzLEnS+3tHqGKue0HUu7GdvxfWS6jbbjWWNHUYaxHUtG8Ky2YwEJwARSd468uukoJSJOicU6YA7PklaQcvYwKV7g+zdncFaD02qd1eD02zE420XNBufYhm4MZHCqdmmLk528ThZ30+Zm1sG6dJCommAsm8Blw8QTTqqrChPtxhOve5gojmPWws3ewkTxNWOj06zE5sOEiaqpy25XU6B4G6RgAOt+XP075kCp6nkZQjdaGQKFDl73mr2VrZuf2EVzVWmICw1ke3oTY9iaA0zzkvxQZlZqiZbO/LDzzcqoFAzCD/Xp2tp9O0Mc6zAYR1gSR3xd8IXG6ZoZOa/ijG4nuVxP5jc5Dk1u3pC00F7ZuQJaONkDvwgtiOa8y2t05oWaZlhD8UJtvuA4MRzpMRgzyPmyJ0IwuV1yqB0NdoPPoJ+XHAy1lRwuElX42oGownRMw7xoVNHVGsc1xEF7sUdVQFt70RCPvswyX8dU0izqAw1ckWZaKWcDCfqxRCmfJXu1FGBkNLHygnTx5klCVXQg6sAZc33p9s9VtJquPk7H6vrlm/3z4/buTqaIWXbFJ/zAdmAaswAYTJjEC9gKpwFBLg0w30YEU77z6RwmvF+0ybrdO0kU3xfCSYJIwGbJYZgWGb3nvWiyuyrT5DV0TT+o9xlDLWkYxHxCKYWxBwlPCabbyMFcP2YoRoTXa34ZR6CL5z3QNAnHoRJ+jTDKsbqEaytUFQsqDeYTdFD4zAygQNzBlOKINQj5gwl0F7PseBDpnf21GZ1knkd0pAD0cXfvS0TZ5UnNIJ5VUY6W/N5Y9jZBw06979B2Z6IH1anfUFHUy2qOzMZXpTmUOxp9qw2dE7x+02qz8+svpTa/XiJStaYS0aGierc6BaTw1FLTYRciRD7t1YGoR8qCnvXhXYCakpgNMYh5Tu9CLkNdQkta6sf/K8qhQ0i1z6ko5kCK0kvVsxclGSAgGVpNQD3fqV+aT+R081U5K1fp5h4KmFqCrgEiJM2+sJ/bdAHzilTn+vzcK9GbBs45q97IeezyLEgTGPd+Mpkw4ooQO2mSNy6Pnfxt5bGzO71e+/pWv6iP9dWXUjtFxeX8VqdoAxOe19kqry70rtPLeBHjdZxtT7N2/4bw1xM+DanCwZwoMne/wkXyIwkxvvvrw78T58v7hpTvpatCx0vEb7cqNO5YFRLbd6wKqaCvspA0Un91oUallBPYqFYufvOln+ZbfFUeGOoUaNxxcG00cMIFsrdLA6reflFDNkC95Z7G8deNawt6PRHURxqYCOTEMl7cEguMxQLw7qZ5hQZ6KgCzn/v/O5DDs//vDeDpPw==</diagram><diagram id="uKXmrOIFqc6csMLveKSW" name="Block-4-2">7X1dd9tGsu2v0WO40N346H7Ml3NyJjPxiXNzZ+blLFmibZ3Ioo9EJ8799RcUCYroahAQhereIEpZK5YokpKwqwvV1XvvujDffvzyw/3lpw9/X10vby90dv3lwnx3obVStqz/2Tzy1/YRa6rtA+/vb653T3p64M3N/1vuHsx2j36+uV4+tJ64Xq1u1zef2g9ere7ullfr1mOX9/erP9tPe7e6bf/UT5fvl+SBN1eXt/TR/3tzvf6w+yt09fT4fyxv3n9ofrIq3fY7Hy+bJ+/+kocPl9erPw8eMt9fmG/vV6v19rOPX75d3m4uXnNd/uOv/OrPT86+zar3P9yqr37657/ef7V9s1fPecn+T7hf3q1Pfuufyr9//+G3H355+L58982V/uUf//Xmt6+aP239V3O9ltf15dt9ubpff1i9X91d3n7/9Og396vPd9fLzbtm9VdPz/lptfpUP6jqB/9nuV7/tYuFy8/rVf3Qh/XH291367/i/q9/bl6/KJov/7V7u8cvvvvS+uqv3Vfvbm5vv13dru4ff1Nzfbm0767qxx/W96vflwffKa/s8u27/Xca3GvEvtn+vZs/0guPnmu7e97D6vP91fLIBd2tlvXl/fvl+sjz1FME1Utvufq4rP/S+oX3y9vL9c0f7d/ucrcG3u+ft3vp1/f3l38dPOHT6uZu/XDwzq83D9RP2C1nXe3ecbeYdalrCFpR89xX1J9sf4vmq4M/5+mhx1h8RlyqucXl8svN+p/Nu9efP/7cRVXsvnz6yZsvmh+MEswuSSxnXmRqyx+X+mhcLm/frv6ECEkaYM8P0nBI6mmEZJUiJEs/V1YxcqWZfUxOIyRtipBUxs+SVc8dn7wiThTnsaO4iaSnm+y/Dr4VDqURQ391t979buq0GgIlrJV6YVwfxNXzw6ZIFDbq4hl1moRNIGx0inSYKy+5VeOmtrz46T+v7tV96f75tzd3/3X77T9+69vMxIrR4yHaH2rxdkcwIWrGCFEag9aLQZu132L7i+1e9RScz4715n33sZ7xx3r0DVIw1ns2KZKPA8Gep8jHReEXm+PG6LFL8sfl7efdn/Xdm9f3qy9/kdh9Cs9NfP354Wa9fPPp8vF6/3l/+akdirt3Xd6vl19OwJJe+qbR4V0jZZsWXPbnU0dbNW3qDwfd7DLrxutFZVdFLuGFLi8/bq7I3duHzT+/fXfz8Du98z18uPy0+fTqr9ub+tLe91/Wt1sMfnq7f+Dy6vf3j8j8/Hldv8ty97i39t4Vm/+Ca+/xY7d6Dx7ffjAi6e2sjAoBWQZwLLhwtATHR9xoFp81biqDA86FgaO9oHkDV8EB15yL+sjR/seskat3anDIqTBytAUxa+SMwkNOh5ErBbkWchYPORNGrhLkDpHLi8xDTiXGLQ/jZgW3Q9wKjbfiCoJcvbW7Xe+uTgu88n8/r5pvfPXw2Ab5un6Czj99efpm/dn7zb+qeZv619q+0/ZxEhH1L3nz6WHZHwiXD5+2jLF3N182m/QTui+r+s1v1purbzbtmxre9c/th/huk23kixDyJoC8YUOe9kXGQV4L8kd3kwDQB/s5I0BvBPqj+1EA6GkLaBzoc4H+6IYWAHraRBoH+kKgP7ojTg+9pl2ocaAvBfqjW2oA6Gkba6oF/rt3y/IqCP115d5mHViv4kBf4iFP22BTLfChkfcZlAjQ0z7aVAv8d/ZqGYb+rS02jbCU0PuiBAToaStuqgU+9Kr3lVII0HP18uIX+NCr3uDt7TRXMy9+gQ8NvU8oBoBeZbSb9+Hy7nr17l394MX3+sLmF64pTw8Q21xFD5rbm/d39edX9SV6PJrZXMCbq8vbr3ff+Hhzfb0lci7rwLl8+/hWm+u9I/7V71t8c1F8t3mvz+vVw44qeTEec5IB0rxsQ+oooFUAUM0HKO3RBQDVAmjXKRscoLTzNrkVOqAYi7VC8wapZIAGWF2TW6EJAfVXaHJATdG9YX74dHk3pH5SNlQ/vblZf75c36w2b/H19omvL+83HPns8bH1h+XmzS8/bv754/rm4feHg4pr+7M7Kq6owcQVCqphOzR+M8F6SsUNhpcX0ypYTL9ZLz+1M/7jk98s767rf365vHu/iYIfH368u15++fnudnOdfvthWf/Eb7dXevOz1VeDa/KzjBCVB0MkpIcYI0SCsqec7rSxNSXeNcyrcpEbchHZRCXBi1hRVl8sDfihBHwvF+vRjl0cKsd2krNRxWPF0l7nofux1W/NI19te71GEY+FwzqgHjsS/7HVY1U7iIuRxWNHL4lIn55XsrehKrNNvsn2H/HoquHMQ5ng08k8Z5l4KujEo/IUmYe29USsFygPFXaqSeVYMkaqqc4y11joXKO9eC5i2DjkHcJg0Ze2c43FzjXpTLWG55pJpgwXK2W8rCvRoRIXrbFnsIG9iqMbPL6oYjgoEvaGemdWMTRQgy//5tcUwfrR5b/vFIMuf7q/A17+6uL8NwyNHxPohiHXCZoTzTURi4XjucZh55rj8yCwc83oqWZA/LCnGo2daqxrp5oYvYnmmognyNFUkxdtbMosXzigVEMPyKaTasY/7EXINQY61xQeZaEoYuSaDv8h8bFp8/cMdFlTcEmgID1t2G4oHsZ5H8ZcqocwxlxaJz0njMnhKRjIXKomSJ8aNpD9UyswkLnMiCAdabhAJocaYCBz2Q5Bes9wgUxa12AgcxkMQbrMsIHs9wyxQG4q+TMorqH9BSrsKOByFYpffkNHgc88RQsD2pudaoEObTvgkwLRwoDLbkg8Z9phgL2TK7n6buI/064Qsfd6JVdrTrxo2g1a7N1gSZt3kzO9SNl+bzrcDbrW9pzncjkmhNEdZDqkBd2uEzRsdAc5EGGv3ZSGNd7arZSBQneQHRH22k3pXmOg0a0oWeGQafNEpRmXbhNtXDTl2mz/Xj6mjB1qi7Dt3MZmyvi7QmvVYrcVe3XyS3j4NZWbW2zuaWgXhwOrq+MDq2ECesv3jB7QXrfL5tVCRYjOZlHE4yiON5j82aEaDsyeSeo4gVmlCMzKz5pllKxpaaE9t7icSljaFGHpHxLZ3I/L/pfECmW6q4hFA68OeeDqaDyNGP8jdSAAYtt17IriiFttKlW0unhG4SaBEwocnSIp+t1x6/I4RSRte0/KGtJm+61gOmtIG+SEiuve8yj7zgShjKrFsB3OQrQOmDVyPhEfAroOOxkxhTpKr0eArqGQihPQ8XtfAQhdh7OCuLgcpcJDQKfD0IkpxlGCOwR0HcJfMRk4ajLgTNZsqtJBR1mHotnu12xDrDoupuCchdquCgIblfvnuIS7ek7Akk0hArJcat1ZS7IhkOWS6M5ahw2BLJcud9biawRkVcalxp215BoDWi6JrQitjwmtMbCnnaSp1s/Q2Ps0GAzwudS0Iqo+KqrGAJ9LQytS6qNSagzwufphIqA+KqDGAJ+rZyay6aOyaQzwB8lpsQWXQGLpevOUhw6mogrxVDZIRasF1I4Tq1y1QdWuWBjrnj5S4ztIR4u9aIFU0vXGxy2UwcFX0eba5NZvSp20v36NzqDWr3p5h00FK6436+Wndog8PvnN8u66/ufX7//47fXl+upD/fnqbvPzr28efh9apEUNLq7Q0E3qbCT0ZbAG2x9tx4kHTdc7NnW/8hZYZvXCHl1gcWn8Kp/WLMMLwzv1A2CYmWresVe2swuV2Lod68W0ijFjaH9VRHDyvFu8X6GrvgQUlV+m8ilMRO7KP+eZfjR2+lF+SDdUHeb8o0n+EZlU4LxOgycc3cEjF81UG0cHjmNONRvIN46FeoTxaQr3Isv2D3TcPh6/er28v6kv2Sb8JnxPyaPdU16YHTqkCiLL8yTp4NlB03Na0egFjlkNOo70yFUEeyGBCTqO9PRU1HsBHD2qY42j6mnIR8axwyxCpHw9B6No65FLcACp6/N4KwFmC1sg+HFQ9MVBXFaLoQcqU6UxQ8cBbYugBQKXlgFSNZgwEEhfBS0QaJ9zqvRm6ECgW2i0QOASOcxLk0h22Ggwc8kZZqVPpBtwNJjPx8gjIWHOpz7AoXw+rh4JUSYnzGgw99jNcw3pCB6AnebTvfewP5k5cfX5/o89ve/U2GE88jK71mb/kZdx4WhkHuHR5Io991e1aBTkBaY6/gIm3kWPH3041lvBcWaBP+lYz5NQhmis6+OxnufHX8AU6+Z8rJsGCAy5bt8avtd2Pj5OCas0Dd9JOx9TJ2i5sEHvpOVchyyiG283YdB7bc2p8JTlqCmbbKVXlaHJjXN6djI5fFPKjX180eSolh6JTE5uDDRSXWXKLKocCF96FjI5fIHGbqvM2M08tsSg0pOPyYGadNF6RVWZ5VAeEIFxbpPDN+mi9fHVWB4fgUlz2Jp/fztqnFqURy9oZM1/YGyYSKkHlKceroXuwzUup7drLhVNfbOGcW8qA4ujKFSHTWhAx7FDgyZaQm8eAzaOWjkCGLTSuOI9cEbQE7uhk6131eMLDpxfmAM69IuiQ20f4fhHtmg5oCnQp5ED1GECOEwIZ5UDBpNOXpwDTiKd5H5M51F8agLjv0QxHRoog51xVGDYlyimQ2dGmY9j1tO+iowjbbaIYnpAGxJsPWq2OV6imD7eZSv74iAqh0OzDf3SEgfH23RogaCZAkEU0z19PrRA4BLKimK6p1GIFghcUlpIxXS6QKDdIrRA4BLbQmqqEwYC2cSjBQKXHndWqmt/iAIcylyyrfhbgpSqa/SdX2DY0FQL/pTqPPR93Qgzh1DK+ZTqPPRdmzruHx5XV65ah3yLrPkyfM7HcVy3+QG+1fgBp8C2OAXNb/hc7/LtFYfgBWxJVi89E6RKc+1JWAqt2++x/dV2L3sK4ueeLhrjn3tkxyXtujF263gBz3GkVlxdMkgRLFc2NeitD0WPs6aSTV+S5aJm4vjZdDDDYpxs+myGReblwNw9Mwd6L+DKgVwNYkj9N1cOJHwauBw4SD2Irf5NOiHcK53AhGZ1wE0f36TDpH18sYRm+79zykLChOtX7ck3e4BVscgPALapAaZ7kckBnLJBSwHOMyyAaaEFLRV11i9gM71wRy9oXKmoDsxDE6nogBMrwko1fcBG1jN0aNNo8ps1jpZ02dBw7NAXiVa0nWj9TgEcjh2qDdGKtnH056Oh4WipPCOZM/dgwegJbc/L+1aRury7Pvzyhe7d/T2mJP1Q3QxN6+2H7tJy7H6oysjy6PH0duRcIYan9/5CQqli03U2VZaDp7WmDBL563arnV7+qhu91IBklORwRik/qHOnFiZCdgmMshMFbFBgg551aL9OJLAhICvnA6l6+nKRgexwNBINbE+DFW5FcilcIEWwXDjT7mDVB3Pck2y2YXF6TjDT5iEazFzqFEgFKxfMtLeIBjOXPAVSn8oGM+mtoMEcdHWdJMssJYuB7orAcGabJjMvTaFDh/l8fEZSDuxFL7UDI2OmWoMl7KA79FI7Px+PkJQMtAy91s6nqyrsmlj7dF6SVgOzvbAYJyppjneV8vu8PZo/mhajaP5yrg3nrPQu9AANLtlRi9LJCSKSbjUrb14WGmE+py2FyQGcslyhAIMR5gvaS/j6al1fIB/R5lzu88fb7ROeEPzp8u3y9nWNyPpmtUHy7Wq9Xn2sn3C7+cY3+9O4Fiabj0AUrFdefl5tj/C+Xd3dbfP0Y8Xx4fJ69Wer/Dh481evsiyjhUkoVNiAd57UKa/UIs8HpWs+Ln9BOwoC9svB3ru9NWCrRkKUDOnAHMtOkN/dLr/UReDmEm9onLtPv7u6vXx4uLlqA9Rf6r+oSKeX+OAKFoEr2Dz2QtOO0i+TSc203TwQ0w5acPvN4zz33unZ9h+tSBijOg/MS2zK6odPl3dDqnMVrM5/fFV/67fvfnzzt/rfb375+W/f/+OgXt++d0e9nrRg4EoMlV+/a1v1jCPfl/xxbv+WFvByRxihG6m8O0LemGGluyMMquTpXQKpkkfSNteru0g+2lg7Wr5PDlUkQXO9KLPkqOaN5jY62b75/JA2z9Aufql26NS0wNg6dkNbx2bbgI3eOtZZu3WsldGLXSXI2wwOTKWdlCZfZ7leHNRs6TX5TocKeNHkP5N1q1VlCJRx6dRdk367d+uzRM4n0iIgRzssorvv58YiICdTmU+huyIg1+GRIKphHIg6hN0ivTxOMk4PnQnMNxSx5QCxZQ2dWtCWQWTwOiTPIrDsafgArLvgbm6SdH4gSaXemy6nYkcZtrGAek7Akr0fALBcIuhZyyYRgOWSPc9aKIkALJfQed7SSARkuajmsxZDIgBLqSlTrZ6A5I8IwHLJ1GcteAQAlm085rwljgjIcrUo5q3nQkCW9igmJ/BBUnBB0ALNOQyiQpJtQdACjRqkttTQqCal8OYeqrZEWKtnQMxOSuElqDqAtappLYxNevQzXmloaRKX5mgC49iE5jigl1N4UFZl6iNOTfcPQnMMNGs0HnJCcxyEnMNDTmiOg258JR5yHSO9ZktzNHgQdTBRhebobaQzPOjoLlpojsEeSO5Dl57m2DU3SmiOPZvi9OvO0B3xVA/qU9Ic/c2dzVIfIQSmLU31oD4lzdHf+wEAS3ftUz2oTwmsvzUEAPZ8vIhT0hz9nSMAsFzE5JnRHP1dS3pkc1rhprOVPnSVbr7zPI+Q/YDOGXmEmKaN0e8Rst2j0gDl9QixPvHI2ta0zpCpSH70FTyOIoZtOtm8OL8WL8u5SWa5l2erU/MkYpYr42a5UFbyA7sw7Td5tgdnR8L0hS62enbCrKIkTC75S/wtekouPVzvpaykLJx+WVhhl4XOZy3bMlvsekPPSHTkRVy5jksQNi95CV47snGvn1iuk+LwcHHayRaH4yUoLmHjrGRSDq+tXhy37UVNUFKMHS5Oh12MBQorE2UHySXanZkC0Keypc9agWnSk9OKpVUAku0RgKooMDx6cqimVQD6qCKoigJDhGVcyxhoN9NR9nQ5pRaaoh11ZIsp6DHX5JSBaUe2eDfbojKLxi8o2RouaC9+cqimHdnio+osAKq064it9/QPf+s6sOnPJpR8FsHeiEg+e81PPTTzKohmXMZ30UHWp7lt1uBZf08IAV5Jt4Ui/AyZmEKC1+EGLtrP46YHGODpMHizlX/6WiYMlDomN4kC1LeMRUTP0sXUf1Qz4umMOjyeyRZZz6TF+ovXy/ub+o/fhE/i6YutE6tTN4aMRzjl7i7Xf4RTpDnCUc3Ut/2SMGqx20B2EWoq/7iTvobp5MfSjJbuVHPK62baS8VNZqlYshmOtlSoh4AsldktlbKjIoq+VPLepaIysn0lL+JaK0jMZVkrqdaKAVkrRda7VhTZh5IX8ayVspL7iqyVXUcWYK3kvfR/R1ra5DVMt5Xmcoq7UzhYnhgPHo0wL+3Cpm7LlJTxIPZOQ87AIZpqJZfKcF4OT/72tVBBbOOyREsuVZWeE7bkqBcDWy5Byqx8nshJMAa2XLz9eVk9+QfFENhWlKIxVU1G2qGWkODqsymmks61hMSWy6YtfjGVdLQlJLZc3lTzsh+ALJQrrk3QrJTb9NAJA1xRuI5STfmnJBDgWhG5vrBd7MkhIaRUVkSuI6OKIKWyInLlQdt5Itdc2eb8Op3I1Q1awxp6DSMNNTUqN4ExLXGXsKPNi8mBijTT1KjSpgeV5mVsiavftHUqaOEQWeLqgs0fkbg+c/CNyfIgmnGPvF0HYYGmtlmD5w+3AQGvY86iSFzb4DlI8DomMIrEtX0TLCHB65jBOFuJq4FEqcO/QSSu3sY6A0Qv17SAPCSNL2/frv5kUre2GOIXg7xH92aj/7po8cKjeY8WS3udh+LK6rfmMa62l5OPCp4343f6qeAujWyCeI86F8F7dH9dhNPdk4e8ib0mMy7kYhc3D2UdXiTC6e7p+UDcRTLayZsqDQloaq/JbBDbqIeieUYbelOlIQEN7gXBlotiNitON219QGDLRTGbF6ebdEYgsOXSUM2M00225BDgcglt5sXptojYKi4xRvxiCmjEIgi2dAs71WIKaKQcCLZcm6BZcbr9aVwg2HIVyjOjdJPTKQhwB81RwCb/Is0tQiAO5oqe6k8OVKSxRQjEwVzRbY8QuscA259a5JoHUrG5c8Uloipmdcf1D1ox7riallPQBGDlTDsb5mW+aPZyyQjAuT5uqAxGzNh/sSVm6H5mxgSZF3pXdvQyL3ZrIDrzosrbkVzk1WJn28PLvdBBybOw1ftST6U8wFww9USmhB333MXOPKMnngFhw594HHjisX4cR0s8HeRTLYmmnWgsYKJp6q5JJprqLDONUeiZxg/kIlKmaa6MyICOZxpbIGYaej/AyzST3BMZEy1hvDAEOiaxiRLM64ZoxPVLUcJbvwcDC6rWxIK9auWcmiHNaR/+wu8QXydVESIufIe48CfVBfXkaeNvESAWPngXVGd+IMdqRpgOsbkoYVuZRmclYqaZVNfz2ULYKfYiwLueWhXtk/V4vYiOrqdIXduJRmkPIKcXZepEk0+q6+klmvEPdhEyTQ7e9XyiazSBbCJlmryj6ymy7Ham0RlgSZNrAt5UhUQpKbD+IX1pgtjGZVnlou7k6HZiYMul7oxPj0yKrUPElms4xKyUu6SFgIHtrIS7GzHFz/UPuFlvMDCPm5P2l3x35BwRfS5Suxb02wwDxHqsqeLnIf5Nib5FRJ9L+g0pD06IvisQ0efaZ0PKmdKhX18mRPS5rAEg5eMp0UfczxXUQW1yUuSUGznl2X2UVRk6ookrRS5oc21yg2lSgqozRFDPwAkioWkAWalVpgBAHeQEgb1SU4Lqr1QMUEN9NA/A5rjx3e3yy9f39xsLho27++7T765uLx8ebq7aEPefTW9/zIknzfQqH1zBInAFm8eed3hMTodLrx9ilIfM9gB896oncKi4pvSZc37dtD0gJ+/UfZ7dCoQxDqSL7i7bw6fLuyHVtgpW2z++qr/13de/fv36619+fVN//tPPb349qMC3795RgSdNJ1zJQWW+FVtRLXJ38EHyhAqFOV+isC9vuYaD4c16+al9A3l88ps6w9T//Pr9H7+9vlxffag/X21C7o/rm4ffh+7WosYKW2zYZhB2c+PYTPotj8WGzqPGRjNcsoNm9cSjGpdr9aQBaUlAnohSYdrU84t3ejN70c1rCE3KDWV+F1lH9DCLQ53XG3baLPLKD8rO29+zXs5EuHJUT3rmYbunJ14ckBN32olOfiJSrGdpYt2r+qy1Cxc7WKPL8k6M00Dl/+zIDcepnkqc5ttpetHjtKr8pGrjJ1Wq9YtFua4OOdfqaKyMGNwjtQYR4rbYzhZ6Qdy+sI5MpUBUF8+4H0rshGMnyQS8J6vAJueZIva9uSiTSeZbccsVtmMPdpxSWOfbAIp+K7d5tmh3hJwuFmUV4w5O+/zgpqIeQdXmob1kQn/RwHBlsWw8gXvqqj5g44p+usYxdx+hzBNH21BHMHEsusbZit/YcT4oHI4dCkrxnTrO7ITDUYdxFBuh4xxNOBw7fODEpOU42xIOR0q8FA+MEG3LMylxlQr0IhLi2GHPJg4DHlPLgK9HLtHyzMwGvO2lylQfzlEJ70XGJWCemfFAgY4zl5h5ViYEpBzGw5lLtjwzQwIHjjPb5GkxJ/Du4BV6JHCJlbVEQrs1iV7LKdrTEtMClmpPoUfCrOabp4wEix4JtK821bofOifoDH0HqLgsBsXYwFfWokfCGUink5ocFIUHcFEsGgJOKpltoQZpp7FRTSqIJ6jaDABV2s+ZnCI+qXeFCazVwmX7j8Tq+ELTRs7kAE5qeUAArpctFMD0bjstrqtSKlvY4tgVjUt2LQLLAXjmwv6LwTMXxmbDpx8zVejBIo/tekkhbPPY8ErZOHT4/cUR8vYLT9d13pep4rIoND11Ffb2AA9gPCA7aPhC3z5O34YDskynHT5p9Kyqb0Ct8sG5nvLh8avXy/ub+optwu98a4qmQxahpnhZ0Bm6xxPRwBAWFVr2MB3qD1EN9NCk4IDUYSBFNtDDg4IDskP/IbqB47oBpY1bGCQgOwQgIhw4LhzAW5Fc58wzVw5o24dz3FNkw6UQmblyAA5nLoXIzJUDcDhzKURmrhyAw5lLIQKpHOC7P/u6ADScc9p0El1AFF0AXCRwKUREF9CjC4CLBNr2mmrNnjL3E9Y/HM5c+g9h/few/uEigUv/Iaz/HtY/XCQMmoKHzQ+HYv0bbXo66HGJpvkZyDqgBAAmt1gAn8F0PCgtgNF64congE1qgGkLbnIAQ2kBTF5BARwYLP/11bq+QD6izRnn54+32yc8IfjT5dvl7esakfXNaoPk29V6vfpYP+F2841v9iebLUw2H4EoWK+8Gmy1PQ79dnV3t63FHslhHy6vH8cwZiFYX73aXNkLn0MWdbRd5aduZftSd6j44lMsBKaKC/AvB940Nc8e99KEBp5GBpvuuQTsl4OdN1pILLDptkrAHmFlG29l51W2UAcieirMjYw73W0J7iMscm+C9SPuJRLudBMmuI9Qwvnz6FVuk6d2uh0TqEeA2pXo1TqlSAjwI+zPM1NRjS0W9M07C/QjN1f9+zoc8NKa4WHHOHTgpZDnAd4v6NCAr+iKn5gxi6nMosqfrqgmVzSyMUsV5J2IkcWzxR5F1odsXFFP1aHO0gLkcSMLOCCP+x88GRwwmSColgvCIusZKFx/4VsZHJgj2PaU4d27JfdGGHAEevX5/o/9/WSLCKNRQrXbz/YbJVRJJmzXlYHzN4Z5ZRe7HuurTs8mp/pfxuTZVB0fJB9eSC3YR1xVdsxVlXZ092D+AMR6sWnWizM08Ive9VKX4HSZkZdxrReqyJf1Mr/10lE/JVgv1vTeX1xF1wt5Gdd6ocYHsl7mt140zHop++8vWU7XC3kZ13oRh77THPrQdqiWtuXELG2IcQQckDoMpJil9ThDwAEpM9ZPs36AA1KGrJ9mllZkeqGRgJQp66eZpcGtSBmzznN+lvfhHFd8a2XMOk/Ni4azjFnnKYnRcOYy0Zq5WRoazs3vdwb3ZyizNDicuSyyNCLOSGZpcJFAu1ZilhbFLA0uEmY1RJ0PZ3LggIazjEhPZJYGFwkyIj2RWRpcJJyBlxaUWVphy54OelwnHifT0scFuFQKC+Az8NKCMksrbNHys8xTA0xbcJMDGMosrfTGfCcGuMxEih3HLC0v+lJ3VGFmmdFmnAA/xk7c992AA5723gT4MTbe1HcDDnoxSIzjuwEHvJglxvHdgANejBPj+G7AAU97a+C+G5X2b59lUUI5b5RZkGAmzhv9myGCbWX7sI3KES0z2skS741g4oOHUnUoKUSk5hcv+FB2aClEpuY3HfCh7FBTiFDNbyPgQ0lrS5GqBdsC+FDSilbEaiEoK6LVr2yBJFcrVUcFK3I170guh1+VzfRwCFuSQ1eS5jsdliQd9iPqIrL9yOAjfT77kbKZo9JrP7Lbr9CAHWw/8sK8waWymZk6kvY4rO5LK1H5dmWjCpC0Mt200jQ84dNK84ueAbE/qRiX7uzh0gptd0tamVpaUdHSyuNLn2uWprOcsCtsvQErq8P4DXis0XY1fRmPWdr+mp6B5CWpfJk2xOAyID0lkgw4tQyowTOgUjQDutMyIHkZVwbUTBlwXgYOgT4yWgbUXDLemVk4WHykuYS8GhHplCYO+A0lTc90xcaBpQsQuImjxQKXyd7MjBwoKQcOaS6bPbFy8Ha++L0/zWXFJ2YO3h4AvwuiB2mFsdX+Sbd5Jd3glw5J718aeqgzOYiTGjoEIHY5FsS0az05xX9SS4fAQUVpoTT/hrblJgdxUlOHAMTOYEEsSl+eBh1N4JXpS+BxlX9GtL5MAkV86EXtG8dTEQ542nwT4Meo1+mZKxz0tBsn0I+R7jN86GnzTaAfAXpLnXzQoM9pfT85eweX91nhRbZ3yINH22Lv8Hzpg86qPmzjKqryDvUxbXLMG0pq74AHZYf6WOwd+uwd8KDsUB+LvUOfCAQPyg71sdg79LHZ8aCk57hi7zCIlgsHZfPOYu/QAyWxd9BZ6RYGCUp6Liv2DoPsHfBW5fmQ5bEU91r1uuvHpUQVXATpuYug8ZA+H1IsltgTD2kuv5S5i9rgkC5p9TzV+zSWqA0PaS6pvkZEGkrUBhcLzSRVEfOfJuYfnGoYxfwNebVfzF/m4YCN5JJkxTtnTsFWvDDYHl/6XOcI5QifYbN57nOO0IrQnAMvY3KOsFrWxdQdVZqD2gHroqNq4F4X9LBblf2OKpZwwAMv41oX9DxN1sXZrgv70uJkvHXRf78I0Obi3S+ae/A8nCjYtmch5MG2ZyXX4cncfSbwkOZyGhKfiT6fCbxY4DpIE5+JPp8JvFigBNDJmRBg+UxoVZgesktc+XJJT1MnBzGWz8TmGAYLYkoYnZwJAZbPRL2K9cKVTxAXqSEeZAiEDTGWz0S9iisoiCu67xYJ4hgluac415nqS99xBYiVGIxEUpzjQS8GI5EU53jQi8FIJMU5HvRc/r7xO+wpqcwBVSZaf6XiIrjOispc38jzFs4ms9miogV7ZHbb+dj3DwA3JdMR/yjN0i4buomItyvSJl8493RFNbmikS1EbNAPWyxE+rVcHrKm7EM2qmav0tRe4pB4s7x9u/rzkHPDRLPZM2t6aDYXBySbx7tObJ5NsbTXeSjGrH5rHmNsez0ZeTZ2V8cM4NlU4ehk5tnU9wL/FmGyOISZqhnj8ywi2YhBbVtRvciK44Fdf/F6eX9T/9mbzOhzylTRjvesL94fv/Lf8OWLoEW0m/SKcGkYmZbq8E0xgHnmXP/LuBiZHW4Z9Fhj3nd4q7Hv8LuQF1+p/g4GOpA6DKS4SvmqemwgK013SXg191mWzs5EK51fGCJIegg1Zk1rQWraSUdnmjJWZ4bWo/mAUc05rX7Jy5jKWEfP88W/L2hQgn7TpBsPvJvmPs+pdpKbZ6PK5dHutqflM2VIo0pHmiFfaWo7OeU7/MsDePQaYdIrYpw7fOBeXBBGhtH7kN+/0/YX3L34aS08f4VltGIwp1UM5GVcFQM9SRab2KDRFXbFULoOb3wxifVFFc4H0ixyJCA7nPHFItbn1aOvSC4zybkZxHo451kfznEJOI5revrc7GHRce5mVD58urw7Hec9o/Lt/RPEe+S3733GyNNuDBjyVVP7nYFuPa1ZLDrOXAaic7OKRcdZM+GsZ4WzBa/MqoyrAp+bnxQ6zudTgad1k0LH+Xw0TQnXs87Ad1qVOp96OyXOCn1fpWi9PTmDoLQeUN6xUZ5XPb3sqMYilaKF9uQATusA5QNcaSyAKeFocuZAaf2fAitYPeGb2uCrUvR8eHIAp3V/CqxgKIDFDISnyM4LQupQfck7qhlIpehJs0A/xl2buj/1Q6/3r4kEPu2NCfhjNFEy9FVPm2UC/AjAW4MOPFf3bG7eT+BdFc3VPdPzwhn91ENPzPhHV5bURWjWP5UW65+TsC2oVzkYH7PSjiYqXE2VmP/UkJndM3sVJLtkGF1TtTfAj27909xXxPpnPtY/z1gPRqdZD42RzjOMf/S+FxHb9md/PcX25/jdvaTOxmh3d6PDUIrxj1eEU39yOCg7XAzE+seDssKHUpxd4jq7DOYbcVYpQ0fjpqpSbOEXHP2+Lroq/NImkqvL/npCubok5LVZOlEJLu9R2TVwr0H8WzaQDbVVTdVrcLnXa4jm3mKo9nzKd/FR3VvaZUDPwpl02I9zsw7cWKuIFi1OlydYtOgqsAGOZNGyR0gsWo6XBS5AuwErC3J6NCsmLSEoLV2lWDYtVd7ROhSblhaUJgtYcKGtSto6nKrsOyWXPXAEjEaZyGlncapC0ZRIBxqPcEhTWvtUJYQpkQ7UVHBIc420nJVphy4cPtJcdg56VkiXE7hPcxk6zMq4I9S1gEOai5Q+K+sObempPhrSBZe51rxMHRw9/IJDmu6nJ6f6T1p0WzJ0GM3YoRik+8eGOGW+DkEMZu1QnIHyP+EqNlkZXMVI2v+CNkkmB3HCVRyEGMzeoRCZN08Cf77aN7bCvxChN8+uWpNVjyb1LuiuWqAfo3VWwEPfvPMZtM6SHnHhN1RKroaKnhXSAW0THNK0odKZyd/dLr98fX+/yaMbXuPu0++ubi8fHm6u2rD0Ex23P+ZE+iG95gdXsAhcweaxlzIKjfIA3ZBQmrlKzftsqZGEUUjfrXDeu+mSvtuz+YmtiBiDJlh2cxuGjidRwXTw46v6W7999+Obv9X/fvPLz3/7/h9Dx5Mk3SiyZYxcewGhVLHRA/lJQuVRt3olbdiA+4FUdPv8RGlPaAJSBskjYgLSyyGg5XGpg4DGJfiVdCcsMu9QKUR3thD4ueNqqrj6k0P5SfOdPi+XtkpkL7eKJq8aWl4z6kyasZv9OpOqDEcps7wq0NapiJkLeZnJMjK8l76MSRlS0jaPeB6E8prDzGsV7dWI0UGoUKTnLBj40X35vIXZBhSnjtnYoqDzDrUyUPw6DBBENufhV5FapNT5QiXHjx5Ci1YuxCfIQdcfFx17ZgK5QHukDMIb93ihOh8OdtIjw0CVigCv5ToRnpkULlAcQcDLdQw8M/0bZeRiwEv3mFM95U8qegO991ouRfrMlG6BkwsIeOmZ7FRLq5Srtwo0cCHgPR+XgaSatkDfEAJe2neanMopaclMleZVZkP9qLiiCEv7UZPDNal6LYCrMQC40j7V5PRMSSVrVFu84UE37b90uNIG1eRwTapTC+Ba33KT4+po60JkKmOUy1S8UFTB/BxXm+LOZ7ObVLEAuhtyXJvd+K2qpPCC9jIcra7Q+eXeBM+qQmCXu+Bxm7DL+49PPThdlv5s3GYdHD5ajc4bvdIfp4uBng6jJwxav+SARE8GhQ1kJkCiR8vFmbNnIVES7uxAeggieprCNOXJPfjz9wBG+ths6Py9XfkTXWrlqH+8rQYM4ckJvZm+jEdqtb+mQkHvSYONJu0pDaYnoNtMCOgDDwoQb2KZ0M95+id5+q6izYR8zrPFQwBXCfWcZ28BAa4Qz0dJyxYSXNoYnepZXlraOSS453MOn5Z0DgmuUM5HAZeceECAK4TzUQoq0mqHAFfo5i/sPXndQWfTk82tErL5qKiaTKWnmlslVPMXdhrJWk1PNLdKiOajolqv1fQ0c6tpF0po5mOUyJl/GlekJ5lbzdWWmhvJHLFE1ufTlkpLMYcEt7tzMdTBPgxu0bzN2/snXIc52J8F3PRQHgJurl5GMa9ehkMEN7Ch7SeevZxr1uG4/cQGC3PDnr9WKTFs+/cSWlc/XWuwM7atzMtChpfWZcqiWPijMWy+MD0O2oNexkTrCuzlzjxM93TMi0MuZnWcixkjtjV0bBeWBmnuFjZKkFI1ymGQLm/frv6EiM8AW/bZERuOT50+PjPo+MzzQBJ1kZIo1dvMLT7Th+cLq0nu9KmC6bOvNAi9LFpUUx0Sc1TvhRLVy2fMnLAUuGbIjBrmeaQwf+Hu57jshi921MUzqrq5xU4BnSIrR3OdKyNVmMcHbMWK1+Ph2h928fZVMcK1jBmuNB7rG6AfjyZTTTyePvi088flNPxVpPCnOqAU4d+zyZlbuq6g07UzgdI0VrxSmgK0E48pAxcr27f40nnx2MAENvHiGQBoTjdmxgQBjSsVbLbB4sbTgx9tZ2Lgp8P4iR9PG7+Cnplg4CeOPMNuiBoUP0RPHkT86GYdAz9x6xmEXxXY3ELgJzYjg/CzhnQLjNH75kQ6/MRoZBB+G18RyPU3K6uRVf3mN+vN9TebNtBGxfBz+yHGArYkAVAFAyAub41tFqaWAPAqYMwAYJuWCSkfSBkADjQAuPQjkIZECQOgKkEDgPagpspLxw4Aa0ADgMseBdK1KmEAuAw0ALgsVCCdrQKIryIFQIC1ihEAXNKk+LsA7AAIEDwxAoD24aa6CxigTUsZAIGDSIgAoI28qe4CsDNAQP2FEQBcnUBIdWrCAKgCR6EQAcDVCYS0WksZAIGzVIQAcLQTODnjrpQWT7lSPq65VqEzurh2QIGps5MzeUqKqyW7dgxcad9ucus1pXmXNaDrlbbjJrdeU+LqUPPwy7tsKlhgvVkvP7WX/OOT3yzvrut/frm8e7+s//3x4ce76+WXn+9uNxfrtx+W9U/8dnu5Nz9bfTW4TosaaGxhUuSEUqPKcBkW4uizxYkLTHlEVzuQK5lXZmHIhYwrdnAmnaK5NfntYpjK6eJQ47QTRx2VOY08ym0Ab2p7PflGublmRmavXGq3SqLLoHKaNYoijgzKBQaIinhn0MaMYFZmdX7Knj4UyVVReXjORLcUGTFVzTNTFeCZqqBRX5SxMlUHLZ9uWmaemSx8ZtKptOJjZKYqemoCmIfrBs/DTZaaaNgXVazU1KE4EcWlT1jHT01uAqnpPDNMFS3DvLCX0qFPEn2ur06AX+3NwSf0aj/wrKlapjV7G8dZFSJ2KmmCkl9EBh7UsOCnieNG7GBpQl3IfsVlQz2yUu1XKhr28VoplJYlDgdBdRV+aqLtr+mkpllmJpWBZyZbmmTHUYryBcW7I8gzIhiVG2NRpMwUfeLCiJkp/kk5RGpS4KnJUVZIvNTUYYsotjS+IBW+aFKaQDlVbWpKEnPAQLHM+5COqj9wikuGrmeFdMBqEQ5pLr05pOcMH9JUSASHNJewHNJchg3pgH0jHNJcCnJIFxk2pC11jodDmksqDmkXw1h74yN9Pu6Q2KYAAVY5XCycj1EkdiwEeLtosaBp53eq9Tu2VUCAKAkXC+djH4mdF8oAIRMtFrh6deIh43vI4MfC+ZhKgsdCgJOFFgu03zc5p4q0zjIZgdhlPSfIUc0NnKaNvsmZVqQ1maHdADSIaYdvcqs4rd8M/iqmrb3JreK01jPwEJvjo9qfuDzj8n2iTaqmZJ7t3/v82b0u3wV6P8Umjzq7+tlUnLIgYbkhDub2MJxOfBkTg8dQFuGZh+meEndxOCO7Oj4jO0ZsF9CxXVgapNouXJQgja6NH28k+rMjNhyfPTPcY8SngY7PPA8kURspiUYXSMPFZ/rwzKHDs6AV6yZ99pUGoZfFiuo8nRC4OmSrq6OhNeJSGKlrwRvmVaQwf9nuJ0+lDlUXz6jq5hY7FjpFVo7muqqIU2E2F3FCXqDkYlV2v2dMaAaaB8+jxDzxhLMHp4KAxtWF5PRMSSzmggcLoPh1WASKD5cv3AHFr8NHTZyVfDkOKH4dzlhieeOLbEDx67AsEl8QXzoDil+Hr4u4J3in5tpR/FSz+0qHX/POIjE/jp/LMNdfwcVbnpuunAzbcYVeJKccFrRxMlW5SloxOSi8XOziuSnIQeHlMgiYm2wcFF4uV4C5acVB4eWyApibQBwUXi79v6jCfVU43ToVwa1T5ADgsgWIX1tjB0CAZYMRAFxeAKL/9vXfmAFQ0ubXVOtz7AwQoOBjBABX90yU3r7SGzQAuPprIu/25d2gATBoSj22GjStppvs7lSWZYumtkomECzPQKyfVOaryIHlI7B5alxf3nRTwYz9Zr38dNGSBz8++c3y7rr+59fv//jt9eX66kP9+epu8/Ovbx5+H5rkowYVXxlfkZCo8mAO13ncmBgk6dfQaz1pEqfnJ49rXbmDj9TrXjT9L8vnLpzPM4Uj6q/oPRtdJlBsRDDti1rpRTM4PJ1OoFmvkYXne4mZbQ1E2VyQo0qh+ovXy/ub+o/frEhfPaTa8qHdux1RED1+5b/hy+enXH2+/2Mfc6eubcZhKk0o9muVqioc2Nwq4rIkSUiRkdkBnSY1Dg+8jkm8VAUrTpHc9G8NaXLUWTA5RuXMqf2N7lnJsbX4JVM+S6AJkRzLwcnRJkmOhTktOZbUaz9icuzQz9BCfObJ0MImw+ND8yacDI3yRuxlykk6bFWAg9KhS5IOy+y0dFhR4/mI6bBDjiZyUJ9vD5sO6Z3rTNKh1IY9Fd+QZGizNMmwOjEZVilrww5tp2irfXUKbDKkMmpJhmefDN3gZJjIySivTkqGlo6nipgMO4TSYlTga7lgk+EpltOSDKedDG02OBnqJMnQqtOSoVMJK0NLidfi+tExMBcxGe7u/GL70a9tzAMAZosiOYA6DKD4fvjqRdQVyGUdMDfjj5Lim9cLNDUx3XJ5B+hZ4VtqVHy5zAPmZv2Bii+Xe8DcvD9Q8eWyD5ib+QcqvlzuAHNz/0DFl0v8L/Yfvv1HYAeVB3dQcSPAcan/41fY2BFQhIjAEBHAJf8XAxDfAAQ1AmgbbKpVOnYOKEOEL4gI4OqjiQWIbwGCGgFcnTbxAPE9QFAjgPbiJucVkdYEJLDJs26hChz9uBvkEYCNcVL9uA6cZFq7cOUTxHRPHxnjM/AISOsDEVrHdlEhYUz7cpPDOOU6dll4HQcaclGBVVlGVy+6+wPlk5pCL5qZfsncH+prGexuima9v5KhkOY2CGlk9qnqYFzRTDdzBC0sgrTTINrAICUHFsGOYZ8iaPJJN7AIyrjIgbQaWAQ7dDRCHfeJM6gINqWwcMePI2iNCSCYLVx6BDtqUSGP07025hps3hpCy3YoZWu+06Fje9KstSRre5+XDr3a6NK0wV06PmlavQx3e79ebVr91ESWVhVxfVeF7Ren5QU1K6Wv4xGnPV3XM6CGpZVWUIVhobOeE4OoJ391IkSyApREeGIiNMMTYSIzKzoDbVAiLOjY8JiJkIsbo2eVCDf+LviJkDaSJRFOLhHmgxOhSWNkVdFZgMMSYcC+PmIi5KKIzU2sN4VEiOSHKonwxEQ41Am/ToSJTKzoTMxBibAskyZCLtXy3FStU0iE9FhTEuHkEuFQ1/s6EeokidDS2bCDEmFlkiZCLnn/3OTfU0iESC6okghPTIRD/e7rRGiSJEJHZyQPSoQ2S5oIuXww5uaTMIFEqLk8McQzwfdMoMSBQgeJA7FDgMs2I/6RAHYIFAEmOkYIGMrfmmozFFsvWwSo7CAhwGWcIbYJvm0CbAhophAQ3wTfNwE2BLjoAWKc4BsnwIYAPRidnKo+rXOCIWrczVH3gW9CasV1DfIZ2GOktk4gy9eZRZEhgTzIH0NDg5zWOyG8kpt1kRDZM3DFSO2YEFq+SidHlvbjvr5a11fGh7KR4Xz+eLt9whN0P12+Xd6+rqFY36w2EL5drderj/UTbjff+GYvvmmBsfkIwL9eeWXXaqvY+XZ1d7ctvx7PDz5cXq/+bL7w8Xz1KssyeswQihG+tWxoJ74uuBYNH7an3mL0dTC0/SaIj9FqyRQo4jnttgniYyBeZeR+DYI4ba4J4mP00gKEEVVki4HnapyI016aID5G30SF1jgE4rR1JoiPgbiFRZx2ygRxplo9N4NNZTkRp20zQZypVgdBnPbQBHGmWh0EcdpbE8SZavVNpk9/G5eWW7RSHQNw6bhFq9QhAC/oTRzdhnhD8G9dyDI3i5I2smK7EDtq6XYonFjevl39eaiZYJJJ7JURPTKJiwORxOOmIrZOolja6zy0YK1+ax4tzLbXk1MnUQzXSWxXSnSdRB7wUql0LL1DEdQ7iLN2P9OFZKmqCmapyFZ4jroWTidJzTRHWfQcVQRsTqosWo6imxTxig96xYPmpKLD5Fis4n2LQUwAVXbcXxXjpnKeN4cG6Rg3hxcu85JyO2SeQNBAD3OZZw25U8YJ9ADoUAHUYQBlmoBvWIQKYMdQHRkm4GsQHAVQL1R6ACkvQmYJBFnoqCuQywkJ0i3dk/wFRIFsEVAUyo8Aq3TPkN3IesCSyyFESzB4JTF+MFR0ezNVowjwYHATCAZaaU/VMgI7GKpyAsHA5acN6Z6YMBismUAwcBWQkA5yCYPBZRMIBi47QUiLOTaoNxwJAnVwtxgb3/PZIKTEt6BsCgh8VXbKPK2Z2eO2rIFPDSrOI7RqdxsYcIRm00xPKAKTROwQ0/CAsSZ9HRMvQ2X6hMUhRtLTXRwaZ3G4AUbSdTHY/zou0lJzXc+gWzTATY6vMqCcJojKILNc3UBIz1A2fEtKeQLBl8sTVgxBfUNQS5QzVrnQXPjYIcDV4xVDUN8QtAyEgAUwBLWUjzE5r8i0hqAZLcDq/+EYRdozsHxN6wZKb+A2XxzM+QDAeJDjK93NImGc1gyUrOKQsjU2qmdg8ZrWCDS4cpthCAmBFRcKnmWsS8K502ahh1VZjMJqKy4UTHm7AgVcXCh4umVlhQi3o70ygXuM5pkp6PpWi4G0B07AxfOVp1GSkfUNATdtlAncY8BdYcLNNSgJku7Ih+4EOO+Oi9kKSWbkg9pRj8/HYy6Ng7XK6LKenFOYy4uFolcyslOYymjDUVyVTnJV0lkVRDS2AQZtSYkFzSALGhQAxYLmRAsaEABVhzeFmIv0mYuAAJjTXJmOLapatkIb5+2jjNH6i9fL+5v6UmzCyWeR2haNtHm3I0zSx6/8NzxL+yLVTJXrp5fuUnR0emmVZd5y2S2CTmJp2RCfO17BxbfOaa4TF64RwnhIbJoxYjMQfKQH0MrWzTttF9ruxcdCM/R+GX2/7V9M3m+8UG2WvThJ9dytiZMUyt2aFsZTvltPMCsNubkyZaWS3BIbnWNPAnn+zbfyf1Lee/M9/gq2my9NXXg3331RutcybVdMNUtjZdXYgUW7wT8//puT0n00myjRrDp85sQosM8oEOT+rDp85sQosM8osAbQLMr0ANIjAjEKHMRSRVmBYhSYyihQ1/evRePAle7AVHGZu2iJgB53QJQI4LJ/EUvAPktAlAjgkvmLD2CfDyBIBDTUknmw4aDM/1AigMsqAJIkB+X4hxIB9LhjqnsBLJu/Gt/gXi82vlyM5/iVPpbNHwq+XHv5uZs1oeDLtVOfu1kTCr5c+3BIsyY2fCtNxnw+FljptQmaa5sNacXEB3C4jZIeX0M30ZNz4UHyWdLK6b2lVjInD2XoznhysGKZK2llyzpH45grKUP3vpOz4UEyV3pcuqZMDusg5ztsWLHcleqlWzTmwymBpXtdEeuPsfn1vFc2FWfAJC2uVl8ZsVaK4sRRo20WBS24YsMtxkpRnDhiw/3PH179+38ffvv4a/7j3X/f/Ph5+T+//PpV98Hjw6fLuyF7YmVDe+I3N+vPl4/g6+yb7RNfX95veKzZ42PrD8v6/9c39e318arWv8L1zcPvm+/XFcLm/4+/+9OD95vnr34/2Gpvf8GOrXbUEoKvrjOkmWL1goZMkXHd///98fP/Xn/+/d//ndvi43/+n4e/9L8DMQPt8aAr276EJmuWUxR7h9AlDJ72irdDH5JF6SGpHEWSjcMZApJuXMXTIQBcaeCAoyx3YC3O/ouhWpx2NTRtYU4IvpAqp/NeFVuSoysS7mOru49cE7En6SkHMrRc1KHDEVsSD7gKDbgO/U1ShXq6o0FtczSAaJ9SJIoB4JxCA462HEWaGAKuGT2zB87aQH8gKnQdLmkiSmz3drICbc2p7mbg1BjICQ9jaZNi39GNwY0JAsslLoAUGMWrP9MDS9tPU6W0pQSWlD/pgeUSC8xKDKILBwcslxm6nhWwJd49NmjXLfKP595j/c5pemC5dD2z0n1o63dd0wPLJeiZlR5AO7/plx5Y2jyaHGM8aTlsMw/RKkQ6YeOcBjGlXaXJYZoy/VJMn+Y6psI04HoxOXp4Stb/nsZ0uE6bXk8yUAfJdbBBTcn5p6DWCzU5qLTPJJTgMXY72gNbmVBWZiOAB7Hmaj1B+pHxQQu339Hn03pKCSyhMqYHtrv1NJTE3+Nc8Pb+CdNh/PuzgJqcrqeHmqsZNS+TikzBAUt7FtCiCuXKytem1qgtSpvtPwy5pHFFFmHDD1FZ9EFb5QTasuqDNipTJeAXILKLAJJWoSNZHp8RJDIMcBnGUB3G9v4WW4ihXHMWsA//sSf9HLsqosToSU8WPT0ZkWYMqwYLeCQ7pm/LNMH2njfT6Ejm9NhIRB0hJB08kvSsSFQeASRVWVIk84UCQrJjQyaijzaS2sCvyfNhHidsHYd6KJXqAzpqxzHnOg6alSgkVPmiAS3HQ9FKZTTouY6L5qUPClTSaEBzcZlnpRdSVQUPNNeUAj0roC1+ecY1rWBWeiLl6AkYGNAFbWdNtQ5PuaIdPUtAA5pLhT8r7o7O4DdcBW2GTbXqTgm0gt9eFYPmG2CLklLuq1SpCcKF7elrRxU+FLRJNjmEU8rOQghbA4Uw7Y5NTq+Ucg3rPLCGq0XlnhDOEyNMm2CTQzjlGg4hbDUUwjKxgmdPlRHkS92XvaOK1UquPdWsxGohIh5Yqd3k5zPoeyYFGr4dVg6quLHv1gkRLjw3FuXyclElvj+XZ1BjJ6zAKKaVSo8praqhBWvaM+9UlckXml7EuBK1MnhuKxK1vhXhYWnLIJZxlUwdluU0rc0aOqUBsXOUOy4qtOmo0Mrd/r9Xhba9acVWoWkS8soudoZmrDq0kvZFRIcW5HDipaSmmBPl2fGyrkTEjjYrRGsWsOEwiNh1KD5FXdbCLs8QseuYxyV6sjZ2TYw/YadD/YS42HWM6hIF2fFeEMS6Ox/2+YDW7ap+85v15vqbzXZo06T8uf0QG/h+HzALYh+1X1+dDyEdGnvSN4EA/3xI6tjg+ztUBPAt3aFOlbgODT7Z4kKAfz5kdmjwyR4ZAnza35gqwR0afLLJhgD/fAwiEp7pW0Rkz8cgPKUSDXGXZmlnbKqFekpJEuIezMp8ujGghdxhcfVWZqUYNYj7J8e1eZ6VRjRH3B05ujWenGYwIYM5b5gPSAxmR3e8k8M04d2VYgrAYHZ0IyuqsREqKeVhbVW+yPIngYkelJ7ZKNeOHlZOToyQUkKkgwoT4/YfNvGyHjT9FxvflO0Liu8mVwPhS88coYUnLig8OXo942pQVBbci4gIpQfYKqhCOQpsVFaOyjrs1WnumzWONqhIQcKRAoanTnm5ruRkfcup/Qg+QYpqVNS9ipTt3Sy2IsVRRYpaGC9SGQQp++siipTjKSmoSEFKSTIXaVDtF1SnIOHYwbpOqlRJaBebBSUpSIB1qIxFntIGUgX1KUhAdmgzRaviARkUqwABqejuWYQrQ/pKaCtScVFaIVUsXDiTrsjjkexRmKOeziqlmWDWc4KZNE3gYOaiqUJKU9hgDtKikGCWKWYc+1Q4mLn4q5CKA75iOigoQMKZi8w6K3FBFVQXIMHMJRiPX4Ml1ZCgw8zFX56V6oAc1sDBzKUCn5UCwaGX2pqrPzIrynq9MUXHmTZIJsd1TlpkBwnsOAQ6pc9gbFVKhiQFGIshqTRtigizfYQ7NGW260XiAShKT8yH3ZQ680dQ6EovGv+3dDRYLV7sJwGaZwTQPAsCGvc8T4sh+zD8Kkz8CnqCjkd7FVP27vU31JV9dw+LTYI1RaXaYW92wczLgdW0XSIc2EBeKnLMvNTUd8J97Sn0FCh+HRx0cWf38LOg+HVwloUC28avKkDxo/ssYb4G8LPKUfzUQifHr2NfJYRXDz8Huv64jusgea4JrVuLXJEAqB9q6v9kBzw5l/eUlgDwKmDQAOA6yYXkxqYMAAsaAJopACBZswkDoCpAA4CLHS/27V4RqEEDgIs3D0moThkADjQAuBj1s2Jam9zQTV4e3ORFhpeLSB+/xk8JbxE424aAl4tAPytmtSkCR98Q8HIR52fFqDZl4AQRAl6uDty8fN0rlS10oLYqUuNbcDXYZsWUN1W4e5IeXjF4fylhsI1qobOF0QTVuKTpgvbEJodqyputpXSPQpmFpQTpyLgOUjtoaFyTrlbrr1blFs48+bknB5i2siYHcNKFS8+pNwu36RCmw5V2qETFMsamt6jaaJtCLxpSzvg6lvrL+9Wmit1/74e6Cv3w99X1cvOM/w8=</diagram><diagram id="XSTu3f3XB3Kw2yURKv7b" name="Mirror-3of4">7V1bd9rI0v01PDpLat0f7eA5k5PJZcaZZOa8nIVBdvgGGwfkxDm//pNADairW0KAuqtFZdYacxXQe6tUXbWrauC9fnj512L09PXdfJLOBsyZvAy84YAx143D/E/xyM/1I7EXrR+4X0wn5Yu2D9xM/5eWDzrlo8/TSbqsvDCbz2fZ9Kn64Hj++JiOs8pjo8Vi/qP6srv5rPqpT6P7FDxwMx7N4KNfppPsa/krWLR9/Nd0ev+Vf7IbJutnHkb8xeUvWX4dTeY/dh7yrgfe68V8nq1vPby8TmfF4vF1ufv4z3+uP7++fljETx+YfxUs3z9crA/2S5u3bH7CIn3MDj608+1q+vfw5Yu3vHqfffhl8fvfsy8Xfnns76PZc7lgw5uPi/nLz/I3Zz/5Qi7mz4+TtDiYO/CufnydZunN02hcPPsjp07+2NfsYVY+vVkrJ79TfkS6yNIXAZSGX+RuljnnZzp/SLNF/sWc8iieH6yP85Njx6H6sUV689jXHZTD8rFRSa77zbG3C5jfKNewzXp6YD0HLBw9FMvzeLss/nweTpf/uGB58xV7Km6Of86m+Tovmtf4dg3Ib7ebB0bjf+5XMH14zvKjpOXjd9PZ7PV8Nl+sPsm7C4r/CpCyxfyfdOeZcPWveMf8Mdt5fP1PF6wuE2BlAYQ1lKAadIZqAFBdocgIRTWKCToUQwBXOsmvFeXddHY7/3G9feBqa/CKdZovsq/z+/njaPbbfP5UYvJ/aZb9LK95o+dsXoU2fZlmfxVvfxWU9/4uD1bcHr7s3vnJ7zzmP/Wv3Tur97yKAn5/+77VPf7GNQ34JY6V8JffzXUggSZBGk98GYFiduutCCQhSrFiFZos58+LcVq37OWVNhst7tNsj2vRocRbpLNRNv1e/XoyFq3eerlYjH7uvOBpPn3MljtH/lg8sOUzA3x2HYGS60NuCbr5bkdwNpJbHo8sj9LysBCd5YktsDx4DUii0y6AE98T6eTGAk/Wlq1825YqbS0M+CDBwsA3ROIbHB0mKZGbJJ9Mknqn4mEzSfzAqE3S1hna8X/+5ge00RkKeAgEjTN0HItg+GBlCwKyBUpb4DvobAEMWqxQhPslQnGDIrj2GkcRXoARW/SNBf97ewQrDXqAzaAf5HsGRlzJQBFXi8jwKA1PIAa9zRseq+JqguFh9lqefsTVQpDESXRYHkVcDQZqyPJskHLRWZ4YoKjE7+Hlvkhwv7p9enh8VT64i1qxXtNxboVGt+ns43w5zabzx/y523mWzR/yF8yKJ642UO4Ct/q3c4zL2fS+eG9WGLSrUXlvnC/+ij9P6WKaL0BxO7eOs+nTMv24fehqvibJ601C3Nk+WBz062L+Y/p4X8D+8+F2XvyAh+ciob6yrhUqQCaN4/T2Dpg1CRsnozS+G2vdByWQTp6ETl5ndIKRLaKTLXSCUX/TdOKmkOhkIZ1gxNY4nWCsrf90Go+y8dce0Anu2ozTiRGdrKUTdMWN08nHTaeH6WSywhgToyZRcuvAIINsb3iXhuNOGYXPQMHgIDHKIkYFkY+NUSFuRuFzyBHRycfnkMMYItHJFjpBsZVxOlEw0146sQTbxS6C1smeyh435lU7P7kzobGyJ777PXz46fyW/eIMkz8//JFObrwLZoWIV1U+0JzmrJ5po0UlxZk+Tnbvts6J7pHaWi8uSIPuR6Q90qVRCV9junR94uhOg7qhV2V8GGhIg0ayaw6Vq7WzVaFTRS5yoK3qKi2qMFVQQIPPVJ3Y4qgsX5PhQ2ipEq2WCpqiKBBMUSQw9UQFD24knDmhr8PmKeoX1N422Th3QwE8Ni6xwMYdXs3ZqTu2h0StayPHa6iwumORQPgw1GCa+KJQtWcL0xT52EyTFbVV/bcw+5ZaHW1hjorT8K9J5ZRtwkMuunMebpHP7ZxHsHWKmR3nPJOf81rLJitkOJ0B6O6cj9Gd83DLiO+cVxW+GN6CYDAWHu4tSBLLwzDdbkGoFviQVlfoTBPcMtpjmig44qO2TMwRA8RagiNQYUrFwk2yBIehs0w2tykwnEbHYJr27XdgyDS5AuG1pNFjRR8DrdXElu3nmJOYM01yFEkebG9FDEwD+JBOWgV4McmDLaYTiDAbp9M5yoP7Uq8H/XDTdEpgvmLFmXezN8P87mNYeAv5I5Np/ruH00WOzpoj6WiZrZdeTbUG0Gs9Fa7U3lvt/apKmdzpfpyMFpNdytynj+liNOsOXTH+YxxdGP4hdA++FGDzLBK4hSZ0D/YbRWWbRnSlERJZ15BwlpVbtArs4bfnOX/iYrmKG1zmL3D9p5ftk/mt++LvTZY+5U8Ortkg9gdJ6K5ffJM+TvI/f4we79P875vlm3y7+fLhcVYs0ud/5Xt89nq9zMVnuxch/zb5j1t/ofXhASVzoLKq2wGcA2VJ7SLNf83odnWogiLltj8/bnA1CIbFsZ6z+bKMlGiLpnHNYUkVJilK8WRFKawrrhhLQO5G0sqo2D5dnCtNnLfRs7077yncthrXb49Oe4fExqRo8C3DbmxM/kIjnfZcocTfO3GjPelPNZaHqjCUCNqSoMwEQZlrgKDG0hEVgkbE0JYM9YwwNK4y1D9xl2TpT4WqMF0JM7abMXOJoe0Y6ptgqBcIDD3xlCrpTzXXADoihh7B0OBIhh61dzFVo+oO6MJ7DGlCE2bNTwy4hqYaPlQZSgRtSVAjwpMgNEBQUyWwVYJS+KctQ2MTDA2FBmperIGhLoxm29OFSOwB7bkw3qu3CZEr6wFN/VraZXyEKJMvSfholcS6DIBKDSkak7LYQFTUzVDpfk0+TYxVGAdRUWJAtdg1F0mGDUSFGJtmktaAmGADEWqxqQyxqUNyiA1ExZA1qthSgxh42ECEumOalNc0TMFBBiIjxeDpzKx4rYwhuFoFgwx6NgTuoWeuaH6Ngws9HgL30KCBjw1c6AkRuIc6TqL3axxc6CERuIdec8Ugg3FwYeqNwD00+CCGAU2Dyw9M4J4gxisG6o2DS4WPpwNXTKUZBxdp3eM+UgVs4II8qXFwkQYxrAQXnVlGGsSwEVyQVzUOLtIgho3ggnyrcXCRBjGsBFcYI2geXKRBDBvBBflZ4+AiDWLYCC7I25oGl6ccCdwTZIXEfK5xcOEMM4tk1UI/HfOzXSNzBYw02rWBRnsUXnDZRGPhxfq00V144Yrdsk/ckrZ2TahS4IggCbbm2ZG5VgDH2ykyUy5qM8XEztknbupfuyZU+9Jm3Ag2s2RVS3/BLJ3VKFb5Ochw26XEhF1icrtE5Vw1aYcQm10y1WIEz0xGDOZFMmatG/NyFFkCRQEn1f7VFchjO+NNtWw5zBPZcT42/avO2hORzD3DaCoUZcJUYVqT/HKwmQpTzXNoeOtJTIVkDhmiTUvgmNi0KCrfqWi6xi6J7f9N2yWeWrTSLlEsJcRtlgS260lFKXo5UBuAGjmJOHPGuFmCiUJ7zNJZjW2Vn4MRarsUCnTn97u1S4r2JNTZokYJJY5FNG2XAqQaRhvLucSz0PgorJBq9U7nUWCbcxbCHD+Be+guVgzEGwcXaSGmjeB6Yl7VOLhICzFtBFdUNxgHN6aBWJiaynN6NG7cEjMDsYRmaImnYeMWm5PB0kSsYyhqZiKWUHaX6IgtxDQSy06KGh2JFdPEIStJY2biUCTYNabDrplSTdDIoeMoambkkNBwM9ExjJKviaW1sdUViyXbQb21sQmNHDpBIaFwHkgqnrVmTRIGQKWyq8ayK2wg0sihA2pUsIFII4eOnstnHkQaOXSAIBwbiDRy6AD1LDYQaeTQAVpDbCDSyKEDhFnYQIQtqpTwSRPBO6AVyzUdj2a/jW7T2cf5clrmk2/nWTZ/yF8wK5642iBZyQkX/3aOcTmb3hfvzYoA1NWovDfO135Fn4b0c5krfj1/fEzHWYnpNoGcfV3Mf0wf7wc7CeSH51k2fVpFwwSZ6Z79z2rCULouzq4jGX2sNS3tOjCsQ3yyhU/itgsDn2RxLeKTHXwSd4AY+ARDav3n03iUjb/2gE+iC4yBTzC6R3yyhU+iN46BT0gHLNgo2hTzmhjgRdqF30Z4RWUFBnhhSGTAwllWhh0qwIffnuf8iYvlSlhxmb/AZU8v2yfzW/fF35ssfcqfHFyzQewPkpCtX3yTPk7yP5+uv38sTLKY8eWfnf+Y9cevDwYomAOTVS9F4IIhXlceppPJWjqS5t99dLs6VEGJUmyQHze4GgTD4ljP2XxZll0OYBWmmYS9hCqbBsC7XGHdcYVK4k4Gr49vo8G/AMF7grMXX1zCpZrHk8ErVidggPcct3V9CTuBJmami7VcF8pHiE620Ak0zDVPJyhkITrZQifQjdU8naCkpv906ksME7SgMk8nKO4hOtlCJ9A5yDydYEzNnkKRhFdi8a2N68H11Fsp4royyQ+VirTCNQoFXDd4mRJybUhExSL7wxh7+GC0qoPj5s6+Da87aMqotZTT5c5MYy1neeHSXcyZAErrKDjeLAtVObVwDhx81gfihc/69MCInKTFChxh6oiM2gyh4wdZm63yfVu2tB6WCj5JMDOSd7jiO3SUmZdLTZV7rQxThM8w2TVTte3IoNajPeJxKk/z3caBHyh0FV1atH076h/vFh1JJEX9oNEi0IPQ7KB23vHxnfiKUkGq96zB0QXXWvM4WjXzre3AFPs90kSb/T7M4WSG3EcYLKVK5UbzI0a9zZsfHna30vw0D0ax3v54Lnb7Azjt6LA/fF2oyL5VHy989ocEhfYmxUGg1pPM09GbFfdIUWgxn8T4GgI+naOksC+iHehwIyAUiQotJhTwoBAQCrmscFMRiYlTqqIfiVPPC4e64xRCIyWr1yVOWcQprg7FwykumMXKKXyeOSZCAV0DAkJRCyx7CZV4+AxUi1DU3Sx9uVwsiqUu5muXN4fj2Wi5nI4HakF9c0R6cExIGa73znLK+jjwx45UWEVCFmrTv5MfYh3pBvoqKNPkcu2NUkuMPSqUWieLLPuyCFLLXiW+rFfJ5+Gbm7f5s18ub/L/X/3x4e31eys6kXRlA2KhHlla3+7K4tHddR+JjMnMBzrmR1ZN0WhRwTk3ZJUGNK2TZ83jhY4ybvvky3g/m+Z8mW9k7FUc+VXO+5GOdJkPw5gWFYslwpp5sfFiMV8WxqNisXbFYuK5EEjsv950pK9Q1ak9QYIx9vHBqFAnUdVNjZV10cEYKEQeVKNQV1mND0bFpC+Slhd7ngAfXorSIJKW16WaGT4cFSUdpNGtw1HcbCDAkQYKHZABC/HhKGswQgkLO6Qf0D0OIJ/0Jiw4e4lPNvIJ+Onm+XSOCdXeSNOg42KeUIwIZS+hoAdlnlDIq0VIl9bEKYRGCnnFCHGqiVO8ORMiTp1j1UhfpGmug3Crd45VI30hVOKjM1BcnkdjWo6HF0h6EMAL/WSb5D6C1Dw03xs6kopCSe7TTu4j4BoZ7z6QwKvortpzK+fsqAVKXOmB8sppUH3md3YMKSYl6Ph58X1jRNrb9DUKHcpCo71loZERWWgiSqEjr23bUOEdHQlJI7izIYFco0AOneGLFNIAEsjVZQDxwaiQq5JAri7xhg9GhcCDBHIrgRw+vBTyYhLI1Qrk0OHIy0hIINcqz4wPR2oGeEh6Fx+ODOCoRJCi3ENcegHgHsfGK/pj5HIB4lMbPx0Bn5BLBUjP1MpxQUCoc9QJ9IZQwINCQCjkOgESMzUK5PBxihpMWs4pFqPjFPIGk/g8c0yEArlHBISi2jp7CZX46AwUnw9KArnj4Y1F4Y95eENoLi7HWb6UKpvx/DBbv6BL87Bjg6Rnfv0J/csvjqM6oXV19BNLhiJJ0bUM6e6Ue4nNUkjXcYQFjSO4oJq1kAlpIY/XQgrVv8yRmETNWkiFJEjtRxGMcYAPRoUkiJRdNf4nwwejQhJEyq66wgF8MCqUQqTsKpybEBtefHYlKbtaJdY8fDgqFEGk7KqLEzv4cGRyHEnZVYdjhA9HUuLYm+iG7rG/X1Sns/jdxlIRn2zkE/DTzfOJhDj28kniuJgnFHIhDhGqnQdlnlCkwrE6x+16CI0UchUOcarJTnkeNk65MELd3FKj0j7ihP01dttr8GfaTVRzB2c+UY3xwa+NrTOYq2Brt60zXMerykuKrPkuj+EMtqT+Dd20ztgsJFZri2/PisnUOviCIFxbQ4SykFBJgO/SDRU7pHk8EN4YCHzMwwtzKaR5PAHSYhVYsmf7x84UesyF8gIUJ/Ie7iy+ExlMtzTd3ZO5UI1A8B4MLzZtOnOhSIHgPRReINA0fxmGQTebBOlca77ZOxtvzstc2caaBOntThShAekWL2OaER/itRtJTGe38x8d9eWttOXdM264GzXcCSKaiRsGaTzxZWyL2a23Ytug67ghtwF7xA1jOTX1ttxl/Bt3GwXky0JlFm0KCNEZJ/7LqMyiTZkvPhhhTITKLJrL//HhSCPma/CK8eFFI+YPqLMI8OGoGAdAdRa1/Xjw4ago/qU6i9peXfhwRK4SxJcSRiQ7Bf6xZzwWzZArBIlPrRx1BIQ6RxFUf3TxoudinlD8yEQoKwklulAICAUje6gIRaL4pkILhEYKhhmJUzZxigkyEQycohJoiwnl4NvreedYA90XQiUhPgMFY1GkNz4e6ZgPDdlEHU33WGXeOW7q+xIliqNCrYTMdvjnuKvvy7UoThKEjGqxBbubpS+Xi0Wx2IUUr7w5HM9Gy+V0PFDLbpuFeuvPPFB2B1d8ZzkDyXLyx9op6YBULkyq/fl9JsC0VgOW79oiBWt1g9B75e3+qxw3SITjrrWD4LgnE+P5srZk4SwrgauwI/z2POdPXCxXkF7mL3D9p5ftk/mt++Lv8PJT8eTHyz8+5X++XN7k///tw80nfvT8664/YP1ywMT8jMyqPAO2RrmHX6T5txvdrg5VUK4UQubHDa4GwbA41nM2X5ak1Oi5uI5Q8+hL8qVuJOGxyLcTmoVz3EX3JXzMHCF8HBmv7vCRlt7ZWEO7rSNBA28M9b2aahIGOxUJryJeoNCumcm2DOFcm5kE5enZXJQQmmlmEglaTV5M3G1RQlzftKfLUhu2f48eoucOPT0j9IwFCSqXLHVMT3hR1UNPd7Brdcnods7qxASrmRMaYTUUu5tgNSNad03ryIgvsS3N5rTWUuAYwDSiRfXtsbBmsaTbh+by9kCm6aby9na4hoIUNokhrnql+oFiNJA6/kQwumHsIoMxpELg9jBG4jbCPIyKAlIqBK67WDJ0MCrqEbXWAdfGYQ/HtDsYE3QwKoZYUnlwDYyJuIs1D6PCxaHq4LqclrhrMw5jpHBxtBYHW2ZUNz3FEcHIAIxKAHuT6u6LSg+4zNtOfMaSoREN9bOYUKLzjoFQJMaxl1DAccFAKFLjnBDfEB++SDth24gvyO1jwBdpK2wr8RWlRRjwPcf2Mr254ItbXAyESshgnA5ffA4dN2GE7ykuCC46fBMYUrBKPBIIC5rABdWsHklke2pSj7RVjwjAMsmZojeamygynWoHinB0Q665RIOjJ5n2Y8+Ui4jGXCQlo5pVoIkZyX4k9ItweYVltzLQRNFRmHRRtUkefNcZhRSDhFG1uRV8OMJQCymjmpVR+HBUaGpIGlUrjcKGo+dC9Qw+v6/37lvZl+ZYpwx6XYl4Jd9s+xt6mxxQvCZ+UqLBv9s09CFtX6sUOT47BCPm+OzQZv/pVvef7Oz3n55TvhLr/pO5Iue17D8360Ki1VYZPXT2yWEARyWCvUkR90ZkCAMakpZfWlNMnkOqVYsJBSMr5glFqlV7CSVxyc0TilRRFhMK+lDGCcV1HqSaOYVLA7QA5vGFex3qtH8KZ0MY0rHZuhprte+557gd6ssYIRirRWA7znE71BtCgeAaAkKd43aoL6MboFrIPKEk/NnNUGxTEBrSFPUNalW9EjdZCz1ZivHz4vtGGI2vcaLn7ts4sdy4aJfMxZHg93leJWUhEdmJpRLCOzpKcrhQBEy21hpbG+MLjjOkkYI9zBa+SEGI8Fpqd32NuB/3ZANktNbXeIzqa05wpoi9SHxJ4ZTevHMAAdPqdcYVdcwrp6FLd35nx5aK3qhbbd1dHq3GI13dEw+o001FoKxhe7upzEjbejfiM+02Z03Y4KbGfFKn4h0duamMatUOsImJO0BmE5miFoRqeuoiKmLbHAQ4KmoIqKantrgbHY78G1FNT5taELFrCQIcFVpVqumpw1EM0iHAUdFEnmoi6nKi4qYeAY4Kf5W043WpSLGbDAIcYQ26EkEKjw9xCemg4xx4kFB6w6eebBQTEcoSQgEPHgGhSOprL6Gg64KAUHBrT4SyhlDAhzJPKJ+0xR3lMoXksL/ncPvuUpk8nUC2w0I1CXPEeAgC20HaYosJ5eLzbnzSFttLKDdK8BGKIkQWEyrGFyHyz3FD35sIEdBcICDUOW7oe2OhIoaOUJKZK/ZIgD0fzByRbJs1S4Aj2SlKEuB2wLridCJZrbXe9CH/RiRbbIEjC1x0ONZLuZG1ONzcoRb7HL/yhGsWYq+vbbqF2J4nTs7jF6BuZdV8XUiO28I+eWKpCQL7BAHDZ5/OwMz42szMkXyBUVCSbzee92LwEcF5r5AZkny7bgMoxvwQ4KgYdUPy7RocA1GMpRPHD5/fzt9///bl8vrv/7z+cZ1/ztWPC0aX4bt4nMob8t3GgR9sCdHlZThG7u2HgLl6vH3FAB+jxQXNfNFjTUIxLIjgqkCDFQ62Qxi2A9xfRGuHIsB5LXYoUURFqTimzj6J4izz9omnOimxaGGmGoaxmAsJpTexmMBwJBHKHkKJ8REEhDpHtV9fahngJQ8BoWBkDUV3Oxv74HtMTNgjwBcG4qhW5RQxV3GkhSsZC6a3VkUyzpiuDbbI4uDWFYHtIJ2lxYQCCgzjhNoIFalteT/alvt7T1ot/UztAUEfDIOIKwFB+A4G5ttX39FNCNGXzGYlW2uNrfXRhZ58B/IHxcbOxrblHjN5LZVn5qGySU8qjd/eleK2voxuL5xGUmlar5xS+CSJNOnrzKTRAnEml3/i0R11a2JrEY6wZJ7OIhzpeso8CirBaVuCI54JpiVZ9Yos6sHfox78Uvwlimzp60IjVw7PiYTzxW/Yb/l8VrjiHd1caxRycfWei2xh7gSHyGyhQixOxT61KglsVzSFVJxKN2qlCdhQVMzBoMKN2sINbCgqBNVUtlFbtoENRcUsE5LFr2Tx2NByYeCD1MPNUip0MFIGx2JpHnCLA9MRfpcRnyzmk+igm+cTSYct5hO43pnnE4y+kLL0FJtCMDRIo7JUjvQ5jnjpkbAUneU4R6Fyb7ROXoCOT6RTtphPPrqdF4MBGeKTNZ4ySFOa5xNFhiy2TyxGxyeq2TwdvEDraB5emG1DAa+Vym2QnDMPr93N5sXoRChZT706V0a95k8gdBV7XUQS/bJeoSvMf5BGr9FZCV1sMMKBdyTSa4yxiP0UdcL47uLH8Nd319/+/dfN9fTu+/Ttn6PRhQ99zg7qjVaHye/yHcpgj0Lejei89z265efXvoJwZqaWyAdUdjTouz2FwJtEpbU1TOh8AGoHfgCMYpcp4zAG0Gezp+/rWU2bkZ+FIe5LTAAIr+USQy3uDwjHiAOUjdsmT1G4QGLpQnUkRnvMw6WoUCCxdB2MYl9C8zBSyt7iFCsMT8SQT1qD6pzPxCcb+QQ3nsb5dI4p+/6IpcH1zjifYBSc+GSNBCQCbrBxPp1jMUdvJEWeOLDPPJ9goJX4ZA2fYI7FOJ9gcIz4ZI3/BFUExvl0jiVIvbFPLMHGp+Ac4wW92d/BLIJxPlG8wGL7BBOIxvnUIl5wN0tfLheLYqmLbHN5cziejZbL6XigFgk356LXn3lgZhmu985qBpLV5I+1SxeDfHD+rwpnIsK0zniXb9siJWlnKRoaxxEOtc6Jg0OdKskcyPb54SwrsaoQIvz2POdPXCxXKF7mL3D9p5ftk/mt++Lv8M3N2/zJL28+/Zr/ef9hePmpePHHyz8+FQ9f3uT/v/rjw9vr9/zz8u+//sj1AQAd85Myq5INmBvRKj1MJ5O14CPNv+/odnWogndlyj8/bnA1CIbFsZ6z+bJkppKopzcNLBB00YkDLQNLJGRmJzAN4+niV/bJeZtdsOX/vv3u/37/+dum+sfK+gEGLt2ezvoB+YLK9qpUP9AOVzB81deYYpbDCiUB1ClbW6fsPSQsRzkXVdmanAClxmBXtiZ9oafgdreyNeb7TDhl3IpsDb4j4LIJxTuOFrrVriOqQhwkCinmiaEN43bPg6ENKrhpgtEXMzLmYYQRBapgaPQuxUSIeRiZHEaqYKiBMRT1QOZhVFQxkti7DkYxrGYeRkVZmFGxN3YYI3HLbh5GRVkYicBrYIxBX0njMJJIwN4knMRl1pg0kfOJRAIW8wn67sb5dI59SvsiEpC4Lcb5dI5FT73hE/SfTPNJUvREHdhPcSXyxVm+yX5Id5aipHIki+VFLAJxEOOWA0YliU/W8ClG59mcZXlbX+TZzBd7P5rnE5W3WcynAF1kSFI+YpVmTuxhLJl1qFcz58tCI6SZa6uZE3CNJOJSrRF5SVkMSX22Up8QG1wKjQhJfWrzFujOOrgbI6lPc7oAHYwKjQhJfWqlPuhgVGhESOpTK/VBB6NCI0JSn9qcGToYFV1WSepTm6pCByNJMyxOfUKXWSId0xrQCc5RmtEbPkHf3TifSJphMZ+g22KaTyEMpBGfrOET9J+M8wlG+kjqc4orEfd4N+MMg/2Q7iyzEsJgIFkOa1KfEYiDGLcc5yjN6A2fYnyeDUkzLOaTn6Djk+E2LbtdWvgzDQPCyjfxbiybeWF65oPVdV3ZY0T3erW767rCQ0WNXVfCUE7VjruuBOJsgLip64onyndjHV1XQurIbbGdDdBFTEOYv1yR5t3szTC/+xgWNi1/ZDLNf/hwusjhWZMkHS2z9dqrudaAem3+hGOxN56vqpzJLerjZLSY7HLmPn1MF6NZZ/B6ATp4YV4TBbx7XJEQwguEecbhhfFpmwSsYpQllkRZ9ApYedsu6g5oojug3qG2UgJwp67RT408I36qFwuN5ZKgwU8N/KT2Hd34qXwdSQl+zAVH7H2QSComtAo2IoW0WL0JOSMluO+I56ZxuGDWgJTgjTCKG3vzMCokxKQEry2PQgejQkJMSvA6CbHYX8M8jAoJMSnB62AU46vmYVRIiEkJXpfIFrfs5mGEEmJSgjfmj8WxDOZhJOWuxco40WVmDoN80hoZjc5RudsbPom+u3k+cft4VnzqjXJXdFsQ8Ilat1nMJ9F/QsAnGOkjJfgprkRiSn8zNdKUEjwm5a7FCqAoRmc5zlG52xs+xfg8G1IoWsynwEHHJ+oRYDOf0EWGEugpE5+s2Xl5fAAiHj6doz/eG/vkhej41MIfv5ulL5eLRbHUhRKzvDkcz0bL5XQ8UGt6m3Wa6888UGgJ13tnNQPJavLH2qknodgx359X4HRdXkjGD7JWgJbv20IlmcMs5sg8XzjUWiMKDnUqCWUic6TDWVaCVWFE+O15zp+4WK5gvMxf4PpPL9sn81v3xd/Pwzc3b/Nnv1ze5P+/+uPD2+v3/Mj5N10ffP1SwLz8/MuqvAKWRTRAD9PJZK2TTvNvNrpdHaqgWCl2zY8bXA2CYXGs52y+LEmoM8rjiZNFmRMDK+DKEqLsBGbgz//+Pr++80fx4tfwIfntz//8NwdRqp9tBz6TgX+TpU/5k4NrNsh3/knI1i++yc1H/ufT9fePxdZY1OpaQQ8tEgg3FqpANgOAdqkis3GnoMq7ix/DX99df/v3XzfX07vv07d/jkYSqthTBOKCUvZQtp5dFYFI11OWmiHpejtYw31qpbpSmEhRVWho1c4VoeiGoLe5aRQVEloStNegGO1TdawVRQZt526ZXTq7nf/Q0NhhU1Anra47ceGb0Vo3KQqSlgxq/+KISrejqKIQW1PpQ51PtU8jTa0nvEJrTZUPNSgmoAe+aRQVUmsqfKhDcZ/Wb1pRhEprqntoChA5+zRc04oi1DdT2UMTiu4+DVy1oujC2I0SQEoeDXElIyUbK40TDeV8OkdZcW/4BP1243xiZ8in3sjUoddinE/nKJboDZ+g/2ScTzAeTGUPp7gSgQb4yX5Id5eUO0dZcW9k6g6Ighi3HDAkSXyyhk8uPs+GZOr28smNQAN843yihhgW8ynGFxmihhj27rwkQhzTfGIUubbYPkUMHZ9gpNEeFStoE7BVrBpTsTJZHQnJWNuWmwm4MkmLer3SOYUgSm19z6oDc4gNLoXyiQSrte3ksJ11PsSLBKtAsLpahvwx7osNutewsjLs0ihiXfsXR6hYV29tP1cMULkMZJ9q+kLtqpBqtlXTQWxWhymkeiSbrYExdNHBqNDqkW62DsYYG4wejHiQcLa58yY6GBUjbEg5W9dmjKGDEcaLlABS/HGIK54t2eHtqS/pLP7onaMyrTd8gr67cT5RW02L+QTdFuN8Iv2bxXyC/pNxPsHANClnT3El4l7wBmlnP6Q7SwR656hM643SMQJxEOOW4xyVab3hU4zPsyFlmsV8ChxsfPJhnJb4ZA+f0EWGfKqptnjn5XmiJMg4nyhybbF9KpQuyPhENbBdIO35YrVzhzWw+d3FvOjTvHnuX8V5/24+SYtX/D8=</diagram><diagram id="3A1HNBjYupDa6VtS3OM_" name="Mirror-3dc">7V1tc9vGzv01mueTM+Ty/aMTxW2aNsnUeZr0funIEuOolS1XkhPn/vpLSqQkLnZJSiYXWAn3zjTWGy3jHGJ3gQNg4L26e/ppMXr4+tt8ks4Gwpk8DbzhQIjIjbP/5k/82DxxEYb+5pnbxXSyec7dPXE9/W9aPOkUzz5OJ+my8sbVfD5bTR+qT47n9/fpeFV5brRYzL9X3/ZlPqv+1ofRbQqeuB6PZvDZT9PJ6uvm2VhEu+d/Tqe3X8vf7IbJ5pW7Ufnm4i9Zfh1N5t/3nvJeD7xXi/l8tfnp7ulVOsuNV9rlr8+/TONvl5PffqQfXj6Nf3vp/PzjYnOxq0M+sv0TFun96uhLC//r1w+T70+rv1z/t4fvSSrG8+IjzrfR7LGwV/G3rn6UBlzMH+8naX4RZ+C9/P51ukqvH0bj/NXvGWWy576u7mbZIzf7cWuj/L3L1WL+z9bq+ctf5vergiJu/o7it6eLVfokwdXwt7pbADLmpvO7dLX4kX2uuEp2nWBzoYK2IvSLK3/fkWCL7Nc9AiTFc6OCd7fbi+9sm/1QmPcAU7unaWo3iciZWpyoqSOPnKldPwS2TSeZBy4eprOb+ffXuydeVg0/X6y+zm/n96PZr/P5Q2HPv9PV6kdh0NHjal4FI32arj4XH89//jP/+YUIiofDp73Xhj/KB/fZH7v+1IugfPjn/mu7j60flZ+TkB8tKkCn95P9h1VmCAUz1tbIHuTcmGZrVP6W6Wz2aj6bL9a28yajNP4y3l5s75VwHKc3X7b0yo18BLkyoOaPi3FaA2mxMmZ/7G26qnlfrCbrIp2NVtNv1S+n4t36o5eLxejH3hse5tP71XLvyh/yJ/bugbjcWJT3QOCGEos319xxevvlnkPzyATNFfxoZL4VxLeX5QkKy5NIZrnnmmB5TJXlEdO8V5q7Dg7PY5nnvhFvnhDi+W7v8ufeS8zzfniu2WL3y3Ph+BLPQ8cxwPPAocRzprk5motn0vx5h8LApcS7iIlnjngeLvFgpAeNeO4e6xo3sEyg7d4MZYEWIKxgZCMaeFT5akdAzWKeByg89wDPjQQWAp95fqY8D3F4DgILJsLEhVH2Ek/D6w+L+dMPwP4dwXM6NuSfBj0llIQjpIRS4MCEkqtKKIV9JZRiYMKBCEd3uUXub5b5P38Mp8t/4AEju+Me8h/HP2bTzLSLZrPebDD49Wb7xGj8z+0amfePq+wqafG8dPd9CfL/K+++9f+K+3fv+c3/ekTSjXyvimQYQCRDBZBBX0AmAMg1cHCDft7AxUCpgA2cC0Uha+TgVvXMkYvJIQc1Jmvk4ObrvJFL6CEn1MgFjFx1wxKQQ85TIwdlMeeNnCtvNfGR89XIQaXHeSMn6CEXqJGD6oUzRy4hh1yoRg7m488bOS+khlwET9y364qB2Zth9vA+zC2YPTOZZn/4cLpIx6vp/D57JR0tV5uA2BrPu6f1x17cPNzdvyiefPmQLqbZ18xRHmbfZfqwTD/snqoG02T0vqThWBkKm0TJjaNQ7L5YhxS3jFiuRveT0WKSv/HH3c08Z89tep8uRrPeNqCxL8ObQHg9Bbxeb/DCcznDe/QuFZwv0OGFh3eG92jnDDZE2PC65ZUZ3y5iqZEcS8XHl1ff7vBNQMgVH1/2zx0uvwTxhSFYxvfoYy0IAuLjCwO1jO/RoUKC+MJwLuN79P4ZhKXw8YVBXxL4thD6kMNXkdLGxxeGhhnfo/dXBP0zDCAzvkfvr+j5Z59ofMNGfBVaMXx8icY3rMQXCpPw8SWaXrASXyhfwseXaPzKRnwVggt8fInGr6zEF0qh8PElGr+yEl8o3jCI783y6vaXt2/ejd99//z36vNPf/37NFS2+QtnOXC5sqWCe/jv47x84WK5LkG6zN7g+g9Puxezn27zf69X6UP24uC1GMT+IAndzZuv0/tJ9s/vo/vbNPv3zfLN/SR9en8/y630x09p9htfbeyc/273Iiy/TfbHbb7Q5vKAkxlS+fN7op/RbHqb83CcIbZmU1n7dVm8cDedTDYlZWn214xu1pfKOVKUEGXXDV4OgmF+rcfVfFkUXQ36lB/LzSD8UrWzzw9V1Y3oiyBQOA5sb2PHPNgHMnGgqXvrmKc0tThNU8M+kPimhtLs0zA16AOJb2qFDN6y0scohDbsrfRRaUNFEMRQK81Ka55tDXRDQfRgvxy6qKPGq4gO0njiqzaKsbjx1iLejXE7qYhW3wLFGWm/JLrmXoHU7LnXGpA1xUG3PahqrcIFvc8M0iYu9E99KcDV7gmunva4J1zv1IJgvXungLZ3ApL4ODThnTSVRJDq5+2NQEof3xsZ6ZzUkzeK2B2FtN0R0AjHkQl3pCmP49YLDRlOfHdkpD98Z+5o77i2bYpJaQwCBQ+l6HDVj4d6XuAF9pzinh9tkub4LoNSs/3jvAimEyEYAIrt8BiwxRr3mmkjc8X3GKTGFhzlMSJCLoPCJiOxw2Vomvlxk6MGZQ++yzDSeLcr/+DuOwcOkwwvyvwj0TAJLMWMYwNhktIq3Lmr3h0BISm+O4KbTHvcEXsjl7Y3Ao1ZjGS4y7+Du9E1eCO5rAzfG6HNrj3AG3Ude9U4t0b9Dr14ixBG3VGLIQNRJFF189WKz+3Yerhnk+8dM55NqD0bd2tsEPyjezZBtF7Hyn4VoJ1fotLrm6znENyOpDN4oS4SHV7u9talcwbbToPwvv3n8d/ruy+j769ePX0cffvP7d1/3yqrsTTrqRLBvVW0jNH/OrpJZx/my2lBhJv5ajW/y94wy194uV1aK2Dm/xvAiqlVvpmFhVUNvClAfjW/v8/oWG6Rt8ivvi7m36f3t/vI3z3OVtOH9c5aUVnRotZPsSst6wUNrvXYdIKbNKaTNXSCSUZsOrUommI6EaWTomoAm06q8g+JTjbW4IHKUk+YrMFTmhpq2U/C1KCylICpoU73NEwtV5YSMLUHTYsnetmP2vLo2Oa4rRJRhbBW+b5EzdW+lf8lu8tbwCncT1ehVjXLCQ0Cr2YnXCZ5nyTf6LcNCLvUtOO53GfKO4Ezf367TzDqXW2fy+05JptT6BdjLuN/7oHcEwZTg+oFRyNX0Yd5zhM5ueSZAHKaUbNcHVpfHUoAOU2bAS7Sqy/SI4CcprySi6Xq8xgEkOOalaNqVvCR0wlqWd5fL+8ngJxmh8JS6HopNAHkNDsUlnrWyz8IIMdawA63MWADip0/F0Qnk9kIL5B6EoCX6GAyG+EFUk8C8KrCppoFldVUQ1pqKsVij82n8sLMJxv5BIMk6HxqMfiB+USUT4rkkkE+xcl9NP34n7dvrj+t/v7z/ehT/P66i0Ezos2gGVEZNPPx9bcPo9X4q5yRJDlWRpYQVOjWn+spj6nbjLKAVHEDBVe6mDmj5Mq5zJzx/AiaujcNpdLU4jRNDZXB+KY+l5kzBExtvarH8xQ+uC9Vj9ox2NABg0fO6OBTNBzV3ynGldQg2uZ73TatqLMJC9WeeZbwY+ic+kpsKIHU5O/1B9TzBA7o1NCBc+EpkHVqbXRq+MhplBesU2vQqaEjp8gj0dvIGR8jQWCL5gpDe7Tn3fgwQsAyxzYRfPQbXycKZ5ljg8wRHznYGoFljm1kjvjIacRyLHNskDmiI6dQU9HbJh3dovXk412qqYF0Al5Qf+Q7BgJeuqGBLN1tUPPgeyNW89irvoB8ChQpdaPqC0XhDPPJGj6BsyU+n2BUgPlkC59gRgefT9xb8kA+6cT2ih1XKdjvzz/JQW98PrVoLsl8osoncFoyySd1yTjTyVo6QbWJSTq5y+Xqu3hz9ebxsxj/fPf3j3/+M27T591GKaAna4lFYrJJqNLUp6ol3g4UomPqFmcCK00deORMDbfLtAWusdQ6VUQm29YpTWjV+Ntz1rcq0Ss2iPvhfv2NYjzcH4Rxle5dD2WrswnrWw917xJYJme0KYG0WXp/TqNwleAp2uwS8kyhLy3EcWjAM2la5ekP0efpiUJwfML2RDZoR3WeKDp7V5SQdkWRzPY4MuCKNLUjXIEguSJ5gDG6Kyp/vyW+aO+Mtp2VYc4XNQvgCbinsqakd//0PN5x5Uu7gCQ9jwH3l0Qm57R2Ipg+ROZcPE7Vfd9u4sAPHBMOw7PCYWjqLlArZtqC10fsNyDnGAjNNjrSMUSUPAOBrYSpePDzeKepDuGKrKrHkAftEfAYVoVkpeoQDoS4pIOyniOvkHFsIBKiG2DCVYaSqIWeN7IqLCt5I3ZGpMOynivT3UjuWjeThwtnJWeUUHNGwoa4bNfh1WPrcOkJaYSpIKzO3UiNOnfCxvIim29WfG5H1sMdm3zrGHFsupFVXINbdWxuSM6xwZwgD+A49kgPFHGJwYbkanhhEJDhPfruBdsSdHhVakiNx+WSnSGtClXFaoDOJxgtYT7ZwieFxhedT6puxBKfbKyWATVgyjmIhguTWszastHWoAiMgK3bzKGy0tZyFRgFW8NSDrw89n4spgiEHKxw2QVMUMqaymVIv5QNeo7GeMWhpDkaE6jZ2rtmV6rML1s/9BpD8eBJHI3nLvPcJM99pIys3NHEDM9h1xK7yno9RzH6x2xZr8dVkJ2ckLztVgYr/uppulfqz93niZxcNUYAOY2yhIts6otsCCAHoxJc7NCi2IEAchoBBavOqSDkw8gIq39bqH8JIKfJ4bNSsl4pSQA5oUaOZWX1sjICyGmqqFg3U58pJYAc3HKwsOLoY3koH8sVjdOMJi59uF9heI++e4HfRYeXdQ726hwUqwE6n1rk3plPRPmkiApj86nctpyavgDqZlRDiMzqC4JzaZ5MwdbwpHoatga6GQK2tr19suf50IZm86wB51m7WVH9BEJp9LwecJ71uDwrPnKcZz0uz4qPHOdZj8uzoiMXwvMH51lpIaTJ1nGetSHPio8cPANxnrVNnhUfOU22jvOsDXlWfOSsmoNzaL+Nk5+DE7ZtfBggdRMChC9OUb0K6UNNo0UWDzSki/DdEQxGaDHjdNGQVroI8mkb/MVKF4UtyqyZT0T5BIOl6HyK4PGf+WTLJGC4G8HnU31j4d3+GqcfefkK6UEF48fFt22Kjl6ta9S2w17kY2zRRRBL47FDp7JFh5+IRVD7iX429RGM67HrtcX1QiErvuuF9GEh67HwxqAIFVsoFhFt39hiRaIHL6x3RIeX2zf2ePfiO2eiRSSncffiw9uiysBGtSNU8UaKjihm1Y7RiXYahCpeArY+1U6DQMWLb+vSqdmr4g0VbtisijcWCr6yivfwwHSkELWbndSuUWDoYyTniRxQ8eIjB7u+sYq3jYoXHTnXgacU7Oi9W1HVvHAaGlRmD/aOPnJUP652rSyuVhPZXz+SL2gy3I+vyInbKnI2a6/xcH8UJ9Jt5DeE+4OGT/QT7o81Gh6WyDcc7tHdYqwpKGKJPBmENIVDLJFvkMjjI6cpHGKJfINEHh85TYNGlsg3SOTRkUs0BUWsJm5Qf+IjpwpvaTBjyciQlvoT8ilWtLYzmrZKYJCN+WQLn2DQFp9PMPTHfLJF0gbVxPh84uoZe/kEJZLofHIdmGO9HK8yk+hY9Xg327yhTwLtsVTJjXrIr64cRwe5sXR6IGmufUVDJxXUvWWCXQeechjqPqCOytGfeFDzscjibSwI+BNYJvhcZPG+A6h/CRDqgI3sl1n6dLlY5LbOk8nFj8PxbLRcTsdVctXI6VSp5s3vPDJXDA2+Z86gJgx2WAIY5mt9KfadbSwloDZJ7OJzO6wUqV/piCMiGfNNmhtcqqsssOuolMLhbFWgVaFE+O/jvHzhYrnG8TJ7g+s/PO1ezH66zf/9Y/jm+m326qfL6+y/L39///b1u/LK2VfdXHzzVkC97A5cVYkFfIvsgu6mk8lG/pFm32x0s75UzrEiYZ9dN3g5CIb5tR5X82XBQpM7FT+MJN7EcKfiqgK4oj9HcKLiZS92pDsrcaDTNSuodR1k9TKdnL0XhPTgceFh/DTuBU9esCgYGx6HSavLPVnYKiJFiZ5ZdbnrQoISbi62fcDNxUr8inNdo5axuF1Mixk9L6iGTLdaq16liVu7cNnEoY5egitxoY8yK+F2rep/KLkoXA/VgmH9e6i2amssD+VLhXMiDo14KI16Wh9GOU+P5INjGL5Hqm8JRdsjReyS3Ii4SwpkzseREZfE0yrauaSInkuCImXKLmnv6IbQjk5RqXdsYKtPL5UY81LP5J5GN49aB4YZmaTnHsqgUr/u4Zj+la09BqbDIOgdyiuS9w7llbnWsIgC+/S8g5EgcK/eIWL3UIXUXFz4mdzTdEVBLXRFdA+RnCQi4B6sCsBKA2g43OEK6hHYWF4S49hEuENoIrBcqS0JYQi6JKsisJJLYo8kqAdgE5nyZrLWQhOA5Q4EkkeK6XkkGwKwXcdRjx36R1BII8xFW3U+J6zO7Nh1z21Qch/u3eTbx5B308SJuUtH1bslATnv5sH4MYlm7zb28oeSuC3gaNU8HtTBMb5H379gd4KPrwCuV+t0ufxvSKueVLEg4BOK60ktJhRU+eITqkWjHStLZ+QyMk8odnOGS2c8GPk0amxC+Ra5jIwEPC1qq628F+QyMhLGhuF+vOT3fiSnCKMcrI3ZhVtQiqLwx7m6XutYzibxhKDvlcazOkUv9X4DMJ6RGGU7qrtMdcNUj1GoHkc4VIexRruKgz1HMb7LcHFwmZbnIsrnHa+87ZYGLYDra1SY+lP7eUInV5tRgA7G7rgqp0VVDgXoNOJGroygA5FmthvL0+lApNHpsUSYDkSagVIspayXUlKAjjVnR2nOKECnGTXFgpr6/CkF6CBGLLg4+rwdyOdtRWM1s+nMkjmMbxf3L3C9+PjCeIrW6bL+YUhM/wAXBHxCwSgPE8oaQsGILzqhIhgf5BXoWHzlHnEU8IVBRBL4tsiKEsQXhP2x8RUO378dLvgg7oKPL9+/3eELgzP4+CJrLPuztewrfcUQYbMiPtFmGs25ClopwKMaEXMK9wIQtBIwtmIIhWXSJ08xatGs9Ekohkuw9OmYg7CfQCyNhtpFOeKCpU/10IEzEAHoNPoZlj41SJ8IQKfR1bD0iQ5EGl0NS5/oQKTR1bD0iQ5EGv0MS58apE8EoNPoZ1j61BBdIwAd9xI6LtOND10Z++FMt42ZbkCowIOEMhtpV7SQZkJZQygQMSJAKKKpOStTr0A6QQBfeMIlga+VqVcQNiSALzw1X45XmS11q8Lj3Wzzhj4XgL1VRunb61321ZXjqO7wgclsV1nPuN1MKhI1Kqj7y9MoOiYz1F1s86RmtBSghsdyEl7bylUZCKIIeG2iJTFWrsogZIOPr1c/RmbXXshAm/TyFdJT58aPi29bcQa93kNbqjT2HioOVJCtPfce8iJpvxIUwa8r3Sd8P6j9RD/diraW5AhG2wiGbmmtWZ772xbL5YcEfC084Tb72oq/OWfHS9HXuq19LU6ftySp95yKTwgcX8uVmxb7WlAKTMDXnmq3XiD0jxQtCQ2Lmz1kJTkhLQwQ+lOAByoqTuNeAEJ/Csa2vsdpqHDehoX+isaYLPQ/Jm0bKSpfzApLwvpO7mYPPW5lKt8Lp6HBdfZgbzvX67A+50VcbaJdfLma89X6kfz96A3xE2Wr1OZTkq+h93OH+PmyPCaMJb53NcTPdxLpBgwbDlxeVP+Jng5cuv61+mPXeXpUkEjH96i6/rVcf9NQf0MAOq6/qa+/IQAR19/U198QgIjrb+rrbwhAxPU3x9Xf4EMX1s8jMntgi599YNs7YbmdnLB6TmtROLAVDGg+sG1YaD6tVaaLWp+ykgDnlMW1bMcJo/DdYHllrmWrhw6UHhGAjpPJFpceAULFir7wZpPJ3AbYZkKBpAgBQsEgAxdIdAG1XAsTtRSi9JfKDLjsyUwtDD7UIdwy8jJhjYgNCIbxl4mQ9x0WEwqoIgkQipsyWLyRBVESAoSCCU0mlD0eSs4+ECBUiwbtTCiqHgqoZQgQqkUhABOKqocCfWrwCRUdEF3+MkufLheL3NZ5ZrD4cTiejZbL6bhKrhpRvSpvuPmdR2YBocH3zBkozFk+90wlppCVmCIQElCb9CRQYsKUXiDthkTkS5fSiDo7y+5FquNZOFu3PXgY3VcoEf77OF8VMF4s1zheZm9w/Yen3YvZT7f5v8PLj5evXr/7+Pr37C2fLq+z//76/vpjefHs226uv3k3YF92E66q3ALuRfZCd9PJZFPan2ZfbnSzvlROsyIXm103eDkIhvm1HlfzZUFEkwEfx5GpE8fAFbixgrwyxY7xBReO8H3v9q9f/hxPVz//9/cvn65+XLSI99hYV+OH8l2aONDr9lZXozR1C6d7HhVmvu+SA6dFmMrK+8CNyJkaBnBIV5f5fiiv0ooZYL1VlylNWN/xJp3dzL8baHazFa819FwYVCRrokmydkrqNCV6RXxgX5umv1FMK9N84VSTCCIOutWZ1dmEqyUPde8SWNt4mAHpkhJIqGC3xzWdvWeKaHsm+bgcd6yArbMJVxnWeyIBDlzYnqi+CIC2J4pwXVELgvXtihLSrsiT2R5HBlyRZjYJV81WXZHcUwXfFZW/3xJftHdGQ+iSt7ZG9qAM8JJ0T+Uc19790/N4B2OO51ys7fv0PAPcR/bgGRS3VHfOglBHTQqOwbPCMWgacJxpiwB/O72MjmMwEuPt1TFE7BkqiJqK+z6Pd5rmIWfamcKXu2ES8AxWhVjdfR/AgY3sfbSDrKG8EpaS7V4jGzzsuKWshZ43sirMKnkjdka0w6yRTHcjuWjdDGhueVJ1RpFHzRkJG+KsffXflXxbo86GnjcSpoKqGncTu9WCbLFtxt5xk10/lm8dI45NaMLB3BCo6tjkARUEHBsME5OYYWjjjEqocNv26zFQkaOGl0sG7S0ZVIjw0Pl0oqODQFmHJxSe2XDlAE8O0tV1UEDnRAcHgcIOCraG+0i8lNX+caw4Cx2czN6dmVD0wEbHQ6rPWa0PZELN1t5leFIHLScOJBr3ER9y6gcC9RhmqLCaSd0zqWnI3D3XhLZUMb7UrrI8z1FMzjNblucJxWaDq5gOPkB5230LVrzF0+ij9Mfy80ROrvoggJxmPhGL5OtF8gSQ0yiPWKxMBiHN1CJWjZJBSKOjYfUeGYQ081RY6VSvdCKAHMtCjpKF4CNXdlXjvHc9cnLemwBy8KjMee+jD2uOfMxWNDIymqcsRzgzvF3cvcDvosPLrbVPSNZAgE/woMnu4ujVQE4z4MML92MM79G7cHB+woa33CRSg7dFSpIgvOQW+zYjK61U2gDNma8YD2pWadNmmuO5as4IoNNiRJGVdwLQnBGwNTxiWCZb8HxoQ7OyBcV8R5YtHHNA2uaDsMJlATerPE62gI+cJi3EsoUG2QI+cpq0EMsWqCCkGGrLsgVaCGkKU1m2QAYheOJk2UIb2QI+chopK8sWGiJq+MhpBJEsW2iQLeAj16ImlBOf1iQ+Aw/yyWhsPYI72MvxKrOIjlSPd7PNG/rkzx5JldSoR/zqynF0M1qNBTQTT/IcimCcCuneYnERdBQkkmSnkQPFv5FhPIcEvKeRA0WH13Xh3my/MnhX+rtfHjx+XHzb5iV6aklWvmJTN3cCtcJR257Nrqsha9/FwkJawcIi/KYbOu3HflD7iZ6maHGHI3tn2EPlMLqnjU61r0sYSKaO0HPsEcxwGLU1ocCs75JDJ4Znw9O4E8AxnICt7WuSIdkwRFebxDCTwGqTNrdDHEq3A3aUM9bEp/U7u/NEToAlHR05TSE/q01ktQk55DQKr/NVm5BDSKPkOl+1CTmENIqt81WbUEMogWcaVpso1SbUkNueJUhEwN3KVI4XTkPHzOzBXnRPjozH1TaaxdVqouPrR/IFew6ZB2k88VUkjcWNZ2Zwe9sBqJvjg/GIeeRH0j0TNkTMPSep/UQ/EfOEdVsts4D0fCCMXbAPPCsf6Lf1gQLFB8au7AO9Bh8ovPpP9OQDNc08WQEpKyCp+cCE870WKyABn2LsfG/SYqIN84kqn0DuAp9PMI7Kitouch2OK61E2IraBMZjGekukI48aki30KnwGkFWY+ZSWyNcB4ahmVDWECqWA0MECNWiRxYTiuouFkQaCRBKMKHs9VCRnL4jQKgWvcuYUFQ9FFCaESAU94C22EOJmB6huAl0l/hWSyjxR5e7Dkd6rV6B6BGqRakYE8oWQuE3unYdmH4msQJZWcQPZw6h4+vy0KEO8SXYpAH6fxL4nsb9SwBfPnKe0oJPgFBQc9us5zznRi4VLSu9ri6u27aty6Ydl3F9pkjkHi1Ogz4ziuo/0Y8+c2tIdrV2RvfouVoY/WG1ThdQh5J/QO906IoDtmlfZulT5r9yG+erUfHjcDwbLZfT8aBt/xLVWvWsxQYafM+cgcKc5XOHLSDA3ws/rOrxsxVAushmFSw+V7d0eNJUHxFGEuabZRJcqrNVRKhWkXC2Pms9jO4rlAj/fZyvChgvlmscL7M3uP7D0+7F7Kfb/N/h5cfLV6/ffXz9e/aWT5fX2X9/fX/9cSBerZHKgBKOuNj+suzbb37f5tOAjdlNuapyDawssv+4m04mm+1gmn3Z0c36Ujntij1Adt3g5SAY5tfKdoDLgpgGXYOXCP+FV9XyiW1J8H6rHV/BZ9GbezjVKVpBIEVEROJCV2y2r5GrGKN1GsYWCUFjt9g0W2lsByR/CBgbbihJt+wKtvPgSiNGETSi2ZZdbgAbL+wHQNLZzfy7gdjHNtzREPsYVGpURVON6smXo7qlLKi5je3mdjEd8AgcSWciYhMl9q5i6hr3omvl6CW4EgF9lNma+3IMm5Uuij1USRayHsqVZgmIODLhoXSz6/ShkvP0SC44VKF7pBCuGvZ4pAjXJbWgWP8uSRB3SULmfBwbcUlC7ZK4eagUenDouSSIEWWXtHd0Q0hir62RPShDukS9VNtWQ8/3Us/knqaXj9HOtb3d7B7Bmx0WSvRwsyvuku7uf0LDiEjc66Et97qmv7jRHsj93etywpTAvW4kQNvrvR7xzV6F1FzM9pnc0zQ8N9pOu7eb3ZfTMQRudqtCne7+bc2Bhex9EfVYZyAvcHFiIrCgmHLMXd9VAhJ6LimyKtYpuSTOvpRTp8l6pFCmvJn8cGkX7sFe75FCl55HsiHU2XXEUuPgGhUwFF2SubimzudIRThiO3+yQRd9sHeL5NvHkHfj7urtvBtokoLv3bgJTnf4AvFZgt6CIOImOPaWMCvkcfiEgv3OAaGs1JzL1RSeUDhnw5rzUx1OD6opSBi7RddvK40tV1NQMHa5EyKRZ9o/ZhVnnIOTyruzEErsh0AvhHIee/NBK/bUhO1d5iaVLjvFwazX09F2Q288gOAyr83yOtE44p55LZ/6PdeIory0i7UFcZ4Tw3XQcEFcrCpO5sKhg09GnlB0HDEbSok1UTD9gfs8oZMrLChABwupWYneQolOATqNQtCoGtgC6GRdMQXoYCjJvLjTBugietBRkOpZAJ0s+qMAHQxzsTiqhTiKAHRlOI5VJPXQySoSCtC5aug4RV6fIqcAHTzAcYr86HO7PAdIKJoSmc1oJtzl+4RS5BQIdapZW5Ai9xNoa8OJxDazmq00NkiR4xtbtJljbKWxQYqcgrHhltGy/IqnGNllNr8iHEhQeuptbjiox68IwTcmX4vbxXjy1QFu2jeQfBXK8bqcNzx8uxg4yOdL4WiST/pTyHlCB/KGBKDTJJ84b9iQNyQAnSb5xHnDhrwhAeg0ySfOGzbkDQlAp0k+cd6wIW+ID12ZU+G8YT10IG9IADpN8onzhg15Q3zoFPWZhOMXh5aLn378ohwRTDZ+AcTjZuIXpV04H97gksBkRXSX5LaYp8X5UmvypUHL+Y295UuFYsw2j+rsAuqoOozP8xX5GKOjOoXbIjXOvoPomF8o3iHgO041/R94kq2jslQMLyMtYAjgNIwtEoLGhof20zA22IFQMDY8jlDXWkhGDH1oRMNaC8E56eNuiLLL/faGSCCWZs94ipHqnJNW5qRlX4YPHdeytsxJ04OOa1lb5qTJQRfBwOV+nH4XiN8P1o8fF9+2O4kOI/duJXT/wmno/JI92DvNyt1g4mo7mOJqNUH99SP5gqcf6RdFIrQ50i9w2h95Zb319q4pprhd6T7huvWf6Ck3ILguvKW+g5wbFFwX3lLfQQ86rgtvqe8gB50Hoxes71DqO+hBpzmtcR5czoPTg45ba1ucBweEij1IKLO5LK9Fa20mFFVCgSguAULB3TgLK7qAOnalxUiRfDErrPBbpGbZd5AVViTkfIffIv3MhKJKqEg+6RAgFDykMqGs2d2AozMBQnHfJosJBTLnBAgFo9Tc6O1ofMu9LpnRVcLn87XVDoMeoVjNfkKEwu8cKALeI58SofDLI4L64dVqWVJPNcTlK4fNn9qWFJspIa5IsugNo9oWUDWrjDbOzLjKyHWkorOyqF6rMvLdoPYTPamMAj4+Wh3goudr+fjYHb6hFB4gsNsP+Pho8eYsdOgRCuqz2GF0tSBQwLdFbTI7DLIOIyZHqJDDAzYTCkwlQo83hTA8wCvQ8SuQIIcvn3FtdhgxPULxlrU7fH0wkhUd3/ILML594Isfo4p4B2nzgiDXvRMgFO8wbCYU6C6KTyiuqrI4LePLHsqkqu/q3Zd/5u6fi79vF794by9+EaPl9cWJtqzzHCma4DuK8FRvXdSUpj7RhnUiisiZusU2zkpTyxNGCZgaNrUj3RlQxEJa0xMBTdhbZ0A1WyE7CQ8x2D44wyEGSviKLf6+5Eh/p5gWHOWbWMllON3Kh+pswq0uD/XvMliKkFdfbRyUQPLwxVbAyckrfOB49GI74GJqwGn6THGTS+nUQQ443QQ41L588ThVJwZu4iCfd9HnTjsgh5Bm0Bu336silwhyyMETPXffU4W96CEHAwTcfE+FXEIOOU2Te+69V0VOLuUmgBw8tWkx4yTRkFbWEfLJdSGfjCaJXE462ssnGFTC5xOXCtmbxPZceaeCzyduNGEvn2CGAp9PMAbHjUG7QDqoFr/7TnnnGmgMqk69ttC/sOegurMBMWB0z9FmKiXziepKFMtZPHw+tRgmb6OSyYsdOWiCrmRSDHU0ams6qRsvACEtfHRaxLSsvBM8ELLHtzWM95AW9Xmgdhhf1OfDBAeL+uwR9anGFdbcK6ZlfZ4XyEoxYUDWpxg9yLq+Vj6emq5PN85Of1w5T+TkIlV85MqzBiv7GpADZSnoyGnkR6jSPsxTBj2EhBqhM5XweaFPDiGNnAhVwoeIkNzXjQBCGtkQS/WkCBg95DQ1AyzVk5AjJz33oSaPcLBh27Ocgw3FjUc82JAAwpuoIVTMe2X9qcIdJeTqLBSDVbWYcRZ0SCurDvmEngX1uFOwxXwCwU90PimmAbO+qwukfVnfpUjDGdV3Kcb0MtJdrBFhQA1pGLrjNcIW5RVMZ+KvES2UV8wnqnwCJ1Z8PnEnUIv3sCDkh88nrvG02T+RUxr7XONpsX8CMhV8PnGNp8X+yZcbjaLzKYAxG8AnG/XffgjC9+j67wC5VTEdfYXv0+sj3WYKupV3gkuvkXRgWXtj35cqdwhUQkRQm0VPnNCxxuDowgp64oSgbX/joJMGx0B94IM5QVtKlxfZfLXiczu2Hqpz8IUjy/Q9AzoH5aR1LqposVxQK6qIoE6YnqfTuaazV2EFgVFHd7h3AiosEyVfAXcAb+WNBDkVVsAtwFshBwuw0ZHTKB/PtFDI9+khBBWO51wo5PvkutcGmgLXMy0U8mFLD2yEyl/GhUL1yIX0kIMRSC4UUiAXgXFt6MhxT+9WyMH2VOjIsd7D3vw85BN6PjVkvYfFfAKxT3w+wTMrK+27QDqhVlMRwrMvI91FBJFcTUXIdXf2arhgNhN9jYjgiZ/5ZA2fYnKawIi7bVu8hwXREXw+wbgW88ka/xSRm5sVcQ2hxf4JZPTx+cQ1hBb7J6DfwedTixodG5XkQeDLR+lSmoemJI9a9ESy0taeQ8/WLeIWVtraASEidFvH8ExPukIi8IKqDb3EgzY0WyGRWNW+8dCShpMXDsfFGxuFw5t7xbRwOHAdOYVgoqwhVgVnuKyh2cdHMlox9E9GpQcxDIuwBFyBnAt2nejIabqlswS8ipxwySGnEWqhSsAJIufRQ05TdoEqDaeIXEwOOU3ZBapknCByfkAOOU05BkvJpbgYPeQ0ZRosJa8iFwpqyCUw8MNSchVyCTnkWDZhb1oS8skVkE9G00gJjA8wn6zhE4g34fMJRi1YYNwF0jFob9MO6f4SHyxosFfQAPMK+J6Di+Qs5lMkn3Hw+cRFchbvbMCZGZ9P3BTZYj6B/B4+n3jQnsXrnS/nrtD55DowqKcl1JdZ+nS5WOS2zlU8xY/D8Wy0XE7Hg7YCOpXGZ/M7j1TsQIPvmTOoCbQ9t1FpLAnGRSQ3Kt1IiUCjUijZccKmS2l6nnal3nEdVVAnnK1y/B5G9xVKhP8+zlcFjBfLNY6X2Rtc/+Fp92L2023+75ur7CVxkV/78l32YWeYfUPn0+X1u//7mP3w6/vrj+Uvyr755ndtPgmYmN2QqyrPgKuRPdLddDLZaOjS7IuObtaXyilXqKiy6wYvB8Ewv9bjar4sSGnwmJ7RKKxiH5Z1bvsiT1dBZJkjx/iFPx6S65+u756+pp9/+/9Pf82uP8WfL1ReYQNRbpVWXBAqLlyv0ofsxcFrMYj9QRKKzZuvM3eS/fPx9bcPo9X4qywb26PH5tdTpEfF4/XFFRFL4mtPVTPuqpxeb1xRyv6YK+hc8YKEHFf0awxzBZMrvh941LiiqqVkruBzJcjz58a4kj1czHP8dpvd/FT723yS5u/4Hw==</diagram></mxfile>
\ No newline at end of file +<mxfile host="drawio.yandex-team.ru" modified="2020-11-13T11:22:48.503Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 YaBrowser/20.9.0.1697 Yowser/2.5 Safari/537.36" etag="Vvx8akMHjgNF9RnE2p6e" version="12.7.0" type="device" pages="4"><diagram id="oeXZIZbNdJ13JvgRRL20" name="General">7VxZl6I4FP41PlYdIGw+lmVNL9NLzak+M/MaICgtEDrErX/9JCxKCKJUg1r21It6SUKS794vd8mpEXiMNu8ITOafsYfCkaZ4mxGYjjRNVW2TfXDJNpfYwMoFMxJ4RaO94CX4iQqhUkiXgYdSoSHFOKRBIgpdHMfIpYIMEoLXYjMfh+JbEzhDkuDFhaEs/Sfw6LxYhWbt5e9RMJuXb1bNcf4kgmXjYiXpHHp4XRGBpxF4JBjT/Fu0eUQh37xyX/J+fxx4upsYQTE9pcO3v80vyXLzl/KsaR5+eHi3/T69M+xicnRbrhh5bAOKn5jQOZ7hGIZPe+mE4GXsIT6swn7t23zCOGFClQm/I0q3BZpwSTETzWkUFk/9IAwfcYhJ9kbgQWT7LpOnlOAFqjwxXRs5/u5JiQDbu0k+bz7Zg/uh7naZqSfCEaJky5oQFEIarMR+sNCT2a7druszDtiImlLqtFIAWmh0iW85QoqXxEVFpyoetXFscRizNgyFZIaoNAz7UlnMXpSB3QX4Qk1XMFwWO/GCYo9v6Qpl0/QJjrjhpgnBm62kJhRtqIgqDINZzL67rD9iAE5WiNCAmdJD8SAKPC9XIJQGP6GTDcVVKOGrzNZtTEbGlI/FdCbN1Ufdgc3HQ5uucBcdxuJ2AyP/ud4btWoUTeYVg66jW9UPAZOuAJj6KyzPXZJVZnjqb2+GujoWzdB4nRnqdm2c0+zwgRC4rTQrVPjk6RpKp1mJzdmX/P29UoJpSJTwGTN8NMUJsZODSt88DxhAP0YE47PygC3t+kgzQ76nPs5UYr/Z5o8lLh/c5dvywBpoerLZP2TfZvxTLYdh08pHyuUy7YQh86Y4Hut5QNFLAl3+ZM0cuhqwaZK7WH6w4bzzCgLpAUDrgOFXAAQNAILBAJRPUkyCWcCY+MZMxzLEnS+3tHqGKue0HUu7GdvxfWS6jbbjWWNHUYaxHUtG8Ky2YwEJwARSd468uukoJSJOicU6YA7PklaQcvYwKV7g+zdncFaD02qd1eD02zE420XNBufYhm4MZHCqdmmLk528ThZ30+Zm1sG6dJCommAsm8Blw8QTTqqrChPtxhOve5gojmPWws3ewkTxNWOj06zE5sOEiaqpy25XU6B4G6RgAOt+XP075kCp6nkZQjdaGQKFDl73mr2VrZuf2EVzVWmICw1ke3oTY9iaA0zzkvxQZlZqiZbO/LDzzcqoFAzCD/Xp2tp9O0Mc6zAYR1gSR3xd8IXG6ZoZOa/ijG4nuVxP5jc5Dk1u3pC00F7ZuQJaONkDvwgtiOa8y2t05oWaZlhD8UJtvuA4MRzpMRgzyPmyJ0IwuV1yqB0NdoPPoJ+XHAy1lRwuElX42oGownRMw7xoVNHVGsc1xEF7sUdVQFt70RCPvswyX8dU0izqAw1ckWZaKWcDCfqxRCmfJXu1FGBkNLHygnTx5klCVXQg6sAZc33p9s9VtJquPk7H6vrlm/3z4/buTqaIWXbFJ/zAdmAaswAYTJjEC9gKpwFBLg0w30YEU77z6RwmvF+0ybrdO0kU3xfCSYJIwGbJYZgWGb3nvWiyuyrT5DV0TT+o9xlDLWkYxHxCKYWxBwlPCabbyMFcP2YoRoTXa34ZR6CL5z3QNAnHoRJ+jTDKsbqEaytUFQsqDeYTdFD4zAygQNzBlOKINQj5gwl0F7PseBDpnf21GZ1knkd0pAD0cXfvS0TZ5UnNIJ5VUY6W/N5Y9jZBw06979B2Z6IH1anfUFHUy2qOzMZXpTmUOxp9qw2dE7x+02qz8+svpTa/XiJStaYS0aGierc6BaTw1FLTYRciRD7t1YGoR8qCnvXhXYCakpgNMYh5Tu9CLkNdQkta6sf/K8qhQ0i1z6ko5kCK0kvVsxclGSAgGVpNQD3fqV+aT+R081U5K1fp5h4KmFqCrgEiJM2+sJ/bdAHzilTn+vzcK9GbBs45q97IeezyLEgTGPd+Mpkw4ooQO2mSNy6Pnfxt5bGzO71e+/pWv6iP9dWXUjtFxeX8VqdoAxOe19kqry70rtPLeBHjdZxtT7N2/4bw1xM+DanCwZwoMne/wkXyIwkxvvvrw78T58v7hpTvpatCx0vEb7cqNO5YFRLbd6wKqaCvspA0Un91oUallBPYqFYufvOln+ZbfFUeGOoUaNxxcG00cMIFsrdLA6reflFDNkC95Z7G8deNawt6PRHURxqYCOTEMl7cEguMxQLw7qZ5hQZ6KgCzn/v/O5DDs//vDeDpPw==</diagram><diagram id="uKXmrOIFqc6csMLveKSW" name="Block-4-2">7X1dd9tGsu2v0WO40N346H7Ml3NyJjPxiXNzZ+blLFmibZ3Ioo9EJ8799RcUCYroahAQhereIEpZK5YokpKwqwvV1XvvujDffvzyw/3lpw9/X10vby90dv3lwnx3obVStqz/2Tzy1/YRa6rtA+/vb653T3p64M3N/1vuHsx2j36+uV4+tJ64Xq1u1zef2g9ere7ullfr1mOX9/erP9tPe7e6bf/UT5fvl+SBN1eXt/TR/3tzvf6w+yt09fT4fyxv3n9ofrIq3fY7Hy+bJ+/+kocPl9erPw8eMt9fmG/vV6v19rOPX75d3m4uXnNd/uOv/OrPT86+zar3P9yqr37657/ef7V9s1fPecn+T7hf3q1Pfuufyr9//+G3H355+L58982V/uUf//Xmt6+aP239V3O9ltf15dt9ubpff1i9X91d3n7/9Og396vPd9fLzbtm9VdPz/lptfpUP6jqB/9nuV7/tYuFy8/rVf3Qh/XH291367/i/q9/bl6/KJov/7V7u8cvvvvS+uqv3Vfvbm5vv13dru4ff1Nzfbm0767qxx/W96vflwffKa/s8u27/Xca3GvEvtn+vZs/0guPnmu7e97D6vP91fLIBd2tlvXl/fvl+sjz1FME1Utvufq4rP/S+oX3y9vL9c0f7d/ucrcG3u+ft3vp1/f3l38dPOHT6uZu/XDwzq83D9RP2C1nXe3ecbeYdalrCFpR89xX1J9sf4vmq4M/5+mhx1h8RlyqucXl8svN+p/Nu9efP/7cRVXsvnz6yZsvmh+MEswuSSxnXmRqyx+X+mhcLm/frv6ECEkaYM8P0nBI6mmEZJUiJEs/V1YxcqWZfUxOIyRtipBUxs+SVc8dn7wiThTnsaO4iaSnm+y/Dr4VDqURQ391t979buq0GgIlrJV6YVwfxNXzw6ZIFDbq4hl1moRNIGx0inSYKy+5VeOmtrz46T+v7tV96f75tzd3/3X77T9+69vMxIrR4yHaH2rxdkcwIWrGCFEag9aLQZu132L7i+1e9RScz4715n33sZ7xx3r0DVIw1ns2KZKPA8Gep8jHReEXm+PG6LFL8sfl7efdn/Xdm9f3qy9/kdh9Cs9NfP354Wa9fPPp8vF6/3l/+akdirt3Xd6vl19OwJJe+qbR4V0jZZsWXPbnU0dbNW3qDwfd7DLrxutFZVdFLuGFLi8/bq7I3duHzT+/fXfz8Du98z18uPy0+fTqr9ub+tLe91/Wt1sMfnq7f+Dy6vf3j8j8/Hldv8ty97i39t4Vm/+Ca+/xY7d6Dx7ffjAi6e2sjAoBWQZwLLhwtATHR9xoFp81biqDA86FgaO9oHkDV8EB15yL+sjR/seskat3anDIqTBytAUxa+SMwkNOh5ErBbkWchYPORNGrhLkDpHLi8xDTiXGLQ/jZgW3Q9wKjbfiCoJcvbW7Xe+uTgu88n8/r5pvfPXw2Ab5un6Czj99efpm/dn7zb+qeZv619q+0/ZxEhH1L3nz6WHZHwiXD5+2jLF3N182m/QTui+r+s1v1purbzbtmxre9c/th/huk23kixDyJoC8YUOe9kXGQV4L8kd3kwDQB/s5I0BvBPqj+1EA6GkLaBzoc4H+6IYWAHraRBoH+kKgP7ojTg+9pl2ocaAvBfqjW2oA6Gkba6oF/rt3y/IqCP115d5mHViv4kBf4iFP22BTLfChkfcZlAjQ0z7aVAv8d/ZqGYb+rS02jbCU0PuiBAToaStuqgU+9Kr3lVII0HP18uIX+NCr3uDt7TRXMy9+gQ8NvU8oBoBeZbSb9+Hy7nr17l394MX3+sLmF64pTw8Q21xFD5rbm/d39edX9SV6PJrZXMCbq8vbr3ff+Hhzfb0lci7rwLl8+/hWm+u9I/7V71t8c1F8t3mvz+vVw44qeTEec5IB0rxsQ+oooFUAUM0HKO3RBQDVAmjXKRscoLTzNrkVOqAYi7VC8wapZIAGWF2TW6EJAfVXaHJATdG9YX74dHk3pH5SNlQ/vblZf75c36w2b/H19omvL+83HPns8bH1h+XmzS8/bv754/rm4feHg4pr+7M7Kq6owcQVCqphOzR+M8F6SsUNhpcX0ypYTL9ZLz+1M/7jk98s767rf365vHu/iYIfH368u15++fnudnOdfvthWf/Eb7dXevOz1VeDa/KzjBCVB0MkpIcYI0SCsqec7rSxNSXeNcyrcpEbchHZRCXBi1hRVl8sDfihBHwvF+vRjl0cKsd2krNRxWPF0l7nofux1W/NI19te71GEY+FwzqgHjsS/7HVY1U7iIuRxWNHL4lIn55XsrehKrNNvsn2H/HoquHMQ5ng08k8Z5l4KujEo/IUmYe29USsFygPFXaqSeVYMkaqqc4y11joXKO9eC5i2DjkHcJg0Ze2c43FzjXpTLWG55pJpgwXK2W8rCvRoRIXrbFnsIG9iqMbPL6oYjgoEvaGemdWMTRQgy//5tcUwfrR5b/vFIMuf7q/A17+6uL8NwyNHxPohiHXCZoTzTURi4XjucZh55rj8yCwc83oqWZA/LCnGo2daqxrp5oYvYnmmognyNFUkxdtbMosXzigVEMPyKaTasY/7EXINQY61xQeZaEoYuSaDv8h8bFp8/cMdFlTcEmgID1t2G4oHsZ5H8ZcqocwxlxaJz0njMnhKRjIXKomSJ8aNpD9UyswkLnMiCAdabhAJocaYCBz2Q5Bes9wgUxa12AgcxkMQbrMsIHs9wyxQG4q+TMorqH9BSrsKOByFYpffkNHgc88RQsD2pudaoEObTvgkwLRwoDLbkg8Z9phgL2TK7n6buI/064Qsfd6JVdrTrxo2g1a7N1gSZt3kzO9SNl+bzrcDbrW9pzncjkmhNEdZDqkBd2uEzRsdAc5EGGv3ZSGNd7arZSBQneQHRH22k3pXmOg0a0oWeGQafNEpRmXbhNtXDTl2mz/Xj6mjB1qi7Dt3MZmyvi7QmvVYrcVe3XyS3j4NZWbW2zuaWgXhwOrq+MDq2ECesv3jB7QXrfL5tVCRYjOZlHE4yiON5j82aEaDsyeSeo4gVmlCMzKz5pllKxpaaE9t7icSljaFGHpHxLZ3I/L/pfECmW6q4hFA68OeeDqaDyNGP8jdSAAYtt17IriiFttKlW0unhG4SaBEwocnSIp+t1x6/I4RSRte0/KGtJm+61gOmtIG+SEiuve8yj7zgShjKrFsB3OQrQOmDVyPhEfAroOOxkxhTpKr0eArqGQihPQ8XtfAQhdh7OCuLgcpcJDQKfD0IkpxlGCOwR0HcJfMRk4ajLgTNZsqtJBR1mHotnu12xDrDoupuCchdquCgIblfvnuIS7ek7Akk0hArJcat1ZS7IhkOWS6M5ahw2BLJcud9biawRkVcalxp215BoDWi6JrQitjwmtMbCnnaSp1s/Q2Ps0GAzwudS0Iqo+KqrGAJ9LQytS6qNSagzwufphIqA+KqDGAJ+rZyay6aOyaQzwB8lpsQWXQGLpevOUhw6mogrxVDZIRasF1I4Tq1y1QdWuWBjrnj5S4ztIR4u9aIFU0vXGxy2UwcFX0eba5NZvSp20v36NzqDWr3p5h00FK6436+Wndog8PvnN8u66/ufX7//47fXl+upD/fnqbvPzr28efh9apEUNLq7Q0E3qbCT0ZbAG2x9tx4kHTdc7NnW/8hZYZvXCHl1gcWn8Kp/WLMMLwzv1A2CYmWresVe2swuV2Lod68W0ijFjaH9VRHDyvFu8X6GrvgQUlV+m8ilMRO7KP+eZfjR2+lF+SDdUHeb8o0n+EZlU4LxOgycc3cEjF81UG0cHjmNONRvIN46FeoTxaQr3Isv2D3TcPh6/er28v6kv2Sb8JnxPyaPdU16YHTqkCiLL8yTp4NlB03Na0egFjlkNOo70yFUEeyGBCTqO9PRU1HsBHD2qY42j6mnIR8axwyxCpHw9B6No65FLcACp6/N4KwFmC1sg+HFQ9MVBXFaLoQcqU6UxQ8cBbYugBQKXlgFSNZgwEEhfBS0QaJ9zqvRm6ECgW2i0QOASOcxLk0h22Ggwc8kZZqVPpBtwNJjPx8gjIWHOpz7AoXw+rh4JUSYnzGgw99jNcw3pCB6AnebTvfewP5k5cfX5/o89ve/U2GE88jK71mb/kZdx4WhkHuHR5Io991e1aBTkBaY6/gIm3kWPH3041lvBcWaBP+lYz5NQhmis6+OxnufHX8AU6+Z8rJsGCAy5bt8avtd2Pj5OCas0Dd9JOx9TJ2i5sEHvpOVchyyiG283YdB7bc2p8JTlqCmbbKVXlaHJjXN6djI5fFPKjX180eSolh6JTE5uDDRSXWXKLKocCF96FjI5fIHGbqvM2M08tsSg0pOPyYGadNF6RVWZ5VAeEIFxbpPDN+mi9fHVWB4fgUlz2Jp/fztqnFqURy9oZM1/YGyYSKkHlKceroXuwzUup7drLhVNfbOGcW8qA4ujKFSHTWhAx7FDgyZaQm8eAzaOWjkCGLTSuOI9cEbQE7uhk6131eMLDpxfmAM69IuiQ20f4fhHtmg5oCnQp5ED1GECOEwIZ5UDBpNOXpwDTiKd5H5M51F8agLjv0QxHRoog51xVGDYlyimQ2dGmY9j1tO+iowjbbaIYnpAGxJsPWq2OV6imD7eZSv74iAqh0OzDf3SEgfH23RogaCZAkEU0z19PrRA4BLKimK6p1GIFghcUlpIxXS6QKDdIrRA4BLbQmqqEwYC2cSjBQKXHndWqmt/iAIcylyyrfhbgpSqa/SdX2DY0FQL/pTqPPR93Qgzh1DK+ZTqPPRdmzruHx5XV65ah3yLrPkyfM7HcVy3+QG+1fgBp8C2OAXNb/hc7/LtFYfgBWxJVi89E6RKc+1JWAqt2++x/dV2L3sK4ueeLhrjn3tkxyXtujF263gBz3GkVlxdMkgRLFc2NeitD0WPs6aSTV+S5aJm4vjZdDDDYpxs+myGReblwNw9Mwd6L+DKgVwNYkj9N1cOJHwauBw4SD2Irf5NOiHcK53AhGZ1wE0f36TDpH18sYRm+79zykLChOtX7ck3e4BVscgPALapAaZ7kckBnLJBSwHOMyyAaaEFLRV11i9gM71wRy9oXKmoDsxDE6nogBMrwko1fcBG1jN0aNNo8ps1jpZ02dBw7NAXiVa0nWj9TgEcjh2qDdGKtnH056Oh4WipPCOZM/dgwegJbc/L+1aRury7Pvzyhe7d/T2mJP1Q3QxN6+2H7tJy7H6oysjy6PH0duRcIYan9/5CQqli03U2VZaDp7WmDBL563arnV7+qhu91IBklORwRik/qHOnFiZCdgmMshMFbFBgg551aL9OJLAhICvnA6l6+nKRgexwNBINbE+DFW5FcilcIEWwXDjT7mDVB3Pck2y2YXF6TjDT5iEazFzqFEgFKxfMtLeIBjOXPAVSn8oGM+mtoMEcdHWdJMssJYuB7orAcGabJjMvTaFDh/l8fEZSDuxFL7UDI2OmWoMl7KA79FI7Px+PkJQMtAy91s6nqyrsmlj7dF6SVgOzvbAYJyppjneV8vu8PZo/mhajaP5yrg3nrPQu9AANLtlRi9LJCSKSbjUrb14WGmE+py2FyQGcslyhAIMR5gvaS/j6al1fIB/R5lzu88fb7ROeEPzp8u3y9nWNyPpmtUHy7Wq9Xn2sn3C7+cY3+9O4Fiabj0AUrFdefl5tj/C+Xd3dbfP0Y8Xx4fJ69Wer/Dh481evsiyjhUkoVNiAd57UKa/UIs8HpWs+Ln9BOwoC9svB3ru9NWCrRkKUDOnAHMtOkN/dLr/UReDmEm9onLtPv7u6vXx4uLlqA9Rf6r+oSKeX+OAKFoEr2Dz2QtOO0i+TSc203TwQ0w5acPvN4zz33unZ9h+tSBijOg/MS2zK6odPl3dDqnMVrM5/fFV/67fvfnzzt/rfb375+W/f/+OgXt++d0e9nrRg4EoMlV+/a1v1jCPfl/xxbv+WFvByRxihG6m8O0LemGGluyMMquTpXQKpkkfSNteru0g+2lg7Wr5PDlUkQXO9KLPkqOaN5jY62b75/JA2z9Aufql26NS0wNg6dkNbx2bbgI3eOtZZu3WsldGLXSXI2wwOTKWdlCZfZ7leHNRs6TX5TocKeNHkP5N1q1VlCJRx6dRdk367d+uzRM4n0iIgRzssorvv58YiICdTmU+huyIg1+GRIKphHIg6hN0ivTxOMk4PnQnMNxSx5QCxZQ2dWtCWQWTwOiTPIrDsafgArLvgbm6SdH4gSaXemy6nYkcZtrGAek7Akr0fALBcIuhZyyYRgOWSPc9aKIkALJfQed7SSARkuajmsxZDIgBLqSlTrZ6A5I8IwHLJ1GcteAQAlm085rwljgjIcrUo5q3nQkCW9igmJ/BBUnBB0ALNOQyiQpJtQdACjRqkttTQqCal8OYeqrZEWKtnQMxOSuElqDqAtappLYxNevQzXmloaRKX5mgC49iE5jigl1N4UFZl6iNOTfcPQnMMNGs0HnJCcxyEnMNDTmiOg258JR5yHSO9ZktzNHgQdTBRhebobaQzPOjoLlpojsEeSO5Dl57m2DU3SmiOPZvi9OvO0B3xVA/qU9Ic/c2dzVIfIQSmLU31oD4lzdHf+wEAS3ftUz2oTwmsvzUEAPZ8vIhT0hz9nSMAsFzE5JnRHP1dS3pkc1rhprOVPnSVbr7zPI+Q/YDOGXmEmKaN0e8Rst2j0gDl9QixPvHI2ta0zpCpSH70FTyOIoZtOtm8OL8WL8u5SWa5l2erU/MkYpYr42a5UFbyA7sw7Td5tgdnR8L0hS62enbCrKIkTC75S/wtekouPVzvpaykLJx+WVhhl4XOZy3bMlvsekPPSHTkRVy5jksQNi95CV47snGvn1iuk+LwcHHayRaH4yUoLmHjrGRSDq+tXhy37UVNUFKMHS5Oh12MBQorE2UHySXanZkC0Keypc9agWnSk9OKpVUAku0RgKooMDx6cqimVQD6qCKoigJDhGVcyxhoN9NR9nQ5pRaaoh11ZIsp6DHX5JSBaUe2eDfbojKLxi8o2RouaC9+cqimHdnio+osAKq064it9/QPf+s6sOnPJpR8FsHeiEg+e81PPTTzKohmXMZ30UHWp7lt1uBZf08IAV5Jt4Ui/AyZmEKC1+EGLtrP46YHGODpMHizlX/6WiYMlDomN4kC1LeMRUTP0sXUf1Qz4umMOjyeyRZZz6TF+ovXy/ub+o/fhE/i6YutE6tTN4aMRzjl7i7Xf4RTpDnCUc3Ut/2SMGqx20B2EWoq/7iTvobp5MfSjJbuVHPK62baS8VNZqlYshmOtlSoh4AsldktlbKjIoq+VPLepaIysn0lL+JaK0jMZVkrqdaKAVkrRda7VhTZh5IX8ayVspL7iqyVXUcWYK3kvfR/R1ra5DVMt5Xmcoq7UzhYnhgPHo0wL+3Cpm7LlJTxIPZOQ87AIZpqJZfKcF4OT/72tVBBbOOyREsuVZWeE7bkqBcDWy5Byqx8nshJMAa2XLz9eVk9+QfFENhWlKIxVU1G2qGWkODqsymmks61hMSWy6YtfjGVdLQlJLZc3lTzsh+ALJQrrk3QrJTb9NAJA1xRuI5STfmnJBDgWhG5vrBd7MkhIaRUVkSuI6OKIKWyInLlQdt5Itdc2eb8Op3I1Q1awxp6DSMNNTUqN4ExLXGXsKPNi8mBijTT1KjSpgeV5mVsiavftHUqaOEQWeLqgs0fkbg+c/CNyfIgmnGPvF0HYYGmtlmD5w+3AQGvY86iSFzb4DlI8DomMIrEtX0TLCHB65jBOFuJq4FEqcO/QSSu3sY6A0Qv17SAPCSNL2/frv5kUre2GOIXg7xH92aj/7po8cKjeY8WS3udh+LK6rfmMa62l5OPCp4343f6qeAujWyCeI86F8F7dH9dhNPdk4e8ib0mMy7kYhc3D2UdXiTC6e7p+UDcRTLayZsqDQloaq/JbBDbqIeieUYbelOlIQEN7gXBlotiNitON219QGDLRTGbF6ebdEYgsOXSUM2M00225BDgcglt5sXptojYKi4xRvxiCmjEIgi2dAs71WIKaKQcCLZcm6BZcbr9aVwg2HIVyjOjdJPTKQhwB81RwCb/Is0tQiAO5oqe6k8OVKSxRQjEwVzRbY8QuscA259a5JoHUrG5c8Uloipmdcf1D1ox7riallPQBGDlTDsb5mW+aPZyyQjAuT5uqAxGzNh/sSVm6H5mxgSZF3pXdvQyL3ZrIDrzosrbkVzk1WJn28PLvdBBybOw1ftST6U8wFww9USmhB333MXOPKMnngFhw594HHjisX4cR0s8HeRTLYmmnWgsYKJp6q5JJprqLDONUeiZxg/kIlKmaa6MyICOZxpbIGYaej/AyzST3BMZEy1hvDAEOiaxiRLM64ZoxPVLUcJbvwcDC6rWxIK9auWcmiHNaR/+wu8QXydVESIufIe48CfVBfXkaeNvESAWPngXVGd+IMdqRpgOsbkoYVuZRmclYqaZVNfz2ULYKfYiwLueWhXtk/V4vYiOrqdIXduJRmkPIKcXZepEk0+q6+klmvEPdhEyTQ7e9XyiazSBbCJlmryj6ymy7Ham0RlgSZNrAt5UhUQpKbD+IX1pgtjGZVnlou7k6HZiYMul7oxPj0yKrUPElms4xKyUu6SFgIHtrIS7GzHFz/UPuFlvMDCPm5P2l3x35BwRfS5Suxb02wwDxHqsqeLnIf5Nib5FRJ9L+g0pD06IvisQ0efaZ0PKmdKhX18mRPS5rAEg5eMp0UfczxXUQW1yUuSUGznl2X2UVRk6ookrRS5oc21yg2lSgqozRFDPwAkioWkAWalVpgBAHeQEgb1SU4Lqr1QMUEN9NA/A5rjx3e3yy9f39xsLho27++7T765uLx8ebq7aEPefTW9/zIknzfQqH1zBInAFm8eed3hMTodLrx9ilIfM9gB896oncKi4pvSZc37dtD0gJ+/UfZ7dCoQxDqSL7i7bw6fLuyHVtgpW2z++qr/13de/fv36619+fVN//tPPb349qMC3795RgSdNJ1zJQWW+FVtRLXJ38EHyhAqFOV+isC9vuYaD4c16+al9A3l88ps6w9T//Pr9H7+9vlxffag/X21C7o/rm4ffh+7WosYKW2zYZhB2c+PYTPotj8WGzqPGRjNcsoNm9cSjGpdr9aQBaUlAnohSYdrU84t3ejN70c1rCE3KDWV+F1lH9DCLQ53XG3baLPLKD8rO29+zXs5EuHJUT3rmYbunJ14ckBN32olOfiJSrGdpYt2r+qy1Cxc7WKPL8k6M00Dl/+zIDcepnkqc5ttpetHjtKr8pGrjJ1Wq9YtFua4OOdfqaKyMGNwjtQYR4rbYzhZ6Qdy+sI5MpUBUF8+4H0rshGMnyQS8J6vAJueZIva9uSiTSeZbccsVtmMPdpxSWOfbAIp+K7d5tmh3hJwuFmUV4w5O+/zgpqIeQdXmob1kQn/RwHBlsWw8gXvqqj5g44p+usYxdx+hzBNH21BHMHEsusbZit/YcT4oHI4dCkrxnTrO7ITDUYdxFBuh4xxNOBw7fODEpOU42xIOR0q8FA+MEG3LMylxlQr0IhLi2GHPJg4DHlPLgK9HLtHyzMwGvO2lylQfzlEJ70XGJWCemfFAgY4zl5h5ViYEpBzGw5lLtjwzQwIHjjPb5GkxJ/Du4BV6JHCJlbVEQrs1iV7LKdrTEtMClmpPoUfCrOabp4wEix4JtK821bofOifoDH0HqLgsBsXYwFfWokfCGUink5ocFIUHcFEsGgJOKpltoQZpp7FRTSqIJ6jaDABV2s+ZnCI+qXeFCazVwmX7j8Tq+ELTRs7kAE5qeUAArpctFMD0bjstrqtSKlvY4tgVjUt2LQLLAXjmwv6LwTMXxmbDpx8zVejBIo/tekkhbPPY8ErZOHT4/cUR8vYLT9d13pep4rIoND11Ffb2AA9gPCA7aPhC3z5O34YDskynHT5p9Kyqb0Ct8sG5nvLh8avXy/ub+optwu98a4qmQxahpnhZ0Bm6xxPRwBAWFVr2MB3qD1EN9NCk4IDUYSBFNtDDg4IDskP/IbqB47oBpY1bGCQgOwQgIhw4LhzAW5Fc58wzVw5o24dz3FNkw6UQmblyAA5nLoXIzJUDcDhzKURmrhyAw5lLIQKpHOC7P/u6ADScc9p0El1AFF0AXCRwKUREF9CjC4CLBNr2mmrNnjL3E9Y/HM5c+g9h/few/uEigUv/Iaz/HtY/XCQMmoKHzQ+HYv0bbXo66HGJpvkZyDqgBAAmt1gAn8F0PCgtgNF64congE1qgGkLbnIAQ2kBTF5BARwYLP/11bq+QD6izRnn54+32yc8IfjT5dvl7esakfXNaoPk29V6vfpYP+F2841v9iebLUw2H4EoWK+8Gmy1PQ79dnV3t63FHslhHy6vH8cwZiFYX73aXNkLn0MWdbRd5aduZftSd6j44lMsBKaKC/AvB940Nc8e99KEBp5GBpvuuQTsl4OdN1pILLDptkrAHmFlG29l51W2UAcieirMjYw73W0J7iMscm+C9SPuJRLudBMmuI9Qwvnz6FVuk6d2uh0TqEeA2pXo1TqlSAjwI+zPM1NRjS0W9M07C/QjN1f9+zoc8NKa4WHHOHTgpZDnAd4v6NCAr+iKn5gxi6nMosqfrqgmVzSyMUsV5J2IkcWzxR5F1odsXFFP1aHO0gLkcSMLOCCP+x88GRwwmSColgvCIusZKFx/4VsZHJgj2PaU4d27JfdGGHAEevX5/o/9/WSLCKNRQrXbz/YbJVRJJmzXlYHzN4Z5ZRe7HuurTs8mp/pfxuTZVB0fJB9eSC3YR1xVdsxVlXZ092D+AMR6sWnWizM08Ive9VKX4HSZkZdxrReqyJf1Mr/10lE/JVgv1vTeX1xF1wt5Gdd6ocYHsl7mt140zHop++8vWU7XC3kZ13oRh77THPrQdqiWtuXELG2IcQQckDoMpJil9ThDwAEpM9ZPs36AA1KGrJ9mllZkeqGRgJQp66eZpcGtSBmzznN+lvfhHFd8a2XMOk/Ni4azjFnnKYnRcOYy0Zq5WRoazs3vdwb3ZyizNDicuSyyNCLOSGZpcJFAu1ZilhbFLA0uEmY1RJ0PZ3LggIazjEhPZJYGFwkyIj2RWRpcJJyBlxaUWVphy54OelwnHifT0scFuFQKC+Az8NKCMksrbNHys8xTA0xbcJMDGMosrfTGfCcGuMxEih3HLC0v+lJ3VGFmmdFmnAA/xk7c992AA5723gT4MTbe1HcDDnoxSIzjuwEHvJglxvHdgANejBPj+G7AAU97a+C+G5X2b59lUUI5b5RZkGAmzhv9myGCbWX7sI3KES0z2skS741g4oOHUnUoKUSk5hcv+FB2aClEpuY3HfCh7FBTiFDNbyPgQ0lrS5GqBdsC+FDSilbEaiEoK6LVr2yBJFcrVUcFK3I170guh1+VzfRwCFuSQ1eS5jsdliQd9iPqIrL9yOAjfT77kbKZo9JrP7Lbr9CAHWw/8sK8waWymZk6kvY4rO5LK1H5dmWjCpC0Mt200jQ84dNK84ueAbE/qRiX7uzh0gptd0tamVpaUdHSyuNLn2uWprOcsCtsvQErq8P4DXis0XY1fRmPWdr+mp6B5CWpfJk2xOAyID0lkgw4tQyowTOgUjQDutMyIHkZVwbUTBlwXgYOgT4yWgbUXDLemVk4WHykuYS8GhHplCYO+A0lTc90xcaBpQsQuImjxQKXyd7MjBwoKQcOaS6bPbFy8Ha++L0/zWXFJ2YO3h4AvwuiB2mFsdX+Sbd5Jd3glw5J718aeqgzOYiTGjoEIHY5FsS0az05xX9SS4fAQUVpoTT/hrblJgdxUlOHAMTOYEEsSl+eBh1N4JXpS+BxlX9GtL5MAkV86EXtG8dTEQ542nwT4Meo1+mZKxz0tBsn0I+R7jN86GnzTaAfAXpLnXzQoM9pfT85eweX91nhRbZ3yINH22Lv8Hzpg86qPmzjKqryDvUxbXLMG0pq74AHZYf6WOwd+uwd8KDsUB+LvUOfCAQPyg71sdg79LHZ8aCk57hi7zCIlgsHZfPOYu/QAyWxd9BZ6RYGCUp6Liv2DoPsHfBW5fmQ5bEU91r1uuvHpUQVXATpuYug8ZA+H1IsltgTD2kuv5S5i9rgkC5p9TzV+zSWqA0PaS6pvkZEGkrUBhcLzSRVEfOfJuYfnGoYxfwNebVfzF/m4YCN5JJkxTtnTsFWvDDYHl/6XOcI5QifYbN57nOO0IrQnAMvY3KOsFrWxdQdVZqD2gHroqNq4F4X9LBblf2OKpZwwAMv41oX9DxN1sXZrgv70uJkvHXRf78I0Obi3S+ae/A8nCjYtmch5MG2ZyXX4cncfSbwkOZyGhKfiT6fCbxY4DpIE5+JPp8JvFigBNDJmRBg+UxoVZgesktc+XJJT1MnBzGWz8TmGAYLYkoYnZwJAZbPRL2K9cKVTxAXqSEeZAiEDTGWz0S9iisoiCu67xYJ4hgluac415nqS99xBYiVGIxEUpzjQS8GI5EU53jQi8FIJMU5HvRc/r7xO+wpqcwBVSZaf6XiIrjOispc38jzFs4ms9miogV7ZHbb+dj3DwA3JdMR/yjN0i4buomItyvSJl8493RFNbmikS1EbNAPWyxE+rVcHrKm7EM2qmav0tRe4pB4s7x9u/rzkHPDRLPZM2t6aDYXBySbx7tObJ5NsbTXeSjGrH5rHmNsez0ZeTZ2V8cM4NlU4ehk5tnU9wL/FmGyOISZqhnj8ywi2YhBbVtRvciK44Fdf/F6eX9T/9mbzOhzylTRjvesL94fv/Lf8OWLoEW0m/SKcGkYmZbq8E0xgHnmXP/LuBiZHW4Z9Fhj3nd4q7Hv8LuQF1+p/g4GOpA6DKS4SvmqemwgK013SXg191mWzs5EK51fGCJIegg1Zk1rQWraSUdnmjJWZ4bWo/mAUc05rX7Jy5jKWEfP88W/L2hQgn7TpBsPvJvmPs+pdpKbZ6PK5dHutqflM2VIo0pHmiFfaWo7OeU7/MsDePQaYdIrYpw7fOBeXBBGhtH7kN+/0/YX3L34aS08f4VltGIwp1UM5GVcFQM9SRab2KDRFXbFULoOb3wxifVFFc4H0ixyJCA7nPHFItbn1aOvSC4zybkZxHo451kfznEJOI5revrc7GHRce5mVD58urw7Hec9o/Lt/RPEe+S3733GyNNuDBjyVVP7nYFuPa1ZLDrOXAaic7OKRcdZM+GsZ4WzBa/MqoyrAp+bnxQ6zudTgad1k0LH+Xw0TQnXs87Ad1qVOp96OyXOCn1fpWi9PTmDoLQeUN6xUZ5XPb3sqMYilaKF9uQATusA5QNcaSyAKeFocuZAaf2fAitYPeGb2uCrUvR8eHIAp3V/CqxgKIDFDISnyM4LQupQfck7qhlIpehJs0A/xl2buj/1Q6/3r4kEPu2NCfhjNFEy9FVPm2UC/AjAW4MOPFf3bG7eT+BdFc3VPdPzwhn91ENPzPhHV5bURWjWP5UW65+TsC2oVzkYH7PSjiYqXE2VmP/UkJndM3sVJLtkGF1TtTfAj27909xXxPpnPtY/z1gPRqdZD42RzjOMf/S+FxHb9md/PcX25/jdvaTOxmh3d6PDUIrxj1eEU39yOCg7XAzE+seDssKHUpxd4jq7DOYbcVYpQ0fjpqpSbOEXHP2+Lroq/NImkqvL/npCubok5LVZOlEJLu9R2TVwr0H8WzaQDbVVTdVrcLnXa4jm3mKo9nzKd/FR3VvaZUDPwpl02I9zsw7cWKuIFi1OlydYtOgqsAGOZNGyR0gsWo6XBS5AuwErC3J6NCsmLSEoLV2lWDYtVd7ROhSblhaUJgtYcKGtSto6nKrsOyWXPXAEjEaZyGlncapC0ZRIBxqPcEhTWvtUJYQpkQ7UVHBIc420nJVphy4cPtJcdg56VkiXE7hPcxk6zMq4I9S1gEOai5Q+K+sObempPhrSBZe51rxMHRw9/IJDmu6nJ6f6T1p0WzJ0GM3YoRik+8eGOGW+DkEMZu1QnIHyP+EqNlkZXMVI2v+CNkkmB3HCVRyEGMzeoRCZN08Cf77aN7bCvxChN8+uWpNVjyb1LuiuWqAfo3VWwEPfvPMZtM6SHnHhN1RKroaKnhXSAW0THNK0odKZyd/dLr98fX+/yaMbXuPu0++ubi8fHm6u2rD0Ex23P+ZE+iG95gdXsAhcweaxlzIKjfIA3ZBQmrlKzftsqZGEUUjfrXDeu+mSvtuz+YmtiBiDJlh2cxuGjidRwXTw46v6W7999+Obv9X/fvPLz3/7/h9Dx5Mk3SiyZYxcewGhVLHRA/lJQuVRt3olbdiA+4FUdPv8RGlPaAJSBskjYgLSyyGg5XGpg4DGJfiVdCcsMu9QKUR3thD4ueNqqrj6k0P5SfOdPi+XtkpkL7eKJq8aWl4z6kyasZv9OpOqDEcps7wq0NapiJkLeZnJMjK8l76MSRlS0jaPeB6E8prDzGsV7dWI0UGoUKTnLBj40X35vIXZBhSnjtnYoqDzDrUyUPw6DBBENufhV5FapNT5QiXHjx5Ci1YuxCfIQdcfFx17ZgK5QHukDMIb93ihOh8OdtIjw0CVigCv5ToRnpkULlAcQcDLdQw8M/0bZeRiwEv3mFM95U8qegO991ouRfrMlG6BkwsIeOmZ7FRLq5Srtwo0cCHgPR+XgaSatkDfEAJe2neanMopaclMleZVZkP9qLiiCEv7UZPDNal6LYCrMQC40j7V5PRMSSVrVFu84UE37b90uNIG1eRwTapTC+Ba33KT4+po60JkKmOUy1S8UFTB/BxXm+LOZ7ObVLEAuhtyXJvd+K2qpPCC9jIcra7Q+eXeBM+qQmCXu+Bxm7DL+49PPThdlv5s3GYdHD5ajc4bvdIfp4uBng6jJwxav+SARE8GhQ1kJkCiR8vFmbNnIVES7uxAeggieprCNOXJPfjz9wBG+ths6Py9XfkTXWrlqH+8rQYM4ckJvZm+jEdqtb+mQkHvSYONJu0pDaYnoNtMCOgDDwoQb2KZ0M95+id5+q6izYR8zrPFQwBXCfWcZ28BAa4Qz0dJyxYSXNoYnepZXlraOSS453MOn5Z0DgmuUM5HAZeceECAK4TzUQoq0mqHAFfo5i/sPXndQWfTk82tErL5qKiaTKWnmlslVPMXdhrJWk1PNLdKiOajolqv1fQ0c6tpF0po5mOUyJl/GlekJ5lbzdWWmhvJHLFE1ufTlkpLMYcEt7tzMdTBPgxu0bzN2/snXIc52J8F3PRQHgJurl5GMa9ehkMEN7Ch7SeevZxr1uG4/cQGC3PDnr9WKTFs+/cSWlc/XWuwM7atzMtChpfWZcqiWPijMWy+MD0O2oNexkTrCuzlzjxM93TMi0MuZnWcixkjtjV0bBeWBmnuFjZKkFI1ymGQLm/frv6EiM8AW/bZERuOT50+PjPo+MzzQBJ1kZIo1dvMLT7Th+cLq0nu9KmC6bOvNAi9LFpUUx0Sc1TvhRLVy2fMnLAUuGbIjBrmeaQwf+Hu57jshi921MUzqrq5xU4BnSIrR3OdKyNVmMcHbMWK1+Ph2h928fZVMcK1jBmuNB7rG6AfjyZTTTyePvi088flNPxVpPCnOqAU4d+zyZlbuq6g07UzgdI0VrxSmgK0E48pAxcr27f40nnx2MAENvHiGQBoTjdmxgQBjSsVbLbB4sbTgx9tZ2Lgp8P4iR9PG7+Cnplg4CeOPMNuiBoUP0RPHkT86GYdAz9x6xmEXxXY3ELgJzYjg/CzhnQLjNH75kQ6/MRoZBB+G18RyPU3K6uRVf3mN+vN9TebNtBGxfBz+yHGArYkAVAFAyAub41tFqaWAPAqYMwAYJuWCSkfSBkADjQAuPQjkIZECQOgKkEDgPagpspLxw4Aa0ADgMseBdK1KmEAuAw0ALgsVCCdrQKIryIFQIC1ihEAXNKk+LsA7AAIEDwxAoD24aa6CxigTUsZAIGDSIgAoI28qe4CsDNAQP2FEQBcnUBIdWrCAKgCR6EQAcDVCYS0WksZAIGzVIQAcLQTODnjrpQWT7lSPq65VqEzurh2QIGps5MzeUqKqyW7dgxcad9ucus1pXmXNaDrlbbjJrdeU+LqUPPwy7tsKlhgvVkvP7WX/OOT3yzvrut/frm8e7+s//3x4ce76+WXn+9uNxfrtx+W9U/8dnu5Nz9bfTW4TosaaGxhUuSEUqPKcBkW4uizxYkLTHlEVzuQK5lXZmHIhYwrdnAmnaK5NfntYpjK6eJQ47QTRx2VOY08ym0Ab2p7PflGublmRmavXGq3SqLLoHKaNYoijgzKBQaIinhn0MaMYFZmdX7Knj4UyVVReXjORLcUGTFVzTNTFeCZqqBRX5SxMlUHLZ9uWmaemSx8ZtKptOJjZKYqemoCmIfrBs/DTZaaaNgXVazU1KE4EcWlT1jHT01uAqnpPDNMFS3DvLCX0qFPEn2ur06AX+3NwSf0aj/wrKlapjV7G8dZFSJ2KmmCkl9EBh7UsOCnieNG7GBpQl3IfsVlQz2yUu1XKhr28VoplJYlDgdBdRV+aqLtr+mkpllmJpWBZyZbmmTHUYryBcW7I8gzIhiVG2NRpMwUfeLCiJkp/kk5RGpS4KnJUVZIvNTUYYsotjS+IBW+aFKaQDlVbWpKEnPAQLHM+5COqj9wikuGrmeFdMBqEQ5pLr05pOcMH9JUSASHNJewHNJchg3pgH0jHNJcCnJIFxk2pC11jodDmksqDmkXw1h74yN9Pu6Q2KYAAVY5XCycj1EkdiwEeLtosaBp53eq9Tu2VUCAKAkXC+djH4mdF8oAIRMtFrh6deIh43vI4MfC+ZhKgsdCgJOFFgu03zc5p4q0zjIZgdhlPSfIUc0NnKaNvsmZVqQ1maHdADSIaYdvcqs4rd8M/iqmrb3JreK01jPwEJvjo9qfuDzj8n2iTaqmZJ7t3/v82b0u3wV6P8Umjzq7+tlUnLIgYbkhDub2MJxOfBkTg8dQFuGZh+meEndxOCO7Oj4jO0ZsF9CxXVgapNouXJQgja6NH28k+rMjNhyfPTPcY8SngY7PPA8kURspiUYXSMPFZ/rwzKHDs6AV6yZ99pUGoZfFiuo8nRC4OmSrq6OhNeJSGKlrwRvmVaQwf9nuJ0+lDlUXz6jq5hY7FjpFVo7muqqIU2E2F3FCXqDkYlV2v2dMaAaaB8+jxDzxhLMHp4KAxtWF5PRMSSzmggcLoPh1WASKD5cv3AHFr8NHTZyVfDkOKH4dzlhieeOLbEDx67AsEl8QXzoDil+Hr4u4J3in5tpR/FSz+0qHX/POIjE/jp/LMNdfwcVbnpuunAzbcYVeJKccFrRxMlW5SloxOSi8XOziuSnIQeHlMgiYm2wcFF4uV4C5acVB4eWyApibQBwUXi79v6jCfVU43ToVwa1T5ADgsgWIX1tjB0CAZYMRAFxeAKL/9vXfmAFQ0ubXVOtz7AwQoOBjBABX90yU3r7SGzQAuPprIu/25d2gATBoSj22GjStppvs7lSWZYumtkomECzPQKyfVOaryIHlI7B5alxf3nRTwYz9Zr38dNGSBz8++c3y7rr+59fv//jt9eX66kP9+epu8/Ovbx5+H5rkowYVXxlfkZCo8mAO13ncmBgk6dfQaz1pEqfnJ49rXbmDj9TrXjT9L8vnLpzPM4Uj6q/oPRtdJlBsRDDti1rpRTM4PJ1OoFmvkYXne4mZbQ1E2VyQo0qh+ovXy/ub+o/frEhfPaTa8qHdux1RED1+5b/hy+enXH2+/2Mfc6eubcZhKk0o9muVqioc2Nwq4rIkSUiRkdkBnSY1Dg+8jkm8VAUrTpHc9G8NaXLUWTA5RuXMqf2N7lnJsbX4JVM+S6AJkRzLwcnRJkmOhTktOZbUaz9icuzQz9BCfObJ0MImw+ND8yacDI3yRuxlykk6bFWAg9KhS5IOy+y0dFhR4/mI6bBDjiZyUJ9vD5sO6Z3rTNKh1IY9Fd+QZGizNMmwOjEZVilrww5tp2irfXUKbDKkMmpJhmefDN3gZJjIySivTkqGlo6nipgMO4TSYlTga7lgk+EpltOSDKedDG02OBnqJMnQqtOSoVMJK0NLidfi+tExMBcxGe7u/GL70a9tzAMAZosiOYA6DKD4fvjqRdQVyGUdMDfjj5Lim9cLNDUx3XJ5B+hZ4VtqVHy5zAPmZv2Bii+Xe8DcvD9Q8eWyD5ib+QcqvlzuAHNz/0DFl0v8L/Yfvv1HYAeVB3dQcSPAcan/41fY2BFQhIjAEBHAJf8XAxDfAAQ1AmgbbKpVOnYOKEOEL4gI4OqjiQWIbwGCGgFcnTbxAPE9QFAjgPbiJucVkdYEJLDJs26hChz9uBvkEYCNcVL9uA6cZFq7cOUTxHRPHxnjM/AISOsDEVrHdlEhYUz7cpPDOOU6dll4HQcaclGBVVlGVy+6+wPlk5pCL5qZfsncH+prGexuima9v5KhkOY2CGlk9qnqYFzRTDdzBC0sgrTTINrAICUHFsGOYZ8iaPJJN7AIyrjIgbQaWAQ7dDRCHfeJM6gINqWwcMePI2iNCSCYLVx6BDtqUSGP07025hps3hpCy3YoZWu+06Fje9KstSRre5+XDr3a6NK0wV06PmlavQx3e79ebVr91ESWVhVxfVeF7Ren5QU1K6Wv4xGnPV3XM6CGpZVWUIVhobOeE4OoJ391IkSyApREeGIiNMMTYSIzKzoDbVAiLOjY8JiJkIsbo2eVCDf+LviJkDaSJRFOLhHmgxOhSWNkVdFZgMMSYcC+PmIi5KKIzU2sN4VEiOSHKonwxEQ41Am/ToSJTKzoTMxBibAskyZCLtXy3FStU0iE9FhTEuHkEuFQ1/s6EeokidDS2bCDEmFlkiZCLnn/3OTfU0iESC6okghPTIRD/e7rRGiSJEJHZyQPSoQ2S5oIuXww5uaTMIFEqLk8McQzwfdMoMSBQgeJA7FDgMs2I/6RAHYIFAEmOkYIGMrfmmozFFsvWwSo7CAhwGWcIbYJvm0CbAhophAQ3wTfNwE2BLjoAWKc4BsnwIYAPRidnKo+rXOCIWrczVH3gW9CasV1DfIZ2GOktk4gy9eZRZEhgTzIH0NDg5zWOyG8kpt1kRDZM3DFSO2YEFq+SidHlvbjvr5a11fGh7KR4Xz+eLt9whN0P12+Xd6+rqFY36w2EL5drderj/UTbjff+GYvvmmBsfkIwL9eeWXXaqvY+XZ1d7ctvx7PDz5cXq/+bL7w8Xz1KssyeswQihG+tWxoJ74uuBYNH7an3mL0dTC0/SaIj9FqyRQo4jnttgniYyBeZeR+DYI4ba4J4mP00gKEEVVki4HnapyI016aID5G30SF1jgE4rR1JoiPgbiFRZx2ygRxplo9N4NNZTkRp20zQZypVgdBnPbQBHGmWh0EcdpbE8SZavVNpk9/G5eWW7RSHQNw6bhFq9QhAC/oTRzdhnhD8G9dyDI3i5I2smK7EDtq6XYonFjevl39eaiZYJJJ7JURPTKJiwORxOOmIrZOolja6zy0YK1+ax4tzLbXk1MnUQzXSWxXSnSdRB7wUql0LL1DEdQ7iLN2P9OFZKmqCmapyFZ4jroWTidJzTRHWfQcVQRsTqosWo6imxTxig96xYPmpKLD5Fis4n2LQUwAVXbcXxXjpnKeN4cG6Rg3hxcu85JyO2SeQNBAD3OZZw25U8YJ9ADoUAHUYQBlmoBvWIQKYMdQHRkm4GsQHAVQL1R6ACkvQmYJBFnoqCuQywkJ0i3dk/wFRIFsEVAUyo8Aq3TPkN3IesCSyyFESzB4JTF+MFR0ezNVowjwYHATCAZaaU/VMgI7GKpyAsHA5acN6Z6YMBismUAwcBWQkA5yCYPBZRMIBi47QUiLOTaoNxwJAnVwtxgb3/PZIKTEt6BsCgh8VXbKPK2Z2eO2rIFPDSrOI7RqdxsYcIRm00xPKAKTROwQ0/CAsSZ9HRMvQ2X6hMUhRtLTXRwaZ3G4AUbSdTHY/zou0lJzXc+gWzTATY6vMqCcJojKILNc3UBIz1A2fEtKeQLBl8sTVgxBfUNQS5QzVrnQXPjYIcDV4xVDUN8QtAyEgAUwBLWUjzE5r8i0hqAZLcDq/+EYRdozsHxN6wZKb+A2XxzM+QDAeJDjK93NImGc1gyUrOKQsjU2qmdg8ZrWCDS4cpthCAmBFRcKnmWsS8K502ahh1VZjMJqKy4UTHm7AgVcXCh4umVlhQi3o70ygXuM5pkp6PpWi4G0B07AxfOVp1GSkfUNATdtlAncY8BdYcLNNSgJku7Ih+4EOO+Oi9kKSWbkg9pRj8/HYy6Ng7XK6LKenFOYy4uFolcyslOYymjDUVyVTnJV0lkVRDS2AQZtSYkFzSALGhQAxYLmRAsaEABVhzeFmIv0mYuAAJjTXJmOLapatkIb5+2jjNH6i9fL+5v6UmzCyWeR2haNtHm3I0zSx6/8NzxL+yLVTJXrp5fuUnR0emmVZd5y2S2CTmJp2RCfO17BxbfOaa4TF64RwnhIbJoxYjMQfKQH0MrWzTttF9ruxcdCM/R+GX2/7V9M3m+8UG2WvThJ9dytiZMUyt2aFsZTvltPMCsNubkyZaWS3BIbnWNPAnn+zbfyf1Lee/M9/gq2my9NXXg3331RutcybVdMNUtjZdXYgUW7wT8//puT0n00myjRrDp85sQosM8oEOT+rDp85sQosM8osAbQLMr0ANIjAjEKHMRSRVmBYhSYyihQ1/evRePAle7AVHGZu2iJgB53QJQI4LJ/EUvAPktAlAjgkvmLD2CfDyBIBDTUknmw4aDM/1AigMsqAJIkB+X4hxIB9LhjqnsBLJu/Gt/gXi82vlyM5/iVPpbNHwq+XHv5uZs1oeDLtVOfu1kTCr5c+3BIsyY2fCtNxnw+FljptQmaa5sNacXEB3C4jZIeX0M30ZNz4UHyWdLK6b2lVjInD2XoznhysGKZK2llyzpH45grKUP3vpOz4UEyV3pcuqZMDusg5ztsWLHcleqlWzTmwymBpXtdEeuPsfn1vFc2FWfAJC2uVl8ZsVaK4sRRo20WBS24YsMtxkpRnDhiw/3PH179+38ffvv4a/7j3X/f/Ph5+T+//PpV98Hjw6fLuyF7YmVDe+I3N+vPl4/g6+yb7RNfX95veKzZ42PrD8v6/9c39e318arWv8L1zcPvm+/XFcLm/4+/+9OD95vnr34/2Gpvf8GOrXbUEoKvrjOkmWL1goZMkXHd///98fP/Xn/+/d//ndvi43/+n4e/9L8DMQPt8aAr276EJmuWUxR7h9AlDJ72irdDH5JF6SGpHEWSjcMZApJuXMXTIQBcaeCAoyx3YC3O/ouhWpx2NTRtYU4IvpAqp/NeFVuSoysS7mOru49cE7En6SkHMrRc1KHDEVsSD7gKDbgO/U1ShXq6o0FtczSAaJ9SJIoB4JxCA462HEWaGAKuGT2zB87aQH8gKnQdLmkiSmz3drICbc2p7mbg1BjICQ9jaZNi39GNwY0JAsslLoAUGMWrP9MDS9tPU6W0pQSWlD/pgeUSC8xKDKILBwcslxm6nhWwJd49NmjXLfKP595j/c5pemC5dD2z0n1o63dd0wPLJeiZlR5AO7/plx5Y2jyaHGM8aTlsMw/RKkQ6YeOcBjGlXaXJYZoy/VJMn+Y6psI04HoxOXp4Stb/nsZ0uE6bXk8yUAfJdbBBTcn5p6DWCzU5qLTPJJTgMXY72gNbmVBWZiOAB7Hmaj1B+pHxQQu339Hn03pKCSyhMqYHtrv1NJTE3+Nc8Pb+CdNh/PuzgJqcrqeHmqsZNS+TikzBAUt7FtCiCuXKytem1qgtSpvtPwy5pHFFFmHDD1FZ9EFb5QTasuqDNipTJeAXILKLAJJWoSNZHp8RJDIMcBnGUB3G9v4WW4ihXHMWsA//sSf9HLsqosToSU8WPT0ZkWYMqwYLeCQ7pm/LNMH2njfT6Ejm9NhIRB0hJB08kvSsSFQeASRVWVIk84UCQrJjQyaijzaS2sCvyfNhHidsHYd6KJXqAzpqxzHnOg6alSgkVPmiAS3HQ9FKZTTouY6L5qUPClTSaEBzcZlnpRdSVQUPNNeUAj0roC1+ecY1rWBWeiLl6AkYGNAFbWdNtQ5PuaIdPUtAA5pLhT8r7o7O4DdcBW2GTbXqTgm0gt9eFYPmG2CLklLuq1SpCcKF7elrRxU+FLRJNjmEU8rOQghbA4Uw7Y5NTq+Ucg3rPLCGq0XlnhDOEyNMm2CTQzjlGg4hbDUUwjKxgmdPlRHkS92XvaOK1UquPdWsxGohIh5Yqd3k5zPoeyYFGr4dVg6quLHv1gkRLjw3FuXyclElvj+XZ1BjJ6zAKKaVSo8praqhBWvaM+9UlckXml7EuBK1MnhuKxK1vhXhYWnLIJZxlUwdluU0rc0aOqUBsXOUOy4qtOmo0Mrd/r9Xhba9acVWoWkS8soudoZmrDq0kvZFRIcW5HDipaSmmBPl2fGyrkTEjjYrRGsWsOEwiNh1KD5FXdbCLs8QseuYxyV6sjZ2TYw/YadD/YS42HWM6hIF2fFeEMS6Ox/2+YDW7ap+85v15vqbzXZo06T8uf0QG/h+HzALYh+1X1+dDyEdGnvSN4EA/3xI6tjg+ztUBPAt3aFOlbgODT7Z4kKAfz5kdmjwyR4ZAnza35gqwR0afLLJhgD/fAwiEp7pW0Rkz8cgPKUSDXGXZmlnbKqFekpJEuIezMp8ujGghdxhcfVWZqUYNYj7J8e1eZ6VRjRH3B05ujWenGYwIYM5b5gPSAxmR3e8k8M04d2VYgrAYHZ0IyuqsREqKeVhbVW+yPIngYkelJ7ZKNeOHlZOToyQUkKkgwoT4/YfNvGyHjT9FxvflO0Liu8mVwPhS88coYUnLig8OXo942pQVBbci4gIpQfYKqhCOQpsVFaOyjrs1WnumzWONqhIQcKRAoanTnm5ruRkfcup/Qg+QYpqVNS9ipTt3Sy2IsVRRYpaGC9SGQQp++siipTjKSmoSEFKSTIXaVDtF1SnIOHYwbpOqlRJaBebBSUpSIB1qIxFntIGUgX1KUhAdmgzRaviARkUqwABqejuWYQrQ/pKaCtScVFaIVUsXDiTrsjjkexRmKOeziqlmWDWc4KZNE3gYOaiqUJKU9hgDtKikGCWKWYc+1Q4mLn4q5CKA75iOigoQMKZi8w6K3FBFVQXIMHMJRiPX4Ml1ZCgw8zFX56V6oAc1sDBzKUCn5UCwaGX2pqrPzIrynq9MUXHmTZIJsd1TlpkBwnsOAQ6pc9gbFVKhiQFGIshqTRtigizfYQ7NGW260XiAShKT8yH3ZQ680dQ6EovGv+3dDRYLV7sJwGaZwTQPAsCGvc8T4sh+zD8Kkz8CnqCjkd7FVP27vU31JV9dw+LTYI1RaXaYW92wczLgdW0XSIc2EBeKnLMvNTUd8J97Sn0FCh+HRx0cWf38LOg+HVwloUC28avKkDxo/ssYb4G8LPKUfzUQifHr2NfJYRXDz8Huv64jusgea4JrVuLXJEAqB9q6v9kBzw5l/eUlgDwKmDQAOA6yYXkxqYMAAsaAJopACBZswkDoCpAA4CLHS/27V4RqEEDgIs3D0moThkADjQAuBj1s2Jam9zQTV4e3ORFhpeLSB+/xk8JbxE424aAl4tAPytmtSkCR98Q8HIR52fFqDZl4AQRAl6uDty8fN0rlS10oLYqUuNbcDXYZsWUN1W4e5IeXjF4fylhsI1qobOF0QTVuKTpgvbEJodqyputpXSPQpmFpQTpyLgOUjtoaFyTrlbrr1blFs48+bknB5i2siYHcNKFS8+pNwu36RCmw5V2qETFMsamt6jaaJtCLxpSzvg6lvrL+9Wmit1/74e6Cv3w99X1cvOM/w8=</diagram><diagram id="XSTu3f3XB3Kw2yURKv7b" name="Mirror-3of4">7V1bd9rI0v01PDpLat0f7eA5k5PJZcaZZOa8nIVBdvgGGwfkxDm//pNADairW0KAuqtFZdYacxXQe6tUXbWrauC9fnj512L09PXdfJLOBsyZvAy84YAx143D/E/xyM/1I7EXrR+4X0wn5Yu2D9xM/5eWDzrlo8/TSbqsvDCbz2fZ9Kn64Hj++JiOs8pjo8Vi/qP6srv5rPqpT6P7FDxwMx7N4KNfppPsa/krWLR9/Nd0ev+Vf7IbJutnHkb8xeUvWX4dTeY/dh7yrgfe68V8nq1vPby8TmfF4vF1ufv4z3+uP7++fljETx+YfxUs3z9crA/2S5u3bH7CIn3MDj608+1q+vfw5Yu3vHqfffhl8fvfsy8Xfnns76PZc7lgw5uPi/nLz/I3Zz/5Qi7mz4+TtDiYO/CufnydZunN02hcPPsjp07+2NfsYVY+vVkrJ79TfkS6yNIXAZSGX+RuljnnZzp/SLNF/sWc8iieH6yP85Njx6H6sUV689jXHZTD8rFRSa77zbG3C5jfKNewzXp6YD0HLBw9FMvzeLss/nweTpf/uGB58xV7Km6Of86m+Tovmtf4dg3Ib7ebB0bjf+5XMH14zvKjpOXjd9PZ7PV8Nl+sPsm7C4r/CpCyxfyfdOeZcPWveMf8Mdt5fP1PF6wuE2BlAYQ1lKAadIZqAFBdocgIRTWKCToUQwBXOsmvFeXddHY7/3G9feBqa/CKdZovsq/z+/njaPbbfP5UYvJ/aZb9LK95o+dsXoU2fZlmfxVvfxWU9/4uD1bcHr7s3vnJ7zzmP/Wv3Tur97yKAn5/+77VPf7GNQ34JY6V8JffzXUggSZBGk98GYFiduutCCQhSrFiFZos58+LcVq37OWVNhst7tNsj2vRocRbpLNRNv1e/XoyFq3eerlYjH7uvOBpPn3MljtH/lg8sOUzA3x2HYGS60NuCbr5bkdwNpJbHo8sj9LysBCd5YktsDx4DUii0y6AE98T6eTGAk/Wlq1825YqbS0M+CDBwsA3ROIbHB0mKZGbJJ9Mknqn4mEzSfzAqE3S1hna8X/+5ge00RkKeAgEjTN0HItg+GBlCwKyBUpb4DvobAEMWqxQhPslQnGDIrj2GkcRXoARW/SNBf97ewQrDXqAzaAf5HsGRlzJQBFXi8jwKA1PIAa9zRseq+JqguFh9lqefsTVQpDESXRYHkVcDQZqyPJskHLRWZ4YoKjE7+Hlvkhwv7p9enh8VT64i1qxXtNxboVGt+ns43w5zabzx/y523mWzR/yF8yKJ642UO4Ct/q3c4zL2fS+eG9WGLSrUXlvnC/+ij9P6WKaL0BxO7eOs+nTMv24fehqvibJ601C3Nk+WBz062L+Y/p4X8D+8+F2XvyAh+ciob6yrhUqQCaN4/T2Dpg1CRsnozS+G2vdByWQTp6ETl5ndIKRLaKTLXSCUX/TdOKmkOhkIZ1gxNY4nWCsrf90Go+y8dce0Anu2ozTiRGdrKUTdMWN08nHTaeH6WSywhgToyZRcuvAIINsb3iXhuNOGYXPQMHgIDHKIkYFkY+NUSFuRuFzyBHRycfnkMMYItHJFjpBsZVxOlEw0146sQTbxS6C1smeyh435lU7P7kzobGyJ777PXz46fyW/eIMkz8//JFObrwLZoWIV1U+0JzmrJ5po0UlxZk+Tnbvts6J7pHaWi8uSIPuR6Q90qVRCV9junR94uhOg7qhV2V8GGhIg0ayaw6Vq7WzVaFTRS5yoK3qKi2qMFVQQIPPVJ3Y4qgsX5PhQ2ipEq2WCpqiKBBMUSQw9UQFD24knDmhr8PmKeoX1N422Th3QwE8Ni6xwMYdXs3ZqTu2h0StayPHa6iwumORQPgw1GCa+KJQtWcL0xT52EyTFbVV/bcw+5ZaHW1hjorT8K9J5ZRtwkMuunMebpHP7ZxHsHWKmR3nPJOf81rLJitkOJ0B6O6cj9Gd83DLiO+cVxW+GN6CYDAWHu4tSBLLwzDdbkGoFviQVlfoTBPcMtpjmig44qO2TMwRA8RagiNQYUrFwk2yBIehs0w2tykwnEbHYJr27XdgyDS5AuG1pNFjRR8DrdXElu3nmJOYM01yFEkebG9FDEwD+JBOWgV4McmDLaYTiDAbp9M5yoP7Uq8H/XDTdEpgvmLFmXezN8P87mNYeAv5I5Np/ruH00WOzpoj6WiZrZdeTbUG0Gs9Fa7U3lvt/apKmdzpfpyMFpNdytynj+liNOsOXTH+YxxdGP4hdA++FGDzLBK4hSZ0D/YbRWWbRnSlERJZ15BwlpVbtArs4bfnOX/iYrmKG1zmL3D9p5ftk/mt++LvTZY+5U8Ortkg9gdJ6K5ffJM+TvI/f4we79P875vlm3y7+fLhcVYs0ud/5Xt89nq9zMVnuxch/zb5j1t/ofXhASVzoLKq2wGcA2VJ7SLNf83odnWogiLltj8/bnA1CIbFsZ6z+bKMlGiLpnHNYUkVJilK8WRFKawrrhhLQO5G0sqo2D5dnCtNnLfRs7077yncthrXb49Oe4fExqRo8C3DbmxM/kIjnfZcocTfO3GjPelPNZaHqjCUCNqSoMwEQZlrgKDG0hEVgkbE0JYM9YwwNK4y1D9xl2TpT4WqMF0JM7abMXOJoe0Y6ptgqBcIDD3xlCrpTzXXADoihh7B0OBIhh61dzFVo+oO6MJ7DGlCE2bNTwy4hqYaPlQZSgRtSVAjwpMgNEBQUyWwVYJS+KctQ2MTDA2FBmperIGhLoxm29OFSOwB7bkw3qu3CZEr6wFN/VraZXyEKJMvSfholcS6DIBKDSkak7LYQFTUzVDpfk0+TYxVGAdRUWJAtdg1F0mGDUSFGJtmktaAmGADEWqxqQyxqUNyiA1ExZA1qthSgxh42ECEumOalNc0TMFBBiIjxeDpzKx4rYwhuFoFgwx6NgTuoWeuaH6Ngws9HgL30KCBjw1c6AkRuIc6TqL3axxc6CERuIdec8Ugg3FwYeqNwD00+CCGAU2Dyw9M4J4gxisG6o2DS4WPpwNXTKUZBxdp3eM+UgVs4II8qXFwkQYxrAQXnVlGGsSwEVyQVzUOLtIgho3ggnyrcXCRBjGsBFcYI2geXKRBDBvBBflZ4+AiDWLYCC7I25oGl6ccCdwTZIXEfK5xcOEMM4tk1UI/HfOzXSNzBYw02rWBRnsUXnDZRGPhxfq00V144Yrdsk/ckrZ2TahS4IggCbbm2ZG5VgDH2ykyUy5qM8XEztknbupfuyZU+9Jm3Ag2s2RVS3/BLJ3VKFb5Ochw26XEhF1icrtE5Vw1aYcQm10y1WIEz0xGDOZFMmatG/NyFFkCRQEn1f7VFchjO+NNtWw5zBPZcT42/avO2hORzD3DaCoUZcJUYVqT/HKwmQpTzXNoeOtJTIVkDhmiTUvgmNi0KCrfqWi6xi6J7f9N2yWeWrTSLlEsJcRtlgS260lFKXo5UBuAGjmJOHPGuFmCiUJ7zNJZjW2Vn4MRarsUCnTn97u1S4r2JNTZokYJJY5FNG2XAqQaRhvLucSz0PgorJBq9U7nUWCbcxbCHD+Be+guVgzEGwcXaSGmjeB6Yl7VOLhICzFtBFdUNxgHN6aBWJiaynN6NG7cEjMDsYRmaImnYeMWm5PB0kSsYyhqZiKWUHaX6IgtxDQSy06KGh2JFdPEIStJY2biUCTYNabDrplSTdDIoeMoambkkNBwM9ExjJKviaW1sdUViyXbQb21sQmNHDpBIaFwHkgqnrVmTRIGQKWyq8ayK2wg0sihA2pUsIFII4eOnstnHkQaOXSAIBwbiDRy6AD1LDYQaeTQAVpDbCDSyKEDhFnYQIQtqpTwSRPBO6AVyzUdj2a/jW7T2cf5clrmk2/nWTZ/yF8wK5642iBZyQkX/3aOcTmb3hfvzYoA1NWovDfO135Fn4b0c5krfj1/fEzHWYnpNoGcfV3Mf0wf7wc7CeSH51k2fVpFwwSZ6Z79z2rCULouzq4jGX2sNS3tOjCsQ3yyhU/itgsDn2RxLeKTHXwSd4AY+ARDav3n03iUjb/2gE+iC4yBTzC6R3yyhU+iN46BT0gHLNgo2hTzmhjgRdqF30Z4RWUFBnhhSGTAwllWhh0qwIffnuf8iYvlSlhxmb/AZU8v2yfzW/fF35ssfcqfHFyzQewPkpCtX3yTPk7yP5+uv38sTLKY8eWfnf+Y9cevDwYomAOTVS9F4IIhXlceppPJWjqS5t99dLs6VEGJUmyQHze4GgTD4ljP2XxZll0OYBWmmYS9hCqbBsC7XGHdcYVK4k4Gr49vo8G/AMF7grMXX1zCpZrHk8ErVidggPcct3V9CTuBJmami7VcF8pHiE620Ak0zDVPJyhkITrZQifQjdU8naCkpv906ksME7SgMk8nKO4hOtlCJ9A5yDydYEzNnkKRhFdi8a2N68H11Fsp4royyQ+VirTCNQoFXDd4mRJybUhExSL7wxh7+GC0qoPj5s6+Da87aMqotZTT5c5MYy1neeHSXcyZAErrKDjeLAtVObVwDhx81gfihc/69MCInKTFChxh6oiM2gyh4wdZm63yfVu2tB6WCj5JMDOSd7jiO3SUmZdLTZV7rQxThM8w2TVTte3IoNajPeJxKk/z3caBHyh0FV1atH076h/vFh1JJEX9oNEi0IPQ7KB23vHxnfiKUkGq96zB0QXXWvM4WjXzre3AFPs90kSb/T7M4WSG3EcYLKVK5UbzI0a9zZsfHna30vw0D0ax3v54Lnb7Azjt6LA/fF2oyL5VHy989ocEhfYmxUGg1pPM09GbFfdIUWgxn8T4GgI+naOksC+iHehwIyAUiQotJhTwoBAQCrmscFMRiYlTqqIfiVPPC4e64xRCIyWr1yVOWcQprg7FwykumMXKKXyeOSZCAV0DAkJRCyx7CZV4+AxUi1DU3Sx9uVwsiqUu5muXN4fj2Wi5nI4HakF9c0R6cExIGa73znLK+jjwx45UWEVCFmrTv5MfYh3pBvoqKNPkcu2NUkuMPSqUWieLLPuyCFLLXiW+rFfJ5+Gbm7f5s18ub/L/X/3x4e31eys6kXRlA2KhHlla3+7K4tHddR+JjMnMBzrmR1ZN0WhRwTk3ZJUGNK2TZ83jhY4ybvvky3g/m+Z8mW9k7FUc+VXO+5GOdJkPw5gWFYslwpp5sfFiMV8WxqNisXbFYuK5EEjsv950pK9Q1ak9QYIx9vHBqFAnUdVNjZV10cEYKEQeVKNQV1mND0bFpC+Slhd7ngAfXorSIJKW16WaGT4cFSUdpNGtw1HcbCDAkQYKHZABC/HhKGswQgkLO6Qf0D0OIJ/0Jiw4e4lPNvIJ+Onm+XSOCdXeSNOg42KeUIwIZS+hoAdlnlDIq0VIl9bEKYRGCnnFCHGqiVO8ORMiTp1j1UhfpGmug3Crd45VI30hVOKjM1BcnkdjWo6HF0h6EMAL/WSb5D6C1Dw03xs6kopCSe7TTu4j4BoZ7z6QwKvortpzK+fsqAVKXOmB8sppUH3md3YMKSYl6Ph58X1jRNrb9DUKHcpCo71loZERWWgiSqEjr23bUOEdHQlJI7izIYFco0AOneGLFNIAEsjVZQDxwaiQq5JAri7xhg9GhcCDBHIrgRw+vBTyYhLI1Qrk0OHIy0hIINcqz4wPR2oGeEh6Fx+ODOCoRJCi3ENcegHgHsfGK/pj5HIB4lMbPx0Bn5BLBUjP1MpxQUCoc9QJ9IZQwINCQCjkOgESMzUK5PBxihpMWs4pFqPjFPIGk/g8c0yEArlHBISi2jp7CZX46AwUnw9KArnj4Y1F4Y95eENoLi7HWb6UKpvx/DBbv6BL87Bjg6Rnfv0J/csvjqM6oXV19BNLhiJJ0bUM6e6Ue4nNUkjXcYQFjSO4oJq1kAlpIY/XQgrVv8yRmETNWkiFJEjtRxGMcYAPRoUkiJRdNf4nwwejQhJEyq66wgF8MCqUQqTsKpybEBtefHYlKbtaJdY8fDgqFEGk7KqLEzv4cGRyHEnZVYdjhA9HUuLYm+iG7rG/X1Sns/jdxlIRn2zkE/DTzfOJhDj28kniuJgnFHIhDhGqnQdlnlCkwrE6x+16CI0UchUOcarJTnkeNk65MELd3FKj0j7ihP01dttr8GfaTVRzB2c+UY3xwa+NrTOYq2Brt60zXMerykuKrPkuj+EMtqT+Dd20ztgsJFZri2/PisnUOviCIFxbQ4SykFBJgO/SDRU7pHk8EN4YCHzMwwtzKaR5PAHSYhVYsmf7x84UesyF8gIUJ/Ie7iy+ExlMtzTd3ZO5UI1A8B4MLzZtOnOhSIHgPRReINA0fxmGQTebBOlca77ZOxtvzstc2caaBOntThShAekWL2OaER/itRtJTGe38x8d9eWttOXdM264GzXcCSKaiRsGaTzxZWyL2a23Ytug67ghtwF7xA1jOTX1ttxl/Bt3GwXky0JlFm0KCNEZJ/7LqMyiTZkvPhhhTITKLJrL//HhSCPma/CK8eFFI+YPqLMI8OGoGAdAdRa1/Xjw4ago/qU6i9peXfhwRK4SxJcSRiQ7Bf6xZzwWzZArBIlPrRx1BIQ6RxFUf3TxoudinlD8yEQoKwklulAICAUje6gIRaL4pkILhEYKhhmJUzZxigkyEQycohJoiwnl4NvreedYA90XQiUhPgMFY1GkNz4e6ZgPDdlEHU33WGXeOW7q+xIliqNCrYTMdvjnuKvvy7UoThKEjGqxBbubpS+Xi0Wx2IUUr7w5HM9Gy+V0PFDLbpuFeuvPPFB2B1d8ZzkDyXLyx9op6YBULkyq/fl9JsC0VgOW79oiBWt1g9B75e3+qxw3SITjrrWD4LgnE+P5srZk4SwrgauwI/z2POdPXCxXkF7mL3D9p5ftk/mt++Lv8PJT8eTHyz8+5X++XN7k///tw80nfvT8664/YP1ywMT8jMyqPAO2RrmHX6T5txvdrg5VUK4UQubHDa4GwbA41nM2X5ak1Oi5uI5Q8+hL8qVuJOGxyLcTmoVz3EX3JXzMHCF8HBmv7vCRlt7ZWEO7rSNBA28M9b2aahIGOxUJryJeoNCumcm2DOFcm5kE5enZXJQQmmlmEglaTV5M3G1RQlzftKfLUhu2f48eoucOPT0j9IwFCSqXLHVMT3hR1UNPd7Brdcnods7qxASrmRMaYTUUu5tgNSNad03ryIgvsS3N5rTWUuAYwDSiRfXtsbBmsaTbh+by9kCm6aby9na4hoIUNokhrnql+oFiNJA6/kQwumHsIoMxpELg9jBG4jbCPIyKAlIqBK67WDJ0MCrqEbXWAdfGYQ/HtDsYE3QwKoZYUnlwDYyJuIs1D6PCxaHq4LqclrhrMw5jpHBxtBYHW2ZUNz3FEcHIAIxKAHuT6u6LSg+4zNtOfMaSoREN9bOYUKLzjoFQJMaxl1DAccFAKFLjnBDfEB++SDth24gvyO1jwBdpK2wr8RWlRRjwPcf2Mr254ItbXAyESshgnA5ffA4dN2GE7ykuCC46fBMYUrBKPBIIC5rABdWsHklke2pSj7RVjwjAMsmZojeamygynWoHinB0Q665RIOjJ5n2Y8+Ui4jGXCQlo5pVoIkZyX4k9ItweYVltzLQRNFRmHRRtUkefNcZhRSDhFG1uRV8OMJQCymjmpVR+HBUaGpIGlUrjcKGo+dC9Qw+v6/37lvZl+ZYpwx6XYl4Jd9s+xt6mxxQvCZ+UqLBv9s09CFtX6sUOT47BCPm+OzQZv/pVvef7Oz3n55TvhLr/pO5Iue17D8360Ki1VYZPXT2yWEARyWCvUkR90ZkCAMakpZfWlNMnkOqVYsJBSMr5glFqlV7CSVxyc0TilRRFhMK+lDGCcV1HqSaOYVLA7QA5vGFex3qtH8KZ0MY0rHZuhprte+557gd6ssYIRirRWA7znE71BtCgeAaAkKd43aoL6MboFrIPKEk/NnNUGxTEBrSFPUNalW9EjdZCz1ZivHz4vtGGI2vcaLn7ts4sdy4aJfMxZHg93leJWUhEdmJpRLCOzpKcrhQBEy21hpbG+MLjjOkkYI9zBa+SEGI8Fpqd32NuB/3ZANktNbXeIzqa05wpoi9SHxJ4ZTevHMAAdPqdcYVdcwrp6FLd35nx5aK3qhbbd1dHq3GI13dEw+o001FoKxhe7upzEjbejfiM+02Z03Y4KbGfFKn4h0duamMatUOsImJO0BmE5miFoRqeuoiKmLbHAQ4KmoIqKantrgbHY78G1FNT5taELFrCQIcFVpVqumpw1EM0iHAUdFEnmoi6nKi4qYeAY4Kf5W043WpSLGbDAIcYQ26EkEKjw9xCemg4xx4kFB6w6eebBQTEcoSQgEPHgGhSOprL6Gg64KAUHBrT4SyhlDAhzJPKJ+0xR3lMoXksL/ncPvuUpk8nUC2w0I1CXPEeAgC20HaYosJ5eLzbnzSFttLKDdK8BGKIkQWEyrGFyHyz3FD35sIEdBcICDUOW7oe2OhIoaOUJKZK/ZIgD0fzByRbJs1S4Aj2SlKEuB2wLridCJZrbXe9CH/RiRbbIEjC1x0ONZLuZG1ONzcoRb7HL/yhGsWYq+vbbqF2J4nTs7jF6BuZdV8XUiO28I+eWKpCQL7BAHDZ5/OwMz42szMkXyBUVCSbzee92LwEcF5r5AZkny7bgMoxvwQ4KgYdUPy7RocA1GMpRPHD5/fzt9///bl8vrv/7z+cZ1/ztWPC0aX4bt4nMob8t3GgR9sCdHlZThG7u2HgLl6vH3FAB+jxQXNfNFjTUIxLIjgqkCDFQ62Qxi2A9xfRGuHIsB5LXYoUURFqTimzj6J4izz9omnOimxaGGmGoaxmAsJpTexmMBwJBHKHkKJ8REEhDpHtV9fahngJQ8BoWBkDUV3Oxv74HtMTNgjwBcG4qhW5RQxV3GkhSsZC6a3VkUyzpiuDbbI4uDWFYHtIJ2lxYQCCgzjhNoIFalteT/alvt7T1ot/UztAUEfDIOIKwFB+A4G5ttX39FNCNGXzGYlW2uNrfXRhZ58B/IHxcbOxrblHjN5LZVn5qGySU8qjd/eleK2voxuL5xGUmlar5xS+CSJNOnrzKTRAnEml3/i0R11a2JrEY6wZJ7OIhzpeso8CirBaVuCI54JpiVZ9Yos6sHfox78Uvwlimzp60IjVw7PiYTzxW/Yb/l8VrjiHd1caxRycfWei2xh7gSHyGyhQixOxT61KglsVzSFVJxKN2qlCdhQVMzBoMKN2sINbCgqBNVUtlFbtoENRcUsE5LFr2Tx2NByYeCD1MPNUip0MFIGx2JpHnCLA9MRfpcRnyzmk+igm+cTSYct5hO43pnnE4y+kLL0FJtCMDRIo7JUjvQ5jnjpkbAUneU4R6Fyb7ROXoCOT6RTtphPPrqdF4MBGeKTNZ4ySFOa5xNFhiy2TyxGxyeq2TwdvEDraB5emG1DAa+Vym2QnDMPr93N5sXoRChZT706V0a95k8gdBV7XUQS/bJeoSvMf5BGr9FZCV1sMMKBdyTSa4yxiP0UdcL47uLH8Nd319/+/dfN9fTu+/Ttn6PRhQ99zg7qjVaHye/yHcpgj0Lejei89z265efXvoJwZqaWyAdUdjTouz2FwJtEpbU1TOh8AGoHfgCMYpcp4zAG0Gezp+/rWU2bkZ+FIe5LTAAIr+USQy3uDwjHiAOUjdsmT1G4QGLpQnUkRnvMw6WoUCCxdB2MYl9C8zBSyt7iFCsMT8SQT1qD6pzPxCcb+QQ3nsb5dI4p+/6IpcH1zjifYBSc+GSNBCQCbrBxPp1jMUdvJEWeOLDPPJ9goJX4ZA2fYI7FOJ9gcIz4ZI3/BFUExvl0jiVIvbFPLMHGp+Ac4wW92d/BLIJxPlG8wGL7BBOIxvnUIl5wN0tfLheLYqmLbHN5cziejZbL6XigFgk356LXn3lgZhmu985qBpLV5I+1SxeDfHD+rwpnIsK0zniXb9siJWlnKRoaxxEOtc6Jg0OdKskcyPb54SwrsaoQIvz2POdPXCxXKF7mL3D9p5ftk/mt++Lv8M3N2/zJL28+/Zr/ef9hePmpePHHyz8+FQ9f3uT/v/rjw9vr9/zz8u+//sj1AQAd85Myq5INmBvRKj1MJ5O14CPNv+/odnWogndlyj8/bnA1CIbFsZ6z+bJkppKopzcNLBB00YkDLQNLJGRmJzAN4+niV/bJeZtdsOX/vv3u/37/+dum+sfK+gEGLt2ezvoB+YLK9qpUP9AOVzB81deYYpbDCiUB1ClbW6fsPSQsRzkXVdmanAClxmBXtiZ9oafgdreyNeb7TDhl3IpsDb4j4LIJxTuOFrrVriOqQhwkCinmiaEN43bPg6ENKrhpgtEXMzLmYYQRBapgaPQuxUSIeRiZHEaqYKiBMRT1QOZhVFQxkti7DkYxrGYeRkVZmFGxN3YYI3HLbh5GRVkYicBrYIxBX0njMJJIwN4knMRl1pg0kfOJRAIW8wn67sb5dI59SvsiEpC4Lcb5dI5FT73hE/SfTPNJUvREHdhPcSXyxVm+yX5Id5aipHIki+VFLAJxEOOWA0YliU/W8ClG59mcZXlbX+TZzBd7P5rnE5W3WcynAF1kSFI+YpVmTuxhLJl1qFcz58tCI6SZa6uZE3CNJOJSrRF5SVkMSX22Up8QG1wKjQhJfWrzFujOOrgbI6lPc7oAHYwKjQhJfWqlPuhgVGhESOpTK/VBB6NCI0JSn9qcGToYFV1WSepTm6pCByNJMyxOfUKXWSId0xrQCc5RmtEbPkHf3TifSJphMZ+g22KaTyEMpBGfrOET9J+M8wlG+kjqc4orEfd4N+MMg/2Q7iyzEsJgIFkOa1KfEYiDGLcc5yjN6A2fYnyeDUkzLOaTn6Djk+E2LbtdWvgzDQPCyjfxbiybeWF65oPVdV3ZY0T3erW767rCQ0WNXVfCUE7VjruuBOJsgLip64onyndjHV1XQurIbbGdDdBFTEOYv1yR5t3szTC/+xgWNi1/ZDLNf/hwusjhWZMkHS2z9dqrudaAem3+hGOxN56vqpzJLerjZLSY7HLmPn1MF6NZZ/B6ATp4YV4TBbx7XJEQwguEecbhhfFpmwSsYpQllkRZ9ApYedsu6g5oojug3qG2UgJwp67RT408I36qFwuN5ZKgwU8N/KT2Hd34qXwdSQl+zAVH7H2QSComtAo2IoW0WL0JOSMluO+I56ZxuGDWgJTgjTCKG3vzMCokxKQEry2PQgejQkJMSvA6CbHYX8M8jAoJMSnB62AU46vmYVRIiEkJXpfIFrfs5mGEEmJSgjfmj8WxDOZhJOWuxco40WVmDoN80hoZjc5RudsbPom+u3k+cft4VnzqjXJXdFsQ8Ilat1nMJ9F/QsAnGOkjJfgprkRiSn8zNdKUEjwm5a7FCqAoRmc5zlG52xs+xfg8G1IoWsynwEHHJ+oRYDOf0EWGEugpE5+s2Xl5fAAiHj6doz/eG/vkhej41MIfv5ulL5eLRbHUhRKzvDkcz0bL5XQ8UGt6m3Wa6888UGgJ13tnNQPJavLH2qknodgx359X4HRdXkjGD7JWgJbv20IlmcMs5sg8XzjUWiMKDnUqCWUic6TDWVaCVWFE+O15zp+4WK5gvMxf4PpPL9sn81v3xd/Pwzc3b/Nnv1ze5P+/+uPD2+v3/Mj5N10ffP1SwLz8/MuqvAKWRTRAD9PJZK2TTvNvNrpdHaqgWCl2zY8bXA2CYXGs52y+LEmoM8rjiZNFmRMDK+DKEqLsBGbgz//+Pr++80fx4tfwIfntz//8NwdRqp9tBz6TgX+TpU/5k4NrNsh3/knI1i++yc1H/ufT9fePxdZY1OpaQQ8tEgg3FqpANgOAdqkis3GnoMq7ix/DX99df/v3XzfX07vv07d/jkYSqthTBOKCUvZQtp5dFYFI11OWmiHpejtYw31qpbpSmEhRVWho1c4VoeiGoLe5aRQVEloStNegGO1TdawVRQZt526ZXTq7nf/Q0NhhU1Anra47ceGb0Vo3KQqSlgxq/+KISrejqKIQW1PpQ51PtU8jTa0nvEJrTZUPNSgmoAe+aRQVUmsqfKhDcZ/Wb1pRhEprqntoChA5+zRc04oi1DdT2UMTiu4+DVy1oujC2I0SQEoeDXElIyUbK40TDeV8OkdZcW/4BP1243xiZ8in3sjUoddinE/nKJboDZ+g/2ScTzAeTGUPp7gSgQb4yX5Id5eUO0dZcW9k6g6Ighi3HDAkSXyyhk8uPs+GZOr28smNQAN843yihhgW8ynGFxmihhj27rwkQhzTfGIUubbYPkUMHZ9gpNEeFStoE7BVrBpTsTJZHQnJWNuWmwm4MkmLer3SOYUgSm19z6oDc4gNLoXyiQSrte3ksJ11PsSLBKtAsLpahvwx7osNutewsjLs0ihiXfsXR6hYV29tP1cMULkMZJ9q+kLtqpBqtlXTQWxWhymkeiSbrYExdNHBqNDqkW62DsYYG4wejHiQcLa58yY6GBUjbEg5W9dmjKGDEcaLlABS/HGIK54t2eHtqS/pLP7onaMyrTd8gr67cT5RW02L+QTdFuN8Iv2bxXyC/pNxPsHANClnT3El4l7wBmlnP6Q7SwR656hM643SMQJxEOOW4xyVab3hU4zPsyFlmsV8ChxsfPJhnJb4ZA+f0EWGfKqptnjn5XmiJMg4nyhybbF9KpQuyPhENbBdIO35YrVzhzWw+d3FvOjTvHnuX8V5/24+SYtX/D8=</diagram><diagram id="3A1HNBjYupDa6VtS3OM_" name="Mirror-3dc">7V1tc9vGzv01mueTM+Ty/aMTxW2aNsnUeZr0funIEuOolS1XkhPn/vpLSqQkLnZJSiYXWAn3zjTWGy3jHGJ3gQNg4L26e/ppMXr4+tt8ks4Gwpk8DbzhQIjIjbP/5k/82DxxEYb+5pnbxXSyec7dPXE9/W9aPOkUzz5OJ+my8sbVfD5bTR+qT47n9/fpeFV5brRYzL9X3/ZlPqv+1ofRbQqeuB6PZvDZT9PJ6uvm2VhEu+d/Tqe3X8vf7IbJ5pW7Ufnm4i9Zfh1N5t/3nvJeD7xXi/l8tfnp7ulVOsuNV9rlr8+/TONvl5PffqQfXj6Nf3vp/PzjYnOxq0M+sv0TFun96uhLC//r1w+T70+rv1z/t4fvSSrG8+IjzrfR7LGwV/G3rn6UBlzMH+8naX4RZ+C9/P51ukqvH0bj/NXvGWWy576u7mbZIzf7cWuj/L3L1WL+z9bq+ctf5vergiJu/o7it6eLVfokwdXwt7pbADLmpvO7dLX4kX2uuEp2nWBzoYK2IvSLK3/fkWCL7Nc9AiTFc6OCd7fbi+9sm/1QmPcAU7unaWo3iciZWpyoqSOPnKldPwS2TSeZBy4eprOb+ffXuydeVg0/X6y+zm/n96PZr/P5Q2HPv9PV6kdh0NHjal4FI32arj4XH89//jP/+YUIiofDp73Xhj/KB/fZH7v+1IugfPjn/mu7j60flZ+TkB8tKkCn95P9h1VmCAUz1tbIHuTcmGZrVP6W6Wz2aj6bL9a28yajNP4y3l5s75VwHKc3X7b0yo18BLkyoOaPi3FaA2mxMmZ/7G26qnlfrCbrIp2NVtNv1S+n4t36o5eLxejH3hse5tP71XLvyh/yJ/bugbjcWJT3QOCGEos319xxevvlnkPzyATNFfxoZL4VxLeX5QkKy5NIZrnnmmB5TJXlEdO8V5q7Dg7PY5nnvhFvnhDi+W7v8ufeS8zzfniu2WL3y3Ph+BLPQ8cxwPPAocRzprk5motn0vx5h8LApcS7iIlnjngeLvFgpAeNeO4e6xo3sEyg7d4MZYEWIKxgZCMaeFT5akdAzWKeByg89wDPjQQWAp95fqY8D3F4DgILJsLEhVH2Ek/D6w+L+dMPwP4dwXM6NuSfBj0llIQjpIRS4MCEkqtKKIV9JZRiYMKBCEd3uUXub5b5P38Mp8t/4AEju+Me8h/HP2bTzLSLZrPebDD49Wb7xGj8z+0amfePq+wqafG8dPd9CfL/K+++9f+K+3fv+c3/ekTSjXyvimQYQCRDBZBBX0AmAMg1cHCDft7AxUCpgA2cC0Uha+TgVvXMkYvJIQc1Jmvk4ObrvJFL6CEn1MgFjFx1wxKQQ85TIwdlMeeNnCtvNfGR89XIQaXHeSMn6CEXqJGD6oUzRy4hh1yoRg7m488bOS+khlwET9y364qB2Zth9vA+zC2YPTOZZn/4cLpIx6vp/D57JR0tV5uA2BrPu6f1x17cPNzdvyiefPmQLqbZ18xRHmbfZfqwTD/snqoG02T0vqThWBkKm0TJjaNQ7L5YhxS3jFiuRveT0WKSv/HH3c08Z89tep8uRrPeNqCxL8ObQHg9Bbxeb/DCcznDe/QuFZwv0OGFh3eG92jnDDZE2PC65ZUZ3y5iqZEcS8XHl1ff7vBNQMgVH1/2zx0uvwTxhSFYxvfoYy0IAuLjCwO1jO/RoUKC+MJwLuN79P4ZhKXw8YVBXxL4thD6kMNXkdLGxxeGhhnfo/dXBP0zDCAzvkfvr+j5Z59ofMNGfBVaMXx8icY3rMQXCpPw8SWaXrASXyhfwseXaPzKRnwVggt8fInGr6zEF0qh8PElGr+yEl8o3jCI783y6vaXt2/ejd99//z36vNPf/37NFS2+QtnOXC5sqWCe/jv47x84WK5LkG6zN7g+g9Puxezn27zf69X6UP24uC1GMT+IAndzZuv0/tJ9s/vo/vbNPv3zfLN/SR9en8/y630x09p9htfbeyc/273Iiy/TfbHbb7Q5vKAkxlS+fN7op/RbHqb83CcIbZmU1n7dVm8cDedTDYlZWn214xu1pfKOVKUEGXXDV4OgmF+rcfVfFkUXQ36lB/LzSD8UrWzzw9V1Y3oiyBQOA5sb2PHPNgHMnGgqXvrmKc0tThNU8M+kPimhtLs0zA16AOJb2qFDN6y0scohDbsrfRRaUNFEMRQK81Ka55tDXRDQfRgvxy6qKPGq4gO0njiqzaKsbjx1iLejXE7qYhW3wLFGWm/JLrmXoHU7LnXGpA1xUG3PahqrcIFvc8M0iYu9E99KcDV7gmunva4J1zv1IJgvXungLZ3ApL4ODThnTSVRJDq5+2NQEof3xsZ6ZzUkzeK2B2FtN0R0AjHkQl3pCmP49YLDRlOfHdkpD98Z+5o77i2bYpJaQwCBQ+l6HDVj4d6XuAF9pzinh9tkub4LoNSs/3jvAimEyEYAIrt8BiwxRr3mmkjc8X3GKTGFhzlMSJCLoPCJiOxw2Vomvlxk6MGZQ++yzDSeLcr/+DuOwcOkwwvyvwj0TAJLMWMYwNhktIq3Lmr3h0BISm+O4KbTHvcEXsjl7Y3Ao1ZjGS4y7+Du9E1eCO5rAzfG6HNrj3AG3Ude9U4t0b9Dr14ixBG3VGLIQNRJFF189WKz+3Yerhnk+8dM55NqD0bd2tsEPyjezZBtF7Hyn4VoJ1fotLrm6znENyOpDN4oS4SHV7u9talcwbbToPwvv3n8d/ruy+j769ePX0cffvP7d1/3yqrsTTrqRLBvVW0jNH/OrpJZx/my2lBhJv5ajW/y94wy194uV1aK2Dm/xvAiqlVvpmFhVUNvClAfjW/v8/oWG6Rt8ivvi7m36f3t/vI3z3OVtOH9c5aUVnRotZPsSst6wUNrvXYdIKbNKaTNXSCSUZsOrUommI6EaWTomoAm06q8g+JTjbW4IHKUk+YrMFTmhpq2U/C1KCylICpoU73NEwtV5YSMLUHTYsnetmP2vLo2Oa4rRJRhbBW+b5EzdW+lf8lu8tbwCncT1ehVjXLCQ0Cr2YnXCZ5nyTf6LcNCLvUtOO53GfKO4Ezf367TzDqXW2fy+05JptT6BdjLuN/7oHcEwZTg+oFRyNX0Yd5zhM5ueSZAHKaUbNcHVpfHUoAOU2bAS7Sqy/SI4CcprySi6Xq8xgEkOOalaNqVvCR0wlqWd5fL+8ngJxmh8JS6HopNAHkNDsUlnrWyz8IIMdawA63MWADip0/F0Qnk9kIL5B6EoCX6GAyG+EFUk8C8KrCppoFldVUQ1pqKsVij82n8sLMJxv5BIMk6HxqMfiB+USUT4rkkkE+xcl9NP34n7dvrj+t/v7z/ehT/P66i0Ezos2gGVEZNPPx9bcPo9X4q5yRJDlWRpYQVOjWn+spj6nbjLKAVHEDBVe6mDmj5Mq5zJzx/AiaujcNpdLU4jRNDZXB+KY+l5kzBExtvarH8xQ+uC9Vj9ox2NABg0fO6OBTNBzV3ynGldQg2uZ73TatqLMJC9WeeZbwY+ic+kpsKIHU5O/1B9TzBA7o1NCBc+EpkHVqbXRq+MhplBesU2vQqaEjp8gj0dvIGR8jQWCL5gpDe7Tn3fgwQsAyxzYRfPQbXycKZ5ljg8wRHznYGoFljm1kjvjIacRyLHNskDmiI6dQU9HbJh3dovXk412qqYF0Al5Qf+Q7BgJeuqGBLN1tUPPgeyNW89irvoB8ChQpdaPqC0XhDPPJGj6BsyU+n2BUgPlkC59gRgefT9xb8kA+6cT2ih1XKdjvzz/JQW98PrVoLsl8osoncFoyySd1yTjTyVo6QbWJSTq5y+Xqu3hz9ebxsxj/fPf3j3/+M27T591GKaAna4lFYrJJqNLUp6ol3g4UomPqFmcCK00deORMDbfLtAWusdQ6VUQm29YpTWjV+Ntz1rcq0Ss2iPvhfv2NYjzcH4Rxle5dD2WrswnrWw917xJYJme0KYG0WXp/TqNwleAp2uwS8kyhLy3EcWjAM2la5ekP0efpiUJwfML2RDZoR3WeKDp7V5SQdkWRzPY4MuCKNLUjXIEguSJ5gDG6Kyp/vyW+aO+Mtp2VYc4XNQvgCbinsqakd//0PN5x5Uu7gCQ9jwH3l0Qm57R2Ipg+ROZcPE7Vfd9u4sAPHBMOw7PCYWjqLlArZtqC10fsNyDnGAjNNjrSMUSUPAOBrYSpePDzeKepDuGKrKrHkAftEfAYVoVkpeoQDoS4pIOyniOvkHFsIBKiG2DCVYaSqIWeN7IqLCt5I3ZGpMOynivT3UjuWjeThwtnJWeUUHNGwoa4bNfh1WPrcOkJaYSpIKzO3UiNOnfCxvIim29WfG5H1sMdm3zrGHFsupFVXINbdWxuSM6xwZwgD+A49kgPFHGJwYbkanhhEJDhPfruBdsSdHhVakiNx+WSnSGtClXFaoDOJxgtYT7ZwieFxhedT6puxBKfbKyWATVgyjmIhguTWszastHWoAiMgK3bzKGy0tZyFRgFW8NSDrw89n4spgiEHKxw2QVMUMqaymVIv5QNeo7GeMWhpDkaE6jZ2rtmV6rML1s/9BpD8eBJHI3nLvPcJM99pIys3NHEDM9h1xK7yno9RzH6x2xZr8dVkJ2ckLztVgYr/uppulfqz93niZxcNUYAOY2yhIts6otsCCAHoxJc7NCi2IEAchoBBavOqSDkw8gIq39bqH8JIKfJ4bNSsl4pSQA5oUaOZWX1sjICyGmqqFg3U58pJYAc3HKwsOLoY3koH8sVjdOMJi59uF9heI++e4HfRYeXdQ726hwUqwE6n1rk3plPRPmkiApj86nctpyavgDqZlRDiMzqC4JzaZ5MwdbwpHoatga6GQK2tr19suf50IZm86wB51m7WVH9BEJp9LwecJ71uDwrPnKcZz0uz4qPHOdZj8uzoiMXwvMH51lpIaTJ1nGetSHPio8cPANxnrVNnhUfOU22jvOsDXlWfOSsmoNzaL+Nk5+DE7ZtfBggdRMChC9OUb0K6UNNo0UWDzSki/DdEQxGaDHjdNGQVroI8mkb/MVKF4UtyqyZT0T5BIOl6HyK4PGf+WTLJGC4G8HnU31j4d3+GqcfefkK6UEF48fFt22Kjl6ta9S2w17kY2zRRRBL47FDp7JFh5+IRVD7iX429RGM67HrtcX1QiErvuuF9GEh67HwxqAIFVsoFhFt39hiRaIHL6x3RIeX2zf2ePfiO2eiRSSncffiw9uiysBGtSNU8UaKjihm1Y7RiXYahCpeArY+1U6DQMWLb+vSqdmr4g0VbtisijcWCr6yivfwwHSkELWbndSuUWDoYyTniRxQ8eIjB7u+sYq3jYoXHTnXgacU7Oi9W1HVvHAaGlRmD/aOPnJUP652rSyuVhPZXz+SL2gy3I+vyInbKnI2a6/xcH8UJ9Jt5DeE+4OGT/QT7o81Gh6WyDcc7tHdYqwpKGKJPBmENIVDLJFvkMjjI6cpHGKJfINEHh85TYNGlsg3SOTRkUs0BUWsJm5Qf+IjpwpvaTBjyciQlvoT8ilWtLYzmrZKYJCN+WQLn2DQFp9PMPTHfLJF0gbVxPh84uoZe/kEJZLofHIdmGO9HK8yk+hY9Xg327yhTwLtsVTJjXrIr64cRwe5sXR6IGmufUVDJxXUvWWCXQeechjqPqCOytGfeFDzscjibSwI+BNYJvhcZPG+A6h/CRDqgI3sl1n6dLlY5LbOk8nFj8PxbLRcTsdVctXI6VSp5s3vPDJXDA2+Z86gJgx2WAIY5mt9KfadbSwloDZJ7OJzO6wUqV/piCMiGfNNmhtcqqsssOuolMLhbFWgVaFE+O/jvHzhYrnG8TJ7g+s/PO1ezH66zf/9Y/jm+m326qfL6+y/L39///b1u/LK2VfdXHzzVkC97A5cVYkFfIvsgu6mk8lG/pFm32x0s75UzrEiYZ9dN3g5CIb5tR5X82XBQpM7FT+MJN7EcKfiqgK4oj9HcKLiZS92pDsrcaDTNSuodR1k9TKdnL0XhPTgceFh/DTuBU9esCgYGx6HSavLPVnYKiJFiZ5ZdbnrQoISbi62fcDNxUr8inNdo5axuF1Mixk9L6iGTLdaq16liVu7cNnEoY5egitxoY8yK+F2rep/KLkoXA/VgmH9e6i2amssD+VLhXMiDo14KI16Wh9GOU+P5INjGL5Hqm8JRdsjReyS3Ii4SwpkzseREZfE0yrauaSInkuCImXKLmnv6IbQjk5RqXdsYKtPL5UY81LP5J5GN49aB4YZmaTnHsqgUr/u4Zj+la09BqbDIOgdyiuS9w7llbnWsIgC+/S8g5EgcK/eIWL3UIXUXFz4mdzTdEVBLXRFdA+RnCQi4B6sCsBKA2g43OEK6hHYWF4S49hEuENoIrBcqS0JYQi6JKsisJJLYo8kqAdgE5nyZrLWQhOA5Q4EkkeK6XkkGwKwXcdRjx36R1BII8xFW3U+J6zO7Nh1z21Qch/u3eTbx5B308SJuUtH1bslATnv5sH4MYlm7zb28oeSuC3gaNU8HtTBMb5H379gd4KPrwCuV+t0ufxvSKueVLEg4BOK60ktJhRU+eITqkWjHStLZ+QyMk8odnOGS2c8GPk0amxC+Ra5jIwEPC1qq628F+QyMhLGhuF+vOT3fiSnCKMcrI3ZhVtQiqLwx7m6XutYzibxhKDvlcazOkUv9X4DMJ6RGGU7qrtMdcNUj1GoHkc4VIexRruKgz1HMb7LcHFwmZbnIsrnHa+87ZYGLYDra1SY+lP7eUInV5tRgA7G7rgqp0VVDgXoNOJGroygA5FmthvL0+lApNHpsUSYDkSagVIspayXUlKAjjVnR2nOKECnGTXFgpr6/CkF6CBGLLg4+rwdyOdtRWM1s+nMkjmMbxf3L3C9+PjCeIrW6bL+YUhM/wAXBHxCwSgPE8oaQsGILzqhIhgf5BXoWHzlHnEU8IVBRBL4tsiKEsQXhP2x8RUO378dLvgg7oKPL9+/3eELgzP4+CJrLPuztewrfcUQYbMiPtFmGs25ClopwKMaEXMK9wIQtBIwtmIIhWXSJ08xatGs9Ekohkuw9OmYg7CfQCyNhtpFOeKCpU/10IEzEAHoNPoZlj41SJ8IQKfR1bD0iQ5EGl0NS5/oQKTR1bD0iQ5EGv0MS58apE8EoNPoZ1j61BBdIwAd9xI6LtOND10Z++FMt42ZbkCowIOEMhtpV7SQZkJZQygQMSJAKKKpOStTr0A6QQBfeMIlga+VqVcQNiSALzw1X45XmS11q8Lj3Wzzhj4XgL1VRunb61321ZXjqO7wgclsV1nPuN1MKhI1Kqj7y9MoOiYz1F1s86RmtBSghsdyEl7bylUZCKIIeG2iJTFWrsogZIOPr1c/RmbXXshAm/TyFdJT58aPi29bcQa93kNbqjT2HioOVJCtPfce8iJpvxIUwa8r3Sd8P6j9RD/diraW5AhG2wiGbmmtWZ772xbL5YcEfC084Tb72oq/OWfHS9HXuq19LU6ftySp95yKTwgcX8uVmxb7WlAKTMDXnmq3XiD0jxQtCQ2Lmz1kJTkhLQwQ+lOAByoqTuNeAEJ/Csa2vsdpqHDehoX+isaYLPQ/Jm0bKSpfzApLwvpO7mYPPW5lKt8Lp6HBdfZgbzvX67A+50VcbaJdfLma89X6kfz96A3xE2Wr1OZTkq+h93OH+PmyPCaMJb53NcTPdxLpBgwbDlxeVP+Jng5cuv61+mPXeXpUkEjH96i6/rVcf9NQf0MAOq6/qa+/IQAR19/U198QgIjrb+rrbwhAxPU3x9Xf4EMX1s8jMntgi599YNs7YbmdnLB6TmtROLAVDGg+sG1YaD6tVaaLWp+ykgDnlMW1bMcJo/DdYHllrmWrhw6UHhGAjpPJFpceAULFir7wZpPJ3AbYZkKBpAgBQsEgAxdIdAG1XAsTtRSi9JfKDLjsyUwtDD7UIdwy8jJhjYgNCIbxl4mQ9x0WEwqoIgkQipsyWLyRBVESAoSCCU0mlD0eSs4+ECBUiwbtTCiqHgqoZQgQqkUhABOKqocCfWrwCRUdEF3+MkufLheL3NZ5ZrD4cTiejZbL6bhKrhpRvSpvuPmdR2YBocH3zBkozFk+90wlppCVmCIQElCb9CRQYsKUXiDthkTkS5fSiDo7y+5FquNZOFu3PXgY3VcoEf77OF8VMF4s1zheZm9w/Yen3YvZT7f5v8PLj5evXr/7+Pr37C2fLq+z//76/vpjefHs226uv3k3YF92E66q3ALuRfZCd9PJZFPan2ZfbnSzvlROsyIXm103eDkIhvm1HlfzZUFEkwEfx5GpE8fAFbixgrwyxY7xBReO8H3v9q9f/hxPVz//9/cvn65+XLSI99hYV+OH8l2aONDr9lZXozR1C6d7HhVmvu+SA6dFmMrK+8CNyJkaBnBIV5f5fiiv0ooZYL1VlylNWN/xJp3dzL8baHazFa819FwYVCRrokmydkrqNCV6RXxgX5umv1FMK9N84VSTCCIOutWZ1dmEqyUPde8SWNt4mAHpkhJIqGC3xzWdvWeKaHsm+bgcd6yArbMJVxnWeyIBDlzYnqi+CIC2J4pwXVELgvXtihLSrsiT2R5HBlyRZjYJV81WXZHcUwXfFZW/3xJftHdGQ+iSt7ZG9qAM8JJ0T+Uc19790/N4B2OO51ys7fv0PAPcR/bgGRS3VHfOglBHTQqOwbPCMWgacJxpiwB/O72MjmMwEuPt1TFE7BkqiJqK+z6Pd5rmIWfamcKXu2ES8AxWhVjdfR/AgY3sfbSDrKG8EpaS7V4jGzzsuKWshZ43sirMKnkjdka0w6yRTHcjuWjdDGhueVJ1RpFHzRkJG+KsffXflXxbo86GnjcSpoKqGncTu9WCbLFtxt5xk10/lm8dI45NaMLB3BCo6tjkARUEHBsME5OYYWjjjEqocNv26zFQkaOGl0sG7S0ZVIjw0Pl0oqODQFmHJxSe2XDlAE8O0tV1UEDnRAcHgcIOCraG+0i8lNX+caw4Cx2czN6dmVD0wEbHQ6rPWa0PZELN1t5leFIHLScOJBr3ER9y6gcC9RhmqLCaSd0zqWnI3D3XhLZUMb7UrrI8z1FMzjNblucJxWaDq5gOPkB5230LVrzF0+ij9Mfy80ROrvoggJxmPhGL5OtF8gSQ0yiPWKxMBiHN1CJWjZJBSKOjYfUeGYQ081RY6VSvdCKAHMtCjpKF4CNXdlXjvHc9cnLemwBy8KjMee+jD2uOfMxWNDIymqcsRzgzvF3cvcDvosPLrbVPSNZAgE/woMnu4ujVQE4z4MML92MM79G7cHB+woa33CRSg7dFSpIgvOQW+zYjK61U2gDNma8YD2pWadNmmuO5as4IoNNiRJGVdwLQnBGwNTxiWCZb8HxoQ7OyBcV8R5YtHHNA2uaDsMJlATerPE62gI+cJi3EsoUG2QI+cpq0EMsWqCCkGGrLsgVaCGkKU1m2QAYheOJk2UIb2QI+chopK8sWGiJq+MhpBJEsW2iQLeAj16ImlBOf1iQ+Aw/yyWhsPYI72MvxKrOIjlSPd7PNG/rkzx5JldSoR/zqynF0M1qNBTQTT/IcimCcCuneYnERdBQkkmSnkQPFv5FhPIcEvKeRA0WH13Xh3my/MnhX+rtfHjx+XHzb5iV6aklWvmJTN3cCtcJR257Nrqsha9/FwkJawcIi/KYbOu3HflD7iZ6maHGHI3tn2EPlMLqnjU61r0sYSKaO0HPsEcxwGLU1ocCs75JDJ4Znw9O4E8AxnICt7WuSIdkwRFebxDCTwGqTNrdDHEq3A3aUM9bEp/U7u/NEToAlHR05TSE/q01ktQk55DQKr/NVm5BDSKPkOl+1CTmENIqt81WbUEMogWcaVpso1SbUkNueJUhEwN3KVI4XTkPHzOzBXnRPjozH1TaaxdVqouPrR/IFew6ZB2k88VUkjcWNZ2Zwe9sBqJvjg/GIeeRH0j0TNkTMPSep/UQ/EfOEdVsts4D0fCCMXbAPPCsf6Lf1gQLFB8au7AO9Bh8ovPpP9OQDNc08WQEpKyCp+cCE870WKyABn2LsfG/SYqIN84kqn0DuAp9PMI7Kitouch2OK61E2IraBMZjGekukI48aki30KnwGkFWY+ZSWyNcB4ahmVDWECqWA0MECNWiRxYTiuouFkQaCRBKMKHs9VCRnL4jQKgWvcuYUFQ9FFCaESAU94C22EOJmB6huAl0l/hWSyjxR5e7Dkd6rV6B6BGqRakYE8oWQuE3unYdmH4msQJZWcQPZw6h4+vy0KEO8SXYpAH6fxL4nsb9SwBfPnKe0oJPgFBQc9us5zznRi4VLSu9ri6u27aty6Ydl3F9pkjkHi1Ogz4ziuo/0Y8+c2tIdrV2RvfouVoY/WG1ThdQh5J/QO906IoDtmlfZulT5r9yG+erUfHjcDwbLZfT8aBt/xLVWvWsxQYafM+cgcKc5XOHLSDA3ws/rOrxsxVAushmFSw+V7d0eNJUHxFGEuabZRJcqrNVRKhWkXC2Pms9jO4rlAj/fZyvChgvlmscL7M3uP7D0+7F7Kfb/N/h5cfLV6/ffXz9e/aWT5fX2X9/fX/9cSBerZHKgBKOuNj+suzbb37f5tOAjdlNuapyDawssv+4m04mm+1gmn3Z0c36Ujntij1Adt3g5SAY5tfKdoDLgpgGXYOXCP+FV9XyiW1J8H6rHV/BZ9GbezjVKVpBIEVEROJCV2y2r5GrGKN1GsYWCUFjt9g0W2lsByR/CBgbbihJt+wKtvPgSiNGETSi2ZZdbgAbL+wHQNLZzfy7gdjHNtzREPsYVGpURVON6smXo7qlLKi5je3mdjEd8AgcSWciYhMl9q5i6hr3omvl6CW4EgF9lNma+3IMm5Uuij1USRayHsqVZgmIODLhoXSz6/ShkvP0SC44VKF7pBCuGvZ4pAjXJbWgWP8uSRB3SULmfBwbcUlC7ZK4eagUenDouSSIEWWXtHd0Q0hir62RPShDukS9VNtWQ8/3Us/knqaXj9HOtb3d7B7Bmx0WSvRwsyvuku7uf0LDiEjc66Et97qmv7jRHsj93etywpTAvW4kQNvrvR7xzV6F1FzM9pnc0zQ8N9pOu7eb3ZfTMQRudqtCne7+bc2Bhex9EfVYZyAvcHFiIrCgmHLMXd9VAhJ6LimyKtYpuSTOvpRTp8l6pFCmvJn8cGkX7sFe75FCl55HsiHU2XXEUuPgGhUwFF2SubimzudIRThiO3+yQRd9sHeL5NvHkHfj7urtvBtokoLv3bgJTnf4AvFZgt6CIOImOPaWMCvkcfiEgv3OAaGs1JzL1RSeUDhnw5rzUx1OD6opSBi7RddvK40tV1NQMHa5EyKRZ9o/ZhVnnIOTyruzEErsh0AvhHIee/NBK/bUhO1d5iaVLjvFwazX09F2Q288gOAyr83yOtE44p55LZ/6PdeIory0i7UFcZ4Tw3XQcEFcrCpO5sKhg09GnlB0HDEbSok1UTD9gfs8oZMrLChABwupWYneQolOATqNQtCoGtgC6GRdMQXoYCjJvLjTBugietBRkOpZAJ0s+qMAHQxzsTiqhTiKAHRlOI5VJPXQySoSCtC5aug4RV6fIqcAHTzAcYr86HO7PAdIKJoSmc1oJtzl+4RS5BQIdapZW5Ai9xNoa8OJxDazmq00NkiR4xtbtJljbKWxQYqcgrHhltGy/IqnGNllNr8iHEhQeuptbjiox68IwTcmX4vbxXjy1QFu2jeQfBXK8bqcNzx8uxg4yOdL4WiST/pTyHlCB/KGBKDTJJ84b9iQNyQAnSb5xHnDhrwhAeg0ySfOGzbkDQlAp0k+cd6wIW+ID12ZU+G8YT10IG9IADpN8onzhg15Q3zoFPWZhOMXh5aLn378ohwRTDZ+AcTjZuIXpV04H97gksBkRXSX5LaYp8X5UmvypUHL+Y295UuFYsw2j+rsAuqoOozP8xX5GKOjOoXbIjXOvoPomF8o3iHgO041/R94kq2jslQMLyMtYAjgNIwtEoLGhof20zA22IFQMDY8jlDXWkhGDH1oRMNaC8E56eNuiLLL/faGSCCWZs94ipHqnJNW5qRlX4YPHdeytsxJ04OOa1lb5qTJQRfBwOV+nH4XiN8P1o8fF9+2O4kOI/duJXT/wmno/JI92DvNyt1g4mo7mOJqNUH99SP5gqcf6RdFIrQ50i9w2h95Zb319q4pprhd6T7huvWf6Ck3ILguvKW+g5wbFFwX3lLfQQ86rgtvqe8gB50Hoxes71DqO+hBpzmtcR5czoPTg45ba1ucBweEij1IKLO5LK9Fa20mFFVCgSguAULB3TgLK7qAOnalxUiRfDErrPBbpGbZd5AVViTkfIffIv3MhKJKqEg+6RAgFDykMqGs2d2AozMBQnHfJosJBTLnBAgFo9Tc6O1ofMu9LpnRVcLn87XVDoMeoVjNfkKEwu8cKALeI58SofDLI4L64dVqWVJPNcTlK4fNn9qWFJspIa5IsugNo9oWUDWrjDbOzLjKyHWkorOyqF6rMvLdoPYTPamMAj4+Wh3goudr+fjYHb6hFB4gsNsP+Pho8eYsdOgRCuqz2GF0tSBQwLdFbTI7DLIOIyZHqJDDAzYTCkwlQo83hTA8wCvQ8SuQIIcvn3FtdhgxPULxlrU7fH0wkhUd3/ILML594Isfo4p4B2nzgiDXvRMgFO8wbCYU6C6KTyiuqrI4LePLHsqkqu/q3Zd/5u6fi79vF794by9+EaPl9cWJtqzzHCma4DuK8FRvXdSUpj7RhnUiisiZusU2zkpTyxNGCZgaNrUj3RlQxEJa0xMBTdhbZ0A1WyE7CQ8x2D44wyEGSviKLf6+5Eh/p5gWHOWbWMllON3Kh+pswq0uD/XvMliKkFdfbRyUQPLwxVbAyckrfOB49GI74GJqwGn6THGTS+nUQQ443QQ41L588ThVJwZu4iCfd9HnTjsgh5Bm0Bu336silwhyyMETPXffU4W96CEHAwTcfE+FXEIOOU2Te+69V0VOLuUmgBw8tWkx4yTRkFbWEfLJdSGfjCaJXE462ssnGFTC5xOXCtmbxPZceaeCzyduNGEvn2CGAp9PMAbHjUG7QDqoFr/7TnnnGmgMqk69ttC/sOegurMBMWB0z9FmKiXziepKFMtZPHw+tRgmb6OSyYsdOWiCrmRSDHU0ams6qRsvACEtfHRaxLSsvBM8ELLHtzWM95AW9Xmgdhhf1OfDBAeL+uwR9anGFdbcK6ZlfZ4XyEoxYUDWpxg9yLq+Vj6emq5PN85Of1w5T+TkIlV85MqzBiv7GpADZSnoyGnkR6jSPsxTBj2EhBqhM5XweaFPDiGNnAhVwoeIkNzXjQBCGtkQS/WkCBg95DQ1AyzVk5AjJz33oSaPcLBh27Ocgw3FjUc82JAAwpuoIVTMe2X9qcIdJeTqLBSDVbWYcRZ0SCurDvmEngX1uFOwxXwCwU90PimmAbO+qwukfVnfpUjDGdV3Kcb0MtJdrBFhQA1pGLrjNcIW5RVMZ+KvES2UV8wnqnwCJ1Z8PnEnUIv3sCDkh88nrvG02T+RUxr7XONpsX8CMhV8PnGNp8X+yZcbjaLzKYAxG8AnG/XffgjC9+j67wC5VTEdfYXv0+sj3WYKupV3gkuvkXRgWXtj35cqdwhUQkRQm0VPnNCxxuDowgp64oSgbX/joJMGx0B94IM5QVtKlxfZfLXiczu2Hqpz8IUjy/Q9AzoH5aR1LqposVxQK6qIoE6YnqfTuaazV2EFgVFHd7h3AiosEyVfAXcAb+WNBDkVVsAtwFshBwuw0ZHTKB/PtFDI9+khBBWO51wo5PvkutcGmgLXMy0U8mFLD2yEyl/GhUL1yIX0kIMRSC4UUiAXgXFt6MhxT+9WyMH2VOjIsd7D3vw85BN6PjVkvYfFfAKxT3w+wTMrK+27QDqhVlMRwrMvI91FBJFcTUXIdXf2arhgNhN9jYjgiZ/5ZA2fYnKawIi7bVu8hwXREXw+wbgW88ka/xSRm5sVcQ2hxf4JZPTx+cQ1hBb7J6DfwedTixodG5XkQeDLR+lSmoemJI9a9ESy0taeQ8/WLeIWVtraASEidFvH8ExPukIi8IKqDb3EgzY0WyGRWNW+8dCShpMXDsfFGxuFw5t7xbRwOHAdOYVgoqwhVgVnuKyh2cdHMlox9E9GpQcxDIuwBFyBnAt2nejIabqlswS8ipxwySGnEWqhSsAJIufRQ05TdoEqDaeIXEwOOU3ZBapknCByfkAOOU05BkvJpbgYPeQ0ZRosJa8iFwpqyCUw8MNSchVyCTnkWDZhb1oS8skVkE9G00gJjA8wn6zhE4g34fMJRi1YYNwF0jFob9MO6f4SHyxosFfQAPMK+J6Di+Qs5lMkn3Hw+cRFchbvbMCZGZ9P3BTZYj6B/B4+n3jQnsXrnS/nrtD55DowqKcl1JdZ+nS5WOS2zlU8xY/D8Wy0XE7Hg7YCOpXGZ/M7j1TsQIPvmTOoCbQ9t1FpLAnGRSQ3Kt1IiUCjUijZccKmS2l6nnal3nEdVVAnnK1y/B5G9xVKhP8+zlcFjBfLNY6X2Rtc/+Fp92L2023+75ur7CVxkV/78l32YWeYfUPn0+X1u//7mP3w6/vrj+Uvyr755ndtPgmYmN2QqyrPgKuRPdLddDLZaOjS7IuObtaXyilXqKiy6wYvB8Ewv9bjar4sSGnwmJ7RKKxiH5Z1bvsiT1dBZJkjx/iFPx6S65+u756+pp9/+/9Pf82uP8WfL1ReYQNRbpVWXBAqLlyv0ofsxcFrMYj9QRKKzZuvM3eS/fPx9bcPo9X4qywb26PH5tdTpEfF4/XFFRFL4mtPVTPuqpxeb1xRyv6YK+hc8YKEHFf0awxzBZMrvh941LiiqqVkruBzJcjz58a4kj1czHP8dpvd/FT723yS5u/4Hw==</diagram></mxfile>
\ No newline at end of file diff --git a/ydb/core/blobstorage/docs/vpatch_actor_protocol.drawio b/ydb/core/blobstorage/docs/vpatch_actor_protocol.drawio index d3fec9c4232..a1e5344132c 100644 --- a/ydb/core/blobstorage/docs/vpatch_actor_protocol.drawio +++ b/ydb/core/blobstorage/docs/vpatch_actor_protocol.drawio @@ -1 +1 @@ -<mxfile host="drawio.yandex-team.ru" modified="2021-05-13T12:45:24.063Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.86 YaBrowser/21.3.0.740 Yowser/2.5 Safari/537.36" etag="wJ5mePPJEHiOP22e9T2I" version="12.7.0" type="device" pages="5"><diagram name="VPatch (datapart)" id="13e1069c-82ec-6db2-03f1-153e76fe0fe0">7V1dc+K4Ev01VO19COVvm8cQJtmtndTNhsnszH25ZUABT4xFbDOB/fUrG8kYSVgmtmyTYR4ytmgJ0zrqPmq15J5+s9zche5qcQ9nwO9pymzT00c9TVNVx0L/JSXbXYk9GOwK5qE3w0L7grH3D8CFCi5dezMQHQjGEPqxtzosnMIgANP4oMwNQ/h2KPYM/cNvXblzwBSMp67Plv7tzeIF+V2Ksv/gd+DNF/irHRN/MHGnL/MQrgP8fT1Nf07/7T5euqQtLB8t3Bl8yxXpn3r6TQhhvLtabm6An+iWqG1X7/bIp9lzhyCIy1SwJ+7EVafGZKrZz8ZEudJ2Lfx0/TXWRU+/vp7GMMQPHG+JktCzr5LL9dL/7D0D3wvQ3XAFQm8JYhCiT3xc/LAvG74tvBiMV+40qfqG4IPKFvHSR3cqukQ9GruoSpjd+767irxJ+q0KKgnBdB1G3k/wCKIdcJJSuI6Tb7rJAJGKJh0BZripTNdK2u7Sm+Jr350Af5j13A30YfL1AUx/UBSH8CWDQdLQM3rGW3fp+Qm6v4Jw5gYuLsZQVpHWhq7vzQN0M0V9kf50tnNwf/0EYQw2uSLcWXcAIrWFWySCP9UMDBw8sBx8+5ZDKQHpIg9QIujikTHPmt6jA11ggPDBMvkR/Plmju7ju+1LaLzOh/+/ur3SWcDQQNl3gyIGAKukoxgtrTXDPFCaqvK0xipNk6Uz0vA5KU3X2laac4ZK447PBpVm2OenNKttpRGyck5Kc4zmlMY8PMcHXEhDd0iDqXWNNLRglfRTtUaTBrNlq6Ra56c0rSRpMKWx0xboaVWlGSWVpktT2hkOT2NQTmmGNKalnp/STLs5pTEPb1xIQ5dJg2V1jTSoGosY2SPMOFVtNGsoOcKkOUC1hVlzVaVpbStNN89PaUZJfirNAeotzJqrKs3U22YNZ4g0q2T41JaltIHK6AjM5mCMb2EYL+AcBq7/aV86PNTiXuYzhCusux8gjrfYr7rrGB5qFmy8+Fvu+nvSVN/Ed6MNbjm92ZKbAP3erFJy8z3/yb5Sekdqle/DCK7DKSjhM2M3nIO4SKe4xUSRhZgIge/GiBMd0ApeF+OqD9BDvyLDkk2xCnpk7R4UV9rj5DoM3W1ObJUIRMe/xuS74dsj4tmKwDvl7YFJoXr3wHuMZxqsAHsOY+66rSCKERpYs7qtYB6eQ30uM4zuzDCy6UN3ZhjNDzDnVK3RA6xlAqO1EGCrqjO95FqJNKVZLUzKqgOtufi3/mTe+NvxS/TtQZ1/eRuOx+aXcxycjtWczljXzSYZjMYPIdxsH9x4ujjqAgMYA7H2sLtSDXSd6Mebuv41dhJxwqgzl+GD5zipgJrygvmXlG1fWQwjb9aj1e+9uDNJXtReBtPhBJ9H47/WAN1curnWhKt2u1ljunn8AnwQw+DSz/Wukbfbzyyp+poa7eLJy6Wr37Wy0WRX8zk009tfPv3kO+lcP9J9N4FxDJepYt0wvk6yoFEpRK2SslvPz4JZwYxITHw4fSEiWM9OPnal9DWzWviKpHCLglBYThiDyvUU6cuKYSnKyqv00sIu0MaEpdjAEMUKVDoIeiS+VVeMiA0RISjtbMc46d+eZvnJmJ4gK2LN43R0hW9uOGOQ1kBENbUPDzDyYg8eBBcItD9TAktvNku/mglH0IMhkzwwXsq+YIhHy0g3iyArDrCWhLZeEtqn4fjUsGi2ZEbgOSiOcwrk5cQ52cmxCMMheF2DKG4Dw1w47U1nLXZTCEKdBeHb/G76v9dX/4cy8r8bYz1+XPxOHE3LINSpNQAyJT4GQoH8IQhZy67xa0uwyCqInsb2Qt0at+HC/iuY3HzXrtgFvj+Ce7B8BC4iRBzMIhITH0KsvLELkyiyuw8v455A7ZrDnjk6Tsn225R2dCtXPkj/ceFZOHppOpZt1sLP18tveOLRNKWvKqZ60HcEwBWd/ZXVt7WDhu3+wD5sBT4/R0CKi1ZZcn8TAhdRd03h8/sWzJhWzSDZZQlfNyySRdE/4vaOWSSBvBy3qLJZmYlfvEtQqvz26AbzBEF/IBxs/tN9CImhYXUDGjY9WRRAo1he4KxsxeTWFs1DasMYm8ODMfYIorXfCYZV0TQRXIkB6HQDgHQA8kTbpDdhmzQ2/pxx9tsEAA+It0c84h6tYBCxUeq2mHsFXDklcUX4aMdYuCEwbAJ5SbgqiGcU4arFoEbNsCJoEcOqGxEGEiGgN7McDZgVy0uCFd/NpbAaeclhDR8YUJxgQeF+67ZDVjRArAYAohfYHT5AOheC4mSrkmtBrmpDQSuN4zC5USs5mamV/aVzor90TpkIaAMqUZ3ORZI8EdDZzaZnZiEbj8GWtatkGbHteQW1UkW2QhydVxTLCye2/NpNrYvp/EUFHDxRd30edyduUtfGgdLoNcui1+4GeqlZrilI2BfIC9BrOvzaja3qsql8Hy0so5cNGZNtZm0DkIKEJVjEEsjLobGcfLHUi687avUqwGdQEj4kU7Jt+FAbiCxRVK9YXhJ8jpDA9ccxOwQOYtx0hLVRswpLkMAhkJeEm4IMDoKcDxwJNspyKaMbXIqeqdrWaTNbu4mIDMkEuGzm6+ZmPqdzm/lM+acYFSP13fvrG9w1xN+eJn+jsRzN2W2fUWqyjOmyD6eWpH3avrSetG8WkJxvMOxckPR98dC2qJHZDWrkUMmjtlFMjQTycqiRdWSWX4jEFterTl+aqhmFksDFoGFg8Yly3YedOM5pBF4gLwmlbNrr+yeFDW6DOgqzFnYp0YBySqYHsggwBQ1JDmhbdpuGRylteN5vYwiHbtmB2dRQJw7qGDAE8sJVvkHf5NZvDFgFmT3HrEzXVq4r0KuyiyhWN3J76OQvW5BTIZCX47gGLL36230BTysuktjC2Rb1MAMuudt8EqSRKWM98R1b5RsGnpdr5mBT4rJOGeo9pK+hk/4l14zM2SUuW/l8ryZ8W+mzArOdXJal2e9hQFWTBQcnua9jtUXOa99QA7vHFHZDYXOw37nKVmeN5QFenBbWUGINeZdM3iUXHW9VfkxpJlmfzXZHmo2MMcoTDAQxEYG8YEyqBk0pByoV0hOOSvIg1F52anDXMma5CayXpatOL13p9JynyaWron36EtdfCnH67sPuS56mXcfyS1Gm+LnpzWjwaEXuGQXy3ywkRW/8kSpHb/+dPZjT6Tfvnyfj4T5avejrp69NvJFCiuKcBl+Ayf0B7NI8OcfuNoQByy8vi6WVPFqTi6VcA8OOkw4fS9WprVNF/EAY5iu7zkXORVH6ikW23BIb+66Vi1PnFAZ9xI9gM4pA/ticolaqzwU6a9g2bUC4FLRKb2Mv4gxt58PSZzkKlj4F8pUjyHyawLV+Iy96ydu9HESs1zWMsV+42nmzaySgqqtN2rnkc2IsH3eL+TdwuUJOFLBWk402510ON/pcgxsyqJk/9xXSDs8PySJsBWvQHyBNgu7Tpt2SUdIt6WWPKshiXYZu02+7asItmdQyuSk4I0MgLwh1XTGnSh3+nFq8FndYyM+UPd12aFRqZ4NTFq6OSryyoaUoODmfV8JKDy9frwhB0t8epRs0KvqK5pi2auz+HjZ4JB3m5CQKhY9FqaSBn8jw65EGi/eGm0ZJA+fAxl9g6A843rwwCCZ97NPJ/qrdpzykpPHu1Lxpjq9G/pEzv96Ad4xB39HbHvNsSHx3Wi+ztI5LZt5Pbuew3UBqRCs3IGXJVv5d++gHIQkl2WSLv9BdJuwqmESrXGX0o/L1c8Xpg5DS6h1c/jTg06OU1JYOgxfOr+nF9+g2hEk37IcsYq2LezgDicS/</diagram><diagram id="fr3X5ynE2yK5susw7VPl" name="VPatch (paritypart)">7V1bc5s8E/41mfm+C3s4Hy6TOE4P6TRN2qTtTYfYik2DjQs4cfrrX2EjbCQhhEGAU/uiNYoQePXsQavd1Yl6PltdBs5i+skfA+9EkcarE3VwoiiybBnwv7jlddNi2vamYRK446TTtuHW/QuSRilpXbpjEGY6Rr7vRe4i2zjy53MwijJtThD4L9luj76XferCmQCi4XbkeGTrvTuOpptWSzG37e+AO5miJ8tG8vtmDuqc/JJw6oz9l50m9eJEPQ98P9p8m63OgRcTD9Hl/fB2OPl+Yf6d+B+Uz5/V4M/rsLcZbFjmlvQnBGAe7T30u9MHcDc8vfr87X715R3QDOP8Yy+hwrPjLRN6rZIfG70iCoIxJGhy6QfR1J/4c8e72LaeBf5yPgbxcyR4te1z5fsL2CjDxt8gil4TdDjLyIdN02jmJX/l/IEIP04wARHjVyWQjV98BwQJWS6BPwNR8Ao7BMBzIvc5ixQnAdwk7Zfeeu278P0UKWEOE2E8YQ1bsrNDhP4yGIHkru3UnAaB87rTbRF3CPOfYxg69TnD/frDL5s3QFc7NNk2rdFDR5Ljf/zz6UzTQ/X9wr3vSdd/70Y9lUBSG0CC+Alevyf3ry9+7F4MVpmrV3S1cqP1TX3LMJPr9X19Q04ut3fGF7s3XoPAhcQDQdJWDssbiGz63c1vfry/Ckbn4Wz4+0oFo59XZs9USdBTJ6Ap0CtZ0JsmH+iJgSy9YKDNL67MPaasZZ+jZeSkGG7QKNxgeHD6zsKFM8+whfFnGWuQs0d/HvXCNbBPYQdZXqy2f4TfJvH/Xy+e766daDS9AeESjpcMCt9xM+6mF2p+CPZ/Uj3v++gHL04wzn1PTEJEYBVlGdrx3Mkcfh9BbopZ7OwZBJEL1ftp8oeZOx5vZAeAL+M8rIeK+TDBBRxXPzvRB7DFcx6Ad+aMniZrOXPue36wfq76uP5QmZcp7OC7gFWGkRIzJnmPjKWQ4cLkLqkvmaqSwWdyVZFNe1rf0OztxzQzD+nJZl9Xs4P6j48hiDDeKMsNVH0sW2/SzJDNduwME0nOmu0M4jka287QVGb/ypKVSnTSzkhl4nc/GLiQjXekHyaFWjBIEtMCff+x8z3HptjLhImlcfIScnmo71ohT4G8/B1K+vLR+Pyrt/pwcflujoyLXZagWiu6xckSWamZAMgw95F9pWEuZc1j3WQYBKRgxYzrnqwLEKPUSSAXa8XQD8CfJQijNqBfHySpUNMomGRhlxeS0Oi3FDs7x8jmFQtMXcvKU13DVvTl+ucBuVagUqdGIq1f9fR0FEFzC8dhOHUW8dflzLtyH4HnzmPsLXbWVF7SvLPOOnuZuhG4XThrgLwEziKLPGiERQ68JUivPc9ZhO7WOAzAaBmEcCZvwMaE3YB8GcVPOk9dUVKGD+KhUieQtB535o6S7znG5dxf/6AwCvyn1AGFuGPozFwvnrw7EIyduYMxjUKzf8txEWGh5sJewcBkJZcvW6eZLCVt012HmSRJ+XyQQVZZGMkkjHD4ZIVUASxI0jHRy006LasRZJlGOpJyiii6KYdJN8VumW4cLqwu0k3VW6YbzdnRRbphDicLefrbopt+IHTD8GbrLdPNOJoXB2de6ErnzAtyOdUI+xllSYebF22Le9KZdxB0UzjNMlUU3ezDpJvatlmGHnZohNM0PsKl64X6KUdZCRw1Zcc1pWF0TlPKLa2MUvzurSupxGtQ5sstrY0qU07hFF74VlR9lGtpdVSZclrbdoZMWR8dBOX0ti1bvSXTtjrm2rbRzEPVEIbKRzk8OKk+bm1pGVqZcmm8VgHlbGFyrqUFVWXK2WrLDjQUj9N+KITU1/mCIdLgTp5giJITWbjVLFNiMJmQFB4RhC0RNAwpNYVO6nRrOm8/WsZ8fGX7I76oK36Ivjl23Jw+vDVxuuDtzppYaWl3OsXv3nqbz+DBRUp9hFMOlHAq58JOHOVaMrIrU86ymjN4qEGjSktL4sqUszkdWOJiSUgH1uD2OvBXr+u4w1yVOfcjUEzGRL3J2gkZ2B/FZmWqYjzwmGQjjNz55Ova5OwZhFnarAZsSNtRl6g0j5IuCgSkL25w+2UJ4MVx/sWH4rU//6T4vH0CHoj8ORlxfIREA+ET7UOCdFttItHZK6kjBurbGGwfA6S7/OvFM90u2JldfEYf/CjyZ2tyO0F0Guf8w1Yfjorahq63TfAdox4Pnj96Ql0S6lu7PiOpr+jV3Ea86WHMFWKhM2hnAtEUV/QPYQpE5kwYIz00mCUim9hAOY6m2pw1pIs3TXm5jae9U7lea2Fy7Ydu5PrUnNUrrEOau8qf5ZqRdNK24SxhooGql0Zyof9T4YU8L+LF5tHggb6yzfZDFvQX44dE0vswoE1FWbViC/wZkfujeSSNtMnT8uH8/qN++rf38WLx7VcaT1aIZjHpvaWd8FiWLdL7uVlh7P4F6Y3pTjK6G/cs8KoO3cgJv2xIdaikf/Y8AA40PxWJbqO2wEaKiIIhvLJa7Qa8sb0sZAKJFb6kDzoWvpdxOqT0vxtnPolh8h5O9ur/bxUn3JuavCndYnFi4DgpUOoF/QXhil6dAOIK1Wt5m1iyeGWO1g0s4Q42pQBL7P6CsER63lMDcRiD4hoaiSEt4z9c+POQdM22ZSbWLbc0TrApncCaiskh1WRjraC/IKyRDn4urHVtRVIz1BRepHUDajJmu6sFKrKgvyCokXsJKdQ6V72nEZDxoqwb9rqCgUZjLS9rAw2j9E3n6t7QfSUNej2oIRC8Tg8xEYxVnR5agdYs6F/S6aFx+strgzd1I+Uflok2J1hRklDb6wkcP0VrU3Z/QTKUvpeS+DzkzfRFb9fdofNiSuoGpvDiXiXXqHoTa1T0zDfm7yjGiJhiy6UxYmXn3CjYGijoX6AldTMndaohLalRcmmPkfsdj9y3sAgCtCtUHLmP19Wsr1xVS8HAKX73zvKkEo+knbgsz5byYytTzkRhzEWlOGrAHKs4/8ERzjD5ICeOWcmV/zH0WFyMIS6uW48x1OkWJme978LY5DbCEiumslYyXDVew7X+mEQbMz51TvORPDgEi4M1cH0nOLJEJzd5750n8G1BBSLZOH6FYoRAotijL+JlEpJ4Aq09U8HSlyilIg2K+Kijigw98YuMAjqomulNCI3iiCLKjiud2ryuE1apdXkfaVP6DADMW2JYMlPmFPTPWzmLL7VuUhbFNVuWNVkWuGVpyFyWpbjFDGPPu7Ro+NfsCAS7DtgRKZCq2hGmjg0k2I4wSYd8GyqnqRNq6lZKvOGLppj4RRJOMibgdIMLTqVVFxYgZFqZqill+1feGKD6OyhFVo7n3x3k+Xds0dXpA/C07OlSPdnqSyLOvKOHxDIcF1nwdiN0s8kAlWpyOkWOItkqZgrUAh1ZwutrSRqXKC80MSwVG6imk/tsS6e+8H5HmumKACahqgmGCf6v80ilM37RxEp91dazPIKslLp5xNQxFzA/j2B7NZbdV7CxamITCytoh945/4hiZv/2Vv06Izcgj3FOICDPrPW/6DvR5+ASDAwRCQa80WfdCD5TsbAMWVLZC4KiG1qE9R76oDZYd+CQWIlbrzQRhMkL7609pqPtpdSZu5eyKR2SLuGA1tkcUHRDgb0k61o/Z4DS2g8vmYp5O2vhtcE9+Px78MX2l2Pzfnp99y54HlJX6BjyO+E3xvPILSSeG6jCSCVczo7S8k2E0e7qPBZs2g58lOmgyA18ZPcv4Hf8XPQUgvUHPlJpruQh7qCDbHf1T5ehhllLVoFuKehfFmqcemUPqFFro1AqzB5DbDseYqtauLeKoiCpIbZ1FMdmltgRaF0w4bv3eVGcp4HUsSnNOqv+0OimGXx0q+PMI1Z246HRTeUsxi6uMPaBEs6QOKNHbEGEa+BgNyGEMyUbrV3bwtyByrg0Uq0BulF/AJl8gYowDwN/Ti4DjvHw9dtWTcbD0y0rRkBrTtnIDoWzlpumYk/9CeG0YJqjhUtJFIGwV6Raacc95vtD0iTXb8/u34jbnmq0HoorUcOi09W2rX2Sl1dtcGk5eu9yGwsO4s9Ay5nNmjeNibLrRUnW7P6V4+2oNFeoWmHghk97xMKtw9ewQLabjRI592cLaHJAtOE4JUPXcOHfmENEw5xoCu10VIumtWtg9POH8ZfbXvAbuOGHr+HP8F4ZPlMk5Pv5JzC7AQ40fyiqufU4wHjudtrt9af6lJUOD1QNu2/bRvrBawrDv5rS9qPmzx+dWXtG31QyY5pwxOwotSjOHz+jwf3X54+KNric3307/eLYFgUWGzSQaYublrH7TGXioujVTVtclmMzPnwu7CHFWwjJA51ZrIPnD+Fi5+acqFrYvH4R7hjWQkEgCG35qMKyXDXawdON+q8YVn3nc9TKzVKhUU+J62e5/Lpt0yNk8dr0WP8WQ3HoFgUzFKdrRck6kTzJXa22G+UdZcx6tQ32dmdB/8rWLp2m9DLbnY0bbqQsMm/d/26U4FZtLFNcKwhaZPevjDOqiqGXZfxnV1WYP9yg+MNFraroRhMZLtqK/6R8+GbNvE/LhWTuWon2zeCF5kxbjG8G9wGZNRfSpBORXsH6KBY2U6BRNxibFQxk0SiChJ30SOcQr8ndWRTbeET3Bt2YIKOdAd8otimHwJcoiNWectzNmZP6ps7Uj7llcBqqOMCMlSmuOMC7lEszJ3QV7etWytIrqz41FRM/Lewoapd3rvln4v26/jw4HY1X6upB2s8xyvB8po5OaQz5Y42sf83nqWHhLDpKRW7C50mdZNLDdJzkmieZWr6zpkmGl4EfK/Yt40O7afrJH4O4x38=</diagram><diagram id="d-BzQ9OEiXnQBmiWa_16" name="Patch (moved patch)">7Zzdc6M2EMD/Gs+0D9fhw8bw6Bgn7UzSycVpLulLRzGKTS0jRwh/9K+vAIEB4YCTYOGc7+XQepGs1U/a1UpORx8uNlcELGc32IGooynOpqPbHU1Tu5rB/gsl21jSt6xYMCWuw5V2grH7H+RChUsD14F+TpFijKi7zAsn2PPghOZkgBC8zqu9YJRvdQmmUBCMJwCJ0h+uQ2ex1NT6O/nv0J3OkpZVg/dvARJl3hN/Bhy8zoj0UUcfEoxp/LTYDCEKjZfYZfOKnux/Rk9rFWm2+2P+9P3a/RZXdnnIK2kXCPTou6sefFvbV5vreyV4HP2lzK8v7adX/oqyAijg9urog8GEYsL7TLeJIVn3l+FjsEDX7gtErsdKF0tI3AWkkLBPEBff7mQX65lL4XgJJuGra4YYk83oArGSyh7ZqFPAXiFpGSGw9N3nqFWFSQicBMR3V/AO+jFcoRQHNGxpmEITqeLAc6DDq0qHS4nqXbgT/ozAM0QXYDKfRi8MMcJh8x6OOuRTgucpKmFFL+w7XoKFi8IZ8ACJAzzAxRx3lVntAiB36rHChA1S1PWao8ZHdwUJhZsMs3wUryBmtiRbpsI/1U0+anxGJsX1Dm9V4bJZBu2uzoWAT6lpWvUOG/bAyTmAIrMvYlTEZzc4SjUWounepLe26bq9nOVUtcx0ouW0pgynKtZpWk7vS7aceV64Tm7h6mmtW7gkTT/zUNMVFy5d+sIlacn/qOV0TbbleueV6+RWLsNo28pllWB0jPmX4vvupasneQJq3fMEPLkJmM6u1kxATdIETPF97wSUHblrYuhuj28J3rCqDcRav3hmE9GYhk+/OFsPhBRriocd+OveCephCqtNzCeT2mXPof3cCUADjjDFywzQCL6E7fusKteb3oef2d+M3HxTjj7fjjS3SmPLEkL0XlOEiFG5Pf4eQFY4j3/z+STp4598gcz4j+cQQYq9S8IscqbgCJtz+RSoeyk4A3CEPY58ADQBgPvR6hbQyUwAIDO6xRF9xpTiRWRuQOggPFJiUsxqTWSXLkpehp6TaDwjPJknKtz6ZqTCOvcYDvBvLAbk5Sc+4FHB3uRKW146bOBYq1P4ZiDI7Q6d3AGYOL6ZAUyGmEAEKNtfZN4rH0Be2y12w2U3jSjzpKRbuqQGHwdkAvlLWuZIqlBPMfJQ+4WKYiMIFUU4pT18P2Hdcxb59LaCZuu2gpakXGj3w2lk2XtBVRGd/GmYrtw9llDXnOkkJSA+bDq95qmr0ZTpeiJ0D1FYwVf+YhZiry84h5uft4xLDzeN88HM6QUDVrd1wYCkI2XjwwczlnSPpp+o6erm1BsznWEIltufU2cbWnpOqTfq3IrLknTnZpbE2Wfn1nLnVsyzyHdupqQ9h3mw7QordE3f1txGV5Xk2z5sOV226UzRcty3laeCz/7r81ce+f5LDHCGBAI2wqyxUi8WpsXHvIgJneEp9gAa7aSFcdvpXONobMMx+BdSuuXWBgHFBXZiFC74AYOtHZ7ljzPlb3Q8yfJXHgeYdY8DDsv9DwgB24zCMszF+5maCyl9rXCKlKQY9x0BVOizh/gbfGre3xTz/vej1cMNXkGHLyrFqPkFkzUgThsw03ufj5lZG7N+KzDrFk+OrLcxq9BvBrPEcx6AGYGvAfTFGw5fBbN+XcysVmBWvBSRRD77MKvQbwgz0Tf+7KuZVRMzqx1O0+gXsNHfxqxCvyHMRKf5c4RgVl3faLXDN5pqgY7e2zRV6DdDU7p9yK9aV7AVvq8RjOr6Pqsdvs8qROZaRSRfod8URtoejO6gH6CvC1NdD5fesJBNU/G4tyKSqtBviibxt0HhlcPgy3KU4lEDJL0dIPUKYFR4twr9pkAqjckZSF97WUohqYFTO0JvAY+KREKFflM4lSesbNef38UZgyFeLBGkUMqeLr0SnRbq3Ic+DpB1g/f0F+2fDKSYIyhs3rpJ+F1x6flQsvuFn/UkKbB9ZFfoN0S2WroNyCQvkgVTzJT5S+z54k/CZOUwuPA6Oiayu5JhbsceQmCqwllX6DfFYGnUV4PBL55GSymq5k1tR3CoF/Kv3Yo9a4V+Q7wl3S75YZO88BBuXPqY+Gn2/JSR7xx4WDiq/24ILHHlKYBgNeSUy5v5LN7+3owDemNswQ9tYD/CPwzzz8eSv/E3xH7JapZIHHeVo894DTD/fPeUe4OR4CWyhytI4/pZu0xDeWC7Ht4gWIQXKrxnf5l5mfUp+35GHH2RPVfwKdzQPMDZixFqp+SaVqiQ3LrosMA9+ncwsodcpCk/DMrdoii7SvOO61usuPtDlDE4uz/nqY/+Bw==</diagram><diagram id="6wJrspvL4pc4MqzzIFbO" name="VPatch(leaked requests)">7Vxbc5s4FP41ntl9SIY7zmNsJ+3utt1s0rRJXzqKkW0ajKiQG3t//QqQuAkjnBqDu85Mp3B8EObo06dzwwN9vFy/wSBYvEcO9Aaa4qwH+mSgaap+YdL/IskmkVwYSiKYY9dhSpngzv0XMiFXW7kODAuKBCGPuEFROEW+D6ekIAMYo5ei2gx5xbsGYA4Fwd0UeKL0s+uQRSIdanYmfwvd+YLfWbUukk+WgCuzJwkXwEEvOZF+NdDHGCGSHC3XY+hFxuN2+aIGK/oP338wV+RWAeNv3vwsGex6l0vSR8DQJ68e+sFfuvefvj7dL6e3f/79dqPdfH1hlyg/gLdi9hrol5dTgjB7ZrLhhqSPH0SHq6X3zp1Bz/Xp2SiA2F1CAjH9xGPim0w2elm4BN4FYBpd+kIhRmULsvTomUoP6awTQC/B6bnngSB0n+K7KlSC4XSFQ/cHvIVhAq5IilYkutM4BU2sila+Ax02VDpdSjzu0p2yYw88QW8Eps/z+IIx8lB0ex/FDxQSjJ5TqEQDzeh3vAZL14tWwCeIHeADJmZwV6nVRsBz5z49mdJJih+dmRZiAtclLEomUk3RRZclRNSWeEOvY6NofAWyFTlkpy8ZvFWFyRY5aJtcEbAlNU+HzmBDDxhydkCRKqKojJ5sbhQ5KhpZbjuet1rOMAuGU9Uqy4mG09qym3acdtO1ju1mV9DWURiucqmKhkuBuXfLWSe+Pzq+N7W+8b3dm+Vn7cT3Zse8NTxOu2kN+d5sDW/GcRrO7NrBUCs8jBPf95zvLatvfG81IHzo0LCbnSJMFmiOfOBdZdJRcYVmOu8QCthUfIOEbJjRwYqgIpioUfHmIX/yGA12bvLTyZoNnpxt2FnyXaMvuNta56snRCs8hXWKeqJIAJ7DWvbYQh8YeoDQNVD4flXTGF96iTHY5BQC5PokzI18EwkyQKklB0JlOZ3rhvr60CxBKPkGGaDSR/kJqtJ7w/HJvDf2KuyOd0erwqk/CsvpVseWU/vjV+xmOa1rzHGGODkWR+RYpF5DbxwLrT+Zw1xKv8ECbLj+jNYMV+HYH4XhdKNry5mC5SZ3NxitNzeATBdb2ctHBMrNyJiGbiz6KDKmOwXeJVvfJHJz09XuwRmJLqBDuf78Y+wC0+iy7CYflowORDx6Q+LRzbZAILpMk7t/VpCenOa//YpV9/MvBrR3z9CDBPknABwghd09AMRU7KeY/+td2BMG9pfW6h4DFwIGPl79qPYCcrNbntEnRAhaxuYGmFxGLStUiuioXHbtelkKy+EaTx6aPnMVZv1hPsulnGtpZutxkE9svTrNVddeIk1eaVvmNzeBfIp3y2cJCajSdqGWw9gkIccuqktklfyONFXDB0qeWRhoXzktXYySKcISormLpn2gWV5EAE+Ucqw5iZcifgHYEQB4gPRqTCY3KHSJiwrxKEf8u5LC0nWc+NZCBFteI6lmgemUTDBii2iimz+PZGm+VmsK+S2Ibzddm2Zt0nStUotyiX476VpdjNxl0Mbw+wqGpAtoV6IsI9oDsqy8ltAQmmYn2NTtItZ47L4NmxL9IjbFfUCrvvpg/C3WJMYYAuoDakq1o9gBlrVDoNJuSph6F6i0Sg4DZ8RtqJTot8SYYrEhYsw3MPpiv90Cfx7B6g8KjvXvvwqupHgZdgIXuxyJSOBSry8hMVsxK6+WebN7w52Y8GS4u4XhyuvFfnwQDhs25TCjDxym78hh+kE4TMybpl7fdQSKG+r5hVWuXxggPxSzq135fq17cUZDsGl98OIMCQFK9FvCmpijbYS1DiPoboLZpkjrBGo8GE2ho9VDTaLfDtQMEWqfwTO8DyrRJQqdDZ11AXAErkkRH83zJDjqWQBZMwMzMx3XHA3MSTQWRR9PNA/a6xmwteqlX5X1O0jnCR/41HlyRJ0n6Ysvvek8MfvTeWLu9tJaZd1ENF1rPeVmfxo1d7NcNeoOaDlbO1LLGRddY06MYXm9/hojX4xhTwXb/ZN25wVbsyb67F85Da5d8pA7fswdZ7WF6ISXFvpZkDCb5vLsWjSdKeeKpRlFRn5VuXjXEMQo1yD0+hBEor8t3ce/DprNQthKJs8Uw5R1F8huG3GNkydmN9WGciOLJcnU1eu3E9KaYoNTxJUTN3zOs2QOOdb3FSJsbzlLdsRLqqCqwTqec/45p9bbpJw7RsuAbsRQ5Fgx/s1vW5XxcFtbmaGWpqDqjdZh1V7Wmi9YUz+fuLNZr7aythc894PlW0y926qcG5ZafsfoEFtMuQXDkFCCRF9SUUr7inhIaBUfp70dyBZjPwlqO2z6aBu13E2Qo1brJ2r1Uq7VtCVlgHr9/qK2ugL6/+RanoSQo3ZLGfSwvnvJdzIlvXAS/YP47n8/uO8/2KO/zNB5/Lh8Aydfxt/OaiD4gPBu3NlFM/L+X7lv2jdSac4t79HvvxX5olT/sMotaE17kYdmNTIP1ctm1yRPekeCe8udtA1UOd1WxLK7ILoYBjHsnKncY/5JdNtDCbq3gHJXGt92n4aeg/ADForWgu+wfVokv3zieW4QQnmaGYRBUvmbuetodYxmlIt5CngQNYZFf2keOPeJFf+xCDonT/7aLP+WJ66i/KtVxM37qARUzof4zs7IA9Nnkb3eIk9sN/o1SvKl/KTFX2nOzYlRMSevyGXQ0+z3bZMllf1KsH71Hw==</diagram><diagram id="znj-D5QoGOLx2BGCd-4G" name="Solution# 1 (Current fix)">7V1bd5s4EP41Pmf3ITnccR59Sbrdk+66cTdp+7KH2LJNg5Er5MTeX7/CiKtkBGlAuPVTYTLIMPpmNPpmoD19tN69Q85m9QHOgdfTlPmup497mqaqfYv8E0r2kcS+uooES+TOqVIqmLr/ASpUqHTrzkGQU8QQetjd5IUz6PtghnMyByH4kldbQC//qxtnCRjBdOZ4rPTBneNVJO1rdir/A7jLVfzLqkWfb+3EyvRJgpUzhy8ZkX7d00cIQhwdrXcj4IXGi+3yYO9G03v0fP/n5Pr91+27gfp1eBENdlPnkuQREPDxq4e2Pn5bu5u76Yf7T/8+WK5tBndX9BLl2fG21F49fTCYYYjoM+N9bEjy+JvwcLv2bt0F8FyfnA03ALlrgAEif/GoeJLKhi8rF4PpxpmFl74QiBHZCq89cqaSQzLr2CGXoOTc85xN4D4eflUhEgRmWxS4z+AOBBG4Qinc4vCXRgloDqpw68/BnA6VTJdyGHftzuix5zwCb+jMnpaHC0bQg+HP+/DwQAFG8CmBSjjQgtzjjbN2vdAD7gGaO75DxRTuKrHa0PHcpU9OZmSSDo9OTQsQBrsCFgUTqSboIm4JILEl2pPr6CiaQWeNemSfnr6k8FYVKltloG3ZVOhQl1omQ6ewIQcUOTVQpLIoKqInnRtFjIpKljuO56OWM8yc4VSVZznWcFpTdtNO0266Jtlu+onajeuprN0SXL654SxOvD8Fy1myPdU4r5Mnt06aWtfWSbMz3mfUWidNyd5nnabdtIrrpNmU3ezTtJspO9r3z9H+5KK9ZXUt2l+JvQ/Ml2BKTyHCK7iEvuNdp9Jh3j9TnVsIN3QmvgGM99TmzhbDPJaITdH+c/bkSzjYpRmfjnd08OhsT8+iew1vsJ6nU98J4BbNQIle7NDYQUtQNqB6ZN4R8BxMPCB3e7xJPFw6QMjZZxQ20PVxkBl5EgpSOKmF5EG9MrOAEOrrfbMAoOgOUjglj/ID++7upPP9WgmFLXlhVLvDWNQynG7JNlx3KItahtOkI47DWZxzio7nFEnC0JmcQuUQEZL8L0J0VQes6H9GY4brzt67nuF0oz3LXd89/T2YgZcJujWX5tDdm4HBKxGdhOEsXfIuUmVpi/F0guBuP3HwbHU07PsQA7EZaYgmAUEfhsZ0Z443oIERh3uDJEx6YIHDC8hQrr/8dNg3XFjM3qLdKN5SxOZigAMC3WwKBCwHM55+3AJycp7/5muj8uef5ZKmT8ADGPpnALRA+ssHAEsD3R/if3nuf8bA21GB0jGgsTnUp+tnfhaQmd3ijD5CjOH6YG4H4UHYHEWkkIway25cL+X95rHGowdnT7EKtX4/Sw0ql1pCB37pZdnAV3ODZY1MYsrvSLkgM4HxFNdjARnarrBcqMX9f0Rj0ovK6L9C3qEWd4DRMzMDvRUTqLGEFkFYFGim4bT3NMsLA8AjCTnWEh9cEb04aM4AsAVO+hBMJjBwsQtzG/kY8bcFhbU7nx9+mtn6F30k0cxFOiUVDKkTjXXzx5EsZrmNqpDnI75ZkjuhuxKSWylFuUC/GZI73m/WgDYC37cgwDKgzUVZGmhbjLJibFaEpn6kVadZbOp2Hmvx3v0YNgX6eWyy64DGv7q1+M3SwyMEHJIDago/UZSAZa0NVJpVA6YUVFqFhCGOiMdQKdBvKGKyJHEYMd+B8MZ+u3P8ZQir9wQcu99/FlwJ8WJJgYtd3IkI4FKuLwhitmJyrxZls2+GO5Zjp7i7A8HW68R63EoMs6rGMK0LMUyvGcP0VmIYS54nWd9NCIoJyfwCXuoXbKAfsOyqrNyv8SxOqwg2KVgrZmWGIAAK9BvCGsvRV8KaxB20FKhVRVpUa229Y+uqAB2tHGoC/YagxpYDHpwn8M+Giy5WON+TWWcAh8EO5/FRnSdBYbOHk3aBUDOTcc1hzxyHYxH0xURzr7lmC1vjuz6P9WulZUfjNHCeW3Y63rKTvGPVmZaduPrQgQaKCNFVGyhU2Q0Uene6NOtZjo+6Ni3HcqanYTnjSrLlzO68M1PPcqZpVrNcU0VOnaUw406HGwR9dvd/LnW//XInvdSt87nHjhYiwc7FnzPHXzLHaVUmPImLMt0s5cTTLtyaHdmZUTRdKJeKFbPH8Vr2qkJ73c2bUaze6OWbN4H+MaI0vh24WASgEQ5UZznQnQxkN424yrSTLqWyzbQAWQKOs1y/GTJA53OcYzd4ykbJDHKs71uI6dpyEa2IA6KgqpvdYc7jv8eh9S4qhI/gekMWYsDGWJY5yC5bXCahqaXMUAtTwHt7us9byxrLokt4wbG7WHRqKWvc4asWZsub9JVLw1KLL7W1scIUe1cMQUQQ6AtKccmKHO+lrfzjNLgAsQyjALQSu2Uaz4uqYlbrJmj1Akdt2oLySbl+d0HLUq2/bqSNyRsham0+aNtN3AuJkyloIRToy0vcjfMnnk6P7e937gVdszsv6Br1PrpTke3vN2U5g10D0jeliuH/L+iPiYMRs+yPeueZV3w7x5LOK5osu9zdDIFHK9Iv7pwYsWhWba80y4MNyaDV+FXIeENlviY/YZseCymvVSwNHencrpvo9AtdcJagH12gL0rFmb7h/EM1lwfFM1762SrPczdBheDqBJsoJ1m4u9ChmmseKcQri9M8onHiVXN1RI0x42nFq1eVQd7iq2I/FK/squGqHE7KpSKjCmKp9WKMQF/eZsrkfEFRBq75376TvaZWfSfQPLLBbxaEdqGkbOnlX74T6DdTOrFK3neNXzboUmt4ZzBlHXltKgl8fTXOAOh8XrREJBX4dktEJJXrS4x9fC7zV63qaeLErLGqHvd7Vq+IHD8x2RzTsdnAcfwzYHUqJBftlEiKbfy2IGcS6Av2ZQ1USMhp+n8IRerp/8SkX/8P</diagram></mxfile>
\ No newline at end of file +<mxfile host="drawio.yandex-team.ru" modified="2021-05-13T12:45:24.063Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.86 YaBrowser/21.3.0.740 Yowser/2.5 Safari/537.36" etag="wJ5mePPJEHiOP22e9T2I" version="12.7.0" type="device" pages="5"><diagram name="VPatch (datapart)" id="13e1069c-82ec-6db2-03f1-153e76fe0fe0">7V1dc+K4Ev01VO19COVvm8cQJtmtndTNhsnszH25ZUABT4xFbDOB/fUrG8kYSVgmtmyTYR4ytmgJ0zrqPmq15J5+s9zche5qcQ9nwO9pymzT00c9TVNVx0L/JSXbXYk9GOwK5qE3w0L7grH3D8CFCi5dezMQHQjGEPqxtzosnMIgANP4oMwNQ/h2KPYM/cNvXblzwBSMp67Plv7tzeIF+V2Ksv/gd+DNF/irHRN/MHGnL/MQrgP8fT1Nf07/7T5euqQtLB8t3Bl8yxXpn3r6TQhhvLtabm6An+iWqG1X7/bIp9lzhyCIy1SwJ+7EVafGZKrZz8ZEudJ2Lfx0/TXWRU+/vp7GMMQPHG+JktCzr5LL9dL/7D0D3wvQ3XAFQm8JYhCiT3xc/LAvG74tvBiMV+40qfqG4IPKFvHSR3cqukQ9GruoSpjd+767irxJ+q0KKgnBdB1G3k/wCKIdcJJSuI6Tb7rJAJGKJh0BZripTNdK2u7Sm+Jr350Af5j13A30YfL1AUx/UBSH8CWDQdLQM3rGW3fp+Qm6v4Jw5gYuLsZQVpHWhq7vzQN0M0V9kf50tnNwf/0EYQw2uSLcWXcAIrWFWySCP9UMDBw8sBx8+5ZDKQHpIg9QIujikTHPmt6jA11ggPDBMvkR/Plmju7ju+1LaLzOh/+/ur3SWcDQQNl3gyIGAKukoxgtrTXDPFCaqvK0xipNk6Uz0vA5KU3X2laac4ZK447PBpVm2OenNKttpRGyck5Kc4zmlMY8PMcHXEhDd0iDqXWNNLRglfRTtUaTBrNlq6Ra56c0rSRpMKWx0xboaVWlGSWVpktT2hkOT2NQTmmGNKalnp/STLs5pTEPb1xIQ5dJg2V1jTSoGosY2SPMOFVtNGsoOcKkOUC1hVlzVaVpbStNN89PaUZJfirNAeotzJqrKs3U22YNZ4g0q2T41JaltIHK6AjM5mCMb2EYL+AcBq7/aV86PNTiXuYzhCusux8gjrfYr7rrGB5qFmy8+Fvu+nvSVN/Ed6MNbjm92ZKbAP3erFJy8z3/yb5Sekdqle/DCK7DKSjhM2M3nIO4SKe4xUSRhZgIge/GiBMd0ApeF+OqD9BDvyLDkk2xCnpk7R4UV9rj5DoM3W1ObJUIRMe/xuS74dsj4tmKwDvl7YFJoXr3wHuMZxqsAHsOY+66rSCKERpYs7qtYB6eQ30uM4zuzDCy6UN3ZhjNDzDnVK3RA6xlAqO1EGCrqjO95FqJNKVZLUzKqgOtufi3/mTe+NvxS/TtQZ1/eRuOx+aXcxycjtWczljXzSYZjMYPIdxsH9x4ujjqAgMYA7H2sLtSDXSd6Mebuv41dhJxwqgzl+GD5zipgJrygvmXlG1fWQwjb9aj1e+9uDNJXtReBtPhBJ9H47/WAN1curnWhKt2u1ljunn8AnwQw+DSz/Wukbfbzyyp+poa7eLJy6Wr37Wy0WRX8zk009tfPv3kO+lcP9J9N4FxDJepYt0wvk6yoFEpRK2SslvPz4JZwYxITHw4fSEiWM9OPnal9DWzWviKpHCLglBYThiDyvUU6cuKYSnKyqv00sIu0MaEpdjAEMUKVDoIeiS+VVeMiA0RISjtbMc46d+eZvnJmJ4gK2LN43R0hW9uOGOQ1kBENbUPDzDyYg8eBBcItD9TAktvNku/mglH0IMhkzwwXsq+YIhHy0g3iyArDrCWhLZeEtqn4fjUsGi2ZEbgOSiOcwrk5cQ52cmxCMMheF2DKG4Dw1w47U1nLXZTCEKdBeHb/G76v9dX/4cy8r8bYz1+XPxOHE3LINSpNQAyJT4GQoH8IQhZy67xa0uwyCqInsb2Qt0at+HC/iuY3HzXrtgFvj+Ce7B8BC4iRBzMIhITH0KsvLELkyiyuw8v455A7ZrDnjk6Tsn225R2dCtXPkj/ceFZOHppOpZt1sLP18tveOLRNKWvKqZ60HcEwBWd/ZXVt7WDhu3+wD5sBT4/R0CKi1ZZcn8TAhdRd03h8/sWzJhWzSDZZQlfNyySRdE/4vaOWSSBvBy3qLJZmYlfvEtQqvz26AbzBEF/IBxs/tN9CImhYXUDGjY9WRRAo1he4KxsxeTWFs1DasMYm8ODMfYIorXfCYZV0TQRXIkB6HQDgHQA8kTbpDdhmzQ2/pxx9tsEAA+It0c84h6tYBCxUeq2mHsFXDklcUX4aMdYuCEwbAJ5SbgqiGcU4arFoEbNsCJoEcOqGxEGEiGgN7McDZgVy0uCFd/NpbAaeclhDR8YUJxgQeF+67ZDVjRArAYAohfYHT5AOheC4mSrkmtBrmpDQSuN4zC5USs5mamV/aVzor90TpkIaAMqUZ3ORZI8EdDZzaZnZiEbj8GWtatkGbHteQW1UkW2QhydVxTLCye2/NpNrYvp/EUFHDxRd30edyduUtfGgdLoNcui1+4GeqlZrilI2BfIC9BrOvzaja3qsql8Hy0so5cNGZNtZm0DkIKEJVjEEsjLobGcfLHUi687avUqwGdQEj4kU7Jt+FAbiCxRVK9YXhJ8jpDA9ccxOwQOYtx0hLVRswpLkMAhkJeEm4IMDoKcDxwJNspyKaMbXIqeqdrWaTNbu4mIDMkEuGzm6+ZmPqdzm/lM+acYFSP13fvrG9w1xN+eJn+jsRzN2W2fUWqyjOmyD6eWpH3avrSetG8WkJxvMOxckPR98dC2qJHZDWrkUMmjtlFMjQTycqiRdWSWX4jEFterTl+aqhmFksDFoGFg8Yly3YedOM5pBF4gLwmlbNrr+yeFDW6DOgqzFnYp0YBySqYHsggwBQ1JDmhbdpuGRylteN5vYwiHbtmB2dRQJw7qGDAE8sJVvkHf5NZvDFgFmT3HrEzXVq4r0KuyiyhWN3J76OQvW5BTIZCX47gGLL36230BTysuktjC2Rb1MAMuudt8EqSRKWM98R1b5RsGnpdr5mBT4rJOGeo9pK+hk/4l14zM2SUuW/l8ryZ8W+mzArOdXJal2e9hQFWTBQcnua9jtUXOa99QA7vHFHZDYXOw37nKVmeN5QFenBbWUGINeZdM3iUXHW9VfkxpJlmfzXZHmo2MMcoTDAQxEYG8YEyqBk0pByoV0hOOSvIg1F52anDXMma5CayXpatOL13p9JynyaWron36EtdfCnH67sPuS56mXcfyS1Gm+LnpzWjwaEXuGQXy3ywkRW/8kSpHb/+dPZjT6Tfvnyfj4T5avejrp69NvJFCiuKcBl+Ayf0B7NI8OcfuNoQByy8vi6WVPFqTi6VcA8OOkw4fS9WprVNF/EAY5iu7zkXORVH6ikW23BIb+66Vi1PnFAZ9xI9gM4pA/ticolaqzwU6a9g2bUC4FLRKb2Mv4gxt58PSZzkKlj4F8pUjyHyawLV+Iy96ydu9HESs1zWMsV+42nmzaySgqqtN2rnkc2IsH3eL+TdwuUJOFLBWk402510ON/pcgxsyqJk/9xXSDs8PySJsBWvQHyBNgu7Tpt2SUdIt6WWPKshiXYZu02+7asItmdQyuSk4I0MgLwh1XTGnSh3+nFq8FndYyM+UPd12aFRqZ4NTFq6OSryyoaUoODmfV8JKDy9frwhB0t8epRs0KvqK5pi2auz+HjZ4JB3m5CQKhY9FqaSBn8jw65EGi/eGm0ZJA+fAxl9g6A843rwwCCZ97NPJ/qrdpzykpPHu1Lxpjq9G/pEzv96Ad4xB39HbHvNsSHx3Wi+ztI5LZt5Pbuew3UBqRCs3IGXJVv5d++gHIQkl2WSLv9BdJuwqmESrXGX0o/L1c8Xpg5DS6h1c/jTg06OU1JYOgxfOr+nF9+g2hEk37IcsYq2LezgDicS/</diagram><diagram id="fr3X5ynE2yK5susw7VPl" name="VPatch (paritypart)">7V1bc5s8E/41mfm+C3s4Hy6TOE4P6TRN2qTtTYfYik2DjQs4cfrrX2EjbCQhhEGAU/uiNYoQePXsQavd1Yl6PltdBs5i+skfA+9EkcarE3VwoiiybBnwv7jlddNi2vamYRK446TTtuHW/QuSRilpXbpjEGY6Rr7vRe4i2zjy53MwijJtThD4L9luj76XferCmQCi4XbkeGTrvTuOpptWSzG37e+AO5miJ8tG8vtmDuqc/JJw6oz9l50m9eJEPQ98P9p8m63OgRcTD9Hl/fB2OPl+Yf6d+B+Uz5/V4M/rsLcZbFjmlvQnBGAe7T30u9MHcDc8vfr87X715R3QDOP8Yy+hwrPjLRN6rZIfG70iCoIxJGhy6QfR1J/4c8e72LaeBf5yPgbxcyR4te1z5fsL2CjDxt8gil4TdDjLyIdN02jmJX/l/IEIP04wARHjVyWQjV98BwQJWS6BPwNR8Ao7BMBzIvc5ixQnAdwk7Zfeeu278P0UKWEOE2E8YQ1bsrNDhP4yGIHkru3UnAaB87rTbRF3CPOfYxg69TnD/frDL5s3QFc7NNk2rdFDR5Ljf/zz6UzTQ/X9wr3vSdd/70Y9lUBSG0CC+Alevyf3ry9+7F4MVpmrV3S1cqP1TX3LMJPr9X19Q04ut3fGF7s3XoPAhcQDQdJWDssbiGz63c1vfry/Ckbn4Wz4+0oFo59XZs9USdBTJ6Ap0CtZ0JsmH+iJgSy9YKDNL67MPaasZZ+jZeSkGG7QKNxgeHD6zsKFM8+whfFnGWuQs0d/HvXCNbBPYQdZXqy2f4TfJvH/Xy+e766daDS9AeESjpcMCt9xM+6mF2p+CPZ/Uj3v++gHL04wzn1PTEJEYBVlGdrx3Mkcfh9BbopZ7OwZBJEL1ftp8oeZOx5vZAeAL+M8rIeK+TDBBRxXPzvRB7DFcx6Ad+aMniZrOXPue36wfq76uP5QmZcp7OC7gFWGkRIzJnmPjKWQ4cLkLqkvmaqSwWdyVZFNe1rf0OztxzQzD+nJZl9Xs4P6j48hiDDeKMsNVH0sW2/SzJDNduwME0nOmu0M4jka287QVGb/ypKVSnTSzkhl4nc/GLiQjXekHyaFWjBIEtMCff+x8z3HptjLhImlcfIScnmo71ohT4G8/B1K+vLR+Pyrt/pwcflujoyLXZagWiu6xckSWamZAMgw95F9pWEuZc1j3WQYBKRgxYzrnqwLEKPUSSAXa8XQD8CfJQijNqBfHySpUNMomGRhlxeS0Oi3FDs7x8jmFQtMXcvKU13DVvTl+ucBuVagUqdGIq1f9fR0FEFzC8dhOHUW8dflzLtyH4HnzmPsLXbWVF7SvLPOOnuZuhG4XThrgLwEziKLPGiERQ68JUivPc9ZhO7WOAzAaBmEcCZvwMaE3YB8GcVPOk9dUVKGD+KhUieQtB535o6S7znG5dxf/6AwCvyn1AGFuGPozFwvnrw7EIyduYMxjUKzf8txEWGh5sJewcBkJZcvW6eZLCVt012HmSRJ+XyQQVZZGMkkjHD4ZIVUASxI0jHRy006LasRZJlGOpJyiii6KYdJN8VumW4cLqwu0k3VW6YbzdnRRbphDicLefrbopt+IHTD8GbrLdPNOJoXB2de6ErnzAtyOdUI+xllSYebF22Le9KZdxB0UzjNMlUU3ezDpJvatlmGHnZohNM0PsKl64X6KUdZCRw1Zcc1pWF0TlPKLa2MUvzurSupxGtQ5sstrY0qU07hFF74VlR9lGtpdVSZclrbdoZMWR8dBOX0ti1bvSXTtjrm2rbRzEPVEIbKRzk8OKk+bm1pGVqZcmm8VgHlbGFyrqUFVWXK2WrLDjQUj9N+KITU1/mCIdLgTp5giJITWbjVLFNiMJmQFB4RhC0RNAwpNYVO6nRrOm8/WsZ8fGX7I76oK36Ivjl23Jw+vDVxuuDtzppYaWl3OsXv3nqbz+DBRUp9hFMOlHAq58JOHOVaMrIrU86ymjN4qEGjSktL4sqUszkdWOJiSUgH1uD2OvBXr+u4w1yVOfcjUEzGRL3J2gkZ2B/FZmWqYjzwmGQjjNz55Ova5OwZhFnarAZsSNtRl6g0j5IuCgSkL25w+2UJ4MVx/sWH4rU//6T4vH0CHoj8ORlxfIREA+ET7UOCdFttItHZK6kjBurbGGwfA6S7/OvFM90u2JldfEYf/CjyZ2tyO0F0Guf8w1Yfjorahq63TfAdox4Pnj96Ql0S6lu7PiOpr+jV3Ea86WHMFWKhM2hnAtEUV/QPYQpE5kwYIz00mCUim9hAOY6m2pw1pIs3TXm5jae9U7lea2Fy7Ydu5PrUnNUrrEOau8qf5ZqRdNK24SxhooGql0Zyof9T4YU8L+LF5tHggb6yzfZDFvQX44dE0vswoE1FWbViC/wZkfujeSSNtMnT8uH8/qN++rf38WLx7VcaT1aIZjHpvaWd8FiWLdL7uVlh7P4F6Y3pTjK6G/cs8KoO3cgJv2xIdaikf/Y8AA40PxWJbqO2wEaKiIIhvLJa7Qa8sb0sZAKJFb6kDzoWvpdxOqT0vxtnPolh8h5O9ur/bxUn3JuavCndYnFi4DgpUOoF/QXhil6dAOIK1Wt5m1iyeGWO1g0s4Q42pQBL7P6CsER63lMDcRiD4hoaiSEt4z9c+POQdM22ZSbWLbc0TrApncCaiskh1WRjraC/IKyRDn4urHVtRVIz1BRepHUDajJmu6sFKrKgvyCokXsJKdQ6V72nEZDxoqwb9rqCgUZjLS9rAw2j9E3n6t7QfSUNej2oIRC8Tg8xEYxVnR5agdYs6F/S6aFx+strgzd1I+Uflok2J1hRklDb6wkcP0VrU3Z/QTKUvpeS+DzkzfRFb9fdofNiSuoGpvDiXiXXqHoTa1T0zDfm7yjGiJhiy6UxYmXn3CjYGijoX6AldTMndaohLalRcmmPkfsdj9y3sAgCtCtUHLmP19Wsr1xVS8HAKX73zvKkEo+knbgsz5byYytTzkRhzEWlOGrAHKs4/8ERzjD5ICeOWcmV/zH0WFyMIS6uW48x1OkWJme978LY5DbCEiumslYyXDVew7X+mEQbMz51TvORPDgEi4M1cH0nOLJEJzd5750n8G1BBSLZOH6FYoRAotijL+JlEpJ4Aq09U8HSlyilIg2K+Kijigw98YuMAjqomulNCI3iiCLKjiud2ryuE1apdXkfaVP6DADMW2JYMlPmFPTPWzmLL7VuUhbFNVuWNVkWuGVpyFyWpbjFDGPPu7Ro+NfsCAS7DtgRKZCq2hGmjg0k2I4wSYd8GyqnqRNq6lZKvOGLppj4RRJOMibgdIMLTqVVFxYgZFqZqill+1feGKD6OyhFVo7n3x3k+Xds0dXpA/C07OlSPdnqSyLOvKOHxDIcF1nwdiN0s8kAlWpyOkWOItkqZgrUAh1ZwutrSRqXKC80MSwVG6imk/tsS6e+8H5HmumKACahqgmGCf6v80ilM37RxEp91dazPIKslLp5xNQxFzA/j2B7NZbdV7CxamITCytoh945/4hiZv/2Vv06Izcgj3FOICDPrPW/6DvR5+ASDAwRCQa80WfdCD5TsbAMWVLZC4KiG1qE9R76oDZYd+CQWIlbrzQRhMkL7609pqPtpdSZu5eyKR2SLuGA1tkcUHRDgb0k61o/Z4DS2g8vmYp5O2vhtcE9+Px78MX2l2Pzfnp99y54HlJX6BjyO+E3xvPILSSeG6jCSCVczo7S8k2E0e7qPBZs2g58lOmgyA18ZPcv4Hf8XPQUgvUHPlJpruQh7qCDbHf1T5ehhllLVoFuKehfFmqcemUPqFFro1AqzB5DbDseYqtauLeKoiCpIbZ1FMdmltgRaF0w4bv3eVGcp4HUsSnNOqv+0OimGXx0q+PMI1Z246HRTeUsxi6uMPaBEs6QOKNHbEGEa+BgNyGEMyUbrV3bwtyByrg0Uq0BulF/AJl8gYowDwN/Ti4DjvHw9dtWTcbD0y0rRkBrTtnIDoWzlpumYk/9CeG0YJqjhUtJFIGwV6Raacc95vtD0iTXb8/u34jbnmq0HoorUcOi09W2rX2Sl1dtcGk5eu9yGwsO4s9Ay5nNmjeNibLrRUnW7P6V4+2oNFeoWmHghk97xMKtw9ewQLabjRI592cLaHJAtOE4JUPXcOHfmENEw5xoCu10VIumtWtg9POH8ZfbXvAbuOGHr+HP8F4ZPlMk5Pv5JzC7AQ40fyiqufU4wHjudtrt9af6lJUOD1QNu2/bRvrBawrDv5rS9qPmzx+dWXtG31QyY5pwxOwotSjOHz+jwf3X54+KNric3307/eLYFgUWGzSQaYublrH7TGXioujVTVtclmMzPnwu7CHFWwjJA51ZrIPnD+Fi5+acqFrYvH4R7hjWQkEgCG35qMKyXDXawdON+q8YVn3nc9TKzVKhUU+J62e5/Lpt0yNk8dr0WP8WQ3HoFgUzFKdrRck6kTzJXa22G+UdZcx6tQ32dmdB/8rWLp2m9DLbnY0bbqQsMm/d/26U4FZtLFNcKwhaZPevjDOqiqGXZfxnV1WYP9yg+MNFraroRhMZLtqK/6R8+GbNvE/LhWTuWon2zeCF5kxbjG8G9wGZNRfSpBORXsH6KBY2U6BRNxibFQxk0SiChJ30SOcQr8ndWRTbeET3Bt2YIKOdAd8otimHwJcoiNWectzNmZP6ps7Uj7llcBqqOMCMlSmuOMC7lEszJ3QV7etWytIrqz41FRM/Lewoapd3rvln4v26/jw4HY1X6upB2s8xyvB8po5OaQz5Y42sf83nqWHhLDpKRW7C50mdZNLDdJzkmieZWr6zpkmGl4EfK/Yt40O7afrJH4O4x38=</diagram><diagram id="d-BzQ9OEiXnQBmiWa_16" name="Patch (moved patch)">7Zzdc6M2EMD/Gs+0D9fhw8bw6Bgn7UzSycVpLulLRzGKTS0jRwh/9K+vAIEB4YCTYOGc7+XQepGs1U/a1UpORx8uNlcELGc32IGooynOpqPbHU1Tu5rB/gsl21jSt6xYMCWuw5V2grH7H+RChUsD14F+TpFijKi7zAsn2PPghOZkgBC8zqu9YJRvdQmmUBCMJwCJ0h+uQ2ex1NT6O/nv0J3OkpZVg/dvARJl3hN/Bhy8zoj0UUcfEoxp/LTYDCEKjZfYZfOKnux/Rk9rFWm2+2P+9P3a/RZXdnnIK2kXCPTou6sefFvbV5vreyV4HP2lzK8v7adX/oqyAijg9urog8GEYsL7TLeJIVn3l+FjsEDX7gtErsdKF0tI3AWkkLBPEBff7mQX65lL4XgJJuGra4YYk83oArGSyh7ZqFPAXiFpGSGw9N3nqFWFSQicBMR3V/AO+jFcoRQHNGxpmEITqeLAc6DDq0qHS4nqXbgT/ozAM0QXYDKfRi8MMcJh8x6OOuRTgucpKmFFL+w7XoKFi8IZ8ACJAzzAxRx3lVntAiB36rHChA1S1PWao8ZHdwUJhZsMs3wUryBmtiRbpsI/1U0+anxGJsX1Dm9V4bJZBu2uzoWAT6lpWvUOG/bAyTmAIrMvYlTEZzc4SjUWounepLe26bq9nOVUtcx0ouW0pgynKtZpWk7vS7aceV64Tm7h6mmtW7gkTT/zUNMVFy5d+sIlacn/qOV0TbbleueV6+RWLsNo28pllWB0jPmX4vvupasneQJq3fMEPLkJmM6u1kxATdIETPF97wSUHblrYuhuj28J3rCqDcRav3hmE9GYhk+/OFsPhBRriocd+OveCephCqtNzCeT2mXPof3cCUADjjDFywzQCL6E7fusKteb3oef2d+M3HxTjj7fjjS3SmPLEkL0XlOEiFG5Pf4eQFY4j3/z+STp4598gcz4j+cQQYq9S8IscqbgCJtz+RSoeyk4A3CEPY58ADQBgPvR6hbQyUwAIDO6xRF9xpTiRWRuQOggPFJiUsxqTWSXLkpehp6TaDwjPJknKtz6ZqTCOvcYDvBvLAbk5Sc+4FHB3uRKW146bOBYq1P4ZiDI7Q6d3AGYOL6ZAUyGmEAEKNtfZN4rH0Be2y12w2U3jSjzpKRbuqQGHwdkAvlLWuZIqlBPMfJQ+4WKYiMIFUU4pT18P2Hdcxb59LaCZuu2gpakXGj3w2lk2XtBVRGd/GmYrtw9llDXnOkkJSA+bDq95qmr0ZTpeiJ0D1FYwVf+YhZiry84h5uft4xLDzeN88HM6QUDVrd1wYCkI2XjwwczlnSPpp+o6erm1BsznWEIltufU2cbWnpOqTfq3IrLknTnZpbE2Wfn1nLnVsyzyHdupqQ9h3mw7QordE3f1txGV5Xk2z5sOV226UzRcty3laeCz/7r81ce+f5LDHCGBAI2wqyxUi8WpsXHvIgJneEp9gAa7aSFcdvpXONobMMx+BdSuuXWBgHFBXZiFC74AYOtHZ7ljzPlb3Q8yfJXHgeYdY8DDsv9DwgB24zCMszF+5maCyl9rXCKlKQY9x0BVOizh/gbfGre3xTz/vej1cMNXkGHLyrFqPkFkzUgThsw03ufj5lZG7N+KzDrFk+OrLcxq9BvBrPEcx6AGYGvAfTFGw5fBbN+XcysVmBWvBSRRD77MKvQbwgz0Tf+7KuZVRMzqx1O0+gXsNHfxqxCvyHMRKf5c4RgVl3faLXDN5pqgY7e2zRV6DdDU7p9yK9aV7AVvq8RjOr6Pqsdvs8qROZaRSRfod8URtoejO6gH6CvC1NdD5fesJBNU/G4tyKSqtBviibxt0HhlcPgy3KU4lEDJL0dIPUKYFR4twr9pkAqjckZSF97WUohqYFTO0JvAY+KREKFflM4lSesbNef38UZgyFeLBGkUMqeLr0SnRbq3Ic+DpB1g/f0F+2fDKSYIyhs3rpJ+F1x6flQsvuFn/UkKbB9ZFfoN0S2WroNyCQvkgVTzJT5S+z54k/CZOUwuPA6Oiayu5JhbsceQmCqwllX6DfFYGnUV4PBL55GSymq5k1tR3CoF/Kv3Yo9a4V+Q7wl3S75YZO88BBuXPqY+Gn2/JSR7xx4WDiq/24ILHHlKYBgNeSUy5v5LN7+3owDemNswQ9tYD/CPwzzz8eSv/E3xH7JapZIHHeVo894DTD/fPeUe4OR4CWyhytI4/pZu0xDeWC7Ht4gWIQXKrxnf5l5mfUp+35GHH2RPVfwKdzQPMDZixFqp+SaVqiQ3LrosMA9+ncwsodcpCk/DMrdoii7SvOO61usuPtDlDE4uz/nqY/+Bw==</diagram><diagram id="6wJrspvL4pc4MqzzIFbO" name="VPatch(leaked requests)">7Vxbc5s4FP41ntl9SIY7zmNsJ+3utt1s0rRJXzqKkW0ajKiQG3t//QqQuAkjnBqDu85Mp3B8EObo06dzwwN9vFy/wSBYvEcO9Aaa4qwH+mSgaap+YdL/IskmkVwYSiKYY9dhSpngzv0XMiFXW7kODAuKBCGPuEFROEW+D6ekIAMYo5ei2gx5xbsGYA4Fwd0UeKL0s+uQRSIdanYmfwvd+YLfWbUukk+WgCuzJwkXwEEvOZF+NdDHGCGSHC3XY+hFxuN2+aIGK/oP338wV+RWAeNv3vwsGex6l0vSR8DQJ68e+sFfuvefvj7dL6e3f/79dqPdfH1hlyg/gLdi9hrol5dTgjB7ZrLhhqSPH0SHq6X3zp1Bz/Xp2SiA2F1CAjH9xGPim0w2elm4BN4FYBpd+kIhRmULsvTomUoP6awTQC/B6bnngSB0n+K7KlSC4XSFQ/cHvIVhAq5IilYkutM4BU2sila+Ax02VDpdSjzu0p2yYw88QW8Eps/z+IIx8lB0ex/FDxQSjJ5TqEQDzeh3vAZL14tWwCeIHeADJmZwV6nVRsBz5z49mdJJih+dmRZiAtclLEomUk3RRZclRNSWeEOvY6NofAWyFTlkpy8ZvFWFyRY5aJtcEbAlNU+HzmBDDxhydkCRKqKojJ5sbhQ5KhpZbjuet1rOMAuGU9Uqy4mG09qym3acdtO1ju1mV9DWURiucqmKhkuBuXfLWSe+Pzq+N7W+8b3dm+Vn7cT3Zse8NTxOu2kN+d5sDW/GcRrO7NrBUCs8jBPf95zvLatvfG81IHzo0LCbnSJMFmiOfOBdZdJRcYVmOu8QCthUfIOEbJjRwYqgIpioUfHmIX/yGA12bvLTyZoNnpxt2FnyXaMvuNta56snRCs8hXWKeqJIAJ7DWvbYQh8YeoDQNVD4flXTGF96iTHY5BQC5PokzI18EwkyQKklB0JlOZ3rhvr60CxBKPkGGaDSR/kJqtJ7w/HJvDf2KuyOd0erwqk/CsvpVseWU/vjV+xmOa1rzHGGODkWR+RYpF5DbxwLrT+Zw1xKv8ECbLj+jNYMV+HYH4XhdKNry5mC5SZ3NxitNzeATBdb2ctHBMrNyJiGbiz6KDKmOwXeJVvfJHJz09XuwRmJLqBDuf78Y+wC0+iy7CYflowORDx6Q+LRzbZAILpMk7t/VpCenOa//YpV9/MvBrR3z9CDBPknABwghd09AMRU7KeY/+td2BMG9pfW6h4DFwIGPl79qPYCcrNbntEnRAhaxuYGmFxGLStUiuioXHbtelkKy+EaTx6aPnMVZv1hPsulnGtpZutxkE9svTrNVddeIk1eaVvmNzeBfIp3y2cJCajSdqGWw9gkIccuqktklfyONFXDB0qeWRhoXzktXYySKcISormLpn2gWV5EAE+Ucqw5iZcifgHYEQB4gPRqTCY3KHSJiwrxKEf8u5LC0nWc+NZCBFteI6lmgemUTDBii2iimz+PZGm+VmsK+S2Ibzddm2Zt0nStUotyiX476VpdjNxl0Mbw+wqGpAtoV6IsI9oDsqy8ltAQmmYn2NTtItZ47L4NmxL9IjbFfUCrvvpg/C3WJMYYAuoDakq1o9gBlrVDoNJuSph6F6i0Sg4DZ8RtqJTot8SYYrEhYsw3MPpiv90Cfx7B6g8KjvXvvwqupHgZdgIXuxyJSOBSry8hMVsxK6+WebN7w52Y8GS4u4XhyuvFfnwQDhs25TCjDxym78hh+kE4TMybpl7fdQSKG+r5hVWuXxggPxSzq135fq17cUZDsGl98OIMCQFK9FvCmpijbYS1DiPoboLZpkjrBGo8GE2ho9VDTaLfDtQMEWqfwTO8DyrRJQqdDZ11AXAErkkRH83zJDjqWQBZMwMzMx3XHA3MSTQWRR9PNA/a6xmwteqlX5X1O0jnCR/41HlyRJ0n6Ysvvek8MfvTeWLu9tJaZd1ENF1rPeVmfxo1d7NcNeoOaDlbO1LLGRddY06MYXm9/hojX4xhTwXb/ZN25wVbsyb67F85Da5d8pA7fswdZ7WF6ISXFvpZkDCb5vLsWjSdKeeKpRlFRn5VuXjXEMQo1yD0+hBEor8t3ce/DprNQthKJs8Uw5R1F8huG3GNkydmN9WGciOLJcnU1eu3E9KaYoNTxJUTN3zOs2QOOdb3FSJsbzlLdsRLqqCqwTqec/45p9bbpJw7RsuAbsRQ5Fgx/s1vW5XxcFtbmaGWpqDqjdZh1V7Wmi9YUz+fuLNZr7aythc894PlW0y926qcG5ZafsfoEFtMuQXDkFCCRF9SUUr7inhIaBUfp70dyBZjPwlqO2z6aBu13E2Qo1brJ2r1Uq7VtCVlgHr9/qK2ugL6/+RanoSQo3ZLGfSwvnvJdzIlvXAS/YP47n8/uO8/2KO/zNB5/Lh8Aydfxt/OaiD4gPBu3NlFM/L+X7lv2jdSac4t79HvvxX5olT/sMotaE17kYdmNTIP1ctm1yRPekeCe8udtA1UOd1WxLK7ILoYBjHsnKncY/5JdNtDCbq3gHJXGt92n4aeg/ADForWgu+wfVokv3zieW4QQnmaGYRBUvmbuetodYxmlIt5CngQNYZFf2keOPeJFf+xCDonT/7aLP+WJ66i/KtVxM37qARUzof4zs7IA9Nnkb3eIk9sN/o1SvKl/KTFX2nOzYlRMSevyGXQ0+z3bZMllf1KsH71Hw==</diagram><diagram id="znj-D5QoGOLx2BGCd-4G" name="Solution# 1 (Current fix)">7V1bd5s4EP41Pmf3ITnccR59Sbrdk+66cTdp+7KH2LJNg5Er5MTeX7/CiKtkBGlAuPVTYTLIMPpmNPpmoD19tN69Q85m9QHOgdfTlPmup497mqaqfYv8E0r2kcS+uooES+TOqVIqmLr/ASpUqHTrzkGQU8QQetjd5IUz6PtghnMyByH4kldbQC//qxtnCRjBdOZ4rPTBneNVJO1rdir/A7jLVfzLqkWfb+3EyvRJgpUzhy8ZkX7d00cIQhwdrXcj4IXGi+3yYO9G03v0fP/n5Pr91+27gfp1eBENdlPnkuQREPDxq4e2Pn5bu5u76Yf7T/8+WK5tBndX9BLl2fG21F49fTCYYYjoM+N9bEjy+JvwcLv2bt0F8FyfnA03ALlrgAEif/GoeJLKhi8rF4PpxpmFl74QiBHZCq89cqaSQzLr2CGXoOTc85xN4D4eflUhEgRmWxS4z+AOBBG4Qinc4vCXRgloDqpw68/BnA6VTJdyGHftzuix5zwCb+jMnpaHC0bQg+HP+/DwQAFG8CmBSjjQgtzjjbN2vdAD7gGaO75DxRTuKrHa0PHcpU9OZmSSDo9OTQsQBrsCFgUTqSboIm4JILEl2pPr6CiaQWeNemSfnr6k8FYVKltloG3ZVOhQl1omQ6ewIQcUOTVQpLIoKqInnRtFjIpKljuO56OWM8yc4VSVZznWcFpTdtNO0266Jtlu+onajeuprN0SXL654SxOvD8Fy1myPdU4r5Mnt06aWtfWSbMz3mfUWidNyd5nnabdtIrrpNmU3ezTtJspO9r3z9H+5KK9ZXUt2l+JvQ/Ml2BKTyHCK7iEvuNdp9Jh3j9TnVsIN3QmvgGM99TmzhbDPJaITdH+c/bkSzjYpRmfjnd08OhsT8+iew1vsJ6nU98J4BbNQIle7NDYQUtQNqB6ZN4R8BxMPCB3e7xJPFw6QMjZZxQ20PVxkBl5EgpSOKmF5EG9MrOAEOrrfbMAoOgOUjglj/ID++7upPP9WgmFLXlhVLvDWNQynG7JNlx3KItahtOkI47DWZxzio7nFEnC0JmcQuUQEZL8L0J0VQes6H9GY4brzt67nuF0oz3LXd89/T2YgZcJujWX5tDdm4HBKxGdhOEsXfIuUmVpi/F0guBuP3HwbHU07PsQA7EZaYgmAUEfhsZ0Z443oIERh3uDJEx6YIHDC8hQrr/8dNg3XFjM3qLdKN5SxOZigAMC3WwKBCwHM55+3AJycp7/5muj8uef5ZKmT8ADGPpnALRA+ssHAEsD3R/if3nuf8bA21GB0jGgsTnUp+tnfhaQmd3ijD5CjOH6YG4H4UHYHEWkkIway25cL+X95rHGowdnT7EKtX4/Sw0ql1pCB37pZdnAV3ODZY1MYsrvSLkgM4HxFNdjARnarrBcqMX9f0Rj0ovK6L9C3qEWd4DRMzMDvRUTqLGEFkFYFGim4bT3NMsLA8AjCTnWEh9cEb04aM4AsAVO+hBMJjBwsQtzG/kY8bcFhbU7nx9+mtn6F30k0cxFOiUVDKkTjXXzx5EsZrmNqpDnI75ZkjuhuxKSWylFuUC/GZI73m/WgDYC37cgwDKgzUVZGmhbjLJibFaEpn6kVadZbOp2Hmvx3v0YNgX6eWyy64DGv7q1+M3SwyMEHJIDago/UZSAZa0NVJpVA6YUVFqFhCGOiMdQKdBvKGKyJHEYMd+B8MZ+u3P8ZQir9wQcu99/FlwJ8WJJgYtd3IkI4FKuLwhitmJyrxZls2+GO5Zjp7i7A8HW68R63EoMs6rGMK0LMUyvGcP0VmIYS54nWd9NCIoJyfwCXuoXbKAfsOyqrNyv8SxOqwg2KVgrZmWGIAAK9BvCGsvRV8KaxB20FKhVRVpUa229Y+uqAB2tHGoC/YagxpYDHpwn8M+Giy5WON+TWWcAh8EO5/FRnSdBYbOHk3aBUDOTcc1hzxyHYxH0xURzr7lmC1vjuz6P9WulZUfjNHCeW3Y63rKTvGPVmZaduPrQgQaKCNFVGyhU2Q0Uene6NOtZjo+6Ni3HcqanYTnjSrLlzO68M1PPcqZpVrNcU0VOnaUw406HGwR9dvd/LnW//XInvdSt87nHjhYiwc7FnzPHXzLHaVUmPImLMt0s5cTTLtyaHdmZUTRdKJeKFbPH8Vr2qkJ73c2bUaze6OWbN4H+MaI0vh24WASgEQ5UZznQnQxkN424yrSTLqWyzbQAWQKOs1y/GTJA53OcYzd4ykbJDHKs71uI6dpyEa2IA6KgqpvdYc7jv8eh9S4qhI/gekMWYsDGWJY5yC5bXCahqaXMUAtTwHt7us9byxrLokt4wbG7WHRqKWvc4asWZsub9JVLw1KLL7W1scIUe1cMQUQQ6AtKccmKHO+lrfzjNLgAsQyjALQSu2Uaz4uqYlbrJmj1Akdt2oLySbl+d0HLUq2/bqSNyRsham0+aNtN3AuJkyloIRToy0vcjfMnnk6P7e937gVdszsv6Br1PrpTke3vN2U5g10D0jeliuH/L+iPiYMRs+yPeueZV3w7x5LOK5osu9zdDIFHK9Iv7pwYsWhWba80y4MNyaDV+FXIeENlviY/YZseCymvVSwNHencrpvo9AtdcJagH12gL0rFmb7h/EM1lwfFM1762SrPczdBheDqBJsoJ1m4u9ChmmseKcQri9M8onHiVXN1RI0x42nFq1eVQd7iq2I/FK/squGqHE7KpSKjCmKp9WKMQF/eZsrkfEFRBq75376TvaZWfSfQPLLBbxaEdqGkbOnlX74T6DdTOrFK3neNXzboUmt4ZzBlHXltKgl8fTXOAOh8XrREJBX4dktEJJXrS4x9fC7zV63qaeLErLGqHvd7Vq+IHD8x2RzTsdnAcfwzYHUqJBftlEiKbfy2IGcS6Av2ZQ1USMhp+n8IRerp/8SkX/8P</diagram></mxfile>
\ No newline at end of file diff --git a/ydb/core/blobstorage/dsproxy/dsproxy.h b/ydb/core/blobstorage/dsproxy/dsproxy.h index 27592e971f9..85406ecf3e6 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy.h @@ -28,7 +28,7 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); constexpr ui32 TypicalPartsInBlob = 6; constexpr ui32 TypicalDisksInSubring = 8; -constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinREALHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme +constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinREALHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme const TDuration ProxyConfigurationTimeout = TDuration::Seconds(20); const ui32 ProxyRetryConfigurationInitialTimeout = 200; @@ -51,11 +51,11 @@ const ui32 MaxRequestSize = 1000; const ui32 MaskSizeBits = 32; -constexpr bool DefaultEnablePutBatching = true; -constexpr bool DefaultEnableVPatch = false; - -constexpr bool WithMovingPatchRequestToStaticNode = true; - +constexpr bool DefaultEnablePutBatching = true; +constexpr bool DefaultEnableVPatch = false; + +constexpr bool WithMovingPatchRequestToStaticNode = true; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Common types //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -302,17 +302,17 @@ public: void CountPuts(const TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>>& q) { for (const auto& item : q) { ++GeneratedSubrequests; - GeneratedSubrequestBytes += item->GetBufferBytes(); + GeneratedSubrequestBytes += item->GetBufferBytes(); } } void CountPuts(const TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>>& q) { - for (const auto& item : q) { - ++GeneratedSubrequests; - GeneratedSubrequestBytes += item->GetBufferBytes(); - } - } - + for (const auto& item : q) { + ++GeneratedSubrequests; + GeneratedSubrequestBytes += item->GetBufferBytes(); + } + } + template<typename T> void SendToQueue(std::unique_ptr<T> event, ui64 cookie, NWilson::TTraceId traceId, bool timeStatsEnabled = false) { const TActorId queueId = GroupQueues->Send(*this, Info->GetTopology(), std::move(event), cookie, std::move(traceId), @@ -346,31 +346,31 @@ public: } TLogoBlobID GetBlobId(std::unique_ptr<TEvBlobStorage::TEvVPut> &ev) { - Y_VERIFY(ev->Record.HasBlobID()); - return LogoBlobIDFromLogoBlobID(ev->Record.GetBlobID()); - } - + Y_VERIFY(ev->Record.HasBlobID()); + return LogoBlobIDFromLogoBlobID(ev->Record.GetBlobID()); + } + TLogoBlobID GetBlobId(std::unique_ptr<TEvBlobStorage::TEvVMultiPut> &ev) { - Y_VERIFY(ev->Record.ItemsSize()); - return LogoBlobIDFromLogoBlobID(ev->Record.GetItems(0).GetBlobID()); - } - + Y_VERIFY(ev->Record.ItemsSize()); + return LogoBlobIDFromLogoBlobID(ev->Record.GetItems(0).GetBlobID()); + } + TLogoBlobID GetBlobId(std::unique_ptr<TEvBlobStorage::TEvVMovedPatch> &ev) { - Y_VERIFY(ev->Record.HasPatchedBlobId()); - return LogoBlobIDFromLogoBlobID(ev->Record.GetPatchedBlobId()); - } - + Y_VERIFY(ev->Record.HasPatchedBlobId()); + return LogoBlobIDFromLogoBlobID(ev->Record.GetPatchedBlobId()); + } + TLogoBlobID GetBlobId(std::unique_ptr<TEvBlobStorage::TEvVPatchStart> &ev) { - Y_VERIFY(ev->Record.HasOriginalBlobId()); - return LogoBlobIDFromLogoBlobID(ev->Record.GetOriginalBlobId()); - } - + Y_VERIFY(ev->Record.HasOriginalBlobId()); + return LogoBlobIDFromLogoBlobID(ev->Record.GetOriginalBlobId()); + } + TLogoBlobID GetBlobId(std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> &ev) { - Y_VERIFY(ev->Record.HasPatchedPartBlobId()); - return LogoBlobIDFromLogoBlobID(ev->Record.GetPatchedPartBlobId()); - } - - template <typename TEvent> + Y_VERIFY(ev->Record.HasPatchedPartBlobId()); + return LogoBlobIDFromLogoBlobID(ev->Record.GetPatchedPartBlobId()); + } + + template <typename TEvent> void SendToQueues(TDeque<std::unique_ptr<TEvent>> &events, bool timeStatsEnabled) { for (auto& request : events) { WILSON_TRACE_FROM_ACTOR(*TlsActivationContext, *this, &TraceId, EvVPutSent); @@ -379,7 +379,7 @@ public: CountEvent(*request); const ui64 cyclesPerUs = NHPTimer::GetCyclesPerSecond() / 1000000; request->Record.MutableTimestamps()->SetSentByDSProxyUs(GetCycleCountFast() / cyclesPerUs); - TLogoBlobID id = GetBlobId(request); + TLogoBlobID id = GetBlobId(request); TVDiskID vDiskId = VDiskIDFromVDiskID(request->Record.GetVDiskID()); LWTRACK(DSProxyPutVPutIsSent, request->Orbit, Info->GetFailDomainOrderNumber(vDiskId), Info->GroupID, id.Channel(), id.PartId(), id.ToString(), id.BlobSize()); @@ -408,7 +408,7 @@ public: Derived().Send(GetProxyActorId(), new TEvDeathNote(Responsiveness)); for (const auto& [queueId, numUnrepliedRequests] : InvolvedQueues) { Derived().Send(queueId, new TEvPruneQueue); - } + } TActorBootstrapped<TDerived>::PassAway(); } @@ -429,14 +429,14 @@ public: XX(Range) XX(CollectGarbage) XX(Status) - XX(Patch) + XX(Patch) default: Y_FAIL(); #undef XX } - // ensure that we are dying for the first time - Y_VERIFY(!Dead); + // ensure that we are dying for the first time + Y_VERIFY(!Dead); if (RequestHandleClass && PoolCounters) { PoolCounters->GetItem(*RequestHandleClass, RequestBytes).Register( RequestBytes, GeneratedSubrequests, GeneratedSubrequestBytes, Timer.Passed()); @@ -446,7 +446,7 @@ public: if (timeStats) { Derived().Send(proxyId, new TEvTimeStats(std::move(*timeStats))); } - + if (LatencyQueueKind) { Derived().Send(proxyId, new TEvLatencyReport(*LatencyQueueKind, now - RequestStartTime)); } @@ -456,14 +456,14 @@ public: static_cast<TEvBlobStorage::TEvGetResult&>(*ev).Sent = now; } - // send the reply to original request sender + // send the reply to original request sender Derived().Send(source, ev.release(), 0, cookie, std::move(traceId)); }; void SendResponse(std::unique_ptr<IEventBase>&& ev, TBlobStorageGroupProxyTimeStats *timeStats = nullptr) { SendResponse(std::move(ev), timeStats, Source, Cookie, std::move(TraceId)); - } - + } + static double GetStartTime(const NKikimrBlobStorage::TTimestamps& timestamps) { return timestamps.GetSentByDSProxyUs() / 1e6; } @@ -533,32 +533,32 @@ IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupIn const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPut *ev, ui64 cookie, NWilson::TTraceId traceId, bool timeStatsEnabled, TDiskResponsivenessTracker::TPerDiskStatsPtr stats, - TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - bool enableRequestMod3x3ForMinLatecy); + TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + bool enableRequestMod3x3ForMinLatecy); -IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, +IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &ev, - bool timeStatsEnabled, - TDiskResponsivenessTracker::TPerDiskStatsPtr stats, - TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, - bool enableRequestMod3x3ForMinLatecy); - + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &ev, + bool timeStatsEnabled, + TDiskResponsivenessTracker::TPerDiskStatsPtr stats, + TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, + bool enableRequestMod3x3ForMinLatecy); + IActor* CreateBlobStorageGroupGetRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvGet *ev, ui64 cookie, NWilson::TTraceId traceId, TNodeLayoutInfoPtr&& nodeLayout, - TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - bool isVMultiPutMode); - -IActor* CreateBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, - const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, - ui64 cookie, NWilson::TTraceId traceId, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - const TActorId &proxyId, bool useVPatch); - + TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + bool isVMultiPutMode); + +IActor* CreateBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, + const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, + ui64 cookie, NWilson::TTraceId traceId, TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + const TActorId &proxyId, bool useVPatch); + IActor* CreateBlobStorageGroupMultiGetRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvGet *ev, @@ -612,9 +612,9 @@ IActor* CreateBlobStorageGroupEjectedProxy(ui32 groupId, TIntrusivePtr<TDsProxyN IActor* CreateBlobStorageGroupProxyConfigured(TIntrusivePtr<TBlobStorageGroupInfo>&& info, bool forceWaitAllDrives, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, TIntrusivePtr<TStoragePoolCounters>&& storagePoolCounters, const TControlWrapper &enablePutBatching, - const TControlWrapper &enableVPatch); + const TControlWrapper &enableVPatch); IActor* CreateBlobStorageGroupProxyUnconfigured(ui32 groupId, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.cpp index eb6d8394f02..a10c48ec0b8 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.cpp @@ -35,28 +35,28 @@ void TBlobState::Init(const TLogoBlobID &id, const TBlobStorageGroupInfo &info) Disks[i].OrderNumber = info.GetOrderNumber(vdisksId[i]); Disks[i].DiskParts.resize(info.Type.TotalPartCount()); } - IsChanged = true; + IsChanged = true; } void TBlobState::AddNeeded(ui64 begin, ui64 size) { Y_VERIFY(bool(Id)); Whole.Needed.Add(begin, begin + size); Whole.NotHere.Add(begin, begin + size); - IsChanged = true; + IsChanged = true; } void TBlobState::AddPartToPut(ui32 partIdx, TString &partData) { Y_VERIFY(bool(Id)); Y_VERIFY(partIdx < Parts.size()); Parts[partIdx].AddPartToPut(partData); - IsChanged = true; + IsChanged = true; } -void TBlobState::MarkBlobReadyToPut(ui8 blobIdx) { +void TBlobState::MarkBlobReadyToPut(ui8 blobIdx) { Y_VERIFY(WholeSituation == ESituation::Unknown || WholeSituation == ESituation::Present); WholeSituation = ESituation::Present; - BlobIdx = blobIdx; - IsChanged = true; + BlobIdx = blobIdx; + IsChanged = true; } bool TBlobState::Restore(const TBlobStorageGroupInfo &info) { @@ -110,7 +110,7 @@ void TBlobState::AddResponseData(const TBlobStorageGroupInfo &info, const TLogoB if (partSize) { Parts[partIdx].AddResponseData(partSize, shift, data); } - IsChanged = true; + IsChanged = true; // Mark part as present for the disk bool isFound = false; for (ui32 diskIdx = 0; diskIdx < Disks.size(); ++diskIdx) { @@ -135,7 +135,7 @@ void TBlobState::AddNoDataResponse(const TBlobStorageGroupInfo &info, const TLog Y_UNUSED(info); Y_VERIFY(id.PartId() != 0); ui32 partIdx = id.PartId() - 1; - IsChanged = true; + IsChanged = true; // Mark part as absent for the disk bool isFound = false; for (ui32 diskIdx = 0; diskIdx < Disks.size(); ++diskIdx) { @@ -157,7 +157,7 @@ void TBlobState::AddPutOkResponse(const TBlobStorageGroupInfo &info, const TLogo Y_UNUSED(info); Y_VERIFY(id.PartId() != 0); ui32 partIdx = id.PartId() - 1; - IsChanged = true; + IsChanged = true; // Mark part as put ok for the disk bool isFound = false; for (ui32 diskIdx = 0; diskIdx < Disks.size(); ++diskIdx) { @@ -178,7 +178,7 @@ void TBlobState::AddErrorResponse(const TBlobStorageGroupInfo &info, const TLogo Y_UNUSED(info); Y_VERIFY(id.PartId() != 0); ui32 partIdx = id.PartId() - 1; - IsChanged = true; + IsChanged = true; // Mark part as error for the disk bool isFound = false; for (ui32 diskIdx = 0; diskIdx < Disks.size(); ++diskIdx) { @@ -200,7 +200,7 @@ void TBlobState::AddNotYetResponse(const TBlobStorageGroupInfo &info, const TLog Y_UNUSED(info); Y_VERIFY(id.PartId() != 0); ui32 partIdx = id.PartId() - 1; - IsChanged = true; + IsChanged = true; // Mark part as error for the disk bool isFound = false; for (ui32 diskIdx = 0; diskIdx < Disks.size(); ++diskIdx) { @@ -260,8 +260,8 @@ TString TBlobState::ToString() const { str << " Disks[" << i << "]# " << Disks[i].ToString(); str << Endl; } - str << " BlobIdx# " << (ui32)BlobIdx; - str << Endl; + str << " BlobIdx# " << (ui32)BlobIdx; + str << Endl; str << "}"; return str.Str(); } @@ -346,9 +346,9 @@ void TGroupDiskRequests::AddGet(const ui32 diskOrderNumber, const TLogoBlobID &i } void TGroupDiskRequests::AddPut(const ui32 diskOrderNumber, const TLogoBlobID &id, TString buffer, - TDiskPutRequest::EPutReason putReason, bool isHandoff, ui8 blobIdx) { + TDiskPutRequest::EPutReason putReason, bool isHandoff, ui8 blobIdx) { Y_VERIFY(diskOrderNumber < DiskRequestsForOrderNumber.size()); - DiskRequestsForOrderNumber[diskOrderNumber].PutsToSend.emplace_back(id, buffer, putReason, isHandoff, blobIdx); + DiskRequestsForOrderNumber[diskOrderNumber].PutsToSend.emplace_back(id, buffer, putReason, isHandoff, blobIdx); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -365,7 +365,7 @@ void TBlackboard::AddNeeded(const TLogoBlobID &id, ui32 inShift, ui32 inSize) { ui64 size = (inSize ? Min(ui64(inSize), maxSize) : maxSize); //Cerr << "size " << size << " shift " << shift << Endl; if (size > 0) { - TBlobState &state = BlobStates[id];; + TBlobState &state = BlobStates[id];; if (!bool(state.Id)) { state.Init(id, *Info); } @@ -389,90 +389,90 @@ void TBlackboard::AddPartToPut(const TLogoBlobID &id, ui32 partIdx, TString &par state.AddPartToPut(partIdx, partData); } -void TBlackboard::MarkBlobReadyToPut(const TLogoBlobID &id, ui8 blobIdx) { +void TBlackboard::MarkBlobReadyToPut(const TLogoBlobID &id, ui8 blobIdx) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() == 0); Y_VERIFY(id.BlobSize() != 0); - TBlobState &state = BlobStates[id];; + TBlobState &state = BlobStates[id];; if (!bool(state.Id)) { state.Init(id, *Info); } - state.MarkBlobReadyToPut(blobIdx); -} - -void TBlackboard::MoveBlobStateToDone(const TLogoBlobID &id) { - Y_VERIFY(bool(id)); - Y_VERIFY(id.PartId() == 0); - Y_VERIFY(id.BlobSize() != 0); - auto it = BlobStates.find(id); - if (it == BlobStates.end()) { - auto doneIt = DoneBlobStates.find(id); - const char *errorMsg = doneIt == DoneBlobStates.end() ? - "This blobId is not in BlobStates or in DoneBlobStates" : - "This blobId is already in DoneBlobStates"; - Y_VERIFY_S(false, errorMsg << " BlobId# " << id << " Blackboard# " << ToString()); - } else { - if (!it->second.IsDone) { - DoneCount++; - it->second.IsDone = true; - } - auto node = BlobStates.extract(it); - DoneBlobStates.insert(std::move(node)); - } -} - + state.MarkBlobReadyToPut(blobIdx); +} + +void TBlackboard::MoveBlobStateToDone(const TLogoBlobID &id) { + Y_VERIFY(bool(id)); + Y_VERIFY(id.PartId() == 0); + Y_VERIFY(id.BlobSize() != 0); + auto it = BlobStates.find(id); + if (it == BlobStates.end()) { + auto doneIt = DoneBlobStates.find(id); + const char *errorMsg = doneIt == DoneBlobStates.end() ? + "This blobId is not in BlobStates or in DoneBlobStates" : + "This blobId is already in DoneBlobStates"; + Y_VERIFY_S(false, errorMsg << " BlobId# " << id << " Blackboard# " << ToString()); + } else { + if (!it->second.IsDone) { + DoneCount++; + it->second.IsDone = true; + } + auto node = BlobStates.extract(it); + DoneBlobStates.insert(std::move(node)); + } +} + void TBlackboard::AddPutOkResponse(const TLogoBlobID &id, ui32 orderNumber) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() != 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); state.AddPutOkResponse(*Info, id, orderNumber); } void TBlackboard::AddResponseData(const TLogoBlobID &id, ui32 orderNumber, ui32 shift, TString &data) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() != 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); state.AddResponseData(*Info, id, orderNumber, shift, data); } void TBlackboard::AddNoDataResponse(const TLogoBlobID &id, ui32 orderNumber) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() != 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); state.AddNoDataResponse(*Info, id, orderNumber); } void TBlackboard::AddNotYetResponse(const TLogoBlobID &id, ui32 orderNumber) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() != 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); state.AddNotYetResponse(*Info, id, orderNumber); } void TBlackboard::AddErrorResponse(const TLogoBlobID &id, ui32 orderNumber) { Y_VERIFY(bool(id)); Y_VERIFY(id.PartId() != 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); state.AddErrorResponse(*Info, id, orderNumber); } EStrategyOutcome TBlackboard::RunStrategy(TLogContext &logCtx, const IStrategy& s, TBatchedVec<TBlobStates::value_type*> *finished) { IStrategy& temp = const_cast<IStrategy&>(s); // better UX - Y_VERIFY(BlobStates.size()); - EStrategyOutcome outcome = EStrategyOutcome::IN_PROGRESS; + Y_VERIFY(BlobStates.size()); + EStrategyOutcome outcome = EStrategyOutcome::IN_PROGRESS; for (auto it = BlobStates.begin(); it != BlobStates.end(); ++it) { auto& blob = it->second; - if (!blob.IsChanged) { - continue; - } - blob.IsChanged = false; + if (!blob.IsChanged) { + continue; + } + blob.IsChanged = false; // recalculate blob outcome if it is not yet determined switch (auto res = temp.Process(logCtx, blob, *Info, *this, GroupDiskRequests)) { case EStrategyOutcome::IN_PROGRESS: - if (blob.IsDone) { - DoneCount--; - blob.IsDone = false; - } + if (blob.IsDone) { + DoneCount--; + blob.IsDone = false; + } outcome = EStrategyOutcome::IN_PROGRESS; break; @@ -484,17 +484,17 @@ EStrategyOutcome TBlackboard::RunStrategy(TLogContext &logCtx, const IStrategy& if (finished) { finished->push_back(&*it); } - if (outcome.ErrorReason) { + if (outcome.ErrorReason) { outcome.ErrorReason += " && "; outcome.ErrorReason += res.ErrorReason; - } else { - outcome.ErrorReason = res.ErrorReason; + } else { + outcome.ErrorReason = res.ErrorReason; } } - if (!blob.IsDone) { - DoneCount++; - blob.IsDone = true; - } + if (!blob.IsDone) { + DoneCount++; + blob.IsDone = true; + } break; case EStrategyOutcome::DONE: @@ -504,36 +504,36 @@ EStrategyOutcome TBlackboard::RunStrategy(TLogContext &logCtx, const IStrategy& finished->push_back(&*it); } } - if (!blob.IsDone) { - DoneCount++; - blob.IsDone = true; - } + if (!blob.IsDone) { + DoneCount++; + blob.IsDone = true; + } break; - } - } - if (DoneCount == (BlobStates.size() + DoneBlobStates.size())) { - outcome = EStrategyOutcome::DONE; + } } + if (DoneCount == (BlobStates.size() + DoneBlobStates.size())) { + outcome = EStrategyOutcome::DONE; + } return outcome; } -TBlobState& TBlackboard::GetState(const TLogoBlobID &id) { +TBlobState& TBlackboard::GetState(const TLogoBlobID &id) { Y_VERIFY(bool(id)); - TLogoBlobID fullId = id.FullID(); - auto it = BlobStates.find(fullId); - if (it == BlobStates.end()) { - it = DoneBlobStates.find(fullId); - Y_VERIFY_S(it != DoneBlobStates.end(), "The blob was not found in BlobStates and DoneBlobStates" - << " blobId# " << fullId - << " BlackBoard# " << ToString()); - } - TBlobState &state = it->second; + TLogoBlobID fullId = id.FullID(); + auto it = BlobStates.find(fullId); + if (it == BlobStates.end()) { + it = DoneBlobStates.find(fullId); + Y_VERIFY_S(it != DoneBlobStates.end(), "The blob was not found in BlobStates and DoneBlobStates" + << " blobId# " << fullId + << " BlackBoard# " << ToString()); + } + TBlobState &state = it->second; return state; } ssize_t TBlackboard::AddPartMap(const TLogoBlobID &id, ui32 diskOrderNumber, ui32 requestIndex) { Y_VERIFY(id); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); ssize_t ret = state.PartMap.size(); state.PartMap.emplace_back(TEvBlobStorage::TEvGetResult::TPartMapItem{ diskOrderNumber, @@ -548,7 +548,7 @@ ssize_t TBlackboard::AddPartMap(const TLogoBlobID &id, ui32 diskOrderNumber, ui3 void TBlackboard::ReportPartMapStatus(const TLogoBlobID &id, ssize_t partMapIndex, ui32 responseIndex, NKikimrProto::EReplyStatus status) { Y_VERIFY(id); Y_VERIFY(partMapIndex >= 0); - TBlobState &state = GetState(id); + TBlobState &state = GetState(id); Y_VERIFY(static_cast<size_t>(partMapIndex) < state.PartMap.size()); TEvBlobStorage::TEvGetResult::TPartMapItem &item = state.PartMap[partMapIndex]; Y_VERIFY(item.ResponseIndex == responseIndex || item.ResponseIndex == Max<ui32>()); @@ -579,7 +579,7 @@ TString TBlackboard::ToString() const { TStringStream str; str << "{BlobStates size# " << BlobStates.size(); str << Endl; - str << " Data# {"; + str << " Data# {"; str << Endl; for (auto it = BlobStates.begin(); it != BlobStates.end(); ++it) { str << "{id# " << it->first.ToString() << " state# {" << it->second.ToString() << "}}"; @@ -587,15 +587,15 @@ TString TBlackboard::ToString() const { } str << "}"; str << Endl; - str << " DoneBlobStates size # " << DoneBlobStates.size(); - str << Endl; - str << " DoneData# {"; - for (auto &kv : DoneBlobStates) { - str << "{id# " << kv.first.ToString() << " state# {" << kv.second.ToString() << "}}"; - str << Endl; - } - str << "}"; - str << Endl; + str << " DoneBlobStates size # " << DoneBlobStates.size(); + str << Endl; + str << " DoneData# {"; + for (auto &kv : DoneBlobStates) { + str << "{id# " << kv.first.ToString() << " state# {" << kv.second.ToString() << "}}"; + str << Endl; + } + str << "}"; + str << Endl; // ... str << " PutHandleClass# " << EPutHandleClass_Name(PutHandleClass); str << Endl; diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h index fe926648a83..f0ae439442e 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h @@ -2,7 +2,7 @@ #include "defs.h" #include "dsproxy.h" - + #include <ydb/core/blobstorage/base/batched_vec.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/util/fragmented_buffer.h> @@ -77,15 +77,15 @@ struct TBlobState { TStackVec<TState, TypicalPartsInBlob> Parts; TStackVec<TDisk, TypicalDisksInSubring> Disks; TVector<TEvBlobStorage::TEvGetResult::TPartMapItem> PartMap; - ui8 BlobIdx; - NKikimrProto::EReplyStatus Status = NKikimrProto::UNKNOWN; - bool IsChanged = false; - bool IsDone = false; + ui8 BlobIdx; + NKikimrProto::EReplyStatus Status = NKikimrProto::UNKNOWN; + bool IsChanged = false; + bool IsDone = false; void Init(const TLogoBlobID &id, const TBlobStorageGroupInfo &Info); void AddNeeded(ui64 begin, ui64 size); void AddPartToPut(ui32 partIdx, TString &partData); - void MarkBlobReadyToPut(ui8 blobIdx = 0); + void MarkBlobReadyToPut(ui8 blobIdx = 0); bool Restore(const TBlobStorageGroupInfo &info); void AddResponseData(const TBlobStorageGroupInfo &info, const TLogoBlobID &id, ui32 diskIdxInSubring, ui32 shift, TString &data); @@ -126,14 +126,14 @@ struct TDiskPutRequest { TString Buffer; EPutReason Reason; bool IsHandoff; - ui8 BlobIdx; + ui8 BlobIdx; - TDiskPutRequest(const TLogoBlobID &id, TString buffer, EPutReason reason, bool isHandoff, ui8 blobIdx = 0) + TDiskPutRequest(const TLogoBlobID &id, TString buffer, EPutReason reason, bool isHandoff, ui8 blobIdx = 0) : Id(id) , Buffer(std::move(buffer)) , Reason(reason) , IsHandoff(isHandoff) - , BlobIdx(blobIdx) + , BlobIdx(blobIdx) {} }; @@ -151,7 +151,7 @@ struct TGroupDiskRequests { void AddGet(const ui32 diskOrderNumber, const TLogoBlobID &id, const TIntervalSet<i32> &intervalSet); void AddGet(const ui32 diskOrderNumber, const TLogoBlobID &id, const ui32 shift, const ui32 size); void AddPut(const ui32 diskOrderNumber, const TLogoBlobID &id, TString buffer, - TDiskPutRequest::EPutReason putReason, bool isHandoff, ui8 blobIdx = 0); + TDiskPutRequest::EPutReason putReason, bool isHandoff, ui8 blobIdx = 0); }; struct TBlackboard; @@ -171,50 +171,50 @@ struct TBlackboard { using TBlobStates = TMap<TLogoBlobID, TBlobState>; TBlobStates BlobStates; - TBlobStates DoneBlobStates; + TBlobStates DoneBlobStates; TGroupDiskRequests GroupDiskRequests; TIntrusivePtr<TBlobStorageGroupInfo> Info; TIntrusivePtr<TGroupQueues> GroupQueues; // To obtain FlowRecords only EAccelerationMode AccelerationMode; const NKikimrBlobStorage::EPutHandleClass PutHandleClass; const NKikimrBlobStorage::EGetHandleClass GetHandleClass; - const bool IsAllRequestsTogether; - ui64 DoneCount = 0; + const bool IsAllRequestsTogether; + ui64 DoneCount = 0; TBlackboard(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &groupQueues, - NKikimrBlobStorage::EPutHandleClass putHandleClass, NKikimrBlobStorage::EGetHandleClass getHandleClass, - bool isAllRequestsTogether = true) + NKikimrBlobStorage::EPutHandleClass putHandleClass, NKikimrBlobStorage::EGetHandleClass getHandleClass, + bool isAllRequestsTogether = true) : GroupDiskRequests(info->GetTotalVDisksNum()) , Info(info) , GroupQueues(groupQueues) , AccelerationMode(AccelerationModeSkipOneSlowest) , PutHandleClass(putHandleClass) , GetHandleClass(getHandleClass) - , IsAllRequestsTogether(isAllRequestsTogether) + , IsAllRequestsTogether(isAllRequestsTogether) {} void AddNeeded(const TLogoBlobID &id, ui32 inShift, ui32 inSize); void AddPartToPut(const TLogoBlobID &id, ui32 partIdx, TString &partData); - void MarkBlobReadyToPut(const TLogoBlobID &id, ui8 blobIdx = 0); - void MoveBlobStateToDone(const TLogoBlobID &id); + void MarkBlobReadyToPut(const TLogoBlobID &id, ui8 blobIdx = 0); + void MoveBlobStateToDone(const TLogoBlobID &id); void AddResponseData(const TLogoBlobID &id, ui32 orderNumber, ui32 shift, TString &data); void AddPutOkResponse(const TLogoBlobID &id, ui32 orderNumber); void AddNoDataResponse(const TLogoBlobID &id, ui32 orderNumber); void AddErrorResponse(const TLogoBlobID &id, ui32 orderNumber); void AddNotYetResponse(const TLogoBlobID &id, ui32 orderNumber); EStrategyOutcome RunStrategy(TLogContext &logCtx, const IStrategy& s, TBatchedVec<TBlobStates::value_type*> *finished = nullptr); - TBlobState& GetState(const TLogoBlobID &id); + TBlobState& GetState(const TLogoBlobID &id); ssize_t AddPartMap(const TLogoBlobID &id, ui32 diskOrderNumber, ui32 requestIndex); void ReportPartMapStatus(const TLogoBlobID &id, ssize_t partMapIndex, ui32 responseIndex, NKikimrProto::EReplyStatus status); void GetWorstPredictedDelaysNs(const TBlobStorageGroupInfo &info, TGroupQueues &groupQueues, NKikimrBlobStorage::EVDiskQueueId queueId, ui64 *outWorstNs, ui64 *outNextToWorstNs, i32 *outWorstOrderNumber) const; TString ToString() const; - void ChangeAll() { - for (auto &[id, blob] : BlobStates) { - blob.IsChanged = true; - } - } + void ChangeAll() { + for (auto &[id, blob] : BlobStates) { + blob.IsChanged = true; + } + } }; }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_block.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_block.cpp index dba0a2fd434..58130fdb1a7 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_block.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_block.cpp @@ -148,7 +148,7 @@ public: {} void Bootstrap() { - A_LOG_DEBUG_S("DSPB05", "bootstrap" + A_LOG_DEBUG_S("DSPB05", "bootstrap" << " ActorId# " << SelfId() << " Group# " << Info->GroupID << " TabletId# " << TabletId diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_get.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_get.cpp index 9b08b418619..0f5c32e24bc 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_get.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_get.cpp @@ -57,8 +57,8 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt bool IsPutAccelerated = false; bool IsPutAccelerateScheduled = false; - const bool IsVMultiPutMode = false; - + const bool IsVMultiPutMode = false; + void Handle(TEvAccelerateGet::TPtr &ev) { RootCauseTrack.OnAccelerate(ev->Get()->CauseIdx); AccelerateGet(); @@ -76,19 +76,19 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt IsGetAccelerated = true; TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; - if (IsVMultiPutMode) { + if (IsVMultiPutMode) { TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts; - GetImpl.AccelerateGet(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vMultiPuts); + GetImpl.AccelerateGet(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vMultiPuts); *Mon->NodeMon->AccelerateEvVMultiPutCount += vMultiPuts.size(); *Mon->NodeMon->AccelerateEvVGetCount += vGets.size(); SendVGetsAndVPuts(vGets, vMultiPuts); - } else { + } else { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - GetImpl.AccelerateGet(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vPuts); + GetImpl.AccelerateGet(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vPuts); *Mon->NodeMon->AccelerateEvVPutCount += vPuts.size(); *Mon->NodeMon->AccelerateEvVGetCount += vGets.size(); SendVGetsAndVPuts(vGets, vPuts); - } + } } void AcceleratePut() { @@ -98,19 +98,19 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt IsPutAccelerated = true; TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; - if (IsVMultiPutMode) { + if (IsVMultiPutMode) { TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts; - GetImpl.AcceleratePut(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vMultiPuts); + GetImpl.AcceleratePut(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vMultiPuts); *Mon->NodeMon->AccelerateEvVMultiPutCount += vMultiPuts.size(); *Mon->NodeMon->AccelerateEvVGetCount += vGets.size(); SendVGetsAndVPuts(vGets, vMultiPuts); - } else { + } else { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - GetImpl.AcceleratePut(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vPuts); + GetImpl.AcceleratePut(LogCtx, GetUnresponsiveDiskOrderNumber(), vGets, vPuts); *Mon->NodeMon->AccelerateEvVPutCount += vPuts.size(); *Mon->NodeMon->AccelerateEvVGetCount += vGets.size(); SendVGetsAndVPuts(vGets, vPuts); - } + } } template <typename TPutEvent> @@ -237,16 +237,16 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; ResponsesReceived++; - if (IsVMultiPutMode) { + if (IsVMultiPutMode) { TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts; - GetImpl.OnVGetResult(LogCtx, *ev->Get(), vGets, vMultiPuts, getResult); + GetImpl.OnVGetResult(LogCtx, *ev->Get(), vGets, vMultiPuts, getResult); SendVGetsAndVPuts(vGets, vMultiPuts); - } else { + } else { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - GetImpl.OnVGetResult(LogCtx, *ev->Get(), vGets, vPuts, getResult); + GetImpl.OnVGetResult(LogCtx, *ev->Get(), vGets, vPuts, getResult); SendVGetsAndVPuts(vGets, vPuts); - } - + } + if (getResult) { SendReplyAndDie(getResult); return; @@ -254,7 +254,7 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt Y_VERIFY(RequestsSent > ResponsesReceived, "RequestsSent# %" PRIu32 " ResponsesReceived# %" PRIu32 " GetImpl.DumpFullState# %s", RequestsSent, ResponsesReceived, GetImpl.DumpFullState().c_str()); - TryScheduleGetAcceleration(); + TryScheduleGetAcceleration(); if (IsPutStarted) { TrySchedulePutAcceleration(); } @@ -274,80 +274,80 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt ReplyAndDie(NKikimrProto::ERROR); } - TLogoBlobID GetFirstBlobId(TEvBlobStorage::TEvVPutResult::TPtr &ev) { - return LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); - } - - TLogoBlobID GetFirstBlobId(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { - Y_VERIFY(ev->Get()->Record.ItemsSize()); - return LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetItems(0).GetBlobID()); - } - - ui64 SumBlobSize(TEvBlobStorage::TEvVPutResult::TPtr &ev) { - return GetFirstBlobId(ev).BlobSize(); - } - - ui64 SumBlobSize(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { - ui64 sum = 0; - for (auto &item : ev->Get()->Record.GetItems()) { - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - sum += blobId.BlobSize(); - } - return sum; - } - + TLogoBlobID GetFirstBlobId(TEvBlobStorage::TEvVPutResult::TPtr &ev) { + return LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); + } + + TLogoBlobID GetFirstBlobId(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { + Y_VERIFY(ev->Get()->Record.ItemsSize()); + return LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetItems(0).GetBlobID()); + } + + ui64 SumBlobSize(TEvBlobStorage::TEvVPutResult::TPtr &ev) { + return GetFirstBlobId(ev).BlobSize(); + } + + ui64 SumBlobSize(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { + ui64 sum = 0; + for (auto &item : ev->Get()->Record.GetItems()) { + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + sum += blobId.BlobSize(); + } + return sum; + } + void Handle(TEvBlobStorage::TEvVPutResult::TPtr &ev) { ProcessReplyFromQueue(ev); HandleVPutResult<TEvBlobStorage::TEvVPut, TEvBlobStorage::TEvVPutResult>(ev); - } - + } + void Handle(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { ProcessReplyFromQueue(ev); HandleVPutResult<TEvBlobStorage::TEvVMultiPut, TEvBlobStorage::TEvVMultiPutResult>(ev); - } - + } + bool HandleVPutInnerErrorStatuses(TEvBlobStorage::TEvVPutResult::TPtr &ev, - TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) - { + TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) + { Y_UNUSED(ev, outGetResult); - return false; - } - + return false; + } + bool HandleVPutInnerErrorStatuses(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev, - TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) - { - const auto &record = ev->Get()->Record; - const NKikimrProto::EReplyStatus status = record.GetStatus(); - const TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) + { + const auto &record = ev->Get()->Record; + const NKikimrProto::EReplyStatus status = record.GetStatus(); + const TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); for (auto &item : record.GetItems()) { - Y_VERIFY(item.HasStatus()); - Y_VERIFY(item.HasBlobID()); - Y_VERIFY(item.HasCookie()); - NKikimrProto::EReplyStatus itemStatus = item.GetStatus(); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + Y_VERIFY(item.HasStatus()); + Y_VERIFY(item.HasBlobID()); + Y_VERIFY(item.HasCookie()); + NKikimrProto::EReplyStatus itemStatus = item.GetStatus(); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); Y_VERIFY(itemStatus != NKikimrProto::RACE); // we should get RACE for the whole request and handle it in CheckForTermErrors if (itemStatus == NKikimrProto::BLOCKED || itemStatus == NKikimrProto::DEADLINE) { - R_LOG_ERROR_S("BPG26", "Handle TEvVMultiPutResultItem" - << " blobId# " << blobId.ToString() - << " status# " << NKikimrProto::EReplyStatus_Name(status) - << " itemStatus# " << NKikimrProto::EReplyStatus_Name(itemStatus)); - TStringStream errorReason; - errorReason << "Got VMultiPutResult itemStatus# " << itemStatus << " from VDiskId# " << vDiskId; - ErrorReason = errorReason.Str(); - GetImpl.PrepareReply(itemStatus, LogCtx, outGetResult); - return true; - } else { - R_LOG_DEBUG_S("BPG27", "Handle TEvVMultiPutResultItem" - << " blobId# " << blobId.ToString() - << " status# " << NKikimrProto::EReplyStatus_Name(status) - << " itemStatus# " << NKikimrProto::EReplyStatus_Name(itemStatus)); - } - } - return false; - } - - template <typename TPutEvent, typename TPutEventResult> + R_LOG_ERROR_S("BPG26", "Handle TEvVMultiPutResultItem" + << " blobId# " << blobId.ToString() + << " status# " << NKikimrProto::EReplyStatus_Name(status) + << " itemStatus# " << NKikimrProto::EReplyStatus_Name(itemStatus)); + TStringStream errorReason; + errorReason << "Got VMultiPutResult itemStatus# " << itemStatus << " from VDiskId# " << vDiskId; + ErrorReason = errorReason.Str(); + GetImpl.PrepareReply(itemStatus, LogCtx, outGetResult); + return true; + } else { + R_LOG_DEBUG_S("BPG27", "Handle TEvVMultiPutResultItem" + << " blobId# " << blobId.ToString() + << " status# " << NKikimrProto::EReplyStatus_Name(status) + << " itemStatus# " << NKikimrProto::EReplyStatus_Name(itemStatus)); + } + } + return false; + } + + template <typename TPutEvent, typename TPutEventResult> void HandleVPutResult(typename TPutEventResult::TPtr &ev) { Y_VERIFY(ev->Get()->Record.HasStatus()); @@ -355,18 +355,18 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt const ui64 cyclesPerUs = NHPTimer::GetCyclesPerSecond() / 1000000; ev->Get()->Record.MutableTimestamps()->SetReceivedByDSProxyUs(GetCycleCountFast() / cyclesPerUs); - const auto &record = ev->Get()->Record; + const auto &record = ev->Get()->Record; const TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); TVDiskIdShort shortId(vDiskId); const NKikimrProto::EReplyStatus status = record.GetStatus(); NActors::NLog::EPriority priority = PriorityForStatusInbound(status); - A_LOG_LOG_S(priority != NActors::NLog::PRI_DEBUG, priority, "BPG30", "Handle VPuEventResult" + A_LOG_LOG_S(priority != NActors::NLog::PRI_DEBUG, priority, "BPG30", "Handle VPuEventResult" << " status# " << NKikimrProto::EReplyStatus_Name(status).data() << " node# " << GetVDiskActorId(shortId).NodeId()); - const TLogoBlobID blob = GetFirstBlobId(ev); - ui64 sumBlobSize = SumBlobSize(ev); - LWPROBE(DSProxyVDiskRequestDuration, TEvBlobStorage::EvVPut, sumBlobSize, blob.TabletID(), + const TLogoBlobID blob = GetFirstBlobId(ev); + ui64 sumBlobSize = SumBlobSize(ev); + LWPROBE(DSProxyVDiskRequestDuration, TEvBlobStorage::EvVPut, sumBlobSize, blob.TabletID(), Info->GroupID, blob.Channel(), Info->GetFailDomainOrderNumber(shortId), GetStartTime(record.GetTimestamps()), GetTotalTimeMs(record.GetTimestamps()), @@ -395,10 +395,10 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt ResponsesReceived++; if (HandleVPutInnerErrorStatuses(ev, getResult)) { - Y_VERIFY(getResult); + Y_VERIFY(getResult); SendReplyAndDie(getResult); - return; - } + return; + } GetImpl.OnVPutResult(LogCtx, *ev->Get(), vGets, vPuts, getResult); SendVGetsAndVPuts(vGets, vPuts); if (getResult) { @@ -412,25 +412,25 @@ class TBlobStorageGroupGetRequest : public TBlobStorageGroupRequestActor<TBlobSt SanityCheck(); // May Die } - void TryScheduleGetAcceleration() { - if (!IsGetAccelerateScheduled && !IsGetAccelerated) { - // Count VDisks that have requests in flight, if there is exactly one such VDisk, Accelerate - if (CountDisksWithActiveRequests() <= 1) { - ui64 timeToAccelerateUs = GetImpl.GetTimeToAccelerateGetNs(LogCtx) / 1000; - TInstant now = TActivationContext::Now(); - TDuration timeSinceStart = (now > StartTime) ? (now - StartTime) : TDuration::MilliSeconds(0); - if (timeSinceStart.MicroSeconds() < timeToAccelerateUs) { - ui64 causeIdx = RootCauseTrack.RegisterAccelerate(); - Schedule(TDuration::MicroSeconds(timeToAccelerateUs - timeSinceStart.MicroSeconds()), - new TEvAccelerateGet(causeIdx)); - IsGetAccelerateScheduled = true; - } else { - AccelerateGet(); - } - } - } - } - + void TryScheduleGetAcceleration() { + if (!IsGetAccelerateScheduled && !IsGetAccelerated) { + // Count VDisks that have requests in flight, if there is exactly one such VDisk, Accelerate + if (CountDisksWithActiveRequests() <= 1) { + ui64 timeToAccelerateUs = GetImpl.GetTimeToAccelerateGetNs(LogCtx) / 1000; + TInstant now = TActivationContext::Now(); + TDuration timeSinceStart = (now > StartTime) ? (now - StartTime) : TDuration::MilliSeconds(0); + if (timeSinceStart.MicroSeconds() < timeToAccelerateUs) { + ui64 causeIdx = RootCauseTrack.RegisterAccelerate(); + Schedule(TDuration::MicroSeconds(timeToAccelerateUs - timeSinceStart.MicroSeconds()), + new TEvAccelerateGet(causeIdx)); + IsGetAccelerateScheduled = true; + } else { + AccelerateGet(); + } + } + } + } + void TrySchedulePutAcceleration() { if (!IsPutAccelerateScheduled && !IsPutAccelerated) { // Count VDisks that have requests in flight, if there is exactly one such VDisk, Accelerate @@ -499,7 +499,7 @@ public: const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvGet *ev, ui64 cookie, NWilson::TTraceId traceId, TNodeLayoutInfoPtr&& nodeLayout, TMaybe<TGroupStat::EKind> latencyQueueKind, - TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, bool isVMultiPutMode) + TInstant now, TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, bool isVMultiPutMode) : TBlobStorageGroupRequestActor(info, state, mon, source, cookie, std::move(traceId), NKikimrServices::BS_PROXY_GET, ev->IsVerboseNoDataEnabled || ev->CollectDebugInfo, latencyQueueKind, now, storagePoolCounters, ev->RestartCounter) @@ -511,7 +511,7 @@ public: , RequestsSent(0) , ResponsesReceived(0) , ReportedBytes(0) - , IsVMultiPutMode(isVMultiPutMode) + , IsVMultiPutMode(isVMultiPutMode) { ReportBytes(sizeof(*this)); MaxSaneRequests = ev->QuerySize * info->Type.TotalPartCount() * (1 + info->Type.Handoff()) * 3; @@ -543,7 +543,7 @@ public: TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; GetImpl.GenerateInitialRequests(LogCtx, vGets); SendVGetsAndVPuts(vGets, vPuts); - TryScheduleGetAcceleration(); + TryScheduleGetAcceleration(); Y_VERIFY(RequestsSent > ResponsesReceived); Become(&TThis::StateWait); @@ -578,9 +578,9 @@ IActor* CreateBlobStorageGroupGetRequest(const TIntrusivePtr<TBlobStorageGroupIn const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvGet *ev, ui64 cookie, NWilson::TTraceId traceId, TNodeLayoutInfoPtr&& nodeLayout, TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, bool isVMultiPutMode) { + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, bool isVMultiPutMode) { return new TBlobStorageGroupGetRequest(info, state, source, mon, ev, cookie, std::move(traceId), - std::move(nodeLayout), latencyQueueKind, now, storagePoolCounters, isVMultiPutMode); + std::move(nodeLayout), latencyQueueKind, now, storagePoolCounters, isVMultiPutMode); } }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.cpp index 0583b454df6..ac8c9b66e39 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.cpp @@ -1,5 +1,5 @@ #include "dsproxy_get_impl.h" -#include "dsproxy_put_impl.h" +#include "dsproxy_put_impl.h" #include "dsproxy_strategy_base.h" #include "dsproxy_blackboard.h" @@ -211,10 +211,10 @@ TString TGetImpl::DumpFullState() const { str << Endl; str << " VPutResponses# " << VPutResponses; str << Endl; - str << " VMultiPutRequests# " << VMultiPutRequests; - str << Endl; - str << " VMultiPutResponses# " << VMultiPutResponses; - str << Endl; + str << " VMultiPutRequests# " << VMultiPutRequests; + str << Endl; + str << " VMultiPutResponses# " << VMultiPutResponses; + str << Endl; str << " IsNoData# " << IsNoData; str << Endl; @@ -304,71 +304,71 @@ void TGetImpl::PrepareVPuts(TLogContext &logCtx, TVDiskID vDiskId = Info->GetVDiskId(diskOrderNumber); for (ui32 idx = beginIdx; idx < endIdx; ++idx) { const TDiskPutRequest &put = requests.PutsToSend[idx]; - ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), - VPutRequests); + ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), + VPutRequests); auto vPut = std::make_unique<TEvBlobStorage::TEvVPut>(put.Id, put.Buffer, vDiskId, true, &cookie, Deadline, Blackboard.PutHandleClass); R_LOG_DEBUG_SX(logCtx, "BPG15", "Send put to orderNumber# " << diskOrderNumber << " idx# " << idx << " vPut# " << vPut->ToString()); outVPuts.push_back(std::move(vPut)); ++VPutRequests; - ReceivedVPutResponses.push_back(false); + ReceivedVPutResponses.push_back(false); } Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber].FirstUnsentPutIdx = endIdx; } } } -void TGetImpl::PrepareVPuts(TLogContext &logCtx, +void TGetImpl::PrepareVPuts(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &outVMultiPuts) { - for (ui32 diskOrderNumber = 0; diskOrderNumber < Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber.size(); - ++diskOrderNumber) { - const TDiskRequests &requests = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber]; - ui32 endIdx = requests.PutsToSend.size(); - ui32 beginIdx = requests.FirstUnsentPutIdx; - if (beginIdx < endIdx) { - TVDiskID vDiskId = Info->GetVDiskId(diskOrderNumber); - // set cookie after adding items + for (ui32 diskOrderNumber = 0; diskOrderNumber < Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber.size(); + ++diskOrderNumber) { + const TDiskRequests &requests = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber]; + ui32 endIdx = requests.PutsToSend.size(); + ui32 beginIdx = requests.FirstUnsentPutIdx; + if (beginIdx < endIdx) { + TVDiskID vDiskId = Info->GetVDiskId(diskOrderNumber); + // set cookie after adding items auto vMultiPut = std::make_unique<TEvBlobStorage::TEvVMultiPut>(vDiskId, Deadline, Blackboard.PutHandleClass, true, nullptr); - ui64 bytes = 0; - ui64 lastItemCount = 0; - for (ui32 idx = beginIdx; idx < endIdx; ++idx) { - const TDiskPutRequest &put = requests.PutsToSend[idx]; - ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), VMultiPutRequests); - ui64 itemSize = vMultiPut->Record.ItemsSize(); - if (itemSize == MaxBatchedPutRequests || bytes + put.Buffer.size() > MaxBatchedPutSize) { - vMultiPut->Record.SetCookie(TVMultiPutCookie(diskOrderNumber, lastItemCount, VMultiPutRequests)); - ++VMultiPutRequests; - ReceivedVMultiPutResponses.push_back(false); - R_LOG_DEBUG_SX(logCtx, "BPG16", "Send multiPut to orderNumber# " << diskOrderNumber << " count# " - << vMultiPut->Record.ItemsSize() << " vMultiPut# " << vMultiPut->ToString()); + ui64 bytes = 0; + ui64 lastItemCount = 0; + for (ui32 idx = beginIdx; idx < endIdx; ++idx) { + const TDiskPutRequest &put = requests.PutsToSend[idx]; + ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), VMultiPutRequests); + ui64 itemSize = vMultiPut->Record.ItemsSize(); + if (itemSize == MaxBatchedPutRequests || bytes + put.Buffer.size() > MaxBatchedPutSize) { + vMultiPut->Record.SetCookie(TVMultiPutCookie(diskOrderNumber, lastItemCount, VMultiPutRequests)); + ++VMultiPutRequests; + ReceivedVMultiPutResponses.push_back(false); + R_LOG_DEBUG_SX(logCtx, "BPG16", "Send multiPut to orderNumber# " << diskOrderNumber << " count# " + << vMultiPut->Record.ItemsSize() << " vMultiPut# " << vMultiPut->ToString()); outVMultiPuts.push_back(std::move(vMultiPut)); - // set cookie after adding items + // set cookie after adding items vMultiPut = std::make_unique<TEvBlobStorage::TEvVMultiPut>(vDiskId, Deadline, Blackboard.PutHandleClass, true, nullptr); - bytes = 0; - lastItemCount = 0; - } - bytes += put.Buffer.size(); - lastItemCount++; - vMultiPut->AddVPut(put.Id, put.Buffer, &cookie); - } - vMultiPut->Record.SetCookie(TVMultiPutCookie(diskOrderNumber, lastItemCount, VMultiPutRequests)); - ++VMultiPutRequests; - ReceivedVMultiPutResponses.push_back(false); - R_LOG_DEBUG_SX(logCtx, "BPG17", "Send multiPut to orderNumber# " << diskOrderNumber << " count# " - << vMultiPut->Record.ItemsSize() << " vMultiPut# " << vMultiPut->ToString()); + bytes = 0; + lastItemCount = 0; + } + bytes += put.Buffer.size(); + lastItemCount++; + vMultiPut->AddVPut(put.Id, put.Buffer, &cookie); + } + vMultiPut->Record.SetCookie(TVMultiPutCookie(diskOrderNumber, lastItemCount, VMultiPutRequests)); + ++VMultiPutRequests; + ReceivedVMultiPutResponses.push_back(false); + R_LOG_DEBUG_SX(logCtx, "BPG17", "Send multiPut to orderNumber# " << diskOrderNumber << " count# " + << vMultiPut->Record.ItemsSize() << " vMultiPut# " << vMultiPut->ToString()); outVMultiPuts.push_back(std::move(vMultiPut)); - Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber].FirstUnsentPutIdx = endIdx; - } - } -} - + Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber].FirstUnsentPutIdx = endIdx; + } + } +} + EStrategyOutcome TGetImpl::RunBoldStrategy(TLogContext &logCtx) { EStrategyOutcome outcome = Blackboard.RunStrategy(logCtx, TBoldStrategy(PhantomCheck)); if (outcome == EStrategyOutcome::DONE && MustRestoreFirst) { - Blackboard.ChangeAll(); + Blackboard.ChangeAll(); outcome = Blackboard.RunStrategy(logCtx, TRestoreStrategy()); } return outcome; @@ -384,7 +384,7 @@ EStrategyOutcome TGetImpl::RunMirror3of4Strategy(TLogContext &logCtx) { // run basic get strategy and, if blob restoration is required and we have successful get, restore the blob to full amount of parts EStrategyOutcome outcome = Blackboard.RunStrategy(logCtx, TMirror3of4GetStrategy()); if (outcome == EStrategyOutcome::DONE && MustRestoreFirst) { - Blackboard.ChangeAll(); + Blackboard.ChangeAll(); outcome = Blackboard.RunStrategy(logCtx, TPut3of4Strategy(TEvBlobStorage::TEvPut::TacticMaxThroughput)); } return outcome; @@ -416,17 +416,17 @@ void TGetImpl::OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVPutResult & ui32 orderNumber = Info->GetOrderNumber(shortId); const TLogoBlobID blob = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - Y_VERIFY(record.HasCookie()); - TBlobCookie cookie(record.GetCookie()); - Y_VERIFY(cookie.GetVDiskOrderNumber() == orderNumber); - Y_VERIFY(cookie.GetPartId() == blob.PartId()); - - ui64 requestIdx = cookie.GetRequestIdx(); - Y_VERIFY_S(!ReceivedVPutResponses[requestIdx], "the response is received twice" - << " Event# " << ev.ToString() - << " State# " << DumpFullState()); - ReceivedVPutResponses[requestIdx] = true; - + Y_VERIFY(record.HasCookie()); + TBlobCookie cookie(record.GetCookie()); + Y_VERIFY(cookie.GetVDiskOrderNumber() == orderNumber); + Y_VERIFY(cookie.GetPartId() == blob.PartId()); + + ui64 requestIdx = cookie.GetRequestIdx(); + Y_VERIFY_S(!ReceivedVPutResponses[requestIdx], "the response is received twice" + << " Event# " << ev.ToString() + << " State# " << DumpFullState()); + ReceivedVPutResponses[requestIdx] = true; + const NKikimrProto::EReplyStatus status = record.GetStatus(); ++VPutResponses; switch (status) { @@ -445,50 +445,50 @@ void TGetImpl::OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVPutResult & Step(logCtx, outVGets, outVPuts, outGetResult); } -void TGetImpl::OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVMultiPutResult &ev, +void TGetImpl::OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVMultiPutResult &ev, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &outVMultiPuts, - TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) { - const NKikimrBlobStorage::TEvVMultiPutResult &record = ev.Record; - Y_VERIFY(record.HasVDiskID()); - TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); - TVDiskIdShort shortId(vdisk); - ui32 orderNumber = Info->GetOrderNumber(shortId); - - Y_VERIFY(record.HasCookie()); - TVMultiPutCookie cookie(record.GetCookie()); - Y_VERIFY(cookie.GetVDiskOrderNumber() == orderNumber); - Y_VERIFY(cookie.GetItemCount() == record.ItemsSize()); - - ui64 requestIdx = cookie.GetRequestIdx(); - Y_VERIFY_S(!ReceivedVMultiPutResponses[requestIdx], "the response is received twice" - << " Event# " << ev.ToString() - << " State# " << DumpFullState()); - ReceivedVMultiPutResponses[requestIdx] = true; - - ++VMultiPutResponses; - for (auto &item : record.GetItems()) { - const NKikimrProto::EReplyStatus status = item.GetStatus(); - const TLogoBlobID blob = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - Y_VERIFY(item.HasCookie()); - TBlobCookie itemCookie(item.GetCookie()); - Y_VERIFY(itemCookie.GetVDiskOrderNumber() == orderNumber); - Y_VERIFY(itemCookie.GetPartId() == blob.PartId()); - switch (status) { - case NKikimrProto::ERROR: - case NKikimrProto::VDISK_ERROR_STATE: - case NKikimrProto::OUT_OF_SPACE: - Blackboard.AddErrorResponse(blob, orderNumber); - break; - case NKikimrProto::OK: - case NKikimrProto::ALREADY: - Blackboard.AddPutOkResponse(blob, orderNumber); - break; - default: - Y_FAIL("Unexpected status# %s", NKikimrProto::EReplyStatus_Name(status).data()); - } - } - Step(logCtx, outVGets, outVMultiPuts, outGetResult); -} - + TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) { + const NKikimrBlobStorage::TEvVMultiPutResult &record = ev.Record; + Y_VERIFY(record.HasVDiskID()); + TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); + TVDiskIdShort shortId(vdisk); + ui32 orderNumber = Info->GetOrderNumber(shortId); + + Y_VERIFY(record.HasCookie()); + TVMultiPutCookie cookie(record.GetCookie()); + Y_VERIFY(cookie.GetVDiskOrderNumber() == orderNumber); + Y_VERIFY(cookie.GetItemCount() == record.ItemsSize()); + + ui64 requestIdx = cookie.GetRequestIdx(); + Y_VERIFY_S(!ReceivedVMultiPutResponses[requestIdx], "the response is received twice" + << " Event# " << ev.ToString() + << " State# " << DumpFullState()); + ReceivedVMultiPutResponses[requestIdx] = true; + + ++VMultiPutResponses; + for (auto &item : record.GetItems()) { + const NKikimrProto::EReplyStatus status = item.GetStatus(); + const TLogoBlobID blob = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + Y_VERIFY(item.HasCookie()); + TBlobCookie itemCookie(item.GetCookie()); + Y_VERIFY(itemCookie.GetVDiskOrderNumber() == orderNumber); + Y_VERIFY(itemCookie.GetPartId() == blob.PartId()); + switch (status) { + case NKikimrProto::ERROR: + case NKikimrProto::VDISK_ERROR_STATE: + case NKikimrProto::OUT_OF_SPACE: + Blackboard.AddErrorResponse(blob, orderNumber); + break; + case NKikimrProto::OK: + case NKikimrProto::ALREADY: + Blackboard.AddPutOkResponse(blob, orderNumber); + break; + default: + Y_FAIL("Unexpected status# %s", NKikimrProto::EReplyStatus_Name(status).data()); + } + } + Step(logCtx, outVGets, outVMultiPuts, outGetResult); +} + }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.h b/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.h index 4908ae9bda4..809900c34ae 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_get_impl.h @@ -32,8 +32,8 @@ class TGetImpl { ui32 BlockedGeneration = 0; ui32 VPutRequests = 0; ui32 VPutResponses = 0; - ui32 VMultiPutRequests = 0; - ui32 VMultiPutResponses = 0; + ui32 VMultiPutRequests = 0; + ui32 VMultiPutResponses = 0; bool IsNoData = false; bool IsReplied = false; @@ -44,9 +44,9 @@ class TGetImpl { ui32 RequestIndex = 0; ui32 ResponseIndex = 0; - TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVPutResponses; - TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVMultiPutResponses; - + TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVPutResponses; + TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVMultiPutResponses; + const TString RequestPrefix; const bool PhantomCheck; @@ -128,155 +128,155 @@ public: return str.Str(); } - ui64 GetVPutRequests() const { - return VPutRequests; - } - - ui64 GetVPutResponses() const { - return VPutResponses; - } - - ui64 GetVMultiPutRequests() const { - return VMultiPutRequests; - } - - ui64 GetVMultiPutResponses() const { - return VMultiPutResponses; - } - - ui64 GetRequestIndex() const { - return RequestIndex; - } - - ui64 GetResponseIndex() const { - return ResponseIndex; - } - - + ui64 GetVPutRequests() const { + return VPutRequests; + } + + ui64 GetVPutResponses() const { + return VPutResponses; + } + + ui64 GetVMultiPutRequests() const { + return VMultiPutRequests; + } + + ui64 GetVMultiPutResponses() const { + return VMultiPutResponses; + } + + ui64 GetRequestIndex() const { + return RequestIndex; + } + + ui64 GetResponseIndex() const { + return ResponseIndex; + } + + void GenerateInitialRequests(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets); - template <typename TVPutEvent> + template <typename TVPutEvent> void OnVGetResult(TLogContext &logCtx, TEvBlobStorage::TEvVGetResult &ev, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts, - TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) { - const NKikimrBlobStorage::TEvVGetResult &record = ev.Record; - Y_VERIFY(record.HasStatus()); - const NKikimrProto::EReplyStatus status = record.GetStatus(); - Y_VERIFY(status != NKikimrProto::RACE && status != NKikimrProto::BLOCKED && status != NKikimrProto::DEADLINE); - R_LOG_DEBUG_SX(logCtx, "BPG57", "handle result# " << ev.ToString()); - - Y_VERIFY(record.HasVDiskID()); - TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); - TVDiskIdShort shortId(vdisk); - ui32 orderNumber = Info->GetOrderNumber(shortId); - { - NActors::NLog::EPriority priority = PriorityForStatusInbound(record.GetStatus()); - A_LOG_LOG_SX(logCtx, priority != NActors::NLog::PRI_DEBUG, priority, "BPG12", "Handle TEvVGetResult" - << " status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data() - << " From# " << vdisk.ToString() - << " orderNumber# " << orderNumber - << " ev " << ev.ToString()); - } - - BlockedGeneration = Max(BlockedGeneration, record.GetBlockedGeneration()); - - Y_VERIFY(record.ResultSize() > 0, "ev# %s vdisk# %s", ev.ToString().data(), vdisk.ToString().data()); - for (ui32 i = 0, e = (ui32)record.ResultSize(); i != e; ++i) { - const NKikimrBlobStorage::TQueryResult &result = record.GetResult(i); - Y_VERIFY(result.HasStatus()); - const NKikimrProto::EReplyStatus replyStatus = result.GetStatus(); - Y_VERIFY(result.HasCookie()); - const ui64 cookie = result.GetCookie(); - Y_VERIFY(result.HasBlobID()); - const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(result.GetBlobID()); - - if (ReportDetailedPartMap) { - Blackboard.ReportPartMapStatus(blobId, - Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[orderNumber].GetsToSend[cookie].PartMapIndex, - ResponseIndex, - replyStatus); - } - - TString resultBuffer = result.HasBuffer() ? result.GetBuffer() : TString(); - ui32 resultShift = result.HasShift() ? result.GetShift() : 0; - - // Currently CRC can be checked only if blob part is fully read - if (resultShift == 0 && resultBuffer.size() == Info->Type.PartSize(blobId)) { - bool isCrcOk = CheckCrcAtTheEnd((TErasureType::ECrcMode)blobId.CrcMode(), resultBuffer); - if (!isCrcOk) { - R_LOG_ERROR_SX(logCtx, "BPG66", "Error in CheckCrcAtTheEnd on TEvVGetResult, blobId# " << blobId - << " resultShift# " << resultShift << " resultBuffer.Size()# " << resultBuffer.size()); - NKikimrBlobStorage::TQueryResult *mutableResult = ev.Record.MutableResult(i); - mutableResult->SetStatus(NKikimrProto::ERROR); - } - } - - if (replyStatus == NKikimrProto::OK) { - // TODO(cthulhu): Verify shift and response size, and cookie - R_LOG_DEBUG_SX(logCtx, "BPG58", "Got# OK orderNumber# " << orderNumber << " vDiskId# " << vdisk.ToString()); - Blackboard.AddResponseData(blobId, orderNumber, resultShift, resultBuffer); - } else if (replyStatus == NKikimrProto::NODATA) { - R_LOG_DEBUG_SX(logCtx, "BPG59", "Got# NODATA orderNumber# " << orderNumber - << " vDiskId# " << vdisk.ToString()); - Blackboard.AddNoDataResponse(blobId, orderNumber); - } else if (replyStatus == NKikimrProto::ERROR - || replyStatus == NKikimrProto::VDISK_ERROR_STATE - || replyStatus == NKikimrProto::CORRUPTED) { - R_LOG_DEBUG_SX(logCtx, "BPG60", "Got# " << NKikimrProto::EReplyStatus_Name(replyStatus).data() - << " orderNumber# " << orderNumber << " vDiskId# " << vdisk.ToString()); - Blackboard.AddErrorResponse(blobId, orderNumber); - } else if (replyStatus == NKikimrProto::NOT_YET) { + TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) { + const NKikimrBlobStorage::TEvVGetResult &record = ev.Record; + Y_VERIFY(record.HasStatus()); + const NKikimrProto::EReplyStatus status = record.GetStatus(); + Y_VERIFY(status != NKikimrProto::RACE && status != NKikimrProto::BLOCKED && status != NKikimrProto::DEADLINE); + R_LOG_DEBUG_SX(logCtx, "BPG57", "handle result# " << ev.ToString()); + + Y_VERIFY(record.HasVDiskID()); + TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); + TVDiskIdShort shortId(vdisk); + ui32 orderNumber = Info->GetOrderNumber(shortId); + { + NActors::NLog::EPriority priority = PriorityForStatusInbound(record.GetStatus()); + A_LOG_LOG_SX(logCtx, priority != NActors::NLog::PRI_DEBUG, priority, "BPG12", "Handle TEvVGetResult" + << " status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data() + << " From# " << vdisk.ToString() + << " orderNumber# " << orderNumber + << " ev " << ev.ToString()); + } + + BlockedGeneration = Max(BlockedGeneration, record.GetBlockedGeneration()); + + Y_VERIFY(record.ResultSize() > 0, "ev# %s vdisk# %s", ev.ToString().data(), vdisk.ToString().data()); + for (ui32 i = 0, e = (ui32)record.ResultSize(); i != e; ++i) { + const NKikimrBlobStorage::TQueryResult &result = record.GetResult(i); + Y_VERIFY(result.HasStatus()); + const NKikimrProto::EReplyStatus replyStatus = result.GetStatus(); + Y_VERIFY(result.HasCookie()); + const ui64 cookie = result.GetCookie(); + Y_VERIFY(result.HasBlobID()); + const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(result.GetBlobID()); + + if (ReportDetailedPartMap) { + Blackboard.ReportPartMapStatus(blobId, + Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[orderNumber].GetsToSend[cookie].PartMapIndex, + ResponseIndex, + replyStatus); + } + + TString resultBuffer = result.HasBuffer() ? result.GetBuffer() : TString(); + ui32 resultShift = result.HasShift() ? result.GetShift() : 0; + + // Currently CRC can be checked only if blob part is fully read + if (resultShift == 0 && resultBuffer.size() == Info->Type.PartSize(blobId)) { + bool isCrcOk = CheckCrcAtTheEnd((TErasureType::ECrcMode)blobId.CrcMode(), resultBuffer); + if (!isCrcOk) { + R_LOG_ERROR_SX(logCtx, "BPG66", "Error in CheckCrcAtTheEnd on TEvVGetResult, blobId# " << blobId + << " resultShift# " << resultShift << " resultBuffer.Size()# " << resultBuffer.size()); + NKikimrBlobStorage::TQueryResult *mutableResult = ev.Record.MutableResult(i); + mutableResult->SetStatus(NKikimrProto::ERROR); + } + } + + if (replyStatus == NKikimrProto::OK) { + // TODO(cthulhu): Verify shift and response size, and cookie + R_LOG_DEBUG_SX(logCtx, "BPG58", "Got# OK orderNumber# " << orderNumber << " vDiskId# " << vdisk.ToString()); + Blackboard.AddResponseData(blobId, orderNumber, resultShift, resultBuffer); + } else if (replyStatus == NKikimrProto::NODATA) { + R_LOG_DEBUG_SX(logCtx, "BPG59", "Got# NODATA orderNumber# " << orderNumber + << " vDiskId# " << vdisk.ToString()); + Blackboard.AddNoDataResponse(blobId, orderNumber); + } else if (replyStatus == NKikimrProto::ERROR + || replyStatus == NKikimrProto::VDISK_ERROR_STATE + || replyStatus == NKikimrProto::CORRUPTED) { + R_LOG_DEBUG_SX(logCtx, "BPG60", "Got# " << NKikimrProto::EReplyStatus_Name(replyStatus).data() + << " orderNumber# " << orderNumber << " vDiskId# " << vdisk.ToString()); + Blackboard.AddErrorResponse(blobId, orderNumber); + } else if (replyStatus == NKikimrProto::NOT_YET) { R_LOG_DEBUG_SX(logCtx, "BPG67", "Got# NOT_YET orderNumber# " << orderNumber - << " vDiskId# " << vdisk.ToString()); - Blackboard.AddNotYetResponse(blobId, orderNumber); - } else { - Y_VERIFY(false, "Unexpected reply status# %s", NKikimrProto::EReplyStatus_Name(replyStatus).data()); - } - } - - ++ResponseIndex; - - Step(logCtx, outVGets, outVPuts, outGetResult); - } - + << " vDiskId# " << vdisk.ToString()); + Blackboard.AddNotYetResponse(blobId, orderNumber); + } else { + Y_VERIFY(false, "Unexpected reply status# %s", NKikimrProto::EReplyStatus_Name(replyStatus).data()); + } + } + + ++ResponseIndex; + + Step(logCtx, outVGets, outVPuts, outGetResult); + } + void OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVPutResult &ev, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> &outVPuts, TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult); - void OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVMultiPutResult &ev, + void OnVPutResult(TLogContext &logCtx, TEvBlobStorage::TEvVMultiPutResult &ev, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &outVMultiPuts, - TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult); - + TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult); + void PrepareReply(NKikimrProto::EReplyStatus status, TLogContext &logCtx, TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult); - template <typename TVPutEvent> + template <typename TVPutEvent> void AccelerateGet(TLogContext &logCtx, i32 slowDiskOrderNumber, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts) { - TAutoPtr<TEvBlobStorage::TEvGetResult> outGetResult; - TBlackboard::EAccelerationMode prevMode = Blackboard.AccelerationMode; - Blackboard.AccelerationMode = TBlackboard::AccelerationModeSkipMarked; - for (auto it = Blackboard.BlobStates.begin(); it != Blackboard.BlobStates.end(); ++it) { - TStackVec<TBlobState::TDisk, TypicalDisksInSubring> &disks = it->second.Disks; - for (ui32 i = 0; i < disks.size(); ++i) { - TBlobState::TDisk &disk = disks[i]; - disk.IsSlow = ((i32)disk.OrderNumber == slowDiskOrderNumber); - } - } + TAutoPtr<TEvBlobStorage::TEvGetResult> outGetResult; + TBlackboard::EAccelerationMode prevMode = Blackboard.AccelerationMode; + Blackboard.AccelerationMode = TBlackboard::AccelerationModeSkipMarked; + for (auto it = Blackboard.BlobStates.begin(); it != Blackboard.BlobStates.end(); ++it) { + TStackVec<TBlobState::TDisk, TypicalDisksInSubring> &disks = it->second.Disks; + for (ui32 i = 0; i < disks.size(); ++i) { + TBlobState::TDisk &disk = disks[i]; + disk.IsSlow = ((i32)disk.OrderNumber == slowDiskOrderNumber); + } + } Blackboard.ChangeAll(); - Step(logCtx, outVGets, outVPuts, outGetResult); - Blackboard.AccelerationMode = prevMode; + Step(logCtx, outVGets, outVPuts, outGetResult); + Blackboard.AccelerationMode = prevMode; Y_VERIFY(!outGetResult, "%s Unexpected get result in AccelerateGet, outGetResult# %s, DumpFullState# %s", RequestPrefix.data(), outGetResult->Print(false).c_str(), DumpFullState().c_str()); - } + } - template <typename TVPutEvent> + template <typename TVPutEvent> void AcceleratePut(TLogContext &logCtx, i32 slowDiskOrderNumber, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts) { - AccelerateGet(logCtx, slowDiskOrderNumber, outVGets, outVPuts); - } + AccelerateGet(logCtx, slowDiskOrderNumber, outVGets, outVPuts); + } ui64 GetTimeToAccelerateGetNs(TLogContext &logCtx); ui64 GetTimeToAcceleratePutNs(TLogContext &logCtx); @@ -289,8 +289,8 @@ protected: EStrategyOutcome RunMirror3of4Strategy(TLogContext &logCtx); EStrategyOutcome RunStrategies(TLogContext &logCtx); - // Returns true if there are additional requests to send - template <typename TVPutEvent> + // Returns true if there are additional requests to send + template <typename TVPutEvent> bool Step(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts, TAutoPtr<TEvBlobStorage::TEvGetResult> &outGetResult) { switch (auto outcome = RunStrategies(logCtx)) { @@ -309,14 +309,14 @@ protected: case EStrategyOutcome::DONE: PrepareReply(NKikimrProto::OK, logCtx, outGetResult); return false; - } - } - + } + } + void PrepareRequests(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &outVGets); void PrepareVPuts(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> &outVPuts); - void PrepareVPuts(TLogContext &logCtx, + void PrepareVPuts(TLogContext &logCtx, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &outVMultiPuts); ui64 GetTimeToAccelerateNs(TLogContext &logCtx, NKikimrBlobStorage::EVDiskQueueId queueId); diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_impl.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_impl.cpp index dbf547ae745..c74b0bcc9a4 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_impl.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_impl.cpp @@ -4,7 +4,7 @@ namespace NKikimr { TBlobStorageGroupProxy::TBlobStorageGroupProxy(TIntrusivePtr<TBlobStorageGroupInfo>&& info, bool forceWaitAllDrives, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, TIntrusivePtr<TStoragePoolCounters>&& storagePoolCounters, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) : GroupId(info->GroupID) , Info(std::move(info)) , Topology(Info->PickTopology()) @@ -12,36 +12,36 @@ namespace NKikimr { , StoragePoolCounters(std::move(storagePoolCounters)) , IsEjected(false) , ForceWaitAllDrives(forceWaitAllDrives) - , EnablePutBatching(enablePutBatching) - , EnableVPatch(enableVPatch) + , EnablePutBatching(enablePutBatching) + , EnableVPatch(enableVPatch) {} TBlobStorageGroupProxy::TBlobStorageGroupProxy(ui32 groupId, bool isEjected, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) : GroupId(groupId) , NodeMon(nodeMon) , IsEjected(isEjected) , ForceWaitAllDrives(false) - , EnablePutBatching(enablePutBatching) - , EnableVPatch(enableVPatch) + , EnablePutBatching(enablePutBatching) + , EnableVPatch(enableVPatch) {} IActor* CreateBlobStorageGroupEjectedProxy(ui32 groupId, TIntrusivePtr<TDsProxyNodeMon> &nodeMon) { - return new TBlobStorageGroupProxy(groupId, true, nodeMon, TControlWrapper(false, false, true), - TControlWrapper(false, false, true)); + return new TBlobStorageGroupProxy(groupId, true, nodeMon, TControlWrapper(false, false, true), + TControlWrapper(false, false, true)); } IActor* CreateBlobStorageGroupProxyConfigured(TIntrusivePtr<TBlobStorageGroupInfo>&& info, bool forceWaitAllDrives, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, TIntrusivePtr<TStoragePoolCounters>&& storagePoolCounters, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) { + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) { Y_VERIFY(info); return new TBlobStorageGroupProxy(std::move(info), forceWaitAllDrives, nodeMon, std::move(storagePoolCounters), - enablePutBatching, enableVPatch); + enablePutBatching, enableVPatch); } IActor* CreateBlobStorageGroupProxyUnconfigured(ui32 groupId, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) { - return new TBlobStorageGroupProxy(groupId, false, nodeMon, enablePutBatching, enableVPatch); + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch) { + return new TBlobStorageGroupProxy(groupId, false, nodeMon, enablePutBatching, enableVPatch); } NActors::NLog::EPriority PriorityForStatusOutbound(NKikimrProto::EReplyStatus status) { diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h index 508eb10a055..22c7736bb35 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h @@ -101,8 +101,8 @@ class TBlobStorageGroupProxy : public TActorBootstrapped<TBlobStorageGroupProxy> TBatchedQueue<TEvBlobStorage::TEvGet::TPtr> BatchedGets[GetHandleClassCount]; TStackVec<NKikimrBlobStorage::EGetHandleClass, GetHandleClassCount> GetBatchedBucketQueue; - TMemorizableControlWrapper EnablePutBatching; - TMemorizableControlWrapper EnableVPatch; + TMemorizableControlWrapper EnablePutBatching; + TMemorizableControlWrapper EnableVPatch; TInstant EstablishingSessionStartTime; @@ -300,10 +300,10 @@ public: TBlobStorageGroupProxy(TIntrusivePtr<TBlobStorageGroupInfo>&& info, bool forceWaitAllDrives, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, TIntrusivePtr<TStoragePoolCounters>&& storagePoolCounters, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); TBlobStorageGroupProxy(ui32 groupId, bool isEjected, TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); + const TControlWrapper &enablePutBatching, const TControlWrapper &enableVPatch); void Bootstrap(); diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp index 73afac8121f..50bfb3e3eb2 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp @@ -44,13 +44,13 @@ TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonito EventIndexRestoreGet = EventGroup->GetCounter("EvIndexRestoreGet", true); EventMultiCollect = EventGroup->GetCounter("EvMultiCollect", true); EventStatus = EventGroup->GetCounter("EvStatus", true); - EventStopPutBatching = EventGroup->GetCounter("EvStopPutBatching", true); - EventStopGetBatching = EventGroup->GetCounter("EvStopGetBatching", true); - EventPatch = EventGroup->GetCounter("EvPatch", true); - - PutsSentViaPutBatching = EventGroup->GetCounter("PutsSentViaPutBatching", true); - PutBatchesSent = EventGroup->GetCounter("PutBatchesSent", true); + EventStopPutBatching = EventGroup->GetCounter("EvStopPutBatching", true); + EventStopGetBatching = EventGroup->GetCounter("EvStopGetBatching", true); + EventPatch = EventGroup->GetCounter("EvPatch", true); + PutsSentViaPutBatching = EventGroup->GetCounter("PutsSentViaPutBatching", true); + PutBatchesSent = EventGroup->GetCounter("PutBatchesSent", true); + auto buckets = EventGroup->GetSubgroup("sensor", "EvPutBytesBuckets"); for (ui32 size : {0, 256, 4096, 65536, 250000, 1000000, 4000000}) { EventPutBytesBuckets.emplace(size, buckets->GetNamedCounter("size", Sprintf("%" PRIu32, (ui32)size), true)); @@ -71,13 +71,13 @@ TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonito ActiveRange = ActiveRequestsGroup->GetCounter("ActiveRange"); ActiveCollectGarbage = ActiveRequestsGroup->GetCounter("ActiveCollectGarbage"); ActiveStatus = ActiveRequestsGroup->GetCounter("ActiveStatus"); - ActivePatch = ActiveRequestsGroup->GetCounter("ActivePatch"); - - // special patch counters - VPatchContinueFailed = ActiveRequestsGroup->GetCounter("VPatchContinueFailed"); - VPatchPartPlacementVerifyFailed = ActiveRequestsGroup->GetCounter("VPatchPartPlacementVerifyFailed"); - PatchesWithFallback = ActiveRequestsGroup->GetCounter("PatchesWithFallback"); + ActivePatch = ActiveRequestsGroup->GetCounter("ActivePatch"); + // special patch counters + VPatchContinueFailed = ActiveRequestsGroup->GetCounter("VPatchContinueFailed"); + VPatchPartPlacementVerifyFailed = ActiveRequestsGroup->GetCounter("VPatchPartPlacementVerifyFailed"); + PatchesWithFallback = ActiveRequestsGroup->GetCounter("PatchesWithFallback"); + // subevents { auto group = Counters->GetSubgroup("subsystem", "subevents"); @@ -85,7 +85,7 @@ TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonito PutGroup.Init(group->GetSubgroup("request", "put")); DiscoverGroup.Init(group->GetSubgroup("request", "discover")); RangeGroup.Init(group->GetSubgroup("request", "range")); - PatchGroup.Init(group->GetSubgroup("request", "patch")); + PatchGroup.Init(group->GetSubgroup("request", "patch")); } ActiveMultiGet = ActiveRequestsGroup->GetCounter("ActiveMultiGet"); @@ -100,7 +100,7 @@ TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonito RespStatRange.emplace(respStatGroup->GetSubgroup("request", "range")); RespStatCollectGarbage.emplace(respStatGroup->GetSubgroup("request", "collectGarbage")); RespStatStatus.emplace(respStatGroup->GetSubgroup("request", "status")); - RespStatPatch.emplace(respStatGroup->GetSubgroup("request", "patch")); + RespStatPatch.emplace(respStatGroup->GetSubgroup("request", "patch")); } void TBlobStorageGroupProxyMon::BecomeFull() { @@ -130,7 +130,7 @@ void TBlobStorageGroupProxyMon::BecomeFull() { IndexRestoreGetResponseTime.Initialize(ResponseGroup, "event", "indexRestoreGet", "Response in millisec", Percentiles1); RangeResponseTime.Initialize(ResponseGroup, "event", "range", "Response in millisec", Percentiles1); - PatchResponseTime.Initialize(ResponseGroup, "event", "patch", "Response in millisec", Percentiles1); + PatchResponseTime.Initialize(ResponseGroup, "event", "patch", "Response in millisec", Percentiles1); } IsLimitedMon = false; } @@ -190,7 +190,7 @@ void TBlobStorageGroupProxyMon::Update() { DiscoverResponseTime.Update(); IndexRestoreGetResponseTime.Update(); RangeResponseTime.Update(); - PatchResponseTime.Update(); + PatchResponseTime.Update(); } BlockResponseTime.Update(); diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h index 941bea6664a..a6c031b88cb 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h @@ -25,7 +25,7 @@ enum class ERequestType { Put, Discover, Range, - Patch, + Patch, }; struct TRequestMonGroup { @@ -35,7 +35,7 @@ struct TRequestMonGroup { NMonitoring::TDynamicCounters::TCounterPtr VGetBlobsReturnedWithNoData; NMonitoring::TDynamicCounters::TCounterPtr VGetBlobsReturnedWithErrors; NMonitoring::TDynamicCounters::TCounterPtr VPutBlobsIssued; - NMonitoring::TDynamicCounters::TCounterPtr VMovedPatchBlobsIssued; + NMonitoring::TDynamicCounters::TCounterPtr VMovedPatchBlobsIssued; void Init(const TIntrusivePtr<NMonitoring::TDynamicCounters> &group) { VGetBlobsIssued = group->GetCounter("VGetBlobsIssued", true); @@ -44,27 +44,27 @@ struct TRequestMonGroup { VGetBlobsReturnedWithNoData = group->GetCounter("VGetBlobsReturnedWithNoData", true); VGetBlobsReturnedWithErrors = group->GetCounter("VGetBlobsReturnedWithErrors", true); VPutBlobsIssued = group->GetCounter("VPutBlobsIssued", true); - VMovedPatchBlobsIssued = group->GetCounter("VMovedPatchBlobsIssued", true); + VMovedPatchBlobsIssued = group->GetCounter("VMovedPatchBlobsIssued", true); } void CountEvent(const TEvBlobStorage::TEvVPut& /*ev*/) { VPutBlobsIssued->Inc(); } - void CountEvent(const TEvBlobStorage::TEvVMultiPut& ev) { - *VPutBlobsIssued += ev.Record.ItemsSize(); - } - - void CountEvent(const TEvBlobStorage::TEvVMovedPatch& /*ev*/) { - VPutBlobsIssued->Inc(); - } - - void CountEvent(const TEvBlobStorage::TEvVPatchStart& /*ev*/) { - } - - void CountEvent(const TEvBlobStorage::TEvVPatchDiff& /*ev*/) { - } - + void CountEvent(const TEvBlobStorage::TEvVMultiPut& ev) { + *VPutBlobsIssued += ev.Record.ItemsSize(); + } + + void CountEvent(const TEvBlobStorage::TEvVMovedPatch& /*ev*/) { + VPutBlobsIssued->Inc(); + } + + void CountEvent(const TEvBlobStorage::TEvVPatchStart& /*ev*/) { + } + + void CountEvent(const TEvBlobStorage::TEvVPatchDiff& /*ev*/) { + } + void CountEvent(const TEvBlobStorage::TEvVGet &ev) { *VGetBlobsIssued += ev.Record.ExtremeQueriesSize(); if (ev.Record.HasRangeQuery()) { @@ -169,7 +169,7 @@ protected: NMonitoring::TPercentileTrackerLg<3, 4, 3> DiscoverResponseTime; NMonitoring::TPercentileTrackerLg<3, 4, 3> IndexRestoreGetResponseTime; NMonitoring::TPercentileTrackerLg<3, 4, 3> RangeResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> PatchResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PatchResponseTime; // event counters TIntrusivePtr<NMonitoring::TDynamicCounters> EventGroup; @@ -187,7 +187,7 @@ protected: TRequestMonGroup PutGroup; TRequestMonGroup DiscoverGroup; TRequestMonGroup RangeGroup; - TRequestMonGroup PatchGroup; + TRequestMonGroup PatchGroup; public: TBlobStorageGroupProxyTimeStats TimeStats; @@ -209,13 +209,13 @@ public: NMonitoring::TDynamicCounters::TCounterPtr EventIndexRestoreGet; NMonitoring::TDynamicCounters::TCounterPtr EventMultiCollect; NMonitoring::TDynamicCounters::TCounterPtr EventStatus; - NMonitoring::TDynamicCounters::TCounterPtr EventStopPutBatching; - NMonitoring::TDynamicCounters::TCounterPtr EventStopGetBatching; - NMonitoring::TDynamicCounters::TCounterPtr EventPatch; - - NMonitoring::TDynamicCounters::TCounterPtr PutsSentViaPutBatching; - NMonitoring::TDynamicCounters::TCounterPtr PutBatchesSent; + NMonitoring::TDynamicCounters::TCounterPtr EventStopPutBatching; + NMonitoring::TDynamicCounters::TCounterPtr EventStopGetBatching; + NMonitoring::TDynamicCounters::TCounterPtr EventPatch; + NMonitoring::TDynamicCounters::TCounterPtr PutsSentViaPutBatching; + NMonitoring::TDynamicCounters::TCounterPtr PutBatchesSent; + // active event counters TIntrusivePtr<NMonitoring::TDynamicCounters> ActiveRequestsGroup; NMonitoring::TDynamicCounters::TCounterPtr ActivePut; @@ -230,7 +230,7 @@ public: NMonitoring::TDynamicCounters::TCounterPtr ActiveIndexRestoreGet; NMonitoring::TDynamicCounters::TCounterPtr ActiveMultiCollect; NMonitoring::TDynamicCounters::TCounterPtr ActiveStatus; - NMonitoring::TDynamicCounters::TCounterPtr ActivePatch; + NMonitoring::TDynamicCounters::TCounterPtr ActivePatch; std::optional<TResponseStatusGroup> RespStatPut; std::optional<TResponseStatusGroup> RespStatGet; @@ -239,20 +239,20 @@ public: std::optional<TResponseStatusGroup> RespStatRange; std::optional<TResponseStatusGroup> RespStatCollectGarbage; std::optional<TResponseStatusGroup> RespStatStatus; - std::optional<TResponseStatusGroup> RespStatPatch; + std::optional<TResponseStatusGroup> RespStatPatch; - // special patch counters - NMonitoring::TDynamicCounters::TCounterPtr VPatchContinueFailed; - NMonitoring::TDynamicCounters::TCounterPtr VPatchPartPlacementVerifyFailed; + // special patch counters + NMonitoring::TDynamicCounters::TCounterPtr VPatchContinueFailed; + NMonitoring::TDynamicCounters::TCounterPtr VPatchPartPlacementVerifyFailed; NMonitoring::TDynamicCounters::TCounterPtr PatchesWithFallback; - + TRequestMonGroup& GetRequestMonGroup(ERequestType request) { switch (request) { case ERequestType::Get: return GetGroup; case ERequestType::Put: return PutGroup; case ERequestType::Discover: return DiscoverGroup; case ERequestType::Range: return RangeGroup; - case ERequestType::Patch: return PatchGroup; + case ERequestType::Patch: return PatchGroup; } Y_FAIL(); } @@ -351,11 +351,11 @@ public: NodeMon->RangeResponseTime.Increment(duration.MilliSeconds()); } - void CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration) { - PatchResponseTime.Increment(duration.MilliSeconds()); - NodeMon->CountPatchResponseTime(type, duration); - } - + void CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration) { + PatchResponseTime.Increment(duration.MilliSeconds()); + NodeMon->CountPatchResponseTime(type, duration); + } + void Update(); void ThroughputUpdate(); }; diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_multiget.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_multiget.cpp index 02d7849855c..3ae7ede0e7c 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_multiget.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_multiget.cpp @@ -1,9 +1,9 @@ #include "dsproxy.h" #include "dsproxy_mon.h" - + #include <ydb/core/blobstorage/base/wilson_events.h> #include <ydb/core/blobstorage/vdisk/query/query_spacetracker.h> - + #include <util/generic/set.h> namespace NKikimr { @@ -162,22 +162,22 @@ public: Y_VERIFY(QuerySize != 0); // reply with error? ui32 beginIdx = 0; TLogoBlobID lastBlobId; - TQueryResultSizeTracker resultSize; - resultSize.Init(); - + TQueryResultSizeTracker resultSize; + resultSize.Init(); + for (ui32 queryIdx = 0; queryIdx < QuerySize; ++queryIdx) { const TEvBlobStorage::TEvGet::TQuery &query = Queries[queryIdx]; if (lastBlobId == query.Id && queryIdx != 0) { continue; } - resultSize.AddAllPartsOfLogoBlob(Info->Type, query.Id); + resultSize.AddAllPartsOfLogoBlob(Info->Type, query.Id); if (queryIdx != beginIdx) { if (resultSize.IsOverflow() || queryIdx - beginIdx == 10000) { PrepareRequest(beginIdx, queryIdx); beginIdx = queryIdx; - resultSize.Init(); - resultSize.AddAllPartsOfLogoBlob(Info->Type, query.Id); + resultSize.Init(); + resultSize.AddAllPartsOfLogoBlob(Info->Type, query.Id); } } } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp index addca8c5330..6b0fdd2eb06 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp @@ -36,7 +36,7 @@ TDsProxyNodeMon::TDsProxyNodeMon(TIntrusivePtr<NMonitoring::TDynamicCounters> &c IndexRestoreGetResponseTime.Initialize(Group, "event", "indexRestoreGet", "latency", percentiles1); RangeResponseTime.Initialize(Group, "event", "range", "latency", percentiles1); - PatchResponseTime.Initialize(Group, "event", "patch", "latency", percentiles4); + PatchResponseTime.Initialize(Group, "event", "patch", "latency", percentiles4); IsCountersPresentedForIdx.fill(false); if (initForAllDeviceTypes) { @@ -51,7 +51,7 @@ TDsProxyNodeMon::TDsProxyNodeMon(TIntrusivePtr<NMonitoring::TDynamicCounters> &c auto group = Group->GetSubgroup("subsystem", "restart"); RestartPut = group->GetCounter("EvPut", true); RestartGet = group->GetCounter("EvGet", true); - RestartPatch = group->GetCounter("EvPatch", true); + RestartPatch = group->GetCounter("EvPatch", true); RestartBlock = group->GetCounter("EvBlock", true); RestartDiscover = group->GetCounter("EvDiscover", true); RestartRange = group->GetCounter("EvRange", true); @@ -170,15 +170,15 @@ void TDsProxyNodeMon::CountGetResponseTime(TPDiskCategory::EDeviceType type, NKi } } -void TDsProxyNodeMon::CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration) { - const ui32 durationMs = duration.MilliSeconds(); - const double durationMsFloat = duration.MicroSeconds() / 1000.0; - PatchResponseTime.Increment(durationMs); - const ui32 idx = IdxForType(type); - Y_VERIFY(IsCountersPresentedForIdx[idx]); - PatchResponseTimeHist[idx]->Collect(durationMsFloat); -} - +void TDsProxyNodeMon::CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration) { + const ui32 durationMs = duration.MilliSeconds(); + const double durationMsFloat = duration.MicroSeconds() / 1000.0; + PatchResponseTime.Increment(durationMs); + const ui32 idx = IdxForType(type); + Y_VERIFY(IsCountersPresentedForIdx[idx]); + PatchResponseTimeHist[idx]->Collect(durationMsFloat); +} + void TDsProxyNodeMon::CheckNodeMonCountersForDeviceType(TPDiskCategory::EDeviceType type) { const ui32 idx = IdxForType(type); @@ -201,7 +201,7 @@ void TDsProxyNodeMon::CheckNodeMonCountersForDeviceType(TPDiskCategory::EDeviceT GetFastReadResponseTimeHistInf[idx] = getNamedHisto("getFastReadInfMs"); GetDiscoverResponseTimeHist[idx] = getNamedHisto("getDiscoverMs"); GetLowReadResponseTimeHist[idx] = getNamedHisto("getLowReadMs"); - PatchResponseTimeHist[idx] = getNamedHisto("patchMs"); + PatchResponseTimeHist[idx] = getNamedHisto("patchMs"); } } } // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h index 150da172fb8..f593d353947 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h @@ -55,9 +55,9 @@ struct TDsProxyNodeMon : public TThrRefBase { NMonitoring::TPercentileTracker<16, 512, 15> GetLowReadResponseTime; THistoPtrForDeviceType GetLowReadResponseTimeHist; - NMonitoring::TPercentileTracker<16, 512, 15> PatchResponseTime; - THistoPtrForDeviceType PatchResponseTimeHist; - + NMonitoring::TPercentileTracker<16, 512, 15> PatchResponseTime; + THistoPtrForDeviceType PatchResponseTimeHist; + NMonitoring::TPercentileTracker<16, 512, 15> BlockResponseTime; NMonitoring::TPercentileTracker<16, 512, 15> DiscoverResponseTime; NMonitoring::TPercentileTracker<16, 512, 15> IndexRestoreGetResponseTime; @@ -74,7 +74,7 @@ struct TDsProxyNodeMon : public TThrRefBase { NMonitoring::TDynamicCounters::TCounterPtr RestartCollectGarbage; NMonitoring::TDynamicCounters::TCounterPtr RestartIndexRestoreGet; NMonitoring::TDynamicCounters::TCounterPtr RestartStatus; - NMonitoring::TDynamicCounters::TCounterPtr RestartPatch; + NMonitoring::TDynamicCounters::TCounterPtr RestartPatch; std::array<NMonitoring::TDynamicCounters::TCounterPtr, 4> RestartHisto; @@ -95,7 +95,7 @@ struct TDsProxyNodeMon : public TThrRefBase { TDuration duration); void CountGetResponseTime(TPDiskCategory::EDeviceType type, NKikimrBlobStorage::EGetHandleClass cls, ui32 size, TDuration duration); - void CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration); + void CountPatchResponseTime(TPDiskCategory::EDeviceType type, TDuration duration); // Called only from NodeWarder void CheckNodeMonCountersForDeviceType(TPDiskCategory::EDeviceType type); diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemonactor.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_nodemonactor.cpp index 5bf4b098362..db723d6d5e1 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemonactor.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemonactor.cpp @@ -67,7 +67,7 @@ public: Mon->DiscoverResponseTime.Update(); Mon->IndexRestoreGetResponseTime.Update(); Mon->RangeResponseTime.Update(); - Mon->PatchResponseTime.Update(); + Mon->PatchResponseTime.Update(); } void Handle(NMon::TEvHttpInfo::TPtr &ev) { diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_patch.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_patch.cpp index fcced0b6f85..3b63445a3bd 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_patch.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_patch.cpp @@ -1,835 +1,835 @@ -#include "dsproxy.h" -#include "dsproxy_mon.h" -#include "root_cause.h" +#include "dsproxy.h" +#include "dsproxy_mon.h" +#include "root_cause.h" #include <ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h> - -#include <util/generic/ymath.h> -#include <util/system/datetime.h> -#include <util/system/hp_timer.h> - -LWTRACE_USING(BLOBSTORAGE_PROVIDER); - -namespace NKikimr { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// PATCH request -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class TBlobStorageGroupPatchRequest : public TBlobStorageGroupRequestActor<TBlobStorageGroupPatchRequest> { - friend class TBlobStorageGroupRequestActor<TBlobStorageGroupPatchRequest>; - - struct TPartPlacement { - ui8 VDiskIdxInSubgroup = 0; - ui8 PartId = 0; - - TString ToString() const { - return TStringBuilder() << "{VDiskIdxInSubgroup# " << (ui32)VDiskIdxInSubgroup << " PartId# " << (ui32)PartId << "}"; - } - }; - - static constexpr ui32 TypicalHandoffCount = 2; - static constexpr ui32 TypicalPartPlacementCount = 1 + TypicalHandoffCount; - static constexpr ui32 TypicalMaxPartsCount = TypicalPartPlacementCount * TypicalPartsInBlob; - - TString Buffer; - TActorId ProxyActorId; - - ui32 OriginalGroupId; - TLogoBlobID OriginalId; - TLogoBlobID PatchedId; - ui32 MaskForCookieBruteForcing; - - ui32 DiffCount = 0; - TArrayHolder<TEvBlobStorage::TEvPatch::TDiff> Diffs; - - TStorageStatusFlags StatusFlags = 0; - float ApproximateFreeSpaceShare = 0; - - TInstant StartTime; - TInstant Deadline; - - NLWTrace::TOrbit Orbit; - TString ErrorReason; - - ui32 SendedGetRequests = 0; - ui32 ReceivedGetResponses = 0; - ui32 SendedPutRequests = 0; - ui32 ReceivedPutResponses = 0; - - TVector<ui32> OkVDisksWithParts; - - ui32 SendedStarts = 0; - ui32 ReceivedFoundParts = 0; - ui32 ErrorResponses = 0; - ui32 ReceivedResults = 0; - - TStackVec<TPartPlacement, TypicalMaxPartsCount> FoundParts; - TStackVec<bool, TypicalDisksInSubring> ReceivedResponseFlags; - TStackVec<bool, TypicalDisksInSubring> EmptyResponseFlags; - TStackVec<bool, TypicalDisksInSubring> ErrorResponseFlags; - TBlobStorageGroupInfo::TVDiskIds VDisks; - - bool UseVPatch = false; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::BS_PROXY_PATCH_ACTOR; - } - - static const auto& ActiveCounter(const TIntrusivePtr<TBlobStorageGroupProxyMon>& mon) { - return mon->ActivePatch; - } - - static constexpr ERequestType RequestType() { - return ERequestType::Patch; - } - - TBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, - const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, - ui64 cookie, NWilson::TTraceId traceId, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - const TActorId &proxyId, bool useVPatch = false) - : TBlobStorageGroupRequestActor(info, state, mon, source, cookie, std::move(traceId), - NKikimrServices::BS_PROXY_PATCH, false, {}, now, storagePoolCounters, - ev->RestartCounter) - , ProxyActorId(proxyId) - , OriginalGroupId(ev->OriginalGroupId) - , OriginalId(ev->OriginalId) - , PatchedId(ev->PatchedId) - , MaskForCookieBruteForcing(ev->MaskForCookieBruteForcing) - , DiffCount(ev->DiffCount) - , Diffs(ev->Diffs.Release()) - , StartTime(now) - , Deadline(ev->Deadline) - , Orbit(std::move(ev->Orbit)) - , UseVPatch(useVPatch) - { - } - - void ReplyAndDie(NKikimrProto::EReplyStatus status) { + +#include <util/generic/ymath.h> +#include <util/system/datetime.h> +#include <util/system/hp_timer.h> + +LWTRACE_USING(BLOBSTORAGE_PROVIDER); + +namespace NKikimr { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// PATCH request +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class TBlobStorageGroupPatchRequest : public TBlobStorageGroupRequestActor<TBlobStorageGroupPatchRequest> { + friend class TBlobStorageGroupRequestActor<TBlobStorageGroupPatchRequest>; + + struct TPartPlacement { + ui8 VDiskIdxInSubgroup = 0; + ui8 PartId = 0; + + TString ToString() const { + return TStringBuilder() << "{VDiskIdxInSubgroup# " << (ui32)VDiskIdxInSubgroup << " PartId# " << (ui32)PartId << "}"; + } + }; + + static constexpr ui32 TypicalHandoffCount = 2; + static constexpr ui32 TypicalPartPlacementCount = 1 + TypicalHandoffCount; + static constexpr ui32 TypicalMaxPartsCount = TypicalPartPlacementCount * TypicalPartsInBlob; + + TString Buffer; + TActorId ProxyActorId; + + ui32 OriginalGroupId; + TLogoBlobID OriginalId; + TLogoBlobID PatchedId; + ui32 MaskForCookieBruteForcing; + + ui32 DiffCount = 0; + TArrayHolder<TEvBlobStorage::TEvPatch::TDiff> Diffs; + + TStorageStatusFlags StatusFlags = 0; + float ApproximateFreeSpaceShare = 0; + + TInstant StartTime; + TInstant Deadline; + + NLWTrace::TOrbit Orbit; + TString ErrorReason; + + ui32 SendedGetRequests = 0; + ui32 ReceivedGetResponses = 0; + ui32 SendedPutRequests = 0; + ui32 ReceivedPutResponses = 0; + + TVector<ui32> OkVDisksWithParts; + + ui32 SendedStarts = 0; + ui32 ReceivedFoundParts = 0; + ui32 ErrorResponses = 0; + ui32 ReceivedResults = 0; + + TStackVec<TPartPlacement, TypicalMaxPartsCount> FoundParts; + TStackVec<bool, TypicalDisksInSubring> ReceivedResponseFlags; + TStackVec<bool, TypicalDisksInSubring> EmptyResponseFlags; + TStackVec<bool, TypicalDisksInSubring> ErrorResponseFlags; + TBlobStorageGroupInfo::TVDiskIds VDisks; + + bool UseVPatch = false; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::BS_PROXY_PATCH_ACTOR; + } + + static const auto& ActiveCounter(const TIntrusivePtr<TBlobStorageGroupProxyMon>& mon) { + return mon->ActivePatch; + } + + static constexpr ERequestType RequestType() { + return ERequestType::Patch; + } + + TBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, + const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, + ui64 cookie, NWilson::TTraceId traceId, TInstant now, + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + const TActorId &proxyId, bool useVPatch = false) + : TBlobStorageGroupRequestActor(info, state, mon, source, cookie, std::move(traceId), + NKikimrServices::BS_PROXY_PATCH, false, {}, now, storagePoolCounters, + ev->RestartCounter) + , ProxyActorId(proxyId) + , OriginalGroupId(ev->OriginalGroupId) + , OriginalId(ev->OriginalId) + , PatchedId(ev->PatchedId) + , MaskForCookieBruteForcing(ev->MaskForCookieBruteForcing) + , DiffCount(ev->DiffCount) + , Diffs(ev->Diffs.Release()) + , StartTime(now) + , Deadline(ev->Deadline) + , Orbit(std::move(ev->Orbit)) + , UseVPatch(useVPatch) + { + } + + void ReplyAndDie(NKikimrProto::EReplyStatus status) { std::unique_ptr<TEvBlobStorage::TEvPatchResult> result = std::make_unique<TEvBlobStorage::TEvPatchResult>(status, PatchedId, - StatusFlags, Info->GroupID, ApproximateFreeSpaceShare); - result->ErrorReason = ErrorReason; - result->Orbit = std::move(Orbit); - TDuration duration = TActivationContext::Now() - StartTime; - Mon->CountPatchResponseTime(Info->GetDeviceType(), duration); - SendResponseAndDie(std::move(result)); - } - - std::unique_ptr<IEventBase> RestartQuery(ui32 counter) { - ++*Mon->NodeMon->RestartPatch; - TEvBlobStorage::TEvPatch *patch; - std::unique_ptr<IEventBase> ev(patch = new TEvBlobStorage::TEvPatch(OriginalGroupId, OriginalId, PatchedId, + StatusFlags, Info->GroupID, ApproximateFreeSpaceShare); + result->ErrorReason = ErrorReason; + result->Orbit = std::move(Orbit); + TDuration duration = TActivationContext::Now() - StartTime; + Mon->CountPatchResponseTime(Info->GetDeviceType(), duration); + SendResponseAndDie(std::move(result)); + } + + std::unique_ptr<IEventBase> RestartQuery(ui32 counter) { + ++*Mon->NodeMon->RestartPatch; + TEvBlobStorage::TEvPatch *patch; + std::unique_ptr<IEventBase> ev(patch = new TEvBlobStorage::TEvPatch(OriginalGroupId, OriginalId, PatchedId, MaskForCookieBruteForcing, std::move(Diffs), DiffCount, Deadline)); - patch->RestartCounter = counter; - patch->Orbit = std::move(Orbit); - return std::move(ev); - } - - void ApplyDiffs() { - for (ui32 idx = 0; idx < DiffCount; ++idx) { - const TEvBlobStorage::TEvPatch::TDiff &diff = Diffs[idx]; - Copy(diff.Buffer.begin(), diff.Buffer.end(), Buffer.begin() + diff.Offset); - } - } - - void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev) { - TEvBlobStorage::TEvGetResult *result = ev->Get(); - Orbit = std::move(result->Orbit); - TraceId = std::move(ev->TraceId); - - ui32 patchedIdHash = PatchedId.Hash(); - bool incorrectCookie = ev->Cookie != patchedIdHash; - bool fail = incorrectCookie - || result->Status != NKikimrProto::OK - || result->ResponseSz != 1 - || result->Responses[0].Status != NKikimrProto::OK; - if (fail) { - if (ev->Cookie != patchedIdHash) { - ErrorReason = "Couldn't get the original blob; Received TEvGetResult with wrong cookie"; - } else if (result->ResponseSz > 1) { - ErrorReason = "Couldn't get the original blob; Received TEvGetResult with more responses than needed"; - } else { - TString getResponseStatus; - if (result->ResponseSz == 1) { - getResponseStatus = TStringBuilder() << " GetResponseStatus# " - << NKikimrProto::EReplyStatus_Name(result->Responses[0].Status); - } - ErrorReason = TStringBuilder() << "Couldn't get the original blob;" - << " GetStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) - << getResponseStatus - << " GetErrorReason# " << result->ErrorReason; - } - R_LOG_ERROR_S("BPPA04", ErrorReason); - ReplyAndDie(NKikimrProto::ERROR); - return; - } - - Buffer = result->Responses[0].Buffer; - ApplyDiffs(); - + patch->RestartCounter = counter; + patch->Orbit = std::move(Orbit); + return std::move(ev); + } + + void ApplyDiffs() { + for (ui32 idx = 0; idx < DiffCount; ++idx) { + const TEvBlobStorage::TEvPatch::TDiff &diff = Diffs[idx]; + Copy(diff.Buffer.begin(), diff.Buffer.end(), Buffer.begin() + diff.Offset); + } + } + + void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev) { + TEvBlobStorage::TEvGetResult *result = ev->Get(); + Orbit = std::move(result->Orbit); + TraceId = std::move(ev->TraceId); + + ui32 patchedIdHash = PatchedId.Hash(); + bool incorrectCookie = ev->Cookie != patchedIdHash; + bool fail = incorrectCookie + || result->Status != NKikimrProto::OK + || result->ResponseSz != 1 + || result->Responses[0].Status != NKikimrProto::OK; + if (fail) { + if (ev->Cookie != patchedIdHash) { + ErrorReason = "Couldn't get the original blob; Received TEvGetResult with wrong cookie"; + } else if (result->ResponseSz > 1) { + ErrorReason = "Couldn't get the original blob; Received TEvGetResult with more responses than needed"; + } else { + TString getResponseStatus; + if (result->ResponseSz == 1) { + getResponseStatus = TStringBuilder() << " GetResponseStatus# " + << NKikimrProto::EReplyStatus_Name(result->Responses[0].Status); + } + ErrorReason = TStringBuilder() << "Couldn't get the original blob;" + << " GetStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) + << getResponseStatus + << " GetErrorReason# " << result->ErrorReason; + } + R_LOG_ERROR_S("BPPA04", ErrorReason); + ReplyAndDie(NKikimrProto::ERROR); + return; + } + + Buffer = result->Responses[0].Buffer; + ApplyDiffs(); + std::unique_ptr<TEvBlobStorage::TEvPut> put = std::make_unique<TEvBlobStorage::TEvPut>(PatchedId, Buffer, Deadline, - NKikimrBlobStorage::AsyncBlob, TEvBlobStorage::TEvPut::TacticDefault); - put->Orbit = std::move(Orbit); + NKikimrBlobStorage::AsyncBlob, TEvBlobStorage::TEvPut::TacticDefault); + put->Orbit = std::move(Orbit); Send(ProxyActorId, put.release(), 0, OriginalId.Hash(), std::move(TraceId)); - } - - void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev) { - TEvBlobStorage::TEvPutResult *result = ev->Get(); - Orbit = std::move(result->Orbit); - TraceId = std::move(ev->TraceId); - - StatusFlags = result->StatusFlags; - ApproximateFreeSpaceShare = result->ApproximateFreeSpaceShare; - - ui32 originalIdHash = OriginalId.Hash(); - bool incorrectCookie = ev->Cookie != originalIdHash; - bool fail = incorrectCookie - || result->Status != NKikimrProto::OK; - if (fail) { - if (incorrectCookie) { - ErrorReason = "Couldn't put the patched blob; Received TEvPutResult with wrong cookie"; - } else { - ErrorReason = TStringBuilder() << "Couldn't put the patched blob;" - << " PutStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) - << " PutErrorReason# " << result->ErrorReason; - } - R_LOG_ERROR_S("BPPA03", ErrorReason); - ReplyAndDie(NKikimrProto::ERROR); - return; - } - - ReplyAndDie(NKikimrProto::OK); - } - - template <typename TEventResultRecord> - void PullOutStatusFlagsAndFressSpace(const TEventResultRecord &record){ - if (record.HasStatusFlags()) { - StatusFlags.Merge(record.GetStatusFlags()); - } - if (record.HasApproximateFreeSpaceShare()) { - float share = record.GetApproximateFreeSpaceShare(); - if (ApproximateFreeSpaceShare == 0.0 || share < ApproximateFreeSpaceShare) { - ApproximateFreeSpaceShare = share; - } - } - } - - void Handle(TEvBlobStorage::TEvVMovedPatchResult::TPtr &ev) { - TEvBlobStorage::TEvVMovedPatchResult *result = ev->Get(); - A_LOG_DEBUG_S("BPPA02", "received " << ev->Get()->ToString() - << " from# " << VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID())); - NKikimrBlobStorage::TEvVMovedPatchResult &record = result->Record; - PullOutStatusFlagsAndFressSpace(record); - Orbit = std::move(result->Orbit); - TraceId = std::move(ev->TraceId); - - ui64 expectedCookie = ((ui64)OriginalId.Hash() << 32) | PatchedId.Hash(); - bool incorrectCookie = ev->Cookie != expectedCookie; - Y_VERIFY(record.HasStatus()); - bool fail = incorrectCookie - || record.GetStatus() != NKikimrProto::OK; - if (fail) { - if (incorrectCookie) { - ErrorReason = "Couldn't put the patched blob; Received TEvVMovedPatchResult with wrong cookie"; - } else { - TString subErrorReason; - if (record.HasErrorReason()) { - subErrorReason = TStringBuilder() << " VMovedPatchErrorReason# " << record.GetErrorReason(); - } - ErrorReason = TStringBuilder() << "Couldn't complete patch;" - << " VMovedPatchStatus# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()) - << subErrorReason; - } - A_LOG_INFO_S("BPPA05", "VMovedPatch failed, NaivePatch started;" - << " OriginalId# " << OriginalId - << " PatchedId# " << PatchedId - << " ErrorReason# " << ErrorReason); - StartNaivePatch(); - return; - } - - ReplyAndDie(NKikimrProto::OK); - } - - void Handle(TEvBlobStorage::TEvVPatchFoundParts::TPtr &ev) { - ReceivedFoundParts++; - - NKikimrBlobStorage::TEvVPatchFoundParts &record = ev->Get()->Record; - - Y_VERIFY(record.HasCookie()); - ui8 subgroupIdx = record.GetCookie(); - - Y_VERIFY(record.HasStatus()); - NKikimrProto::EReplyStatus status = record.GetStatus(); - - TString errorReason; - if (record.HasErrorReason()) { - errorReason = record.GetErrorReason(); - } - - bool wasReceived = std::exchange(ReceivedResponseFlags[subgroupIdx], true); - Y_VERIFY(!wasReceived); - - if (status == NKikimrProto::ERROR) { - ErrorResponses++; - ErrorResponseFlags[subgroupIdx] = true; - } - - EmptyResponseFlags[subgroupIdx] = !record.OriginalPartsSize(); - for (auto &partId : record.GetOriginalParts()) { - FoundParts.push_back({subgroupIdx, (ui8)partId}); - } - - if (record.OriginalPartsSize()) { - OkVDisksWithParts.push_back(subgroupIdx); - } - - A_LOG_INFO_S("BPPA07", "received VPatchFoundParts" - << " Status# " << status - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline - << " SubgroupIdx# " << (ui32)subgroupIdx - << " PartsCount# " << record.OriginalPartsSize() - << " ReceivedFoundParts# " << ReceivedFoundParts << '/' << SendedStarts - << " ErrorReason# " << errorReason); - - if (ReceivedFoundParts == SendedStarts) { - bool continueVPatch = VerifyPartPlacement(); - if (continueVPatch) { - continueVPatch = ContinueVPatch(); - } else { - Mon->VPatchPartPlacementVerifyFailed->Inc(); - } - if (!continueVPatch) { - StopVPatch(); - StartFallback(); - } - } - } - - void Handle(TEvBlobStorage::TEvVPatchResult::TPtr &ev) { - ReceivedResults++; - NKikimrBlobStorage::TEvVPatchResult &record = ev->Get()->Record; - PullOutStatusFlagsAndFressSpace(record); - Y_VERIFY(record.HasStatus()); - NKikimrProto::EReplyStatus status = record.GetStatus(); - TString errorReason; - if (record.HasErrorReason()) { - errorReason = record.GetErrorReason(); - } - - A_LOG_INFO_S("BPPA06", "received VPatchResult" - << " Status# " << status - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline - << " ReceivedResults# " << ReceivedResults << '/' << Info->Type.TotalPartCount() - << " ErrorReason# " << errorReason); - - Y_VERIFY(record.HasCookie()); - ui8 subgroupIdx = record.GetCookie(); - - bool wasReceived = std::exchange(ReceivedResponseFlags[subgroupIdx], true); - Y_VERIFY(!wasReceived); - - if (status != NKikimrProto::OK) { - StartFallback(); - return; - } - - if (ReceivedResults == Info->Type.TotalPartCount()) { - ReplyAndDie(NKikimrProto::OK); - } - } - - bool VerifyPartPlacementForMirror3dc() const { - constexpr ui32 DCCount = 3; - constexpr ui32 VDiskByDC = 3; - ui32 countByDC[DCCount] = {0, 0, 0}; - - for (auto &[subgroupIdx, partId] : FoundParts) { - countByDC[subgroupIdx / VDiskByDC]++; - } - - if (countByDC[0] && countByDC[1] && countByDC[2]) { - return true; - } - - ui32 x2Count = 0; - for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { - if (countByDC[dcIdx] >= 2) { - x2Count++; - } - } - return x2Count >= 2; - } - - bool VerifyPartPlacement() const { - if (Info->Type.GetErasure() == TErasureType::ErasureMirror3dc) { - return VerifyPartPlacementForMirror3dc(); - } else { - TSubgroupPartLayout layout; - - for (auto &[subgroupIdx, partId] : FoundParts) { - layout.AddItem(subgroupIdx, partId - 1, Info->Type); - } - return layout.CountEffectiveReplicas(Info->Type) == Info->Type.TotalPartCount(); - } - } - - void SendStopDiffs() { + } + + void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev) { + TEvBlobStorage::TEvPutResult *result = ev->Get(); + Orbit = std::move(result->Orbit); + TraceId = std::move(ev->TraceId); + + StatusFlags = result->StatusFlags; + ApproximateFreeSpaceShare = result->ApproximateFreeSpaceShare; + + ui32 originalIdHash = OriginalId.Hash(); + bool incorrectCookie = ev->Cookie != originalIdHash; + bool fail = incorrectCookie + || result->Status != NKikimrProto::OK; + if (fail) { + if (incorrectCookie) { + ErrorReason = "Couldn't put the patched blob; Received TEvPutResult with wrong cookie"; + } else { + ErrorReason = TStringBuilder() << "Couldn't put the patched blob;" + << " PutStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) + << " PutErrorReason# " << result->ErrorReason; + } + R_LOG_ERROR_S("BPPA03", ErrorReason); + ReplyAndDie(NKikimrProto::ERROR); + return; + } + + ReplyAndDie(NKikimrProto::OK); + } + + template <typename TEventResultRecord> + void PullOutStatusFlagsAndFressSpace(const TEventResultRecord &record){ + if (record.HasStatusFlags()) { + StatusFlags.Merge(record.GetStatusFlags()); + } + if (record.HasApproximateFreeSpaceShare()) { + float share = record.GetApproximateFreeSpaceShare(); + if (ApproximateFreeSpaceShare == 0.0 || share < ApproximateFreeSpaceShare) { + ApproximateFreeSpaceShare = share; + } + } + } + + void Handle(TEvBlobStorage::TEvVMovedPatchResult::TPtr &ev) { + TEvBlobStorage::TEvVMovedPatchResult *result = ev->Get(); + A_LOG_DEBUG_S("BPPA02", "received " << ev->Get()->ToString() + << " from# " << VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID())); + NKikimrBlobStorage::TEvVMovedPatchResult &record = result->Record; + PullOutStatusFlagsAndFressSpace(record); + Orbit = std::move(result->Orbit); + TraceId = std::move(ev->TraceId); + + ui64 expectedCookie = ((ui64)OriginalId.Hash() << 32) | PatchedId.Hash(); + bool incorrectCookie = ev->Cookie != expectedCookie; + Y_VERIFY(record.HasStatus()); + bool fail = incorrectCookie + || record.GetStatus() != NKikimrProto::OK; + if (fail) { + if (incorrectCookie) { + ErrorReason = "Couldn't put the patched blob; Received TEvVMovedPatchResult with wrong cookie"; + } else { + TString subErrorReason; + if (record.HasErrorReason()) { + subErrorReason = TStringBuilder() << " VMovedPatchErrorReason# " << record.GetErrorReason(); + } + ErrorReason = TStringBuilder() << "Couldn't complete patch;" + << " VMovedPatchStatus# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()) + << subErrorReason; + } + A_LOG_INFO_S("BPPA05", "VMovedPatch failed, NaivePatch started;" + << " OriginalId# " << OriginalId + << " PatchedId# " << PatchedId + << " ErrorReason# " << ErrorReason); + StartNaivePatch(); + return; + } + + ReplyAndDie(NKikimrProto::OK); + } + + void Handle(TEvBlobStorage::TEvVPatchFoundParts::TPtr &ev) { + ReceivedFoundParts++; + + NKikimrBlobStorage::TEvVPatchFoundParts &record = ev->Get()->Record; + + Y_VERIFY(record.HasCookie()); + ui8 subgroupIdx = record.GetCookie(); + + Y_VERIFY(record.HasStatus()); + NKikimrProto::EReplyStatus status = record.GetStatus(); + + TString errorReason; + if (record.HasErrorReason()) { + errorReason = record.GetErrorReason(); + } + + bool wasReceived = std::exchange(ReceivedResponseFlags[subgroupIdx], true); + Y_VERIFY(!wasReceived); + + if (status == NKikimrProto::ERROR) { + ErrorResponses++; + ErrorResponseFlags[subgroupIdx] = true; + } + + EmptyResponseFlags[subgroupIdx] = !record.OriginalPartsSize(); + for (auto &partId : record.GetOriginalParts()) { + FoundParts.push_back({subgroupIdx, (ui8)partId}); + } + + if (record.OriginalPartsSize()) { + OkVDisksWithParts.push_back(subgroupIdx); + } + + A_LOG_INFO_S("BPPA07", "received VPatchFoundParts" + << " Status# " << status + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline + << " SubgroupIdx# " << (ui32)subgroupIdx + << " PartsCount# " << record.OriginalPartsSize() + << " ReceivedFoundParts# " << ReceivedFoundParts << '/' << SendedStarts + << " ErrorReason# " << errorReason); + + if (ReceivedFoundParts == SendedStarts) { + bool continueVPatch = VerifyPartPlacement(); + if (continueVPatch) { + continueVPatch = ContinueVPatch(); + } else { + Mon->VPatchPartPlacementVerifyFailed->Inc(); + } + if (!continueVPatch) { + StopVPatch(); + StartFallback(); + } + } + } + + void Handle(TEvBlobStorage::TEvVPatchResult::TPtr &ev) { + ReceivedResults++; + NKikimrBlobStorage::TEvVPatchResult &record = ev->Get()->Record; + PullOutStatusFlagsAndFressSpace(record); + Y_VERIFY(record.HasStatus()); + NKikimrProto::EReplyStatus status = record.GetStatus(); + TString errorReason; + if (record.HasErrorReason()) { + errorReason = record.GetErrorReason(); + } + + A_LOG_INFO_S("BPPA06", "received VPatchResult" + << " Status# " << status + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline + << " ReceivedResults# " << ReceivedResults << '/' << Info->Type.TotalPartCount() + << " ErrorReason# " << errorReason); + + Y_VERIFY(record.HasCookie()); + ui8 subgroupIdx = record.GetCookie(); + + bool wasReceived = std::exchange(ReceivedResponseFlags[subgroupIdx], true); + Y_VERIFY(!wasReceived); + + if (status != NKikimrProto::OK) { + StartFallback(); + return; + } + + if (ReceivedResults == Info->Type.TotalPartCount()) { + ReplyAndDie(NKikimrProto::OK); + } + } + + bool VerifyPartPlacementForMirror3dc() const { + constexpr ui32 DCCount = 3; + constexpr ui32 VDiskByDC = 3; + ui32 countByDC[DCCount] = {0, 0, 0}; + + for (auto &[subgroupIdx, partId] : FoundParts) { + countByDC[subgroupIdx / VDiskByDC]++; + } + + if (countByDC[0] && countByDC[1] && countByDC[2]) { + return true; + } + + ui32 x2Count = 0; + for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { + if (countByDC[dcIdx] >= 2) { + x2Count++; + } + } + return x2Count >= 2; + } + + bool VerifyPartPlacement() const { + if (Info->Type.GetErasure() == TErasureType::ErasureMirror3dc) { + return VerifyPartPlacementForMirror3dc(); + } else { + TSubgroupPartLayout layout; + + for (auto &[subgroupIdx, partId] : FoundParts) { + layout.AddItem(subgroupIdx, partId - 1, Info->Type); + } + return layout.CountEffectiveReplicas(Info->Type) == Info->Type.TotalPartCount(); + } + } + + void SendStopDiffs() { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPatchDiff>> events; - for (ui32 vdiskIdx = 0; vdiskIdx < VDisks.size(); ++vdiskIdx) { - if (!ErrorResponseFlags[vdiskIdx] && !EmptyResponseFlags[vdiskIdx] && ReceivedResponseFlags[vdiskIdx]) { + for (ui32 vdiskIdx = 0; vdiskIdx < VDisks.size(); ++vdiskIdx) { + if (!ErrorResponseFlags[vdiskIdx] && !EmptyResponseFlags[vdiskIdx] && ReceivedResponseFlags[vdiskIdx]) { std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> ev = std::make_unique<TEvBlobStorage::TEvVPatchDiff>( - OriginalId, PatchedId, VDisks[vdiskIdx], 0, Deadline, vdiskIdx); - ev->SetForceEnd(); + OriginalId, PatchedId, VDisks[vdiskIdx], 0, Deadline, vdiskIdx); + ev->SetForceEnd(); events.emplace_back(std::move(ev)); - } - } - SendToQueues(events, false); - } - - bool WithXorDiffs() const { - return Info->Type.ErasureFamily() != TErasureType::ErasureMirror; - } - - void SendDiffs(const TStackVec<TPartPlacement, TypicalPartsInBlob> &placement) { + } + } + SendToQueues(events, false); + } + + bool WithXorDiffs() const { + return Info->Type.ErasureFamily() != TErasureType::ErasureMirror; + } + + void SendDiffs(const TStackVec<TPartPlacement, TypicalPartsInBlob> &placement) { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPatchDiff>> events; - - TPartDiffSet diffSet; - TVector<TDiff> diffs; - diffs.reserve(DiffCount); - for (ui32 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { - diffs.emplace_back(Diffs[diffIdx].Buffer, Diffs[diffIdx].Offset, false, false); - } - Info->Type.SplitDiffs(TErasureType::CrcModeNone, OriginalId.BlobSize(), diffs, diffSet); - - ui32 dataParts = Info->Type.DataParts(); - ui32 dataPartCount = 0; - TStackVec<TPartPlacement, TypicalPartsInBlob> parityPlacements; - if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { - for (const TPartPlacement &partPlacement : placement) { - if (partPlacement.PartId <= dataParts) { - dataPartCount++; - } else { - parityPlacements.emplace_back(partPlacement); - } - } - } - - for (const TPartPlacement &partPlacement : placement) { - ui32 idxInSubgroup = partPlacement.VDiskIdxInSubgroup; - ui32 patchedPartId = partPlacement.PartId; - Y_VERIFY_S(idxInSubgroup < VDisks.size(), "vdisidxInSubgroupkIdx# " << idxInSubgroup << "/" << VDisks.size()); - - Y_VERIFY(Info->GetIdxInSubgroup(VDisks[idxInSubgroup], OriginalId.Hash()) == idxInSubgroup); - ui32 patchedIdxInSubgroup = Info->GetIdxInSubgroup(VDisks[idxInSubgroup], PatchedId.Hash()); - if (patchedIdxInSubgroup != idxInSubgroup) { - // now only mirror3dc has this case (has 9 vdisks instead of 4 or 8) - Y_VERIFY(Info->Type.GetErasure() == TErasureType::ErasureMirror3dc); - patchedPartId = 1 + patchedIdxInSubgroup / 3;; - } - - ReceivedResponseFlags[idxInSubgroup] = false; - TLogoBlobID originalPartBlobId(OriginalId, partPlacement.PartId); - TLogoBlobID partchedPartBlobId(PatchedId, partPlacement.PartId); - - ui32 waitedXorDiffs = (partPlacement.PartId > dataParts) ? dataPartCount : 0; + + TPartDiffSet diffSet; + TVector<TDiff> diffs; + diffs.reserve(DiffCount); + for (ui32 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { + diffs.emplace_back(Diffs[diffIdx].Buffer, Diffs[diffIdx].Offset, false, false); + } + Info->Type.SplitDiffs(TErasureType::CrcModeNone, OriginalId.BlobSize(), diffs, diffSet); + + ui32 dataParts = Info->Type.DataParts(); + ui32 dataPartCount = 0; + TStackVec<TPartPlacement, TypicalPartsInBlob> parityPlacements; + if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { + for (const TPartPlacement &partPlacement : placement) { + if (partPlacement.PartId <= dataParts) { + dataPartCount++; + } else { + parityPlacements.emplace_back(partPlacement); + } + } + } + + for (const TPartPlacement &partPlacement : placement) { + ui32 idxInSubgroup = partPlacement.VDiskIdxInSubgroup; + ui32 patchedPartId = partPlacement.PartId; + Y_VERIFY_S(idxInSubgroup < VDisks.size(), "vdisidxInSubgroupkIdx# " << idxInSubgroup << "/" << VDisks.size()); + + Y_VERIFY(Info->GetIdxInSubgroup(VDisks[idxInSubgroup], OriginalId.Hash()) == idxInSubgroup); + ui32 patchedIdxInSubgroup = Info->GetIdxInSubgroup(VDisks[idxInSubgroup], PatchedId.Hash()); + if (patchedIdxInSubgroup != idxInSubgroup) { + // now only mirror3dc has this case (has 9 vdisks instead of 4 or 8) + Y_VERIFY(Info->Type.GetErasure() == TErasureType::ErasureMirror3dc); + patchedPartId = 1 + patchedIdxInSubgroup / 3;; + } + + ReceivedResponseFlags[idxInSubgroup] = false; + TLogoBlobID originalPartBlobId(OriginalId, partPlacement.PartId); + TLogoBlobID partchedPartBlobId(PatchedId, partPlacement.PartId); + + ui32 waitedXorDiffs = (partPlacement.PartId > dataParts) ? dataPartCount : 0; auto ev = std::make_unique<TEvBlobStorage::TEvVPatchDiff>(originalPartBlobId, partchedPartBlobId, VDisks[idxInSubgroup], waitedXorDiffs, Deadline, idxInSubgroup); - - ui32 diffForPartIdx = 0; - if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { - diffForPartIdx = partPlacement.PartId - 1; - } - auto &diffsForPart = diffSet.PartDiffs[diffForPartIdx].Diffs; - for (auto &diff : diffsForPart) { - ev->AddDiff(diff.Offset, diff.Buffer); - } - - for (const TPartPlacement &parity : parityPlacements) { - ev->AddXorReceiver(VDisks[parity.VDiskIdxInSubgroup], parity.PartId); - } - + + ui32 diffForPartIdx = 0; + if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { + diffForPartIdx = partPlacement.PartId - 1; + } + auto &diffsForPart = diffSet.PartDiffs[diffForPartIdx].Diffs; + for (auto &diff : diffsForPart) { + ev->AddDiff(diff.Offset, diff.Buffer); + } + + for (const TPartPlacement &parity : parityPlacements) { + ev->AddXorReceiver(VDisks[parity.VDiskIdxInSubgroup], parity.PartId); + } + events.push_back(std::move(ev)); - } - SendToQueues(events, false); - SendStopDiffs(); - ReceivedResponseFlags.assign(VDisks.size(), false); - } - - void SendDiffs(const TStackVec<bool, TypicalPartsInBlob> &inPrimary, - const TStackVec<ui32, TypicalHandoffCount> &choosenHandoffForParts) - { - ui32 handoffPartIdx = 0; - TStackVec<ui32, TypicalPartsInBlob> vdiskIdxForParts(Info->Type.TotalPartCount()); - for (ui32 partIdx = 0; partIdx < Info->Type.TotalPartCount(); ++partIdx) { - vdiskIdxForParts[partIdx] = partIdx; - if (!inPrimary[partIdx]) { - vdiskIdxForParts[partIdx] = choosenHandoffForParts[handoffPartIdx]; - handoffPartIdx++; - } - } - TStackVec<TPartPlacement, TypicalPartsInBlob> placements; - ui32 dataParts = Info->Type.DataParts(); - for (ui32 partIdx = 0; partIdx < Info->Type.TotalPartCount(); ++partIdx) { - ui32 vdiskIdx = vdiskIdxForParts[partIdx]; - Y_VERIFY_S(vdiskIdx == partIdx || vdiskIdx >= dataParts, "vdiskIdx# " << vdiskIdx << " partIdx# " << partIdx); - placements.push_back(TPartPlacement{static_cast<ui8>(vdiskIdx), static_cast<ui8>(partIdx + 1)}); - } - SendDiffs(placements); - } - - void StartMovedPatch() { - A_LOG_DEBUG_S("BPPA12", "StartMovedPatch" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline); - Become(&TThis::MovedPatchState); - - ui32 subgroupIdx = 0; - if (OkVDisksWithParts) { - ui32 okVDiskIdx = RandomNumber<ui32>(OkVDisksWithParts.size()); - subgroupIdx = OkVDisksWithParts[okVDiskIdx]; - } else { - subgroupIdx = RandomNumber<ui32>(Info->Type.TotalPartCount()); - } - TVDiskID vDisk = Info->GetVDiskInSubgroup(subgroupIdx, OriginalId.Hash()); + } + SendToQueues(events, false); + SendStopDiffs(); + ReceivedResponseFlags.assign(VDisks.size(), false); + } + + void SendDiffs(const TStackVec<bool, TypicalPartsInBlob> &inPrimary, + const TStackVec<ui32, TypicalHandoffCount> &choosenHandoffForParts) + { + ui32 handoffPartIdx = 0; + TStackVec<ui32, TypicalPartsInBlob> vdiskIdxForParts(Info->Type.TotalPartCount()); + for (ui32 partIdx = 0; partIdx < Info->Type.TotalPartCount(); ++partIdx) { + vdiskIdxForParts[partIdx] = partIdx; + if (!inPrimary[partIdx]) { + vdiskIdxForParts[partIdx] = choosenHandoffForParts[handoffPartIdx]; + handoffPartIdx++; + } + } + TStackVec<TPartPlacement, TypicalPartsInBlob> placements; + ui32 dataParts = Info->Type.DataParts(); + for (ui32 partIdx = 0; partIdx < Info->Type.TotalPartCount(); ++partIdx) { + ui32 vdiskIdx = vdiskIdxForParts[partIdx]; + Y_VERIFY_S(vdiskIdx == partIdx || vdiskIdx >= dataParts, "vdiskIdx# " << vdiskIdx << " partIdx# " << partIdx); + placements.push_back(TPartPlacement{static_cast<ui8>(vdiskIdx), static_cast<ui8>(partIdx + 1)}); + } + SendDiffs(placements); + } + + void StartMovedPatch() { + A_LOG_DEBUG_S("BPPA12", "StartMovedPatch" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline); + Become(&TThis::MovedPatchState); + + ui32 subgroupIdx = 0; + if (OkVDisksWithParts) { + ui32 okVDiskIdx = RandomNumber<ui32>(OkVDisksWithParts.size()); + subgroupIdx = OkVDisksWithParts[okVDiskIdx]; + } else { + subgroupIdx = RandomNumber<ui32>(Info->Type.TotalPartCount()); + } + TVDiskID vDisk = Info->GetVDiskInSubgroup(subgroupIdx, OriginalId.Hash()); TDeque<std::unique_ptr<TEvBlobStorage::TEvVMovedPatch>> events; - - ui64 cookie = ((ui64)OriginalId.Hash() << 32) | PatchedId.Hash(); - events.emplace_back(new TEvBlobStorage::TEvVMovedPatch(OriginalGroupId, Info->GroupID, - OriginalId, PatchedId, vDisk, false, cookie, Deadline)); - events.back()->Orbit = std::move(Orbit); - for (ui64 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { - auto &diff = Diffs[diffIdx]; - events.back()->AddDiff(diff.Offset, diff.Buffer); - } - SendToQueues(events, false); - } - - void StartNaivePatch() { - A_LOG_DEBUG_S("BPPA11", "StartNaivePatch" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline); - Become(&TThis::NaiveState); + + ui64 cookie = ((ui64)OriginalId.Hash() << 32) | PatchedId.Hash(); + events.emplace_back(new TEvBlobStorage::TEvVMovedPatch(OriginalGroupId, Info->GroupID, + OriginalId, PatchedId, vDisk, false, cookie, Deadline)); + events.back()->Orbit = std::move(Orbit); + for (ui64 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { + auto &diff = Diffs[diffIdx]; + events.back()->AddDiff(diff.Offset, diff.Buffer); + } + SendToQueues(events, false); + } + + void StartNaivePatch() { + A_LOG_DEBUG_S("BPPA11", "StartNaivePatch" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline); + Become(&TThis::NaiveState); auto get = std::make_unique<TEvBlobStorage::TEvGet>(OriginalId, 0, OriginalId.BlobSize(), Deadline, NKikimrBlobStorage::AsyncRead); - get->Orbit = std::move(Orbit); - if (OriginalGroupId == Info->GroupID) { + get->Orbit = std::move(Orbit); + if (OriginalGroupId == Info->GroupID) { Send(ProxyActorId, get.release(), 0, PatchedId.Hash(), std::move(TraceId)); - } else { + } else { SendToBSProxy(SelfId(), OriginalGroupId, get.release(), PatchedId.Hash(), std::move(TraceId)); - } - } - - void StartFallback() { - Mon->PatchesWithFallback->Inc(); - if (WithMovingPatchRequestToStaticNode && UseVPatch) { - StartMovedPatch(); - } else { - StartNaivePatch(); - } - } - - void StartVPatch() { - Become(&TThis::VPatchState); - - Info->PickSubgroup(OriginalId.Hash(), &VDisks, nullptr); - ReceivedResponseFlags.assign(VDisks.size(), false); - ErrorResponseFlags.assign(VDisks.size(), false); - EmptyResponseFlags.assign(VDisks.size(), false); - + } + } + + void StartFallback() { + Mon->PatchesWithFallback->Inc(); + if (WithMovingPatchRequestToStaticNode && UseVPatch) { + StartMovedPatch(); + } else { + StartNaivePatch(); + } + } + + void StartVPatch() { + Become(&TThis::VPatchState); + + Info->PickSubgroup(OriginalId.Hash(), &VDisks, nullptr); + ReceivedResponseFlags.assign(VDisks.size(), false); + ErrorResponseFlags.assign(VDisks.size(), false); + EmptyResponseFlags.assign(VDisks.size(), false); + TDeque<std::unique_ptr<TEvBlobStorage::TEvVPatchStart>> events; - - for (ui32 idx = 0; idx < VDisks.size(); ++idx) { + + for (ui32 idx = 0; idx < VDisks.size(); ++idx) { std::unique_ptr<TEvBlobStorage::TEvVPatchStart> ev = std::make_unique<TEvBlobStorage::TEvVPatchStart>( - OriginalId, PatchedId, VDisks[idx], Deadline, idx, true); - events.emplace_back(std::move(ev)); - SendedStarts++; - } - - A_LOG_DEBUG_S("BPPA08", "StartVPatcn" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline - << " SendedStarts# " << SendedStarts); - - SendToQueues(events, false); - } - - bool FindHandoffs(const TStackVec<TStackVec<ui32, TypicalHandoffCount>, TypicalPartsInBlob>& handoffForParts, - const TStackVec<ui32, TypicalPartsInBlob> &handoffParts, - TStackVec<ui32, TypicalHandoffCount> *choosenHandoffForParts, ui8 depth = 0) - { - auto &choosen = *choosenHandoffForParts; - Y_VERIFY_DEBUG(choosen.size() == handoffParts.size()); - if (depth >= handoffParts.size()) { - return true; - } - ui32 partIdx = handoffParts[depth]; - for (ui32 idx = 0; idx < handoffForParts[partIdx].size(); ++idx) { - Y_VERIFY_DEBUG(depth < choosen.size()); - choosen[depth] = handoffForParts[partIdx][idx]; - bool isCorrect = true; - for (ui32 prevDepth = 0; prevDepth < depth; ++prevDepth) { - Y_VERIFY_DEBUG(prevDepth < choosen.size()); - isCorrect &= (choosen[depth] != choosen[prevDepth]); - } - bool hasAnswer = false; - if (isCorrect) { - hasAnswer = FindHandoffs(handoffForParts, handoffParts, choosenHandoffForParts, depth + 1); - } - if (hasAnswer) { - return true; - } - } - return false; - } - - TString ConvertFoundPartsToString() const { - TStringBuilder str; - str << "["; - for (auto &a : FoundParts) { - str << a.ToString() << ' '; - } - str << ']'; - return str; - } - - bool ContinueVPatchForMirror3dc() { - constexpr ui32 DCCount = 3; - constexpr ui32 VDiskByDC = 3; - ui32 countByDC[DCCount] = {0, 0, 0}; - TPartPlacement diskByDC[DCCount][VDiskByDC]; - - for (auto &[subgroupIdx, partId] : FoundParts) { - ui32 dc = subgroupIdx / VDiskByDC; - ui32 idx = countByDC[dc]; - diskByDC[dc][idx] = TPartPlacement{subgroupIdx, partId}; - countByDC[dc]++; - } - - if (countByDC[0] && countByDC[1] && countByDC[2]) { - SendDiffs({diskByDC[0][0], diskByDC[1][0], diskByDC[2][0]}); - return true; - } - - ui32 x2Count = 0; - for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { - if (countByDC[dcIdx] >= 2) { - x2Count++; - } - } - if (x2Count < 2) { - return false; - } - TStackVec<TPartPlacement, TypicalPartsInBlob> placements; - for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { - if (countByDC[dcIdx] >= 2) { - placements.push_back(diskByDC[dcIdx][0]); - placements.push_back(diskByDC[dcIdx][1]); - } - } - SendDiffs(placements); - return true; - } - - bool ContinueVPatch() { - A_LOG_DEBUG_S("BPPA09", "ContinueVPatch" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " FoundParts# " << ConvertFoundPartsToString() - << " Deadline# " << Deadline); - - if (Info->Type.GetErasure() == TErasureType::ErasureMirror3dc) { - return ContinueVPatchForMirror3dc(); - } - - TStackVec<bool, TypicalPartsInBlob> inPrimary; - TStackVec<TStackVec<ui32, TypicalHandoffCount>, TypicalPartsInBlob> handoffForParts; - - inPrimary.resize(Info->Type.TotalPartCount()); - handoffForParts.resize(inPrimary.size()); - - for (auto &[subgroupIdx, partId] : FoundParts) { - if (subgroupIdx == partId) { - inPrimary[subgroupIdx] = true; - } else { - handoffForParts[partId - 1].push_back(subgroupIdx); - } - } - - TStackVec<ui32, TypicalPartsInBlob> handoffParts; - for (ui32 idx = 0; idx < inPrimary.size(); ++idx) { - if (!inPrimary[idx]) { - handoffParts.push_back(idx); - } - } - - TStackVec<ui32, TypicalHandoffCount> choosenHandoffForParts(handoffParts.size()); - if (handoffParts.size()) { - bool find = FindHandoffs(handoffForParts, handoffParts, &choosenHandoffForParts); - Y_VERIFY_DEBUG_S(find, "handoffParts# " << FormatList(handoffParts) - << " FoundParts# " << ConvertFoundPartsToString() - << " choosenHandoffForParts# " << FormatList(choosenHandoffForParts) - << " inPrimary# " << FormatList(inPrimary)); - if (!find) { - Mon->VPatchContinueFailed->Inc(); - return false; - } - } - - SendDiffs(inPrimary, choosenHandoffForParts); - return true; - } - - void StopVPatch() { - A_LOG_DEBUG_S("BPPA10", "StopVPatch" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline); - SendStopDiffs(); - ReceivedResponseFlags.assign(VDisks.size(), false); - } - - bool CheckDiffs() { - for (ui32 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { - bool ok = Diffs[diffIdx].Offset < OriginalId.BlobSize(); - ok &= Diffs[diffIdx].Offset + Diffs[diffIdx].Buffer.size() <= OriginalId.BlobSize(); - if (!ok) { - TStringBuilder str; - str << "Diff at index " << diffIdx << " went beyound the blob;" - << " BlobSize# " << OriginalId.BlobSize() - << " DiffStart# " << Diffs[diffIdx].Offset - << " DiffEnd# " << Diffs[diffIdx].Offset + Diffs[diffIdx].Buffer.size() << Endl; - ErrorReason = str; - return false; - } - } - for (ui32 diffIdx = 1; diffIdx < DiffCount; ++diffIdx) { - ui32 prevIdx = diffIdx - 1; - bool ok = Diffs[prevIdx].Offset + Diffs[prevIdx].Buffer.size() <= Diffs[diffIdx].Offset; - if (!ok) { - TStringBuilder str; - str << "the end of the diff at index " << prevIdx << " righter than" - << " the start of the diff at index " << prevIdx << ';' - << " PrevDiffEnd# " << Diffs[prevIdx].Offset + Diffs[prevIdx].Buffer.size() - << " DiffStart# " << Diffs[diffIdx].Offset << Endl; - ErrorReason = str; - return false; - } - } - return true; - } - - void Bootstrap() { - A_LOG_INFO_S("BPPA01", "bootstrap" - << " ActorId# " << SelfId() - << " Group# " << Info->GroupID - << " DiffCount# " << DiffCount - << " OriginalBlob# " << OriginalId - << " PatchedBlob# " << PatchedId - << " Deadline# " << Deadline - << " RestartCounter# " << RestartCounter); - TLogoBlobID truePatchedBlobId = PatchedId; - bool result = true; - if (Info->Type.ErasureFamily() == TErasureType::ErasureParityBlock) { - result = TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(OriginalId, &truePatchedBlobId, - MaskForCookieBruteForcing, OriginalGroupId, Info->GroupID); - if (result && PatchedId != truePatchedBlobId) { - TStringBuilder str; - str << "PatchedId wasn't from TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement;"; - str << " OriginalId# " << OriginalId; - str << " PatchedId# " << PatchedId; - ErrorReason = str; - ReplyAndDie(NKikimrProto::ERROR); - return; - } - } - - if (!CheckDiffs()) { - ReplyAndDie(NKikimrProto::ERROR); - return; - } - - bool isAllowedErasure = Info->Type.ErasureFamily() == TErasureType::ErasureParityBlock - || Info->Type.GetErasure() == TErasureType::ErasureNone - || Info->Type.GetErasure() == TErasureType::ErasureMirror3 - || Info->Type.GetErasure() == TErasureType::ErasureMirror3dc; - if (result && isAllowedErasure && UseVPatch && OriginalGroupId == Info->GroupID) { - StartVPatch(); - } else { - StartFallback(); - } - } - - STATEFN(NaiveState) { - if (ProcessEvent(ev)) { - return; - } - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvGetResult, Handle); - hFunc(TEvBlobStorage::TEvPutResult, Handle); - IgnoreFunc(TEvBlobStorage::TEvVPatchResult); - default: - Y_FAIL("Received unknown event"); - }; - } - - STATEFN(MovedPatchState) { - if (ProcessEvent(ev)) { - return; - } - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVMovedPatchResult, Handle); - IgnoreFunc(TEvBlobStorage::TEvVPatchResult); - default: - Y_FAIL("Received unknown event"); - }; - } - - STATEFN(VPatchState) { - if (ProcessEvent(ev)) { - return; - } - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVPatchFoundParts, Handle); - hFunc(TEvBlobStorage::TEvVPatchResult, Handle); - default: - Y_FAIL("Received unknown event"); - }; - } -}; - -IActor* CreateBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, - const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, - ui64 cookie, NWilson::TTraceId traceId, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - const TActorId &proxyId, bool useVPatch) { - return new TBlobStorageGroupPatchRequest(info, state, source, mon, ev, cookie, std::move(traceId), - now, storagePoolCounters, proxyId, useVPatch); -} - -}//NKikimr + OriginalId, PatchedId, VDisks[idx], Deadline, idx, true); + events.emplace_back(std::move(ev)); + SendedStarts++; + } + + A_LOG_DEBUG_S("BPPA08", "StartVPatcn" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline + << " SendedStarts# " << SendedStarts); + + SendToQueues(events, false); + } + + bool FindHandoffs(const TStackVec<TStackVec<ui32, TypicalHandoffCount>, TypicalPartsInBlob>& handoffForParts, + const TStackVec<ui32, TypicalPartsInBlob> &handoffParts, + TStackVec<ui32, TypicalHandoffCount> *choosenHandoffForParts, ui8 depth = 0) + { + auto &choosen = *choosenHandoffForParts; + Y_VERIFY_DEBUG(choosen.size() == handoffParts.size()); + if (depth >= handoffParts.size()) { + return true; + } + ui32 partIdx = handoffParts[depth]; + for (ui32 idx = 0; idx < handoffForParts[partIdx].size(); ++idx) { + Y_VERIFY_DEBUG(depth < choosen.size()); + choosen[depth] = handoffForParts[partIdx][idx]; + bool isCorrect = true; + for (ui32 prevDepth = 0; prevDepth < depth; ++prevDepth) { + Y_VERIFY_DEBUG(prevDepth < choosen.size()); + isCorrect &= (choosen[depth] != choosen[prevDepth]); + } + bool hasAnswer = false; + if (isCorrect) { + hasAnswer = FindHandoffs(handoffForParts, handoffParts, choosenHandoffForParts, depth + 1); + } + if (hasAnswer) { + return true; + } + } + return false; + } + + TString ConvertFoundPartsToString() const { + TStringBuilder str; + str << "["; + for (auto &a : FoundParts) { + str << a.ToString() << ' '; + } + str << ']'; + return str; + } + + bool ContinueVPatchForMirror3dc() { + constexpr ui32 DCCount = 3; + constexpr ui32 VDiskByDC = 3; + ui32 countByDC[DCCount] = {0, 0, 0}; + TPartPlacement diskByDC[DCCount][VDiskByDC]; + + for (auto &[subgroupIdx, partId] : FoundParts) { + ui32 dc = subgroupIdx / VDiskByDC; + ui32 idx = countByDC[dc]; + diskByDC[dc][idx] = TPartPlacement{subgroupIdx, partId}; + countByDC[dc]++; + } + + if (countByDC[0] && countByDC[1] && countByDC[2]) { + SendDiffs({diskByDC[0][0], diskByDC[1][0], diskByDC[2][0]}); + return true; + } + + ui32 x2Count = 0; + for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { + if (countByDC[dcIdx] >= 2) { + x2Count++; + } + } + if (x2Count < 2) { + return false; + } + TStackVec<TPartPlacement, TypicalPartsInBlob> placements; + for (ui32 dcIdx = 0; dcIdx < DCCount; ++dcIdx) { + if (countByDC[dcIdx] >= 2) { + placements.push_back(diskByDC[dcIdx][0]); + placements.push_back(diskByDC[dcIdx][1]); + } + } + SendDiffs(placements); + return true; + } + + bool ContinueVPatch() { + A_LOG_DEBUG_S("BPPA09", "ContinueVPatch" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " FoundParts# " << ConvertFoundPartsToString() + << " Deadline# " << Deadline); + + if (Info->Type.GetErasure() == TErasureType::ErasureMirror3dc) { + return ContinueVPatchForMirror3dc(); + } + + TStackVec<bool, TypicalPartsInBlob> inPrimary; + TStackVec<TStackVec<ui32, TypicalHandoffCount>, TypicalPartsInBlob> handoffForParts; + + inPrimary.resize(Info->Type.TotalPartCount()); + handoffForParts.resize(inPrimary.size()); + + for (auto &[subgroupIdx, partId] : FoundParts) { + if (subgroupIdx == partId) { + inPrimary[subgroupIdx] = true; + } else { + handoffForParts[partId - 1].push_back(subgroupIdx); + } + } + + TStackVec<ui32, TypicalPartsInBlob> handoffParts; + for (ui32 idx = 0; idx < inPrimary.size(); ++idx) { + if (!inPrimary[idx]) { + handoffParts.push_back(idx); + } + } + + TStackVec<ui32, TypicalHandoffCount> choosenHandoffForParts(handoffParts.size()); + if (handoffParts.size()) { + bool find = FindHandoffs(handoffForParts, handoffParts, &choosenHandoffForParts); + Y_VERIFY_DEBUG_S(find, "handoffParts# " << FormatList(handoffParts) + << " FoundParts# " << ConvertFoundPartsToString() + << " choosenHandoffForParts# " << FormatList(choosenHandoffForParts) + << " inPrimary# " << FormatList(inPrimary)); + if (!find) { + Mon->VPatchContinueFailed->Inc(); + return false; + } + } + + SendDiffs(inPrimary, choosenHandoffForParts); + return true; + } + + void StopVPatch() { + A_LOG_DEBUG_S("BPPA10", "StopVPatch" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline); + SendStopDiffs(); + ReceivedResponseFlags.assign(VDisks.size(), false); + } + + bool CheckDiffs() { + for (ui32 diffIdx = 0; diffIdx < DiffCount; ++diffIdx) { + bool ok = Diffs[diffIdx].Offset < OriginalId.BlobSize(); + ok &= Diffs[diffIdx].Offset + Diffs[diffIdx].Buffer.size() <= OriginalId.BlobSize(); + if (!ok) { + TStringBuilder str; + str << "Diff at index " << diffIdx << " went beyound the blob;" + << " BlobSize# " << OriginalId.BlobSize() + << " DiffStart# " << Diffs[diffIdx].Offset + << " DiffEnd# " << Diffs[diffIdx].Offset + Diffs[diffIdx].Buffer.size() << Endl; + ErrorReason = str; + return false; + } + } + for (ui32 diffIdx = 1; diffIdx < DiffCount; ++diffIdx) { + ui32 prevIdx = diffIdx - 1; + bool ok = Diffs[prevIdx].Offset + Diffs[prevIdx].Buffer.size() <= Diffs[diffIdx].Offset; + if (!ok) { + TStringBuilder str; + str << "the end of the diff at index " << prevIdx << " righter than" + << " the start of the diff at index " << prevIdx << ';' + << " PrevDiffEnd# " << Diffs[prevIdx].Offset + Diffs[prevIdx].Buffer.size() + << " DiffStart# " << Diffs[diffIdx].Offset << Endl; + ErrorReason = str; + return false; + } + } + return true; + } + + void Bootstrap() { + A_LOG_INFO_S("BPPA01", "bootstrap" + << " ActorId# " << SelfId() + << " Group# " << Info->GroupID + << " DiffCount# " << DiffCount + << " OriginalBlob# " << OriginalId + << " PatchedBlob# " << PatchedId + << " Deadline# " << Deadline + << " RestartCounter# " << RestartCounter); + TLogoBlobID truePatchedBlobId = PatchedId; + bool result = true; + if (Info->Type.ErasureFamily() == TErasureType::ErasureParityBlock) { + result = TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(OriginalId, &truePatchedBlobId, + MaskForCookieBruteForcing, OriginalGroupId, Info->GroupID); + if (result && PatchedId != truePatchedBlobId) { + TStringBuilder str; + str << "PatchedId wasn't from TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement;"; + str << " OriginalId# " << OriginalId; + str << " PatchedId# " << PatchedId; + ErrorReason = str; + ReplyAndDie(NKikimrProto::ERROR); + return; + } + } + + if (!CheckDiffs()) { + ReplyAndDie(NKikimrProto::ERROR); + return; + } + + bool isAllowedErasure = Info->Type.ErasureFamily() == TErasureType::ErasureParityBlock + || Info->Type.GetErasure() == TErasureType::ErasureNone + || Info->Type.GetErasure() == TErasureType::ErasureMirror3 + || Info->Type.GetErasure() == TErasureType::ErasureMirror3dc; + if (result && isAllowedErasure && UseVPatch && OriginalGroupId == Info->GroupID) { + StartVPatch(); + } else { + StartFallback(); + } + } + + STATEFN(NaiveState) { + if (ProcessEvent(ev)) { + return; + } + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvGetResult, Handle); + hFunc(TEvBlobStorage::TEvPutResult, Handle); + IgnoreFunc(TEvBlobStorage::TEvVPatchResult); + default: + Y_FAIL("Received unknown event"); + }; + } + + STATEFN(MovedPatchState) { + if (ProcessEvent(ev)) { + return; + } + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVMovedPatchResult, Handle); + IgnoreFunc(TEvBlobStorage::TEvVPatchResult); + default: + Y_FAIL("Received unknown event"); + }; + } + + STATEFN(VPatchState) { + if (ProcessEvent(ev)) { + return; + } + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVPatchFoundParts, Handle); + hFunc(TEvBlobStorage::TEvVPatchResult, Handle); + default: + Y_FAIL("Received unknown event"); + }; + } +}; + +IActor* CreateBlobStorageGroupPatchRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, + const TIntrusivePtr<TGroupQueues> &state, const TActorId &source, + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TEvBlobStorage::TEvPatch *ev, + ui64 cookie, NWilson::TTraceId traceId, TInstant now, + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + const TActorId &proxyId, bool useVPatch) { + return new TBlobStorageGroupPatchRequest(info, state, source, mon, ev, cookie, std::move(traceId), + now, storagePoolCounters, proxyId, useVPatch); +} + +}//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp index d9e0e2b3202..1a0be7f70e2 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp @@ -21,7 +21,7 @@ struct TEvResume : public TEventLocal<TEvResume, TEvBlobStorage::EvResume> { double WaitSec; double SplitSec; size_t Count; - TBatchedVec<TDataPartSet> PartSets; + TBatchedVec<TDataPartSet> PartSets; }; struct TEvAccelerate : public TEventLocal<TEvAccelerate, TEvBlobStorage::EvAccelerate> { @@ -37,43 +37,43 @@ struct TEvAccelerate : public TEventLocal<TEvAccelerate, TEvBlobStorage::EvAccel //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobStorageGroupPutRequest> { - struct TMultiPutItemInfo { - TLogoBlobID BlobId; - TString Buffer; - ui64 BufferSize; + struct TMultiPutItemInfo { + TLogoBlobID BlobId; + TString Buffer; + ui64 BufferSize; TActorId Recipient; - ui64 Cookie; - NWilson::TTraceId TraceId; + ui64 Cookie; + NWilson::TTraceId TraceId; NLWTrace::TOrbit Orbit; bool Replied = false; - + TMultiPutItemInfo(TLogoBlobID id, const TString& buffer, TActorId recipient, ui64 cookie, NWilson::TTraceId traceId, NLWTrace::TOrbit &&orbit) - : BlobId(id) - , Buffer(buffer) - , BufferSize(buffer.size()) + : BlobId(id) + , Buffer(buffer) + , BufferSize(buffer.size()) , Recipient(recipient) - , Cookie(cookie) - , TraceId(std::move(traceId)) + , Cookie(cookie) + , TraceId(std::move(traceId)) , Orbit(std::move(orbit)) - {} - }; - + {} + }; + TPutImpl PutImpl; TRootCause RootCauseTrack; - TStackVec<ui64, TypicalDisksInGroup> WaitingVDiskResponseCount; - ui64 WaitingVDiskCount = 0; - - TBatchedVec<TMultiPutItemInfo> ItemsInfo; - - bool IsManyPuts = false; - + TStackVec<ui64, TypicalDisksInGroup> WaitingVDiskResponseCount; + ui64 WaitingVDiskCount = 0; + + TBatchedVec<TMultiPutItemInfo> ItemsInfo; + + bool IsManyPuts = false; + TInstant Deadline; ui64 RequestsSent = 0; ui64 ResponsesReceived = 0; ui64 MaxSaneRequests = 0; - ui64 ResponsesSent = 0; + ui64 ResponsesSent = 0; TInstant StartTime; NKikimrBlobStorage::EPutHandleClass HandleClass; @@ -91,8 +91,8 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt bool IsAccelerated; bool IsAccelerateScheduled; - const bool IsMultiPutMode; - + const bool IsMultiPutMode; + void SanityCheck() { if (RequestsSent <= MaxSaneRequests) { return; @@ -118,23 +118,23 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt } IsAccelerated = true; - if (IsMultiPutMode) { + if (IsMultiPutMode) { TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts; - PutImpl.Accelerate(LogCtx, vMultiPuts); - UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVMultiPut, TVMultiPutCookie>(vMultiPuts); - RequestsSent += vMultiPuts.size(); + PutImpl.Accelerate(LogCtx, vMultiPuts); + UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVMultiPut, TVMultiPutCookie>(vMultiPuts); + RequestsSent += vMultiPuts.size(); *Mon->NodeMon->AccelerateEvVMultiPutCount += vMultiPuts.size(); - CountPuts(vMultiPuts); + CountPuts(vMultiPuts); SendToQueues(vMultiPuts, TimeStatsEnabled); - } else { + } else { TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - PutImpl.Accelerate(LogCtx, vPuts); - UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVPut, TBlobCookie>(vPuts); - RequestsSent += vPuts.size(); + PutImpl.Accelerate(LogCtx, vPuts); + UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVPut, TBlobCookie>(vPuts); + RequestsSent += vPuts.size(); *Mon->NodeMon->AccelerateEvVPutCount += vPuts.size(); - CountPuts(vPuts); + CountPuts(vPuts); SendToQueues(vPuts, TimeStatsEnabled); - } + } } void Handle(TEvBlobStorage::TEvVPutResult::TPtr &ev) { @@ -149,24 +149,24 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt const ui64 cyclesPerUs = NHPTimer::GetCyclesPerSecond() / 1000000; ev->Get()->Record.MutableTimestamps()->SetReceivedByDSProxyUs(GetCycleCountFast() / cyclesPerUs); const NKikimrBlobStorage::TEvVPutResult &record = ev->Get()->Record; - const TLogoBlobID blob = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - const TLogoBlobID origBlobId = TLogoBlobID(blob, 0); - Y_VERIFY(record.HasCookie()); - TBlobCookie cookie(record.GetCookie()); - const ui64 idx = cookie.GetBlobIdx(); - const ui64 vdisk = cookie.GetVDiskOrderNumber(); + const TLogoBlobID blob = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + const TLogoBlobID origBlobId = TLogoBlobID(blob, 0); + Y_VERIFY(record.HasCookie()); + TBlobCookie cookie(record.GetCookie()); + const ui64 idx = cookie.GetBlobIdx(); + const ui64 vdisk = cookie.GetVDiskOrderNumber(); const NKikimrProto::EReplyStatus status = record.GetStatus(); - - Y_VERIFY(vdisk < WaitingVDiskResponseCount.size(), "blobIdx# %" PRIu64 " vdisk# %" PRIu64, idx, vdisk); - if (WaitingVDiskResponseCount[vdisk] == 1) { - WaitingVDiskCount--; - } - WaitingVDiskResponseCount[vdisk]--; - - Y_VERIFY(idx < ItemsInfo.size()); - Y_VERIFY(origBlobId == ItemsInfo[idx].BlobId); + + Y_VERIFY(vdisk < WaitingVDiskResponseCount.size(), "blobIdx# %" PRIu64 " vdisk# %" PRIu64, idx, vdisk); + if (WaitingVDiskResponseCount[vdisk] == 1) { + WaitingVDiskCount--; + } + WaitingVDiskResponseCount[vdisk]--; + + Y_VERIFY(idx < ItemsInfo.size()); + Y_VERIFY(origBlobId == ItemsInfo[idx].BlobId); if (TimeStatsEnabled && record.GetMsgQoS().HasExecTimeStats()) { - TimeStats.ApplyPut(ItemsInfo[idx].BufferSize, record.GetMsgQoS().GetExecTimeStats()); + TimeStats.ApplyPut(ItemsInfo[idx].BufferSize, record.GetMsgQoS().GetExecTimeStats()); } Y_VERIFY(record.HasVDiskID()); @@ -174,7 +174,7 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt const TVDiskIdShort shortId(vDiskId); LWPROBE(DSProxyVDiskRequestDuration, TEvBlobStorage::EvVPut, blob.BlobSize(), blob.TabletID(), - Info->GroupID, blob.Channel(), Info->GetFailDomainOrderNumber(shortId), + Info->GroupID, blob.Channel(), Info->GetFailDomainOrderNumber(shortId), GetStartTime(record.GetTimestamps()), GetTotalTimeMs(record.GetTimestamps()), GetVDiskTimeMs(record.GetTimestamps()), @@ -188,9 +188,9 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt } TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - TPutImpl::TPutResultVec putResults; - PutImpl.OnVPutEventResult(LogCtx, ev->Sender, *ev->Get(), vPuts, putResults); - UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVPut, TBlobCookie>(vPuts); + TPutImpl::TPutResultVec putResults; + PutImpl.OnVPutEventResult(LogCtx, ev->Sender, *ev->Get(), vPuts, putResults); + UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVPut, TBlobCookie>(vPuts); RequestsSent += vPuts.size(); CountPuts(vPuts); SendToQueues(vPuts, TimeStatsEnabled); @@ -201,7 +201,7 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt ui64(RequestsSent), ui64(ResponsesReceived)); if (!IsAccelerateScheduled && !IsAccelerated) { - if (WaitingVDiskCount == 1 && RequestsSent > 1) { + if (WaitingVDiskCount == 1 && RequestsSent > 1) { ui64 timeToAccelerateUs = PutImpl.GetTimeToAccelerateNs(LogCtx) / 1000; TDuration timeSinceStart = TActivationContext::Now() - StartTime; if (timeSinceStart.MicroSeconds() < timeToAccelerateUs) { @@ -220,27 +220,27 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt void Handle(TEvBlobStorage::TEvVMultiPutResult::TPtr &ev) { ProcessReplyFromQueue(ev); - // generate wilson event about request completion + // generate wilson event about request completion WILSON_TRACE_FROM_ACTOR(*TlsActivationContext, *this, &TraceId, EvVPutResultReceived, MergedNode = std::move(ev->TraceId)); - ResponsesReceived++; - - const ui64 cyclesPerUs = NHPTimer::GetCyclesPerSecond() / 1000000; + ResponsesReceived++; + + const ui64 cyclesPerUs = NHPTimer::GetCyclesPerSecond() / 1000000; ev->Get()->Record.MutableTimestamps()->SetReceivedByDSProxyUs(GetCycleCountFast() / cyclesPerUs); - const NKikimrBlobStorage::TEvVMultiPutResult &record = ev->Get()->Record; - - if (TimeStatsEnabled && record.GetMsgQoS().HasExecTimeStats()) { + const NKikimrBlobStorage::TEvVMultiPutResult &record = ev->Get()->Record; + + if (TimeStatsEnabled && record.GetMsgQoS().HasExecTimeStats()) { TimeStats.ApplyPut(RequestBytes, record.GetMsgQoS().GetExecTimeStats()); - } - - Y_VERIFY(record.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - const TVDiskIdShort shortId(vDiskId); - - Y_VERIFY(record.HasCookie()); - TVMultiPutCookie cookie(record.GetCookie()); - const ui64 vdisk = cookie.GetVDiskOrderNumber(); + } + + Y_VERIFY(record.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + const TVDiskIdShort shortId(vDiskId); + + Y_VERIFY(record.HasCookie()); + TVMultiPutCookie cookie(record.GetCookie()); + const ui64 vdisk = cookie.GetVDiskOrderNumber(); const NKikimrProto::EReplyStatus status = record.GetStatus(); - + auto prio = NLog::PRI_DEBUG; if (status != NKikimrProto::OK) { prio = NLog::PRI_INFO; @@ -253,12 +253,12 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt A_LOG_LOG_S(false, prio, "BPP02", "received " << ev->Get()->ToString() << " from# " << VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID())); - Y_VERIFY(vdisk < WaitingVDiskResponseCount.size(), " vdisk# %" PRIu64, vdisk); - if (WaitingVDiskResponseCount[vdisk] == 1) { - WaitingVDiskCount--; - } - WaitingVDiskResponseCount[vdisk]--; - + Y_VERIFY(vdisk < WaitingVDiskResponseCount.size(), " vdisk# %" PRIu64, vdisk); + if (WaitingVDiskResponseCount[vdisk] == 1) { + WaitingVDiskCount--; + } + WaitingVDiskResponseCount[vdisk]--; + // Trace put request duration if (LWPROBE_ENABLED(DSProxyVDiskRequestDuration)) { for (auto &item : record.GetItems()) { @@ -278,67 +278,67 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt // Handle put results bool isCauseRegistered = !RootCauseTrack.IsOn; TPutImpl::TPutResultVec putResults; - for (auto &item : record.GetItems()) { + for (auto &item : record.GetItems()) { if (!isCauseRegistered) { isCauseRegistered = RootCauseTrack.OnReply(cookie.GetCauseIdx(), GetTotalTimeMs(record.GetTimestamps()) - GetVDiskTimeMs(record.GetTimestamps()), GetVDiskTimeMs(record.GetTimestamps())); } - - Y_VERIFY(item.HasStatus()); - Y_VERIFY(item.HasBlobID()); - Y_VERIFY(item.HasCookie()); + + Y_VERIFY(item.HasStatus()); + Y_VERIFY(item.HasBlobID()); + Y_VERIFY(item.HasCookie()); ui64 blobIdx = TBlobCookie(item.GetCookie()).GetBlobIdx(); - NKikimrProto::EReplyStatus itemStatus = item.GetStatus(); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + NKikimrProto::EReplyStatus itemStatus = item.GetStatus(); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); Y_VERIFY(itemStatus != NKikimrProto::RACE); // we should get RACE for the whole request and handle it in CheckForTermErrors if (itemStatus == NKikimrProto::BLOCKED || itemStatus == NKikimrProto::DEADLINE) { - TStringStream errorReason; - errorReason << "Got VMultiPutResult itemStatus# " << itemStatus << " from VDiskId# " << vDiskId; - ErrorReason = errorReason.Str(); - PutImpl.PrepareOneReply(itemStatus, TLogoBlobID(blobId, 0), blobIdx, LogCtx, ErrorReason, putResults); - } - } + TStringStream errorReason; + errorReason << "Got VMultiPutResult itemStatus# " << itemStatus << " from VDiskId# " << vDiskId; + ErrorReason = errorReason.Str(); + PutImpl.PrepareOneReply(itemStatus, TLogoBlobID(blobId, 0), blobIdx, LogCtx, ErrorReason, putResults); + } + } if (ReplyAndDieWithLastResponse(putResults)) { - return; - } - putResults.clear(); - + return; + } + putResults.clear(); + TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts; - PutImpl.OnVPutEventResult(LogCtx, ev->Sender, *ev->Get(), vMultiPuts, putResults); - UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVMultiPut, TVMultiPutCookie>(vMultiPuts); - RequestsSent += vMultiPuts.size(); - CountPuts(vMultiPuts); + PutImpl.OnVPutEventResult(LogCtx, ev->Sender, *ev->Get(), vMultiPuts, putResults); + UpdatePengingVDiskResponseCount<TEvBlobStorage::TEvVMultiPut, TVMultiPutCookie>(vMultiPuts); + RequestsSent += vMultiPuts.size(); + CountPuts(vMultiPuts); SendToQueues(vMultiPuts, TimeStatsEnabled); if (ReplyAndDieWithLastResponse(putResults)) { - return; - } - Y_VERIFY(RequestsSent > ResponsesReceived, "RequestsSent# %" PRIu64 " ResponsesReceived# %" PRIu64 - " ResponsesSent# %" PRIu64 " BlobsCount# %" PRIu64 " TPutImpl# %s", ui64(RequestsSent), ui64(ResponsesReceived), - (ui64)ResponsesSent, (ui64)ItemsInfo.size(), PutImpl.DumpFullState().data()); - - if (!IsAccelerateScheduled && !IsAccelerated) { - if (WaitingVDiskCount == 1 && RequestsSent > 1) { - ui64 timeToAccelerateUs = PutImpl.GetTimeToAccelerateNs(LogCtx) / 1000; + return; + } + Y_VERIFY(RequestsSent > ResponsesReceived, "RequestsSent# %" PRIu64 " ResponsesReceived# %" PRIu64 + " ResponsesSent# %" PRIu64 " BlobsCount# %" PRIu64 " TPutImpl# %s", ui64(RequestsSent), ui64(ResponsesReceived), + (ui64)ResponsesSent, (ui64)ItemsInfo.size(), PutImpl.DumpFullState().data()); + + if (!IsAccelerateScheduled && !IsAccelerated) { + if (WaitingVDiskCount == 1 && RequestsSent > 1) { + ui64 timeToAccelerateUs = PutImpl.GetTimeToAccelerateNs(LogCtx) / 1000; TDuration timeSinceStart = TActivationContext::Now() - StartTime; - if (timeSinceStart.MicroSeconds() < timeToAccelerateUs) { + if (timeSinceStart.MicroSeconds() < timeToAccelerateUs) { ui64 causeIdx = RootCauseTrack.RegisterAccelerate(); Schedule(TDuration::MicroSeconds(timeToAccelerateUs - timeSinceStart.MicroSeconds()), new TEvAccelerate(causeIdx)); - IsAccelerateScheduled = true; - } else { + IsAccelerateScheduled = true; + } else { Accelerate(); - } - } - } + } + } + } SanityCheck(); // May Die - } - + } + friend class TBlobStorageGroupRequestActor<TBlobStorageGroupPutRequest>; void ReplyAndDie(NKikimrProto::EReplyStatus status) { - TPutImpl::TPutResultVec putResults; - PutImpl.PrepareReply(status, LogCtx, ErrorReason, putResults); + TPutImpl::TPutResultVec putResults; + PutImpl.PrepareReply(status, LogCtx, ErrorReason, putResults); Y_VERIFY(ReplyAndDieWithLastResponse(putResults)); } @@ -346,14 +346,14 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt for (auto& [blobIdx, result] : putResults) { Y_VERIFY(ResponsesSent != ItemsInfo.size()); SendReply(result, blobIdx); - } + } if (ResponsesSent == ItemsInfo.size()) { PassAway(); return true; } - return false; - } - + return false; + } + void SendReply(std::unique_ptr<TEvBlobStorage::TEvPutResult> &putResult, ui64 blobIdx) { NKikimrProto::EReplyStatus status = putResult->Status; A_LOG_LOG_S(false, status == NKikimrProto::OK ? NLog::PRI_DEBUG : NLog::PRI_NOTICE, "BPP21", @@ -361,47 +361,47 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt << " ItemsInfo.size# " << ItemsInfo.size() << " Last# " << (ResponsesSent + 1 == ItemsInfo.size() ? "true" : "false")); const TDuration duration = TActivationContext::Now() - StartTime; - TLogoBlobID blobId = putResult->Id; - TLogoBlobID origBlobId = TLogoBlobID(blobId, 0); + TLogoBlobID blobId = putResult->Id; + TLogoBlobID origBlobId = TLogoBlobID(blobId, 0); WILSON_TRACE_FROM_ACTOR(*TlsActivationContext, *this, &ItemsInfo[blobIdx].TraceId, EvPutResultSent, ReplyStatus = status); - Mon->CountPutPesponseTime(Info->GetDeviceType(), HandleClass, ItemsInfo[blobIdx].BufferSize, duration); + Mon->CountPutPesponseTime(Info->GetDeviceType(), HandleClass, ItemsInfo[blobIdx].BufferSize, duration); *Mon->ActivePutCapacity -= ReportedBytes; - Y_VERIFY(PutImpl.GetHandoffPartsSent() <= Info->Type.TotalPartCount() * MaxHandoffNodes * ItemsInfo.size()); + Y_VERIFY(PutImpl.GetHandoffPartsSent() <= Info->Type.TotalPartCount() * MaxHandoffNodes * ItemsInfo.size()); ++*Mon->HandoffPartsSent[Min(Mon->HandoffPartsSent.size() - 1, (size_t)PutImpl.GetHandoffPartsSent())]; ReportedBytes = 0; const bool success = (status == NKikimrProto::OK); LWPROBE(DSProxyRequestDuration, TEvBlobStorage::EvPut, - blobId.BlobSize(), + blobId.BlobSize(), duration.SecondsFloat() * 1000.0, - blobId.TabletID(), Info->GroupID, blobId.Channel(), - NKikimrBlobStorage::EPutHandleClass_Name(HandleClass), success); - ResponsesSent++; - Y_VERIFY(ResponsesSent <= ItemsInfo.size()); + blobId.TabletID(), Info->GroupID, blobId.Channel(), + NKikimrBlobStorage::EPutHandleClass_Name(HandleClass), success); + ResponsesSent++; + Y_VERIFY(ResponsesSent <= ItemsInfo.size()); RootCauseTrack.RenderTrack(ItemsInfo[blobIdx].Orbit); LWTRACK(DSProxyPutReply, ItemsInfo[blobIdx].Orbit); putResult->Orbit = std::move(ItemsInfo[blobIdx].Orbit); - if (!IsManyPuts) { + if (!IsManyPuts) { SendResponse(std::move(putResult), TimeStatsEnabled ? &TimeStats : nullptr); - } else { + } else { SendResponse(std::move(putResult), TimeStatsEnabled ? &TimeStats : nullptr, ItemsInfo[blobIdx].Recipient, ItemsInfo[blobIdx].Cookie, - std::move(ItemsInfo[blobIdx].TraceId)); + std::move(ItemsInfo[blobIdx].TraceId)); ItemsInfo[blobIdx].Replied = true; - } + } } - TString BlobIdSequenceToString() const { - TStringBuilder blobIdsStr; - blobIdsStr << '['; - for (ui64 blobIdx = 0; blobIdx < ItemsInfo.size(); ++blobIdx) { - if (blobIdx) { - blobIdsStr << ' '; - } - blobIdsStr << ItemsInfo[blobIdx].BlobId.ToString(); - } + TString BlobIdSequenceToString() const { + TStringBuilder blobIdsStr; + blobIdsStr << '['; + for (ui64 blobIdx = 0; blobIdx < ItemsInfo.size(); ++blobIdx) { + if (blobIdx) { + blobIdsStr << ' '; + } + blobIdsStr << ItemsInfo[blobIdx].BlobId.ToString(); + } return blobIdsStr << ']'; - } - + } + std::unique_ptr<IEventBase> RestartQuery(ui32 counter) { ++*Mon->NodeMon->RestartPut; auto ev = std::make_unique<TEvBlobStorage::TEvBunchOfEvents>(); @@ -448,13 +448,13 @@ public: ui64 cookie, NWilson::TTraceId traceId, bool timeStatsEnabled, TDiskResponsivenessTracker::TPerDiskStatsPtr stats, TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - bool enableRequestMod3x3ForMinLatecy) + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + bool enableRequestMod3x3ForMinLatecy) : TBlobStorageGroupRequestActor(info, state, mon, source, cookie, std::move(traceId), NKikimrServices::BS_PROXY_PUT, false, latencyQueueKind, now, storagePoolCounters, ev->RestartCounter) - , PutImpl(info, state, ev, mon, enableRequestMod3x3ForMinLatecy) - , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) + , PutImpl(info, state, ev, mon, enableRequestMod3x3ForMinLatecy) + , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) , Deadline(ev->Deadline) , StartTime(now) , HandleClass(ev->HandleClass) @@ -464,15 +464,15 @@ public: , Stats(std::move(stats)) , IsAccelerated(false) , IsAccelerateScheduled(false) - , IsMultiPutMode(false) + , IsMultiPutMode(false) { if (ev->Orbit.HasShuttles()) { RootCauseTrack.IsOn = true; } ItemsInfo.emplace_back(ev->Id, ev->Buffer, source, cookie, NWilson::TTraceId(), std::move(ev->Orbit)); - LWPROBE(DSProxyBlobPutTactics, ItemsInfo[0].BlobId.TabletID(), Info->GroupID, ItemsInfo[0].BlobId.ToString(), - Tactic, NKikimrBlobStorage::EPutHandleClass_Name(HandleClass)); - ReportBytes(ItemsInfo[0].Buffer.capacity() + sizeof(*this)); + LWPROBE(DSProxyBlobPutTactics, ItemsInfo[0].BlobId.TabletID(), Info->GroupID, ItemsInfo[0].BlobId.ToString(), + Tactic, NKikimrBlobStorage::EPutHandleClass_Name(HandleClass)); + ReportBytes(ItemsInfo[0].Buffer.capacity() + sizeof(*this)); RequestBytes = ev->Buffer.size(); RequestHandleClass = HandleClassToHandleClass(HandleClass); @@ -487,59 +487,59 @@ public: return res; } - TBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, + TBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &events, - bool timeStatsEnabled, TDiskResponsivenessTracker::TPerDiskStatsPtr stats, - TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, - bool enableRequestMod3x3ForMinLatecy) + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &events, + bool timeStatsEnabled, TDiskResponsivenessTracker::TPerDiskStatsPtr stats, + TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, + bool enableRequestMod3x3ForMinLatecy) : TBlobStorageGroupRequestActor(info, state, mon, TActorId(), 0, NWilson::TTraceId(), NKikimrServices::BS_PROXY_PUT, false, latencyQueueKind, now, storagePoolCounters, MaxRestartCounter(events)) - , PutImpl(info, state, events, mon, handleClass, tactic, enableRequestMod3x3ForMinLatecy) - , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) - , IsManyPuts(true) - , Deadline(TInstant::Zero()) + , PutImpl(info, state, events, mon, handleClass, tactic, enableRequestMod3x3ForMinLatecy) + , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) + , IsManyPuts(true) + , Deadline(TInstant::Zero()) , StartTime(now) - , HandleClass(handleClass) - , ReportedBytes(0) - , TimeStatsEnabled(timeStatsEnabled) - , Tactic(tactic) - , Stats(std::move(stats)) - , IsAccelerated(false) - , IsAccelerateScheduled(false) - , IsMultiPutMode(true) - { - Y_VERIFY_DEBUG(events.size() <= MaxBatchedPutRequests); - for (auto &ev : events) { - Deadline = Max(Deadline, ev->Get()->Deadline); + , HandleClass(handleClass) + , ReportedBytes(0) + , TimeStatsEnabled(timeStatsEnabled) + , Tactic(tactic) + , Stats(std::move(stats)) + , IsAccelerated(false) + , IsAccelerateScheduled(false) + , IsMultiPutMode(true) + { + Y_VERIFY_DEBUG(events.size() <= MaxBatchedPutRequests); + for (auto &ev : events) { + Deadline = Max(Deadline, ev->Get()->Deadline); if (ev->Get()->Orbit.HasShuttles()) { RootCauseTrack.IsOn = true; } ItemsInfo.emplace_back( - ev->Get()->Id, - ev->Get()->Buffer, - ev->Sender, - ev->Cookie, + ev->Get()->Id, + ev->Get()->Buffer, + ev->Sender, + ev->Cookie, std::move(ev->TraceId), std::move(ev->Get()->Orbit) ); - LWPROBE(DSProxyBlobPutTactics, ItemsInfo.back().BlobId.TabletID(), Info->GroupID, - ItemsInfo.back().BlobId.ToString(), Tactic, NKikimrBlobStorage::EPutHandleClass_Name(HandleClass)); - } - - RequestBytes = 0; - for (auto &item: ItemsInfo) { - ReportBytes(item.Buffer.capacity()); - RequestBytes += item.BufferSize; - } - ReportBytes(sizeof(*this)); - RequestHandleClass = HandleClassToHandleClass(HandleClass); + LWPROBE(DSProxyBlobPutTactics, ItemsInfo.back().BlobId.TabletID(), Info->GroupID, + ItemsInfo.back().BlobId.ToString(), Tactic, NKikimrBlobStorage::EPutHandleClass_Name(HandleClass)); + } + + RequestBytes = 0; + for (auto &item: ItemsInfo) { + ReportBytes(item.Buffer.capacity()); + RequestBytes += item.BufferSize; + } + ReportBytes(sizeof(*this)); + RequestHandleClass = HandleClassToHandleClass(HandleClass); MaxSaneRequests = info->Type.TotalPartCount() * (1ull + info->Type.Handoff()) * 2; - } - + } + void ReportBytes(i64 bytes) { *Mon->ActivePutCapacity += bytes; ReportedBytes += bytes; @@ -549,8 +549,8 @@ public: A_LOG_INFO_S("BPP13", "bootstrap" << " ActorId# " << SelfId() << " Group# " << Info->GroupID - << " BlobCount# " << ItemsInfo.size() - << " BlobIDs# " << BlobIdSequenceToString() + << " BlobCount# " << ItemsInfo.size() + << " BlobIDs# " << BlobIdSequenceToString() << " HandleClass# " << NKikimrBlobStorage::EPutHandleClass_Name(HandleClass) << " Tactic# " << TEvBlobStorage::TEvPut::TacticName(Tactic) << " Deadline# " << Deadline @@ -564,33 +564,33 @@ public: Timer.Reset(); - // TODO: how correct rewrite this? + // TODO: how correct rewrite this? WILSON_TRACE_FROM_ACTOR(*TlsActivationContext, *this, &TraceId, EvPutReceived, Size = RequestBytes, - LogoBlobId = ItemsInfo[0].BlobId); + LogoBlobId = ItemsInfo[0].BlobId); double wilsonSec = Timer.PassedReset(); const ui32 totalParts = Info->Type.TotalPartCount(); TAutoPtr<TEvResume> resume(new TEvResume()); - resume->PartSets.resize(ItemsInfo.size()); + resume->PartSets.resize(ItemsInfo.size()); - for (ui64 idx = 0; idx < ItemsInfo.size(); ++idx) { - TLogoBlobID blobId = ItemsInfo[idx].BlobId; + for (ui64 idx = 0; idx < ItemsInfo.size(); ++idx) { + TLogoBlobID blobId = ItemsInfo[idx].BlobId; - const ui64 partSize = Info->Type.PartSize(blobId); + const ui64 partSize = Info->Type.PartSize(blobId); ui64 bufferSize = ItemsInfo[idx].BufferSize; - - char *data = ItemsInfo[idx].Buffer.Detach(); - Encrypt(data, data, 0, bufferSize, blobId, *Info); - TDataPartSet &partSet = resume->PartSets[idx]; - - partSet.Parts.resize(totalParts); - if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { - for (ui32 i = 0; i < totalParts; ++i) { - partSet.Parts[i].UninitializedOwnedWhole(partSize); - - } + + char *data = ItemsInfo[idx].Buffer.Detach(); + Encrypt(data, data, 0, bufferSize, blobId, *Info); + TDataPartSet &partSet = resume->PartSets[idx]; + + partSet.Parts.resize(totalParts); + if (Info->Type.ErasureFamily() != TErasureType::ErasureMirror) { + for (ui32 i = 0; i < totalParts; ++i) { + partSet.Parts[i].UninitializedOwnedWhole(partSize); + + } } } double allocateSec = Timer.PassedReset(); @@ -629,7 +629,7 @@ public: A_LOG_WARN_S("BPP14", "Wakeup " << " ActorId# " << SelfId() << " Group# " << Info->GroupID - << " BlobIDs# " << BlobIdSequenceToString() + << " BlobIDs# " << BlobIdSequenceToString() << " Not answered in " << (TActivationContext::Now() - StartTime).Seconds() << " seconds"); if (TInstant::Now() > Deadline) { @@ -640,41 +640,41 @@ public: Schedule(TDuration::MilliSeconds(DsPutWakeupMs), new TKikimrEvents::TEvWakeup); } - template <typename TPutEvent, typename TCookie> + template <typename TPutEvent, typename TCookie> void UpdatePengingVDiskResponseCount(const TDeque<std::unique_ptr<TPutEvent>> &putEvents) { - for (auto &event : putEvents) { - Y_VERIFY(event->Record.HasCookie()); - TCookie cookie(event->Record.GetCookie()); + for (auto &event : putEvents) { + Y_VERIFY(event->Record.HasCookie()); + TCookie cookie(event->Record.GetCookie()); if (RootCauseTrack.IsOn) { cookie.SetCauseIdx(RootCauseTrack.RegisterCause()); event->Record.SetCookie(cookie); } - ui64 vdisk = cookie.GetVDiskOrderNumber(); - Y_VERIFY(vdisk < WaitingVDiskResponseCount.size()); - if (!WaitingVDiskResponseCount[vdisk]) { - WaitingVDiskCount++; - } - WaitingVDiskResponseCount[vdisk]++; - } - } - + ui64 vdisk = cookie.GetVDiskOrderNumber(); + Y_VERIFY(vdisk < WaitingVDiskResponseCount.size()); + if (!WaitingVDiskResponseCount[vdisk]) { + WaitingVDiskCount++; + } + WaitingVDiskResponseCount[vdisk]++; + } + } + void ResumeBootstrap(TAutoPtr<TEvResume> resume) { double waitSec = Timer.PassedReset(); resume->WaitSec += waitSec; Y_VERIFY(ItemsInfo.size() == resume->PartSets.size()); bool splitDone = true; - for (ui64 idx = 0; idx < ItemsInfo.size(); ++idx) { + for (ui64 idx = 0; idx < ItemsInfo.size(); ++idx) { TDataPartSet &partSet = resume->PartSets[idx]; - TString &buffer = ItemsInfo[idx].Buffer; - TLogoBlobID blobId = ItemsInfo[idx].BlobId; + TString &buffer = ItemsInfo[idx].Buffer; + TLogoBlobID blobId = ItemsInfo[idx].BlobId; Info->Type.IncrementalSplitData((TErasureType::ECrcMode)blobId.CrcMode(), buffer, partSet); if (partSet.IsSplitDone()) { ReportBytes(partSet.MemoryConsumed - ItemsInfo[idx].BufferSize); } else { splitDone = false; } - } + } double splitSec = Timer.PassedReset(); resume->SplitSec += splitSec; resume->Count++; @@ -701,9 +701,9 @@ public: CountPuts(vPuts); SendToQueues(vPuts, TimeStatsEnabled); } - } else { + } else { Send(SelfId(), resume.Release()); - } + } } STATEFN(StateWait) { @@ -726,24 +726,24 @@ IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupIn ui64 cookie, NWilson::TTraceId traceId, bool timeStatsEnabled, TDiskResponsivenessTracker::TPerDiskStatsPtr stats, TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - bool enableRequestMod3x3ForMinLatecy) { + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + bool enableRequestMod3x3ForMinLatecy) { return new TBlobStorageGroupPutRequest(info, state, source, mon, ev, cookie, std::move(traceId), timeStatsEnabled, - std::move(stats), latencyQueueKind, now, storagePoolCounters, enableRequestMod3x3ForMinLatecy); + std::move(stats), latencyQueueKind, now, storagePoolCounters, enableRequestMod3x3ForMinLatecy); } -IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, +IActor* CreateBlobStorageGroupPutRequest(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, - const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &ev, - bool timeStatsEnabled, - TDiskResponsivenessTracker::TPerDiskStatsPtr stats, - TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, - TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, - NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, - bool enableRequestMod3x3ForMinLatecy) { - return new TBlobStorageGroupPutRequest(info, state, mon, ev, timeStatsEnabled, - std::move(stats), latencyQueueKind, now, storagePoolCounters, handleClass, tactic, - enableRequestMod3x3ForMinLatecy); -} - + const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &ev, + bool timeStatsEnabled, + TDiskResponsivenessTracker::TPerDiskStatsPtr stats, + TMaybe<TGroupStat::EKind> latencyQueueKind, TInstant now, + TIntrusivePtr<TStoragePoolCounters> &storagePoolCounters, + NKikimrBlobStorage::EPutHandleClass handleClass, TEvBlobStorage::TEvPut::ETactic tactic, + bool enableRequestMod3x3ForMinLatecy) { + return new TBlobStorageGroupPutRequest(info, state, mon, ev, timeStatsEnabled, + std::move(stats), latencyQueueKind, now, storagePoolCounters, handleClass, tactic, + enableRequestMod3x3ForMinLatecy); +} + }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.cpp index 70b774636ca..cd3c55e5614 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.cpp @@ -9,12 +9,12 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); namespace NKikimr { -using TPutResultVec = TPutImpl::TPutResultVec; +using TPutResultVec = TPutImpl::TPutResultVec; -bool TPutImpl::RunStrategies(TLogContext &logCtx, TPutResultVec &outPutResults) { +bool TPutImpl::RunStrategies(TLogContext &logCtx, TPutResultVec &outPutResults) { switch (Info->Type.GetErasure()) { case TBlobStorageGroupType::ErasureMirror3dc: - return RunStrategy(logCtx, TPut3dcStrategy(Tactic, EnableRequestMod3x3ForMinLatecy), outPutResults); + return RunStrategy(logCtx, TPut3dcStrategy(Tactic, EnableRequestMod3x3ForMinLatecy), outPutResults); case TBlobStorageGroupType::ErasureMirror3of4: return RunStrategy(logCtx, TPut3of4Strategy(Tactic), outPutResults); default: @@ -32,90 +32,90 @@ bool TPutImpl::RunStrategy(TLogContext &logCtx, const IStrategy& strategy, TPutR return false; } -NLog::EPriority GetPriorityForReply(TAtomicLogPriorityMuteChecker<NLog::PRI_ERROR, NLog::PRI_DEBUG> &checker, - NKikimrProto::EReplyStatus status) { - NLog::EPriority priority = PriorityForStatusOutbound(status); - if (priority == NLog::PRI_ERROR) { - return checker.Register(TActivationContext::Now(), TDuration::Seconds(5)); - } else { - checker.Unmute(); - return priority; - } -} - -void TPutImpl::PrepareOneReply(NKikimrProto::EReplyStatus status, TLogoBlobID blobId, ui64 blobIdx, TLogContext &logCtx, - TString errorReason, TPutResultVec &outPutResults) { - Y_VERIFY(IsInitialized); - if (!IsDone[blobIdx]) { - outPutResults.emplace_back(blobIdx, new TEvBlobStorage::TEvPutResult(status, blobId, StatusFlags, Info->GroupID, - ApproximateFreeSpaceShare)); - outPutResults.back().second->ErrorReason = errorReason; - NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, status); - A_LOG_LOG_SX(logCtx, true, priority, "BPP12", "Result# " << outPutResults.back().second->Print(false)); - MarkBlobAsSent(blobIdx); - } -} - +NLog::EPriority GetPriorityForReply(TAtomicLogPriorityMuteChecker<NLog::PRI_ERROR, NLog::PRI_DEBUG> &checker, + NKikimrProto::EReplyStatus status) { + NLog::EPriority priority = PriorityForStatusOutbound(status); + if (priority == NLog::PRI_ERROR) { + return checker.Register(TActivationContext::Now(), TDuration::Seconds(5)); + } else { + checker.Unmute(); + return priority; + } +} + +void TPutImpl::PrepareOneReply(NKikimrProto::EReplyStatus status, TLogoBlobID blobId, ui64 blobIdx, TLogContext &logCtx, + TString errorReason, TPutResultVec &outPutResults) { + Y_VERIFY(IsInitialized); + if (!IsDone[blobIdx]) { + outPutResults.emplace_back(blobIdx, new TEvBlobStorage::TEvPutResult(status, blobId, StatusFlags, Info->GroupID, + ApproximateFreeSpaceShare)); + outPutResults.back().second->ErrorReason = errorReason; + NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, status); + A_LOG_LOG_SX(logCtx, true, priority, "BPP12", "Result# " << outPutResults.back().second->Print(false)); + MarkBlobAsSent(blobIdx); + } +} + void TPutImpl::PrepareReply(NKikimrProto::EReplyStatus status, TLogContext &logCtx, TString errorReason, - TPutResultVec &outPutResults) { - A_LOG_DEBUG_SX(logCtx, "BPP34", "PrepareReply status# " << status << " errorReason# " << errorReason); - for (ui64 idx = 0; idx < BlobIds.size(); ++idx) { - if (IsDone[idx]) { - A_LOG_DEBUG_SX(logCtx, "BPP35", "blob# " << BlobIds[idx].ToString() << - " idx# " << idx << " is sent, skipped"); - continue; - } - - outPutResults.emplace_back(idx, new TEvBlobStorage::TEvPutResult(status, BlobIds[idx], StatusFlags, Info->GroupID, - ApproximateFreeSpaceShare)); - outPutResults.back().second->ErrorReason = errorReason; - - NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, status); - A_LOG_LOG_SX(logCtx, true, priority, "BPP38", - "PrepareReply Result# " << outPutResults.back().second->Print(false)); - - if (IsInitialized) { - MarkBlobAsSent(idx); - } - } + TPutResultVec &outPutResults) { + A_LOG_DEBUG_SX(logCtx, "BPP34", "PrepareReply status# " << status << " errorReason# " << errorReason); + for (ui64 idx = 0; idx < BlobIds.size(); ++idx) { + if (IsDone[idx]) { + A_LOG_DEBUG_SX(logCtx, "BPP35", "blob# " << BlobIds[idx].ToString() << + " idx# " << idx << " is sent, skipped"); + continue; + } + + outPutResults.emplace_back(idx, new TEvBlobStorage::TEvPutResult(status, BlobIds[idx], StatusFlags, Info->GroupID, + ApproximateFreeSpaceShare)); + outPutResults.back().second->ErrorReason = errorReason; + + NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, status); + A_LOG_LOG_SX(logCtx, true, priority, "BPP38", + "PrepareReply Result# " << outPutResults.back().second->Print(false)); + + if (IsInitialized) { + MarkBlobAsSent(idx); + } + } } -void TPutImpl::PrepareReply(TLogContext &logCtx, TString errorReason, +void TPutImpl::PrepareReply(TLogContext &logCtx, TString errorReason, TBatchedVec<TBlackboard::TBlobStates::value_type*>& finished, TPutResultVec &outPutResults) { - A_LOG_DEBUG_SX(logCtx, "BPP36", "PrepareReply errorReason# " << errorReason); - Y_VERIFY(IsInitialized); + A_LOG_DEBUG_SX(logCtx, "BPP36", "PrepareReply errorReason# " << errorReason); + Y_VERIFY(IsInitialized); for (auto item : finished) { auto &[blobId, state] = *item; const ui64 idx = state.BlobIdx; - Y_VERIFY(blobId == BlobIds[idx], "BlobIdx# %" PRIu64 " BlobState# %s Blackboard# %s", - idx, state.ToString().c_str(), Blackboard.ToString().c_str()); - Y_VERIFY(!IsDone[idx]); + Y_VERIFY(blobId == BlobIds[idx], "BlobIdx# %" PRIu64 " BlobState# %s Blackboard# %s", + idx, state.ToString().c_str(), Blackboard.ToString().c_str()); + Y_VERIFY(!IsDone[idx]); Y_VERIFY(state.Status != NKikimrProto::UNKNOWN); - outPutResults.emplace_back(idx, new TEvBlobStorage::TEvPutResult(state.Status, blobId, StatusFlags, - Info->GroupID, ApproximateFreeSpaceShare)); - outPutResults.back().second->ErrorReason = errorReason; - - NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, state.Status); - A_LOG_LOG_SX(logCtx, true, priority, "BPP37", - "PrepareReply Result# " << outPutResults.back().second->Print(false)); + outPutResults.emplace_back(idx, new TEvBlobStorage::TEvPutResult(state.Status, blobId, StatusFlags, + Info->GroupID, ApproximateFreeSpaceShare)); + outPutResults.back().second->ErrorReason = errorReason; + + NLog::EPriority priority = GetPriorityForReply(Info->PutErrorMuteChecker, state.Status); + A_LOG_LOG_SX(logCtx, true, priority, "BPP37", + "PrepareReply Result# " << outPutResults.back().second->Print(false)); MarkBlobAsSent(idx); - } -} - + } +} + ui64 TPutImpl::GetTimeToAccelerateNs(TLogContext &logCtx) { Y_UNUSED(logCtx); - Y_VERIFY(!Blackboard.BlobStates.empty()); - TBatchedVec<ui64> nextToWorstPredictedNsVec(Blackboard.BlobStates.size()); - ui64 idx = 0; - for (auto &[_, state] : Blackboard.BlobStates) { - // Find the slowest disk - i32 worstSubgroupIdx = -1; - ui64 worstPredictedNs = 0; + Y_VERIFY(!Blackboard.BlobStates.empty()); + TBatchedVec<ui64> nextToWorstPredictedNsVec(Blackboard.BlobStates.size()); + ui64 idx = 0; + for (auto &[_, state] : Blackboard.BlobStates) { + // Find the slowest disk + i32 worstSubgroupIdx = -1; + ui64 worstPredictedNs = 0; state.GetWorstPredictedDelaysNs(*Info, *Blackboard.GroupQueues, HandleClassToQueueId(Blackboard.PutHandleClass), - &worstPredictedNs, &nextToWorstPredictedNsVec[idx], &worstSubgroupIdx); - idx++; + &worstPredictedNs, &nextToWorstPredictedNsVec[idx], &worstSubgroupIdx); + idx++; } - return *MaxElement(nextToWorstPredictedNsVec.begin(), nextToWorstPredictedNsVec.end()); + return *MaxElement(nextToWorstPredictedNsVec.begin(), nextToWorstPredictedNsVec.end()); } TString TPutImpl::DumpFullState() const { @@ -126,33 +126,33 @@ TString TPutImpl::DumpFullState() const { str << Endl; str << " Blackboard# " << Blackboard.ToString(); str << Endl; - str << " BlobIds# " << BlobIds.ToString(); - str << Endl; - str << "IsDone# " << IsDone.ToString(); + str << " BlobIds# " << BlobIds.ToString(); str << Endl; + str << "IsDone# " << IsDone.ToString(); + str << Endl; str << " HandoffPartsSent# " << HandoffPartsSent; str << Endl; str << " VPutRequests# " << VPutRequests; str << Endl; str << " VPutResponses# " << VPutResponses; str << Endl; - str << " VMultiPutRequests# " << VMultiPutRequests; - str << Endl; - str << " VMultiPutResponses# " << VMultiPutResponses; - str << Endl; + str << " VMultiPutRequests# " << VMultiPutRequests; + str << Endl; + str << " VMultiPutResponses# " << VMultiPutResponses; + str << Endl; str << " Tactic# " << TEvBlobStorage::TEvPut::TacticName(Tactic); str << Endl; str << "}"; return str.Str(); } -bool TPutImpl::MarkBlobAsSent(ui64 idx) { - Y_VERIFY(idx < BlobIds.size()); - Y_VERIFY(!IsDone[idx]); - Blackboard.MoveBlobStateToDone(BlobIds[idx]); - IsDone[idx] = true; - DoneBlobs++; - return true; -} - +bool TPutImpl::MarkBlobAsSent(ui64 idx) { + Y_VERIFY(idx < BlobIds.size()); + Y_VERIFY(!IsDone[idx]); + Blackboard.MoveBlobStateToDone(BlobIds[idx]); + IsDone[idx] = true; + DoneBlobs++; + return true; +} + }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.h b/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.h index 34f675032b5..a922d1b91bf 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_put_impl.h @@ -4,10 +4,10 @@ #include "dsproxy_blackboard.h" #include "dsproxy_cookies.h" #include "dsproxy_mon.h" -#include "dsproxy_strategy_accelerate_put.h" -#include "dsproxy_strategy_accelerate_put_m3dc.h" -#include "dsproxy_strategy_restore.h" -#include "dsproxy_strategy_put_m3dc.h" +#include "dsproxy_strategy_accelerate_put.h" +#include "dsproxy_strategy_accelerate_put_m3dc.h" +#include "dsproxy_strategy_restore.h" +#include "dsproxy_strategy_put_m3dc.h" #include "dsproxy_strategy_put_m3of4.h" #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> #include <ydb/core/blobstorage/base/wilson_events.h> @@ -18,86 +18,86 @@ namespace NKikimr { class TStrategyBase; class TPutImpl { -public: +public: using TPutResultVec = TBatchedVec<std::pair<ui64, std::unique_ptr<TEvBlobStorage::TEvPutResult>>>; - -private: + +private: TBlobStorageGroupInfo::TServiceIds VDisksSvc; TBlobStorageGroupInfo::TVDiskIds VDisksId; - TInstant Deadline; + TInstant Deadline; const TIntrusivePtr<TBlobStorageGroupInfo> Info; TBlackboard Blackboard; - TBatchedVec<bool> IsDone; + TBatchedVec<bool> IsDone; TStorageStatusFlags StatusFlags; float ApproximateFreeSpaceShare; TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; ui32 HandoffPartsSent = 0; ui32 VPutRequests = 0; ui32 VPutResponses = 0; - ui32 VMultiPutRequests = 0; - ui32 VMultiPutResponses = 0; + ui32 VMultiPutRequests = 0; + ui32 VMultiPutResponses = 0; bool AtLeastOneResponseWasNotOk = false; - bool EnableRequestMod3x3ForMinLatecy = false; - - ui64 DoneBlobs = 0; + bool EnableRequestMod3x3ForMinLatecy = false; + ui64 DoneBlobs = 0; + const TEvBlobStorage::TEvPut::ETactic Tactic; - TBatchedVec<TLogoBlobID> BlobIds; - - TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVPutResponses; - TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVMultiPutResponses; - - bool IsInitialized = false; - - TString ErrorDescription; - + TBatchedVec<TLogoBlobID> BlobIds; + + TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVPutResponses; + TStackVec<bool, MaxBatchedPutRequests * TypicalDisksInSubring> ReceivedVMultiPutResponses; + + bool IsInitialized = false; + + TString ErrorDescription; + public: TPutImpl(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, - TEvBlobStorage::TEvPut *ev, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, - bool enableRequestMod3x3ForMinLatecy) + TEvBlobStorage::TEvPut *ev, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, + bool enableRequestMod3x3ForMinLatecy) : Deadline(ev->Deadline) , Info(info) - , Blackboard(info, state, ev->HandleClass, NKikimrBlobStorage::EGetHandleClass::AsyncRead, false) - , IsDone(1) + , Blackboard(info, state, ev->HandleClass, NKikimrBlobStorage::EGetHandleClass::AsyncRead, false) + , IsDone(1) , StatusFlags(0) , ApproximateFreeSpaceShare(0.f) , Mon(mon) - , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) + , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) , Tactic(ev->Tactic) - , BlobIds({ev->Id}) + , BlobIds({ev->Id}) { - Y_VERIFY(BlobIds.size()); - Y_VERIFY(BlobIds.size() <= MaxBatchedPutRequests); + Y_VERIFY(BlobIds.size()); + Y_VERIFY(BlobIds.size() <= MaxBatchedPutRequests); } TPutImpl(const TIntrusivePtr<TBlobStorageGroupInfo> &info, const TIntrusivePtr<TGroupQueues> &state, - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &events, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, - NKikimrBlobStorage::EPutHandleClass putHandleClass, TEvBlobStorage::TEvPut::ETactic tactic, - bool enableRequestMod3x3ForMinLatecy) - : Deadline(TInstant::Zero()) - , Info(info) - , Blackboard(info, state, putHandleClass, NKikimrBlobStorage::EGetHandleClass::AsyncRead, false) - , IsDone(events.size()) - , StatusFlags(0) - , ApproximateFreeSpaceShare(0.f) - , Mon(mon) - , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) - , Tactic(tactic) - { - Y_VERIFY(events.size(), "TEvPut vector is empty"); - for (auto &ev : events) { - Y_VERIFY(ev->Get()->HandleClass == putHandleClass); - Y_VERIFY(ev->Get()->Tactic == tactic); - BlobIds.push_back(ev->Get()->Id); - Deadline = Max(Deadline, ev->Get()->Deadline); - } - Y_VERIFY(BlobIds.size()); - Y_VERIFY(BlobIds.size() <= MaxBatchedPutRequests); - } - + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &events, const TIntrusivePtr<TBlobStorageGroupProxyMon> &mon, + NKikimrBlobStorage::EPutHandleClass putHandleClass, TEvBlobStorage::TEvPut::ETactic tactic, + bool enableRequestMod3x3ForMinLatecy) + : Deadline(TInstant::Zero()) + , Info(info) + , Blackboard(info, state, putHandleClass, NKikimrBlobStorage::EGetHandleClass::AsyncRead, false) + , IsDone(events.size()) + , StatusFlags(0) + , ApproximateFreeSpaceShare(0.f) + , Mon(mon) + , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) + , Tactic(tactic) + { + Y_VERIFY(events.size(), "TEvPut vector is empty"); + for (auto &ev : events) { + Y_VERIFY(ev->Get()->HandleClass == putHandleClass); + Y_VERIFY(ev->Get()->Tactic == tactic); + BlobIds.push_back(ev->Get()->Id); + Deadline = Max(Deadline, ev->Get()->Deadline); + } + Y_VERIFY(BlobIds.size()); + Y_VERIFY(BlobIds.size() <= MaxBatchedPutRequests); + } + NKikimrBlobStorage::EPutHandleClass GetPutHandleClass() const { return Blackboard.PutHandleClass; } @@ -106,247 +106,247 @@ public: return HandoffPartsSent; } - template <typename TVPutEvent> - void GenerateInitialRequests(TLogContext &logCtx, TBatchedVec<TDataPartSet> &partSets, + template <typename TVPutEvent> + void GenerateInitialRequests(TLogContext &logCtx, TBatchedVec<TDataPartSet> &partSets, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts) { - Y_UNUSED(logCtx); - Y_VERIFY_S(partSets.size() == BlobIds.size(), "partSets.size# " << partSets.size() - << " BlobIds.size# " << BlobIds.size()); - const ui32 totalParts = Info->Type.TotalPartCount(); - for (ui64 blobIdx = 0; blobIdx < BlobIds.size(); ++blobIdx) { - TLogoBlobID &blobId = BlobIds[blobIdx]; - for (ui32 i = 0; i < totalParts; ++i) { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(partSets[blobIdx].Parts[i].OwnedString.Data(), - partSets[blobIdx].Parts[i].OwnedString.Size()); - Blackboard.AddPartToPut(blobId, i, partSets[blobIdx].Parts[i].OwnedString); - } - Blackboard.MarkBlobReadyToPut(blobId, blobIdx); - } - - TPutResultVec putResults; - bool workDone = Step(logCtx, outVPuts, putResults); - IsInitialized = true; - Y_VERIFY(!outVPuts.empty()); - Y_VERIFY(putResults.empty()); - Y_VERIFY(workDone); - } - - template <typename TEvent> - bool MarkRequest(const TEvent &event, ui32 orderNumber) { - constexpr bool isVPut = std::is_same_v<TEvent, TEvBlobStorage::TEvVPutResult>; - constexpr bool isVMultiPut = std::is_same_v<TEvent, TEvBlobStorage::TEvVMultiPutResult>; + Y_UNUSED(logCtx); + Y_VERIFY_S(partSets.size() == BlobIds.size(), "partSets.size# " << partSets.size() + << " BlobIds.size# " << BlobIds.size()); + const ui32 totalParts = Info->Type.TotalPartCount(); + for (ui64 blobIdx = 0; blobIdx < BlobIds.size(); ++blobIdx) { + TLogoBlobID &blobId = BlobIds[blobIdx]; + for (ui32 i = 0; i < totalParts; ++i) { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(partSets[blobIdx].Parts[i].OwnedString.Data(), + partSets[blobIdx].Parts[i].OwnedString.Size()); + Blackboard.AddPartToPut(blobId, i, partSets[blobIdx].Parts[i].OwnedString); + } + Blackboard.MarkBlobReadyToPut(blobId, blobIdx); + } + + TPutResultVec putResults; + bool workDone = Step(logCtx, outVPuts, putResults); + IsInitialized = true; + Y_VERIFY(!outVPuts.empty()); + Y_VERIFY(putResults.empty()); + Y_VERIFY(workDone); + } + + template <typename TEvent> + bool MarkRequest(const TEvent &event, ui32 orderNumber) { + constexpr bool isVPut = std::is_same_v<TEvent, TEvBlobStorage::TEvVPutResult>; + constexpr bool isVMultiPut = std::is_same_v<TEvent, TEvBlobStorage::TEvVMultiPutResult>; static_assert(isVPut || isVMultiPut); - using TCookie = std::conditional_t<isVPut, TBlobCookie, TVMultiPutCookie>; - - auto responses = isVPut ? ReceivedVPutResponses : ReceivedVMultiPutResponses; + using TCookie = std::conditional_t<isVPut, TBlobCookie, TVMultiPutCookie>; + + auto responses = isVPut ? ReceivedVPutResponses : ReceivedVMultiPutResponses; auto putType = std::is_same_v<TCookie, TBlobCookie> ? "TEvVPut" : "TEvVMultiPut"; - - const auto &record = event.Record; - if (!record.HasCookie()) { - ErrorDescription = TStringBuilder() << putType << " doesn't have cookie"; - return true; - } - TCookie cookie(record.GetCookie()); - if (cookie.GetVDiskOrderNumber() != orderNumber) { - ErrorDescription = TStringBuilder() << putType << " has wrong cookie; unexpected orderNumber;" - << " expected# " << orderNumber - << " received# " << cookie.GetVDiskOrderNumber() - << " cookie# " << ui64(cookie); - return true; - } - - ui64 requestIdx = cookie.GetRequestIdx(); - if (responses[requestIdx]) { - ErrorDescription = TStringBuilder() << putType << "is recieved twice" - << " Event# " << event.ToString() - << " State# " << DumpFullState(); - return true; - } - responses[requestIdx] = true; - return false; - } - - void PackToStringImpl(TStringBuilder &) { - } - - template <typename TArg, typename... TArgs> - void PackToStringImpl(TStringBuilder &builder, TArg arg, TArgs... args) { - builder << arg; - PackToStringImpl(builder, args...); - } - - template <typename... TArgs> - TString PackToString(TArgs... args) { - TStringBuilder builder; - PackToStringImpl(builder, args...); - return builder; - } - - template <typename TPutRecord, typename ...TArgs> - bool ProcessOneVPutResult(TLogContext &logCtx, const TPutRecord &record, TVDiskID vDiskId, ui32 orderNumber, - TArgs ...resultTypeArgs) - { - Y_VERIFY(record.HasStatus()); - Y_VERIFY(record.HasBlobID()); - Y_VERIFY(record.HasCookie()); - NKikimrProto::EReplyStatus status = record.GetStatus(); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - ui32 diskIdx = Info->GetTopology().GetIdxInSubgroup(vDiskId, blobId.Hash()); - - TBlobCookie cookie(record.GetCookie()); - if (cookie.GetPartId() != blobId.PartId()) { - ErrorDescription = TStringBuilder() - << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected PartId;" - << " expected# " << blobId.PartId() - << " received# " << cookie.GetPartId() - << " cookie# " << ui64(cookie); - return true; - } - if (cookie.GetVDiskOrderNumber() != orderNumber) { - ErrorDescription = TStringBuilder() - << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected orderNumber;" - << " expected# " << orderNumber - << " received# " << cookie.GetVDiskOrderNumber() - << " cookie# " << ui64(cookie); - return true; - } - - ui64 blobIdx = cookie.GetBlobIdx(); - ui32 partIdx = blobId.PartId() - 1; - - A_LOG_LOG_SX(logCtx, false, PriorityForStatusInbound(status), "BPP11", - "Got " << PackToString(resultTypeArgs...) - << " part# " << partIdx - << " diskIdx# " << diskIdx - << " vDiskId# " << vDiskId - << " blob# " << blobId - << " status# " << status); - - if (IsDone.size() <= blobIdx) { - ErrorDescription = TStringBuilder() - << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected blobIdx;" - << " received# " << cookie.GetVDiskOrderNumber() - << " blobCount# " << IsDone.size() - << " cookie# " << ui64(cookie); - return true; - } - - if (IsDone[blobIdx]) { - return false; - } - switch (status) { - case NKikimrProto::ERROR: - case NKikimrProto::VDISK_ERROR_STATE: - case NKikimrProto::OUT_OF_SPACE: - Blackboard.AddErrorResponse(blobId, orderNumber); - AtLeastOneResponseWasNotOk = true; - break; - case NKikimrProto::OK: - case NKikimrProto::ALREADY: - Blackboard.AddPutOkResponse(blobId, orderNumber); - break; - default: - ErrorDescription = TStringBuilder() << "Unexpected status# " << status; - return true; - } - return false; - } - - template <typename TVPutEvent, typename TVPutEventResult> + + const auto &record = event.Record; + if (!record.HasCookie()) { + ErrorDescription = TStringBuilder() << putType << " doesn't have cookie"; + return true; + } + TCookie cookie(record.GetCookie()); + if (cookie.GetVDiskOrderNumber() != orderNumber) { + ErrorDescription = TStringBuilder() << putType << " has wrong cookie; unexpected orderNumber;" + << " expected# " << orderNumber + << " received# " << cookie.GetVDiskOrderNumber() + << " cookie# " << ui64(cookie); + return true; + } + + ui64 requestIdx = cookie.GetRequestIdx(); + if (responses[requestIdx]) { + ErrorDescription = TStringBuilder() << putType << "is recieved twice" + << " Event# " << event.ToString() + << " State# " << DumpFullState(); + return true; + } + responses[requestIdx] = true; + return false; + } + + void PackToStringImpl(TStringBuilder &) { + } + + template <typename TArg, typename... TArgs> + void PackToStringImpl(TStringBuilder &builder, TArg arg, TArgs... args) { + builder << arg; + PackToStringImpl(builder, args...); + } + + template <typename... TArgs> + TString PackToString(TArgs... args) { + TStringBuilder builder; + PackToStringImpl(builder, args...); + return builder; + } + + template <typename TPutRecord, typename ...TArgs> + bool ProcessOneVPutResult(TLogContext &logCtx, const TPutRecord &record, TVDiskID vDiskId, ui32 orderNumber, + TArgs ...resultTypeArgs) + { + Y_VERIFY(record.HasStatus()); + Y_VERIFY(record.HasBlobID()); + Y_VERIFY(record.HasCookie()); + NKikimrProto::EReplyStatus status = record.GetStatus(); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + ui32 diskIdx = Info->GetTopology().GetIdxInSubgroup(vDiskId, blobId.Hash()); + + TBlobCookie cookie(record.GetCookie()); + if (cookie.GetPartId() != blobId.PartId()) { + ErrorDescription = TStringBuilder() + << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected PartId;" + << " expected# " << blobId.PartId() + << " received# " << cookie.GetPartId() + << " cookie# " << ui64(cookie); + return true; + } + if (cookie.GetVDiskOrderNumber() != orderNumber) { + ErrorDescription = TStringBuilder() + << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected orderNumber;" + << " expected# " << orderNumber + << " received# " << cookie.GetVDiskOrderNumber() + << " cookie# " << ui64(cookie); + return true; + } + + ui64 blobIdx = cookie.GetBlobIdx(); + ui32 partIdx = blobId.PartId() - 1; + + A_LOG_LOG_SX(logCtx, false, PriorityForStatusInbound(status), "BPP11", + "Got " << PackToString(resultTypeArgs...) + << " part# " << partIdx + << " diskIdx# " << diskIdx + << " vDiskId# " << vDiskId + << " blob# " << blobId + << " status# " << status); + + if (IsDone.size() <= blobIdx) { + ErrorDescription = TStringBuilder() + << PackToString(resultTypeArgs...) << " has wrong cookie; unexpected blobIdx;" + << " received# " << cookie.GetVDiskOrderNumber() + << " blobCount# " << IsDone.size() + << " cookie# " << ui64(cookie); + return true; + } + + if (IsDone[blobIdx]) { + return false; + } + switch (status) { + case NKikimrProto::ERROR: + case NKikimrProto::VDISK_ERROR_STATE: + case NKikimrProto::OUT_OF_SPACE: + Blackboard.AddErrorResponse(blobId, orderNumber); + AtLeastOneResponseWasNotOk = true; + break; + case NKikimrProto::OK: + case NKikimrProto::ALREADY: + Blackboard.AddPutOkResponse(blobId, orderNumber); + break; + default: + ErrorDescription = TStringBuilder() << "Unexpected status# " << status; + return true; + } + return false; + } + + template <typename TVPutEvent, typename TVPutEventResult> void OnVPutEventResult(TLogContext &logCtx, TActorId sender, TVPutEventResult &ev, - TDeque<std::unique_ptr<TVPutEvent>> &outVPutEvents, TPutResultVec &outPutResults) - { - constexpr bool isVPut = std::is_same_v<TVPutEvent, TEvBlobStorage::TEvVPut>; - constexpr bool isVMultiPut = std::is_same_v<TVPutEvent, TEvBlobStorage::TEvVMultiPut>; - static_assert(isVPut || isVMultiPut); + TDeque<std::unique_ptr<TVPutEvent>> &outVPutEvents, TPutResultVec &outPutResults) + { + constexpr bool isVPut = std::is_same_v<TVPutEvent, TEvBlobStorage::TEvVPut>; + constexpr bool isVMultiPut = std::is_same_v<TVPutEvent, TEvBlobStorage::TEvVMultiPut>; + static_assert(isVPut || isVMultiPut); auto putType = isVPut ? "TEvVPut" : "TEvVMultiPut"; - - auto &record = ev.Record; - Y_VERIFY(record.HasStatus()); - NKikimrProto::EReplyStatus status = record.GetStatus(); - Y_VERIFY(status != NKikimrProto::BLOCKED && status != NKikimrProto::RACE && status != NKikimrProto::DEADLINE); - if (record.HasStatusFlags()) { - StatusFlags.Merge(record.GetStatusFlags()); - } - if (record.HasApproximateFreeSpaceShare()) { - float share = record.GetApproximateFreeSpaceShare(); - if (ApproximateFreeSpaceShare == 0.f || share < ApproximateFreeSpaceShare) { - ApproximateFreeSpaceShare = share; - } - } - - Y_VERIFY(record.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - TVDiskIdShort shortId(vDiskId); - ui32 orderNumber = Info->GetOrderNumber(shortId); - - Y_VERIFY_S(!MarkRequest(ev, orderNumber), ErrorDescription); - - if constexpr (isVPut) { - VPutResponses++; - bool error = ProcessOneVPutResult(logCtx, ev.Record, vDiskId, orderNumber, "TEvVPutResult"); - Y_VERIFY_S(!error, ErrorDescription); - } else { - TVMultiPutCookie cookie(record.GetCookie()); - if (cookie.GetItemCount() != record.ItemsSize()) { - ErrorDescription = TStringBuilder() << putType << " has wrong cookie; unexpected ItemCount;" - << " expected# " << record.ItemsSize() - << " received# " << cookie.GetItemCount() - << " cookie# " << ui64(cookie); - Y_VERIFY_S(false, ErrorDescription); - return; - } - - VMultiPutResponses++; - for (ui32 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { - bool error = ProcessOneVPutResult(logCtx, ev.Record.GetItems(itemIdx), vDiskId, orderNumber, - "TEvVMultiPutResult item# ", itemIdx); - Y_VERIFY_S(!error, ErrorDescription); - } - A_LOG_LOG_SX(logCtx, false, PriorityForStatusInbound(status), "BPP23", "Got VMultiPutResult " - << " vDiskId# " << vDiskId - << " from# " << sender - << " status# " << status - << " vMultiPutResult# " << ev.ToString()); - } - - const auto &requests = isVPut ? VPutRequests : VMultiPutRequests; - const auto &responses = isVPut ? VPutResponses : VMultiPutResponses; - - if (Info->Type.GetErasure() == TBlobStorageGroupType::Erasure4Plus2Block - && !AtLeastOneResponseWasNotOk - && requests >= 6 && responses <= 4) { - // 6 == Info->Type.TotalPartCount() - // There is no need to run strategy since no new information is recieved - return; - } - - Step(logCtx, outVPutEvents, outPutResults); - Y_VERIFY_S(DoneBlobs == BlobIds.size() || requests > responses, - "No put result while" - << " Type# " << putType - << " DoneBlobs# " << DoneBlobs - << " requests# " << requests - << " responses# " << responses - << " Blackboard# " << Blackboard.ToString()); - } - + + auto &record = ev.Record; + Y_VERIFY(record.HasStatus()); + NKikimrProto::EReplyStatus status = record.GetStatus(); + Y_VERIFY(status != NKikimrProto::BLOCKED && status != NKikimrProto::RACE && status != NKikimrProto::DEADLINE); + if (record.HasStatusFlags()) { + StatusFlags.Merge(record.GetStatusFlags()); + } + if (record.HasApproximateFreeSpaceShare()) { + float share = record.GetApproximateFreeSpaceShare(); + if (ApproximateFreeSpaceShare == 0.f || share < ApproximateFreeSpaceShare) { + ApproximateFreeSpaceShare = share; + } + } + + Y_VERIFY(record.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + TVDiskIdShort shortId(vDiskId); + ui32 orderNumber = Info->GetOrderNumber(shortId); + + Y_VERIFY_S(!MarkRequest(ev, orderNumber), ErrorDescription); + + if constexpr (isVPut) { + VPutResponses++; + bool error = ProcessOneVPutResult(logCtx, ev.Record, vDiskId, orderNumber, "TEvVPutResult"); + Y_VERIFY_S(!error, ErrorDescription); + } else { + TVMultiPutCookie cookie(record.GetCookie()); + if (cookie.GetItemCount() != record.ItemsSize()) { + ErrorDescription = TStringBuilder() << putType << " has wrong cookie; unexpected ItemCount;" + << " expected# " << record.ItemsSize() + << " received# " << cookie.GetItemCount() + << " cookie# " << ui64(cookie); + Y_VERIFY_S(false, ErrorDescription); + return; + } + + VMultiPutResponses++; + for (ui32 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { + bool error = ProcessOneVPutResult(logCtx, ev.Record.GetItems(itemIdx), vDiskId, orderNumber, + "TEvVMultiPutResult item# ", itemIdx); + Y_VERIFY_S(!error, ErrorDescription); + } + A_LOG_LOG_SX(logCtx, false, PriorityForStatusInbound(status), "BPP23", "Got VMultiPutResult " + << " vDiskId# " << vDiskId + << " from# " << sender + << " status# " << status + << " vMultiPutResult# " << ev.ToString()); + } + + const auto &requests = isVPut ? VPutRequests : VMultiPutRequests; + const auto &responses = isVPut ? VPutResponses : VMultiPutResponses; + + if (Info->Type.GetErasure() == TBlobStorageGroupType::Erasure4Plus2Block + && !AtLeastOneResponseWasNotOk + && requests >= 6 && responses <= 4) { + // 6 == Info->Type.TotalPartCount() + // There is no need to run strategy since no new information is recieved + return; + } + + Step(logCtx, outVPutEvents, outPutResults); + Y_VERIFY_S(DoneBlobs == BlobIds.size() || requests > responses, + "No put result while" + << " Type# " << putType + << " DoneBlobs# " << DoneBlobs + << " requests# " << requests + << " responses# " << responses + << " Blackboard# " << Blackboard.ToString()); + } + void PrepareReply(NKikimrProto::EReplyStatus status, TLogContext &logCtx, TString errorReason, - TPutResultVec &outPutResults); + TPutResultVec &outPutResults); void PrepareReply(TLogContext &logCtx, TString errorReason, TBatchedVec<TBlackboard::TBlobStates::value_type*>& finished, TPutResultVec &outPutResults); - void PrepareOneReply(NKikimrProto::EReplyStatus status, TLogoBlobID blobId, ui64 blobIdx, TLogContext &logCtx, - TString errorReason, TPutResultVec &outPutResults); + void PrepareOneReply(NKikimrProto::EReplyStatus status, TLogoBlobID blobId, ui64 blobIdx, TLogContext &logCtx, + TString errorReason, TPutResultVec &outPutResults); ui64 GetTimeToAccelerateNs(TLogContext &logCtx); - - template <typename TVPutEvent> + + template <typename TVPutEvent> void Accelerate(TLogContext &logCtx, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts) { Blackboard.ChangeAll(); switch (Info->Type.GetErasure()) { case TBlobStorageGroupType::ErasureMirror3dc: - Blackboard.RunStrategy(logCtx, TAcceleratePut3dcStrategy(Tactic, EnableRequestMod3x3ForMinLatecy)); + Blackboard.RunStrategy(logCtx, TAcceleratePut3dcStrategy(Tactic, EnableRequestMod3x3ForMinLatecy)); break; case TBlobStorageGroupType::ErasureMirror3of4: Blackboard.RunStrategy(logCtx, TPut3of4Strategy(Tactic, true)); @@ -354,103 +354,103 @@ public: default: Blackboard.RunStrategy(logCtx, TAcceleratePutStrategy()); break; - } - PrepareVPuts(logCtx, outVPuts); - } - + } + PrepareVPuts(logCtx, outVPuts); + } + TString DumpFullState() const; - bool MarkBlobAsSent(ui64 blobIdx); - bool MarkBlobAsSent(TMap<TLogoBlobID, TBlobState>::iterator it); - - TString ToString() const; - + bool MarkBlobAsSent(ui64 blobIdx); + bool MarkBlobAsSent(TMap<TLogoBlobID, TBlobState>::iterator it); + + TString ToString() const; + protected: - bool RunStrategies(TLogContext &logCtx, TPutResultVec &outPutResults); + bool RunStrategies(TLogContext &logCtx, TPutResultVec &outPutResults); bool RunStrategy(TLogContext &logCtx, const IStrategy& strategy, TPutResultVec &outPutResults); - - // Returns true if there are additional requests to send - template <typename TVPutEvent> + + // Returns true if there are additional requests to send + template <typename TVPutEvent> bool Step(TLogContext &logCtx, TDeque<std::unique_ptr<TVPutEvent>> &outVPuts, - TPutResultVec &outPutResults) { - if (!RunStrategies(logCtx, outPutResults)) { - const ui32 numRequests = outVPuts.size(); - PrepareVPuts(logCtx, outVPuts); - return outVPuts.size() > numRequests; - } else { - Y_VERIFY(outPutResults.size()); - PrepareVPuts(logCtx, outVPuts); - return false; - } - } - - - template <typename TVPutEvent> - void PrepareVPuts(TLogContext &logCtx, TDeque<std::unique_ptr<TVPutEvent>> &outVPutEvents) { - constexpr bool isVPut = std::is_same_v<TEvBlobStorage::TEvVPut, TVPutEvent>; - constexpr bool isVMultiPut = std::is_same_v<TEvBlobStorage::TEvVMultiPut, TVPutEvent>; - static_assert(isVPut || isVMultiPut); - - const ui32 diskCount = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber.size(); - for (ui32 diskOrderNumber = 0; diskOrderNumber < diskCount; ++diskOrderNumber) { - const TDiskRequests &requests = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber]; - ui32 endIdx = requests.PutsToSend.size(); - ui32 beginIdx = requests.FirstUnsentPutIdx; - - if (beginIdx >= endIdx) { - continue; - } - - TVDiskID vDiskId = Info->GetVDiskId(diskOrderNumber); - - if constexpr (isVMultiPut) { - ui64 cookie = TVMultiPutCookie(diskOrderNumber, endIdx - beginIdx, VMultiPutRequests); - auto vMultiPut = std::make_unique<TEvBlobStorage::TEvVMultiPut>(vDiskId, Deadline, Blackboard.PutHandleClass, - false, &cookie); - vMultiPut->ReservePayload(endIdx - beginIdx); - outVPutEvents.push_back(std::move(vMultiPut)); - ++VMultiPutRequests; - ReceivedVMultiPutResponses.push_back(false); - } - - for (ui32 idx = beginIdx; idx < endIdx; ++idx) { - const TDiskPutRequest &put = requests.PutsToSend[idx]; - ui32 counter = isVPut ? VPutRequests : VMultiPutRequests; - ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), counter); - - if constexpr (isVPut) { - auto vPut = std::make_unique<TEvBlobStorage::TEvVPut>(put.Id, put.Buffer, vDiskId, false, &cookie, - Deadline, Blackboard.PutHandleClass); - R_LOG_DEBUG_SX(logCtx, "BPP20", "Send put to orderNumber# " << diskOrderNumber << " idx# " << idx - << " vPut# " << vPut->ToString()); - outVPutEvents.push_back(std::move(vPut)); - ++VPutRequests; - ReceivedVPutResponses.push_back(false); - } else if constexpr (isVMultiPut) { - outVPutEvents.back()->AddVPut(put.Id, put.Buffer, &cookie); - } - - if (put.IsHandoff) { - ++HandoffPartsSent; - } - - LWPROBE(DSProxyPutVPut, put.Id.TabletID(), Info->GroupID, put.Id.Channel(), put.Id.PartId(), - put.Id.ToString(), Tactic, - NKikimrBlobStorage::EPutHandleClass_Name(Blackboard.PutHandleClass), - put.Id.BlobSize(), put.Buffer.size(), Info->GetFailDomainOrderNumber(vDiskId), - NKikimrBlobStorage::EVDiskQueueId_Name(TGroupQueues::TVDisk::TQueues::VDiskQueueId(*outVPutEvents.back())), - Blackboard.GroupQueues->GetPredictedDelayNsForEvent(*outVPutEvents.back(), Info->GetTopology()) * 0.000001); - } - - if constexpr (isVMultiPut) { - R_LOG_DEBUG_SX(logCtx, "BPP39", "Send put to orderNumber# " << diskOrderNumber - << " count# " << outVPutEvents.back()->Record.ItemsSize() - << " vMultiPut# " << outVPutEvents.back()->ToString()); - } - - Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber].FirstUnsentPutIdx = endIdx; - } - } + TPutResultVec &outPutResults) { + if (!RunStrategies(logCtx, outPutResults)) { + const ui32 numRequests = outVPuts.size(); + PrepareVPuts(logCtx, outVPuts); + return outVPuts.size() > numRequests; + } else { + Y_VERIFY(outPutResults.size()); + PrepareVPuts(logCtx, outVPuts); + return false; + } + } + + + template <typename TVPutEvent> + void PrepareVPuts(TLogContext &logCtx, TDeque<std::unique_ptr<TVPutEvent>> &outVPutEvents) { + constexpr bool isVPut = std::is_same_v<TEvBlobStorage::TEvVPut, TVPutEvent>; + constexpr bool isVMultiPut = std::is_same_v<TEvBlobStorage::TEvVMultiPut, TVPutEvent>; + static_assert(isVPut || isVMultiPut); + + const ui32 diskCount = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber.size(); + for (ui32 diskOrderNumber = 0; diskOrderNumber < diskCount; ++diskOrderNumber) { + const TDiskRequests &requests = Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber]; + ui32 endIdx = requests.PutsToSend.size(); + ui32 beginIdx = requests.FirstUnsentPutIdx; + + if (beginIdx >= endIdx) { + continue; + } + + TVDiskID vDiskId = Info->GetVDiskId(diskOrderNumber); + + if constexpr (isVMultiPut) { + ui64 cookie = TVMultiPutCookie(diskOrderNumber, endIdx - beginIdx, VMultiPutRequests); + auto vMultiPut = std::make_unique<TEvBlobStorage::TEvVMultiPut>(vDiskId, Deadline, Blackboard.PutHandleClass, + false, &cookie); + vMultiPut->ReservePayload(endIdx - beginIdx); + outVPutEvents.push_back(std::move(vMultiPut)); + ++VMultiPutRequests; + ReceivedVMultiPutResponses.push_back(false); + } + + for (ui32 idx = beginIdx; idx < endIdx; ++idx) { + const TDiskPutRequest &put = requests.PutsToSend[idx]; + ui32 counter = isVPut ? VPutRequests : VMultiPutRequests; + ui64 cookie = TBlobCookie(diskOrderNumber, put.BlobIdx, put.Id.PartId(), counter); + + if constexpr (isVPut) { + auto vPut = std::make_unique<TEvBlobStorage::TEvVPut>(put.Id, put.Buffer, vDiskId, false, &cookie, + Deadline, Blackboard.PutHandleClass); + R_LOG_DEBUG_SX(logCtx, "BPP20", "Send put to orderNumber# " << diskOrderNumber << " idx# " << idx + << " vPut# " << vPut->ToString()); + outVPutEvents.push_back(std::move(vPut)); + ++VPutRequests; + ReceivedVPutResponses.push_back(false); + } else if constexpr (isVMultiPut) { + outVPutEvents.back()->AddVPut(put.Id, put.Buffer, &cookie); + } + + if (put.IsHandoff) { + ++HandoffPartsSent; + } + + LWPROBE(DSProxyPutVPut, put.Id.TabletID(), Info->GroupID, put.Id.Channel(), put.Id.PartId(), + put.Id.ToString(), Tactic, + NKikimrBlobStorage::EPutHandleClass_Name(Blackboard.PutHandleClass), + put.Id.BlobSize(), put.Buffer.size(), Info->GetFailDomainOrderNumber(vDiskId), + NKikimrBlobStorage::EVDiskQueueId_Name(TGroupQueues::TVDisk::TQueues::VDiskQueueId(*outVPutEvents.back())), + Blackboard.GroupQueues->GetPredictedDelayNsForEvent(*outVPutEvents.back(), Info->GetTopology()) * 0.000001); + } + + if constexpr (isVMultiPut) { + R_LOG_DEBUG_SX(logCtx, "BPP39", "Send put to orderNumber# " << diskOrderNumber + << " count# " << outVPutEvents.back()->Record.ItemsSize() + << " vMultiPut# " << outVPutEvents.back()->ToString()); + } + + Blackboard.GroupDiskRequests.DiskRequestsForOrderNumber[diskOrderNumber].FirstUnsentPutIdx = endIdx; + } + } }; //TPutImpl }//NKikimr diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_range.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_range.cpp index e234f8c18e1..c7e734120a1 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_range.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_range.cpp @@ -15,7 +15,7 @@ namespace NKikimr { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TBlobStorageGroupRangeRequest : public TBlobStorageGroupRequestActor<TBlobStorageGroupRangeRequest> { - static constexpr ui32 MaxBlobsToQueryAtOnce = 8096; + static constexpr ui32 MaxBlobsToQueryAtOnce = 8096; const ui64 TabletId; const TLogoBlobID From; @@ -180,7 +180,7 @@ class TBlobStorageGroupRangeRequest : public TBlobStorageGroupRequestActor<TBlob } } - from = TLogoBlobID(TabletId, generation, step, channel, TLogoBlobID::MaxBlobSize, cookie); + from = TLogoBlobID(TabletId, generation, step, channel, TLogoBlobID::MaxBlobSize, cookie); send = send && to < from; } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp index 49986464a2f..b50f81aeb3e 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp @@ -14,7 +14,7 @@ namespace NKikimr { TActivationContext::Send(StopGetBatchingEvent.Release()); } BatchedGetRequestCount++; - + EnsureMonitoring(true); LWTRACK(DSProxyGetHandle, ev->Get()->Orbit); EnableWilsonTracing(ev, Mon->GetSamplePPM); @@ -107,7 +107,7 @@ namespace NKikimr { Send(MonActor, new TEvThroughputAddRequest(ev->Get()->HandleClass, bytes)); EnableWilsonTracing(ev, Mon->PutSamplePPM); - if (EnablePutBatching && bytes < MaxBatchedPutSize) { + if (EnablePutBatching && bytes < MaxBatchedPutSize) { NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->HandleClass; TEvBlobStorage::TEvPut::ETactic tactic = ev->Get()->Tactic; Y_VERIFY((ui64)handleClass <= PutHandleClassCount); @@ -119,8 +119,8 @@ namespace NKikimr { } if (batchedPuts.Queue.size() == MaxBatchedPutRequests || batchedPuts.Bytes + bytes > MaxBatchedPutSize) { - *Mon->PutsSentViaPutBatching += batchedPuts.Queue.size(); - ++*Mon->PutBatchesSent; + *Mon->PutsSentViaPutBatching += batchedPuts.Queue.size(); + ++*Mon->PutBatchesSent; ProcessBatchedPutRequests(batchedPuts, handleClass, tactic); } @@ -129,14 +129,14 @@ namespace NKikimr { } else { TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(ev->Get()->HandleClass); - TAppData *app = NKikimr::AppData(TActivationContext::AsActorContext()); - bool enableRequestMod3x3ForMinLatency = app->FeatureFlags.GetEnable3x3RequestsForMirror3DCMinLatencyPut(); + TAppData *app = NKikimr::AppData(TActivationContext::AsActorContext()); + bool enableRequestMod3x3ForMinLatency = app->FeatureFlags.GetEnable3x3RequestsForMirror3DCMinLatencyPut(); // TODO(alexvru): MinLatency support const TActorId reqID = Register( CreateBlobStorageGroupPutRequest(Info, Sessions->GroupQueues, ev->Sender, Mon, ev->Get(), ev->Cookie, std::move(ev->TraceId), Mon->TimeStats.IsEnabled(), - PerDiskStats, kind, TActivationContext::Now(), StoragePoolCounters, - enableRequestMod3x3ForMinLatency)); + PerDiskStats, kind, TActivationContext::Now(), StoragePoolCounters, + enableRequestMod3x3ForMinLatency)); ActiveRequests.insert(reqID); } } @@ -158,11 +158,11 @@ namespace NKikimr { } EnsureMonitoring(true); Mon->EventPatch->Inc(); - TInstant now = TActivationContext::Now(); + TInstant now = TActivationContext::Now(); const TActorId reqId = Register( CreateBlobStorageGroupPatchRequest(Info, Sessions->GroupQueues, ev->Sender, Mon, - ev->Get(), ev->Cookie, std::move(ev->TraceId), now, - StoragePoolCounters, SelfId(), EnableVPatch.Update(now))); + ev->Get(), ev->Cookie, std::move(ev->TraceId), now, + StoragePoolCounters, SelfId(), EnableVPatch.Update(now))); ActiveRequests.insert(reqId); } @@ -257,21 +257,21 @@ namespace NKikimr { TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(handleClass); if (Info) { - TAppData *app = NKikimr::AppData(TActivationContext::AsActorContext()); - bool enableRequestMod3x3ForMinLatency = app->FeatureFlags.GetEnable3x3RequestsForMirror3DCMinLatencyPut(); + TAppData *app = NKikimr::AppData(TActivationContext::AsActorContext()); + bool enableRequestMod3x3ForMinLatency = app->FeatureFlags.GetEnable3x3RequestsForMirror3DCMinLatencyPut(); // TODO(alexvru): MinLatency support if (batchedPuts.Queue.size() == 1) { const TActorId reqID = Register( CreateBlobStorageGroupPutRequest(Info, Sessions->GroupQueues, batchedPuts.Queue.front()->Sender, Mon, batchedPuts.Queue.front()->Get(), batchedPuts.Queue.front()->Cookie, std::move(batchedPuts.Queue.front()->TraceId), Mon->TimeStats.IsEnabled(), PerDiskStats, kind, - TActivationContext::Now(), StoragePoolCounters, enableRequestMod3x3ForMinLatency)); + TActivationContext::Now(), StoragePoolCounters, enableRequestMod3x3ForMinLatency)); ActiveRequests.insert(reqID); } else { const TActorId reqID = Register( CreateBlobStorageGroupPutRequest(Info, Sessions->GroupQueues, Mon, batchedPuts.Queue, Mon->TimeStats.IsEnabled(), PerDiskStats, kind, TActivationContext::Now(), - StoragePoolCounters, handleClass, tactic, enableRequestMod3x3ForMinLatency)); + StoragePoolCounters, handleClass, tactic, enableRequestMod3x3ForMinLatency)); ActiveRequests.insert(reqID); } } else { @@ -289,15 +289,15 @@ namespace NKikimr { for (auto &bucket : PutBatchedBucketQueue) { auto &batchedPuts = BatchedPuts[bucket.HandleClass][bucket.Tactic]; Y_VERIFY(!batchedPuts.Queue.empty()); - *Mon->PutsSentViaPutBatching += batchedPuts.Queue.size(); - ++*Mon->PutBatchesSent; + *Mon->PutsSentViaPutBatching += batchedPuts.Queue.size(); + ++*Mon->PutBatchesSent; ProcessBatchedPutRequests(batchedPuts, bucket.HandleClass, bucket.Tactic); } PutBatchedBucketQueue.clear(); ++*Mon->EventStopPutBatching; LWPROBE(DSProxyBatchedPutRequest, BatchedPutRequestCount, GroupId); BatchedPutRequestCount = 0; - EnablePutBatching.Update(TActivationContext::Now()); + EnablePutBatching.Update(TActivationContext::Now()); } void TBlobStorageGroupProxy::Handle(TEvStopBatchingGetRequests::TPtr& ev) { diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp index af2739699df..6a8f38d00f7 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp @@ -296,8 +296,8 @@ namespace NKikimr { } void TBlobStorageGroupProxy::Handle(TEvRequestProxySessionsState::TPtr &ev) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, "RequestProxySessionsState Group# " << GroupId - << " Marker# DSP59"); + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, "RequestProxySessionsState Group# " << GroupId + << " Marker# DSP59"); Send(ev->Sender, new TEvProxySessionsState(Sessions ? Sessions->GroupQueues : nullptr)); } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_accelerate_put_m3dc.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_accelerate_put_m3dc.h index a9e36aa68ba..c207098fa71 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_accelerate_put_m3dc.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_accelerate_put_m3dc.h @@ -14,19 +14,19 @@ public: static constexpr size_t NumFailDomainsPerFailRealm = 3; const TEvBlobStorage::TEvPut::ETactic Tactic; - const bool EnableRequestMod3x3ForMinLatecy; + const bool EnableRequestMod3x3ForMinLatecy; - TAcceleratePut3dcStrategy(TEvBlobStorage::TEvPut::ETactic tactic, bool enableRequestMod3x3ForMinLatecy) + TAcceleratePut3dcStrategy(TEvBlobStorage::TEvPut::ETactic tactic, bool enableRequestMod3x3ForMinLatecy) : Tactic(tactic) - , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) + , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) {} ui8 PreferredReplicasPerRealm(bool isDegraded) const { // calculate the least number of replicas we have to provide per each realm - if (Tactic == TEvBlobStorage::TEvPut::TacticMinLatency) { - return EnableRequestMod3x3ForMinLatecy ? 3 : 2; - } - return isDegraded ? 2 : 1; + if (Tactic == TEvBlobStorage::TEvPut::TacticMinLatency) { + return EnableRequestMod3x3ForMinLatecy ? 3 : 2; + } + return isDegraded ? 2 : 1; } EStrategyOutcome Process(TLogContext &logCtx, TBlobState &state, const TBlobStorageGroupInfo &info, diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.cpp index c73c20712ec..f09f3eb6a8a 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.cpp @@ -212,7 +212,7 @@ bool TStrategyBase::VerifyTheWholeSituation(TBlobState &state) { } void TStrategyBase::PreparePartLayout(const TBlobState &state, const TBlobStorageGroupInfo &info, - TBlobStorageGroupType::TPartLayout *layout, ui32 slowDiskIdx) { + TBlobStorageGroupType::TPartLayout *layout, ui32 slowDiskIdx) { Y_VERIFY(layout); const ui32 totalPartCount = info.Type.TotalPartCount(); const ui32 blobSubringSize = info.Type.BlobSubgroupSize(); @@ -235,19 +235,19 @@ void TStrategyBase::PreparePartLayout(const TBlobState &state, const TBlobStorag for (ui32 partIdx = beginPartIdx; partIdx < endPartIdx; ++partIdx) { TBlobState::ESituation partSituation = disk.DiskParts[partIdx].Situation; if (partSituation == TBlobState::ESituation::Present || - (diskIdx != slowDiskIdx && partSituation == TBlobState::ESituation::Sent)) { + (diskIdx != slowDiskIdx && partSituation == TBlobState::ESituation::Sent)) { layout->VDiskPartMask[diskIdx] |= (1ul << partIdx); } layout->VDiskMask |= (1ul << diskIdx); } } } - if (slowDiskIdx == InvalidVDiskIdx) { - layout->SlowVDiskMask = 0; - } else { - Y_VERIFY_DEBUG(slowDiskIdx < sizeof(layout->SlowVDiskMask) * 8); - layout->SlowVDiskMask = (1ull << slowDiskIdx); - } + if (slowDiskIdx == InvalidVDiskIdx) { + layout->SlowVDiskMask = 0; + } else { + Y_VERIFY_DEBUG(slowDiskIdx < sizeof(layout->SlowVDiskMask) * 8); + layout->SlowVDiskMask = (1ull << slowDiskIdx); + } } bool TStrategyBase::IsPutNeeded(const TBlobState &state, const TBlobStorageGroupType::TPartPlacement &partPlacement) { @@ -307,9 +307,9 @@ void TStrategyBase::PreparePutsForPartPlacement(TLogContext &logCtx, TBlobState // send record.PartIdx to record.VDiskIdx if needed TBlobState::TDisk &disk = state.Disks[record.VDiskIdx]; TBlobState::ESituation partSituation = disk.DiskParts[record.PartIdx].Situation; - A_LOG_DEBUG_SX(logCtx, "BPG33 ", "partPlacement record partSituation# " << TBlobState::SituationToString(partSituation) - << " to# " << (ui32)record.VDiskIdx - << " blob Id# " << TLogoBlobID(state.Id, record.PartIdx + 1).ToString()); + A_LOG_DEBUG_SX(logCtx, "BPG33 ", "partPlacement record partSituation# " << TBlobState::SituationToString(partSituation) + << " to# " << (ui32)record.VDiskIdx + << " blob Id# " << TLogoBlobID(state.Id, record.PartIdx + 1).ToString()); bool isNeeded = false; switch (partSituation) { case TBlobState::ESituation::Unknown: @@ -333,7 +333,7 @@ void TStrategyBase::PreparePutsForPartPlacement(TLogContext &logCtx, TBlobState << " blob Id# " << partId.ToString()); Y_VERIFY(state.Parts[record.PartIdx].Data.IsMonolith()); groupDiskRequests.AddPut(disk.OrderNumber, partId, state.Parts[record.PartIdx].Data.GetMonolith(), - TDiskPutRequest::ReasonInitial, info.Type.IsHandoffInSubgroup(record.VDiskIdx), state.BlobIdx); + TDiskPutRequest::ReasonInitial, info.Type.IsHandoffInSubgroup(record.VDiskIdx), state.BlobIdx); disk.DiskParts[record.PartIdx].Situation = TBlobState::ESituation::Sent; } } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.h index ca005dc9367..6dc97f822cf 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_base.h @@ -15,8 +15,8 @@ protected: EDE_NORMAL }; - static constexpr ui32 InvalidVDiskIdx = Max<ui32>(); - + static constexpr ui32 InvalidVDiskIdx = Max<ui32>(); + void Finish(NKikimrProto::EReplyStatus status); // Altruistic - suppose all ERROR and non-responding disks have all the data // Optimistic - suppose all non-responding disks have the data, but ERROR disks are wiped @@ -35,7 +35,7 @@ protected: TBlobState::TDisk &disk, TIntervalSet<i32> &intervalSet, const char *logMarker); bool VerifyTheWholeSituation(TBlobState &state); void PreparePartLayout(const TBlobState &state, const TBlobStorageGroupInfo &info, - TBlobStorageGroupType::TPartLayout *layout, ui32 slowDiskIdx); + TBlobStorageGroupType::TPartLayout *layout, ui32 slowDiskIdx); bool IsPutNeeded(const TBlobState &state, const TBlobStorageGroupType::TPartPlacement &partPlacement); void PreparePutsForPartPlacement(TLogContext &logCtx, TBlobState &state, const TBlobStorageGroupInfo &info, TGroupDiskRequests &groupDiskRequests, diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_m3dc_restore.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_m3dc_restore.h index 161a1b3f9f5..70fc5e0238f 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_m3dc_restore.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_m3dc_restore.h @@ -55,7 +55,7 @@ namespace NKikimr { "Blob Id# %s unexpected whole situation %" PRIu32, state.Id.ToString().c_str(), ui32(state.WholeSituation)); state.WholeSituation = TBlobState::ESituation::Present; - const EStrategyOutcome outcome = TPut3dcStrategy(TEvBlobStorage::TEvPut::TacticMaxThroughput, false).Process(logCtx, + const EStrategyOutcome outcome = TPut3dcStrategy(TEvBlobStorage::TEvPut::TacticMaxThroughput, false).Process(logCtx, state, info, blackboard, groupDiskRequests); switch (outcome) { case EStrategyOutcome::IN_PROGRESS: diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_min_iops_block.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_min_iops_block.h index d191f110060..36d95f6adb1 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_min_iops_block.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_get_min_iops_block.h @@ -53,7 +53,7 @@ public: const ui32 totalPartCount = info.Type.TotalPartCount(); const i32 handoff = info.Type.Handoff(); ui32 partsMissing = 0; - ui32 responsesPending = 0; + ui32 responsesPending = 0; for (ui32 partIdx = 0; partIdx < totalPartCount; ++partIdx) { bool isMissing = true; for (i32 niche = -1; niche < handoff; ++niche) { @@ -64,16 +64,16 @@ public: if (partSituation == TBlobState::ESituation::Present) { isMissing = false; } - if (!requested.IsEmpty()) { - responsesPending++; - } + if (!requested.IsEmpty()) { + responsesPending++; + } } if (isMissing) { partsMissing++; } } - if (partsMissing > info.Type.ParityParts() && responsesPending > 0) { + if (partsMissing > info.Type.ParityParts() && responsesPending > 0) { return std::nullopt; } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_put_m3dc.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_put_m3dc.h index 77cc7529605..603dd957e55 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_put_m3dc.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_put_m3dc.h @@ -14,18 +14,18 @@ public: static constexpr size_t NumFailDomainsPerFailRealm = 3; const TEvBlobStorage::TEvPut::ETactic Tactic; - const bool EnableRequestMod3x3ForMinLatecy; + const bool EnableRequestMod3x3ForMinLatecy; - TPut3dcStrategy(TEvBlobStorage::TEvPut::ETactic tactic, bool enableRequestMod3x3ForMinLatecy) - : Tactic(tactic) - , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) - {} + TPut3dcStrategy(TEvBlobStorage::TEvPut::ETactic tactic, bool enableRequestMod3x3ForMinLatecy) + : Tactic(tactic) + , EnableRequestMod3x3ForMinLatecy(enableRequestMod3x3ForMinLatecy) + {} ui8 PreferredReplicasPerRealm(bool isDegraded) const { // calculate the least number of replicas we have to provide per each realm ui8 preferredReplicasPerRealm = (isDegraded ? 2 : 1); if (Tactic == TEvBlobStorage::TEvPut::TacticMinLatency) { - preferredReplicasPerRealm = (EnableRequestMod3x3ForMinLatecy ? 3 : 2); + preferredReplicasPerRealm = (EnableRequestMod3x3ForMinLatecy ? 3 : 2); } return preferredReplicasPerRealm; } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_restore.h b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_restore.h index 97c8fe3e847..827bea4a6de 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_strategy_restore.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_strategy_restore.h @@ -149,7 +149,7 @@ public: if (slowDiskSubgroupIdx >= 0) { // If there is an exceptionally slow disk, try not touching it, mark isDone TBlobStorageGroupType::TPartLayout layout; - PreparePartLayout(state, info, &layout, slowDiskSubgroupIdx); + PreparePartLayout(state, info, &layout, slowDiskSubgroupIdx); TBlobStorageGroupType::TPartPlacement partPlacement; bool isCorrectable = info.Type.CorrectLayout(layout, partPlacement); @@ -163,7 +163,7 @@ public: if (!isDone) { // Fill in the part layout TBlobStorageGroupType::TPartLayout layout; - PreparePartLayout(state, info, &layout, InvalidVDiskIdx); + PreparePartLayout(state, info, &layout, InvalidVDiskIdx); TBlobStorageGroupType::TPartPlacement partPlacement; bool isCorrectable = info.Type.CorrectLayout(layout, partPlacement); Y_VERIFY(isCorrectable); diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_counters_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_counters_ut.cpp index bb64b2711ae..ffb87f1b44c 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_counters_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_counters_ut.cpp @@ -1,146 +1,146 @@ -#include "defs.h" - -#include "dsproxy_env_mock_ut.h" -#include "dsproxy_test_state_ut.h" -#include "dsproxy_vdisk_mock_ut.h" - +#include "defs.h" + +#include "dsproxy_env_mock_ut.h" +#include "dsproxy_test_state_ut.h" +#include "dsproxy_vdisk_mock_ut.h" + #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h> #include <ydb/core/util/stlog.h> - -#include <cstring> - - -namespace NKikimr { - -namespace NDSProxyCountersTest { - - -void SetLogPriorities(TTestBasicRuntime &runtime) { - bool IsVerbose = false; - if (IsVerbose) { - runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::BS_PROXY_GET, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::BS_PROXY_PUT, NLog::PRI_DEBUG); - runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); - } - runtime.SetLogPriority(NKikimrServices::BS_PROXY, NLog::PRI_CRIT); - runtime.SetLogPriority(NKikimrServices::BS_QUEUE, NLog::PRI_CRIT); -} - - -Y_UNIT_TEST_SUITE(DSProxyCounters) { - -Y_UNIT_TEST(PutGeneratedSubrequestBytes) { - NKikimr::TBlobStorageGroupType erasure = TErasureType::Erasure4Plus2Block; - TTestBasicRuntime runtime(1, false); - SetLogPriorities(runtime); - SetupRuntime(runtime); - TDSProxyEnv env; - env.Configure(runtime, erasure, 1, 0); - TTestState testState(runtime, erasure, env.Info); - - TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; - - - TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 863, 1, 254, 24576); - ui32 requestBytes = blobId.BlobSize(); - - TRequestMonItem &requestMonItem = env.StoragePoolCounters->GetItem(HandleClassToHandleClass(handleClass), requestBytes); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 0); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 0); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 0); - - TString buffer = TString::Uninitialized(blobId.BlobSize()); - for (char &ch : buffer) { - ch = 'a'; - } - TVector<TBlobTestSet::TBlob> blobs { - TBlobTestSet::TBlob(blobId, buffer) - }; - - TEvBlobStorage::TEvPut::TPtr put = testState.CreatePutRequest(blobs[0], tactic, handleClass); + +#include <cstring> + + +namespace NKikimr { + +namespace NDSProxyCountersTest { + + +void SetLogPriorities(TTestBasicRuntime &runtime) { + bool IsVerbose = false; + if (IsVerbose) { + runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::BS_PROXY_GET, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::BS_PROXY_PUT, NLog::PRI_DEBUG); + runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); + } + runtime.SetLogPriority(NKikimrServices::BS_PROXY, NLog::PRI_CRIT); + runtime.SetLogPriority(NKikimrServices::BS_QUEUE, NLog::PRI_CRIT); +} + + +Y_UNIT_TEST_SUITE(DSProxyCounters) { + +Y_UNIT_TEST(PutGeneratedSubrequestBytes) { + NKikimr::TBlobStorageGroupType erasure = TErasureType::Erasure4Plus2Block; + TTestBasicRuntime runtime(1, false); + SetLogPriorities(runtime); + SetupRuntime(runtime); + TDSProxyEnv env; + env.Configure(runtime, erasure, 1, 0); + TTestState testState(runtime, erasure, env.Info); + + TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; + + + TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 863, 1, 254, 24576); + ui32 requestBytes = blobId.BlobSize(); + + TRequestMonItem &requestMonItem = env.StoragePoolCounters->GetItem(HandleClassToHandleClass(handleClass), requestBytes); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 0); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 0); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 0); + + TString buffer = TString::Uninitialized(blobId.BlobSize()); + for (char &ch : buffer) { + ch = 'a'; + } + TVector<TBlobTestSet::TBlob> blobs { + TBlobTestSet::TBlob(blobId, buffer) + }; + + TEvBlobStorage::TEvPut::TPtr put = testState.CreatePutRequest(blobs[0], tactic, handleClass); runtime.Register(env.CreatePutRequestActor(put).release()); - - TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; - for (ui64 part = 1; part <= env.Info->Type.TotalPartCount(); ++part) { - TLogoBlobID partBlobId(blobId, part); - TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); - specialStatuses[id] = NKikimrProto::OK; - } - - TGroupMock &groupMock = testState.GetGroupMock(); - groupMock.SetSpecialStatuses(specialStatuses); - - testState.HandleVPutsWithMock(env.Info->Type.TotalPartCount()); - - TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; - expectedStatus[blobId] = NKikimrProto::OK; - testState.ReceivePutResults(expectedStatus.size(), expectedStatus); - - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 254); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 6); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 64 * 6); -} - -Y_UNIT_TEST(MultiPutGeneratedSubrequestBytes) { - NKikimr::TBlobStorageGroupType erasure = TErasureType::Erasure4Plus2Block; - TTestBasicRuntime runtime(1, false); - SetLogPriorities(runtime); - SetupRuntime(runtime); - TDSProxyEnv env; - env.Configure(runtime, erasure, 1, 0); - TTestState testState(runtime, erasure, env.Info); - - TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; - - - TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 863, 1, 254, 24576); - ui32 requestBytes = blobId.BlobSize(); - - TRequestMonItem &requestMonItem = env.StoragePoolCounters->GetItem(HandleClassToHandleClass(handleClass), requestBytes); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 0); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 0); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 0); - - TString buffer = TString::Uninitialized(blobId.BlobSize()); - for (char &ch : buffer) { - ch = 'a'; - } - TVector<TBlobTestSet::TBlob> blobs { - TBlobTestSet::TBlob(blobId, buffer) - }; - - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; - testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); + + TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; + for (ui64 part = 1; part <= env.Info->Type.TotalPartCount(); ++part) { + TLogoBlobID partBlobId(blobId, part); + TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); + specialStatuses[id] = NKikimrProto::OK; + } + + TGroupMock &groupMock = testState.GetGroupMock(); + groupMock.SetSpecialStatuses(specialStatuses); + + testState.HandleVPutsWithMock(env.Info->Type.TotalPartCount()); + + TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; + expectedStatus[blobId] = NKikimrProto::OK; + testState.ReceivePutResults(expectedStatus.size(), expectedStatus); + + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 254); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 6); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 64 * 6); +} + +Y_UNIT_TEST(MultiPutGeneratedSubrequestBytes) { + NKikimr::TBlobStorageGroupType erasure = TErasureType::Erasure4Plus2Block; + TTestBasicRuntime runtime(1, false); + SetLogPriorities(runtime); + SetupRuntime(runtime); + TDSProxyEnv env; + env.Configure(runtime, erasure, 1, 0); + TTestState testState(runtime, erasure, env.Info); + + TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; + + + TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 863, 1, 254, 24576); + ui32 requestBytes = blobId.BlobSize(); + + TRequestMonItem &requestMonItem = env.StoragePoolCounters->GetItem(HandleClassToHandleClass(handleClass), requestBytes); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 0); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 0); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 0); + + TString buffer = TString::Uninitialized(blobId.BlobSize()); + for (char &ch : buffer) { + ch = 'a'; + } + TVector<TBlobTestSet::TBlob> blobs { + TBlobTestSet::TBlob(blobId, buffer) + }; + + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; + testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); runtime.Register(env.CreatePutRequestActor(batched, tactic, handleClass).release()); - - TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; - for (ui64 part = 1; part <= env.Info->Type.TotalPartCount(); ++part) { - TLogoBlobID partBlobId(blobId, part); - TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); - specialStatuses[id] = NKikimrProto::OK; - } - - TGroupMock &groupMock = testState.GetGroupMock(); - groupMock.SetSpecialStatuses(specialStatuses); - - testState.HandleVMultiPutsWithMock(env.Info->Type.TotalPartCount()); - - TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; - expectedStatus[blobId] = NKikimrProto::OK; - testState.ReceivePutResults(expectedStatus.size(), expectedStatus); - - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 254); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 6); - UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 64 * 6); -} - - -} - - -} // NDSProxyPatchTest - -} // NKikimr + + TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; + for (ui64 part = 1; part <= env.Info->Type.TotalPartCount(); ++part) { + TLogoBlobID partBlobId(blobId, part); + TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); + specialStatuses[id] = NKikimrProto::OK; + } + + TGroupMock &groupMock = testState.GetGroupMock(); + groupMock.SetSpecialStatuses(specialStatuses); + + testState.HandleVMultiPutsWithMock(env.Info->Type.TotalPartCount()); + + TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; + expectedStatus[blobId] = NKikimrProto::OK; + testState.ReceivePutResults(expectedStatus.size(), expectedStatus); + + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.RequestBytes->Val(), 254); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequests->Val(), 6); + UNIT_ASSERT_VALUES_EQUAL(requestMonItem.GeneratedSubrequestBytes->Val(), 64 * 6); +} + + +} + + +} // NDSProxyPatchTest + +} // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_env_mock_ut.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_env_mock_ut.h index 0f2350daf85..5deb47a575a 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_env_mock_ut.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_env_mock_ut.h @@ -1,162 +1,162 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/blobstorage/dsproxy/dsproxy.h> #include <ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h> - + #include <ydb/core/blobstorage/base/utility.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> - + #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/basics/appdata.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - -inline TMaybe<TGroupStat::EKind> PutHandleClassToGroupStatKind(NKikimrBlobStorage::EPutHandleClass handleClass) { - switch (handleClass) { - case NKikimrBlobStorage::TabletLog: - return TGroupStat::EKind::PUT_TABLET_LOG; - - case NKikimrBlobStorage::UserData: - return TGroupStat::EKind::PUT_USER_DATA; - - default: - return {}; - } -} - -struct TDSProxyEnv { - TVector<TActorId> VDisks; - TIntrusivePtr<TBlobStorageGroupInfo> Info; - TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; - TBSProxyContextPtr BSProxyCtxPtr; - TIntrusivePtr<NMonitoring::TDynamicCounters> DynCounters; - TIntrusivePtr<NKikimr::TStoragePoolCounters> StoragePoolCounters; - TDiskResponsivenessTracker::TPerDiskStatsPtr PerDiskStatsPtr; - TNodeLayoutInfoPtr NodeLayoutInfo; - TIntrusivePtr<TGroupQueues> GroupQueues; - ui64 GroupId; - ui64 NodeIdx; - TActorId RealProxyActorId; - TActorId FakeProxyActorId; - - TActorId MakeVDiskId(ui32 failDomainIdx, ui32 driveIdx, ui32 drivesPerFailDomain) const { - ui32 i = failDomainIdx * drivesPerFailDomain + driveIdx; - return VDisks[i]; - } - - void Configure(TTestActorRuntime& runtime, TBlobStorageGroupType type, ui64 groupId, ui64 nodeIndex) { - GroupId = groupId; - NodeIdx = nodeIndex; - - ui32 vDiskCount = type.BlobSubgroupSize(); - VDisks.clear(); - for (ui32 i = 0; i < vDiskCount; ++i) { - VDisks.push_back(runtime.AllocateEdgeActor(nodeIndex)); - } - - if (type.GetErasure() == TErasureType::ErasureMirror3dc) { - Info = new TBlobStorageGroupInfo(type.GetErasure(), 1, 3, 3, &VDisks); - } else { - Info = new TBlobStorageGroupInfo(type.GetErasure(), 1, type.BlobSubgroupSize(), 1, &VDisks); - } - - RealProxyActorId = MakeBlobStorageProxyID(groupId); - TIntrusivePtr<TDsProxyNodeMon> nodeMon = new TDsProxyNodeMon(runtime.GetAppData(nodeIndex).Counters, true); - TString name = Sprintf("%09" PRIu64, groupId); - TIntrusivePtr<NMonitoring::TDynamicCounters> group = GetServiceCounters( - runtime.GetAppData(0).Counters, "dsproxy")->GetSubgroup("blobstorageproxy", name); - TIntrusivePtr<NMonitoring::TDynamicCounters> percentileGroup = GetServiceCounters( - runtime.GetAppData(0).Counters, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", name); - TIntrusivePtr<NMonitoring::TDynamicCounters> overviewGroup = GetServiceCounters( - runtime.GetAppData(0).Counters, "dsproxy_overview"); - BSProxyCtxPtr.Reset(new TBSProxyContext(group->GetSubgroup("subsystem", "memproxy"))); - Mon = new TBlobStorageGroupProxyMon(group, percentileGroup, overviewGroup, Info, nodeMon, false); - TDsProxyPerPoolCounters perPoolCounters(runtime.GetAppData(nodeIndex).Counters); - TIntrusivePtr<TStoragePoolCounters> storagePoolCounters = perPoolCounters.GetPoolCounters("pool_name"); - TControlWrapper enablePutBatching(DefaultEnablePutBatching, false, true); - TControlWrapper enableVPatch(DefaultEnableVPatch, false, true); + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { + +inline TMaybe<TGroupStat::EKind> PutHandleClassToGroupStatKind(NKikimrBlobStorage::EPutHandleClass handleClass) { + switch (handleClass) { + case NKikimrBlobStorage::TabletLog: + return TGroupStat::EKind::PUT_TABLET_LOG; + + case NKikimrBlobStorage::UserData: + return TGroupStat::EKind::PUT_USER_DATA; + + default: + return {}; + } +} + +struct TDSProxyEnv { + TVector<TActorId> VDisks; + TIntrusivePtr<TBlobStorageGroupInfo> Info; + TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; + TBSProxyContextPtr BSProxyCtxPtr; + TIntrusivePtr<NMonitoring::TDynamicCounters> DynCounters; + TIntrusivePtr<NKikimr::TStoragePoolCounters> StoragePoolCounters; + TDiskResponsivenessTracker::TPerDiskStatsPtr PerDiskStatsPtr; + TNodeLayoutInfoPtr NodeLayoutInfo; + TIntrusivePtr<TGroupQueues> GroupQueues; + ui64 GroupId; + ui64 NodeIdx; + TActorId RealProxyActorId; + TActorId FakeProxyActorId; + + TActorId MakeVDiskId(ui32 failDomainIdx, ui32 driveIdx, ui32 drivesPerFailDomain) const { + ui32 i = failDomainIdx * drivesPerFailDomain + driveIdx; + return VDisks[i]; + } + + void Configure(TTestActorRuntime& runtime, TBlobStorageGroupType type, ui64 groupId, ui64 nodeIndex) { + GroupId = groupId; + NodeIdx = nodeIndex; + + ui32 vDiskCount = type.BlobSubgroupSize(); + VDisks.clear(); + for (ui32 i = 0; i < vDiskCount; ++i) { + VDisks.push_back(runtime.AllocateEdgeActor(nodeIndex)); + } + + if (type.GetErasure() == TErasureType::ErasureMirror3dc) { + Info = new TBlobStorageGroupInfo(type.GetErasure(), 1, 3, 3, &VDisks); + } else { + Info = new TBlobStorageGroupInfo(type.GetErasure(), 1, type.BlobSubgroupSize(), 1, &VDisks); + } + + RealProxyActorId = MakeBlobStorageProxyID(groupId); + TIntrusivePtr<TDsProxyNodeMon> nodeMon = new TDsProxyNodeMon(runtime.GetAppData(nodeIndex).Counters, true); + TString name = Sprintf("%09" PRIu64, groupId); + TIntrusivePtr<NMonitoring::TDynamicCounters> group = GetServiceCounters( + runtime.GetAppData(0).Counters, "dsproxy")->GetSubgroup("blobstorageproxy", name); + TIntrusivePtr<NMonitoring::TDynamicCounters> percentileGroup = GetServiceCounters( + runtime.GetAppData(0).Counters, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", name); + TIntrusivePtr<NMonitoring::TDynamicCounters> overviewGroup = GetServiceCounters( + runtime.GetAppData(0).Counters, "dsproxy_overview"); + BSProxyCtxPtr.Reset(new TBSProxyContext(group->GetSubgroup("subsystem", "memproxy"))); + Mon = new TBlobStorageGroupProxyMon(group, percentileGroup, overviewGroup, Info, nodeMon, false); + TDsProxyPerPoolCounters perPoolCounters(runtime.GetAppData(nodeIndex).Counters); + TIntrusivePtr<TStoragePoolCounters> storagePoolCounters = perPoolCounters.GetPoolCounters("pool_name"); + TControlWrapper enablePutBatching(DefaultEnablePutBatching, false, true); + TControlWrapper enableVPatch(DefaultEnableVPatch, false, true); IActor *dsproxy = CreateBlobStorageGroupProxyConfigured(TIntrusivePtr(Info), true, nodeMon, std::move(storagePoolCounters), enablePutBatching, enableVPatch); - TActorId actorId = runtime.Register(dsproxy, nodeIndex); - runtime.RegisterService(RealProxyActorId, actorId, nodeIndex); - - FakeProxyActorId = runtime.AllocateEdgeActor(0); - TAutoPtr <IEventHandle> handle; - runtime.Send(new IEventHandle(RealProxyActorId, FakeProxyActorId, new TEvRequestProxySessionsState)); - auto queues = runtime.GrabEdgeEventRethrow<TEvProxySessionsState>(handle); - GroupQueues = queues->GroupQueues; - NodeLayoutInfo = nullptr; - DynCounters = new NMonitoring::TDynamicCounters(); - StoragePoolCounters = new NKikimr::TStoragePoolCounters(DynCounters, "", {}); - PerDiskStatsPtr = new TDiskResponsivenessTracker::TPerDiskStats; - } - - void SetGroupGeneration(ui32 generation) { - Info = new TBlobStorageGroupInfo(Info, TVDiskID(Info->GroupID, generation, 0, 0, 0), VDisks[0]); - } - + TActorId actorId = runtime.Register(dsproxy, nodeIndex); + runtime.RegisterService(RealProxyActorId, actorId, nodeIndex); + + FakeProxyActorId = runtime.AllocateEdgeActor(0); + TAutoPtr <IEventHandle> handle; + runtime.Send(new IEventHandle(RealProxyActorId, FakeProxyActorId, new TEvRequestProxySessionsState)); + auto queues = runtime.GrabEdgeEventRethrow<TEvProxySessionsState>(handle); + GroupQueues = queues->GroupQueues; + NodeLayoutInfo = nullptr; + DynCounters = new NMonitoring::TDynamicCounters(); + StoragePoolCounters = new NKikimr::TStoragePoolCounters(DynCounters, "", {}); + PerDiskStatsPtr = new TDiskResponsivenessTracker::TPerDiskStats; + } + + void SetGroupGeneration(ui32 generation) { + Info = new TBlobStorageGroupInfo(Info, TVDiskID(Info->GroupID, generation, 0, 0, 0), VDisks[0]); + } + std::unique_ptr<IActor> CreatePutRequestActor(TEvBlobStorage::TEvPut::TPtr &ev) { - TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(ev->Get()->HandleClass); + TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(ev->Get()->HandleClass); return std::unique_ptr<IActor>(CreateBlobStorageGroupPutRequest(Info, GroupQueues, ev->Sender, Mon, ev->Get(), - ev->Cookie, std::move(ev->TraceId), Mon->TimeStats.IsEnabled(), PerDiskStatsPtr, kind, + ev->Cookie, std::move(ev->TraceId), Mon->TimeStats.IsEnabled(), PerDiskStatsPtr, kind, TInstant::Now(), StoragePoolCounters, false)); - } - + } + std::unique_ptr<IActor> CreatePutRequestActor(TBatchedVec<TEvBlobStorage::TEvPut::TPtr> &batched, - TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) - { - TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(handleClass); + TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) + { + TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(handleClass); return std::unique_ptr<IActor>(CreateBlobStorageGroupPutRequest(Info, GroupQueues, - Mon, batched, Mon->TimeStats.IsEnabled(), PerDiskStatsPtr, kind,TInstant::Now(), + Mon, batched, Mon->TimeStats.IsEnabled(), PerDiskStatsPtr, kind,TInstant::Now(), StoragePoolCounters, handleClass, tactic, false)); - } - + } + std::unique_ptr<IActor> CreateGetRequestActor(TEvBlobStorage::TEvGet::TPtr &ev, - NKikimrBlobStorage::EPutHandleClass handleClass, bool withMultiPut) - { - TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(handleClass); + NKikimrBlobStorage::EPutHandleClass handleClass, bool withMultiPut) + { + TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(handleClass); return std::unique_ptr<IActor>(CreateBlobStorageGroupGetRequest(Info, GroupQueues, ev->Sender, Mon, - ev->Get(), ev->Cookie, std::move(ev->TraceId), TNodeLayoutInfoPtr(NodeLayoutInfo), + ev->Get(), ev->Cookie, std::move(ev->TraceId), TNodeLayoutInfoPtr(NodeLayoutInfo), kind, TInstant::Now(), StoragePoolCounters, withMultiPut)); - } - + } + std::unique_ptr<IActor> CreatePatchRequestActor(TEvBlobStorage::TEvPatch::TPtr &ev, bool useVPatch = false) { return std::unique_ptr<IActor>(CreateBlobStorageGroupPatchRequest(Info, GroupQueues, ev->Sender, Mon, - ev->Get(), ev->Cookie, std::move(ev->TraceId), TInstant::Now(), StoragePoolCounters, + ev->Get(), ev->Cookie, std::move(ev->TraceId), TInstant::Now(), StoragePoolCounters, FakeProxyActorId, useVPatch)); - } -}; - - -inline bool ScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, - TDuration delay, TInstant& deadline) { - if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { - deadline = runtime.GetTimeProvider()->Now() + delay; - return false; - } - return true; -} - -inline void SetupRuntime(TTestActorRuntime& runtime) { - TAppPrepare app; - app.ClearDomainsAndHive(); - runtime.SetScheduledEventFilter(&ScheduledFilterFunc); - - runtime.SetEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { - if (ev->GetTypeRewrite() == TEvBlobStorage::EvVCheckReadiness) { - runtime.Send(new IEventHandle( - ev->Sender, ev->Recipient, new TEvBlobStorage::TEvVCheckReadinessResult(NKikimrProto::OK), 0, - ev->Cookie), 0, true); - return true; - } - return false; - }); - - runtime.Initialize(app.Unwrap()); -} - -} // NKikimr + } +}; + + +inline bool ScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, + TDuration delay, TInstant& deadline) { + if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { + deadline = runtime.GetTimeProvider()->Now() + delay; + return false; + } + return true; +} + +inline void SetupRuntime(TTestActorRuntime& runtime) { + TAppPrepare app; + app.ClearDomainsAndHive(); + runtime.SetScheduledEventFilter(&ScheduledFilterFunc); + + runtime.SetEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() == TEvBlobStorage::EvVCheckReadiness) { + runtime.Send(new IEventHandle( + ev->Sender, ev->Recipient, new TEvBlobStorage::TEvVCheckReadinessResult(NKikimrProto::OK), 0, + ev->Cookie), 0, true); + return true; + } + return false; + }); + + runtime.Initialize(app.Unwrap()); +} + +} // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut.cpp index 6a8fbff0e28..fdd7d015001 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut.cpp @@ -56,14 +56,14 @@ Y_UNIT_TEST_SUITE(TBsProxyFaultToleranceTest) { FAULT_TOLERANCE_TEST(ERASURE, TDiscoverFaultToleranceTest) \ FAULT_TOLERANCE_TEST(ERASURE, TPutFaultToleranceTest) - //ERASURE_TEST(ErasureMirror3) - //ERASURE_TEST(Erasure3Plus1Block) - //ERASURE_TEST(Erasure3Plus1Stripe) + //ERASURE_TEST(ErasureMirror3) + //ERASURE_TEST(Erasure3Plus1Block) + //ERASURE_TEST(Erasure3Plus1Stripe) ERASURE_TEST(Erasure4Plus2Block) - //ERASURE_TEST(Erasure3Plus2Block) - //ERASURE_TEST(Erasure4Plus2Stripe) - //ERASURE_TEST(Erasure3Plus2Stripe) - //ERASURE_TEST(ErasureMirror3Plus2) + //ERASURE_TEST(Erasure3Plus2Block) + //ERASURE_TEST(Erasure4Plus2Stripe) + //ERASURE_TEST(Erasure3Plus2Stripe) + //ERASURE_TEST(ErasureMirror3Plus2) ERASURE_TEST(ErasureMirror3dc) ERASURE_TEST(ErasureMirror3of4) diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_base.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_base.h index c4b5c76f39b..a6da8445f69 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_base.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_base.h @@ -185,8 +185,8 @@ public: } } while (responsesPending--) { - auto resp = WaitForSpecificEvent<TEvVMockCtlResponse>(); - // Cerr << (TStringBuilder() << "]] SpecEventDelete(wipe=" << wipe << "): " << resp->Get()->ToString() << Endl); + auto resp = WaitForSpecificEvent<TEvVMockCtlResponse>(); + // Cerr << (TStringBuilder() << "]] SpecEventDelete(wipe=" << wipe << "): " << resp->Get()->ToString() << Endl); } } diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_range.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_range.h index c055d68eabb..61b43711f53 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_range.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_range.h @@ -15,7 +15,7 @@ public: void Check(ui64 tabletId, const TBlobStorageGroupInfo::TGroupVDisks& disks, NKikimrProto::EReplyStatus defaultExpectedStatus = NKikimrProto::OK) { - // Cerr << (TStringBuilder() << "]] " << disks.ToString() << Endl); + // Cerr << (TStringBuilder() << "]] " << disks.ToString() << Endl); for (ui32 generation = 1; generation <= 4; ++generation) { NKikimrProto::EReplyStatus expectedStatus = defaultExpectedStatus; TVector<TEvBlobStorage::TEvRangeResult::TResponse> expectedResponse; @@ -26,11 +26,11 @@ public: for (ui32 step = 1; step <= 2; ++step) { TString buffer = Sprintf("%256s/%" PRIu64 "/%" PRIu32 "/%" PRIu32, "kikimr", tabletId, generation, step); - TStringBuilder b; - for (int i = 0; i < 1024; ++i) { - b << 'a'; - } - buffer += b; + TStringBuilder b; + for (int i = 0; i < 1024; ++i) { + b << 'a'; + } + buffer += b; TLogoBlobID id(tabletId, generation, step, 0 /*channel*/, buffer.size(), 0); if (defaultExpectedStatus == NKikimrProto::OK) { UNIT_ASSERT_VALUES_EQUAL(NKikimrProto::OK, PutWithResult(id, buffer, TEvBlobStorage::TEvPut::TacticMaxThroughput)); @@ -77,7 +77,7 @@ public: } for (ui32 i = 0; i < vdisks.size(); ++i) { auto event = WaitForSpecificEvent<TEvBlobStorage::TEvVGetResult>(); - // Cerr << (TStringBuilder() << "]] Get: " << event->Get()->ToString() << Endl); + // Cerr << (TStringBuilder() << "]] Get: " << event->Get()->ToString() << Endl); if (event->Get()->Record.GetStatus() == NKikimrProto::OK) { for (const auto& item : event->Get()->Record.GetResult()) { if (item.GetStatus() == NKikimrProto::OK) { diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_runtime.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_runtime.h index bc33af4b86c..53ff008cca7 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_runtime.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_fault_tolerance_ut_runtime.h @@ -85,12 +85,12 @@ public: TIntrusivePtr<TDsProxyNodeMon> nodeMon(new TDsProxyNodeMon(Counters, true)); TDsProxyPerPoolCounters perPoolCounters(Counters); TIntrusivePtr<TStoragePoolCounters> storagePoolCounters = perPoolCounters.GetPoolCounters("pool_name"); - TControlWrapper enablePutBatching(DefaultEnablePutBatching, false, true); - TControlWrapper enableVPatch(DefaultEnableVPatch, false, true); + TControlWrapper enablePutBatching(DefaultEnablePutBatching, false, true); + TControlWrapper enableVPatch(DefaultEnableVPatch, false, true); IActor *dsproxy = CreateBlobStorageGroupProxyConfigured(TIntrusivePtr(GroupInfo), false, nodeMon, std::move(storagePoolCounters), enablePutBatching, enableVPatch); - setup->LocalServices.emplace_back(MakeBlobStorageProxyID(GroupInfo->GroupID), - TActorSetupCmd(dsproxy, TMailboxType::Simple, 0)); + setup->LocalServices.emplace_back(MakeBlobStorageProxyID(GroupInfo->GroupID), + TActorSetupCmd(dsproxy, TMailboxType::Simple, 0)); ActorSystem.reset(new TActorSystem(setup, AppData.get(), logSettings)); LOG_NOTICE(*ActorSystem, NActorsServices::TEST, "Actor system created"); diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_get_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_get_ut.cpp index 85e16c2aef3..13c07049899 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_get_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_get_ut.cpp @@ -1,5 +1,5 @@ #include "defs.h" -#include "dsproxy_vdisk_mock_ut.h" +#include "dsproxy_vdisk_mock_ut.h" #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/actor_helpers.h> @@ -153,309 +153,309 @@ Y_UNIT_TEST(TestMirror32GetBlobCrcCheck) { TestIntervalsAndCrcAllOk(TErasureType::ErasureMirror3Plus2, false, true); } -class TTestWipedAllOkStep { - enum ETVPutEventKind { - TVPEK_VPUT, - TVPEK_VMULTIPUT - }; - -public: - struct TVPutInfo { - TVDiskID VDiskId; - TLogoBlobID BlobId; - ui64 BlobIdx; - - bool operator<(const TVPutInfo &other) const { - return std::tie(VDiskId, BlobId, BlobIdx) < std::tie(other.VDiskId, other.BlobId, other.BlobIdx); - } - - bool operator==(const TVPutInfo &other) const { - return std::tie(VDiskId, BlobId, BlobIdx) == std::tie(other.VDiskId, other.BlobId, other.BlobIdx); - } - }; - - ui32 GroupId; - TErasureType::EErasureSpecies ErasureSpecies; - ui32 DomainCount; - ui32 GenerateBlobsMode = 0; - const TVector<ui64> &QueryCounts; - ui64 MaxQueryCount; - - const TVector<TBlobTestSet::TBlob> *Blobs = nullptr; - - bool IsVerboseNoDataEnabled; - bool IsRestore; - - TVector<TVPutInfo> SendVPuts; - -private: - TMaybe<TGroupMock> Group; +class TTestWipedAllOkStep { + enum ETVPutEventKind { + TVPEK_VPUT, + TVPEK_VMULTIPUT + }; + +public: + struct TVPutInfo { + TVDiskID VDiskId; + TLogoBlobID BlobId; + ui64 BlobIdx; + + bool operator<(const TVPutInfo &other) const { + return std::tie(VDiskId, BlobId, BlobIdx) < std::tie(other.VDiskId, other.BlobId, other.BlobIdx); + } + + bool operator==(const TVPutInfo &other) const { + return std::tie(VDiskId, BlobId, BlobIdx) == std::tie(other.VDiskId, other.BlobId, other.BlobIdx); + } + }; + + ui32 GroupId; + TErasureType::EErasureSpecies ErasureSpecies; + ui32 DomainCount; + ui32 GenerateBlobsMode = 0; + const TVector<ui64> &QueryCounts; + ui64 MaxQueryCount; + + const TVector<TBlobTestSet::TBlob> *Blobs = nullptr; + + bool IsVerboseNoDataEnabled; + bool IsRestore; + + TVector<TVPutInfo> SendVPuts; + +private: + TMaybe<TGroupMock> Group; TIntrusivePtr<TGroupQueues> GroupQueues; - TMaybe<TBlobTestSet> BlobSet; - - ui64 VPutRequests = 0; - ui64 VPutResponses = 0; - ui64 VMultiPutRequests = 0; - ui64 VMultiPutResponses = 0; - ui64 RequestIndex = 0; - ui64 ResponseIndex = 0; - -public: - TTestWipedAllOkStep(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 domainCount, - const TVector<ui64> &queryCounts, bool isVerboseNoDataEnabled, bool isRestore) - : GroupId(groupId) - , ErasureSpecies(erasureSpecies) - , DomainCount(domainCount) - , QueryCounts(queryCounts) - , MaxQueryCount(*MaxElement(QueryCounts.begin(), QueryCounts.end())) - , IsVerboseNoDataEnabled(isVerboseNoDataEnabled) - , IsRestore(isRestore) - { - } - - void SetGenerateBlobsMode(ui64 generateBlobsMode) { - GenerateBlobsMode = generateBlobsMode; - } - - void SetBlobs(const TVector<TBlobTestSet::TBlob> &blobs) { - Blobs = &blobs; - } - - void AddWipedVDisk(ui64 idx, ui64 error) { - switch (error) { - case 0: - Group->Wipe(idx); - break; - case 1: - Group->SetError(idx, NKikimrProto::ERROR); - break; - case 2: - Group->SetError(idx, NKikimrProto::CORRUPTED); - break; - case 3: - Group->SetError(idx, NKikimrProto::VDISK_ERROR_STATE); - break; - case 4: - Group->SetError(idx, NKikimrProto::NOT_YET); - break; - } - } - - void Run(bool useVMultiPut) { - for (ui64 qci = 0; qci < QueryCounts.size(); ++qci) { - ui64 queryCount = QueryCounts[qci]; - for (ui64 bci = 0; bci < QueryCounts.size(); ++bci) { - ui64 blobCount = QueryCounts[bci]; - if (useVMultiPut) { - SubStep<TVPEK_VMULTIPUT>(queryCount, blobCount); - } else { - SubStep<TVPEK_VPUT>(queryCount, blobCount); - } - } - } - } - - void Init() { - SendVPuts.clear(); - Group.Clear(); - Group.ConstructInPlace(GroupId, ErasureSpecies, DomainCount, 1); + TMaybe<TBlobTestSet> BlobSet; + + ui64 VPutRequests = 0; + ui64 VPutResponses = 0; + ui64 VMultiPutRequests = 0; + ui64 VMultiPutResponses = 0; + ui64 RequestIndex = 0; + ui64 ResponseIndex = 0; + +public: + TTestWipedAllOkStep(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 domainCount, + const TVector<ui64> &queryCounts, bool isVerboseNoDataEnabled, bool isRestore) + : GroupId(groupId) + , ErasureSpecies(erasureSpecies) + , DomainCount(domainCount) + , QueryCounts(queryCounts) + , MaxQueryCount(*MaxElement(QueryCounts.begin(), QueryCounts.end())) + , IsVerboseNoDataEnabled(isVerboseNoDataEnabled) + , IsRestore(isRestore) + { + } + + void SetGenerateBlobsMode(ui64 generateBlobsMode) { + GenerateBlobsMode = generateBlobsMode; + } + + void SetBlobs(const TVector<TBlobTestSet::TBlob> &blobs) { + Blobs = &blobs; + } + + void AddWipedVDisk(ui64 idx, ui64 error) { + switch (error) { + case 0: + Group->Wipe(idx); + break; + case 1: + Group->SetError(idx, NKikimrProto::ERROR); + break; + case 2: + Group->SetError(idx, NKikimrProto::CORRUPTED); + break; + case 3: + Group->SetError(idx, NKikimrProto::VDISK_ERROR_STATE); + break; + case 4: + Group->SetError(idx, NKikimrProto::NOT_YET); + break; + } + } + + void Run(bool useVMultiPut) { + for (ui64 qci = 0; qci < QueryCounts.size(); ++qci) { + ui64 queryCount = QueryCounts[qci]; + for (ui64 bci = 0; bci < QueryCounts.size(); ++bci) { + ui64 blobCount = QueryCounts[bci]; + if (useVMultiPut) { + SubStep<TVPEK_VMULTIPUT>(queryCount, blobCount); + } else { + SubStep<TVPEK_VPUT>(queryCount, blobCount); + } + } + } + } + + void Init() { + SendVPuts.clear(); + Group.Clear(); + Group.ConstructInPlace(GroupId, ErasureSpecies, DomainCount, 1); GroupQueues = Group->MakeGroupQueues(); - BlobSet.Clear(); - BlobSet.ConstructInPlace(); - if (Blobs) { - BlobSet->AddBlobs(*Blobs); - } else { - BlobSet->GenerateSet(GenerateBlobsMode, MaxQueryCount); - } - Group->PutBlobSet(*BlobSet); - } - -private: - void ClearCounters() { - VPutRequests = 0; - VPutResponses = 0; - VMultiPutRequests = 0; - VMultiPutResponses = 0; - RequestIndex = 0; - ResponseIndex = 0; - } - - void AssertCounters(TGetImpl &getImpl) { - UNIT_ASSERT_C(VPutResponses == getImpl.GetVPutResponses() - && VPutRequests == getImpl.GetVPutRequests() - && VMultiPutResponses == getImpl.GetVMultiPutResponses() - && VMultiPutRequests == getImpl.GetVMultiPutRequests() - && RequestIndex == getImpl.GetRequestIndex() - && ResponseIndex == getImpl.GetResponseIndex(), - "Not equal expected VPutRequest and VPutResponse with given" - << " VPutRequests# " << VPutRequests - << " VPutResponses# " << VPutResponses - << " getImpl.VPutRequests# " << getImpl.GetVPutRequests() - << " getImpl.VPutResponses# " << getImpl.GetVPutResponses() - << " VMultiPutRequests# " << VMultiPutRequests - << " VMultiPutResponses# " << VMultiPutResponses - << " getImpl.VMultiPutRequests# " << getImpl.GetVMultiPutRequests() - << " getImpl.VMultiPutResponses# " << getImpl.GetVMultiPutResponses() - << " RequestIndex# " << RequestIndex - << " ResponseIndex# " << ResponseIndex - << " getImpl.RequestIndex# " << getImpl.GetRequestIndex() - << " getImpl.ResponseIndex# " << getImpl.GetResponseIndex()); - } - - void ProcessVPuts(TLogContext &logCtx, TGetImpl &getImpl, + BlobSet.Clear(); + BlobSet.ConstructInPlace(); + if (Blobs) { + BlobSet->AddBlobs(*Blobs); + } else { + BlobSet->GenerateSet(GenerateBlobsMode, MaxQueryCount); + } + Group->PutBlobSet(*BlobSet); + } + +private: + void ClearCounters() { + VPutRequests = 0; + VPutResponses = 0; + VMultiPutRequests = 0; + VMultiPutResponses = 0; + RequestIndex = 0; + ResponseIndex = 0; + } + + void AssertCounters(TGetImpl &getImpl) { + UNIT_ASSERT_C(VPutResponses == getImpl.GetVPutResponses() + && VPutRequests == getImpl.GetVPutRequests() + && VMultiPutResponses == getImpl.GetVMultiPutResponses() + && VMultiPutRequests == getImpl.GetVMultiPutRequests() + && RequestIndex == getImpl.GetRequestIndex() + && ResponseIndex == getImpl.GetResponseIndex(), + "Not equal expected VPutRequest and VPutResponse with given" + << " VPutRequests# " << VPutRequests + << " VPutResponses# " << VPutResponses + << " getImpl.VPutRequests# " << getImpl.GetVPutRequests() + << " getImpl.VPutResponses# " << getImpl.GetVPutResponses() + << " VMultiPutRequests# " << VMultiPutRequests + << " VMultiPutResponses# " << VMultiPutResponses + << " getImpl.VMultiPutRequests# " << getImpl.GetVMultiPutRequests() + << " getImpl.VMultiPutResponses# " << getImpl.GetVMultiPutResponses() + << " RequestIndex# " << RequestIndex + << " ResponseIndex# " << ResponseIndex + << " getImpl.RequestIndex# " << getImpl.GetRequestIndex() + << " getImpl.ResponseIndex# " << getImpl.GetResponseIndex()); + } + + void ProcessVPuts(TLogContext &logCtx, TGetImpl &getImpl, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &vGets, TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> &vPuts, - TAutoPtr<TEvBlobStorage::TEvGetResult> &getResult) { - for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - auto &putRequest = vPuts[vPutIdx]->Record; - Y_VERIFY(putRequest.HasCookie()); - auto vdisk = VDiskIDFromVDiskID(putRequest.GetVDiskID()); - auto blobId = LogoBlobIDFromLogoBlobID(putRequest.GetBlobID()); - TBlobCookie cookie(putRequest.GetCookie()); - SendVPuts.push_back({vdisk, blobId, cookie.GetBlobIdx()}); - TEvBlobStorage::TEvVPutResult vPutResult; + TAutoPtr<TEvBlobStorage::TEvGetResult> &getResult) { + for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + auto &putRequest = vPuts[vPutIdx]->Record; + Y_VERIFY(putRequest.HasCookie()); + auto vdisk = VDiskIDFromVDiskID(putRequest.GetVDiskID()); + auto blobId = LogoBlobIDFromLogoBlobID(putRequest.GetBlobID()); + TBlobCookie cookie(putRequest.GetCookie()); + SendVPuts.push_back({vdisk, blobId, cookie.GetBlobIdx()}); + TEvBlobStorage::TEvVPutResult vPutResult; vPutResult.MakeError(NKikimrProto::OK, TString(), putRequest); - + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); - VPutResponses++; - RequestIndex += nextVGets.size(); - VPutRequests += nextVPuts.size(); - AssertCounters(getImpl); - + getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); + VPutResponses++; + RequestIndex += nextVGets.size(); + VPutRequests += nextVPuts.size(); + AssertCounters(getImpl); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - if (getResult) { - break; - } - } - vPuts.clear(); - } - - void ProcessVPuts(TLogContext &logCtx, TGetImpl &getImpl, + if (getResult) { + break; + } + } + vPuts.clear(); + } + + void ProcessVPuts(TLogContext &logCtx, TGetImpl &getImpl, TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> &vGets, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &vPuts, - TAutoPtr<TEvBlobStorage::TEvGetResult> &getResult) { - for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - auto &multiPutRequest = vPuts[vPutIdx]->Record; - Y_VERIFY(multiPutRequest.HasCookie()); - - auto vdisk = VDiskIDFromVDiskID(multiPutRequest.GetVDiskID()); - UNIT_ASSERT(multiPutRequest.ItemsSize() <= MaxBatchedPutRequests); - ui64 sendBytes = 0; - for (auto &item : multiPutRequest.GetItems()) { - Y_VERIFY(item.HasCookie()); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - TString buffer = item.GetBuffer(); - sendBytes += buffer.size(); - TBlobCookie cookie(item.GetCookie()); - SendVPuts.push_back({vdisk, blobId, cookie.GetBlobIdx()}); - } - UNIT_ASSERT(sendBytes <= MaxBatchedPutSize); - - TEvBlobStorage::TEvVMultiPutResult vMultiPutResult; + TAutoPtr<TEvBlobStorage::TEvGetResult> &getResult) { + for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + auto &multiPutRequest = vPuts[vPutIdx]->Record; + Y_VERIFY(multiPutRequest.HasCookie()); + + auto vdisk = VDiskIDFromVDiskID(multiPutRequest.GetVDiskID()); + UNIT_ASSERT(multiPutRequest.ItemsSize() <= MaxBatchedPutRequests); + ui64 sendBytes = 0; + for (auto &item : multiPutRequest.GetItems()) { + Y_VERIFY(item.HasCookie()); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + TString buffer = item.GetBuffer(); + sendBytes += buffer.size(); + TBlobCookie cookie(item.GetCookie()); + SendVPuts.push_back({vdisk, blobId, cookie.GetBlobIdx()}); + } + UNIT_ASSERT(sendBytes <= MaxBatchedPutSize); + + TEvBlobStorage::TEvVMultiPutResult vMultiPutResult; vMultiPutResult.MakeError(NKikimrProto::OK, TString(), multiPutRequest); - + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> nextVPuts; - getImpl.OnVPutResult(logCtx, vMultiPutResult, - nextVGets, nextVPuts, getResult); - VMultiPutResponses++; - RequestIndex += nextVGets.size(); - VMultiPutRequests += nextVPuts.size(); - AssertCounters(getImpl); - + getImpl.OnVPutResult(logCtx, vMultiPutResult, + nextVGets, nextVPuts, getResult); + VMultiPutResponses++; + RequestIndex += nextVGets.size(); + VMultiPutRequests += nextVPuts.size(); + AssertCounters(getImpl); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - if (getResult) { - break; - } - } - vPuts.clear(); - } - - template <ETVPutEventKind TVPutEventKind> - void SubStep(ui64 queryCount, ui64 blobCount) { - using TVPutEvent = std::conditional_t<TVPutEventKind == TVPEK_VPUT, TEvBlobStorage::TEvVPut, - TEvBlobStorage::TEvVMultiPut>; - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( - new TEvBlobStorage::TEvGet::TQuery[MaxQueryCount]); - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( - new TEvBlobStorage::TEvGet::TQuery[MaxQueryCount]); - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; - q.Id = BlobSet->Get(queryIdx % blobCount).Id; - q.Shift = (queryIdx * 177) % q.Id.BlobSize(); - q.Size = Min((ui64)70, (ui64)q.Id.BlobSize() - (ui64)q.Shift); - queriesB[queryIdx] = q; - } - TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead, IsRestore, false); - ev.IsVerboseNoDataEnabled = IsVerboseNoDataEnabled; + if (getResult) { + break; + } + } + vPuts.clear(); + } + + template <ETVPutEventKind TVPutEventKind> + void SubStep(ui64 queryCount, ui64 blobCount) { + using TVPutEvent = std::conditional_t<TVPutEventKind == TVPEK_VPUT, TEvBlobStorage::TEvVPut, + TEvBlobStorage::TEvVMultiPut>; + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( + new TEvBlobStorage::TEvGet::TQuery[MaxQueryCount]); + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( + new TEvBlobStorage::TEvGet::TQuery[MaxQueryCount]); + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; + q.Id = BlobSet->Get(queryIdx % blobCount).Id; + q.Shift = (queryIdx * 177) % q.Id.BlobSize(); + q.Size = Min((ui64)70, (ui64)q.Id.BlobSize() - (ui64)q.Shift); + queriesB[queryIdx] = q; + } + TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead, IsRestore, false); + ev.IsVerboseNoDataEnabled = IsVerboseNoDataEnabled; TGetImpl getImpl(Group->GetInfo(), GroupQueues, &ev, nullptr); - ClearCounters(); + ClearCounters(); TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; TDeque<std::unique_ptr<TVPutEvent>> vPuts; - TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); - logCtx.LogAcc.IsLogEnabled = false; - getImpl.GenerateInitialRequests(logCtx, vGets); - RequestIndex += vGets.size(); - AssertCounters(getImpl); - - TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; - for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { - - bool isLast = (vGetIdx == vGets.size() - 1); - auto &request = vGets[vGetIdx]->Record; - Y_VERIFY(request.HasCookie()); - //ui64 messageCookie = request->Record.GetCookie(); - TEvBlobStorage::TEvVGetResult vGetResult; - Group->OnVGet(*vGets[vGetIdx], vGetResult); - - // TODO: generate result + TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); + logCtx.LogAcc.IsLogEnabled = false; + getImpl.GenerateInitialRequests(logCtx, vGets); + RequestIndex += vGets.size(); + AssertCounters(getImpl); + + TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; + for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { + + bool isLast = (vGetIdx == vGets.size() - 1); + auto &request = vGets[vGetIdx]->Record; + Y_VERIFY(request.HasCookie()); + //ui64 messageCookie = request->Record.GetCookie(); + TEvBlobStorage::TEvVGetResult vGetResult; + Group->OnVGet(*vGets[vGetIdx], vGetResult); + + // TODO: generate result TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TVPutEvent>> nextVPuts; - getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); - ResponseIndex++; - RequestIndex += nextVGets.size(); - if constexpr (TVPutEventKind == TVPEK_VPUT) { - VPutRequests += nextVPuts.size(); - } else { - VMultiPutRequests += nextVPuts.size(); - } - AssertCounters(getImpl); - + getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); + ResponseIndex++; + RequestIndex += nextVGets.size(); + if constexpr (TVPutEventKind == TVPEK_VPUT) { + VPutRequests += nextVPuts.size(); + } else { + VMultiPutRequests += nextVPuts.size(); + } + AssertCounters(getImpl); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); - if (IsRestore) { + if (IsRestore) { std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - } else { - UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); - } - if (getResult) { - break; - } - ProcessVPuts(logCtx, getImpl, vGets, vPuts, getResult); - if (getResult) { - break; - } - if (!isLast) { - UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); - } - } - UNIT_ASSERT(getResult); - UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); - // TODO: check the data vs queriesB - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; - TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; - UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); - UNIT_ASSERT_VALUES_EQUAL(a.Status, NKikimrProto::OK); - UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); - UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); - BlobSet->Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); - } - } -}; - + } else { + UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); + } + if (getResult) { + break; + } + ProcessVPuts(logCtx, getImpl, vGets, vPuts, getResult); + if (getResult) { + break; + } + if (!isLast) { + UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); + } + } + UNIT_ASSERT(getResult); + UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); + // TODO: check the data vs queriesB + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; + TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; + UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); + UNIT_ASSERT_VALUES_EQUAL(a.Status, NKikimrProto::OK); + UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); + UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); + BlobSet->Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); + } + } +}; + void TestIntervalsWipedAllOk(TErasureType::EErasureSpecies erasureSpecies, bool isVerboseNoDataEnabled = false) { TActorSystemStub actorSystemStub; @@ -463,113 +463,113 @@ void TestIntervalsWipedAllOk(TErasureType::EErasureSpecies erasureSpecies, bool TBlobStorageGroupType groupType(erasureSpecies); const ui32 domainCount = groupType.BlobSubgroupSize(); - TVector<ui64> queryCounts = {1, 2, 3, 13, 34}; + TVector<ui64> queryCounts = {1, 2, 3, 13, 34}; - for (bool isRestore : {false, true}) { - for (ui32 generateMode = 0; generateMode < 2; ++generateMode) { + for (bool isRestore : {false, true}) { + for (ui32 generateMode = 0; generateMode < 2; ++generateMode) { for (ui64 wiped1 = 0; wiped1 < domainCount; ++wiped1) { for (ui64 wiped2 = 0; wiped2 <= wiped1; ++wiped2) { ui64 maxErrorMask = (wiped1 == wiped2 ? 4 : 24); for (ui64 errorMask = 0; errorMask <= maxErrorMask; ++errorMask) { ui64 error1 = errorMask % 5; ui64 error2 = errorMask / 5; - TTestWipedAllOkStep testStep( - groupId, erasureSpecies, domainCount, queryCounts, - isVerboseNoDataEnabled, isRestore); - testStep.SetGenerateBlobsMode(generateMode); - testStep.Init(); - testStep.AddWipedVDisk(wiped1, error1); - testStep.AddWipedVDisk(wiped2, error2); - testStep.Run(false); - } - } - } - } - } -} - -void TestIntervalsWipedAllOkVMultiPut(TErasureType::EErasureSpecies erasureSpecies, bool isVerboseNoDataEnabled = false) { - TActorSystemStub actorSystemStub; - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); + TTestWipedAllOkStep testStep( + groupId, erasureSpecies, domainCount, queryCounts, + isVerboseNoDataEnabled, isRestore); + testStep.SetGenerateBlobsMode(generateMode); + testStep.Init(); + testStep.AddWipedVDisk(wiped1, error1); + testStep.AddWipedVDisk(wiped2, error2); + testStep.Run(false); + } + } + } + } + } +} + +void TestIntervalsWipedAllOkVMultiPut(TErasureType::EErasureSpecies erasureSpecies, bool isVerboseNoDataEnabled = false) { + TActorSystemStub actorSystemStub; + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); const ui32 domainCount = groupType.BlobSubgroupSize(); - - TVector<ui64> queryCounts = {1, 2, 3, 13, 34, 55}; - - for (bool isRestore : {false, true}) { - for (ui32 generateMode = 0; generateMode < 2; ++generateMode) { - for (ui64 wiped1 = 0; wiped1 < domainCount; ++wiped1) { - for (ui64 wiped2 = 0; wiped2 <= wiped1; ++wiped2) { - ui64 maxErrorMask = (wiped1 == wiped2 ? 4 : 24); - for (ui64 errorMask = 0; errorMask <= maxErrorMask; ++errorMask) { - ui64 error1 = errorMask % 5; - ui64 error2 = errorMask / 5; - TTestWipedAllOkStep testStep( - groupId, erasureSpecies, domainCount, queryCounts, - isVerboseNoDataEnabled, isRestore); - testStep.SetGenerateBlobsMode(generateMode); - testStep.Init(); - testStep.AddWipedVDisk(wiped1, error1); - testStep.AddWipedVDisk(wiped2, error2); - testStep.Run(true); - } - } - } - } - } -} - -void TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::EErasureSpecies erasureSpecies, - bool isVerboseNoDataEnabled = false) { - TActorSystemStub actorSystemStub; - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); + + TVector<ui64> queryCounts = {1, 2, 3, 13, 34, 55}; + + for (bool isRestore : {false, true}) { + for (ui32 generateMode = 0; generateMode < 2; ++generateMode) { + for (ui64 wiped1 = 0; wiped1 < domainCount; ++wiped1) { + for (ui64 wiped2 = 0; wiped2 <= wiped1; ++wiped2) { + ui64 maxErrorMask = (wiped1 == wiped2 ? 4 : 24); + for (ui64 errorMask = 0; errorMask <= maxErrorMask; ++errorMask) { + ui64 error1 = errorMask % 5; + ui64 error2 = errorMask / 5; + TTestWipedAllOkStep testStep( + groupId, erasureSpecies, domainCount, queryCounts, + isVerboseNoDataEnabled, isRestore); + testStep.SetGenerateBlobsMode(generateMode); + testStep.Init(); + testStep.AddWipedVDisk(wiped1, error1); + testStep.AddWipedVDisk(wiped2, error2); + testStep.Run(true); + } + } + } + } + } +} + +void TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::EErasureSpecies erasureSpecies, + bool isVerboseNoDataEnabled = false) { + TActorSystemStub actorSystemStub; + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); const ui32 domainCount = groupType.BlobSubgroupSize(); - const TVector<ui64> queryCounts = {1, 2}; - - TVector<TLogoBlobID> blobIDs = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIDs) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - blobs.emplace_back(id, builder); - } - - for (ui64 wiped1 = 0; wiped1 < domainCount; ++wiped1) { - for (ui64 wiped2 = 0; wiped2 <= wiped1; ++wiped2) { - ui64 maxErrorMask = (wiped1 == wiped2 ? 4 : 24); - for (ui64 errorMask = 0; errorMask <= maxErrorMask; ++errorMask) { - ui64 error1 = errorMask % 5; - ui64 error2 = errorMask / 5; - TTestWipedAllOkStep testStep( - groupId, erasureSpecies, domainCount, queryCounts, - isVerboseNoDataEnabled, true); - testStep.SetBlobs(blobs); - - testStep.Init(); - testStep.AddWipedVDisk(wiped1, error1); - testStep.AddWipedVDisk(wiped2, error2); - testStep.Run(false); - auto sendVPuts = std::move(testStep.SendVPuts); - - testStep.Init(); - testStep.AddWipedVDisk(wiped1, error1); - testStep.AddWipedVDisk(wiped2, error2); - testStep.Run(true); - auto sendVMultiPuts = std::move(testStep.SendVPuts); - - Sort(sendVPuts.begin(), sendVPuts.end()); - Sort(sendVMultiPuts.begin(), sendVMultiPuts.end()); - UNIT_ASSERT(sendVPuts == sendVMultiPuts); + const TVector<ui64> queryCounts = {1, 2}; + + TVector<TLogoBlobID> blobIDs = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIDs) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + blobs.emplace_back(id, builder); + } + + for (ui64 wiped1 = 0; wiped1 < domainCount; ++wiped1) { + for (ui64 wiped2 = 0; wiped2 <= wiped1; ++wiped2) { + ui64 maxErrorMask = (wiped1 == wiped2 ? 4 : 24); + for (ui64 errorMask = 0; errorMask <= maxErrorMask; ++errorMask) { + ui64 error1 = errorMask % 5; + ui64 error2 = errorMask / 5; + TTestWipedAllOkStep testStep( + groupId, erasureSpecies, domainCount, queryCounts, + isVerboseNoDataEnabled, true); + testStep.SetBlobs(blobs); + + testStep.Init(); + testStep.AddWipedVDisk(wiped1, error1); + testStep.AddWipedVDisk(wiped2, error2); + testStep.Run(false); + auto sendVPuts = std::move(testStep.SendVPuts); + + testStep.Init(); + testStep.AddWipedVDisk(wiped1, error1); + testStep.AddWipedVDisk(wiped2, error2); + testStep.Run(true); + auto sendVMultiPuts = std::move(testStep.SendVPuts); + + Sort(sendVPuts.begin(), sendVPuts.end()); + Sort(sendVMultiPuts.begin(), sendVMultiPuts.end()); + UNIT_ASSERT(sendVPuts == sendVMultiPuts); } } } @@ -592,23 +592,23 @@ public: Group.PutBlobSet(BlobSet); } - void SetBlobSet(const TVector<TBlobTestSet::TBlob>& blobs) { - BlobSet.AddBlobs(blobs); - Group.PutBlobSet(BlobSet); - } - + void SetBlobSet(const TVector<TBlobTestSet::TBlob>& blobs) { + BlobSet.AddBlobs(blobs); + Group.PutBlobSet(BlobSet); + } + void SetError(ui32 domainIdx, NKikimrProto::EReplyStatus status) { Group.SetError(domainIdx, status); } - void SetPredictedDelayNs(ui32 domainIdx, ui64 predictedDelayNs) { - Group.SetPredictedDelayNs(domainIdx, predictedDelayNs); - } - + void SetPredictedDelayNs(ui32 domainIdx, ui64 predictedDelayNs) { + Group.SetPredictedDelayNs(domainIdx, predictedDelayNs); + } + TIntrusivePtr<TGroupQueues> GetSessionsState() const { return GroupQueues; - } - + } + TAutoPtr<TEvBlobStorage::TEvGetResult> Simulate(TEvBlobStorage::TEvGet *ev) { TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; @@ -669,291 +669,291 @@ public: }; -Y_UNIT_TEST(TestBlock42VGetCountWithErasure) { - bool isVerboseNoDataEnabled = false; - TErasureType::EErasureSpecies erasureSpecies = TErasureType::Erasure4Plus2Block; - TActorSystemStub actorSystemStub; - - TVector<TLogoBlobID> blobIDs = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - }; - - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIDs) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - blobs.emplace_back(id, builder); - } - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); - const ui32 domainCount = groupType.BlobSubgroupSize(); - - const ui64 queryCount = 1; - - const ui32 isRestore = 0; - - TGroupMock group(groupId, erasureSpecies, domainCount, 1); +Y_UNIT_TEST(TestBlock42VGetCountWithErasure) { + bool isVerboseNoDataEnabled = false; + TErasureType::EErasureSpecies erasureSpecies = TErasureType::Erasure4Plus2Block; + TActorSystemStub actorSystemStub; + + TVector<TLogoBlobID> blobIDs = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + }; + + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIDs) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + blobs.emplace_back(id, builder); + } + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); + const ui32 domainCount = groupType.BlobSubgroupSize(); + + const ui64 queryCount = 1; + + const ui32 isRestore = 0; + + TGroupMock group(groupId, erasureSpecies, domainCount, 1); TIntrusivePtr<TGroupQueues> groupQueues = group.MakeGroupQueues(); - TBlobTestSet blobSet; - blobSet.AddBlobs(blobs); - group.PutBlobSet(blobSet); - - for (ui32 idx = 0; idx < domainCount; ++idx) { - group.SetPredictedDelayNs(idx, 1); - } - group.SetPredictedDelayNs(7, 10); - - group.SetNotYetBlob(0, TLogoBlobID(blobIDs[0], 1)); - - ui64 blobCount = queryCount; - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; - q.Id = blobSet.Get(queryIdx % blobCount).Id; - q.Shift = 0; - q.Size = (ui64)q.Id.BlobSize(); - queriesB[queryIdx] = q; - } - TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); - ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; - - TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; - + TBlobTestSet blobSet; + blobSet.AddBlobs(blobs); + group.PutBlobSet(blobSet); + + for (ui32 idx = 0; idx < domainCount; ++idx) { + group.SetPredictedDelayNs(idx, 1); + } + group.SetPredictedDelayNs(7, 10); + + group.SetNotYetBlob(0, TLogoBlobID(blobIDs[0], 1)); + + ui64 blobCount = queryCount; + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; + q.Id = blobSet.Get(queryIdx % blobCount).Id; + q.Shift = 0; + q.Size = (ui64)q.Id.BlobSize(); + queriesB[queryIdx] = q; + } + TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); + ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; + + TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; + TGetImpl getImpl(group.GetInfo(), groupQueues, &ev, nullptr); TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); - logCtx.LogAcc.IsLogEnabled = false; - getImpl.GenerateInitialRequests(logCtx, vGets); - - for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { - if (vGetIdx == 3) { + TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); + logCtx.LogAcc.IsLogEnabled = false; + getImpl.GenerateInitialRequests(logCtx, vGets); + + for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { + if (vGetIdx == 3) { vGets.push_back(std::move(vGets[3])); - continue; - } - bool isLast = (vGetIdx == vGets.size() - 1); - auto &request = vGets[vGetIdx]->Record; - - Y_VERIFY(request.HasCookie()); - TEvBlobStorage::TEvVGetResult vGetResult; - group.OnVGet(*vGets[vGetIdx], vGetResult); - - // TODO: generate result + continue; + } + bool isLast = (vGetIdx == vGets.size() - 1); + auto &request = vGets[vGetIdx]->Record; + + Y_VERIFY(request.HasCookie()); + TEvBlobStorage::TEvVGetResult vGetResult; + group.OnVGet(*vGets[vGetIdx], vGetResult); + + // TODO: generate result TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - - getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); - + + getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); - if (ev.MustRestoreFirst) { + if (ev.MustRestoreFirst) { std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - } else { - UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); - } - if (getResult) { - break; - } - for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - auto &putRequest = vPuts[vPutIdx]->Record; - Y_VERIFY(putRequest.HasCookie()); - TEvBlobStorage::TEvVPutResult vPutResult; + } else { + UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); + } + if (getResult) { + break; + } + for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + auto &putRequest = vPuts[vPutIdx]->Record; + Y_VERIFY(putRequest.HasCookie()); + TEvBlobStorage::TEvVPutResult vPutResult; vPutResult.MakeError(NKikimrProto::OK, TString(), putRequest); - + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); + getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - if (getResult) { - break; - } - } - vPuts.clear(); - if (getResult) { - break; - } - if (!isLast) { - UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); - } - } - + if (getResult) { + break; + } + } + vPuts.clear(); + if (getResult) { + break; + } + if (!isLast) { + UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); + } + } + UNIT_ASSERT_VALUES_EQUAL(vGets.size(), 8); - - UNIT_ASSERT(getResult); - UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); - // TODO: check the data vs queriesB - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; - TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; - UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); - if (a.Status != NKikimrProto::ERROR) { - if (a.Status == NKikimrProto::OK) { - UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); - UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); - blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); - } else { - TStringStream str; - str << " isRestore# " << isRestore - << " status# " << a.Status; - UNIT_ASSERT_C(false, str.Str()); - } - } - } // for queryIdx - - return; -} - -Y_UNIT_TEST(TestBlock42WipedOneDiskAndErrorDurringGet) { - bool isVerboseNoDataEnabled = false; - TErasureType::EErasureSpecies erasureSpecies = TErasureType::Erasure4Plus2Block; - TActorSystemStub actorSystemStub; - - TVector<TLogoBlobID> blobIDs = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - // TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIDs) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - - blobs.emplace_back(id, builder); - } - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); - const ui32 domainCount = groupType.BlobSubgroupSize(); - - const ui64 queryCount = 1; - - const ui32 isRestore = 0; - - TGroupMock group(groupId, erasureSpecies, domainCount, 1); + + UNIT_ASSERT(getResult); + UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); + // TODO: check the data vs queriesB + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; + TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; + UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); + if (a.Status != NKikimrProto::ERROR) { + if (a.Status == NKikimrProto::OK) { + UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); + UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); + blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); + } else { + TStringStream str; + str << " isRestore# " << isRestore + << " status# " << a.Status; + UNIT_ASSERT_C(false, str.Str()); + } + } + } // for queryIdx + + return; +} + +Y_UNIT_TEST(TestBlock42WipedOneDiskAndErrorDurringGet) { + bool isVerboseNoDataEnabled = false; + TErasureType::EErasureSpecies erasureSpecies = TErasureType::Erasure4Plus2Block; + TActorSystemStub actorSystemStub; + + TVector<TLogoBlobID> blobIDs = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + // TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIDs) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + + blobs.emplace_back(id, builder); + } + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); + const ui32 domainCount = groupType.BlobSubgroupSize(); + + const ui64 queryCount = 1; + + const ui32 isRestore = 0; + + TGroupMock group(groupId, erasureSpecies, domainCount, 1); TIntrusivePtr<TGroupQueues> groupQueues = group.MakeGroupQueues(); - TBlobTestSet blobSet; - blobSet.AddBlobs(blobs); - group.PutBlobSet(blobSet); - - for (ui32 idx = 0; idx < domainCount; ++idx) { - group.SetPredictedDelayNs(idx, 1); - } - group.SetPredictedDelayNs(7, 10); - - group.SetNotYetBlob(0, TLogoBlobID(blobIDs[0], 1)); - - ui64 blobCount = queryCount; - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; - q.Id = blobSet.Get(queryIdx % blobCount).Id; - q.Shift = 0; - q.Size = (ui64)q.Id.BlobSize(); - queriesB[queryIdx] = q; - } - TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); - ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; - - TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; - + TBlobTestSet blobSet; + blobSet.AddBlobs(blobs); + group.PutBlobSet(blobSet); + + for (ui32 idx = 0; idx < domainCount; ++idx) { + group.SetPredictedDelayNs(idx, 1); + } + group.SetPredictedDelayNs(7, 10); + + group.SetNotYetBlob(0, TLogoBlobID(blobIDs[0], 1)); + + ui64 blobCount = queryCount; + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; + q.Id = blobSet.Get(queryIdx % blobCount).Id; + q.Shift = 0; + q.Size = (ui64)q.Id.BlobSize(); + queriesB[queryIdx] = q; + } + TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); + ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; + + TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; + TGetImpl getImpl(group.GetInfo(), groupQueues, &ev, nullptr); TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); - logCtx.LogAcc.IsLogEnabled = false; - getImpl.GenerateInitialRequests(logCtx, vGets); - - for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { - if (vGetIdx == 3) { + TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); + logCtx.LogAcc.IsLogEnabled = false; + getImpl.GenerateInitialRequests(logCtx, vGets); + + for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { + if (vGetIdx == 3) { vGets.push_back(std::move(vGets[3])); - continue; - } - if (vGetIdx == 5) { - group.SetError(2, NKikimrProto::ERROR); - group.SetPredictedDelayNs(7, 1); - } - bool isLast = (vGetIdx == vGets.size() - 1); - auto &request = vGets[vGetIdx]->Record; - - Y_VERIFY(request.HasCookie()); - TEvBlobStorage::TEvVGetResult vGetResult; - group.OnVGet(*vGets[vGetIdx], vGetResult); - + continue; + } + if (vGetIdx == 5) { + group.SetError(2, NKikimrProto::ERROR); + group.SetPredictedDelayNs(7, 1); + } + bool isLast = (vGetIdx == vGets.size() - 1); + auto &request = vGets[vGetIdx]->Record; + + Y_VERIFY(request.HasCookie()); + TEvBlobStorage::TEvVGetResult vGetResult; + group.OnVGet(*vGets[vGetIdx], vGetResult); + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - - getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); - + + getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); - if (ev.MustRestoreFirst) { + if (ev.MustRestoreFirst) { std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - } else { - UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); - } - if (getResult) { - break; - } - for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - auto &putRequest = vPuts[vPutIdx]->Record; - Y_VERIFY(putRequest.HasCookie()); - TEvBlobStorage::TEvVPutResult vPutResult; + } else { + UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); + } + if (getResult) { + break; + } + for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + auto &putRequest = vPuts[vPutIdx]->Record; + Y_VERIFY(putRequest.HasCookie()); + TEvBlobStorage::TEvVPutResult vPutResult; vPutResult.MakeError(NKikimrProto::OK, TString(), putRequest); - + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); + getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - if (getResult) { - break; - } - } - vPuts.clear(); - if (getResult) { - break; - } - if (!isLast) { - UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); - } - } - - UNIT_ASSERT(getResult); - UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); - // TODO: check the data vs queriesB - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; - TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; - UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); - if (a.Status != NKikimrProto::ERROR) { - if (a.Status == NKikimrProto::OK) { - UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); - UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); - blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); - } else { - TStringStream str; - str << " isRestore# " << isRestore - << " status# " << a.Status; - UNIT_ASSERT_C(false, str.Str()); - } - } - } // for queryIdx - - return; -} - + if (getResult) { + break; + } + } + vPuts.clear(); + if (getResult) { + break; + } + if (!isLast) { + UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); + } + } + + UNIT_ASSERT(getResult); + UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); + // TODO: check the data vs queriesB + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; + TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; + UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); + if (a.Status != NKikimrProto::ERROR) { + if (a.Status == NKikimrProto::OK) { + UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); + UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); + blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); + } else { + TStringStream str; + str << " isRestore# " << isRestore + << " status# " << a.Status; + UNIT_ASSERT_C(false, str.Str()); + } + } + } // for queryIdx + + return; +} + void ApplyError(TGetSimulator &simulator, ui64 idx, ui64 error) { switch (error) { case 0: @@ -1065,173 +1065,173 @@ void TestIntervalsWipedError(TErasureType::EErasureSpecies erasureSpecies, bool return; } -void TestWipedErrorWithTwoBlobs(TErasureType::EErasureSpecies erasureSpecies, bool isVerboseNoDataEnabled = false) { - TActorSystemStub actorSystemStub; - - TVector<TLogoBlobID> blobIDs = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIDs) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - - blobs.emplace_back(id, builder); - } - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); - const ui32 domainCount = groupType.BlobSubgroupSize(); - - const ui64 queryCount = 1; - const ui32 isRestore = 0; - ui64 seed = 0; - - for (i32 slowDisk = -1; slowDisk < (i32)domainCount; ++slowDisk) { - for (i32 wipedDisk = -1; wipedDisk < (i32) domainCount; ++wipedDisk) { - if (wipedDisk == slowDisk && slowDisk != -1) { - continue; - } - i32 endMayErrorSendIteration = domainCount; - for (i32 errorIteration = -1; errorIteration < endMayErrorSendIteration; ++errorIteration) { - for (ui32 errorDisk = 0; errorDisk < domainCount; ++errorDisk) { - if (errorIteration == -1 && errorDisk > 0) { - break; - } - if (errorIteration != -1 && (errorDisk == (ui32)slowDisk || errorDisk == (ui32)wipedDisk)) { - continue; - } - - for (ui64 it = 0; it < 100; ++it, ++seed) { - SetRandomSeed(seed); - TGroupMock group(groupId, erasureSpecies, domainCount, 1); +void TestWipedErrorWithTwoBlobs(TErasureType::EErasureSpecies erasureSpecies, bool isVerboseNoDataEnabled = false) { + TActorSystemStub actorSystemStub; + + TVector<TLogoBlobID> blobIDs = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIDs) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + + blobs.emplace_back(id, builder); + } + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); + const ui32 domainCount = groupType.BlobSubgroupSize(); + + const ui64 queryCount = 1; + const ui32 isRestore = 0; + ui64 seed = 0; + + for (i32 slowDisk = -1; slowDisk < (i32)domainCount; ++slowDisk) { + for (i32 wipedDisk = -1; wipedDisk < (i32) domainCount; ++wipedDisk) { + if (wipedDisk == slowDisk && slowDisk != -1) { + continue; + } + i32 endMayErrorSendIteration = domainCount; + for (i32 errorIteration = -1; errorIteration < endMayErrorSendIteration; ++errorIteration) { + for (ui32 errorDisk = 0; errorDisk < domainCount; ++errorDisk) { + if (errorIteration == -1 && errorDisk > 0) { + break; + } + if (errorIteration != -1 && (errorDisk == (ui32)slowDisk || errorDisk == (ui32)wipedDisk)) { + continue; + } + + for (ui64 it = 0; it < 100; ++it, ++seed) { + SetRandomSeed(seed); + TGroupMock group(groupId, erasureSpecies, domainCount, 1); TIntrusivePtr<TGroupQueues> groupQueues = group.MakeGroupQueues(); - TBlobTestSet blobSet; - blobSet.AddBlobs(blobs); - group.PutBlobSet(blobSet); - - for (ui32 idx = 0; idx < domainCount; ++idx) { - group.SetPredictedDelayNs(idx, 1); - } - if (slowDisk != -1) { - group.SetPredictedDelayNs(slowDisk, 10); - } - - if (wipedDisk != -1) { - group.SetError(wipedDisk, NKikimrProto::NOT_YET); - } - - ui64 blobCount = queryCount; - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( - new TEvBlobStorage::TEvGet::TQuery[queryCount]); - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; - q.Id = blobSet.Get(queryIdx % blobCount).Id; - q.Shift = 0; - q.Size = (ui64)q.Id.BlobSize(); - queriesB[queryIdx] = q; - } - TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); - ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; - - TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; - + TBlobTestSet blobSet; + blobSet.AddBlobs(blobs); + group.PutBlobSet(blobSet); + + for (ui32 idx = 0; idx < domainCount; ++idx) { + group.SetPredictedDelayNs(idx, 1); + } + if (slowDisk != -1) { + group.SetPredictedDelayNs(slowDisk, 10); + } + + if (wipedDisk != -1) { + group.SetError(wipedDisk, NKikimrProto::NOT_YET); + } + + ui64 blobCount = queryCount; + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesA( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queriesB( + new TEvBlobStorage::TEvGet::TQuery[queryCount]); + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGet::TQuery &q = queriesA[queryIdx]; + q.Id = blobSet.Get(queryIdx % blobCount).Id; + q.Shift = 0; + q.Size = (ui64)q.Id.BlobSize(); + queriesB[queryIdx] = q; + } + TEvBlobStorage::TEvGet ev(queriesA, queryCount, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead, isRestore, false); + ev.IsVerboseNoDataEnabled = isVerboseNoDataEnabled; + + TAutoPtr<TEvBlobStorage::TEvGetResult> getResult; + TGetImpl getImpl(group.GetInfo(), groupQueues, &ev, nullptr); TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> vGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); - logCtx.LogAcc.IsLogEnabled = false; - getImpl.GenerateInitialRequests(logCtx, vGets); - - for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { - ui32 needIdx = RandomNumber<ui32>(vGets.size() - vGetIdx); - std::swap(vGets[vGetIdx], vGets[vGetIdx + needIdx]); - - bool isLast = (vGetIdx == vGets.size() - 1); - auto &request = vGets[vGetIdx]->Record; - - if ((ui32)errorIteration == vGetIdx) { - group.SetError(errorDisk, NKikimrProto::ERROR); - } - - Y_VERIFY(request.HasCookie()); - TEvBlobStorage::TEvVGetResult vGetResult; - group.OnVGet(*vGets[vGetIdx], vGetResult); - - // TODO: generate result + TLogContext logCtx(NKikimrServices::BS_PROXY_GET, false); + logCtx.LogAcc.IsLogEnabled = false; + getImpl.GenerateInitialRequests(logCtx, vGets); + + for (ui64 vGetIdx = 0; vGetIdx < vGets.size(); ++vGetIdx) { + ui32 needIdx = RandomNumber<ui32>(vGets.size() - vGetIdx); + std::swap(vGets[vGetIdx], vGets[vGetIdx + needIdx]); + + bool isLast = (vGetIdx == vGets.size() - 1); + auto &request = vGets[vGetIdx]->Record; + + if ((ui32)errorIteration == vGetIdx) { + group.SetError(errorDisk, NKikimrProto::ERROR); + } + + Y_VERIFY(request.HasCookie()); + TEvBlobStorage::TEvVGetResult vGetResult; + group.OnVGet(*vGets[vGetIdx], vGetResult); + + // TODO: generate result TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - - getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); - + + getImpl.OnVGetResult(logCtx, vGetResult, nextVGets, nextVPuts, getResult); + std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); - if (ev.MustRestoreFirst) { + if (ev.MustRestoreFirst) { std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - } else { - UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); - } - if (getResult) { - break; - } - for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - auto &putRequest = vPuts[vPutIdx]->Record; - Y_VERIFY(putRequest.HasCookie()); - TEvBlobStorage::TEvVPutResult vPutResult; + } else { + UNIT_ASSERT_VALUES_EQUAL(nextVPuts.size(), 0); + } + if (getResult) { + break; + } + for (ui64 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + auto &putRequest = vPuts[vPutIdx]->Record; + Y_VERIFY(putRequest.HasCookie()); + TEvBlobStorage::TEvVPutResult vPutResult; vPutResult.MakeError(NKikimrProto::OK, TString(), putRequest); - + TDeque<std::unique_ptr<TEvBlobStorage::TEvVGet>> nextVGets; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); + getImpl.OnVPutResult(logCtx, vPutResult, nextVGets, nextVPuts, getResult); std::move(nextVGets.begin(), nextVGets.end(), std::back_inserter(vGets)); std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - if (getResult) { - break; - } - } - vPuts.clear(); - if (getResult) { - break; - } - if (!isLast) { - UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); - } - } - - UNIT_ASSERT(getResult); - UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); - // TODO: check the data vs queriesB - for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { - TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; - TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; - UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); - if (a.Status != NKikimrProto::ERROR) { - if (a.Status == NKikimrProto::OK) { - UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); - UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); - blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); - } else { - TStringStream str; - str << " isRestore# " << isRestore - << " status# " << a.Status; - UNIT_ASSERT_C(false, str.Str()); - } - } - } // for queryIdx - } - } - } - } - } - return; -} - + if (getResult) { + break; + } + } + vPuts.clear(); + if (getResult) { + break; + } + if (!isLast) { + UNIT_ASSERT_VALUES_EQUAL(getResult, nullptr); + } + } + + UNIT_ASSERT(getResult); + UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, queryCount); + // TODO: check the data vs queriesB + for (ui64 queryIdx = 0; queryIdx < queryCount; ++queryIdx) { + TEvBlobStorage::TEvGetResult::TResponse &a = getResult->Responses[queryIdx]; + TEvBlobStorage::TEvGet::TQuery &q = queriesB[queryIdx]; + UNIT_ASSERT_VALUES_EQUAL(q.Id, a.Id); + if (a.Status != NKikimrProto::ERROR) { + if (a.Status == NKikimrProto::OK) { + UNIT_ASSERT_VALUES_EQUAL(q.Shift, a.Shift); + UNIT_ASSERT_VALUES_EQUAL(q.Size, a.RequestedSize); + blobSet.Check(queryIdx % blobCount, q.Id, q.Shift, q.Size, a.Buffer); + } else { + TStringStream str; + str << " isRestore# " << isRestore + << " status# " << a.Status; + UNIT_ASSERT_C(false, str.Str()); + } + } + } // for queryIdx + } + } + } + } + } + return; +} + Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOk) { TestIntervalsWipedAllOk(TErasureType::Erasure4Plus2Block); } @@ -1240,42 +1240,42 @@ Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVerbose) { TestIntervalsWipedAllOk(TErasureType::Erasure4Plus2Block, true); } -Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVMultiPut) { - TestIntervalsWipedAllOkVMultiPut(TErasureType::Erasure4Plus2Block); -} - -Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVerboseVMultiPut) { - TestIntervalsWipedAllOkVMultiPut(TErasureType::Erasure4Plus2Block, true); -} - -Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkComparisonVMultiPutAndVPut) { - TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::Erasure4Plus2Block); -} - -Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVerboseComparisonVMultiPutAndVPut) { - TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::Erasure4Plus2Block, true); -} - +Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVMultiPut) { + TestIntervalsWipedAllOkVMultiPut(TErasureType::Erasure4Plus2Block); +} + +Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVerboseVMultiPut) { + TestIntervalsWipedAllOkVMultiPut(TErasureType::Erasure4Plus2Block, true); +} + +Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkComparisonVMultiPutAndVPut) { + TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::Erasure4Plus2Block); +} + +Y_UNIT_TEST(TestBlock42GetIntervalsWipedAllOkVerboseComparisonVMultiPutAndVPut) { + TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::Erasure4Plus2Block, true); +} + Y_UNIT_TEST(TestBlock42GetIntervalsWipedError) { TestIntervalsWipedError(TErasureType::Erasure4Plus2Block); } -Y_UNIT_TEST(TestBlock42WipedErrorWithTwoBlobs) { - TestWipedErrorWithTwoBlobs(TErasureType::Erasure4Plus2Block); -} - +Y_UNIT_TEST(TestBlock42WipedErrorWithTwoBlobs) { + TestWipedErrorWithTwoBlobs(TErasureType::Erasure4Plus2Block); +} + Y_UNIT_TEST(TestMirror32GetIntervalsWipedAllOk) { TestIntervalsWipedAllOk(TErasureType::ErasureMirror3Plus2); } -Y_UNIT_TEST(TestMirror32GetIntervalsWipedAllOkVMultiPut) { - TestIntervalsWipedAllOkVMultiPut(TErasureType::ErasureMirror3Plus2); -} - -Y_UNIT_TEST(TestMirror32GetIntervalsWipedAllOkComparisonVMultiPutAndVPut) { - TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::ErasureMirror3Plus2); -} +Y_UNIT_TEST(TestMirror32GetIntervalsWipedAllOkVMultiPut) { + TestIntervalsWipedAllOkVMultiPut(TErasureType::ErasureMirror3Plus2); +} +Y_UNIT_TEST(TestMirror32GetIntervalsWipedAllOkComparisonVMultiPutAndVPut) { + TestIntervalsWipedAllOkComparisonVMultiPutAndVPut(TErasureType::ErasureMirror3Plus2); +} + void SpecificTest(ui32 badA, ui32 badB, ui32 blobSize, TMap<i64, i64> sizeForOffset) { TActorSystemStub actorSystemStub; TErasureType::EErasureSpecies erasureSpecies = TErasureType::Erasure4Plus2Block; diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_patch_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_patch_ut.cpp index 74bd0ed7ee8..f9b4eed80de 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_patch_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_patch_ut.cpp @@ -1,825 +1,825 @@ -#include "defs.h" - -#include "dsproxy_env_mock_ut.h" -#include "dsproxy_test_state_ut.h" -#include "dsproxy_vdisk_mock_ut.h" - +#include "defs.h" + +#include "dsproxy_env_mock_ut.h" +#include "dsproxy_test_state_ut.h" +#include "dsproxy_vdisk_mock_ut.h" + #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h> #include <ydb/core/util/stlog.h> - -#include <cstring> - -namespace NKikimr { -namespace NDSProxyPatchTest { - -struct TVDiskPointer { - ui32 VDiskIdx; - - static TVDiskPointer GetVDiskIdx(ui32 vdiskIdx) { - return {vdiskIdx}; - } - - std::pair<ui32, ui32> GetIndecies(const TDSProxyEnv &env, ui64 hash) const { - TVDiskID vdisk = env.Info->GetVDiskId(VDiskIdx); - ui32 idxInSubgroup = env.Info->GetIdxInSubgroup(vdisk, hash); - return {VDiskIdx, idxInSubgroup}; - } -}; - -struct TTestArgs { - TLogoBlobID OriginalId; - TLogoBlobID PatchedId; - ui64 OriginalGroupId; - ui64 CurrentGroupId; - TString Buffer; - TVector<TDiff> Diffs; - TBlobStorageGroupType GType; - ui32 MaskForCookieBruteForcing; - - ui32 StatusFlags = 1; - float ApproximateFreeSpaceShare = 0.1; - TVector<TVector<ui64>> PartPlacement; - TVector<bool> ErrorVDisks; - - void MakeDefaultPartPlacement() { - for (TVector<ui64> &pl : PartPlacement) { - pl.clear(); - } - switch (GType.GetErasure()) { - case TErasureType::Erasure4Plus2Block: - for (ui64 idx = 0; idx < GType.TotalPartCount(); ++idx) { - PartPlacement[idx].push_back(idx + 1); - } - break; - case TErasureType::ErasureNone: - PartPlacement[0].push_back(1); - break; - case TErasureType::ErasureMirror3: - PartPlacement[0].push_back(1); - PartPlacement[1].push_back(1); - PartPlacement[2].push_back(1); - break; - case TErasureType::ErasureMirror3dc: - PartPlacement[0].push_back(1); - PartPlacement[3].push_back(2); - PartPlacement[6].push_back(3); - break; - default: - UNIT_ASSERT(false); // not implemented - } - } - - void ResetDefaultPlacement() { - ErrorVDisks.assign(GType.BlobSubgroupSize(), false); - PartPlacement.resize(GType.BlobSubgroupSize()); - MakeDefaultPartPlacement(); - } - - void MakeDefault(TBlobStorageGroupType type, bool sameGroupId = true, bool goodPatchedId = true) { - constexpr ui32 bufferSize = 1000; - OriginalId = TLogoBlobID(1, 2, 3, 4, bufferSize, 5); - PatchedId = TLogoBlobID(1, 3, 3, 4, bufferSize, 6); - OriginalGroupId = 0; - CurrentGroupId = OriginalGroupId + !sameGroupId; - Buffer = TString::Uninitialized(bufferSize); - for (char &c : Buffer) { - c = 'a'; - } - for (ui32 idx = 0; idx < bufferSize; idx += 2) { - Diffs.emplace_back("b", idx); - } - GType = type; - - ErrorVDisks.assign(type.BlobSubgroupSize(), false); - PartPlacement.resize(type.BlobSubgroupSize()); - MakeDefaultPartPlacement(); - - MaskForCookieBruteForcing = (Max<ui32>() << 17) & TLogoBlobID::MaxCookie; - if (goodPatchedId) { - UNIT_ASSERT(TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(OriginalId, &PatchedId, - MaskForCookieBruteForcing, OriginalGroupId, CurrentGroupId)); - } - } -}; - -enum class ENaivePatchCase { - Ok, - ErrorOnGetItem, - ErrorOnGet, - ErrorOnPut, -}; - -NKikimrProto::EReplyStatus GetPatchResultStatus(ENaivePatchCase naiveCase) { - switch (naiveCase) { - case ENaivePatchCase::Ok: - return NKikimrProto::OK; - case ENaivePatchCase::ErrorOnGetItem: - case ENaivePatchCase::ErrorOnGet: - case ENaivePatchCase::ErrorOnPut: - return NKikimrProto::ERROR; - } -} - -NKikimrProto::EReplyStatus GetGetResultStatus(ENaivePatchCase naiveCase) { - switch (naiveCase) { - case ENaivePatchCase::Ok: - return NKikimrProto::OK; - case ENaivePatchCase::ErrorOnGetItem: - return NKikimrProto::OK; - case ENaivePatchCase::ErrorOnGet: - return NKikimrProto::ERROR; - case ENaivePatchCase::ErrorOnPut: - return NKikimrProto::OK; - } -} - -NKikimrProto::EReplyStatus GetPutResultStatus(ENaivePatchCase naiveCase) { - switch (naiveCase) { - case ENaivePatchCase::Ok: - return NKikimrProto::OK; - case ENaivePatchCase::ErrorOnGetItem: - return NKikimrProto::UNKNOWN; - case ENaivePatchCase::ErrorOnGet: - return NKikimrProto::UNKNOWN; - case ENaivePatchCase::ErrorOnPut: - return NKikimrProto::ERROR; - } -} - -enum class EVPatchCase { - Ok, - OneErrorAndAllPartExistInStart, - OnePartLostInStart, - DeadGroupInStart, - ErrorDuringVPatchDiff, - Custom, -}; - -NKikimrProto::EReplyStatus GetPatchResultStatus(EVPatchCase vpatchCase) { - switch (vpatchCase) { - case EVPatchCase::Ok: - case EVPatchCase::OneErrorAndAllPartExistInStart: - return NKikimrProto::OK; - case EVPatchCase::OnePartLostInStart: - case EVPatchCase::DeadGroupInStart: - case EVPatchCase::ErrorDuringVPatchDiff: - case EVPatchCase::Custom: - return NKikimrProto::UNKNOWN; - } -} - -struct TFoundPartsResult { - NKikimrProto::EReplyStatus Status; - TVector<ui64> Parts; -}; - -TFoundPartsResult GetVPatchFoundPartsStatus(const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase vpatchCase, const TVDiskPointer &vdisk) { - - TVector<ui64> foundPartsOk; - auto [vdiskIdx, idxInSubgroup] = vdisk.GetIndecies(env, args.OriginalId.Hash()); - if (args.GType.GetErasure() == TErasureType::ErasureMirror3dc) { - foundPartsOk.push_back(idxInSubgroup / 3); - } else if (idxInSubgroup < args.GType.TotalPartCount()) { - foundPartsOk.push_back(idxInSubgroup + 1); - } - TFoundPartsResult okResult{NKikimrProto::OK, foundPartsOk}; - TFoundPartsResult lostResult{NKikimrProto::OK, {}}; - TFoundPartsResult errorResult{NKikimrProto::ERROR, {}}; - TFoundPartsResult customOkResult{NKikimrProto::OK, args.PartPlacement[idxInSubgroup]}; - - if (args.GType.GetErasure() == TErasureType::ErasureMirror3dc) { - TFoundPartsResult correctResult = (idxInSubgroup % 3 == 0) ? okResult : lostResult; - switch (vpatchCase) { - case EVPatchCase::Ok: - case EVPatchCase::ErrorDuringVPatchDiff: - return correctResult; - case EVPatchCase::OneErrorAndAllPartExistInStart: - return (idxInSubgroup % 3 == 2 && idxInSubgroup / 3 == 2) ? errorResult : correctResult; - case EVPatchCase::OnePartLostInStart: - return (idxInSubgroup == 0) ? lostResult : correctResult; - case EVPatchCase::DeadGroupInStart: - return (idxInSubgroup / 3 < 2) ? errorResult : correctResult; - case EVPatchCase::Custom: - return args.ErrorVDisks[idxInSubgroup] ? errorResult : customOkResult; - } - } else { - switch (vpatchCase) { - case EVPatchCase::Ok: - case EVPatchCase::ErrorDuringVPatchDiff: - return okResult; - case EVPatchCase::OneErrorAndAllPartExistInStart: - return (idxInSubgroup == args.GType.TotalPartCount()) ? errorResult : okResult; - case EVPatchCase::OnePartLostInStart: - return (idxInSubgroup == 0) ? lostResult : okResult; - case EVPatchCase::DeadGroupInStart: - return (idxInSubgroup <= args.GType.Handoff()) ? errorResult : okResult; - case EVPatchCase::Custom: - return args.ErrorVDisks[vdiskIdx] ? errorResult : customOkResult; - } - } -} - -NKikimrProto::EReplyStatus GetVPatchResultStatus(const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase vpatchCase, const TVDiskPointer &vdisk) -{ - auto [prevStatus, parts] = GetVPatchFoundPartsStatus(env, args, vpatchCase, vdisk); - if (!parts) { - return NKikimrProto::UNKNOWN; - } - if (prevStatus != NKikimrProto::OK) { - return NKikimrProto::UNKNOWN; - } - - switch (vpatchCase) { - case EVPatchCase::Ok: - case EVPatchCase::OneErrorAndAllPartExistInStart: - case EVPatchCase::OnePartLostInStart: - case EVPatchCase::DeadGroupInStart: - case EVPatchCase::Custom: - return NKikimrProto::OK; - case EVPatchCase::ErrorDuringVPatchDiff: - return NKikimrProto::ERROR; - } -} - -enum class EMovedPatchCase { - Ok, - Error -}; - -NKikimrProto::EReplyStatus GetPatchResultStatus(EMovedPatchCase movedCase) { - switch (movedCase) { - case EMovedPatchCase::Ok: - return NKikimrProto::OK; - case EMovedPatchCase::Error: - return NKikimrProto::UNKNOWN; - } -} - -NKikimrProto::EReplyStatus GetVMovedPatchResultStatus(EMovedPatchCase movedCase) { - switch (movedCase) { - case EMovedPatchCase::Ok: - return NKikimrProto::OK; - case EMovedPatchCase::Error: - return NKikimrProto::ERROR; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -template <typename THandle, typename TEventHolder> -void SendByHandle(TTestBasicRuntime &runtime, const THandle &oldHandle, TEventHolder &&ev) { + +#include <cstring> + +namespace NKikimr { +namespace NDSProxyPatchTest { + +struct TVDiskPointer { + ui32 VDiskIdx; + + static TVDiskPointer GetVDiskIdx(ui32 vdiskIdx) { + return {vdiskIdx}; + } + + std::pair<ui32, ui32> GetIndecies(const TDSProxyEnv &env, ui64 hash) const { + TVDiskID vdisk = env.Info->GetVDiskId(VDiskIdx); + ui32 idxInSubgroup = env.Info->GetIdxInSubgroup(vdisk, hash); + return {VDiskIdx, idxInSubgroup}; + } +}; + +struct TTestArgs { + TLogoBlobID OriginalId; + TLogoBlobID PatchedId; + ui64 OriginalGroupId; + ui64 CurrentGroupId; + TString Buffer; + TVector<TDiff> Diffs; + TBlobStorageGroupType GType; + ui32 MaskForCookieBruteForcing; + + ui32 StatusFlags = 1; + float ApproximateFreeSpaceShare = 0.1; + TVector<TVector<ui64>> PartPlacement; + TVector<bool> ErrorVDisks; + + void MakeDefaultPartPlacement() { + for (TVector<ui64> &pl : PartPlacement) { + pl.clear(); + } + switch (GType.GetErasure()) { + case TErasureType::Erasure4Plus2Block: + for (ui64 idx = 0; idx < GType.TotalPartCount(); ++idx) { + PartPlacement[idx].push_back(idx + 1); + } + break; + case TErasureType::ErasureNone: + PartPlacement[0].push_back(1); + break; + case TErasureType::ErasureMirror3: + PartPlacement[0].push_back(1); + PartPlacement[1].push_back(1); + PartPlacement[2].push_back(1); + break; + case TErasureType::ErasureMirror3dc: + PartPlacement[0].push_back(1); + PartPlacement[3].push_back(2); + PartPlacement[6].push_back(3); + break; + default: + UNIT_ASSERT(false); // not implemented + } + } + + void ResetDefaultPlacement() { + ErrorVDisks.assign(GType.BlobSubgroupSize(), false); + PartPlacement.resize(GType.BlobSubgroupSize()); + MakeDefaultPartPlacement(); + } + + void MakeDefault(TBlobStorageGroupType type, bool sameGroupId = true, bool goodPatchedId = true) { + constexpr ui32 bufferSize = 1000; + OriginalId = TLogoBlobID(1, 2, 3, 4, bufferSize, 5); + PatchedId = TLogoBlobID(1, 3, 3, 4, bufferSize, 6); + OriginalGroupId = 0; + CurrentGroupId = OriginalGroupId + !sameGroupId; + Buffer = TString::Uninitialized(bufferSize); + for (char &c : Buffer) { + c = 'a'; + } + for (ui32 idx = 0; idx < bufferSize; idx += 2) { + Diffs.emplace_back("b", idx); + } + GType = type; + + ErrorVDisks.assign(type.BlobSubgroupSize(), false); + PartPlacement.resize(type.BlobSubgroupSize()); + MakeDefaultPartPlacement(); + + MaskForCookieBruteForcing = (Max<ui32>() << 17) & TLogoBlobID::MaxCookie; + if (goodPatchedId) { + UNIT_ASSERT(TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(OriginalId, &PatchedId, + MaskForCookieBruteForcing, OriginalGroupId, CurrentGroupId)); + } + } +}; + +enum class ENaivePatchCase { + Ok, + ErrorOnGetItem, + ErrorOnGet, + ErrorOnPut, +}; + +NKikimrProto::EReplyStatus GetPatchResultStatus(ENaivePatchCase naiveCase) { + switch (naiveCase) { + case ENaivePatchCase::Ok: + return NKikimrProto::OK; + case ENaivePatchCase::ErrorOnGetItem: + case ENaivePatchCase::ErrorOnGet: + case ENaivePatchCase::ErrorOnPut: + return NKikimrProto::ERROR; + } +} + +NKikimrProto::EReplyStatus GetGetResultStatus(ENaivePatchCase naiveCase) { + switch (naiveCase) { + case ENaivePatchCase::Ok: + return NKikimrProto::OK; + case ENaivePatchCase::ErrorOnGetItem: + return NKikimrProto::OK; + case ENaivePatchCase::ErrorOnGet: + return NKikimrProto::ERROR; + case ENaivePatchCase::ErrorOnPut: + return NKikimrProto::OK; + } +} + +NKikimrProto::EReplyStatus GetPutResultStatus(ENaivePatchCase naiveCase) { + switch (naiveCase) { + case ENaivePatchCase::Ok: + return NKikimrProto::OK; + case ENaivePatchCase::ErrorOnGetItem: + return NKikimrProto::UNKNOWN; + case ENaivePatchCase::ErrorOnGet: + return NKikimrProto::UNKNOWN; + case ENaivePatchCase::ErrorOnPut: + return NKikimrProto::ERROR; + } +} + +enum class EVPatchCase { + Ok, + OneErrorAndAllPartExistInStart, + OnePartLostInStart, + DeadGroupInStart, + ErrorDuringVPatchDiff, + Custom, +}; + +NKikimrProto::EReplyStatus GetPatchResultStatus(EVPatchCase vpatchCase) { + switch (vpatchCase) { + case EVPatchCase::Ok: + case EVPatchCase::OneErrorAndAllPartExistInStart: + return NKikimrProto::OK; + case EVPatchCase::OnePartLostInStart: + case EVPatchCase::DeadGroupInStart: + case EVPatchCase::ErrorDuringVPatchDiff: + case EVPatchCase::Custom: + return NKikimrProto::UNKNOWN; + } +} + +struct TFoundPartsResult { + NKikimrProto::EReplyStatus Status; + TVector<ui64> Parts; +}; + +TFoundPartsResult GetVPatchFoundPartsStatus(const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase vpatchCase, const TVDiskPointer &vdisk) { + + TVector<ui64> foundPartsOk; + auto [vdiskIdx, idxInSubgroup] = vdisk.GetIndecies(env, args.OriginalId.Hash()); + if (args.GType.GetErasure() == TErasureType::ErasureMirror3dc) { + foundPartsOk.push_back(idxInSubgroup / 3); + } else if (idxInSubgroup < args.GType.TotalPartCount()) { + foundPartsOk.push_back(idxInSubgroup + 1); + } + TFoundPartsResult okResult{NKikimrProto::OK, foundPartsOk}; + TFoundPartsResult lostResult{NKikimrProto::OK, {}}; + TFoundPartsResult errorResult{NKikimrProto::ERROR, {}}; + TFoundPartsResult customOkResult{NKikimrProto::OK, args.PartPlacement[idxInSubgroup]}; + + if (args.GType.GetErasure() == TErasureType::ErasureMirror3dc) { + TFoundPartsResult correctResult = (idxInSubgroup % 3 == 0) ? okResult : lostResult; + switch (vpatchCase) { + case EVPatchCase::Ok: + case EVPatchCase::ErrorDuringVPatchDiff: + return correctResult; + case EVPatchCase::OneErrorAndAllPartExistInStart: + return (idxInSubgroup % 3 == 2 && idxInSubgroup / 3 == 2) ? errorResult : correctResult; + case EVPatchCase::OnePartLostInStart: + return (idxInSubgroup == 0) ? lostResult : correctResult; + case EVPatchCase::DeadGroupInStart: + return (idxInSubgroup / 3 < 2) ? errorResult : correctResult; + case EVPatchCase::Custom: + return args.ErrorVDisks[idxInSubgroup] ? errorResult : customOkResult; + } + } else { + switch (vpatchCase) { + case EVPatchCase::Ok: + case EVPatchCase::ErrorDuringVPatchDiff: + return okResult; + case EVPatchCase::OneErrorAndAllPartExistInStart: + return (idxInSubgroup == args.GType.TotalPartCount()) ? errorResult : okResult; + case EVPatchCase::OnePartLostInStart: + return (idxInSubgroup == 0) ? lostResult : okResult; + case EVPatchCase::DeadGroupInStart: + return (idxInSubgroup <= args.GType.Handoff()) ? errorResult : okResult; + case EVPatchCase::Custom: + return args.ErrorVDisks[vdiskIdx] ? errorResult : customOkResult; + } + } +} + +NKikimrProto::EReplyStatus GetVPatchResultStatus(const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase vpatchCase, const TVDiskPointer &vdisk) +{ + auto [prevStatus, parts] = GetVPatchFoundPartsStatus(env, args, vpatchCase, vdisk); + if (!parts) { + return NKikimrProto::UNKNOWN; + } + if (prevStatus != NKikimrProto::OK) { + return NKikimrProto::UNKNOWN; + } + + switch (vpatchCase) { + case EVPatchCase::Ok: + case EVPatchCase::OneErrorAndAllPartExistInStart: + case EVPatchCase::OnePartLostInStart: + case EVPatchCase::DeadGroupInStart: + case EVPatchCase::Custom: + return NKikimrProto::OK; + case EVPatchCase::ErrorDuringVPatchDiff: + return NKikimrProto::ERROR; + } +} + +enum class EMovedPatchCase { + Ok, + Error +}; + +NKikimrProto::EReplyStatus GetPatchResultStatus(EMovedPatchCase movedCase) { + switch (movedCase) { + case EMovedPatchCase::Ok: + return NKikimrProto::OK; + case EMovedPatchCase::Error: + return NKikimrProto::UNKNOWN; + } +} + +NKikimrProto::EReplyStatus GetVMovedPatchResultStatus(EMovedPatchCase movedCase) { + switch (movedCase) { + case EMovedPatchCase::Ok: + return NKikimrProto::OK; + case EMovedPatchCase::Error: + return NKikimrProto::ERROR; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <typename THandle, typename TEventHolder> +void SendByHandle(TTestBasicRuntime &runtime, const THandle &oldHandle, TEventHolder &&ev) { auto handle = std::make_unique<IEventHandle>(oldHandle->Sender, oldHandle->Recipient, ev.release(), oldHandle->Flags, oldHandle->Cookie); runtime.Send(handle.release()); -} - -void ReceivePatchResult(TTestBasicRuntime &runtime, const TTestArgs &args, NKikimrProto::EReplyStatus status) { - CTEST << "ReceivePatchResult: Start\n"; - TAutoPtr<IEventHandle> handle; - TEvBlobStorage::TEvPatchResult *result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPatchResult>(handle); - UNIT_ASSERT_VALUES_EQUAL(result->Status, status); - UNIT_ASSERT_VALUES_EQUAL(result->Id, args.PatchedId); - if (status == NKikimrProto::OK) { - UNIT_ASSERT_VALUES_EQUAL(result->StatusFlags.Raw, args.StatusFlags); - UNIT_ASSERT_VALUES_EQUAL(result->ApproximateFreeSpaceShare, args.ApproximateFreeSpaceShare); - } - CTEST << "ReceivePatchResult: Finish\n"; -} - -void ConductGet(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { - CTEST << "ConductGet: Start\n"; - NKikimrProto::EReplyStatus resultStatus = GetGetResultStatus(naiveCase); - TAutoPtr<IEventHandle> handle; - TEvBlobStorage::TEvGet *get = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGet>(handle); - UNIT_ASSERT_VALUES_EQUAL(get->QuerySize, 1); - UNIT_ASSERT_VALUES_EQUAL(get->Queries[0].Id, args.OriginalId); - UNIT_ASSERT_VALUES_EQUAL(handle->Cookie, args.PatchedId.Hash()); - +} + +void ReceivePatchResult(TTestBasicRuntime &runtime, const TTestArgs &args, NKikimrProto::EReplyStatus status) { + CTEST << "ReceivePatchResult: Start\n"; + TAutoPtr<IEventHandle> handle; + TEvBlobStorage::TEvPatchResult *result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPatchResult>(handle); + UNIT_ASSERT_VALUES_EQUAL(result->Status, status); + UNIT_ASSERT_VALUES_EQUAL(result->Id, args.PatchedId); + if (status == NKikimrProto::OK) { + UNIT_ASSERT_VALUES_EQUAL(result->StatusFlags.Raw, args.StatusFlags); + UNIT_ASSERT_VALUES_EQUAL(result->ApproximateFreeSpaceShare, args.ApproximateFreeSpaceShare); + } + CTEST << "ReceivePatchResult: Finish\n"; +} + +void ConductGet(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { + CTEST << "ConductGet: Start\n"; + NKikimrProto::EReplyStatus resultStatus = GetGetResultStatus(naiveCase); + TAutoPtr<IEventHandle> handle; + TEvBlobStorage::TEvGet *get = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGet>(handle); + UNIT_ASSERT_VALUES_EQUAL(get->QuerySize, 1); + UNIT_ASSERT_VALUES_EQUAL(get->Queries[0].Id, args.OriginalId); + UNIT_ASSERT_VALUES_EQUAL(handle->Cookie, args.PatchedId.Hash()); + std::unique_ptr<TEvBlobStorage::TEvGetResult> getResult; - if (resultStatus == NKikimrProto::OK) { + if (resultStatus == NKikimrProto::OK) { getResult = std::make_unique<TEvBlobStorage::TEvGetResult>(NKikimrProto::OK, 1, args.CurrentGroupId); - if (naiveCase == ENaivePatchCase::ErrorOnGetItem) { - getResult->Responses[0].Id = args.OriginalId; - getResult->Responses[0].Status = NKikimrProto::ERROR; - } else { - getResult->Responses[0].Buffer = args.Buffer; - getResult->Responses[0].Id = args.OriginalId; - getResult->Responses[0].Status = NKikimrProto::OK; - } - } else { + if (naiveCase == ENaivePatchCase::ErrorOnGetItem) { + getResult->Responses[0].Id = args.OriginalId; + getResult->Responses[0].Status = NKikimrProto::ERROR; + } else { + getResult->Responses[0].Buffer = args.Buffer; + getResult->Responses[0].Id = args.OriginalId; + getResult->Responses[0].Status = NKikimrProto::OK; + } + } else { getResult = std::make_unique<TEvBlobStorage::TEvGetResult>(NKikimrProto::ERROR, 0, args.CurrentGroupId); - } - - SendByHandle(runtime, handle, std::move(getResult)); - CTEST << "ConductGet: Finish\n"; -} - -TString MakePatchedBuffer(const TTestArgs &args) { - TString buffer = TString::Uninitialized(args.Buffer.size()); - memcpy(buffer.begin(), args.Buffer.begin(), args.Buffer.size()); - for (auto &diff : args.Diffs) { - memcpy(buffer.begin() + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); - } - return buffer; -} - -void ConductPut(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { - NKikimrProto::EReplyStatus resultStatus = GetPutResultStatus(naiveCase); - if (resultStatus == NKikimrProto::UNKNOWN) { - CTEST << "ConductPut: Skip\n"; - return; - } - CTEST << "ConductPut: Start\n"; - TAutoPtr<IEventHandle> handle; - TEvBlobStorage::TEvPut *put = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPut>(handle); - UNIT_ASSERT_VALUES_EQUAL(put->Id, args.PatchedId); - UNIT_ASSERT_VALUES_EQUAL(handle->Cookie, args.OriginalId.Hash()); - TString patchedBuffer = MakePatchedBuffer(args); - UNIT_ASSERT_VALUES_EQUAL(put->Buffer, patchedBuffer); - + } + + SendByHandle(runtime, handle, std::move(getResult)); + CTEST << "ConductGet: Finish\n"; +} + +TString MakePatchedBuffer(const TTestArgs &args) { + TString buffer = TString::Uninitialized(args.Buffer.size()); + memcpy(buffer.begin(), args.Buffer.begin(), args.Buffer.size()); + for (auto &diff : args.Diffs) { + memcpy(buffer.begin() + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); + } + return buffer; +} + +void ConductPut(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { + NKikimrProto::EReplyStatus resultStatus = GetPutResultStatus(naiveCase); + if (resultStatus == NKikimrProto::UNKNOWN) { + CTEST << "ConductPut: Skip\n"; + return; + } + CTEST << "ConductPut: Start\n"; + TAutoPtr<IEventHandle> handle; + TEvBlobStorage::TEvPut *put = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPut>(handle); + UNIT_ASSERT_VALUES_EQUAL(put->Id, args.PatchedId); + UNIT_ASSERT_VALUES_EQUAL(handle->Cookie, args.OriginalId.Hash()); + TString patchedBuffer = MakePatchedBuffer(args); + UNIT_ASSERT_VALUES_EQUAL(put->Buffer, patchedBuffer); + std::unique_ptr<TEvBlobStorage::TEvPutResult> putResult = std::make_unique<TEvBlobStorage::TEvPutResult>( - resultStatus, args.PatchedId, args.StatusFlags, args.CurrentGroupId, args.ApproximateFreeSpaceShare); - SendByHandle(runtime, handle, std::move(putResult)); - CTEST << "ConductPut: Finish\n"; -} - -void ConductNaivePatch(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { - CTEST << "ConductNaivePatch: Start\n"; - ConductGet(runtime, args, naiveCase); - ConductPut(runtime, args, naiveCase); - NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(naiveCase); - ReceivePatchResult(runtime, args, resultStatus); - CTEST << "ConductNaivePatch: Finish\n"; -} - - -void ConductVPatchStart(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase naiveCase, TVDiskPointer vdiskPointer) -{ - auto [vdiskIdx, idxInSubgroup] = vdiskPointer.GetIndecies(env, args.OriginalId.Hash()); - CTEST << "ConductVPatchStart: Start vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; - TVDiskID vdisk = env.Info->GetVDiskInSubgroup(idxInSubgroup, args.OriginalId.Hash()); - auto [status, parts] = GetVPatchFoundPartsStatus(env, args, naiveCase, vdiskPointer); - - auto start = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchStart>({env.VDisks[vdiskIdx]}); - auto &startRecord = start->Get()->Record; - UNIT_ASSERT(vdisk == VDiskIDFromVDiskID(startRecord.GetVDiskID())); - UNIT_ASSERT(args.OriginalId == LogoBlobIDFromLogoBlobID(startRecord.GetOriginalBlobId())); - - TInstant now = runtime.GetCurrentTime(); - UNIT_ASSERT(startRecord.HasCookie()); + resultStatus, args.PatchedId, args.StatusFlags, args.CurrentGroupId, args.ApproximateFreeSpaceShare); + SendByHandle(runtime, handle, std::move(putResult)); + CTEST << "ConductPut: Finish\n"; +} + +void ConductNaivePatch(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { + CTEST << "ConductNaivePatch: Start\n"; + ConductGet(runtime, args, naiveCase); + ConductPut(runtime, args, naiveCase); + NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(naiveCase); + ReceivePatchResult(runtime, args, resultStatus); + CTEST << "ConductNaivePatch: Finish\n"; +} + + +void ConductVPatchStart(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase naiveCase, TVDiskPointer vdiskPointer) +{ + auto [vdiskIdx, idxInSubgroup] = vdiskPointer.GetIndecies(env, args.OriginalId.Hash()); + CTEST << "ConductVPatchStart: Start vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; + TVDiskID vdisk = env.Info->GetVDiskInSubgroup(idxInSubgroup, args.OriginalId.Hash()); + auto [status, parts] = GetVPatchFoundPartsStatus(env, args, naiveCase, vdiskPointer); + + auto start = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchStart>({env.VDisks[vdiskIdx]}); + auto &startRecord = start->Get()->Record; + UNIT_ASSERT(vdisk == VDiskIDFromVDiskID(startRecord.GetVDiskID())); + UNIT_ASSERT(args.OriginalId == LogoBlobIDFromLogoBlobID(startRecord.GetOriginalBlobId())); + + TInstant now = runtime.GetCurrentTime(); + UNIT_ASSERT(startRecord.HasCookie()); std::unique_ptr<TEvBlobStorage::TEvVPatchFoundParts> foundParts = std::make_unique<TEvBlobStorage::TEvVPatchFoundParts>( - status, args.OriginalId, args.PatchedId, vdisk, startRecord.GetCookie(), now, "", &startRecord, - nullptr, nullptr, nullptr, NWilson::TTraceId(), 0); - for (auto partId : parts) { - foundParts->AddPart(partId); - } - SendByHandle(runtime, start, std::move(foundParts)); - CTEST << "ConductVPatchStart: Finish vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; -} - -void ConductVPatchDiff(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase naiveCase, TVDiskPointer vdiskPointer) -{ - auto [vdiskIdx, idxInSubgroup] = vdiskPointer.GetIndecies(env, args.PatchedId.Hash()); - TVDiskID vdisk = env.Info->GetVDiskInSubgroup(idxInSubgroup, args.PatchedId.Hash()); - NKikimrProto::EReplyStatus resultStatus = GetVPatchResultStatus(env, args, naiveCase, vdiskPointer); - if (resultStatus == NKikimrProto::UNKNOWN) { - CTEST << "ConductVPatchDiff: Skip vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; - return; - } - CTEST << "ConductVPatchDiff: Start vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; - - auto diffEv = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchDiff>({env.VDisks[vdiskIdx]}); - auto &diffRecord = diffEv->Get()->Record; - UNIT_ASSERT(vdisk == VDiskIDFromVDiskID(diffRecord.GetVDiskID())); - UNIT_ASSERT_VALUES_EQUAL(args.OriginalId, LogoBlobIDFromLogoBlobID(diffRecord.GetOriginalPartBlobId()).FullID()); - UNIT_ASSERT_VALUES_EQUAL(args.PatchedId, LogoBlobIDFromLogoBlobID(diffRecord.GetPatchedPartBlobId()).FullID()); - - if (env.Info->Type.ErasureFamily() == TErasureType::ErasureMirror) { - UNIT_ASSERT_C(!diffEv->Get()->IsXorReceiver(), "it can't be xorreceiver"); - UNIT_ASSERT_C(!diffRecord.XorReceiversSize(), "it can't has xorreceivers"); - } - - TInstant now = runtime.GetCurrentTime(); - UNIT_ASSERT(diffRecord.HasCookie()); + status, args.OriginalId, args.PatchedId, vdisk, startRecord.GetCookie(), now, "", &startRecord, + nullptr, nullptr, nullptr, NWilson::TTraceId(), 0); + for (auto partId : parts) { + foundParts->AddPart(partId); + } + SendByHandle(runtime, start, std::move(foundParts)); + CTEST << "ConductVPatchStart: Finish vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; +} + +void ConductVPatchDiff(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase naiveCase, TVDiskPointer vdiskPointer) +{ + auto [vdiskIdx, idxInSubgroup] = vdiskPointer.GetIndecies(env, args.PatchedId.Hash()); + TVDiskID vdisk = env.Info->GetVDiskInSubgroup(idxInSubgroup, args.PatchedId.Hash()); + NKikimrProto::EReplyStatus resultStatus = GetVPatchResultStatus(env, args, naiveCase, vdiskPointer); + if (resultStatus == NKikimrProto::UNKNOWN) { + CTEST << "ConductVPatchDiff: Skip vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; + return; + } + CTEST << "ConductVPatchDiff: Start vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; + + auto diffEv = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchDiff>({env.VDisks[vdiskIdx]}); + auto &diffRecord = diffEv->Get()->Record; + UNIT_ASSERT(vdisk == VDiskIDFromVDiskID(diffRecord.GetVDiskID())); + UNIT_ASSERT_VALUES_EQUAL(args.OriginalId, LogoBlobIDFromLogoBlobID(diffRecord.GetOriginalPartBlobId()).FullID()); + UNIT_ASSERT_VALUES_EQUAL(args.PatchedId, LogoBlobIDFromLogoBlobID(diffRecord.GetPatchedPartBlobId()).FullID()); + + if (env.Info->Type.ErasureFamily() == TErasureType::ErasureMirror) { + UNIT_ASSERT_C(!diffEv->Get()->IsXorReceiver(), "it can't be xorreceiver"); + UNIT_ASSERT_C(!diffRecord.XorReceiversSize(), "it can't has xorreceivers"); + } + + TInstant now = runtime.GetCurrentTime(); + UNIT_ASSERT(diffRecord.HasCookie()); std::unique_ptr<TEvBlobStorage::TEvVPatchResult> result = std::make_unique<TEvBlobStorage::TEvVPatchResult>( - resultStatus, args.OriginalId, args.PatchedId, vdisk, diffRecord.GetCookie(), now, - &diffRecord, nullptr, nullptr, nullptr,NWilson::TTraceId(), 0); - result->SetStatusFlagsAndFreeSpace(args.StatusFlags, args.ApproximateFreeSpaceShare); - - SendByHandle(runtime, diffEv, std::move(result)); - CTEST << "ConductVPatchDiff: Finish vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; -} - -void ConductFailedVPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args) { - CTEST << "ConductFailedVPatch: Start\n"; - for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { - TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); - ConductVPatchStart(runtime, env, args, EVPatchCase::OnePartLostInStart, vdisk); - } - for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { - TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); - ConductVPatchDiff(runtime, env, args, EVPatchCase::OnePartLostInStart, vdisk); - } - CTEST << "ConductFailedVPatch: Finish\n"; -} - - -void ConductVMovedPatch(TTestBasicRuntime &runtime, const TTestArgs &args, EMovedPatchCase movedCase) { - CTEST << "ConductVMovedPatch: Start\n"; - NKikimrProto::EReplyStatus resultStatus = GetVMovedPatchResultStatus(movedCase); - TAutoPtr<IEventHandle> handle; - TEvBlobStorage::TEvVMovedPatch *vPatch = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVMovedPatch>(handle); - - NKikimrBlobStorage::TEvVMovedPatch &vPatchRecord = vPatch->Record; - UNIT_ASSERT(args.OriginalId == LogoBlobIDFromLogoBlobID(vPatchRecord.GetOriginalBlobId())); - UNIT_ASSERT(args.PatchedId == LogoBlobIDFromLogoBlobID(vPatchRecord.GetPatchedBlobId())); - UNIT_ASSERT_VALUES_EQUAL(vPatchRecord.DiffsSize(), args.Diffs.size()); - for (ui32 diffIdx = 0; diffIdx < args.Diffs.size(); ++diffIdx) { - UNIT_ASSERT_VALUES_EQUAL(vPatchRecord.GetDiffs(diffIdx).GetOffset(), args.Diffs[diffIdx].Offset); - UNIT_ASSERT_EQUAL(vPatchRecord.GetDiffs(diffIdx).GetBuffer(), args.Diffs[diffIdx].Buffer); - } - ui64 expectedCookie = ((ui64)args.OriginalId.Hash() << 32) | args.PatchedId.Hash(); - UNIT_ASSERT(vPatchRecord.HasCookie()); - UNIT_ASSERT(vPatchRecord.GetCookie() == expectedCookie); - - TVDiskID vDiskId = VDiskIDFromVDiskID(vPatchRecord.GetVDiskID()); - TOutOfSpaceStatus oos(args.StatusFlags, args.ApproximateFreeSpaceShare); + resultStatus, args.OriginalId, args.PatchedId, vdisk, diffRecord.GetCookie(), now, + &diffRecord, nullptr, nullptr, nullptr,NWilson::TTraceId(), 0); + result->SetStatusFlagsAndFreeSpace(args.StatusFlags, args.ApproximateFreeSpaceShare); + + SendByHandle(runtime, diffEv, std::move(result)); + CTEST << "ConductVPatchDiff: Finish vdiskIdx# " << vdiskIdx << " idxInSubgroup# " << idxInSubgroup << "\n"; +} + +void ConductFailedVPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args) { + CTEST << "ConductFailedVPatch: Start\n"; + for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { + TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); + ConductVPatchStart(runtime, env, args, EVPatchCase::OnePartLostInStart, vdisk); + } + for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { + TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); + ConductVPatchDiff(runtime, env, args, EVPatchCase::OnePartLostInStart, vdisk); + } + CTEST << "ConductFailedVPatch: Finish\n"; +} + + +void ConductVMovedPatch(TTestBasicRuntime &runtime, const TTestArgs &args, EMovedPatchCase movedCase) { + CTEST << "ConductVMovedPatch: Start\n"; + NKikimrProto::EReplyStatus resultStatus = GetVMovedPatchResultStatus(movedCase); + TAutoPtr<IEventHandle> handle; + TEvBlobStorage::TEvVMovedPatch *vPatch = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVMovedPatch>(handle); + + NKikimrBlobStorage::TEvVMovedPatch &vPatchRecord = vPatch->Record; + UNIT_ASSERT(args.OriginalId == LogoBlobIDFromLogoBlobID(vPatchRecord.GetOriginalBlobId())); + UNIT_ASSERT(args.PatchedId == LogoBlobIDFromLogoBlobID(vPatchRecord.GetPatchedBlobId())); + UNIT_ASSERT_VALUES_EQUAL(vPatchRecord.DiffsSize(), args.Diffs.size()); + for (ui32 diffIdx = 0; diffIdx < args.Diffs.size(); ++diffIdx) { + UNIT_ASSERT_VALUES_EQUAL(vPatchRecord.GetDiffs(diffIdx).GetOffset(), args.Diffs[diffIdx].Offset); + UNIT_ASSERT_EQUAL(vPatchRecord.GetDiffs(diffIdx).GetBuffer(), args.Diffs[diffIdx].Buffer); + } + ui64 expectedCookie = ((ui64)args.OriginalId.Hash() << 32) | args.PatchedId.Hash(); + UNIT_ASSERT(vPatchRecord.HasCookie()); + UNIT_ASSERT(vPatchRecord.GetCookie() == expectedCookie); + + TVDiskID vDiskId = VDiskIDFromVDiskID(vPatchRecord.GetVDiskID()); + TOutOfSpaceStatus oos(args.StatusFlags, args.ApproximateFreeSpaceShare); std::unique_ptr<TEvBlobStorage::TEvVMovedPatchResult> vPatchResult = std::make_unique<TEvBlobStorage::TEvVMovedPatchResult>( - resultStatus, args.OriginalId, args.PatchedId, vDiskId, expectedCookie, oos, - TAppData::TimeProvider->Now(), 0, &vPatchRecord, nullptr, nullptr, nullptr, NWilson::TTraceId(), 0, TString()); - - SendByHandle(runtime, handle, std::move(vPatchResult)); - CTEST << "ConductVMovedPatch: Finish\n"; -} - -void ConductMovedPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EMovedPatchCase movedCase) -{ - CTEST << "ConductMovedPatch: Start\n"; - ConductFailedVPatch(runtime, env, args); - ConductVMovedPatch(runtime, args, movedCase); - NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(movedCase); - if (resultStatus == NKikimrProto::UNKNOWN) { - ConductNaivePatch(runtime, args, ENaivePatchCase::Ok); - } else { - ReceivePatchResult(runtime, args, resultStatus); - } - CTEST << "ConductMovedPatch: Finish\n"; -} - -void ConductFallbackPatch(TTestBasicRuntime &runtime, const TTestArgs &args) { - CTEST << "ConductFallbackPatch: Start\n"; - ConductVMovedPatch(runtime, args, EMovedPatchCase::Ok); - ReceivePatchResult(runtime, args, NKikimrProto::OK); - CTEST << "ConductFallbackPatch: Finish\n"; -} - -void ConductVPatchEvents(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase vpatchCase) -{ - CTEST << "ConductVPatchEvents: Start\n"; - for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { - TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); - ConductVPatchStart(runtime, env, args, vpatchCase, vdisk); - } - for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { - TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); - ConductVPatchDiff(runtime, env, args, vpatchCase, vdisk); - } - CTEST << "ConductVPatchEvents: Finish\n"; -} - -void ConductVPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EVPatchCase vpatchCase) -{ - CTEST << "ConductFallbackPatch: Start\n"; - ConductVPatchEvents(runtime, env, args, vpatchCase); - NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(vpatchCase); - if (resultStatus == NKikimrProto::UNKNOWN) { - ConductFallbackPatch(runtime, args); - } else { - ReceivePatchResult(runtime, args, resultStatus); - } - CTEST << "ConductFallbackPatch: Finish\n"; -} - -//////////////////////////////////////////////////////////////////////////////// - -TEvBlobStorage::TEvPatch::TPtr CreatePatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args) { - TArrayHolder<TEvBlobStorage::TEvPatch::TDiff> diffs(new TEvBlobStorage::TEvPatch::TDiff[args.Diffs.size()]); - for (ui32 diffIdx = 0; diffIdx < args.Diffs.size(); ++diffIdx) { - const TDiff &diff = args.Diffs[diffIdx]; - UNIT_ASSERT(!diff.IsXor && !diff.IsAligned); - diffs[diffIdx].Offset = diff.Offset; - diffs[diffIdx].Buffer = diff.Buffer; - } - TInstant deadline = runtime.GetCurrentTime() + TDuration::Seconds(5); + resultStatus, args.OriginalId, args.PatchedId, vDiskId, expectedCookie, oos, + TAppData::TimeProvider->Now(), 0, &vPatchRecord, nullptr, nullptr, nullptr, NWilson::TTraceId(), 0, TString()); + + SendByHandle(runtime, handle, std::move(vPatchResult)); + CTEST << "ConductVMovedPatch: Finish\n"; +} + +void ConductMovedPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EMovedPatchCase movedCase) +{ + CTEST << "ConductMovedPatch: Start\n"; + ConductFailedVPatch(runtime, env, args); + ConductVMovedPatch(runtime, args, movedCase); + NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(movedCase); + if (resultStatus == NKikimrProto::UNKNOWN) { + ConductNaivePatch(runtime, args, ENaivePatchCase::Ok); + } else { + ReceivePatchResult(runtime, args, resultStatus); + } + CTEST << "ConductMovedPatch: Finish\n"; +} + +void ConductFallbackPatch(TTestBasicRuntime &runtime, const TTestArgs &args) { + CTEST << "ConductFallbackPatch: Start\n"; + ConductVMovedPatch(runtime, args, EMovedPatchCase::Ok); + ReceivePatchResult(runtime, args, NKikimrProto::OK); + CTEST << "ConductFallbackPatch: Finish\n"; +} + +void ConductVPatchEvents(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase vpatchCase) +{ + CTEST << "ConductVPatchEvents: Start\n"; + for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { + TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); + ConductVPatchStart(runtime, env, args, vpatchCase, vdisk); + } + for (ui32 idxInSubgroup = 0; idxInSubgroup < args.GType.BlobSubgroupSize(); ++idxInSubgroup) { + TVDiskPointer vdisk = TVDiskPointer::GetVDiskIdx(idxInSubgroup); + ConductVPatchDiff(runtime, env, args, vpatchCase, vdisk); + } + CTEST << "ConductVPatchEvents: Finish\n"; +} + +void ConductVPatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EVPatchCase vpatchCase) +{ + CTEST << "ConductFallbackPatch: Start\n"; + ConductVPatchEvents(runtime, env, args, vpatchCase); + NKikimrProto::EReplyStatus resultStatus = GetPatchResultStatus(vpatchCase); + if (resultStatus == NKikimrProto::UNKNOWN) { + ConductFallbackPatch(runtime, args); + } else { + ReceivePatchResult(runtime, args, resultStatus); + } + CTEST << "ConductFallbackPatch: Finish\n"; +} + +//////////////////////////////////////////////////////////////////////////////// + +TEvBlobStorage::TEvPatch::TPtr CreatePatch(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args) { + TArrayHolder<TEvBlobStorage::TEvPatch::TDiff> diffs(new TEvBlobStorage::TEvPatch::TDiff[args.Diffs.size()]); + for (ui32 diffIdx = 0; diffIdx < args.Diffs.size(); ++diffIdx) { + const TDiff &diff = args.Diffs[diffIdx]; + UNIT_ASSERT(!diff.IsXor && !diff.IsAligned); + diffs[diffIdx].Offset = diff.Offset; + diffs[diffIdx].Buffer = diff.Buffer; + } + TInstant deadline = runtime.GetCurrentTime() + TDuration::Seconds(5); std::unique_ptr<TEvBlobStorage::TEvPatch> patch = std::make_unique<TEvBlobStorage::TEvPatch>( - args.OriginalGroupId, args.OriginalId, args.PatchedId, args.MaskForCookieBruteForcing, std::move(diffs), - args.Diffs.size(), deadline); - TEvBlobStorage::TEvPatch::TPtr ev(static_cast<TEventHandle<TEvBlobStorage::TEvPatch>*>( + args.OriginalGroupId, args.OriginalId, args.PatchedId, args.MaskForCookieBruteForcing, std::move(diffs), + args.Diffs.size(), deadline); + TEvBlobStorage::TEvPatch::TPtr ev(static_cast<TEventHandle<TEvBlobStorage::TEvPatch>*>( new IEventHandle(env.FakeProxyActorId, env.FakeProxyActorId, patch.release()))); - return ev; -} - -void RunNaivePatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { - TDSProxyEnv env; - env.Configure(runtime, args.GType, args.CurrentGroupId, 0); - TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); + return ev; +} + +void RunNaivePatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, ENaivePatchCase naiveCase) { + TDSProxyEnv env; + env.Configure(runtime, args.GType, args.CurrentGroupId, 0); + TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); std::unique_ptr<IActor> patchActor = env.CreatePatchRequestActor(patch, false); runtime.Register(patchActor.release()); - ConductNaivePatch(runtime, args, naiveCase); -} - -void RunMovedPatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, EMovedPatchCase movedCase) { - TDSProxyEnv env; - env.Configure(runtime, args.GType, args.CurrentGroupId, 0); - TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); + ConductNaivePatch(runtime, args, naiveCase); +} + +void RunMovedPatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, EMovedPatchCase movedCase) { + TDSProxyEnv env; + env.Configure(runtime, args.GType, args.CurrentGroupId, 0); + TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); std::unique_ptr<IActor> patchActor = env.CreatePatchRequestActor(patch, true); runtime.Register(patchActor.release()); - ConductMovedPatch(runtime, env, args, movedCase); -} - -void RunVPatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, EVPatchCase vpatchCase) { - TDSProxyEnv env; - env.Configure(runtime, args.GType, args.CurrentGroupId, 0); - TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); + ConductMovedPatch(runtime, env, args, movedCase); +} + +void RunVPatchTest(TTestBasicRuntime &runtime, const TTestArgs &args, EVPatchCase vpatchCase) { + TDSProxyEnv env; + env.Configure(runtime, args.GType, args.CurrentGroupId, 0); + TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); std::unique_ptr<IActor> patchActor = env.CreatePatchRequestActor(patch, true); runtime.Register(patchActor.release()); - ConductVPatch(runtime, env, args, vpatchCase); -} - -void SetLogPriorities(TTestBasicRuntime &runtime) { - bool IsVerbose = false; - if (IsVerbose) { - runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); - runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); - } - runtime.SetLogPriority(NKikimrServices::BS_PROXY, NLog::PRI_CRIT); - runtime.SetLogPriority(NKikimrServices::BS_QUEUE, NLog::PRI_CRIT); -} - -//////////////////////////////////////////////////////////////////////////////// - -Y_UNIT_TEST_SUITE(TDSProxyPatchTest) { - -template <typename ECase> -void RunGeneralTest(void(*runner)(TTestBasicRuntime &runtime, const TTestArgs &args, ECase testCase), - TErasureType::TErasureType::EErasureSpecies erasure, - ECase testCase) -{ - TTestArgs args; - args.MakeDefault(erasure, true); - TTestBasicRuntime runtime(1, false); - SetLogPriorities(runtime); - SetupRuntime(runtime); - runner(runtime, args, testCase); -} - -#define Y_UNIT_TEST_NAIVE(naiveCase, erasure) \ - Y_UNIT_TEST(Naive ## naiveCase ## _ ## erasure) { \ - RunGeneralTest(RunNaivePatchTest, TErasureType::TErasureType:: erasure, ENaivePatchCase:: naiveCase); \ - } \ -// end Y_UNIT_TEST_NAIVE - -#define Y_UNIT_TEST_MOVED(movedCase, erasure) \ - Y_UNIT_TEST(Moved ## movedCase ## _ ## erasure) { \ - RunGeneralTest(RunMovedPatchTest, TErasureType::TErasureType:: erasure, EMovedPatchCase:: movedCase); \ - } \ -// end Y_UNIT_TEST_MOVED - -#define Y_UNIT_TEST_VPATCH(vpatchCase, erasure) \ - Y_UNIT_TEST(VPatch ## vpatchCase ## _ ## erasure) { \ - RunGeneralTest(RunVPatchTest, TErasureType::TErasureType:: erasure, EVPatchCase:: vpatchCase); \ - } \ -// end Y_UNIT_TEST_VPATCH - -#define Y_UNIT_TEST_PATCH_PACK(erasure) \ - Y_UNIT_TEST_NAIVE(Ok, erasure) \ - Y_UNIT_TEST_NAIVE(ErrorOnGetItem, erasure) \ - Y_UNIT_TEST_NAIVE(ErrorOnGet, erasure) \ - Y_UNIT_TEST_NAIVE(ErrorOnPut, erasure) \ - Y_UNIT_TEST_MOVED(Ok, erasure) \ - Y_UNIT_TEST_MOVED(Error, erasure) \ - Y_UNIT_TEST_VPATCH(Ok, erasure) \ - Y_UNIT_TEST_VPATCH(OneErrorAndAllPartExistInStart, erasure) \ - Y_UNIT_TEST_VPATCH(OnePartLostInStart, erasure) \ - Y_UNIT_TEST_VPATCH(DeadGroupInStart, erasure) \ - Y_UNIT_TEST_VPATCH(ErrorDuringVPatchDiff, erasure) \ -// end Y_UNIT_TEST_PATCH_PACK - - Y_UNIT_TEST_PATCH_PACK(ErasureNone) - Y_UNIT_TEST_PATCH_PACK(Erasure4Plus2Block) - Y_UNIT_TEST_PATCH_PACK(ErasureMirror3) - Y_UNIT_TEST_PATCH_PACK(ErasureMirror3dc) - -} - -//////////////////////////////////////////////////////////////////////////////// - -void MakeFaultToleranceArgsForBlock4Plus2(TTestArgs &args, i64 wiped1, i64 wiped2, - ui32 errorMask, i64 handoff1, i64 handoff2, i64 extraHandoff) -{ - args.ResetDefaultPlacement(); - if (wiped1 != -1) { - args.PartPlacement[wiped1].clear(); - } - if (wiped2 != -1) { - args.PartPlacement[wiped2].clear(); - } - if (handoff1 > 0) { - args.PartPlacement[6].push_back(handoff1); - } - if (handoff2 > 0) { - args.PartPlacement[7].push_back(handoff2); - } - if (extraHandoff < 0) { - args.PartPlacement[6].push_back(-extraHandoff); - } - if (extraHandoff > 0) { - args.PartPlacement[7].push_back(extraHandoff); - } - for (ui32 bit = 1, idx = 0; bit < errorMask; idx++, bit <<= 1) { - args.ErrorVDisks[idx] = (errorMask & bit); - } -} - -void MakeFaultToleranceArgsForMirror3dc(TTestArgs &args, bool is2x2, ui32 errorMask, i64 wipedDc, i64 startIdxInDc[]) { - args.ResetDefaultPlacement(); - for (auto &pl : args.PartPlacement) { - pl.clear(); - } - - for (ui32 dcIdx = 0; dcIdx < 3; ++dcIdx) { - ui32 replCnt = is2x2 ? 2 : 1; - ui32 startIdx = dcIdx * 3; - for (ui32 replIdx = 0; replIdx < replCnt; ++replIdx) { - ui32 orderIdx = (startIdxInDc[dcIdx] + replIdx) % 3; - ui32 diskIdx = startIdx + orderIdx; - args.PartPlacement[diskIdx].push_back(dcIdx + 1); - } - } - - if (wipedDc >= 0) { - ui32 startIdx = wipedDc * 3; - for (ui32 orderIdx = 0; orderIdx < 3; ++orderIdx) { - ui32 diskIdx = startIdx + orderIdx; - args.PartPlacement[diskIdx].clear(); - } - } - - for (ui32 bit = 1, idx = 0; bit < errorMask; idx++, bit <<= 1) { - args.ErrorVDisks[idx] = (errorMask & bit); - } -} - -enum class EFaultToleranceCase { - Ok, - Fallback -}; - -EFaultToleranceCase GetFaultToleranceCaseForBlock4Plus2(const TDSProxyEnv &env, const TTestArgs &args) { - UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), args.ErrorVDisks.size()); - UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), env.Info->Type.BlobSubgroupSize()); - TSubgroupPartLayout layout; - for (ui32 vdiskIdx = 0; vdiskIdx < env.Info->Type.BlobSubgroupSize(); ++vdiskIdx) { - if (!args.ErrorVDisks[vdiskIdx]) { - auto [_, idxInSubgroup] = TVDiskPointer::GetVDiskIdx(vdiskIdx).GetIndecies(env, args.OriginalId.Hash()); - for (ui64 partId : args.PartPlacement[idxInSubgroup]) { - layout.AddItem(idxInSubgroup, partId - 1, env.Info->Type); - } - } - } - if (layout.CountEffectiveReplicas(env.Info->Type) == env.Info->Type.TotalPartCount()) { - return EFaultToleranceCase::Ok; - } else { - return EFaultToleranceCase::Fallback; - } -} - - -EFaultToleranceCase GetFaultToleranceCaseForMirror3dc(const TDSProxyEnv &env, const TTestArgs &args) { - UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), args.ErrorVDisks.size()); - UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), env.Info->Type.BlobSubgroupSize()); - TSubgroupPartLayout layout; - constexpr ui32 dcCnt = 3; - ui32 replInDc[dcCnt] = {0, 0, 0}; - for (ui32 vdiskIdx = 0; vdiskIdx < env.Info->Type.BlobSubgroupSize(); ++vdiskIdx) { - if (!args.ErrorVDisks[vdiskIdx] && args.PartPlacement[vdiskIdx]) { - ui32 dcIdx = vdiskIdx / 3; - replInDc[dcIdx]++; - } - } - ui32 x2cnt = 0; - for (ui32 dcIdx = 0; dcIdx < dcCnt; ++dcIdx) { - x2cnt += (replInDc[dcIdx] >= 2); - } - if ((replInDc[0] && replInDc[1] && replInDc[2]) || x2cnt >= 2) { - return EFaultToleranceCase::Ok; - } else { - return EFaultToleranceCase::Fallback; - } -} - -void ConductFaultTolerance(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, - EFaultToleranceCase faultCase) -{ - CTEST << "ConductFallbackPatch: Start\n"; - ConductVPatchEvents(runtime, env, args, EVPatchCase::Custom); - if (faultCase == EFaultToleranceCase::Fallback) { - ConductFallbackPatch(runtime, args); - } else { - ReceivePatchResult(runtime, args, NKikimrProto::OK); - } - CTEST << "ConductFallbackPatch: Finish\n"; -} - -void RunFaultToleranceBlock4Plus2(TTestBasicRuntime &runtime, const TTestArgs &args) { - TDSProxyEnv env; - env.Configure(runtime, args.GType, args.CurrentGroupId, 0); - TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); + ConductVPatch(runtime, env, args, vpatchCase); +} + +void SetLogPriorities(TTestBasicRuntime &runtime) { + bool IsVerbose = false; + if (IsVerbose) { + runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); + runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); + } + runtime.SetLogPriority(NKikimrServices::BS_PROXY, NLog::PRI_CRIT); + runtime.SetLogPriority(NKikimrServices::BS_QUEUE, NLog::PRI_CRIT); +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TDSProxyPatchTest) { + +template <typename ECase> +void RunGeneralTest(void(*runner)(TTestBasicRuntime &runtime, const TTestArgs &args, ECase testCase), + TErasureType::TErasureType::EErasureSpecies erasure, + ECase testCase) +{ + TTestArgs args; + args.MakeDefault(erasure, true); + TTestBasicRuntime runtime(1, false); + SetLogPriorities(runtime); + SetupRuntime(runtime); + runner(runtime, args, testCase); +} + +#define Y_UNIT_TEST_NAIVE(naiveCase, erasure) \ + Y_UNIT_TEST(Naive ## naiveCase ## _ ## erasure) { \ + RunGeneralTest(RunNaivePatchTest, TErasureType::TErasureType:: erasure, ENaivePatchCase:: naiveCase); \ + } \ +// end Y_UNIT_TEST_NAIVE + +#define Y_UNIT_TEST_MOVED(movedCase, erasure) \ + Y_UNIT_TEST(Moved ## movedCase ## _ ## erasure) { \ + RunGeneralTest(RunMovedPatchTest, TErasureType::TErasureType:: erasure, EMovedPatchCase:: movedCase); \ + } \ +// end Y_UNIT_TEST_MOVED + +#define Y_UNIT_TEST_VPATCH(vpatchCase, erasure) \ + Y_UNIT_TEST(VPatch ## vpatchCase ## _ ## erasure) { \ + RunGeneralTest(RunVPatchTest, TErasureType::TErasureType:: erasure, EVPatchCase:: vpatchCase); \ + } \ +// end Y_UNIT_TEST_VPATCH + +#define Y_UNIT_TEST_PATCH_PACK(erasure) \ + Y_UNIT_TEST_NAIVE(Ok, erasure) \ + Y_UNIT_TEST_NAIVE(ErrorOnGetItem, erasure) \ + Y_UNIT_TEST_NAIVE(ErrorOnGet, erasure) \ + Y_UNIT_TEST_NAIVE(ErrorOnPut, erasure) \ + Y_UNIT_TEST_MOVED(Ok, erasure) \ + Y_UNIT_TEST_MOVED(Error, erasure) \ + Y_UNIT_TEST_VPATCH(Ok, erasure) \ + Y_UNIT_TEST_VPATCH(OneErrorAndAllPartExistInStart, erasure) \ + Y_UNIT_TEST_VPATCH(OnePartLostInStart, erasure) \ + Y_UNIT_TEST_VPATCH(DeadGroupInStart, erasure) \ + Y_UNIT_TEST_VPATCH(ErrorDuringVPatchDiff, erasure) \ +// end Y_UNIT_TEST_PATCH_PACK + + Y_UNIT_TEST_PATCH_PACK(ErasureNone) + Y_UNIT_TEST_PATCH_PACK(Erasure4Plus2Block) + Y_UNIT_TEST_PATCH_PACK(ErasureMirror3) + Y_UNIT_TEST_PATCH_PACK(ErasureMirror3dc) + +} + +//////////////////////////////////////////////////////////////////////////////// + +void MakeFaultToleranceArgsForBlock4Plus2(TTestArgs &args, i64 wiped1, i64 wiped2, + ui32 errorMask, i64 handoff1, i64 handoff2, i64 extraHandoff) +{ + args.ResetDefaultPlacement(); + if (wiped1 != -1) { + args.PartPlacement[wiped1].clear(); + } + if (wiped2 != -1) { + args.PartPlacement[wiped2].clear(); + } + if (handoff1 > 0) { + args.PartPlacement[6].push_back(handoff1); + } + if (handoff2 > 0) { + args.PartPlacement[7].push_back(handoff2); + } + if (extraHandoff < 0) { + args.PartPlacement[6].push_back(-extraHandoff); + } + if (extraHandoff > 0) { + args.PartPlacement[7].push_back(extraHandoff); + } + for (ui32 bit = 1, idx = 0; bit < errorMask; idx++, bit <<= 1) { + args.ErrorVDisks[idx] = (errorMask & bit); + } +} + +void MakeFaultToleranceArgsForMirror3dc(TTestArgs &args, bool is2x2, ui32 errorMask, i64 wipedDc, i64 startIdxInDc[]) { + args.ResetDefaultPlacement(); + for (auto &pl : args.PartPlacement) { + pl.clear(); + } + + for (ui32 dcIdx = 0; dcIdx < 3; ++dcIdx) { + ui32 replCnt = is2x2 ? 2 : 1; + ui32 startIdx = dcIdx * 3; + for (ui32 replIdx = 0; replIdx < replCnt; ++replIdx) { + ui32 orderIdx = (startIdxInDc[dcIdx] + replIdx) % 3; + ui32 diskIdx = startIdx + orderIdx; + args.PartPlacement[diskIdx].push_back(dcIdx + 1); + } + } + + if (wipedDc >= 0) { + ui32 startIdx = wipedDc * 3; + for (ui32 orderIdx = 0; orderIdx < 3; ++orderIdx) { + ui32 diskIdx = startIdx + orderIdx; + args.PartPlacement[diskIdx].clear(); + } + } + + for (ui32 bit = 1, idx = 0; bit < errorMask; idx++, bit <<= 1) { + args.ErrorVDisks[idx] = (errorMask & bit); + } +} + +enum class EFaultToleranceCase { + Ok, + Fallback +}; + +EFaultToleranceCase GetFaultToleranceCaseForBlock4Plus2(const TDSProxyEnv &env, const TTestArgs &args) { + UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), args.ErrorVDisks.size()); + UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), env.Info->Type.BlobSubgroupSize()); + TSubgroupPartLayout layout; + for (ui32 vdiskIdx = 0; vdiskIdx < env.Info->Type.BlobSubgroupSize(); ++vdiskIdx) { + if (!args.ErrorVDisks[vdiskIdx]) { + auto [_, idxInSubgroup] = TVDiskPointer::GetVDiskIdx(vdiskIdx).GetIndecies(env, args.OriginalId.Hash()); + for (ui64 partId : args.PartPlacement[idxInSubgroup]) { + layout.AddItem(idxInSubgroup, partId - 1, env.Info->Type); + } + } + } + if (layout.CountEffectiveReplicas(env.Info->Type) == env.Info->Type.TotalPartCount()) { + return EFaultToleranceCase::Ok; + } else { + return EFaultToleranceCase::Fallback; + } +} + + +EFaultToleranceCase GetFaultToleranceCaseForMirror3dc(const TDSProxyEnv &env, const TTestArgs &args) { + UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), args.ErrorVDisks.size()); + UNIT_ASSERT_VALUES_EQUAL(args.PartPlacement.size(), env.Info->Type.BlobSubgroupSize()); + TSubgroupPartLayout layout; + constexpr ui32 dcCnt = 3; + ui32 replInDc[dcCnt] = {0, 0, 0}; + for (ui32 vdiskIdx = 0; vdiskIdx < env.Info->Type.BlobSubgroupSize(); ++vdiskIdx) { + if (!args.ErrorVDisks[vdiskIdx] && args.PartPlacement[vdiskIdx]) { + ui32 dcIdx = vdiskIdx / 3; + replInDc[dcIdx]++; + } + } + ui32 x2cnt = 0; + for (ui32 dcIdx = 0; dcIdx < dcCnt; ++dcIdx) { + x2cnt += (replInDc[dcIdx] >= 2); + } + if ((replInDc[0] && replInDc[1] && replInDc[2]) || x2cnt >= 2) { + return EFaultToleranceCase::Ok; + } else { + return EFaultToleranceCase::Fallback; + } +} + +void ConductFaultTolerance(TTestBasicRuntime &runtime, const TDSProxyEnv &env, const TTestArgs &args, + EFaultToleranceCase faultCase) +{ + CTEST << "ConductFallbackPatch: Start\n"; + ConductVPatchEvents(runtime, env, args, EVPatchCase::Custom); + if (faultCase == EFaultToleranceCase::Fallback) { + ConductFallbackPatch(runtime, args); + } else { + ReceivePatchResult(runtime, args, NKikimrProto::OK); + } + CTEST << "ConductFallbackPatch: Finish\n"; +} + +void RunFaultToleranceBlock4Plus2(TTestBasicRuntime &runtime, const TTestArgs &args) { + TDSProxyEnv env; + env.Configure(runtime, args.GType, args.CurrentGroupId, 0); + TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); std::unique_ptr<IActor> patchActor = env.CreatePatchRequestActor(patch, true); runtime.Register(patchActor.release()); - ConductFaultTolerance(runtime, env, args, GetFaultToleranceCaseForBlock4Plus2(env, args)); -} - -void RunFaultToleranceMirror3dc(TTestBasicRuntime &runtime, const TTestArgs &args) { - TDSProxyEnv env; - env.Configure(runtime, args.GType, args.CurrentGroupId, 0); - TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); + ConductFaultTolerance(runtime, env, args, GetFaultToleranceCaseForBlock4Plus2(env, args)); +} + +void RunFaultToleranceMirror3dc(TTestBasicRuntime &runtime, const TTestArgs &args) { + TDSProxyEnv env; + env.Configure(runtime, args.GType, args.CurrentGroupId, 0); + TEvBlobStorage::TEvPatch::TPtr patch = CreatePatch(runtime, env, args); std::unique_ptr<IActor> patchActor = env.CreatePatchRequestActor(patch, true); runtime.Register(patchActor.release()); - ConductFaultTolerance(runtime, env, args, GetFaultToleranceCaseForMirror3dc(env, args)); -} - - -Y_UNIT_TEST_SUITE(TDSProxyFaultTolerancePatchTest) { - Y_UNIT_TEST(mirror3dc) { - TBlobStorageGroupType type = TBlobStorageGroupType::ErasureMirror3dc; - ui64 errorMaskEnd = (1 << type.BlobSubgroupSize()); - - TTestArgs args; - args.MakeDefault(type); - for (bool is2x2 : {false, true}) { - for (i64 wipedDc = -1; wipedDc < 3; ++wipedDc) { - //for (i64 i = 0; i < 27; ++i) { // don't use because this test alreade is very fat - i64 startIdxInDC[3] = {0, 0, 0}; // {i % 3, i / 3 % 3, i / 9}; - TTestBasicRuntime runtime; - SetupRuntime(runtime); - for (ui32 errorMask = 0; errorMask < errorMaskEnd; ++errorMask) { - MakeFaultToleranceArgsForMirror3dc(args, is2x2, errorMask, wipedDc, startIdxInDC); - RunFaultToleranceMirror3dc(runtime, args); - } - //} - } - } - } - - - Y_UNIT_TEST(block42) { - TBlobStorageGroupType type = TBlobStorageGroupType::Erasure4Plus2Block; - ui64 errorMaskEnd = (1 << type.BlobSubgroupSize()); - - TTestArgs args; - args.MakeDefault(type); - for (i64 wiped1 = -1; wiped1 < type.TotalPartCount(); ++wiped1) { - for (i64 wiped2 = -1; wiped2 < type.TotalPartCount(); ++wiped2) { - if (wiped1 != -1 && wiped1 >= wiped2) { - continue; - } - TTestBasicRuntime runtime; - SetupRuntime(runtime); - for (ui32 errorMask = 0; errorMask < errorMaskEnd; ++errorMask) { - for (i64 extra = -type.TotalPartCount(); extra <= type.TotalPartCount(); extra++) { - if (extra == wiped1 + 1 || extra == wiped2 + 1) { - continue; - } - if (-extra == wiped1 + 1 || -extra == wiped2 + 1) { - continue; - } - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, 0, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped1 + 1, 0, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped2 + 1, 0, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, wiped1 + 1, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, wiped2 + 1, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped1 + 1, wiped2 + 1, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped2 + 1, wiped1 + 1, extra); - RunFaultToleranceBlock4Plus2(runtime, args); - } - } - } - } - } -} - -} // NDSProxyPatchTest -} // NKikimr + ConductFaultTolerance(runtime, env, args, GetFaultToleranceCaseForMirror3dc(env, args)); +} + + +Y_UNIT_TEST_SUITE(TDSProxyFaultTolerancePatchTest) { + Y_UNIT_TEST(mirror3dc) { + TBlobStorageGroupType type = TBlobStorageGroupType::ErasureMirror3dc; + ui64 errorMaskEnd = (1 << type.BlobSubgroupSize()); + + TTestArgs args; + args.MakeDefault(type); + for (bool is2x2 : {false, true}) { + for (i64 wipedDc = -1; wipedDc < 3; ++wipedDc) { + //for (i64 i = 0; i < 27; ++i) { // don't use because this test alreade is very fat + i64 startIdxInDC[3] = {0, 0, 0}; // {i % 3, i / 3 % 3, i / 9}; + TTestBasicRuntime runtime; + SetupRuntime(runtime); + for (ui32 errorMask = 0; errorMask < errorMaskEnd; ++errorMask) { + MakeFaultToleranceArgsForMirror3dc(args, is2x2, errorMask, wipedDc, startIdxInDC); + RunFaultToleranceMirror3dc(runtime, args); + } + //} + } + } + } + + + Y_UNIT_TEST(block42) { + TBlobStorageGroupType type = TBlobStorageGroupType::Erasure4Plus2Block; + ui64 errorMaskEnd = (1 << type.BlobSubgroupSize()); + + TTestArgs args; + args.MakeDefault(type); + for (i64 wiped1 = -1; wiped1 < type.TotalPartCount(); ++wiped1) { + for (i64 wiped2 = -1; wiped2 < type.TotalPartCount(); ++wiped2) { + if (wiped1 != -1 && wiped1 >= wiped2) { + continue; + } + TTestBasicRuntime runtime; + SetupRuntime(runtime); + for (ui32 errorMask = 0; errorMask < errorMaskEnd; ++errorMask) { + for (i64 extra = -type.TotalPartCount(); extra <= type.TotalPartCount(); extra++) { + if (extra == wiped1 + 1 || extra == wiped2 + 1) { + continue; + } + if (-extra == wiped1 + 1 || -extra == wiped2 + 1) { + continue; + } + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, 0, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped1 + 1, 0, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped2 + 1, 0, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, wiped1 + 1, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, 0, wiped2 + 1, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped1 + 1, wiped2 + 1, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + MakeFaultToleranceArgsForBlock4Plus2(args, wiped1, wiped2, errorMask, wiped2 + 1, wiped1 + 1, extra); + RunFaultToleranceBlock4Plus2(runtime, args); + } + } + } + } + } +} + +} // NDSProxyPatchTest +} // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp index d2d39a35504..f1b6e602f82 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp @@ -1,15 +1,15 @@ #include "defs.h" -#include "dsproxy_vdisk_mock_ut.h" -#include "dsproxy_env_mock_ut.h" +#include "dsproxy_vdisk_mock_ut.h" +#include "dsproxy_env_mock_ut.h" #include <ydb/core/blobstorage/dsproxy/dsproxy_put_impl.h> #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> - + #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/actor_helpers.h> #include <library/cpp/containers/stack_vector/stack_vec.h> -#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/registar.h> namespace NKikimr { namespace NDSProxyPutTest { @@ -25,264 +25,264 @@ TString AlphaData(ui32 size) { return data; } -void TestPutMaxPartCountOnHandoff(TErasureType::EErasureSpecies erasureSpecies) { - TActorSystemStub actorSystemStub; - i32 size = 786; - TLogoBlobID blobId(72075186224047637, 1, 863, 1, size, 24576); - TString data = AlphaData(size); - - const ui32 groupId = 0; - TBlobStorageGroupType groupType(erasureSpecies); - const ui32 domainCount = groupType.BlobSubgroupSize();; - - TGroupMock group(groupId, erasureSpecies, domainCount, 1); +void TestPutMaxPartCountOnHandoff(TErasureType::EErasureSpecies erasureSpecies) { + TActorSystemStub actorSystemStub; + i32 size = 786; + TLogoBlobID blobId(72075186224047637, 1, 863, 1, size, 24576); + TString data = AlphaData(size); + + const ui32 groupId = 0; + TBlobStorageGroupType groupType(erasureSpecies); + const ui32 domainCount = groupType.BlobSubgroupSize();; + + TGroupMock group(groupId, erasureSpecies, domainCount, 1); TIntrusivePtr<TGroupQueues> groupQueues = group.MakeGroupQueues(); - - TIntrusivePtr<NMonitoring::TDynamicCounters> counters(new NMonitoring::TDynamicCounters()); + + TIntrusivePtr<NMonitoring::TDynamicCounters> counters(new NMonitoring::TDynamicCounters()); TIntrusivePtr<TDsProxyNodeMon> nodeMon(new TDsProxyNodeMon(counters, true)); - TIntrusivePtr<TBlobStorageGroupProxyMon> mon(new TBlobStorageGroupProxyMon(counters, counters, counters, + TIntrusivePtr<TBlobStorageGroupProxyMon> mon(new TBlobStorageGroupProxyMon(counters, counters, counters, group.GetInfo(), nodeMon, false)); - - TLogContext logCtx(NKikimrServices::BS_PROXY_PUT, false); - logCtx.LogAcc.IsLogEnabled = false; - - const ui32 hash = blobId.Hash(); - const ui32 totalvd = group.GetInfo()->Type.BlobSubgroupSize(); - const ui32 totalParts = group.GetInfo()->Type.TotalPartCount(); - Y_VERIFY(blobId.BlobSize() == data.size()); - Y_VERIFY(totalvd >= totalParts); - TBlobStorageGroupInfo::TServiceIds vDisksSvc; - TBlobStorageGroupInfo::TVDiskIds vDisksId; - group.GetInfo()->PickSubgroup(hash, &vDisksId, &vDisksSvc); - - TString encryptedData = data; - char *dataBytes = encryptedData.Detach(); - Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *group.GetInfo()); - - TBatchedVec<TDataPartSet> partSetSingleton(1); - partSetSingleton[0].Parts.resize(totalParts); - group.GetInfo()->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, partSetSingleton[0]); - - TEvBlobStorage::TEvPut ev(blobId, data, TInstant::Max(), NKikimrBlobStorage::TabletLog, - TEvBlobStorage::TEvPut::TacticDefault); - - TPutImpl putImpl(group.GetInfo(), groupQueues, &ev, mon, false); - - for (ui32 idx = 0; idx < domainCount; ++idx) { - group.SetPredictedDelayNs(idx, 1); - } - group.SetPredictedDelayNs(7, 10); - + + TLogContext logCtx(NKikimrServices::BS_PROXY_PUT, false); + logCtx.LogAcc.IsLogEnabled = false; + + const ui32 hash = blobId.Hash(); + const ui32 totalvd = group.GetInfo()->Type.BlobSubgroupSize(); + const ui32 totalParts = group.GetInfo()->Type.TotalPartCount(); + Y_VERIFY(blobId.BlobSize() == data.size()); + Y_VERIFY(totalvd >= totalParts); + TBlobStorageGroupInfo::TServiceIds vDisksSvc; + TBlobStorageGroupInfo::TVDiskIds vDisksId; + group.GetInfo()->PickSubgroup(hash, &vDisksId, &vDisksSvc); + + TString encryptedData = data; + char *dataBytes = encryptedData.Detach(); + Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *group.GetInfo()); + + TBatchedVec<TDataPartSet> partSetSingleton(1); + partSetSingleton[0].Parts.resize(totalParts); + group.GetInfo()->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, partSetSingleton[0]); + + TEvBlobStorage::TEvPut ev(blobId, data, TInstant::Max(), NKikimrBlobStorage::TabletLog, + TEvBlobStorage::TEvPut::TacticDefault); + + TPutImpl putImpl(group.GetInfo(), groupQueues, &ev, mon, false); + + for (ui32 idx = 0; idx < domainCount; ++idx) { + group.SetPredictedDelayNs(idx, 1); + } + group.SetPredictedDelayNs(7, 10); + TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - putImpl.GenerateInitialRequests(logCtx, partSetSingleton, vPuts); - group.SetError(0, NKikimrProto::ERROR); - - TPutImpl::TPutResultVec putResults; - - TVector<ui32> diskSequence = {0, 7, 7, 7, 7, 6, 3, 4, 5, 1, 2}; - TVector<ui32> slowDiskSequence = {3, 4, 5, 6, 1, 2}; - - for (ui32 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { - ui32 nextVPut = vPutIdx; - ui32 diskPos = (ui32)-1; - for (ui32 i = vPutIdx; i < vPuts.size(); ++i) { - auto& record = vPuts[i]->Record; - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - ui32 diskIdx = group.VDiskIdx(vDiskId); - auto it = Find(diskSequence, diskIdx); - if (it != diskSequence.end()) { - ui32 pos = it - diskSequence.begin(); - if (pos < diskPos) { - nextVPut = i; - diskPos = pos; - } - } - } - CTEST << "vdisk exp# " << (diskSequence.size() ? diskSequence.front() : -1) << " get# " << group.VDiskIdx(VDiskIDFromVDiskID(vPuts[nextVPut]->Record.GetVDiskID())) << Endl; - if (diskPos != (ui32)-1) { - diskSequence.erase(diskSequence.begin() + diskPos); - } - std::swap(vPuts[vPutIdx], vPuts[nextVPut]); - - for (ui32 idx = 0; idx < domainCount; ++idx) { - group.SetPredictedDelayNs(idx, 1); - } - if (vPutIdx < slowDiskSequence.size()) { - group.SetPredictedDelayNs(slowDiskSequence[vPutIdx], 10); - } - + putImpl.GenerateInitialRequests(logCtx, partSetSingleton, vPuts); + group.SetError(0, NKikimrProto::ERROR); + + TPutImpl::TPutResultVec putResults; + + TVector<ui32> diskSequence = {0, 7, 7, 7, 7, 6, 3, 4, 5, 1, 2}; + TVector<ui32> slowDiskSequence = {3, 4, 5, 6, 1, 2}; + + for (ui32 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { + ui32 nextVPut = vPutIdx; + ui32 diskPos = (ui32)-1; + for (ui32 i = vPutIdx; i < vPuts.size(); ++i) { + auto& record = vPuts[i]->Record; + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + ui32 diskIdx = group.VDiskIdx(vDiskId); + auto it = Find(diskSequence, diskIdx); + if (it != diskSequence.end()) { + ui32 pos = it - diskSequence.begin(); + if (pos < diskPos) { + nextVPut = i; + diskPos = pos; + } + } + } + CTEST << "vdisk exp# " << (diskSequence.size() ? diskSequence.front() : -1) << " get# " << group.VDiskIdx(VDiskIDFromVDiskID(vPuts[nextVPut]->Record.GetVDiskID())) << Endl; + if (diskPos != (ui32)-1) { + diskSequence.erase(diskSequence.begin() + diskPos); + } + std::swap(vPuts[vPutIdx], vPuts[nextVPut]); + + for (ui32 idx = 0; idx < domainCount; ++idx) { + group.SetPredictedDelayNs(idx, 1); + } + if (vPutIdx < slowDiskSequence.size()) { + group.SetPredictedDelayNs(slowDiskSequence[vPutIdx], 10); + } + TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> nextVPuts; - - TEvBlobStorage::TEvVPut& vPut = *vPuts[vPutIdx]; + + TEvBlobStorage::TEvVPut& vPut = *vPuts[vPutIdx]; TActorId sender; - TEvBlobStorage::TEvVPutResult vPutResult; - NKikimrProto::EReplyStatus status = group.OnVPut(vPut); + TEvBlobStorage::TEvVPutResult vPutResult; + NKikimrProto::EReplyStatus status = group.OnVPut(vPut); vPutResult.MakeError(status, TString(), vPut.Record); - - putImpl.OnVPutEventResult(logCtx, sender, vPutResult, nextVPuts, putResults); - - if (putResults.size()) { - break; - } - + + putImpl.OnVPutEventResult(logCtx, sender, vPutResult, nextVPuts, putResults); + + if (putResults.size()) { + break; + } + std::move(nextVPuts.begin(), nextVPuts.end(), std::back_inserter(vPuts)); - } - UNIT_ASSERT(putResults.size() == 1); - auto& [_, result] = putResults.front(); - UNIT_ASSERT(result->Status == NKikimrProto::OK); - UNIT_ASSERT(result->Id == blobId); - UNIT_ASSERT(putImpl.GetHandoffPartsSent() == 7); -} - -Y_UNIT_TEST(TestBlock42MaxPartCountOnHandoff) { - TestPutMaxPartCountOnHandoff(TErasureType::Erasure4Plus2Block); -} - -enum ETestPutAllOkMode { - TPAOM_VPUT, - TPAOM_VMULTIPUT -}; - -template <TErasureType::EErasureSpecies ErasureSpecies, ETestPutAllOkMode TestMode> -struct TTestPutAllOk { - static constexpr ui32 GroupId = 0; - static constexpr i32 DataSize = 100500; - static constexpr bool IsVPut = TestMode == TPAOM_VPUT; - static constexpr ui64 BlobCount = IsVPut ? 1 : 2; - static constexpr ui32 MaxIterations = 10000; - - TActorSystemStub ActorSystemStub; - TBlobStorageGroupType GroupType; - TGroupMock Group; + } + UNIT_ASSERT(putResults.size() == 1); + auto& [_, result] = putResults.front(); + UNIT_ASSERT(result->Status == NKikimrProto::OK); + UNIT_ASSERT(result->Id == blobId); + UNIT_ASSERT(putImpl.GetHandoffPartsSent() == 7); +} + +Y_UNIT_TEST(TestBlock42MaxPartCountOnHandoff) { + TestPutMaxPartCountOnHandoff(TErasureType::Erasure4Plus2Block); +} + +enum ETestPutAllOkMode { + TPAOM_VPUT, + TPAOM_VMULTIPUT +}; + +template <TErasureType::EErasureSpecies ErasureSpecies, ETestPutAllOkMode TestMode> +struct TTestPutAllOk { + static constexpr ui32 GroupId = 0; + static constexpr i32 DataSize = 100500; + static constexpr bool IsVPut = TestMode == TPAOM_VPUT; + static constexpr ui64 BlobCount = IsVPut ? 1 : 2; + static constexpr ui32 MaxIterations = 10000; + + TActorSystemStub ActorSystemStub; + TBlobStorageGroupType GroupType; + TGroupMock Group; TIntrusivePtr<TGroupQueues> GroupQueues; - TBatchedVec<TLogoBlobID> BlobIds; - TString Data; + TBatchedVec<TLogoBlobID> BlobIds; + TString Data; - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TIntrusivePtr<TDsProxyNodeMon> NodeMon; - TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + TIntrusivePtr<TDsProxyNodeMon> NodeMon; + TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; - TLogContext LogCtx; + TLogContext LogCtx; - TBatchedVec<TDataPartSet> PartSets; + TBatchedVec<TDataPartSet> PartSets; - TStackVec<ui32, 16> CheckStack; + TStackVec<ui32, 16> CheckStack; - TTestPutAllOk() - : GroupType(ErasureSpecies) - , Group(GroupId, ErasureSpecies, GroupType.BlobSubgroupSize(), 1) + TTestPutAllOk() + : GroupType(ErasureSpecies) + , Group(GroupId, ErasureSpecies, GroupType.BlobSubgroupSize(), 1) , GroupQueues(Group.MakeGroupQueues()) - , BlobIds({TLogoBlobID(743284823, 10, 12345, 0, DataSize, 0), TLogoBlobID(743284823, 9, 12346, 0, DataSize, 0)}) - , Data(AlphaData(DataSize)) - , Counters(new NMonitoring::TDynamicCounters()) - , NodeMon(new TDsProxyNodeMon(Counters, true)) + , BlobIds({TLogoBlobID(743284823, 10, 12345, 0, DataSize, 0), TLogoBlobID(743284823, 9, 12346, 0, DataSize, 0)}) + , Data(AlphaData(DataSize)) + , Counters(new NMonitoring::TDynamicCounters()) + , NodeMon(new TDsProxyNodeMon(Counters, true)) , Mon(new TBlobStorageGroupProxyMon(Counters, Counters, Counters, Group.GetInfo(), NodeMon, false)) - , LogCtx(NKikimrServices::BS_PROXY_PUT, false) - , PartSets(BlobCount) - { - LogCtx.LogAcc.IsLogEnabled = false; - - const ui32 totalvd = Group.GetInfo()->Type.BlobSubgroupSize(); - const ui32 totalParts = Group.GetInfo()->Type.TotalPartCount(); - Y_VERIFY(totalvd >= totalParts); - - for (ui64 blobIdx = 0; blobIdx < BlobCount; ++blobIdx) { - TLogoBlobID blobId = BlobIds[blobIdx]; - Y_VERIFY(blobId.BlobSize() == Data.size()); - TBlobStorageGroupInfo::TServiceIds vDisksSvc; - TBlobStorageGroupInfo::TVDiskIds vDisksId; - const ui32 hash = blobId.Hash(); - Group.GetInfo()->PickSubgroup(hash, &vDisksId, &vDisksSvc); - - TString encryptedData = Data; - char *dataBytes = encryptedData.Detach(); - Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *Group.GetInfo()); - - PartSets[blobIdx].Parts.resize(totalParts); - Group.GetInfo()->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, PartSets[blobIdx]); - } - } + , LogCtx(NKikimrServices::BS_PROXY_PUT, false) + , PartSets(BlobCount) + { + LogCtx.LogAcc.IsLogEnabled = false; + + const ui32 totalvd = Group.GetInfo()->Type.BlobSubgroupSize(); + const ui32 totalParts = Group.GetInfo()->Type.TotalPartCount(); + Y_VERIFY(totalvd >= totalParts); + + for (ui64 blobIdx = 0; blobIdx < BlobCount; ++blobIdx) { + TLogoBlobID blobId = BlobIds[blobIdx]; + Y_VERIFY(blobId.BlobSize() == Data.size()); + TBlobStorageGroupInfo::TServiceIds vDisksSvc; + TBlobStorageGroupInfo::TVDiskIds vDisksId; + const ui32 hash = blobId.Hash(); + Group.GetInfo()->PickSubgroup(hash, &vDisksId, &vDisksSvc); + + TString encryptedData = Data; + char *dataBytes = encryptedData.Detach(); + Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *Group.GetInfo()); + + PartSets[blobIdx].Parts.resize(totalParts); + Group.GetInfo()->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, PartSets[blobIdx]); + } + } void InitVPutResults(TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> &vPuts, TDeque<std::unique_ptr<TEvBlobStorage::TEvVPutResult>> &vPutResults) - { + { vPutResults.resize(vPuts.size()); for (ui32 vPutIdx = 0; vPutIdx < vPuts.size(); ++vPutIdx) { TEvBlobStorage::TEvVPut &vPut = *vPuts[vPutIdx]; - NKikimrProto::EReplyStatus status = Group.OnVPut(vPut); + NKikimrProto::EReplyStatus status = Group.OnVPut(vPut); UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); vPutResults[vPutIdx].reset(new TEvBlobStorage::TEvVPutResult()); TEvBlobStorage::TEvVPutResult &vPutResult = *vPutResults[vPutIdx]; vPutResult.MakeError(status, TString(), vPut.Record); } - } + } void InitVPutResults(TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> &vMultiPuts, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult>> &vMultiPutResults) - { - vMultiPutResults.resize(vMultiPuts.size()); - for (ui32 vMultiPutIdx = 0; vMultiPutIdx < vMultiPuts.size(); ++vMultiPutIdx) { - TEvBlobStorage::TEvVMultiPut &vMultiPut = *vMultiPuts[vMultiPutIdx]; - TVector<NKikimrProto::EReplyStatus> statuses = Group.OnVMultiPut(vMultiPut); - for (auto status : statuses) { - UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); - } + { + vMultiPutResults.resize(vMultiPuts.size()); + for (ui32 vMultiPutIdx = 0; vMultiPutIdx < vMultiPuts.size(); ++vMultiPutIdx) { + TEvBlobStorage::TEvVMultiPut &vMultiPut = *vMultiPuts[vMultiPutIdx]; + TVector<NKikimrProto::EReplyStatus> statuses = Group.OnVMultiPut(vMultiPut); + for (auto status : statuses) { + UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); + } vMultiPutResults[vMultiPutIdx].reset(new TEvBlobStorage::TEvVMultiPutResult()); - Y_VERIFY(vMultiPut.Record.ItemsSize() == statuses.size()); - TEvBlobStorage::TEvVMultiPutResult &vMultiPutResult = *vMultiPutResults[vMultiPutIdx]; + Y_VERIFY(vMultiPut.Record.ItemsSize() == statuses.size()); + TEvBlobStorage::TEvVMultiPutResult &vMultiPutResult = *vMultiPutResults[vMultiPutIdx]; vMultiPutResult.MakeError(NKikimrProto::OK, TString(), vMultiPut.Record); - for (ui64 itemIdx = 0; itemIdx < statuses.size(); ++itemIdx) { - NKikimrBlobStorage::TVMultiPutResultItem &item = *vMultiPutResult.Record.MutableItems(itemIdx); - NKikimrProto::EReplyStatus status = statuses[itemIdx]; - item.SetStatus(status); - } - Y_VERIFY(vMultiPutResult.Record.ItemsSize() == statuses.size()); - } - } - - template <typename TVPutResultEvent> + for (ui64 itemIdx = 0; itemIdx < statuses.size(); ++itemIdx) { + NKikimrBlobStorage::TVMultiPutResultItem &item = *vMultiPutResult.Record.MutableItems(itemIdx); + NKikimrProto::EReplyStatus status = statuses[itemIdx]; + item.SetStatus(status); + } + Y_VERIFY(vMultiPutResult.Record.ItemsSize() == statuses.size()); + } + } + + template <typename TVPutResultEvent> void PermutateVPutResults(ui64 resIdx, bool &isAborted, TDeque<std::unique_ptr<TVPutResultEvent>> &vPutResults) { - // select result in range [resIdx, vPutResults.size()) - if (resIdx + 1 < CheckStack.size()) { - ui32 tgt = CheckStack[resIdx]; - UNIT_ASSERT(tgt < vPutResults.size()); - UNIT_ASSERT(tgt >= resIdx); - std::swap(vPutResults[resIdx], vPutResults[tgt]); - } else if (resIdx + 1 == CheckStack.size()) { - ui32 &tgt = CheckStack[resIdx]; - tgt++; - if (tgt >= vPutResults.size()) { - isAborted = true; - CheckStack.pop_back(); - return; - } - } else { - CheckStack.push_back(resIdx); - } - } - - bool Step(TPutImpl &putImpl, + // select result in range [resIdx, vPutResults.size()) + if (resIdx + 1 < CheckStack.size()) { + ui32 tgt = CheckStack[resIdx]; + UNIT_ASSERT(tgt < vPutResults.size()); + UNIT_ASSERT(tgt >= resIdx); + std::swap(vPutResults[resIdx], vPutResults[tgt]); + } else if (resIdx + 1 == CheckStack.size()) { + ui32 &tgt = CheckStack[resIdx]; + tgt++; + if (tgt >= vPutResults.size()) { + isAborted = true; + CheckStack.pop_back(); + return; + } + } else { + CheckStack.push_back(resIdx); + } + } + + bool Step(TPutImpl &putImpl, TDeque<std::unique_ptr<TEvBlobStorage::TEvVPutResult>> &vPutResults, - TPutImpl::TPutResultVec &putResults) - { + TPutImpl::TPutResultVec &putResults) + { bool isAborted = false; - for (ui64 resIdx = 0; resIdx < vPutResults.size(); ++resIdx) { - PermutateVPutResults(resIdx, isAborted, vPutResults); - if (isAborted) { - break; + for (ui64 resIdx = 0; resIdx < vPutResults.size(); ++resIdx) { + PermutateVPutResults(resIdx, isAborted, vPutResults); + if (isAborted) { + break; } TActorId sender; TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts2; - putImpl.OnVPutEventResult(LogCtx, sender, *vPutResults[resIdx], vPuts2, putResults); - if (putResults.size()) { + putImpl.OnVPutEventResult(LogCtx, sender, *vPutResults[resIdx], vPuts2, putResults); + if (putResults.size()) { break; } - + for (ui32 vPutIdx = 0; vPutIdx < vPuts2.size(); ++vPutIdx) { TEvBlobStorage::TEvVPut &vPut = *vPuts2[vPutIdx]; - NKikimrProto::EReplyStatus status = Group.OnVPut(vPut); + NKikimrProto::EReplyStatus status = Group.OnVPut(vPut); UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); vPutResults.emplace_back(new TEvBlobStorage::TEvVPutResult()); TEvBlobStorage::TEvVPutResult &vPutResult = *vPutResults.back(); @@ -290,158 +290,158 @@ struct TTestPutAllOk { } } - return isAborted; - } - - bool Step(TPutImpl &putImpl, + return isAborted; + } + + bool Step(TPutImpl &putImpl, TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult>> &vMultiPutResults, - TPutImpl::TPutResultVec &putResults) - { - bool isAborted = false; - for (ui64 resIdx = 0; resIdx < vMultiPutResults.size(); ++resIdx) { - PermutateVPutResults(resIdx, isAborted, vMultiPutResults); - if (isAborted) { + TPutImpl::TPutResultVec &putResults) + { + bool isAborted = false; + for (ui64 resIdx = 0; resIdx < vMultiPutResults.size(); ++resIdx) { + PermutateVPutResults(resIdx, isAborted, vMultiPutResults); + if (isAborted) { break; } - + TActorId sender; TDeque<std::unique_ptr<TEvBlobStorage::TEvVMultiPut>> vMultiPuts2; - putImpl.OnVPutEventResult(LogCtx, sender, *vMultiPutResults[resIdx], vMultiPuts2, putResults); - if (putResults.size() == BlobIds.size()) { - break; - } - - for (ui32 vMultiPutIdx = 0; vMultiPutIdx < vMultiPuts2.size(); ++vMultiPutIdx) { - TEvBlobStorage::TEvVMultiPut &vMultiPut = *vMultiPuts2[vMultiPutIdx]; - TVector<NKikimrProto::EReplyStatus> statuses = Group.OnVMultiPut(vMultiPut); - for (auto status : statuses) { - UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); - } - + putImpl.OnVPutEventResult(LogCtx, sender, *vMultiPutResults[resIdx], vMultiPuts2, putResults); + if (putResults.size() == BlobIds.size()) { + break; + } + + for (ui32 vMultiPutIdx = 0; vMultiPutIdx < vMultiPuts2.size(); ++vMultiPutIdx) { + TEvBlobStorage::TEvVMultiPut &vMultiPut = *vMultiPuts2[vMultiPutIdx]; + TVector<NKikimrProto::EReplyStatus> statuses = Group.OnVMultiPut(vMultiPut); + for (auto status : statuses) { + UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::OK); + } + vMultiPutResults[vMultiPutIdx].reset(new TEvBlobStorage::TEvVMultiPutResult()); - Y_VERIFY(vMultiPut.Record.ItemsSize() == statuses.size()); - TEvBlobStorage::TEvVMultiPutResult &vMultiPutResult = *vMultiPutResults[vMultiPutIdx]; + Y_VERIFY(vMultiPut.Record.ItemsSize() == statuses.size()); + TEvBlobStorage::TEvVMultiPutResult &vMultiPutResult = *vMultiPutResults[vMultiPutIdx]; vMultiPutResult.MakeError(NKikimrProto::OK, TString(), vMultiPut.Record); - - for (ui64 itemIdx = 0; itemIdx < statuses.size(); ++itemIdx) { - auto &item = vMultiPut.Record.GetItems(itemIdx); - NKikimrProto::EReplyStatus status = statuses[itemIdx]; - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - - ui64 cookieValue = 0; - ui64 *cookie = nullptr; - if (item.HasCookie()) { - cookieValue = item.GetCookie(); - cookie = &cookieValue; - } - + + for (ui64 itemIdx = 0; itemIdx < statuses.size(); ++itemIdx) { + auto &item = vMultiPut.Record.GetItems(itemIdx); + NKikimrProto::EReplyStatus status = statuses[itemIdx]; + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + + ui64 cookieValue = 0; + ui64 *cookie = nullptr; + if (item.HasCookie()) { + cookieValue = item.GetCookie(); + cookie = &cookieValue; + } + vMultiPutResult.AddVPutResult(status, TString(), blobId, cookie); - } - } + } + } } - - return isAborted; + + return isAborted; } - void Run() { - using TVPutEvent = std::conditional_t<IsVPut, TEvBlobStorage::TEvVPut, TEvBlobStorage::TEvVMultiPut>; - using TVPutResultEvent = std::conditional_t<IsVPut, TEvBlobStorage::TEvVPutResult, - TEvBlobStorage::TEvVMultiPutResult>; - - ui64 i = 0; - for (; i < MaxIterations; ++i) { - Group.Wipe(); - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> events; - for (auto &blobId : BlobIds) { + void Run() { + using TVPutEvent = std::conditional_t<IsVPut, TEvBlobStorage::TEvVPut, TEvBlobStorage::TEvVMultiPut>; + using TVPutResultEvent = std::conditional_t<IsVPut, TEvBlobStorage::TEvVPutResult, + TEvBlobStorage::TEvVMultiPutResult>; + + ui64 i = 0; + for (; i < MaxIterations; ++i) { + Group.Wipe(); + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> events; + for (auto &blobId : BlobIds) { std::unique_ptr<TEvBlobStorage::TEvPut> vPut(new TEvBlobStorage::TEvPut(blobId, Data, TInstant::Max(), - NKikimrBlobStorage::TabletLog, TEvBlobStorage::TEvPut::TacticDefault)); - events.emplace_back(static_cast<TEventHandle<TEvBlobStorage::TEvPut> *>( + NKikimrBlobStorage::TabletLog, TEvBlobStorage::TEvPut::TacticDefault)); + events.emplace_back(static_cast<TEventHandle<TEvBlobStorage::TEvPut> *>( new IEventHandle(TActorId(), TActorId(), vPut.release()))); - } - - TMaybe<TPutImpl> putImpl; - TPutImpl::TPutResultVec putResults; - if constexpr (IsVPut) { - putImpl.ConstructInPlace(Group.GetInfo(), GroupQueues, events[0]->Get(), Mon, false); - } else { + } + + TMaybe<TPutImpl> putImpl; + TPutImpl::TPutResultVec putResults; + if constexpr (IsVPut) { + putImpl.ConstructInPlace(Group.GetInfo(), GroupQueues, events[0]->Get(), Mon, false); + } else { putImpl.ConstructInPlace(Group.GetInfo(), GroupQueues, events, Mon, - NKikimrBlobStorage::TabletLog, TEvBlobStorage::TEvPut::TacticDefault, false); - } - + NKikimrBlobStorage::TabletLog, TEvBlobStorage::TEvPut::TacticDefault, false); + } + TDeque<std::unique_ptr<TVPutEvent>> vPuts; - putImpl->GenerateInitialRequests(LogCtx, PartSets, vPuts); - UNIT_ASSERT(vPuts.size() == 6 || !IsVPut); + putImpl->GenerateInitialRequests(LogCtx, PartSets, vPuts); + UNIT_ASSERT(vPuts.size() == 6 || !IsVPut); TDeque<std::unique_ptr<TVPutResultEvent>> vPutResults; - InitVPutResults(vPuts, vPutResults); - - bool isAborted = Step(*putImpl, vPutResults, putResults); - if (!isAborted) { - UNIT_ASSERT(putResults.size() == BlobCount); - for (auto& [blobIdx, result] : putResults) { - UNIT_ASSERT(result->Status == NKikimrProto::OK); - UNIT_ASSERT(result->Id == BlobIds[blobIdx]); - } - } else { - if (CheckStack.size() == 0) { - break; - } - } - } - - UNIT_ASSERT(i != MaxIterations || !IsVPut); - } -}; - + InitVPutResults(vPuts, vPutResults); + + bool isAborted = Step(*putImpl, vPutResults, putResults); + if (!isAborted) { + UNIT_ASSERT(putResults.size() == BlobCount); + for (auto& [blobIdx, result] : putResults) { + UNIT_ASSERT(result->Status == NKikimrProto::OK); + UNIT_ASSERT(result->Id == BlobIds[blobIdx]); + } + } else { + if (CheckStack.size() == 0) { + break; + } + } + } + + UNIT_ASSERT(i != MaxIterations || !IsVPut); + } +}; + Y_UNIT_TEST(TestBlock42PutAllOk) { - TTestPutAllOk<TErasureType::Erasure4Plus2Block, TPAOM_VPUT>().Run(); + TTestPutAllOk<TErasureType::Erasure4Plus2Block, TPAOM_VPUT>().Run(); } -Y_UNIT_TEST(TestBlock42MultiPutAllOk) { - TTestPutAllOk<TErasureType::Erasure4Plus2Block, TPAOM_VMULTIPUT>().Run(); -} - -Y_UNIT_TEST(TestMirror3dcWith3x3MinLatencyMod) { - TTestBasicRuntime runtime; - SetupRuntime(runtime); - TDSProxyEnv env; - env.Configure(runtime, TErasureType::ErasureMirror3dc, 1, 0); - - i32 size = 786; - TLogoBlobID blobId(72075186224047637, 1, 863, 1, size, 24576); - TString data = AlphaData(size); - TEvBlobStorage::TEvPut ev(blobId, data, TInstant::Max(), NKikimrBlobStorage::TabletLog, - TEvBlobStorage::TEvPut::TacticMinLatency); - TPutImpl putImpl(env.Info, env.GroupQueues, &ev, env.Mon, true); +Y_UNIT_TEST(TestBlock42MultiPutAllOk) { + TTestPutAllOk<TErasureType::Erasure4Plus2Block, TPAOM_VMULTIPUT>().Run(); +} + +Y_UNIT_TEST(TestMirror3dcWith3x3MinLatencyMod) { + TTestBasicRuntime runtime; + SetupRuntime(runtime); + TDSProxyEnv env; + env.Configure(runtime, TErasureType::ErasureMirror3dc, 1, 0); + + i32 size = 786; + TLogoBlobID blobId(72075186224047637, 1, 863, 1, size, 24576); + TString data = AlphaData(size); + TEvBlobStorage::TEvPut ev(blobId, data, TInstant::Max(), NKikimrBlobStorage::TabletLog, + TEvBlobStorage::TEvPut::TacticMinLatency); + TPutImpl putImpl(env.Info, env.GroupQueues, &ev, env.Mon, true); TDeque<std::unique_ptr<TEvBlobStorage::TEvVPut>> vPuts; - - TLogContext logCtx(NKikimrServices::BS_PROXY_PUT, false); - logCtx.LogAcc.IsLogEnabled = false; - - const ui32 totalParts = env.Info->Type.TotalPartCount(); - TBatchedVec<TDataPartSet> partSetSingleton(1); - partSetSingleton[0].Parts.resize(totalParts); - - TString encryptedData = data; - char *dataBytes = encryptedData.Detach(); - Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *env.Info); - env.Info->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, partSetSingleton[0]); - putImpl.GenerateInitialRequests(logCtx, partSetSingleton, vPuts); - - UNIT_ASSERT_VALUES_EQUAL(vPuts.size(), 9); - using TVDiskIDTuple = decltype(std::declval<TVDiskID>().ConvertToTuple()); - THashSet<TVDiskIDTuple> vDiskIds; - for (auto &vPut : vPuts) { - TVDiskID vDiskId = VDiskIDFromVDiskID(vPut->Record.GetVDiskID()); - bool inserted = vDiskIds.insert(vDiskId.ConvertToTuple()).second; - UNIT_ASSERT(inserted); - } - for (ui32 diskOrderNumber = 0; diskOrderNumber < env.Info->Type.BlobSubgroupSize(); ++diskOrderNumber) { - TVDiskID vDiskId = env.Info->GetVDiskId(diskOrderNumber); - auto it = vDiskIds.find(vDiskId.ConvertToTuple()); - UNIT_ASSERT(it != vDiskIds.end()); - } -} - + + TLogContext logCtx(NKikimrServices::BS_PROXY_PUT, false); + logCtx.LogAcc.IsLogEnabled = false; + + const ui32 totalParts = env.Info->Type.TotalPartCount(); + TBatchedVec<TDataPartSet> partSetSingleton(1); + partSetSingleton[0].Parts.resize(totalParts); + + TString encryptedData = data; + char *dataBytes = encryptedData.Detach(); + Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), blobId, *env.Info); + env.Info->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedData, partSetSingleton[0]); + putImpl.GenerateInitialRequests(logCtx, partSetSingleton, vPuts); + + UNIT_ASSERT_VALUES_EQUAL(vPuts.size(), 9); + using TVDiskIDTuple = decltype(std::declval<TVDiskID>().ConvertToTuple()); + THashSet<TVDiskIDTuple> vDiskIds; + for (auto &vPut : vPuts) { + TVDiskID vDiskId = VDiskIDFromVDiskID(vPut->Record.GetVDiskID()); + bool inserted = vDiskIds.insert(vDiskId.ConvertToTuple()).second; + UNIT_ASSERT(inserted); + } + for (ui32 diskOrderNumber = 0; diskOrderNumber < env.Info->Type.BlobSubgroupSize(); ++diskOrderNumber) { + TVDiskID vDiskId = env.Info->GetVDiskId(diskOrderNumber); + auto it = vDiskIds.find(vDiskId.ConvertToTuple()); + UNIT_ASSERT(it != vDiskIds.end()); + } +} + } // Y_UNIT_TEST_SUITE TDSProxyPutTest } // namespace NDSProxyPutTest } // namespace NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_sequence_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_sequence_ut.cpp index fdd7a92ea30..8e6de594ad1 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_sequence_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_sequence_ut.cpp @@ -1,11 +1,11 @@ #include "defs.h" -#include "dsproxy_env_mock_ut.h" -#include "dsproxy_vdisk_mock_ut.h" -#include "dsproxy_test_state_ut.h" +#include "dsproxy_env_mock_ut.h" +#include "dsproxy_vdisk_mock_ut.h" +#include "dsproxy_test_state_ut.h" #include <ydb/core/blobstorage/dsproxy/dsproxy.h> #include <ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h> - + #include <ydb/core/blobstorage/base/utility.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> @@ -13,28 +13,28 @@ #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/basics/appdata.h> - + #include <library/cpp/testing/unittest/registar.h> namespace NKikimr { namespace NBlobStorageProxySequenceTest { constexpr ui32 GROUP_ID = 0; -TDSProxyEnv DSProxyEnv; +TDSProxyEnv DSProxyEnv; -void Setup(TTestActorRuntime& runtime, TBlobStorageGroupType type) { +void Setup(TTestActorRuntime& runtime, TBlobStorageGroupType type) { if (IsVerbose) { runtime.SetLogPriority(NKikimrServices::BS_PROXY_PUT, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::BS_PROXY_GET, NLog::PRI_DEBUG); - runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); + runtime.SetLogPriority(NKikimrServices::BS_PROXY_PATCH, NLog::PRI_DEBUG); } else { runtime.SetLogPriority(NKikimrServices::BS_PROXY, NLog::PRI_CRIT); runtime.SetLogPriority(NKikimrServices::BS_PROXY_PUT, NLog::PRI_CRIT); runtime.SetLogPriority(NKikimrServices::BS_PROXY_GET, NLog::PRI_CRIT); runtime.SetLogPriority(NKikimrServices::BS_QUEUE, NLog::PRI_CRIT); } - SetupRuntime(runtime); - DSProxyEnv.Configure(runtime, type, GROUP_ID, 0); + SetupRuntime(runtime); + DSProxyEnv.Configure(runtime, type, GROUP_ID, 0); } struct TGetQuery { @@ -174,25 +174,25 @@ struct TVDiskState { IsValid = true; } - void SetFrom(const IEventHandle *handle, const TEvBlobStorage::TEvVMultiPut *vput, ui64 itemIdx) { - Sender = handle->Sender; - ActorId = handle->Recipient; - auto &item = vput->Record.GetItems(itemIdx); - LogoBlobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - VDiskId = VDiskIDFromVDiskID(vput->Record.GetVDiskID()); - LastCookie = handle->Cookie; - InnerCookie = item.GetCookie(); - Data = item.GetBuffer(); - if (!item.HasBuffer()) { - const TRope& rope = vput->GetPayload(itemIdx); - Data = TString::Uninitialized(rope.GetSize()); - rope.Begin().ExtractPlainDataAndAdvance(Data.Detach(), Data.size()); - } - MsgId = vput->Record.GetMsgQoS().GetMsgId().GetMsgId(); - SequenceId = vput->Record.GetMsgQoS().GetMsgId().GetSequenceId(); - IsValid = true; - } - + void SetFrom(const IEventHandle *handle, const TEvBlobStorage::TEvVMultiPut *vput, ui64 itemIdx) { + Sender = handle->Sender; + ActorId = handle->Recipient; + auto &item = vput->Record.GetItems(itemIdx); + LogoBlobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + VDiskId = VDiskIDFromVDiskID(vput->Record.GetVDiskID()); + LastCookie = handle->Cookie; + InnerCookie = item.GetCookie(); + Data = item.GetBuffer(); + if (!item.HasBuffer()) { + const TRope& rope = vput->GetPayload(itemIdx); + Data = TString::Uninitialized(rope.GetSize()); + rope.Begin().ExtractPlainDataAndAdvance(Data.Detach(), Data.size()); + } + MsgId = vput->Record.GetMsgQoS().GetMsgId().GetMsgId(); + SequenceId = vput->Record.GetMsgQoS().GetMsgId().GetSequenceId(); + IsValid = true; + } + void SetCookiesAndSenderFrom(const IEventHandle *handle, const TEvBlobStorage::TEvVGet *vget) { Sender = handle->Sender; LastCookie = handle->Cookie; @@ -209,38 +209,38 @@ struct TVDiskState { } }; - -void SetPredictedDelaysForAllQueues(const THashMap<TVDiskID, ui32> &latencies) { - UNIT_ASSERT(DSProxyEnv.GroupQueues); - for (TGroupQueues::TFailDomain &domain : DSProxyEnv.GroupQueues->FailDomains) { - for (TGroupQueues::TVDisk &vDisk : domain.VDisks) { - ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; - ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; - for (ui32 id = begin; id < end; ++id) { - NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id); - auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); - if (flowRecord) { - flowRecord->SetPredictedDelayNs(0); - } - } - } - } - ui64 changedCount = 0; - for (auto &[vDiskId, latency] : latencies) { - changedCount++; - TGroupQueues::TVDisk &vDisk = DSProxyEnv.GroupQueues->FailDomains[vDiskId.FailDomain].VDisks[vDiskId.VDisk]; - ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; - ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; - for (ui32 id = begin; id < end; ++id) { - NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id);; - auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); - if (flowRecord) { - flowRecord->SetPredictedDelayNs(latency); - } - } - } -} - + +void SetPredictedDelaysForAllQueues(const THashMap<TVDiskID, ui32> &latencies) { + UNIT_ASSERT(DSProxyEnv.GroupQueues); + for (TGroupQueues::TFailDomain &domain : DSProxyEnv.GroupQueues->FailDomains) { + for (TGroupQueues::TVDisk &vDisk : domain.VDisks) { + ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; + ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; + for (ui32 id = begin; id < end; ++id) { + NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id); + auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); + if (flowRecord) { + flowRecord->SetPredictedDelayNs(0); + } + } + } + } + ui64 changedCount = 0; + for (auto &[vDiskId, latency] : latencies) { + changedCount++; + TGroupQueues::TVDisk &vDisk = DSProxyEnv.GroupQueues->FailDomains[vDiskId.FailDomain].VDisks[vDiskId.VDisk]; + ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; + ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; + for (ui32 id = begin; id < end; ++id) { + NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id);; + auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); + if (flowRecord) { + flowRecord->SetPredictedDelayNs(latency); + } + } + } +} + void SendVGetResult(ui32 vDiskIdx, NKikimrProto::EReplyStatus status, ui32 partId, TVector<TVDiskState> &subgroup, TTestActorRuntime &runtime) { @@ -249,7 +249,7 @@ void SendVGetResult(ui32 vDiskIdx, NKikimrProto::EReplyStatus status, ui32 partI std::unique_ptr<TEvBlobStorage::TEvVGetResult> result(new TEvBlobStorage::TEvVGetResult(NKikimrProto::OK, from->VDiskId, TAppData::TimeProvider->Now(), 0, nullptr, nullptr, nullptr, nullptr, NWilson::TTraceId(), {}, 0U, 0U)); - SetPredictedDelaysForAllQueues({}); + SetPredictedDelaysForAllQueues({}); ui64 queryCookie = from->QueryCookies.size() ? from->QueryCookies[0] : 0; if (status == NKikimrProto::OK) { result->Record.SetCookie(from->InnerCookie); @@ -282,7 +282,7 @@ void SendVGetResult(ui32 blobIdx, ui32 vDiskIdx, NKikimrProto::EReplyStatus stat return; } - SetPredictedDelaysForAllQueues({}); + SetPredictedDelaysForAllQueues({}); if (status == NKikimrProto::ERROR || status == NKikimrProto::TRYLATER || status == NKikimrProto::TRYLATER_SIZE || status == NKikimrProto::TRYLATER_TIME || status == NKikimrProto::VDISK_ERROR_STATE) { @@ -347,55 +347,55 @@ void SendVGetResult(ui32 blobIdx, ui32 vDiskIdx, NKikimrProto::EReplyStatus stat } } - -void GrabVPutEvent(TTestActorRuntime &runtime, TVector<TVDiskState> &subgroup, ui32 vDiskIdxShift = 0) { - TAutoPtr <IEventHandle> handle; - auto vput = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPut>(handle); - UNIT_ASSERT(vput); - TLogoBlobID id = LogoBlobIDFromLogoBlobID(vput->Record.GetBlobID()); - ui32 idx = id.PartId(); - Y_VERIFY(idx); - idx = idx - 1 + vDiskIdxShift; - Y_VERIFY_S(idx < subgroup.size(), idx << ' ' << subgroup.size() << ' ' << vDiskIdxShift << ' ' << vput->ToString()); - Y_VERIFY(!subgroup[idx].IsValid); - subgroup[idx].SetFrom(handle.Get(), vput); -} - -void SendVPutResultEvent(TTestActorRuntime &runtime, TVDiskState &vdisk, NKikimrProto::EReplyStatus status) { - Y_VERIFY(vdisk.IsValid); + +void GrabVPutEvent(TTestActorRuntime &runtime, TVector<TVDiskState> &subgroup, ui32 vDiskIdxShift = 0) { + TAutoPtr <IEventHandle> handle; + auto vput = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPut>(handle); + UNIT_ASSERT(vput); + TLogoBlobID id = LogoBlobIDFromLogoBlobID(vput->Record.GetBlobID()); + ui32 idx = id.PartId(); + Y_VERIFY(idx); + idx = idx - 1 + vDiskIdxShift; + Y_VERIFY_S(idx < subgroup.size(), idx << ' ' << subgroup.size() << ' ' << vDiskIdxShift << ' ' << vput->ToString()); + Y_VERIFY(!subgroup[idx].IsValid); + subgroup[idx].SetFrom(handle.Get(), vput); +} + +void SendVPutResultEvent(TTestActorRuntime &runtime, TVDiskState &vdisk, NKikimrProto::EReplyStatus status) { + Y_VERIFY(vdisk.IsValid); std::unique_ptr<TEvBlobStorage::TEvVPutResult> vPutResult(new TEvBlobStorage::TEvVPutResult( - status, vdisk.LogoBlobId, vdisk.VDiskId, - &vdisk.InnerCookie, TOutOfSpaceStatus(0u, 0.0), TAppData::TimeProvider->Now(), + status, vdisk.LogoBlobId, vdisk.VDiskId, + &vdisk.InnerCookie, TOutOfSpaceStatus(0u, 0.0), TAppData::TimeProvider->Now(), 0, nullptr, nullptr, nullptr, nullptr, 0, NWilson::TTraceId(), 0, TString())); - vPutResult->Record.MutableMsgQoS()->MutableMsgId()->SetMsgId(vdisk.MsgId); - vPutResult->Record.MutableMsgQoS()->MutableMsgId()->SetSequenceId(vdisk.SequenceId); - SetPredictedDelaysForAllQueues({}); + vPutResult->Record.MutableMsgQoS()->MutableMsgId()->SetMsgId(vdisk.MsgId); + vPutResult->Record.MutableMsgQoS()->MutableMsgId()->SetSequenceId(vdisk.SequenceId); + SetPredictedDelaysForAllQueues({}); runtime.Send(new IEventHandle(vdisk.Sender, vdisk.ActorId, vPutResult.release(), 0, vdisk.LastCookie)); -} - +} + void PrepareBlobSubgroup(TLogoBlobID logoblobid, TString data, TVector<TVDiskState> &subgroup, - TTestActorRuntime &runtime, TBlobStorageGroupType type) { + TTestActorRuntime &runtime, TBlobStorageGroupType type) { TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); runtime.Send(new IEventHandle(proxy, sender, new TEvBlobStorage::TEvPut(logoblobid, data, TInstant::Max()))); - subgroup.resize(type.BlobSubgroupSize()); + subgroup.resize(type.BlobSubgroupSize()); - ui32 partCount = type.TotalPartCount(); - for (ui32 i = 0; i < partCount; ++i) { - GrabVPutEvent(runtime, subgroup); + ui32 partCount = type.TotalPartCount(); + for (ui32 i = 0; i < partCount; ++i) { + GrabVPutEvent(runtime, subgroup); } - for (ui32 i = type.Handoff(); i < partCount; ++i) { - SendVPutResultEvent(runtime, subgroup[i], NKikimrProto::OK); + for (ui32 i = type.Handoff(); i < partCount; ++i) { + SendVPutResultEvent(runtime, subgroup[i], NKikimrProto::OK); } - for (ui32 i = 0; i < type.Handoff(); ++i) { - SendVPutResultEvent(runtime, subgroup[i], NKikimrProto::ERROR); + for (ui32 i = 0; i < type.Handoff(); ++i) { + SendVPutResultEvent(runtime, subgroup[i], NKikimrProto::ERROR); } - for (ui32 i = 0; i < type.Handoff(); ++i) { - GrabVPutEvent(runtime, subgroup, type.TotalPartCount()); + for (ui32 i = 0; i < type.Handoff(); ++i) { + GrabVPutEvent(runtime, subgroup, type.TotalPartCount()); } - for (ui32 i = 0; i < type.Handoff(); ++i) { - SendVPutResultEvent(runtime, subgroup[i + type.TotalPartCount()], NKikimrProto::OK); + for (ui32 i = 0; i < type.Handoff(); ++i) { + SendVPutResultEvent(runtime, subgroup[i + type.TotalPartCount()], NKikimrProto::OK); } TAutoPtr<IEventHandle> handle; @@ -404,352 +404,352 @@ void PrepareBlobSubgroup(TLogoBlobID logoblobid, TString data, TVector<TVDiskSta UNIT_ASSERT(putResult->Status == NKikimrProto::OK); } - - + + Y_UNIT_TEST_SUITE(TBlobStorageProxySequenceTest) { - - -struct TGeneralDecorator : public TDecorator { - using TAction = std::function<bool(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx)>; - - TAction Action; - - TGeneralDecorator(THolder<IActor> &&actor, TAction action) - : TDecorator(std::move(actor)) - , Action(action) - { - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) override { - return Action(ev, ctx); - } -}; - - -Y_UNIT_TEST(TestBlock42PutWithChangingSlowDisk) { - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - TTestBasicRuntime runtime(1, false); - Setup(runtime, type); - TTestState testState(runtime, type, DSProxyEnv.Info); - - TLogoBlobID blobId(72075186224047637, 1, 863, 1, 786, 24576); - - TStringBuilder dataBuilder; - for (size_t i = 0; i < blobId.BlobSize(); ++i) { - dataBuilder << 'a'; - } - TBlobTestSet::TBlob blob(blobId, dataBuilder); - - - TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; - - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; - - TEvBlobStorage::TEvPut::TPtr ev = testState.CreatePutRequest(blob, tactic, handleClass); + + +struct TGeneralDecorator : public TDecorator { + using TAction = std::function<bool(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx)>; + + TAction Action; + + TGeneralDecorator(THolder<IActor> &&actor, TAction action) + : TDecorator(std::move(actor)) + , Action(action) + { + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) override { + return Action(ev, ctx); + } +}; + + +Y_UNIT_TEST(TestBlock42PutWithChangingSlowDisk) { + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + TTestBasicRuntime runtime(1, false); + Setup(runtime, type); + TTestState testState(runtime, type, DSProxyEnv.Info); + + TLogoBlobID blobId(72075186224047637, 1, 863, 1, 786, 24576); + + TStringBuilder dataBuilder; + for (size_t i = 0; i < blobId.BlobSize(); ++i) { + dataBuilder << 'a'; + } + TBlobTestSet::TBlob blob(blobId, dataBuilder); + + + TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; + + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; + + TEvBlobStorage::TEvPut::TPtr ev = testState.CreatePutRequest(blob, tactic, handleClass); std::unique_ptr<IActor> putActor = DSProxyEnv.CreatePutRequestActor(ev); - - TGroupMock &groupMock = testState.GetGroupMock(); - groupMock.SetError(0, NKikimrProto::ERROR); - groupMock.SetError(5, NKikimrProto::ERROR); - - THashMap<TVDiskID, ui32> latencies = testState.MakePredictedDelaysForVDisks(blobId); - for (auto &[vDiskId, latency] : latencies) { - latency = 1; - } - std::optional<TVDiskID> prevVDiskId; - TSet<TLogoBlobID> okParts; - - auto action = [&](TAutoPtr<IEventHandle>& ev, const TActorContext&) { - if (ev->Type == TEvents::TSystem::Bootstrap) { - return true; - } - UNIT_ASSERT(ev->Type == TEvBlobStorage::EvVPutResult); - TEvBlobStorage::TEvVPutResult *putResult = ev->Get<TEvBlobStorage::TEvVPutResult>(); - UNIT_ASSERT(putResult); - TVDiskID vDiskId = VDiskIDFromVDiskID(putResult->Record.GetVDiskID()); - TLogoBlobID part = LogoBlobIDFromLogoBlobID(putResult->Record.GetBlobID()); - NKikimrProto::EReplyStatus status = putResult->Record.GetStatus(); - CTEST << "Receive LogoBlobId# " << part << " vdisk " << vDiskId << " " << NKikimrProto::EReplyStatus_Name(status) << Endl; - if (prevVDiskId) { - latencies[*prevVDiskId] = 1; - } - latencies[vDiskId] = 10; - prevVDiskId = vDiskId; - SetPredictedDelaysForAllQueues(latencies); - return true; - }; - + + TGroupMock &groupMock = testState.GetGroupMock(); + groupMock.SetError(0, NKikimrProto::ERROR); + groupMock.SetError(5, NKikimrProto::ERROR); + + THashMap<TVDiskID, ui32> latencies = testState.MakePredictedDelaysForVDisks(blobId); + for (auto &[vDiskId, latency] : latencies) { + latency = 1; + } + std::optional<TVDiskID> prevVDiskId; + TSet<TLogoBlobID> okParts; + + auto action = [&](TAutoPtr<IEventHandle>& ev, const TActorContext&) { + if (ev->Type == TEvents::TSystem::Bootstrap) { + return true; + } + UNIT_ASSERT(ev->Type == TEvBlobStorage::EvVPutResult); + TEvBlobStorage::TEvVPutResult *putResult = ev->Get<TEvBlobStorage::TEvVPutResult>(); + UNIT_ASSERT(putResult); + TVDiskID vDiskId = VDiskIDFromVDiskID(putResult->Record.GetVDiskID()); + TLogoBlobID part = LogoBlobIDFromLogoBlobID(putResult->Record.GetBlobID()); + NKikimrProto::EReplyStatus status = putResult->Record.GetStatus(); + CTEST << "Receive LogoBlobId# " << part << " vdisk " << vDiskId << " " << NKikimrProto::EReplyStatus_Name(status) << Endl; + if (prevVDiskId) { + latencies[*prevVDiskId] = 1; + } + latencies[vDiskId] = 10; + prevVDiskId = vDiskId; + SetPredictedDelaysForAllQueues(latencies); + return true; + }; + runtime.Register(new TGeneralDecorator(THolder<IActor>(putActor.release()), action)); - - for (ui64 idx = 0; idx < 8; ++idx) { - TEvBlobStorage::TEvVPut::TPtr ev = testState.GrabEventPtr<TEvBlobStorage::TEvVPut>(); - TLogoBlobID part = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID()); - UNIT_ASSERT(okParts.count(part) == 0); - NKikimrProto::EReplyStatus status = groupMock.OnVPut(*ev->Get()); - if (status == NKikimrProto::OK) { - okParts.insert(part); - } - TEvBlobStorage::TEvVPutResult::TPtr result = testState.CreateEventResultPtr(ev, status, vDiskId); - runtime.Send(result.Release()); - } - - - TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus { - {blobId, NKikimrProto::OK} - }; - testState.ReceivePutResults(1, expectedStatus); -} - -void MakeTestMultiPutItemStatuses(TTestBasicRuntime &runtime, const TBlobStorageGroupType &type, - const TBatchedVec<NKikimrProto::EReplyStatus> &statuses) { - TString data("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - TTestState testState(runtime, type, DSProxyEnv.Info); - - TVector<TLogoBlobID> blobIds = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIds) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - blobs.emplace_back(id, builder); - } - - TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; - - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; - testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); + + for (ui64 idx = 0; idx < 8; ++idx) { + TEvBlobStorage::TEvVPut::TPtr ev = testState.GrabEventPtr<TEvBlobStorage::TEvVPut>(); + TLogoBlobID part = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID()); + UNIT_ASSERT(okParts.count(part) == 0); + NKikimrProto::EReplyStatus status = groupMock.OnVPut(*ev->Get()); + if (status == NKikimrProto::OK) { + okParts.insert(part); + } + TEvBlobStorage::TEvVPutResult::TPtr result = testState.CreateEventResultPtr(ev, status, vDiskId); + runtime.Send(result.Release()); + } + + + TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus { + {blobId, NKikimrProto::OK} + }; + testState.ReceivePutResults(1, expectedStatus); +} + +void MakeTestMultiPutItemStatuses(TTestBasicRuntime &runtime, const TBlobStorageGroupType &type, + const TBatchedVec<NKikimrProto::EReplyStatus> &statuses) { + TString data("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + TTestState testState(runtime, type, DSProxyEnv.Info); + + TVector<TLogoBlobID> blobIds = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIds) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + blobs.emplace_back(id, builder); + } + + TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; + + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; + testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); runtime.Register(DSProxyEnv.CreatePutRequestActor(batched, tactic, handleClass).release()); - - TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; - for (ui64 idx = 0; idx < blobIds.size(); ++idx) { - for (ui64 part = 1; part <= type.TotalPartCount(); ++part) { - TLogoBlobID partBlobId(blobIds[idx], part); - TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); - specialStatuses[id] = statuses[idx]; - } - } - - TGroupMock &groupMock = testState.GetGroupMock(); - groupMock.SetSpecialStatuses(specialStatuses); - - testState.HandleVMultiPutsWithMock(type.BlobSubgroupSize()); - - TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; - for (ui64 idx = 0; idx < blobIds.size(); ++idx) { - expectedStatus[blobIds[idx]] = statuses[idx]; - } - testState.ReceivePutResults(expectedStatus.size(), expectedStatus); -} - -Y_UNIT_TEST(TestGivenBlock42MultiPut2ItemsStatuses) { - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - TTestBasicRuntime runtime(1, false); - Setup(runtime, type); + + TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; + for (ui64 idx = 0; idx < blobIds.size(); ++idx) { + for (ui64 part = 1; part <= type.TotalPartCount(); ++part) { + TLogoBlobID partBlobId(blobIds[idx], part); + TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); + specialStatuses[id] = statuses[idx]; + } + } + + TGroupMock &groupMock = testState.GetGroupMock(); + groupMock.SetSpecialStatuses(specialStatuses); + + testState.HandleVMultiPutsWithMock(type.BlobSubgroupSize()); + + TMap<TLogoBlobID, NKikimrProto::EReplyStatus> expectedStatus; + for (ui64 idx = 0; idx < blobIds.size(); ++idx) { + expectedStatus[blobIds[idx]] = statuses[idx]; + } + testState.ReceivePutResults(expectedStatus.size(), expectedStatus); +} + +Y_UNIT_TEST(TestGivenBlock42MultiPut2ItemsStatuses) { + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + TTestBasicRuntime runtime(1, false); + Setup(runtime, type); constexpr ui64 statusCount = 3; - NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { - NKikimrProto::OK, - NKikimrProto::BLOCKED, - NKikimrProto::DEADLINE - }; - Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); - for (ui64 fstIdx = 0; fstIdx < statusCount; ++fstIdx) { - for (ui64 sndIdx = 0; sndIdx < statusCount; ++sndIdx) { - MakeTestMultiPutItemStatuses(runtime, type, {maybeStatuses[fstIdx], maybeStatuses[sndIdx]}); - } - } -} - -enum { - Begin = EventSpaceBegin(TEvents::ES_USERSPACE), - EvRequestEnd -}; - -struct TEvRequestEnd : TEventLocal<TEvRequestEnd, EvRequestEnd> { - TEvRequestEnd() = default; -}; - -struct TDyingDecorator : public TTestDecorator { - TActorId ParentId; - - TDyingDecorator(THolder<IActor> &&actor, TActorId parentId) - : TTestDecorator(std::move(actor)) - , ParentId(parentId) - { - } - - virtual ~TDyingDecorator() { - if (NActors::TlsActivationContext) { - std::unique_ptr<IEventBase> ev = std::make_unique<TEvRequestEnd>(); - std::unique_ptr<IEventHandle> handle = std::make_unique<IEventHandle>(ParentId, ParentId, ev.release()); - TActivationContext::Send(handle.release()); - } - } -}; - -Y_UNIT_TEST(TestGivenBlock42GroupGenerationGreaterThanVDiskGenerations) { - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - TTestBasicRuntime runtime(1, false); - Setup(runtime, type); - - TString data("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - TTestState testState(runtime, type, DSProxyEnv.Info); - - TVector<TLogoBlobID> blobIds = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIds) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - blobs.emplace_back(id, builder); - } - - TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; - - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; - testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); - - DSProxyEnv.SetGroupGeneration(2); - - THolder<IActor> putActor(DSProxyEnv.CreatePutRequestActor(batched, tactic, handleClass).release()); - runtime.Register(new TDyingDecorator( - std::move(putActor), testState.EdgeActor)); - - TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; - for (ui64 idx = 0; idx < blobIds.size(); ++idx) { - for (ui64 part = 1; part <= type.TotalPartCount(); ++part) { - TLogoBlobID partBlobId(blobIds[idx], part); - TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); - specialStatuses[id] = NKikimrProto::RACE; - } - } - - TGroupMock &groupMock = testState.GetGroupMock(); - groupMock.SetSpecialStatuses(specialStatuses); - - testState.HandleVMultiPutsWithMock(8); - testState.GrabEventPtr<TEvRequestEnd>(); -} - -void MakeTestGivenBlock42GetRecoverMultiPutStatuses(NKikimrProto::EReplyStatus expectedStatus, - const TVector<NKikimrProto::EReplyStatus> &statuses = {}) { - Y_UNUSED(expectedStatus); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - TTestBasicRuntime runtime(1, false); - Setup(runtime, type); - - constexpr ui64 blobCount = 2; - TVector<TLogoBlobID> blobIds = { - TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) - }; - Y_VERIFY(blobIds.size() == blobCount); - Y_VERIFY(statuses.empty() || statuses.size() == blobCount); - - TVector<TBlobTestSet::TBlob> blobs; - for (const auto& id : blobIds) { - TStringBuilder builder; - for (size_t i = 0; i < id.BlobSize(); ++i) { - builder << 'a'; - } - blobs.emplace_back(id, builder); - } - - TTestState testState(runtime, type, DSProxyEnv.Info); - - TGroupMock &groupMock = testState.GetGroupMock(); - testState.PutBlobsToGroupMock(blobs); - groupMock.Wipe(3); - groupMock.Wipe(4); - - TEvBlobStorage::TEvGet::TPtr ev = testState.CreateGetRequest(blobIds, true); + NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { + NKikimrProto::OK, + NKikimrProto::BLOCKED, + NKikimrProto::DEADLINE + }; + Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); + for (ui64 fstIdx = 0; fstIdx < statusCount; ++fstIdx) { + for (ui64 sndIdx = 0; sndIdx < statusCount; ++sndIdx) { + MakeTestMultiPutItemStatuses(runtime, type, {maybeStatuses[fstIdx], maybeStatuses[sndIdx]}); + } + } +} + +enum { + Begin = EventSpaceBegin(TEvents::ES_USERSPACE), + EvRequestEnd +}; + +struct TEvRequestEnd : TEventLocal<TEvRequestEnd, EvRequestEnd> { + TEvRequestEnd() = default; +}; + +struct TDyingDecorator : public TTestDecorator { + TActorId ParentId; + + TDyingDecorator(THolder<IActor> &&actor, TActorId parentId) + : TTestDecorator(std::move(actor)) + , ParentId(parentId) + { + } + + virtual ~TDyingDecorator() { + if (NActors::TlsActivationContext) { + std::unique_ptr<IEventBase> ev = std::make_unique<TEvRequestEnd>(); + std::unique_ptr<IEventHandle> handle = std::make_unique<IEventHandle>(ParentId, ParentId, ev.release()); + TActivationContext::Send(handle.release()); + } + } +}; + +Y_UNIT_TEST(TestGivenBlock42GroupGenerationGreaterThanVDiskGenerations) { + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + TTestBasicRuntime runtime(1, false); + Setup(runtime, type); + + TString data("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + TTestState testState(runtime, type, DSProxyEnv.Info); + + TVector<TLogoBlobID> blobIds = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIds) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + blobs.emplace_back(id, builder); + } + + TEvBlobStorage::TEvPut::ETactic tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass handleClass = NKikimrBlobStorage::TabletLog; + + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched; + testState.CreatePutRequests(blobs, std::back_inserter(batched), tactic, handleClass); + + DSProxyEnv.SetGroupGeneration(2); + + THolder<IActor> putActor(DSProxyEnv.CreatePutRequestActor(batched, tactic, handleClass).release()); + runtime.Register(new TDyingDecorator( + std::move(putActor), testState.EdgeActor)); + + TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; + for (ui64 idx = 0; idx < blobIds.size(); ++idx) { + for (ui64 part = 1; part <= type.TotalPartCount(); ++part) { + TLogoBlobID partBlobId(blobIds[idx], part); + TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); + specialStatuses[id] = NKikimrProto::RACE; + } + } + + TGroupMock &groupMock = testState.GetGroupMock(); + groupMock.SetSpecialStatuses(specialStatuses); + + testState.HandleVMultiPutsWithMock(8); + testState.GrabEventPtr<TEvRequestEnd>(); +} + +void MakeTestGivenBlock42GetRecoverMultiPutStatuses(NKikimrProto::EReplyStatus expectedStatus, + const TVector<NKikimrProto::EReplyStatus> &statuses = {}) { + Y_UNUSED(expectedStatus); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + TTestBasicRuntime runtime(1, false); + Setup(runtime, type); + + constexpr ui64 blobCount = 2; + TVector<TLogoBlobID> blobIds = { + TLogoBlobID(72075186224047637, 1, 863, 1, 786, 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288) + }; + Y_VERIFY(blobIds.size() == blobCount); + Y_VERIFY(statuses.empty() || statuses.size() == blobCount); + + TVector<TBlobTestSet::TBlob> blobs; + for (const auto& id : blobIds) { + TStringBuilder builder; + for (size_t i = 0; i < id.BlobSize(); ++i) { + builder << 'a'; + } + blobs.emplace_back(id, builder); + } + + TTestState testState(runtime, type, DSProxyEnv.Info); + + TGroupMock &groupMock = testState.GetGroupMock(); + testState.PutBlobsToGroupMock(blobs); + groupMock.Wipe(3); + groupMock.Wipe(4); + + TEvBlobStorage::TEvGet::TPtr ev = testState.CreateGetRequest(blobIds, true); runtime.Register(DSProxyEnv.CreateGetRequestActor(ev, NKikimrBlobStorage::TabletLog, true).release()); - - testState.HandleVGetsWithMock(type.BlobSubgroupSize()); - - if (statuses.empty()) { - groupMock.SetError(3, expectedStatus); - groupMock.SetError(4, expectedStatus); - } else { - TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; - for (ui64 idx = 0; idx < blobIds.size(); ++idx) { - for (ui64 part = 3; part <= 4; ++part) { - TLogoBlobID partBlobId(blobIds[idx], part - 2 * idx); - TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); - specialStatuses[id] = statuses[idx]; - } - } - groupMock.SetSpecialStatuses(specialStatuses); - } - - testState.HandleVMultiPutsWithMock(3); - - TAutoPtr<IEventHandle> handle; - auto getResult = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGetResult>(handle); - UNIT_ASSERT(getResult); - UNIT_ASSERT(getResult->Status == expectedStatus); -} - -Y_UNIT_TEST(TestGivenBlock42GetRecoverMultiPutStatuses) { + + testState.HandleVGetsWithMock(type.BlobSubgroupSize()); + + if (statuses.empty()) { + groupMock.SetError(3, expectedStatus); + groupMock.SetError(4, expectedStatus); + } else { + TMap<TPartLocation, NKikimrProto::EReplyStatus> specialStatuses; + for (ui64 idx = 0; idx < blobIds.size(); ++idx) { + for (ui64 part = 3; part <= 4; ++part) { + TLogoBlobID partBlobId(blobIds[idx], part - 2 * idx); + TPartLocation id = testState.PrimaryVDiskForBlobPart(partBlobId); + specialStatuses[id] = statuses[idx]; + } + } + groupMock.SetSpecialStatuses(specialStatuses); + } + + testState.HandleVMultiPutsWithMock(3); + + TAutoPtr<IEventHandle> handle; + auto getResult = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGetResult>(handle); + UNIT_ASSERT(getResult); + UNIT_ASSERT(getResult->Status == expectedStatus); +} + +Y_UNIT_TEST(TestGivenBlock42GetRecoverMultiPutStatuses) { constexpr ui64 statusCount = 3; - NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { - NKikimrProto::OK, - NKikimrProto::BLOCKED, - NKikimrProto::DEADLINE - }; - Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); - for (ui64 idx = 0; idx < statusCount; ++idx) { - MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx]); - } -} - -Y_UNIT_TEST(TestGivenBlock42GetRecoverMultiPut2ItemsStatuses) { + NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { + NKikimrProto::OK, + NKikimrProto::BLOCKED, + NKikimrProto::DEADLINE + }; + Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); + for (ui64 idx = 0; idx < statusCount; ++idx) { + MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx]); + } +} + +Y_UNIT_TEST(TestGivenBlock42GetRecoverMultiPut2ItemsStatuses) { constexpr ui64 statusCount = 3; - NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { - NKikimrProto::OK, - NKikimrProto::BLOCKED, - NKikimrProto::DEADLINE - }; - Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); - for (ui64 idx = 1; idx < statusCount; ++idx) { - MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx], {maybeStatuses[idx], maybeStatuses[0]}); - MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx], {maybeStatuses[0], maybeStatuses[idx]}); - } -} - -Y_UNIT_TEST(TestGivenMirror3DCGetWithFirstSlowDisk) { - TBlobStorageGroupType type = {TErasureType::ErasureMirror3dc}; - TTestBasicRuntime runtime(1, false); - Setup(runtime, type); - - TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288); - - TTestState testState(runtime, type, DSProxyEnv.Info); - - - TEvBlobStorage::TEvGet::TPtr ev = testState.CreateGetRequest({blobId}, false); + NKikimrProto::EReplyStatus maybeStatuses[statusCount] = { + NKikimrProto::OK, + NKikimrProto::BLOCKED, + NKikimrProto::DEADLINE + }; + Y_VERIFY(maybeStatuses[statusCount - 1] == NKikimrProto::DEADLINE); + for (ui64 idx = 1; idx < statusCount; ++idx) { + MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx], {maybeStatuses[idx], maybeStatuses[0]}); + MakeTestGivenBlock42GetRecoverMultiPutStatuses(maybeStatuses[idx], {maybeStatuses[0], maybeStatuses[idx]}); + } +} + +Y_UNIT_TEST(TestGivenMirror3DCGetWithFirstSlowDisk) { + TBlobStorageGroupType type = {TErasureType::ErasureMirror3dc}; + TTestBasicRuntime runtime(1, false); + Setup(runtime, type); + + TLogoBlobID blobId = TLogoBlobID(72075186224047637, 1, 2194, 1, 142, 12288); + + TTestState testState(runtime, type, DSProxyEnv.Info); + + + TEvBlobStorage::TEvGet::TPtr ev = testState.CreateGetRequest({blobId}, false); TActorId getActorId = runtime.Register(DSProxyEnv.CreateGetRequestActor(ev, NKikimrBlobStorage::TabletLog, false).release()); - runtime.EnableScheduleForActor(getActorId); - - testState.GrabEventPtr<TEvBlobStorage::TEvVGet>(); - TEvBlobStorage::TEvVGet::TPtr vget = testState.GrabEventPtr<TEvBlobStorage::TEvVGet>(); -} - + runtime.EnableScheduleForActor(getActorId); + + testState.GrabEventPtr<TEvBlobStorage::TEvVGet>(); + TEvBlobStorage::TEvVGet::TPtr vget = testState.GrabEventPtr<TEvBlobStorage::TEvVGet>(); +} + Y_UNIT_TEST(TestGivenBlock42GetThenVGetResponseParts2523Nodata4ThenGetOk) { TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -758,7 +758,7 @@ Y_UNIT_TEST(TestGivenBlock42GetThenVGetResponseParts2523Nodata4ThenGetOk) { TLogoBlobID logoblobid(1, 0, 0, 0, (ui32)data.size(), 0); TVector<TVDiskState> subgroup; - PrepareBlobSubgroup(logoblobid, data, subgroup, runtime, type); + PrepareBlobSubgroup(logoblobid, data, subgroup, runtime, type); runtime.Send(new IEventHandle(proxy, sender, new TEvBlobStorage::TEvGet(logoblobid, 0, 0, TInstant::Max(), NKikimrBlobStorage::EGetHandleClass::FastRead))); @@ -790,194 +790,194 @@ Y_UNIT_TEST(TestGivenBlock42GetThenVGetResponseParts2523Nodata4ThenGetOk) { UNIT_ASSERT(getResult->Responses[0].Status == NKikimrProto::OK); } - -struct TBlobPack { - ui32 Count; - ui32 DataLength; - - static TString ToString(const TBlobPack &pack) { - return TStringBuilder() << "{" - << "Count# " << pack.Count - << " DataLength# " << pack.DataLength - << '}'; - } - - TString ToString() const { - return ToString(*this); - } -}; - -TString VectorToString(const TVector<ui32> &vec) { - TStringBuilder str; - str << "{" << Endl; - str << "Size# " << vec.size() << Endl; - for (ui64 idx = 0; idx < vec.size(); ++idx) { - str << " [" << idx << "]# " << ToString(vec[idx]); - } - str << "}"; - return str; -} - -template <typename Type> -TString VectorToString(const TVector<Type> &vec) { - TStringBuilder str; - str << "{" << Endl; - str << "Size# " << vec.size() << Endl; - for (ui64 idx = 0; idx < vec.size(); ++idx) { - str << " [" << idx << "]# " << Type::ToString(vec[idx]); - } - str << "}"; - return str; -} - -void MakeTestProtobufSizeWithMultiGet(const TVector<TBlobPack> &packs, TVector<ui32> vGetQueryCounts) { - TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type(TErasureType::ErasureNone); - Setup(runtime, type); - + +struct TBlobPack { + ui32 Count; + ui32 DataLength; + + static TString ToString(const TBlobPack &pack) { + return TStringBuilder() << "{" + << "Count# " << pack.Count + << " DataLength# " << pack.DataLength + << '}'; + } + + TString ToString() const { + return ToString(*this); + } +}; + +TString VectorToString(const TVector<ui32> &vec) { + TStringBuilder str; + str << "{" << Endl; + str << "Size# " << vec.size() << Endl; + for (ui64 idx = 0; idx < vec.size(); ++idx) { + str << " [" << idx << "]# " << ToString(vec[idx]); + } + str << "}"; + return str; +} + +template <typename Type> +TString VectorToString(const TVector<Type> &vec) { + TStringBuilder str; + str << "{" << Endl; + str << "Size# " << vec.size() << Endl; + for (ui64 idx = 0; idx < vec.size(); ++idx) { + str << " [" << idx << "]# " << Type::ToString(vec[idx]); + } + str << "}"; + return str; +} + +void MakeTestProtobufSizeWithMultiGet(const TVector<TBlobPack> &packs, TVector<ui32> vGetQueryCounts) { + TTestBasicRuntime runtime(1, false); + TBlobStorageGroupType type(TErasureType::ErasureNone); + Setup(runtime, type); + TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); - - TVector<TString> datas(packs.size()); - for (ui32 idx = 0; idx < packs.size(); ++idx) { - for (ui32 chIdx = 0; chIdx < packs[idx].DataLength; ++chIdx) { - datas[idx] += 'a' + chIdx % 26; - } - } - ui32 blobCount = 0; - for (const TBlobPack &pack : packs) { - blobCount += pack.Count; - } - - TVector<TVector<TVDiskState>> blobSubgroups(blobCount); - TVector<TLogoBlobID> blobIds(blobCount); - for (ui32 packIdx = 0, blobIdx = 0; packIdx < packs.size(); ++packIdx) { - const TBlobPack &pack = packs[packIdx]; - for (ui32 interIdx = 0; interIdx < pack.Count; ++interIdx, ++blobIdx) { - blobIds[blobIdx] = TLogoBlobID(1, 0, blobIdx, 0, datas[packIdx].size(), 0); - PrepareBlobSubgroup(blobIds[blobIdx], datas[packIdx], blobSubgroups[blobIdx], runtime, type); - } - } - - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queries(new TEvBlobStorage::TEvGet::TQuery[blobCount]); - for (ui32 i = 0; i < blobCount; ++i) { - auto &q = queries[i]; - q.Id = blobIds[i]; - q.Shift = 0; - q.Size = q.Id.BlobSize(); - } - runtime.Send(new IEventHandle(proxy, sender, new TEvBlobStorage::TEvGet( - queries, blobCount, TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::AsyncRead, false))); - - struct TTestVGetInfo { - ui32 Count; - ui32 ProtobufSize; - - static TString ToString(const TTestVGetInfo& self) { - return TStringBuilder() << "{" << "Count# " << self.Count << " ProtobufSize# " << self.ProtobufSize << "}"; - } - - TString ToString() const { - return ToString(*this); - } - }; - - TVector<TTestVGetInfo> vGetInformations; - vGetInformations.push_back({0, 0}); - TQueryResultSizeTracker resultSize; - resultSize.Init(); - TMap<ui64, ui64> firstQueryStepToVGetIdx = { {0, 0} }; - TVector<ui64> firstSteps(1, 0); - for (ui32 i = 0; i < blobCount; ++i) { - ui32 size = blobIds[i].BlobSize(); - resultSize.AddLogoBlobIndex(); - resultSize.AddLogoBlobData(size, 0, 0); - if (resultSize.IsOverflow()) { - firstSteps.push_back(blobIds[i].Step()); - firstQueryStepToVGetIdx[firstSteps.back()] = firstSteps.size() - 1; - resultSize.Init(); - resultSize.AddLogoBlobIndex(); - resultSize.AddLogoBlobData(size, 0, 0); - vGetInformations.push_back({0, 0}); - } - vGetInformations.back().Count++; - vGetInformations.back().ProtobufSize = resultSize.GetSize(); - } - - UNIT_ASSERT_C(vGetInformations.size() == vGetQueryCounts.size(), - "not equal vGetCount for multiget" - << " vGetInformations# " << VectorToString(vGetInformations) - << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); - for (ui32 vGetIdx = 0; vGetIdx < vGetInformations.size(); ++vGetIdx) { - UNIT_ASSERT_C(vGetInformations[vGetIdx].Count == vGetQueryCounts[vGetIdx], - "not equal query count " - << " vGetIdx# " << vGetIdx - << " vGetInformations# " << VectorToString(vGetInformations) - << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); - } - - for (ui64 idx = 0; idx < vGetInformations.size(); ++idx) { + + TVector<TString> datas(packs.size()); + for (ui32 idx = 0; idx < packs.size(); ++idx) { + for (ui32 chIdx = 0; chIdx < packs[idx].DataLength; ++chIdx) { + datas[idx] += 'a' + chIdx % 26; + } + } + ui32 blobCount = 0; + for (const TBlobPack &pack : packs) { + blobCount += pack.Count; + } + + TVector<TVector<TVDiskState>> blobSubgroups(blobCount); + TVector<TLogoBlobID> blobIds(blobCount); + for (ui32 packIdx = 0, blobIdx = 0; packIdx < packs.size(); ++packIdx) { + const TBlobPack &pack = packs[packIdx]; + for (ui32 interIdx = 0; interIdx < pack.Count; ++interIdx, ++blobIdx) { + blobIds[blobIdx] = TLogoBlobID(1, 0, blobIdx, 0, datas[packIdx].size(), 0); + PrepareBlobSubgroup(blobIds[blobIdx], datas[packIdx], blobSubgroups[blobIdx], runtime, type); + } + } + + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queries(new TEvBlobStorage::TEvGet::TQuery[blobCount]); + for (ui32 i = 0; i < blobCount; ++i) { + auto &q = queries[i]; + q.Id = blobIds[i]; + q.Shift = 0; + q.Size = q.Id.BlobSize(); + } + runtime.Send(new IEventHandle(proxy, sender, new TEvBlobStorage::TEvGet( + queries, blobCount, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::AsyncRead, false))); + + struct TTestVGetInfo { + ui32 Count; + ui32 ProtobufSize; + + static TString ToString(const TTestVGetInfo& self) { + return TStringBuilder() << "{" << "Count# " << self.Count << " ProtobufSize# " << self.ProtobufSize << "}"; + } + + TString ToString() const { + return ToString(*this); + } + }; + + TVector<TTestVGetInfo> vGetInformations; + vGetInformations.push_back({0, 0}); + TQueryResultSizeTracker resultSize; + resultSize.Init(); + TMap<ui64, ui64> firstQueryStepToVGetIdx = { {0, 0} }; + TVector<ui64> firstSteps(1, 0); + for (ui32 i = 0; i < blobCount; ++i) { + ui32 size = blobIds[i].BlobSize(); + resultSize.AddLogoBlobIndex(); + resultSize.AddLogoBlobData(size, 0, 0); + if (resultSize.IsOverflow()) { + firstSteps.push_back(blobIds[i].Step()); + firstQueryStepToVGetIdx[firstSteps.back()] = firstSteps.size() - 1; + resultSize.Init(); + resultSize.AddLogoBlobIndex(); + resultSize.AddLogoBlobData(size, 0, 0); + vGetInformations.push_back({0, 0}); + } + vGetInformations.back().Count++; + vGetInformations.back().ProtobufSize = resultSize.GetSize(); + } + + UNIT_ASSERT_C(vGetInformations.size() == vGetQueryCounts.size(), + "not equal vGetCount for multiget" + << " vGetInformations# " << VectorToString(vGetInformations) + << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); + for (ui32 vGetIdx = 0; vGetIdx < vGetInformations.size(); ++vGetIdx) { + UNIT_ASSERT_C(vGetInformations[vGetIdx].Count == vGetQueryCounts[vGetIdx], + "not equal query count " + << " vGetIdx# " << vGetIdx + << " vGetInformations# " << VectorToString(vGetInformations) + << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); + } + + for (ui64 idx = 0; idx < vGetInformations.size(); ++idx) { TMap<TActorId, TGetRequest> lastRequest; - TAutoPtr<IEventHandle> handle; - auto vget = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>(handle); - UNIT_ASSERT_C(vget->Record.ExtremeQueriesSize(), "vget without queries vGet# " << vget->ToString()); - auto firstQuery = vget->Record.GetExtremeQueries(0); - TLogoBlobID firstBlobId = LogoBlobIDFromLogoBlobID(firstQuery.GetId()); - ui64 step = firstBlobId.Step(); - UNIT_ASSERT_C(firstQueryStepToVGetIdx.count(step), - "not expected first blobId step" - << " idx# " << idx - << " step# " << step - << " firstSteps# " << FormatList(firstSteps) - << " vGet# " << vget->ToString() - << " vGetInformations# " << VectorToString(vGetInformations) - << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); - ui64 vGetIdx = firstQueryStepToVGetIdx[step]; - TTestVGetInfo &vGetInfo = vGetInformations[vGetIdx]; - lastRequest[handle->Recipient].SetFrom(handle.Get(), vget); - UNIT_ASSERT_C(vget->Record.ExtremeQueriesSize() == vGetInfo.Count, - "vget query count not equal predicted count" - << " idx# " << idx - << " vGetIdx# " << vGetIdx - << " realQueryCount# " << vget->Record.ExtremeQueriesSize() - << " predictedQueryCount# " << vGetInfo.Count - << " vGet# " << vget->ToString() - << " vGetInformations# " << VectorToString(vGetInformations) - << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); - UNIT_ASSERT_C(vget->Record.ByteSizeLong() <= vGetInfo.ProtobufSize, - "real protobuf size greater than predicted size" - << " idx# " << idx - << " vGetIdx# " << vGetIdx - << " realProtoBufSize# " << vget->Record.ByteSizeLong() - << " predictedProtoBufSize# " << vGetInfo.ProtobufSize - << " vGet# " << vget->ToString() - << " vGetInformations# " << VectorToString(vGetInformations) - << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); - SendVGetResult(0, 0, NKikimrProto::OK, blobSubgroups, lastRequest, runtime); - } - - TAutoPtr<IEventHandle> handle; - auto getResult = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGetResult>(handle); - UNIT_ASSERT(getResult); - UNIT_ASSERT(getResult->Status == NKikimrProto::OK); - UNIT_ASSERT(getResult->ResponseSz == blobCount); - UNIT_ASSERT(getResult->Responses[0].Status == NKikimrProto::OK); -} - -Y_UNIT_TEST(TestProtobufSizeWithMultiGet) { - MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'332}}, {224}); // MaxProtobufSize - 1 - MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'333}}, {224}); // MaxProtobufSize - MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'334}}, {223, 1}); // MaxProtobufSize + 1 - - auto blobPacks = {TBlobPack{223, 300'000}, TBlobPack{1, 186'333}, TBlobPack{223, 300'000}, TBlobPack{1, 186'334}}; - MakeTestProtobufSizeWithMultiGet(blobPacks, {224, 223, 1}); -} - + TAutoPtr<IEventHandle> handle; + auto vget = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>(handle); + UNIT_ASSERT_C(vget->Record.ExtremeQueriesSize(), "vget without queries vGet# " << vget->ToString()); + auto firstQuery = vget->Record.GetExtremeQueries(0); + TLogoBlobID firstBlobId = LogoBlobIDFromLogoBlobID(firstQuery.GetId()); + ui64 step = firstBlobId.Step(); + UNIT_ASSERT_C(firstQueryStepToVGetIdx.count(step), + "not expected first blobId step" + << " idx# " << idx + << " step# " << step + << " firstSteps# " << FormatList(firstSteps) + << " vGet# " << vget->ToString() + << " vGetInformations# " << VectorToString(vGetInformations) + << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); + ui64 vGetIdx = firstQueryStepToVGetIdx[step]; + TTestVGetInfo &vGetInfo = vGetInformations[vGetIdx]; + lastRequest[handle->Recipient].SetFrom(handle.Get(), vget); + UNIT_ASSERT_C(vget->Record.ExtremeQueriesSize() == vGetInfo.Count, + "vget query count not equal predicted count" + << " idx# " << idx + << " vGetIdx# " << vGetIdx + << " realQueryCount# " << vget->Record.ExtremeQueriesSize() + << " predictedQueryCount# " << vGetInfo.Count + << " vGet# " << vget->ToString() + << " vGetInformations# " << VectorToString(vGetInformations) + << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); + UNIT_ASSERT_C(vget->Record.ByteSizeLong() <= vGetInfo.ProtobufSize, + "real protobuf size greater than predicted size" + << " idx# " << idx + << " vGetIdx# " << vGetIdx + << " realProtoBufSize# " << vget->Record.ByteSizeLong() + << " predictedProtoBufSize# " << vGetInfo.ProtobufSize + << " vGet# " << vget->ToString() + << " vGetInformations# " << VectorToString(vGetInformations) + << " vGetQueryCounts# " << VectorToString(vGetQueryCounts)); + SendVGetResult(0, 0, NKikimrProto::OK, blobSubgroups, lastRequest, runtime); + } + + TAutoPtr<IEventHandle> handle; + auto getResult = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvGetResult>(handle); + UNIT_ASSERT(getResult); + UNIT_ASSERT(getResult->Status == NKikimrProto::OK); + UNIT_ASSERT(getResult->ResponseSz == blobCount); + UNIT_ASSERT(getResult->Responses[0].Status == NKikimrProto::OK); +} + +Y_UNIT_TEST(TestProtobufSizeWithMultiGet) { + MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'332}}, {224}); // MaxProtobufSize - 1 + MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'333}}, {224}); // MaxProtobufSize + MakeTestProtobufSizeWithMultiGet({TBlobPack{223, 300'000}, TBlobPack{1, 186'334}}, {223, 1}); // MaxProtobufSize + 1 + + auto blobPacks = {TBlobPack{223, 300'000}, TBlobPack{1, 186'333}, TBlobPack{223, 300'000}, TBlobPack{1, 186'334}}; + MakeTestProtobufSizeWithMultiGet(blobPacks, {224, 223, 1}); +} + Y_UNIT_TEST(TestGivenStripe42GetThenVGetResponsePartsNodata263451ThenGetOk) { TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Stripe}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Stripe}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -987,7 +987,7 @@ Y_UNIT_TEST(TestGivenStripe42GetThenVGetResponsePartsNodata263451ThenGetOk) { TLogoBlobID logoblobid(0x10010000001000Bull, 5, 58949, 1, 1209816, 10); TVector<TVDiskState> subgroup; - PrepareBlobSubgroup(logoblobid, data, subgroup, runtime, type); + PrepareBlobSubgroup(logoblobid, data, subgroup, runtime, type); runtime.Send(new IEventHandle(proxy, sender, new TEvBlobStorage::TEvGet(logoblobid, 0, 0, TInstant::Max(), NKikimrBlobStorage::EGetHandleClass::FastRead))); @@ -1019,11 +1019,11 @@ Y_UNIT_TEST(TestGivenStripe42GetThenVGetResponsePartsNodata263451ThenGetOk) { UNIT_ASSERT(getResult->Responses[0].Status == NKikimrProto::OK); } -Y_UNIT_TEST(TestGivenStripe42WhenGet2PartsOfBlobThenGetOk) { +Y_UNIT_TEST(TestGivenStripe42WhenGet2PartsOfBlobThenGetOk) { // Arrange TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Stripe}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Stripe}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -1047,7 +1047,7 @@ Y_UNIT_TEST(TestGivenStripe42WhenGet2PartsOfBlobThenGetOk) { TMap<TActorId, TGetRequest> lastRequest; for (ui32 i = 0; i < logoblobids.size(); ++i) { - PrepareBlobSubgroup(logoblobids[i], data, blobSubgroups[i], runtime, type); + PrepareBlobSubgroup(logoblobids[i], data, blobSubgroups[i], runtime, type); } // Act @@ -1087,8 +1087,8 @@ Y_UNIT_TEST(TestGivenStripe42WhenGet2PartsOfBlobThenGetOk) { Y_UNIT_TEST(TestGivenBlock42IntersectingPutWhenNodataOkThenOk) { TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -1108,7 +1108,7 @@ Y_UNIT_TEST(TestGivenBlock42IntersectingPutWhenNodataOkThenOk) { TMap<TActorId, TGetRequest> lastRequest; for (ui32 i = 0; i < logoblobids.size(); ++i) { - PrepareBlobSubgroup(logoblobids[i], data, blobSubgroups[i], runtime, type); + PrepareBlobSubgroup(logoblobids[i], data, blobSubgroups[i], runtime, type); } TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queries(new TEvBlobStorage::TEvGet::TQuery[4]); @@ -1147,8 +1147,8 @@ Y_UNIT_TEST(TestGivenBlock42IntersectingPutWhenNodataOkThenOk) { Y_UNIT_TEST(TestGivenBlock42PutWhenPartialGetThenSingleDiskRequestOk) { TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -1160,7 +1160,7 @@ Y_UNIT_TEST(TestGivenBlock42PutWhenPartialGetThenSingleDiskRequestOk) { } TLogoBlobID logoblobid(1, 2, 3, 4, (ui32)data.size(), 5); TVector<TVDiskState> blobSubgroup; - PrepareBlobSubgroup(logoblobid, data, blobSubgroup, runtime, type); + PrepareBlobSubgroup(logoblobid, data, blobSubgroup, runtime, type); for (ui32 step = 0; step < 32; ++step) { for (ui32 part = 0; part < 4; ++part) { @@ -1256,8 +1256,8 @@ Y_UNIT_TEST(TestGivenBlock42PutWhenPartialGetThenSingleDiskRequestOk) { Y_UNIT_TEST(TestGivenBlock42Put6PartsOnOneVDiskWhenDiscoverThenRecoverFirst) { TTestBasicRuntime runtime(1, false); - TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; - Setup(runtime, type); + TBlobStorageGroupType type = {TErasureType::Erasure4Plus2Block}; + Setup(runtime, type); TActorId proxy = MakeBlobStorageProxyID(GROUP_ID); TActorId sender = runtime.AllocateEdgeActor(0); @@ -1270,7 +1270,7 @@ Y_UNIT_TEST(TestGivenBlock42Put6PartsOnOneVDiskWhenDiscoverThenRecoverFirst) { TLogoBlobID logoblobid(1, 2, 3, 4, (ui32)data.size(), 5); TVector<TVector<TVDiskState>> blobSubgroups; blobSubgroups.resize(1); - PrepareBlobSubgroup(logoblobid, data, blobSubgroups[0], runtime, type); + PrepareBlobSubgroup(logoblobid, data, blobSubgroups[0], runtime, type); const ui64 tabletId = 1; @@ -1305,7 +1305,7 @@ Y_UNIT_TEST(TestGivenBlock42Put6PartsOnOneVDiskWhenDiscoverThenRecoverFirst) { TIngress ingress; for (ui32 partIdx = 0; partIdx < 6; ++partIdx) { TLogoBlobID blobPartId(logoblobid, partIdx + 1); - TIngress partIngress(*TIngress::CreateIngressWithLocal(&DSProxyEnv.Info->GetTopology(), req.VDiskId, blobPartId)); + TIngress partIngress(*TIngress::CreateIngressWithLocal(&DSProxyEnv.Info->GetTopology(), req.VDiskId, blobPartId)); ingress.Merge(partIngress); } const ui64 ingressRaw = ingress.Raw(); diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_test_state_ut.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_test_state_ut.h index bc805ac12b7..7b550ddf97f 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_test_state_ut.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_test_state_ut.h @@ -1,289 +1,289 @@ -#pragma once - -#include "defs.h" -#include "dsproxy_vdisk_mock_ut.h" - -#include <library/cpp/testing/unittest/registar.h> - - -namespace NKikimr { - -struct TTestState { - TTestActorRuntime &Runtime; - TActorId EdgeActor; - TBlobStorageGroupType Type; - TGroupMock GroupMock; - TIntrusivePtr<TBlobStorageGroupInfo> Info; - - - TTestState(TTestActorRuntime &runtime, const TBlobStorageGroupType &type, - TIntrusivePtr<TBlobStorageGroupInfo> &info, ui64 nodeIndex = 0) - : Runtime(runtime) - , EdgeActor(runtime.AllocateEdgeActor(nodeIndex)) - , Type(type) - , GroupMock(0, Type.GetErasure(), Type.BlobSubgroupSize(), 1, info) - , Info(info) - { - } - - TTestState(TTestActorRuntime &runtime, const TBlobStorageGroupType &type, ui64 nodeIndex = 0) - : Runtime(runtime) - , EdgeActor(runtime.AllocateEdgeActor(nodeIndex)) - , Type(type) - , GroupMock(0, Type.GetErasure(), Type.BlobSubgroupSize(), 1) - , Info(GroupMock.GetInfo()) - { - } - - TGroupMock& GetGroupMock() { - return GroupMock; - } - - //////////////////////////////////////////////////////////////////////////// - // HELPER FUNCTIONS - //////////////////////////////////////////////////////////////////////////// - TPartLocation PrimaryVDiskForBlobPart(TLogoBlobID blobId) { - Y_VERIFY(blobId.PartId()); - TLogoBlobID origBlobId(blobId, 0); - TVDiskID vDiskId = Info->GetVDiskInSubgroup(blobId.PartId() - 1, origBlobId.Hash()); - return {blobId, vDiskId}; - } - - TPartLocation HandoffVDiskForBlobPart(TLogoBlobID blobId, ui64 handoffIdx) { - Y_VERIFY(blobId.PartId()); - Y_VERIFY(handoffIdx < Type.Handoff()); - TLogoBlobID origBlobId(blobId, 0); - TVDiskID vDiskId = Info->GetVDiskInSubgroup(Type.TotalPartCount() + handoffIdx, origBlobId.Hash()); - return {blobId, vDiskId}; - } - - THashMap<TVDiskID, ui32> MakePredictedDelaysForVDisks(TLogoBlobID blobId) { - TBlobStorageGroupInfo::TVDiskIds vDiskIds; - TBlobStorageGroupInfo::TServiceIds serviceIds; - THashMap<TVDiskID, ui32> result; - Info->PickSubgroup(blobId.Hash(), &vDiskIds, &serviceIds); - for (ui32 idx = 0; idx < vDiskIds.size(); ++idx) { - result.emplace(vDiskIds[idx], 0); - } - return result; - } - - //////////////////////////////////////////////////////////////////////////// - // PUT BLOBS TO GROUP MOCK - //////////////////////////////////////////////////////////////////////////// - TTestState& PutBlobsToGroupMock(const TBlobTestSet &blobSet) { - GroupMock.PutBlobSet(blobSet); - return *this; - } - - template <typename TIter> - TTestState& PutBlobsToGroupMock(TIter begin, TIter end) { - TBlobTestSet blobSet; - blobSet.AddBlobs(begin, end); - return PutBlobsToGroupMock(blobSet); - } - - TTestState& PutBlobsToGroupMock(const TVector<TBlobTestSet::TBlob> &blobs) { - return PutBlobsToGroupMock(blobs.begin(), blobs.end()); - } - - //////////////////////////////////////////////////////////////////////////// - // CREATE REQUESTS - //////////////////////////////////////////////////////////////////////////// - TEvBlobStorage::TEvPut::TPtr CreatePutRequest(const TBlobTestSet::TBlob &blob, - TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) - { +#pragma once + +#include "defs.h" +#include "dsproxy_vdisk_mock_ut.h" + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr { + +struct TTestState { + TTestActorRuntime &Runtime; + TActorId EdgeActor; + TBlobStorageGroupType Type; + TGroupMock GroupMock; + TIntrusivePtr<TBlobStorageGroupInfo> Info; + + + TTestState(TTestActorRuntime &runtime, const TBlobStorageGroupType &type, + TIntrusivePtr<TBlobStorageGroupInfo> &info, ui64 nodeIndex = 0) + : Runtime(runtime) + , EdgeActor(runtime.AllocateEdgeActor(nodeIndex)) + , Type(type) + , GroupMock(0, Type.GetErasure(), Type.BlobSubgroupSize(), 1, info) + , Info(info) + { + } + + TTestState(TTestActorRuntime &runtime, const TBlobStorageGroupType &type, ui64 nodeIndex = 0) + : Runtime(runtime) + , EdgeActor(runtime.AllocateEdgeActor(nodeIndex)) + , Type(type) + , GroupMock(0, Type.GetErasure(), Type.BlobSubgroupSize(), 1) + , Info(GroupMock.GetInfo()) + { + } + + TGroupMock& GetGroupMock() { + return GroupMock; + } + + //////////////////////////////////////////////////////////////////////////// + // HELPER FUNCTIONS + //////////////////////////////////////////////////////////////////////////// + TPartLocation PrimaryVDiskForBlobPart(TLogoBlobID blobId) { + Y_VERIFY(blobId.PartId()); + TLogoBlobID origBlobId(blobId, 0); + TVDiskID vDiskId = Info->GetVDiskInSubgroup(blobId.PartId() - 1, origBlobId.Hash()); + return {blobId, vDiskId}; + } + + TPartLocation HandoffVDiskForBlobPart(TLogoBlobID blobId, ui64 handoffIdx) { + Y_VERIFY(blobId.PartId()); + Y_VERIFY(handoffIdx < Type.Handoff()); + TLogoBlobID origBlobId(blobId, 0); + TVDiskID vDiskId = Info->GetVDiskInSubgroup(Type.TotalPartCount() + handoffIdx, origBlobId.Hash()); + return {blobId, vDiskId}; + } + + THashMap<TVDiskID, ui32> MakePredictedDelaysForVDisks(TLogoBlobID blobId) { + TBlobStorageGroupInfo::TVDiskIds vDiskIds; + TBlobStorageGroupInfo::TServiceIds serviceIds; + THashMap<TVDiskID, ui32> result; + Info->PickSubgroup(blobId.Hash(), &vDiskIds, &serviceIds); + for (ui32 idx = 0; idx < vDiskIds.size(); ++idx) { + result.emplace(vDiskIds[idx], 0); + } + return result; + } + + //////////////////////////////////////////////////////////////////////////// + // PUT BLOBS TO GROUP MOCK + //////////////////////////////////////////////////////////////////////////// + TTestState& PutBlobsToGroupMock(const TBlobTestSet &blobSet) { + GroupMock.PutBlobSet(blobSet); + return *this; + } + + template <typename TIter> + TTestState& PutBlobsToGroupMock(TIter begin, TIter end) { + TBlobTestSet blobSet; + blobSet.AddBlobs(begin, end); + return PutBlobsToGroupMock(blobSet); + } + + TTestState& PutBlobsToGroupMock(const TVector<TBlobTestSet::TBlob> &blobs) { + return PutBlobsToGroupMock(blobs.begin(), blobs.end()); + } + + //////////////////////////////////////////////////////////////////////////// + // CREATE REQUESTS + //////////////////////////////////////////////////////////////////////////// + TEvBlobStorage::TEvPut::TPtr CreatePutRequest(const TBlobTestSet::TBlob &blob, + TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) + { std::unique_ptr<TEvBlobStorage::TEvPut> put = std::make_unique<TEvBlobStorage::TEvPut>(blob.Id, blob.Data, TInstant::Max(), - handleClass, tactic); - return static_cast<TEventHandle<TEvBlobStorage::TEvPut>*>( + handleClass, tactic); + return static_cast<TEventHandle<TEvBlobStorage::TEvPut>*>( new IEventHandle(EdgeActor, EdgeActor, put.release())); - } - - template <typename TIter, typename TPutIter> - void CreatePutRequests(TIter begin, TIter end, TPutIter out, - TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) - { - for (auto it = begin; it != end; ++it) { - *out++ = CreatePutRequest(*it, tactic, handleClass); - } - } - - template <typename TPutIter> - void CreatePutRequests(const TVector<TBlobTestSet::TBlob> &blobs, TPutIter out, TEvBlobStorage::TEvPut::ETactic tactic, - NKikimrBlobStorage::EPutHandleClass handleClass) - { - CreatePutRequests(blobs.begin(), blobs.end(), out, tactic, handleClass); - } - - TEvBlobStorage::TEvGet::TPtr CreateGetRequest(const TVector<TLogoBlobID> &blobs, bool mustRestore) - { - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queries(new TEvBlobStorage::TEvGet::TQuery[blobs.size()]); - for (ui64 queryIdx = 0; queryIdx < blobs.size(); ++queryIdx) { - TEvBlobStorage::TEvGet::TQuery &q = queries[queryIdx]; - q.Id = blobs[queryIdx]; - q.Shift = 0; - q.Size = q.Id.BlobSize(); - } + } + + template <typename TIter, typename TPutIter> + void CreatePutRequests(TIter begin, TIter end, TPutIter out, + TEvBlobStorage::TEvPut::ETactic tactic, NKikimrBlobStorage::EPutHandleClass handleClass) + { + for (auto it = begin; it != end; ++it) { + *out++ = CreatePutRequest(*it, tactic, handleClass); + } + } + + template <typename TPutIter> + void CreatePutRequests(const TVector<TBlobTestSet::TBlob> &blobs, TPutIter out, TEvBlobStorage::TEvPut::ETactic tactic, + NKikimrBlobStorage::EPutHandleClass handleClass) + { + CreatePutRequests(blobs.begin(), blobs.end(), out, tactic, handleClass); + } + + TEvBlobStorage::TEvGet::TPtr CreateGetRequest(const TVector<TLogoBlobID> &blobs, bool mustRestore) + { + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> queries(new TEvBlobStorage::TEvGet::TQuery[blobs.size()]); + for (ui64 queryIdx = 0; queryIdx < blobs.size(); ++queryIdx) { + TEvBlobStorage::TEvGet::TQuery &q = queries[queryIdx]; + q.Id = blobs[queryIdx]; + q.Shift = 0; + q.Size = q.Id.BlobSize(); + } std::unique_ptr<TEvBlobStorage::TEvGet> get = std::make_unique<TEvBlobStorage::TEvGet>(queries, blobs.size(), TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead, mustRestore, false); - return static_cast<TEventHandle<TEvBlobStorage::TEvGet>*>( + NKikimrBlobStorage::EGetHandleClass::FastRead, mustRestore, false); + return static_cast<TEventHandle<TEvBlobStorage::TEvGet>*>( new IEventHandle(EdgeActor, EdgeActor, get.release())); - } - - //////////////////////////////////////////////////////////////////////////// - // CREATE EVENTS - //////////////////////////////////////////////////////////////////////////// - template <typename TEvent> - typename TEvent::TPtr CreateEventPtr(TActorId recipient, TActorId sender, TEvent *ev, ui64 cookie) { - TAutoPtr<IEventHandle> handle = new IEventHandle(recipient, sender, ev, 0, cookie); - return static_cast<TEventHandle<TEvent>*>(handle.Release()); - } - - template <typename TEvent> + } + + //////////////////////////////////////////////////////////////////////////// + // CREATE EVENTS + //////////////////////////////////////////////////////////////////////////// + template <typename TEvent> + typename TEvent::TPtr CreateEventPtr(TActorId recipient, TActorId sender, TEvent *ev, ui64 cookie) { + TAutoPtr<IEventHandle> handle = new IEventHandle(recipient, sender, ev, 0, cookie); + return static_cast<TEventHandle<TEvent>*>(handle.Release()); + } + + template <typename TEvent> typename TEvent::TPtr CreateEventPtr(TActorId recipient, TActorId sender, std::unique_ptr<TEvent> &ev, ui64 cookie) { return CreateEventPtr(recipient, sender, ev.release(), cookie); - } - - template <typename TEvent> - typename TEvent::TPtr GrabEventPtr() { - TAutoPtr<IEventHandle> handle; - auto ev = Runtime.GrabEdgeEventRethrow<TEvent>(handle); - UNIT_ASSERT(ev); - return static_cast<TEventHandle<TEvent>*>(handle.Release()); - } - - template <typename TEventPtr, typename ...TArgs> - auto CreateEventResultPtr(TEventPtr &ev, TArgs ...args) { - auto result = CreateEventResult(ev->Get(), args...); - auto handle = CreateEventPtr(ev->Sender, ev->Recipient, result, ev->Cookie); - ui64 msgId = ev->Get()->Record.GetMsgQoS().GetMsgId().GetMsgId(); - ui64 sequenceId = ev->Get()->Record.GetMsgQoS().GetMsgId().GetSequenceId(); - handle->Get()->Record.MutableMsgQoS()->MutableMsgId()->SetMsgId(msgId); - handle->Get()->Record.MutableMsgQoS()->MutableMsgId()->SetSequenceId(sequenceId); - return handle; - } - + } + + template <typename TEvent> + typename TEvent::TPtr GrabEventPtr() { + TAutoPtr<IEventHandle> handle; + auto ev = Runtime.GrabEdgeEventRethrow<TEvent>(handle); + UNIT_ASSERT(ev); + return static_cast<TEventHandle<TEvent>*>(handle.Release()); + } + + template <typename TEventPtr, typename ...TArgs> + auto CreateEventResultPtr(TEventPtr &ev, TArgs ...args) { + auto result = CreateEventResult(ev->Get(), args...); + auto handle = CreateEventPtr(ev->Sender, ev->Recipient, result, ev->Cookie); + ui64 msgId = ev->Get()->Record.GetMsgQoS().GetMsgId().GetMsgId(); + ui64 sequenceId = ev->Get()->Record.GetMsgQoS().GetMsgId().GetSequenceId(); + handle->Get()->Record.MutableMsgQoS()->MutableMsgId()->SetMsgId(msgId); + handle->Get()->Record.MutableMsgQoS()->MutableMsgId()->SetSequenceId(sequenceId); + return handle; + } + std::unique_ptr<TEvBlobStorage::TEvVPutResult> CreateEventResult(TEvBlobStorage::TEvVPut *ev, - NKikimrProto::EReplyStatus status, TVDiskID vDiskId) - { - NKikimrBlobStorage::TEvVPut &record = ev->Record; - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - ui64 cookieValue = record.GetCookie(); - ui64 *cookie = record.HasCookie() ? &cookieValue : nullptr; + NKikimrProto::EReplyStatus status, TVDiskID vDiskId) + { + NKikimrBlobStorage::TEvVPut &record = ev->Record; + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + ui64 cookieValue = record.GetCookie(); + ui64 *cookie = record.HasCookie() ? &cookieValue : nullptr; std::unique_ptr<TEvBlobStorage::TEvVPutResult> result(new TEvBlobStorage::TEvVPutResult(status, blobId, vDiskId, - cookie, TOutOfSpaceStatus(0u, 0.0), TAppData::TimeProvider->Now(), - 0, nullptr, nullptr, nullptr, nullptr, 0, NWilson::TTraceId(), 0, TString())); - return result; - } - + cookie, TOutOfSpaceStatus(0u, 0.0), TAppData::TimeProvider->Now(), + 0, nullptr, nullptr, nullptr, nullptr, 0, NWilson::TTraceId(), 0, TString())); + return result; + } + std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> CreateEventResult(TEvBlobStorage::TEvVMultiPut *ev, - NKikimrProto::EReplyStatus status, TVDiskID vDiskId) - { - NKikimrBlobStorage::TEvVMultiPut &record = ev->Record; - ui64 cookieValue = record.GetCookie(); - ui64 *cookie = record.HasCookie() ? &cookieValue : nullptr; + NKikimrProto::EReplyStatus status, TVDiskID vDiskId) + { + NKikimrBlobStorage::TEvVMultiPut &record = ev->Record; + ui64 cookieValue = record.GetCookie(); + ui64 *cookie = record.HasCookie() ? &cookieValue : nullptr; std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> result( - new TEvBlobStorage::TEvVMultiPutResult(status, vDiskId, cookie, TAppData::TimeProvider->Now(), 0, - nullptr, nullptr, nullptr, nullptr, 0, NWilson::TTraceId(), 0, TString())); - result->Record.SetStatusFlags(TOutOfSpaceStatus(0u, 0.0).Flags); - return result; - } - - template <typename TIter> + new TEvBlobStorage::TEvVMultiPutResult(status, vDiskId, cookie, TAppData::TimeProvider->Now(), 0, + nullptr, nullptr, nullptr, nullptr, 0, NWilson::TTraceId(), 0, TString())); + result->Record.SetStatusFlags(TOutOfSpaceStatus(0u, 0.0).Flags); + return result; + } + + template <typename TIter> std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> CreateEventResult(TEvBlobStorage::TEvVMultiPut *ev, - NKikimrProto::EReplyStatus status, TIter begin, TIter end, TVDiskID vDiskId) - { - NKikimrBlobStorage::TEvVMultiPut &record = ev->Record; - std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> result = CreateEventResult(ev, status, vDiskId); - - ui64 itemCount = record.ItemsSize(); - for (ui64 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - UNIT_ASSERT(begin != end); - auto &item = record.GetItems(itemIdx); - UNIT_ASSERT(item.HasBlobID()); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - ui64 cookieValue = item.GetCookie(); - ui64 *cookie = item.HasCookie() ? &cookieValue : nullptr; - result->AddVPutResult(*begin, TString(), blobId, cookie, TOutOfSpaceStatus(0u, 0.0).Flags); - begin++; - } - UNIT_ASSERT(begin == end); - return result; - } - - template <typename TCont> + NKikimrProto::EReplyStatus status, TIter begin, TIter end, TVDiskID vDiskId) + { + NKikimrBlobStorage::TEvVMultiPut &record = ev->Record; + std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> result = CreateEventResult(ev, status, vDiskId); + + ui64 itemCount = record.ItemsSize(); + for (ui64 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + UNIT_ASSERT(begin != end); + auto &item = record.GetItems(itemIdx); + UNIT_ASSERT(item.HasBlobID()); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + ui64 cookieValue = item.GetCookie(); + ui64 *cookie = item.HasCookie() ? &cookieValue : nullptr; + result->AddVPutResult(*begin, TString(), blobId, cookie, TOutOfSpaceStatus(0u, 0.0).Flags); + begin++; + } + UNIT_ASSERT(begin == end); + return result; + } + + template <typename TCont> std::unique_ptr<TEvBlobStorage::TEvVMultiPutResult> CreateEventResult(TEvBlobStorage::TEvVMultiPut *ev, - NKikimrProto::EReplyStatus status, const TCont &cont, TVDiskID vDiskId) - { - return CreateEventResult(ev, status, cont.cbegin(), cont.cend(), vDiskId); - } - + NKikimrProto::EReplyStatus status, const TCont &cont, TVDiskID vDiskId) + { + return CreateEventResult(ev, status, cont.cbegin(), cont.cend(), vDiskId); + } + std::unique_ptr<TEvBlobStorage::TEvVGetResult> CreateEventResult(TEvBlobStorage::TEvVGet *ev, - NKikimrProto::EReplyStatus status = NKikimrProto::OK) - { - TVDiskID vDiskId = VDiskIDFromVDiskID(ev->Record.GetVDiskID()); + NKikimrProto::EReplyStatus status = NKikimrProto::OK) + { + TVDiskID vDiskId = VDiskIDFromVDiskID(ev->Record.GetVDiskID()); std::unique_ptr<TEvBlobStorage::TEvVGetResult> result(new TEvBlobStorage::TEvVGetResult( - status, vDiskId, TAppData::TimeProvider->Now(), 0, nullptr, - nullptr, nullptr, nullptr, NWilson::TTraceId(), {}, 0U, 0U)); - return result; - } - - //////////////////////////////////////////////////////////////////////////// - // HANDLE WITH MOCK - //////////////////////////////////////////////////////////////////////////// - void HandleVGetsWithMock(ui64 vGetCount) { - for (ui64 idx = 0; idx < vGetCount; ++idx) { - TEvBlobStorage::TEvVGet::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVGet>(); - TEvBlobStorage::TEvVGetResult::TPtr result = CreateEventResultPtr(ev, NKikimrProto::UNKNOWN); - GroupMock.OnVGet(*ev->Get(), *result->Get()); - Runtime.Send(result.Release()); - } - } - - void HandleVPutsWithMock(ui64 vPutCount) { - for (ui64 idx = 0; idx < vPutCount; ++idx) { - TEvBlobStorage::TEvVPut::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVPut>(); - TVDiskID vDiskId = GroupMock.GetVDiskID(*ev->Get()); - NKikimrProto::EReplyStatus status = GroupMock.OnVPut(*ev->Get()); - TEvBlobStorage::TEvVPutResult::TPtr result = CreateEventResultPtr(ev, status, vDiskId); - Runtime.Send(result.Release()); - } - } - - void HandleVMultiPutsWithMock(ui64 vMultiPutCount) { - for (ui64 idx = 0; idx < vMultiPutCount; ++idx) { - TEvBlobStorage::TEvVMultiPut::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVMultiPut>(); - TVDiskID vDiskId = GroupMock.GetVDiskID(*ev->Get()); - TVector<NKikimrProto::EReplyStatus> statuses = GroupMock.OnVMultiPut(*ev->Get()); - NKikimrProto::EReplyStatus status = NKikimrProto::OK; - if (std::all_of(statuses.begin(), statuses.end(), [](auto st) { return st == NKikimrProto::RACE; })) { - status = NKikimrProto::RACE; - } - TEvBlobStorage::TEvVMultiPutResult::TPtr result = CreateEventResultPtr(ev, status, statuses, vDiskId); - Runtime.Send(result.Release()); - } - } - - //////////////////////////////////////////////////////////////////////////// - // RECEIVE RESPONSES - //////////////////////////////////////////////////////////////////////////// - void ReceivePutResults(ui64 count, const TMap<TLogoBlobID, NKikimrProto::EReplyStatus> &specialStatus) - { - TSet<TLogoBlobID> seenBlobs; - for (ui64 idx = 0; idx < count; ++idx) { - TAutoPtr<IEventHandle> handle; - TEvBlobStorage::TEvPutResult *putResult = Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPutResult>(handle); - Y_VERIFY(!seenBlobs.count(putResult->Id)); - seenBlobs.insert(putResult->Id); - auto it = specialStatus.find(putResult->Id); - Y_VERIFY(it != specialStatus.end()); - NKikimrProto::EReplyStatus expectedStatus = it->second; - Y_VERIFY_S(putResult->Status == expectedStatus, "expected status " - << NKikimrProto::EReplyStatus_Name(expectedStatus) - << ", but given " - << NKikimrProto::EReplyStatus_Name(putResult->Status)); - } - } -}; - + status, vDiskId, TAppData::TimeProvider->Now(), 0, nullptr, + nullptr, nullptr, nullptr, NWilson::TTraceId(), {}, 0U, 0U)); + return result; + } + + //////////////////////////////////////////////////////////////////////////// + // HANDLE WITH MOCK + //////////////////////////////////////////////////////////////////////////// + void HandleVGetsWithMock(ui64 vGetCount) { + for (ui64 idx = 0; idx < vGetCount; ++idx) { + TEvBlobStorage::TEvVGet::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVGet>(); + TEvBlobStorage::TEvVGetResult::TPtr result = CreateEventResultPtr(ev, NKikimrProto::UNKNOWN); + GroupMock.OnVGet(*ev->Get(), *result->Get()); + Runtime.Send(result.Release()); + } + } + + void HandleVPutsWithMock(ui64 vPutCount) { + for (ui64 idx = 0; idx < vPutCount; ++idx) { + TEvBlobStorage::TEvVPut::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVPut>(); + TVDiskID vDiskId = GroupMock.GetVDiskID(*ev->Get()); + NKikimrProto::EReplyStatus status = GroupMock.OnVPut(*ev->Get()); + TEvBlobStorage::TEvVPutResult::TPtr result = CreateEventResultPtr(ev, status, vDiskId); + Runtime.Send(result.Release()); + } + } + + void HandleVMultiPutsWithMock(ui64 vMultiPutCount) { + for (ui64 idx = 0; idx < vMultiPutCount; ++idx) { + TEvBlobStorage::TEvVMultiPut::TPtr ev = GrabEventPtr<TEvBlobStorage::TEvVMultiPut>(); + TVDiskID vDiskId = GroupMock.GetVDiskID(*ev->Get()); + TVector<NKikimrProto::EReplyStatus> statuses = GroupMock.OnVMultiPut(*ev->Get()); + NKikimrProto::EReplyStatus status = NKikimrProto::OK; + if (std::all_of(statuses.begin(), statuses.end(), [](auto st) { return st == NKikimrProto::RACE; })) { + status = NKikimrProto::RACE; + } + TEvBlobStorage::TEvVMultiPutResult::TPtr result = CreateEventResultPtr(ev, status, statuses, vDiskId); + Runtime.Send(result.Release()); + } + } + + //////////////////////////////////////////////////////////////////////////// + // RECEIVE RESPONSES + //////////////////////////////////////////////////////////////////////////// + void ReceivePutResults(ui64 count, const TMap<TLogoBlobID, NKikimrProto::EReplyStatus> &specialStatus) + { + TSet<TLogoBlobID> seenBlobs; + for (ui64 idx = 0; idx < count; ++idx) { + TAutoPtr<IEventHandle> handle; + TEvBlobStorage::TEvPutResult *putResult = Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvPutResult>(handle); + Y_VERIFY(!seenBlobs.count(putResult->Id)); + seenBlobs.insert(putResult->Id); + auto it = specialStatus.find(putResult->Id); + Y_VERIFY(it != specialStatus.end()); + NKikimrProto::EReplyStatus expectedStatus = it->second; + Y_VERIFY_S(putResult->Status == expectedStatus, "expected status " + << NKikimrProto::EReplyStatus_Name(expectedStatus) + << ", but given " + << NKikimrProto::EReplyStatus_Name(putResult->Status)); + } + } +}; + } // namespace NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_vdisk_mock_ut.h b/ydb/core/blobstorage/dsproxy/ut/dsproxy_vdisk_mock_ut.h index 0a17c046e71..3ddcb3c88e4 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_vdisk_mock_ut.h +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_vdisk_mock_ut.h @@ -1,542 +1,542 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/blobstorage/dsproxy/dsproxy.h> - + #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> - + #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/actor_helpers.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - -class TVDiskMock { - TIntrusivePtr<TBlobStorageGroupInfo> Info; - const TVDiskID VDiskId; - TMap<TLogoBlobID, TString> Blobs; - TMap<TLogoBlobID, TString> NotYetBlobs; - bool IsError; - NKikimrProto::EReplyStatus Status; - TMap<TLogoBlobID, NKikimrProto::EReplyStatus> SpecialStatuses; -public: - TIntrusivePtr<NBackpressure::TFlowRecord> FlowRecord; - - TVDiskMock(TVDiskID vDiskId) - : VDiskId(vDiskId) - , IsError(false) - , Status(NKikimrProto::OK) - , FlowRecord(new NBackpressure::TFlowRecord()) - { - } - - void SetInfo(TIntrusivePtr<TBlobStorageGroupInfo> info) { - Info = info; - } - + +namespace NKikimr { + +class TVDiskMock { + TIntrusivePtr<TBlobStorageGroupInfo> Info; + const TVDiskID VDiskId; + TMap<TLogoBlobID, TString> Blobs; + TMap<TLogoBlobID, TString> NotYetBlobs; + bool IsError; + NKikimrProto::EReplyStatus Status; + TMap<TLogoBlobID, NKikimrProto::EReplyStatus> SpecialStatuses; +public: + TIntrusivePtr<NBackpressure::TFlowRecord> FlowRecord; + + TVDiskMock(TVDiskID vDiskId) + : VDiskId(vDiskId) + , IsError(false) + , Status(NKikimrProto::OK) + , FlowRecord(new NBackpressure::TFlowRecord()) + { + } + + void SetInfo(TIntrusivePtr<TBlobStorageGroupInfo> info) { + Info = info; + } + TActorId GetActorId() { - return Info->GetActorId(Info->GetOrderNumber(VDiskId)); - } - - TVDiskID GetVDiskId() const { - return VDiskId; - } - - NKikimrProto::EReplyStatus Put(const TLogoBlobID id, const TString &data) { - if (IsError) { - return Status; - } - auto it = SpecialStatuses.find(id); - if (it != SpecialStatuses.end()) { - return it->second; - } - if (Blobs.count(id)) { - return NKikimrProto::ALREADY; - } - Blobs[id] = data; - return NKikimrProto::OK; - } - - void Wipe() { - Blobs.clear(); - FlowRecord->SetPredictedDelayNs(0); - } - - void SetError(NKikimrProto::EReplyStatus status) { - if (status == NKikimrProto::NOT_YET) { - for (auto it = Blobs.begin(); it != Blobs.end(); ++it) { - NotYetBlobs[it->first] = it->second; - } - } else { - IsError = true; - Status = status; - } - } - - void SetNotYet(const TLogoBlobID blobID) { - auto it = Blobs.find(blobID); - if (it != Blobs.end()) { - NotYetBlobs[blobID] = it->second; - } - } - - void SetPredictedDelayNs(ui64 predictDelaysNs) { - FlowRecord->SetPredictedDelayNs(predictDelaysNs); - } - - void UnsetError() { - NotYetBlobs.clear(); - IsError = false; - Status = NKikimrProto::OK; - } - - void SetSpecialStatus(const TLogoBlobID blobID, NKikimrProto::EReplyStatus status) { - SpecialStatuses[blobID] = status; - } - - void OnVGet(const TEvBlobStorage::TEvVGet &vGet, TEvBlobStorage::TEvVGetResult &outVGetResult) { - auto &request = vGet.Record; - Y_VERIFY(request.HasCookie()); - if (IsError) { + return Info->GetActorId(Info->GetOrderNumber(VDiskId)); + } + + TVDiskID GetVDiskId() const { + return VDiskId; + } + + NKikimrProto::EReplyStatus Put(const TLogoBlobID id, const TString &data) { + if (IsError) { + return Status; + } + auto it = SpecialStatuses.find(id); + if (it != SpecialStatuses.end()) { + return it->second; + } + if (Blobs.count(id)) { + return NKikimrProto::ALREADY; + } + Blobs[id] = data; + return NKikimrProto::OK; + } + + void Wipe() { + Blobs.clear(); + FlowRecord->SetPredictedDelayNs(0); + } + + void SetError(NKikimrProto::EReplyStatus status) { + if (status == NKikimrProto::NOT_YET) { + for (auto it = Blobs.begin(); it != Blobs.end(); ++it) { + NotYetBlobs[it->first] = it->second; + } + } else { + IsError = true; + Status = status; + } + } + + void SetNotYet(const TLogoBlobID blobID) { + auto it = Blobs.find(blobID); + if (it != Blobs.end()) { + NotYetBlobs[blobID] = it->second; + } + } + + void SetPredictedDelayNs(ui64 predictDelaysNs) { + FlowRecord->SetPredictedDelayNs(predictDelaysNs); + } + + void UnsetError() { + NotYetBlobs.clear(); + IsError = false; + Status = NKikimrProto::OK; + } + + void SetSpecialStatus(const TLogoBlobID blobID, NKikimrProto::EReplyStatus status) { + SpecialStatuses[blobID] = status; + } + + void OnVGet(const TEvBlobStorage::TEvVGet &vGet, TEvBlobStorage::TEvVGetResult &outVGetResult) { + auto &request = vGet.Record; + Y_VERIFY(request.HasCookie()); + if (IsError) { outVGetResult.MakeError(Status, TString(), request); - return; - } - //ui64 messageCookie = request->Record.GetCookie(); - - outVGetResult.Record.SetStatus(NKikimrProto::OK); - // Ignore RangeQuery (pretend there are no results) - - // TODO: Check for overlapping / out of order queries - size_t size = request.ExtremeQueriesSize(); - for (unsigned i = 0; i < size; i++) { - const NKikimrBlobStorage::TExtremeQuery &query = request.GetExtremeQueries(i); - Y_VERIFY(request.HasVDiskID()); - TLogoBlobID id = LogoBlobIDFromLogoBlobID(query.GetId()); - ui64 partId = id.PartId(); - ui64 partBegin = partId ? partId : 1; - ui64 partEnd = partId ? partId : 7; - - ui64 shift = (query.HasShift() ? query.GetShift() : 0); - ui64 *cookie = nullptr; - ui64 cookieValue = 0; - if (query.HasCookie()) { - cookieValue = query.GetCookie(); - cookie = &cookieValue; - } - - ui64 partSize = Info->Type.PartSize(id); - - ui64 rShift = (shift > partSize ? partSize : shift); - ui64 resultSize = query.HasSize() ? query.GetSize() : 0; - if (!resultSize) { - resultSize = partSize; - } - resultSize = Min(partSize - rShift, resultSize); - - bool isNoData = true; - TLogoBlobID idFirst(id, partBegin); - TLogoBlobID idLast(id, partEnd); - for (auto it = Blobs.lower_bound(idFirst); it != Blobs.end() && it->first <= idLast; it++) { - isNoData = false; - if (NotYetBlobs.find(it->first) != NotYetBlobs.end()) { - outVGetResult.AddResult(NKikimrProto::NOT_YET, it->first, shift, - nullptr, 0, cookie); - } else { - TString resultData; - resultData.resize(resultSize); - memcpy((void*)resultData.data(), ((char*)(void*)it->second.data()) + rShift, resultSize); - outVGetResult.AddResult(NKikimrProto::OK, it->first, shift, - resultData.data(), resultSize, cookie); - } - } - if (isNoData) { - CTEST << "VDisk# " << VDiskId.ToString() << " blob# " << id.ToString() << " NODATA" << Endl; - outVGetResult.AddResult(NKikimrProto::NODATA, id, shift, nullptr, 0, cookie); - } - } - Y_VERIFY(request.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); - VDiskIDFromVDiskID(vDiskId, outVGetResult.Record.MutableVDiskID()); - if (request.HasCookie()) { - outVGetResult.Record.SetCookie(request.GetCookie()); - } - } -}; - -class TBlobTestSet { -public: - struct TBlob { - TLogoBlobID Id; - TString Data; - - TBlob(TLogoBlobID id, TString data) - : Id(id) - , Data(data) - {} - }; - -private: - TVector<TBlob> Blobs; - TMap<TLogoBlobID, TString> DataById; - - TString AlphaData(ui32 size) { - TString data = TString::Uninitialized(size); - ui8 *p = (ui8*)(void*)data.Detach(); - for (ui32 offset = 0; offset < size; ++offset) { - p[offset] = (ui8)offset; - } - return data; - } -public: - TBlobTestSet() - {} - - template <typename TIter> - void AddBlobs(TIter begin, TIter end) { - for (auto it = begin; it != end; ++it) { - Blobs.emplace_back(*it); - DataById[it->Id] = it->Data; - } - } - - template <typename TCont> - void AddBlobs(const TCont& blobs) { - AddBlobs(blobs.begin(), blobs.end()); - } - - void GenerateSet(ui32 setIdx, ui32 count, ui32 forceSize = 0) { - switch (setIdx) { - case 0: - for (ui64 i = 0; i < count; ++i) { - ui32 step = i + 1; - ui32 size = forceSize ? forceSize : 750 + i * 31; - TLogoBlobID id(1, 2, step, 0, size, 0); - TString data = AlphaData(size); - Blobs.emplace_back(id, data); - DataById[id] = data; - } - break; - case 1: - for (ui64 i = 0; i < count; ++i) { - ui32 step = i + 1; - ui32 size = forceSize ? forceSize : 92 + i * 31; - TLogoBlobID id(1, 2, step, 0, size, 0); - TString data = AlphaData(size); - Blobs.emplace_back(id, data); - DataById[id] = data; - } - break; - case 2: - for (ui64 i = 0; i < count; ++i) { - ui32 step = i + 1; - ui32 size = forceSize ? forceSize : 92 + i * 31; - TLogoBlobID id(1, 2, step, 0, size, 0, 0, TErasureType::CrcModeWholePart); - TString data = AlphaData(size); - Blobs.emplace_back(id, data); - DataById[id] = data; - } - break; - default: - Y_VERIFY(false, "Unexpected setIdx# %" PRIu32, setIdx); - break; - } - } - - ui64 Size() const { - return Blobs.size(); - } - - const TBlob& Get(ui32 idx) const { - Y_VERIFY(idx < Blobs.size()); - return Blobs[idx]; - } - - TString Get(TLogoBlobID id, ui32 shift, ui32 size) const { - auto it = DataById.find(id); - Y_VERIFY(it != DataById.end()); - TString data = it->second; - shift = Min(shift, id.BlobSize()); - ui32 rSize = size ? size : id.BlobSize(); - rSize = Min(rSize, id.BlobSize() - shift); - TString result; - result.resize(rSize); - memcpy((void*)result.data(), (char*)(void*)data.data() + shift, rSize); - return result; - } - - void Check(ui32 idx, TLogoBlobID id, ui32 shift, ui32 size, TString buffer) const { - const TBlob& blob = Get(idx); - Y_VERIFY(id == blob.Id); - if (size != buffer.size()) { - UNIT_ASSERT_VALUES_EQUAL(size, buffer.size()); - } - const ui8 *a = (const ui8*)(void*)blob.Data.data(); - const ui8 *b = (const ui8*)(void*)buffer.data(); - if (memcmp(a + shift, b, size) != 0) { - for (ui32 offset = 0; offset < size; ++offset) { - UNIT_ASSERT_VALUES_EQUAL_C((ui32)a[shift + offset], (ui32)b[offset], - "Id# " << id.ToString() << " offset# " << offset - << " shift# " << shift << " s+o# " << (shift + offset) << " size# " << size - << " canonic a# " << (ui32)a[shift+offset] << " actual b# " << (ui32)b[offset]); - } - } - } -}; - -struct TPartLocation { - TLogoBlobID BlobId; - TVDiskID VDiskId; - - static TString ToString(const TPartLocation &self) { - return TStringBuilder() - << "{BlobId# " << self.BlobId.ToString() - << " VDiskId# " << self.VDiskId.ToString() - << "}"; - } - - TString ToString() const { - return ToString(*this); - } - - bool operator<(const TPartLocation &other) const { - return std::tie(BlobId, VDiskId) < std::tie(other.BlobId, other.VDiskId); - } -}; - -class TGroupMock { - const ui32 GroupId; - const TErasureType::EErasureSpecies ErasureSpecies; - const ui32 FailDomains; - const ui32 DrivesPerFailDomain; - - TVector<TVDiskMock> VDisks; - TIntrusivePtr<TBlobStorageGroupInfo> Info; - - TVDiskMock& GetVDisk(ui32 failDomainIdx, ui32 driveIdx) { - ui32 i = failDomainIdx * DrivesPerFailDomain + driveIdx; - Y_VERIFY(i < VDisks.size(), "i# %" PRIu32 " size# %" PRIu32, (ui32)i, (ui32)VDisks.size()); - return VDisks[i]; - } - - void InitBsInfo() { - for (auto& mock : VDisks) { - mock.SetInfo(Info); - } - } - - void InitVDisks() { - for (ui64 domainIdx = 0; domainIdx < FailDomains; ++domainIdx) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - // Node = domainIdx - // PoolId = driveIdx - // LocalId = index in VDisks - TVDiskID vDiskId(GroupId, 1, 0, domainIdx, driveIdx); - VDisks.emplace_back(vDiskId); - } - } - } - -public: - TGroupMock(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 failDomains, ui32 drivesPerFailDomain, - TIntrusivePtr<TBlobStorageGroupInfo> info) - : GroupId(groupId) - , ErasureSpecies(erasureSpecies) - , FailDomains(failDomains) - , DrivesPerFailDomain(drivesPerFailDomain) - , Info(info) - { - Y_UNUSED(ErasureSpecies); - InitVDisks(); - InitBsInfo(); - } - - TGroupMock(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 failDomains, ui32 drivesPerFailDomain) - : TGroupMock(groupId, erasureSpecies, failDomains, drivesPerFailDomain, - new TBlobStorageGroupInfo(erasureSpecies, drivesPerFailDomain, failDomains)) - { - } - - ui32 VDiskIdx(const TVDiskID &id) { - ui32 idx = (ui32)id.FailDomain * DrivesPerFailDomain + (ui32)id.VDisk; - return idx; - } - - TIntrusivePtr<TBlobStorageGroupInfo> GetInfo() { - return Info; - } - - void OnVGet(const TEvBlobStorage::TEvVGet &vGet, TEvBlobStorage::TEvVGetResult &outVGetResult) { - Y_VERIFY(vGet.Record.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(vGet.Record.GetVDiskID()); - GetVDisk(vDiskId.FailDomain, vDiskId.VDisk).OnVGet(vGet, outVGetResult); - } - - NKikimrProto::EReplyStatus OnVPut(TEvBlobStorage::TEvVPut &vPut) { - const NKikimrBlobStorage::TEvVPut &record = vPut.Record; - Y_VERIFY(record.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - ui32 idx = VDiskIdx(vDiskId); - TVDiskMock &disk = VDisks[idx]; - if (disk.GetVDiskId() != vDiskId) { - return NKikimrProto::RACE; - } - const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - TString buffer = vPut.GetBuffer().ConvertToString(); - NKikimrProto::EReplyStatus status = disk.Put(blobId, buffer); - return status; - } - - template <typename TEvent> - TVDiskID GetVDiskID(TEvent &vPut) { - const auto &record = vPut.Record; - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - ui32 idx = VDiskIdx(vDiskId); - TVDiskMock &disk = VDisks[idx]; - return disk.GetVDiskId(); - } - - TVector<NKikimrProto::EReplyStatus> OnVMultiPut(TEvBlobStorage::TEvVMultiPut &vMultiPut) { - const NKikimrBlobStorage::TEvVMultiPut &record = vMultiPut.Record; - Y_VERIFY(record.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - ui32 idx = VDiskIdx(vDiskId); - TVDiskMock &disk = VDisks[idx]; - if (disk.GetVDiskId() != vDiskId) { - return TVector<NKikimrProto::EReplyStatus>(record.ItemsSize(), NKikimrProto::RACE); - } - Y_VERIFY(disk.GetVDiskId() == vDiskId); - TVector<NKikimrProto::EReplyStatus> statuses; - for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { - auto &item = record.GetItems(itemIdx); - const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - TString buffer = vMultiPut.GetItemBuffer(itemIdx).ConvertToString(); - statuses.push_back(disk.Put(blobId, buffer)); - } - return statuses; - } - - void Wipe(ui32 domainIdx) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).Wipe(); - } - } - - void Wipe() { - for (ui32 idx = 0; idx < VDisks.size(); ++idx) { - VDisks[idx].Wipe(); - } - } - - void SetError(ui32 domainIdx, NKikimrProto::EReplyStatus status) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).SetError(status); - } - } - - void SetPredictedDelayNs(ui32 domainIdx, ui64 predictedDelayNs) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).SetPredictedDelayNs(predictedDelayNs); - } - } - - void UnsetError(ui32 domainIdx) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).UnsetError(); - } - } - - void SetSpecialStatus(ui32 domainIdx, TLogoBlobID blobID, NKikimrProto::EReplyStatus status) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).SetSpecialStatus(blobID, status); - } - } - - void SetSpecialStatuses(const TMap<TPartLocation, NKikimrProto::EReplyStatus> &statuses) { - for (auto &[partLocation, status] : statuses) { - ui64 vDiskIdx = VDiskIdx(partLocation.VDiskId); - VDisks[vDiskIdx].SetSpecialStatus(partLocation.BlobId, status); - } - } - - void SetNotYetBlob(ui32 domainIdx, const TLogoBlobID blobID) { - for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - GetVDisk(domainIdx, driveIdx).SetNotYet(blobID); - } - } - - void Put(const TLogoBlobID id, const TString &data, ui32 handoffsToUse = 0) { - const ui32 hash = id.Hash(); - const ui32 totalvd = Info->Type.BlobSubgroupSize(); - const ui32 totalParts = Info->Type.TotalPartCount(); - Y_VERIFY(id.BlobSize() == data.size()); - Y_VERIFY(totalvd >= totalParts); - TBlobStorageGroupInfo::TServiceIds vDisksSvc; - TBlobStorageGroupInfo::TVDiskIds vDisksId; - Info->PickSubgroup(hash, &vDisksId, &vDisksSvc); - - TString encryptedData = data; - char *dataBytes = encryptedData.Detach(); - Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), id, *Info); - - TDataPartSet partSet; - partSet.Parts.resize(totalParts); - Info->Type.SplitData((TErasureType::ECrcMode)id.CrcMode(), encryptedData, partSet); - for (ui32 i = 0; i < totalParts; ++i) { - TLogoBlobID pId(id, i + 1); - TString pData = partSet.Parts[i].OwnedString; - if (i < handoffsToUse) { - Y_VERIFY(totalParts + i < totalvd); - GetVDisk(vDisksId[totalParts + i].FailDomain, vDisksId[totalParts + i].VDisk).Put(pId, pData); - } else { - GetVDisk(vDisksId[i].FailDomain, vDisksId[i].VDisk).Put(pId, pData); - } - } - } - - i32 DomainIdxForBlobSubgroupIdx(const TLogoBlobID id, i32 subgroupIdx) { - const ui32 hash = id.Hash(); - const ui32 totalvd = Info->Type.BlobSubgroupSize(); - const ui32 totalParts = Info->Type.TotalPartCount(); - Y_VERIFY(totalvd >= totalParts); - TBlobStorageGroupInfo::TServiceIds vDisksSvc; - TBlobStorageGroupInfo::TVDiskIds vDisksId; - Info->PickSubgroup(hash, &vDisksId, &vDisksSvc); - - return vDisksId[subgroupIdx].FailDomain; - } - - void PutBlobSet(const TBlobTestSet &blobSet, ui32 handoffsToUse = 0) { - for (ui64 i = 0; i < blobSet.Size(); ++i) { - const auto &blob = blobSet.Get(i); - Put(blob.Id, blob.Data, handoffsToUse); - } - } - + return; + } + //ui64 messageCookie = request->Record.GetCookie(); + + outVGetResult.Record.SetStatus(NKikimrProto::OK); + // Ignore RangeQuery (pretend there are no results) + + // TODO: Check for overlapping / out of order queries + size_t size = request.ExtremeQueriesSize(); + for (unsigned i = 0; i < size; i++) { + const NKikimrBlobStorage::TExtremeQuery &query = request.GetExtremeQueries(i); + Y_VERIFY(request.HasVDiskID()); + TLogoBlobID id = LogoBlobIDFromLogoBlobID(query.GetId()); + ui64 partId = id.PartId(); + ui64 partBegin = partId ? partId : 1; + ui64 partEnd = partId ? partId : 7; + + ui64 shift = (query.HasShift() ? query.GetShift() : 0); + ui64 *cookie = nullptr; + ui64 cookieValue = 0; + if (query.HasCookie()) { + cookieValue = query.GetCookie(); + cookie = &cookieValue; + } + + ui64 partSize = Info->Type.PartSize(id); + + ui64 rShift = (shift > partSize ? partSize : shift); + ui64 resultSize = query.HasSize() ? query.GetSize() : 0; + if (!resultSize) { + resultSize = partSize; + } + resultSize = Min(partSize - rShift, resultSize); + + bool isNoData = true; + TLogoBlobID idFirst(id, partBegin); + TLogoBlobID idLast(id, partEnd); + for (auto it = Blobs.lower_bound(idFirst); it != Blobs.end() && it->first <= idLast; it++) { + isNoData = false; + if (NotYetBlobs.find(it->first) != NotYetBlobs.end()) { + outVGetResult.AddResult(NKikimrProto::NOT_YET, it->first, shift, + nullptr, 0, cookie); + } else { + TString resultData; + resultData.resize(resultSize); + memcpy((void*)resultData.data(), ((char*)(void*)it->second.data()) + rShift, resultSize); + outVGetResult.AddResult(NKikimrProto::OK, it->first, shift, + resultData.data(), resultSize, cookie); + } + } + if (isNoData) { + CTEST << "VDisk# " << VDiskId.ToString() << " blob# " << id.ToString() << " NODATA" << Endl; + outVGetResult.AddResult(NKikimrProto::NODATA, id, shift, nullptr, 0, cookie); + } + } + Y_VERIFY(request.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); + VDiskIDFromVDiskID(vDiskId, outVGetResult.Record.MutableVDiskID()); + if (request.HasCookie()) { + outVGetResult.Record.SetCookie(request.GetCookie()); + } + } +}; + +class TBlobTestSet { +public: + struct TBlob { + TLogoBlobID Id; + TString Data; + + TBlob(TLogoBlobID id, TString data) + : Id(id) + , Data(data) + {} + }; + +private: + TVector<TBlob> Blobs; + TMap<TLogoBlobID, TString> DataById; + + TString AlphaData(ui32 size) { + TString data = TString::Uninitialized(size); + ui8 *p = (ui8*)(void*)data.Detach(); + for (ui32 offset = 0; offset < size; ++offset) { + p[offset] = (ui8)offset; + } + return data; + } +public: + TBlobTestSet() + {} + + template <typename TIter> + void AddBlobs(TIter begin, TIter end) { + for (auto it = begin; it != end; ++it) { + Blobs.emplace_back(*it); + DataById[it->Id] = it->Data; + } + } + + template <typename TCont> + void AddBlobs(const TCont& blobs) { + AddBlobs(blobs.begin(), blobs.end()); + } + + void GenerateSet(ui32 setIdx, ui32 count, ui32 forceSize = 0) { + switch (setIdx) { + case 0: + for (ui64 i = 0; i < count; ++i) { + ui32 step = i + 1; + ui32 size = forceSize ? forceSize : 750 + i * 31; + TLogoBlobID id(1, 2, step, 0, size, 0); + TString data = AlphaData(size); + Blobs.emplace_back(id, data); + DataById[id] = data; + } + break; + case 1: + for (ui64 i = 0; i < count; ++i) { + ui32 step = i + 1; + ui32 size = forceSize ? forceSize : 92 + i * 31; + TLogoBlobID id(1, 2, step, 0, size, 0); + TString data = AlphaData(size); + Blobs.emplace_back(id, data); + DataById[id] = data; + } + break; + case 2: + for (ui64 i = 0; i < count; ++i) { + ui32 step = i + 1; + ui32 size = forceSize ? forceSize : 92 + i * 31; + TLogoBlobID id(1, 2, step, 0, size, 0, 0, TErasureType::CrcModeWholePart); + TString data = AlphaData(size); + Blobs.emplace_back(id, data); + DataById[id] = data; + } + break; + default: + Y_VERIFY(false, "Unexpected setIdx# %" PRIu32, setIdx); + break; + } + } + + ui64 Size() const { + return Blobs.size(); + } + + const TBlob& Get(ui32 idx) const { + Y_VERIFY(idx < Blobs.size()); + return Blobs[idx]; + } + + TString Get(TLogoBlobID id, ui32 shift, ui32 size) const { + auto it = DataById.find(id); + Y_VERIFY(it != DataById.end()); + TString data = it->second; + shift = Min(shift, id.BlobSize()); + ui32 rSize = size ? size : id.BlobSize(); + rSize = Min(rSize, id.BlobSize() - shift); + TString result; + result.resize(rSize); + memcpy((void*)result.data(), (char*)(void*)data.data() + shift, rSize); + return result; + } + + void Check(ui32 idx, TLogoBlobID id, ui32 shift, ui32 size, TString buffer) const { + const TBlob& blob = Get(idx); + Y_VERIFY(id == blob.Id); + if (size != buffer.size()) { + UNIT_ASSERT_VALUES_EQUAL(size, buffer.size()); + } + const ui8 *a = (const ui8*)(void*)blob.Data.data(); + const ui8 *b = (const ui8*)(void*)buffer.data(); + if (memcmp(a + shift, b, size) != 0) { + for (ui32 offset = 0; offset < size; ++offset) { + UNIT_ASSERT_VALUES_EQUAL_C((ui32)a[shift + offset], (ui32)b[offset], + "Id# " << id.ToString() << " offset# " << offset + << " shift# " << shift << " s+o# " << (shift + offset) << " size# " << size + << " canonic a# " << (ui32)a[shift+offset] << " actual b# " << (ui32)b[offset]); + } + } + } +}; + +struct TPartLocation { + TLogoBlobID BlobId; + TVDiskID VDiskId; + + static TString ToString(const TPartLocation &self) { + return TStringBuilder() + << "{BlobId# " << self.BlobId.ToString() + << " VDiskId# " << self.VDiskId.ToString() + << "}"; + } + + TString ToString() const { + return ToString(*this); + } + + bool operator<(const TPartLocation &other) const { + return std::tie(BlobId, VDiskId) < std::tie(other.BlobId, other.VDiskId); + } +}; + +class TGroupMock { + const ui32 GroupId; + const TErasureType::EErasureSpecies ErasureSpecies; + const ui32 FailDomains; + const ui32 DrivesPerFailDomain; + + TVector<TVDiskMock> VDisks; + TIntrusivePtr<TBlobStorageGroupInfo> Info; + + TVDiskMock& GetVDisk(ui32 failDomainIdx, ui32 driveIdx) { + ui32 i = failDomainIdx * DrivesPerFailDomain + driveIdx; + Y_VERIFY(i < VDisks.size(), "i# %" PRIu32 " size# %" PRIu32, (ui32)i, (ui32)VDisks.size()); + return VDisks[i]; + } + + void InitBsInfo() { + for (auto& mock : VDisks) { + mock.SetInfo(Info); + } + } + + void InitVDisks() { + for (ui64 domainIdx = 0; domainIdx < FailDomains; ++domainIdx) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + // Node = domainIdx + // PoolId = driveIdx + // LocalId = index in VDisks + TVDiskID vDiskId(GroupId, 1, 0, domainIdx, driveIdx); + VDisks.emplace_back(vDiskId); + } + } + } + +public: + TGroupMock(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 failDomains, ui32 drivesPerFailDomain, + TIntrusivePtr<TBlobStorageGroupInfo> info) + : GroupId(groupId) + , ErasureSpecies(erasureSpecies) + , FailDomains(failDomains) + , DrivesPerFailDomain(drivesPerFailDomain) + , Info(info) + { + Y_UNUSED(ErasureSpecies); + InitVDisks(); + InitBsInfo(); + } + + TGroupMock(ui32 groupId, TErasureType::EErasureSpecies erasureSpecies, ui32 failDomains, ui32 drivesPerFailDomain) + : TGroupMock(groupId, erasureSpecies, failDomains, drivesPerFailDomain, + new TBlobStorageGroupInfo(erasureSpecies, drivesPerFailDomain, failDomains)) + { + } + + ui32 VDiskIdx(const TVDiskID &id) { + ui32 idx = (ui32)id.FailDomain * DrivesPerFailDomain + (ui32)id.VDisk; + return idx; + } + + TIntrusivePtr<TBlobStorageGroupInfo> GetInfo() { + return Info; + } + + void OnVGet(const TEvBlobStorage::TEvVGet &vGet, TEvBlobStorage::TEvVGetResult &outVGetResult) { + Y_VERIFY(vGet.Record.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(vGet.Record.GetVDiskID()); + GetVDisk(vDiskId.FailDomain, vDiskId.VDisk).OnVGet(vGet, outVGetResult); + } + + NKikimrProto::EReplyStatus OnVPut(TEvBlobStorage::TEvVPut &vPut) { + const NKikimrBlobStorage::TEvVPut &record = vPut.Record; + Y_VERIFY(record.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + ui32 idx = VDiskIdx(vDiskId); + TVDiskMock &disk = VDisks[idx]; + if (disk.GetVDiskId() != vDiskId) { + return NKikimrProto::RACE; + } + const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + TString buffer = vPut.GetBuffer().ConvertToString(); + NKikimrProto::EReplyStatus status = disk.Put(blobId, buffer); + return status; + } + + template <typename TEvent> + TVDiskID GetVDiskID(TEvent &vPut) { + const auto &record = vPut.Record; + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + ui32 idx = VDiskIdx(vDiskId); + TVDiskMock &disk = VDisks[idx]; + return disk.GetVDiskId(); + } + + TVector<NKikimrProto::EReplyStatus> OnVMultiPut(TEvBlobStorage::TEvVMultiPut &vMultiPut) { + const NKikimrBlobStorage::TEvVMultiPut &record = vMultiPut.Record; + Y_VERIFY(record.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + ui32 idx = VDiskIdx(vDiskId); + TVDiskMock &disk = VDisks[idx]; + if (disk.GetVDiskId() != vDiskId) { + return TVector<NKikimrProto::EReplyStatus>(record.ItemsSize(), NKikimrProto::RACE); + } + Y_VERIFY(disk.GetVDiskId() == vDiskId); + TVector<NKikimrProto::EReplyStatus> statuses; + for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { + auto &item = record.GetItems(itemIdx); + const TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + TString buffer = vMultiPut.GetItemBuffer(itemIdx).ConvertToString(); + statuses.push_back(disk.Put(blobId, buffer)); + } + return statuses; + } + + void Wipe(ui32 domainIdx) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).Wipe(); + } + } + + void Wipe() { + for (ui32 idx = 0; idx < VDisks.size(); ++idx) { + VDisks[idx].Wipe(); + } + } + + void SetError(ui32 domainIdx, NKikimrProto::EReplyStatus status) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).SetError(status); + } + } + + void SetPredictedDelayNs(ui32 domainIdx, ui64 predictedDelayNs) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).SetPredictedDelayNs(predictedDelayNs); + } + } + + void UnsetError(ui32 domainIdx) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).UnsetError(); + } + } + + void SetSpecialStatus(ui32 domainIdx, TLogoBlobID blobID, NKikimrProto::EReplyStatus status) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).SetSpecialStatus(blobID, status); + } + } + + void SetSpecialStatuses(const TMap<TPartLocation, NKikimrProto::EReplyStatus> &statuses) { + for (auto &[partLocation, status] : statuses) { + ui64 vDiskIdx = VDiskIdx(partLocation.VDiskId); + VDisks[vDiskIdx].SetSpecialStatus(partLocation.BlobId, status); + } + } + + void SetNotYetBlob(ui32 domainIdx, const TLogoBlobID blobID) { + for (ui64 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + GetVDisk(domainIdx, driveIdx).SetNotYet(blobID); + } + } + + void Put(const TLogoBlobID id, const TString &data, ui32 handoffsToUse = 0) { + const ui32 hash = id.Hash(); + const ui32 totalvd = Info->Type.BlobSubgroupSize(); + const ui32 totalParts = Info->Type.TotalPartCount(); + Y_VERIFY(id.BlobSize() == data.size()); + Y_VERIFY(totalvd >= totalParts); + TBlobStorageGroupInfo::TServiceIds vDisksSvc; + TBlobStorageGroupInfo::TVDiskIds vDisksId; + Info->PickSubgroup(hash, &vDisksId, &vDisksSvc); + + TString encryptedData = data; + char *dataBytes = encryptedData.Detach(); + Encrypt(dataBytes, dataBytes, 0, encryptedData.size(), id, *Info); + + TDataPartSet partSet; + partSet.Parts.resize(totalParts); + Info->Type.SplitData((TErasureType::ECrcMode)id.CrcMode(), encryptedData, partSet); + for (ui32 i = 0; i < totalParts; ++i) { + TLogoBlobID pId(id, i + 1); + TString pData = partSet.Parts[i].OwnedString; + if (i < handoffsToUse) { + Y_VERIFY(totalParts + i < totalvd); + GetVDisk(vDisksId[totalParts + i].FailDomain, vDisksId[totalParts + i].VDisk).Put(pId, pData); + } else { + GetVDisk(vDisksId[i].FailDomain, vDisksId[i].VDisk).Put(pId, pData); + } + } + } + + i32 DomainIdxForBlobSubgroupIdx(const TLogoBlobID id, i32 subgroupIdx) { + const ui32 hash = id.Hash(); + const ui32 totalvd = Info->Type.BlobSubgroupSize(); + const ui32 totalParts = Info->Type.TotalPartCount(); + Y_VERIFY(totalvd >= totalParts); + TBlobStorageGroupInfo::TServiceIds vDisksSvc; + TBlobStorageGroupInfo::TVDiskIds vDisksId; + Info->PickSubgroup(hash, &vDisksId, &vDisksSvc); + + return vDisksId[subgroupIdx].FailDomain; + } + + void PutBlobSet(const TBlobTestSet &blobSet, ui32 handoffsToUse = 0) { + for (ui64 i = 0; i < blobSet.Size(); ++i) { + const auto &blob = blobSet.Get(i); + Put(blob.Id, blob.Data, handoffsToUse); + } + } + TIntrusivePtr<TGroupQueues> MakeGroupQueues() { TIntrusivePtr<TGroupQueues> groupQueues(new TGroupQueues(Info->GetTopology())); ui32 idx = 0; for (auto& domain : groupQueues->FailDomains) { - for (auto& vDisk : domain.VDisks) { - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutTabletLog).Reset( - VDisks[idx].FlowRecord); - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob).Reset( - VDisks[idx].FlowRecord); - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutUserData).Reset( - VDisks[idx].FlowRecord); - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetAsyncRead).Reset( - VDisks[idx].FlowRecord); - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetFastRead).Reset( - VDisks[idx].FlowRecord); - vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetDiscover).Reset( - VDisks[idx].FlowRecord); + for (auto& vDisk : domain.VDisks) { + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutTabletLog).Reset( + VDisks[idx].FlowRecord); + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob).Reset( + VDisks[idx].FlowRecord); + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::PutUserData).Reset( + VDisks[idx].FlowRecord); + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetAsyncRead).Reset( + VDisks[idx].FlowRecord); + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetFastRead).Reset( + VDisks[idx].FlowRecord); + vDisk.Queues.FlowRecordForQueueId(NKikimrBlobStorage::EVDiskQueueId::GetDiscover).Reset( + VDisks[idx].FlowRecord); ++idx; - } - } + } + } return groupQueues; - } -}; - -} // namespace NKikimr + } +}; + +} // namespace NKikimr diff --git a/ydb/core/blobstorage/dsproxy/ut/ya.make b/ydb/core/blobstorage/dsproxy/ut/ya.make index 76825e572bb..baef8d115a9 100644 --- a/ydb/core/blobstorage/dsproxy/ut/ya.make +++ b/ydb/core/blobstorage/dsproxy/ut/ya.make @@ -41,8 +41,8 @@ SRCS( dsproxy_put_ut.cpp dsproxy_quorum_tracker_ut.cpp dsproxy_sequence_ut.cpp - dsproxy_patch_ut.cpp - dsproxy_counters_ut.cpp + dsproxy_patch_ut.cpp + dsproxy_counters_ut.cpp ) diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp index 13b1bec3563..b9dccb8a11f 100644 --- a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp @@ -30,9 +30,9 @@ #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/core/scheduler_basic.h> #include <library/cpp/actors/interconnect/interconnect.h> -#include <library/cpp/actors/interconnect/poller_tcp.h> -#include <library/cpp/actors/interconnect/poller_actor.h> -#include <library/cpp/actors/interconnect/mock/ic_mock.h> +#include <library/cpp/actors/interconnect/poller_tcp.h> +#include <library/cpp/actors/interconnect/poller_actor.h> +#include <library/cpp/actors/interconnect/mock/ic_mock.h> #include <library/cpp/actors/protos/services_common.pb.h> #include <library/cpp/actors/util/affinity.h> #include <library/cpp/svnversion/svnversion.h> @@ -41,13 +41,13 @@ #include <util/folder/dirut.h> #include <util/folder/tempdir.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/generic/hash.h> #include <util/generic/string.h> #include <util/generic/yexception.h> -#include <util/random/mersenne.h> -#include <util/random/random.h> -#include <util/random/shuffle.h> +#include <util/random/mersenne.h> +#include <util/random/random.h> +#include <util/random/shuffle.h> #include <util/string/escape.h> #include <util/string/printf.h> #include <util/system/backtrace.h> @@ -64,7 +64,7 @@ namespace NKikimr { do { \ UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::msg, "Unexpected message " << (int)LastResponse.Message); \ UNIT_ASSERT_EQUAL_C(LastResponse.Status, NKikimrProto::st, "Unexpected status " \ - << StatusToString(LastResponse.Status) << " deadVDisks " << Env->DeadVDisksMask); \ + << StatusToString(LastResponse.Status) << " deadVDisks " << Env->DeadVDisksMask); \ UNIT_ASSERT_VALUES_EQUAL_C(LastResponse.Data.size(), sz, "Data size " \ << (int)LastResponse.Data.size() << " while expected " << (sz)); \ } while(false) @@ -76,16 +76,16 @@ do { \ << EscapeC(TString(LastResponse.Data[0])).c_str() << "' instead of '" << EscapeC(TString(d)).c_str() << "'"); \ } while(false) - -#define TEST_RESPONSE_FULLCHECK(msg, st, sz, d) \ -do { \ - TEST_RESPONSE_3(msg, st, sz); \ - for (size_t i = 0; i < sz; ++i) { \ - UNIT_ASSERT_C(TString(LastResponse.Data[i]) == TString(d[i]), "Got " << i << "'th element '" \ - << EscapeC(TString(LastResponse.Data[i])).c_str() << "' instead of '" << EscapeC(TString(d[i])).c_str() << "' at test step " << TestStep); \ - } \ -} while (false) - + +#define TEST_RESPONSE_FULLCHECK(msg, st, sz, d) \ +do { \ + TEST_RESPONSE_3(msg, st, sz); \ + for (size_t i = 0; i < sz; ++i) { \ + UNIT_ASSERT_C(TString(LastResponse.Data[i]) == TString(d[i]), "Got " << i << "'th element '" \ + << EscapeC(TString(LastResponse.Data[i])).c_str() << "' instead of '" << EscapeC(TString(d[i])).c_str() << "' at test step " << TestStep); \ + } \ +} while (false) + #define VERBOSE_COUT(str) \ do { \ if (IsVerbose) { \ @@ -107,105 +107,105 @@ static bool IsVerbose = false; static bool IsProfilerEnabled = true; -TString TestData2( - "But I must explain to you how all this mistaken idea of denouncing pleasure and praising pain was born.."); +TString TestData2( + "But I must explain to you how all this mistaken idea of denouncing pleasure and praising pain was born.."); static TString StatusToString(const NKikimrProto::EReplyStatus status) { return NKikimrProto::EReplyStatus_Name(status); } - -struct ITestParametrs : public TThrRefBase { - virtual ~ITestParametrs() - { - } -}; - -struct TTestArgs { - ui64 BadVDiskMask = 0; - TBlobStorageGroupType::EErasureSpecies ErasureSpecies; - ui32 SleepMilliseconds = 0; - bool StartBadDisks = true; - ui32 FailDomainCount = 0; - ui32 DrivesPerFailDomain = 0; - TIntrusivePtr<ITestParametrs> Parametrs = nullptr; - bool EnablePutBatching = DefaultEnablePutBatching; - TPDiskCategory::EDeviceType DeviceType = TPDiskCategory::DEVICE_TYPE_ROT; -}; - -struct TTestEnvironment : public TThrRefBase { - ui32 VDiskCount; - ui64 DeadVDisksMask; + +struct ITestParametrs : public TThrRefBase { + virtual ~ITestParametrs() + { + } +}; + +struct TTestArgs { + ui64 BadVDiskMask = 0; + TBlobStorageGroupType::EErasureSpecies ErasureSpecies; + ui32 SleepMilliseconds = 0; + bool StartBadDisks = true; + ui32 FailDomainCount = 0; + ui32 DrivesPerFailDomain = 0; + TIntrusivePtr<ITestParametrs> Parametrs = nullptr; + bool EnablePutBatching = DefaultEnablePutBatching; + TPDiskCategory::EDeviceType DeviceType = TPDiskCategory::DEVICE_TYPE_ROT; +}; + +struct TTestEnvironment : public TThrRefBase { + ui32 VDiskCount; + ui64 DeadVDisksMask; TVector<TActorId> VDisks; TVector<TActorId> PDisks; - TVector<TVDiskID> VDiskIds; - - bool ShouldBeUnwritable; - bool ShouldBeUndiscoverable; - - ui64 FailDomainCount; - ui64 DrivesPerFailDomain; - ui32 FailedDiskCount; - TBlobStorageGroupType GroupType; - + TVector<TVDiskID> VDiskIds; + + bool ShouldBeUnwritable; + bool ShouldBeUndiscoverable; + + ui64 FailDomainCount; + ui64 DrivesPerFailDomain; + ui32 FailedDiskCount; + TBlobStorageGroupType GroupType; + TActorId ProxyId; TActorId ProxyTestId; - - TSystemEvent DoneEvent; - yexception LastException; - volatile bool IsLastExceptionSet = false; - - - TTestEnvironment(const TTestArgs &args) - : DeadVDisksMask(args.StartBadDisks ? 0 : args.BadVDiskMask) - , FailedDiskCount(0) - , GroupType(args.ErasureSpecies) - , ProxyId(MakeBlobStorageProxyID(0)) - , ProxyTestId(MakeBlobStorageProxyID(1)) - { - for (int i = 0; i < 32; ++i) { - if (args.BadVDiskMask & (1 << i)) { - ++FailedDiskCount; - } - } - - ShouldBeUnwritable = FailedDiskCount > GroupType.Handoff(); - ShouldBeUndiscoverable = FailedDiskCount > GroupType.ParityParts(); - - VDiskCount = args.FailDomainCount * args.DrivesPerFailDomain; - if (VDiskCount == 0) { - VDiskCount = GroupType.BlobSubgroupSize(); - DrivesPerFailDomain = 1; - FailDomainCount = VDiskCount; - } else { - DrivesPerFailDomain = args.DrivesPerFailDomain; - FailDomainCount = args.FailDomainCount; - } - - VDisks.resize(VDiskCount); - PDisks.resize(VDiskCount); - VDiskIds.reserve(VDiskCount); - for (ui32 failDomainIdx = 0; failDomainIdx < FailDomainCount; ++failDomainIdx) { - for (ui32 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { - ui32 i = GetVDiskTestIdx(failDomainIdx, driveIdx); - VDisks[i] = MakeBlobStorageVDiskID(2, i + 1, 1); - PDisks[i] = MakeBlobStoragePDiskID(2, i + 1); - VDiskIds.emplace_back(0, 1, 0, failDomainIdx, driveIdx); - } - } - } - - ui64 GetVDiskTestIdx(ui64 failDomain, ui64 orderNumber) const { - return failDomain * DrivesPerFailDomain + orderNumber; - } - - ui64 GetVDiskTestIdx(const TVDiskID &vDiskId) const { - ui64 failDomain = vDiskId.FailDomain; - ui64 orderNumber = vDiskId.VDisk; - return GetVDiskTestIdx(failDomain, orderNumber); - } -}; - + + TSystemEvent DoneEvent; + yexception LastException; + volatile bool IsLastExceptionSet = false; + + + TTestEnvironment(const TTestArgs &args) + : DeadVDisksMask(args.StartBadDisks ? 0 : args.BadVDiskMask) + , FailedDiskCount(0) + , GroupType(args.ErasureSpecies) + , ProxyId(MakeBlobStorageProxyID(0)) + , ProxyTestId(MakeBlobStorageProxyID(1)) + { + for (int i = 0; i < 32; ++i) { + if (args.BadVDiskMask & (1 << i)) { + ++FailedDiskCount; + } + } + + ShouldBeUnwritable = FailedDiskCount > GroupType.Handoff(); + ShouldBeUndiscoverable = FailedDiskCount > GroupType.ParityParts(); + + VDiskCount = args.FailDomainCount * args.DrivesPerFailDomain; + if (VDiskCount == 0) { + VDiskCount = GroupType.BlobSubgroupSize(); + DrivesPerFailDomain = 1; + FailDomainCount = VDiskCount; + } else { + DrivesPerFailDomain = args.DrivesPerFailDomain; + FailDomainCount = args.FailDomainCount; + } + + VDisks.resize(VDiskCount); + PDisks.resize(VDiskCount); + VDiskIds.reserve(VDiskCount); + for (ui32 failDomainIdx = 0; failDomainIdx < FailDomainCount; ++failDomainIdx) { + for (ui32 driveIdx = 0; driveIdx < DrivesPerFailDomain; ++driveIdx) { + ui32 i = GetVDiskTestIdx(failDomainIdx, driveIdx); + VDisks[i] = MakeBlobStorageVDiskID(2, i + 1, 1); + PDisks[i] = MakeBlobStoragePDiskID(2, i + 1); + VDiskIds.emplace_back(0, 1, 0, failDomainIdx, driveIdx); + } + } + } + + ui64 GetVDiskTestIdx(ui64 failDomain, ui64 orderNumber) const { + return failDomain * DrivesPerFailDomain + orderNumber; + } + + ui64 GetVDiskTestIdx(const TVDiskID &vDiskId) const { + ui64 failDomain = vDiskId.FailDomain; + ui64 orderNumber = vDiskId.VDisk; + return GetVDiskTestIdx(failDomain, orderNumber); + } +}; + class TTestBlobStorageProxy : public TActor<TTestBlobStorageProxy> { protected: struct TResponseData { @@ -225,8 +225,8 @@ protected: , MessageStopProfilerResult , MessageVStatusResult , MessageVCompactResult - , MessageProxyQueueState - , MessageProxySessionsState + , MessageProxyQueueState + , MessageProxySessionsState }; EMessage Message; NKikimrProto::EReplyStatus Status; @@ -238,8 +238,8 @@ protected: bool LogoBlobsCompacted; TStorageStatusFlags StatusFlags; TActorId Sender; - TVDiskID VDiskId; - bool IsConnected; + TVDiskID VDiskId; + bool IsConnected; TIntrusivePtr<TGroupQueues> GroupQueues; TResponseData() @@ -273,84 +273,84 @@ protected: int InitStep; ui32 InitVDiskIdx; - ui32 ReadyQueueCount = 0; - ui32 QueueCount = 0; + ui32 ReadyQueueCount = 0; + ui32 QueueCount = 0; TIntrusivePtr<TGroupQueues> GroupQueues; - - const TIntrusivePtr<TTestEnvironment> Env; - const TIntrusivePtr<ITestParametrs> Parametrs; - + + const TIntrusivePtr<TTestEnvironment> Env; + const TIntrusivePtr<ITestParametrs> Parametrs; + virtual void TestFSM(const TActorContext &ctx) = 0; - void ActTestFSM(const TActorContext &ctx) { + void ActTestFSM(const TActorContext &ctx) { try { switch (InitStep) { case 0: - { - ctx.Send(Env->ProxyId, new TEvRequestProxySessionsState); - InitStep = 3; - break; - - case 3: - UNIT_ASSERT(LastResponse.Message == TResponseData::MessageProxySessionsState); + { + ctx.Send(Env->ProxyId, new TEvRequestProxySessionsState); + InitStep = 3; + break; + + case 3: + UNIT_ASSERT(LastResponse.Message == TResponseData::MessageProxySessionsState); GroupQueues = std::move(LastResponse.GroupQueues); UNIT_ASSERT(GroupQueues); auto &failDomains = GroupQueues->FailDomains; - for (ui64 failDomainIdx = 0; failDomainIdx < failDomains.size(); ++failDomainIdx) { - auto &failDomain = failDomains[failDomainIdx]; - for (ui64 vDiskIdx = 0; vDiskIdx < failDomain.VDisks.size(); ++vDiskIdx) { - ui64 vDiskTestIdx = Env->GetVDiskTestIdx(failDomainIdx, vDiskIdx); - if (Env->DeadVDisksMask & (1ull << vDiskTestIdx)) { - continue; - } - auto &queues = failDomain.VDisks[vDiskIdx].Queues; - ctx.Send(queues.PutTabletLog.ActorId, new TEvRequestProxyQueueState); - ctx.Send(queues.PutAsyncBlob.ActorId, new TEvRequestProxyQueueState); - ctx.Send(queues.PutUserData.ActorId, new TEvRequestProxyQueueState); - ctx.Send(queues.GetAsyncRead.ActorId, new TEvRequestProxyQueueState); - ctx.Send(queues.GetFastRead.ActorId, new TEvRequestProxyQueueState); - ctx.Send(queues.GetDiscover.ActorId, new TEvRequestProxyQueueState); - QueueCount += 6; - } - } - InitStep = 6; - break; - } - case 6: - UNIT_ASSERT(LastResponse.Message == TResponseData::MessageProxyQueueState); - if (LastResponse.IsConnected) { - ReadyQueueCount++; - } else { - ctx.Send(LastResponse.Sender, new TEvRequestProxyQueueState); - } - - if (ReadyQueueCount != QueueCount) { - break; - } - InitStep = 8; + for (ui64 failDomainIdx = 0; failDomainIdx < failDomains.size(); ++failDomainIdx) { + auto &failDomain = failDomains[failDomainIdx]; + for (ui64 vDiskIdx = 0; vDiskIdx < failDomain.VDisks.size(); ++vDiskIdx) { + ui64 vDiskTestIdx = Env->GetVDiskTestIdx(failDomainIdx, vDiskIdx); + if (Env->DeadVDisksMask & (1ull << vDiskTestIdx)) { + continue; + } + auto &queues = failDomain.VDisks[vDiskIdx].Queues; + ctx.Send(queues.PutTabletLog.ActorId, new TEvRequestProxyQueueState); + ctx.Send(queues.PutAsyncBlob.ActorId, new TEvRequestProxyQueueState); + ctx.Send(queues.PutUserData.ActorId, new TEvRequestProxyQueueState); + ctx.Send(queues.GetAsyncRead.ActorId, new TEvRequestProxyQueueState); + ctx.Send(queues.GetFastRead.ActorId, new TEvRequestProxyQueueState); + ctx.Send(queues.GetDiscover.ActorId, new TEvRequestProxyQueueState); + QueueCount += 6; + } + } + InitStep = 6; + break; + } + case 6: + UNIT_ASSERT(LastResponse.Message == TResponseData::MessageProxyQueueState); + if (LastResponse.IsConnected) { + ReadyQueueCount++; + } else { + ctx.Send(LastResponse.Sender, new TEvRequestProxyQueueState); + } + + if (ReadyQueueCount != QueueCount) { + break; + } + InitStep = 8; [[fallthrough]]; - - case 8: - while (Env->DeadVDisksMask & (1ull << InitVDiskIdx)) { + + case 8: + while (Env->DeadVDisksMask & (1ull << InitVDiskIdx)) { ++InitVDiskIdx; } [[fallthrough]]; - + case 10: { bool isSendNeeded = false; - if (InitStep == 8 || LastResponse.Status == NKikimrProto::NOTREADY || LastResponse.Status == NKikimrProto::ERROR) { - if (LastResponse.Status == NKikimrProto::NOTREADY || LastResponse.Status == NKikimrProto::ERROR) { + if (InitStep == 8 || LastResponse.Status == NKikimrProto::NOTREADY || LastResponse.Status == NKikimrProto::ERROR) { + if (LastResponse.Status == NKikimrProto::NOTREADY || LastResponse.Status == NKikimrProto::ERROR) { Sleep(TDuration::MilliSeconds(50)); } isSendNeeded = true; InitStep = 10; } else { ++InitVDiskIdx; - while (Env->DeadVDisksMask & (1ull << InitVDiskIdx)) { + while (Env->DeadVDisksMask & (1ull << InitVDiskIdx)) { ++InitVDiskIdx; } - if (InitVDiskIdx < Env->VDiskCount) { + if (InitVDiskIdx < Env->VDiskCount) { isSendNeeded = true; } else { InitStep = 20; @@ -359,7 +359,7 @@ protected: } if (isSendNeeded) { ui32 domains = BsInfo->GetTotalFailDomainsNum(); - ui32 drives = Env->VDiskCount / domains; + ui32 drives = Env->VDiskCount / domains; ui32 domainIdx = InitVDiskIdx / drives; ui32 driveIdx = InitVDiskIdx - domainIdx * drives; @@ -373,12 +373,12 @@ protected: {}, from, to); - + auto& msgId = *x->Record.MutableMsgQoS()->MutableMsgId(); msgId.SetMsgId(1); msgId.SetSequenceId(1); GroupQueues->Send(*this, BsInfo->GetTopology(), std::move(x), 0, NWilson::TTraceId(), false); - break; + break; } [[fallthrough]]; } @@ -389,11 +389,11 @@ protected: LastResponse.Clear(); } catch (yexception ex) { - Env->IsLastExceptionSet = true; - Env->LastException = ex; - Env->DoneEvent.Signal(); + Env->IsLastExceptionSet = true; + Env->LastException = ex; + Env->DoneEvent.Signal(); } - VERBOSE_COUT("InitStep# " << InitStep); + VERBOSE_COUT("InitStep# " << InitStep); } void HandleStartProfilerResult(TEvProfiler::TEvStartResult::TPtr &ev, const TActorContext &ctx) { @@ -597,26 +597,26 @@ protected: Y_UNUSED(ev); } - void HandleProxyQueueState(TEvProxyQueueState::TPtr &ev, const TActorContext &ctx) { - VERBOSE_COUT("HandleProxyQueueState"); - LastResponse.Message = TResponseData::MessageProxyQueueState; - LastResponse.VDiskId = ev->Get()->VDiskId; - LastResponse.IsConnected = ev->Get()->IsConnected; - LastResponse.Sender = ev->Sender; - ActTestFSM(ctx); - } - - void HandleProxySessionsState(TEvProxySessionsState::TPtr &ev, const TActorContext &ctx) { - VERBOSE_COUT("HandleProxySessionsState"); - LastResponse.Message = TResponseData::MessageProxySessionsState; + void HandleProxyQueueState(TEvProxyQueueState::TPtr &ev, const TActorContext &ctx) { + VERBOSE_COUT("HandleProxyQueueState"); + LastResponse.Message = TResponseData::MessageProxyQueueState; + LastResponse.VDiskId = ev->Get()->VDiskId; + LastResponse.IsConnected = ev->Get()->IsConnected; + LastResponse.Sender = ev->Sender; + ActTestFSM(ctx); + } + + void HandleProxySessionsState(TEvProxySessionsState::TPtr &ev, const TActorContext &ctx) { + VERBOSE_COUT("HandleProxySessionsState"); + LastResponse.Message = TResponseData::MessageProxySessionsState; LastResponse.GroupQueues = std::move(ev->Get()->GroupQueues); - LastResponse.Sender = ev->Sender; - ActTestFSM(ctx); - } - + LastResponse.Sender = ev->Sender; + ActTestFSM(ctx); + } + public: TTestBlobStorageProxy(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) : TActor(&TThis::StateRegister) , Proxy(proxy) , ErasureSpecies(bsInfo->Type.GetErasure()) @@ -624,8 +624,8 @@ public: , TestStep(0) , InitStep(0) , InitVDiskIdx(0) - , Env(env) - , Parametrs(parametrs) + , Env(env) + , Parametrs(parametrs) {} STFUNC(StateRegister) { @@ -645,8 +645,8 @@ public: HFunc(TEvBlobStorage::TEvBlockResult, HandleBlockResult); HFunc(TEvProfiler::TEvStartResult, HandleStartProfilerResult); HFunc(TEvProfiler::TEvStopResult, HandleStopProfilerResult); - HFunc(TEvProxyQueueState, HandleProxyQueueState); - HFunc(TEvProxySessionsState, HandleProxySessionsState); + HFunc(TEvProxyQueueState, HandleProxyQueueState); + HFunc(TEvProxySessionsState, HandleProxySessionsState); } } }; @@ -666,7 +666,7 @@ class TTestBlobStorageProxyBlockSet : public TTestBlobStorageProxy { TEST_RESPONSE(MessageBlockResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -676,8 +676,8 @@ class TTestBlobStorageProxyBlockSet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyBlockSet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -717,7 +717,7 @@ class TTestBlobStorageProxyBlockCheck : public TTestBlobStorageProxy { TEST_RESPONSE(MessagePutResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -727,179 +727,179 @@ class TTestBlobStorageProxyBlockCheck : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyBlockCheck(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; -char DefaultCharGenerator(int i) { - return i % 256; -} - -template <const TVector<size_t>& Order> -class TTestIterativePut: public TTestBlobStorageProxy { - void TestFSM(const TActorContext& ctx) { - VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - - if (!TestStep) { - UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, - "Unexpected " << (int)LastResponse.Message); - for (auto idx : Order) { - char ch = DefaultCharGenerator(idx); - TLogoBlobID logoblobid(1, idx, 0, 0, 1, 0); - ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, TString(ch), TInstant::Max())); - } - } else if (TestStep < (int)Order.size()) { - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - } else { - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - } - TestStep++; - } - -public: +char DefaultCharGenerator(int i) { + return i % 256; +} + +template <const TVector<size_t>& Order> +class TTestIterativePut: public TTestBlobStorageProxy { + void TestFSM(const TActorContext& ctx) { + VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); + + if (!TestStep) { + UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, + "Unexpected " << (int)LastResponse.Message); + for (auto idx : Order) { + char ch = DefaultCharGenerator(idx); + TLogoBlobID logoblobid(1, idx, 0, 0, 1, 0); + ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, TString(ch), TInstant::Max())); + } + } else if (TestStep < (int)Order.size()) { + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + } else { + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + } + TestStep++; + } + +public: TTestIterativePut(const TActorId& proxy, const TIntrusivePtr<TBlobStorageGroupInfo>& bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) - { - } -}; - -template <ui64 PutInFlight, ui64 MaxSendPut> -class TTestInFlightPuts: public TTestBlobStorageProxy { - TString Data; - - void TestFSM(const TActorContext& ctx) { - VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - ui64 testStep = TestStep; - if (!testStep) { - UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, - "Unexpected " << (int)LastResponse.Message); - for (ui64 idx = 0; idx < PutInFlight && idx < MaxSendPut; ++idx) { - TLogoBlobID logoblobid(1, idx, 0, 0, Data.size(), 0); - ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, Data, TInstant::Max())); - } - } else if (testStep <= MaxSendPut - PutInFlight) { - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - ui64 idx = testStep + PutInFlight - 1; - TLogoBlobID logoblobid(1, idx, 0, 0, Data.size(), 0); - ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, Data, TInstant::Max())); - } else if (testStep < MaxSendPut) { - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - } else { - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - } - TestStep++; - } - -public: + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + { + } +}; + +template <ui64 PutInFlight, ui64 MaxSendPut> +class TTestInFlightPuts: public TTestBlobStorageProxy { + TString Data; + + void TestFSM(const TActorContext& ctx) { + VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); + ui64 testStep = TestStep; + if (!testStep) { + UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, + "Unexpected " << (int)LastResponse.Message); + for (ui64 idx = 0; idx < PutInFlight && idx < MaxSendPut; ++idx) { + TLogoBlobID logoblobid(1, idx, 0, 0, Data.size(), 0); + ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, Data, TInstant::Max())); + } + } else if (testStep <= MaxSendPut - PutInFlight) { + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + ui64 idx = testStep + PutInFlight - 1; + TLogoBlobID logoblobid(1, idx, 0, 0, Data.size(), 0); + ctx.Send(Proxy, new TEvBlobStorage::TEvPut(logoblobid, Data, TInstant::Max())); + } else if (testStep < MaxSendPut) { + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + } else { + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + } + TestStep++; + } + +public: TTestInFlightPuts(const TActorId& proxy, const TIntrusivePtr<TBlobStorageGroupInfo>& bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) - { - Data.reserve(4096); - for (ui64 i = 0; i < 4096; ++i) { - Data.push_back('a'); - } - } -}; - -template <const TVector<size_t>& Order> -class TTestOrderGet: public TTestBlobStorageProxy { - size_t GetIdx(int i) { - return Order ? Order[i] : i; - } - - void InitExpectedValues() { - ExpectedValues.resize(Order.size()); - for (size_t idx = 0; idx != Order.size(); idx++) { - ExpectedValues[idx] = TString(DefaultCharGenerator(GetIdx(idx))); - } - } - - void TestFSM(const TActorContext& ctx) { - VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - - if (!TestStep) { - UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, - "Unexpected " << (int)LastResponse.Message); - - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> q(new TEvBlobStorage::TEvGet::TQuery[Order.size()]); - for (size_t idx = 0; idx < Order.size(); ++idx) { - q[idx].Set(TLogoBlobID(1, GetIdx(idx), 0, 0, 1, 0)); - } - - ctx.Send(Proxy, new TEvBlobStorage::TEvGet(q, Order.size(), TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::FastRead)); - } else { - InitExpectedValues(); - TEST_RESPONSE_FULLCHECK(MessageGetResult, OK, Order.size(), ExpectedValues); - - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - } - TestStep++; - } - -public: + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + { + Data.reserve(4096); + for (ui64 i = 0; i < 4096; ++i) { + Data.push_back('a'); + } + } +}; + +template <const TVector<size_t>& Order> +class TTestOrderGet: public TTestBlobStorageProxy { + size_t GetIdx(int i) { + return Order ? Order[i] : i; + } + + void InitExpectedValues() { + ExpectedValues.resize(Order.size()); + for (size_t idx = 0; idx != Order.size(); idx++) { + ExpectedValues[idx] = TString(DefaultCharGenerator(GetIdx(idx))); + } + } + + void TestFSM(const TActorContext& ctx) { + VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); + + if (!TestStep) { + UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, + "Unexpected " << (int)LastResponse.Message); + + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> q(new TEvBlobStorage::TEvGet::TQuery[Order.size()]); + for (size_t idx = 0; idx < Order.size(); ++idx) { + q[idx].Set(TLogoBlobID(1, GetIdx(idx), 0, 0, 1, 0)); + } + + ctx.Send(Proxy, new TEvBlobStorage::TEvGet(q, Order.size(), TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead)); + } else { + InitExpectedValues(); + TEST_RESPONSE_FULLCHECK(MessageGetResult, OK, Order.size(), ExpectedValues); + + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + } + TestStep++; + } + +public: TTestOrderGet(const TActorId& proxy, const TIntrusivePtr<TBlobStorageGroupInfo>& bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) - { - } - -private: - TVector<TString> ExpectedValues; -}; - -template <int StartIdx, int RangeLength> -class TTestRangeGet: public TTestBlobStorageProxy { - void InitExpectedValues() { - ExpectedValues.resize(RangeSize); - for (int i = 0; i != RangeSize; i++) { - ExpectedValues[i] = TString(DefaultCharGenerator(StartIdx + i * DIdx)); - } - } - - void TestFSM(const TActorContext& ctx) { - VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - - if (!TestStep) { - UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, - "Unexpected " << (int)LastResponse.Message); - - TLogoBlobID from(1, StartIdx, 0, 0, 1, 0); - TLogoBlobID to(1, StartIdx + RangeLength, 0, 0, 1, 0); - ctx.Send(Proxy, new TEvBlobStorage::TEvRange(MakeTabletID(0, 0, 1), from, to, true, TInstant::Max())); - } else { - InitExpectedValues(); - TEST_RESPONSE_FULLCHECK(MessageRangeResult, OK, (size_t)RangeSize, ExpectedValues); - - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - } - TestStep++; - } - -public: + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + { + } + +private: + TVector<TString> ExpectedValues; +}; + +template <int StartIdx, int RangeLength> +class TTestRangeGet: public TTestBlobStorageProxy { + void InitExpectedValues() { + ExpectedValues.resize(RangeSize); + for (int i = 0; i != RangeSize; i++) { + ExpectedValues[i] = TString(DefaultCharGenerator(StartIdx + i * DIdx)); + } + } + + void TestFSM(const TActorContext& ctx) { + VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); + + if (!TestStep) { + UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, + "Unexpected " << (int)LastResponse.Message); + + TLogoBlobID from(1, StartIdx, 0, 0, 1, 0); + TLogoBlobID to(1, StartIdx + RangeLength, 0, 0, 1, 0); + ctx.Send(Proxy, new TEvBlobStorage::TEvRange(MakeTabletID(0, 0, 1), from, to, true, TInstant::Max())); + } else { + InitExpectedValues(); + TEST_RESPONSE_FULLCHECK(MessageRangeResult, OK, (size_t)RangeSize, ExpectedValues); + + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + } + TestStep++; + } + +public: TTestRangeGet(const TActorId& proxy, const TIntrusivePtr<TBlobStorageGroupInfo>& bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) - { - } - -private: - const int RangeSize = std::abs(RangeLength); - const int DIdx = RangeLength > 0 ? 1 : -1; - TVector<TString> ExpectedValues; -}; - - + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + { + } + +private: + const int RangeSize = std::abs(RangeLength); + const int DIdx = RangeLength > 0 ? 1 : -1; + TVector<TString> ExpectedValues; +}; + + class TTestBlobStorageProxyBlock : public TTestBlobStorageProxy { void TestFSM(const TActorContext &ctx) { VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); @@ -989,7 +989,7 @@ class TTestBlobStorageProxyBlock : public TTestBlobStorageProxy { case 90: TEST_RESPONSE(MessagePutResult, OK, 0, ""); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -999,8 +999,8 @@ class TTestBlobStorageProxyBlock : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyBlock(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1027,7 +1027,7 @@ class TTestBlobStorageProxyPut : public TTestBlobStorageProxy { TEST_RESPONSE(MessagePutResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1037,8 +1037,8 @@ class TTestBlobStorageProxyPut : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPut(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1065,7 +1065,7 @@ class TTestBlobStorageProxyPutInvalidSize : public TTestBlobStorageProxy { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1075,8 +1075,8 @@ class TTestBlobStorageProxyPutInvalidSize : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPutInvalidSize(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1100,7 +1100,7 @@ class TTestBlobStorageProxyPutFail : public TTestBlobStorageProxy { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1110,8 +1110,8 @@ class TTestBlobStorageProxyPutFail : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPutFail(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1137,7 +1137,7 @@ class TTestBlobStorageProxyGet : public TTestBlobStorageProxy { TEST_RESPONSE(MessageGetResult, OK, 1, testData2); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1147,8 +1147,8 @@ class TTestBlobStorageProxyGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1172,7 +1172,7 @@ class TTestBlobStorageProxyGetFail : public TTestBlobStorageProxy { TEST_RESPONSE(MessageGetResult, OK, 1, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1182,8 +1182,8 @@ class TTestBlobStorageProxyGetFail : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGetFail(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1207,7 +1207,7 @@ class TTestBlobStorageProxyGetTimeout : public TTestBlobStorageProxy { TEST_RESPONSE(MessageGetResult, ERROR, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1217,8 +1217,8 @@ class TTestBlobStorageProxyGetTimeout : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGetTimeout(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1246,7 +1246,7 @@ class TTestBlobStorageProxyPutGetMany: public TTestBlobStorageProxy { } else if (TestStep == int(count) + 1) { TEST_RESPONSE_3(MessageGetResult, OK, count); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); ++TestStep; return; } else { @@ -1272,8 +1272,8 @@ class TTestBlobStorageProxyPutGetMany: public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPutGetMany(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1344,7 +1344,7 @@ class TTestBlobStorageProxyPutGetStatus: public TTestBlobStorageProxy { UNIT_ASSERT(LastResponse.StatusFlags.Raw & ui32(NKikimrBlobStorage::StatusIsValid)); } VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); ++TestStep; return; } else { @@ -1365,8 +1365,8 @@ class TTestBlobStorageProxyPutGetStatus: public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPutGetStatus(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1397,7 +1397,7 @@ class TTestBlobStorageProxyVPutVGet : public TTestBlobStorageProxy { TAutoPtr<TEvBlobStorage::TEvVPut> vPut( new TEvBlobStorage::TEvVPut(logoblobid, partSet.Parts[0].OwnedString, vDiskId, false, nullptr, TInstant::Max(), NKikimrBlobStorage::AsyncBlob)); - ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); break; } case 10: @@ -1423,7 +1423,7 @@ class TTestBlobStorageProxyVPutVGet : public TTestBlobStorageProxy { TEST_RESPONSE(MessageVGetResult, OK, 1, partSet.Parts[0].OwnedString); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -1434,8 +1434,8 @@ class TTestBlobStorageProxyVPutVGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVPutVGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1490,7 +1490,7 @@ class TTestBlobStorageProxyVPutVGetLimit : public TTestBlobStorageProxy { auto& msgId = *vPut->Record.MutableMsgQoS()->MutableMsgId(); msgId.SetMsgId(i); msgId.SetSequenceId(1); - ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); } break; } @@ -1520,7 +1520,7 @@ class TTestBlobStorageProxyVPutVGetLimit : public TTestBlobStorageProxy { { TEST_RESPONSE_3(MessageVGetResult, OK, 5); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -1531,102 +1531,102 @@ class TTestBlobStorageProxyVPutVGetLimit : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVPutVGetLimit(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) , Iteration(0) {} }; -enum EVDiskIdxMode { - VDIM_TestIdx, - VDIM_IdxInSubgroup -}; - +enum EVDiskIdxMode { + VDIM_TestIdx, + VDIM_IdxInSubgroup +}; + class TTestBlobStorageProxyVPut : public TTestBlobStorageProxy { -public: - struct TParametrs : public ITestParametrs { - using TPtr = TIntrusivePtr<TParametrs>; - - TString TestData; - ui64 PartId; - EVDiskIdxMode Mode = VDIM_IdxInSubgroup; - ui64 VDiskIdx = Max<ui64>(); // default value for VDIM_IdxInSubgroup - - TParametrs(TString testData = "", ui64 partId = 0, EVDiskIdxMode mode = VDIM_IdxInSubgroup, - ui64 vDiskIdx = Max<ui64>()) - : TestData(testData) - , PartId(partId) - , Mode(mode) - , VDiskIdx(vDiskIdx) - {} - - virtual ~TParametrs() - {} - }; - -private: - TParametrs& GetParametrs() { - TParametrs *param = dynamic_cast<TParametrs*>(Parametrs.Get()); - UNIT_ASSERT(param); - return *param; - } - +public: + struct TParametrs : public ITestParametrs { + using TPtr = TIntrusivePtr<TParametrs>; + + TString TestData; + ui64 PartId; + EVDiskIdxMode Mode = VDIM_IdxInSubgroup; + ui64 VDiskIdx = Max<ui64>(); // default value for VDIM_IdxInSubgroup + + TParametrs(TString testData = "", ui64 partId = 0, EVDiskIdxMode mode = VDIM_IdxInSubgroup, + ui64 vDiskIdx = Max<ui64>()) + : TestData(testData) + , PartId(partId) + , Mode(mode) + , VDiskIdx(vDiskIdx) + {} + + virtual ~TParametrs() + {} + }; + +private: + TParametrs& GetParametrs() { + TParametrs *param = dynamic_cast<TParametrs*>(Parametrs.Get()); + UNIT_ASSERT(param); + return *param; + } + void TestFSM(const TActorContext &ctx) { - TParametrs ¶metrs = GetParametrs(); - UNIT_ASSERT(parametrs.PartId <= BsInfo->Type.TotalPartCount()); - UNIT_ASSERT(parametrs.PartId > 0); - TLogoBlobID blobId(1, 0, 0, 0, parametrs.TestData.size(), 0); - TLogoBlobID logoblobid(1, 0, 0, 0, parametrs.TestData.size(), 0, parametrs.PartId); - UNIT_ASSERT(parametrs.VDiskIdx < BsInfo->Type.BlobSubgroupSize() || parametrs.VDiskIdx == Max<ui64>()); - TMaybe<TVDiskID> vDiskId; - ui64 realVDiskIdx = parametrs.VDiskIdx; - - switch (parametrs.Mode) { - case VDIM_TestIdx: - UNIT_ASSERT(parametrs.VDiskIdx != Max<ui64>()); - vDiskId = Env->VDiskIds[parametrs.VDiskIdx]; - break; - case VDIM_IdxInSubgroup: - ui64 idxInSubgroup = (parametrs.VDiskIdx == Max<ui64>()) ? (parametrs.PartId - 1) : parametrs.VDiskIdx; - vDiskId = BsInfo->GetVDiskInSubgroup(idxInSubgroup, blobId.Hash()); - realVDiskIdx = Env->GetVDiskTestIdx(*vDiskId); - if (*vDiskId != Env->VDiskIds[realVDiskIdx]) { - TStringBuilder str; - str << "partId# " << parametrs.PartId; - str << " idxInSubgroup# " << idxInSubgroup; - str << " vDiskId# " << vDiskId->ToString(); - str << " realVDiskIdx# " << realVDiskIdx; - str << " vDisks# ["; - for (ui64 idx = 0; idx < Env->VDiskIds.size(); ++idx) { - str << (idx ? " " : "") << Env->VDiskIds[idx].ToString(); - } - str << "]"; - UNIT_ASSERT_C(*vDiskId == Env->VDiskIds[realVDiskIdx], str); - } - break; - } - UNIT_ASSERT(vDiskId); + TParametrs ¶metrs = GetParametrs(); + UNIT_ASSERT(parametrs.PartId <= BsInfo->Type.TotalPartCount()); + UNIT_ASSERT(parametrs.PartId > 0); + TLogoBlobID blobId(1, 0, 0, 0, parametrs.TestData.size(), 0); + TLogoBlobID logoblobid(1, 0, 0, 0, parametrs.TestData.size(), 0, parametrs.PartId); + UNIT_ASSERT(parametrs.VDiskIdx < BsInfo->Type.BlobSubgroupSize() || parametrs.VDiskIdx == Max<ui64>()); + TMaybe<TVDiskID> vDiskId; + ui64 realVDiskIdx = parametrs.VDiskIdx; + + switch (parametrs.Mode) { + case VDIM_TestIdx: + UNIT_ASSERT(parametrs.VDiskIdx != Max<ui64>()); + vDiskId = Env->VDiskIds[parametrs.VDiskIdx]; + break; + case VDIM_IdxInSubgroup: + ui64 idxInSubgroup = (parametrs.VDiskIdx == Max<ui64>()) ? (parametrs.PartId - 1) : parametrs.VDiskIdx; + vDiskId = BsInfo->GetVDiskInSubgroup(idxInSubgroup, blobId.Hash()); + realVDiskIdx = Env->GetVDiskTestIdx(*vDiskId); + if (*vDiskId != Env->VDiskIds[realVDiskIdx]) { + TStringBuilder str; + str << "partId# " << parametrs.PartId; + str << " idxInSubgroup# " << idxInSubgroup; + str << " vDiskId# " << vDiskId->ToString(); + str << " realVDiskIdx# " << realVDiskIdx; + str << " vDisks# ["; + for (ui64 idx = 0; idx < Env->VDiskIds.size(); ++idx) { + str << (idx ? " " : "") << Env->VDiskIds[idx].ToString(); + } + str << "]"; + UNIT_ASSERT_C(*vDiskId == Env->VDiskIds[realVDiskIdx], str); + } + break; + } + UNIT_ASSERT(vDiskId); VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - TString encryptedTestData; - encryptedTestData.resize(parametrs.TestData.size()); - Encrypt(encryptedTestData.Detach(), parametrs.TestData.data(), 0, parametrs.TestData.size(), blobId, *BsInfo); - + TString encryptedTestData; + encryptedTestData.resize(parametrs.TestData.size()); + Encrypt(encryptedTestData.Detach(), parametrs.TestData.data(), 0, parametrs.TestData.size(), blobId, *BsInfo); + switch (TestStep) { case 0: { UNIT_ASSERT_EQUAL_C(LastResponse.Message, TResponseData::MessageNone, "Unexpected " << (int)LastResponse.Message); - VERBOSE_COUT(" Sending TEvVPut to " << vDiskId->ToString() << " blob " << logoblobid.ToString()); + VERBOSE_COUT(" Sending TEvVPut to " << vDiskId->ToString() << " blob " << logoblobid.ToString()); NKikimr::TDataPartSet partSet; - BsInfo->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedTestData, partSet); + BsInfo->Type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedTestData, partSet); TAutoPtr<TEvBlobStorage::TEvVPut> vPut( - new TEvBlobStorage::TEvVPut(logoblobid, partSet.Parts[parametrs.PartId-1].OwnedString, *vDiskId, - false, nullptr, TInstant::Max(), NKikimrBlobStorage::AsyncBlob)); - ctx.Send(Env->VDisks[realVDiskIdx], vPut.Release()); + new TEvBlobStorage::TEvVPut(logoblobid, partSet.Parts[parametrs.PartId-1].OwnedString, *vDiskId, + false, nullptr, TInstant::Max(), NKikimrBlobStorage::AsyncBlob)); + ctx.Send(Env->VDisks[realVDiskIdx], vPut.Release()); break; } case 10: @@ -1634,7 +1634,7 @@ private: TEST_RESPONSE(MessageVPutResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -1645,12 +1645,12 @@ private: } public: TTestBlobStorageProxyVPut(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; -template <int vDiskIdx, int partId, bool &isNoData> +template <int vDiskIdx, int partId, bool &isNoData> class TTestBlobStorageProxyVGet : public TTestBlobStorageProxy { void TestFSM(const TActorContext &ctx) { VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); @@ -1699,13 +1699,13 @@ class TTestBlobStorageProxyVGet : public TTestBlobStorageProxy { {id}); ctx.Send(Env->VDisks[vDiskIdx], x.release()); break; - } else if (LastResponse.ItemStatus[0] == NKikimrProto::NODATA) { - isNoData = true; - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - break; - } else if (LastResponse.ItemStatus[0] != NKikimrProto::OK) { - UNIT_ASSERT(false); + } else if (LastResponse.ItemStatus[0] == NKikimrProto::NODATA) { + isNoData = true; + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + break; + } else if (LastResponse.ItemStatus[0] != NKikimrProto::OK) { + UNIT_ASSERT(false); } } } @@ -1715,9 +1715,9 @@ class TTestBlobStorageProxyVGet : public TTestBlobStorageProxy { type.SplitData((TErasureType::ECrcMode)blobId.CrcMode(), encryptedTestData2, partSet); TEST_RESPONSE(MessageVGetResult, OK, 1, partSet.Parts[0].OwnedString); - isNoData = false; + isNoData = false; VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -1728,8 +1728,8 @@ class TTestBlobStorageProxyVGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1785,7 +1785,7 @@ class TTestBlobStorageProxyVGetFail : public TTestBlobStorageProxy { ctx.Send(Env->VDisks[vDiskIdx], x.release()); } else { VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); } } break; @@ -1804,7 +1804,7 @@ class TTestBlobStorageProxyVGetFail : public TTestBlobStorageProxy { } VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); } break; default: @@ -1815,8 +1815,8 @@ class TTestBlobStorageProxyVGetFail : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVGetFail(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1840,7 +1840,7 @@ class TTestBlobStorageProxyVBlockVPutVGet : public TTestBlobStorageProxy { msgId.SetMsgId(0); msgId.SetSequenceId(1); - ctx.Send(Env->VDisks[0], x.Release()); + ctx.Send(Env->VDisks[0], x.Release()); break; } case 10: @@ -1860,7 +1860,7 @@ class TTestBlobStorageProxyVBlockVPutVGet : public TTestBlobStorageProxy { msgId.SetMsgId(0); msgId.SetSequenceId(1); - ctx.Send(Env->VDisks[0], x.Release()); + ctx.Send(Env->VDisks[0], x.Release()); break; } case 20: @@ -1888,7 +1888,7 @@ class TTestBlobStorageProxyVBlockVPutVGet : public TTestBlobStorageProxy { { TEST_RESPONSE(MessageVGetResult, OK, 1, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -1899,8 +1899,8 @@ class TTestBlobStorageProxyVBlockVPutVGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVBlockVPutVGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1928,7 +1928,7 @@ class TTestBlobStorageProxyGarbageMark : public TTestBlobStorageProxy { TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1938,8 +1938,8 @@ class TTestBlobStorageProxyGarbageMark : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGarbageMark(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1965,7 +1965,7 @@ class TTestBlobStorageProxyGarbageUnmark : public TTestBlobStorageProxy { TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -1975,8 +1975,8 @@ class TTestBlobStorageProxyGarbageUnmark : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGarbageUnmark(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -1998,7 +1998,7 @@ class TTestBlobStorageProxyGarbageCollect : public TTestBlobStorageProxy { TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2008,8 +2008,8 @@ class TTestBlobStorageProxyGarbageCollect : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGarbageCollect(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2045,7 +2045,7 @@ class TTestBlobStorageProxyGarbageCollectHuge : public TTestBlobStorageProxy { TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2055,8 +2055,8 @@ class TTestBlobStorageProxyGarbageCollectHuge : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyGarbageCollectHuge(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2088,7 +2088,7 @@ class TTestBlobStorageProxyEmptyGet : public TTestBlobStorageProxy { TEST_RESPONSE(MessagePutResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2098,8 +2098,8 @@ class TTestBlobStorageProxyEmptyGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyEmptyGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2220,7 +2220,7 @@ class TTestBlobStorageProxyGarbageCollectComplex : public TTestBlobStorageProxy TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -2231,9 +2231,9 @@ class TTestBlobStorageProxyGarbageCollectComplex : public TTestBlobStorageProxy } public: TTestBlobStorageProxyGarbageCollectComplex(const TActorId &proxy, - const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2284,7 +2284,7 @@ class TTestBlobStorageProxyGarbageCollectAfterLargeData : public TTestBlobStorag TEST_RESPONSE(MessageCollectGarbageResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -2297,9 +2297,9 @@ class TTestBlobStorageProxyGarbageCollectAfterLargeData : public TTestBlobStorag } public: TTestBlobStorageProxyGarbageCollectAfterLargeData(const TActorId &proxy, - const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2349,7 +2349,7 @@ class TTestBlobStorageProxySimpleDiscover : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, OK, 1, testData2); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2359,8 +2359,8 @@ class TTestBlobStorageProxySimpleDiscover : public TTestBlobStorageProxy { } public: TTestBlobStorageProxySimpleDiscover(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2405,7 +2405,7 @@ class TTestBlobStorageProxyDiscover : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, OK, 1, testData2); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2415,8 +2415,8 @@ class TTestBlobStorageProxyDiscover : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyDiscover(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2441,7 +2441,7 @@ class TTestBlobStorageProxyDiscoverFail : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2451,8 +2451,8 @@ class TTestBlobStorageProxyDiscoverFail : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyDiscoverFail(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2477,7 +2477,7 @@ class TTestBlobStorageProxyDiscoverEmpty : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, NODATA, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2487,8 +2487,8 @@ class TTestBlobStorageProxyDiscoverEmpty : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyDiscoverEmpty(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2513,7 +2513,7 @@ class TTestBlobStorageProxyDiscoverTimeout : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, TIMEOUT, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2523,8 +2523,8 @@ class TTestBlobStorageProxyDiscoverTimeout : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyDiscoverTimeout(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2605,7 +2605,7 @@ class TTestBlobStorageProxyLongTailDiscoverPut : public TTestBlobStorageProxy { partIdx = 0; } - ui32 vDiskIdx = vDiskId.FailDomain * BsInfo->GetNumVDisksPerFailDomain() + vDiskId.VDisk; + ui32 vDiskIdx = vDiskId.FailDomain * BsInfo->GetNumVDisksPerFailDomain() + vDiskId.VDisk; TLogoBlobID from(1, 0, 2 + Iteration/*step*/, 0, testData3.size(), cookie, partIdx + 1); VERBOSE_COUT(" Sending TEvVPut partId# " << (partIdx + 1) << " cookie# " << cookie); @@ -2617,7 +2617,7 @@ class TTestBlobStorageProxyLongTailDiscoverPut : public TTestBlobStorageProxy { msgId.SetMsgId(MsgIdx); msgId.SetSequenceId(9990); ++MsgIdx; - ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); ++Iteration; if (Iteration < 100) { return; @@ -2655,8 +2655,8 @@ class TTestBlobStorageProxyLongTailDiscoverPut : public TTestBlobStorageProxy { msgId.SetMsgId(MsgIdx); msgId.SetSequenceId(9990); ++MsgIdx; - ui32 vDiskIdx = vDiskId.FailDomain * BsInfo->GetNumVDisksPerFailDomain() + vDiskId.VDisk; - ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); + ui32 vDiskIdx = vDiskId.FailDomain * BsInfo->GetNumVDisksPerFailDomain() + vDiskId.VDisk; + ctx.Send(Env->VDisks[vDiskIdx], vPut.Release()); ++Iteration; if (Iteration < 200) { return; @@ -2666,7 +2666,7 @@ class TTestBlobStorageProxyLongTailDiscoverPut : public TTestBlobStorageProxy { case 60: TEST_RESPONSE(MessageVPutResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2676,8 +2676,8 @@ class TTestBlobStorageProxyLongTailDiscoverPut : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyLongTailDiscoverPut(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) , MsgIdx(0) , Iteration(0) {} @@ -2711,7 +2711,7 @@ class TTestBlobStorageProxyLongTailDiscover : public TTestBlobStorageProxy { TEST_RESPONSE(MessageDiscoverResult, OK, 1, testData2); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2721,8 +2721,8 @@ class TTestBlobStorageProxyLongTailDiscover : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyLongTailDiscover(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2765,7 +2765,7 @@ class TTestBlobStorageProxyPartialGet : public TTestBlobStorageProxy { TEST_RESPONSE(MessageGetResult, OK, 1, TString(testData, 7, testData.size() - 7)); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2775,8 +2775,8 @@ class TTestBlobStorageProxyPartialGet : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyPartialGet(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2799,7 +2799,7 @@ class TTestEmptyRange : public TTestBlobStorageProxy { case 10: TEST_RESPONSE(MessageRangeResult, OK, 0, ""); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -2809,8 +2809,8 @@ class TTestEmptyRange : public TTestBlobStorageProxy { } public: TTestEmptyRange(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -2852,7 +2852,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 10: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageGetResult, OK, 1, ""); } // Put. @@ -2863,7 +2863,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 20: { - if (Env->ShouldBeUnwritable) { + if (Env->ShouldBeUnwritable) { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); } else { TEST_RESPONSE(MessagePutResult, OK, 0, ""); @@ -2883,11 +2883,11 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { case 60: { if (isFirstFallthrough) { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageGetResult, OK, 1, testData); } } else { - if (Env->ShouldBeUnwritable) { + if (Env->ShouldBeUnwritable) { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); } else { TEST_RESPONSE(MessagePutResult, OK, 0, ""); @@ -2909,13 +2909,13 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { { ui32 iteration = (TestStep - 70)/10; if (isFirstFallthrough) { - if (Env->ShouldBeUnwritable) { + if (Env->ShouldBeUnwritable) { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); } else { TEST_RESPONSE(MessagePutResult, OK, 0, ""); } } else { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageGetResult, OK, 1, testData); } } @@ -2928,7 +2928,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 110: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageGetResult, OK, 1, testData); } // Get range. @@ -2940,7 +2940,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 120: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageRangeResult, OK, 1, testData); } // Get range. @@ -2952,7 +2952,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 130: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageRangeResult, OK, 4, testData); for (int i = 0; i < 4; ++i) { if (LastResponse.Data[i] != testData) { @@ -2971,7 +2971,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 140: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageRangeResult, OK, 4, testData); for (int i = 0; i < 4; ++i) { if (LastResponse.Data[i] != testData) { @@ -2990,7 +2990,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 150: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageRangeResult, OK, 0, ""); } // Get range. @@ -3009,7 +3009,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { { ui32 iteration = (TestStep - 160)/10; if (isFirstFallthrough) { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageRangeResult, OK, 1, testData); } } else { @@ -3018,7 +3018,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } // Get 1 part ui32 domains = BsInfo->GetTotalFailDomainsNum(); - ui32 drives = Env->VDiskCount / domains; + ui32 drives = Env->VDiskCount / domains; ui32 domainIdx = iteration / drives; ui32 driveIdx = iteration - domainIdx * drives; @@ -3049,7 +3049,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } case 210: { - if (!Env->ShouldBeUnwritable) { + if (!Env->ShouldBeUnwritable) { TEST_RESPONSE(MessageGetResult, OK, 1, TString(testData, 7, 11)); } // Put. @@ -3059,7 +3059,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { break; } case 220: - if (Env->ShouldBeUnwritable) { + if (Env->ShouldBeUnwritable) { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); } else { TEST_RESPONSE(MessagePutResult, OK, 0, ""); @@ -3068,7 +3068,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { ctx.Send(Proxy, new TEvBlobStorage::TEvDiscover(1, 0, true, false, TInstant::Max(), 0)); break; case 230: - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, nullptr); } else { TEST_RESPONSE(MessageDiscoverResult, OK, 1, testData2); @@ -3077,7 +3077,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { ctx.Send(Proxy, new TEvBlobStorage::TEvDiscover(1, 0, false, false, TInstant::Max(), 0)); break; case 240: - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, 0); } else { TEST_RESPONSE(MessageDiscoverResult, OK, 1, ""); @@ -3086,7 +3086,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { ctx.Send(Proxy, new TEvBlobStorage::TEvDiscover(2/*tabletId*/, 0, true, false, TInstant::Max(), 0)); break; case 250: - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, 0); } else { TEST_RESPONSE(MessageDiscoverResult, NODATA, 0, ""); @@ -3095,7 +3095,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { ctx.Send(Proxy, new TEvBlobStorage::TEvDiscover(1, 0, true, true, TInstant::Max(), 0)); break; case 260: - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, 0); } else { TEST_RESPONSE(MessageDiscoverResult, OK, 1, testData2); @@ -3104,7 +3104,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { ctx.Send(Proxy, new TEvBlobStorage::TEvDiscover(1, 0, false, true, TInstant::Max(), 0)); break; case 270: - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, 0); } else { TEST_RESPONSE(MessageDiscoverResult, OK, 1, ""); @@ -3114,7 +3114,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { break; case 280: { - if (Env->ShouldBeUndiscoverable) { + if (Env->ShouldBeUndiscoverable) { TEST_RESPONSE(MessageDiscoverResult, ERROR, 0, 0); } else { TEST_RESPONSE(MessageDiscoverResult, NODATA, 0, ""); @@ -3128,7 +3128,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { break; } case 290: - if (Env->ShouldBeUnwritable) { + if (Env->ShouldBeUnwritable) { TEST_RESPONSE(MessagePutResult, ERROR, 0, ""); } else { TEST_RESPONSE(MessagePutResult, OK, 0, ""); @@ -3146,7 +3146,7 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { VERBOSE_COUT(LastResponse.Profile); } VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; default: ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; @@ -3156,8 +3156,8 @@ class TTestBlobStorageProxyBasic1 : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyBasic1(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -3192,7 +3192,7 @@ class TTestGetMultipart : public TTestBlobStorageProxy { "Got '" << LastResponse.Data[1] << "' instead of '" << p2 << "'"); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -3203,8 +3203,8 @@ class TTestGetMultipart : public TTestBlobStorageProxy { } public: TTestGetMultipart(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -3219,7 +3219,7 @@ class TTestVDiskCompacted : public TTestBlobStorageProxy { TAutoPtr<TEvBlobStorage::TEvVCompact> vCompact(new TEvBlobStorage::TEvVCompact( vDiskId, NKikimrBlobStorage::TEvVCompact::ASYNC)); VERBOSE_COUT("Sending EvVCompact to vDiskIdx: " << vDiskIdx); - ctx.Send(Env->VDisks[vDiskIdx], vCompact.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vCompact.Release()); break; } case 10: @@ -3228,7 +3228,7 @@ class TTestVDiskCompacted : public TTestBlobStorageProxy { TVDiskID vDiskId(0, 1, 0, vDiskIdx , 0); TAutoPtr<TEvBlobStorage::TEvVStatus> vStatus(new TEvBlobStorage::TEvVStatus(vDiskId)); VERBOSE_COUT("Sending EvVStatus to vDiskIdx: " << vDiskIdx); - ctx.Send(Env->VDisks[vDiskIdx], vStatus.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vStatus.Release()); break; } case 20: @@ -3236,14 +3236,14 @@ class TTestVDiskCompacted : public TTestBlobStorageProxy { TEST_RESPONSE(MessageVStatusResult, OK, 0, ""); if (LastResponse.LogoBlobsCompacted) { VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } Sleep(TDuration::MilliSeconds(50)); TVDiskID vDiskId(0, 1, 0, vDiskIdx , 0); TAutoPtr<TEvBlobStorage::TEvVStatus> vStatus(new TEvBlobStorage::TEvVStatus(vDiskId)); VERBOSE_COUT("Sending EvVStatus to vDiskIdx: " << vDiskIdx); - ctx.Send(Env->VDisks[vDiskIdx], vStatus.Release()); + ctx.Send(Env->VDisks[vDiskIdx], vStatus.Release()); return; } default: @@ -3254,8 +3254,8 @@ class TTestVDiskCompacted : public TTestBlobStorageProxy { } public: TTestVDiskCompacted(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; @@ -3281,7 +3281,7 @@ class TTestBlobStorageProxyVPutVCollectVGetRace : public TTestBlobStorageProxy { msgId.SetMsgId(0); msgId.SetSequenceId(1); - ctx.Send(Env->VDisks[0], x.Release()); + ctx.Send(Env->VDisks[0], x.Release()); break; } case 10: @@ -3327,7 +3327,7 @@ class TTestBlobStorageProxyVPutVCollectVGetRace : public TTestBlobStorageProxy { { TEST_RESPONSE(MessageVGetResult, OK, 1, testData); VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); + Env->DoneEvent.Signal(); break; } default: @@ -3338,113 +3338,113 @@ class TTestBlobStorageProxyVPutVCollectVGetRace : public TTestBlobStorageProxy { } public: TTestBlobStorageProxyVPutVCollectVGetRace(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) - {} -}; - -TMaybe<TGroupStat::EKind> PutHandleClassToGroupStatKind(NKikimrBlobStorage::EPutHandleClass handleClass) { - switch (handleClass) { - case NKikimrBlobStorage::TabletLog: - return TGroupStat::EKind::PUT_TABLET_LOG; - - case NKikimrBlobStorage::UserData: - return TGroupStat::EKind::PUT_USER_DATA; - - default: - return {}; - } -} - -class TTestBlobStorageProxyForRequest : public TTestBlobStorageProxy { -protected: - void TestFSM(const TActorContext &ctx) override { - if (!TestStep) { - TIntrusivePtr<TDsProxyNodeMon> nodeMon = new TDsProxyNodeMon(NKikimr::AppData(ctx)->Counters, true); - TString name = Sprintf("%09" PRIu32, 0); - TIntrusivePtr<NMonitoring::TDynamicCounters> group = GetServiceCounters( - NKikimr::AppData(ctx)->Counters, "dsproxy")->GetSubgroup("blobstorageproxy", name); - TIntrusivePtr<NMonitoring::TDynamicCounters> percentileGroup = GetServiceCounters( - NKikimr::AppData(ctx)->Counters, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", name); - TIntrusivePtr<NMonitoring::TDynamicCounters> overviewGroup = GetServiceCounters( - NKikimr::AppData(ctx)->Counters, "dsproxy_overview"); - Mon = new TBlobStorageGroupProxyMon(group, percentileGroup, overviewGroup, BsInfo, nodeMon, false); - - TIntrusivePtr<NMonitoring::TDynamicCounters> DynCounters = new NMonitoring::TDynamicCounters(); - StoragePoolCounters = new NKikimr::TStoragePoolCounters(DynCounters, "", {}); - PerDiskStatsPtr = new TDiskResponsivenessTracker::TPerDiskStats; - } - } - - TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; - TIntrusivePtr<NKikimr::TStoragePoolCounters> StoragePoolCounters; - TDiskResponsivenessTracker::TPerDiskStatsPtr PerDiskStatsPtr; - - TTestBlobStorageProxyForRequest(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) {} }; -class TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob : public TTestBlobStorageProxyForRequest { - void TestFSM(const TActorContext &ctx) { - TTestBlobStorageProxyForRequest::TestFSM(ctx); - - VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); - TVector<TLogoBlobID> blobIds = { - TLogoBlobID(72075186224047637, 1, 863, 1, Data1.size(), 24576), - TLogoBlobID(72075186224047637, 1, 2194, 1, Data2.size(), 12288) - }; - - switch (TestStep) { - case 0: { - TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched(2); - batched[0] = GetPut(blobIds[0], Data1); - batched[1] = GetPut(blobIds[1], Data2); - - TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(HandleClass); +TMaybe<TGroupStat::EKind> PutHandleClassToGroupStatKind(NKikimrBlobStorage::EPutHandleClass handleClass) { + switch (handleClass) { + case NKikimrBlobStorage::TabletLog: + return TGroupStat::EKind::PUT_TABLET_LOG; + + case NKikimrBlobStorage::UserData: + return TGroupStat::EKind::PUT_USER_DATA; + + default: + return {}; + } +} + +class TTestBlobStorageProxyForRequest : public TTestBlobStorageProxy { +protected: + void TestFSM(const TActorContext &ctx) override { + if (!TestStep) { + TIntrusivePtr<TDsProxyNodeMon> nodeMon = new TDsProxyNodeMon(NKikimr::AppData(ctx)->Counters, true); + TString name = Sprintf("%09" PRIu32, 0); + TIntrusivePtr<NMonitoring::TDynamicCounters> group = GetServiceCounters( + NKikimr::AppData(ctx)->Counters, "dsproxy")->GetSubgroup("blobstorageproxy", name); + TIntrusivePtr<NMonitoring::TDynamicCounters> percentileGroup = GetServiceCounters( + NKikimr::AppData(ctx)->Counters, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", name); + TIntrusivePtr<NMonitoring::TDynamicCounters> overviewGroup = GetServiceCounters( + NKikimr::AppData(ctx)->Counters, "dsproxy_overview"); + Mon = new TBlobStorageGroupProxyMon(group, percentileGroup, overviewGroup, BsInfo, nodeMon, false); + + TIntrusivePtr<NMonitoring::TDynamicCounters> DynCounters = new NMonitoring::TDynamicCounters(); + StoragePoolCounters = new NKikimr::TStoragePoolCounters(DynCounters, "", {}); + PerDiskStatsPtr = new TDiskResponsivenessTracker::TPerDiskStats; + } + } + + TIntrusivePtr<TBlobStorageGroupProxyMon> Mon; + TIntrusivePtr<NKikimr::TStoragePoolCounters> StoragePoolCounters; + TDiskResponsivenessTracker::TPerDiskStatsPtr PerDiskStatsPtr; + + TTestBlobStorageProxyForRequest(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxy(proxy, bsInfo, env, parametrs) + {} +}; + +class TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob : public TTestBlobStorageProxyForRequest { + void TestFSM(const TActorContext &ctx) { + TTestBlobStorageProxyForRequest::TestFSM(ctx); + + VERBOSE_COUT("Test step " << TestStep << " Line " << __LINE__); + TVector<TLogoBlobID> blobIds = { + TLogoBlobID(72075186224047637, 1, 863, 1, Data1.size(), 24576), + TLogoBlobID(72075186224047637, 1, 2194, 1, Data2.size(), 12288) + }; + + switch (TestStep) { + case 0: { + TBatchedVec<TEvBlobStorage::TEvPut::TPtr> batched(2); + batched[0] = GetPut(blobIds[0], Data1); + batched[1] = GetPut(blobIds[1], Data2); + + TMaybe<TGroupStat::EKind> kind = PutHandleClassToGroupStatKind(HandleClass); IActor *reqActor = CreateBlobStorageGroupPutRequest(BsInfo, GroupQueues, - Mon, batched, false, PerDiskStatsPtr, kind,TInstant::Now(), - StoragePoolCounters, HandleClass, Tactic, false); - - ctx.Register(reqActor); - break; - } - case 10: - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - break; - case 20: - TEST_RESPONSE(MessagePutResult, OK, 0, ""); - VERBOSE_COUT("Done"); - Env->DoneEvent.Signal(); - break; - default: - ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; - break; - } - TestStep += 10; - } - - TEvBlobStorage::TEvPut::TPtr GetPut(TLogoBlobID id, const TString &data) { + Mon, batched, false, PerDiskStatsPtr, kind,TInstant::Now(), + StoragePoolCounters, HandleClass, Tactic, false); + + ctx.Register(reqActor); + break; + } + case 10: + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + break; + case 20: + TEST_RESPONSE(MessagePutResult, OK, 0, ""); + VERBOSE_COUT("Done"); + Env->DoneEvent.Signal(); + break; + default: + ythrow TWithBackTrace<yexception>() << "Unexpected TestStep " << TestStep << Endl; + break; + } + TestStep += 10; + } + + TEvBlobStorage::TEvPut::TPtr GetPut(TLogoBlobID id, const TString &data) { std::unique_ptr<TEvBlobStorage::TEvPut> put = std::make_unique<TEvBlobStorage::TEvPut>(id, data, - TInstant::Max(), HandleClass, Tactic); + TInstant::Max(), HandleClass, Tactic); return static_cast<TEventHandle<TEvBlobStorage::TEvPut>*>(new IEventHandle(SelfId(), SelfId(), put.release())); - } - - TEvBlobStorage::TEvPut::ETactic Tactic = TEvBlobStorage::TEvPut::TacticDefault; - NKikimrBlobStorage::EPutHandleClass HandleClass = NKikimrBlobStorage::TabletLog; - TString Data1; - TString Data2; -public: - TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, - const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) - : TTestBlobStorageProxyForRequest(proxy, bsInfo, env, parametrs) - { - Data1.resize(MaxBatchedPutSize - 1, 'a'); - Data2.resize(1, 'a'); - } -}; - + } + + TEvBlobStorage::TEvPut::ETactic Tactic = TEvBlobStorage::TEvPut::TacticDefault; + NKikimrBlobStorage::EPutHandleClass HandleClass = NKikimrBlobStorage::TabletLog; + TString Data1; + TString Data2; +public: + TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob(const TActorId &proxy, const TIntrusivePtr<TBlobStorageGroupInfo> &bsInfo, + const TIntrusivePtr<TTestEnvironment> &env, const TIntrusivePtr<ITestParametrs> ¶metrs) + : TTestBlobStorageProxyForRequest(proxy, bsInfo, env, parametrs) + { + Data1.resize(MaxBatchedPutSize - 1, 'a'); + Data2.resize(1, 'a'); + } +}; + #define PROXY_UNIT_TEST(a) UNIT_TEST(a) class TBlobStorageProxyTest: public TTestBase { @@ -3459,8 +3459,8 @@ class TBlobStorageProxyTest: public TTestBase { PROXY_UNIT_TEST(TestPartialGetMirror); PROXY_UNIT_TEST(TestBlock); PROXY_UNIT_TEST(TestBlockPersistence); - PROXY_UNIT_TEST(TestGetAndRangeGetManyBlobs); - PROXY_UNIT_TEST(TestInFlightPuts); + PROXY_UNIT_TEST(TestGetAndRangeGetManyBlobs); + PROXY_UNIT_TEST(TestInFlightPuts); PROXY_UNIT_TEST(TestCollectGarbage); PROXY_UNIT_TEST(TestHugeCollectGarbage); PROXY_UNIT_TEST(TestCollectGarbageAfterLargeData); @@ -3510,7 +3510,7 @@ class TBlobStorageProxyTest: public TTestBase { PROXY_UNIT_TEST(TestEmptyDiscover); PROXY_UNIT_TEST(TestEmptyDiscoverMaxi); PROXY_UNIT_TEST(TestCompactedGetMultipart); - PROXY_UNIT_TEST(TestBatchedPutRequestDoesNotContainAHugeBlob); + PROXY_UNIT_TEST(TestBatchedPutRequestDoesNotContainAHugeBlob); //TODO: Think of another way to emulate VDisk errors // PROXY_UNIT_TEST(TestProxyGetDoubleTimeout); // PROXY_UNIT_TEST(TestProxyDiscoverDoubleTimeout); @@ -3518,7 +3518,7 @@ class TBlobStorageProxyTest: public TTestBase { // PROXY_UNIT_TEST(TestProxyPutDoubleTimeout); UNIT_TEST_SUITE_END(); - TIntrusivePtr<TTestEnvironment> LastTestEnv; + TIntrusivePtr<TTestEnvironment> LastTestEnv; TMap<TString, TIntrusivePtr<NPDisk::TSectorMap>> SectorMapByPath; public: @@ -3546,13 +3546,13 @@ public: void TestVPutVGetPersistence() { TTempDir tempDir; TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure3Plus1Block; - ui64 partId = 1; - ui64 vDiskIdx = 1; - TTestArgs args{0, erasureSpecies}; - args.Parametrs = new TTestBlobStorageProxyVPut::TParametrs(TestData2, partId, VDIM_TestIdx, vDiskIdx); - TestBlobStorage<TTestBlobStorageProxyVPut>(tempDir().c_str(), args); - static bool isNoData = false; - TestBlobStorage<TTestBlobStorageProxyVGet<1, 1, isNoData>>(0, erasureSpecies, tempDir().c_str()); + ui64 partId = 1; + ui64 vDiskIdx = 1; + TTestArgs args{0, erasureSpecies}; + args.Parametrs = new TTestBlobStorageProxyVPut::TParametrs(TestData2, partId, VDIM_TestIdx, vDiskIdx); + TestBlobStorage<TTestBlobStorageProxyVPut>(tempDir().c_str(), args); + static bool isNoData = false; + TestBlobStorage<TTestBlobStorageProxyVGet<1, 1, isNoData>>(0, erasureSpecies, tempDir().c_str()); SectorMapByPath.clear(); } @@ -3726,43 +3726,43 @@ public: SectorMapByPath.clear(); } - void TestGetAndRangeGetManyBlobs() { - TTempDir tempDir; - TMersenne<ui64> prng(42); - TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure4Plus2Stripe; - - constexpr int actionCount = 17'000; - constexpr int startIndx = 1; - - static TVector<size_t> order(actionCount); - Iota(order.begin(), order.end(), (size_t)startIndx); - - // Put - Shuffle(order.begin(), order.end(), prng); - TestBlobStorage<TTestIterativePut<order>>(0, erasureSpecies, tempDir().c_str()); - - // Get - Shuffle(order.begin(), order.end(), prng); - TestBlobStorage<TTestOrderGet<order>>(0, erasureSpecies, tempDir().c_str()); - - order.clear(); - - // Forward range - TestBlobStorage<TTestRangeGet<startIndx, actionCount>>(0, erasureSpecies, tempDir().c_str()); - - // Backward range - TestBlobStorage<TTestRangeGet<actionCount, -actionCount>>(0, erasureSpecies, tempDir().c_str()); - - SectorMapByPath.clear(); - } - - void TestInFlightPuts() { - TTempDir tempDir; - TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure4Plus2Stripe; - TestBlobStorage<TTestInFlightPuts<256, 1'000>>(0, erasureSpecies, tempDir().c_str()); - SectorMapByPath.clear(); - } - + void TestGetAndRangeGetManyBlobs() { + TTempDir tempDir; + TMersenne<ui64> prng(42); + TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure4Plus2Stripe; + + constexpr int actionCount = 17'000; + constexpr int startIndx = 1; + + static TVector<size_t> order(actionCount); + Iota(order.begin(), order.end(), (size_t)startIndx); + + // Put + Shuffle(order.begin(), order.end(), prng); + TestBlobStorage<TTestIterativePut<order>>(0, erasureSpecies, tempDir().c_str()); + + // Get + Shuffle(order.begin(), order.end(), prng); + TestBlobStorage<TTestOrderGet<order>>(0, erasureSpecies, tempDir().c_str()); + + order.clear(); + + // Forward range + TestBlobStorage<TTestRangeGet<startIndx, actionCount>>(0, erasureSpecies, tempDir().c_str()); + + // Backward range + TestBlobStorage<TTestRangeGet<actionCount, -actionCount>>(0, erasureSpecies, tempDir().c_str()); + + SectorMapByPath.clear(); + } + + void TestInFlightPuts() { + TTempDir tempDir; + TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure4Plus2Stripe; + TestBlobStorage<TTestInFlightPuts<256, 1'000>>(0, erasureSpecies, tempDir().c_str()); + SectorMapByPath.clear(); + } + void TestHugeCollectGarbage() { TBlobStorageGroupType::EErasureSpecies erasureSpecies = TBlobStorageGroupType::Erasure3Plus1Block; TestBlobStorage<TTestBlobStorageProxyGarbageCollectHuge>(0, erasureSpecies, nullptr); @@ -3818,33 +3818,33 @@ public: } } - + template <int vDiskIdx, int partId> void TestProxyRestoreOnGet(TBlobStorageGroupType::EErasureSpecies erasureSpecies) { TTempDir tempDir; - - TBlobStorageGroupType type(erasureSpecies); - TTestArgs args{0, erasureSpecies}; - TIntrusivePtr<TTestBlobStorageProxyVPut::TParametrs> param = - new TTestBlobStorageProxyVPut::TParametrs(TestData2, 0); - args.Parametrs = param.Get(); - - for (param->PartId = 1; param->PartId <= type.TotalPartCount(); ++param->PartId ) { - TestBlobStorage<TTestBlobStorageProxyVPut>(tempDir().c_str(), args); - } - - static bool isNoData = false; - TestBlobStorage<TTestBlobStorageProxyVGet<vDiskIdx, partId, isNoData>>(0, erasureSpecies, tempDir().c_str()); - if (isNoData) { - SectorMapByPath.clear(); - return; - } + + TBlobStorageGroupType type(erasureSpecies); + TTestArgs args{0, erasureSpecies}; + TIntrusivePtr<TTestBlobStorageProxyVPut::TParametrs> param = + new TTestBlobStorageProxyVPut::TParametrs(TestData2, 0); + args.Parametrs = param.Get(); + + for (param->PartId = 1; param->PartId <= type.TotalPartCount(); ++param->PartId ) { + TestBlobStorage<TTestBlobStorageProxyVPut>(tempDir().c_str(), args); + } + + static bool isNoData = false; + TestBlobStorage<TTestBlobStorageProxyVGet<vDiskIdx, partId, isNoData>>(0, erasureSpecies, tempDir().c_str()); + if (isNoData) { + SectorMapByPath.clear(); + return; + } RemoveVDiskData(vDiskIdx, tempDir()); //TestBlobStorage<TTestBlobStorageProxyVGetFail<vDiskIdx>>(0, erasureSpecies, tempDir().c_str()); - TestBlobStorage<TTestBlobStorageProxyGet>(0, erasureSpecies, tempDir().c_str()); + TestBlobStorage<TTestBlobStorageProxyGet>(0, erasureSpecies, tempDir().c_str()); // Hands here - TestBlobStorage<TTestBlobStorageProxyVGet<vDiskIdx, partId, isNoData>>(0, erasureSpecies, tempDir().c_str()); + TestBlobStorage<TTestBlobStorageProxyVGet<vDiskIdx, partId, isNoData>>(0, erasureSpecies, tempDir().c_str()); SectorMapByPath.clear(); } @@ -3860,10 +3860,10 @@ public: TestBlobStorage<TTestBlobStorageProxyVGetFail<handoffVDiskIdx>>(badDiskMask, erasureSpecies, tempDir().c_str()); TestBlobStorage<TTestBlobStorageProxyDiscover>(badDiskMask, erasureSpecies, tempDir().c_str()); - static bool isNoData = false; - TestBlobStorage<TTestBlobStorageProxyVGet<handoffVDiskIdx, 1, isNoData>>(badDiskMask, erasureSpecies, + static bool isNoData = false; + TestBlobStorage<TTestBlobStorageProxyVGet<handoffVDiskIdx, 1, isNoData>>(badDiskMask, erasureSpecies, tempDir().c_str()); - UNIT_ASSERT(!isNoData); + UNIT_ASSERT(!isNoData); TestBlobStorage<TTestBlobStorageProxyBlockCheck>(badDiskMask, erasureSpecies, tempDir().c_str()); SectorMapByPath.clear(); } @@ -4044,72 +4044,72 @@ public: SectorMapByPath.clear(); } - void TestBatchedPutRequestDoesNotContainAHugeBlob() { - TTestArgs args{0, TBlobStorageGroupType::ErasureNone}; - args.DeviceType = TPDiskCategory::DEVICE_TYPE_NVME; - TestBlobStorage<TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob>(nullptr, args); - SectorMapByPath.clear(); + void TestBatchedPutRequestDoesNotContainAHugeBlob() { + TTestArgs args{0, TBlobStorageGroupType::ErasureNone}; + args.DeviceType = TPDiskCategory::DEVICE_TYPE_NVME; + TestBlobStorage<TTestBlobStorageProxyBatchedPutRequestDoesNotContainAHugeBlob>(nullptr, args); + SectorMapByPath.clear(); } THolder<TActorSystemSetup> BuildActorSystemSetup(ui32 nodeId, NMonitoring::TDynamicCounters &counters, TIntrusivePtr<TTableNameserverSetup> &nameserverTable, TInterconnectMock &interconnectMock) { - auto setup = MakeHolder<TActorSystemSetup>(); - setup->NodeId = nodeId; - setup->ExecutorsCount = 4; - setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); -#if RT_EXECUTOR_POOL - setup->Executors[0].Reset(new TRtExecutorPool(0, 1)); - setup->Executors[1].Reset(new TRtExecutorPool(1, 2)); - setup->Executors[2].Reset(new TRtExecutorPool(2, 10)); - setup->Executors[3].Reset(new TRtExecutorPool(3, 2)); -#else - setup->Executors[0].Reset(new TBasicExecutorPool(0, 1, 20)); - setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20)); - setup->Executors[2].Reset(new TIOExecutorPool(2, 10)); - setup->Executors[3].Reset(new TBasicExecutorPool(3, 2, 20)); -#endif - setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(512, 100))); + auto setup = MakeHolder<TActorSystemSetup>(); + setup->NodeId = nodeId; + setup->ExecutorsCount = 4; + setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); +#if RT_EXECUTOR_POOL + setup->Executors[0].Reset(new TRtExecutorPool(0, 1)); + setup->Executors[1].Reset(new TRtExecutorPool(1, 2)); + setup->Executors[2].Reset(new TRtExecutorPool(2, 10)); + setup->Executors[3].Reset(new TRtExecutorPool(3, 2)); +#else + setup->Executors[0].Reset(new TBasicExecutorPool(0, 1, 20)); + setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20)); + setup->Executors[2].Reset(new TIOExecutorPool(2, 10)); + setup->Executors[3].Reset(new TBasicExecutorPool(3, 2, 20)); +#endif + setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(512, 100))); setup->LocalServices.emplace_back(MakePollerActorId(), TActorSetupCmd(CreatePollerActor(), TMailboxType::ReadAsFilled, 0)); - - ui64 nodeCount = nameserverTable->StaticNodeTable.size() + 1; - setup->LocalServices.emplace_back( + + ui64 nodeCount = nameserverTable->StaticNodeTable.size() + 1; + setup->LocalServices.emplace_back( GetNameserviceActorId(), - TActorSetupCmd(CreateNameserverTable(nameserverTable), TMailboxType::ReadAsFilled, 0) - ); - TIntrusivePtr<TInterconnectProxyCommon> icCommon = new TInterconnectProxyCommon(); + TActorSetupCmd(CreateNameserverTable(nameserverTable), TMailboxType::ReadAsFilled, 0) + ); + TIntrusivePtr<TInterconnectProxyCommon> icCommon = new TInterconnectProxyCommon(); icCommon->NameserviceId = GetNameserviceActorId(); - icCommon->MonCounters = counters.GetSubgroup("counters", "interconnect"); - icCommon->TechnicalSelfHostName = "127.0.0.1"; - setup->Interconnect.ProxyActors.resize(nodeCount); - //ui64 basePort = 12000; - for (ui32 xnode : xrange<ui32>(1, nodeCount)) { - if (xnode != nodeId) { - //IActor *actor = new TInterconnectProxyTCP(xnode, icCommon); - IActor *actor = interconnectMock.CreateProxyMock(nodeId, xnode, icCommon); - setup->Interconnect.ProxyActors[xnode] = TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0); - } - } - - TActorSetupCmd profilerSetup(CreateProfilerActor(nullptr, "."), TMailboxType::Simple, 0); + icCommon->MonCounters = counters.GetSubgroup("counters", "interconnect"); + icCommon->TechnicalSelfHostName = "127.0.0.1"; + setup->Interconnect.ProxyActors.resize(nodeCount); + //ui64 basePort = 12000; + for (ui32 xnode : xrange<ui32>(1, nodeCount)) { + if (xnode != nodeId) { + //IActor *actor = new TInterconnectProxyTCP(xnode, icCommon); + IActor *actor = interconnectMock.CreateProxyMock(nodeId, xnode, icCommon); + setup->Interconnect.ProxyActors[xnode] = TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0); + } + } + + TActorSetupCmd profilerSetup(CreateProfilerActor(nullptr, "."), TMailboxType::Simple, 0); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeProfilerID(nodeId), profilerSetup)); - + const TActorId nameserviceId = GetNameserviceActorId(); - TActorSetupCmd nameserviceSetup(CreateNameserverTable(nameserverTable), TMailboxType::Simple, 0); + TActorSetupCmd nameserviceSetup(CreateNameserverTable(nameserverTable), TMailboxType::Simple, 0); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(nameserviceId, nameserviceSetup)); - - return setup; - } - - TIntrusivePtr<NActors::NLog::TSettings> AddLoggerActor(THolder<TActorSystemSetup> &setup, - NMonitoring::TDynamicCounters &counters) { - + + return setup; + } + + TIntrusivePtr<NActors::NLog::TSettings> AddLoggerActor(THolder<TActorSystemSetup> &setup, + NMonitoring::TDynamicCounters &counters) { + NActors::TActorId loggerActorId = NActors::TActorId(setup->NodeId, "logger"); - TIntrusivePtr<NActors::NLog::TSettings> logSettings( - new NActors::NLog::TSettings(loggerActorId, NKikimrServices::LOGGER, - IsVerbose ? NLog::PRI_ERROR : NLog::PRI_CRIT, - IsVerbose ? NLog::PRI_ERROR : NLog::PRI_CRIT, - 0)); + TIntrusivePtr<NActors::NLog::TSettings> logSettings( + new NActors::NLog::TSettings(loggerActorId, NKikimrServices::LOGGER, + IsVerbose ? NLog::PRI_ERROR : NLog::PRI_CRIT, + IsVerbose ? NLog::PRI_ERROR : NLog::PRI_CRIT, + 0)); logSettings->Append( NActorsServices::EServiceCommon_MIN, NActorsServices::EServiceCommon_MAX, @@ -4117,67 +4117,67 @@ public: ); logSettings->Append( NKikimrServices::EServiceKikimr_MIN, - NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_MAX, NKikimrServices::EServiceKikimr_Name ); - - if (IsVerbose) { - TString explanation; - //logSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::BS_LOCALRECOVERY, explanation); - //logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_COLLECT, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_DISCOVER, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_PUT, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_GET, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_VDISK_GET, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_VDISK_PUT, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_SKELETON, explanation); - logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_QUEUE, explanation); - // logSettings->TimeThresholdMs = 5000; - // logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_IFACE, explanation); - //logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_LOCALRECOVERY, explanation); - } - NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(logSettings, - NActors::CreateStderrBackend(), - counters.GetSubgroup("counters", "utils")); - NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 2); + + if (IsVerbose) { + TString explanation; + //logSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::BS_LOCALRECOVERY, explanation); + //logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_COLLECT, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_DISCOVER, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_PUT, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_PROXY_GET, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_VDISK_GET, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_VDISK_PUT, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_SKELETON, explanation); + logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_QUEUE, explanation); + // logSettings->TimeThresholdMs = 5000; + // logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_IFACE, explanation); + //logSettings->SetLevel(NLog::PRI_DEBUG, NKikimrServices::BS_LOCALRECOVERY, explanation); + } + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(logSettings, + NActors::CreateStderrBackend(), + counters.GetSubgroup("counters", "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 2); std::pair<NActors::TActorId, NActors::TActorSetupCmd> loggerActorPair(loggerActorId, loggerActorCmd); - setup->LocalServices.push_back(loggerActorPair); - - return logSettings; - } - + setup->LocalServices.push_back(loggerActorPair); + + return logSettings; + } + template <class T> void TestBlobStorage(ui64 badVDiskMask, TBlobStorageGroupType::EErasureSpecies erasureSpecies, const char* dir, ui32 sleepMilliseconds = 0, bool startBadDisks = true, ui32 failDomains = 0, ui32 drivesPerFailDomain = 0) { - TTestArgs args; - args.BadVDiskMask = badVDiskMask; - args.ErasureSpecies = erasureSpecies; - args.SleepMilliseconds = sleepMilliseconds; - args.StartBadDisks = startBadDisks; - args.FailDomainCount = failDomains; - args.DrivesPerFailDomain = drivesPerFailDomain; - TestBlobStorage<T>(dir, args); - } - - template <class T> - void TestBlobStorage(const char* dir, const TTestArgs &args = TTestArgs()) { - TIntrusivePtr<TTestEnvironment> env = new TTestEnvironment(args); + TTestArgs args; + args.BadVDiskMask = badVDiskMask; + args.ErasureSpecies = erasureSpecies; + args.SleepMilliseconds = sleepMilliseconds; + args.StartBadDisks = startBadDisks; + args.FailDomainCount = failDomains; + args.DrivesPerFailDomain = drivesPerFailDomain; + TestBlobStorage<T>(dir, args); + } + + template <class T> + void TestBlobStorage(const char* dir, const TTestArgs &args = TTestArgs()) { + TIntrusivePtr<TTestEnvironment> env = new TTestEnvironment(args); TIntrusivePtr<TTableNameserverSetup> nameserverTable(new TTableNameserverSetup()); TPortManager pm; nameserverTable->StaticNodeTable[1] = std::pair<TString, ui32>("127.0.0.1", pm.GetPort(12001)); nameserverTable->StaticNodeTable[2] = std::pair<TString, ui32>("127.0.0.1", pm.GetPort(12002)); - TIntrusivePtr<TBlobStorageGroupInfo> bsInfo(new TBlobStorageGroupInfo(args.ErasureSpecies, - env->DrivesPerFailDomain, env->FailDomainCount, 1, &env->VDisks)); + TIntrusivePtr<TBlobStorageGroupInfo> bsInfo(new TBlobStorageGroupInfo(args.ErasureSpecies, + env->DrivesPerFailDomain, env->FailDomainCount, 1, &env->VDisks)); // first node TIntrusivePtr<NMonitoring::TDynamicCounters> counters(new NMonitoring::TDynamicCounters()); - TInterconnectMock interconnect; - - TAppData appData(0, 0, 0, 0, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr); + TInterconnectMock interconnect; + + TAppData appData(0, 0, 0, 0, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr); appData.Counters = counters; auto ioContext = std::make_shared<NKikimr::NPDisk::TIoContextFactoryOSS>(); appData.IoContextFactory = ioContext.get(); @@ -4196,11 +4196,11 @@ public: TTempDir tempDir; NPDisk::TKey mainKey = 123; NPDisk::TKey badMainKey = 124; - for (ui32 failDomainIdx = 0; failDomainIdx < env->FailDomainCount; ++failDomainIdx) { - for (ui32 driveIdx = 0; driveIdx < env->DrivesPerFailDomain; ++driveIdx) { - ui32 i = env->GetVDiskTestIdx(failDomainIdx, driveIdx); - bool isBad = (args.BadVDiskMask & (1 << i)); - if (!args.StartBadDisks && isBad) { + for (ui32 failDomainIdx = 0; failDomainIdx < env->FailDomainCount; ++failDomainIdx) { + for (ui32 driveIdx = 0; driveIdx < env->DrivesPerFailDomain; ++driveIdx) { + ui32 i = env->GetVDiskTestIdx(failDomainIdx, driveIdx); + bool isBad = (args.BadVDiskMask & (1 << i)); + if (!args.StartBadDisks && isBad) { continue; } TString databaseDirectory = Sprintf(isBad ? "%s/vdisk_bad_%d" : "%s/vdisk_%d", @@ -4233,11 +4233,11 @@ public: setup2->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(pdiskId, pDiskSetup)); TVDiskConfig::TBaseInfo baseInfo( - env->VDiskIds[i], - pdiskId, + env->VDiskIds[i], + pdiskId, mainKey, i + 1, - args.DeviceType, + args.DeviceType, 0, NKikimrBlobStorage::TVDiskKind::Default, 2, @@ -4258,14 +4258,14 @@ public: } } - TActorSetupCmd proxyTestSetup(new T(env->ProxyId, bsInfo.Get(), env, args.Parametrs), - TMailboxType::Revolving, 0); + TActorSetupCmd proxyTestSetup(new T(env->ProxyId, bsInfo.Get(), env, args.Parametrs), + TMailboxType::Revolving, 0); setup1->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(env->ProxyTestId, proxyTestSetup)); - ////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////// GetServiceCounters(counters, "utils"); - TIntrusivePtr<NActors::NLog::TSettings> logSettings1 = AddLoggerActor(setup1, *counters); - TIntrusivePtr<NActors::NLog::TSettings> logSettings2 = AddLoggerActor(setup2, *counters); + TIntrusivePtr<NActors::NLog::TSettings> logSettings1 = AddLoggerActor(setup1, *counters); + TIntrusivePtr<NActors::NLog::TSettings> logSettings2 = AddLoggerActor(setup2, *counters); ////////////////////////////////////////////////////////////////////////////// std::unique_ptr<TActorSystem> actorSystem1; @@ -4273,7 +4273,7 @@ public: actorSystem1->Start(); std::unique_ptr<TActorSystem> actorSystem2; actorSystem2.reset(new TActorSystem(setup2, &appData, logSettings2)); - actorSystem2->Start(); + actorSystem2->Start(); EnableActorCallstack(); try { VERBOSE_COUT("Sending TEvBoot to testproxy"); @@ -4281,38 +4281,38 @@ public: nullptr)); VERBOSE_COUT("Done"); - bool isDone = env->DoneEvent.Wait(TEST_TIMEOUT); - UNIT_ASSERT_C(isDone, "test timeout, badVDiskMask = " << args.BadVDiskMask); + bool isDone = env->DoneEvent.Wait(TEST_TIMEOUT); + UNIT_ASSERT_C(isDone, "test timeout, badVDiskMask = " << args.BadVDiskMask); if (isDone) { - Sleep(TDuration::MilliSeconds(args.SleepMilliseconds)); + Sleep(TDuration::MilliSeconds(args.SleepMilliseconds)); } - for (ui32 i = 0; i != env->VDiskCount; ++i) { - actorSystem2->Send(env->VDisks[i], new NActors::TEvents::TEvPoisonPill()); + for (ui32 i = 0; i != env->VDiskCount; ++i) { + actorSystem2->Send(env->VDisks[i], new NActors::TEvents::TEvPoisonPill()); } actorSystem1->Stop(); - actorSystem2->Stop(); - env->DoneEvent.Reset(); + actorSystem2->Stop(); + env->DoneEvent.Reset(); DisableActorCallstack(); } catch (yexception ex) { - env->IsLastExceptionSet = true; - env->LastException = ex; + env->IsLastExceptionSet = true; + env->LastException = ex; DisableActorCallstack(); actorSystem1->Stop(); - actorSystem2->Stop(); + actorSystem2->Stop(); } catch (...) { DisableActorCallstack(); actorSystem1->Stop(); - actorSystem2->Stop(); + actorSystem2->Stop(); throw; } - if (env->IsLastExceptionSet) { - env->IsLastExceptionSet = false; - ythrow env->LastException; + if (env->IsLastExceptionSet) { + env->IsLastExceptionSet = false; + ythrow env->LastException; } - LastTestEnv = env; + LastTestEnv = env; } }; diff --git a/ydb/core/blobstorage/dsproxy/ya.make b/ydb/core/blobstorage/dsproxy/ya.make index 6e165c7f5d1..1a7a9db5737 100644 --- a/ydb/core/blobstorage/dsproxy/ya.make +++ b/ydb/core/blobstorage/dsproxy/ya.make @@ -35,7 +35,7 @@ SRCS( dsproxy_nodemon.h dsproxy_nodemonactor.cpp dsproxy_nodemonactor.h - dsproxy_patch.cpp + dsproxy_patch.cpp dsproxy_put.cpp dsproxy_put_impl.cpp dsproxy_put_impl.h diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp index 054d5be6653..d11a6711d15 100644 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp +++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp @@ -787,17 +787,17 @@ void TBlobStorageGroupInfo::PickSubgroup(ui32 hash, TVDiskIds *outVDisk, TServic } bool TBlobStorageGroupInfo::BelongsToSubgroup(const TVDiskID &vdisk, ui32 hash) const { - Y_VERIFY_DEBUG_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); - Y_VERIFY_DEBUG_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration - << ", given GroupGeneration# " << vdisk.GroupGeneration); + Y_VERIFY_DEBUG_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); + Y_VERIFY_DEBUG_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration + << ", given GroupGeneration# " << vdisk.GroupGeneration); return Topology->BelongsToSubgroup(TVDiskIdShort(vdisk), hash); } // Returns either vdisk idx in the blob subgroup, or BlobSubgroupSize if the vdisk is not in the blob subgroup ui32 TBlobStorageGroupInfo::GetIdxInSubgroup(const TVDiskID &vdisk, ui32 hash) const { - Y_VERIFY_DEBUG_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); - Y_VERIFY_DEBUG_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration - << ", given GroupGeneration# " << vdisk.GroupGeneration); + Y_VERIFY_DEBUG_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); + Y_VERIFY_DEBUG_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration + << ", given GroupGeneration# " << vdisk.GroupGeneration); return Topology->GetIdxInSubgroup(vdisk, hash); } @@ -807,9 +807,9 @@ TVDiskID TBlobStorageGroupInfo::GetVDiskInSubgroup(ui32 idxInSubgroup, ui32 hash } ui32 TBlobStorageGroupInfo::GetOrderNumber(const TVDiskID &vdisk) const { - Y_VERIFY_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); - Y_VERIFY_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration - << ", given GroupGeneration# " << vdisk.GroupGeneration); + Y_VERIFY_S(vdisk.GroupID == GroupID, "Expected GroupID# " << GroupID << ", given GroupID# " << vdisk.GroupID); + Y_VERIFY_S(vdisk.GroupGeneration == GroupGeneration, "Expected GroupGeeration# " << GroupGeneration + << ", given GroupGeneration# " << vdisk.GroupGeneration); return Topology->GetOrderNumber(vdisk); } diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h index e2d1445be4e..c1de55152c8 100644 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h +++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h @@ -413,7 +413,7 @@ public: // origin of the group info content std::optional<NKikimrBlobStorage::TGroupInfo> Group; - TAtomicLogPriorityMuteChecker<NLog::PRI_ERROR, NLog::PRI_DEBUG> PutErrorMuteChecker; + TAtomicLogPriorityMuteChecker<NLog::PRI_ERROR, NLog::PRI_DEBUG> PutErrorMuteChecker; private: // group topology -- all disks inside this group diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap.cpp index 572c36fbaba..2abb19d4ce6 100644 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap.cpp +++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap.cpp @@ -15,24 +15,24 @@ namespace NKikimr { ui32 domainIdx = hash % numFailDomains; // get shift of this disk inside ring - ui32 shift = failDomainOrderNumber + numFailDomains - domainIdx; - if (shift >= numFailDomains) { - shift -= numFailDomains; - } + ui32 shift = failDomainOrderNumber + numFailDomains - domainIdx; + if (shift >= numFailDomains) { + shift -= numFailDomains; + } if (shift >= BlobSubgroupSize) { return BlobSubgroupSize; } - const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(vdisk); - if (domain.VDisks.size() == 1) { - return shift; - } else { - TReallyFastRng32 rng(hash); - for (ui32 i = 0; i < shift; ++i) { - rng(); - } - - return vdisk.VDisk == rng() % domain.VDisks.size() ? shift : BlobSubgroupSize; + const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(vdisk); + if (domain.VDisks.size() == 1) { + return shift; + } else { + TReallyFastRng32 rng(hash); + for (ui32 i = 0; i < shift; ++i) { + rng(); + } + + return vdisk.VDisk == rng() % domain.VDisks.size() ? shift : BlobSubgroupSize; } } @@ -48,28 +48,28 @@ namespace NKikimr { const ui32 numFailDomains = Topology->GetTotalFailDomainsNum(); ui32 domainIdx = hash % numFailDomains; - if (Topology->GetNumVDisksPerFailDomain() == 1) { - for (ui32 i = 0; i < BlobSubgroupSize; ++i) { - const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(domainIdx); - if (++domainIdx == numFailDomains) { - domainIdx = 0; - } - Y_VERIFY_DEBUG(domain.VDisks.size() == 1); - const TBlobStorageGroupInfo::TVDiskInfo& vdisk = domain.VDisks[0]; - orderNums.push_back(vdisk.OrderNumber); - } - } else { - TReallyFastRng32 rng(hash); - - for (ui32 i = 0; i < BlobSubgroupSize; ++i) { - const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(domainIdx); - if (++domainIdx == numFailDomains) { - domainIdx = 0; - } - const ui32 vdiskIdx = rng() % domain.VDisks.size(); - const TBlobStorageGroupInfo::TVDiskInfo& vdisk = domain.VDisks[vdiskIdx]; - orderNums.push_back(vdisk.OrderNumber); + if (Topology->GetNumVDisksPerFailDomain() == 1) { + for (ui32 i = 0; i < BlobSubgroupSize; ++i) { + const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(domainIdx); + if (++domainIdx == numFailDomains) { + domainIdx = 0; + } + Y_VERIFY_DEBUG(domain.VDisks.size() == 1); + const TBlobStorageGroupInfo::TVDiskInfo& vdisk = domain.VDisks[0]; + orderNums.push_back(vdisk.OrderNumber); } + } else { + TReallyFastRng32 rng(hash); + + for (ui32 i = 0; i < BlobSubgroupSize; ++i) { + const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(domainIdx); + if (++domainIdx == numFailDomains) { + domainIdx = 0; + } + const ui32 vdiskIdx = rng() % domain.VDisks.size(); + const TBlobStorageGroupInfo::TVDiskInfo& vdisk = domain.VDisks[vdiskIdx]; + orderNums.push_back(vdisk.OrderNumber); + } } } @@ -83,19 +83,19 @@ namespace NKikimr { TVDiskIdShort GetVDiskInSubgroup(ui32 idxInSubgroup, ui32 hash) override final { const ui32 numFailDomains = Topology->GetTotalFailDomainsNum(); - ui32 domainIdx = (static_cast<ui64>(hash) + idxInSubgroup) % numFailDomains; + ui32 domainIdx = (static_cast<ui64>(hash) + idxInSubgroup) % numFailDomains; const TBlobStorageGroupInfo::TFailDomain& domain = Topology->GetFailDomain(domainIdx); - if (domain.VDisks.size() == 1) { - return domain.VDisks[0].VDiskIdShort; - } else { - TReallyFastRng32 rng(hash); - for (ui32 i = 0; i < idxInSubgroup; ++i) { - rng(); - } - - const ui32 vdiskIdx = rng() % domain.VDisks.size(); - return domain.VDisks[vdiskIdx].VDiskIdShort; + if (domain.VDisks.size() == 1) { + return domain.VDisks[0].VDiskIdShort; + } else { + TReallyFastRng32 rng(hash); + for (ui32 i = 0; i < idxInSubgroup; ++i) { + rng(); + } + + const ui32 vdiskIdx = rng() % domain.VDisks.size(); + return domain.VDisks[vdiskIdx].VDiskIdShort; } } }; diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp index 9147c6a8715..20bc84ca779 100644 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp +++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp @@ -97,8 +97,8 @@ namespace { Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { - void MakeBelongsToSubgroupBenchmark(TBlobStorageGroupType::EErasureSpecies erasure, ui32 numFailDomains, - NUnitTest::TTestContext& ut_context) { + void MakeBelongsToSubgroupBenchmark(TBlobStorageGroupType::EErasureSpecies erasure, ui32 numFailDomains, + NUnitTest::TTestContext& ut_context) { auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(erasure, 1, numFailDomains, 1); const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); @@ -110,10 +110,10 @@ Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { } } - constexpr ui64 blobCount = 10'000'000; - UNIT_ASSERT(blobCount == ids.size()); - ui64 iterationCount = numFailDomains * blobCount; - + constexpr ui64 blobCount = 10'000'000; + UNIT_ASSERT(blobCount == ids.size()); + ui64 iterationCount = numFailDomains * blobCount; + THPTimer timer; ui32 num = 0; for (const auto& vdisk : groupInfo->GetVDisks()) { @@ -122,14 +122,14 @@ Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { num += groupInfo->BelongsToSubgroup(vd, id.Hash()) ? 1 : 0; } } - double newMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; - TString newMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) - << " domains " << numFailDomains - << " new (ns)"; - ut_context.Metrics[newMetricsName] = newMetric; - Cerr << newMetricsName << ": " << newMetric<< Endl; - - timer.Reset(); + double newMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; + TString newMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) + << " domains " << numFailDomains + << " new (ns)"; + ut_context.Metrics[newMetricsName] = newMetric; + Cerr << newMetricsName << ": " << newMetric<< Endl; + + timer.Reset(); ui32 num2 = 0; for (const auto& vdisk : groupInfo->GetVDisks()) { for (const TLogoBlobID& id : ids) { @@ -137,75 +137,75 @@ Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { num2 += orig.BelongsToSubgroup(vd, id.Hash()) ? 1 : 0; } } - double oldMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; - TString oldMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) - << " domains " << numFailDomains - << " old (ns)"; - ut_context.Metrics[oldMetricsName] = oldMetric; - Cerr << oldMetricsName << ": " << oldMetric << Endl; + double oldMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; + TString oldMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) + << " domains " << numFailDomains + << " old (ns)"; + ut_context.Metrics[oldMetricsName] = oldMetric; + Cerr << oldMetricsName << ": " << oldMetric << Endl; UNIT_ASSERT_VALUES_EQUAL(num, num2); } - Y_UNIT_TEST(BelongsToSubgroupBenchmark) { - auto erasures = {TBlobStorageGroupType::ErasureNone, - TBlobStorageGroupType::ErasureMirror3, - TBlobStorageGroupType::Erasure4Plus2Block, - TBlobStorageGroupType::ErasureMirror3of4}; - for (auto erasure : erasures) { - TBlobStorageGroupType type(erasure); - for (ui32 domains : {type.BlobSubgroupSize(), 9u}) { - MakeBelongsToSubgroupBenchmark(erasure, domains, ut_context); - } - } - } - + Y_UNIT_TEST(BelongsToSubgroupBenchmark) { + auto erasures = {TBlobStorageGroupType::ErasureNone, + TBlobStorageGroupType::ErasureMirror3, + TBlobStorageGroupType::Erasure4Plus2Block, + TBlobStorageGroupType::ErasureMirror3of4}; + for (auto erasure : erasures) { + TBlobStorageGroupType type(erasure); + for (ui32 domains : {type.BlobSubgroupSize(), 9u}) { + MakeBelongsToSubgroupBenchmark(erasure, domains, ut_context); + } + } + } + void BasicCheck(const std::unique_ptr<TBlobStorageGroupInfo> &groupInfo, TOriginalBlobStorageGroupInfo &orig, - TLogoBlobID id, ui32 blobSubgroupSize) { - std::array<TVDiskID, 8> vdisks; - std::array<TActorId, 8> services; - orig.PickSubgroup(id.Hash(), blobSubgroupSize, vdisks.data(), services.data()); - - TBlobStorageGroupInfo::TVDiskIds vdisks2; - TBlobStorageGroupInfo::TServiceIds services2; - groupInfo->PickSubgroup(id.Hash(), &vdisks2, &services2); - - UNIT_ASSERT_EQUAL(vdisks2.size(), blobSubgroupSize); - UNIT_ASSERT_EQUAL(services2.size(), blobSubgroupSize); - UNIT_ASSERT(std::equal(vdisks2.begin(), vdisks2.end(), vdisks.begin())); - UNIT_ASSERT(std::equal(services2.begin(), services2.end(), services.begin())); - - for (ui32 i = 0; i < blobSubgroupSize; ++i) { - const TVDiskID& vdisk = vdisks[i]; - - UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), - orig.GetVDiskInSubgroup(i, id.Hash())); - UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), vdisk); - - UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), - orig.GetIdxInSubgroup(vdisk, id.Hash())); - UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), i); - } - - THashMap<TVDiskID, ui32> disk2index; - for (ui32 i = 0; i < blobSubgroupSize; ++i) { - disk2index[vdisks2[i]] = i; - } - - for (const auto& vdisk : groupInfo->GetVDisks()) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - auto it = disk2index.find(vd); - bool isReplicaFor = it != disk2index.end(); - - UNIT_ASSERT_VALUES_EQUAL(orig.BelongsToSubgroup(vd, id.Hash()), isReplicaFor); - UNIT_ASSERT_VALUES_EQUAL(groupInfo->BelongsToSubgroup(vd, id.Hash()), isReplicaFor); - - const ui32 index = isReplicaFor ? it->second : blobSubgroupSize; - UNIT_ASSERT_VALUES_EQUAL(orig.GetIdxInSubgroup(vd, id.Hash()), index); - UNIT_ASSERT_VALUES_EQUAL(groupInfo->GetIdxInSubgroup(vd, id.Hash()), index); - } - } - + TLogoBlobID id, ui32 blobSubgroupSize) { + std::array<TVDiskID, 8> vdisks; + std::array<TActorId, 8> services; + orig.PickSubgroup(id.Hash(), blobSubgroupSize, vdisks.data(), services.data()); + + TBlobStorageGroupInfo::TVDiskIds vdisks2; + TBlobStorageGroupInfo::TServiceIds services2; + groupInfo->PickSubgroup(id.Hash(), &vdisks2, &services2); + + UNIT_ASSERT_EQUAL(vdisks2.size(), blobSubgroupSize); + UNIT_ASSERT_EQUAL(services2.size(), blobSubgroupSize); + UNIT_ASSERT(std::equal(vdisks2.begin(), vdisks2.end(), vdisks.begin())); + UNIT_ASSERT(std::equal(services2.begin(), services2.end(), services.begin())); + + for (ui32 i = 0; i < blobSubgroupSize; ++i) { + const TVDiskID& vdisk = vdisks[i]; + + UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), + orig.GetVDiskInSubgroup(i, id.Hash())); + UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), vdisk); + + UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), + orig.GetIdxInSubgroup(vdisk, id.Hash())); + UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), i); + } + + THashMap<TVDiskID, ui32> disk2index; + for (ui32 i = 0; i < blobSubgroupSize; ++i) { + disk2index[vdisks2[i]] = i; + } + + for (const auto& vdisk : groupInfo->GetVDisks()) { + auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); + auto it = disk2index.find(vd); + bool isReplicaFor = it != disk2index.end(); + + UNIT_ASSERT_VALUES_EQUAL(orig.BelongsToSubgroup(vd, id.Hash()), isReplicaFor); + UNIT_ASSERT_VALUES_EQUAL(groupInfo->BelongsToSubgroup(vd, id.Hash()), isReplicaFor); + + const ui32 index = isReplicaFor ? it->second : blobSubgroupSize; + UNIT_ASSERT_VALUES_EQUAL(orig.GetIdxInSubgroup(vd, id.Hash()), index); + UNIT_ASSERT_VALUES_EQUAL(groupInfo->GetIdxInSubgroup(vd, id.Hash()), index); + } + } + Y_UNIT_TEST(BasicChecks) { for (auto erasure : {TBlobStorageGroupType::ErasureNone, TBlobStorageGroupType::ErasureMirror3, TBlobStorageGroupType::Erasure3Plus1Block, TBlobStorageGroupType::Erasure3Plus1Stripe, @@ -219,20 +219,20 @@ Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { for (ui32 i = 0; i < 100; ++i) { for (ui32 j = 0; j < 10; ++j) { TLogoBlobID id(i, 1, j, 1, 1000, 1); - BasicCheck(groupInfo, orig, id, blobSubgroupSize); + BasicCheck(groupInfo, orig, id, blobSubgroupSize); } } } } - Y_UNIT_TEST(CheckCorrectBehaviourWithHashOverlow) { + Y_UNIT_TEST(CheckCorrectBehaviourWithHashOverlow) { auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TErasureType::ErasureMirror3, 1U, 5U, 1U); - const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); - TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); - TLogoBlobID id(4550843067551373890, 2564314201, 2840555155, 59, 0, 2230444); - BasicCheck(groupInfo, orig, id, blobSubgroupSize); - } - + const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); + TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); + TLogoBlobID id(4550843067551373890, 2564314201, 2840555155, 59, 0, 2230444); + BasicCheck(groupInfo, orig, id, blobSubgroupSize); + } + Y_UNIT_TEST(Mirror3dcMapper) { auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureMirror3, 3U, 5U, 4U); diff --git a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.cpp b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.cpp index 0f6a440f57f..fad6cb8b091 100644 --- a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.cpp +++ b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.cpp @@ -36,7 +36,7 @@ void TBlobPutTactics::ToString(ui64 value, TString *out) { void TEventTypeField::ToString(ui64 value, TString* out) { #define CASE(EVENT) case TEvBlobStorage::EVENT: *out = #EVENT; break; switch(TEvBlobStorage::EEv(value)) { - CASE(EvPatch); + CASE(EvPatch); CASE(EvPut); CASE(EvVPut); CASE(EvGet); diff --git a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h index 7e20a7bd81a..27638251f5a 100644 --- a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h +++ b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h @@ -56,12 +56,12 @@ struct TEventTypeField { } #define BLOBSTORAGE_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ - PROBE(DSProxyBatchedPutRequest, GROUPS("DSProxy"), \ - TYPES(ui64, ui32), \ - NAMES("count" , "groupId")) \ - PROBE(DSProxyBatchedGetRequest, GROUPS("DSProxy"), \ - TYPES(ui64, ui32), \ - NAMES("count" , "groupId")) \ + PROBE(DSProxyBatchedPutRequest, GROUPS("DSProxy"), \ + TYPES(ui64, ui32), \ + NAMES("count" , "groupId")) \ + PROBE(DSProxyBatchedGetRequest, GROUPS("DSProxy"), \ + TYPES(ui64, ui32), \ + NAMES("count" , "groupId")) \ PROBE(ProxyPutBootstrapPart, GROUPS("Durations"), \ TYPES(ui64, double, double, ui64, double), \ NAMES("size", "waitMs", "splitMs", "splitCount", "splitElapsedMs")) \ @@ -72,18 +72,18 @@ struct TEventTypeField { TYPES(NKikimr::TEventTypeField, ui64, ui64, ui32, ui32, ui32, double, double, double, double, TString, TString), \ NAMES("type", "size", "tabletId", "groupId", "channel", "vdiskOrderNum", "startTime", "totalDurationMs", \ "vdiskDurationMs", "transferDurationMs", "handleClass", "status")) \ - PROBE(VDiskSkeletonFrontVMovedPatchRecieved, GROUPS("VDisk", "DSProxy"), \ - TYPES(ui32, ui32, ui32, ui64, ui64), \ - NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ - PROBE(VDiskSkeletonFrontVPatchStartRecieved, GROUPS("VDisk", "DSProxy"), \ - TYPES(ui32, ui32, ui32, ui64, ui64), \ - NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ - PROBE(VDiskSkeletonFrontVPatchDiffRecieved, GROUPS("VDisk", "DSProxy"), \ - TYPES(ui32, ui32, ui32, ui64, ui64), \ - NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ - PROBE(VDiskSkeletonFrontVPatchXorDiffRecieved, GROUPS("VDisk", "DSProxy"), \ - TYPES(ui32, ui32, ui32, ui64, ui64), \ - NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ + PROBE(VDiskSkeletonFrontVMovedPatchRecieved, GROUPS("VDisk", "DSProxy"), \ + TYPES(ui32, ui32, ui32, ui64, ui64), \ + NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ + PROBE(VDiskSkeletonFrontVPatchStartRecieved, GROUPS("VDisk", "DSProxy"), \ + TYPES(ui32, ui32, ui32, ui64, ui64), \ + NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ + PROBE(VDiskSkeletonFrontVPatchDiffRecieved, GROUPS("VDisk", "DSProxy"), \ + TYPES(ui32, ui32, ui32, ui64, ui64), \ + NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ + PROBE(VDiskSkeletonFrontVPatchXorDiffRecieved, GROUPS("VDisk", "DSProxy"), \ + TYPES(ui32, ui32, ui32, ui64, ui64), \ + NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ PROBE(VDiskSkeletonFrontVPutRecieved, GROUPS("VDisk", "DSProxy"), \ TYPES(ui32, ui32, ui32, ui64, ui64), \ NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ @@ -93,9 +93,9 @@ struct TEventTypeField { PROBE(VDiskSkeletonVPutRecieved, GROUPS("VDisk", "DSProxy"), \ TYPES(ui32, ui32, ui32, ui64, ui64), \ NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ - PROBE(VDiskSkeletonVMultiPutRecieved, GROUPS("VDisk", "DSProxy"), \ - TYPES(ui32, ui32, ui32, ui64, ui64), \ - NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ + PROBE(VDiskSkeletonVMultiPutRecieved, GROUPS("VDisk", "DSProxy"), \ + TYPES(ui32, ui32, ui32, ui64, ui64), \ + NAMES("nodeId", "groupId", "vdiskOrderNum", "tabletId", "size")) \ PROBE(VDiskRecoveryLogWriterVPutIsRecieved, GROUPS("VDisk"), \ TYPES(ui32, ui64), \ NAMES("owner", "lsn")) \ diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index e3c74cce7b4..2a942f00074 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -483,23 +483,23 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { BlockGroup(runtime, sender0, tabletId, groupId, generation++, true, NKikimrProto::EReplyStatus::NO_GROUP); } - CUSTOM_UNIT_TEST(TestSendToInvalidGroupId) { - TTestBasicRuntime runtime(1, false); - Setup(runtime, "", nullptr); - - auto sender = runtime.AllocateEdgeActor(0); - - CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); - ui32 groupId = Max<ui32>(); - - ui64 tabletId = 1234; - ui32 generation = 1; + CUSTOM_UNIT_TEST(TestSendToInvalidGroupId) { + TTestBasicRuntime runtime(1, false); + Setup(runtime, "", nullptr); + + auto sender = runtime.AllocateEdgeActor(0); + + CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); + ui32 groupId = Max<ui32>(); + + ui64 tabletId = 1234; + ui32 generation = 1; BlockGroup(runtime, sender, tabletId, groupId, generation, true, NKikimrProto::ERROR); - Put(runtime, sender, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", - NKikimrProto::EReplyStatus::ERROR); + Put(runtime, sender, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", + NKikimrProto::EReplyStatus::ERROR); CollectGroup(runtime, sender, tabletId, groupId, true, NKikimrProto::EReplyStatus::ERROR); - } - + } + CUSTOM_UNIT_TEST(TestBlockEncriptedGroup) { TTestBasicRuntime runtime(2, false); Setup(runtime, "", nullptr); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index 47093a108da..c61cfcfcddf 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -33,14 +33,14 @@ void TNodeWarden::StartInvalidGroupProxy() { STLOG(PRI_DEBUG, BS_NODE, NW11, "StartInvalidGroupProxy", (GroupId, groupId)); TlsActivationContext->ExecutorThread.ActorSystem->RegisterLocalService(MakeBlobStorageProxyID(groupId), Register( CreateBlobStorageGroupEjectedProxy(groupId, DsProxyNodeMon), TMailboxType::ReadAsFilled, AppData()->SystemPoolId)); -} - +} + void TNodeWarden::StopInvalidGroupProxy() { - ui32 groupId = Max<ui32>(); + ui32 groupId = Max<ui32>(); STLOG(PRI_DEBUG, BS_NODE, NW15, "StopInvalidGroupProxy", (GroupId, groupId)); TActivationContext::Send(new IEventHandle(TEvents::TSystem::Poison, 0, MakeBlobStorageProxyID(groupId), {}, nullptr, 0)); -} - +} + void TNodeWarden::PassAway() { STLOG(PRI_DEBUG, BS_NODE, NW25, "PassAway"); NTabletPipe::CloseClient(SelfId(), PipeClientId); @@ -74,11 +74,11 @@ void TNodeWarden::Bootstrap() { DsProxyNodeMonActor = Register(CreateDsProxyNodeMon(DsProxyNodeMon)); DsProxyPerPoolCounters = new TDsProxyPerPoolCounters(AppData()->Counters); - if (actorSystem && actorSystem->AppData<TAppData>() && actorSystem->AppData<TAppData>()->Icb) { - actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching"); - actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch"); - } - + if (actorSystem && actorSystem->AppData<TAppData>() && actorSystem->AppData<TAppData>()->Icb) { + actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching"); + actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch"); + } + // start replication broker const auto& replBrokerConfig = Cfg->ServiceSet.GetReplBrokerConfig(); @@ -147,7 +147,7 @@ void TNodeWarden::HandleReadCache() { if (!proto.HasInstanceId() && !proto.HasAvailDomain() && !proto.HasServiceSet()) { return; } - + Y_VERIFY(proto.HasInstanceId()); Y_VERIFY(proto.HasAvailDomain() && proto.GetAvailDomain() == AvailDomainId); if (!InstanceId) { diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index be67e5ef670..b4c28eade64 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -5,7 +5,7 @@ #include <ydb/core/blobstorage/dsproxy/group_sessions.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> - + namespace NKikimr::NStorage { constexpr ui32 ProxyConfigurationTimeoutMilliseconds = 200; @@ -105,9 +105,9 @@ namespace NKikimr::NStorage { struct TEvUpdateNodeDrives : TEventLocal<TEvUpdateNodeDrives, EvUpdateNodeDrives> {}; }; - TControlWrapper EnablePutBatching; - TControlWrapper EnableVPatch; - + TControlWrapper EnablePutBatching; + TControlWrapper EnableVPatch; + TReplQuoter::TPtr ReplNodeRequestQuoter; TReplQuoter::TPtr ReplNodeResponseQuoter; @@ -118,8 +118,8 @@ namespace NKikimr::NStorage { TNodeWarden(const TIntrusivePtr<TNodeWardenConfig> &cfg) : Cfg(cfg) - , EnablePutBatching(Cfg->FeatureFlags.GetEnablePutBatchingForBlobStorage(), false, true) - , EnableVPatch(Cfg->FeatureFlags.GetEnableVPatch(), false, true) + , EnablePutBatching(Cfg->FeatureFlags.GetEnablePutBatchingForBlobStorage(), false, true) + , EnableVPatch(Cfg->FeatureFlags.GetEnableVPatch(), false, true) { Y_VERIFY(Cfg->ServiceSet.AvailabilityDomainsSize() <= 1); AvailDomainId = 1; @@ -409,7 +409,7 @@ namespace NKikimr::NStorage { void RenderWholePage(IOutputStream&); void RenderLocalDrives(IOutputStream&); void RenderDSProxies(IOutputStream& out); - + void SendDiskMetrics(bool reportMetrics); void Handle(TEvStatusUpdate::TPtr ev); @@ -418,7 +418,7 @@ namespace NKikimr::NStorage { void Handle(TEvBlobStorage::TEvRestartPDiskResult::TPtr ev); void FillInVDiskStatus(google::protobuf::RepeatedPtrField<NKikimrBlobStorage::TVDiskStatus> *pb, bool initial); - + void HandleForwarded(TAutoPtr<::NActors::IEventHandle> &ev); void HandleIncrHugeInit(NIncrHuge::TEvIncrHugeInit::TPtr ev); @@ -449,13 +449,13 @@ namespace NKikimr::NStorage { fFunc(TEvBlobStorage::TEvPut::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvGet::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvBlock::EventType, HandleForwarded); - fFunc(TEvBlobStorage::TEvPatch::EventType, HandleForwarded); + fFunc(TEvBlobStorage::TEvPatch::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvDiscover::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvRange::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvCollectGarbage::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvStatus::EventType, HandleForwarded); fFunc(TEvBlobStorage::TEvBunchOfEvents::EventType, HandleForwarded); - fFunc(TEvRequestProxySessionsState::EventType, HandleForwarded); + fFunc(TEvRequestProxySessionsState::EventType, HandleForwarded); hFunc(NIncrHuge::TEvIncrHugeInit, HandleIncrHugeInit); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 8e722ce42a0..fb8230b2260 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -329,37 +329,37 @@ struct TEvLog : public TEventLocal<TEvLog, TEvBlobStorage::EvLog> { mutable NLWTrace::TOrbit Orbit; }; -struct TEvMultiLog : public TEventLocal<TEvMultiLog, TEvBlobStorage::EvMultiLog> { - TBatchedVec<THolder<TEvLog>> Logs; - TLsnSeg LsnSeg; - - void AddLog(THolder<TEvLog> &&ev) { - Logs.emplace_back(std::move(ev)); - auto &log = *Logs.back(); - if (Logs.size() == 1) { - LsnSeg = TLsnSeg(log.LsnSegmentStart, log.Lsn); - } else { - Y_VERIFY_S(LsnSeg.Last + 1 == log.LsnSegmentStart, "LastLsn# " << LsnSeg.Last << - " NewLsnStart# " << log.LsnSegmentStart); - LsnSeg.Last = log.Lsn; - } - } - - TString ToString() const { - return ToString(*this); - } - - static TString ToString(const TEvMultiLog &record) { - TStringBuilder str; - str << '{'; - for (ui64 idx = 0; idx < record.Logs.size(); ++idx) { - str << (idx ? ", " : "") << idx << "# " << record.Logs[idx]->ToString(); - } - str << '}'; - return str; - } -}; - +struct TEvMultiLog : public TEventLocal<TEvMultiLog, TEvBlobStorage::EvMultiLog> { + TBatchedVec<THolder<TEvLog>> Logs; + TLsnSeg LsnSeg; + + void AddLog(THolder<TEvLog> &&ev) { + Logs.emplace_back(std::move(ev)); + auto &log = *Logs.back(); + if (Logs.size() == 1) { + LsnSeg = TLsnSeg(log.LsnSegmentStart, log.Lsn); + } else { + Y_VERIFY_S(LsnSeg.Last + 1 == log.LsnSegmentStart, "LastLsn# " << LsnSeg.Last << + " NewLsnStart# " << log.LsnSegmentStart); + LsnSeg.Last = log.Lsn; + } + } + + TString ToString() const { + return ToString(*this); + } + + static TString ToString(const TEvMultiLog &record) { + TStringBuilder str; + str << '{'; + for (ui64 idx = 0; idx < record.Logs.size(); ++idx) { + str << (idx ? ", " : "") << idx << "# " << record.Logs[idx]->ToString(); + } + str << '}'; + return str; + } +}; + struct TEvLogResult : public TEventLocal<TEvLogResult, TEvBlobStorage::EvLogResult> { struct TRecord { ui64 Lsn; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp index d92b19c6cca..84517e0c77a 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp @@ -539,27 +539,27 @@ public: } void ErrorHandle(NPDisk::TEvMultiLog::TPtr &ev) { - const NPDisk::TEvMultiLog &evMultiLog = *ev->Get(); - TStringStream str; - str << "PDiskId# " << PDisk->PDiskId; - str << " TEvBatchedLogs error because PDisk State# "; - if (CurrentStateFunc() == &TPDiskActor::StateInit) { - str << "Init, wait for PDisk to initialize. Did you ckeck EvYardInit result? Marker# BSY10"; - } else if (CurrentStateFunc() == &TPDiskActor::StateError) { - str << "Error, there is a terminal internal error in PDisk. Did you check EvYardInit result? Marker# BSY11"; - } else { - str << "Unknown, something went very wrong in PDisk. Marker# BSY12"; - } - str << " StateErrorReason# " << StateErrorReason; - THolder<NPDisk::TEvLogResult> result(new NPDisk::TEvLogResult(NKikimrProto::CORRUPTED, 0, str.Str())); - for (auto &log : evMultiLog.Logs) { - result->Results.push_back(NPDisk::TEvLogResult::TRecord(log->Lsn, log->Cookie)); - } - PDisk->Mon.WriteLog.CountRequest(0); + const NPDisk::TEvMultiLog &evMultiLog = *ev->Get(); + TStringStream str; + str << "PDiskId# " << PDisk->PDiskId; + str << " TEvBatchedLogs error because PDisk State# "; + if (CurrentStateFunc() == &TPDiskActor::StateInit) { + str << "Init, wait for PDisk to initialize. Did you ckeck EvYardInit result? Marker# BSY10"; + } else if (CurrentStateFunc() == &TPDiskActor::StateError) { + str << "Error, there is a terminal internal error in PDisk. Did you check EvYardInit result? Marker# BSY11"; + } else { + str << "Unknown, something went very wrong in PDisk. Marker# BSY12"; + } + str << " StateErrorReason# " << StateErrorReason; + THolder<NPDisk::TEvLogResult> result(new NPDisk::TEvLogResult(NKikimrProto::CORRUPTED, 0, str.Str())); + for (auto &log : evMultiLog.Logs) { + result->Results.push_back(NPDisk::TEvLogResult::TRecord(log->Lsn, log->Cookie)); + } + PDisk->Mon.WriteLog.CountRequest(0); Send(ev->Sender, result.Release()); - PDisk->Mon.WriteLog.CountResponse(); - } - + PDisk->Mon.WriteLog.CountResponse(); + } + void ErrorHandle(NPDisk::TEvReadLog::TPtr &ev) { const NPDisk::TEvReadLog &evReadLog = *ev->Get(); TStringStream str; @@ -677,15 +677,15 @@ public: } void Handle(NPDisk::TEvMultiLog::TPtr &ev) { - for (auto &log : ev->Get()->Logs) { - double burstMs; - TLogWrite* request = PDisk->ReqCreator.CreateLogWrite(*log, ev->Sender, burstMs, std::move(ev->TraceId)); - CheckBurst(request->IsSensitive, burstMs); - request->Orbit = std::move(log->Orbit); - PDisk->InputRequest(request); - } - } - + for (auto &log : ev->Get()->Logs) { + double burstMs; + TLogWrite* request = PDisk->ReqCreator.CreateLogWrite(*log, ev->Sender, burstMs, std::move(ev->TraceId)); + CheckBurst(request->IsSensitive, burstMs); + request->Orbit = std::move(log->Orbit); + PDisk->InputRequest(request); + } + } + void Handle(NPDisk::TEvReadLog::TPtr &ev) { LOG_DEBUG(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# %" PRIu32 " %s Marker# BSY01", (ui32)PDisk->PDiskId, ev->Get()->ToString().c_str()); diff --git a/ydb/core/blobstorage/testload/test_load_actor.cpp b/ydb/core/blobstorage/testload/test_load_actor.cpp index c8788b13104..7d4768fef3e 100644 --- a/ydb/core/blobstorage/testload/test_load_actor.cpp +++ b/ydb/core/blobstorage/testload/test_load_actor.cpp @@ -1,10 +1,10 @@ #include "test_load_actor.h" - + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/blobstorage/base/blobstorage_events.h> #include <ydb/public/lib/base/msgbus.h> - + #include <google/protobuf/text_format.h> #include <library/cpp/monlib/service/pages/templates.h> @@ -238,7 +238,7 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Load actor with tag# " << msg->Tag << " finished"); LoadActors.erase(iter); - FinishedTests.push_back({msg->Tag, msg->ErrorReason, TAppData::TimeProvider->Now()}); + FinishedTests.push_back({msg->Tag, msg->ErrorReason, TAppData::TimeProvider->Now()}); auto it = InfoRequests.begin(); while (it != InfoRequests.end()) { diff --git a/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp b/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp index 3b446a74c2d..c70b7b00189 100644 --- a/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp +++ b/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp @@ -373,7 +373,7 @@ public: if (IntervalMs) { // Enforce intervals between requests - TInstant now = TAppData::TimeProvider->Now(); + TInstant now = TAppData::TimeProvider->Now(); TInstant nextRequest = LastRequest + TDuration::MilliSeconds(IntervalMs); if (now < nextRequest) { // Suspend sending until interval will elapse diff --git a/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp b/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp index 8ea16e014cb..a18244d0fcc 100644 --- a/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp @@ -387,7 +387,7 @@ public: if (IntervalMs) { // Enforce intervals between requests - TInstant now = TAppData::TimeProvider->Now(); + TInstant now = TAppData::TimeProvider->Now(); TInstant nextRequest = LastRequest + TDuration::MilliSeconds(IntervalMs); if (now < nextRequest) { // Suspend sending until interval will elapse diff --git a/ydb/core/blobstorage/testload/test_load_vdisk_write.cpp b/ydb/core/blobstorage/testload/test_load_vdisk_write.cpp index 39ecca9dbda..6cd952ccf2f 100644 --- a/ydb/core/blobstorage/testload/test_load_vdisk_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_vdisk_write.cpp @@ -148,7 +148,7 @@ namespace NKikimr { } void TryToIssuePuts(const TActorContext& ctx) { - const TInstant now = TAppData::TimeProvider->Now(); + const TInstant now = TAppData::TimeProvider->Now(); while (IsConnected && now >= NextWriteRequestTimestamp && @@ -218,7 +218,7 @@ namespace NKikimr { } void TryToCollect(const TActorContext& ctx) { - const TInstant now = TAppData::TimeProvider->Now(); + const TInstant now = TAppData::TimeProvider->Now(); if (IsConnected && now >= NextCollectRequestTimestamp) { if (CollectStep < BlobStep + StepDistance) { const ui32 collectStep = CollectStep++; diff --git a/ydb/core/blobstorage/testload/test_load_write.cpp b/ydb/core/blobstorage/testload/test_load_write.cpp index 3169c1609ed..572278f9ae3 100644 --- a/ydb/core/blobstorage/testload/test_load_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_write.cpp @@ -41,7 +41,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo public: void Wakeup(const TActorContext& ctx) { - while (Events && TAppData::TimeProvider->Now() >= Events.top().Timestamp) { + while (Events && TAppData::TimeProvider->Now() >= Events.top().Timestamp) { TCallback callback = std::move(Events.top().Callback); Events.pop(); callback(ctx); @@ -248,7 +248,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo // Issue TEvDiscover void Bootstrap(const TActorContext& ctx) { - NextWriteTimestamp = TAppData::TimeProvider->Now(); + NextWriteTimestamp = TAppData::TimeProvider->Now(); auto ev = std::make_unique<TEvBlobStorage::TEvDiscover>(TabletId, Generation, false, true, TInstant::Max(), 0); LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, PrintMe() << " is bootstrapped, going to send " << ev->ToString()); @@ -326,8 +326,8 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo } void StartWorking(const TActorContext& ctx) { - StartTimestamp = TAppData::TimeProvider->Now(); - InitializeTrackers(StartTimestamp); + StartTimestamp = TAppData::TimeProvider->Now(); + InitializeTrackers(StartTimestamp); IssueWriteIfPossible(ctx); ScheduleGarbageCollect(ctx); ExposeCounters(ctx); @@ -353,7 +353,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo SendToBSProxy(ctx, GroupId, ev.Release(), QueryDispatcher.ObtainCookie(std::move(callback))); } - void InitializeTrackers(TInstant now) { + void InitializeTrackers(TInstant now) { LastLatencyTrackerUpdate = now; MegabytesPerSecondST.Add(now, 0); @@ -407,8 +407,8 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo ReadBytesInFlightQT.CalculateQuantiles(); using namespace std::placeholders; - WakeupQueue.Put(TAppData::TimeProvider->Now() + ExposePeriod, - std::bind(&TTabletWriter::ExposeCounters, this, _1), ctx); + WakeupQueue.Put(TAppData::TimeProvider->Now() + ExposePeriod, + std::bind(&TTabletWriter::ExposeCounters, this, _1), ctx); } void DumpState(IOutputStream& str) { @@ -503,7 +503,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo } void IssueWriteIfPossible(const TActorContext& ctx) { - const TInstant now = TAppData::TimeProvider->Now(); + const TInstant now = TAppData::TimeProvider->Now(); while ((WritesInFlight < MaxWritesInFlight || !MaxWritesInFlight) && (WriteBytesInFlight < MaxWriteBytesInFlight || !MaxWriteBytesInFlight) && @@ -575,7 +575,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo if (ConfirmedBlobIds.size() == 1) { if (NextReadTimestamp == TInstant()) { - NextReadTimestamp = TAppData::TimeProvider->Now(); + NextReadTimestamp = TAppData::TimeProvider->Now(); } IssueReadIfPossible(ctx); } @@ -584,7 +584,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo const auto nowCycles = GetCycleCountFast(); WritesInFlightTimestamps.emplace_back(writeQueryId, nowCycles); SentTimestamp.emplace(writeQueryId, nowCycles); - IssuedWriteTimestamp.push_back(TAppData::TimeProvider->Now()); + IssuedWriteTimestamp.push_back(TAppData::TimeProvider->Now()); while (IssuedWriteTimestamp.size() > 10000 || IssuedWriteTimestamp.back() - IssuedWriteTimestamp.front() >= TDuration::Seconds(5)) { IssuedWriteTimestamp.pop_front(); } @@ -638,8 +638,8 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo TDuration duration = GarbageCollectIntervalGen.Generate(); if (duration != TDuration()) { using namespace std::placeholders; - WakeupQueue.Put(TAppData::TimeProvider->Now() + duration, - std::bind(&TTabletWriter::IssueGarbageCollectRequest, this, _1), ctx); + WakeupQueue.Put(TAppData::TimeProvider->Now() + duration, + std::bind(&TTabletWriter::IssueGarbageCollectRequest, this, _1), ctx); } } @@ -665,7 +665,7 @@ class TLogWriterTestLoadActor : public TActorBootstrapped<TLogWriterTestLoadActo } void IssueReadIfPossible(const TActorContext& ctx) { - const TInstant now = TAppData::TimeProvider->Now(); + const TInstant now = TAppData::TimeProvider->Now(); while (ReadsInFlight < MaxReadsInFlight && (ReadBytesInFlight < MaxReadBytesInFlight || !MaxReadBytesInFlight) && @@ -877,7 +877,7 @@ public: } void HandleUpdateQuantile(const TActorContext& ctx) { - TInstant now = TAppData::TimeProvider->Now(); + TInstant now = TAppData::TimeProvider->Now(); for (auto& writer : TabletWriters) { writer.UpdateQuantile(now); } @@ -887,7 +887,7 @@ public: void HandleWakeup(const TActorContext& ctx) { // erase all scheduled items before this time point, including it WakeupScheduledAt.erase(WakeupScheduledAt.begin(), std::upper_bound(WakeupScheduledAt.begin(), - WakeupScheduledAt.end(), TAppData::TimeProvider->Now())); + WakeupScheduledAt.end(), TAppData::TimeProvider->Now())); WakeupQueue.Wakeup(ctx); ScheduleWakeup(ctx); } @@ -907,7 +907,7 @@ public: const TInstant scheduledWakeupTime = WakeupScheduledAt ? WakeupScheduledAt.front() : TInstant::Max(); if (nextWakeupTime && *nextWakeupTime < scheduledWakeupTime) { WakeupScheduledAt.push_front(*nextWakeupTime); - TDuration delta = *nextWakeupTime - TAppData::TimeProvider->Now(); + TDuration delta = *nextWakeupTime - TAppData::TimeProvider->Now(); if (delta >= WakeupScheduleThreshold) { ctx.Schedule(delta, new TEvents::TEvWakeup); } else { diff --git a/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp b/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp index 1f1b9e27be5..01c0f79c834 100644 --- a/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp @@ -2,7 +2,7 @@ #include <ydb/core/blobstorage/dsproxy/group_sessions.h> - + Y_UNIT_TEST_SUITE(CountingEvents) { struct TTestInfo { @@ -56,34 +56,34 @@ Y_UNIT_TEST_SUITE(CountingEvents) { UNIT_ASSERT_EQUAL(collectResult->Status, status); } - TIntrusivePtr<TGroupQueues> ReceiveGroupQueues(const TTestInfo &test) { - test.Runtime->WrapInActorContext(test.Edge, [&] { - SendToBSProxy(test.Edge, test.Info->GroupID, new TEvRequestProxySessionsState); - }); - std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); - UNIT_ASSERT_EQUAL_C(handle->Type, TEvBlobStorage::EvProxySessionsState, "expected# " << (ui64)TEvBlobStorage::EvProxySessionsState - << " given# " << handle->Type); - TEvProxySessionsState *state = handle->Get<TEvProxySessionsState>(); - return state->GroupQueues; - } - - void NormalizePredictedDelays(const TIntrusivePtr<TGroupQueues> &queues) { - for (TGroupQueues::TFailDomain &domain : queues->FailDomains) { - for (TGroupQueues::TVDisk &vDisk : domain.VDisks) { - ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; - ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; - for (ui32 id = begin; id < end; ++id) { - NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id); - auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); - if (flowRecord) { - flowRecord->SetPredictedDelayNs(1'000'000); - } - } - } - } - } - - + TIntrusivePtr<TGroupQueues> ReceiveGroupQueues(const TTestInfo &test) { + test.Runtime->WrapInActorContext(test.Edge, [&] { + SendToBSProxy(test.Edge, test.Info->GroupID, new TEvRequestProxySessionsState); + }); + std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); + UNIT_ASSERT_EQUAL_C(handle->Type, TEvBlobStorage::EvProxySessionsState, "expected# " << (ui64)TEvBlobStorage::EvProxySessionsState + << " given# " << handle->Type); + TEvProxySessionsState *state = handle->Get<TEvProxySessionsState>(); + return state->GroupQueues; + } + + void NormalizePredictedDelays(const TIntrusivePtr<TGroupQueues> &queues) { + for (TGroupQueues::TFailDomain &domain : queues->FailDomains) { + for (TGroupQueues::TVDisk &vDisk : domain.VDisks) { + ui32 begin = NKikimrBlobStorage::EVDiskQueueId::Begin; + ui32 end = NKikimrBlobStorage::EVDiskQueueId::End; + for (ui32 id = begin; id < end; ++id) { + NKikimrBlobStorage::EVDiskQueueId vDiskQueueId = static_cast<NKikimrBlobStorage::EVDiskQueueId>(id); + auto flowRecord = vDisk.Queues.FlowRecordForQueueId(vDiskQueueId); + if (flowRecord) { + flowRecord->SetPredictedDelayNs(1'000'000); + } + } + } + } + } + + void CountingEventsTest(TString typeOperation, ui32 eventsCount, TBlobStorageGroupType groupType) { TEnvironmentSetup env(true, groupType); @@ -98,9 +98,9 @@ Y_UNIT_TEST_SUITE(CountingEvents) { const TActorId& edge = runtime->AllocateEdgeActor(1); TTestInfo test{runtime, edge, info}; - // set predicted a response time - TIntrusivePtr<TGroupQueues> queues = ReceiveGroupQueues(test); - + // set predicted a response time + TIntrusivePtr<TGroupQueues> queues = ReceiveGroupQueues(test); + constexpr ui32 size = 100; TString data(size, 'a'); ui64 tabletId = 1; @@ -110,12 +110,12 @@ Y_UNIT_TEST_SUITE(CountingEvents) { if (typeOperation == "put") { TLogoBlobID originalBlobId(tabletId, 1, 0, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendPut(test, originalBlobId, data, NKikimrProto::OK); startEventsCount = test.Runtime->GetEventsProcessed(); TLogoBlobID originalBlobId2(tabletId, 1, 1, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendPut(test, originalBlobId2, data, NKikimrProto::OK); finishEventsCount = test.Runtime->GetEventsProcessed(); @@ -123,39 +123,39 @@ Y_UNIT_TEST_SUITE(CountingEvents) { startEventsCount = test.Runtime->GetEventsProcessed(); TLogoBlobID originalBlobId3(tabletId, 1, 2, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendPut(test, originalBlobId3, data, NKikimrProto::OK); finishEventsCount = test.Runtime->GetEventsProcessed(); UNIT_ASSERT_VALUES_EQUAL(finishEventsCount - startEventsCount, eventsCount); } else if (typeOperation == "get") { TLogoBlobID originalBlobId(tabletId, 1, 0, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendPut(test, originalBlobId, data, NKikimrProto::OK); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendGet(test, originalBlobId, data, NKikimrProto::OK); startEventsCount = test.Runtime->GetEventsProcessed(); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendGet(test, originalBlobId, data, NKikimrProto::OK); finishEventsCount = test.Runtime->GetEventsProcessed(); UNIT_ASSERT_VALUES_EQUAL(finishEventsCount - startEventsCount, eventsCount); startEventsCount = test.Runtime->GetEventsProcessed(); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendGet(test, originalBlobId, data, NKikimrProto::OK); finishEventsCount = test.Runtime->GetEventsProcessed(); UNIT_ASSERT_VALUES_EQUAL(finishEventsCount - startEventsCount, eventsCount); } else if (typeOperation == "collect") { TLogoBlobID originalBlobId(tabletId, 1, 0, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendPut(test, originalBlobId, data, NKikimrProto::OK); startEventsCount = test.Runtime->GetEventsProcessed(); TLogoBlobID originalBlobId2(tabletId, 1, 1, 0, size, 0); - NormalizePredictedDelays(queues); + NormalizePredictedDelays(queues); SendCollect(test, originalBlobId2, NKikimrProto::OK); finishEventsCount = test.Runtime->GetEventsProcessed(); diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 1cc05f00bf0..a09f8c16f90 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -61,7 +61,7 @@ struct TEnvironmentSetup { }) {} - TEnvironmentSetup(ui32 nodeCount, TNodeWardenMockActor::TSetup::TPtr nodeWardenMockSetup, + TEnvironmentSetup(ui32 nodeCount, TNodeWardenMockActor::TSetup::TPtr nodeWardenMockSetup, TBlobStorageGroupType erasure = TBlobStorageGroupType::ErasureNone) : TEnvironmentSetup(TSettings{ .NodeCount = nodeCount, @@ -308,7 +308,7 @@ struct TEnvironmentSetup { } auto response = Invoke(request); - UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); } std::vector<ui32> GetGroups() { @@ -446,18 +446,18 @@ struct TEnvironmentSetup { return WaitForEdgeActorEvent<TEvBlobStorage::TEvVDbStatResult>(edge)->Get()->Record.GetData(); } - void WithQueueId(const TVDiskID& vdiskId, NKikimrBlobStorage::EVDiskQueueId queue, std::function<void(TActorId)> action) { - const TActorId& queueId = CreateQueueActor(vdiskId, queue, 1000); - action(queueId); - Runtime->Send(new IEventHandle(TEvents::TSystem::Poison, 0, queueId, {}, nullptr, 0), queueId.NodeId()); - } - + void WithQueueId(const TVDiskID& vdiskId, NKikimrBlobStorage::EVDiskQueueId queue, std::function<void(TActorId)> action) { + const TActorId& queueId = CreateQueueActor(vdiskId, queue, 1000); + action(queueId); + Runtime->Send(new IEventHandle(TEvents::TSystem::Poison, 0, queueId, {}, nullptr, 0), queueId.NodeId()); + } + void CheckBlob(const TActorId& vdiskActorId, const TVDiskID& vdiskId, const TLogoBlobID& blobId, const TString& part, NKikimrProto::EReplyStatus status = NKikimrProto::OK) { - WithQueueId(vdiskId, NKikimrBlobStorage::EVDiskQueueId::GetFastRead, [&](TActorId queueId) { - const TActorId& edge = Runtime->AllocateEdgeActor(queueId.NodeId(), __FILE__, __LINE__); - Runtime->Send(new IEventHandle(queueId, edge, TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(vdiskId, - TInstant::Max(), NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, + WithQueueId(vdiskId, NKikimrBlobStorage::EVDiskQueueId::GetFastRead, [&](TActorId queueId) { + const TActorId& edge = Runtime->AllocateEdgeActor(queueId.NodeId(), __FILE__, __LINE__); + Runtime->Send(new IEventHandle(queueId, edge, TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(vdiskId, + TInstant::Max(), NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, Nothing(), {{blobId, 1u, ui32(part.size() - 2)}}).release()), queueId.NodeId()); auto r = WaitForEdgeActorEvent<TEvBlobStorage::TEvVGetResult>(edge, false); { @@ -483,22 +483,22 @@ struct TEnvironmentSetup { UNIT_ASSERT(!res.HasBuffer()); UNIT_ASSERT_VALUES_EQUAL(LogoBlobIDFromLogoBlobID(res.GetBlobID()), blobId.FullID()); UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), status); - } - }); + } + }); } void PutBlob(const TVDiskID& vdiskId, const TLogoBlobID& blobId, const TString& part) { - WithQueueId(vdiskId, NKikimrBlobStorage::EVDiskQueueId::PutTabletLog, [&](TActorId queueId) { - const TActorId& edge = Runtime->AllocateEdgeActor(queueId.NodeId(), __FILE__, __LINE__); - Runtime->Send(new IEventHandle(queueId, edge, new TEvBlobStorage::TEvVPut(blobId, part, vdiskId, false, nullptr, - TInstant::Max(), NKikimrBlobStorage::EPutHandleClass::TabletLog)), queueId.NodeId()); - auto r = WaitForEdgeActorEvent<TEvBlobStorage::TEvVPutResult>(edge); - - auto& record = r->Get()->Record; - Cerr << "*** PUT BLOB " << blobId << " TO " << vdiskId << " FINISHED WITH " - << NKikimrProto::EReplyStatus_Name(record.GetStatus()) << " ***" << Endl; - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrProto::OK); - }); + WithQueueId(vdiskId, NKikimrBlobStorage::EVDiskQueueId::PutTabletLog, [&](TActorId queueId) { + const TActorId& edge = Runtime->AllocateEdgeActor(queueId.NodeId(), __FILE__, __LINE__); + Runtime->Send(new IEventHandle(queueId, edge, new TEvBlobStorage::TEvVPut(blobId, part, vdiskId, false, nullptr, + TInstant::Max(), NKikimrBlobStorage::EPutHandleClass::TabletLog)), queueId.NodeId()); + auto r = WaitForEdgeActorEvent<TEvBlobStorage::TEvVPutResult>(edge); + + auto& record = r->Get()->Record; + Cerr << "*** PUT BLOB " << blobId << " TO " << vdiskId << " FINISHED WITH " + << NKikimrProto::EReplyStatus_Name(record.GetStatus()) << " ***" << Endl; + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrProto::OK); + }); } void CommenceReplication() { diff --git a/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp b/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp index b9ffea9da2c..5122f6c14fe 100644 --- a/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp @@ -6,7 +6,7 @@ Y_UNIT_TEST_SUITE(Mirror3of4) { SetRandomSeed(1); TEnvironmentSetup env(false, TBlobStorageGroupType::ErasureMirror3of4); auto& runtime = env.Runtime; - env.CreateBoxAndPool(); + env.CreateBoxAndPool(); env.Sim(TDuration::Minutes(1)); // runtime->SetLogPriority(NKikimrServices::BS_PROXY_PUT, NLog::PRI_DEBUG); // runtime->SetLogPriority(NKikimrServices::BS_VDISK_PUT, NLog::PRI_DEBUG); diff --git a/ydb/core/blobstorage/ut_blobstorage/patch.cpp b/ydb/core/blobstorage/ut_blobstorage/patch.cpp index b68c05b80ef..5f1559a5c89 100644 --- a/ydb/core/blobstorage/ut_blobstorage/patch.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/patch.cpp @@ -2,68 +2,68 @@ #include <ydb/core/blobstorage/ut_blobstorage/lib/common.h> Y_UNIT_TEST_SUITE(BlobPatching) { - - void SendPut(const TTestInfo &test, const TLogoBlobID &blobId, const TString &data, - NKikimrProto::EReplyStatus status) - { + + void SendPut(const TTestInfo &test, const TLogoBlobID &blobId, const TString &data, + NKikimrProto::EReplyStatus status) + { std::unique_ptr<IEventBase> ev = std::make_unique<TEvBlobStorage::TEvPut>(blobId, data, TInstant::Max()); - - test.Runtime->WrapInActorContext(test.Edge, [&] { + + test.Runtime->WrapInActorContext(test.Edge, [&] { SendToBSProxy(test.Edge, test.Info->GroupID, ev.release()); - }); - std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); - - UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvPutResult); - TEvBlobStorage::TEvPutResult *putResult = handle->Get<TEvBlobStorage::TEvPutResult>(); - UNIT_ASSERT_EQUAL(putResult->Status, status); - }; - - void SendGet(const TTestInfo &test, const TLogoBlobID &blobId, const TString &data, - NKikimrProto::EReplyStatus status) - { + }); + std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); + + UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvPutResult); + TEvBlobStorage::TEvPutResult *putResult = handle->Get<TEvBlobStorage::TEvPutResult>(); + UNIT_ASSERT_EQUAL(putResult->Status, status); + }; + + void SendGet(const TTestInfo &test, const TLogoBlobID &blobId, const TString &data, + NKikimrProto::EReplyStatus status) + { TArrayHolder<TEvBlobStorage::TEvGet::TQuery> getQueries{new TEvBlobStorage::TEvGet::TQuery[1]}; - getQueries[0].Id = blobId; + getQueries[0].Id = blobId; std::unique_ptr<IEventBase> ev = std::make_unique<TEvBlobStorage::TEvGet>(getQueries, 1, TInstant::Max(), - NKikimrBlobStorage::AsyncRead); - test.Runtime->WrapInActorContext(test.Edge, [&] { + NKikimrBlobStorage::AsyncRead); + test.Runtime->WrapInActorContext(test.Edge, [&] { SendToBSProxy(test.Edge, test.Info->GroupID, ev.release()); - }); - std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); - UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvGetResult); - TEvBlobStorage::TEvGetResult *getResult = handle->Get<TEvBlobStorage::TEvGetResult>(); - UNIT_ASSERT(getResult); - UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, 1); - UNIT_ASSERT_VALUES_EQUAL(getResult->Responses[0].Status, status); - UNIT_ASSERT_VALUES_EQUAL(getResult->Responses[0].Buffer, data); - }; - - void SendPatch(const TTestInfo &test, const TLogoBlobID &originalBlobId, const TLogoBlobID &patchedBlobId, ui32 mask, - const TVector<TEvBlobStorage::TEvPatch::TDiff> &diffs, NKikimrProto::EReplyStatus status) - { + }); + std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); + UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvGetResult); + TEvBlobStorage::TEvGetResult *getResult = handle->Get<TEvBlobStorage::TEvGetResult>(); + UNIT_ASSERT(getResult); + UNIT_ASSERT_VALUES_EQUAL(getResult->ResponseSz, 1); + UNIT_ASSERT_VALUES_EQUAL(getResult->Responses[0].Status, status); + UNIT_ASSERT_VALUES_EQUAL(getResult->Responses[0].Buffer, data); + }; + + void SendPatch(const TTestInfo &test, const TLogoBlobID &originalBlobId, const TLogoBlobID &patchedBlobId, ui32 mask, + const TVector<TEvBlobStorage::TEvPatch::TDiff> &diffs, NKikimrProto::EReplyStatus status) + { TArrayHolder<TEvBlobStorage::TEvPatch::TDiff> diffArr{new TEvBlobStorage::TEvPatch::TDiff[diffs.size()]}; - for (ui32 idx = 0; idx < diffs.size(); ++idx) { - diffArr[idx] = diffs[idx]; - } + for (ui32 idx = 0; idx < diffs.size(); ++idx) { + diffArr[idx] = diffs[idx]; + } std::unique_ptr<IEventBase> ev = std::make_unique<TEvBlobStorage::TEvPatch>(test.Info->GroupID, originalBlobId, patchedBlobId, - mask, std::move(diffArr), diffs.size(), TInstant::Max()); - test.Runtime->WrapInActorContext(test.Edge, [&] { + mask, std::move(diffArr), diffs.size(), TInstant::Max()); + test.Runtime->WrapInActorContext(test.Edge, [&] { SendToBSProxy(test.Edge, test.Info->GroupID, ev.release()); - }); - std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); - UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvPatchResult); - TEvBlobStorage::TEvPatchResult *patchResult = handle->Get<TEvBlobStorage::TEvPatchResult>(); - UNIT_ASSERT_EQUAL(patchResult->Status, status); - }; - - TString ApplyDiffs(const TString &source, const TVector<TEvBlobStorage::TEvPatch::TDiff> &diffs) { - TString result = TString::Uninitialized(source.size()); - Copy(source.begin(), source.end(), result.begin()); - for (auto &diff : diffs) { - Copy(diff.Buffer.begin(), diff.Buffer.end(), result.begin() + diff.Offset); - } - return result; - }; - + }); + std::unique_ptr<IEventHandle> handle = test.Runtime->WaitForEdgeActorEvent({test.Edge}); + UNIT_ASSERT_EQUAL(handle->Type, TEvBlobStorage::EvPatchResult); + TEvBlobStorage::TEvPatchResult *patchResult = handle->Get<TEvBlobStorage::TEvPatchResult>(); + UNIT_ASSERT_EQUAL(patchResult->Status, status); + }; + + TString ApplyDiffs(const TString &source, const TVector<TEvBlobStorage::TEvPatch::TDiff> &diffs) { + TString result = TString::Uninitialized(source.size()); + Copy(source.begin(), source.end(), result.begin()); + for (auto &diff : diffs) { + Copy(diff.Buffer.begin(), diff.Buffer.end(), result.begin() + diff.Offset); + } + return result; + }; + void MakePatchingTest(TString erasure) { TEnvironmentSetup env(true, GetErasureTypeByString(erasure)); TTestInfo test = InitTest(env); @@ -73,133 +73,133 @@ Y_UNIT_TEST_SUITE(BlobPatching) { TLogoBlobID originalBlobId(1, 1, 0, 0, size, 0); std::unique_ptr<IEventHandle> handle; - SendPut(test, originalBlobId, data, NKikimrProto::OK); - SendGet(test, originalBlobId, data, NKikimrProto::OK); + SendPut(test, originalBlobId, data, NKikimrProto::OK); + SendGet(test, originalBlobId, data, NKikimrProto::OK); TVector<TEvBlobStorage::TEvPatch::TDiff> diffs1(1); diffs1[0].Set("b", 0); - TString patchedData1 = ApplyDiffs(data, diffs1); + TString patchedData1 = ApplyDiffs(data, diffs1); TLogoBlobID patchedBlobId1(1, 1, 1, 0, size, 0); TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(originalBlobId, &patchedBlobId1, 0, test.Info->GroupID, test.Info->GroupID); - SendPatch(test, originalBlobId, patchedBlobId1, 0, diffs1, NKikimrProto::OK); - SendGet(test, patchedBlobId1, patchedData1, NKikimrProto::OK); + SendPatch(test, originalBlobId, patchedBlobId1, 0, diffs1, NKikimrProto::OK); + SendGet(test, patchedBlobId1, patchedData1, NKikimrProto::OK); TVector<TEvBlobStorage::TEvPatch::TDiff> diffs2(2); diffs2[0].Set("b", 0); diffs2[1].Set("b", 99); - TString patchedData2 = ApplyDiffs(data, diffs2); + TString patchedData2 = ApplyDiffs(data, diffs2); TLogoBlobID patchedBlobId2(1, 1, 2, 0, size, 0); TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(originalBlobId, &patchedBlobId2, 0, test.Info->GroupID, test.Info->GroupID); - SendPatch(test, originalBlobId, patchedBlobId2, 0, diffs2, NKikimrProto::OK); - SendGet(test, patchedBlobId2, patchedData2, NKikimrProto::OK); + SendPatch(test, originalBlobId, patchedBlobId2, 0, diffs2, NKikimrProto::OK); + SendGet(test, patchedBlobId2, patchedData2, NKikimrProto::OK); TLogoBlobID patchedBlobId3(1, 1, 3, 0, size, 0); TLogoBlobID truePatchedBlobId3(1, 1, 3, 0, size, 0); TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(originalBlobId, &truePatchedBlobId3, TLogoBlobID::MaxCookie, test.Info->GroupID, test.Info->GroupID); UNIT_ASSERT(patchedBlobId3 != truePatchedBlobId3); - NKikimrProto::EReplyStatus statusWhenNotMatchingCookie = (erasure == "block-4-2" ? NKikimrProto::ERROR : NKikimrProto::OK); - SendPatch(test, originalBlobId, patchedBlobId3, TLogoBlobID::MaxCookie, diffs2, statusWhenNotMatchingCookie); - SendPatch(test, originalBlobId, truePatchedBlobId3, TLogoBlobID::MaxCookie, diffs2, NKikimrProto::OK); - SendGet(test, truePatchedBlobId3, patchedData2, NKikimrProto::OK); + NKikimrProto::EReplyStatus statusWhenNotMatchingCookie = (erasure == "block-4-2" ? NKikimrProto::ERROR : NKikimrProto::OK); + SendPatch(test, originalBlobId, patchedBlobId3, TLogoBlobID::MaxCookie, diffs2, statusWhenNotMatchingCookie); + SendPatch(test, originalBlobId, truePatchedBlobId3, TLogoBlobID::MaxCookie, diffs2, NKikimrProto::OK); + SendGet(test, truePatchedBlobId3, patchedData2, NKikimrProto::OK); } - void MakeStressPatchingTest(TString erasure) { + void MakeStressPatchingTest(TString erasure) { TEnvironmentSetup env(true, GetErasureTypeByString(erasure)); TTestInfo test = InitTest(env); - - constexpr ui32 size = 100; + + constexpr ui32 size = 100; TString data(size, 'a'); - TLogoBlobID originalBlobId(1, 1, 0, 0, size, 0); - std::unique_ptr<IEventHandle> handle; - - SendPut(test, originalBlobId, data, NKikimrProto::OK); - SendGet(test, originalBlobId, data, NKikimrProto::OK); - - TVector<TEvBlobStorage::TEvPatch::TDiff> diffs(50); - for (ui32 idx = 0; idx < diffs.size(); ++idx) { - diffs[idx].Set("b", idx * 2); - } - TString patchedData = ApplyDiffs(data, diffs); - - constexpr ui32 patchCount = 1'000; - for (ui32 patchIdx = 0; patchIdx < patchCount; ++patchIdx) { - TLogoBlobID patchedBlobId(1, 1, patchIdx + 1, 0, size, 0); - TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(originalBlobId, &patchedBlobId, TLogoBlobID::MaxCookie, + TLogoBlobID originalBlobId(1, 1, 0, 0, size, 0); + std::unique_ptr<IEventHandle> handle; + + SendPut(test, originalBlobId, data, NKikimrProto::OK); + SendGet(test, originalBlobId, data, NKikimrProto::OK); + + TVector<TEvBlobStorage::TEvPatch::TDiff> diffs(50); + for (ui32 idx = 0; idx < diffs.size(); ++idx) { + diffs[idx].Set("b", idx * 2); + } + TString patchedData = ApplyDiffs(data, diffs); + + constexpr ui32 patchCount = 1'000; + for (ui32 patchIdx = 0; patchIdx < patchCount; ++patchIdx) { + TLogoBlobID patchedBlobId(1, 1, patchIdx + 1, 0, size, 0); + TEvBlobStorage::TEvPatch::GetBlobIdWithSamePlacement(originalBlobId, &patchedBlobId, TLogoBlobID::MaxCookie, test.Info->GroupID, test.Info->GroupID); - SendPatch(test, originalBlobId, patchedBlobId, 0, diffs, NKikimrProto::OK); - SendGet(test, patchedBlobId, patchedData, NKikimrProto::OK); - } - } - - - Y_UNIT_TEST(Mirror3of4) { + SendPatch(test, originalBlobId, patchedBlobId, 0, diffs, NKikimrProto::OK); + SendGet(test, patchedBlobId, patchedData, NKikimrProto::OK); + } + } + + + Y_UNIT_TEST(Mirror3of4) { MakePatchingTest("mirror-3of4"); } - Y_UNIT_TEST(Mirror3dc) { - MakePatchingTest("mirror-3-dc"); - } - - Y_UNIT_TEST(Mirror3) { + Y_UNIT_TEST(Mirror3dc) { + MakePatchingTest("mirror-3-dc"); + } + + Y_UNIT_TEST(Mirror3) { MakePatchingTest("mirror-3"); } - Y_UNIT_TEST(Block42) { + Y_UNIT_TEST(Block42) { MakePatchingTest("block-4-2"); } - Y_UNIT_TEST(None) { + Y_UNIT_TEST(None) { MakePatchingTest("none"); } - - - Y_UNIT_TEST(StressMirror3of4) { - MakeStressPatchingTest("mirror-3of4"); - } - - Y_UNIT_TEST(StressMirror3dc) { - MakeStressPatchingTest("mirror-3-dc"); - } - - Y_UNIT_TEST(StressMirror3) { - MakeStressPatchingTest("mirror-3"); - } - - Y_UNIT_TEST(StressBlock42) { - MakeStressPatchingTest("block-4-2"); - } - - Y_UNIT_TEST(StressNone) { - MakeStressPatchingTest("none"); - } - - Y_UNIT_TEST(DiffsWithIncorectPatchedBlobPartId) { - TEnvironmentSetup env(true); - auto& runtime = env.Runtime; - env.CreateBoxAndPool(); - env.CommenceReplication(); - auto groups = env.GetGroups(); - auto info = env.GetGroupInfo(groups[0]); - constexpr ui32 size = 100; - TLogoBlobID originalBlobId(1, 1, 0, 0, size, 0); - - env.WithQueueId(info->GetVDiskId(0), NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob, [&](TActorId queueId) { - TActorId edge = runtime->AllocateEdgeActor(queueId.NodeId()); - constexpr ui32 diffCount = 100; - for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { - TLogoBlobID patchedBlobId(1, 1, diffIdx + 1, 0, size, 0); + + + Y_UNIT_TEST(StressMirror3of4) { + MakeStressPatchingTest("mirror-3of4"); + } + + Y_UNIT_TEST(StressMirror3dc) { + MakeStressPatchingTest("mirror-3-dc"); + } + + Y_UNIT_TEST(StressMirror3) { + MakeStressPatchingTest("mirror-3"); + } + + Y_UNIT_TEST(StressBlock42) { + MakeStressPatchingTest("block-4-2"); + } + + Y_UNIT_TEST(StressNone) { + MakeStressPatchingTest("none"); + } + + Y_UNIT_TEST(DiffsWithIncorectPatchedBlobPartId) { + TEnvironmentSetup env(true); + auto& runtime = env.Runtime; + env.CreateBoxAndPool(); + env.CommenceReplication(); + auto groups = env.GetGroups(); + auto info = env.GetGroupInfo(groups[0]); + constexpr ui32 size = 100; + TLogoBlobID originalBlobId(1, 1, 0, 0, size, 0); + + env.WithQueueId(info->GetVDiskId(0), NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob, [&](TActorId queueId) { + TActorId edge = runtime->AllocateEdgeActor(queueId.NodeId()); + constexpr ui32 diffCount = 100; + for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { + TLogoBlobID patchedBlobId(1, 1, diffIdx + 1, 0, size, 0); std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> ev = std::make_unique<TEvBlobStorage::TEvVPatchDiff>( - originalBlobId, patchedBlobId, info->GetVDiskId(0), 0, TInstant::Max(), 0); + originalBlobId, patchedBlobId, info->GetVDiskId(0), 0, TInstant::Max(), 0); runtime->Send(new IEventHandle(queueId, edge, ev.release()), queueId.NodeId()); - } - for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { - auto r = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvVPatchResult>(edge, false); - UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetStatus(), NKikimrProto::ERROR); - } - }); - } + } + for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { + auto r = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvVPatchResult>(edge, false); + UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetStatus(), NKikimrProto::ERROR); + } + }); + } } diff --git a/ydb/core/blobstorage/ut_blobstorage/scrub.cpp b/ydb/core/blobstorage/ut_blobstorage/scrub.cpp index 8a355a5bd34..334781cb530 100644 --- a/ydb/core/blobstorage/ut_blobstorage/scrub.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/scrub.cpp @@ -161,9 +161,9 @@ Y_UNIT_TEST_SUITE(BlobScrubbing) { void ScrubTest(TBlobStorageGroupType erasure) { SetRandomSeed(1); - TEnvironmentSetup env(false, erasure); + TEnvironmentSetup env(false, erasure); auto& runtime = env.Runtime; - env.CreateBoxAndPool(); + env.CreateBoxAndPool(); env.Sim(TDuration::Minutes(1)); auto groups = env.GetGroups(); auto info = env.GetGroupInfo(groups[0]); diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp index b73da3df8db..e269040cb16 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp @@ -185,26 +185,26 @@ IActor *CreateRangeGet(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstanc /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TManyPuts : public TActorBootstrapped<TManyPuts> { - struct TPut { - ui64 Step; - TString Data; - }; - + struct TPut { + ui64 Step; + TString Data; + }; + TConfiguration *Conf; TActorId NotifyID; const TAllVDisks::TVDiskInstance VDiskInfo; - + std::shared_ptr<TVector<TMsgPackInfo>> MsgPacks; - ui32 MsgNum; - + ui32 MsgNum; + const ui64 TabletId; const ui32 Channel; const ui32 Gen; std::shared_ptr<IPutHandleClassGenerator> HandleClassGen; std::shared_ptr<TSet<ui32>> BadSteps; const TDuration RequestTimeout; - TVector<TPut> Puts; - ui32 PutIdx; + TVector<TPut> Puts; + ui32 PutIdx; TActorId QueueActorId; bool Started = false; // how many deadline statuses we got @@ -243,30 +243,30 @@ class TManyPuts : public TActorBootstrapped<TManyPuts> { Die(ctx); } - bool IsEnd() { - return PutIdx == MsgNum; - } - + bool IsEnd() { + return PutIdx == MsgNum; + } + void SendPut(const TActorContext &ctx) { // put logo blob - while (!IsEnd()) { - if (PutIdx % 100 == 0) - LOG_NOTICE(ctx, NActorsServices::TEST, "PUT PutIdx=%u", PutIdx); - - const TPut &put = Puts[PutIdx]; + while (!IsEnd()) { + if (PutIdx % 100 == 0) + LOG_NOTICE(ctx, NActorsServices::TEST, "PUT PutIdx=%u", PutIdx); - TLogoBlobID logoBlobID(TabletId, Gen, put.Step, Channel, put.Data.size(), 0, 1); + const TPut &put = Puts[PutIdx]; + + TLogoBlobID logoBlobID(TabletId, Gen, put.Step, Channel, put.Data.size(), 0, 1); TVDiskIdShort mainVDiskId = TIngress::GetMainReplica(&Conf->GroupInfo->GetTopology(), logoBlobID); if (mainVDiskId == VDiskInfo.VDiskID) { const bool noTimeout = RequestTimeout == TDuration::Seconds(0); const TInstant deadline = noTimeout ? TInstant::Max() : TInstant::Now() + RequestTimeout; ctx.Send(QueueActorId, - new TEvBlobStorage::TEvVPut(logoBlobID, put.Data, VDiskInfo.VDiskID, false, + new TEvBlobStorage::TEvVPut(logoBlobID, put.Data, VDiskInfo.VDiskID, false, nullptr, deadline, HandleClassGen->GetHandleClass())); return; } else { - BadSteps->insert(put.Step); - PutIdx++; + BadSteps->insert(put.Step); + PutIdx++; } } @@ -286,8 +286,8 @@ class TManyPuts : public TActorBootstrapped<TManyPuts> { } } - PutIdx++; - if (IsEnd()) { + PutIdx++; + if (IsEnd()) { Finish(ctx); } else { SendPut(ctx); @@ -299,23 +299,23 @@ class TManyPuts : public TActorBootstrapped<TManyPuts> { HFunc(TEvProxyQueueState, Handle); ) - void Init() { - MsgNum = 0; - for (auto & el: *MsgPacks) { - MsgNum += el.Count; - } - Puts.reserve(MsgNum); - ui64 msgIdx = 0; - for (ui32 packIdx = 0; packIdx < MsgPacks->size(); ++packIdx) { - for (ui32 i = 0; i < MsgPacks->at(packIdx).Count; ++i) { - Puts.push_back({msgIdx, MsgPacks->at(packIdx).MsgData}); - msgIdx++; - } - } - Shuffle(Puts.begin(), Puts.end()); - Y_VERIFY(Puts.size() == MsgNum); - } - + void Init() { + MsgNum = 0; + for (auto & el: *MsgPacks) { + MsgNum += el.Count; + } + Puts.reserve(MsgNum); + ui64 msgIdx = 0; + for (ui32 packIdx = 0; packIdx < MsgPacks->size(); ++packIdx) { + for (ui32 i = 0; i < MsgPacks->at(packIdx).Count; ++i) { + Puts.push_back({msgIdx, MsgPacks->at(packIdx).MsgData}); + msgIdx++; + } + } + Shuffle(Puts.begin(), Puts.end()); + Y_VERIFY(Puts.size() == MsgNum); + } + public: TManyPuts(TConfiguration *conf, const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, ui32 msgDataSize, ui32 msgNum, ui64 tabletId, ui32 channel, ui32 gen, @@ -325,36 +325,36 @@ public: , Conf(conf) , NotifyID(notifyID) , VDiskInfo(vdiskInfo) - , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgDataSize, msgNum)}) + , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgDataSize, msgNum)}) , TabletId(tabletId) , Channel(channel) , Gen(gen) , HandleClassGen(cls) , BadSteps(badSteps) , RequestTimeout(requestTimeout) - , PutIdx(0) + , PutIdx(0) { - Init(); - } + Init(); + } TManyPuts(TConfiguration *conf, const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, ui64 tabletId, ui32 channel, ui32 gen, std::shared_ptr<IPutHandleClassGenerator> cls, std::shared_ptr<TSet<ui32>> badSteps, - TDuration requestTimeout) - : TActorBootstrapped<TManyPuts>() - , Conf(conf) - , NotifyID(notifyID) - , VDiskInfo(vdiskInfo) - , MsgPacks(msgPacks) - , TabletId(tabletId) - , Channel(channel) - , Gen(gen) - , HandleClassGen(cls) - , BadSteps(badSteps) - , RequestTimeout(requestTimeout) - , PutIdx(0) - { - Init(); + TDuration requestTimeout) + : TActorBootstrapped<TManyPuts>() + , Conf(conf) + , NotifyID(notifyID) + , VDiskInfo(vdiskInfo) + , MsgPacks(msgPacks) + , TabletId(tabletId) + , Channel(channel) + , Gen(gen) + , HandleClassGen(cls) + , BadSteps(badSteps) + , RequestTimeout(requestTimeout) + , PutIdx(0) + { + Init(); } }; @@ -369,10 +369,10 @@ IActor *CreateManyPuts(TConfiguration *conf, const TActorId ¬ifyID, const TAl IActor *CreateManyPuts(TConfiguration *conf, const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, ui64 tabletId, ui32 channel, ui32 gen, std::shared_ptr<IPutHandleClassGenerator> cls, std::shared_ptr<TSet<ui32>> badSteps, - TDuration requestTimeout) { - return new TManyPuts(conf, notifyID, vdiskInfo, msgPacks, tabletId, channel, gen, cls, badSteps, requestTimeout); -} - + TDuration requestTimeout) { + return new TManyPuts(conf, notifyID, vdiskInfo, msgPacks, tabletId, channel, gen, cls, badSteps, requestTimeout); +} + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> { TConfiguration *Conf; @@ -650,152 +650,152 @@ IActor *CreateManyGets(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstanc } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class TGet : public TActorBootstrapped<TGet> { +class TGet : public TActorBootstrapped<TGet> { TActorId NotifyID; - const TAllVDisks::TVDiskInstance VDiskInfo; - ui32 MsgNum; + const TAllVDisks::TVDiskInstance VDiskInfo; + ui32 MsgNum; std::shared_ptr<TVector<TMsgPackInfo>> MsgPacks; - const ui64 TabletId; - const ui32 Channel; - const ui32 Gen; - const ui64 Shift; - TVector<TString> Data; - - bool WithErrorResponse; - - friend class TActorBootstrapped<TGet>; - - void Bootstrap(const TActorContext &ctx) { - Become(&TThis::StateReadFunc); - - // get logo blob - SendGet(ctx); - } - - void SendGet(const TActorContext &ctx) { - // get logo blob - auto req = TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(VDiskInfo.VDiskID, - TInstant::Max(), - NKikimrBlobStorage::EGetHandleClass::AsyncRead, - TEvBlobStorage::TEvVGet::EFlags::None, - {}, - {}); - for (ui64 i = 0; i < MsgNum; ++i) { - const TString data = Data[i]; - TLogoBlobID logoBlobID(TabletId, Gen, i, Channel, data.size(), 0, 1); - req->AddExtremeQuery(logoBlobID, Shift, data.size() - Shift, &i); - } + const ui64 TabletId; + const ui32 Channel; + const ui32 Gen; + const ui64 Shift; + TVector<TString> Data; + + bool WithErrorResponse; + + friend class TActorBootstrapped<TGet>; + + void Bootstrap(const TActorContext &ctx) { + Become(&TThis::StateReadFunc); + + // get logo blob + SendGet(ctx); + } + + void SendGet(const TActorContext &ctx) { + // get logo blob + auto req = TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(VDiskInfo.VDiskID, + TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::AsyncRead, + TEvBlobStorage::TEvVGet::EFlags::None, + {}, + {}); + for (ui64 i = 0; i < MsgNum; ++i) { + const TString data = Data[i]; + TLogoBlobID logoBlobID(TabletId, Gen, i, Channel, data.size(), 0, 1); + req->AddExtremeQuery(logoBlobID, Shift, data.size() - Shift, &i); + } ctx.Send(VDiskInfo.ActorID, req.release()); - } - - void Check(const TActorContext &ctx, const NKikimrBlobStorage::TEvVGetResult &rec) { - Y_UNUSED(ctx); - ui32 size = rec.GetResult().size(); - Y_VERIFY(size == MsgNum, "size=%d", size); - for (ui64 i = 0; i < MsgNum; ++i) { - const NKikimrBlobStorage::TQueryResult &q = rec.GetResult(i); - const TString &data = Data[i]; - if (q.GetBuffer() != data.substr(Shift)) { - fprintf(stderr, "Original: %s\n", data.data()); - fprintf(stderr, "Received: %s\n", q.GetBuffer().data()); - Y_VERIFY(q.GetBuffer() == data.substr(Shift)); - } - } - auto serial = rec.SerializeAsString(); - if (serial.size() > MaxProtobufSize) { - Y_FAIL(); - } - } - - void Handle(TEvBlobStorage::TEvVGetResult::TPtr &ev, const TActorContext &ctx) { - NKikimrProto::EReplyStatus status = ev->Get()->Record.GetStatus(); - switch (status) { - case NKikimrProto::OK: - if (WithErrorResponse) { - Y_FAIL("Expected ERROR status but given OK"); - } - Check(ctx, ev->Get()->Record); - break; - - case NKikimrProto::ERROR: - if (!WithErrorResponse) { - Y_FAIL("Expected OK status but given ERROR"); - } - break; - - default: - LOG_NOTICE(ctx, NActorsServices::TEST, "ERROR"); - fprintf(stderr, "ERROR\n"); - } - - ctx.Send(NotifyID, new TEvents::TEvCompleted()); - Die(ctx); - } - - STRICT_STFUNC(StateReadFunc, - HFunc(TEvBlobStorage::TEvVGetResult, Handle); - IgnoreFunc(TEvBlobStorage::TEvVWindowChange); - ) - - void Init() { - MsgNum = 0; - for (auto &el: *MsgPacks) { - MsgNum += el.Count; - } - Data.reserve(MsgNum); - for (ui32 packIdx = 0; packIdx < MsgPacks->size(); ++packIdx) { - for (ui32 i = 0; i < MsgPacks->at(packIdx).Count; ++i) { - Data.emplace_back(MsgPacks->at(packIdx).MsgData); - } - } - Y_VERIFY(Data.size() == MsgNum); - } - -public: + } + + void Check(const TActorContext &ctx, const NKikimrBlobStorage::TEvVGetResult &rec) { + Y_UNUSED(ctx); + ui32 size = rec.GetResult().size(); + Y_VERIFY(size == MsgNum, "size=%d", size); + for (ui64 i = 0; i < MsgNum; ++i) { + const NKikimrBlobStorage::TQueryResult &q = rec.GetResult(i); + const TString &data = Data[i]; + if (q.GetBuffer() != data.substr(Shift)) { + fprintf(stderr, "Original: %s\n", data.data()); + fprintf(stderr, "Received: %s\n", q.GetBuffer().data()); + Y_VERIFY(q.GetBuffer() == data.substr(Shift)); + } + } + auto serial = rec.SerializeAsString(); + if (serial.size() > MaxProtobufSize) { + Y_FAIL(); + } + } + + void Handle(TEvBlobStorage::TEvVGetResult::TPtr &ev, const TActorContext &ctx) { + NKikimrProto::EReplyStatus status = ev->Get()->Record.GetStatus(); + switch (status) { + case NKikimrProto::OK: + if (WithErrorResponse) { + Y_FAIL("Expected ERROR status but given OK"); + } + Check(ctx, ev->Get()->Record); + break; + + case NKikimrProto::ERROR: + if (!WithErrorResponse) { + Y_FAIL("Expected OK status but given ERROR"); + } + break; + + default: + LOG_NOTICE(ctx, NActorsServices::TEST, "ERROR"); + fprintf(stderr, "ERROR\n"); + } + + ctx.Send(NotifyID, new TEvents::TEvCompleted()); + Die(ctx); + } + + STRICT_STFUNC(StateReadFunc, + HFunc(TEvBlobStorage::TEvVGetResult, Handle); + IgnoreFunc(TEvBlobStorage::TEvVWindowChange); + ) + + void Init() { + MsgNum = 0; + for (auto &el: *MsgPacks) { + MsgNum += el.Count; + } + Data.reserve(MsgNum); + for (ui32 packIdx = 0; packIdx < MsgPacks->size(); ++packIdx) { + for (ui32 i = 0; i < MsgPacks->at(packIdx).Count; ++i) { + Data.emplace_back(MsgPacks->at(packIdx).MsgData); + } + } + Y_VERIFY(Data.size() == MsgNum); + } + +public: TGet(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, ui32 msgDataSize, ui32 msgNum, - ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) - : TActorBootstrapped<TGet>() - , NotifyID(notifyID) - , VDiskInfo(vdiskInfo) - , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgDataSize, msgNum)}) - , TabletId(tabletId) - , Channel(channel) - , Gen(gen) - , Shift(shift) - , WithErrorResponse(withErrorResponse) - { - Init(); - } - + ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) + : TActorBootstrapped<TGet>() + , NotifyID(notifyID) + , VDiskInfo(vdiskInfo) + , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgDataSize, msgNum)}) + , TabletId(tabletId) + , Channel(channel) + , Gen(gen) + , Shift(shift) + , WithErrorResponse(withErrorResponse) + { + Init(); + } + TGet(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, - ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) - : TActorBootstrapped<TGet>() - , NotifyID(notifyID) - , VDiskInfo(vdiskInfo) - , MsgPacks(msgPacks) - , TabletId(tabletId) - , Channel(channel) - , Gen(gen) - , Shift(shift) - , WithErrorResponse(withErrorResponse) - { - Init(); - } -}; - + ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) + : TActorBootstrapped<TGet>() + , NotifyID(notifyID) + , VDiskInfo(vdiskInfo) + , MsgPacks(msgPacks) + , TabletId(tabletId) + , Channel(channel) + , Gen(gen) + , Shift(shift) + , WithErrorResponse(withErrorResponse) + { + Init(); + } +}; + IActor *CreateGet(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, ui32 msgDataSize, - ui32 msgNum, ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) { - return new TGet(notifyID, vdiskInfo, msgDataSize, msgNum, tabletId, channel, gen, shift, withErrorResponse); -} - + ui32 msgNum, ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, bool withErrorResponse) { + return new TGet(notifyID, vdiskInfo, msgDataSize, msgNum, tabletId, channel, gen, shift, withErrorResponse); +} + IActor *CreateGet(const TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, - bool withErrorResponse) { - return new TGet(notifyID, vdiskInfo, msgPacks, tabletId, channel, gen, shift, withErrorResponse); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bool withErrorResponse) { + return new TGet(notifyID, vdiskInfo, msgPacks, tabletId, channel, gen, shift, withErrorResponse); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TPutGC : public TActorBootstrapped<TPutGC> { const NActors::TActorId NotifyID; const TAllVDisks::TVDiskInstance VDiskInfo; diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.h b/ydb/core/blobstorage/ut_vdisk/lib/helpers.h index ed8792b838d..0c5e20a6438 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.h +++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.h @@ -80,27 +80,27 @@ NActors::IActor *CreateManyPuts(TConfiguration *conf, const NActors::TActorId &n std::shared_ptr<IPutHandleClassGenerator> cls, std::shared_ptr<TSet<ui32>> badSteps, TDuration requestTimeout); -struct TMsgPackInfo { - ui32 Count; - TString MsgData; - - TMsgPackInfo(ui32 dataSize, ui32 count) - : Count(count) - { - MsgData.reserve(dataSize); - for (ui32 i = 0; i < dataSize; i++) { - MsgData.append('a' + i % 26); - } - } -}; - - +struct TMsgPackInfo { + ui32 Count; + TString MsgData; + + TMsgPackInfo(ui32 dataSize, ui32 count) + : Count(count) + { + MsgData.reserve(dataSize); + for (ui32 i = 0; i < dataSize; i++) { + MsgData.append('a' + i % 26); + } + } +}; + + NActors::IActor *CreateManyPuts(TConfiguration *conf, const NActors::TActorId ¬ifyID, - const TAllVDisks::TVDiskInstance &vdiskInfo, + const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, ui64 tabletId, ui32 channel, ui32 gen, std::shared_ptr<IPutHandleClassGenerator> cls, std::shared_ptr<TSet<ui32>> badSteps, - TDuration requestTimeout); - + TDuration requestTimeout); + NActors::IActor *CreateManyMultiPuts(TConfiguration *conf, const NActors::TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, ui32 msgDataSize, ui32 msgNum, ui32 batchSize, @@ -113,13 +113,13 @@ NActors::IActor *CreateManyGets(const NActors::TActorId ¬ifyID, const TAllVDi std::shared_ptr<TSet<ui32>> badSteps); NActors::IActor *CreateGet(const NActors::TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, - ui32 msgDataSize, ui32 msgNum, ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, - bool withErrorResponse); - + ui32 msgDataSize, ui32 msgNum, ui64 tabletId, ui32 channel, ui32 gen, ui64 shift, + bool withErrorResponse); + NActors::IActor *CreateGet(const NActors::TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, ui64 tabletId, ui32 channel, ui32 gen, - ui64 shift, bool withErrorResponse); - + ui64 shift, bool withErrorResponse); + NActors::IActor *CreatePutGC(const NActors::TActorId ¬ifyID, const TAllVDisks::TVDiskInstance &vdiskInfo, ui64 tabletID, ui32 recGen, ui32 recGenCounter, ui32 channel, bool collect, ui32 collectGen, ui32 collectStep, TAutoPtr<TVector<NKikimr::TLogoBlobID>> keep, diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_many.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_many.cpp index d542dfc32a9..8b4140726b5 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/test_many.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/test_many.cpp @@ -7,69 +7,69 @@ using namespace NKikimr; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class TManyPutOneGetActor : public TSyncTestBase { -protected: - const bool WaitForCompaction; +class TManyPutOneGetActor : public TSyncTestBase { +protected: + const bool WaitForCompaction; std::shared_ptr<TVector<TMsgPackInfo>> MsgPacks; - const ui64 TabletId; - const ui64 Shift; + const ui64 TabletId; + const ui64 Shift; std::shared_ptr<IPutHandleClassGenerator> HandleClassGen; std::shared_ptr<TSet<ui32>> BadSteps; - const bool WithErrorResponse; - - virtual void Scenario(const TActorContext &ctx) { - // load data - SyncRunner->Run(ctx, CreateManyPuts(Conf, SyncRunner->NotifyID(), Conf->VDisks->Get(0), MsgPacks, - TabletId, 0, 1, HandleClassGen, BadSteps, TDuration::Seconds(0))); - LOG_NOTICE(ctx, NActorsServices::TEST, " Data is loaded"); - - // wait for compaction - if (WaitForCompaction) { - SyncRunner->Run(ctx, CreateWaitForCompaction(SyncRunner->NotifyID(), Conf)); - LOG_NOTICE(ctx, NActorsServices::TEST, " COMPACTION done"); - } - - // read - SyncRunner->Run(ctx, CreateGet(SyncRunner->NotifyID(), Conf->VDisks->Get(0), MsgPacks, TabletId, 0, 1, Shift, - WithErrorResponse)); - LOG_NOTICE(ctx, NActorsServices::TEST, " GET done"); - } - - -public: - TManyPutOneGetActor(TConfiguration *conf, bool waitForCompaction, ui32 msgNum, ui32 msgSize, - ui64 tabletId, ui64 shift, NKikimrBlobStorage::EPutHandleClass cls, - bool withErrorResponse) - : TSyncTestBase(conf) - , WaitForCompaction(waitForCompaction) - , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgSize, msgNum)}) - , TabletId(tabletId) - , Shift(shift) + const bool WithErrorResponse; + + virtual void Scenario(const TActorContext &ctx) { + // load data + SyncRunner->Run(ctx, CreateManyPuts(Conf, SyncRunner->NotifyID(), Conf->VDisks->Get(0), MsgPacks, + TabletId, 0, 1, HandleClassGen, BadSteps, TDuration::Seconds(0))); + LOG_NOTICE(ctx, NActorsServices::TEST, " Data is loaded"); + + // wait for compaction + if (WaitForCompaction) { + SyncRunner->Run(ctx, CreateWaitForCompaction(SyncRunner->NotifyID(), Conf)); + LOG_NOTICE(ctx, NActorsServices::TEST, " COMPACTION done"); + } + + // read + SyncRunner->Run(ctx, CreateGet(SyncRunner->NotifyID(), Conf->VDisks->Get(0), MsgPacks, TabletId, 0, 1, Shift, + WithErrorResponse)); + LOG_NOTICE(ctx, NActorsServices::TEST, " GET done"); + } + + +public: + TManyPutOneGetActor(TConfiguration *conf, bool waitForCompaction, ui32 msgNum, ui32 msgSize, + ui64 tabletId, ui64 shift, NKikimrBlobStorage::EPutHandleClass cls, + bool withErrorResponse) + : TSyncTestBase(conf) + , WaitForCompaction(waitForCompaction) + , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgSize, msgNum)}) + , TabletId(tabletId) + , Shift(shift) , HandleClassGen(std::make_shared<TPutHandleClassGenerator>(cls)) , BadSteps(std::make_shared<TSet<ui32>>()) - , WithErrorResponse(withErrorResponse) - {} - + , WithErrorResponse(withErrorResponse) + {} + TManyPutOneGetActor(TConfiguration *conf, bool waitForCompaction, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, - ui64 tabletId, ui64 shift, NKikimrBlobStorage::EPutHandleClass cls, - bool withErrorResponse) - : TSyncTestBase(conf) - , WaitForCompaction(waitForCompaction) - , MsgPacks(msgPacks) - , TabletId(tabletId) - , Shift(shift) + ui64 tabletId, ui64 shift, NKikimrBlobStorage::EPutHandleClass cls, + bool withErrorResponse) + : TSyncTestBase(conf) + , WaitForCompaction(waitForCompaction) + , MsgPacks(msgPacks) + , TabletId(tabletId) + , Shift(shift) , HandleClassGen(std::make_shared<TPutHandleClassGenerator>(cls)) , BadSteps(std::make_shared<TSet<ui32>>()) - , WithErrorResponse(withErrorResponse) - {} -}; - -void TManyPutOneGet::operator ()(TConfiguration *conf) { - conf->ActorSystem1->Register(new TManyPutOneGetActor(conf, WaitForCompaction, MsgPacks, TabletId, Shift, - HandleClass, WithErrorResponse)); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + , WithErrorResponse(withErrorResponse) + {} +}; + +void TManyPutOneGet::operator ()(TConfiguration *conf) { + conf->ActorSystem1->Register(new TManyPutOneGetActor(conf, WaitForCompaction, MsgPacks, TabletId, Shift, + HandleClass, WithErrorResponse)); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TManyPutGetActor : public TSyncTestBase { protected: const bool WaitForCompaction; diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_many.h b/ydb/core/blobstorage/ut_vdisk/lib/test_many.h index acfe4549df2..20539ae2aef 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/test_many.h +++ b/ydb/core/blobstorage/ut_vdisk/lib/test_many.h @@ -2,42 +2,42 @@ #include "defs.h" #include "prepare.h" -#include "helpers.h" +#include "helpers.h" /////////////////////////////////////////////////////////////////////////// -struct TManyPutOneGet { - const bool WaitForCompaction; +struct TManyPutOneGet { + const bool WaitForCompaction; std::shared_ptr<TVector<TMsgPackInfo>> MsgPacks; - const NKikimrBlobStorage::EPutHandleClass HandleClass; - const ui64 TabletId; - const ui64 Shift; - const bool WithErrorResponse; - - TManyPutOneGet(bool waitForCompaction, ui32 msgNum, ui32 msgSize, NKikimrBlobStorage::EPutHandleClass cls, - ui64 tabletId = 0, ui64 shift = 0, bool withErrorResponse = false) - : WaitForCompaction(waitForCompaction) - , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgSize, msgNum)}) - , HandleClass(cls) - , TabletId(tabletId) - , Shift(shift) - , WithErrorResponse(withErrorResponse) - {} - + const NKikimrBlobStorage::EPutHandleClass HandleClass; + const ui64 TabletId; + const ui64 Shift; + const bool WithErrorResponse; + + TManyPutOneGet(bool waitForCompaction, ui32 msgNum, ui32 msgSize, NKikimrBlobStorage::EPutHandleClass cls, + ui64 tabletId = 0, ui64 shift = 0, bool withErrorResponse = false) + : WaitForCompaction(waitForCompaction) + , MsgPacks(new TVector<TMsgPackInfo>{TMsgPackInfo(msgSize, msgNum)}) + , HandleClass(cls) + , TabletId(tabletId) + , Shift(shift) + , WithErrorResponse(withErrorResponse) + {} + TManyPutOneGet(bool waitForCompaction, std::shared_ptr<TVector<TMsgPackInfo>> msgPacks, - NKikimrBlobStorage::EPutHandleClass cls, ui64 tabletId = 0, ui64 shift = 0, - bool withErrorResponse = false) - : WaitForCompaction(waitForCompaction) - , MsgPacks(msgPacks) - , HandleClass(cls) - , TabletId(tabletId) - , Shift(shift) - , WithErrorResponse(withErrorResponse) - {} - - void operator ()(TConfiguration *conf); -}; - -/////////////////////////////////////////////////////////////////////////// + NKikimrBlobStorage::EPutHandleClass cls, ui64 tabletId = 0, ui64 shift = 0, + bool withErrorResponse = false) + : WaitForCompaction(waitForCompaction) + , MsgPacks(msgPacks) + , HandleClass(cls) + , TabletId(tabletId) + , Shift(shift) + , WithErrorResponse(withErrorResponse) + {} + + void operator ()(TConfiguration *conf); +}; + +/////////////////////////////////////////////////////////////////////////// struct TManyPutGet { const bool WaitForCompaction; const ui32 MsgNum; diff --git a/ydb/core/blobstorage/ut_vdisk/vdisk_test.cpp b/ydb/core/blobstorage/ut_vdisk/vdisk_test.cpp index 694828c9f8c..bd20ab3c70d 100644 --- a/ydb/core/blobstorage/ut_vdisk/vdisk_test.cpp +++ b/ydb/core/blobstorage/ut_vdisk/vdisk_test.cpp @@ -257,22 +257,22 @@ Y_UNIT_TEST_SUITE(TBsVDiskRangeHuge) { #endif Y_UNIT_TEST_SUITE(TBsVDiskManyPutGetCheckSize) { - Y_UNIT_TEST(ManyPutGetCheckSize) { + Y_UNIT_TEST(ManyPutGetCheckSize) { std::shared_ptr<TVector<TMsgPackInfo>> msgPacks(std::unique_ptr<TVector<TMsgPackInfo>>(new TVector<TMsgPackInfo>{ - TMsgPackInfo(100'000, 672), - TMsgPackInfo(17'026, 1) + TMsgPackInfo(100'000, 672), + TMsgPackInfo(17'026, 1) })); - TManyPutOneGet testOk(false, msgPacks, UNK, 0, 257, false); - TestRun<TManyPutOneGet, TFastVDiskSetupHndOff>(&testOk, TDuration::Minutes(100), DefChunkSize, DefDiskSize, - 1, 1, NKikimr::TErasureType::ErasureNone); + TManyPutOneGet testOk(false, msgPacks, UNK, 0, 257, false); + TestRun<TManyPutOneGet, TFastVDiskSetupHndOff>(&testOk, TDuration::Minutes(100), DefChunkSize, DefDiskSize, + 1, 1, NKikimr::TErasureType::ErasureNone); std::shared_ptr<TVector<TMsgPackInfo>> failMsgPacks(std::unique_ptr<TVector<TMsgPackInfo>>(new TVector<TMsgPackInfo>{ - TMsgPackInfo(100'000, 672), - TMsgPackInfo(17'027, 1) + TMsgPackInfo(100'000, 672), + TMsgPackInfo(17'027, 1) })); - TManyPutOneGet testError(false, failMsgPacks, UNK, 0, 257, true); - TestRun<TManyPutOneGet, TFastVDiskSetupHndOff>(&testError, TDuration::Minutes(100), DefChunkSize, DefDiskSize, - 1, 1, NKikimr::TErasureType::ErasureNone); - } + TManyPutOneGet testError(false, failMsgPacks, UNK, 0, 257, true); + TestRun<TManyPutOneGet, TFastVDiskSetupHndOff>(&testError, TDuration::Minutes(100), DefChunkSize, DefDiskSize, + 1, 1, NKikimr::TErasureType::ErasureNone); + } } Y_UNIT_TEST_SUITE(TBsVDiskManyPutGet) { //// Group diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h index ff4bb4f0d61..9f9341c4e56 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h @@ -201,7 +201,7 @@ namespace NKikimr { bool BarrierValidation; TDuration WhiteboardUpdateInterval; bool EnableVDiskCooldownTimeout; - TControlWrapper EnableVPatch = true; + TControlWrapper EnableVPatch = true; TVDiskConfig(const TBaseInfo &baseInfo); void Merge(const NKikimrBlobStorage::TVDiskConfig &update); diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp index 3acfe26ea3f..ca0969d554b 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp @@ -4,24 +4,24 @@ namespace NKikimr { TCostModel::TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, - ui64 writeBlockSize, ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType) + ui64 writeBlockSize, ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType) : SeekTimeUs(seekTimeUs) , ReadSpeedBps(readSpeedBps) , WriteSpeedBps(writeSpeedBps) , ReadBlockSize(readBlockSize) , WriteBlockSize(writeBlockSize) , MinREALHugeBlobInBytes(minREALHugeBlobInBytes) - , GType(gType) + , GType(gType) {} - TCostModel::TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType) + TCostModel::TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType) : SeekTimeUs(settings.GetSeekTimeUs()) , ReadSpeedBps(settings.GetReadSpeedBps()) , WriteSpeedBps(settings.GetWriteSpeedBps()) , ReadBlockSize(settings.GetReadBlockSize()) , WriteBlockSize(settings.GetWriteBlockSize()) , MinREALHugeBlobInBytes(settings.GetMinREALHugeBlobInBytes()) - , GType(gType) + , GType(gType) {} void TCostModel::FillInSettings(NKikimrBlobStorage::TVDiskCostSettings &settings) const { @@ -33,26 +33,26 @@ namespace NKikimr { settings.SetMinREALHugeBlobInBytes(MinREALHugeBlobInBytes); } - ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchStart &) const { - return InMemReadCost(); - } - - ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchDiff &ev) const { - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(ev.Record.GetPatchedPartBlobId()); - ui32 size = blobId.BlobSize(); - return ReadCostBySize(size) + HugeWriteCost(size); - } - - ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchXorDiff &ev) const { - const ui64 bufSize = ev.DiffSizeSum(); - return HugeWriteCost(bufSize); - } - - ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVMovedPatch &ev) const { - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(ev.Record.GetPatchedBlobId()); - return MovedPatchCostBySize(blobId.BlobSize()); - } - + ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchStart &) const { + return InMemReadCost(); + } + + ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchDiff &ev) const { + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(ev.Record.GetPatchedPartBlobId()); + ui32 size = blobId.BlobSize(); + return ReadCostBySize(size) + HugeWriteCost(size); + } + + ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPatchXorDiff &ev) const { + const ui64 bufSize = ev.DiffSizeSum(); + return HugeWriteCost(bufSize); + } + + ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVMovedPatch &ev) const { + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(ev.Record.GetPatchedBlobId()); + return MovedPatchCostBySize(blobId.BlobSize()); + } + ui64 TCostModel::GetCost(const TEvBlobStorage::TEvVPut &ev, bool *logPutInternalQueue) const { const auto &record = ev.Record; const NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass(); @@ -86,18 +86,18 @@ namespace NKikimr { return cost; } - ui64 TCostModel::MovedPatchCostBySize(ui32 blobSize) const { - ui32 partSize = GType.TErasureType::PartSize(TErasureType::CrcModeNone, blobSize); - ui32 memMultiply = 3; // in the worst case (buffer + parts + serealized events) - return memMultiply * GType.TotalPartCount() * (ReadCostBySize(partSize) + HugeWriteCost(partSize)); - } - - ui64 TCostModel::ReadCostBySize(ui64 size) const { - ui64 seekCost = (size / ReadBlockSize + 1) * SeekTimeUs; - ui64 readCost = size * ui64(1000000000) / ReadSpeedBps; - return seekCost + readCost; - } - + ui64 TCostModel::MovedPatchCostBySize(ui32 blobSize) const { + ui32 partSize = GType.TErasureType::PartSize(TErasureType::CrcModeNone, blobSize); + ui32 memMultiply = 3; // in the worst case (buffer + parts + serealized events) + return memMultiply * GType.TotalPartCount() * (ReadCostBySize(partSize) + HugeWriteCost(partSize)); + } + + ui64 TCostModel::ReadCostBySize(ui64 size) const { + ui64 seekCost = (size / ReadBlockSize + 1) * SeekTimeUs; + ui64 readCost = size * ui64(1000000000) / ReadSpeedBps; + return seekCost + readCost; + } + ui64 TCostModel::ReadCost(const TEvBlobStorage::TEvVGet &ev) const { const auto &record = ev.Record; ui64 cost = 0; @@ -123,7 +123,7 @@ namespace NKikimr { size = id.BlobSize(); } - cost += ReadCostBySize(size); + cost += ReadCostBySize(size); } Y_VERIFY_DEBUG(cost); @@ -133,14 +133,14 @@ namespace NKikimr { ui64 TCostModel::CalculateCost(const TMessageCostEssence& essence) const { ui64 cost = essence.BaseCost; for (ui64 size : essence.ReadSizes) { - cost += ReadCostBySize(size); + cost += ReadCostBySize(size); } if (essence.SmallWriteSize != -1) { cost += SmallWriteCost(essence.SmallWriteSize); } - if (essence.MovedPatchBlobSize != -1) { - cost += MovedPatchCostBySize(essence.MovedPatchBlobSize); - } + if (essence.MovedPatchBlobSize != -1) { + cost += MovedPatchCostBySize(essence.MovedPatchBlobSize); + } for (ui64 size : essence.PutBufferSizes) { NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, essence.HandleClass, size); if (handleType == NPriPut::Log) { diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h index eb16996c30f..4717a5f3d0b 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h @@ -17,10 +17,10 @@ namespace NKikimr { friend class TCostModel; ui64 BaseCost = 0; - i32 SmallWriteSize = -1; - i32 MovedPatchBlobSize = -1; - TStackVec<i32, 1> PutBufferSizes; - TVector<ui32> ReadSizes; // FIXME: optimize + i32 SmallWriteSize = -1; + i32 MovedPatchBlobSize = -1; + TStackVec<i32, 1> PutBufferSizes; + TVector<ui32> ReadSizes; // FIXME: optimize // handle class for TEvPut and TEvMultiPut only NKikimrBlobStorage::EPutHandleClass HandleClass = NKikimrBlobStorage::TabletLog; @@ -70,34 +70,34 @@ namespace NKikimr { : SmallWriteSize(ev.GetCachedByteSize()) {} - TMessageCostEssence(const TEvBlobStorage::TEvVMovedPatch& ev) - : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) - { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev.Record.GetOriginalBlobId()); - MovedPatchBlobSize = id.BlobSize(); - } - - TMessageCostEssence(const TEvBlobStorage::TEvVPatchStart&) - : BaseCost(TCostModel::InMemReadCost(EInMemType::Read)) - , HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) - { - } - - TMessageCostEssence(const TEvBlobStorage::TEvVPatchDiff& ev) - : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) - { - // it has range vget subquery for finding parts - TLogoBlobID id(LogoBlobIDFromLogoBlobID(ev.Record.GetOriginalPartBlobId())); - ReadSizes.push_back(id.BlobSize()); - PutBufferSizes.push_back(id.BlobSize()); - } - - TMessageCostEssence(const TEvBlobStorage::TEvVPatchXorDiff& ev) - : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) - { - PutBufferSizes.push_back(ev.DiffSizeSum()); - } - + TMessageCostEssence(const TEvBlobStorage::TEvVMovedPatch& ev) + : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) + { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev.Record.GetOriginalBlobId()); + MovedPatchBlobSize = id.BlobSize(); + } + + TMessageCostEssence(const TEvBlobStorage::TEvVPatchStart&) + : BaseCost(TCostModel::InMemReadCost(EInMemType::Read)) + , HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) + { + } + + TMessageCostEssence(const TEvBlobStorage::TEvVPatchDiff& ev) + : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) + { + // it has range vget subquery for finding parts + TLogoBlobID id(LogoBlobIDFromLogoBlobID(ev.Record.GetOriginalPartBlobId())); + ReadSizes.push_back(id.BlobSize()); + PutBufferSizes.push_back(id.BlobSize()); + } + + TMessageCostEssence(const TEvBlobStorage::TEvVPatchXorDiff& ev) + : HandleClass(NKikimrBlobStorage::EPutHandleClass::AsyncBlob) + { + PutBufferSizes.push_back(ev.DiffSizeSum()); + } + TMessageCostEssence(const TEvBlobStorage::TEvVPut& ev) : HandleClass(ev.Record.GetHandleClass()) { @@ -124,11 +124,11 @@ namespace NKikimr { ui64 ReadBlockSize; ui64 WriteBlockSize; ui32 MinREALHugeBlobInBytes; - TBlobStorageGroupType GType; + TBlobStorageGroupType GType; TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, ui64 writeBlockSize, - ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType); - TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType); + ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType); + TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType); /// SETTINGS void FillInSettings(NKikimrBlobStorage::TVDiskCostSettings &settings) const; @@ -164,11 +164,11 @@ namespace NKikimr { return GetCost(ev, &logPutInternalQueue); } - ui64 GetCost(const TEvBlobStorage::TEvVPatchStart &ev) const; - ui64 GetCost(const TEvBlobStorage::TEvVPatchDiff &ev) const; - ui64 GetCost(const TEvBlobStorage::TEvVPatchXorDiff &ev) const; - ui64 GetCost(const TEvBlobStorage::TEvVMovedPatch &ev) const; - + ui64 GetCost(const TEvBlobStorage::TEvVPatchStart &ev) const; + ui64 GetCost(const TEvBlobStorage::TEvVPatchDiff &ev) const; + ui64 GetCost(const TEvBlobStorage::TEvVPatchXorDiff &ev) const; + ui64 GetCost(const TEvBlobStorage::TEvVMovedPatch &ev) const; + ui64 GetCost(const TEvBlobStorage::TEvVPut &ev, bool *logPutInternalQueue) const; ui64 GetCost(const TEvBlobStorage::TEvVMultiPut &ev, bool *logPutInternalQueue) const; @@ -213,8 +213,8 @@ namespace NKikimr { return costs.at(size_t(t)); } - ui64 MovedPatchCostBySize(ui32 blobSize) const; - ui64 ReadCostBySize(ui64 size) const; + ui64 MovedPatchCostBySize(ui32 blobSize) const; + ui64 ReadCostBySize(ui64 size) const; ui64 ReadCost(const TEvBlobStorage::TEvVGet &ev) const; }; diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_events.cpp index 25a0e24320d..9d972a37bc9 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_events.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_events.cpp @@ -42,26 +42,26 @@ namespace NKikimr { } } - void TEvBlobStorage::TEvVPut::StorePayload(TRope&& buffer) { - Y_VERIFY(KIKIMR_USE_PROTOBUF_WITH_PAYLOAD); - AddPayload(std::move(buffer)); - } - - void TEvBlobStorage::TEvVMultiPut::StorePayload(NKikimrBlobStorage::TVMultiPutItem &item, const TString& buffer) { - if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { + void TEvBlobStorage::TEvVPut::StorePayload(TRope&& buffer) { + Y_VERIFY(KIKIMR_USE_PROTOBUF_WITH_PAYLOAD); + AddPayload(std::move(buffer)); + } + + void TEvBlobStorage::TEvVMultiPut::StorePayload(NKikimrBlobStorage::TVMultiPutItem &item, const TString& buffer) { + if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { AddPayload(TRope(buffer)); Y_VERIFY_DEBUG(Record.ItemsSize() == GetPayloadCount()); - } else { - item.SetBuffer(buffer); - } - } - - TRope TEvBlobStorage::TEvVMultiPut::GetItemBuffer(ui64 itemIdx) const { - auto &item = Record.GetItems(itemIdx); - if (item.HasBuffer()) { + } else { + item.SetBuffer(buffer); + } + } + + TRope TEvBlobStorage::TEvVMultiPut::GetItemBuffer(ui64 itemIdx) const { + auto &item = Record.GetItems(itemIdx); + if (item.HasBuffer()) { return TRope(item.GetBuffer()); - } else { - return GetPayload(itemIdx); - } - } + } else { + return GetPayload(itemIdx); + } + } } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h index 47fbd959c96..6173a1fe394 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_events.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_events.h @@ -38,7 +38,7 @@ namespace NKikimr { ReplJob, DSProxy, VDiskLoad, - VPatch, + VPatch, }; inline const char *EQueueClientType2String(EQueueClientType t) { @@ -47,7 +47,7 @@ namespace NKikimr { case EQueueClientType::ReplJob: return "ReplJob"; case EQueueClientType::DSProxy: return "DSProxy"; case EQueueClientType::VDiskLoad: return "VDiskLoad"; - case EQueueClientType::VPatch: return "VPatch"; + case EQueueClientType::VPatch: return "VPatch"; } Y_FAIL("unexpected EQueueClientType"); @@ -87,11 +87,11 @@ namespace NKikimr { Identifier = msgQoS.GetVDiskLoadId(); break; - case NKikimrBlobStorage::TMsgQoS::ClientIdCase::kVPatchVDiskId: - Type = EQueueClientType::VPatch; - Identifier = msgQoS.GetVPatchVDiskId(); - break; - + case NKikimrBlobStorage::TMsgQoS::ClientIdCase::kVPatchVDiskId: + Type = EQueueClientType::VPatch; + Identifier = msgQoS.GetVPatchVDiskId(); + break; + case NKikimrBlobStorage::TMsgQoS::ClientIdCase::CLIENTID_NOT_SET: Type = EQueueClientType::None; Identifier = 0; @@ -118,10 +118,10 @@ namespace NKikimr { case EQueueClientType::VDiskLoad: msgQoS->SetVDiskLoadId(Identifier); break; - - case EQueueClientType::VPatch: - msgQoS->SetVPatchVDiskId(Identifier); - break; + + case EQueueClientType::VPatch: + msgQoS->SetVPatchVDiskId(Identifier); + break; } } @@ -491,10 +491,10 @@ namespace NKikimr { } }; - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // TEvVPut - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // TEvVPut + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + struct TEvBlobStorage::TEvVPut : public TEventPB<TEvBlobStorage::TEvVPut, NKikimrBlobStorage::TEvVPut, TEvBlobStorage::EvVPut> { // In current realization it is intentionaly lost on event serialization since @@ -508,22 +508,22 @@ namespace NKikimr { const bool ignoreBlock, const ui64 *cookie, TInstant deadline, NKikimrBlobStorage::EPutHandleClass cls) { - InitWithoutBuffer(logoBlobId, vdisk, ignoreBlock, cookie, deadline, cls); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.Data(), buffer.size()); - StorePayload(buffer); - } - - TEvVPut(const TLogoBlobID &logoBlobId, TRope buffer, const TVDiskID &vdisk, - const bool ignoreBlock, const ui64 *cookie, TInstant deadline, - NKikimrBlobStorage::EPutHandleClass cls) - { - InitWithoutBuffer(logoBlobId, vdisk, ignoreBlock, cookie, deadline, cls); - StorePayload(std::move(buffer)); - } - - void InitWithoutBuffer(const TLogoBlobID &logoBlobId, const TVDiskID &vdisk, const bool ignoreBlock, - const ui64 *cookie, TInstant deadline, NKikimrBlobStorage::EPutHandleClass cls) - { + InitWithoutBuffer(logoBlobId, vdisk, ignoreBlock, cookie, deadline, cls); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.Data(), buffer.size()); + StorePayload(buffer); + } + + TEvVPut(const TLogoBlobID &logoBlobId, TRope buffer, const TVDiskID &vdisk, + const bool ignoreBlock, const ui64 *cookie, TInstant deadline, + NKikimrBlobStorage::EPutHandleClass cls) + { + InitWithoutBuffer(logoBlobId, vdisk, ignoreBlock, cookie, deadline, cls); + StorePayload(std::move(buffer)); + } + + void InitWithoutBuffer(const TLogoBlobID &logoBlobId, const TVDiskID &vdisk, const bool ignoreBlock, + const ui64 *cookie, TInstant deadline, NKikimrBlobStorage::EPutHandleClass cls) + { REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&logoBlobId, sizeof(logoBlobId)); REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&vdisk, sizeof(vdisk)); REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&ignoreBlock, sizeof(ignoreBlock)); @@ -551,14 +551,14 @@ namespace NKikimr { return Record.GetIgnoreBlock(); } - TRope GetBuffer() const { - return Record.HasBuffer() ? TRope(Record.GetBuffer()) : GetPayload(0); - } - + TRope GetBuffer() const { + return Record.HasBuffer() ? TRope(Record.GetBuffer()) : GetPayload(0); + } + void StorePayload(const TString& buffer); - void StorePayload(TRope&& buffer); - + void StorePayload(TRope&& buffer); + ui64 GetBufferBytes() const { if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { ui64 sizeBytes = 0; @@ -760,32 +760,32 @@ namespace NKikimr { struct TEvBlobStorage::TEvVMultiPut : public TEventPB<TEvBlobStorage::TEvVMultiPut, NKikimrBlobStorage::TEvVMultiPut, TEvBlobStorage::EvVMultiPut> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVMultiPut() = default; - - TEvVMultiPut(const TVDiskID &vdisk, TInstant deadline, NKikimrBlobStorage::EPutHandleClass cls, - bool ignoreBlock, const ui64 *cookie = nullptr) - { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&vdisk, sizeof(vdisk)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&ignoreBlock, sizeof(ignoreBlock)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&cookie, sizeof(cookie)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&deadline, sizeof(deadline)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&cls, sizeof(cls)); - - VDiskIDFromVDiskID(vdisk, Record.MutableVDiskID()); - Record.SetIgnoreBlock(ignoreBlock); - if (cookie) { - Record.SetCookie(*cookie); - } - if (deadline != TInstant::Max()) { - Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); - } - Record.SetHandleClass(cls); - Record.MutableMsgQoS()->SetExtQueueId(HandleClassToQueueId(cls)); - } - + { + mutable NLWTrace::TOrbit Orbit; + + TEvVMultiPut() = default; + + TEvVMultiPut(const TVDiskID &vdisk, TInstant deadline, NKikimrBlobStorage::EPutHandleClass cls, + bool ignoreBlock, const ui64 *cookie = nullptr) + { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&vdisk, sizeof(vdisk)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&ignoreBlock, sizeof(ignoreBlock)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&cookie, sizeof(cookie)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&deadline, sizeof(deadline)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&cls, sizeof(cls)); + + VDiskIDFromVDiskID(vdisk, Record.MutableVDiskID()); + Record.SetIgnoreBlock(ignoreBlock); + if (cookie) { + Record.SetCookie(*cookie); + } + if (deadline != TInstant::Max()) { + Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); + } + Record.SetHandleClass(cls); + Record.MutableMsgQoS()->SetExtQueueId(HandleClassToQueueId(cls)); + } + ui64 GetBufferBytes(ui64 idx) const { Y_VERIFY_DEBUG(idx < Record.ItemsSize()); if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { @@ -795,46 +795,46 @@ namespace NKikimr { } } - ui64 GetBufferBytes() const { - ui64 bytes = 0; - if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { + ui64 GetBufferBytes() const { + ui64 bytes = 0; + if (KIKIMR_USE_PROTOBUF_WITH_PAYLOAD) { ui32 size = GetPayloadCount(); - for (ui32 i = 0; i < size; ++i) { - bytes += GetPayload(i).GetSize(); - } - } else { - for (const auto &item : Record.GetItems()) { - bytes += item.GetBuffer().size(); - } - } - return bytes; - } - - ui64 GetSumBlobSize() const { - ui64 sum = 0; - for (auto &item : Record.GetItems()) { - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - sum += blobId.BlobSize(); - } - return sum; - } - - void StorePayload(NKikimrBlobStorage::TVMultiPutItem &item, const TString &buffer); - - - TRope GetItemBuffer(ui64 itemIdx) const; - - void AddVPut(const TLogoBlobID &logoBlobId, const TString &buffer, ui64 *cookie) { - NKikimrBlobStorage::TVMultiPutItem *item = Record.AddItems(); - LogoBlobIDFromLogoBlobID(logoBlobId, item->MutableBlobID()); - item->SetFullDataSize(logoBlobId.BlobSize()); - StorePayload(*item, buffer); - item->SetFullDataSize(logoBlobId.BlobSize()); - if (cookie) { - item->SetCookie(*cookie); - } - } - + for (ui32 i = 0; i < size; ++i) { + bytes += GetPayload(i).GetSize(); + } + } else { + for (const auto &item : Record.GetItems()) { + bytes += item.GetBuffer().size(); + } + } + return bytes; + } + + ui64 GetSumBlobSize() const { + ui64 sum = 0; + for (auto &item : Record.GetItems()) { + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + sum += blobId.BlobSize(); + } + return sum; + } + + void StorePayload(NKikimrBlobStorage::TVMultiPutItem &item, const TString &buffer); + + + TRope GetItemBuffer(ui64 itemIdx) const; + + void AddVPut(const TLogoBlobID &logoBlobId, const TString &buffer, ui64 *cookie) { + NKikimrBlobStorage::TVMultiPutItem *item = Record.AddItems(); + LogoBlobIDFromLogoBlobID(logoBlobId, item->MutableBlobID()); + item->SetFullDataSize(logoBlobId.BlobSize()); + StorePayload(*item, buffer); + item->SetFullDataSize(logoBlobId.BlobSize()); + if (cookie) { + item->SetCookie(*cookie); + } + } + bool Validate(TString& errorReason) { if (Record.ItemsSize() == 0) { errorReason = "TEvVMultiPut rejected by VDisk. It has 0 blobs to put"; @@ -853,172 +853,172 @@ namespace NKikimr { return false; } - TString ToString() const override { - TStringStream str; - str << "{EvVMultiPut"; - ui64 size = Record.ItemsSize(); - for (ui64 itemIdx = 0; itemIdx < size; ++itemIdx) { - const NKikimrBlobStorage::TVMultiPutItem &item = Record.GetItems(itemIdx); - str << " Item# {VMultiPutItem"; - TLogoBlobID id = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - str << " ID# " << id.ToString(); - str << " FullDataSize# " << item.GetFullDataSize(); - TString data = GetItemBuffer(itemIdx).ConvertToString(); - str << " DataSize# " << data.size(); - if (data.size() > 16) { - str << " Data# <too_large>"; - } else { - str << " Data# " << data; - } - if (item.HasCookie()) { - str << " Cookie# " << item.GetCookie(); - } - str << "}"; - } - - str << " VDiskId# " << VDiskIDFromVDiskID(Record.GetVDiskID()); - - if (Record.GetIgnoreBlock()) { - str << " IgnoreBlock"; - } - if (Record.HasHandleClass()) { - str << " HandleClass# " << Record.GetHandleClass(); - } - if (Record.HasMsgQoS()) { - str << " "; - TEvBlobStorage::TEvVPut::OutMsgQos(Record.GetMsgQoS(), str); - } - if (Record.HasCookie()) { - str << " Cookie# " << Record.GetCookie(); - } - str << "}"; - return str.Str(); - } - }; - + TString ToString() const override { + TStringStream str; + str << "{EvVMultiPut"; + ui64 size = Record.ItemsSize(); + for (ui64 itemIdx = 0; itemIdx < size; ++itemIdx) { + const NKikimrBlobStorage::TVMultiPutItem &item = Record.GetItems(itemIdx); + str << " Item# {VMultiPutItem"; + TLogoBlobID id = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + str << " ID# " << id.ToString(); + str << " FullDataSize# " << item.GetFullDataSize(); + TString data = GetItemBuffer(itemIdx).ConvertToString(); + str << " DataSize# " << data.size(); + if (data.size() > 16) { + str << " Data# <too_large>"; + } else { + str << " Data# " << data; + } + if (item.HasCookie()) { + str << " Cookie# " << item.GetCookie(); + } + str << "}"; + } + + str << " VDiskId# " << VDiskIDFromVDiskID(Record.GetVDiskID()); + + if (Record.GetIgnoreBlock()) { + str << " IgnoreBlock"; + } + if (Record.HasHandleClass()) { + str << " HandleClass# " << Record.GetHandleClass(); + } + if (Record.HasMsgQoS()) { + str << " "; + TEvBlobStorage::TEvVPut::OutMsgQos(Record.GetMsgQoS(), str); + } + if (Record.HasCookie()) { + str << " Cookie# " << Record.GetCookie(); + } + str << "}"; + return str.Str(); + } + }; + struct TEvBlobStorage::TEvVMultiPutResult : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVMultiPutResult, NKikimrBlobStorage::TEvVMultiPutResult, - TEvBlobStorage::EvVMultiPutResult> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVMultiPutResult() = default; - + TEvBlobStorage::EvVMultiPutResult> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVMultiPutResult() = default; + TEvVMultiPutResult(const NKikimrProto::EReplyStatus status, const TVDiskID &vdisk, const ui64 *cookie, const TInstant &now, ui32 recByteSize, NKikimrBlobStorage::TEvVMultiPut *record, const TActorIDPtr &skeletonFrontIDPtr, const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, const ui64 bufferSizeBytes, NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) - : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), - TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, recByteSize, record, skeletonFrontIDPtr) - { + : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), + TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, recByteSize, record, skeletonFrontIDPtr) + { IncrementSize(bufferSizeBytes); - Record.SetStatus(status); - VDiskIDFromVDiskID(vdisk, Record.MutableVDiskID()); - if (cookie) { - Record.SetCookie(*cookie); - } - if (record && record->HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); - } + Record.SetStatus(status); + VDiskIDFromVDiskID(vdisk, Record.MutableVDiskID()); + if (cookie) { + Record.SetCookie(*cookie); + } + if (record && record->HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); + } if (status == NKikimrProto::OK) { Record.SetIncarnationGuid(incarnationGuid); } if (errorReason && status != NKikimrProto::OK) { Record.SetErrorReason(errorReason); } - } - + } + void AddVPutResult(NKikimrProto::EReplyStatus status, const TString& errorReason, const TLogoBlobID &logoBlobId, ui64 *cookie, ui32 statusFlags = 0) - { - NKikimrBlobStorage::TVMultiPutResultItem *item = Record.AddItems(); - item->SetStatus(status); + { + NKikimrBlobStorage::TVMultiPutResultItem *item = Record.AddItems(); + item->SetStatus(status); if (errorReason && status != NKikimrProto::OK) { item->SetErrorReason(errorReason); } - LogoBlobIDFromLogoBlobID(logoBlobId, item->MutableBlobID()); - if (cookie) { - item->SetCookie(*cookie); - } - item->SetStatusFlags(statusFlags); - } - + LogoBlobIDFromLogoBlobID(logoBlobId, item->MutableBlobID()); + if (cookie) { + item->SetCookie(*cookie); + } + item->SetStatusFlags(statusFlags); + } + void MakeError(NKikimrProto::EReplyStatus status, const TString& errorReason, const NKikimrBlobStorage::TEvVMultiPut &request) { - Record.SetStatus(status); + Record.SetStatus(status); if (errorReason && status != NKikimrProto::OK) { Record.SetErrorReason(errorReason); } if (status == NKikimrProto::NOTREADY) { status = NKikimrProto::ERROR; // treat BS_QUEUE internally generated NOTREADY as ERROR } - - for (auto &item : request.GetItems()) { - Y_VERIFY(item.HasBlobID()); - TLogoBlobID logoBlobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - ui64 cookieValue = 0; - ui64 *cookie = nullptr; - if (item.HasCookie()) { - cookieValue = item.GetCookie(); - cookie = &cookieValue; - } + + for (auto &item : request.GetItems()) { + Y_VERIFY(item.HasBlobID()); + TLogoBlobID logoBlobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + ui64 cookieValue = 0; + ui64 *cookie = nullptr; + if (item.HasCookie()) { + cookieValue = item.GetCookie(); + cookie = &cookieValue; + } AddVPutResult(status, errorReason, logoBlobId, cookie); - } + } if (request.HasVDiskID()) { Record.MutableVDiskID()->CopyFrom(request.GetVDiskID()); } - if (request.HasCookie()) { - Record.SetCookie(request.GetCookie()); - } - if (request.HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); - } - } - - TString ToString() const override { - return ToString(Record); - } - - static TString ToString(const NKikimrBlobStorage::TEvVMultiPutResult &record) { - TStringStream str; - str << "{EvVMultiPutResult Status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data(); + if (request.HasCookie()) { + Record.SetCookie(request.GetCookie()); + } + if (request.HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); + } + } + + TString ToString() const override { + return ToString(Record); + } + + static TString ToString(const NKikimrBlobStorage::TEvVMultiPutResult &record) { + TStringStream str; + str << "{EvVMultiPutResult Status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data(); if (record.HasErrorReason()) { str << " ErrorReason# " << '"' << EscapeC(record.GetErrorReason()) << '"'; } - ui64 size = record.ItemsSize(); - for (ui64 itemIdx = 0; itemIdx < size; ++itemIdx) { - const NKikimrBlobStorage::TVMultiPutResultItem &item = record.GetItems(itemIdx); - str << " Item# {VMultiPutResultItem"; - str << " Status# " << NKikimrProto::EReplyStatus_Name(item.GetStatus()).data(); + ui64 size = record.ItemsSize(); + for (ui64 itemIdx = 0; itemIdx < size; ++itemIdx) { + const NKikimrBlobStorage::TVMultiPutResultItem &item = record.GetItems(itemIdx); + str << " Item# {VMultiPutResultItem"; + str << " Status# " << NKikimrProto::EReplyStatus_Name(item.GetStatus()).data(); if (item.HasErrorReason()) { str << " ErrorReason# " << '"' << EscapeC(item.GetErrorReason()) << '"'; } - TLogoBlobID id = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - str << " ID# " << id.ToString(); - if (item.HasCookie()) { - str << " Cookie# " << item.GetCookie(); - } - str << "}"; - } - if (record.HasMsgQoS()) { - str << " "; - TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); - } - if (record.HasCookie()) { - str << " Cookie# " << record.GetCookie(); - } - str << "}"; - return str.Str(); - } - }; - + TLogoBlobID id = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + str << " ID# " << id.ToString(); + if (item.HasCookie()) { + str << " Cookie# " << item.GetCookie(); + } + str << "}"; + } + if (record.HasMsgQoS()) { + str << " "; + TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); + } + if (record.HasCookie()) { + str << " Cookie# " << record.GetCookie(); + } + str << "}"; + return str.Str(); + } + }; + ////////////////////////////////////////////////////////////////////////////////////////////// // VGet ////////////////////////////////////////////////////////////////////////////////////////////// - + struct TEvBlobStorage::TEvVGet : public TEventPB<TEvBlobStorage::TEvVGet, NKikimrBlobStorage::TEvVGet, TEvBlobStorage::EvVGet> { @@ -1383,297 +1383,297 @@ namespace NKikimr { } }; - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // VPatch - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - template <typename TEv, typename TRecord, ui32 EventType> - struct TEvVSpecialPatchBase - : public TEventPB<TEv, TRecord, EventType> { - mutable NLWTrace::TOrbit Orbit; - - TEvVSpecialPatchBase() = default; - - TEvVSpecialPatchBase(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, - TInstant deadline) - { - InitWithoutBuffer(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, - cookie, deadline); - } - - void InitWithoutBuffer(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, - TInstant deadline) - { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&originalBlobId, sizeof(originalBlobId)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&patchedBlobId, sizeof(patchedBlobId)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&vdisk, sizeof(vdisk)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&ignoreBlock, sizeof(ignoreBlock)); - - this->Record.SetOriginalGroupId(originalGroupId); - this->Record.SetPatchedGroupId(patchedGroupId); - - LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); - LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); - - VDiskIDFromVDiskID(vdisk, this->Record.MutableVDiskID()); - if (ignoreBlock) { - this->Record.SetIgnoreBlock(ignoreBlock); - } - if (cookie) { - this->Record.SetCookie(*cookie); - } - if (deadline != TInstant::Max()) { - this->Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); - } - this->Record.MutableMsgQoS()->SetExtQueueId(HandleClassToQueueId(NKikimrBlobStorage::AsyncBlob)); - } - - bool GetIgnoreBlock() const { - return this->Record.GetIgnoreBlock(); - } - - void AddDiff(ui64 startIdx, const TString &buffer) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(this->Record.GetOriginalBlobId()); - Y_VERIFY(startIdx < id.BlobSize()); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&buffer, sizeof(buffer)); - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size() + 1); - Y_VERIFY(startIdx + buffer.size() <= id.BlobSize()); - - NKikimrBlobStorage::TDiffBlock *diffBlock = this->Record.AddDiffs(); - diffBlock->SetOffset(startIdx); - diffBlock->SetBuffer(buffer); - } - - TString ToString() const override { - return ToString(this->Record); - } - - static TString ToString(const TRecord &record) { - TStringStream str; - TLogoBlobID originalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); - TLogoBlobID patchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); - str << "{TEvVMovedPatch"; - str << " OriginalBlobId# " << originalId.ToString(); - str << " PatchedBlobId# " << patchedId.ToString(); - str << " OriginalGroupId# " << record.GetOriginalBlobId(); - str << " PatchedGroupId# " << record.GetPatchedBlobId(); - if (record.GetIgnoreBlock()) { - str << " IgnoreBlock"; - } - if (record.HasMsgQoS()) { - str << " "; - TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); - } - str << " DiffCount# " << record.DiffsSize(); - str << "}"; - return str.Str(); - } - - ui32 DiffSizeSum() const { - ui32 result = 0; - for (auto &diff : this->Record.GetDiffs()) { - result += diff.GetBuffer().size(); - } - return result; - } - }; - - template <typename TEv, typename TRecord, ui32 EventType, typename TReqRecord> - struct TEvVSpecialPatchBaseResult - : public TEvVResultBaseWithQoSPB<TEv, TRecord, EventType> { - using TBase = TEvVResultBaseWithQoSPB<TEv, TRecord, EventType>; - mutable NLWTrace::TOrbit Orbit; - - TEvVSpecialPatchBaseResult() = default; - - TEvVSpecialPatchBaseResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, - TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, - TReqRecord *record, const TActorIDPtr &skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, - NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) - : TBase(now, counterPtr, histoPtr, std::move(traceId), - TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, recByteSize, record, skeletonFrontIDPtr) - { - this->Record.SetStatus(status); - LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); - LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); - VDiskIDFromVDiskID(vdisk, this->Record.MutableVDiskID()); - if (cookie) { - this->Record.SetCookie(*cookie); - } - this->Record.SetStatusFlags(oosStatus.Flags); - this->Record.SetApproximateFreeSpaceShare(oosStatus.ApproximateFreeSpaceShare); - if (record && record->HasTimestamps()) { - this->Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); - } - if (status == NKikimrProto::OK) { - this->Record.SetIncarnationGuid(incarnationGuid); - } - if (errorReason && status != NKikimrProto::OK) { - this->Record.SetErrorReason(errorReason); - } - } - - void UpdateStatus(const NKikimrProto::EReplyStatus status) { - this->Record.SetStatus(status); - } - - TString ToString() const override { - return ToString(this->Record); - } - - static TString ToString(const TRecord &record) { - TStringStream str; - TLogoBlobID originalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); - TLogoBlobID patchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); - TVDiskID vdiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - str << "{EvVMovedPatchResult Status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data(); - if (record.HasErrorReason()) { - str << " ErrorReason# " << '"' << EscapeC(record.GetErrorReason()) << '"'; - } - str << " OriginalBlobId# " << originalId; - str << " PatchedBlobId# " << patchedId; - str << " VDiskId# " << vdiskId; - if (record.HasCookie()) { - str << " Cookie# " << record.GetCookie(); - } - if (record.HasMsgQoS()) { - str << " "; - TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); - } - str << "}"; - - return str.Str(); - } - - void MakeError(NKikimrProto::EReplyStatus status, const TString& errorReason, - const TReqRecord &request) { - this->Record.SetStatus(status); - if (status != NKikimrProto::OK && errorReason) { - this->Record.SetErrorReason(errorReason); - } - Y_VERIFY(request.HasOriginalBlobId()); - Y_VERIFY(request.HasPatchedBlobId()); - TLogoBlobID originalBlobId = LogoBlobIDFromLogoBlobID(request.GetOriginalBlobId()); - TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(request.GetPatchedBlobId()); - LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); - LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); - - Y_VERIFY(request.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); - VDiskIDFromVDiskID(vDiskId, this->Record.MutableVDiskID()); - if (request.HasCookie()) { - this->Record.SetCookie(request.GetCookie()); - } - if (request.HasTimestamps()) { - this->Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); - } - } - }; - - - struct TEvBlobStorage::TEvVMovedPatch - : TEvVSpecialPatchBase< - TEvBlobStorage::TEvVMovedPatch, - NKikimrBlobStorage::TEvVMovedPatch, - TEvBlobStorage::EvVMovedPatch> - { - using TBase = TEvVSpecialPatchBase< - TEvBlobStorage::TEvVMovedPatch, - NKikimrBlobStorage::TEvVMovedPatch, - TEvBlobStorage::EvVMovedPatch>; - - TEvVMovedPatch() = default; - - TEvVMovedPatch(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, - TInstant deadline) - : TBase(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, - cookie, deadline) - {} - }; - - struct TEvBlobStorage::TEvVMovedPatchResult - : TEvVSpecialPatchBaseResult< - TEvBlobStorage::TEvVMovedPatchResult, - NKikimrBlobStorage::TEvVMovedPatchResult, - TEvBlobStorage::EvVMovedPatchResult, - NKikimrBlobStorage::TEvVMovedPatch> - { - using TBase = TEvVSpecialPatchBaseResult< - TEvBlobStorage::TEvVMovedPatchResult, - NKikimrBlobStorage::TEvVMovedPatchResult, - TEvBlobStorage::EvVMovedPatchResult, - NKikimrBlobStorage::TEvVMovedPatch>; - - TEvVMovedPatchResult() = default; - - TEvVMovedPatchResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, - TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, - NKikimrBlobStorage::TEvVMovedPatch *record, const TActorIDPtr &skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, - NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) - : TBase(status, originalBlobId, patchedBlobId, vdisk, cookie, oosStatus, now, recByteSize, - record, skeletonFrontIDPtr, counterPtr, histoPtr, std::move(traceId), incarnationGuid, errorReason) - {} - }; - - - struct TEvBlobStorage::TEvVInplacePatch - : TEvVSpecialPatchBase< - TEvBlobStorage::TEvVInplacePatch, - NKikimrBlobStorage::TEvVInplacePatch, - TEvBlobStorage::EvVInplacePatch> - { - using TBase = TEvVSpecialPatchBase< - TEvBlobStorage::TEvVInplacePatch, - NKikimrBlobStorage::TEvVInplacePatch, - TEvBlobStorage::EvVInplacePatch>; - - TEvVInplacePatch() = default; - - TEvVInplacePatch(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, - TInstant deadline) - : TBase(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, - cookie, deadline) - {} - }; - - struct TEvBlobStorage::TEvVInplacePatchResult - : TEvVSpecialPatchBaseResult< - TEvBlobStorage::TEvVInplacePatchResult, - NKikimrBlobStorage::TEvVInplacePatchResult, - TEvBlobStorage::EvVInplacePatchResult, - NKikimrBlobStorage::TEvVInplacePatch> - { - using TBase = TEvVSpecialPatchBaseResult< - TEvBlobStorage::TEvVInplacePatchResult, - NKikimrBlobStorage::TEvVInplacePatchResult, - TEvBlobStorage::EvVInplacePatchResult, - NKikimrBlobStorage::TEvVInplacePatch>; - - TEvVInplacePatchResult() = default; - - TEvVInplacePatchResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, - TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, - NKikimrBlobStorage::TEvVInplacePatch *record, const TActorIDPtr &skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, - NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) - : TBase(status, originalBlobId, patchedBlobId, vdisk, cookie, oosStatus, now, recByteSize, - record, skeletonFrontIDPtr, counterPtr, histoPtr, std::move(traceId), incarnationGuid, errorReason) - {} - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // VBLOCK - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // VPatch + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + template <typename TEv, typename TRecord, ui32 EventType> + struct TEvVSpecialPatchBase + : public TEventPB<TEv, TRecord, EventType> { + mutable NLWTrace::TOrbit Orbit; + + TEvVSpecialPatchBase() = default; + + TEvVSpecialPatchBase(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, + TInstant deadline) + { + InitWithoutBuffer(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, + cookie, deadline); + } + + void InitWithoutBuffer(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, + TInstant deadline) + { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&originalBlobId, sizeof(originalBlobId)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&patchedBlobId, sizeof(patchedBlobId)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&vdisk, sizeof(vdisk)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&ignoreBlock, sizeof(ignoreBlock)); + + this->Record.SetOriginalGroupId(originalGroupId); + this->Record.SetPatchedGroupId(patchedGroupId); + + LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); + LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); + + VDiskIDFromVDiskID(vdisk, this->Record.MutableVDiskID()); + if (ignoreBlock) { + this->Record.SetIgnoreBlock(ignoreBlock); + } + if (cookie) { + this->Record.SetCookie(*cookie); + } + if (deadline != TInstant::Max()) { + this->Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); + } + this->Record.MutableMsgQoS()->SetExtQueueId(HandleClassToQueueId(NKikimrBlobStorage::AsyncBlob)); + } + + bool GetIgnoreBlock() const { + return this->Record.GetIgnoreBlock(); + } + + void AddDiff(ui64 startIdx, const TString &buffer) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(this->Record.GetOriginalBlobId()); + Y_VERIFY(startIdx < id.BlobSize()); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(&buffer, sizeof(buffer)); + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size() + 1); + Y_VERIFY(startIdx + buffer.size() <= id.BlobSize()); + + NKikimrBlobStorage::TDiffBlock *diffBlock = this->Record.AddDiffs(); + diffBlock->SetOffset(startIdx); + diffBlock->SetBuffer(buffer); + } + + TString ToString() const override { + return ToString(this->Record); + } + + static TString ToString(const TRecord &record) { + TStringStream str; + TLogoBlobID originalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); + TLogoBlobID patchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); + str << "{TEvVMovedPatch"; + str << " OriginalBlobId# " << originalId.ToString(); + str << " PatchedBlobId# " << patchedId.ToString(); + str << " OriginalGroupId# " << record.GetOriginalBlobId(); + str << " PatchedGroupId# " << record.GetPatchedBlobId(); + if (record.GetIgnoreBlock()) { + str << " IgnoreBlock"; + } + if (record.HasMsgQoS()) { + str << " "; + TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); + } + str << " DiffCount# " << record.DiffsSize(); + str << "}"; + return str.Str(); + } + + ui32 DiffSizeSum() const { + ui32 result = 0; + for (auto &diff : this->Record.GetDiffs()) { + result += diff.GetBuffer().size(); + } + return result; + } + }; + + template <typename TEv, typename TRecord, ui32 EventType, typename TReqRecord> + struct TEvVSpecialPatchBaseResult + : public TEvVResultBaseWithQoSPB<TEv, TRecord, EventType> { + using TBase = TEvVResultBaseWithQoSPB<TEv, TRecord, EventType>; + mutable NLWTrace::TOrbit Orbit; + + TEvVSpecialPatchBaseResult() = default; + + TEvVSpecialPatchBaseResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, + TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, + TReqRecord *record, const TActorIDPtr &skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, + NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) + : TBase(now, counterPtr, histoPtr, std::move(traceId), + TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, recByteSize, record, skeletonFrontIDPtr) + { + this->Record.SetStatus(status); + LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); + LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); + VDiskIDFromVDiskID(vdisk, this->Record.MutableVDiskID()); + if (cookie) { + this->Record.SetCookie(*cookie); + } + this->Record.SetStatusFlags(oosStatus.Flags); + this->Record.SetApproximateFreeSpaceShare(oosStatus.ApproximateFreeSpaceShare); + if (record && record->HasTimestamps()) { + this->Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); + } + if (status == NKikimrProto::OK) { + this->Record.SetIncarnationGuid(incarnationGuid); + } + if (errorReason && status != NKikimrProto::OK) { + this->Record.SetErrorReason(errorReason); + } + } + + void UpdateStatus(const NKikimrProto::EReplyStatus status) { + this->Record.SetStatus(status); + } + + TString ToString() const override { + return ToString(this->Record); + } + + static TString ToString(const TRecord &record) { + TStringStream str; + TLogoBlobID originalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); + TLogoBlobID patchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); + TVDiskID vdiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + str << "{EvVMovedPatchResult Status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()).data(); + if (record.HasErrorReason()) { + str << " ErrorReason# " << '"' << EscapeC(record.GetErrorReason()) << '"'; + } + str << " OriginalBlobId# " << originalId; + str << " PatchedBlobId# " << patchedId; + str << " VDiskId# " << vdiskId; + if (record.HasCookie()) { + str << " Cookie# " << record.GetCookie(); + } + if (record.HasMsgQoS()) { + str << " "; + TEvBlobStorage::TEvVPut::OutMsgQos(record.GetMsgQoS(), str); + } + str << "}"; + + return str.Str(); + } + + void MakeError(NKikimrProto::EReplyStatus status, const TString& errorReason, + const TReqRecord &request) { + this->Record.SetStatus(status); + if (status != NKikimrProto::OK && errorReason) { + this->Record.SetErrorReason(errorReason); + } + Y_VERIFY(request.HasOriginalBlobId()); + Y_VERIFY(request.HasPatchedBlobId()); + TLogoBlobID originalBlobId = LogoBlobIDFromLogoBlobID(request.GetOriginalBlobId()); + TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(request.GetPatchedBlobId()); + LogoBlobIDFromLogoBlobID(originalBlobId, this->Record.MutableOriginalBlobId()); + LogoBlobIDFromLogoBlobID(patchedBlobId, this->Record.MutablePatchedBlobId()); + + Y_VERIFY(request.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); + VDiskIDFromVDiskID(vDiskId, this->Record.MutableVDiskID()); + if (request.HasCookie()) { + this->Record.SetCookie(request.GetCookie()); + } + if (request.HasTimestamps()) { + this->Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); + } + } + }; + + + struct TEvBlobStorage::TEvVMovedPatch + : TEvVSpecialPatchBase< + TEvBlobStorage::TEvVMovedPatch, + NKikimrBlobStorage::TEvVMovedPatch, + TEvBlobStorage::EvVMovedPatch> + { + using TBase = TEvVSpecialPatchBase< + TEvBlobStorage::TEvVMovedPatch, + NKikimrBlobStorage::TEvVMovedPatch, + TEvBlobStorage::EvVMovedPatch>; + + TEvVMovedPatch() = default; + + TEvVMovedPatch(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, + TInstant deadline) + : TBase(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, + cookie, deadline) + {} + }; + + struct TEvBlobStorage::TEvVMovedPatchResult + : TEvVSpecialPatchBaseResult< + TEvBlobStorage::TEvVMovedPatchResult, + NKikimrBlobStorage::TEvVMovedPatchResult, + TEvBlobStorage::EvVMovedPatchResult, + NKikimrBlobStorage::TEvVMovedPatch> + { + using TBase = TEvVSpecialPatchBaseResult< + TEvBlobStorage::TEvVMovedPatchResult, + NKikimrBlobStorage::TEvVMovedPatchResult, + TEvBlobStorage::EvVMovedPatchResult, + NKikimrBlobStorage::TEvVMovedPatch>; + + TEvVMovedPatchResult() = default; + + TEvVMovedPatchResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, + TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, + NKikimrBlobStorage::TEvVMovedPatch *record, const TActorIDPtr &skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, + NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) + : TBase(status, originalBlobId, patchedBlobId, vdisk, cookie, oosStatus, now, recByteSize, + record, skeletonFrontIDPtr, counterPtr, histoPtr, std::move(traceId), incarnationGuid, errorReason) + {} + }; + + + struct TEvBlobStorage::TEvVInplacePatch + : TEvVSpecialPatchBase< + TEvBlobStorage::TEvVInplacePatch, + NKikimrBlobStorage::TEvVInplacePatch, + TEvBlobStorage::EvVInplacePatch> + { + using TBase = TEvVSpecialPatchBase< + TEvBlobStorage::TEvVInplacePatch, + NKikimrBlobStorage::TEvVInplacePatch, + TEvBlobStorage::EvVInplacePatch>; + + TEvVInplacePatch() = default; + + TEvVInplacePatch(ui32 originalGroupId, ui32 patchedGroupId, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, bool ignoreBlock, TMaybe<ui64> cookie, + TInstant deadline) + : TBase(originalGroupId, patchedGroupId, originalBlobId, patchedBlobId, vdisk, ignoreBlock, + cookie, deadline) + {} + }; + + struct TEvBlobStorage::TEvVInplacePatchResult + : TEvVSpecialPatchBaseResult< + TEvBlobStorage::TEvVInplacePatchResult, + NKikimrBlobStorage::TEvVInplacePatchResult, + TEvBlobStorage::EvVInplacePatchResult, + NKikimrBlobStorage::TEvVInplacePatch> + { + using TBase = TEvVSpecialPatchBaseResult< + TEvBlobStorage::TEvVInplacePatchResult, + NKikimrBlobStorage::TEvVInplacePatchResult, + TEvBlobStorage::EvVInplacePatchResult, + NKikimrBlobStorage::TEvVInplacePatch>; + + TEvVInplacePatchResult() = default; + + TEvVInplacePatchResult(const NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vdisk, TMaybe<ui64> cookie, + TOutOfSpaceStatus oosStatus, const TInstant &now, ui32 recByteSize, + NKikimrBlobStorage::TEvVInplacePatch *record, const TActorIDPtr &skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, + NWilson::TTraceId traceId, ui64 incarnationGuid, const TString& errorReason) + : TBase(status, originalBlobId, patchedBlobId, vdisk, cookie, oosStatus, now, recByteSize, + record, skeletonFrontIDPtr, counterPtr, histoPtr, std::move(traceId), incarnationGuid, errorReason) + {} + }; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // VBLOCK + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + struct TEvBlobStorage::TEvVBlock : public TEventPB<TEvBlobStorage::TEvVBlock, NKikimrBlobStorage::TEvVBlock, TEvBlobStorage::EvVBlock> { TEvVBlock() @@ -1762,303 +1762,303 @@ namespace NKikimr { } }; - ////////////////////////////////////////////////////////////////////////////////////////////// - // VPatch - ////////////////////////////////////////////////////////////////////////////////////////////// - - struct TEvBlobStorage::TEvVPatchStart - : public TEventPB<TEvBlobStorage::TEvVPatchStart, - NKikimrBlobStorage::TEvVPatchStart, - TEvBlobStorage::EvVPatchStart> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchStart() = default; - - TEvVPatchStart(const TLogoBlobID &originalBlobId, const TLogoBlobID &patchedBlobId, const TVDiskID &vDiskId, - TInstant deadline, TMaybe<ui64> cookie, bool notifyIfNotReady) - { - LogoBlobIDFromLogoBlobID(originalBlobId, Record.MutableOriginalBlobId()); - LogoBlobIDFromLogoBlobID(patchedBlobId, Record.MutablePatchedBlobId()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - if (cookie) { - Record.SetCookie(*cookie); - } - if (deadline != TInstant::Max()) { - Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); - } - if (notifyIfNotReady) { - Record.SetNotifyIfNotReady(true); - } - Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::GetFastRead); - } - }; - - struct TEvBlobStorage::TEvVPatchFoundParts - : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchFoundParts, - NKikimrBlobStorage::TEvVPatchFoundParts, - TEvBlobStorage::EvVPatchFoundParts> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchFoundParts() = default; - - TEvVPatchFoundParts(NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, - const TLogoBlobID &patchedBlobId, const TVDiskID &vDiskId, TMaybe<ui64> cookie, const TInstant &now, - const TString &errorReason, NKikimrBlobStorage::TEvVPatchStart *record, - const TActorIDPtr &skeletonFrontIDPtr, const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, - const NVDiskMon::TLtcHistoPtr &histoPtr, NWilson::TTraceId traceId, ui64 incarnationGuid) - : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), - TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), - record, skeletonFrontIDPtr) - { - Record.SetStatus(status); - LogoBlobIDFromLogoBlobID(originalBlobId, Record.MutableOriginalBlobId()); - LogoBlobIDFromLogoBlobID(patchedBlobId, Record.MutablePatchedBlobId()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - - if (cookie) { - Record.SetCookie(*cookie); - } - if (record && record->HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); - } - if (status == NKikimrProto::OK) { - Record.SetIncarnationGuid(incarnationGuid); - } - Record.SetErrorReason(errorReason); - } - - void AddPart(ui8 part) { - Record.AddOriginalParts(part); - } - - void SetStatus(NKikimrProto::EReplyStatus status) { - Record.SetStatus(status); - } - - void MakeError(NKikimrProto::EReplyStatus status, const TString& errorReason, - const NKikimrBlobStorage::TEvVPatchStart &request) { - Record.SetErrorReason(errorReason); - Record.SetStatus(status); - Y_VERIFY(request.HasOriginalBlobId()); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(request.GetOriginalBlobId()); - LogoBlobIDFromLogoBlobID(blobId, Record.MutableOriginalBlobId()); - Y_VERIFY(request.HasVDiskID()); - TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - Y_VERIFY(request.HasCookie()); - Record.SetCookie(request.GetCookie()); - } - }; - - struct TEvBlobStorage::TEvVPatchDiff - : public TEventPB<TEvBlobStorage::TEvVPatchDiff, - NKikimrBlobStorage::TEvVPatchDiff, - TEvBlobStorage::EvVPatchDiff> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchDiff() = default; - - TEvVPatchDiff(const TLogoBlobID &originalPartBlobId, const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, - ui32 expectedXorDiffs, TInstant deadline, TMaybe<ui64> cookie) - { - LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); - LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - if (expectedXorDiffs) { - Record.SetExpectedXorDiffs(expectedXorDiffs); - } - if (cookie) { - Record.SetCookie(*cookie); - } - if (deadline != TInstant::Max()) { - Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); - } - Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob); - } - - void AddDiff(ui64 startIdx, const TString &buffer) { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size()); - - NKikimrBlobStorage::TDiffBlock *r = Record.AddDiffs(); - r->SetOffset(startIdx); - r->SetBuffer(buffer.data(), buffer.size()); - } - - void AddXorReceiver(const TVDiskID &vDiskId, ui8 partId) { - NKikimrBlobStorage::TXorDiffReceiver *r = Record.AddXorReceivers(); - VDiskIDFromVDiskID(vDiskId, r->MutableVDiskID()); - r->SetPartId(partId); - } - - void SetForceEnd() { - Record.SetForceEnd(true); - } - - bool IsForceEnd() const { - return Record.HasForceEnd() && Record.GetForceEnd(); - } - - bool IsXorReceiver() const { - return Record.HasExpectedXorDiffs() && Record.GetExpectedXorDiffs(); - } - - ui32 GetExpectedXorDiffs() const { - return Record.HasExpectedXorDiffs() ? Record.GetExpectedXorDiffs() : 0; - } - - ui32 DiffSizeSum() const { - ui32 result = 0; - for (auto &diff : Record.GetDiffs()) { - result += diff.GetBuffer().size(); - } - return result; - } - }; - - - struct TEvBlobStorage::TEvVPatchXorDiff - : public TEventPB<TEvBlobStorage::TEvVPatchXorDiff, - NKikimrBlobStorage::TEvVPatchXorDiff, - TEvBlobStorage::EvVPatchXorDiff> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchXorDiff() = default; - - TEvVPatchXorDiff(const TLogoBlobID &originalPartBlobId, const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, - ui8 partId, TInstant deadline, TMaybe<ui64> cookie) - { - LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); - LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - Record.SetFromPartId(partId); - if (cookie) { - Record.SetCookie(*cookie); - } - if (deadline != TInstant::Max()) { - Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); - } - Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob); - } - - void AddDiff(ui64 startIdx, const TString &buffer) { - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size()); - - NKikimrBlobStorage::TDiffBlock *r = Record.AddDiffs(); - r->SetOffset(startIdx); - r->SetBuffer(buffer.data(), buffer.size()); - } - - ui32 DiffSizeSum() const { - ui32 result = 0; - for (auto &diff : Record.GetDiffs()) { - result += diff.GetBuffer().size(); - } - return result; - } - }; - - struct TEvBlobStorage::TEvVPatchXorDiffResult - : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchXorDiffResult, - NKikimrBlobStorage::TEvVPatchXorDiffResult, - TEvBlobStorage::EvVPatchXorDiffResult> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchXorDiffResult() = default; - - TEvVPatchXorDiffResult(NKikimrProto::EReplyStatus status, TInstant now, - NKikimrBlobStorage::TEvVPatchXorDiff *record, const TActorIDPtr &skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, - const NVDiskMon::TLtcHistoPtr &histoPtr, NWilson::TTraceId traceId) - : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), - TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), - record, skeletonFrontIDPtr) - { - Record.SetStatus(status); - if (record && record->HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); - } - } - - void MakeError(NKikimrProto::EReplyStatus status, const TString& /*errorReason*/, - const NKikimrBlobStorage::TEvVPatchXorDiff &request) - { - Record.SetStatus(status); - if (request.HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); - } - } - }; - - struct TEvBlobStorage::TEvVPatchResult - : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchResult, - NKikimrBlobStorage::TEvVPatchResult, - TEvBlobStorage::EvVPatchResult> - { - mutable NLWTrace::TOrbit Orbit; - - TEvVPatchResult() = default; - - TEvVPatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &originalPartBlobId, - const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, TMaybe<ui64> cookie, TInstant now, - NKikimrBlobStorage::TEvVPatchDiff *record, const TActorIDPtr &skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, - NWilson::TTraceId traceId, ui64 incarnationGuid) - : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), - TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), - record, skeletonFrontIDPtr) - { - Record.SetStatus(status); - LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); - LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); - VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); - - if (cookie) { - Record.SetCookie(*cookie); - } - if (record && record->HasTimestamps()) { - Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); - } - if (status == NKikimrProto::OK) { - Record.SetIncarnationGuid(incarnationGuid); - } - } - - void SetStatus(NKikimrProto::EReplyStatus status, const TString &errorReason) { - Record.SetStatus(status); - Record.SetErrorReason(errorReason); - } - - void SetStatusFlagsAndFreeSpace(ui32 statusFlags, float approximateFreeSpaceShare) { - Record.SetStatusFlags(statusFlags); - Record.SetApproximateFreeSpaceShare(approximateFreeSpaceShare); - } - - void MakeError(NKikimrProto::EReplyStatus status, const TString &errorReason, - const NKikimrBlobStorage::TEvVPatchDiff &request) - { - Record.SetErrorReason(errorReason); - Record.SetStatus(status); - - Y_VERIFY(request.HasOriginalPartBlobId()); - TLogoBlobID originalPartBlobId = LogoBlobIDFromLogoBlobID(request.GetOriginalPartBlobId()); - LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); - - Y_VERIFY(request.HasPatchedPartBlobId()); - TLogoBlobID patchedPartBlobId = LogoBlobIDFromLogoBlobID(request.GetPatchedPartBlobId()); - LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); - - Y_VERIFY(request.HasVDiskID()); + ////////////////////////////////////////////////////////////////////////////////////////////// + // VPatch + ////////////////////////////////////////////////////////////////////////////////////////////// + + struct TEvBlobStorage::TEvVPatchStart + : public TEventPB<TEvBlobStorage::TEvVPatchStart, + NKikimrBlobStorage::TEvVPatchStart, + TEvBlobStorage::EvVPatchStart> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchStart() = default; + + TEvVPatchStart(const TLogoBlobID &originalBlobId, const TLogoBlobID &patchedBlobId, const TVDiskID &vDiskId, + TInstant deadline, TMaybe<ui64> cookie, bool notifyIfNotReady) + { + LogoBlobIDFromLogoBlobID(originalBlobId, Record.MutableOriginalBlobId()); + LogoBlobIDFromLogoBlobID(patchedBlobId, Record.MutablePatchedBlobId()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + if (cookie) { + Record.SetCookie(*cookie); + } + if (deadline != TInstant::Max()) { + Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); + } + if (notifyIfNotReady) { + Record.SetNotifyIfNotReady(true); + } + Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::GetFastRead); + } + }; + + struct TEvBlobStorage::TEvVPatchFoundParts + : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchFoundParts, + NKikimrBlobStorage::TEvVPatchFoundParts, + TEvBlobStorage::EvVPatchFoundParts> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchFoundParts() = default; + + TEvVPatchFoundParts(NKikimrProto::EReplyStatus status, const TLogoBlobID &originalBlobId, + const TLogoBlobID &patchedBlobId, const TVDiskID &vDiskId, TMaybe<ui64> cookie, const TInstant &now, + const TString &errorReason, NKikimrBlobStorage::TEvVPatchStart *record, + const TActorIDPtr &skeletonFrontIDPtr, const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, + const NVDiskMon::TLtcHistoPtr &histoPtr, NWilson::TTraceId traceId, ui64 incarnationGuid) + : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), + TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), + record, skeletonFrontIDPtr) + { + Record.SetStatus(status); + LogoBlobIDFromLogoBlobID(originalBlobId, Record.MutableOriginalBlobId()); + LogoBlobIDFromLogoBlobID(patchedBlobId, Record.MutablePatchedBlobId()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + + if (cookie) { + Record.SetCookie(*cookie); + } + if (record && record->HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); + } + if (status == NKikimrProto::OK) { + Record.SetIncarnationGuid(incarnationGuid); + } + Record.SetErrorReason(errorReason); + } + + void AddPart(ui8 part) { + Record.AddOriginalParts(part); + } + + void SetStatus(NKikimrProto::EReplyStatus status) { + Record.SetStatus(status); + } + + void MakeError(NKikimrProto::EReplyStatus status, const TString& errorReason, + const NKikimrBlobStorage::TEvVPatchStart &request) { + Record.SetErrorReason(errorReason); + Record.SetStatus(status); + Y_VERIFY(request.HasOriginalBlobId()); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(request.GetOriginalBlobId()); + LogoBlobIDFromLogoBlobID(blobId, Record.MutableOriginalBlobId()); + Y_VERIFY(request.HasVDiskID()); + TVDiskID vDiskId = VDiskIDFromVDiskID(request.GetVDiskID()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + Y_VERIFY(request.HasCookie()); + Record.SetCookie(request.GetCookie()); + } + }; + + struct TEvBlobStorage::TEvVPatchDiff + : public TEventPB<TEvBlobStorage::TEvVPatchDiff, + NKikimrBlobStorage::TEvVPatchDiff, + TEvBlobStorage::EvVPatchDiff> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchDiff() = default; + + TEvVPatchDiff(const TLogoBlobID &originalPartBlobId, const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, + ui32 expectedXorDiffs, TInstant deadline, TMaybe<ui64> cookie) + { + LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); + LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + if (expectedXorDiffs) { + Record.SetExpectedXorDiffs(expectedXorDiffs); + } + if (cookie) { + Record.SetCookie(*cookie); + } + if (deadline != TInstant::Max()) { + Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); + } + Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob); + } + + void AddDiff(ui64 startIdx, const TString &buffer) { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size()); + + NKikimrBlobStorage::TDiffBlock *r = Record.AddDiffs(); + r->SetOffset(startIdx); + r->SetBuffer(buffer.data(), buffer.size()); + } + + void AddXorReceiver(const TVDiskID &vDiskId, ui8 partId) { + NKikimrBlobStorage::TXorDiffReceiver *r = Record.AddXorReceivers(); + VDiskIDFromVDiskID(vDiskId, r->MutableVDiskID()); + r->SetPartId(partId); + } + + void SetForceEnd() { + Record.SetForceEnd(true); + } + + bool IsForceEnd() const { + return Record.HasForceEnd() && Record.GetForceEnd(); + } + + bool IsXorReceiver() const { + return Record.HasExpectedXorDiffs() && Record.GetExpectedXorDiffs(); + } + + ui32 GetExpectedXorDiffs() const { + return Record.HasExpectedXorDiffs() ? Record.GetExpectedXorDiffs() : 0; + } + + ui32 DiffSizeSum() const { + ui32 result = 0; + for (auto &diff : Record.GetDiffs()) { + result += diff.GetBuffer().size(); + } + return result; + } + }; + + + struct TEvBlobStorage::TEvVPatchXorDiff + : public TEventPB<TEvBlobStorage::TEvVPatchXorDiff, + NKikimrBlobStorage::TEvVPatchXorDiff, + TEvBlobStorage::EvVPatchXorDiff> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchXorDiff() = default; + + TEvVPatchXorDiff(const TLogoBlobID &originalPartBlobId, const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, + ui8 partId, TInstant deadline, TMaybe<ui64> cookie) + { + LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); + LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + Record.SetFromPartId(partId); + if (cookie) { + Record.SetCookie(*cookie); + } + if (deadline != TInstant::Max()) { + Record.MutableMsgQoS()->SetDeadlineSeconds((ui32)deadline.Seconds()); + } + Record.MutableMsgQoS()->SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob); + } + + void AddDiff(ui64 startIdx, const TString &buffer) { + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer.data(), buffer.size()); + + NKikimrBlobStorage::TDiffBlock *r = Record.AddDiffs(); + r->SetOffset(startIdx); + r->SetBuffer(buffer.data(), buffer.size()); + } + + ui32 DiffSizeSum() const { + ui32 result = 0; + for (auto &diff : Record.GetDiffs()) { + result += diff.GetBuffer().size(); + } + return result; + } + }; + + struct TEvBlobStorage::TEvVPatchXorDiffResult + : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchXorDiffResult, + NKikimrBlobStorage::TEvVPatchXorDiffResult, + TEvBlobStorage::EvVPatchXorDiffResult> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchXorDiffResult() = default; + + TEvVPatchXorDiffResult(NKikimrProto::EReplyStatus status, TInstant now, + NKikimrBlobStorage::TEvVPatchXorDiff *record, const TActorIDPtr &skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, + const NVDiskMon::TLtcHistoPtr &histoPtr, NWilson::TTraceId traceId) + : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), + TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), + record, skeletonFrontIDPtr) + { + Record.SetStatus(status); + if (record && record->HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); + } + } + + void MakeError(NKikimrProto::EReplyStatus status, const TString& /*errorReason*/, + const NKikimrBlobStorage::TEvVPatchXorDiff &request) + { + Record.SetStatus(status); + if (request.HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(request.GetTimestamps()); + } + } + }; + + struct TEvBlobStorage::TEvVPatchResult + : public TEvVResultBaseWithQoSPB<TEvBlobStorage::TEvVPatchResult, + NKikimrBlobStorage::TEvVPatchResult, + TEvBlobStorage::EvVPatchResult> + { + mutable NLWTrace::TOrbit Orbit; + + TEvVPatchResult() = default; + + TEvVPatchResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &originalPartBlobId, + const TLogoBlobID &patchedPartBlobId, const TVDiskID &vDiskId, TMaybe<ui64> cookie, TInstant now, + NKikimrBlobStorage::TEvVPatchDiff *record, const TActorIDPtr &skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr, const NVDiskMon::TLtcHistoPtr &histoPtr, + NWilson::TTraceId traceId, ui64 incarnationGuid) + : TEvVResultBaseWithQoSPB(now, counterPtr, histoPtr, std::move(traceId), + TInterconnectChannels::IC_BLOBSTORAGE_SMALL_MSG, record->GetCachedSize(), + record, skeletonFrontIDPtr) + { + Record.SetStatus(status); + LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); + LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); + VDiskIDFromVDiskID(vDiskId, Record.MutableVDiskID()); + + if (cookie) { + Record.SetCookie(*cookie); + } + if (record && record->HasTimestamps()) { + Record.MutableTimestamps()->CopyFrom(record->GetTimestamps()); + } + if (status == NKikimrProto::OK) { + Record.SetIncarnationGuid(incarnationGuid); + } + } + + void SetStatus(NKikimrProto::EReplyStatus status, const TString &errorReason) { + Record.SetStatus(status); + Record.SetErrorReason(errorReason); + } + + void SetStatusFlagsAndFreeSpace(ui32 statusFlags, float approximateFreeSpaceShare) { + Record.SetStatusFlags(statusFlags); + Record.SetApproximateFreeSpaceShare(approximateFreeSpaceShare); + } + + void MakeError(NKikimrProto::EReplyStatus status, const TString &errorReason, + const NKikimrBlobStorage::TEvVPatchDiff &request) + { + Record.SetErrorReason(errorReason); + Record.SetStatus(status); + + Y_VERIFY(request.HasOriginalPartBlobId()); + TLogoBlobID originalPartBlobId = LogoBlobIDFromLogoBlobID(request.GetOriginalPartBlobId()); + LogoBlobIDFromLogoBlobID(originalPartBlobId, Record.MutableOriginalPartBlobId()); + + Y_VERIFY(request.HasPatchedPartBlobId()); + TLogoBlobID patchedPartBlobId = LogoBlobIDFromLogoBlobID(request.GetPatchedPartBlobId()); + LogoBlobIDFromLogoBlobID(patchedPartBlobId, Record.MutablePatchedPartBlobId()); + + Y_VERIFY(request.HasVDiskID()); Record.MutableVDiskID()->CopyFrom(request.GetVDiskID()); - Y_VERIFY(request.HasCookie()); - Record.SetCookie(request.GetCookie()); - } - }; - + Y_VERIFY(request.HasCookie()); + Record.SetCookie(request.GetCookie()); + } + }; + struct TEvBlobStorage::TEvVGetBlock : public TEventPB<TEvBlobStorage::TEvVGetBlock, NKikimrBlobStorage::TEvVGetBlock, @@ -2717,15 +2717,15 @@ namespace NKikimr { Record.SetFrontQueueId(queueId); wstatus.Serialize(*Record.MutableWindow()); } - + TEvVWindowChange(NKikimrBlobStorage::EVDiskQueueId queueId, TDropConnection) { Record.SetFrontQueueId(queueId); Record.SetDropConnection(true); } - TString ToString() const override { + TString ToString() const override { return SingleLineProto(Record); - } + } }; struct TEvVGenerationChange: public TEventLocal<TEvVGenerationChange, TEvBlobStorage::EvVGenerationChange> { diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_lsnmngr.h b/ydb/core/blobstorage/vdisk/common/vdisk_lsnmngr.h index dde57d68b2e..28deb2f1cf4 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_lsnmngr.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_lsnmngr.h @@ -172,26 +172,26 @@ namespace NKikimr { AllocForSyncLog->Allocated(seg); return seg; } - - // hull db update - TLsnSeg AllocDiscreteLsnBatchForHull(ui64 lsnAdvance) { - TLsnSeg seg = AllocLsn(lsnAdvance); - for (ui64 lsn = seg.First; lsn <= seg.Last; ++lsn) { - TLsnSeg point(lsn, lsn); - AllocForHull->Allocated(point); - } - return seg; - } - - // hull db and synclog update - TLsnSeg AllocDiscreteLsnBatchForHullAndSyncLog(ui64 lsnAdvance) { - TLsnSeg seg = AllocDiscreteLsnBatchForHull(lsnAdvance); - for (ui64 lsn = seg.First; lsn <= seg.Last; ++lsn) { - TLsnSeg point(lsn, lsn); - AllocForSyncLog->Allocated(point); - } - return seg; - } + + // hull db update + TLsnSeg AllocDiscreteLsnBatchForHull(ui64 lsnAdvance) { + TLsnSeg seg = AllocLsn(lsnAdvance); + for (ui64 lsn = seg.First; lsn <= seg.Last; ++lsn) { + TLsnSeg point(lsn, lsn); + AllocForHull->Allocated(point); + } + return seg; + } + + // hull db and synclog update + TLsnSeg AllocDiscreteLsnBatchForHullAndSyncLog(ui64 lsnAdvance) { + TLsnSeg seg = AllocDiscreteLsnBatchForHull(lsnAdvance); + for (ui64 lsn = seg.First; lsn <= seg.Last; ++lsn) { + TLsnSeg point(lsn, lsn); + AllocForSyncLog->Allocated(point); + } + return seg; + } ////////////////////////////// LSN ALLOCATION ////////////////////////////////////// ////////////////////////////// LSN CONFIRMATION //////////////////////////////////// diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h b/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h index aa1bff8f84e..de4caf504eb 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h @@ -96,38 +96,38 @@ public: public: GROUP_CONSTRUCTOR(TSkeletonOverloadGroup) { - COUNTER_INIT(EmergencyMovedPatchQueueItems, false); - COUNTER_INIT(EmergencyPatchStartQueueItems, false); + COUNTER_INIT(EmergencyMovedPatchQueueItems, false); + COUNTER_INIT(EmergencyPatchStartQueueItems, false); COUNTER_INIT(EmergencyPutQueueItems, false); COUNTER_INIT(EmergencyMultiPutQueueItems, false); COUNTER_INIT(EmergencyLocalSyncDataQueueItems, false); COUNTER_INIT(EmergencyAnubisOsirisPutQueueItems, false); - - COUNTER_INIT(EmergencyMovedPatchQueueBytes, false); - COUNTER_INIT(EmergencyPatchStartQueueBytes, false); + + COUNTER_INIT(EmergencyMovedPatchQueueBytes, false); + COUNTER_INIT(EmergencyPatchStartQueueBytes, false); COUNTER_INIT(EmergencyPutQueueBytes, false); COUNTER_INIT(EmergencyMultiPutQueueBytes, false); COUNTER_INIT(EmergencyLocalSyncDataQueueBytes, false); COUNTER_INIT(EmergencyAnubisOsirisPutQueueBytes, false); - + COUNTER_INIT(FreshSatisfactionRankPercent, false); COUNTER_INIT(LevelSatisfactionRankPercent, false); } - COUNTER_DEF(EmergencyMovedPatchQueueItems); - COUNTER_DEF(EmergencyPatchStartQueueItems); + COUNTER_DEF(EmergencyMovedPatchQueueItems); + COUNTER_DEF(EmergencyPatchStartQueueItems); COUNTER_DEF(EmergencyPutQueueItems); COUNTER_DEF(EmergencyMultiPutQueueItems); COUNTER_DEF(EmergencyLocalSyncDataQueueItems); COUNTER_DEF(EmergencyAnubisOsirisPutQueueItems); - - COUNTER_DEF(EmergencyMovedPatchQueueBytes); - COUNTER_DEF(EmergencyPatchStartQueueBytes); + + COUNTER_DEF(EmergencyMovedPatchQueueBytes); + COUNTER_DEF(EmergencyPatchStartQueueBytes); COUNTER_DEF(EmergencyPutQueueBytes); COUNTER_DEF(EmergencyMultiPutQueueBytes); COUNTER_DEF(EmergencyLocalSyncDataQueueBytes); COUNTER_DEF(EmergencyAnubisOsirisPutQueueBytes); - + COUNTER_DEF(FreshSatisfactionRankPercent); COUNTER_DEF(LevelSatisfactionRankPercent); }; @@ -422,12 +422,12 @@ public: public: GROUP_CONSTRUCTOR(TVDiskIFaceGroup) { - COUNTER_INIT(MovedPatchMsgs, true); - COUNTER_INIT(PatchStartMsgs, true); - COUNTER_INIT(PatchDiffMsgs, true); - COUNTER_INIT(PatchXorDiffMsgs, true); + COUNTER_INIT(MovedPatchMsgs, true); + COUNTER_INIT(PatchStartMsgs, true); + COUNTER_INIT(PatchDiffMsgs, true); + COUNTER_INIT(PatchXorDiffMsgs, true); COUNTER_INIT(PutMsgs, true); - COUNTER_INIT(MultiPutMsgs, true); + COUNTER_INIT(MultiPutMsgs, true); COUNTER_INIT(GetMsgs, true); COUNTER_INIT(BlockMsgs, true); COUNTER_INIT(GetBlockMsgs, true); @@ -441,12 +441,12 @@ public: COUNTER_INIT(AnubisPutMsgs, true); COUNTER_INIT(OsirisPutMsgs, true); - COUNTER_INIT_PRIVATE(MovedPatchResMsgs, true); - COUNTER_INIT_PRIVATE(PatchFoundPartsMsgs, true); - COUNTER_INIT_PRIVATE(PatchXorDiffResMsgs, true); - COUNTER_INIT_PRIVATE(PatchResMsgs, true); + COUNTER_INIT_PRIVATE(MovedPatchResMsgs, true); + COUNTER_INIT_PRIVATE(PatchFoundPartsMsgs, true); + COUNTER_INIT_PRIVATE(PatchXorDiffResMsgs, true); + COUNTER_INIT_PRIVATE(PatchResMsgs, true); COUNTER_INIT_PRIVATE(PutResMsgs, true); - COUNTER_INIT_PRIVATE(MultiPutResMsgs, true); + COUNTER_INIT_PRIVATE(MultiPutResMsgs, true); COUNTER_INIT_PRIVATE(GetResMsgs, true); COUNTER_INIT_PRIVATE(BlockResMsgs, true); COUNTER_INIT_PRIVATE(GetBlockResMsgs, true); @@ -464,12 +464,12 @@ public: COUNTER_INIT(GetTotalBytes, true); } - COUNTER_DEF(MovedPatchMsgs); - COUNTER_DEF(PatchStartMsgs); - COUNTER_DEF(PatchDiffMsgs); - COUNTER_DEF(PatchXorDiffMsgs); + COUNTER_DEF(MovedPatchMsgs); + COUNTER_DEF(PatchStartMsgs); + COUNTER_DEF(PatchDiffMsgs); + COUNTER_DEF(PatchXorDiffMsgs); COUNTER_DEF(PutMsgs); - COUNTER_DEF(MultiPutMsgs); + COUNTER_DEF(MultiPutMsgs); COUNTER_DEF(GetMsgs); COUNTER_DEF(BlockMsgs); COUNTER_DEF(GetBlockMsgs); @@ -483,12 +483,12 @@ public: COUNTER_DEF(AnubisPutMsgs); COUNTER_DEF(OsirisPutMsgs); - COUNTER_DEF(MovedPatchResMsgs); - COUNTER_DEF(PatchFoundPartsMsgs); - COUNTER_DEF(PatchXorDiffResMsgs); - COUNTER_DEF(PatchResMsgs); + COUNTER_DEF(MovedPatchResMsgs); + COUNTER_DEF(PatchFoundPartsMsgs); + COUNTER_DEF(PatchXorDiffResMsgs); + COUNTER_DEF(PatchResMsgs); COUNTER_DEF(PutResMsgs); - COUNTER_DEF(MultiPutResMsgs); + COUNTER_DEF(MultiPutResMsgs); COUNTER_DEF(GetResMsgs); COUNTER_DEF(BlockResMsgs); COUNTER_DEF(GetBlockResMsgs); diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_queues.h b/ydb/core/blobstorage/vdisk/common/vdisk_queues.h index 9deed7fb435..9377b95f53f 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_queues.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_queues.h @@ -1,73 +1,73 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_iter.h> #include <ydb/core/blobstorage/backpressure/queue_backpressure_client.h> #include <ydb/core/blobstorage/backpressure/queue_backpressure_common.h> - - -namespace NKikimr { - - inline TVDiskIdShort GetVDiskID(const TBlobStorageGroupInfo::TVDiskInfo &vdisk) { - return vdisk.VDiskIdShort; - } - - inline TVDiskID GetVDiskID(const std::pair<TVDiskID, TActorId> &p) { - return p.first; - } - - inline TActorId GetVDiskActorId(const TBlobStorageGroupInfo::TVDiskInfo &vdisk, - const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &gInfo) - { - return gInfo->GetActorId(vdisk.VDiskIdShort); - } - + + +namespace NKikimr { + + inline TVDiskIdShort GetVDiskID(const TBlobStorageGroupInfo::TVDiskInfo &vdisk) { + return vdisk.VDiskIdShort; + } + + inline TVDiskID GetVDiskID(const std::pair<TVDiskID, TActorId> &p) { + return p.first; + } + + inline TActorId GetVDiskActorId(const TBlobStorageGroupInfo::TVDiskInfo &vdisk, + const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &gInfo) + { + return gInfo->GetActorId(vdisk.VDiskIdShort); + } + inline TActorId GetVDiskActorId(const std::pair<TVDiskID, TActorId> &p, - const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &) - { - return p.second; - } - - struct TBasicQueueActorIdWrapper { - TActorId Wrap(TActorId &&id) const { - return id; - } - }; - - template <typename TContainer, typename TWrappedQueueActorId> - void EmplaceToContainer(TContainer &cont, const TVDiskIdShort &vdisk, TWrappedQueueActorId &&actorId) { - cont.emplace(vdisk, std::move(actorId)); - } - - inline void EmplaceToContainer(std::deque<std::pair<TVDiskID, TActorId>> &cont, const TVDiskID &vdisk, TActorId &&actorId) { - cont.emplace_back(vdisk, std::move(actorId)); - } - - template <typename TContainer, typename TVDiskContainer, typename TWrapper = TBasicQueueActorIdWrapper> - void CreateQueuesForVDisks(TContainer &cont, const TActorId &parent, - const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &gInfo, - const TIntrusivePtr<TVDiskContext> &vCtx, const TVDiskContainer &disks, - const TIntrusivePtr<NMonitoring::TDynamicCounters> &groupCounters, - const NBackpressure::TQueueClientId &queueClientId, NKikimrBlobStorage::EVDiskQueueId vDiskQueueId, - const TString &queueName, TInterconnectChannels::EInterconnectChannels interconnectChannel, - TWrapper wrapper = {}) - { - TBlobStorageGroupType type = gInfo->Type; - for (auto &vdiskInfo : disks) { - auto vdisk = GetVDiskID(vdiskInfo); - if (vdisk != vCtx->ShortSelfVDisk) { - TIntrusivePtr<NBackpressure::TFlowRecord> flowRecord = MakeIntrusive<NBackpressure::TFlowRecord>(); - TActorId vdiskActorId = GetVDiskActorId(vdiskInfo, gInfo); + const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &) + { + return p.second; + } + + struct TBasicQueueActorIdWrapper { + TActorId Wrap(TActorId &&id) const { + return id; + } + }; + + template <typename TContainer, typename TWrappedQueueActorId> + void EmplaceToContainer(TContainer &cont, const TVDiskIdShort &vdisk, TWrappedQueueActorId &&actorId) { + cont.emplace(vdisk, std::move(actorId)); + } + + inline void EmplaceToContainer(std::deque<std::pair<TVDiskID, TActorId>> &cont, const TVDiskID &vdisk, TActorId &&actorId) { + cont.emplace_back(vdisk, std::move(actorId)); + } + + template <typename TContainer, typename TVDiskContainer, typename TWrapper = TBasicQueueActorIdWrapper> + void CreateQueuesForVDisks(TContainer &cont, const TActorId &parent, + const TIntrusivePtr<NKikimr::TBlobStorageGroupInfo> &gInfo, + const TIntrusivePtr<TVDiskContext> &vCtx, const TVDiskContainer &disks, + const TIntrusivePtr<NMonitoring::TDynamicCounters> &groupCounters, + const NBackpressure::TQueueClientId &queueClientId, NKikimrBlobStorage::EVDiskQueueId vDiskQueueId, + const TString &queueName, TInterconnectChannels::EInterconnectChannels interconnectChannel, + TWrapper wrapper = {}) + { + TBlobStorageGroupType type = gInfo->Type; + for (auto &vdiskInfo : disks) { + auto vdisk = GetVDiskID(vdiskInfo); + if (vdisk != vCtx->ShortSelfVDisk) { + TIntrusivePtr<NBackpressure::TFlowRecord> flowRecord = MakeIntrusive<NBackpressure::TFlowRecord>(); + TActorId vdiskActorId = GetVDiskActorId(vdiskInfo, gInfo); std::unique_ptr<IActor> queue; queue.reset(CreateVDiskBackpressureClient(gInfo, vdisk, - vDiskQueueId, groupCounters, vCtx, queueClientId, queueName, - interconnectChannel, vdiskActorId.NodeId() == parent.NodeId(), + vDiskQueueId, groupCounters, vCtx, queueClientId, queueName, + interconnectChannel, vdiskActorId.NodeId() == parent.NodeId(), TDuration::Minutes(1), flowRecord, NMonitoring::TCountableBase::EVisibility::Private)); TActorId serviceId = TActivationContext::Register(queue.release(), parent); - EmplaceToContainer(cont, vdisk, wrapper.Wrap(std::move(serviceId))); - } - } - } - -} // NKikimr + EmplaceToContainer(cont, vdisk, wrapper.Wrap(std::move(serviceId))); + } + } + } + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_recoverylogwriter.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_recoverylogwriter.cpp index f0ec9c15e40..2cfa5d0bc44 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_recoverylogwriter.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_recoverylogwriter.cpp @@ -106,8 +106,8 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); while (!Queue.empty() && ((item = &Queue.top())->LsnSegmentStart == CurSentLsn + 1)) { CurSentLsn = item->Lsn; std::unique_ptr<IEventHandle> ev = std::move(const_cast<TQueueItem*>(item)->Ev); - ui32 type = ev->Type; - switch (type) { + ui32 type = ev->Type; + switch (type) { case TEvBlobStorage::EvLog: LWTRACK(VDiskRecoveryLogWriterVPutIsSent, ev->Get<NPDisk::TEvLog>()->Orbit, Owner, item->Lsn); break; @@ -118,9 +118,9 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); LWTRACK(VDiskRecoveryLogWriterVPutIsSent, log->Orbit, Owner, log->Lsn); } break; - } - } - Queue.pop(); + } + } + Queue.pop(); ctx.ExecutorThread.Send(ev.release()); } } @@ -150,39 +150,39 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); } } - void Handle(NPDisk::TEvMultiLog::TPtr &ev, const TActorContext &ctx) { - NPDisk::TEvMultiLog *logs = ev->Get(); - ui64 lsnSegmentStart = logs->LsnSeg.First; - ui64 lsn = logs->LsnSeg.Last; - Y_VERIFY_DEBUG_S(lsnSegmentStart == logs->Logs.front()->LsnSegmentStart && lsn == logs->Logs.back()->Lsn, - "LsnSeg not match with inner logs" - << "LsnSeg# " << logs->LsnSeg.ToString() - << "Logs.front().LsnSegmentStart# " << logs->Logs.front()->LsnSegmentStart - << "Logs.back().Lsn# " << logs->Logs.back()->Lsn); - for (auto &log : logs->Logs) { - LWTRACK(VDiskRecoveryLogWriterVPutIsRecieved, log->Orbit, Owner, log->Lsn); + void Handle(NPDisk::TEvMultiLog::TPtr &ev, const TActorContext &ctx) { + NPDisk::TEvMultiLog *logs = ev->Get(); + ui64 lsnSegmentStart = logs->LsnSeg.First; + ui64 lsn = logs->LsnSeg.Last; + Y_VERIFY_DEBUG_S(lsnSegmentStart == logs->Logs.front()->LsnSegmentStart && lsn == logs->Logs.back()->Lsn, + "LsnSeg not match with inner logs" + << "LsnSeg# " << logs->LsnSeg.ToString() + << "Logs.front().LsnSegmentStart# " << logs->Logs.front()->LsnSegmentStart + << "Logs.back().Lsn# " << logs->Logs.back()->Lsn); + for (auto &log : logs->Logs) { + LWTRACK(VDiskRecoveryLogWriterVPutIsRecieved, log->Orbit, Owner, log->Lsn); TLogSignature signature = log->Signature.GetUnmasked(); Y_VERIFY(TLogSignature::First < signature && signature < TLogSignature::Max); - i64 msgSize = log->ApproximateSize(); - // count written bytes - *LsmLogBytesWritten += msgSize; - // update generic counters + i64 msgSize = log->ApproximateSize(); + // count written bytes + *LsmLogBytesWritten += msgSize; + // update generic counters Counters.Update(signature, msgSize); - LWTRACK(VDiskRecoveryLogWriterVPutIsSent, log->Orbit, Owner, lsn); - } + LWTRACK(VDiskRecoveryLogWriterVPutIsSent, log->Orbit, Owner, lsn); + } std::unique_ptr<IEventHandle> converted(ev->Forward(YardID).Release()); - - if (lsnSegmentStart == CurSentLsn + 1) { - // rewrite and send message; + + if (lsnSegmentStart == CurSentLsn + 1) { + // rewrite and send message; ctx.ExecutorThread.Send(converted.release()); - CurSentLsn = lsn; - // proceed with elements waiting in the queue - ProcessQueue(ctx); - } else { + CurSentLsn = lsn; + // proceed with elements waiting in the queue + ProcessQueue(ctx); + } else { Queue.push(TQueueItem(std::move(converted), lsnSegmentStart, lsn)); - } - } - + } + } + void Handle(TEvBlobStorage::TEvVCompact::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ev); ui64 lsn = CurSentLsn + 1; @@ -196,7 +196,7 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER); STRICT_STFUNC(StateFunc, HFunc(NPDisk::TEvLog, Handle) - HFunc(NPDisk::TEvMultiLog, Handle) + HFunc(NPDisk::TEvMultiLog, Handle) HFunc(TEvBlobStorage::TEvVCompact, Handle) HFunc(TEvents::TEvPoisonPill, HandlePoison) ) diff --git a/ydb/core/blobstorage/vdisk/common/ya.make b/ydb/core/blobstorage/vdisk/common/ya.make index 4fb4364ee2a..11194b0b74a 100644 --- a/ydb/core/blobstorage/vdisk/common/ya.make +++ b/ydb/core/blobstorage/vdisk/common/ya.make @@ -52,7 +52,7 @@ SRCS( vdisk_pdisk_error.h vdisk_pdiskctx.h vdisk_private_events.h - vdisk_queues.h + vdisk_queues.h vdisk_recoverylogwriter.cpp vdisk_recoverylogwriter.h vdisk_response.cpp diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp index 30d2aeece40..db0ec52ffcb 100644 --- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp +++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp @@ -164,7 +164,7 @@ namespace NKikimr { case NKikimrBlobStorage::EPutHandleClass::TabletLog: return NPriWrite::HullHugeUserData; case NKikimrBlobStorage::EPutHandleClass::AsyncBlob: return NPriWrite::HullHugeAsyncBlob; case NKikimrBlobStorage::EPutHandleClass::UserData: return NPriWrite::HullHugeUserData; - default: Y_FAIL_S("Unexpected HandleClass# " << int(Item->HandleClass)); + default: Y_FAIL_S("Unexpected HandleClass# " << int(Item->HandleClass)); } } @@ -685,9 +685,9 @@ namespace NKikimr { // notify log cutter if the FirstLsnToKeep has changed since last reporting if (firstLsnToKeep != State.LastReportedFirstLsnToKeep) { - Y_VERIFY_S(firstLsnToKeep > State.LastReportedFirstLsnToKeep, "huge keeper log rollback" - << " firstLsnToKeep#" << firstLsnToKeep - << " State.LastReportedFirstLsnToKeep# " << State.LastReportedFirstLsnToKeep); + Y_VERIFY_S(firstLsnToKeep > State.LastReportedFirstLsnToKeep, "huge keeper log rollback" + << " firstLsnToKeep#" << firstLsnToKeep + << " State.LastReportedFirstLsnToKeep# " << State.LastReportedFirstLsnToKeep); ctx.Send(HugeKeeperCtx->LogCutterId, new TEvVDiskCutLog(TEvVDiskCutLog::HugeKeeper, firstLsnToKeep)); State.LastReportedFirstLsnToKeep = firstLsnToKeep; @@ -777,10 +777,10 @@ namespace NKikimr { auto checkAndSet = [this, msg] (ui64 &dbLsn) { ui64 origRecoveredLsn = HugeKeeperCtx->LsnMngr->GetOriginallyRecoveredLsn(); - Y_VERIFY_S(dbLsn <= msg->DeletionLsn, HugeKeeperCtx->VCtx->VDiskLogPrefix << " Check failed:" - << " dbLsn# " << dbLsn << " origRecoveredLsn# " << origRecoveredLsn - << " recovInfo# " << HugeKeeperCtx->LocalRecoveryInfoDbg - << " msg# " << msg->ToString()); + Y_VERIFY_S(dbLsn <= msg->DeletionLsn, HugeKeeperCtx->VCtx->VDiskLogPrefix << " Check failed:" + << " dbLsn# " << dbLsn << " origRecoveredLsn# " << origRecoveredLsn + << " recovInfo# " << HugeKeeperCtx->LocalRecoveryInfoDbg + << " msg# " << msg->ToString()); dbLsn = msg->DeletionLsn; }; @@ -842,8 +842,8 @@ namespace NKikimr { Y_VERIFY(nErased == 1); // depending on SlotIsUsed... if (msg->SlotIsUsed) { - Y_VERIFY_S(State.Pers->LogPos.HugeBlobLoggedLsn < msg->RecLsn, - "pers# " << State.Pers->ToString() << " msg# " << msg->ToString()); + Y_VERIFY_S(State.Pers->LogPos.HugeBlobLoggedLsn < msg->RecLsn, + "pers# " << State.Pers->ToString() << " msg# " << msg->ToString()); // ...update HugeBlobLoggedLsn (monotonically incremented) State.Pers->LogPos.HugeBlobLoggedLsn = msg->RecLsn; } else { diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp index e60200de987..9985255928e 100644 --- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp +++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp @@ -84,9 +84,9 @@ namespace NKikimr { // TChain //////////////////////////////////////////////////////////////////////////// TMask TChain::BuildConstMask(const TString &prefix, ui32 slotsInChunk) { - Y_VERIFY_S(1 < slotsInChunk && slotsInChunk <= MaxNumberOfSlots, - prefix << "It's not a good idea to have so many slots in chunk;" - << " slotsInChunk# " << slotsInChunk); + Y_VERIFY_S(1 < slotsInChunk && slotsInChunk <= MaxNumberOfSlots, + prefix << "It's not a good idea to have so many slots in chunk;" + << " slotsInChunk# " << slotsInChunk); TMask mask; mask.Reserve(slotsInChunk); mask.Set(0, slotsInChunk); @@ -101,8 +101,8 @@ namespace NKikimr { TFreeSpace::iterator it = FreeSpace.begin(); TMask &mask = it->second; - Y_VERIFY_S(!mask.Empty(), VDiskLogPrefix << "TChain::Allocate1:" - << " id# " << id->ToString() << " State# " << ToString()); + Y_VERIFY_S(!mask.Empty(), VDiskLogPrefix << "TChain::Allocate1:" + << " id# " << id->ToString() << " State# " << ToString()); ui32 slot = mask.FirstNonZeroBit(); mask.Reset(slot); @@ -118,30 +118,30 @@ namespace NKikimr { // allocate id, but we know that this chain doesn't have free slots, so add a chunk to it void TChain::Allocate(NPrivate::TChunkSlot *id, TChunkID chunkId) { - Y_VERIFY_S(FreeSpace.empty(), VDiskLogPrefix << "Empty"); + Y_VERIFY_S(FreeSpace.empty(), VDiskLogPrefix << "Empty"); FreeSpace.emplace(chunkId, ConstMask); FreeSlotsInFreeSpace += SlotsInChunk; bool res = Allocate(id); - Y_VERIFY_S(res, VDiskLogPrefix << "TChain::Allocate2:" - << " id# " << id->ToString() << " chunkId# " << chunkId << " State# " << ToString()); + Y_VERIFY_S(res, VDiskLogPrefix << "TChain::Allocate2:" + << " id# " << id->ToString() << " chunkId# " << chunkId << " State# " << ToString()); } TFreeRes TChain::Free(const NPrivate::TChunkSlot &id) { - Y_VERIFY_S(id.GetSlotId() < SlotsInChunk && AllocatedSlots > 0, VDiskLogPrefix - << " id# " << id.ToString() << " SlotsInChunk# " << SlotsInChunk - << " AllocatedSlots# " << AllocatedSlots << " State# " << ToString()); + Y_VERIFY_S(id.GetSlotId() < SlotsInChunk && AllocatedSlots > 0, VDiskLogPrefix + << " id# " << id.ToString() << " SlotsInChunk# " << SlotsInChunk + << " AllocatedSlots# " << AllocatedSlots << " State# " << ToString()); AllocatedSlots--; ui32 chunkId = id.GetChunkId(); ui32 slotId = id.GetSlotId(); - Y_VERIFY_S(chunkId, VDiskLogPrefix << "chunkId# " << chunkId); + Y_VERIFY_S(chunkId, VDiskLogPrefix << "chunkId# " << chunkId); TFreeSpace::iterator it; auto freeFoundSlot = [&] (TFreeSpace &container, const char *containerName) { TMask &mask = it->second; Y_VERIFY_S(!mask.Get(slotId), VDiskLogPrefix << "TChain::Free: containerName# " << containerName - << " id# " << id.ToString() << " State# " << ToString()); + << " id# " << id.ToString() << " State# " << ToString()); mask.Set(slotId); ++FreeSlotsInFreeSpace; if (mask == ConstMask) { @@ -208,8 +208,8 @@ namespace NKikimr { TFreeSpace::iterator it = FreeSpace.find(chunkId); if (it != FreeSpace.end()) { TMask &mask = it->second; - Y_VERIFY_S(mask.Get(slotId), VDiskLogPrefix << "RecoveryModeAllocate:" - << " id# " << id.ToString() << " State# " << ToString()); + Y_VERIFY_S(mask.Get(slotId), VDiskLogPrefix << "RecoveryModeAllocate:" + << " id# " << id.ToString() << " State# " << ToString()); mask.Reset(slotId); if (mask.Empty()) { @@ -225,15 +225,15 @@ namespace NKikimr { } void TChain::RecoveryModeAllocate(const NPrivate::TChunkSlot &id, TChunkID chunkId) { - Y_VERIFY_S(id.GetChunkId() == chunkId && FreeSpace.find(chunkId) == FreeSpace.end(), - VDiskLogPrefix << " id# " << id.ToString() << " chunkId# " << chunkId << " State# " << ToString()); + Y_VERIFY_S(id.GetChunkId() == chunkId && FreeSpace.find(chunkId) == FreeSpace.end(), + VDiskLogPrefix << " id# " << id.ToString() << " chunkId# " << chunkId << " State# " << ToString()); FreeSpace.emplace(chunkId, ConstMask); FreeSlotsInFreeSpace += SlotsInChunk; bool res = RecoveryModeAllocate(id); - Y_VERIFY_S(res, VDiskLogPrefix << "RecoveryModeAllocate:" - << " id# " << id.ToString() << " chunkId# " << chunkId << " State# " << ToString()); + Y_VERIFY_S(res, VDiskLogPrefix << "RecoveryModeAllocate:" + << " id# " << id.ToString() << " chunkId# " << chunkId << " State# " << ToString()); } void TChain::Save(IOutputStream *s) const { @@ -252,8 +252,8 @@ namespace NKikimr { FreeSpace.clear(); ui32 slotsInChunk = 0; ::Load(s, slotsInChunk); - Y_VERIFY_S(slotsInChunk == SlotsInChunk, VDiskLogPrefix - << "slotsInChunk# " << slotsInChunk << " SlotsInChunk# " << SlotsInChunk); + Y_VERIFY_S(slotsInChunk == SlotsInChunk, VDiskLogPrefix + << "slotsInChunk# " << slotsInChunk << " SlotsInChunk# " << SlotsInChunk); ::Load(s, AllocatedSlots); ::Load(s, FreeSpace); FreeSlotsInFreeSpace = 0; @@ -331,9 +331,9 @@ namespace NKikimr { { ui32 slotSizeInBlocks = Blocks + ShiftInBlocks; ui32 blocksInChunk = chunkSize / appendBlockSize; - Y_VERIFY_S(appendBlockSize * blocksInChunk == chunkSize, VDiskLogPrefix - << "Blocks# " << Blocks << " ShiftInBlocks# " << ShiftInBlocks - << " chunkSize# " << chunkSize << " appendBlockSize# " << appendBlockSize); + Y_VERIFY_S(appendBlockSize * blocksInChunk == chunkSize, VDiskLogPrefix + << "Blocks# " << Blocks << " ShiftInBlocks# " << ShiftInBlocks + << " chunkSize# " << chunkSize << " appendBlockSize# " << appendBlockSize); SlotsInChunk = blocksInChunk / slotSizeInBlocks; SlotSize = slotSizeInBlocks * appendBlockSize; @@ -347,8 +347,8 @@ namespace NKikimr { NPrivate::TChunkSlot TChainDelegator::Convert(const TDiskPart &addr) const { ui32 slotId = addr.Offset / SlotSize; - Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix - << "slotId# " << slotId << " addr# " << addr.ToString() << " State# " << ToString()); + Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix + << "slotId# " << slotId << " addr# " << addr.ToString() << " State# " << ToString()); return NPrivate::TChunkSlot(addr.ChunkIdx, slotId); } @@ -416,12 +416,12 @@ namespace NKikimr { , Overhead(overhead) , OldMapCompatible(oldMapCompatible) { - Y_VERIFY_S(MinHugeBlobInBytes != 0 && + Y_VERIFY_S(MinHugeBlobInBytes != 0 && MinHugeBlobInBytes <= MilestoneBlobInBytes && MilestoneBlobInBytes < MaxBlobInBytes, "INVALID CONFIGURATION! (SETTINGS ARE:" - << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes - << " MilestoneBlobInBytes# " << MilestoneBlobInBytes << " ChunkSize# " << ChunkSize - << " AppendBlockSize# " << AppendBlockSize << ")"); + << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes + << " MilestoneBlobInBytes# " << MilestoneBlobInBytes << " ChunkSize# " << ChunkSize + << " AppendBlockSize# " << AppendBlockSize << ")"); BuildLayout(OldMapCompatible); } @@ -511,8 +511,8 @@ namespace NKikimr { // map size has been changed, run migration StartMode = EStartMode::Migrated; TBuiltChainDelegators b = BuildChains(MilestoneBlobInBytes); - Y_VERIFY_S(size == b.ChainDelegators.size(), "size# " << size - << " b.ChainDelegators.size()# " << b.ChainDelegators.size()); + Y_VERIFY_S(size == b.ChainDelegators.size(), "size# " << size + << " b.ChainDelegators.size()# " << b.ChainDelegators.size()); // load into temporary delegators for (auto &x : b.ChainDelegators) { @@ -535,10 +535,10 @@ namespace NKikimr { // entry point size rollback case Y_VERIFY(size > ChainDelegators.size()); ui32 curChainDelegatorsSize = ChainDelegators.size(); - Y_FAIL_S("Impossible case; MinHugeBlobInBytes# " << MinHugeBlobInBytes - << " MilestoneBlobInBytes# " << MilestoneBlobInBytes - << " loadedSize# " << size - << " curChainDelegatorsSize# " << curChainDelegatorsSize); + Y_FAIL_S("Impossible case; MinHugeBlobInBytes# " << MinHugeBlobInBytes + << " MilestoneBlobInBytes# " << MilestoneBlobInBytes + << " loadedSize# " << size + << " curChainDelegatorsSize# " << curChainDelegatorsSize); } } @@ -676,11 +676,11 @@ namespace NKikimr { Y_VERIFY(!ChainDelegators.empty()); BuildSearchTable(); - Y_VERIFY_S(GetMinREALHugeBlobInBytes() != 0, "INVALID CONFIGURATION: MinREALHugeBlobInBytes IS 0" + Y_VERIFY_S(GetMinREALHugeBlobInBytes() != 0, "INVALID CONFIGURATION: MinREALHugeBlobInBytes IS 0" << " (SETTINGS ARE: MaxBlobInBytes# " << MaxBlobInBytes - << " MinHugeBlobInBytes# " << MinHugeBlobInBytes - << " ChunkSize# " << ChunkSize - << " AppendBlockSize# " << AppendBlockSize << ')'); + << " MinHugeBlobInBytes# " << MinHugeBlobInBytes + << " ChunkSize# " << ChunkSize + << " AppendBlockSize# " << AppendBlockSize << ')'); } inline ui32 TAllChains::SizeToBlocks(ui32 size) const { @@ -721,14 +721,14 @@ namespace NKikimr { ////////////////////////////////////////////////////////////////////////////////////////// THugeSlot THeap::ConvertDiskPartToHugeSlot(const TDiskPart &addr) const { const TChainDelegator *chainD = Chains.GetChain(addr.Size); - Y_VERIFY_S(chainD && (addr.Offset / chainD->SlotSize * chainD->SlotSize == addr.Offset), VDiskLogPrefix - << "chainD# " << (chainD ? chainD->ToString() : "nullptr") << " addr# " << addr.ToString()); + Y_VERIFY_S(chainD && (addr.Offset / chainD->SlotSize * chainD->SlotSize == addr.Offset), VDiskLogPrefix + << "chainD# " << (chainD ? chainD->ToString() : "nullptr") << " addr# " << addr.ToString()); return THugeSlot(addr.ChunkIdx, addr.Offset, chainD->SlotSize); } bool THeap::Allocate(ui32 size, THugeSlot *hugeSlot, ui32 *slotSize) { TChainDelegator *chainD = Chains.GetChain(size); - Y_VERIFY_S(chainD, VDiskLogPrefix << "size# " << size << " Heap# " << ToString()); + Y_VERIFY_S(chainD, VDiskLogPrefix << "size# " << size << " Heap# " << ToString()); *slotSize = chainD->SlotSize; NPrivate::TChunkSlot id; @@ -761,7 +761,7 @@ namespace NKikimr { ui32 THeap::RemoveChunk() { if (FreeChunks.size() > FreeChunksReservation) { ui32 chunkId = GetChunkIdFromFreeChunks(); - Y_VERIFY_S(chunkId, VDiskLogPrefix << "State# " << ToString()); + Y_VERIFY_S(chunkId, VDiskLogPrefix << "State# " << ToString()); return chunkId; } else { return 0; @@ -787,7 +787,7 @@ namespace NKikimr { void THeap::RecoveryModeAllocate(const TDiskPart &addr) { ui32 size = addr.Size; TChainDelegator *chainD = Chains.GetChain(size); - Y_VERIFY_S(chainD, VDiskLogPrefix << "State# " << ToString()); + Y_VERIFY_S(chainD, VDiskLogPrefix << "State# " << ToString()); NPrivate::TChunkSlot id(chainD->Convert(addr)); bool allocated = chainD->ChainPtr->RecoveryModeAllocate(id); @@ -887,7 +887,7 @@ namespace NKikimr { // THeap: Private ////////////////////////////////////////////////////////////////////////////////////////// inline ui32 THeap::GetChunkIdFromFreeChunks() { - Y_VERIFY_S(!FreeChunks.empty(), VDiskLogPrefix << "State# " << ToString()); + Y_VERIFY_S(!FreeChunks.empty(), VDiskLogPrefix << "State# " << ToString()); TFreeChunks::iterator it = FreeChunks.begin(); ui32 chunkId = *it; FreeChunks.erase(it); @@ -896,7 +896,7 @@ namespace NKikimr { inline void THeap::PutChunkIdToFreeChunks(ui32 chunkId) { bool res = FreeChunks.insert(chunkId).second; - Y_VERIFY_S(res, VDiskLogPrefix << "State# " << ToString()); + Y_VERIFY_S(res, VDiskLogPrefix << "State# " << ToString()); } } // NHuge diff --git a/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp b/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp index 4e9f1f03c3a..8f57ff44775 100644 --- a/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp +++ b/ydb/core/blobstorage/vdisk/ingress/blobstorage_ingress.cpp @@ -37,10 +37,10 @@ namespace NKikimr { Y_VERIFY(handoff < domainsNum); // barrierIngressValueMask - ui32 barrierIngressValueMask = (1ull << totalVDisks) - 1; + ui32 barrierIngressValueMask = (1ull << totalVDisks) - 1; // barrierIngressDomainMask - ui32 barrierIngressDomainMask = (1ull << disksInDomain) - 1; + ui32 barrierIngressDomainMask = (1ull << disksInDomain) - 1; return new TIngressCache(vdiskOrderNum, totalVDisks, domainsNum, disksInDomain, handoff, barrierIngressValueMask, barrierIngressDomainMask, std::move(top)); diff --git a/ydb/core/blobstorage/vdisk/query/query_base.h b/ydb/core/blobstorage/vdisk/query/query_base.h index 092b93964f2..a4e337643dc 100644 --- a/ydb/core/blobstorage/vdisk/query/query_base.h +++ b/ydb/core/blobstorage/vdisk/query/query_base.h @@ -76,7 +76,7 @@ namespace NKikimr { void SendResponseAndDie(const TActorContext &ctx, T *self) { bool hasNotYet = false; - if (ResultSize.IsOverflow()) { + if (ResultSize.IsOverflow()) { Result->Record.SetStatus(NKikimrProto::ERROR); Result->Record.MutableResult()->Clear(); // for every 'extreme' query add ERROR result, for 'range' would empty vec for now diff --git a/ydb/core/blobstorage/vdisk/query/query_extr.cpp b/ydb/core/blobstorage/vdisk/query/query_extr.cpp index 196b6d2054a..ee7a26d0c3f 100644 --- a/ydb/core/blobstorage/vdisk/query/query_extr.cpp +++ b/ydb/core/blobstorage/vdisk/query/query_extr.cpp @@ -105,7 +105,7 @@ namespace NKikimr { void MainCycle(const TActorContext &ctx) { TQuery *query = nullptr; - while ((query = FetchNextQuery()) && !ResultSize.IsOverflow()) { + while ((query = FetchNextQuery()) && !ResultSize.IsOverflow()) { Y_VERIFY(query->PartId == 0); // only full blobs (w/o specifying a part) are allowed const ui64 *cookiePtr = query->HasCookie ? &query->CookieVal : nullptr; ResultSize.AddLogoBlobIndex(); @@ -170,7 +170,7 @@ namespace NKikimr { friend class TLevelIndexExtremeQueryViaBatcherBase; friend class TLevelIndexQueryBase; - const TBlobStorageGroupType GType; + const TBlobStorageGroupType GType; TReadBatcher Batcher; TRecordMergerCallback Merger; TActiveActors ActiveActors; @@ -312,7 +312,7 @@ namespace NKikimr { void MainCycle(const TActorContext &ctx) { TQuery *query = nullptr; - while ((query = FetchNextQuery()) && !ResultSize.IsOverflow()) { + while ((query = FetchNextQuery()) && !ResultSize.IsOverflow()) { const TLogoBlobID &fullId = query->LogoBlobID; // full blob id we are looking for const TLogoBlobID partId = TLogoBlobID(fullId, query->PartId); bool found = false; @@ -320,7 +320,7 @@ namespace NKikimr { ResultSize.AddLogoBlobIndex(); if (BlobInIndex) { - ResultSize.AddLogoBlobData(GType.PartSize(partId), query->Shift, query->Size); + ResultSize.AddLogoBlobData(GType.PartSize(partId), query->Shift, query->Size); Batcher.StartTraverse(fullId, query, query->PartId, query->Shift, query->Size); ForwardIt->PutToMerger(&Merger); Merger.Finish(); @@ -342,7 +342,7 @@ namespace NKikimr { } } - if (ResultSize.IsOverflow()) { + if (ResultSize.IsOverflow()) { SendResponseAndDie(ctx, this); } else { ui8 priority = PDiskPriority(); @@ -391,9 +391,9 @@ namespace NKikimr { : TLevelIndexExtremeQueryViaBatcherBase(queryCtx, parentId, std::move(logoBlobsSnapshot), std::move(barrierSnapshot), ev, std::move(result), replSchedulerId) , TActorBootstrapped<TLevelIndexExtremeQueryViaBatcherMergeData>() - , GType(QueryCtx->HullCtx->VCtx->Top->GType) + , GType(QueryCtx->HullCtx->VCtx->Top->GType) , Batcher(BatcherCtx) - , Merger(&Batcher, GType) + , Merger(&Batcher, GType) {} }; diff --git a/ydb/core/blobstorage/vdisk/query/query_range.cpp b/ydb/core/blobstorage/vdisk/query/query_range.cpp index 4513187b550..722b25d6542 100644 --- a/ydb/core/blobstorage/vdisk/query/query_range.cpp +++ b/ydb/core/blobstorage/vdisk/query/query_range.cpp @@ -95,7 +95,7 @@ namespace NKikimr { void MainCycleIndexOnly(const TActorContext &ctx) { if (DirectionForward) { // forward direction - while (Counter > 0 && !ResultSize.IsOverflow() && ForwardIt.Valid() && ForwardIt.GetCurKey() <= Last) { + while (Counter > 0 && !ResultSize.IsOverflow() && ForwardIt.Valid() && ForwardIt.GetCurKey() <= Last) { ForwardIt.PutToMerger(&Merger); Merger.Finish(); AddIndexOnly(ForwardIt.GetCurKey().LogoBlobID(), Merger); @@ -104,7 +104,7 @@ namespace NKikimr { } } else { // backward direction - while (Counter > 0 && !ResultSize.IsOverflow() && BackwardIt.Valid() && BackwardIt.GetCurKey() >= First) { + while (Counter > 0 && !ResultSize.IsOverflow() && BackwardIt.Valid() && BackwardIt.GetCurKey() >= First) { BackwardIt.PutToMerger(&Merger); Merger.Finish(); AddIndexOnly(BackwardIt.GetCurKey().LogoBlobID(), Merger); diff --git a/ydb/core/blobstorage/vdisk/query/query_spacetracker.h b/ydb/core/blobstorage/vdisk/query/query_spacetracker.h index 374107424c7..20d5aefa55a 100644 --- a/ydb/core/blobstorage/vdisk/query/query_spacetracker.h +++ b/ydb/core/blobstorage/vdisk/query/query_spacetracker.h @@ -15,7 +15,7 @@ namespace NKikimr { void AddLogoBlobIndex() { Size += LogoBlobIndexSize; } - + void AddLogoBlobData(ui64 blobSize, ui64 queryShift, ui64 querySize) { if (querySize) { Size += querySize; @@ -25,14 +25,14 @@ namespace NKikimr { } } - void AddAllPartsOfLogoBlob(const TBlobStorageGroupType &type, TLogoBlobID blobId) { - for (ui32 partIdx = 0; partIdx < type.TotalPartCount(); ++partIdx) { - AddLogoBlobIndex(); + void AddAllPartsOfLogoBlob(const TBlobStorageGroupType &type, TLogoBlobID blobId) { + for (ui32 partIdx = 0; partIdx < type.TotalPartCount(); ++partIdx) { + AddLogoBlobIndex(); AddLogoBlobData(type.PartSize(TLogoBlobID(blobId, partIdx + 1)), 0, 0); - } - } - - bool IsOverflow() const { + } + } + + bool IsOverflow() const { return Size > MaxProtobufSize; } @@ -40,34 +40,34 @@ namespace NKikimr { return Size; } - void Init() { - Size = VGetResultIndexSize; - } - + void Init() { + Size = VGetResultIndexSize; + } + private: // Result size we have counted ui64 Size = 0; - // Max size of TQueryResult without Buffer data + // Max size of TQueryResult without Buffer data static constexpr ui64 LogoBlobIndexSize = - 1 + 2 // Status - + 1 + 1 + 24 // BlobId - + 1 + 10 // Shift - + 1 + 10 // Size - + 1 + 10 // Buffer filed byte and varint - + 1 + 10 // Cookie - + 1 + 10 // FullDataSize - + 1 + 10; // Ingress - - // Max size of TEvVGetResult without TQueryResult data - static constexpr ui64 VGetResultIndexSize = - 1 + 2 // Status - + 1 + 10 // TQueryResult field byte and varint - + 1 + 1 + 30 // VDiskID - + 1 + 10 // Cookier - + 1 + 2 + 274 // MsgQoS - + 1 + 5 // BlockedGeneration - + 1 + 1 + 45; // Timestamp= + 1 + 2 // Status + + 1 + 1 + 24 // BlobId + + 1 + 10 // Shift + + 1 + 10 // Size + + 1 + 10 // Buffer filed byte and varint + + 1 + 10 // Cookie + + 1 + 10 // FullDataSize + + 1 + 10; // Ingress + + // Max size of TEvVGetResult without TQueryResult data + static constexpr ui64 VGetResultIndexSize = + 1 + 2 // Status + + 1 + 10 // TQueryResult field byte and varint + + 1 + 1 + 30 // VDiskID + + 1 + 10 // Cookier + + 1 + 2 + 274 // MsgQoS + + 1 + 5 // BlockedGeneration + + 1 + 1 + 45; // Timestamp= // total limit on result }; diff --git a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp index 8f5b39bab81..5ad9528fa15 100644 --- a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp +++ b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp @@ -1,205 +1,205 @@ -#include "query_spacetracker.h" - +#include "query_spacetracker.h" + #include <ydb/core/protos/base.pb.h> #include <ydb/core/protos/blobstorage.pb.h> - + #include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - - NKikimrBlobStorage::TVDiskID GetMaxTVDiskID() { - NKikimrBlobStorage::TVDiskID diskId; - diskId.SetGroupID(Max<ui32>()); - diskId.SetGroupGeneration(Max<ui32>()); - diskId.SetRing(Max<ui32>()); - diskId.SetDomain(Max<ui32>()); - diskId.SetVDisk(Max<ui32>()); - return diskId; - } - - NKikimrBlobStorage::TMessageId GetMaxTMessageId() { - NKikimrBlobStorage::TMessageId msgId; - msgId.SetSequenceId(Max<ui64>()); - msgId.SetMsgId(Max<ui64>()); - return msgId; - } - - NKikimrBlobStorage::TVDiskCostSettings GetMaxTVDiskCostSettings() { - NKikimrBlobStorage::TVDiskCostSettings costSettings; - costSettings.SetSeekTimeUs(Max<ui64>()); - costSettings.SetReadSpeedBps(Max<ui64>()); - costSettings.SetWriteSpeedBps(Max<ui64>()); - costSettings.SetReadBlockSize(Max<ui64>()); - costSettings.SetWriteBlockSize(Max<ui64>()); - costSettings.SetMinREALHugeBlobInBytes(Max<ui32>()); - return costSettings; - } - - NKikimrBlobStorage::TWindowFeedback GetMaxTWindowFeedback() { - NKikimrBlobStorage::TWindowFeedback feedback; - feedback.SetStatus(NKikimrBlobStorage::TWindowFeedback_EStatus_EStatus_MAX); - feedback.SetActualWindowSize(Max<ui64>()); - feedback.SetMaxWindowSize(Max<ui64>()); - *feedback.MutableExpectedMsgId() = GetMaxTMessageId(); - *feedback.MutableFailedMsgId() = GetMaxTMessageId(); - return feedback; - } - - NKikimrBlobStorage::TExecTimeStats GetMaxTExecTimeStats() { - NKikimrBlobStorage::TExecTimeStats stats; - stats.SetSubmitTimestamp(Max<ui64>()); - stats.SetInSenderQueue(Max<ui64>()); - stats.SetReceivedTimestamp(Max<ui64>()); - stats.SetTotal(Max<ui64>()); - stats.SetInQueue(Max<ui64>()); - stats.SetExecution(Max<ui64>()); - stats.SetHugeWriteTime(Max<ui64>()); - return stats; - } - - NKikimrBlobStorage::TMsgQoS GetMaxTMsgQoS() { - NKikimrBlobStorage::TMsgQoS msgQoS; - msgQoS.SetDeadlineSeconds(Max<ui32>()); - *msgQoS.MutableMsgId() = GetMaxTMessageId(); - msgQoS.SetCost(Max<ui64>()); - msgQoS.SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId_MAX); - msgQoS.SetIntQueueId(NKikimrBlobStorage::EVDiskInternalQueueId_MAX); - *msgQoS.MutableCostSettings() = GetMaxTVDiskCostSettings(); - msgQoS.SetSendMeCostSettings(true); - *msgQoS.MutableWindow() = GetMaxTWindowFeedback(); - msgQoS.SetVDiskLoadId(Max<ui64>()); - *msgQoS.MutableExecTimeStats() = GetMaxTExecTimeStats(); - return msgQoS; - } - - NKikimrBlobStorage::TTimestamps GetMaxTTimestamps() { - NKikimrBlobStorage::TTimestamps ts; - ts.SetSentByDSProxyUs(Max<ui64>()); - ts.SetReceivedByDSProxyUs(Max<ui64>()); - ts.SetSentByVDiskUs(Max<ui64>()); - ts.SetReceivedByVDiskUs(Max<ui64>()); - return ts; - } - - constexpr ui64 DefaultDataSize = 200'000; - constexpr ui64 DefaultQueryMaxCount = MaxProtobufSize / DefaultDataSize; - - NKikimrBlobStorage::TQueryResult GetMaxTQueryResult(ui64 size = DefaultDataSize) { - static TString data; - if (data.size() != size) { - data.resize(size, 'a'); - } - NKikimrBlobStorage::TQueryResult res; - res.SetStatus(NKikimrProto::EReplyStatus_MAX); - res.MutableBlobID(); // in only fixuint64 - res.SetShift(Max<ui64>()); - res.SetSize(Max<ui64>()); - res.SetBuffer(data); - res.SetCookie(Max<ui64>()); - res.SetFullDataSize(Max<ui64>()); - res.SetIngress(Max<ui64>()); - return res; - } - - Y_UNIT_TEST_SUITE(TQueryResultSizeTrackerTest) { - Y_UNIT_TEST(CheckWithoutQueryResult) { - TQueryResultSizeTracker resultSize; - NKikimrBlobStorage::TEvVGetResult result; - - // check without QueryResult - resultSize.Init(); - - result.SetStatus(NKikimrProto::EReplyStatus_MAX); - *result.MutableVDiskID() = GetMaxTVDiskID(); - result.SetCookie(Max<ui64>()); - *result.MutableMsgQoS() = GetMaxTMsgQoS(); - result.SetBlockedGeneration(Max<ui32>()); - *result.MutableTimestamps() = GetMaxTTimestamps(); - - UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); - UNIT_ASSERT(!resultSize.IsOverflow()); - } - - Y_UNIT_TEST(CheckOnlyQueryResult) { - TQueryResultSizeTracker resultSize; - NKikimrBlobStorage::TEvVGetResult result; - - constexpr ui64 queryCount = (MaxProtobufSize + DefaultDataSize - 1) / DefaultDataSize; - UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); - - for (ui64 idx = 0; idx < queryCount; ++idx) { - UNIT_ASSERT(!resultSize.IsOverflow()); - resultSize.AddLogoBlobIndex(); - resultSize.AddLogoBlobData(DefaultDataSize, 0, 0); - *result.AddResult() = GetMaxTQueryResult(); - UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); - } - - UNIT_ASSERT(resultSize.IsOverflow()); - } - - Y_UNIT_TEST(CheckAll) { - TQueryResultSizeTracker resultSize; - NKikimrBlobStorage::TEvVGetResult result; - - // check without QueryResult - resultSize.Init(); - - result.SetStatus(NKikimrProto::EReplyStatus_MAX); - *result.MutableVDiskID() = GetMaxTVDiskID(); - result.SetCookie(Max<ui64>()); - *result.MutableMsgQoS() = GetMaxTMsgQoS(); - result.SetBlockedGeneration(Max<ui32>()); - *result.MutableTimestamps() = GetMaxTTimestamps(); - - constexpr ui64 queryCount = (MaxProtobufSize + DefaultDataSize - 1) / DefaultDataSize; - UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); - - for (ui64 idx = 0; idx < queryCount; ++idx) { - UNIT_ASSERT(!resultSize.IsOverflow()); - resultSize.AddLogoBlobIndex(); - resultSize.AddLogoBlobData(DefaultDataSize, 0, 0); - *result.AddResult() = GetMaxTQueryResult(); - UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); - } - - UNIT_ASSERT(resultSize.IsOverflow()); - } - - void MakeTestSerializeDeserialize(ui64 defaultQueryCount, ui64 lastDataSize, ui64 protobufSize) { - NKikimrBlobStorage::TEvVGetResult result; - - result.SetStatus(NKikimrProto::EReplyStatus_MAX); - *result.MutableVDiskID() = GetMaxTVDiskID(); - result.SetCookie(Max<ui64>()); - *result.MutableMsgQoS() = GetMaxTMsgQoS(); - result.SetBlockedGeneration(Max<ui32>()); - *result.MutableTimestamps() = GetMaxTTimestamps(); - - for (ui64 idx = 0; idx < defaultQueryCount; ++idx) { + +namespace NKikimr { + + NKikimrBlobStorage::TVDiskID GetMaxTVDiskID() { + NKikimrBlobStorage::TVDiskID diskId; + diskId.SetGroupID(Max<ui32>()); + diskId.SetGroupGeneration(Max<ui32>()); + diskId.SetRing(Max<ui32>()); + diskId.SetDomain(Max<ui32>()); + diskId.SetVDisk(Max<ui32>()); + return diskId; + } + + NKikimrBlobStorage::TMessageId GetMaxTMessageId() { + NKikimrBlobStorage::TMessageId msgId; + msgId.SetSequenceId(Max<ui64>()); + msgId.SetMsgId(Max<ui64>()); + return msgId; + } + + NKikimrBlobStorage::TVDiskCostSettings GetMaxTVDiskCostSettings() { + NKikimrBlobStorage::TVDiskCostSettings costSettings; + costSettings.SetSeekTimeUs(Max<ui64>()); + costSettings.SetReadSpeedBps(Max<ui64>()); + costSettings.SetWriteSpeedBps(Max<ui64>()); + costSettings.SetReadBlockSize(Max<ui64>()); + costSettings.SetWriteBlockSize(Max<ui64>()); + costSettings.SetMinREALHugeBlobInBytes(Max<ui32>()); + return costSettings; + } + + NKikimrBlobStorage::TWindowFeedback GetMaxTWindowFeedback() { + NKikimrBlobStorage::TWindowFeedback feedback; + feedback.SetStatus(NKikimrBlobStorage::TWindowFeedback_EStatus_EStatus_MAX); + feedback.SetActualWindowSize(Max<ui64>()); + feedback.SetMaxWindowSize(Max<ui64>()); + *feedback.MutableExpectedMsgId() = GetMaxTMessageId(); + *feedback.MutableFailedMsgId() = GetMaxTMessageId(); + return feedback; + } + + NKikimrBlobStorage::TExecTimeStats GetMaxTExecTimeStats() { + NKikimrBlobStorage::TExecTimeStats stats; + stats.SetSubmitTimestamp(Max<ui64>()); + stats.SetInSenderQueue(Max<ui64>()); + stats.SetReceivedTimestamp(Max<ui64>()); + stats.SetTotal(Max<ui64>()); + stats.SetInQueue(Max<ui64>()); + stats.SetExecution(Max<ui64>()); + stats.SetHugeWriteTime(Max<ui64>()); + return stats; + } + + NKikimrBlobStorage::TMsgQoS GetMaxTMsgQoS() { + NKikimrBlobStorage::TMsgQoS msgQoS; + msgQoS.SetDeadlineSeconds(Max<ui32>()); + *msgQoS.MutableMsgId() = GetMaxTMessageId(); + msgQoS.SetCost(Max<ui64>()); + msgQoS.SetExtQueueId(NKikimrBlobStorage::EVDiskQueueId_MAX); + msgQoS.SetIntQueueId(NKikimrBlobStorage::EVDiskInternalQueueId_MAX); + *msgQoS.MutableCostSettings() = GetMaxTVDiskCostSettings(); + msgQoS.SetSendMeCostSettings(true); + *msgQoS.MutableWindow() = GetMaxTWindowFeedback(); + msgQoS.SetVDiskLoadId(Max<ui64>()); + *msgQoS.MutableExecTimeStats() = GetMaxTExecTimeStats(); + return msgQoS; + } + + NKikimrBlobStorage::TTimestamps GetMaxTTimestamps() { + NKikimrBlobStorage::TTimestamps ts; + ts.SetSentByDSProxyUs(Max<ui64>()); + ts.SetReceivedByDSProxyUs(Max<ui64>()); + ts.SetSentByVDiskUs(Max<ui64>()); + ts.SetReceivedByVDiskUs(Max<ui64>()); + return ts; + } + + constexpr ui64 DefaultDataSize = 200'000; + constexpr ui64 DefaultQueryMaxCount = MaxProtobufSize / DefaultDataSize; + + NKikimrBlobStorage::TQueryResult GetMaxTQueryResult(ui64 size = DefaultDataSize) { + static TString data; + if (data.size() != size) { + data.resize(size, 'a'); + } + NKikimrBlobStorage::TQueryResult res; + res.SetStatus(NKikimrProto::EReplyStatus_MAX); + res.MutableBlobID(); // in only fixuint64 + res.SetShift(Max<ui64>()); + res.SetSize(Max<ui64>()); + res.SetBuffer(data); + res.SetCookie(Max<ui64>()); + res.SetFullDataSize(Max<ui64>()); + res.SetIngress(Max<ui64>()); + return res; + } + + Y_UNIT_TEST_SUITE(TQueryResultSizeTrackerTest) { + Y_UNIT_TEST(CheckWithoutQueryResult) { + TQueryResultSizeTracker resultSize; + NKikimrBlobStorage::TEvVGetResult result; + + // check without QueryResult + resultSize.Init(); + + result.SetStatus(NKikimrProto::EReplyStatus_MAX); + *result.MutableVDiskID() = GetMaxTVDiskID(); + result.SetCookie(Max<ui64>()); + *result.MutableMsgQoS() = GetMaxTMsgQoS(); + result.SetBlockedGeneration(Max<ui32>()); + *result.MutableTimestamps() = GetMaxTTimestamps(); + + UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); + UNIT_ASSERT(!resultSize.IsOverflow()); + } + + Y_UNIT_TEST(CheckOnlyQueryResult) { + TQueryResultSizeTracker resultSize; + NKikimrBlobStorage::TEvVGetResult result; + + constexpr ui64 queryCount = (MaxProtobufSize + DefaultDataSize - 1) / DefaultDataSize; + UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); + + for (ui64 idx = 0; idx < queryCount; ++idx) { + UNIT_ASSERT(!resultSize.IsOverflow()); + resultSize.AddLogoBlobIndex(); + resultSize.AddLogoBlobData(DefaultDataSize, 0, 0); + *result.AddResult() = GetMaxTQueryResult(); + UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); + } + + UNIT_ASSERT(resultSize.IsOverflow()); + } + + Y_UNIT_TEST(CheckAll) { + TQueryResultSizeTracker resultSize; + NKikimrBlobStorage::TEvVGetResult result; + + // check without QueryResult + resultSize.Init(); + + result.SetStatus(NKikimrProto::EReplyStatus_MAX); + *result.MutableVDiskID() = GetMaxTVDiskID(); + result.SetCookie(Max<ui64>()); + *result.MutableMsgQoS() = GetMaxTMsgQoS(); + result.SetBlockedGeneration(Max<ui32>()); + *result.MutableTimestamps() = GetMaxTTimestamps(); + + constexpr ui64 queryCount = (MaxProtobufSize + DefaultDataSize - 1) / DefaultDataSize; + UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); + + for (ui64 idx = 0; idx < queryCount; ++idx) { + UNIT_ASSERT(!resultSize.IsOverflow()); + resultSize.AddLogoBlobIndex(); + resultSize.AddLogoBlobData(DefaultDataSize, 0, 0); + *result.AddResult() = GetMaxTQueryResult(); + UNIT_ASSERT_LE(result.ByteSizeLong(), resultSize.GetSize()); + } + + UNIT_ASSERT(resultSize.IsOverflow()); + } + + void MakeTestSerializeDeserialize(ui64 defaultQueryCount, ui64 lastDataSize, ui64 protobufSize) { + NKikimrBlobStorage::TEvVGetResult result; + + result.SetStatus(NKikimrProto::EReplyStatus_MAX); + *result.MutableVDiskID() = GetMaxTVDiskID(); + result.SetCookie(Max<ui64>()); + *result.MutableMsgQoS() = GetMaxTMsgQoS(); + result.SetBlockedGeneration(Max<ui32>()); + *result.MutableTimestamps() = GetMaxTTimestamps(); + + for (ui64 idx = 0; idx < defaultQueryCount; ++idx) { Y_PROTOBUF_SUPPRESS_NODISCARD result.ParseFromString(result.SerializeAsString()); - *result.AddResult() = GetMaxTQueryResult(); - } - *result.AddResult() = GetMaxTQueryResult(lastDataSize); - UNIT_ASSERT(result.ByteSizeLong() == protobufSize); - - NKikimrBlobStorage::TEvVGetResult desResult; + *result.AddResult() = GetMaxTQueryResult(); + } + *result.AddResult() = GetMaxTQueryResult(lastDataSize); + UNIT_ASSERT(result.ByteSizeLong() == protobufSize); + + NKikimrBlobStorage::TEvVGetResult desResult; TArrayHolder<char> buffer(new char[protobufSize]); Y_PROTOBUF_SUPPRESS_NODISCARD result.SerializeToArray(buffer.Get(), protobufSize); - UNIT_ASSERT(desResult.ParseFromArray(buffer.Get(), protobufSize)); - } - - Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSizeMinusOne) { - MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'775, MaxProtobufSize - 1); - } - - Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSize) { - MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'776, MaxProtobufSize); - } - - Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSizePlusOne) { - MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'777, MaxProtobufSize + 1); - } - } - -} // NKikimr + UNIT_ASSERT(desResult.ParseFromArray(buffer.Get(), protobufSize)); + } + + Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSizeMinusOne) { + MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'775, MaxProtobufSize - 1); + } + + Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSize) { + MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'776, MaxProtobufSize); + } + + Y_UNIT_TEST(SerializeDeserializeMaxPtotobufSizePlusOne) { + MakeTestSerializeDeserialize(DefaultQueryMaxCount, 84'777, MaxProtobufSize + 1); + } + } + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/query/ut/ya.make b/ydb/core/blobstorage/vdisk/query/ut/ya.make index 53d39a8af66..c94445e8ee3 100644 --- a/ydb/core/blobstorage/vdisk/query/ut/ya.make +++ b/ydb/core/blobstorage/vdisk/query/ut/ya.make @@ -1,7 +1,7 @@ UNITTEST_FOR(ydb/core/blobstorage/vdisk/query) - -OWNER(g:kikimr) - + +OWNER(g:kikimr) + IF (WITH_VALGRIND) FORK_SUBTESTS() TIMEOUT(1800) @@ -12,14 +12,14 @@ ELSE() TIMEOUT(600) SIZE(MEDIUM) ENDIF() - -PEERDIR( + +PEERDIR( ydb/core/blobstorage/vdisk/huge ydb/core/protos -) - -SRCS( +) + +SRCS( query_spacetracker_ut.cpp -) - -END() +) + +END() diff --git a/ydb/core/blobstorage/vdisk/query/ya.make b/ydb/core/blobstorage/vdisk/query/ya.make index 4d638cf8037..2238e892692 100644 --- a/ydb/core/blobstorage/vdisk/query/ya.make +++ b/ydb/core/blobstorage/vdisk/query/ya.make @@ -38,7 +38,7 @@ SRCS( ) END() - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp index 34be219abce..1bac48eda3a 100644 --- a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp +++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp @@ -190,17 +190,17 @@ namespace NKikimr { void Bootstrap() { QueueActorMapPtr = std::make_shared<TQueueActorMap>(); - auto replInterconnectChannel = static_cast<TInterconnectChannels::EInterconnectChannels>( + auto replInterconnectChannel = static_cast<TInterconnectChannels::EInterconnectChannels>( ReplCtx->VDiskCfg->ReplInterconnectChannel); - + const TBlobStorageGroupInfo::TTopology& topology = ReplCtx->GInfo->GetTopology(); - NBackpressure::TQueueClientId replQueueClientId(NBackpressure::EQueueClientType::ReplJob, - topology.GetOrderNumber(ReplCtx->VCtx->ShortSelfVDisk)); - CreateQueuesForVDisks(*QueueActorMapPtr, SelfId(), ReplCtx->GInfo, ReplCtx->VCtx, - ReplCtx->GInfo->GetVDisks(), ReplCtx->MonGroup.GetGroup(), - replQueueClientId, NKikimrBlobStorage::EVDiskQueueId::GetAsyncRead, - "PeerRepl", replInterconnectChannel); - + NBackpressure::TQueueClientId replQueueClientId(NBackpressure::EQueueClientType::ReplJob, + topology.GetOrderNumber(ReplCtx->VCtx->ShortSelfVDisk)); + CreateQueuesForVDisks(*QueueActorMapPtr, SelfId(), ReplCtx->GInfo, ReplCtx->VCtx, + ReplCtx->GInfo->GetVDisks(), ReplCtx->MonGroup.GetGroup(), + replQueueClientId, NKikimrBlobStorage::EVDiskQueueId::GetAsyncRead, + "PeerRepl", replInterconnectChannel); + for (const auto& [vdiskId, vdiskActorId] : ReplCtx->VDiskCfg->BaseInfo.DonorDiskIds) { TIntrusivePtr<NBackpressure::TFlowRecord> flowRecord(new NBackpressure::TFlowRecord); auto info = MakeIntrusive<TBlobStorageGroupInfo>(ReplCtx->GInfo, vdiskId, vdiskActorId); diff --git a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp index 6e4c2b66758..bf5d5851d34 100644 --- a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp @@ -1,20 +1,20 @@ #include "blob_recovery_impl.h" #include <ydb/core/blobstorage/vdisk/common/vdisk_queues.h> - + namespace NKikimr { void TBlobRecoveryActor::StartQueues() { - struct TQueueActorIdWrapper { - TQueueInfo Wrap(TActorId &&id) const { - return {std::move(id)}; - } - }; + struct TQueueActorIdWrapper { + TQueueInfo Wrap(TActorId &&id) const { + return {std::move(id)}; + } + }; const NBackpressure::TQueueClientId clientId(NBackpressure::EQueueClientType::ReplJob, Info->GetTotalVDisksNum() + Info->GetOrderNumber(VCtx->ShortSelfVDisk)); // distinct queue client id - CreateQueuesForVDisks(Queues, SelfId(), Info, VCtx, Info->GetVDisks(), Counters, - clientId, NKikimrBlobStorage::EVDiskQueueId::GetLowRead, "PeerScrub", - TInterconnectChannels::IC_BLOBSTORAGE_ASYNC_DATA, TQueueActorIdWrapper()); + CreateQueuesForVDisks(Queues, SelfId(), Info, VCtx, Info->GetVDisks(), Counters, + clientId, NKikimrBlobStorage::EVDiskQueueId::GetLowRead, "PeerScrub", + TInterconnectChannels::IC_BLOBSTORAGE_ASYNC_DATA, TQueueActorIdWrapper()); } void TBlobRecoveryActor::StopQueues() { diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp index 96b813d3172..445d0b5cb3a 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp @@ -7,8 +7,8 @@ #include "blobstorage_takedbsnap.h" #include "skeleton_loggedrec.h" #include "skeleton_vmultiput_actor.h" -#include "skeleton_vmovedpatch_actor.h" -#include "skeleton_vpatch_actor.h" +#include "skeleton_vmovedpatch_actor.h" +#include "skeleton_vpatch_actor.h" #include "skeleton_oos_logic.h" #include "skeleton_oos_tracker.h" #include "skeleton_overload_handler.h" @@ -143,129 +143,129 @@ namespace NKikimr { } //////////////////////////////////////////////////////////////////////// - // PATCH SECTOR - //////////////////////////////////////////////////////////////////////// - - void Handle(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx) { - const bool postpone = OverloadHandler->PostponeEvent(ev, ctx, this); - if (!postpone) { - PrivateHandle(ev, ctx); - } - } - - void PrivateHandle(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVMovedPatch: register actor;" - << " Event# " << ev->Get()->ToString()); - IFaceMonGroup->MovedPatchMsgs()++; - TOutOfSpaceStatus oosStatus = VCtx->GetOutOfSpaceState().GetGlobalStatusFlags(); - Register(CreateSkeletonVMovedPatchActor(SelfId(), oosStatus, ev, SkeletonFrontIDPtr, - IFaceMonGroup->MovedPatchResMsgsPtr(), Db->GetVDiskIncarnationGuid(), VCtx)); - } - - void UpdateVPatchCtx() { - if (!VPatchCtx) { - TIntrusivePtr<NMonitoring::TDynamicCounters> patchGroup = VCtx->VDiskCounters->GetSubgroup("subsystem", "patch"); - VPatchCtx = MakeIntrusive<TVPatchCtx>(); - NBackpressure::TQueueClientId patchQueueClientId(NBackpressure::EQueueClientType::VPatch, - VCtx->Top->GetOrderNumber(VCtx->ShortSelfVDisk)); - CreateQueuesForVDisks(VPatchCtx->AsyncBlobQueues, SelfId(), GInfo, VCtx, GInfo->GetVDisks(), patchGroup, - patchQueueClientId, NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob, - "PeerVPatch", TInterconnectChannels::IC_BLOBSTORAGE_ASYNC_DATA); - } - } - - void Handle(TEvProxyQueueState::TPtr &/*ev*/, const TActorContext &/*ctx*/) { - // TODO(kruall): Make it better - } - - template <typename TEvPtr> - void ReplyVPatchError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvPtr &ev) { - using namespace NErrBuilder; + // PATCH SECTOR + //////////////////////////////////////////////////////////////////////// + + void Handle(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx) { + const bool postpone = OverloadHandler->PostponeEvent(ev, ctx, this); + if (!postpone) { + PrivateHandle(ev, ctx); + } + } + + void PrivateHandle(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx) { + LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVMovedPatch: register actor;" + << " Event# " << ev->Get()->ToString()); + IFaceMonGroup->MovedPatchMsgs()++; + TOutOfSpaceStatus oosStatus = VCtx->GetOutOfSpaceState().GetGlobalStatusFlags(); + Register(CreateSkeletonVMovedPatchActor(SelfId(), oosStatus, ev, SkeletonFrontIDPtr, + IFaceMonGroup->MovedPatchResMsgsPtr(), Db->GetVDiskIncarnationGuid(), VCtx)); + } + + void UpdateVPatchCtx() { + if (!VPatchCtx) { + TIntrusivePtr<NMonitoring::TDynamicCounters> patchGroup = VCtx->VDiskCounters->GetSubgroup("subsystem", "patch"); + VPatchCtx = MakeIntrusive<TVPatchCtx>(); + NBackpressure::TQueueClientId patchQueueClientId(NBackpressure::EQueueClientType::VPatch, + VCtx->Top->GetOrderNumber(VCtx->ShortSelfVDisk)); + CreateQueuesForVDisks(VPatchCtx->AsyncBlobQueues, SelfId(), GInfo, VCtx, GInfo->GetVDisks(), patchGroup, + patchQueueClientId, NKikimrBlobStorage::EVDiskQueueId::PutAsyncBlob, + "PeerVPatch", TInterconnectChannels::IC_BLOBSTORAGE_ASYNC_DATA); + } + } + + void Handle(TEvProxyQueueState::TPtr &/*ev*/, const TActorContext &/*ctx*/) { + // TODO(kruall): Make it better + } + + template <typename TEvPtr> + void ReplyVPatchError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvPtr &ev) { + using namespace NErrBuilder; std::unique_ptr<IEventBase> res = ErroneousResult(VCtx, status, errorReason, ev, TActivationContext::Now(), - SkeletonFrontIDPtr, SelfVDiskId, Db->GetVDiskIncarnationGuid(), GInfo); + SkeletonFrontIDPtr, SelfVDiskId, Db->GetVDiskIncarnationGuid(), GInfo); SendReply(TActivationContext::AsActorContext(), std::move(res), ev, BS_VDISK_PATCH); - } - - void Handle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { - const bool postpone = OverloadHandler->PostponeEvent(ev, ctx, this); - if (!postpone) { - PrivateHandle(ev, ctx); - } - } - - void PrivateHandle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { - TInstant now = ctx.Now(); - if (!EnableVPatch.Update(now)) { - ReplyVPatchError(NKikimrProto::ERROR, "VPatch is disabled", ev); - return; - } - - TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetPatchedBlobId()); - - if (VPatchActors.count(patchedBlobId)) { - ReplyVPatchError(NKikimrProto::ERROR, "The patching request already is running", ev); - return; - } - - LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: register actor;" - << " Event# " << ev->Get()->ToString()); - IFaceMonGroup->PatchStartMsgs()++; - UpdateVPatchCtx(); + } + + void Handle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { + const bool postpone = OverloadHandler->PostponeEvent(ev, ctx, this); + if (!postpone) { + PrivateHandle(ev, ctx); + } + } + + void PrivateHandle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { + TInstant now = ctx.Now(); + if (!EnableVPatch.Update(now)) { + ReplyVPatchError(NKikimrProto::ERROR, "VPatch is disabled", ev); + return; + } + + TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetPatchedBlobId()); + + if (VPatchActors.count(patchedBlobId)) { + ReplyVPatchError(NKikimrProto::ERROR, "The patching request already is running", ev); + return; + } + + LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: register actor;" + << " Event# " << ev->Get()->ToString()); + IFaceMonGroup->PatchStartMsgs()++; + UpdateVPatchCtx(); std::unique_ptr<IActor> actor{CreateSkeletonVPatchActor(SelfId(), GInfo->Type, ev, now, SkeletonFrontIDPtr, - IFaceMonGroup->PatchFoundPartsMsgsPtr(), IFaceMonGroup->PatchResMsgsPtr(), VPatchCtx, + IFaceMonGroup->PatchFoundPartsMsgsPtr(), IFaceMonGroup->PatchResMsgsPtr(), VPatchCtx, VCtx->VDiskLogPrefix, Db->GetVDiskIncarnationGuid())}; TActorId vPatchActor = Register(actor.release()); - VPatchActors.emplace(patchedBlobId, vPatchActor); - } - - template <typename TEvDiffPtr> - void HandleVPatchDiffResending(TEvDiffPtr &ev, const TActorContext &ctx) { - if constexpr (std::is_same_v<TEvDiffPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { - LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: recieve diff;" - << " Event# " << ev->Get()->ToString()); - IFaceMonGroup->PatchDiffMsgs()++; - } - if constexpr (std::is_same_v<TEvDiffPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { - LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: recieve xor diff;" - << " Event# " << ev->Get()->ToString()); - IFaceMonGroup->PatchXorDiffMsgs()++; - } - TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetPatchedPartBlobId()).FullID(); - auto it = VPatchActors.find(patchedBlobId); - if (it != VPatchActors.end()) { + VPatchActors.emplace(patchedBlobId, vPatchActor); + } + + template <typename TEvDiffPtr> + void HandleVPatchDiffResending(TEvDiffPtr &ev, const TActorContext &ctx) { + if constexpr (std::is_same_v<TEvDiffPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { + LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: recieve diff;" + << " Event# " << ev->Get()->ToString()); + IFaceMonGroup->PatchDiffMsgs()++; + } + if constexpr (std::is_same_v<TEvDiffPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { + LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: recieve xor diff;" + << " Event# " << ev->Get()->ToString()); + IFaceMonGroup->PatchXorDiffMsgs()++; + } + TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetPatchedPartBlobId()).FullID(); + auto it = VPatchActors.find(patchedBlobId); + if (it != VPatchActors.end()) { TActivationContext::Send(ev->Forward(it->second)); - } else { - ReplyVPatchError(NKikimrProto::ERROR, "VPatchActor doesn't exist", ev); - } - } - - void Handle(TEvVPatchDyingRequest::TPtr &ev) { - auto it = VPatchActors.find(ev->Get()->PatchedBlobId); - if (it != VPatchActors.end()) { - VPatchActors.erase(it); - } - Send(ev->Sender, new TEvVPatchDyingConfirm); - } - - //////////////////////////////////////////////////////////////////////// + } else { + ReplyVPatchError(NKikimrProto::ERROR, "VPatchActor doesn't exist", ev); + } + } + + void Handle(TEvVPatchDyingRequest::TPtr &ev) { + auto it = VPatchActors.find(ev->Get()->PatchedBlobId); + if (it != VPatchActors.end()) { + VPatchActors.erase(it); + } + Send(ev->Sender, new TEvVPatchDyingConfirm); + } + + //////////////////////////////////////////////////////////////////////// // MULTIPUT SECTOR //////////////////////////////////////////////////////////////////////// void ReplyError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx, TInstant now) { - using namespace NErrBuilder; + using namespace NErrBuilder; std::unique_ptr<IEventBase> res(ErroneousResult(VCtx, status, errorReason, ev, now, SkeletonFrontIDPtr, SelfVDiskId, Db->GetVDiskIncarnationGuid(), GInfo)); SendReply(ctx, std::move(res), ev, BS_VDISK_PUT); - } - + } + void ReplyError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx, TInstant now, const TBatchedVec<NKikimrProto::EReplyStatus> &statuses) { - using namespace NErrBuilder; + using namespace NErrBuilder; std::unique_ptr<IEventBase> res(ErroneousResult(VCtx, status, errorReason, ev, now, SkeletonFrontIDPtr, SelfVDiskId, statuses, Db->GetVDiskIncarnationGuid(), GInfo)); SendReply(ctx, std::move(res), ev, BS_VDISK_PUT); - } - + } + void Handle(TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx) { const bool postpone = OverloadHandler->PostponeEvent(ev, ctx, this); if (!postpone) { @@ -273,98 +273,98 @@ namespace NKikimr { } } - struct TVPutInfo { - TRope Buffer = {}; - TLogoBlobID BlobId = {}; - TIngress Ingress = {}; - TLsnSeg Lsn = {}; - THullCheckStatus HullStatus; - bool IsHugeBlob = false; - - TVPutInfo(TLogoBlobID blobId, TRope &&buffer) - : Buffer(std::move(buffer)) - , BlobId(blobId) - , HullStatus({NKikimrProto::UNKNOWN, 0 ,false}) - {} - }; - + struct TVPutInfo { + TRope Buffer = {}; + TLogoBlobID BlobId = {}; + TIngress Ingress = {}; + TLsnSeg Lsn = {}; + THullCheckStatus HullStatus; + bool IsHugeBlob = false; + + TVPutInfo(TLogoBlobID blobId, TRope &&buffer) + : Buffer(std::move(buffer)) + , BlobId(blobId) + , HullStatus({NKikimrProto::UNKNOWN, 0 ,false}) + {} + }; + void UpdatePDiskWriteBytes(size_t size) { *PDiskWriteBytes += size; // actual size for small blobs may be up to one block, but it may be // batched along with other VDisk log entries on the PDisk } - TLoggedRecVPut* CreateLoggedRec(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, + TLoggedRecVPut* CreateLoggedRec(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, const TIngress &ingress, TRope &&buffer, std::unique_ptr<TEvBlobStorage::TEvVPutResult> res, const TActorId &sender, ui64 cookie) - { + { return new TLoggedRecVPut(seg, confirmSyncLogAlso, id, ingress, std::move(buffer), std::move(res), sender, cookie); - } - - TLoggedRecVMultiPutItem* CreateLoggedRec(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, + } + + TLoggedRecVMultiPutItem* CreateLoggedRec(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, const TIngress &ingress, TRope &&buffer, std::unique_ptr<TEvVMultiPutItemResult> res, const TActorId &sender, ui64 cookie) - { + { return new TLoggedRecVMultiPutItem(seg, confirmSyncLogAlso, id, ingress, std::move(buffer), std::move(res), - sender, cookie); - } - - template <typename TEvResult> + sender, cookie); + } + + template <typename TEvResult> std::unique_ptr<NPDisk::TEvLog> CreatePutLogEvent(const TActorContext &ctx, TString evPrefix, NActors::TActorId sender, - ui64 cookie, NLWTrace::TOrbit &&orbit, TVPutInfo &info, + ui64 cookie, NLWTrace::TOrbit &&orbit, TVPutInfo &info, std::unique_ptr<TEvResult> result) - { - Y_VERIFY_DEBUG(info.HullStatus.Status == NKikimrProto::OK); - const TLogoBlobID &id = info.BlobId; - TRope &buffer = info.Buffer; - const TLsnSeg &seg = info.Lsn; - const TIngress &ingress = info.Ingress; - -#ifdef OPTIMIZE_SYNC - // nothing to do, don't create synclog record + { + Y_VERIFY_DEBUG(info.HullStatus.Status == NKikimrProto::OK); + const TLogoBlobID &id = info.BlobId; + TRope &buffer = info.Buffer; + const TLsnSeg &seg = info.Lsn; + const TIngress &ingress = info.Ingress; + +#ifdef OPTIMIZE_SYNC + // nothing to do, don't create synclog record std::unique_ptr<NSyncLog::TEvSyncLogPut> syncLogMsg; -#else +#else std::unique_ptr<NSyncLog::TEvSyncLogPut> syncLogMsg( - new NSyncLog::TEvSyncLogPut(Db->GType, seg.Point(), TLogoBlobID(id, 0), info.Ingress)); -#endif - + new NSyncLog::TEvSyncLogPut(Db->GType, seg.Point(), TLogoBlobID(id, 0), info.Ingress)); +#endif + // prepare message to recovery log TString dataToWrite = TPutRecoveryLogRecOpt::Serialize(Db->GType, id, buffer); - LOG_DEBUG_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix - << evPrefix << ": userDataSize# " << buffer.GetSize() - << " writtenSize# " << dataToWrite.size() - << " channel# " << id.Channel() - << " Marker# BSVS04"); + LOG_DEBUG_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix + << evPrefix << ": userDataSize# " << buffer.GetSize() + << " writtenSize# " << dataToWrite.size() + << " channel# " << id.Channel() + << " Marker# BSVS04"); UpdatePDiskWriteBytes(dataToWrite.size()); - - bool confirmSyncLogAlso = static_cast<bool>(syncLogMsg); - intptr_t loggedRecId = LoggedRecsVault.Put( + + bool confirmSyncLogAlso = static_cast<bool>(syncLogMsg); + intptr_t loggedRecId = LoggedRecsVault.Put( CreateLoggedRec(seg, confirmSyncLogAlso, id, ingress, std::move(buffer), std::move(result), sender, cookie)); - void *loggedRecCookie = reinterpret_cast<void *>(loggedRecId); - // create log msg + void *loggedRecCookie = reinterpret_cast<void *>(loggedRecId); + // create log msg auto logMsg = CreateHullUpdate(HullLogCtx, TLogSignature::SignatureLogoBlobOpt, dataToWrite, seg, loggedRecCookie, std::move(syncLogMsg), nullptr); - // send prepared message to recovery log - logMsg->Orbit = std::move(orbit); - return logMsg; - } - + // send prepared message to recovery log + logMsg->Orbit = std::move(orbit); + return logMsg; + } + std::unique_ptr<TEvHullWriteHugeBlob> CreateHullWriteHugeBlob(const TActorContext &ctx, NActors::TActorId sender, - ui64 cookie, NWilson::TTraceId &traceId, bool ignoreBlock, - NKikimrBlobStorage::EPutHandleClass handleClass, TVPutInfo &info, + ui64 cookie, NWilson::TTraceId &traceId, bool ignoreBlock, + NKikimrBlobStorage::EPutHandleClass handleClass, TVPutInfo &info, std::unique_ptr<TEvBlobStorage::TEvVPutResult> res) - { - Y_VERIFY_DEBUG(info.HullStatus.Status == NKikimrProto::OK); - WILSON_TRACE_FROM_ACTOR(ctx, *this, &traceId, EvHullWriteHugeBlobSent); + { + Y_VERIFY_DEBUG(info.HullStatus.Status == NKikimrProto::OK); + WILSON_TRACE_FROM_ACTOR(ctx, *this, &traceId, EvHullWriteHugeBlobSent); info.Buffer = TDiskBlob::Create(info.BlobId.BlobSize(), info.BlobId.PartId(), Db->GType.TotalPartCount(), std::move(info.Buffer), *Arena); UpdatePDiskWriteBytes(info.Buffer.GetSize()); return std::make_unique<TEvHullWriteHugeBlob>(sender, cookie, info.BlobId, info.Ingress, std::move(info.Buffer), ignoreBlock, handleClass, std::move(res)); - } - + } + THullCheckStatus ValidateVPut(const TActorContext &ctx, TString evPrefix, - TLogoBlobID id, ui64 bufSize, bool ignoreBlock) - { + TLogoBlobID id, ui64 bufSize, bool ignoreBlock) + { ui64 blobPartSize = 0; try { blobPartSize = GInfo->Type.PartSize(id); @@ -378,91 +378,91 @@ namespace NKikimr { << evPrefix << ": buffer size does not match with part size;" << " buffer size# " << bufSize << " PartSize# " << blobPartSize - << " id# " << id - << " Marker# BSVS01"); + << " id# " << id + << " Marker# BSVS01"); return {NKikimrProto::ERROR, "buffer size mismatch"}; } - - if (bufSize > Config->MaxLogoBlobDataSize) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << evPrefix << ": data is too large;" - << " id# " << id - << " size# " << bufSize - << " chunkSize# " << PDiskCtx->Dsk->ChunkSize - << " Marker# BSVS02"); + + if (bufSize > Config->MaxLogoBlobDataSize) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << evPrefix << ": data is too large;" + << " id# " << id + << " size# " << bufSize + << " chunkSize# " << PDiskCtx->Dsk->ChunkSize + << " Marker# BSVS02"); return {NKikimrProto::ERROR, "buffer is too large"}; - } - + } + auto status = Hull->CheckLogoBlob(ctx, id, ignoreBlock); if (status.Status != NKikimrProto::OK) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << evPrefix << ": failed to pass the Hull check;" - << " id# " << id - << " status# " << status - << " Marker# BSVS03"); - } - return status; - } - - TString GetHugeBlobsForErrorMsg(const TBatchedVec<TVPutInfo> &putsInfo) { - TStringBuilder hugeBlobs; - bool hasHugeBlob = false; - for (auto &item : putsInfo) { - if (item.IsHugeBlob) { - hugeBlobs << (hasHugeBlob ? " " : "") << "{" - << "BlobId# " << item.BlobId - << " BufferSize# " << item.Buffer.GetSize() << "}"; - hasHugeBlob = true; - } - } - return hugeBlobs; - } - + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << evPrefix << ": failed to pass the Hull check;" + << " id# " << id + << " status# " << status + << " Marker# BSVS03"); + } + return status; + } + + TString GetHugeBlobsForErrorMsg(const TBatchedVec<TVPutInfo> &putsInfo) { + TStringBuilder hugeBlobs; + bool hasHugeBlob = false; + for (auto &item : putsInfo) { + if (item.IsHugeBlob) { + hugeBlobs << (hasHugeBlob ? " " : "") << "{" + << "BlobId# " << item.BlobId + << " BufferSize# " << item.Buffer.GetSize() << "}"; + hasHugeBlob = true; + } + } + return hugeBlobs; + } + void PrivateHandle(TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx) { - WILSON_TRACE_FROM_ACTOR(ctx, *this, &ev->TraceId, EvVPutReceived, VDiskId = SelfVDiskId, - PDiskId = Config->BaseInfo.PDiskId, VDiskSlotId = Config->BaseInfo.VDiskSlotId); - IFaceMonGroup->MultiPutMsgs()++; + WILSON_TRACE_FROM_ACTOR(ctx, *this, &ev->TraceId, EvVPutReceived, VDiskId = SelfVDiskId, + PDiskId = Config->BaseInfo.PDiskId, VDiskSlotId = Config->BaseInfo.VDiskSlotId); + IFaceMonGroup->MultiPutMsgs()++; IFaceMonGroup->PutTotalBytes() += ev->GetSize(); - - NKikimrBlobStorage::TEvVMultiPut &record = ev->Get()->Record; - TInstant now = TAppData::TimeProvider->Now(); - - if (!record.ItemsSize()) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: empty multiput;" - << " event# " << ev->Get()->ToString() - << " sender actorId# " << ev->Sender - << " Marker# BSVS05"); + + NKikimrBlobStorage::TEvVMultiPut &record = ev->Get()->Record; + TInstant now = TAppData::TimeProvider->Now(); + + if (!record.ItemsSize()) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: empty multiput;" + << " event# " << ev->Get()->ToString() + << " sender actorId# " << ev->Sender + << " Marker# BSVS05"); ReplyError(NKikimrProto::ERROR, "empty multiput", ev, ctx, now); - return; - } - - TLogoBlobID firstBlobId = LogoBlobIDFromLogoBlobID(record.GetItems(0).GetBlobID()); - LWTRACK(VDiskSkeletonVMultiPutRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), - firstBlobId.TabletID(), ev->Get()->GetSumBlobSize()); - - if (!OutOfSpaceLogic->Allow(ctx, ev)) { + return; + } + + TLogoBlobID firstBlobId = LogoBlobIDFromLogoBlobID(record.GetItems(0).GetBlobID()); + LWTRACK(VDiskSkeletonVMultiPutRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), + firstBlobId.TabletID(), ev->Get()->GetSumBlobSize()); + + if (!OutOfSpaceLogic->Allow(ctx, ev)) { ReplyError(NKikimrProto::OUT_OF_SPACE, "out of space", ev, ctx, now); - return; - } - - if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: race;" - << " Marker# BSVS06"); + return; + } + + if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: race;" + << " Marker# BSVS06"); ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); - return; - } - - bool hasPostponed = false; - bool hasHugeBlob = false; - bool ignoreBlock = record.GetIgnoreBlock(); - - TBatchedVec<TVPutInfo> putsInfo; - ui64 lsnCount = 0; - for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { - auto &item = record.GetItems(itemIdx); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - putsInfo.emplace_back(blobId, ev->Get()->GetItemBuffer(itemIdx)); - TVPutInfo &info = putsInfo.back(); - + return; + } + + bool hasPostponed = false; + bool hasHugeBlob = false; + bool ignoreBlock = record.GetIgnoreBlock(); + + TBatchedVec<TVPutInfo> putsInfo; + ui64 lsnCount = 0; + for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { + auto &item = record.GetItems(itemIdx); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + putsInfo.emplace_back(blobId, ev->Get()->GetItemBuffer(itemIdx)); + TVPutInfo &info = putsInfo.back(); + try { info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, blobId.FullID()); } catch (yexception ex) { @@ -474,95 +474,95 @@ namespace NKikimr { info.HullStatus = ValidateVPut(ctx, "TEvVMultiPut", blobId, info.Buffer.GetSize(), ignoreBlock); } - if (info.HullStatus.Status == NKikimrProto::OK) { + if (info.HullStatus.Status == NKikimrProto::OK) { auto ingressOpt = TIngress::CreateIngressWithLocal(VCtx->Top.get(), VCtx->ShortSelfVDisk, blobId); - if (!ingressOpt) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: ingress mismatch;" - << " id# " << blobId - << " Marker# BSVS07"); - info.HullStatus = {NKikimrProto::ERROR, 0, false}; - } else { - info.Ingress = *ingressOpt; - } - } - hasPostponed |= info.HullStatus.Postponed; - - if (info.IsHugeBlob) { - hasHugeBlob = true; - } else { - lsnCount += (info.HullStatus.Status == NKikimrProto::OK); - } - } - - if (hasHugeBlob) { - LOG_CRIT_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix - << "TEvVMultiPut: TEvVMultiPut has huge blobs# " << GetHugeBlobsForErrorMsg(putsInfo) - << " Marker# BSVS08"); - } - - TBatchedVec<NKikimrProto::EReplyStatus> statuses; - for (auto &info : putsInfo) { - if (info.HullStatus.Postponed) { - statuses.push_back(NKikimrProto::OK); - } else { - statuses.push_back(info.HullStatus.Status); - } - } - if (!hasHugeBlob && !lsnCount && !hasPostponed) { - LOG_INFO_S(ctx, BS_VDISK_PUT, Db->VCtx->VDiskLogPrefix << "TEvVMultiPut: all items have errors" - << " Marker# BSVS09"); + if (!ingressOpt) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: ingress mismatch;" + << " id# " << blobId + << " Marker# BSVS07"); + info.HullStatus = {NKikimrProto::ERROR, 0, false}; + } else { + info.Ingress = *ingressOpt; + } + } + hasPostponed |= info.HullStatus.Postponed; + + if (info.IsHugeBlob) { + hasHugeBlob = true; + } else { + lsnCount += (info.HullStatus.Status == NKikimrProto::OK); + } + } + + if (hasHugeBlob) { + LOG_CRIT_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix + << "TEvVMultiPut: TEvVMultiPut has huge blobs# " << GetHugeBlobsForErrorMsg(putsInfo) + << " Marker# BSVS08"); + } + + TBatchedVec<NKikimrProto::EReplyStatus> statuses; + for (auto &info : putsInfo) { + if (info.HullStatus.Postponed) { + statuses.push_back(NKikimrProto::OK); + } else { + statuses.push_back(info.HullStatus.Status); + } + } + if (!hasHugeBlob && !lsnCount && !hasPostponed) { + LOG_INFO_S(ctx, BS_VDISK_PUT, Db->VCtx->VDiskLogPrefix << "TEvVMultiPut: all items have errors" + << " Marker# BSVS09"); ReplyError(NKikimrProto::OK, TString(), ev, ctx, now, statuses); - return; - } - - TOutOfSpaceStatus oosStatus = VCtx->GetOutOfSpaceState().GetGlobalStatusFlags(); - NLWTrace::TOrbit orbit = std::move(ev->Get()->Orbit); - NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->Record.GetHandleClass(); - TVDiskID vdisk = VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID()); - + return; + } + + TOutOfSpaceStatus oosStatus = VCtx->GetOutOfSpaceState().GetGlobalStatusFlags(); + NLWTrace::TOrbit orbit = std::move(ev->Get()->Orbit); + NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->Record.GetHandleClass(); + TVDiskID vdisk = VDiskIDFromVDiskID(ev->Get()->Record.GetVDiskID()); + std::unique_ptr<NPDisk::TEvMultiLog> evLogs = std::make_unique<NPDisk::TEvMultiLog>(); - ui64 cookie = ev->Cookie; - - IActor* vMultiPutActor = CreateSkeletonVMultiPutActor(SelfId(), statuses, oosStatus, ev, - SkeletonFrontIDPtr, IFaceMonGroup->MultiPutResMsgsPtr(), Db->GetVDiskIncarnationGuid()); + ui64 cookie = ev->Cookie; + + IActor* vMultiPutActor = CreateSkeletonVMultiPutActor(SelfId(), statuses, oosStatus, ev, + SkeletonFrontIDPtr, IFaceMonGroup->MultiPutResMsgsPtr(), Db->GetVDiskIncarnationGuid()); NActors::TActorId vMultiPutActorId = ctx.Register(vMultiPutActor); - - TLsnSeg lsnBatch; - if (lsnCount) { -#ifdef OPTIMIZE_SYNC - lsnBatch = Db->LsnMngr->AllocDiscreteLsnBatchForHull(lsnCount); -#else - lsnBatch = Db->LsnMngr->AllocDiscreteLsnBatchForHullAndSyncLog(lsnCount); -#endif - } - - for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { - TVPutInfo &info = putsInfo[itemIdx]; - NKikimrProto::EReplyStatus status = info.HullStatus.Status; + + TLsnSeg lsnBatch; + if (lsnCount) { +#ifdef OPTIMIZE_SYNC + lsnBatch = Db->LsnMngr->AllocDiscreteLsnBatchForHull(lsnCount); +#else + lsnBatch = Db->LsnMngr->AllocDiscreteLsnBatchForHullAndSyncLog(lsnCount); +#endif + } + + for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { + TVPutInfo &info = putsInfo[itemIdx]; + NKikimrProto::EReplyStatus status = info.HullStatus.Status; const TString& errorReason = info.HullStatus.ErrorReason; - - if (info.HullStatus.Postponed) { + + if (info.HullStatus.Postponed) { auto result = std::make_unique<TEvVMultiPutItemResult>(info.BlobId, itemIdx, status, errorReason); Hull->PostponeReplyUntilCommitted(result.release(), vMultiPutActorId, itemIdx, info.HullStatus.Lsn); - continue; - } - - if (status != NKikimrProto::OK) { - continue; - } - - if (info.IsHugeBlob) { - // pass the work to huge blob writer - NWilson::TTraceId traceId; - TInstant deadline = (record.HasMsgQoS() && record.GetMsgQoS().HasDeadlineSeconds()) ? - TInstant::Seconds(record.GetMsgQoS().GetDeadlineSeconds()) : - TInstant::Max(); - TEvBlobStorage::TEvVPut vPut(info.BlobId, TRope(), vdisk, ignoreBlock, - &itemIdx, deadline, handleClass); + continue; + } + + if (status != NKikimrProto::OK) { + continue; + } + + if (info.IsHugeBlob) { + // pass the work to huge blob writer + NWilson::TTraceId traceId; + TInstant deadline = (record.HasMsgQoS() && record.GetMsgQoS().HasDeadlineSeconds()) ? + TInstant::Seconds(record.GetMsgQoS().GetDeadlineSeconds()) : + TInstant::Max(); + TEvBlobStorage::TEvVPut vPut(info.BlobId, TRope(), vdisk, ignoreBlock, + &itemIdx, deadline, handleClass); std::unique_ptr<TEvBlobStorage::TEvVPutResult> result( - new TEvBlobStorage::TEvVPutResult(status, info.BlobId, SelfVDiskId, &itemIdx, oosStatus, now, - vPut.GetCachedByteSize(), &vPut.Record, SkeletonFrontIDPtr, nullptr, - VCtx->Histograms.GetHistogram(handleClass), info.Buffer.GetSize(), + new TEvBlobStorage::TEvVPutResult(status, info.BlobId, SelfVDiskId, &itemIdx, oosStatus, now, + vPut.GetCachedByteSize(), &vPut.Record, SkeletonFrontIDPtr, nullptr, + VCtx->Histograms.GetHistogram(handleClass), info.Buffer.GetSize(), NWilson::TTraceId(), Db->GetVDiskIncarnationGuid(), errorReason)); if (info.Buffer) { auto hugeWrite = CreateHullWriteHugeBlob(ctx, vMultiPutActorId, cookie, traceId, ignoreBlock, @@ -572,25 +572,25 @@ namespace NKikimr { ctx.Send(SelfId(), new TEvHullLogHugeBlob(0, info.BlobId, info.Ingress, TDiskPart(), ignoreBlock, vMultiPutActorId, cookie, std::move(result))); } - } else { - Y_VERIFY(lsnBatch.First <= lsnBatch.Last); - - info.Lsn = TLsnSeg(lsnBatch.First, lsnBatch.First); - lsnBatch.First++; + } else { + Y_VERIFY(lsnBatch.First <= lsnBatch.Last); + + info.Lsn = TLsnSeg(lsnBatch.First, lsnBatch.First); + lsnBatch.First++; std::unique_ptr<TEvVMultiPutItemResult> evItemResult( new TEvVMultiPutItemResult(info.BlobId, itemIdx, status, errorReason)); auto logMsg = CreatePutLogEvent(ctx, "TEvVMultiPut", vMultiPutActorId, cookie, std::move(orbit), info, std::move(evItemResult)); evLogs->AddLog(THolder<NPDisk::TEvLog>(logMsg.release())); - } - } - - // Manage PDisk scheduler weights - OverloadHandler->ActualizeWeights(ctx, Mask(EHullDbType::LogoBlobs)); - - if (lsnCount) { + } + } + + // Manage PDisk scheduler weights + OverloadHandler->ActualizeWeights(ctx, Mask(EHullDbType::LogoBlobs)); + + if (lsnCount) { ctx.Send(Db->LoggerID, evLogs.release()); - } + } } //////////////////////////////////////////////////////////////////////// @@ -623,11 +623,11 @@ namespace NKikimr { IFaceMonGroup->PutTotalBytes() += ev->GetSize(); TInstant now = TAppData::TimeProvider->Now(); NKikimrBlobStorage::TEvVPut &record = ev->Get()->Record; - const TLogoBlobID id = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + const TLogoBlobID id = LogoBlobIDFromLogoBlobID(record.GetBlobID()); LWTRACK(VDiskSkeletonVPutRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - TVPutInfo info(id, ev->Get()->GetBuffer()); - const ui64 bufSize = info.Buffer.GetSize(); + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + TVPutInfo info(id, ev->Get()->GetBuffer()); + const ui64 bufSize = info.Buffer.GetSize(); try { info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, id.FullID()); @@ -644,38 +644,38 @@ namespace NKikimr { ReplyError({NKikimrProto::OUT_OF_SPACE, "out of space", 0, false}, ev, ctx, now); return; } - - if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVPut: race; id# " << id - << " Marker# BSVS10"); + + if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVPut: race; id# " << id + << " Marker# BSVS10"); ReplyError({NKikimrProto::RACE, "group generation mismatch", 0, false}, ev, ctx, now); return; } - - info.HullStatus = ValidateVPut(ctx, "TEvVPut", id, bufSize, ignoreBlock); - if (info.HullStatus.Status != NKikimrProto::OK) { - ReplyError(info.HullStatus, ev, ctx, now); + + info.HullStatus = ValidateVPut(ctx, "TEvVPut", id, bufSize, ignoreBlock); + if (info.HullStatus.Status != NKikimrProto::OK) { + ReplyError(info.HullStatus, ev, ctx, now); return; } - + auto ingressOpt = TIngress::CreateIngressWithLocal(VCtx->Top.get(), VCtx->ShortSelfVDisk, id); - if (!ingressOpt) { - LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVPut: ingress mismatch; id# " << id - << " Marker# BSVS11"); + if (!ingressOpt) { + LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVPut: ingress mismatch; id# " << id + << " Marker# BSVS11"); ReplyError({NKikimrProto::ERROR, "ingress mismatch", 0, false}, ev, ctx, now); return; } - info.Ingress = *ingressOpt; + info.Ingress = *ingressOpt; - LOG_DEBUG_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix <<"TEvVPut: " << " result# " << ev->Get()->ToString() - << " Marker# BSVS12"); + LOG_DEBUG_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix <<"TEvVPut: " << " result# " << ev->Get()->ToString() + << " Marker# BSVS12"); - if (!info.IsHugeBlob) { + if (!info.IsHugeBlob) { #ifdef OPTIMIZE_SYNC - info.Lsn = Db->LsnMngr->AllocLsnForHull(); + info.Lsn = Db->LsnMngr->AllocLsnForHull(); #else - info.Lsn = Db->LsnMngr->AllocLsnForHullAndSyncLog(); + info.Lsn = Db->LsnMngr->AllocLsnForHullAndSyncLog(); #endif } @@ -692,8 +692,8 @@ namespace NKikimr { ctx.Send(Db->LoggerID, logMsg.release()); } else if (info.Buffer) { // pass the work to huge blob writer - NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass(); - auto hugeWrite = CreateHullWriteHugeBlob(ctx, ev->Sender, ev->Cookie, ev->TraceId, ignoreBlock, + NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass(); + auto hugeWrite = CreateHullWriteHugeBlob(ctx, ev->Sender, ev->Cookie, ev->TraceId, ignoreBlock, handleClass, info, std::move(result)); ctx.Send(Db->HugeKeeperID, hugeWrite.release()); } else { @@ -713,8 +713,8 @@ namespace NKikimr { if (status.Status != NKikimrProto::OK) { msg->Result->UpdateStatus(status.Status); // modify status in result LOG_DEBUG_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix - << "TEvVPut: realtime# false result# " << msg->Result->ToString() - << " Marker# BSVS13"); + << "TEvVPut: realtime# false result# " << msg->Result->ToString() + << " Marker# BSVS13"); if (msg->HugeBlob != TDiskPart()) { ctx.Send(Db->HugeKeeperID, new TEvHullHugeBlobLogged(msg->WriteId, msg->HugeBlob, 0, false)); } @@ -832,8 +832,8 @@ namespace NKikimr { // FIXME: check PartId() is not null and is not too large LOG_DEBUG_S(ctx, BS_VDISK_GET, VCtx->VDiskLogPrefix - << "TEvVGet: " << TEvBlobStorage::TEvVGet::ToString(record) - << " Marker# BSVS14"); + << "TEvVGet: " << TEvBlobStorage::TEvVGet::ToString(record) + << " Marker# BSVS14"); if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); @@ -908,8 +908,8 @@ namespace NKikimr { } LOG_DEBUG_S(ctx, BS_VDISK_BLOCK, VCtx->VDiskLogPrefix - << "TEvVBlock: tabletId# " << tabletId << " gen# " << gen - << " Marker# BSVS14"); + << "TEvVBlock: tabletId# " << tabletId << " gen# " << gen + << " Marker# BSVS14"); TLsnSeg seg; ui32 actGen = 0; @@ -925,8 +925,8 @@ namespace NKikimr { if (postponed) { Hull->PostponeReplyUntilCommitted(result.release(), ev->Sender, ev->Cookie, postponeUntilLsn); } else { - LOG_DEBUG_S(ctx, BS_VDISK_BLOCK, VCtx->VDiskLogPrefix << "TEvVBlockResult: " << result->ToString() - << " Marker# BSVS15"); + LOG_DEBUG_S(ctx, BS_VDISK_BLOCK, VCtx->VDiskLogPrefix << "TEvVBlockResult: " << result->ToString() + << " Marker# BSVS15"); SendReply(ctx, std::move(result), ev, BS_VDISK_BLOCK); } @@ -960,8 +960,8 @@ namespace NKikimr { const ui64 tabletId = record.GetTabletId(); LOG_DEBUG_S(ctx, BS_VDISK_BLOCK, VCtx->VDiskLogPrefix - << "TEvVGetBlock: tabletId# " << tabletId - << " Marker# BSVS16"); + << "TEvVGetBlock: tabletId# " << tabletId + << " Marker# BSVS16"); std::unique_ptr<TEvBlobStorage::TEvVGetBlockResult> result; if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { @@ -983,8 +983,8 @@ namespace NKikimr { } LOG_DEBUG_S(ctx, BS_VDISK_BLOCK, VCtx->VDiskLogPrefix - << "TEvVGetBlockResult: " << result->ToString() - << " Marker# BSVS17"); + << "TEvVGetBlockResult: " << result->ToString() + << " Marker# BSVS17"); SendVDiskResponse(ctx, ev->Sender, result.release(), *this, ev->Cookie); } @@ -1019,8 +1019,8 @@ namespace NKikimr { } LOG_DEBUG_S(ctx, BS_VDISK_GC, VCtx->VDiskLogPrefix - << "TEvVCollectGarbage: " << ev->Get()->ToString() - << " Marker# BSVS18"); + << "TEvVCollectGarbage: " << ev->Get()->ToString() + << " Marker# BSVS18"); TLsnSeg seg; TBarrierIngress ingress(HullCtx->IngressCache.Get()); @@ -1067,8 +1067,8 @@ namespace NKikimr { TInstant now = TAppData::TimeProvider->Now(); NKikimrBlobStorage::TEvVGetBarrier &record = ev->Get()->Record; LOG_DEBUG_S(ctx, BS_VDISK_GC, VCtx->VDiskLogPrefix - << "TEvVGetBarrier: " << ev->Get()->ToString() - << " Marker# BSVS19"); + << "TEvVGetBarrier: " << ev->Get()->ToString() + << " Marker# BSVS19"); if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); @@ -1096,8 +1096,8 @@ namespace NKikimr { void Handle(TEvBlobStorage::TEvVStatus::TPtr &ev, const TActorContext &ctx) { IFaceMonGroup->StatusMsgs()++; TInstant now = TAppData::TimeProvider->Now(); - LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVStatus" - << " Marker# BSVS20"); + LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVStatus" + << " Marker# BSVS20"); auto aid = ctx.Register(CreateStatusRequestHandler(VCtx, Db->SkeletonID, Db->SyncerID, Db->SyncLogID, IFaceMonGroup, SelfVDiskId, Db->GetVDiskIncarnationGuid(), GInfo, ev, ctx.SelfID, now, ReplDone)); ActiveActors.Insert(aid); @@ -1129,8 +1129,8 @@ namespace NKikimr { IFaceMonGroup->DbStatMsgs()++; TInstant now = TAppData::TimeProvider->Now(); const NKikimrBlobStorage::TEvVDbStat &record = ev->Get()->Record; - LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVDbStat" - << " Marker# BSVS21"); + LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVDbStat" + << " Marker# BSVS21"); if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); @@ -1187,8 +1187,8 @@ namespace NKikimr { void Handle(TEvBlobStorage::TEvVCompact::TPtr &ev, const TActorContext &ctx) { TInstant now = TAppData::TimeProvider->Now(); const NKikimrBlobStorage::TEvVCompact &record = ev->Get()->Record; - LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVCompact" - << " Marker# BSVS22"); + LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVCompact" + << " Marker# BSVS22"); if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); @@ -1254,8 +1254,8 @@ namespace NKikimr { void Handle(TEvBlobStorage::TEvVBaldSyncLog::TPtr &ev, const TActorContext &ctx) { TInstant now = TAppData::TimeProvider->Now(); const NKikimrBlobStorage::TEvVBaldSyncLog &record = ev->Get()->Record; - LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVBaldSyncLog" - << " Marker# BSVS23"); + LOG_DEBUG_S(ctx, BS_VDISK_OTHER, VCtx->VDiskLogPrefix << "TEvVBaldSyncLog" + << " Marker# BSVS23"); if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); @@ -1303,17 +1303,17 @@ namespace NKikimr { if (!SelfVDiskId.SameGroupAndGeneration(record.GetSourceVDiskID())) { auto protoVDisk = VDiskIDFromVDiskID(record.GetSourceVDiskID()); LOG_WARN_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix - << "TSkeleton::Handle(TEvBlobStorage::TEvVSyncGuid): Source:" - << " Self# " << SelfVDiskId << " Source# " << protoVDisk - << " Marker# BSVS24"); + << "TSkeleton::Handle(TEvBlobStorage::TEvVSyncGuid): Source:" + << " Self# " << SelfVDiskId << " Source# " << protoVDisk + << " Marker# BSVS24"); ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); } if (!SelfVDiskId.SameDisk(record.GetTargetVDiskID())) { auto protoVDisk = VDiskIDFromVDiskID(record.GetTargetVDiskID()); LOG_WARN_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix - << "TSkeleton::Handle(TEvBlobStorage::TEvVSyncGuid): Target:" - << " Self# " << SelfVDiskId << " Source# " << protoVDisk - << " Marker# BSVS25"); + << "TSkeleton::Handle(TEvBlobStorage::TEvVSyncGuid): Target:" + << " Self# " << SelfVDiskId << " Source# " << protoVDisk + << " Marker# BSVS25"); ReplyError(NKikimrProto::RACE, "group generation mismatch", ev, ctx, now); } @@ -1499,8 +1499,8 @@ namespace NKikimr { const TEvRecoveredHugeBlob *msg = ev->Get(); const TLogoBlobID& id = msg->Id; - LOG_DEBUG_S(ctx, BS_REPL, VCtx->VDiskLogPrefix << "TSkeleton::Handle(TEvRecoveredHugeBlob): id# " << id - << " Marker# BSVS26"); + LOG_DEBUG_S(ctx, BS_REPL, VCtx->VDiskLogPrefix << "TSkeleton::Handle(TEvRecoveredHugeBlob): id# " << id + << " Marker# BSVS26"); TRope buf = std::move(msg->Data); const ui64 bufSize = buf.GetSize(); @@ -1528,8 +1528,8 @@ namespace NKikimr { for (const TLogoBlobID& logoBlobId : msg->Phantoms) { LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix - << "adding DoNotKeep to phantom LogoBlobId# " << logoBlobId - << " Marker# BSVS27"); + << "adding DoNotKeep to phantom LogoBlobId# " << logoBlobId + << " Marker# BSVS27"); } TLsnSeg seg = Hull->AllocateLsnForPhantoms(msg->Phantoms); @@ -1583,8 +1583,8 @@ namespace NKikimr { void SkeletonIsUpAndRunning(const TActorContext &ctx, bool runRepl = false) { Become(&TThis::StateNormal); VDiskMonGroup.VDiskState(NKikimrWhiteboard::EVDiskState::OK); - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON IS UP AND RUNNING" - << " Marker# BSVS28"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON IS UP AND RUNNING" + << " Marker# BSVS28"); // notify SkeletonFront auto msg = std::make_unique<TEvFrontRecoveryStatus>(TEvFrontRecoveryStatus::SyncGuidRecoveryDone, NKikimrProto::OK, @@ -1655,8 +1655,8 @@ namespace NKikimr { ctx.Send(*SkeletonFrontIDPtr, new TEv(TEv::UpdateIncarnationGuid, Db->GetVDiskIncarnationGuid())); // we got a recovered local DB here - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON LOCAL RECOVERY SUCCEEDED" - << " Marker# BSVS29"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON LOCAL RECOVERY SUCCEEDED" + << " Marker# BSVS29"); // run logger forwarder auto logWriter = CreateRecoveryLogWriter(PDiskCtx->PDiskId, Db->SkeletonID, @@ -1731,12 +1731,12 @@ namespace NKikimr { QueryCtx = std::make_shared<TQueryCtx>(HullCtx, PDiskCtx, SelfId()); // create overload handler - auto vMovedPatch = [this] (const TActorContext &ctx, TEvBlobStorage::TEvVMovedPatch::TPtr ev) { - this->PrivateHandle(ev, ctx); - }; - auto vPatchStart = [this] (const TActorContext &ctx, TEvBlobStorage::TEvVPatchStart::TPtr ev) { - this->PrivateHandle(ev, ctx); - }; + auto vMovedPatch = [this] (const TActorContext &ctx, TEvBlobStorage::TEvVMovedPatch::TPtr ev) { + this->PrivateHandle(ev, ctx); + }; + auto vPatchStart = [this] (const TActorContext &ctx, TEvBlobStorage::TEvVPatchStart::TPtr ev) { + this->PrivateHandle(ev, ctx); + }; auto vput = [this] (const TActorContext &ctx, TEvBlobStorage::TEvVPut::TPtr ev) { this->PrivateHandle(ev, ctx); }; @@ -1751,8 +1751,8 @@ namespace NKikimr { }; NMonGroup::TSkeletonOverloadGroup overloadMonGroup(VCtx->VDiskCounters, "subsystem", "emergency"); OverloadHandler = std::make_unique<TOverloadHandler>(VCtx, PDiskCtx, Hull, - std::move(overloadMonGroup), std::move(vMovedPatch), std::move(vPatchStart), std::move(vput), - std::move(vMultiPutHandler), std::move(loc), std::move(aoput)); + std::move(overloadMonGroup), std::move(vMovedPatch), std::move(vPatchStart), std::move(vput), + std::move(vMultiPutHandler), std::move(loc), std::move(aoput)); ScheduleWakeupEmergencyPutQueue(ctx); // actualize weights before we start @@ -1816,8 +1816,8 @@ namespace NKikimr { // Deliver CutLog that we may receive if not initialized DeliverDelayedCutLogIfAny(ctx); } else { - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON LOCAL RECOVERY FAILED" - << " Marker# BSVS30"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON LOCAL RECOVERY FAILED" + << " Marker# BSVS30"); auto phase = TEvFrontRecoveryStatus::LocalRecoveryDone; auto state = NKikimrWhiteboard::EVDiskState::LocalRecoveryError; SkeletonErrorState(ctx, phase, state); @@ -1826,8 +1826,8 @@ namespace NKikimr { void Handle(TEvSyncGuidRecoveryDone::TPtr &ev, const TActorContext &ctx) { if (ev->Get()->Status == NKikimrProto::OK) { - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON SYNC GUID RECOVERY SUCCEEDED" - << " Marker# BSVS31"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON SYNC GUID RECOVERY SUCCEEDED" + << " Marker# BSVS31"); DbBirthLsn = ev->Get()->DbBirthLsn; SkeletonIsUpAndRunning(ctx, Config->RunRepl); if (Config->RunRepl) { @@ -1841,8 +1841,8 @@ namespace NKikimr { } } } else { - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON SYNC GUID RECOVERY FAILED" - << " Marker# BSVS32"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON SYNC GUID RECOVERY FAILED" + << " Marker# BSVS32"); auto phase = TEvFrontRecoveryStatus::SyncGuidRecoveryDone; auto state = NKikimrWhiteboard::EVDiskState::SyncGuidRecoveryError; SkeletonErrorState(ctx, phase, state); @@ -2044,14 +2044,14 @@ namespace NKikimr { Y_VERIFY(!CutLogDelayedMsg); LOG_DEBUG_S(ctx, BS_LOGCUTTER, VCtx->VDiskLogPrefix << "Handle " << msg->ToString() - << " actorid# " << ctx.SelfID.ToString() - << " Marker# BSVS33"); + << " actorid# " << ctx.SelfID.ToString() + << " Marker# BSVS33"); SpreadCutLog(std::move(msg), ctx); } else { LOG_DEBUG_S(ctx, BS_LOGCUTTER, VCtx->VDiskLogPrefix << "Handle " << msg->ToString() - << " DELAYED actorid# " << ctx.SelfID.ToString() - << " Marker# BSVS34"); + << " DELAYED actorid# " << ctx.SelfID.ToString() + << " Marker# BSVS34"); CutLogDelayedMsg = std::move(msg); } } @@ -2091,8 +2091,8 @@ namespace NKikimr { LOG_DEBUG_S(ctx, BS_LOGCUTTER, VCtx->VDiskLogPrefix << "SpreadCutLog: Handle " << msg->ToString() << " DELAYED; counter# " << counter - << " actorid# " << ctx.SelfID.ToString() - << " Marker# BSVS35"); + << " actorid# " << ctx.SelfID.ToString() + << " Marker# BSVS35"); } // NOTE: We can get NPDisk::TEvCutLog when local recovery is not finished. @@ -2101,8 +2101,8 @@ namespace NKikimr { void DeliverDelayedCutLogIfAny(const TActorContext &ctx) { LOG_DEBUG_S(ctx, BS_LOGCUTTER, VCtx->VDiskLogPrefix << "DeliverDelayedCutLogIfAny: hasMsg# " << (CutLogDelayedMsg ? "true" : "false") - << " actorid# " << ctx.SelfID.ToString() - << " Marker# BSVS36"); + << " actorid# " << ctx.SelfID.ToString() + << " Marker# BSVS36"); LocalDbInitialized = true; if (CutLogDelayedMsg) { @@ -2122,9 +2122,9 @@ namespace NKikimr { GInfo = msg->NewInfo; SelfVDiskId = msg->NewVDiskId; - // clear VPatchCtx - VPatchCtx = nullptr; - + // clear VPatchCtx + VPatchCtx = nullptr; + // send command to Synclog ctx.Send(Db->SyncLogID, ev->Get()->Clone()); // send command to Syncer @@ -2162,8 +2162,8 @@ namespace NKikimr { friend class TActorBootstrapped<TSkeleton>; void Bootstrap(const TActorContext &ctx) { - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON START" - << " Marker# BSVS37"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "SKELETON START" + << " Marker# BSVS37"); Become(&TThis::StateLocalRecovery); Db->SkeletonID.Set(ctx.SelfID); // generation independent self VDisk Id @@ -2272,7 +2272,7 @@ namespace NKikimr { FFunc(TEvBlobStorage::EvScrubAwait, ForwardToScrubActor) FFunc(TEvBlobStorage::EvRecoverBlob, ForwardToScrubActor) FFunc(TEvBlobStorage::EvNonrestoredCorruptedBlobNotify, ForwardToScrubActor) - HFunc(TEvProxyQueueState, Handle) + HFunc(TEvProxyQueueState, Handle) ) STRICT_STFUNC(StateSyncGuidRecovery, @@ -2320,15 +2320,15 @@ namespace NKikimr { HFunc(TEvReportScrubStatus, Handle) HFunc(TEvRestoreCorruptedBlob, Handle) HFunc(TEvBlobStorage::TEvCaptureVDiskLayout, Handle) - HFunc(TEvProxyQueueState, Handle) + HFunc(TEvProxyQueueState, Handle) ) STRICT_STFUNC(StateNormal, - HFunc(TEvBlobStorage::TEvVMovedPatch, Handle) - HFunc(TEvBlobStorage::TEvVPatchStart, Handle) - HFunc(TEvBlobStorage::TEvVPatchDiff, HandleVPatchDiffResending) - HFunc(TEvBlobStorage::TEvVPatchXorDiff, HandleVPatchDiffResending) - hFunc(TEvVPatchDyingRequest, Handle) + HFunc(TEvBlobStorage::TEvVMovedPatch, Handle) + HFunc(TEvBlobStorage::TEvVPatchStart, Handle) + HFunc(TEvBlobStorage::TEvVPatchDiff, HandleVPatchDiffResending) + HFunc(TEvBlobStorage::TEvVPatchXorDiff, HandleVPatchDiffResending) + hFunc(TEvVPatchDyingRequest, Handle) HFunc(TEvBlobStorage::TEvVPut, Handle) HFunc(TEvBlobStorage::TEvVMultiPut, Handle) HFunc(TEvHullLogHugeBlob, Handle) @@ -2379,7 +2379,7 @@ namespace NKikimr { HFunc(TEvReportScrubStatus, Handle) HFunc(TEvRestoreCorruptedBlob, Handle) HFunc(TEvBlobStorage::TEvCaptureVDiskLayout, Handle) - HFunc(TEvProxyQueueState, Handle) + HFunc(TEvProxyQueueState, Handle) ) STRICT_STFUNC(StateDatabaseError, @@ -2401,8 +2401,8 @@ namespace NKikimr { HFunc(TEvReportScrubStatus, Handle) HFunc(TEvRestoreCorruptedBlob, Handle) HFunc(TEvBlobStorage::TEvCaptureVDiskLayout, Handle) - HFunc(TEvProxyQueueState, Handle) - hFunc(TEvVPatchDyingRequest, Handle) + HFunc(TEvProxyQueueState, Handle) + hFunc(TEvVPatchDyingRequest, Handle) ) PDISK_TERMINATE_STATE_FUNC_DEF; @@ -2432,7 +2432,7 @@ namespace NKikimr { , SyncLogIFaceGroup(VCtx->VDiskCounters, "subsystem", "synclog") , IFaceMonGroup(std::make_shared<NMonGroup::TVDiskIFaceGroup>( VCtx->VDiskCounters, "subsystem", "interface")) - , EnableVPatch(cfg->EnableVPatch) + , EnableVPatch(cfg->EnableVPatch) {} virtual ~TSkeleton() { @@ -2450,7 +2450,7 @@ namespace NKikimr { std::shared_ptr<THull> Hull; // run it after local recovery std::shared_ptr<TOutOfSpaceLogic> OutOfSpaceLogic; std::shared_ptr<TQueryCtx> QueryCtx; - TIntrusivePtr<TVPatchCtx> VPatchCtx; + TIntrusivePtr<TVPatchCtx> VPatchCtx; TIntrusivePtr<TLocalRecoveryInfo> LocalRecovInfo; // just info we got after local recovery std::unique_ptr<TOverloadHandler> OverloadHandler; TActorIDPtr SkeletonFrontIDPtr; @@ -2475,8 +2475,8 @@ namespace NKikimr { TActorId DefragId; bool HasUnreadableBlobs = false; std::unique_ptr<TVDiskCompactionState> VDiskCompactionState; - TMemorizableControlWrapper EnableVPatch; - THashMap<TLogoBlobID, TActorId> VPatchActors; + TMemorizableControlWrapper EnableVPatch; + THashMap<TLogoBlobID, TActorId> VPatchActors; }; //////////////////////////////////////////////////////////////////////////// diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonerr.h b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonerr.h index 7e899b32368..1d19dda98a1 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonerr.h +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonerr.h @@ -56,26 +56,26 @@ namespace NKikimr { return vctx->IFaceMonGroup->GetBarrierResMsgsPtr(); } - inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, - TEvBlobStorage::TEvVMovedPatch::TPtr &) { - return vctx->IFaceMonGroup->MovedPatchMsgsPtr(); - } - - inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, - TEvBlobStorage::TEvVPatchFoundParts::TPtr &) { - return vctx->IFaceMonGroup->PatchFoundPartsMsgsPtr(); - } - - inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, - TEvBlobStorage::TEvVPatchXorDiffResult::TPtr &) { - return vctx->IFaceMonGroup->PatchXorDiffResMsgsPtr(); - } - - inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, - TEvBlobStorage::TEvVPatchResult::TPtr &) { - return vctx->IFaceMonGroup->PatchResMsgsPtr(); - } - + inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, + TEvBlobStorage::TEvVMovedPatch::TPtr &) { + return vctx->IFaceMonGroup->MovedPatchMsgsPtr(); + } + + inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, + TEvBlobStorage::TEvVPatchFoundParts::TPtr &) { + return vctx->IFaceMonGroup->PatchFoundPartsMsgsPtr(); + } + + inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, + TEvBlobStorage::TEvVPatchXorDiffResult::TPtr &) { + return vctx->IFaceMonGroup->PatchXorDiffResMsgsPtr(); + } + + inline const NMonitoring::TDynamicCounters::TCounterPtr &ResultingCounterForEvent(const TVDiskContextPtr &vctx, + TEvBlobStorage::TEvVPatchResult::TPtr &) { + return vctx->IFaceMonGroup->PatchResMsgsPtr(); + } + template<typename TRequest, typename TResponse> inline void SetRacingGroupInfo(const TRequest& request, TResponse& response, const TIntrusivePtr<TBlobStorageGroupInfo>& groupInfo) { @@ -87,11 +87,11 @@ namespace NKikimr { } } - inline void SetRacingGroupInfo(const NKikimrBlobStorage::TEvVPatchXorDiff&, - NKikimrBlobStorage::TEvVPatchXorDiffResult&, const TIntrusivePtr<TBlobStorageGroupInfo>&) - {} - + inline void SetRacingGroupInfo(const NKikimrBlobStorage::TEvVPatchXorDiff&, + NKikimrBlobStorage::TEvVPatchXorDiffResult&, const TIntrusivePtr<TBlobStorageGroupInfo>&) + {} + //////////////////////////////////////////////////////////////////////////////////////////// // CreateResult -- create result from original message and status //////////////////////////////////////////////////////////////////////////////////////////// @@ -108,12 +108,12 @@ namespace NKikimr { } const auto oosStatus = vctx->GetOutOfSpaceState().GetGlobalStatusFlags(); const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr = ResultingCounterForEvent(vctx, ev); - + return std::make_unique<TEvBlobStorage::TEvVMovedPatchResult>(status, originalId, patchedId, vdiskID, cookie, oosStatus, now, ev->Get()->GetCachedByteSize(), &record, skeletonFrontIDPtr, counterPtr, nullptr, std::move(ev->TraceId), vdiskIncarnationGuid, errorReason); } - + static inline std::unique_ptr<TEvBlobStorage::TEvVPatchFoundParts> CreateResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, @@ -124,50 +124,50 @@ namespace NKikimr { TMaybe<ui64> cookie; if (record.HasCookie()) { cookie = record.GetCookie(); - } + } const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr = ResultingCounterForEvent(vctx, ev); - + return std::make_unique<TEvBlobStorage::TEvVPatchFoundParts>(status, originalId, patchedId, vdiskID, cookie, now, errorReason, &record, skeletonFrontIDPtr, counterPtr, nullptr, std::move(ev->TraceId), vdiskIncarnationGuid); } - + static inline std::unique_ptr<TEvBlobStorage::TEvVPatchResult> CreateResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVPatchDiff::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid) { - + auto &record = ev->Get()->Record; TLogoBlobID originalId = LogoBlobIDFromLogoBlobID(record.GetOriginalPartBlobId()); TLogoBlobID patchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()); TMaybe<ui64> cookie; if (record.HasCookie()) { cookie = record.GetCookie(); - } + } const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr = ResultingCounterForEvent(vctx, ev); - + auto res = std::make_unique<TEvBlobStorage::TEvVPatchResult>(status, originalId, patchedId, vdiskID, cookie, now, &record, skeletonFrontIDPtr, counterPtr, nullptr, std::move(ev->TraceId), vdiskIncarnationGuid); res->SetStatus(status, errorReason); return res; } - + static inline std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiffResult> CreateResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVPatchXorDiff::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid) { - + Y_UNUSED(errorReason, vdiskID, vdiskIncarnationGuid); auto &record = ev->Get()->Record; const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr = ResultingCounterForEvent(vctx, ev); return std::make_unique<TEvBlobStorage::TEvVPatchXorDiffResult>(status, now, &record, skeletonFrontIDPtr, counterPtr, nullptr, std::move(ev->TraceId)); } - + static inline std::unique_ptr<TEvBlobStorage::TEvVPutResult> CreateResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVPut::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid) { - + NKikimrBlobStorage::TEvVPut &record = ev->Get()->Record; const TLogoBlobID id = LogoBlobIDFromLogoBlobID(record.GetBlobID()); const ui64 bufferSizeBytes = ev->Get()->GetBufferBytes(); @@ -177,7 +177,7 @@ namespace NKikimr { const auto handleClass = record.GetHandleClass(); const NVDiskMon::TLtcHistoPtr &histoPtr = vctx->Histograms.GetHistogram(handleClass); const NMonitoring::TDynamicCounters::TCounterPtr &counterPtr = ResultingCounterForEvent(vctx, ev); - + return std::make_unique<TEvBlobStorage::TEvVPutResult>(status, id, vdiskID, cookie, oosStatus, now, ev->Get()->GetCachedByteSize(), &record, skeletonFrontIDPtr, counterPtr, histoPtr, bufferSizeBytes, std::move(ev->TraceId), vdiskIncarnationGuid, errorReason); @@ -214,47 +214,47 @@ namespace NKikimr { // NErrBuilder -- routines for error results //////////////////////////////////////////////////////////////////////////// namespace NErrBuilder { - template <typename TEvPtr> + template <typename TEvPtr> static inline std::unique_ptr<IEventBase> ErroneousResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, - TEvPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, - const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid, - const TIntrusivePtr<TBlobStorageGroupInfo>& groupInfo) - { - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr>) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalBlobId()); - LWTRACK(VDiskSkeletonFrontVMovedPatchRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, - vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - } - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalBlobId()); - LWTRACK(VDiskSkeletonFrontVPatchStartRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, - vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - } - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalPartBlobId()); - LWTRACK(VDiskSkeletonFrontVPatchDiffRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, - vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - } - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalPartBlobId()); - LWTRACK(VDiskSkeletonFrontVPatchXorDiffRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, - vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - } - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPut::TPtr>) { - TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); - LWTRACK(VDiskSkeletonFrontVPutRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, - vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); - } - + TEvPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, + const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid, + const TIntrusivePtr<TBlobStorageGroupInfo>& groupInfo) + { + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr>) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalBlobId()); + LWTRACK(VDiskSkeletonFrontVMovedPatchRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, + vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + } + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalBlobId()); + LWTRACK(VDiskSkeletonFrontVPatchStartRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, + vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + } + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalPartBlobId()); + LWTRACK(VDiskSkeletonFrontVPatchDiffRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, + vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + } + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetOriginalPartBlobId()); + LWTRACK(VDiskSkeletonFrontVPatchXorDiffRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, + vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + } + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPut::TPtr>) { + TLogoBlobID id = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetBlobID()); + LWTRACK(VDiskSkeletonFrontVPutRecieved, ev->Get()->Orbit, vctx->NodeId, vctx->GroupId, + vctx->Top->GetFailDomainOrderNumber(vctx->ShortSelfVDisk), id.TabletID(), id.BlobSize()); + } + auto result = CreateResult(vctx, status, errorReason, ev, now, skeletonFrontIDPtr, vdiskID, vdiskIncarnationGuid); - SetRacingGroupInfo(ev->Get()->Record, result->Record, groupInfo); + SetRacingGroupInfo(ev->Get()->Record, result->Record, groupInfo); return result; - } - + } + static inline std::unique_ptr<IEventBase> - ErroneousResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, + ErroneousResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, const TVDiskID &vdiskID, const TBatchedVec<NKikimrProto::EReplyStatus> &statuses, ui64 vdiskIncarnationGuid, @@ -273,9 +273,9 @@ namespace NKikimr { auto result = std::make_unique<TEvBlobStorage::TEvVMultiPutResult>(status, vdiskID, cookie, now, ev->Get()->GetCachedByteSize(), &record, skeletonFrontIDPtr, counterPtr, histoPtr, bufferSizeBytes, std::move(ev->TraceId), vdiskIncarnationGuid, errorReason); - Y_VERIFY(record.ItemsSize() == statuses.size()); - for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { - auto &item = record.GetItems(itemIdx); + Y_VERIFY(record.ItemsSize() == statuses.size()); + for (ui64 itemIdx = 0; itemIdx < record.ItemsSize(); ++itemIdx) { + auto &item = record.GetItems(itemIdx); ui64 cookieValue = 0; ui64 *cookiePtr = nullptr; if (item.HasCookie()) { @@ -291,17 +291,17 @@ namespace NKikimr { static inline std::unique_ptr<IEventBase> ErroneousResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& errorReason, - TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TInstant &now, - const TActorIDPtr &skeletonFrontIDPtr, + TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TInstant &now, + const TActorIDPtr &skeletonFrontIDPtr, const TVDiskID &vdiskID, ui64 vdiskIncarnationGuid, const TIntrusivePtr<TBlobStorageGroupInfo>& groupInfo) - { - NKikimrBlobStorage::TEvVMultiPut &record = ev->Get()->Record; - TBatchedVec<NKikimrProto::EReplyStatus> statuses(record.ItemsSize(), status); + { + NKikimrBlobStorage::TEvVMultiPut &record = ev->Get()->Record; + TBatchedVec<NKikimrProto::EReplyStatus> statuses(record.ItemsSize(), status); return ErroneousResult(vctx, status, errorReason, ev, now, skeletonFrontIDPtr, vdiskID, statuses, vdiskIncarnationGuid, groupInfo); - } - + } + static inline std::unique_ptr<IEventBase> ErroneousResult(const TVDiskContextPtr &vctx, const NKikimrProto::EReplyStatus status, const TString& /*errorReason*/, TEvBlobStorage::TEvVGet::TPtr &ev, const TInstant &now, const TActorIDPtr &skeletonFrontIDPtr, diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp index 08c32a8921e..89d125e457d 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp @@ -55,10 +55,10 @@ namespace NKikimr { void ApplyToRecord(IEventHandle& event, Func&& callback) { switch (event.GetTypeRewrite()) { #define EVENT_TYPE(EVENT) case TEvBlobStorage::EVENT::EventType: callback(static_cast<TEvBlobStorage::EVENT&>(*event.GetBase()).Record); return; - EVENT_TYPE(TEvVMovedPatch) - EVENT_TYPE(TEvVPatchStart) - EVENT_TYPE(TEvVPatchDiff) - EVENT_TYPE(TEvVPatchXorDiff) + EVENT_TYPE(TEvVMovedPatch) + EVENT_TYPE(TEvVPatchStart) + EVENT_TYPE(TEvVPatchDiff) + EVENT_TYPE(TEvVPatchXorDiff) EVENT_TYPE(TEvVPut) EVENT_TYPE(TEvVMultiPut) EVENT_TYPE(TEvVGet) @@ -517,10 +517,10 @@ namespace NKikimr { switch (ev->Type()) { #define UPDATE_WINDOW_STATUS(TYPE) case TYPE::EventType: msgQoS = static_cast<TYPE&>(*ev).Record.MutableMsgQoS(); break; // all message types that have MsgQoS structure - UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVMovedPatchResult) - UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchFoundParts) - UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchXorDiffResult) - UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchResult) + UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVMovedPatchResult) + UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchFoundParts) + UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchXorDiffResult) + UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPatchResult) UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVPutResult) UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVMultiPutResult) UPDATE_WINDOW_STATUS(TEvBlobStorage::TEvVGetResult) @@ -724,14 +724,14 @@ namespace NKikimr { } else { switch (msg->Phase) { case TEvFrontRecoveryStatus::LocalRecoveryDone: - { + { Become(&TThis::StateSyncGuidRecoveryInProgress); - TBlobStorageGroupType type = (GInfo ? GInfo->Type : TErasureType::ErasureNone); + TBlobStorageGroupType type = (GInfo ? GInfo->Type : TErasureType::ErasureNone); CostModel = std::make_unique<TCostModel>(msg->Dsk->SeekTimeUs, msg->Dsk->ReadSpeedBps, msg->Dsk->WriteSpeedBps, msg->Dsk->ReadBlockSize, msg->Dsk->WriteBlockSize, msg->HugeBlobCtx->MinREALHugeBlobInBytes, type); break; - } + } case TEvFrontRecoveryStatus::SyncGuidRecoveryDone: Become(&TThis::StateFunc); SendNotifications(ctx); @@ -940,8 +940,8 @@ namespace NKikimr { << "Access denied Type# " << Sprintf("0x%08" PRIx32, ev->GetTypeRewrite()) << " Sender# " << ev->Sender.ToString() << " OriginScopeId# " << ScopeIdToString(ev->OriginScopeId) - << " LocalScopeId# " << ScopeIdToString(AppData(ctx)->LocalScopeId.GetInterconnectScopeId()) - << " Marker# BSVSF01"); + << " LocalScopeId# " << ScopeIdToString(AppData(ctx)->LocalScopeId.GetInterconnectScopeId()) + << " Marker# BSVSF01"); ++*AccessDeniedMessages; TInstant now = TAppData::TimeProvider->Now(); FillInCostSettingsAndTimestampIfApplicable(ev->Get()->Record, now); @@ -984,26 +984,26 @@ namespace NKikimr { return val ? TInstant::Seconds(val) : TInstant::Max(); } - template <typename TRecord, typename Dimmy = void> - struct THasMsgQoS { - static constexpr bool value = false; - }; - - template <typename TRecord> - struct THasMsgQoS<TRecord, TRecord> { - static constexpr bool value = true; - - static auto Checking(TRecord &r) { - return r.MutableMsgQoS(); - } - }; - - template<typename TRecord> - void FillInCostSettingsAndTimestampIfApplicable(TRecord& record, TInstant now) const - { - if constexpr (THasMsgQoS<TRecord>::value) { - FillInCostSettingsAndTimestampIfRequired(record.MutableMsgQoS(), now); - } + template <typename TRecord, typename Dimmy = void> + struct THasMsgQoS { + static constexpr bool value = false; + }; + + template <typename TRecord> + struct THasMsgQoS<TRecord, TRecord> { + static constexpr bool value = true; + + static auto Checking(TRecord &r) { + return r.MutableMsgQoS(); + } + }; + + template<typename TRecord> + void FillInCostSettingsAndTimestampIfApplicable(TRecord& record, TInstant now) const + { + if constexpr (THasMsgQoS<TRecord>::value) { + FillInCostSettingsAndTimestampIfRequired(record.MutableMsgQoS(), now); + } } void FillInCostSettingsAndTimestampIfRequired(NKikimrBlobStorage::TMsgQoS *qos, TInstant now) const { @@ -1077,50 +1077,50 @@ namespace NKikimr { return compatibilityMatrix[extId][intId]; } - template <typename TEvPtr> - void HandlePatchEvent(TEvPtr &ev) { - const ui64 cost = CostModel->GetCost(*ev->Get()); - - auto &record = ev->Get()->Record; - - TLogoBlobID blob; - TLogoBlobID patchedBlob; - constexpr bool blobIdWithoutPartId = std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr> - || std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>; - if constexpr (blobIdWithoutPartId) { - blob = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); - patchedBlob = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); - } else { - blob = LogoBlobIDFromLogoBlobID(record.GetOriginalPartBlobId()); - patchedBlob = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()); - } - - const char* name = "Unknown"; + template <typename TEvPtr> + void HandlePatchEvent(TEvPtr &ev) { + const ui64 cost = CostModel->GetCost(*ev->Get()); + + auto &record = ev->Get()->Record; + + TLogoBlobID blob; + TLogoBlobID patchedBlob; + constexpr bool blobIdWithoutPartId = std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr> + || std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>; + if constexpr (blobIdWithoutPartId) { + blob = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); + patchedBlob = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); + } else { + blob = LogoBlobIDFromLogoBlobID(record.GetOriginalPartBlobId()); + patchedBlob = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()); + } + + const char* name = "Unknown"; TIntQueueClass *queue = IntQueueHugePutsBackground.get(); - if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr>) { - LWTRACK(VDiskSkeletonFrontVMovedPatchRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); - name = "TEvVMovedPatch"; - } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>) { - LWTRACK(VDiskSkeletonFrontVPatchStartRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); + if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVMovedPatch::TPtr>) { + LWTRACK(VDiskSkeletonFrontVMovedPatchRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); + name = "TEvVMovedPatch"; + } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchStart::TPtr>) { + LWTRACK(VDiskSkeletonFrontVPatchStartRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); queue = IntQueueFastGets.get(); - name = "TEvVPatchStart"; - } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { - LWTRACK(VDiskSkeletonFrontVPatchDiffRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); - name = "TEvVPatchDiff"; - } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { - LWTRACK(VDiskSkeletonFrontVPatchXorDiffRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, - VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); - name = "TEvVPatchXorDiff"; - } - - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix - << name << ": received;" << " OriginalBlobId# " << blob << " PatchedBlobId# " << patchedBlob); - HandleRequestWithQoS(TActivationContext::AsActorContext(), ev, name, cost, *queue); - } - + name = "TEvVPatchStart"; + } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { + LWTRACK(VDiskSkeletonFrontVPatchDiffRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); + name = "TEvVPatchDiff"; + } else if constexpr (std::is_same_v<TEvPtr, TEvBlobStorage::TEvVPatchXorDiff::TPtr>) { + LWTRACK(VDiskSkeletonFrontVPatchXorDiffRecieved, ev->Get()->Orbit, VCtx->NodeId, VCtx->GroupId, + VCtx->Top->GetFailDomainOrderNumber(VCtx->ShortSelfVDisk), blob.TabletID(), blob.BlobSize()); + name = "TEvVPatchXorDiff"; + } + + LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix + << name << ": received;" << " OriginalBlobId# " << blob << " PatchedBlobId# " << patchedBlob); + HandleRequestWithQoS(TActivationContext::AsActorContext(), ev, name, cost, *queue); + } + void Handle(TEvBlobStorage::TEvVPut::TPtr &ev, const TActorContext &ctx) { bool logPutInternalQueue = true; const ui64 cost = CostModel->GetCost(*ev->Get(), &logPutInternalQueue); @@ -1270,11 +1270,11 @@ namespace NKikimr { template <typename TPtr> void SetReceivedTime(TPtr& ev) { using TRecord = decltype(ev->Get()->Record); - if constexpr (std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVMovedPatch> - || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchStart> - || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchDiff> - || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchXorDiff> - || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPut> + if constexpr (std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVMovedPatch> + || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchStart> + || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchDiff> + || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPatchXorDiff> + || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVPut> || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVMultiPut> || std::is_convertible_v<TRecord, NKikimrBlobStorage::TEvVGet>) { @@ -1393,9 +1393,9 @@ namespace NKikimr { } // all checks passed - LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "VDisk Generation Change success;" - << " new VDiskId# " << vdiskId - << " Marker# BSVSF02"); + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "VDisk Generation Change success;" + << " new VDiskId# " << vdiskId + << " Marker# BSVSF02"); // update GroupInfo-related fields GInfo = info; @@ -1418,8 +1418,8 @@ namespace NKikimr { void Handle(TEvPDiskErrorStateChange::TPtr &ev, const TActorContext &ctx) { LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix << "SkeletonFront: got TEvPDiskErrorStateChange;" - << " state# " << TPDiskErrorState::StateToString(ev->Get()->State) - << " Marker# BSVSF03"); + << " state# " << TPDiskErrorState::StateToString(ev->Get()->State) + << " Marker# BSVSF03"); // switch skeleton state to PDiskError @@ -1438,11 +1438,11 @@ namespace NKikimr { for (auto *q : {&IntQueueAsyncGets, &IntQueueFastGets, &IntQueueDiscover, &IntQueueLowGets, &IntQueueLogPuts, &IntQueueHugePutsForeground, &IntQueueHugePutsBackground}) { (*q)->DropWithError(ctx, *this); - } + } // drop external queues DisconnectClients(ctx); } - + void DisconnectClients(const TActorContext& ctx) { const auto& base = Config->BaseInfo; const TActorId& serviceId = MakeBlobStorageVDiskID(SelfId().NodeId(), base.PDiskId, base.VDiskSlotId); @@ -1540,10 +1540,10 @@ namespace NKikimr { // while local db recovery is in progress, we use this state function to handle requests STRICT_STFUNC(StateLocalRecoveryInProgress, - HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVPut, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVMultiPut, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVGet, DatabaseNotReadyHandle) @@ -1579,10 +1579,10 @@ namespace NKikimr { // while recovering sync guid we use this state function to handle requests STRICT_STFUNC(StateSyncGuidRecoveryInProgress, - HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseNotReadyHandle) - HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseNotReadyHandle) + HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVPut, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVMultiPut, DatabaseNotReadyHandle) HFunc(TEvBlobStorage::TEvVGet, DatabaseNotReadyHandle) @@ -1621,10 +1621,10 @@ namespace NKikimr { ) STRICT_STFUNC(StateDatabaseError, - HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseErrorHandle) - HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseErrorHandle) - HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseErrorHandle) - HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseErrorHandle) + HFunc(TEvBlobStorage::TEvVMovedPatch, DatabaseErrorHandle) + HFunc(TEvBlobStorage::TEvVPatchStart, DatabaseErrorHandle) + HFunc(TEvBlobStorage::TEvVPatchDiff, DatabaseErrorHandle) + HFunc(TEvBlobStorage::TEvVPatchXorDiff, DatabaseErrorHandle) HFunc(TEvBlobStorage::TEvVPut, DatabaseErrorHandle) HFunc(TEvBlobStorage::TEvVMultiPut, DatabaseErrorHandle) HFunc(TEvBlobStorage::TEvVGet, DatabaseErrorHandle) @@ -1663,33 +1663,33 @@ namespace NKikimr { // Events checking: RACE and access control //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - template <typename TEv> - static constexpr bool IsPatchEvent = std::is_same_v<TEv, TEvBlobStorage::TEvVMovedPatch> - || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchStart> - || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchDiff> - || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchXorDiff>; + template <typename TEv> + static constexpr bool IsPatchEvent = std::is_same_v<TEv, TEvBlobStorage::TEvVMovedPatch> + || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchStart> + || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchDiff> + || std::is_same_v<TEv, TEvBlobStorage::TEvVPatchXorDiff>; - template <typename TEv> - static constexpr bool IsWithoutQoS = std::is_same_v<TEv, TEvBlobStorage::TEvVStatus> - || std::is_same_v<TEv, TEvBlobStorage::TEvVDbStat> - || std::is_same_v<TEv, TEvBlobStorage::TEvVCompact> + template <typename TEv> + static constexpr bool IsWithoutQoS = std::is_same_v<TEv, TEvBlobStorage::TEvVStatus> + || std::is_same_v<TEv, TEvBlobStorage::TEvVDbStat> + || std::is_same_v<TEv, TEvBlobStorage::TEvVCompact> || std::is_same_v<TEv, TEvBlobStorage::TEvVDefrag> - || std::is_same_v<TEv, TEvBlobStorage::TEvVBaldSyncLog>; + || std::is_same_v<TEv, TEvBlobStorage::TEvVBaldSyncLog>; template <typename TEv> static constexpr bool IsValidatable = std::is_same_v<TEv, TEvBlobStorage::TEvVMultiPut> || std::is_same_v<TEv, TEvBlobStorage::TEvVGet> || std::is_same_v<TEv, TEvBlobStorage::TEvVPut>; - template<typename TEventType> - void CheckExecute(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) { - if constexpr (IsPatchEvent<TEventType>) { - HandlePatchEvent(ev); - } else if constexpr (IsWithoutQoS<TEventType>) { - HandleRequestWithoutQoS(ev, ctx); - } else { - Handle(ev, ctx); - } + template<typename TEventType> + void CheckExecute(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) { + if constexpr (IsPatchEvent<TEventType>) { + HandlePatchEvent(ev); + } else if constexpr (IsWithoutQoS<TEventType>) { + HandleRequestWithoutQoS(ev, ctx); + } else { + Handle(ev, ctx); + } } template <typename TEv> @@ -1700,8 +1700,8 @@ namespace NKikimr { return true; } - template <typename TEventType> - void Check(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) { + template <typename TEventType> + void Check(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (!SelfVDiskId.SameDisk(record.GetVDiskID())) { return Reply(ev, ctx, NKikimrProto::RACE, "group generation mismatch", TAppData::TimeProvider->Now()); @@ -1734,10 +1734,10 @@ namespace NKikimr { } STRICT_STFUNC(StateFunc, - HFunc(TEvBlobStorage::TEvVMovedPatch, Check) - HFunc(TEvBlobStorage::TEvVPatchStart, Check) - HFunc(TEvBlobStorage::TEvVPatchDiff, Check) - HFunc(TEvBlobStorage::TEvVPatchXorDiff, Check) + HFunc(TEvBlobStorage::TEvVMovedPatch, Check) + HFunc(TEvBlobStorage::TEvVPatchStart, Check) + HFunc(TEvBlobStorage::TEvVPatchDiff, Check) + HFunc(TEvBlobStorage::TEvVPatchXorDiff, Check) HFunc(TEvBlobStorage::TEvVPut, ValidateEvent) HFunc(TEvBlobStorage::TEvVMultiPut, ValidateEvent) HFunc(TEvBlobStorage::TEvVGet, ValidateEvent) @@ -1786,10 +1786,10 @@ namespace NKikimr { NKikimrProto::EReplyStatus status, const TString& errorReason, TInstant now, const TWindowStatus &wstatus) { switch (ev->GetTypeRewrite()) { - HFuncStatus(TEvBlobStorage::TEvVMovedPatch, status, errorReason, now, wstatus); - HFuncStatus(TEvBlobStorage::TEvVPatchStart, status, errorReason, now, wstatus); - HFuncStatus(TEvBlobStorage::TEvVPatchDiff, status, errorReason, now, wstatus); - HFuncStatus(TEvBlobStorage::TEvVPatchXorDiff, status, errorReason, now, wstatus); + HFuncStatus(TEvBlobStorage::TEvVMovedPatch, status, errorReason, now, wstatus); + HFuncStatus(TEvBlobStorage::TEvVPatchStart, status, errorReason, now, wstatus); + HFuncStatus(TEvBlobStorage::TEvVPatchDiff, status, errorReason, now, wstatus); + HFuncStatus(TEvBlobStorage::TEvVPatchXorDiff, status, errorReason, now, wstatus); HFuncStatus(TEvBlobStorage::TEvVPut, status, errorReason, now, wstatus); HFuncStatus(TEvBlobStorage::TEvVMultiPut, status, errorReason, now, wstatus); HFuncStatus(TEvBlobStorage::TEvVGet, status, errorReason, now, wstatus); diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_syncfullhandler.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_syncfullhandler.cpp index 11f708e717d..baf982c98ea 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_syncfullhandler.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_syncfullhandler.cpp @@ -49,8 +49,8 @@ namespace NKikimr { LOG_DEBUG_S(ctx, BS_SYNCJOB, Db->VCtx->VDiskLogPrefix << "TVSyncFullHandler: Bootstrap: fromVDisk# " << VDiskIDFromVDiskID(Record.GetSourceVDiskID()) - << " fromSyncState# " << clientSyncState.ToString() - << " Marker# BSVSFH01"); + << " fromSyncState# " << clientSyncState.ToString() + << " Marker# BSVSFH01"); // check that the disk is from this group @@ -67,10 +67,10 @@ namespace NKikimr { // check disk guid and start from the beginning if it has changed if (Db->GetVDiskIncarnationGuid() != clientSyncState.Guid) { LOG_DEBUG_S(ctx, BS_SYNCJOB, Db->VCtx->VDiskLogPrefix - << "TVSyncFullHandler: GUID CHANGED;" - << " SourceVDisk# " << SourceVDisk - << " DbBirthLsn# " << DbBirthLsn - << " Marker# BSVSFH02"); + << "TVSyncFullHandler: GUID CHANGED;" + << " SourceVDisk# " << SourceVDisk + << " DbBirthLsn# " << DbBirthLsn + << " Marker# BSVSFH02"); auto result = std::make_unique<TEvBlobStorage::TEvVSyncFullResult>(NKikimrProto::NODATA, SelfVDiskId, TSyncState(Db->GetVDiskIncarnationGuid(), DbBirthLsn), Record.GetCookie(), Now, IFaceMonGroup->SyncFullResMsgsPtr(), nullptr, std::move(Ev->TraceId), Ev->GetChannel()); @@ -113,8 +113,8 @@ namespace NKikimr { LOG_DEBUG_S(ctx, BS_SYNCJOB, Db->VCtx->VDiskLogPrefix << "TVSyncFullHandler: ourConfirmedLsn# " << ConfirmedLsn << " syncedLsn# " << syncedLsn - << " SourceVDisk# " << SourceVDisk - << " Marker# BSVSFH03"); + << " SourceVDisk# " << SourceVDisk + << " Marker# BSVSFH03"); IActor *actor = CreateHullSyncFullActor( Db->Config, diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.cpp index ce4615fb5d4..6fc6c5fb854 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.cpp @@ -39,50 +39,50 @@ namespace NKikimr { TLogoBlobID genId(Id, 0); hull.AddLogoBlob(ctx, genId, Id.PartId(), Ingress, Buffer, Seg.Point()); - LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_PUT, hull.GetHullCtx()->VCtx->VDiskLogPrefix << "TEvVPut: reply;" - << " id# " << Id - << " msg# " << Result->ToString() - << " Marker# BSVSLR01"); + LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_PUT, hull.GetHullCtx()->VCtx->VDiskLogPrefix << "TEvVPut: reply;" + << " id# " << Id + << " msg# " << Result->ToString() + << " Marker# BSVSLR01"); SendVDiskResponse(ctx, Recipient, Result.release(), actor, RecipientCookie); } /////////////////////////////////////////////////////////////////////////////////////////////////////// - // TLoggedRecVPut -- incapsulates TEvVPut replay action (for small blobs) - /////////////////////////////////////////////////////////////////////////////////////////////////////// - TLoggedRecVMultiPutItem::TLoggedRecVMultiPutItem( - TLsnSeg seg, - bool confirmSyncLogAlso, - const TLogoBlobID &id, - const TIngress &ingress, - TRope &&buffer, + // TLoggedRecVPut -- incapsulates TEvVPut replay action (for small blobs) + /////////////////////////////////////////////////////////////////////////////////////////////////////// + TLoggedRecVMultiPutItem::TLoggedRecVMultiPutItem( + TLsnSeg seg, + bool confirmSyncLogAlso, + const TLogoBlobID &id, + const TIngress &ingress, + TRope &&buffer, std::unique_ptr<TEvVMultiPutItemResult> result, const TActorId &recipient, - ui64 recipientCookie) - : ILoggedRec(seg, confirmSyncLogAlso) - , Id(id) - , Ingress(ingress) - , Buffer(std::move(buffer)) + ui64 recipientCookie) + : ILoggedRec(seg, confirmSyncLogAlso) + , Id(id) + , Ingress(ingress) + , Buffer(std::move(buffer)) , Result(std::move(result)) - , Recipient(recipient) - , RecipientCookie(recipientCookie) - {} - - void TLoggedRecVMultiPutItem::Replay(THull &hull, const TActorContext &ctx, const IActor& actor) { - Y_UNUSED(actor); - TLogoBlobID genId(Id, 0); - hull.AddLogoBlob(ctx, genId, Id.PartId(), Ingress, Buffer, Seg.Point()); - - LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_PUT, hull.GetHullCtx()->VCtx->VDiskLogPrefix - << "TEvVMultiPut: item reply;" - << " id# " << Id - << " msg# " << Result->ToString() - << " Marker# BSVSLR02"); - + , Recipient(recipient) + , RecipientCookie(recipientCookie) + {} + + void TLoggedRecVMultiPutItem::Replay(THull &hull, const TActorContext &ctx, const IActor& actor) { + Y_UNUSED(actor); + TLogoBlobID genId(Id, 0); + hull.AddLogoBlob(ctx, genId, Id.PartId(), Ingress, Buffer, Seg.Point()); + + LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_PUT, hull.GetHullCtx()->VCtx->VDiskLogPrefix + << "TEvVMultiPut: item reply;" + << " id# " << Id + << " msg# " << Result->ToString() + << " Marker# BSVSLR02"); + ctx.Send(Recipient, Result.release(), RecipientCookie); - } - - /////////////////////////////////////////////////////////////////////////////////////////////////////// + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////// // TLoggedRecVPut -- incapsulates TEvVPut replay action (for huge blobs) /////////////////////////////////////////////////////////////////////////////////////////////////////// TLoggedRecVPutHuge::TLoggedRecVPutHuge( @@ -106,8 +106,8 @@ namespace NKikimr { } LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_PUT, hull.GetHullCtx()->VCtx->VDiskLogPrefix - << "TEvVPut: realtime# false result# " << msg->Result->ToString() - << " Marker# BSVSLR03"); + << "TEvVPut: realtime# false result# " << msg->Result->ToString() + << " Marker# BSVSLR03"); SendVDiskResponse(ctx, msg->OrigClient, msg->Result.release(), actor, msg->OrigCookie); } @@ -140,8 +140,8 @@ namespace NKikimr { hull.AddBlockCmd(ctx, TabletId, Gen, IssuerGuid, Seg.Point(), replySender); LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_BLOCK, hull.GetHullCtx()->VCtx->VDiskLogPrefix - << "TEvVBlock: result# " << Result->ToString() - << " Marker# BSVSLR04"); + << "TEvVBlock: result# " << Result->ToString() + << " Marker# BSVSLR04"); SendVDiskResponse(ctx, Recipient, Result.release(), actor, RecipientCookie); } @@ -165,8 +165,8 @@ namespace NKikimr { hull.AddGCCmd(ctx, record, Ingress, Seg); LOG_DEBUG_S(ctx, NKikimrServices::BS_VDISK_GC, hull.GetHullCtx()->VCtx->VDiskLogPrefix - << "TEvVCollectGarbage: result# " << Result->ToString() - << " Marker# BSVSLR05"); + << "TEvVCollectGarbage: result# " << Result->ToString() + << " Marker# BSVSLR05"); SendVDiskResponse(ctx, OrigEv->Sender, Result.release(), actor, OrigEv->Cookie); } diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.h index 50a7a06d2af..33ad29d596d 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_loggedrec.h @@ -1,6 +1,6 @@ #pragma once #include "defs.h" -#include "skeleton_vmultiput_actor.h" +#include "skeleton_vmultiput_actor.h" #include <ydb/core/blobstorage/vdisk/common/vdisk_private_events.h> #include <ydb/core/blobstorage/vdisk/hulldb/hulldb_bulksst_add.h> #include <ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer_localwriter.h> @@ -62,25 +62,25 @@ namespace NKikimr { }; /////////////////////////////////////////////////////////////////////////////////////////////////////// - // TLoggedRecVMultiPutItem -- incapsulates TEvVMultiPut item replay action (for small blobs) - /////////////////////////////////////////////////////////////////////////////////////////////////////// - class TLoggedRecVMultiPutItem : public ILoggedRec { - public: - TLoggedRecVMultiPutItem(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, const TIngress &ingress, + // TLoggedRecVMultiPutItem -- incapsulates TEvVMultiPut item replay action (for small blobs) + /////////////////////////////////////////////////////////////////////////////////////////////////////// + class TLoggedRecVMultiPutItem : public ILoggedRec { + public: + TLoggedRecVMultiPutItem(TLsnSeg seg, bool confirmSyncLogAlso, const TLogoBlobID &id, const TIngress &ingress, TRope &&buffer, std::unique_ptr<TEvVMultiPutItemResult> result, const TActorId &recipient, - ui64 recipientCookie); - void Replay(THull &hull, const TActorContext &ctx, const IActor& actor) override; - - private: - TLogoBlobID Id; - TIngress Ingress; - TRope Buffer; + ui64 recipientCookie); + void Replay(THull &hull, const TActorContext &ctx, const IActor& actor) override; + + private: + TLogoBlobID Id; + TIngress Ingress; + TRope Buffer; std::unique_ptr<TEvVMultiPutItemResult> Result; TActorId Recipient; - ui64 RecipientCookie; - }; - - /////////////////////////////////////////////////////////////////////////////////////////////////////// + ui64 RecipientCookie; + }; + + /////////////////////////////////////////////////////////////////////////////////////////////////////// // TLoggedRecVPut -- incapsulates TEvVPut replay action (for huge blobs) /////////////////////////////////////////////////////////////////////////////////////////////////////// class TLoggedRecVPutHuge : public ILoggedRec { diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.cpp index 5592abd5c08..5bc3e10b05e 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.cpp @@ -132,8 +132,8 @@ namespace NKikimr { TOutOfSpaceLogic::~TOutOfSpaceLogic() {} - template <typename TPutEventPtr> - bool AllowImpl(const TOutOfSpaceLogic &logic, const TActorContext &ctx, TPutEventPtr &ev) { + template <typename TPutEventPtr> + bool AllowImpl(const TOutOfSpaceLogic &logic, const TActorContext &ctx, TPutEventPtr &ev) { Y_UNUSED(ctx); auto color = logic.VCtx->GetOutOfSpaceState().GetGlobalColor(); auto &stat = logic.Stat->Lookup(TOutOfSpaceLogic::TStat::Put, color).HandleMsg(ev->Get()->GetCachedByteSize()); @@ -147,7 +147,7 @@ namespace NKikimr { case TSpaceColor::ORANGE: { // allow writes with IgnoreBlock=true - auto &record = ev->Get()->Record; + auto &record = ev->Get()->Record; const bool allow = record.GetIgnoreBlock(); return stat.Pass(allow); } @@ -159,14 +159,14 @@ namespace NKikimr { } } - bool TOutOfSpaceLogic::Allow(const TActorContext &ctx, TEvBlobStorage::TEvVPut::TPtr &ev) const { - return AllowImpl(*this, ctx, ev); - } - - bool TOutOfSpaceLogic::Allow(const TActorContext &ctx, TEvBlobStorage::TEvVMultiPut::TPtr &ev) const { - return AllowImpl(*this, ctx, ev); - } - + bool TOutOfSpaceLogic::Allow(const TActorContext &ctx, TEvBlobStorage::TEvVPut::TPtr &ev) const { + return AllowImpl(*this, ctx, ev); + } + + bool TOutOfSpaceLogic::Allow(const TActorContext &ctx, TEvBlobStorage::TEvVMultiPut::TPtr &ev) const { + return AllowImpl(*this, ctx, ev); + } + bool TOutOfSpaceLogic::Allow(const TActorContext &ctx, TEvBlobStorage::TEvVBlock::TPtr &ev) const { Y_UNUSED(ctx); auto color = VCtx->GetOutOfSpaceState().GetGlobalColor(); @@ -227,16 +227,16 @@ namespace NKikimr { if (msg->IsAnubis()) { LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix << "OUT OF SPACE while removing LogoBlob we got from Anubis;" - << " LogoBlobId# " << msg->LogoBlobId - << " Marker# BSVSOOSL01"); + << " LogoBlobId# " << msg->LogoBlobId + << " Marker# BSVSOOSL01"); return stat.NotAllow(); } else { // We MUST allow Osiris writes. W/o Osiris we can't work. // There should not be too much of them. LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix << "OUT OF SPACE while adding resurrected by Osiris LogoBlob;" - << " FORCING addition: LogoBlobId# " << msg->LogoBlobId - << " Marker# BSVSOOSL02"); + << " FORCING addition: LogoBlobId# " << msg->LogoBlobId + << " Marker# BSVSOOSL02"); return stat.Allow(); } } diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.h index 8def77caa1f..7963f07eded 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_logic.h @@ -21,7 +21,7 @@ namespace NKikimr { // Check if we allow this write bool Allow(const TActorContext &ctx, TEvBlobStorage::TEvVPut::TPtr &ev) const; - bool Allow(const TActorContext &ctx, TEvBlobStorage::TEvVMultiPut::TPtr &ev) const; + bool Allow(const TActorContext &ctx, TEvBlobStorage::TEvVMultiPut::TPtr &ev) const; bool Allow(const TActorContext &ctx, TEvBlobStorage::TEvVBlock::TPtr &ev) const; bool Allow(const TActorContext &ctx, TEvBlobStorage::TEvVCollectGarbage::TPtr &ev) const; bool Allow(const TActorContext &ctx, TEvLocalSyncData::TPtr &ev) const; @@ -38,9 +38,9 @@ namespace NKikimr { mutable std::unique_ptr<TStat> Stat; bool DefaultAllow(ESpaceColor color) const; - - template <typename TEvPtr> - friend bool AllowImpl(const TOutOfSpaceLogic &logic, const TActorContext &ctx, TEvPtr &ev); + + template <typename TEvPtr> + friend bool AllowImpl(const TOutOfSpaceLogic &logic, const TActorContext &ctx, TEvPtr &ev); }; } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp index 8af5bd8dc22..925fbd00c38 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp @@ -75,8 +75,8 @@ namespace NKikimr { void Handle(NPDisk::TEvCheckSpaceResult::TPtr &ev, const TActorContext &ctx) { const auto *msg = ev->Get(); LOG_DEBUG_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix - << "TDskSpaceTrackerActor:handle TEvCheckSpaceResult; msg# " << msg->ToString() - << " Marker# BSVSOOST02"); + << "TDskSpaceTrackerActor:handle TEvCheckSpaceResult; msg# " << msg->ToString() + << " Marker# BSVSOOST02"); CHECK_PDISK_RESPONSE(VCtx, ev, ctx); diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp index 45968b8d54f..169322f913d 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp @@ -27,8 +27,8 @@ namespace NKikimr { NMonGroup::TSkeletonOverloadGroup &Mon; TQueueType Queue; - TVMovedPatchHandler VMovedPatchHandler; - TVPatchStartHandler VPatchStartHandler; + TVMovedPatchHandler VMovedPatchHandler; + TVPatchStartHandler VPatchStartHandler; TVPutHandler VPutHandler; TVMultiPutHandler VMultiPutHandler; TLocalSyncDataHandler LocalSyncDataHandler; @@ -37,33 +37,33 @@ namespace NKikimr { public: TEmergencyQueue( NMonGroup::TSkeletonOverloadGroup &mon, - TVMovedPatchHandler &&vMovedPatch, - TVPatchStartHandler &&vPatchStart, + TVMovedPatchHandler &&vMovedPatch, + TVPatchStartHandler &&vPatchStart, TVPutHandler &&vput, TVMultiPutHandler &&vMultiPut, TLocalSyncDataHandler &&loc, TAnubisOsirisPutHandler &&aoput) : Mon(mon) - , VMovedPatchHandler(std::move(vMovedPatch)) - , VPatchStartHandler(std::move(vPatchStart)) + , VMovedPatchHandler(std::move(vMovedPatch)) + , VPatchStartHandler(std::move(vPatchStart)) , VPutHandler(std::move(vput)) , VMultiPutHandler(std::move(vMultiPut)) , LocalSyncDataHandler(std::move(loc)) , AnubisOsirisPutHandler(std::move(aoput)) {} - void Push(TEvBlobStorage::TEvVMovedPatch::TPtr ev) { - ++Mon.EmergencyMovedPatchQueueItems(); - Mon.EmergencyMovedPatchQueueBytes() += ev->Get()->Record.ByteSize(); - Queue.Push(TItem(ev)); - } - - void Push(TEvBlobStorage::TEvVPatchStart::TPtr ev) { - ++Mon.EmergencyPatchStartQueueItems(); - Mon.EmergencyPatchStartQueueBytes() += ev->Get()->Record.ByteSize(); - Queue.Push(TItem(ev)); - } - + void Push(TEvBlobStorage::TEvVMovedPatch::TPtr ev) { + ++Mon.EmergencyMovedPatchQueueItems(); + Mon.EmergencyMovedPatchQueueBytes() += ev->Get()->Record.ByteSize(); + Queue.Push(TItem(ev)); + } + + void Push(TEvBlobStorage::TEvVPatchStart::TPtr ev) { + ++Mon.EmergencyPatchStartQueueItems(); + Mon.EmergencyPatchStartQueueBytes() += ev->Get()->Record.ByteSize(); + Queue.Push(TItem(ev)); + } + void Push(TEvBlobStorage::TEvVPut::TPtr ev) { ++Mon.EmergencyPutQueueItems(); Mon.EmergencyPutQueueBytes() += ev->Get()->Record.ByteSize(); @@ -71,8 +71,8 @@ namespace NKikimr { } void Push(TEvBlobStorage::TEvVMultiPut::TPtr ev) { - ++Mon.EmergencyMultiPutQueueItems(); - Mon.EmergencyMultiPutQueueBytes() += ev->Get()->Record.ByteSize(); + ++Mon.EmergencyMultiPutQueueItems(); + Mon.EmergencyMultiPutQueueBytes() += ev->Get()->Record.ByteSize(); Queue.Push(TItem(ev)); } @@ -98,20 +98,20 @@ namespace NKikimr { TAutoPtr<IEventHandle> ev = item->Ev.release(); Queue.Pop(); switch (ev->GetTypeRewrite()) { - case TEvBlobStorage::EvVMovedPatch: { - auto *evMovedPatch = reinterpret_cast<TEvBlobStorage::TEvVMovedPatch::TPtr*>(&ev); - --Mon.EmergencyMovedPatchQueueItems(); - Mon.EmergencyMovedPatchQueueBytes() -= (*evMovedPatch)->Get()->Record.ByteSize(); - VMovedPatchHandler(ctx, *evMovedPatch); - break; - } - case TEvBlobStorage::EvVPatchStart: { - auto *evPatchStart = reinterpret_cast<TEvBlobStorage::TEvVPatchStart::TPtr*>(&ev); - --Mon.EmergencyPatchStartQueueItems(); - Mon.EmergencyPatchStartQueueBytes() -= (*evPatchStart)->Get()->Record.ByteSize(); - VPatchStartHandler(ctx, *evPatchStart); - break; - } + case TEvBlobStorage::EvVMovedPatch: { + auto *evMovedPatch = reinterpret_cast<TEvBlobStorage::TEvVMovedPatch::TPtr*>(&ev); + --Mon.EmergencyMovedPatchQueueItems(); + Mon.EmergencyMovedPatchQueueBytes() -= (*evMovedPatch)->Get()->Record.ByteSize(); + VMovedPatchHandler(ctx, *evMovedPatch); + break; + } + case TEvBlobStorage::EvVPatchStart: { + auto *evPatchStart = reinterpret_cast<TEvBlobStorage::TEvVPatchStart::TPtr*>(&ev); + --Mon.EmergencyPatchStartQueueItems(); + Mon.EmergencyPatchStartQueueBytes() -= (*evPatchStart)->Get()->Record.ByteSize(); + VPatchStartHandler(ctx, *evPatchStart); + break; + } case TEvBlobStorage::EvVPut: { auto *evPut = reinterpret_cast<TEvBlobStorage::TEvVPut::TPtr*>(&ev); --Mon.EmergencyPutQueueItems(); @@ -154,16 +154,16 @@ namespace NKikimr { const TPDiskCtxPtr &pdiskCtx, std::shared_ptr<THull> hull, NMonGroup::TSkeletonOverloadGroup &&mon, - TVMovedPatchHandler &&vMovedPatch, - TVPatchStartHandler &&vPatchStart, + TVMovedPatchHandler &&vMovedPatch, + TVPatchStartHandler &&vPatchStart, TVPutHandler &&vput, TVMultiPutHandler &&vMultiPut, TLocalSyncDataHandler &&loc, TAnubisOsirisPutHandler &&aoput) : Hull(std::move(hull)) , Mon(std::move(mon)) - , EmergencyQueue(new TEmergencyQueue(Mon, std::move(vMovedPatch), std::move(vPatchStart), std::move(vput), - std::move(vMultiPut), std::move(loc), std::move(aoput))) + , EmergencyQueue(new TEmergencyQueue(Mon, std::move(vMovedPatch), std::move(vPatchStart), std::move(vput), + std::move(vMultiPut), std::move(loc), std::move(aoput))) , DynamicPDiskWeightsManager(std::make_shared<TDynamicPDiskWeightsManager>(vctx, pdiskCtx)) {} @@ -210,14 +210,14 @@ namespace NKikimr { return proceedFurther; } - bool TOverloadHandler::PostponeEvent(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx, IActor *skeleton) { - return PostponeEventPrivate(ev, ctx, skeleton); - } - - bool TOverloadHandler::PostponeEvent(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx, IActor *skeleton) { - return PostponeEventPrivate(ev, ctx, skeleton); - } - + bool TOverloadHandler::PostponeEvent(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx, IActor *skeleton) { + return PostponeEventPrivate(ev, ctx, skeleton); + } + + bool TOverloadHandler::PostponeEvent(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx, IActor *skeleton) { + return PostponeEventPrivate(ev, ctx, skeleton); + } + bool TOverloadHandler::PostponeEvent(TEvBlobStorage::TEvVPut::TPtr &ev, const TActorContext &ctx, IActor *skeleton) { return PostponeEventPrivate(ev, ctx, skeleton); } diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h index 00707bb803c..8d84422f93e 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h @@ -36,10 +36,10 @@ namespace NKikimr { /////////////////////////////////////////////////////////////////////////////////////////////////// // Handlers for postponed events /////////////////////////////////////////////////////////////////////////////////////////////////// - using TVMovedPatchHandler = std::function<void(const TActorContext &ctx, - TEvBlobStorage::TEvVMovedPatch::TPtr ev)>; - using TVPatchStartHandler = std::function<void(const TActorContext &ctx, - TEvBlobStorage::TEvVPatchStart::TPtr ev)>; + using TVMovedPatchHandler = std::function<void(const TActorContext &ctx, + TEvBlobStorage::TEvVMovedPatch::TPtr ev)>; + using TVPatchStartHandler = std::function<void(const TActorContext &ctx, + TEvBlobStorage::TEvVPatchStart::TPtr ev)>; using TVPutHandler = std::function<void(const TActorContext &ctx, TEvBlobStorage::TEvVPut::TPtr ev)>; using TVMultiPutHandler = std::function<void(const TActorContext &ctx, @@ -63,8 +63,8 @@ namespace NKikimr { const TPDiskCtxPtr &pdiskCtx, std::shared_ptr<THull> hull, NMonGroup::TSkeletonOverloadGroup &&mon, - TVMovedPatchHandler &&vMovedPatch, - TVPatchStartHandler &&vPatchStart, + TVMovedPatchHandler &&vMovedPatch, + TVPatchStartHandler &&vPatchStart, TVPutHandler &&vput, TVMultiPutHandler &&vMultiPut, TLocalSyncDataHandler &&loc, @@ -78,8 +78,8 @@ namespace NKikimr { bool ProcessPostponedEvents(const TActorContext &ctx, int batchSize, bool actualizeLevels); // Postpone event in case of overload - bool PostponeEvent(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx, IActor *skeleton); - bool PostponeEvent(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx, IActor *skeleton); + bool PostponeEvent(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx, IActor *skeleton); + bool PostponeEvent(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx, IActor *skeleton); bool PostponeEvent(TEvBlobStorage::TEvVPut::TPtr &ev, const TActorContext &ctx, IActor *skeleton); bool PostponeEvent(TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx, IActor *skeleton); bool PostponeEvent(TEvLocalSyncData::TPtr &ev, const TActorContext &ctx, IActor *skeleton); diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.cpp index cc24115b3f0..cc55bb6038a 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.cpp @@ -1,208 +1,208 @@ -#include "skeleton_vmovedpatch_actor.h" - +#include "skeleton_vmovedpatch_actor.h" + #include <ydb/core/blobstorage/vdisk/common/vdisk_response.h> - - -namespace NKikimr { - namespace NPrivate { - - class TVMovedPatchActor : public TActorBootstrapped<TVMovedPatchActor> { - friend TActorBootstrapped<TVMovedPatchActor>; - - static constexpr ui64 SubRequestDurationMs = 1000; - - ui32 OriginalGroupId; - ui32 PatchedGroupId; - TLogoBlobID OriginalId; - TLogoBlobID PatchedId; - - TString Buffer; - TString ErrorReason; - - ui32 DiffCount = 0; + + +namespace NKikimr { + namespace NPrivate { + + class TVMovedPatchActor : public TActorBootstrapped<TVMovedPatchActor> { + friend TActorBootstrapped<TVMovedPatchActor>; + + static constexpr ui64 SubRequestDurationMs = 1000; + + ui32 OriginalGroupId; + ui32 PatchedGroupId; + TLogoBlobID OriginalId; + TLogoBlobID PatchedId; + + TString Buffer; + TString ErrorReason; + + ui32 DiffCount = 0; std::unique_ptr<TEvBlobStorage::TEvPatch::TDiff[]> Diffs; - - TActorIDPtr SkeletonFrontIDPtr; - NMonitoring::TDynamicCounters::TCounterPtr MovedPatchResMsgsPtr; - - TEvBlobStorage::TEvVMovedPatch::TPtr Event; + + TActorIDPtr SkeletonFrontIDPtr; + NMonitoring::TDynamicCounters::TCounterPtr MovedPatchResMsgsPtr; + + TEvBlobStorage::TEvVMovedPatch::TPtr Event; TActorId LeaderId; - TOutOfSpaceStatus OOSStatus; - - - NWilson::TTraceId TraceId; - NLWTrace::TOrbit Orbit; - - const ui64 IncarnationGuid; - - TVDiskContextPtr VCtx; - - public: + TOutOfSpaceStatus OOSStatus; + + + NWilson::TTraceId TraceId; + NLWTrace::TOrbit Orbit; + + const ui64 IncarnationGuid; + + TVDiskContextPtr VCtx; + + public: TVMovedPatchActor(TActorId leaderId, TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMovedPatch::TPtr &ev, - TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr movedPatchResMsgsPtr, - ui64 incarnationGuid, const TVDiskContextPtr &vCtx) - : TActorBootstrapped() - , SkeletonFrontIDPtr(skeletonFrontIDPtr) - , MovedPatchResMsgsPtr(movedPatchResMsgsPtr) - , Event(ev) + TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr movedPatchResMsgsPtr, + ui64 incarnationGuid, const TVDiskContextPtr &vCtx) + : TActorBootstrapped() + , SkeletonFrontIDPtr(skeletonFrontIDPtr) + , MovedPatchResMsgsPtr(movedPatchResMsgsPtr) + , Event(ev) , LeaderId(leaderId) - , OOSStatus(oosStatus) - , IncarnationGuid(incarnationGuid) - , VCtx(vCtx) - { - NKikimrBlobStorage::TEvVMovedPatch &record = Event->Get()->Record; - Y_VERIFY(record.HasOriginalGroupId()); - OriginalGroupId = record.GetOriginalGroupId(); - Y_VERIFY(record.HasPatchedGroupId()); - PatchedGroupId = record.GetPatchedGroupId(); - Y_VERIFY(record.HasOriginalBlobId()); - OriginalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); - Y_VERIFY(record.HasPatchedBlobId()); - PatchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); - - DiffCount = record.DiffsSize(); + , OOSStatus(oosStatus) + , IncarnationGuid(incarnationGuid) + , VCtx(vCtx) + { + NKikimrBlobStorage::TEvVMovedPatch &record = Event->Get()->Record; + Y_VERIFY(record.HasOriginalGroupId()); + OriginalGroupId = record.GetOriginalGroupId(); + Y_VERIFY(record.HasPatchedGroupId()); + PatchedGroupId = record.GetPatchedGroupId(); + Y_VERIFY(record.HasOriginalBlobId()); + OriginalId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); + Y_VERIFY(record.HasPatchedBlobId()); + PatchedId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); + + DiffCount = record.DiffsSize(); Diffs.reset(new TEvBlobStorage::TEvPatch::TDiff[DiffCount]); - for (ui32 idx = 0; idx < DiffCount; ++idx) { - const NKikimrBlobStorage::TDiffBlock &diff = record.GetDiffs(idx); - Y_VERIFY(diff.HasOffset()); - Diffs[idx].Offset = diff.GetOffset(); - Y_VERIFY(diff.HasBuffer()); - Diffs[idx].Buffer = diff.GetBuffer(); - } - } - - private: - void SendResponseAndDie(const TActorContext &ctx, NKikimrProto::EReplyStatus status, - const TString &errorSubMsg = "") - { - NKikimrBlobStorage::TEvVMovedPatch &record = Event->Get()->Record; - TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); - - TMaybe<ui64> cookie; - if (record.HasCookie()) { - cookie = record.GetCookie(); - } - - TInstant now = TAppData::TimeProvider->Now(); + for (ui32 idx = 0; idx < DiffCount; ++idx) { + const NKikimrBlobStorage::TDiffBlock &diff = record.GetDiffs(idx); + Y_VERIFY(diff.HasOffset()); + Diffs[idx].Offset = diff.GetOffset(); + Y_VERIFY(diff.HasBuffer()); + Diffs[idx].Buffer = diff.GetBuffer(); + } + } + + private: + void SendResponseAndDie(const TActorContext &ctx, NKikimrProto::EReplyStatus status, + const TString &errorSubMsg = "") + { + NKikimrBlobStorage::TEvVMovedPatch &record = Event->Get()->Record; + TVDiskID vdisk = VDiskIDFromVDiskID(record.GetVDiskID()); + + TMaybe<ui64> cookie; + if (record.HasCookie()) { + cookie = record.GetCookie(); + } + + TInstant now = TAppData::TimeProvider->Now(); auto vMovedPatchResult = std::make_unique<TEvBlobStorage::TEvVMovedPatchResult>(status, OriginalId, - PatchedId, vdisk, cookie, OOSStatus, now, Event->Get()->GetCachedByteSize(), &record, - SkeletonFrontIDPtr, MovedPatchResMsgsPtr, nullptr, std::move(TraceId), IncarnationGuid, - ErrorReason); - vMovedPatchResult->Orbit = std::move(Orbit); - - if (status == NKikimrProto::ERROR) { - LOG_ERROR_S(ctx, NKikimrServices::BS_VDISK_PATCH, VCtx->VDiskLogPrefix - << "TEvVMovedPatch: " << errorSubMsg << ';' - << " OriginalBlobId# " << OriginalId - << " PatchedBlobId# " << PatchedId - << " ErrorReason# " << ErrorReason - << " Marker# BSVSP01"); - } + PatchedId, vdisk, cookie, OOSStatus, now, Event->Get()->GetCachedByteSize(), &record, + SkeletonFrontIDPtr, MovedPatchResMsgsPtr, nullptr, std::move(TraceId), IncarnationGuid, + ErrorReason); + vMovedPatchResult->Orbit = std::move(Orbit); + + if (status == NKikimrProto::ERROR) { + LOG_ERROR_S(ctx, NKikimrServices::BS_VDISK_PATCH, VCtx->VDiskLogPrefix + << "TEvVMovedPatch: " << errorSubMsg << ';' + << " OriginalBlobId# " << OriginalId + << " PatchedBlobId# " << PatchedId + << " ErrorReason# " << ErrorReason + << " Marker# BSVSP01"); + } SendVDiskResponse(ctx, Event->Sender, vMovedPatchResult.release(), *this, Event->Cookie); - PassAway(); - } - - void ApplyDiffs() { - for (ui32 idx = 0; idx < DiffCount; ++idx) { - const TEvBlobStorage::TEvPatch::TDiff &diff = Diffs[idx]; - memcpy(Buffer.begin() + diff.Offset, diff.Buffer.begin(), diff.Buffer.size()); - } - } - - void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev, const TActorContext &ctx) { - TEvBlobStorage::TEvGetResult *result = ev->Get(); - Orbit = std::move(result->Orbit); - TraceId = std::move(ev->TraceId); - - ui32 patchedIdHash = PatchedId.Hash(); - - constexpr auto errorSubMsg = "failed on VGet"; - if (ev->Cookie != patchedIdHash) { - ErrorReason = "Couldn't get the original blob; Received TEvGetResult with wrong cookie"; - SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); - return; - } else if (result->ResponseSz > 1) { - ErrorReason = "Couldn't get the original blob; Received TEvGetResult with more responses than needed"; - SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); - return; - } else if (result->Status != NKikimrProto::OK || result->ResponseSz != 1 || result->Responses[0].Status != NKikimrProto::OK) { - TString getResponseStatus; - if (result->ResponseSz == 1) { - getResponseStatus = TStringBuilder() << " GetResponseStatus# " - << NKikimrProto::EReplyStatus_Name(result->Responses[0].Status); - } - ErrorReason = TStringBuilder() << "Couldn't get the original blob;" - << " GetStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) - << getResponseStatus - << " GetErrorReason# " << result->ErrorReason; - SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); - return; - } - - Buffer = result->Responses[0].Buffer; - ApplyDiffs(); - TInstant deadline = TActivationContext::Now() + TDuration::MilliSeconds(SubRequestDurationMs); - - // We have chosen UserData as PutHandleClass on purpose. - // If VMovedPatch and Put were AsyncWrite, it would become a deadlock - // because the put subrequest may not send and the moved patch request will end by timeout. + PassAway(); + } + + void ApplyDiffs() { + for (ui32 idx = 0; idx < DiffCount; ++idx) { + const TEvBlobStorage::TEvPatch::TDiff &diff = Diffs[idx]; + memcpy(Buffer.begin() + diff.Offset, diff.Buffer.begin(), diff.Buffer.size()); + } + } + + void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev, const TActorContext &ctx) { + TEvBlobStorage::TEvGetResult *result = ev->Get(); + Orbit = std::move(result->Orbit); + TraceId = std::move(ev->TraceId); + + ui32 patchedIdHash = PatchedId.Hash(); + + constexpr auto errorSubMsg = "failed on VGet"; + if (ev->Cookie != patchedIdHash) { + ErrorReason = "Couldn't get the original blob; Received TEvGetResult with wrong cookie"; + SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); + return; + } else if (result->ResponseSz > 1) { + ErrorReason = "Couldn't get the original blob; Received TEvGetResult with more responses than needed"; + SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); + return; + } else if (result->Status != NKikimrProto::OK || result->ResponseSz != 1 || result->Responses[0].Status != NKikimrProto::OK) { + TString getResponseStatus; + if (result->ResponseSz == 1) { + getResponseStatus = TStringBuilder() << " GetResponseStatus# " + << NKikimrProto::EReplyStatus_Name(result->Responses[0].Status); + } + ErrorReason = TStringBuilder() << "Couldn't get the original blob;" + << " GetStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) + << getResponseStatus + << " GetErrorReason# " << result->ErrorReason; + SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); + return; + } + + Buffer = result->Responses[0].Buffer; + ApplyDiffs(); + TInstant deadline = TActivationContext::Now() + TDuration::MilliSeconds(SubRequestDurationMs); + + // We have chosen UserData as PutHandleClass on purpose. + // If VMovedPatch and Put were AsyncWrite, it would become a deadlock + // because the put subrequest may not send and the moved patch request will end by timeout. std::unique_ptr<TEvBlobStorage::TEvPut> put = std::make_unique<TEvBlobStorage::TEvPut>(PatchedId, Buffer, deadline, - NKikimrBlobStorage::UserData, TEvBlobStorage::TEvPut::TacticDefault); - put->Orbit = std::move(Orbit); - + NKikimrBlobStorage::UserData, TEvBlobStorage::TEvPut::TacticDefault); + put->Orbit = std::move(Orbit); + SendToBSProxy(SelfId(), PatchedGroupId, put.release(), OriginalId.Hash(), std::move(Event->TraceId)); - } - - void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev, const TActorContext &ctx) { - TEvBlobStorage::TEvPutResult *result = ev->Get(); - Orbit = std::move(result->Orbit); - TraceId = std::move(ev->TraceId); - - ui32 originalIdHash = OriginalId.Hash(); - - constexpr auto errorSubMsg = "failed on VPut"; - if (ev->Cookie != originalIdHash) { - ErrorReason = "Couldn't put the patched blob; Received TEvPutResult with wrong cookie"; - SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); - return; - } else if (result->Status != NKikimrProto::OK) { - ErrorReason = TStringBuilder() << "Couldn't put the patched blob;" - << " PutStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) - << " PutErrorReason# " << result->ErrorReason; - SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); - return; - } - - SendResponseAndDie(ctx, NKikimrProto::OK); - } - - void Bootstrap() { - TInstant deadline = TActivationContext::Now() + TDuration::MilliSeconds(SubRequestDurationMs); + } + + void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev, const TActorContext &ctx) { + TEvBlobStorage::TEvPutResult *result = ev->Get(); + Orbit = std::move(result->Orbit); + TraceId = std::move(ev->TraceId); + + ui32 originalIdHash = OriginalId.Hash(); + + constexpr auto errorSubMsg = "failed on VPut"; + if (ev->Cookie != originalIdHash) { + ErrorReason = "Couldn't put the patched blob; Received TEvPutResult with wrong cookie"; + SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); + return; + } else if (result->Status != NKikimrProto::OK) { + ErrorReason = TStringBuilder() << "Couldn't put the patched blob;" + << " PutStatus# " << NKikimrProto::EReplyStatus_Name(result->Status) + << " PutErrorReason# " << result->ErrorReason; + SendResponseAndDie(ctx, NKikimrProto::ERROR, errorSubMsg); + return; + } + + SendResponseAndDie(ctx, NKikimrProto::OK); + } + + void Bootstrap() { + TInstant deadline = TActivationContext::Now() + TDuration::MilliSeconds(SubRequestDurationMs); std::unique_ptr<TEvBlobStorage::TEvGet> get = std::make_unique<TEvBlobStorage::TEvGet>(OriginalId, 0, - OriginalId.BlobSize(), deadline, NKikimrBlobStorage::AsyncRead); - get->Orbit = std::move(Event->Get()->Orbit); - + OriginalId.BlobSize(), deadline, NKikimrBlobStorage::AsyncRead); + get->Orbit = std::move(Event->Get()->Orbit); + SendToBSProxy(SelfId(), OriginalGroupId, get.release(), PatchedId.Hash(), std::move(Event->TraceId)); - Become(&TThis::StateWait); - } - - STFUNC(StateWait) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvBlobStorage::TEvGetResult, Handle); - HFunc(TEvBlobStorage::TEvPutResult, Handle); - } - } - }; - - } // NPrivate - + Become(&TThis::StateWait); + } + + STFUNC(StateWait) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvBlobStorage::TEvGetResult, Handle); + HFunc(TEvBlobStorage::TEvPutResult, Handle); + } + } + }; + + } // NPrivate + IActor* CreateSkeletonVMovedPatchActor(TActorId leaderId, TOutOfSpaceStatus oosStatus, - TEvBlobStorage::TEvVMovedPatch::TPtr &ev, TActorIDPtr skeletonFrontIDPtr, - NMonitoring::TDynamicCounters::TCounterPtr counterPtr, ui64 incarnationGuid, - const TVDiskContextPtr &vCtx) - { + TEvBlobStorage::TEvVMovedPatch::TPtr &ev, TActorIDPtr skeletonFrontIDPtr, + NMonitoring::TDynamicCounters::TCounterPtr counterPtr, ui64 incarnationGuid, + const TVDiskContextPtr &vCtx) + { return new NPrivate::TVMovedPatchActor(leaderId, oosStatus, ev, skeletonFrontIDPtr, - counterPtr, incarnationGuid, vCtx); - } - -} // NKikimr + counterPtr, incarnationGuid, vCtx); + } + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.h index 91f8109385b..396bc91ec93 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmovedpatch_actor.h @@ -1,14 +1,14 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> - -namespace NKikimr { - + +namespace NKikimr { + IActor* CreateSkeletonVMovedPatchActor(TActorId leaderId, TOutOfSpaceStatus oosStatus, - TEvBlobStorage::TEvVMovedPatch::TPtr &ev, TActorIDPtr skeletonFrontIDPtr, - NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, - ui64 incarnationGuid, const TVDiskContextPtr &vCtx); - -} // NKikimr + TEvBlobStorage::TEvVMovedPatch::TPtr &ev, TActorIDPtr skeletonFrontIDPtr, + NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, + ui64 incarnationGuid, const TVDiskContextPtr &vCtx); + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp index ac7dcd7d58a..526c4b48a75 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp @@ -2,7 +2,7 @@ #include <ydb/core/blobstorage/vdisk/common/vdisk_response.h> #include <ydb/core/blobstorage/base/batched_vec.h> - + namespace NKikimr { namespace NPrivate { @@ -17,55 +17,55 @@ namespace NKikimr { ui32 StatusFlags = 0; bool Received = false; bool HasCookie = false; - - TString ToString() const { - return TStringBuilder() - << "{" - << " Status# " << NKikimrProto::EReplyStatus_Name(Status) + + TString ToString() const { + return TStringBuilder() + << "{" + << " Status# " << NKikimrProto::EReplyStatus_Name(Status) << " ErrorReason# " << '"' << EscapeC(ErrorReason) << '"' - << " BlobId# " << BlobId.ToString() - << " HasCookie# " << HasCookie - << " Cookie# " << Cookie - << " StatusFlags# " << NPDisk::StatusFlagsToString(StatusFlags) - << " Received# " << Received - << " }"; - } + << " BlobId# " << BlobId.ToString() + << " HasCookie# " << HasCookie + << " Cookie# " << Cookie + << " StatusFlags# " << NPDisk::StatusFlagsToString(StatusFlags) + << " Received# " << Received + << " }"; + } }; - TBatchedVec<TItem> Items; + TBatchedVec<TItem> Items; ui64 ReceivedResults; TActorIDPtr SkeletonFrontIDPtr; - NMonitoring::TDynamicCounters::TCounterPtr MultiPutResMsgsPtr; + NMonitoring::TDynamicCounters::TCounterPtr MultiPutResMsgsPtr; TEvBlobStorage::TEvVMultiPut::TPtr Event; TActorId LeaderId; - TOutOfSpaceStatus OOSStatus; + TOutOfSpaceStatus OOSStatus; const ui64 IncarnationGuid; public: TBufferVMultiPutActor(TActorId leaderId, const TBatchedVec<NKikimrProto::EReplyStatus> &statuses, - TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, - TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, + TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, + TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, ui64 incarnationGuid) : TActorBootstrapped() , Items(ev->Get()->Record.ItemsSize()) , ReceivedResults(0) , SkeletonFrontIDPtr(skeletonFrontIDPtr) - , MultiPutResMsgsPtr(multiPutResMsgsPtr) + , MultiPutResMsgsPtr(multiPutResMsgsPtr) , Event(ev) , LeaderId(leaderId) - , OOSStatus(oosStatus) + , OOSStatus(oosStatus) , IncarnationGuid(incarnationGuid) { - Y_VERIFY(statuses.size() == Items.size()); - for (ui64 idx = 0; idx < Items.size(); ++idx) { - Items[idx].Status = statuses[idx]; - } + Y_VERIFY(statuses.size() == Items.size()); + for (ui64 idx = 0; idx < Items.size(); ++idx) { + Items[idx].Status = statuses[idx]; + } } private: - void SendResponseAndDie(const TActorContext &ctx) { + void SendResponseAndDie(const TActorContext &ctx) { NKikimrBlobStorage::TEvVMultiPut &vMultiPutRecord = Event->Get()->Record; TVDiskID vdisk = VDiskIDFromVDiskID(vMultiPutRecord.GetVDiskID()); @@ -88,56 +88,56 @@ namespace NKikimr { result.HasCookie ? &result.Cookie : nullptr, result.StatusFlags); } - vMultiPutResult->Record.SetStatusFlags(OOSStatus.Flags); + vMultiPutResult->Record.SetStatusFlags(OOSStatus.Flags); SendVDiskResponse(ctx, Event->Sender, vMultiPutResult.release(), *this, Event->Cookie); PassAway(); } - void Handle(TEvVMultiPutItemResult::TPtr &ev, const TActorContext &ctx) { - TLogoBlobID blobId = ev->Get()->BlobId; - ui64 idx = ev->Get()->ItemIdx; - Y_VERIFY(idx < Items.size(), "itemIdx# %" PRIu64 " ItemsSize# %" PRIu64, idx, (ui64)Items.size()); - TItem &item = Items[idx]; - Y_VERIFY(blobId == item.BlobId, "itemIdx# %" PRIu64 " blobId# %s item# %s", idx, blobId.ToString().data(), item.ToString().data()); - - Y_VERIFY(!item.Received, "itemIdx# %" PRIu64 " item# %s", idx, item.ToString().data()); - item.Received = true; - item.Status = ev->Get()->Status; + void Handle(TEvVMultiPutItemResult::TPtr &ev, const TActorContext &ctx) { + TLogoBlobID blobId = ev->Get()->BlobId; + ui64 idx = ev->Get()->ItemIdx; + Y_VERIFY(idx < Items.size(), "itemIdx# %" PRIu64 " ItemsSize# %" PRIu64, idx, (ui64)Items.size()); + TItem &item = Items[idx]; + Y_VERIFY(blobId == item.BlobId, "itemIdx# %" PRIu64 " blobId# %s item# %s", idx, blobId.ToString().data(), item.ToString().data()); + + Y_VERIFY(!item.Received, "itemIdx# %" PRIu64 " item# %s", idx, item.ToString().data()); + item.Received = true; + item.Status = ev->Get()->Status; item.ErrorReason = ev->Get()->ErrorReason; - - ReceivedResults++; - - if (ReceivedResults == Items.size()) { - SendResponseAndDie(ctx); - } - } - + + ReceivedResults++; + + if (ReceivedResults == Items.size()) { + SendResponseAndDie(ctx); + } + } + void Handle(TEvBlobStorage::TEvVPutResult::TPtr &ev, const TActorContext &ctx) { NKikimrBlobStorage::TEvVPutResult &record = ev->Get()->Record; TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); Y_VERIFY(record.HasCookie()); ui64 idx = record.GetCookie(); - Y_VERIFY(idx < Items.size(), "itemIdx# %" PRIu64 " ItemsSize# %" PRIu64, idx, (ui64)Items.size()); + Y_VERIFY(idx < Items.size(), "itemIdx# %" PRIu64 " ItemsSize# %" PRIu64, idx, (ui64)Items.size()); TItem &item = Items[idx]; - Y_VERIFY(blobId == item.BlobId, "itemIdx# %" PRIu64 " blobId# %s item# %s", idx, blobId.ToString().data(), item.ToString().data()); + Y_VERIFY(blobId == item.BlobId, "itemIdx# %" PRIu64 " blobId# %s item# %s", idx, blobId.ToString().data(), item.ToString().data()); - Y_VERIFY(!item.Received, "itemIdx# %" PRIu64 " item# %s", idx, item.ToString().data()); + Y_VERIFY(!item.Received, "itemIdx# %" PRIu64 " item# %s", idx, item.ToString().data()); item.Received = true; Y_VERIFY(record.HasStatus()); item.Status = record.GetStatus(); item.ErrorReason = record.GetErrorReason(); - + ReceivedResults++; if (ReceivedResults == Items.size()) { - SendResponseAndDie(ctx); + SendResponseAndDie(ctx); } } void Bootstrap(const TActorContext &ctx) { - Y_UNUSED(ctx); + Y_UNUSED(ctx); NKikimrBlobStorage::TEvVMultiPut &record = Event->Get()->Record; for (ui64 idx = 0; idx < record.ItemsSize(); ++idx) { @@ -152,10 +152,10 @@ namespace NKikimr { item.Cookie = recItem.GetCookie(); } - if (item.Status != NKikimrProto::OK) { - item.Received = true; - ReceivedResults++; - } + if (item.Status != NKikimrProto::OK) { + item.Received = true; + ReceivedResults++; + } } Become(&TThis::StateWait); @@ -164,7 +164,7 @@ namespace NKikimr { STFUNC(StateWait) { Y_UNUSED(ctx); switch (ev->GetTypeRewrite()) { - HFunc(TEvVMultiPutItemResult, Handle); + HFunc(TEvVMultiPutItemResult, Handle); HFunc(TEvBlobStorage::TEvVPutResult, Handle); } } @@ -173,11 +173,11 @@ namespace NKikimr { } // NPrivate IActor* CreateSkeletonVMultiPutActor(TActorId leaderId, const TBatchedVec<NKikimrProto::EReplyStatus> &statuses, - TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, - TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr counterPtr, - ui64 incarnationGuid) { + TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, + TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr counterPtr, + ui64 incarnationGuid) { return new NPrivate::TBufferVMultiPutActor(leaderId, statuses, oosStatus, ev, - skeletonFrontIDPtr, counterPtr, incarnationGuid); + skeletonFrontIDPtr, counterPtr, incarnationGuid); } } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.h index 1eebe004567..beeccfff1cb 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.h @@ -9,23 +9,23 @@ namespace NKikimr { struct TEvVMultiPutItemResult : TEventLocal<TEvVMultiPutItemResult, TEvBlobStorage::EvVMultiPutItemResult> { - TLogoBlobID BlobId; - ui64 ItemIdx; - NKikimrProto::EReplyStatus Status; + TLogoBlobID BlobId; + ui64 ItemIdx; + NKikimrProto::EReplyStatus Status; TString ErrorReason; - + TEvVMultiPutItemResult(TLogoBlobID id, ui64 itemIdx, NKikimrProto::EReplyStatus status, TString errorReason) - : TEventLocal() - , BlobId(id) - , ItemIdx(itemIdx) - , Status(status) + : TEventLocal() + , BlobId(id) + , ItemIdx(itemIdx) + , Status(status) , ErrorReason(std::move(errorReason)) - {} -}; - + {} +}; + IActor* CreateSkeletonVMultiPutActor(TActorId leaderId, const TBatchedVec<NKikimrProto::EReplyStatus> &statuses, - TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, - TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, + TOutOfSpaceStatus oosStatus, TEvBlobStorage::TEvVMultiPut::TPtr &ev, + TActorIDPtr skeletonFrontIDPtr, NMonitoring::TDynamicCounters::TCounterPtr multiPutResMsgsPtr, ui64 incarnationGuid); } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.cpp index b04e8d6d8fe..2426398998d 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.cpp @@ -1,683 +1,683 @@ -#include "skeleton_vpatch_actor.h" - +#include "skeleton_vpatch_actor.h" + #include <ydb/core/blobstorage/vdisk/common/vdisk_response.h> #include <ydb/core/util/stlog.h> - -#include <util/generic/serialized_enum.h> - - -namespace NKikimr::NPrivate { - - class TSkeletonVPatchActor : public TActorBootstrapped<TSkeletonVPatchActor> { - friend TActorBootstrapped<TSkeletonVPatchActor>; - // When the actor is created with VPatchStart(request) - // Send VGet for finding parts - // Go to StartState - - // When the actor in StartState - // Receive VGetResult - // if it has error then send Error in VPatchFoundParts(response VPatchStart) and die - // if it doesn't find parts send Ok in VPatchFoundParts(response VPatchStart) without parts and die - // otherwise it send Ok in VPatchFoundParts(response VPatchStart) and fo to WaitState - - // When the actor in WaitState - // Receive VPatchDiff(request) - // if it has a force end flag then send Ok in VPatchResult(response for VPatchDiff) and die - // Send VGet for pulling part - // if it has expected xor diffs then part is parity and actor go to ParityState - // otherwise part is data and actor go to DataState - - // When the actor in DataState - // Receive VGetResult - // if it has some troubles send Error in VPatchResult(response for VPatchDiff) and die - // Send VPatchXorDiffs to vdisk with parity parts in future ignore their responses(VPatchResult) - // Apply the patch - // Send VPut with the patched part - // Receive VPutResult - // Send VPatchResult(response VPatchDiff) and die - - // When the actor in ParityState - // Receive VGetResult - // if it has some troubles send Error in VPatchResult(response for VPatchDiff) and die - // Receive VPatchXorDiffs from vdisk with data parts, apply them and send responses(VPatchXorDiffs) - // Send VPut with the patched part - // Receive VPutResult - // Send VPatchResult(response VPatchDiff) and die - - struct TXorReceiver { - TVDiskID VDiskId; - ui8 PartId; - - TXorReceiver(const TVDiskID &vDiskId, ui8 partId) - : VDiskId(vDiskId) - , PartId(partId) - { - } - }; - - struct TXorDiffs { - TVector<TDiff> Diffs; - ui8 PartId; + +#include <util/generic/serialized_enum.h> + + +namespace NKikimr::NPrivate { + + class TSkeletonVPatchActor : public TActorBootstrapped<TSkeletonVPatchActor> { + friend TActorBootstrapped<TSkeletonVPatchActor>; + // When the actor is created with VPatchStart(request) + // Send VGet for finding parts + // Go to StartState + + // When the actor in StartState + // Receive VGetResult + // if it has error then send Error in VPatchFoundParts(response VPatchStart) and die + // if it doesn't find parts send Ok in VPatchFoundParts(response VPatchStart) without parts and die + // otherwise it send Ok in VPatchFoundParts(response VPatchStart) and fo to WaitState + + // When the actor in WaitState + // Receive VPatchDiff(request) + // if it has a force end flag then send Ok in VPatchResult(response for VPatchDiff) and die + // Send VGet for pulling part + // if it has expected xor diffs then part is parity and actor go to ParityState + // otherwise part is data and actor go to DataState + + // When the actor in DataState + // Receive VGetResult + // if it has some troubles send Error in VPatchResult(response for VPatchDiff) and die + // Send VPatchXorDiffs to vdisk with parity parts in future ignore their responses(VPatchResult) + // Apply the patch + // Send VPut with the patched part + // Receive VPutResult + // Send VPatchResult(response VPatchDiff) and die + + // When the actor in ParityState + // Receive VGetResult + // if it has some troubles send Error in VPatchResult(response for VPatchDiff) and die + // Receive VPatchXorDiffs from vdisk with data parts, apply them and send responses(VPatchXorDiffs) + // Send VPut with the patched part + // Receive VPutResult + // Send VPatchResult(response VPatchDiff) and die + + struct TXorReceiver { + TVDiskID VDiskId; + ui8 PartId; + + TXorReceiver(const TVDiskID &vDiskId, ui8 partId) + : VDiskId(vDiskId) + , PartId(partId) + { + } + }; + + struct TXorDiffs { + TVector<TDiff> Diffs; + ui8 PartId; std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiffResult> ResultEvent; - TActorId Sender; - ui64 Cookie; - - + TActorId Sender; + ui64 Cookie; + + TXorDiffs(TVector<TDiff> &&diffs, ui8 partId, std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiffResult> &&result, - const TActorId &sender, ui64 cookie) - : Diffs(std::move(diffs)) - , PartId(partId) - , ResultEvent(std::move(result)) - , Sender(sender) - , Cookie(cookie) - { - } - }; - - static constexpr TDuration CommonLiveTime = TDuration::Seconds(57); - // 60s is timeout in backpressure queue, try to be nearer to it - - TActorId ProxyId; - - TLogoBlobID OriginalBlobId; - TLogoBlobID PatchedBlobId; - ui8 OriginalPartId = 0; - ui8 PatchedPartId = 0; - TVDiskID VDiskId; - - TInstant Deadline; - - TActorId Sender; - ui64 Cookie; - TActorIDPtr SkeletonFrontIDPtr; - NMonitoring::TDynamicCounters::TCounterPtr VPatchFoundPartsMsgsPtr; - NMonitoring::TDynamicCounters::TCounterPtr VPatchResMsgsPtr; - const TIntrusivePtr<TVPatchCtx> VPatchCtx; - TString VDiskLogPrefix; - + const TActorId &sender, ui64 cookie) + : Diffs(std::move(diffs)) + , PartId(partId) + , ResultEvent(std::move(result)) + , Sender(sender) + , Cookie(cookie) + { + } + }; + + static constexpr TDuration CommonLiveTime = TDuration::Seconds(57); + // 60s is timeout in backpressure queue, try to be nearer to it + + TActorId ProxyId; + + TLogoBlobID OriginalBlobId; + TLogoBlobID PatchedBlobId; + ui8 OriginalPartId = 0; + ui8 PatchedPartId = 0; + TVDiskID VDiskId; + + TInstant Deadline; + + TActorId Sender; + ui64 Cookie; + TActorIDPtr SkeletonFrontIDPtr; + NMonitoring::TDynamicCounters::TCounterPtr VPatchFoundPartsMsgsPtr; + NMonitoring::TDynamicCounters::TCounterPtr VPatchResMsgsPtr; + const TIntrusivePtr<TVPatchCtx> VPatchCtx; + TString VDiskLogPrefix; + TActorId LeaderId; - - const ui64 IncarnationGuid; - - TStackVec<ui32, 1> FoundOriginalParts; - - TStackVec<TXorReceiver, 2> XorReceivers; - TString Buffer; - TVector<TDiff> Diffs; - TVector<TXorDiffs> ReceivedXorDiffs; - - TString ErrorReason; - + + const ui64 IncarnationGuid; + + TStackVec<ui32, 1> FoundOriginalParts; + + TStackVec<TXorReceiver, 2> XorReceivers; + TString Buffer; + TVector<TDiff> Diffs; + TVector<TXorDiffs> ReceivedXorDiffs; + + TString ErrorReason; + std::unique_ptr<TEvBlobStorage::TEvVPatchFoundParts> FoundPartsEvent; std::unique_ptr<TEvBlobStorage::TEvVPatchResult> ResultEvent; - - TBlobStorageGroupType GType; - - ui64 ReceivedXorDiffCount = 0; - ui64 WaitedXorDiffCount = 0; - - public: + + TBlobStorageGroupType GType; + + ui64 ReceivedXorDiffCount = 0; + ui64 WaitedXorDiffCount = 0; + + public: TSkeletonVPatchActor(TActorId leaderId, const TBlobStorageGroupType &gType, - TEvBlobStorage::TEvVPatchStart::TPtr &ev, TInstant now, TActorIDPtr skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, - const TIntrusivePtr<TVPatchCtx> &vPatchCtx, const TString &vDiskLogPrefix, ui64 incarnationGuid) - : TActorBootstrapped() - , Sender(ev->Sender) - , Cookie(ev->Cookie) - , SkeletonFrontIDPtr(skeletonFrontIDPtr) - , VPatchFoundPartsMsgsPtr(vPatchFoundPartsMsgsPtr) - , VPatchResMsgsPtr(vPatchResMsgsPtr) - , VPatchCtx(vPatchCtx) - , VDiskLogPrefix(vDiskLogPrefix) + TEvBlobStorage::TEvVPatchStart::TPtr &ev, TInstant now, TActorIDPtr skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, + const TIntrusivePtr<TVPatchCtx> &vPatchCtx, const TString &vDiskLogPrefix, ui64 incarnationGuid) + : TActorBootstrapped() + , Sender(ev->Sender) + , Cookie(ev->Cookie) + , SkeletonFrontIDPtr(skeletonFrontIDPtr) + , VPatchFoundPartsMsgsPtr(vPatchFoundPartsMsgsPtr) + , VPatchResMsgsPtr(vPatchResMsgsPtr) + , VPatchCtx(vPatchCtx) + , VDiskLogPrefix(vDiskLogPrefix) , LeaderId(leaderId) - , IncarnationGuid(incarnationGuid) - , GType(gType) - { - NKikimrBlobStorage::TEvVPatchStart &record = ev->Get()->Record; - if (record.HasMsgQoS() && record.GetMsgQoS().HasDeadlineSeconds()) { - Deadline = TInstant::Seconds(record.GetMsgQoS().HasDeadlineSeconds()); - } - if (!Deadline) { - Deadline = now + CommonLiveTime; - } - - Y_VERIFY(record.HasOriginalBlobId()); - OriginalBlobId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); - Y_VERIFY(record.HasPatchedBlobId()); - PatchedBlobId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); - Y_VERIFY(record.HasVDiskID()); - VDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); - Y_VERIFY(record.HasCookie()); + , IncarnationGuid(incarnationGuid) + , GType(gType) + { + NKikimrBlobStorage::TEvVPatchStart &record = ev->Get()->Record; + if (record.HasMsgQoS() && record.GetMsgQoS().HasDeadlineSeconds()) { + Deadline = TInstant::Seconds(record.GetMsgQoS().HasDeadlineSeconds()); + } + if (!Deadline) { + Deadline = now + CommonLiveTime; + } + + Y_VERIFY(record.HasOriginalBlobId()); + OriginalBlobId = LogoBlobIDFromLogoBlobID(record.GetOriginalBlobId()); + Y_VERIFY(record.HasPatchedBlobId()); + PatchedBlobId = LogoBlobIDFromLogoBlobID(record.GetPatchedBlobId()); + Y_VERIFY(record.HasVDiskID()); + VDiskId = VDiskIDFromVDiskID(record.GetVDiskID()); + Y_VERIFY(record.HasCookie()); FoundPartsEvent = std::make_unique<TEvBlobStorage::TEvVPatchFoundParts>( - NKikimrProto::OK, OriginalBlobId, PatchedBlobId, VDiskId, record.GetCookie(), now, ErrorReason, &record, - SkeletonFrontIDPtr, VPatchFoundPartsMsgsPtr, nullptr, - std::move(ev->TraceId), IncarnationGuid); - } - - void Bootstrap() { - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP03, - VDiskLogPrefix << " TEvVPatch: bootsrapped;", - (OriginalBlobId, OriginalBlobId), - (Deadline, Deadline)); - ui32 cookie = 0; + NKikimrProto::OK, OriginalBlobId, PatchedBlobId, VDiskId, record.GetCookie(), now, ErrorReason, &record, + SkeletonFrontIDPtr, VPatchFoundPartsMsgsPtr, nullptr, + std::move(ev->TraceId), IncarnationGuid); + } + + void Bootstrap() { + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP03, + VDiskLogPrefix << " TEvVPatch: bootsrapped;", + (OriginalBlobId, OriginalBlobId), + (Deadline, Deadline)); + ui32 cookie = 0; std::unique_ptr<TEvBlobStorage::TEvVGet> msg = TEvBlobStorage::TEvVGet::CreateRangeIndexQuery(VDiskId, Deadline, - NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, cookie, - OriginalBlobId, TLogoBlobID(OriginalBlobId, TLogoBlobID::MaxPartId), - TLogoBlobID::MaxPartId, nullptr, false); + NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, cookie, + OriginalBlobId, TLogoBlobID(OriginalBlobId, TLogoBlobID::MaxPartId), + TLogoBlobID::MaxPartId, nullptr, false); Send(LeaderId, msg.release()); - - Become(&TThis::StartState); - - TDuration liveDuration = Deadline - TActivationContext::Now(); - Schedule(liveDuration, new TKikimrEvents::TEvWakeup); - } - - void SendVPatchFoundParts(NKikimrProto::EReplyStatus status) - { - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP04, - VDiskLogPrefix << " TEvVPatch: sended found parts;", - (OriginalBlobId, OriginalBlobId), - (FoundParts, FormatList(FoundOriginalParts)), - (Status, status)); - for (ui8 part : FoundOriginalParts) { - FoundPartsEvent->AddPart(part); - } - FoundPartsEvent->SetStatus(status); + + Become(&TThis::StartState); + + TDuration liveDuration = Deadline - TActivationContext::Now(); + Schedule(liveDuration, new TKikimrEvents::TEvWakeup); + } + + void SendVPatchFoundParts(NKikimrProto::EReplyStatus status) + { + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP04, + VDiskLogPrefix << " TEvVPatch: sended found parts;", + (OriginalBlobId, OriginalBlobId), + (FoundParts, FormatList(FoundOriginalParts)), + (Status, status)); + for (ui8 part : FoundOriginalParts) { + FoundPartsEvent->AddPart(part); + } + FoundPartsEvent->SetStatus(status); SendVDiskResponse(TActivationContext::AsActorContext(), Sender, FoundPartsEvent.release(), *this, Cookie); - } - - void PullOriginalPart(ui64 pullingPart) { - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP05, - VDiskLogPrefix << " TEvVPatch: send vGet for pulling part data;", - (OriginalBlobId, OriginalBlobId), - (PullingPart, pullingPart)); - ui32 cookie = 0; + } + + void PullOriginalPart(ui64 pullingPart) { + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP05, + VDiskLogPrefix << " TEvVPatch: send vGet for pulling part data;", + (OriginalBlobId, OriginalBlobId), + (PullingPart, pullingPart)); + ui32 cookie = 0; std::unique_ptr<TEvBlobStorage::TEvVGet> msg = TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(VDiskId, Deadline, - NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, cookie, - {}, false); - TLogoBlobID id(OriginalBlobId, pullingPart); - msg->AddExtremeQuery(id, 0, 0, &pullingPart); + NKikimrBlobStorage::EGetHandleClass::FastRead, TEvBlobStorage::TEvVGet::EFlags::None, cookie, + {}, false); + TLogoBlobID id(OriginalBlobId, pullingPart); + msg->AddExtremeQuery(id, 0, 0, &pullingPart); Send(LeaderId, msg.release()); - } - - void HandleVGetRangeResult(TEvBlobStorage::TEvVGetResult::TPtr &ev) { - Become(&TThis::WaitState); - NKikimrBlobStorage::TEvVGetResult &record = ev->Get()->Record; - Y_VERIFY(record.HasStatus()); - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP06, - VDiskLogPrefix << " TEvVPatch: received parts index;", - (OriginalBlobId, OriginalBlobId), - (Status, record.GetStatus()), - (ResultSize, record.ResultSize())); - if (record.GetStatus() != NKikimrProto::OK) { - ErrorReason = TStringBuilder() << "Recieve not OK status from VGetRange," - << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); - SendVPatchFoundParts(NKikimrProto::ERROR); - ConfirmDying(true); - return; - } - if (record.ResultSize() != 1) { - ErrorReason = TStringBuilder() << "Expected only one result, but given " << record.ResultSize() - << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); - SendVPatchFoundParts(NKikimrProto::ERROR); - ConfirmDying(true); - return; - } - - // it has to have only one result - auto &item = record.GetResult(0); - FoundOriginalParts.reserve(item.PartsSize()); - Y_VERIFY(item.HasStatus()); - if (item.GetStatus() == NKikimrProto::OK) { - for (ui32 partId : item.GetParts()) { - FoundOriginalParts.push_back(partId); - } - } - - SendVPatchFoundParts(NKikimrProto::OK); - if (FoundOriginalParts.empty()) { - ConfirmDying(true); - } - } - - void SendVPatchResult(NKikimrProto::EReplyStatus status) - { - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP07, - VDiskLogPrefix << " TEvVPatch: send patch result;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId), - (Status, status), - (ErrorReason, ErrorReason)); - Y_VERIFY(ResultEvent); - ResultEvent->SetStatus(status, ErrorReason); + } + + void HandleVGetRangeResult(TEvBlobStorage::TEvVGetResult::TPtr &ev) { + Become(&TThis::WaitState); + NKikimrBlobStorage::TEvVGetResult &record = ev->Get()->Record; + Y_VERIFY(record.HasStatus()); + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP06, + VDiskLogPrefix << " TEvVPatch: received parts index;", + (OriginalBlobId, OriginalBlobId), + (Status, record.GetStatus()), + (ResultSize, record.ResultSize())); + if (record.GetStatus() != NKikimrProto::OK) { + ErrorReason = TStringBuilder() << "Recieve not OK status from VGetRange," + << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); + SendVPatchFoundParts(NKikimrProto::ERROR); + ConfirmDying(true); + return; + } + if (record.ResultSize() != 1) { + ErrorReason = TStringBuilder() << "Expected only one result, but given " << record.ResultSize() + << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); + SendVPatchFoundParts(NKikimrProto::ERROR); + ConfirmDying(true); + return; + } + + // it has to have only one result + auto &item = record.GetResult(0); + FoundOriginalParts.reserve(item.PartsSize()); + Y_VERIFY(item.HasStatus()); + if (item.GetStatus() == NKikimrProto::OK) { + for (ui32 partId : item.GetParts()) { + FoundOriginalParts.push_back(partId); + } + } + + SendVPatchFoundParts(NKikimrProto::OK); + if (FoundOriginalParts.empty()) { + ConfirmDying(true); + } + } + + void SendVPatchResult(NKikimrProto::EReplyStatus status) + { + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP07, + VDiskLogPrefix << " TEvVPatch: send patch result;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId), + (Status, status), + (ErrorReason, ErrorReason)); + Y_VERIFY(ResultEvent); + ResultEvent->SetStatus(status, ErrorReason); SendVDiskResponse(TActivationContext::AsActorContext(), Sender, ResultEvent.release(), *this, Cookie); - } - - void HandleVGetResult(TEvBlobStorage::TEvVGetResult::TPtr &ev) { - NKikimrBlobStorage::TEvVGetResult &record = ev->Get()->Record; - Y_VERIFY(record.HasStatus()); - if (record.GetStatus() != NKikimrProto::OK) { - ErrorReason = TStringBuilder() << "Recieve not OK status from VGetResult," - << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - return; - } - if (record.ResultSize() != 1) { - ErrorReason = TStringBuilder() << "Recieve not correct result count from VGetResult," - << " expetced 1 but given " << record.ResultSize(); - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - return; - } - - auto &item = *record.MutableResult(0); - Y_VERIFY(item.HasStatus()); - if (item.GetStatus() != NKikimrProto::OK) { - ErrorReason = TStringBuilder() << "Recieve not OK status from VGetResult," - << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()) - << " response status# " << NKikimrProto::EReplyStatus_Name(item.GetStatus()); - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - return; - } - - Y_VERIFY(item.HasBlobID()); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); - - Y_VERIFY(item.HasBuffer()); - Buffer = item.GetBuffer(); - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP08, - VDiskLogPrefix << " TEvVPatch: received part data;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId), - (ReceivedBlobId, blobId), - (Status, record.GetStatus()), - (ResultSize, record.ResultSize())); - - ui8 *buffer = reinterpret_cast<ui8*>(const_cast<char*>(Buffer.data())); - if (blobId.PartId() <= GType.DataParts()) { - if (GType.ErasureFamily() != TErasureType::ErasureMirror) { - SendXorDiff(); - } - GType.ApplyDiff(TErasureType::CrcModeNone, buffer, Diffs); - SendVPut(); - } else { - ui8 toPart = blobId.PartId(); - ui32 dataSize = blobId.BlobSize(); - - for (ui32 idx = ReceivedXorDiffs.size(); idx != 0; --idx) { - auto &[diffs, partId, result, sender, cookie] = ReceivedXorDiffs.back(); - GType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, buffer, diffs, partId - 1, toPart - 1); + } + + void HandleVGetResult(TEvBlobStorage::TEvVGetResult::TPtr &ev) { + NKikimrBlobStorage::TEvVGetResult &record = ev->Get()->Record; + Y_VERIFY(record.HasStatus()); + if (record.GetStatus() != NKikimrProto::OK) { + ErrorReason = TStringBuilder() << "Recieve not OK status from VGetResult," + << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + return; + } + if (record.ResultSize() != 1) { + ErrorReason = TStringBuilder() << "Recieve not correct result count from VGetResult," + << " expetced 1 but given " << record.ResultSize(); + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + return; + } + + auto &item = *record.MutableResult(0); + Y_VERIFY(item.HasStatus()); + if (item.GetStatus() != NKikimrProto::OK) { + ErrorReason = TStringBuilder() << "Recieve not OK status from VGetResult," + << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()) + << " response status# " << NKikimrProto::EReplyStatus_Name(item.GetStatus()); + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + return; + } + + Y_VERIFY(item.HasBlobID()); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(item.GetBlobID()); + + Y_VERIFY(item.HasBuffer()); + Buffer = item.GetBuffer(); + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP08, + VDiskLogPrefix << " TEvVPatch: received part data;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId), + (ReceivedBlobId, blobId), + (Status, record.GetStatus()), + (ResultSize, record.ResultSize())); + + ui8 *buffer = reinterpret_cast<ui8*>(const_cast<char*>(Buffer.data())); + if (blobId.PartId() <= GType.DataParts()) { + if (GType.ErasureFamily() != TErasureType::ErasureMirror) { + SendXorDiff(); + } + GType.ApplyDiff(TErasureType::CrcModeNone, buffer, Diffs); + SendVPut(); + } else { + ui8 toPart = blobId.PartId(); + ui32 dataSize = blobId.BlobSize(); + + for (ui32 idx = ReceivedXorDiffs.size(); idx != 0; --idx) { + auto &[diffs, partId, result, sender, cookie] = ReceivedXorDiffs.back(); + GType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, buffer, diffs, partId - 1, toPart - 1); SendVDiskResponse(TActivationContext::AsActorContext(), sender, result.release(), *this, cookie); - ReceivedXorDiffs.pop_back(); - } - - if (ReceivedXorDiffCount == WaitedXorDiffCount) { - SendVPut(); - } - } - } - - template <typename TDiffEvent> - TVector<TDiff> PullDiff(const TDiffEvent &diffRecord, bool isXor) { - TVector<TDiff> diffs; - diffs.reserve(diffRecord.DiffsSize()); - for (auto &diff : diffRecord.GetDiffs()) { - TString buffer = diff.GetBuffer(); - bool isAligned = (GType.ErasureFamily() != TErasureType::ErasureMirror); - diffs.emplace_back(buffer, diff.GetOffset(), isXor, isAligned); - } - return std::move(diffs); - } - - bool CheckDiff(const TVector<TDiff> &diffs, const TString &diffName) { - for (ui32 diffIdx = 0; diffIdx < diffs.size(); ++diffIdx) { - const TDiff &diff = diffs[diffIdx]; - bool ok = (diff.Offset < GType.PartSize(OriginalBlobId)); - ok &= (diff.Offset + diff.GetDiffLength() <= GType.PartSize(OriginalBlobId)); - if (!ok) { - ErrorReason = TStringBuilder() << "The diff at index " << diffIdx << " went beyound the blob part;" - << " DiffStart# " << diff.Offset - << " DiffEnd# " << diff.Offset + diff.GetDiffLength() - << " BlobPartSize# " << GType.PartSize(OriginalBlobId); - return false; - } - } - for (ui32 diffIdx = 1; diffIdx < diffs.size(); ++diffIdx) { - ui32 prevIdx = diffIdx - 1; - bool ok = diffs[prevIdx].Offset < diffs[diffIdx].Offset; - if (!ok) { - ErrorReason = TStringBuilder() << "[" << diffName << "]" - << " the start of the diff at index " << prevIdx << " righter than" - << " the start of the diff at index " << diffIdx << ';' - << " PrevDiffStart# " << diffs[prevIdx].Offset + diffs[prevIdx].GetDiffLength() - << " DiffStart# " << diffs[diffIdx].Offset; - return false; - } - } - return true; - } - - void SendXorDiff() { - TVector<TDiff> xorDiffs; - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP14, - VDiskLogPrefix << " TEvVPatch: send xor diffs;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId), - (XorDiffCount, XorReceivers.size())); - - const ui8 *buffer = reinterpret_cast<const ui8*>(Buffer.data()); - GType.MakeXorDiff(TErasureType::CrcModeNone, OriginalBlobId.BlobSize(), buffer, Diffs, &xorDiffs); - - for (TXorReceiver &xorReceiver : XorReceivers) { + ReceivedXorDiffs.pop_back(); + } + + if (ReceivedXorDiffCount == WaitedXorDiffCount) { + SendVPut(); + } + } + } + + template <typename TDiffEvent> + TVector<TDiff> PullDiff(const TDiffEvent &diffRecord, bool isXor) { + TVector<TDiff> diffs; + diffs.reserve(diffRecord.DiffsSize()); + for (auto &diff : diffRecord.GetDiffs()) { + TString buffer = diff.GetBuffer(); + bool isAligned = (GType.ErasureFamily() != TErasureType::ErasureMirror); + diffs.emplace_back(buffer, diff.GetOffset(), isXor, isAligned); + } + return std::move(diffs); + } + + bool CheckDiff(const TVector<TDiff> &diffs, const TString &diffName) { + for (ui32 diffIdx = 0; diffIdx < diffs.size(); ++diffIdx) { + const TDiff &diff = diffs[diffIdx]; + bool ok = (diff.Offset < GType.PartSize(OriginalBlobId)); + ok &= (diff.Offset + diff.GetDiffLength() <= GType.PartSize(OriginalBlobId)); + if (!ok) { + ErrorReason = TStringBuilder() << "The diff at index " << diffIdx << " went beyound the blob part;" + << " DiffStart# " << diff.Offset + << " DiffEnd# " << diff.Offset + diff.GetDiffLength() + << " BlobPartSize# " << GType.PartSize(OriginalBlobId); + return false; + } + } + for (ui32 diffIdx = 1; diffIdx < diffs.size(); ++diffIdx) { + ui32 prevIdx = diffIdx - 1; + bool ok = diffs[prevIdx].Offset < diffs[diffIdx].Offset; + if (!ok) { + ErrorReason = TStringBuilder() << "[" << diffName << "]" + << " the start of the diff at index " << prevIdx << " righter than" + << " the start of the diff at index " << diffIdx << ';' + << " PrevDiffStart# " << diffs[prevIdx].Offset + diffs[prevIdx].GetDiffLength() + << " DiffStart# " << diffs[diffIdx].Offset; + return false; + } + } + return true; + } + + void SendXorDiff() { + TVector<TDiff> xorDiffs; + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP14, + VDiskLogPrefix << " TEvVPatch: send xor diffs;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId), + (XorDiffCount, XorReceivers.size())); + + const ui8 *buffer = reinterpret_cast<const ui8*>(Buffer.data()); + GType.MakeXorDiff(TErasureType::CrcModeNone, OriginalBlobId.BlobSize(), buffer, Diffs, &xorDiffs); + + for (TXorReceiver &xorReceiver : XorReceivers) { std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiff> xorDiff = std::make_unique<TEvBlobStorage::TEvVPatchXorDiff>( - TLogoBlobID(OriginalBlobId, xorReceiver.PartId), - TLogoBlobID(PatchedBlobId, xorReceiver.PartId), - xorReceiver.VDiskId, OriginalPartId, Deadline, 0); - for (auto &diff : xorDiffs) { - Y_VERIFY(diff.Offset < GType.PartSize(PatchedBlobId)); - Y_VERIFY(diff.Offset + diff.GetDiffLength() <= GType.PartSize(PatchedBlobId)); - xorDiff->AddDiff(diff.Offset, diff.Buffer); - } - TVDiskIdShort shortId(xorReceiver.VDiskId); - Y_VERIFY(VPatchCtx); - Y_VERIFY(VPatchCtx->AsyncBlobQueues); - auto it = VPatchCtx->AsyncBlobQueues.find(shortId); - Y_VERIFY(it != VPatchCtx->AsyncBlobQueues.end()); - - TInstant now = TActivationContext::Now(); - NKikimrBlobStorage::TEvVPatchXorDiff &record = xorDiff->Record; - NKikimrBlobStorage::TMsgQoS &msgQoS = *record.MutableMsgQoS(); - NKikimrBlobStorage::TExecTimeStats &execTimeStats = *msgQoS.MutableExecTimeStats(); - execTimeStats.SetSubmitTimestamp(now.GetValue()); - + TLogoBlobID(OriginalBlobId, xorReceiver.PartId), + TLogoBlobID(PatchedBlobId, xorReceiver.PartId), + xorReceiver.VDiskId, OriginalPartId, Deadline, 0); + for (auto &diff : xorDiffs) { + Y_VERIFY(diff.Offset < GType.PartSize(PatchedBlobId)); + Y_VERIFY(diff.Offset + diff.GetDiffLength() <= GType.PartSize(PatchedBlobId)); + xorDiff->AddDiff(diff.Offset, diff.Buffer); + } + TVDiskIdShort shortId(xorReceiver.VDiskId); + Y_VERIFY(VPatchCtx); + Y_VERIFY(VPatchCtx->AsyncBlobQueues); + auto it = VPatchCtx->AsyncBlobQueues.find(shortId); + Y_VERIFY(it != VPatchCtx->AsyncBlobQueues.end()); + + TInstant now = TActivationContext::Now(); + NKikimrBlobStorage::TEvVPatchXorDiff &record = xorDiff->Record; + NKikimrBlobStorage::TMsgQoS &msgQoS = *record.MutableMsgQoS(); + NKikimrBlobStorage::TExecTimeStats &execTimeStats = *msgQoS.MutableExecTimeStats(); + execTimeStats.SetSubmitTimestamp(now.GetValue()); + Send(it->second, xorDiff.release()); - } - } - - void SendVPut() { - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP15, - VDiskLogPrefix << " TEvVPatch: send vPut;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId)); - ui64 cookie = OriginalBlobId.Hash(); + } + } + + void SendVPut() { + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP15, + VDiskLogPrefix << " TEvVPatch: send vPut;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId)); + ui64 cookie = OriginalBlobId.Hash(); std::unique_ptr<IEventBase> put = std::make_unique<TEvBlobStorage::TEvVPut>(TLogoBlobID(PatchedBlobId, PatchedPartId), - Buffer, VDiskId, false, &cookie, Deadline, NKikimrBlobStorage::AsyncBlob); + Buffer, VDiskId, false, &cookie, Deadline, NKikimrBlobStorage::AsyncBlob); Send(LeaderId, put.release()); - } - - void HandleError(TEvBlobStorage::TEvVPatchDiff::TPtr &ev) { - NKikimrBlobStorage::TEvVPatchDiff &record = ev->Get()->Record; - TInstant now = TActivationContext::Now(); + } + + void HandleError(TEvBlobStorage::TEvVPatchDiff::TPtr &ev) { + NKikimrBlobStorage::TEvVPatchDiff &record = ev->Get()->Record; + TInstant now = TActivationContext::Now(); ResultEvent = std::make_unique<TEvBlobStorage::TEvVPatchResult>( - NKikimrProto::OK, TLogoBlobID(OriginalBlobId, OriginalPartId), - TLogoBlobID(PatchedBlobId, PatchedPartId), VDiskId, record.GetCookie(), now, - &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, - std::move(ev->TraceId), IncarnationGuid); - Sender = ev->Sender; - Cookie = ev->Cookie; - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - } - - void Handle(TEvBlobStorage::TEvVPatchDiff::TPtr &ev) { - NKikimrBlobStorage::TEvVPatchDiff &record = ev->Get()->Record; - Y_VERIFY(record.HasCookie()); - - TLogoBlobID originalPartBlobId = LogoBlobIDFromLogoBlobID(record.GetOriginalPartBlobId()); - TLogoBlobID patchedPartBlobId = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()); - OriginalPartId = originalPartBlobId.PartId(); - PatchedPartId = patchedPartBlobId.PartId(); - - bool forceEnd = ev->Get()->IsForceEnd(); - - bool isXorReceiver = ev->Get()->IsXorReceiver(); - WaitedXorDiffCount = ev->Get()->GetExpectedXorDiffs(); - - if (isXorReceiver) { - Become(&TThis::ParityState); - } else { - Become(&TThis::DataState); - } - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP09, - VDiskLogPrefix << " TEvVPatch: received diff;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId), - (XorReceiver, (isXorReceiver ? "yes" : "no")), - (ForceEnd, (forceEnd ? "yes" : "no"))); - - Y_VERIFY(!ResultEvent); - TInstant now = TActivationContext::Now(); - + NKikimrProto::OK, TLogoBlobID(OriginalBlobId, OriginalPartId), + TLogoBlobID(PatchedBlobId, PatchedPartId), VDiskId, record.GetCookie(), now, + &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, + std::move(ev->TraceId), IncarnationGuid); + Sender = ev->Sender; + Cookie = ev->Cookie; + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + } + + void Handle(TEvBlobStorage::TEvVPatchDiff::TPtr &ev) { + NKikimrBlobStorage::TEvVPatchDiff &record = ev->Get()->Record; + Y_VERIFY(record.HasCookie()); + + TLogoBlobID originalPartBlobId = LogoBlobIDFromLogoBlobID(record.GetOriginalPartBlobId()); + TLogoBlobID patchedPartBlobId = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()); + OriginalPartId = originalPartBlobId.PartId(); + PatchedPartId = patchedPartBlobId.PartId(); + + bool forceEnd = ev->Get()->IsForceEnd(); + + bool isXorReceiver = ev->Get()->IsXorReceiver(); + WaitedXorDiffCount = ev->Get()->GetExpectedXorDiffs(); + + if (isXorReceiver) { + Become(&TThis::ParityState); + } else { + Become(&TThis::DataState); + } + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP09, + VDiskLogPrefix << " TEvVPatch: received diff;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId), + (XorReceiver, (isXorReceiver ? "yes" : "no")), + (ForceEnd, (forceEnd ? "yes" : "no"))); + + Y_VERIFY(!ResultEvent); + TInstant now = TActivationContext::Now(); + ResultEvent = std::make_unique<TEvBlobStorage::TEvVPatchResult>( - NKikimrProto::OK, originalPartBlobId, patchedPartBlobId, VDiskId, record.GetCookie(), now, - &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, - std::move(ev->TraceId), IncarnationGuid); - Sender = ev->Sender; - Cookie = ev->Cookie; - - if (forceEnd) { - SendVPatchResult(NKikimrProto::OK); - PassAway(); - return; - } - - for (auto &protoXorReceiver : record.GetXorReceivers()) { - Y_VERIFY(protoXorReceiver.HasVDiskID()); - Y_VERIFY(protoXorReceiver.HasPartId()); - XorReceivers.emplace_back( - VDiskIDFromVDiskID(protoXorReceiver.GetVDiskID()), - protoXorReceiver.GetPartId()); - } - - Diffs = PullDiff(record, false); - - if (!CheckDiff(Diffs, "Diff from DSProxy")) { - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - return; - } - - PullOriginalPart(OriginalPartId); - } - - void Handle(TEvBlobStorage::TEvVPutResult::TPtr &ev) { - NKikimrBlobStorage::TEvVPutResult &record = ev->Get()->Record; - Y_VERIFY(record.HasStatus()); - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP10, - VDiskLogPrefix << " TEvVPatch: received put result;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (OriginalPartId, (ui32)OriginalPartId), - (PatchedPartId, (ui32)PatchedPartId), - (Status, record.GetStatus())); - - NKikimrProto::EReplyStatus status = NKikimrProto::OK; - if (record.GetStatus() != NKikimrProto::OK) { - ErrorReason = TStringBuilder() << "Recieve not OK status from VPutResult," - << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); - status = NKikimrProto::ERROR; - } - - ResultEvent->SetStatusFlagsAndFreeSpace(record.GetStatusFlags(), record.GetApproximateFreeSpaceShare()); - - SendVPatchResult(status); - PassAway(); - } - - void HandleError(TEvBlobStorage::TEvVPatchXorDiff::TPtr &ev) { - NKikimrBlobStorage::TEvVPatchXorDiff &record = ev->Get()->Record; - TInstant now = TActivationContext::Now(); + NKikimrProto::OK, originalPartBlobId, patchedPartBlobId, VDiskId, record.GetCookie(), now, + &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, + std::move(ev->TraceId), IncarnationGuid); + Sender = ev->Sender; + Cookie = ev->Cookie; + + if (forceEnd) { + SendVPatchResult(NKikimrProto::OK); + PassAway(); + return; + } + + for (auto &protoXorReceiver : record.GetXorReceivers()) { + Y_VERIFY(protoXorReceiver.HasVDiskID()); + Y_VERIFY(protoXorReceiver.HasPartId()); + XorReceivers.emplace_back( + VDiskIDFromVDiskID(protoXorReceiver.GetVDiskID()), + protoXorReceiver.GetPartId()); + } + + Diffs = PullDiff(record, false); + + if (!CheckDiff(Diffs, "Diff from DSProxy")) { + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + return; + } + + PullOriginalPart(OriginalPartId); + } + + void Handle(TEvBlobStorage::TEvVPutResult::TPtr &ev) { + NKikimrBlobStorage::TEvVPutResult &record = ev->Get()->Record; + Y_VERIFY(record.HasStatus()); + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP10, + VDiskLogPrefix << " TEvVPatch: received put result;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (OriginalPartId, (ui32)OriginalPartId), + (PatchedPartId, (ui32)PatchedPartId), + (Status, record.GetStatus())); + + NKikimrProto::EReplyStatus status = NKikimrProto::OK; + if (record.GetStatus() != NKikimrProto::OK) { + ErrorReason = TStringBuilder() << "Recieve not OK status from VPutResult," + << " received status# " << NKikimrProto::EReplyStatus_Name(record.GetStatus()); + status = NKikimrProto::ERROR; + } + + ResultEvent->SetStatusFlagsAndFreeSpace(record.GetStatusFlags(), record.GetApproximateFreeSpaceShare()); + + SendVPatchResult(status); + PassAway(); + } + + void HandleError(TEvBlobStorage::TEvVPatchXorDiff::TPtr &ev) { + NKikimrBlobStorage::TEvVPatchXorDiff &record = ev->Get()->Record; + TInstant now = TActivationContext::Now(); auto resultEvent = std::make_unique<TEvBlobStorage::TEvVPatchXorDiffResult>( - NKikimrProto::ERROR, now, &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, - std::move(ev->TraceId)); + NKikimrProto::ERROR, now, &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, + std::move(ev->TraceId)); SendVDiskResponse(TActivationContext::AsActorContext(), ev->Sender, resultEvent.release(), *this, ev->Cookie); - } - - void Handle(TEvBlobStorage::TEvVPatchXorDiff::TPtr &ev) { - NKikimrBlobStorage::TEvVPatchXorDiff &record = ev->Get()->Record; - Y_VERIFY(record.HasFromPartId()); - ui8 fromPart = record.GetFromPartId(); - ui8 toPart = OriginalPartId; - TVector<TDiff> xorDiffs = PullDiff(record, true); - ReceivedXorDiffCount++; - - STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP13, - VDiskLogPrefix << " TEvVPatch: received xor diff;", - (OriginalBlobId, OriginalBlobId), - (PatchedBlobId, PatchedBlobId), - (FromPart, (ui32)fromPart), - (ToPart, (ui32)toPart), - (HasBuffer, (Buffer.empty() ? "no" : "yes")), - (ReceivedXorDiffCount, TStringBuilder() << ReceivedXorDiffCount << '/' << WaitedXorDiffCount)); - - TInstant now = TActivationContext::Now(); + } + + void Handle(TEvBlobStorage::TEvVPatchXorDiff::TPtr &ev) { + NKikimrBlobStorage::TEvVPatchXorDiff &record = ev->Get()->Record; + Y_VERIFY(record.HasFromPartId()); + ui8 fromPart = record.GetFromPartId(); + ui8 toPart = OriginalPartId; + TVector<TDiff> xorDiffs = PullDiff(record, true); + ReceivedXorDiffCount++; + + STLOG(PRI_INFO, BS_VDISK_PATCH, BSVSP13, + VDiskLogPrefix << " TEvVPatch: received xor diff;", + (OriginalBlobId, OriginalBlobId), + (PatchedBlobId, PatchedBlobId), + (FromPart, (ui32)fromPart), + (ToPart, (ui32)toPart), + (HasBuffer, (Buffer.empty() ? "no" : "yes")), + (ReceivedXorDiffCount, TStringBuilder() << ReceivedXorDiffCount << '/' << WaitedXorDiffCount)); + + TInstant now = TActivationContext::Now(); std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiffResult> resultEvent = std::make_unique<TEvBlobStorage::TEvVPatchXorDiffResult>( - NKikimrProto::OK, now, &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, std::move(ev->TraceId)); - - if (!CheckDiff(xorDiffs, "XorDiff from datapart")) { - for (auto &[diffs, partId, result, sender, cookie] : ReceivedXorDiffs) { + NKikimrProto::OK, now, &record, SkeletonFrontIDPtr, VPatchResMsgsPtr, nullptr, std::move(ev->TraceId)); + + if (!CheckDiff(xorDiffs, "XorDiff from datapart")) { + for (auto &[diffs, partId, result, sender, cookie] : ReceivedXorDiffs) { SendVDiskResponse(TActivationContext::AsActorContext(), sender, result.release(), *this, cookie); - } + } SendVDiskResponse(TActivationContext::AsActorContext(), ev->Sender, resultEvent.release(), *this, ev->Cookie); - - if (ResultEvent) { - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - } else { - Become(&TThis::ErrorState); - } - return; - } - - if (Buffer) { - ui8 *buffer = reinterpret_cast<ui8*>(const_cast<char*>(Buffer.data())); - ui32 dataSize = OriginalBlobId.BlobSize(); - GType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, buffer, xorDiffs, fromPart - 1, toPart - 1); - - if (ReceivedXorDiffCount == WaitedXorDiffCount) { - SendVPut(); - } - - xorDiffs.clear(); + + if (ResultEvent) { + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + } else { + Become(&TThis::ErrorState); + } + return; + } + + if (Buffer) { + ui8 *buffer = reinterpret_cast<ui8*>(const_cast<char*>(Buffer.data())); + ui32 dataSize = OriginalBlobId.BlobSize(); + GType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, buffer, xorDiffs, fromPart - 1, toPart - 1); + + if (ReceivedXorDiffCount == WaitedXorDiffCount) { + SendVPut(); + } + + xorDiffs.clear(); SendVDiskResponse(TActivationContext::AsActorContext(), ev->Sender, resultEvent.release(), *this, ev->Cookie); - } else { - ReceivedXorDiffs.emplace_back(std::move(xorDiffs), fromPart, std::move(resultEvent), - ev->Sender, ev->Cookie); - } - } - - void ConfirmDying(bool forceDeath) { + } else { + ReceivedXorDiffs.emplace_back(std::move(xorDiffs), fromPart, std::move(resultEvent), + ev->Sender, ev->Cookie); + } + } + + void ConfirmDying(bool forceDeath) { Send(LeaderId, new TEvVPatchDyingRequest(PatchedBlobId)); - if (forceDeath) { - PassAway(); - } else { - Schedule(CommonLiveTime, new TEvVPatchDyingConfirm); - } - } - - void HandleInStartState(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { - ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, before receiving diff"; - STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP11, VDiskLogPrefix << " " << ErrorReason << ";"); - SendVPatchFoundParts(NKikimrProto::ERROR); - ConfirmDying(true); - } - - void HandleInWaitState(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { - ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, before receiving diff"; - STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP16, VDiskLogPrefix << " " << ErrorReason << ";"); - ConfirmDying(false); - Become(&TThis::ErrorState); - } - - void HandleInDataOrParityStates(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { - ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, after receiving diff"; - STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP12, VDiskLogPrefix << " " << ErrorReason << ";"); - SendVPatchResult(NKikimrProto::ERROR); - PassAway(); - } - - STATEFN(StartState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetRangeResult) - hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) - hFunc(TKikimrEvents::TEvWakeup, HandleInStartState) - default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); - } - } - - STATEFN(WaitState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVPatchDiff, Handle) - hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) - hFunc(TKikimrEvents::TEvWakeup, HandleInWaitState) - sFunc(TEvVPatchDyingConfirm, PassAway) - default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); - } - } - - STATEFN(ErrorState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVPatchDiff, HandleError) - hFunc(TEvBlobStorage::TEvVPatchXorDiff, HandleError) - hFunc(TKikimrEvents::TEvWakeup, HandleInWaitState) - sFunc(TEvVPatchDyingConfirm, PassAway) - default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); - } - } - - STATEFN(DataState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetResult) - hFunc(TEvBlobStorage::TEvVPutResult, Handle) - IgnoreFunc(TEvBlobStorage::TEvVPatchXorDiffResult) - hFunc(TKikimrEvents::TEvWakeup, HandleInDataOrParityStates) - IgnoreFunc(TEvVPatchDyingConfirm) - default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); - } - } - - STATEFN(ParityState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetResult) - hFunc(TEvBlobStorage::TEvVPutResult, Handle) - hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) - hFunc(TKikimrEvents::TEvWakeup, HandleInDataOrParityStates) - IgnoreFunc(TEvVPatchDyingConfirm) - default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); - } - } - }; - -} // NKikimr::NPrivate - -namespace NKikimr { - + if (forceDeath) { + PassAway(); + } else { + Schedule(CommonLiveTime, new TEvVPatchDyingConfirm); + } + } + + void HandleInStartState(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { + ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, before receiving diff"; + STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP11, VDiskLogPrefix << " " << ErrorReason << ";"); + SendVPatchFoundParts(NKikimrProto::ERROR); + ConfirmDying(true); + } + + void HandleInWaitState(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { + ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, before receiving diff"; + STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP16, VDiskLogPrefix << " " << ErrorReason << ";"); + ConfirmDying(false); + Become(&TThis::ErrorState); + } + + void HandleInDataOrParityStates(TKikimrEvents::TEvWakeup::TPtr &/*ev*/) { + ErrorReason = "TEvVPatch: the vpatch actor died due to a deadline, after receiving diff"; + STLOG(PRI_ERROR, BS_VDISK_PATCH, BSVSP12, VDiskLogPrefix << " " << ErrorReason << ";"); + SendVPatchResult(NKikimrProto::ERROR); + PassAway(); + } + + STATEFN(StartState) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetRangeResult) + hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) + hFunc(TKikimrEvents::TEvWakeup, HandleInStartState) + default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); + } + } + + STATEFN(WaitState) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVPatchDiff, Handle) + hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) + hFunc(TKikimrEvents::TEvWakeup, HandleInWaitState) + sFunc(TEvVPatchDyingConfirm, PassAway) + default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); + } + } + + STATEFN(ErrorState) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVPatchDiff, HandleError) + hFunc(TEvBlobStorage::TEvVPatchXorDiff, HandleError) + hFunc(TKikimrEvents::TEvWakeup, HandleInWaitState) + sFunc(TEvVPatchDyingConfirm, PassAway) + default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); + } + } + + STATEFN(DataState) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetResult) + hFunc(TEvBlobStorage::TEvVPutResult, Handle) + IgnoreFunc(TEvBlobStorage::TEvVPatchXorDiffResult) + hFunc(TKikimrEvents::TEvWakeup, HandleInDataOrParityStates) + IgnoreFunc(TEvVPatchDyingConfirm) + default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); + } + } + + STATEFN(ParityState) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvVGetResult, HandleVGetResult) + hFunc(TEvBlobStorage::TEvVPutResult, Handle) + hFunc(TEvBlobStorage::TEvVPatchXorDiff, Handle) + hFunc(TKikimrEvents::TEvWakeup, HandleInDataOrParityStates) + IgnoreFunc(TEvVPatchDyingConfirm) + default: Y_FAIL_S(VDiskLogPrefix << " unexpected event " << ToString(ev->GetTypeRewrite())); + } + } + }; + +} // NKikimr::NPrivate + +namespace NKikimr { + IActor* CreateSkeletonVPatchActor(TActorId leaderId, const TBlobStorageGroupType &gType, - TEvBlobStorage::TEvVPatchStart::TPtr &ev, TInstant now, TActorIDPtr skeletonFrontIDPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, - const TIntrusivePtr<TVPatchCtx> &vPatchCtx, const TString &vDiskLogPrefix, ui64 incarnationGuid) - { + TEvBlobStorage::TEvVPatchStart::TPtr &ev, TInstant now, TActorIDPtr skeletonFrontIDPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, + const TIntrusivePtr<TVPatchCtx> &vPatchCtx, const TString &vDiskLogPrefix, ui64 incarnationGuid) + { return new NPrivate::TSkeletonVPatchActor(leaderId, gType, ev, now, skeletonFrontIDPtr, - vPatchFoundPartsMsgsPtr, vPatchResMsgsPtr, vPatchCtx, vDiskLogPrefix, incarnationGuid); - } - -} // NKikimr + vPatchFoundPartsMsgsPtr, vPatchResMsgsPtr, vPatchCtx, vDiskLogPrefix, incarnationGuid); + } + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.h b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.h index 8c2c0d658c3..d5b2fdefae9 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.h +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor.h @@ -1,39 +1,39 @@ -#pragma once - -#include "defs.h" - +#pragma once + +#include "defs.h" + #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> #include <ydb/core/blobstorage/base/utility.h> #include <ydb/core/blobstorage/base/vdisk_sync_common.h> - - -namespace NKikimr { - -struct TEvVPatchDyingRequest : TEventLocal< - TEvVPatchDyingRequest, - TEvBlobStorage::EvVPatchDyingRequest> -{ - TEvVPatchDyingRequest(TLogoBlobID id) - : PatchedBlobId(id) - {} - - TLogoBlobID PatchedBlobId; -}; - -struct TEvVPatchDyingConfirm : TEventLocal< - TEvVPatchDyingConfirm, - TEvBlobStorage::EvVPatchDyingConfirm> -{}; - -struct TVPatchCtx : public TThrRefBase, TNonCopyable { - TQueueActorMap AsyncBlobQueues; - - TVPatchCtx() = default; -}; - + + +namespace NKikimr { + +struct TEvVPatchDyingRequest : TEventLocal< + TEvVPatchDyingRequest, + TEvBlobStorage::EvVPatchDyingRequest> +{ + TEvVPatchDyingRequest(TLogoBlobID id) + : PatchedBlobId(id) + {} + + TLogoBlobID PatchedBlobId; +}; + +struct TEvVPatchDyingConfirm : TEventLocal< + TEvVPatchDyingConfirm, + TEvBlobStorage::EvVPatchDyingConfirm> +{}; + +struct TVPatchCtx : public TThrRefBase, TNonCopyable { + TQueueActorMap AsyncBlobQueues; + + TVPatchCtx() = default; +}; + IActor* CreateSkeletonVPatchActor(TActorId leaderId, const TBlobStorageGroupType &gType, TEvBlobStorage::TEvVPatchStart::TPtr &ev, - TInstant now, TActorIDPtr skeletonFrontIDPtr, const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, - const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, const TIntrusivePtr<TVPatchCtx> &vPatchCtx, - const TString &vDiskLogPrefix, ui64 incarnationGuid); - -} // NKikimr + TInstant now, TActorIDPtr skeletonFrontIDPtr, const NMonitoring::TDynamicCounters::TCounterPtr &vPatchFoundPartsMsgsPtr, + const NMonitoring::TDynamicCounters::TCounterPtr &vPatchResMsgsPtr, const TIntrusivePtr<TVPatchCtx> &vPatchCtx, + const TString &vDiskLogPrefix, ui64 incarnationGuid); + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor_ut.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor_ut.cpp index ca54eeec88c..84610009694 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor_ut.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vpatch_actor_ut.cpp @@ -1,880 +1,880 @@ -#include "skeleton_vpatch_actor.h" - +#include "skeleton_vpatch_actor.h" + #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> - + #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/basics/appdata.h> - - -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { - - enum { - Begin = EventSpaceBegin(TEvents::ES_USERSPACE), - EvRequestEnd - }; - - struct TEvRequestEnd : TEventLocal<TEvRequestEnd, EvRequestEnd> { - TEvRequestEnd() = default; - }; - - struct TVPatchDecoratorArgs { - TActorId EdgeActor; - bool IsCheckingEvents; - TVector<ui64> SequenceOfReceivingEvents; - TVector<ui64> SequenceOfSendingEvents; - }; - - struct TVPatchDecorator : public TTestDecorator { - - static constexpr bool CheckDyingState = false; - bool InStateFunc = false; - - TActorId EdgeActor; - - TVector<ui64> SequenceOfReceivingEvents; - TVector<ui64> SequenceOfSendingEvents; - ui64 ReceivingIdx = 0; - ui64 SendingIdx = 0; - bool IsCheckingEvents = false; - - TVPatchDecorator(THolder<IActor> &&actor, TVPatchDecoratorArgs &args) - : TTestDecorator(std::move(actor)) - , EdgeActor(args.EdgeActor) - , SequenceOfReceivingEvents(std::move(args.SequenceOfReceivingEvents)) - , SequenceOfSendingEvents(std::move(args.SequenceOfSendingEvents)) - , IsCheckingEvents(args.IsCheckingEvents) - { - } - - virtual ~TVPatchDecorator() { - if (NActors::TlsActivationContext) { + + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { + + enum { + Begin = EventSpaceBegin(TEvents::ES_USERSPACE), + EvRequestEnd + }; + + struct TEvRequestEnd : TEventLocal<TEvRequestEnd, EvRequestEnd> { + TEvRequestEnd() = default; + }; + + struct TVPatchDecoratorArgs { + TActorId EdgeActor; + bool IsCheckingEvents; + TVector<ui64> SequenceOfReceivingEvents; + TVector<ui64> SequenceOfSendingEvents; + }; + + struct TVPatchDecorator : public TTestDecorator { + + static constexpr bool CheckDyingState = false; + bool InStateFunc = false; + + TActorId EdgeActor; + + TVector<ui64> SequenceOfReceivingEvents; + TVector<ui64> SequenceOfSendingEvents; + ui64 ReceivingIdx = 0; + ui64 SendingIdx = 0; + bool IsCheckingEvents = false; + + TVPatchDecorator(THolder<IActor> &&actor, TVPatchDecoratorArgs &args) + : TTestDecorator(std::move(actor)) + , EdgeActor(args.EdgeActor) + , SequenceOfReceivingEvents(std::move(args.SequenceOfReceivingEvents)) + , SequenceOfSendingEvents(std::move(args.SequenceOfSendingEvents)) + , IsCheckingEvents(args.IsCheckingEvents) + { + } + + virtual ~TVPatchDecorator() { + if (NActors::TlsActivationContext) { std::unique_ptr<IEventBase> ev = std::make_unique<TEvRequestEnd>(); std::unique_ptr<IEventHandle> handle = std::make_unique<IEventHandle>(EdgeActor, EdgeActor, ev.release()); TActivationContext::Send(handle.release()); - } - } - - bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { - if (IsCheckingEvents) { - UNIT_ASSERT_LT_C(SendingIdx, SequenceOfSendingEvents.size(), "SequenceOfSendingEvents overbounded"); - UNIT_ASSERT_VALUES_EQUAL_C(SequenceOfSendingEvents[SendingIdx], ev->Type, "sending idx " << SendingIdx); - } - - SendingIdx++; - return true; - } - - bool DoBeforeReceiving(TAutoPtr<IEventHandle> &ev, const TActorContext &/*ctx*/) override { - if (ev->Type == TEvents::TSystem::PoisonPill) { - PassAway(); - return false; - } - - InStateFunc = true; - if (IsCheckingEvents) { - UNIT_ASSERT_LT_C(ReceivingIdx, SequenceOfReceivingEvents.size(), "SequenceOfReceivingEvents overbounded"); - UNIT_ASSERT_VALUES_EQUAL_C(SequenceOfReceivingEvents[ReceivingIdx], ev->Type, "receive idx " << ReceivingIdx); - } - - ReceivingIdx++; - return true; - } - - void DoAfterReceiving(const TActorContext &/*ctx*/) override { - InStateFunc = false; - } - }; - - bool ScheduledFilterFunc(NActors::TTestActorRuntimeBase& runtime, TAutoPtr<NActors::IEventHandle>& event, - TDuration delay, TInstant& deadline) { - if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { - deadline = runtime.GetTimeProvider()->Now() + delay; - return false; - } - return true; - } - - struct TVPatchTestGeneralData { - TBlobStorageGroupType GType; - TTestBasicRuntime Runtime; // TODO(kruall): change to lighter - TVector<TActorId> EdgeActors; - TLogoBlobID OriginalBlobId; - TLogoBlobID PatchedBlobId; - - TInstant Deadline; - TInstant Now; - - TVector<ui64> SequenceOfReceivingEvents; - TVector<ui64> SequenceOfSendingEvents; - bool IsCheckingEventsByDecorator = false; - - TVector<TActorId> VPatchActorIds; - TVector<TVDiskID> VDiskIds; - - float ApproximateFreeSpaceShare = 0.1; - ui32 StatusFlags = 1; - - TVPatchTestGeneralData(const TBlobStorageGroupType &gType, ui32 blobSize, ui32 nodeCount = 1) - : GType(gType) - , Runtime(nodeCount, false) - , OriginalBlobId(1, 2, 3, 4, blobSize, 6) - , PatchedBlobId(1, 3, 3, 4, blobSize, 6) - , Deadline() - { - InitLogLevels(); - Runtime.SetScheduledEventFilter(&ScheduledFilterFunc); - Runtime.SetEventFilter([](NActors::TTestActorRuntimeBase&, TAutoPtr<NActors::IEventHandle>&) { - return false; - }); - - TAppPrepare app; - app.ClearDomainsAndHive(); - Runtime.Initialize(app.Unwrap()); - - for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { - EdgeActors.push_back(Runtime.AllocateEdgeActor(nodeIdx)); - TVDiskIdShort shortId(0, nodeIdx, 0); - VDiskIds.emplace_back(0, 1, shortId); - } - - Now = Runtime.GetCurrentTime(); - } - - void InitLogLevels() { - Runtime.SetLogPriority(NKikimrServices::BS_VDISK_PATCH, NLog::PRI_DEBUG); - Runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); - } - + } + } + + bool DoBeforeSending(TAutoPtr<IEventHandle> &ev) override { + if (IsCheckingEvents) { + UNIT_ASSERT_LT_C(SendingIdx, SequenceOfSendingEvents.size(), "SequenceOfSendingEvents overbounded"); + UNIT_ASSERT_VALUES_EQUAL_C(SequenceOfSendingEvents[SendingIdx], ev->Type, "sending idx " << SendingIdx); + } + + SendingIdx++; + return true; + } + + bool DoBeforeReceiving(TAutoPtr<IEventHandle> &ev, const TActorContext &/*ctx*/) override { + if (ev->Type == TEvents::TSystem::PoisonPill) { + PassAway(); + return false; + } + + InStateFunc = true; + if (IsCheckingEvents) { + UNIT_ASSERT_LT_C(ReceivingIdx, SequenceOfReceivingEvents.size(), "SequenceOfReceivingEvents overbounded"); + UNIT_ASSERT_VALUES_EQUAL_C(SequenceOfReceivingEvents[ReceivingIdx], ev->Type, "receive idx " << ReceivingIdx); + } + + ReceivingIdx++; + return true; + } + + void DoAfterReceiving(const TActorContext &/*ctx*/) override { + InStateFunc = false; + } + }; + + bool ScheduledFilterFunc(NActors::TTestActorRuntimeBase& runtime, TAutoPtr<NActors::IEventHandle>& event, + TDuration delay, TInstant& deadline) { + if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { + deadline = runtime.GetTimeProvider()->Now() + delay; + return false; + } + return true; + } + + struct TVPatchTestGeneralData { + TBlobStorageGroupType GType; + TTestBasicRuntime Runtime; // TODO(kruall): change to lighter + TVector<TActorId> EdgeActors; + TLogoBlobID OriginalBlobId; + TLogoBlobID PatchedBlobId; + + TInstant Deadline; + TInstant Now; + + TVector<ui64> SequenceOfReceivingEvents; + TVector<ui64> SequenceOfSendingEvents; + bool IsCheckingEventsByDecorator = false; + + TVector<TActorId> VPatchActorIds; + TVector<TVDiskID> VDiskIds; + + float ApproximateFreeSpaceShare = 0.1; + ui32 StatusFlags = 1; + + TVPatchTestGeneralData(const TBlobStorageGroupType &gType, ui32 blobSize, ui32 nodeCount = 1) + : GType(gType) + , Runtime(nodeCount, false) + , OriginalBlobId(1, 2, 3, 4, blobSize, 6) + , PatchedBlobId(1, 3, 3, 4, blobSize, 6) + , Deadline() + { + InitLogLevels(); + Runtime.SetScheduledEventFilter(&ScheduledFilterFunc); + Runtime.SetEventFilter([](NActors::TTestActorRuntimeBase&, TAutoPtr<NActors::IEventHandle>&) { + return false; + }); + + TAppPrepare app; + app.ClearDomainsAndHive(); + Runtime.Initialize(app.Unwrap()); + + for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { + EdgeActors.push_back(Runtime.AllocateEdgeActor(nodeIdx)); + TVDiskIdShort shortId(0, nodeIdx, 0); + VDiskIds.emplace_back(0, 1, shortId); + } + + Now = Runtime.GetCurrentTime(); + } + + void InitLogLevels() { + Runtime.SetLogPriority(NKikimrServices::BS_VDISK_PATCH, NLog::PRI_DEBUG); + Runtime.SetLogPriority(NActorsServices::TEST, NLog::PRI_DEBUG); + } + std::unique_ptr<TEvBlobStorage::TEvVPatchStart> CreateVPatchStart(TMaybe<ui64> cookie, ui32 nodeId = 0) const { return std::make_unique<TEvBlobStorage::TEvVPatchStart>(OriginalBlobId, PatchedBlobId, VDiskIds[nodeId], Deadline, - cookie, false); - } - + cookie, false); + } + std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> CreateVPatchDiff(ui8 partId, ui8 waitedXorDiffs, - const TVector<TDiff> &diffs, TMaybe<ui64> cookie, ui8 nodeId = 0) const { + const TVector<TDiff> &diffs, TMaybe<ui64> cookie, ui8 nodeId = 0) const { std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = std::make_unique<TEvBlobStorage::TEvVPatchDiff>(TLogoBlobID(OriginalBlobId, partId), - TLogoBlobID(PatchedBlobId, partId), VDiskIds[nodeId], waitedXorDiffs, Deadline, cookie); - for (auto &diffBlock : diffs) { - diff->AddDiff(diffBlock.Offset, diffBlock.Buffer); - } - return std::move(diff); - } - + TLogoBlobID(PatchedBlobId, partId), VDiskIds[nodeId], waitedXorDiffs, Deadline, cookie); + for (auto &diffBlock : diffs) { + diff->AddDiff(diffBlock.Offset, diffBlock.Buffer); + } + return std::move(diff); + } + std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> CreateForceEndVPatchDiff(ui8 partId, TMaybe<ui64> cookie) const { std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = std::make_unique<TEvBlobStorage::TEvVPatchDiff>(TLogoBlobID(OriginalBlobId, partId), - TLogoBlobID(PatchedBlobId, partId), VDiskIds[partId - 1], false, Deadline, cookie); - diff->SetForceEnd(); - return std::move(diff); - } - - template <typename DecoratorType = void> - TActorId CreateTVPatchActor(TEvBlobStorage::TEvVPatchStart::TPtr &&ev, ui32 nodeId = 0) { - TIntrusivePtr<TVPatchCtx> patchCtx = MakeIntrusive<TVPatchCtx>(); - for (ui32 idx = 0; idx < VDiskIds.size(); ++idx) { - TVDiskIdShort id(VDiskIds[idx]); - patchCtx->AsyncBlobQueues.emplace(id, EdgeActors[idx]); - } - + TLogoBlobID(PatchedBlobId, partId), VDiskIds[partId - 1], false, Deadline, cookie); + diff->SetForceEnd(); + return std::move(diff); + } + + template <typename DecoratorType = void> + TActorId CreateTVPatchActor(TEvBlobStorage::TEvVPatchStart::TPtr &&ev, ui32 nodeId = 0) { + TIntrusivePtr<TVPatchCtx> patchCtx = MakeIntrusive<TVPatchCtx>(); + for (ui32 idx = 0; idx < VDiskIds.size(); ++idx) { + TVDiskIdShort id(VDiskIds[idx]); + patchCtx->AsyncBlobQueues.emplace(id, EdgeActors[idx]); + } + THolder<IActor> actor{CreateSkeletonVPatchActor(EdgeActors[nodeId], GType, ev, TInstant(), nullptr, nullptr, nullptr, patchCtx, VDiskIds[nodeId].ToString(), 0)}; - - if constexpr (!std::is_void_v<DecoratorType>) { - TVPatchDecoratorArgs args{EdgeActors[nodeId], IsCheckingEventsByDecorator, - SequenceOfReceivingEvents, SequenceOfSendingEvents}; - actor = MakeHolder<DecoratorType>(std::move(actor), args); - } - - VPatchActorIds.emplace_back(Runtime.Register(actor.Release(), nodeId)); - return VPatchActorIds.back(); - } - - void WaitEndTest() { - for (TActorId &edgeActor : EdgeActors) { - Runtime.GrabEdgeEventRethrow<TEvRequestEnd>({edgeActor}); - } - } - - void ForceEndTest() { + + if constexpr (!std::is_void_v<DecoratorType>) { + TVPatchDecoratorArgs args{EdgeActors[nodeId], IsCheckingEventsByDecorator, + SequenceOfReceivingEvents, SequenceOfSendingEvents}; + actor = MakeHolder<DecoratorType>(std::move(actor), args); + } + + VPatchActorIds.emplace_back(Runtime.Register(actor.Release(), nodeId)); + return VPatchActorIds.back(); + } + + void WaitEndTest() { + for (TActorId &edgeActor : EdgeActors) { + Runtime.GrabEdgeEventRethrow<TEvRequestEnd>({edgeActor}); + } + } + + void ForceEndTest() { std::unique_ptr<IEventHandle> handle; - ui32 nodeCount = Runtime.GetNodeCount(); - for (ui32 nodeId = 0; nodeId < nodeCount; ++nodeId) { + ui32 nodeCount = Runtime.GetNodeCount(); + for (ui32 nodeId = 0; nodeId < nodeCount; ++nodeId) { handle = std::make_unique<IEventHandle>(VPatchActorIds[nodeId], EdgeActors[nodeId], - new NActors::TEvents::TEvPoisonPill); + new NActors::TEvents::TEvPoisonPill); Runtime.Send(handle.release()); - } - WaitEndTest(); - } - }; - - template<typename EventType> - typename EventType::TPtr CreateEventHandle(const TActorId &recipient, const TActorId &sender, + } + WaitEndTest(); + } + }; + + template<typename EventType> + typename EventType::TPtr CreateEventHandle(const TActorId &recipient, const TActorId &sender, std::unique_ptr<EventType> &&ev) - { + { return static_cast<TEventHandle<EventType>*>(new IEventHandle(recipient, sender, ev.release())); - } - - - - Y_UNIT_TEST_SUITE(TVPatchTests) { - - struct TBlob { - TLogoBlobID BlobId; - TString Buffer; - - TBlob(const TLogoBlobID &blob, ui8 partId, const TString &buffer = "") - : BlobId(blob, partId) - , Buffer(buffer) - { - } - - TBlob(const TLogoBlobID &blob, ui8 partId, ui32 bufferSize) - : BlobId(blob, partId) - { - TStringBuilder str; - for (ui32 idx = 0; idx < bufferSize; ++idx) { - str << 'a'; - } - Buffer = str; - } - }; - - bool PassFindingParts(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vGetStatus, - const TVector<ui8> &foundParts, ui32 nodeId = 0) { - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[nodeId]; - TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; - - TAutoPtr<IEventHandle> handle; - auto evVGetRange = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>(handle); - - UNIT_ASSERT(evVGetRange->Record.HasCookie()); - UNIT_ASSERT(evVGetRange->Record.HasIndexOnly() && evVGetRange->Record.GetIndexOnly()); + } + + + + Y_UNIT_TEST_SUITE(TVPatchTests) { + + struct TBlob { + TLogoBlobID BlobId; + TString Buffer; + + TBlob(const TLogoBlobID &blob, ui8 partId, const TString &buffer = "") + : BlobId(blob, partId) + , Buffer(buffer) + { + } + + TBlob(const TLogoBlobID &blob, ui8 partId, ui32 bufferSize) + : BlobId(blob, partId) + { + TStringBuilder str; + for (ui32 idx = 0; idx < bufferSize; ++idx) { + str << 'a'; + } + Buffer = str; + } + }; + + bool PassFindingParts(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vGetStatus, + const TVector<ui8> &foundParts, ui32 nodeId = 0) { + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[nodeId]; + TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; + + TAutoPtr<IEventHandle> handle; + auto evVGetRange = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>(handle); + + UNIT_ASSERT(evVGetRange->Record.HasCookie()); + UNIT_ASSERT(evVGetRange->Record.HasIndexOnly() && evVGetRange->Record.GetIndexOnly()); std::unique_ptr<TEvBlobStorage::TEvVGetResult> evVGetRangeResult = std::make_unique<TEvBlobStorage::TEvVGetResult>( - vGetStatus, testData.VDiskIds[nodeId], testData.Now, evVGetRange->GetCachedByteSize(), &evVGetRange->Record, - nullptr, nullptr, nullptr, std::move(handle->TraceId), evVGetRange->Record.GetCookie(), - handle->GetChannel(), 0); - - evVGetRangeResult->AddResult(NKikimrProto::OK, TLogoBlobID(testData.OriginalBlobId, 0)); - for (ui8 partId : foundParts) { - evVGetRangeResult->Record.MutableResult(0)->AddParts(partId); - } + vGetStatus, testData.VDiskIds[nodeId], testData.Now, evVGetRange->GetCachedByteSize(), &evVGetRange->Record, + nullptr, nullptr, nullptr, std::move(handle->TraceId), evVGetRange->Record.GetCookie(), + handle->GetChannel(), 0); + + evVGetRangeResult->AddResult(NKikimrProto::OK, TLogoBlobID(testData.OriginalBlobId, 0)); + for (ui8 partId : foundParts) { + evVGetRangeResult->Record.MutableResult(0)->AddParts(partId); + } handle = MakeHolder<IEventHandle>(vPatchActorId, edgeActor, evVGetRangeResult.release()); - runtime.Send(handle.Release()); - - auto evVPatchFoundParts = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchFoundParts>(handle); - NKikimrBlobStorage::TEvVPatchFoundParts &vPatchFoundParts = evVPatchFoundParts->Record; - UNIT_ASSERT(vPatchFoundParts.HasCookie()); - UNIT_ASSERT(vPatchFoundParts.GetCookie() == nodeId); - if (vPatchFoundParts.OriginalPartsSize()) { - UNIT_ASSERT(vPatchFoundParts.HasStatus()); - UNIT_ASSERT(vPatchFoundParts.GetStatus() == NKikimrProto::OK); - - TVector<ui8> parts; - parts.reserve(vPatchFoundParts.OriginalPartsSize()); - for (ui64 part : vPatchFoundParts.GetOriginalParts()) { - UNIT_ASSERT(part <= TLogoBlobID::MaxPartId); - parts.push_back(part); - } - UNIT_ASSERT(foundParts == parts); - return false; - } else { - UNIT_ASSERT(vPatchFoundParts.HasStatus()); - return true; - } - } - - void MakeVPatchFindingPartsTest(NKikimrProto::EReplyStatus vGetStatus, const TVector<ui8> &foundParts, - TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) - { - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - TVPatchTestGeneralData testData(type, 10); - TActorId edgeActor = testData.EdgeActors[0]; - - testData.IsCheckingEventsByDecorator = true; - testData.SequenceOfReceivingEvents = std::move(receivingEvents); - testData.SequenceOfSendingEvents = std::move(sendingEvents); - + runtime.Send(handle.Release()); + + auto evVPatchFoundParts = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchFoundParts>(handle); + NKikimrBlobStorage::TEvVPatchFoundParts &vPatchFoundParts = evVPatchFoundParts->Record; + UNIT_ASSERT(vPatchFoundParts.HasCookie()); + UNIT_ASSERT(vPatchFoundParts.GetCookie() == nodeId); + if (vPatchFoundParts.OriginalPartsSize()) { + UNIT_ASSERT(vPatchFoundParts.HasStatus()); + UNIT_ASSERT(vPatchFoundParts.GetStatus() == NKikimrProto::OK); + + TVector<ui8> parts; + parts.reserve(vPatchFoundParts.OriginalPartsSize()); + for (ui64 part : vPatchFoundParts.GetOriginalParts()) { + UNIT_ASSERT(part <= TLogoBlobID::MaxPartId); + parts.push_back(part); + } + UNIT_ASSERT(foundParts == parts); + return false; + } else { + UNIT_ASSERT(vPatchFoundParts.HasStatus()); + return true; + } + } + + void MakeVPatchFindingPartsTest(NKikimrProto::EReplyStatus vGetStatus, const TVector<ui8> &foundParts, + TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) + { + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + TVPatchTestGeneralData testData(type, 10); + TActorId edgeActor = testData.EdgeActors[0]; + + testData.IsCheckingEventsByDecorator = true; + testData.SequenceOfReceivingEvents = std::move(receivingEvents); + testData.SequenceOfSendingEvents = std::move(sendingEvents); + std::unique_ptr<TEvBlobStorage::TEvVPatchStart> start = testData.CreateVPatchStart(0); - TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); - TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); - - bool isKilled = PassFindingParts(testData, vGetStatus, foundParts); - TAutoPtr<IEventHandle> handle; - if (!isKilled) { + TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); + TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); + + bool isKilled = PassFindingParts(testData, vGetStatus, foundParts); + TAutoPtr<IEventHandle> handle; + if (!isKilled) { std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateForceEndVPatchDiff(1, 0); handle = MakeHolder<IEventHandle>(vPatchActorId, edgeActor, diff.release()); - testData.Runtime.Send(handle.Release()); - - auto result = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); - UNIT_ASSERT(result->Record.GetStatus() == NKikimrProto::OK); - } else { - auto diyngRequest = testData.Runtime.GrabEdgeEventRethrow<TEvVPatchDyingRequest>(handle); - UNIT_ASSERT(diyngRequest->PatchedBlobId == testData.PatchedBlobId); - } - - testData.WaitEndTest(); - } - - Y_UNIT_TEST(FindingPartsWhenPartsAreDontExist) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchDyingRequest}; - MakeVPatchFindingPartsTest(NKikimrProto::OK, {}, std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(FindingPartsWhenOnlyOnePartExists) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchDiff}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchResult}; - MakeVPatchFindingPartsTest(NKikimrProto::OK, {1}, std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(FindingPartsWhenSeveralPartsExist) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchDiff}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchResult}; - MakeVPatchFindingPartsTest(NKikimrProto::OK, {1, 2}, std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(FindingPartsWhenError) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchDyingRequest}; - MakeVPatchFindingPartsTest(NKikimrProto::ERROR, {}, std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(FindingPartsWithTimeout) { - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - TVPatchTestGeneralData testData(type, 10); - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[0]; - - testData.IsCheckingEventsByDecorator = true; - testData.SequenceOfReceivingEvents = {TEvents::TSystem::Bootstrap, TKikimrEvents::TSystem::Wakeup}; - testData.SequenceOfSendingEvents = { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchDyingRequest}; - + testData.Runtime.Send(handle.Release()); + + auto result = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); + UNIT_ASSERT(result->Record.GetStatus() == NKikimrProto::OK); + } else { + auto diyngRequest = testData.Runtime.GrabEdgeEventRethrow<TEvVPatchDyingRequest>(handle); + UNIT_ASSERT(diyngRequest->PatchedBlobId == testData.PatchedBlobId); + } + + testData.WaitEndTest(); + } + + Y_UNIT_TEST(FindingPartsWhenPartsAreDontExist) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchDyingRequest}; + MakeVPatchFindingPartsTest(NKikimrProto::OK, {}, std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(FindingPartsWhenOnlyOnePartExists) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchDiff}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchResult}; + MakeVPatchFindingPartsTest(NKikimrProto::OK, {1}, std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(FindingPartsWhenSeveralPartsExist) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchDiff}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchResult}; + MakeVPatchFindingPartsTest(NKikimrProto::OK, {1, 2}, std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(FindingPartsWhenError) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchDyingRequest}; + MakeVPatchFindingPartsTest(NKikimrProto::ERROR, {}, std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(FindingPartsWithTimeout) { + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + TVPatchTestGeneralData testData(type, 10); + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[0]; + + testData.IsCheckingEventsByDecorator = true; + testData.SequenceOfReceivingEvents = {TEvents::TSystem::Bootstrap, TKikimrEvents::TSystem::Wakeup}; + testData.SequenceOfSendingEvents = { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchDyingRequest}; + std::unique_ptr<TEvBlobStorage::TEvVPatchStart> start = testData.CreateVPatchStart(0); - TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); - TActorId actorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); - - runtime.EnableScheduleForActor(actorId); - - TAutoPtr<IEventHandle> handle; - auto evVPatchFoundParts = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchFoundParts>(handle); - NKikimrBlobStorage::TEvVPatchFoundParts &vPatchFoundParts = evVPatchFoundParts->Record; - UNIT_ASSERT_VALUES_EQUAL(vPatchFoundParts.GetStatus(), NKikimrProto::ERROR); - - auto dyingRequest = runtime.GrabEdgeEventRethrow<TEvVPatchDyingRequest>(handle); - UNIT_ASSERT_VALUES_EQUAL(dyingRequest->PatchedBlobId, testData.PatchedBlobId); - testData.WaitEndTest(); - } - - bool PassPullingPart(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vGetStatus, - const TBlob &blob, ui32 nodeId = 0) { - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[nodeId]; - TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; - - - auto vGetHandle = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>({edgeActor}); - auto evVGet = vGetHandle->Get(); - - UNIT_ASSERT(evVGet->Record.HasCookie()); - UNIT_ASSERT(!evVGet->Record.HasIndexOnly() || !evVGet->Record.GetIndexOnly()); - + TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); + TActorId actorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); + + runtime.EnableScheduleForActor(actorId); + + TAutoPtr<IEventHandle> handle; + auto evVPatchFoundParts = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchFoundParts>(handle); + NKikimrBlobStorage::TEvVPatchFoundParts &vPatchFoundParts = evVPatchFoundParts->Record; + UNIT_ASSERT_VALUES_EQUAL(vPatchFoundParts.GetStatus(), NKikimrProto::ERROR); + + auto dyingRequest = runtime.GrabEdgeEventRethrow<TEvVPatchDyingRequest>(handle); + UNIT_ASSERT_VALUES_EQUAL(dyingRequest->PatchedBlobId, testData.PatchedBlobId); + testData.WaitEndTest(); + } + + bool PassPullingPart(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vGetStatus, + const TBlob &blob, ui32 nodeId = 0) { + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[nodeId]; + TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; + + + auto vGetHandle = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVGet>({edgeActor}); + auto evVGet = vGetHandle->Get(); + + UNIT_ASSERT(evVGet->Record.HasCookie()); + UNIT_ASSERT(!evVGet->Record.HasIndexOnly() || !evVGet->Record.GetIndexOnly()); + std::unique_ptr<TEvBlobStorage::TEvVGetResult> evVGetResult = std::make_unique<TEvBlobStorage::TEvVGetResult>( - vGetStatus, testData.VDiskIds[nodeId], testData.Now, evVGet->GetCachedByteSize(), &evVGet->Record, - nullptr, nullptr, nullptr, std::move(vGetHandle->TraceId), evVGet->Record.GetCookie(), - vGetHandle->GetChannel(), 0); - evVGetResult->AddResult(NKikimrProto::OK, blob.BlobId, 0, blob.Buffer.data(), blob.Buffer.size()); - + vGetStatus, testData.VDiskIds[nodeId], testData.Now, evVGet->GetCachedByteSize(), &evVGet->Record, + nullptr, nullptr, nullptr, std::move(vGetHandle->TraceId), evVGet->Record.GetCookie(), + vGetHandle->GetChannel(), 0); + evVGetResult->AddResult(NKikimrProto::OK, blob.BlobId, 0, blob.Buffer.data(), blob.Buffer.size()); + std::unique_ptr<IEventHandle> handle = std::make_unique<IEventHandle>(vPatchActorId, edgeActor, evVGetResult.release()); runtime.Send(handle.release()); - - return vGetStatus != NKikimrProto::OK; - } - - bool PassStoringPart(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vPutStatus, const TBlob &blob, - ui32 nodeId = 0) - { - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[nodeId]; - TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; - - TAutoPtr<IEventHandle> handle; - auto vPut = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPut>(handle); - NKikimrBlobStorage::TEvVPut &record = vPut->Record; - - UNIT_ASSERT(record.HasCookie()); - ui64 cookie = record.GetCookie(); - TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); - UNIT_ASSERT_VALUES_EQUAL(blobId, blob.BlobId); - UNIT_ASSERT_C(vPut->GetBuffer() == blob.Buffer, "NodeId# " << nodeId); - - TOutOfSpaceStatus oos = TOutOfSpaceStatus(testData.StatusFlags, testData.ApproximateFreeSpaceShare); + + return vGetStatus != NKikimrProto::OK; + } + + bool PassStoringPart(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus vPutStatus, const TBlob &blob, + ui32 nodeId = 0) + { + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[nodeId]; + TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; + + TAutoPtr<IEventHandle> handle; + auto vPut = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPut>(handle); + NKikimrBlobStorage::TEvVPut &record = vPut->Record; + + UNIT_ASSERT(record.HasCookie()); + ui64 cookie = record.GetCookie(); + TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(record.GetBlobID()); + UNIT_ASSERT_VALUES_EQUAL(blobId, blob.BlobId); + UNIT_ASSERT_C(vPut->GetBuffer() == blob.Buffer, "NodeId# " << nodeId); + + TOutOfSpaceStatus oos = TOutOfSpaceStatus(testData.StatusFlags, testData.ApproximateFreeSpaceShare); std::unique_ptr<TEvBlobStorage::TEvVPutResult> vPutResult = std::make_unique<TEvBlobStorage::TEvVPutResult>( - vPutStatus, blobId, testData.VDiskIds[nodeId], &cookie, oos, testData.Now, - 0, &record, nullptr, nullptr, nullptr, vPut->GetBufferBytes(), std::move(handle->TraceId), - 0, ""); - + vPutStatus, blobId, testData.VDiskIds[nodeId], &cookie, oos, testData.Now, + 0, &record, nullptr, nullptr, nullptr, vPut->GetBufferBytes(), std::move(handle->TraceId), + 0, ""); + handle = MakeHolder<IEventHandle>(vPatchActorId, edgeActor, vPutResult.release()); - runtime.Send(handle.Release()); - return true; - } - - void MakeVPatchTest(NKikimrProto::EReplyStatus pullingStatus, NKikimrProto::EReplyStatus storingStatus, - ui32 partSize, const TVector<ui8> &foundPartIds, ui8 pullingPart, ui32 xorReceiverCount, - TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) - { - Y_UNUSED(xorReceiverCount); - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - TVPatchTestGeneralData testData(type, 10); - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[0]; - - testData.IsCheckingEventsByDecorator = true; - testData.SequenceOfReceivingEvents = std::move(receivingEvents); - testData.SequenceOfSendingEvents = std::move(sendingEvents); - + runtime.Send(handle.Release()); + return true; + } + + void MakeVPatchTest(NKikimrProto::EReplyStatus pullingStatus, NKikimrProto::EReplyStatus storingStatus, + ui32 partSize, const TVector<ui8> &foundPartIds, ui8 pullingPart, ui32 xorReceiverCount, + TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) + { + Y_UNUSED(xorReceiverCount); + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + TVPatchTestGeneralData testData(type, 10); + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[0]; + + testData.IsCheckingEventsByDecorator = true; + testData.SequenceOfReceivingEvents = std::move(receivingEvents); + testData.SequenceOfSendingEvents = std::move(sendingEvents); + std::unique_ptr<TEvBlobStorage::TEvVPatchStart> start = testData.CreateVPatchStart(0); - TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); - TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); - - - bool isKilled = false; - isKilled = PassFindingParts(testData, NKikimrProto::OK, foundPartIds); - UNIT_ASSERT(!isKilled); - + TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); + TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); + + + bool isKilled = false; + isKilled = PassFindingParts(testData, NKikimrProto::OK, foundPartIds); + UNIT_ASSERT(!isKilled); + std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateVPatchDiff(pullingPart, false, {}, 0); - TAutoPtr<IEventHandle> handle; - + TAutoPtr<IEventHandle> handle; + handle = MakeHolder<IEventHandle>(vPatchActorId, edgeActor, diff.release()); - runtime.Send(handle.Release()); - TBlob pullingBlob(testData.OriginalBlobId, pullingPart, partSize); - - isKilled = PassPullingPart(testData, pullingStatus, pullingBlob); - if (isKilled) { - auto result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); - UNIT_ASSERT(result->Record.GetStatus() == NKikimrProto::ERROR); - testData.WaitEndTest(); - return; - } - - TBlob storingBlob = pullingBlob; - storingBlob.BlobId = TLogoBlobID(testData.PatchedBlobId, pullingPart); - isKilled = PassStoringPart(testData, storingStatus, storingBlob); - NKikimrProto::EReplyStatus expectedResultStatus = - (storingStatus != NKikimrProto::OK) ? NKikimrProto::ERROR : NKikimrProto::OK; - - auto result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); - UNIT_ASSERT(result->Record.GetStatus() == expectedResultStatus); - UNIT_ASSERT(result->Record.GetStatusFlags() == testData.StatusFlags); - UNIT_ASSERT(result->Record.GetApproximateFreeSpaceShare() == testData.ApproximateFreeSpaceShare); - testData.WaitEndTest(); - } - - Y_UNIT_TEST(PatchPartOk) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchDiff, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPutResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPut, - TEvBlobStorage::EvVPatchResult}; - - TVector<ui8> foundPartIds = {1}; - MakeVPatchTest(NKikimrProto::OK, NKikimrProto::OK, 100, {1}, 1, 0, - std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(PatchPartGetError) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchDiff, - TEvBlobStorage::EvVGetResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchResult}; - - TVector<ui8> foundPartIds = {1}; - MakeVPatchTest(NKikimrProto::ERROR, NKikimrProto::OK, 100, {1}, 1, 0, - std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(PatchPartPutError) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchDiff, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPutResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPut, - TEvBlobStorage::EvVPatchResult}; - - TVector<ui8> foundPartIds = {1}; - MakeVPatchTest(NKikimrProto::OK, NKikimrProto::ERROR, 100, {1}, 1, 0, - std::move(receivingEvents), std::move(sendingEvents)); - } - - void SendXorDiff(TVPatchTestGeneralData &testData, const TVector<TDiff> &xorDiffs, ui32 toPart, ui32 nodeId = 0) { - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[nodeId]; - TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; - TVDiskID vDiskId = testData.VDiskIds[nodeId]; - + runtime.Send(handle.Release()); + TBlob pullingBlob(testData.OriginalBlobId, pullingPart, partSize); + + isKilled = PassPullingPart(testData, pullingStatus, pullingBlob); + if (isKilled) { + auto result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); + UNIT_ASSERT(result->Record.GetStatus() == NKikimrProto::ERROR); + testData.WaitEndTest(); + return; + } + + TBlob storingBlob = pullingBlob; + storingBlob.BlobId = TLogoBlobID(testData.PatchedBlobId, pullingPart); + isKilled = PassStoringPart(testData, storingStatus, storingBlob); + NKikimrProto::EReplyStatus expectedResultStatus = + (storingStatus != NKikimrProto::OK) ? NKikimrProto::ERROR : NKikimrProto::OK; + + auto result = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); + UNIT_ASSERT(result->Record.GetStatus() == expectedResultStatus); + UNIT_ASSERT(result->Record.GetStatusFlags() == testData.StatusFlags); + UNIT_ASSERT(result->Record.GetApproximateFreeSpaceShare() == testData.ApproximateFreeSpaceShare); + testData.WaitEndTest(); + } + + Y_UNIT_TEST(PatchPartOk) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchDiff, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPutResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPut, + TEvBlobStorage::EvVPatchResult}; + + TVector<ui8> foundPartIds = {1}; + MakeVPatchTest(NKikimrProto::OK, NKikimrProto::OK, 100, {1}, 1, 0, + std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(PatchPartGetError) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchDiff, + TEvBlobStorage::EvVGetResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchResult}; + + TVector<ui8> foundPartIds = {1}; + MakeVPatchTest(NKikimrProto::ERROR, NKikimrProto::OK, 100, {1}, 1, 0, + std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(PatchPartPutError) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchDiff, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPutResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPut, + TEvBlobStorage::EvVPatchResult}; + + TVector<ui8> foundPartIds = {1}; + MakeVPatchTest(NKikimrProto::OK, NKikimrProto::ERROR, 100, {1}, 1, 0, + std::move(receivingEvents), std::move(sendingEvents)); + } + + void SendXorDiff(TVPatchTestGeneralData &testData, const TVector<TDiff> &xorDiffs, ui32 toPart, ui32 nodeId = 0) { + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[nodeId]; + TActorId vPatchActorId = testData.VPatchActorIds[nodeId]; + TVDiskID vDiskId = testData.VDiskIds[nodeId]; + std::unique_ptr<TEvBlobStorage::TEvVPatchXorDiff> xorDiff = std::make_unique<TEvBlobStorage::TEvVPatchXorDiff>( - TLogoBlobID(testData.OriginalBlobId, toPart), - TLogoBlobID(testData.PatchedBlobId, toPart), - vDiskId, toPart, testData.Deadline, 0); - for (auto &diff : xorDiffs) { - xorDiff->AddDiff(diff.Offset, diff.Buffer); - } - + TLogoBlobID(testData.OriginalBlobId, toPart), + TLogoBlobID(testData.PatchedBlobId, toPart), + vDiskId, toPart, testData.Deadline, 0); + for (auto &diff : xorDiffs) { + xorDiff->AddDiff(diff.Offset, diff.Buffer); + } + std::unique_ptr<IEventHandle> handle = std::make_unique<IEventHandle>(vPatchActorId, edgeActor, xorDiff.release()); runtime.Send(handle.release()); - } - - void ReceiveVPatchResult(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus status) { - TAutoPtr<IEventHandle> handle; - auto result = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); - UNIT_ASSERT(result->Record.GetStatus() == status); - } - - void MakeXorDiffFaultToleranceTest(NKikimrProto::EReplyStatus status, ui64 partSize, - const TVector<TDiff> &diffs, TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) - { - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - TVPatchTestGeneralData testData(type, partSize); - TTestActorRuntimeBase &runtime = testData.Runtime; - TActorId edgeActor = testData.EdgeActors[0]; - - testData.IsCheckingEventsByDecorator = true; - testData.SequenceOfReceivingEvents = std::move(receivingEvents); - testData.SequenceOfSendingEvents = std::move(sendingEvents); - + } + + void ReceiveVPatchResult(TVPatchTestGeneralData &testData, NKikimrProto::EReplyStatus status) { + TAutoPtr<IEventHandle> handle; + auto result = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchResult>(handle); + UNIT_ASSERT(result->Record.GetStatus() == status); + } + + void MakeXorDiffFaultToleranceTest(NKikimrProto::EReplyStatus status, ui64 partSize, + const TVector<TDiff> &diffs, TVector<ui64> &&receivingEvents, TVector<ui64> &&sendingEvents) + { + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + TVPatchTestGeneralData testData(type, partSize); + TTestActorRuntimeBase &runtime = testData.Runtime; + TActorId edgeActor = testData.EdgeActors[0]; + + testData.IsCheckingEventsByDecorator = true; + testData.SequenceOfReceivingEvents = std::move(receivingEvents); + testData.SequenceOfSendingEvents = std::move(sendingEvents); + std::unique_ptr<TEvBlobStorage::TEvVPatchStart> start = testData.CreateVPatchStart(0); - TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); - TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); - - ui8 partId = type.DataParts() + 1; - bool isKilled = false; - isKilled = PassFindingParts(testData, NKikimrProto::OK, {partId}); - UNIT_ASSERT(!isKilled); - - SendXorDiff(testData, diffs, type.DataParts()); + TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); + TActorId vPatchActorId = testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev)); + + ui8 partId = type.DataParts() + 1; + bool isKilled = false; + isKilled = PassFindingParts(testData, NKikimrProto::OK, {partId}); + UNIT_ASSERT(!isKilled); + + SendXorDiff(testData, diffs, type.DataParts()); std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateVPatchDiff(partId, 1, {}, 0); std::unique_ptr<IEventHandle> handle; - + handle = std::make_unique<IEventHandle>(vPatchActorId, edgeActor, diff.release()); runtime.Send(handle.release()); - - if (status != NKikimrProto::OK) { - TAutoPtr<IEventHandle> handle; - testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiffResult>(handle); - ReceiveVPatchResult(testData, status); - testData.WaitEndTest(); - } else { - testData.ForceEndTest(); - } - - } - - Y_UNIT_TEST(PatchPartFastXorDiffWithEmptyDiffBuffer) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchXorDiff, - TEvBlobStorage::EvVPatchDiff, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPutResult}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPut, - TEvBlobStorage::EvVPatchResult, - TEvBlobStorage::EvVPatchResult}; - - TVector<TDiff> diffs; - diffs.emplace_back("", 0, true, false); - MakeXorDiffFaultToleranceTest(NKikimrProto::OK, 100, diffs, - std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(PatchPartFastXorDiffBeyoundBlob) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchXorDiff, - TEvBlobStorage::EvVPatchDiff,}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchXorDiffResult, - TEvBlobStorage::EvVPatchResult}; - - TVector<TDiff> diffs; - diffs.emplace_back("", 100, true, false); - MakeXorDiffFaultToleranceTest(NKikimrProto::ERROR, 100, diffs, - std::move(receivingEvents), std::move(sendingEvents)); - } - - Y_UNIT_TEST(PatchPartFastXorDiffDisorder) { - TVector<ui64> receivingEvents { - TEvents::TSystem::Bootstrap, - TEvBlobStorage::EvVGetResult, - TEvBlobStorage::EvVPatchXorDiff, - TEvBlobStorage::EvVPatchDiff,}; - TVector<ui64> sendingEvents { - TEvBlobStorage::EvVGet, - TEvBlobStorage::EvVPatchFoundParts, - TEvBlobStorage::EvVPatchXorDiffResult, - TEvBlobStorage::EvVPatchResult}; - - TVector<TDiff> diffs; - diffs.emplace_back("aa", 3, true, false); - diffs.emplace_back("aa", 0, true, false); - MakeXorDiffFaultToleranceTest(NKikimrProto::ERROR, 100, diffs, - std::move(receivingEvents), std::move(sendingEvents)); - } - - void MakeFullVPatchTest(const TBlobStorageGroupType &type, const TString &data, const TVector<TDiff> &diffs, - bool quickXorDiffs = false) - { - ui32 nodeCount = type.TotalPartCount(); - TVPatchTestGeneralData testData(type, data.Size(), nodeCount); - - for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { + + if (status != NKikimrProto::OK) { + TAutoPtr<IEventHandle> handle; + testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiffResult>(handle); + ReceiveVPatchResult(testData, status); + testData.WaitEndTest(); + } else { + testData.ForceEndTest(); + } + + } + + Y_UNIT_TEST(PatchPartFastXorDiffWithEmptyDiffBuffer) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchXorDiff, + TEvBlobStorage::EvVPatchDiff, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPutResult}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPut, + TEvBlobStorage::EvVPatchResult, + TEvBlobStorage::EvVPatchResult}; + + TVector<TDiff> diffs; + diffs.emplace_back("", 0, true, false); + MakeXorDiffFaultToleranceTest(NKikimrProto::OK, 100, diffs, + std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(PatchPartFastXorDiffBeyoundBlob) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchXorDiff, + TEvBlobStorage::EvVPatchDiff,}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchXorDiffResult, + TEvBlobStorage::EvVPatchResult}; + + TVector<TDiff> diffs; + diffs.emplace_back("", 100, true, false); + MakeXorDiffFaultToleranceTest(NKikimrProto::ERROR, 100, diffs, + std::move(receivingEvents), std::move(sendingEvents)); + } + + Y_UNIT_TEST(PatchPartFastXorDiffDisorder) { + TVector<ui64> receivingEvents { + TEvents::TSystem::Bootstrap, + TEvBlobStorage::EvVGetResult, + TEvBlobStorage::EvVPatchXorDiff, + TEvBlobStorage::EvVPatchDiff,}; + TVector<ui64> sendingEvents { + TEvBlobStorage::EvVGet, + TEvBlobStorage::EvVPatchFoundParts, + TEvBlobStorage::EvVPatchXorDiffResult, + TEvBlobStorage::EvVPatchResult}; + + TVector<TDiff> diffs; + diffs.emplace_back("aa", 3, true, false); + diffs.emplace_back("aa", 0, true, false); + MakeXorDiffFaultToleranceTest(NKikimrProto::ERROR, 100, diffs, + std::move(receivingEvents), std::move(sendingEvents)); + } + + void MakeFullVPatchTest(const TBlobStorageGroupType &type, const TString &data, const TVector<TDiff> &diffs, + bool quickXorDiffs = false) + { + ui32 nodeCount = type.TotalPartCount(); + TVPatchTestGeneralData testData(type, data.Size(), nodeCount); + + for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { std::unique_ptr<TEvBlobStorage::TEvVPatchStart> start = testData.CreateVPatchStart(nodeIdx, nodeIdx); - TActorId edgeActor = testData.EdgeActors[nodeIdx]; - TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); - testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev), nodeIdx); - } - - TString savedData = TString::Uninitialized(data.size()); - memcpy(savedData.begin(), data.begin(), data.size()); - - TString result = TString::Uninitialized(data.size()); - memcpy(result.begin(), data.begin(), data.size()); - ui8 *resultBytes = reinterpret_cast<ui8*>(const_cast<char*>(result.data())); - type.ApplyDiff(TErasureType::CrcModeNone, resultBytes, diffs); - TDataPartSet partSet; - TDataPartSet resultPartSet; - TDataPartSet savedPartSet; - type.SplitData(TErasureType::CrcModeNone, data, partSet); - type.SplitData(TErasureType::CrcModeNone, savedData, savedPartSet); - type.SplitData(TErasureType::CrcModeNone, result, resultPartSet); - - TPartDiffSet diffSet; - type.SplitDiffs(TErasureType::CrcModeNone, data.size(), diffs, diffSet); - - for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { - ui8 partId = nodeIdx + 1; - PassFindingParts(testData, NKikimrProto::OK, {partId}, nodeIdx);; - } - - ui32 dataPartCount = type.DataParts(); - ui32 totalPartCount = type.TotalPartCount(); - - ui32 dataDiffCount = 0; - for (ui32 partIdx = 0; partIdx < dataPartCount; ++partIdx) { - ui32 partId = partIdx + 1; + TActorId edgeActor = testData.EdgeActors[nodeIdx]; + TEvBlobStorage::TEvVPatchStart::TPtr ev = CreateEventHandle(edgeActor, edgeActor, std::move(start)); + testData.CreateTVPatchActor<TVPatchDecorator>(std::move(ev), nodeIdx); + } + + TString savedData = TString::Uninitialized(data.size()); + memcpy(savedData.begin(), data.begin(), data.size()); + + TString result = TString::Uninitialized(data.size()); + memcpy(result.begin(), data.begin(), data.size()); + ui8 *resultBytes = reinterpret_cast<ui8*>(const_cast<char*>(result.data())); + type.ApplyDiff(TErasureType::CrcModeNone, resultBytes, diffs); + TDataPartSet partSet; + TDataPartSet resultPartSet; + TDataPartSet savedPartSet; + type.SplitData(TErasureType::CrcModeNone, data, partSet); + type.SplitData(TErasureType::CrcModeNone, savedData, savedPartSet); + type.SplitData(TErasureType::CrcModeNone, result, resultPartSet); + + TPartDiffSet diffSet; + type.SplitDiffs(TErasureType::CrcModeNone, data.size(), diffs, diffSet); + + for (ui32 nodeIdx = 0; nodeIdx < nodeCount; ++nodeIdx) { + ui8 partId = nodeIdx + 1; + PassFindingParts(testData, NKikimrProto::OK, {partId}, nodeIdx);; + } + + ui32 dataPartCount = type.DataParts(); + ui32 totalPartCount = type.TotalPartCount(); + + ui32 dataDiffCount = 0; + for (ui32 partIdx = 0; partIdx < dataPartCount; ++partIdx) { + ui32 partId = partIdx + 1; std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateVPatchDiff(partId, 0, - diffSet.PartDiffs[partIdx].Diffs, 0, partIdx); - - for (ui32 parityPartIdx = dataPartCount; parityPartIdx < totalPartCount; ++parityPartIdx) { - diff->AddXorReceiver(testData.VDiskIds[parityPartIdx], parityPartIdx + 1); - } - + diffSet.PartDiffs[partIdx].Diffs, 0, partIdx); + + for (ui32 parityPartIdx = dataPartCount; parityPartIdx < totalPartCount; ++parityPartIdx) { + diff->AddXorReceiver(testData.VDiskIds[parityPartIdx], parityPartIdx + 1); + } + std::unique_ptr<IEventHandle> handle; handle = std::make_unique<IEventHandle>(testData.VPatchActorIds[partIdx], testData.EdgeActors[partIdx], diff.release()); testData.Runtime.Send(handle.release()); - dataDiffCount++; - } - - for (ui32 partIdx = 0; partIdx < dataPartCount; ++partIdx) { - ui32 partId = partIdx + 1; - TBlob pullingBlob(testData.OriginalBlobId, partId, partSet.Parts[partIdx].OwnedString); - PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); - } - - if (!quickXorDiffs) { - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - ui32 partId = partIdx + 1; + dataDiffCount++; + } + + for (ui32 partIdx = 0; partIdx < dataPartCount; ++partIdx) { + ui32 partId = partIdx + 1; + TBlob pullingBlob(testData.OriginalBlobId, partId, partSet.Parts[partIdx].OwnedString); + PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); + } + + if (!quickXorDiffs) { + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + ui32 partId = partIdx + 1; std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateVPatchDiff(partId, dataDiffCount, - {}, 0, partIdx); + {}, 0, partIdx); std::unique_ptr<IEventHandle> handle; handle = std::make_unique<IEventHandle>(testData.VPatchActorIds[partIdx], testData.EdgeActors[partIdx], diff.release()); testData.Runtime.Send(handle.release()); - } - - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - ui32 partId = partIdx + 1; - TBlob pullingBlob(testData.OriginalBlobId, partId, savedPartSet.Parts[partIdx].OwnedString); - PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); - } - } - - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - for (ui32 dataDiffIdx = 0; dataDiffIdx < dataDiffCount; ++dataDiffIdx) { - TActorId edgeActor = testData.EdgeActors[partIdx]; - auto handle = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiff>({edgeActor}); - auto &record = handle->Get()->Record; - - ui8 fromPartId = record.GetFromPartId(); - ui32 patchedPartId = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()).PartId(); - - TVector<TDiff> xorDiffs; - const ui8 *buffer = reinterpret_cast<const ui8*>(partSet.Parts[fromPartId - 1].OwnedString.data()); - testData.GType.MakeXorDiff(TErasureType::CrcModeNone, data.size(), buffer, diffSet.PartDiffs[fromPartId - 1].Diffs, &xorDiffs); - - UNIT_ASSERT_VALUES_EQUAL_C(xorDiffs.size(), record.DiffsSize(), "from# " << (ui32)fromPartId); - for (ui32 idx = 0; idx < xorDiffs.size(); ++idx) { - UNIT_ASSERT_VALUES_EQUAL_C(xorDiffs[idx].Offset, record.GetDiffs(idx).GetOffset(), "from# " << (ui32)fromPartId); - UNIT_ASSERT_EQUAL(xorDiffs[idx].Buffer, record.GetDiffs(idx).GetBuffer()); - } - - TActorId patchActor = testData.VPatchActorIds[patchedPartId - 1]; + } + + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + ui32 partId = partIdx + 1; + TBlob pullingBlob(testData.OriginalBlobId, partId, savedPartSet.Parts[partIdx].OwnedString); + PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); + } + } + + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + for (ui32 dataDiffIdx = 0; dataDiffIdx < dataDiffCount; ++dataDiffIdx) { + TActorId edgeActor = testData.EdgeActors[partIdx]; + auto handle = testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiff>({edgeActor}); + auto &record = handle->Get()->Record; + + ui8 fromPartId = record.GetFromPartId(); + ui32 patchedPartId = LogoBlobIDFromLogoBlobID(record.GetPatchedPartBlobId()).PartId(); + + TVector<TDiff> xorDiffs; + const ui8 *buffer = reinterpret_cast<const ui8*>(partSet.Parts[fromPartId - 1].OwnedString.data()); + testData.GType.MakeXorDiff(TErasureType::CrcModeNone, data.size(), buffer, diffSet.PartDiffs[fromPartId - 1].Diffs, &xorDiffs); + + UNIT_ASSERT_VALUES_EQUAL_C(xorDiffs.size(), record.DiffsSize(), "from# " << (ui32)fromPartId); + for (ui32 idx = 0; idx < xorDiffs.size(); ++idx) { + UNIT_ASSERT_VALUES_EQUAL_C(xorDiffs[idx].Offset, record.GetDiffs(idx).GetOffset(), "from# " << (ui32)fromPartId); + UNIT_ASSERT_EQUAL(xorDiffs[idx].Buffer, record.GetDiffs(idx).GetBuffer()); + } + + TActorId patchActor = testData.VPatchActorIds[patchedPartId - 1]; auto handle2 = std::make_unique<IEventHandle>(patchActor, edgeActor, handle->Release().Release(), handle->Flags, - handle->Cookie, nullptr, std::move(handle->TraceId)); + handle->Cookie, nullptr, std::move(handle->TraceId)); testData.Runtime.Send(handle2.release()); - } - } - - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - UNIT_ASSERT_EQUAL(partSet.Parts[partIdx].OwnedString, savedPartSet.Parts[partIdx].OwnedString); - } - - if (quickXorDiffs) { - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - ui32 partId = partIdx + 1; + } + } + + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + UNIT_ASSERT_EQUAL(partSet.Parts[partIdx].OwnedString, savedPartSet.Parts[partIdx].OwnedString); + } + + if (quickXorDiffs) { + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + ui32 partId = partIdx + 1; std::unique_ptr<TEvBlobStorage::TEvVPatchDiff> diff = testData.CreateVPatchDiff(partId, dataDiffCount, - {}, 0, partIdx); + {}, 0, partIdx); std::unique_ptr<IEventHandle> handle; handle = std::make_unique<IEventHandle>(testData.VPatchActorIds[partIdx], testData.EdgeActors[partIdx], diff.release()); testData.Runtime.Send(handle.release()); - } - - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - ui32 partId = partIdx + 1; - TBlob pullingBlob(testData.OriginalBlobId, partId, savedPartSet.Parts[partIdx].OwnedString); - PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); - } - } - - // receive xor diff's results - for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { - for (ui32 dataDiffIdx = 0; dataDiffIdx < dataDiffCount; ++dataDiffIdx) { - TActorId edgeActor = testData.EdgeActors[partIdx]; - testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiffResult>({edgeActor}); - } - } - - for (ui32 partIdx = 0; partIdx < totalPartCount; ++partIdx) { - ui32 partId = partIdx + 1; - TBlob storingBlob(testData.PatchedBlobId, partId, resultPartSet.Parts[partIdx].OwnedString); - PassStoringPart(testData, NKikimrProto::OK, storingBlob, partIdx); - } - - testData.ForceEndTest(); - Y_UNUSED(partSet, resultPartSet, diffSet); - } - - Y_UNIT_TEST(FullPatchTest) { - ui32 dataSize = 2079; - TString data = TString::Uninitialized(dataSize); - Fill(data.begin(), data.vend(), 'a'); - - ui32 diffSize = 31; - UNIT_ASSERT(dataSize % (diffSize + 1) == diffSize); - ui32 diffCount = dataSize / (diffSize + 1) + 1; - TVector<TDiff> diffs; - diffs.reserve(diffCount); - ui32 left = 0; - for (ui32 idx = 0; idx < diffCount; ++idx) { - TString buffer = TString::Uninitialized(diffSize); - Fill(buffer.begin(), buffer.vend(), 'a' + 1 + (idx % 25)); - diffs.emplace_back(buffer, left); - left += diffSize + 1; - } - - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - MakeFullVPatchTest(type, data, diffs); - } - - Y_UNIT_TEST(FullPatchTestXorDiffFasterVGetResult) { - ui32 dataSize = 2079; - TString data = TString::Uninitialized(dataSize); - Fill(data.begin(), data.vend(), 'a'); - - ui32 diffSize = 31; - UNIT_ASSERT(dataSize % (diffSize + 1) == diffSize); - ui32 diffCount = dataSize / (diffSize + 1) + 1; - TVector<TDiff> diffs; - diffs.reserve(diffCount); - ui32 left = 0; - for (ui32 idx = 0; idx < diffCount; ++idx) { - TString buffer = TString::Uninitialized(diffSize); - Fill(buffer.begin(), buffer.vend(), 'a' + 1 + (idx % 25)); - diffs.emplace_back(buffer, left); - left += diffSize + 1; - } - - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - MakeFullVPatchTest(type, data, diffs, true); - } - - Y_UNIT_TEST(FullPatchTestSpecialCase1) { - ui32 dataSize = 100; - TString data = TString::Uninitialized(dataSize); - Fill(data.begin(), data.vend(), 'a'); - - TVector<TDiff> diffs; - diffs.reserve(2); - diffs.emplace_back("b", 0); - diffs.emplace_back("b", 99); - - TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); - MakeFullVPatchTest(type, data, diffs, true); - } - } - -} // NKikimr + } + + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + ui32 partId = partIdx + 1; + TBlob pullingBlob(testData.OriginalBlobId, partId, savedPartSet.Parts[partIdx].OwnedString); + PassPullingPart(testData, NKikimrProto::OK, pullingBlob, partIdx); + } + } + + // receive xor diff's results + for (ui32 partIdx = dataPartCount; partIdx < totalPartCount; ++partIdx) { + for (ui32 dataDiffIdx = 0; dataDiffIdx < dataDiffCount; ++dataDiffIdx) { + TActorId edgeActor = testData.EdgeActors[partIdx]; + testData.Runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvVPatchXorDiffResult>({edgeActor}); + } + } + + for (ui32 partIdx = 0; partIdx < totalPartCount; ++partIdx) { + ui32 partId = partIdx + 1; + TBlob storingBlob(testData.PatchedBlobId, partId, resultPartSet.Parts[partIdx].OwnedString); + PassStoringPart(testData, NKikimrProto::OK, storingBlob, partIdx); + } + + testData.ForceEndTest(); + Y_UNUSED(partSet, resultPartSet, diffSet); + } + + Y_UNIT_TEST(FullPatchTest) { + ui32 dataSize = 2079; + TString data = TString::Uninitialized(dataSize); + Fill(data.begin(), data.vend(), 'a'); + + ui32 diffSize = 31; + UNIT_ASSERT(dataSize % (diffSize + 1) == diffSize); + ui32 diffCount = dataSize / (diffSize + 1) + 1; + TVector<TDiff> diffs; + diffs.reserve(diffCount); + ui32 left = 0; + for (ui32 idx = 0; idx < diffCount; ++idx) { + TString buffer = TString::Uninitialized(diffSize); + Fill(buffer.begin(), buffer.vend(), 'a' + 1 + (idx % 25)); + diffs.emplace_back(buffer, left); + left += diffSize + 1; + } + + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + MakeFullVPatchTest(type, data, diffs); + } + + Y_UNIT_TEST(FullPatchTestXorDiffFasterVGetResult) { + ui32 dataSize = 2079; + TString data = TString::Uninitialized(dataSize); + Fill(data.begin(), data.vend(), 'a'); + + ui32 diffSize = 31; + UNIT_ASSERT(dataSize % (diffSize + 1) == diffSize); + ui32 diffCount = dataSize / (diffSize + 1) + 1; + TVector<TDiff> diffs; + diffs.reserve(diffCount); + ui32 left = 0; + for (ui32 idx = 0; idx < diffCount; ++idx) { + TString buffer = TString::Uninitialized(diffSize); + Fill(buffer.begin(), buffer.vend(), 'a' + 1 + (idx % 25)); + diffs.emplace_back(buffer, left); + left += diffSize + 1; + } + + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + MakeFullVPatchTest(type, data, diffs, true); + } + + Y_UNIT_TEST(FullPatchTestSpecialCase1) { + ui32 dataSize = 100; + TString data = TString::Uninitialized(dataSize); + Fill(data.begin(), data.vend(), 'a'); + + TVector<TDiff> diffs; + diffs.reserve(2); + diffs.emplace_back("b", 0); + diffs.emplace_back("b", 99); + + TBlobStorageGroupType type(TErasureType::Erasure4Plus2Block); + MakeFullVPatchTest(type, data, diffs, true); + } + } + +} // NKikimr diff --git a/ydb/core/blobstorage/vdisk/skeleton/ut/ya.make b/ydb/core/blobstorage/vdisk/skeleton/ut/ya.make index 67d78277da5..426f765888b 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/ut/ya.make +++ b/ydb/core/blobstorage/vdisk/skeleton/ut/ya.make @@ -20,9 +20,9 @@ PEERDIR( SRCS( skeleton_oos_logic_ut.cpp - skeleton_vpatch_actor_ut.cpp + skeleton_vpatch_actor_ut.cpp ) -YQL_LAST_ABI_VERSION() - +YQL_LAST_ABI_VERSION() + END() diff --git a/ydb/core/blobstorage/vdisk/skeleton/ya.make b/ydb/core/blobstorage/vdisk/skeleton/ya.make index 892b79e6e12..e649cec219e 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/ya.make +++ b/ydb/core/blobstorage/vdisk/skeleton/ya.make @@ -41,10 +41,10 @@ SRCS( skeleton_overload_handler.h skeleton_vmultiput_actor.cpp skeleton_vmultiput_actor.h - skeleton_vmovedpatch_actor.cpp - skeleton_vmovedpatch_actor.h - skeleton_vpatch_actor.cpp - skeleton_vpatch_actor.h + skeleton_vmovedpatch_actor.cpp + skeleton_vmovedpatch_actor.h + skeleton_vpatch_actor.cpp + skeleton_vpatch_actor.h ) END() diff --git a/ydb/core/control/immediate_control_board_wrapper.h b/ydb/core/control/immediate_control_board_wrapper.h index ce8a6adde58..69e210355db 100644 --- a/ydb/core/control/immediate_control_board_wrapper.h +++ b/ydb/core/control/immediate_control_board_wrapper.h @@ -31,34 +31,34 @@ public: } }; -class TMemorizableControlWrapper { - static constexpr i32 RequestCountWithRelevantValue = 1024; - static constexpr TDuration TimeDurationWithRelevantValue = TDuration::Seconds(15); - TControlWrapper Control; - TInstant CheckingRelevantDeadline; - i32 CheckingCounter = 0; - i64 CurrentValue = 0; - -public: - TMemorizableControlWrapper(const TControlWrapper &control) - : Control(control) - , CurrentValue(Control) - { - } - - i64 Update(TInstant now) { - CheckingCounter--; - if (now > CheckingRelevantDeadline || CheckingCounter <= 0) { - CurrentValue = Control; - CheckingRelevantDeadline = now + TimeDurationWithRelevantValue; - CheckingCounter = RequestCountWithRelevantValue; - } - return CurrentValue; - } - - operator i64() const { - return CurrentValue; - } -}; - +class TMemorizableControlWrapper { + static constexpr i32 RequestCountWithRelevantValue = 1024; + static constexpr TDuration TimeDurationWithRelevantValue = TDuration::Seconds(15); + TControlWrapper Control; + TInstant CheckingRelevantDeadline; + i32 CheckingCounter = 0; + i64 CurrentValue = 0; + +public: + TMemorizableControlWrapper(const TControlWrapper &control) + : Control(control) + , CurrentValue(Control) + { + } + + i64 Update(TInstant now) { + CheckingCounter--; + if (now > CheckingRelevantDeadline || CheckingCounter <= 0) { + CurrentValue = Control; + CheckingRelevantDeadline = now + TimeDurationWithRelevantValue; + CheckingCounter = RequestCountWithRelevantValue; + } + return CurrentValue; + } + + operator i64() const { + return CurrentValue; + } +}; + } diff --git a/ydb/core/erasure/erasure.cpp b/ydb/core/erasure/erasure.cpp index a41b0279328..0a737fdb033 100644 --- a/ydb/core/erasure/erasure.cpp +++ b/ydb/core/erasure/erasure.cpp @@ -2642,332 +2642,332 @@ void TErasureType::IncrementalSplitData(ECrcMode crcMode, const TString& buffer, } } -void MirrorSplitDiff(const TErasureType &type, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) { - outDiffSet.PartDiffs.resize(type.TotalPartCount()); - ui32 parityParts = type.ParityParts(); - for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) { - outDiffSet.PartDiffs[partIdx].Diffs = diffs; - } -} - -void EoBlockSplitDiff(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) { - TBlockParams p(crcMode, type, dataSize); - outDiffSet.PartDiffs.resize(type.TotalPartCount()); - ui32 dataParts = type.DataParts(); - - ui32 partOffset = 0; - ui32 diffIdx = 0; - for (ui32 partIdx = 0; partIdx < dataParts && diffIdx < diffs.size(); ++partIdx) { - ui32 nextOffset = partOffset; - if (partIdx < p.FirstSmallPartIdx) { - nextOffset += p.LargePartSize; - } else { - nextOffset += p.SmallPartSize; - } - if (partIdx + 1 == dataParts) { - nextOffset = dataSize; - } - if (partOffset == nextOffset) { - continue; - } - TPartDiff &part = outDiffSet.PartDiffs[partIdx]; - - while (diffIdx < diffs.size()) { - const TDiff &diff = diffs[diffIdx]; - ui32 lineOffset = diff.Offset % sizeof(ui64); - ui32 diffEnd = diff.Offset + diff.GetDiffLength(); - - if (diff.Offset <= partOffset && diffEnd >= partOffset) { - ui32 diffEndForThisPart = Min(diffEnd, nextOffset); - ui32 diffShift = partOffset - diff.Offset; - - ui32 bufferSize = diffEndForThisPart - partOffset; - Y_VERIFY(bufferSize); - Y_VERIFY_S(diffShift + bufferSize <= diff.Buffer.size(), "diffShift# " << diffShift - << " bufferSize# " << bufferSize << " diff.GetDiffLength()# " << diff.GetDiffLength()); - TString newBuffer = TString::Uninitialized(bufferSize); - memcpy(newBuffer.begin(), diff.Buffer.begin() + diffShift, bufferSize); - part.Diffs.emplace_back(newBuffer, 0, false, true); - - if (diffEnd <= nextOffset) { - diffIdx++; - } else { - break; - } - } else if (diffEnd <= nextOffset) { - TString buffer; - ui32 bufferSize = 0; - if (lineOffset && !diff.IsAligned) { - bufferSize = diff.GetDiffLength() + lineOffset; - buffer = TString::Uninitialized(bufferSize); - memcpy(buffer.begin() + lineOffset, diff.Buffer.begin(), diff.GetDiffLength()); - } else { - buffer = diff.Buffer; - bufferSize = diff.Buffer.size(); - } - Y_VERIFY(bufferSize); - part.Diffs.emplace_back(buffer, diff.Offset - partOffset, false, true); - diffIdx++; - } else if (diff.Offset < nextOffset) { - ui32 bufferSize = nextOffset - diff.Offset + lineOffset; - TString newBuffer = TString::Uninitialized(bufferSize); - memcpy(newBuffer.begin() + lineOffset, diff.Buffer.begin(), bufferSize - lineOffset); - Y_VERIFY(bufferSize); - part.Diffs.emplace_back(newBuffer, diff.Offset - partOffset, false, true); - break; - } else { - break; - } - } - - partOffset = nextOffset; - } -} - -void TErasureType::SplitDiffs(ECrcMode crcMode, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) const { - Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); - const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; - - // change crc part only in during of applying diffs - switch (erasure.ErasureFamily) { - case TErasureType::ErasureMirror: - MirrorSplitDiff(*this, diffs, outDiffSet); - break; - case TErasureType::ErasureParityStripe: - Y_FAIL("Not implemented"); - break; - case TErasureType::ErasureParityBlock: - Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); - EoBlockSplitDiff(crcMode, *this, dataSize, diffs, outDiffSet); - break; - } -} - -template <typename Bucket> -void XorCpy(Bucket *dest, const Bucket *orig, const Bucket *diff, ui32 count) { - for (ui32 idx = 0; idx < count; ++idx) { - dest[idx] = orig[idx] ^ diff[idx]; - } -} - -void MakeEoBlockXorDiff(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, - const ui8 *src, const TVector<TDiff> &inDiffs, TVector<TDiff> *outDiffs) -{ - Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); - TBlockParams p(crcMode, type, dataSize); - - for (const TDiff &diff : inDiffs) { - const ui8 *diffBufferBytes = reinterpret_cast<const ui8*>(diff.Buffer.data()); - - ui32 lineOffset = diff.Offset % sizeof(ui64); - ui32 lowerStartPos = diff.Offset - lineOffset; - ui32 upperStartPos = lowerStartPos + (lineOffset ? sizeof(ui64) : 0); - - ui32 end = diff.Offset + diff.GetDiffLength(); - ui32 endLineOffset = end % sizeof(ui64); - ui32 lowerEndPos = end - endLineOffset; - ui32 upperEndPos = lowerEndPos + (endLineOffset ? sizeof(ui64) : 0); - - ui32 bufferSize = upperEndPos - lowerStartPos; - Y_VERIFY(bufferSize); - TString xorDiffBuffer = TString::Uninitialized(bufferSize); - ui8 *xorDiffBufferBytes = reinterpret_cast<ui8*>(const_cast<char*>(xorDiffBuffer.data())); - - if (lowerEndPos == lowerStartPos) { - ui64 &val = *reinterpret_cast<ui64*>(xorDiffBufferBytes); - val = 0; - ui32 byteCount = diff.GetDiffLength(); - XorCpy(xorDiffBufferBytes + lineOffset, src + diff.Offset, diffBufferBytes + lineOffset, byteCount); - outDiffs->emplace_back(xorDiffBuffer, diff.Offset, true, true); - continue; - } - - if (lineOffset) { - ui64 &val = *reinterpret_cast<ui64*>(xorDiffBufferBytes); - val = 0; - ui32 byteCount = Min<ui32>(sizeof(ui64) - lineOffset, diff.GetDiffLength()); - XorCpy(xorDiffBufferBytes + lineOffset, src + diff.Offset, diffBufferBytes + lineOffset, byteCount); - } - - ui32 firstLine = lowerStartPos / sizeof(ui64); - ui32 lineStart = upperStartPos / sizeof(ui64); - ui32 lineEnd = lowerEndPos / sizeof(ui64); - ui64 *xorUI64 = reinterpret_cast<ui64*>(xorDiffBufferBytes) + (lineStart - firstLine); - const ui64 *srcUI64 = reinterpret_cast<const ui64*>(src) + lineStart; - const ui64 *diffUI64 = reinterpret_cast<const ui64*>(diffBufferBytes)+ (lineStart - firstLine); - ui32 countUI64 = lineEnd - lineStart; - XorCpy(xorUI64, srcUI64, diffUI64, countUI64); - - if (endLineOffset) { - ui64 &val = reinterpret_cast<ui64*>(xorDiffBufferBytes)[lineEnd - firstLine]; - val = 0; - ui32 diffOffset = lowerEndPos - lowerStartPos; - XorCpy(xorDiffBufferBytes + diffOffset, src + lowerEndPos, diffBufferBytes + diffOffset, endLineOffset); - } - outDiffs->emplace_back(xorDiffBuffer, diff.Offset, true, true); - } -} - -void TErasureType::MakeXorDiff(ECrcMode crcMode, ui32 dataSize, const ui8 *src, - const TVector<TDiff> &inDiffs, TVector<TDiff> *outDiffs) const -{ - Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); - const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; - switch (erasure.ErasureFamily) { - case TErasureType::ErasureMirror: - Y_FAIL("unreachable"); - break; - case TErasureType::ErasureParityStripe: - Y_FAIL("Not implemented"); - break; - case TErasureType::ErasureParityBlock: - Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); - MakeEoBlockXorDiff(crcMode, *this, dataSize, src, inDiffs, outDiffs); - break; - } -} - -void TErasureType::ApplyDiff(ECrcMode crcMode, ui8 *dst, const TVector<TDiff> &diffs) const { - Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); - for (auto &diff : diffs) { - memcpy(dst + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); - } -} - -template <bool forSecondParity> -void ApllyXorDiffForEoBlock(const TBlockParams &p, ui64 *buffer, const ui64 *diff, ui64 adj, - ui32 begin, ui32 end, ui32 diffOffset, ui8 fromPart = 0) - { - ui32 blockBegin = begin / p.LineCount; - ui32 blockEnd = (end + p.LineCount - 1) / p.LineCount; - - Y_VERIFY(!forSecondParity || (blockBegin == 0 && blockEnd == 1)); - - if (forSecondParity && adj) { - for (ui32 idx = 0; idx < p.LineCount; ++idx) { - buffer[idx] ^= adj; - } - } - - Y_VERIFY(begin < end); - if constexpr (forSecondParity) { - ui32 lineBorder = p.LineCount - fromPart; - - if (begin < lineBorder) { - ui32 bufferBegin = begin + fromPart; - ui32 diffBegin = begin - diffOffset; - ui32 count = Min(lineBorder, end) - begin; - XorCpy(buffer + bufferBegin, buffer + bufferBegin, diff + diffBegin, count); - } - - if (end > lineBorder + 1) { - ui32 currentBegin = Max(lineBorder + 1, begin); - ui32 bufferBegin = currentBegin + fromPart - p.Prime; - ui32 count = end - currentBegin; - ui32 diffBegin = currentBegin - diffOffset; - XorCpy(buffer + bufferBegin, buffer + bufferBegin, diff + diffBegin, count); - } - } else { - ui32 count = end - begin; - ui32 diffBegin = begin - diffOffset; - XorCpy(buffer + begin, buffer + begin, diff + diffBegin, count); - } -} - -void ApplyEoBlockXorDiffForFirstParityPart(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, - ui64 *dst, const TVector<TDiff> &xorDiffs) -{ - Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); - TBlockParams p(crcMode, type, dataSize); - - for (const TDiff &diff : xorDiffs) { - Y_VERIFY(diff.Buffer.size() % sizeof(ui64) == 0); - Y_VERIFY(diff.IsXor && diff.IsAligned); - const ui64 *diffBuffer = reinterpret_cast<const ui64*>(diff.GetBufferBegin()); - ui32 diffOffset = diff.Offset / sizeof(ui64); - ui32 offset = diff.Offset / sizeof(ui64); - ui32 lineCount = diff.Buffer.size() / sizeof(ui64); - Y_VERIFY(diff.Offset < type.PartSize(crcMode, dataSize)); - Y_VERIFY(diff.Offset + diff.GetDiffLength() <= type.PartSize(crcMode, dataSize)); - ApllyXorDiffForEoBlock<false>(p, dst, diffBuffer, 0, offset, offset + lineCount, diffOffset); - } -} - -void ApplyXorForSecondParityPart(const TBlockParams &p, ui64 *startBufferBlock, const ui64 *startDiffBlock, - ui32 begin, ui32 end, ui8 fromPart, ui32 diffOffset) -{ - ui32 m = p.Prime; - const ui32 mint = (m - 2 < p.LineCount ? 1 : m - 2 - p.LineCount); - ui64 adj = 0; - ui32 adjRelatedBytesBegin = m - 1 - fromPart; - bool isAdjChanged = (fromPart >= mint) - && (adjRelatedBytesBegin >= begin) - && (adjRelatedBytesBegin < end); - - if (isAdjChanged) { - adj = startDiffBlock[adjRelatedBytesBegin - diffOffset]; - } - - ApllyXorDiffForEoBlock<true>(p, startBufferBlock, startDiffBlock, adj, begin, end, diffOffset, fromPart); -} - -void ApplyEoBlockXorDiffForSecondParityPart(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, - ui8 fromPart, ui64 *dst, const TVector<TDiff> &xorDiffs) -{ - Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); - TBlockParams p(crcMode, type, dataSize); - - ui64 bytesInBlock = p.LineCount * sizeof(ui64); - - for (const TDiff &diff : xorDiffs) { - Y_VERIFY(diff.Buffer.size() % sizeof(ui64) == 0); - Y_VERIFY(diff.IsXor && diff.IsAligned); - - const ui64 *diffBuffer = reinterpret_cast<const ui64*>(diff.GetBufferBegin()); - - ui32 firstBlock = diff.Offset / bytesInBlock * p.LineCount; - ui32 begin = diff.Offset / sizeof(ui64); - ui32 end = begin + diff.Buffer.size() / sizeof(ui64); - ui32 endBlock = (end + p.LineCount - 1) / p.LineCount * p.LineCount; - - for (ui32 idx = firstBlock; idx < endBlock; idx += p.LineCount) { - ui32 lineBegin = Max(idx, begin) - idx; - ui32 lineEnd = Min<ui32>(idx + p.LineCount, end) - idx; - if (idx == firstBlock) { - ui32 diffOffset = begin - idx; - ApplyXorForSecondParityPart(p, dst + idx, diffBuffer, lineBegin, lineEnd, fromPart, diffOffset); - } else { - ui32 diffOffset = idx - begin; - ApplyXorForSecondParityPart(p, dst + idx, diffBuffer + diffOffset, lineBegin, lineEnd, fromPart, 0); - } - } - } -} - -void TErasureType::ApplyXorDiff(ECrcMode crcMode, ui32 dataSize, ui8 *dst, - const TVector<TDiff> &diffs, ui8 fromPart, ui8 toPart) const -{ - Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); - const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; - switch (erasure.ErasureFamily) { - case TErasureType::ErasureMirror: - Y_FAIL("unreachable"); - break; - case TErasureType::ErasureParityStripe: - Y_FAIL("Not implemented"); - break; - case TErasureType::ErasureParityBlock: - Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); - ui64 *lineDst = reinterpret_cast<ui64*>(dst); - if (toPart + 1 != TotalPartCount()) { - ApplyEoBlockXorDiffForFirstParityPart(crcMode, *this, dataSize, lineDst, diffs); - } else { - ApplyEoBlockXorDiffForSecondParityPart(crcMode, *this, dataSize, fromPart, lineDst, diffs); - } - break; - } -} - +void MirrorSplitDiff(const TErasureType &type, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) { + outDiffSet.PartDiffs.resize(type.TotalPartCount()); + ui32 parityParts = type.ParityParts(); + for (ui32 partIdx = 0; partIdx <= parityParts; ++partIdx) { + outDiffSet.PartDiffs[partIdx].Diffs = diffs; + } +} + +void EoBlockSplitDiff(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) { + TBlockParams p(crcMode, type, dataSize); + outDiffSet.PartDiffs.resize(type.TotalPartCount()); + ui32 dataParts = type.DataParts(); + + ui32 partOffset = 0; + ui32 diffIdx = 0; + for (ui32 partIdx = 0; partIdx < dataParts && diffIdx < diffs.size(); ++partIdx) { + ui32 nextOffset = partOffset; + if (partIdx < p.FirstSmallPartIdx) { + nextOffset += p.LargePartSize; + } else { + nextOffset += p.SmallPartSize; + } + if (partIdx + 1 == dataParts) { + nextOffset = dataSize; + } + if (partOffset == nextOffset) { + continue; + } + TPartDiff &part = outDiffSet.PartDiffs[partIdx]; + + while (diffIdx < diffs.size()) { + const TDiff &diff = diffs[diffIdx]; + ui32 lineOffset = diff.Offset % sizeof(ui64); + ui32 diffEnd = diff.Offset + diff.GetDiffLength(); + + if (diff.Offset <= partOffset && diffEnd >= partOffset) { + ui32 diffEndForThisPart = Min(diffEnd, nextOffset); + ui32 diffShift = partOffset - diff.Offset; + + ui32 bufferSize = diffEndForThisPart - partOffset; + Y_VERIFY(bufferSize); + Y_VERIFY_S(diffShift + bufferSize <= diff.Buffer.size(), "diffShift# " << diffShift + << " bufferSize# " << bufferSize << " diff.GetDiffLength()# " << diff.GetDiffLength()); + TString newBuffer = TString::Uninitialized(bufferSize); + memcpy(newBuffer.begin(), diff.Buffer.begin() + diffShift, bufferSize); + part.Diffs.emplace_back(newBuffer, 0, false, true); + + if (diffEnd <= nextOffset) { + diffIdx++; + } else { + break; + } + } else if (diffEnd <= nextOffset) { + TString buffer; + ui32 bufferSize = 0; + if (lineOffset && !diff.IsAligned) { + bufferSize = diff.GetDiffLength() + lineOffset; + buffer = TString::Uninitialized(bufferSize); + memcpy(buffer.begin() + lineOffset, diff.Buffer.begin(), diff.GetDiffLength()); + } else { + buffer = diff.Buffer; + bufferSize = diff.Buffer.size(); + } + Y_VERIFY(bufferSize); + part.Diffs.emplace_back(buffer, diff.Offset - partOffset, false, true); + diffIdx++; + } else if (diff.Offset < nextOffset) { + ui32 bufferSize = nextOffset - diff.Offset + lineOffset; + TString newBuffer = TString::Uninitialized(bufferSize); + memcpy(newBuffer.begin() + lineOffset, diff.Buffer.begin(), bufferSize - lineOffset); + Y_VERIFY(bufferSize); + part.Diffs.emplace_back(newBuffer, diff.Offset - partOffset, false, true); + break; + } else { + break; + } + } + + partOffset = nextOffset; + } +} + +void TErasureType::SplitDiffs(ECrcMode crcMode, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) const { + Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); + const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; + + // change crc part only in during of applying diffs + switch (erasure.ErasureFamily) { + case TErasureType::ErasureMirror: + MirrorSplitDiff(*this, diffs, outDiffSet); + break; + case TErasureType::ErasureParityStripe: + Y_FAIL("Not implemented"); + break; + case TErasureType::ErasureParityBlock: + Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); + EoBlockSplitDiff(crcMode, *this, dataSize, diffs, outDiffSet); + break; + } +} + +template <typename Bucket> +void XorCpy(Bucket *dest, const Bucket *orig, const Bucket *diff, ui32 count) { + for (ui32 idx = 0; idx < count; ++idx) { + dest[idx] = orig[idx] ^ diff[idx]; + } +} + +void MakeEoBlockXorDiff(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, + const ui8 *src, const TVector<TDiff> &inDiffs, TVector<TDiff> *outDiffs) +{ + Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); + TBlockParams p(crcMode, type, dataSize); + + for (const TDiff &diff : inDiffs) { + const ui8 *diffBufferBytes = reinterpret_cast<const ui8*>(diff.Buffer.data()); + + ui32 lineOffset = diff.Offset % sizeof(ui64); + ui32 lowerStartPos = diff.Offset - lineOffset; + ui32 upperStartPos = lowerStartPos + (lineOffset ? sizeof(ui64) : 0); + + ui32 end = diff.Offset + diff.GetDiffLength(); + ui32 endLineOffset = end % sizeof(ui64); + ui32 lowerEndPos = end - endLineOffset; + ui32 upperEndPos = lowerEndPos + (endLineOffset ? sizeof(ui64) : 0); + + ui32 bufferSize = upperEndPos - lowerStartPos; + Y_VERIFY(bufferSize); + TString xorDiffBuffer = TString::Uninitialized(bufferSize); + ui8 *xorDiffBufferBytes = reinterpret_cast<ui8*>(const_cast<char*>(xorDiffBuffer.data())); + + if (lowerEndPos == lowerStartPos) { + ui64 &val = *reinterpret_cast<ui64*>(xorDiffBufferBytes); + val = 0; + ui32 byteCount = diff.GetDiffLength(); + XorCpy(xorDiffBufferBytes + lineOffset, src + diff.Offset, diffBufferBytes + lineOffset, byteCount); + outDiffs->emplace_back(xorDiffBuffer, diff.Offset, true, true); + continue; + } + + if (lineOffset) { + ui64 &val = *reinterpret_cast<ui64*>(xorDiffBufferBytes); + val = 0; + ui32 byteCount = Min<ui32>(sizeof(ui64) - lineOffset, diff.GetDiffLength()); + XorCpy(xorDiffBufferBytes + lineOffset, src + diff.Offset, diffBufferBytes + lineOffset, byteCount); + } + + ui32 firstLine = lowerStartPos / sizeof(ui64); + ui32 lineStart = upperStartPos / sizeof(ui64); + ui32 lineEnd = lowerEndPos / sizeof(ui64); + ui64 *xorUI64 = reinterpret_cast<ui64*>(xorDiffBufferBytes) + (lineStart - firstLine); + const ui64 *srcUI64 = reinterpret_cast<const ui64*>(src) + lineStart; + const ui64 *diffUI64 = reinterpret_cast<const ui64*>(diffBufferBytes)+ (lineStart - firstLine); + ui32 countUI64 = lineEnd - lineStart; + XorCpy(xorUI64, srcUI64, diffUI64, countUI64); + + if (endLineOffset) { + ui64 &val = reinterpret_cast<ui64*>(xorDiffBufferBytes)[lineEnd - firstLine]; + val = 0; + ui32 diffOffset = lowerEndPos - lowerStartPos; + XorCpy(xorDiffBufferBytes + diffOffset, src + lowerEndPos, diffBufferBytes + diffOffset, endLineOffset); + } + outDiffs->emplace_back(xorDiffBuffer, diff.Offset, true, true); + } +} + +void TErasureType::MakeXorDiff(ECrcMode crcMode, ui32 dataSize, const ui8 *src, + const TVector<TDiff> &inDiffs, TVector<TDiff> *outDiffs) const +{ + Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); + const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; + switch (erasure.ErasureFamily) { + case TErasureType::ErasureMirror: + Y_FAIL("unreachable"); + break; + case TErasureType::ErasureParityStripe: + Y_FAIL("Not implemented"); + break; + case TErasureType::ErasureParityBlock: + Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); + MakeEoBlockXorDiff(crcMode, *this, dataSize, src, inDiffs, outDiffs); + break; + } +} + +void TErasureType::ApplyDiff(ECrcMode crcMode, ui8 *dst, const TVector<TDiff> &diffs) const { + Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); + for (auto &diff : diffs) { + memcpy(dst + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); + } +} + +template <bool forSecondParity> +void ApllyXorDiffForEoBlock(const TBlockParams &p, ui64 *buffer, const ui64 *diff, ui64 adj, + ui32 begin, ui32 end, ui32 diffOffset, ui8 fromPart = 0) + { + ui32 blockBegin = begin / p.LineCount; + ui32 blockEnd = (end + p.LineCount - 1) / p.LineCount; + + Y_VERIFY(!forSecondParity || (blockBegin == 0 && blockEnd == 1)); + + if (forSecondParity && adj) { + for (ui32 idx = 0; idx < p.LineCount; ++idx) { + buffer[idx] ^= adj; + } + } + + Y_VERIFY(begin < end); + if constexpr (forSecondParity) { + ui32 lineBorder = p.LineCount - fromPart; + + if (begin < lineBorder) { + ui32 bufferBegin = begin + fromPart; + ui32 diffBegin = begin - diffOffset; + ui32 count = Min(lineBorder, end) - begin; + XorCpy(buffer + bufferBegin, buffer + bufferBegin, diff + diffBegin, count); + } + + if (end > lineBorder + 1) { + ui32 currentBegin = Max(lineBorder + 1, begin); + ui32 bufferBegin = currentBegin + fromPart - p.Prime; + ui32 count = end - currentBegin; + ui32 diffBegin = currentBegin - diffOffset; + XorCpy(buffer + bufferBegin, buffer + bufferBegin, diff + diffBegin, count); + } + } else { + ui32 count = end - begin; + ui32 diffBegin = begin - diffOffset; + XorCpy(buffer + begin, buffer + begin, diff + diffBegin, count); + } +} + +void ApplyEoBlockXorDiffForFirstParityPart(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, + ui64 *dst, const TVector<TDiff> &xorDiffs) +{ + Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); + TBlockParams p(crcMode, type, dataSize); + + for (const TDiff &diff : xorDiffs) { + Y_VERIFY(diff.Buffer.size() % sizeof(ui64) == 0); + Y_VERIFY(diff.IsXor && diff.IsAligned); + const ui64 *diffBuffer = reinterpret_cast<const ui64*>(diff.GetBufferBegin()); + ui32 diffOffset = diff.Offset / sizeof(ui64); + ui32 offset = diff.Offset / sizeof(ui64); + ui32 lineCount = diff.Buffer.size() / sizeof(ui64); + Y_VERIFY(diff.Offset < type.PartSize(crcMode, dataSize)); + Y_VERIFY(diff.Offset + diff.GetDiffLength() <= type.PartSize(crcMode, dataSize)); + ApllyXorDiffForEoBlock<false>(p, dst, diffBuffer, 0, offset, offset + lineCount, diffOffset); + } +} + +void ApplyXorForSecondParityPart(const TBlockParams &p, ui64 *startBufferBlock, const ui64 *startDiffBlock, + ui32 begin, ui32 end, ui8 fromPart, ui32 diffOffset) +{ + ui32 m = p.Prime; + const ui32 mint = (m - 2 < p.LineCount ? 1 : m - 2 - p.LineCount); + ui64 adj = 0; + ui32 adjRelatedBytesBegin = m - 1 - fromPart; + bool isAdjChanged = (fromPart >= mint) + && (adjRelatedBytesBegin >= begin) + && (adjRelatedBytesBegin < end); + + if (isAdjChanged) { + adj = startDiffBlock[adjRelatedBytesBegin - diffOffset]; + } + + ApllyXorDiffForEoBlock<true>(p, startBufferBlock, startDiffBlock, adj, begin, end, diffOffset, fromPart); +} + +void ApplyEoBlockXorDiffForSecondParityPart(TErasureType::ECrcMode crcMode, const TErasureType &type, ui32 dataSize, + ui8 fromPart, ui64 *dst, const TVector<TDiff> &xorDiffs) +{ + Y_VERIFY(crcMode == TErasureType::CrcModeNone, "crc's not implemented"); + TBlockParams p(crcMode, type, dataSize); + + ui64 bytesInBlock = p.LineCount * sizeof(ui64); + + for (const TDiff &diff : xorDiffs) { + Y_VERIFY(diff.Buffer.size() % sizeof(ui64) == 0); + Y_VERIFY(diff.IsXor && diff.IsAligned); + + const ui64 *diffBuffer = reinterpret_cast<const ui64*>(diff.GetBufferBegin()); + + ui32 firstBlock = diff.Offset / bytesInBlock * p.LineCount; + ui32 begin = diff.Offset / sizeof(ui64); + ui32 end = begin + diff.Buffer.size() / sizeof(ui64); + ui32 endBlock = (end + p.LineCount - 1) / p.LineCount * p.LineCount; + + for (ui32 idx = firstBlock; idx < endBlock; idx += p.LineCount) { + ui32 lineBegin = Max(idx, begin) - idx; + ui32 lineEnd = Min<ui32>(idx + p.LineCount, end) - idx; + if (idx == firstBlock) { + ui32 diffOffset = begin - idx; + ApplyXorForSecondParityPart(p, dst + idx, diffBuffer, lineBegin, lineEnd, fromPart, diffOffset); + } else { + ui32 diffOffset = idx - begin; + ApplyXorForSecondParityPart(p, dst + idx, diffBuffer + diffOffset, lineBegin, lineEnd, fromPart, 0); + } + } + } +} + +void TErasureType::ApplyXorDiff(ECrcMode crcMode, ui32 dataSize, ui8 *dst, + const TVector<TDiff> &diffs, ui8 fromPart, ui8 toPart) const +{ + Y_VERIFY(crcMode == CrcModeNone, "crc's not implemented"); + const TErasureParameters& erasure = ErasureSpeciesParameters[ErasureSpecies]; + switch (erasure.ErasureFamily) { + case TErasureType::ErasureMirror: + Y_FAIL("unreachable"); + break; + case TErasureType::ErasureParityStripe: + Y_FAIL("Not implemented"); + break; + case TErasureType::ErasureParityBlock: + Y_VERIFY(erasure.ParityParts == 2, "Other is not implemented"); + ui64 *lineDst = reinterpret_cast<ui64*>(dst); + if (toPart + 1 != TotalPartCount()) { + ApplyEoBlockXorDiffForFirstParityPart(crcMode, *this, dataSize, lineDst, diffs); + } else { + ApplyEoBlockXorDiffForSecondParityPart(crcMode, *this, dataSize, fromPart, lineDst, diffs); + } + break; + } +} + void TErasureType::RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TString& outBuffer, bool restoreParts, bool restoreFullData, bool restoreParityParts) const { partSet.FullDataFragment.ReferenceTo(outBuffer); diff --git a/ydb/core/erasure/erasure.h b/ydb/core/erasure/erasure.h index 35bd68c2d2f..7704a11b0e4 100644 --- a/ydb/core/erasure/erasure.h +++ b/ydb/core/erasure/erasure.h @@ -15,46 +15,46 @@ namespace NKikimr { -struct TDiff { - TString Buffer; - ui32 Offset = 0; - bool IsXor = false; - bool IsAligned = false; - - TDiff(const TString &buffer, ui32 offset, bool isXor, bool isAligned) - : Buffer(buffer) - , Offset(offset) - , IsXor(isXor) - , IsAligned(isAligned) - { - } - - TDiff(const TString &buffer, ui32 offset) - : TDiff(buffer, offset, false, false) - { - } - - ui32 GetDiffLength() const { - return (IsAligned ? Buffer.size() - Offset % sizeof(ui64) : Buffer.size()); - } - - const ui8* GetBufferBegin() const { - return reinterpret_cast<const ui8*>((Buffer.data())); - } - - const ui8* GetDataBegin() const { - return GetBufferBegin() + (IsAligned ? Offset % sizeof(ui64) : 0); - } -}; - -struct TPartDiff { - TVector<TDiff> Diffs; -}; - -struct TPartDiffSet { - TVector<TPartDiff> PartDiffs; -}; - +struct TDiff { + TString Buffer; + ui32 Offset = 0; + bool IsXor = false; + bool IsAligned = false; + + TDiff(const TString &buffer, ui32 offset, bool isXor, bool isAligned) + : Buffer(buffer) + , Offset(offset) + , IsXor(isXor) + , IsAligned(isAligned) + { + } + + TDiff(const TString &buffer, ui32 offset) + : TDiff(buffer, offset, false, false) + { + } + + ui32 GetDiffLength() const { + return (IsAligned ? Buffer.size() - Offset % sizeof(ui64) : Buffer.size()); + } + + const ui8* GetBufferBegin() const { + return reinterpret_cast<const ui8*>((Buffer.data())); + } + + const ui8* GetDataBegin() const { + return GetBufferBegin() + (IsAligned ? Offset % sizeof(ui64) : 0); + } +}; + +struct TPartDiff { + TVector<TDiff> Diffs; +}; + +struct TPartDiffSet { + TVector<TPartDiff> PartDiffs; +}; + // Part fragment, contains only some data struct TPartFragment { TString OwnedString; // Used for ownership only @@ -320,13 +320,13 @@ struct TErasureType { void SplitData(ECrcMode crcMode, const TString& buffer, TDataPartSet& outPartSet) const; void IncrementalSplitData(ECrcMode crcMode, const TString& buffer, TDataPartSet& outPartSet) const; - void SplitDiffs(ECrcMode crcMode, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) const; - void ApplyDiff(ECrcMode crcMode, ui8 *dst, const TVector<TDiff> &diffs) const; - void MakeXorDiff(ECrcMode crcMode, ui32 dataSize, const ui8 *src, const TVector<TDiff> &inDiffs, - TVector<TDiff> *outDiffs) const; - void ApplyXorDiff(ECrcMode crcMode, ui32 dataSize, ui8 *dst, - const TVector<TDiff> &diffs, ui8 fromPart, ui8 toPart) const; - + void SplitDiffs(ECrcMode crcMode, ui32 dataSize, const TVector<TDiff> &diffs, TPartDiffSet& outDiffSet) const; + void ApplyDiff(ECrcMode crcMode, ui8 *dst, const TVector<TDiff> &diffs) const; + void MakeXorDiff(ECrcMode crcMode, ui32 dataSize, const ui8 *src, const TVector<TDiff> &inDiffs, + TVector<TDiff> *outDiffs) const; + void ApplyXorDiff(ECrcMode crcMode, ui32 dataSize, ui8 *dst, + const TVector<TDiff> &diffs, ui8 fromPart, ui8 toPart) const; + void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, TString& outBuffer, bool restoreParts, bool restoreFullData, bool restoreParityParts) const; void RestoreData(ECrcMode crcMode, TDataPartSet& partSet, bool restoreParts, bool restoreFullData, diff --git a/ydb/core/erasure/erasure_rope_ut.cpp b/ydb/core/erasure/erasure_rope_ut.cpp index 62a6242ea92..8f924f1a40f 100644 --- a/ydb/core/erasure/erasure_rope_ut.cpp +++ b/ydb/core/erasure/erasure_rope_ut.cpp @@ -1,13 +1,13 @@ #include "erasure_rope.h" -#include "ut_util.h" +#include "ut_util.h" namespace NKikimr { namespace NErasureRope { -TRope GenerateRandomRope(size_t dataSize) { +TRope GenerateRandomRope(size_t dataSize) { NPrivate::TMersenne64 randGen(Seed()); - return TRopeHelpers::RopeFromStringMemcpy(GenerateRandomString(randGen, dataSize)); + return TRopeHelpers::RopeFromStringMemcpy(GenerateRandomString(randGen, dataSize)); } void TestMissingPartWithRandomData(TRopeErasureType &groupType, ui32 *missingPartIdx, ui32 missingParts, diff --git a/ydb/core/erasure/erasure_ut.cpp b/ydb/core/erasure/erasure_ut.cpp index 2473bd16506..bff83d9e602 100644 --- a/ydb/core/erasure/erasure_ut.cpp +++ b/ydb/core/erasure/erasure_ut.cpp @@ -1,5 +1,5 @@ #include "erasure.h" -#include "ut_util.h" +#include "ut_util.h" namespace NKikimr { @@ -122,112 +122,112 @@ void TestAllLossesDifferentSizes(TErasureType &groupType, ui32 maxParts) { } // missingVariant } -void PrintBuffer(const TString &buffer) { - Cerr << " ["; - for (ui32 idx = 0; idx < buffer.size(); ++idx) { - if (idx) { - Cerr << " "; - } - Cerr << (ui32)(ui8)buffer[idx]; - } - Cerr << "]\n"; -} - -void PrintDiff(const TDiff &diff) { - Cerr << "Offset# " << diff.Offset - << " IsXor# " << (diff.IsXor ? "yes" : "no") - << " IsAligned# " << (diff.IsAligned ? "yes" : "no"); - PrintBuffer(diff.Buffer); -} - -void RunTestDiff(TErasureType &groupType, ui32 dataSize, const TString &testString, const TVector<TDiff> &diffs) { - TDataPartSet partSet; - groupType.SplitData(TErasureType::CrcModeNone, testString, partSet); - ui64 partSize = groupType.PartSize(TErasureType::CrcModeNone, dataSize); - for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) { - UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size()); - } - - TString result = TString::Uninitialized(dataSize); - memcpy(result.begin(), testString.begin(), dataSize); - for (const auto &diff : diffs) { - memcpy(result.begin() + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); - } - - TDataPartSet resultPartSet; - groupType.SplitData(TErasureType::CrcModeNone, result, resultPartSet); - - TPartDiffSet diffSet; - groupType.SplitDiffs(TErasureType::CrcModeNone, dataSize, diffs, diffSet); - - ui32 dataParts = groupType.DataParts(); - ui32 parityParts = groupType.ErasureFamily() == TErasureType::ErasureMirror ? 0 : groupType.ParityParts(); - - for (ui32 partIdx = 0; partIdx < dataParts; ++partIdx) { - if (!partSet.Parts[partIdx].Size) { - continue; - } - const ui8 *src = reinterpret_cast<ui8*>(partSet.Parts[partIdx].GetDataAt(0)); - const TVector<TDiff> &diffs = diffSet.PartDiffs[partIdx].Diffs; - if (diffs.empty()) { - continue; - } - - for (ui32 parityPartIdx = dataParts; parityPartIdx < dataParts + parityParts; ++parityPartIdx) { - TVector<TDiff> xorDiffs; - if (!partSet.Parts[parityPartIdx].Size) { - continue; - } - ui8 *dst = reinterpret_cast<ui8*>(partSet.Parts[parityPartIdx].GetDataAt(0)); - groupType.MakeXorDiff(TErasureType::CrcModeNone, dataSize, src, diffs, &xorDiffs); - - groupType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, dst, - xorDiffs, partIdx, parityPartIdx); - - } - - ui8 *dst = reinterpret_cast<ui8*>(partSet.Parts[partIdx].GetDataAt(0));; - groupType.ApplyDiff(TErasureType::CrcModeNone, dst, diffs); - } - - for (ui32 partIdx = 0; partIdx < dataParts + parityParts; ++partIdx) { - UNIT_ASSERT_STRINGS_EQUAL(partSet.Parts[partIdx].OwnedString, resultPartSet.Parts[partIdx].OwnedString); - } -} - -TVector<TDiff> GenerateRandomDiff(NPrivate::TMersenne64 &randGen, ui32 dataSize, ui32 diffCount, ui32 diffSize, - ui32 offset = Max<ui32>()) { - UNIT_ASSERT(dataSize >= diffCount * diffSize + diffCount - 1); - TVector<TDiff> diffs; - int leftPosition = 0; - for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { - ui32 diffPosition = 0; - if (offset != Max<ui32>()) { - diffPosition = leftPosition + offset; - } else { - ui32 nextDiffCount = diffCount - 1 - diffIdx; - UNIT_ASSERT(nextDiffCount < diffCount); - UNIT_ASSERT(dataSize >= leftPosition + nextDiffCount * (diffSize + 1) + diffSize); - ui32 maxDiffOffset = dataSize - leftPosition - nextDiffCount * (diffSize + 1) - diffSize; - UNIT_ASSERT(maxDiffOffset < dataSize); - ui32 diffOffset = (maxDiffOffset ? randGen.GenRand() % maxDiffOffset : 0); - UNIT_ASSERT(diffOffset < dataSize); - UNIT_ASSERT(diffOffset + diffSize <= dataSize); - diffPosition = leftPosition + diffOffset; - } - UNIT_ASSERT(diffPosition < dataSize); - UNIT_ASSERT(diffPosition + diffSize <= dataSize); - - TString buffer = TString::Uninitialized(diffSize); - for (ui32 i = 0; i < diffSize; ++i) { - buffer[i] = (char)randGen.GenRand(); - } - diffs.emplace_back(buffer, diffPosition, false, false); - leftPosition = diffPosition + 1 + diffSize; - } - return diffs; -} - +void PrintBuffer(const TString &buffer) { + Cerr << " ["; + for (ui32 idx = 0; idx < buffer.size(); ++idx) { + if (idx) { + Cerr << " "; + } + Cerr << (ui32)(ui8)buffer[idx]; + } + Cerr << "]\n"; +} + +void PrintDiff(const TDiff &diff) { + Cerr << "Offset# " << diff.Offset + << " IsXor# " << (diff.IsXor ? "yes" : "no") + << " IsAligned# " << (diff.IsAligned ? "yes" : "no"); + PrintBuffer(diff.Buffer); +} + +void RunTestDiff(TErasureType &groupType, ui32 dataSize, const TString &testString, const TVector<TDiff> &diffs) { + TDataPartSet partSet; + groupType.SplitData(TErasureType::CrcModeNone, testString, partSet); + ui64 partSize = groupType.PartSize(TErasureType::CrcModeNone, dataSize); + for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) { + UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size()); + } + + TString result = TString::Uninitialized(dataSize); + memcpy(result.begin(), testString.begin(), dataSize); + for (const auto &diff : diffs) { + memcpy(result.begin() + diff.Offset, diff.GetDataBegin(), diff.GetDiffLength()); + } + + TDataPartSet resultPartSet; + groupType.SplitData(TErasureType::CrcModeNone, result, resultPartSet); + + TPartDiffSet diffSet; + groupType.SplitDiffs(TErasureType::CrcModeNone, dataSize, diffs, diffSet); + + ui32 dataParts = groupType.DataParts(); + ui32 parityParts = groupType.ErasureFamily() == TErasureType::ErasureMirror ? 0 : groupType.ParityParts(); + + for (ui32 partIdx = 0; partIdx < dataParts; ++partIdx) { + if (!partSet.Parts[partIdx].Size) { + continue; + } + const ui8 *src = reinterpret_cast<ui8*>(partSet.Parts[partIdx].GetDataAt(0)); + const TVector<TDiff> &diffs = diffSet.PartDiffs[partIdx].Diffs; + if (diffs.empty()) { + continue; + } + + for (ui32 parityPartIdx = dataParts; parityPartIdx < dataParts + parityParts; ++parityPartIdx) { + TVector<TDiff> xorDiffs; + if (!partSet.Parts[parityPartIdx].Size) { + continue; + } + ui8 *dst = reinterpret_cast<ui8*>(partSet.Parts[parityPartIdx].GetDataAt(0)); + groupType.MakeXorDiff(TErasureType::CrcModeNone, dataSize, src, diffs, &xorDiffs); + + groupType.ApplyXorDiff(TErasureType::CrcModeNone, dataSize, dst, + xorDiffs, partIdx, parityPartIdx); + + } + + ui8 *dst = reinterpret_cast<ui8*>(partSet.Parts[partIdx].GetDataAt(0));; + groupType.ApplyDiff(TErasureType::CrcModeNone, dst, diffs); + } + + for (ui32 partIdx = 0; partIdx < dataParts + parityParts; ++partIdx) { + UNIT_ASSERT_STRINGS_EQUAL(partSet.Parts[partIdx].OwnedString, resultPartSet.Parts[partIdx].OwnedString); + } +} + +TVector<TDiff> GenerateRandomDiff(NPrivate::TMersenne64 &randGen, ui32 dataSize, ui32 diffCount, ui32 diffSize, + ui32 offset = Max<ui32>()) { + UNIT_ASSERT(dataSize >= diffCount * diffSize + diffCount - 1); + TVector<TDiff> diffs; + int leftPosition = 0; + for (ui32 diffIdx = 0; diffIdx < diffCount; ++diffIdx) { + ui32 diffPosition = 0; + if (offset != Max<ui32>()) { + diffPosition = leftPosition + offset; + } else { + ui32 nextDiffCount = diffCount - 1 - diffIdx; + UNIT_ASSERT(nextDiffCount < diffCount); + UNIT_ASSERT(dataSize >= leftPosition + nextDiffCount * (diffSize + 1) + diffSize); + ui32 maxDiffOffset = dataSize - leftPosition - nextDiffCount * (diffSize + 1) - diffSize; + UNIT_ASSERT(maxDiffOffset < dataSize); + ui32 diffOffset = (maxDiffOffset ? randGen.GenRand() % maxDiffOffset : 0); + UNIT_ASSERT(diffOffset < dataSize); + UNIT_ASSERT(diffOffset + diffSize <= dataSize); + diffPosition = leftPosition + diffOffset; + } + UNIT_ASSERT(diffPosition < dataSize); + UNIT_ASSERT(diffPosition + diffSize <= dataSize); + + TString buffer = TString::Uninitialized(diffSize); + for (ui32 i = 0; i < diffSize; ++i) { + buffer[i] = (char)randGen.GenRand(); + } + diffs.emplace_back(buffer, diffPosition, false, false); + leftPosition = diffPosition + 1 + diffSize; + } + return diffs; +} + Y_UNIT_TEST_SUITE(TErasureTypeTest) { // Test if new version is capable to restore data splited by current version (which is right by definition) Y_UNIT_TEST(isSplittedDataEqualsToOldVerion) { @@ -438,62 +438,62 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { } } - void BaseCheckDiffSpliting(TErasureType type, ui32 dataSize, ui32 diffCount, - ui32 diffSize, ui32 diffOffset) - { - NPrivate::TMersenne64 randGen(0); - TString testString = GenerateRandomString(randGen, dataSize); - TVector<TDiff> diffs = GenerateRandomDiff(randGen, dataSize, diffCount, diffSize, diffOffset); - RunTestDiff(type, dataSize, testString, diffs); - } - - void CheckDifferentCasesInDiffSpliting(TErasureType type) { - struct TTestCase { - ui32 DataSize; - ui32 DiffCount; - ui32 DiffSize; - ui32 DiffOffset; - }; - TVector<TTestCase> testCases = { - TTestCase{31, 16, 1, 0}, - TTestCase{120, 16, 3, 3}, - TTestCase{511, 32, 10, 3}, - TTestCase{50000, 100, 401, 89}, - TTestCase{31, 1, 31, 0}, - TTestCase{120, 1, 120, 0}, - TTestCase{511, 1, 511, 0}, - TTestCase{50000, 1, 50000, 0}, - TTestCase{250, 1, 10, 240} - }; - for (auto [dataSize, diffCount, diffSize, diffOffset] : testCases) { - BaseCheckDiffSpliting(type, dataSize, diffCount, diffSize, diffOffset); - } - } - - Y_UNIT_TEST(TestDifferentCasesInDiffSplitingMirror3Of4) { - CheckDifferentCasesInDiffSpliting(TErasureType::EErasureSpecies::ErasureMirror3of4); - } - - Y_UNIT_TEST(TestDifferentCasesInDiffSplitingBlock4Plus2) { - CheckDifferentCasesInDiffSpliting(TErasureType::EErasureSpecies::Erasure4Plus2Block); - } - - - Y_UNIT_TEST(TestSplitDiffBlock4Plus2SpecialCase1) { - TErasureType groupType(TErasureType::EErasureSpecies::Erasure4Plus2Block); - ui32 dataSize = 100; - TStringBuilder dataBuilder; - for (ui32 idx = 0; idx < 100; ++idx) { - dataBuilder << 'a'; - } - TString testString = dataBuilder; - TVector<TDiff> diffs; - diffs.reserve(2); - diffs.emplace_back("b", 0); - diffs.emplace_back("b", 99); - RunTestDiff(groupType, dataSize, testString, diffs); - } - + void BaseCheckDiffSpliting(TErasureType type, ui32 dataSize, ui32 diffCount, + ui32 diffSize, ui32 diffOffset) + { + NPrivate::TMersenne64 randGen(0); + TString testString = GenerateRandomString(randGen, dataSize); + TVector<TDiff> diffs = GenerateRandomDiff(randGen, dataSize, diffCount, diffSize, diffOffset); + RunTestDiff(type, dataSize, testString, diffs); + } + + void CheckDifferentCasesInDiffSpliting(TErasureType type) { + struct TTestCase { + ui32 DataSize; + ui32 DiffCount; + ui32 DiffSize; + ui32 DiffOffset; + }; + TVector<TTestCase> testCases = { + TTestCase{31, 16, 1, 0}, + TTestCase{120, 16, 3, 3}, + TTestCase{511, 32, 10, 3}, + TTestCase{50000, 100, 401, 89}, + TTestCase{31, 1, 31, 0}, + TTestCase{120, 1, 120, 0}, + TTestCase{511, 1, 511, 0}, + TTestCase{50000, 1, 50000, 0}, + TTestCase{250, 1, 10, 240} + }; + for (auto [dataSize, diffCount, diffSize, diffOffset] : testCases) { + BaseCheckDiffSpliting(type, dataSize, diffCount, diffSize, diffOffset); + } + } + + Y_UNIT_TEST(TestDifferentCasesInDiffSplitingMirror3Of4) { + CheckDifferentCasesInDiffSpliting(TErasureType::EErasureSpecies::ErasureMirror3of4); + } + + Y_UNIT_TEST(TestDifferentCasesInDiffSplitingBlock4Plus2) { + CheckDifferentCasesInDiffSpliting(TErasureType::EErasureSpecies::Erasure4Plus2Block); + } + + + Y_UNIT_TEST(TestSplitDiffBlock4Plus2SpecialCase1) { + TErasureType groupType(TErasureType::EErasureSpecies::Erasure4Plus2Block); + ui32 dataSize = 100; + TStringBuilder dataBuilder; + for (ui32 idx = 0; idx < 100; ++idx) { + dataBuilder << 'a'; + } + TString testString = dataBuilder; + TVector<TDiff> diffs; + diffs.reserve(2); + diffs.emplace_back("b", 0); + diffs.emplace_back("b", 99); + RunTestDiff(groupType, dataSize, testString, diffs); + } + // Mirror tests Y_UNIT_TEST(TestMirror3LossOfAllPossible3) { // Set up the erasure diff --git a/ydb/core/erasure/ut_util.h b/ydb/core/erasure/ut_util.h index ef2f7236553..b5ba9a6bbb7 100644 --- a/ydb/core/erasure/ut_util.h +++ b/ydb/core/erasure/ut_util.h @@ -1,70 +1,70 @@ -#pragma once - -#include <library/cpp/testing/unittest/registar.h> -#include <util/random/entropy.h> -#include <util/random/mersenne64.h> +#pragma once + +#include <library/cpp/testing/unittest/registar.h> +#include <util/random/entropy.h> +#include <util/random/mersenne64.h> #include <util/stream/null.h> -#include <util/string/printf.h> - +#include <util/string/printf.h> + IOutputStream& Ctest = Cnull; - + #define VERBOSE_COUT(a) Ctest << a - + inline TString PrintArr(ui32 *arr, ui32 n) { - TStringStream out; - if (n == 0) { - out << "-"; - } - for (ui32 i = 0; i < n; ++i) { - out << arr[i] << " "; - } - out << Endl; - return out.Str(); -} - + TStringStream out; + if (n == 0) { + out << "-"; + } + for (ui32 i = 0; i < n; ++i) { + out << arr[i] << " "; + } + out << Endl; + return out.Str(); +} + inline const char *BoolToStr(bool val) { - return val ? "true " : "false"; -} - -inline ui32 Fact(ui32 n) { - ui32 res = 1; - while (n > 1) { - res *= n; - n--; - } - return res; -} - -inline void GenFirstCombination(ui32 *variants, ui32 const k) { - for (ui32 i = 0; i < k; ++i) { - variants[i] = i; - } -} - -inline void GenNextCombination(ui32 *variants, ui32 const k, ui32 const n) { - for (ui32 i = k-1; i != (ui32)-1; --i) { - if ( variants[i] < n - 1 - (k - 1 - i)) { - ui32 tmp = ++variants[i]; - for (ui32 j = i+1; j < k; ++j) { - variants[j] = ++tmp; - } - break; - } - } -} - -inline TString GenerateRandomString(NPrivate::TMersenne64 &randGen, size_t dataSize) { - TString testString; - testString.resize(dataSize); - char *writePosChar = (char *)testString.data(); - ui32 charParts = testString.size() % sizeof(ui64); - for (ui32 i = 0; i < charParts; ++i) { - writePosChar[i] = (char)randGen.GenRand(); - } - ui64 *writePos64 = (ui64 *)writePosChar; - ui32 ui64Parts = testString.size() / sizeof(ui64); - for (ui32 i = 0; i < ui64Parts; ++i) { - writePos64[i] = randGen.GenRand(); - } - return testString; -} + return val ? "true " : "false"; +} + +inline ui32 Fact(ui32 n) { + ui32 res = 1; + while (n > 1) { + res *= n; + n--; + } + return res; +} + +inline void GenFirstCombination(ui32 *variants, ui32 const k) { + for (ui32 i = 0; i < k; ++i) { + variants[i] = i; + } +} + +inline void GenNextCombination(ui32 *variants, ui32 const k, ui32 const n) { + for (ui32 i = k-1; i != (ui32)-1; --i) { + if ( variants[i] < n - 1 - (k - 1 - i)) { + ui32 tmp = ++variants[i]; + for (ui32 j = i+1; j < k; ++j) { + variants[j] = ++tmp; + } + break; + } + } +} + +inline TString GenerateRandomString(NPrivate::TMersenne64 &randGen, size_t dataSize) { + TString testString; + testString.resize(dataSize); + char *writePosChar = (char *)testString.data(); + ui32 charParts = testString.size() % sizeof(ui64); + for (ui32 i = 0; i < charParts; ++i) { + writePosChar[i] = (char)randGen.GenRand(); + } + ui64 *writePos64 = (ui64 *)writePosChar; + ui32 ui64Parts = testString.size() / sizeof(ui64); + for (ui32 i = 0; i < ui64Parts; ++i) { + writePos64[i] = randGen.GenRand(); + } + return testString; +} diff --git a/ydb/core/keyvalue/keyvalue_const.h b/ydb/core/keyvalue/keyvalue_const.h index 09b2327ef5e..854668b687e 100644 --- a/ydb/core/keyvalue/keyvalue_const.h +++ b/ydb/core/keyvalue/keyvalue_const.h @@ -7,8 +7,8 @@ namespace NKeyValue { constexpr ui32 BLOB_CHANNEL = 2; constexpr ui64 KEYVALUE_VERSION = 1ull; -constexpr ui64 InlineStorageChannelInPublicApi = 1; -constexpr ui64 MainStorageChannelInPublicApi = 2; - +constexpr ui64 InlineStorageChannelInPublicApi = 1; +constexpr ui64 MainStorageChannelInPublicApi = 2; + } // NKeyValue } // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_events.h b/ydb/core/keyvalue/keyvalue_events.h index 03bc81ad8fc..564aef6563c 100644 --- a/ydb/core/keyvalue/keyvalue_events.h +++ b/ydb/core/keyvalue/keyvalue_events.h @@ -5,7 +5,7 @@ #include <ydb/public/lib/base/msgbus.h> #include <ydb/core/keyvalue/protos/events.pb.h> - + namespace NKikimr { namespace NKeyValue { @@ -24,20 +24,20 @@ struct TEvKeyValue { EvReportWriteLatency, EvUpdateWeights, - EvRead = EvRequest + 16, - EvReadRange, - EvExecuteTransaction, - EvGetStatus, - EvObtainLock, - + EvRead = EvRequest + 16, + EvReadRange, + EvExecuteTransaction, + EvGetStatus, + EvObtainLock, + EvResponse = EvRequest + 512, - EvReadResponse = EvResponse + 16, - EvReadRangeResponse, - EvExecuteTransactionResponse, - EvGetStatusResponse, - EvObtainLockResponse, - + EvReadResponse = EvResponse + 16, + EvReadRangeResponse, + EvExecuteTransactionResponse, + EvGetStatusResponse, + EvObtainLockResponse, + EvEnd }; @@ -45,87 +45,87 @@ struct TEvKeyValue { EvEnd < EventSpaceEnd(TKikimrEvents::ES_KEYVALUE), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_KEYVALUE)"); - struct TEvReadResponse; - - struct TEvRead : public TEventPB<TEvRead, - NKikimrKeyValue::ReadRequest, EvRead> { - - using TResponse = TEvReadResponse; - TEvRead() { } - }; - - struct TEvReadResponse : public TEventPB<TEvReadResponse, - NKikimrKeyValue::ReadResult, EvReadResponse> { - TEvReadResponse() { } - }; - - struct TEvReadRangeResponse; - - struct TEvReadRange : public TEventPB<TEvReadRange, - NKikimrKeyValue::ReadRangeRequest, EvReadRange> { - - using TResponse = TEvReadRangeResponse; - TEvReadRange() { } - }; - - struct TEvReadRangeResponse : public TEventPB<TEvReadRangeResponse, - NKikimrKeyValue::ReadRangeResult, EvReadRangeResponse> { - TEvReadRangeResponse() { } - }; - - struct TEvExecuteTransactionResponse; - - struct TEvExecuteTransaction : public TEventPB<TEvExecuteTransaction, - NKikimrKeyValue::ExecuteTransactionRequest, EvExecuteTransaction> { - - using TResponse = TEvExecuteTransactionResponse; - TEvExecuteTransaction() { } - }; - - struct TEvExecuteTransactionResponse : public TEventPB<TEvExecuteTransactionResponse, - NKikimrKeyValue::ExecuteTransactionResult, EvExecuteTransactionResponse> { - TEvExecuteTransactionResponse() { } - }; - - struct TEvGetStatusResponse; - - struct TEvGetStatus : public TEventPB<TEvGetStatus, - NKikimrKeyValue::GetStatusRequest, EvGetStatus> { - - using TResponse = TEvGetStatusResponse; - TEvGetStatus() { } - }; - - struct TEvGetStatusResponse : public TEventPB<TEvGetStatusResponse, - NKikimrKeyValue::GetStatusResult, EvGetStatusResponse> { - TEvGetStatusResponse() { } - }; - - struct TEvObtainLockResponse; - - struct TEvObtainLock : public TEventPB<TEvObtainLock, - NKikimrKeyValue::ObtainLockRequest, EvObtainLock> { - - using TResponse = TEvObtainLockResponse; - TEvObtainLock() { } - }; - - struct TEvObtainLockResponse : public TEventPB<TEvObtainLockResponse, - NKikimrKeyValue::ObtainLockResult, EvObtainLockResponse> { - TEvObtainLockResponse() { } - }; - - struct TEvRequest : public TEventPB<TEvRequest, + struct TEvReadResponse; + + struct TEvRead : public TEventPB<TEvRead, + NKikimrKeyValue::ReadRequest, EvRead> { + + using TResponse = TEvReadResponse; + TEvRead() { } + }; + + struct TEvReadResponse : public TEventPB<TEvReadResponse, + NKikimrKeyValue::ReadResult, EvReadResponse> { + TEvReadResponse() { } + }; + + struct TEvReadRangeResponse; + + struct TEvReadRange : public TEventPB<TEvReadRange, + NKikimrKeyValue::ReadRangeRequest, EvReadRange> { + + using TResponse = TEvReadRangeResponse; + TEvReadRange() { } + }; + + struct TEvReadRangeResponse : public TEventPB<TEvReadRangeResponse, + NKikimrKeyValue::ReadRangeResult, EvReadRangeResponse> { + TEvReadRangeResponse() { } + }; + + struct TEvExecuteTransactionResponse; + + struct TEvExecuteTransaction : public TEventPB<TEvExecuteTransaction, + NKikimrKeyValue::ExecuteTransactionRequest, EvExecuteTransaction> { + + using TResponse = TEvExecuteTransactionResponse; + TEvExecuteTransaction() { } + }; + + struct TEvExecuteTransactionResponse : public TEventPB<TEvExecuteTransactionResponse, + NKikimrKeyValue::ExecuteTransactionResult, EvExecuteTransactionResponse> { + TEvExecuteTransactionResponse() { } + }; + + struct TEvGetStatusResponse; + + struct TEvGetStatus : public TEventPB<TEvGetStatus, + NKikimrKeyValue::GetStatusRequest, EvGetStatus> { + + using TResponse = TEvGetStatusResponse; + TEvGetStatus() { } + }; + + struct TEvGetStatusResponse : public TEventPB<TEvGetStatusResponse, + NKikimrKeyValue::GetStatusResult, EvGetStatusResponse> { + TEvGetStatusResponse() { } + }; + + struct TEvObtainLockResponse; + + struct TEvObtainLock : public TEventPB<TEvObtainLock, + NKikimrKeyValue::ObtainLockRequest, EvObtainLock> { + + using TResponse = TEvObtainLockResponse; + TEvObtainLock() { } + }; + + struct TEvObtainLockResponse : public TEventPB<TEvObtainLockResponse, + NKikimrKeyValue::ObtainLockResult, EvObtainLockResponse> { + TEvObtainLockResponse() { } + }; + + struct TEvRequest : public TEventPB<TEvRequest, NKikimrClient::TKeyValueRequest, EvRequest> { TEvRequest() { } }; - struct TEvResponse : public TEventPB<TEvResponse, + struct TEvResponse : public TEventPB<TEvResponse, NKikimrClient::TResponse, EvResponse> { TEvResponse() { } }; - struct TEvIntermediate : public TEventLocal<TEvIntermediate, EvIntermediate> { + struct TEvIntermediate : public TEventLocal<TEvIntermediate, EvIntermediate> { THolder<NKeyValue::TIntermediate> Intermediate; TEvIntermediate() { } @@ -135,7 +135,7 @@ struct TEvKeyValue { {} }; - struct TEvNotify : public TEventLocal<TEvNotify, EvNotify> { + struct TEvNotify : public TEventLocal<TEvNotify, EvNotify> { ui64 RequestUid; ui64 Generation; ui64 Step; @@ -152,45 +152,45 @@ struct TEvKeyValue { , Stat(stat) , Status(status) {} - - TEvNotify(ui64 requestUid, ui64 generation, ui64 step, const NKeyValue::TRequestStat &stat, - NKikimrKeyValue::Statuses::ReplyStatus status) - : RequestUid(requestUid) - , Generation(generation) - , Step(step) - , Stat(stat) - , Status(ConvertStatus(status)) - {} - - static NMsgBusProxy::EResponseStatus ConvertStatus(NKikimrKeyValue::Statuses::ReplyStatus status) { - switch (status) { - case NKikimrKeyValue::Statuses::RSTATUS_OK: - return NMsgBusProxy::MSTATUS_OK; - case NKikimrKeyValue::Statuses::RSTATUS_ERROR: - return NMsgBusProxy::MSTATUS_ERROR; - case NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT: - return NMsgBusProxy::MSTATUS_TIMEOUT; - case NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR: - return NMsgBusProxy::MSTATUS_INTERNALERROR; - default: - return NMsgBusProxy::MSTATUS_INTERNALERROR; - } - } - }; - - struct TEvStoreCollect : public TEventLocal<TEvStoreCollect, EvStoreCollect> { + + TEvNotify(ui64 requestUid, ui64 generation, ui64 step, const NKeyValue::TRequestStat &stat, + NKikimrKeyValue::Statuses::ReplyStatus status) + : RequestUid(requestUid) + , Generation(generation) + , Step(step) + , Stat(stat) + , Status(ConvertStatus(status)) + {} + + static NMsgBusProxy::EResponseStatus ConvertStatus(NKikimrKeyValue::Statuses::ReplyStatus status) { + switch (status) { + case NKikimrKeyValue::Statuses::RSTATUS_OK: + return NMsgBusProxy::MSTATUS_OK; + case NKikimrKeyValue::Statuses::RSTATUS_ERROR: + return NMsgBusProxy::MSTATUS_ERROR; + case NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT: + return NMsgBusProxy::MSTATUS_TIMEOUT; + case NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR: + return NMsgBusProxy::MSTATUS_INTERNALERROR; + default: + return NMsgBusProxy::MSTATUS_INTERNALERROR; + } + } + }; + + struct TEvStoreCollect : public TEventLocal<TEvStoreCollect, EvStoreCollect> { TEvStoreCollect() { } }; - struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> { + struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> { TEvCollect() { } }; - struct TEvEraseCollect : public TEventLocal<TEvEraseCollect, EvEraseCollect> { + struct TEvEraseCollect : public TEventLocal<TEvEraseCollect, EvEraseCollect> { TEvEraseCollect() { } }; - struct TEvPeriodicRefresh : public TEventLocal<TEvPeriodicRefresh, EvPeriodicRefresh> { + struct TEvPeriodicRefresh : public TEventLocal<TEvPeriodicRefresh, EvPeriodicRefresh> { TEvPeriodicRefresh() { } }; }; diff --git a/ydb/core/keyvalue/keyvalue_flat_impl.h b/ydb/core/keyvalue/keyvalue_flat_impl.h index 292f2feff93..518771a5d27 100644 --- a/ydb/core/keyvalue/keyvalue_flat_impl.h +++ b/ydb/core/keyvalue/keyvalue_flat_impl.h @@ -279,29 +279,29 @@ protected: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // gRPC - - void Handle(TEvKeyValue::TEvRead::TPtr &ev) { - State.OnEvReadRequest(ev, TActivationContext::AsActorContext(), Info()); - } - - void Handle(TEvKeyValue::TEvReadRange::TPtr &ev) { - State.OnEvReadRangeRequest(ev, TActivationContext::AsActorContext(), Info()); - } - - void Handle(TEvKeyValue::TEvExecuteTransaction::TPtr &ev) { - State.OnEvExecuteTransaction(ev, TActivationContext::AsActorContext(), Info()); - } - - void Handle(TEvKeyValue::TEvGetStatus::TPtr &ev) { - State.OnEvGetStatus(ev, TActivationContext::AsActorContext(), Info()); - } - - void Handle(TEvKeyValue::TEvObtainLock::TPtr &ev) { - State.OnEvObtainLock(ev, TActivationContext::AsActorContext(), Info()); - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // gRPC + + void Handle(TEvKeyValue::TEvRead::TPtr &ev) { + State.OnEvReadRequest(ev, TActivationContext::AsActorContext(), Info()); + } + + void Handle(TEvKeyValue::TEvReadRange::TPtr &ev) { + State.OnEvReadRangeRequest(ev, TActivationContext::AsActorContext(), Info()); + } + + void Handle(TEvKeyValue::TEvExecuteTransaction::TPtr &ev) { + State.OnEvExecuteTransaction(ev, TActivationContext::AsActorContext(), Info()); + } + + void Handle(TEvKeyValue::TEvGetStatus::TPtr &ev) { + State.OnEvGetStatus(ev, TActivationContext::AsActorContext(), Info()); + } + + void Handle(TEvKeyValue::TEvObtainLock::TPtr &ev) { + State.OnEvObtainLock(ev, TActivationContext::AsActorContext(), Info()); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Online state void Handle(TEvKeyValue::TEvEraseCollect::TPtr &ev, const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() @@ -339,19 +339,19 @@ protected: Execute(new TTxStoreCollect(this), ctx); } - void CheckYellowChannels(TRequestStat& stat) { - IExecutor* executor = Executor(); + void CheckYellowChannels(TRequestStat& stat) { + IExecutor* executor = Executor(); if ((stat.YellowMoveChannels || stat.YellowStopChannels) && executor) { executor->OnYellowChannels(std::move(stat.YellowMoveChannels), std::move(stat.YellowStopChannels)); - } - } - + } + } + void Handle(TEvKeyValue::TEvIntermediate::TPtr &ev, const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() << " Handle TEvIntermediate " << ev->Get()->ToString()); - - CheckYellowChannels(ev->Get()->Intermediate->Stat); - + + CheckYellowChannels(ev->Get()->Intermediate->Stat); + State.OnEvIntermediate(*(ev->Get()->Intermediate), ctx); Execute(new TTxRequest(std::move(ev->Get()->Intermediate), this), ctx); } @@ -360,8 +360,8 @@ protected: TEvKeyValue::TEvNotify &event = *ev->Get(); LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() << " Handle TEvNotify " << event.ToString()); - - CheckYellowChannels(ev->Get()->Stat); + + CheckYellowChannels(ev->Get()->Stat); State.OnRequestComplete(event.RequestUid, event.Generation, event.Step, ctx, Info(), event.Status, event.Stat); } @@ -469,12 +469,12 @@ public: return; RestoreActorActivity(); switch (ev->GetTypeRewrite()) { - hFunc(TEvKeyValue::TEvRead, Handle); - hFunc(TEvKeyValue::TEvReadRange, Handle); - hFunc(TEvKeyValue::TEvExecuteTransaction, Handle); - hFunc(TEvKeyValue::TEvGetStatus, Handle); - hFunc(TEvKeyValue::TEvObtainLock, Handle); - + hFunc(TEvKeyValue::TEvRead, Handle); + hFunc(TEvKeyValue::TEvReadRange, Handle); + hFunc(TEvKeyValue::TEvExecuteTransaction, Handle); + hFunc(TEvKeyValue::TEvGetStatus, Handle); + hFunc(TEvKeyValue::TEvObtainLock, Handle); + HFunc(TEvKeyValue::TEvEraseCollect, Handle); HFunc(TEvKeyValue::TEvCollect, Handle); HFunc(TEvKeyValue::TEvStoreCollect, Handle); @@ -535,9 +535,9 @@ public: } - bool ReassignChannelsEnabled() const override { - return true; - } + bool ReassignChannelsEnabled() const override { + return true; + } }; diff --git a/ydb/core/keyvalue/keyvalue_intermediate.cpp b/ydb/core/keyvalue/keyvalue_intermediate.cpp index 82a78d9fa29..d7dd7f2c75d 100644 --- a/ydb/core/keyvalue/keyvalue_intermediate.cpp +++ b/ydb/core/keyvalue/keyvalue_intermediate.cpp @@ -76,15 +76,15 @@ TIntermediate::TIntermediate(TActorId respondTo, TActorId keyValueActorId, ui64 } void TIntermediate::UpdateStat() { - auto checkRead = [&] (const auto &read) { + auto checkRead = [&] (const auto &read) { if (read.Status == NKikimrProto::NODATA) { Stat.ReadNodata++; } else if (read.Status == NKikimrProto::OK) { Stat.Reads++; Stat.ReadBytes += read.Value.size(); } - }; - auto checkRangeRead = [&] (const auto &range) { + }; + auto checkRangeRead = [&] (const auto &range) { if (range.IncludeData) { for (const auto &read: range.Reads) { if (read.Status == NKikimrProto::NODATA) { @@ -95,50 +95,50 @@ void TIntermediate::UpdateStat() { } } } else { - Stat.IndexRangeRead++; + Stat.IndexRangeRead++; } - }; - - if (ReadCommand) { - auto checkReadCommand = [&] (auto &cmd) { - using Type = std::decay_t<decltype(cmd)>; - if constexpr (std::is_same_v<Type, TIntermediate::TRead>) { - checkRead(cmd); - } - if constexpr (std::is_same_v<Type, TIntermediate::TRangeRead>) { - checkRangeRead(cmd); - } - }; - std::visit(checkReadCommand, *ReadCommand); - } - - for (const auto &read: Reads) { - checkRead(read); - } - for (const auto &range: RangeReads) { - checkRangeRead(range); + }; + + if (ReadCommand) { + auto checkReadCommand = [&] (auto &cmd) { + using Type = std::decay_t<decltype(cmd)>; + if constexpr (std::is_same_v<Type, TIntermediate::TRead>) { + checkRead(cmd); + } + if constexpr (std::is_same_v<Type, TIntermediate::TRangeRead>) { + checkRangeRead(cmd); + } + }; + std::visit(checkReadCommand, *ReadCommand); } + + for (const auto &read: Reads) { + checkRead(read); + } + for (const auto &range: RangeReads) { + checkRangeRead(range); + } for (const auto &write: Writes) { if (write.Status == NKikimrProto::OK) { Stat.WriteBytes += write.Data.size(); } } - for (const auto &cmd : Commands) { - if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { - continue; - } - const auto &write = std::get<TIntermediate::TWrite>(cmd); - if (write.Status == NKikimrProto::OK) { - Stat.WriteBytes += write.Data.size(); - } - } - - Stat.Writes = WriteCount; + for (const auto &cmd : Commands) { + if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { + continue; + } + const auto &write = std::get<TIntermediate::TWrite>(cmd); + if (write.Status == NKikimrProto::OK) { + Stat.WriteBytes += write.Data.size(); + } + } + + Stat.Writes = WriteCount; Stat.GetStatuses = GetStatuses.size(); - Stat.Renames = RenameCount; - Stat.CopyRanges = CopyRangeCount; - Stat.Concats = ConcatCount; + Stat.Renames = RenameCount; + Stat.CopyRanges = CopyRangeCount; + Stat.Concats = ConcatCount; } } // NKeyValue diff --git a/ydb/core/keyvalue/keyvalue_intermediate.h b/ydb/core/keyvalue/keyvalue_intermediate.h index c28c14b9c37..bc446c8d61e 100644 --- a/ydb/core/keyvalue/keyvalue_intermediate.h +++ b/ydb/core/keyvalue/keyvalue_intermediate.h @@ -38,7 +38,7 @@ struct TIntermediate { ui32 Offset; ui32 Size; ui32 ValueSize; - ui32 RequestedSize = 0; + ui32 RequestedSize = 0; ui64 CreationUnixTime; NKikimrClient::TKeyValueRequest::EStorageChannel StorageChannel; NKikimrBlobStorage::EGetHandleClass HandleClass; @@ -100,9 +100,9 @@ struct TIntermediate { bool IsAllowed; }; - using TCmd = std::variant<TWrite, TDelete, TRename, TCopyRange, TConcat>; - using TReadCmd = std::variant<TRead, TRangeRead>; - + using TCmd = std::variant<TWrite, TDelete, TRename, TCopyRange, TConcat>; + using TReadCmd = std::variant<TRead, TRangeRead>; + TDeque<TRead> Reads; TDeque<TRangeRead> RangeReads; TDeque<TWrite> Writes; @@ -114,16 +114,16 @@ struct TIntermediate { TMaybe<TTrimLeakedBlobs> TrimLeakedBlobs; TMaybe<TSetExecutorFastLogPolicy> SetExecutorFastLogPolicy; - TStackVec<TCmd, 1> Commands; - TStackVec<ui32, 1> WriteIndices; - std::optional<TReadCmd> ReadCommand; - - ui64 WriteCount = 0; - ui64 DeleteCount = 0; - ui64 RenameCount = 0; - ui64 CopyRangeCount = 0; - ui64 ConcatCount = 0; - + TStackVec<TCmd, 1> Commands; + TStackVec<ui32, 1> WriteIndices; + std::optional<TReadCmd> ReadCommand; + + ui64 WriteCount = 0; + ui64 DeleteCount = 0; + ui64 RenameCount = 0; + ui64 CopyRangeCount = 0; + ui64 ConcatCount = 0; + ui64 Cookie; ui64 Generation; ui64 RequestUid; @@ -150,13 +150,13 @@ struct TIntermediate { TRequestStat Stat; NKikimrClient::TResponse Response; - NKikimrKeyValue::ExecuteTransactionResult ExecuteTransactionResponse; - NKikimrKeyValue::GetStatusResult GetStatusResponse; - - THashMap<ui32, NKikimrKeyValue::Channel*> Channels; - - ui32 EvType = 0; + NKikimrKeyValue::ExecuteTransactionResult ExecuteTransactionResponse; + NKikimrKeyValue::GetStatusResult GetStatusResponse; + THashMap<ui32, NKikimrKeyValue::Channel*> Channels; + + ui32 EvType = 0; + TIntermediate(TActorId respondTo, TActorId keyValueActorId, ui64 channelGeneration, ui64 channelStep, TRequestType::EType requestType); diff --git a/ydb/core/keyvalue/keyvalue_request_stat.h b/ydb/core/keyvalue/keyvalue_request_stat.h index 070ffcbecac..d65fbcd0190 100644 --- a/ydb/core/keyvalue/keyvalue_request_stat.h +++ b/ydb/core/keyvalue/keyvalue_request_stat.h @@ -37,7 +37,7 @@ struct TRequestStat { TVector<ui32> YellowStopChannels; TVector<ui32> YellowMoveChannels; - + void Clear() { ReadBytes = 0; Reads = 0; diff --git a/ydb/core/keyvalue/keyvalue_state.cpp b/ydb/core/keyvalue/keyvalue_state.cpp index 6b31c463f83..fda119eeebd 100644 --- a/ydb/core/keyvalue/keyvalue_state.cpp +++ b/ydb/core/keyvalue/keyvalue_state.cpp @@ -1,6 +1,6 @@ #include "keyvalue_state.h" #include "keyvalue_data.h" -#include "keyvalue_storage_read_request.h" +#include "keyvalue_storage_read_request.h" #include "keyvalue_storage_request.h" #include "keyvalue_trash_key_arbitrary.h" #include <ydb/core/base/tablet.h> @@ -14,7 +14,7 @@ #include <library/cpp/monlib/service/pages/templates.h> #include <library/cpp/json/writer/json_value.h> #include <util/string/escape.h> -#include <util/charset/utf8.h> +#include <util/charset/utf8.h> // Set to 1 in order for tablet to reboot instead of failing a Y_VERIFY on database damage #define KIKIMR_KEYVALUE_ALLOW_DAMAGE 0 @@ -24,41 +24,41 @@ namespace NKeyValue { constexpr ui64 KeyValuePairSizeEstimation = 1 + 5 // Key id, length + 1 + 5 // Value id, length - + 1 + 4 // ValueSize id, value - + 1 + 8 // CreationUnixTime id, value + + 1 + 4 // ValueSize id, value + + 1 + 8 // CreationUnixTime id, value + 1 + 1 // StorageChannel id, value + 1 + 1 // Status id, value ; -constexpr ui64 KeyValuePairSizeEstimationNewApi = 1 + 5 // Key id, length - + 1 + 5 // Value id, length - + 1 + 4 // ValueSize id, value - + 1 + 8 // CreationUnixTime id, value - + 1 + 4 // StorageChannel id, value - + 1 + 1 // Status id, value - ; - -constexpr ui64 KeyInfoSizeEstimation = 1 + 5 // Key id, length - + 1 + 4 // ValueSize id, value - + 1 + 8 // CreationUnixTime id, value - + 1 + 4 // StorageChannel id, value - ; - -constexpr ui64 ReadRangeRequestMetaDataSizeEstimation = 1 + 5 // pair id, length - + 1 + 1 // Status id, value - ; - +constexpr ui64 KeyValuePairSizeEstimationNewApi = 1 + 5 // Key id, length + + 1 + 5 // Value id, length + + 1 + 4 // ValueSize id, value + + 1 + 8 // CreationUnixTime id, value + + 1 + 4 // StorageChannel id, value + + 1 + 1 // Status id, value + ; + +constexpr ui64 KeyInfoSizeEstimation = 1 + 5 // Key id, length + + 1 + 4 // ValueSize id, value + + 1 + 8 // CreationUnixTime id, value + + 1 + 4 // StorageChannel id, value + ; + +constexpr ui64 ReadRangeRequestMetaDataSizeEstimation = 1 + 5 // pair id, length + + 1 + 1 // Status id, value + ; + constexpr ui64 ReadResultSizeEstimation = 1 + 1 // Status id, value + 1 + 5 // Value id, length OR Message id, length ; -constexpr ui64 ReadResultSizeEstimationNewApi = 1 + 5 // Key id, length - + 1 + 5 // Value id, length - + 1 + 8 // Offset id, value - + 1 + 8 // Size id, value - + 1 + 1 // Status id, value - ; - +constexpr ui64 ReadResultSizeEstimationNewApi = 1 + 5 // Key id, length + + 1 + 5 // Value id, length + + 1 + 8 // Offset id, value + + 1 + 8 // Size id, value + + 1 + 1 // Status id, value + ; + constexpr ui64 ErrorMessageSizeEstimation = 128; // Guideline: @@ -134,9 +134,9 @@ void TKeyValueState::SetupResourceMetrics(NMetrics::TResourceMetrics* resourceMe ResourceMetrics = resourceMetrics; } -void TKeyValueState::CountRequestComplete(NMsgBusProxy::EResponseStatus status, - const TRequestStat &stat, const TActorContext &ctx) -{ +void TKeyValueState::CountRequestComplete(NMsgBusProxy::EResponseStatus status, + const TRequestStat &stat, const TActorContext &ctx) +{ ui64 fullLatencyMs = (TAppData::TimeProvider->Now() - stat.IntermediateCreatedAt).MilliSeconds(); if (stat.RequestType == TRequestType::WriteOnly) { TabletCounters->Percentile()[COUNTER_LATENCY_FULL_WO].IncrementFor(fullLatencyMs); @@ -807,8 +807,8 @@ void TKeyValueState::RequestExecute(THolder<TIntermediate> &intermediate, ISimpl // Process CmdIncrementGeneration() if (intermediate->HasIncrementGeneration) { - bool IsOk = intermediate->Commands.size() == 0 - && intermediate->Deletes.size() == 0 && intermediate->RangeReads.size() == 0 + bool IsOk = intermediate->Commands.size() == 0 + && intermediate->Deletes.size() == 0 && intermediate->RangeReads.size() == 0 && intermediate->Reads.size() == 0 && intermediate->Renames.size() == 0 && intermediate->Writes.size() == 0 && intermediate->GetStatuses.size() == 0; @@ -819,7 +819,7 @@ void TKeyValueState::RequestExecute(THolder<TIntermediate> &intermediate, ISimpl TStringStream str; str << "KeyValue# " << TabletId; str << " CmdIncrementGeneration can't be grouped with any other Cmd!"; - str << " Commands# " << intermediate->Commands.size(); + str << " Commands# " << intermediate->Commands.size(); str << " Deletes# " << intermediate->Deletes.size(); str << " RangeReads# " << intermediate->RangeReads.size(); str << " Reads# " << intermediate->Reads.size(); @@ -852,35 +852,35 @@ void TKeyValueState::RequestComplete(THolder<TIntermediate> &intermediate, const /////////////////////////////////////////////////////////////////////////////// // Request processing // - + void TKeyValueState::Reply(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info) { if (!intermediate->IsReplied) { - if (intermediate->EvType == TEvKeyValue::TEvRequest::EventType) { - THolder<TEvKeyValue::TEvResponse> response(new TEvKeyValue::TEvResponse); - response->Record = intermediate->Response; - ResourceMetrics->Network.Increment(response->Record.ByteSize()); - ctx.Send(intermediate->RespondTo, response.Release()); - } - if (intermediate->EvType == TEvKeyValue::TEvExecuteTransaction::EventType) { - THolder<TEvKeyValue::TEvExecuteTransactionResponse> response(new TEvKeyValue::TEvExecuteTransactionResponse); - response->Record = intermediate->ExecuteTransactionResponse; - ResourceMetrics->Network.Increment(response->Record.ByteSize()); - ctx.Send(intermediate->RespondTo, response.Release()); - } - if (intermediate->EvType == TEvKeyValue::TEvGetStatus::EventType) { - THolder<TEvKeyValue::TEvGetStatusResponse> response(new TEvKeyValue::TEvGetStatusResponse); - response->Record = intermediate->GetStatusResponse; - ResourceMetrics->Network.Increment(response->Record.ByteSize()); - ctx.Send(intermediate->RespondTo, response.Release()); - } - if (intermediate->EvType == TEvKeyValue::TEvObtainLock::EventType) { - THolder<TEvKeyValue::TEvObtainLockResponse> response(new TEvKeyValue::TEvObtainLockResponse); - response->Record.set_lock_generation(StoredState.GetUserGeneration()); - response->Record.set_cookie(intermediate->Cookie); - ResourceMetrics->Network.Increment(response->Record.ByteSize()); - ctx.Send(intermediate->RespondTo, response.Release()); - } + if (intermediate->EvType == TEvKeyValue::TEvRequest::EventType) { + THolder<TEvKeyValue::TEvResponse> response(new TEvKeyValue::TEvResponse); + response->Record = intermediate->Response; + ResourceMetrics->Network.Increment(response->Record.ByteSize()); + ctx.Send(intermediate->RespondTo, response.Release()); + } + if (intermediate->EvType == TEvKeyValue::TEvExecuteTransaction::EventType) { + THolder<TEvKeyValue::TEvExecuteTransactionResponse> response(new TEvKeyValue::TEvExecuteTransactionResponse); + response->Record = intermediate->ExecuteTransactionResponse; + ResourceMetrics->Network.Increment(response->Record.ByteSize()); + ctx.Send(intermediate->RespondTo, response.Release()); + } + if (intermediate->EvType == TEvKeyValue::TEvGetStatus::EventType) { + THolder<TEvKeyValue::TEvGetStatusResponse> response(new TEvKeyValue::TEvGetStatusResponse); + response->Record = intermediate->GetStatusResponse; + ResourceMetrics->Network.Increment(response->Record.ByteSize()); + ctx.Send(intermediate->RespondTo, response.Release()); + } + if (intermediate->EvType == TEvKeyValue::TEvObtainLock::EventType) { + THolder<TEvKeyValue::TEvObtainLockResponse> response(new TEvKeyValue::TEvObtainLockResponse); + response->Record.set_lock_generation(StoredState.GetUserGeneration()); + response->Record.set_cookie(intermediate->Cookie); + ResourceMetrics->Network.Increment(response->Record.ByteSize()); + ctx.Send(intermediate->RespondTo, response.Release()); + } intermediate->IsReplied = true; intermediate->UpdateStat(); @@ -891,338 +891,338 @@ void TKeyValueState::Reply(THolder<TIntermediate> &intermediate, const TActorCon } } -void TKeyValueState::ProcessCmd(TIntermediate::TRead &request, - NKikimrClient::TKeyValueResponse::TReadResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &/*db*/, const TActorContext &/*ctx*/, TRequestStat &/*stat*/, ui64 /*unixTime*/) -{ - NKikimrProto::EReplyStatus outStatus = request.CumulativeStatus(); - request.Status = outStatus; - legacyResponse->SetStatus(outStatus); - if (outStatus == NKikimrProto::OK) { - legacyResponse->SetValue(request.Value); - Y_VERIFY(request.Value.size() == request.ValueSize); - } else { - legacyResponse->SetMessage(request.Message); - if (outStatus == NKikimrProto::NODATA) { - for (ui32 itemIdx = 0; itemIdx < request.ReadItems.size(); ++itemIdx) { - TIntermediate::TRead::TReadItem &item = request.ReadItems[itemIdx]; - // Make sure the blob is not referenced anymore - auto refCountIt = RefCounts.find(item.LogoBlobId); - if (refCountIt != RefCounts.end()) { - TStringStream str; - str << "KeyValue# " << TabletId - << " CmdRead " - //<< " ReadIdx# " << i - << " key# " << EscapeC(request.Key) - << " ItemIdx# " << itemIdx - << " BlobId# " << item.LogoBlobId.ToString() - << " Status# " << NKikimrProto::EReplyStatus_Name(item.Status) - << " outStatus# " << NKikimrProto::EReplyStatus_Name(outStatus) - << " but blob has RefCount# " << refCountIt->second - << " ! KEYVALUE CONSISTENCY ERROR!" - << " Message# " << request.Message - << " Marker# KV46"; - Y_VERIFY(false, "%s", str.Str().c_str()); - } - } - } - } -} - -void TKeyValueState::ProcessCmd(TIntermediate::TRangeRead &request, - NKikimrClient::TKeyValueResponse::TReadRangeResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &/*db*/, const TActorContext &/*ctx*/, TRequestStat &/*stat*/, ui64 /*unixTime*/) -{ - for (ui64 r = 0; r < request.Reads.size(); ++r) { - auto &read = request.Reads[r]; - auto *resultKv = legacyResponse->AddPair(); - - NKikimrProto::EReplyStatus outStatus = read.CumulativeStatus(); - read.Status = outStatus; - if (outStatus != NKikimrProto::OK && outStatus != NKikimrProto::OVERRUN) { - // LOG_ERROR_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " CmdReadRange " << r - // << " status " << NKikimrProto::EReplyStatus_Name(outStatus) - // << " message " << read.Message - // << " key " << EscapeC(read.Key)); - +void TKeyValueState::ProcessCmd(TIntermediate::TRead &request, + NKikimrClient::TKeyValueResponse::TReadResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &/*db*/, const TActorContext &/*ctx*/, TRequestStat &/*stat*/, ui64 /*unixTime*/) +{ + NKikimrProto::EReplyStatus outStatus = request.CumulativeStatus(); + request.Status = outStatus; + legacyResponse->SetStatus(outStatus); + if (outStatus == NKikimrProto::OK) { + legacyResponse->SetValue(request.Value); + Y_VERIFY(request.Value.size() == request.ValueSize); + } else { + legacyResponse->SetMessage(request.Message); + if (outStatus == NKikimrProto::NODATA) { + for (ui32 itemIdx = 0; itemIdx < request.ReadItems.size(); ++itemIdx) { + TIntermediate::TRead::TReadItem &item = request.ReadItems[itemIdx]; + // Make sure the blob is not referenced anymore + auto refCountIt = RefCounts.find(item.LogoBlobId); + if (refCountIt != RefCounts.end()) { + TStringStream str; + str << "KeyValue# " << TabletId + << " CmdRead " + //<< " ReadIdx# " << i + << " key# " << EscapeC(request.Key) + << " ItemIdx# " << itemIdx + << " BlobId# " << item.LogoBlobId.ToString() + << " Status# " << NKikimrProto::EReplyStatus_Name(item.Status) + << " outStatus# " << NKikimrProto::EReplyStatus_Name(outStatus) + << " but blob has RefCount# " << refCountIt->second + << " ! KEYVALUE CONSISTENCY ERROR!" + << " Message# " << request.Message + << " Marker# KV46"; + Y_VERIFY(false, "%s", str.Str().c_str()); + } + } + } + } +} + +void TKeyValueState::ProcessCmd(TIntermediate::TRangeRead &request, + NKikimrClient::TKeyValueResponse::TReadRangeResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &/*db*/, const TActorContext &/*ctx*/, TRequestStat &/*stat*/, ui64 /*unixTime*/) +{ + for (ui64 r = 0; r < request.Reads.size(); ++r) { + auto &read = request.Reads[r]; + auto *resultKv = legacyResponse->AddPair(); + + NKikimrProto::EReplyStatus outStatus = read.CumulativeStatus(); + read.Status = outStatus; + if (outStatus != NKikimrProto::OK && outStatus != NKikimrProto::OVERRUN) { + // LOG_ERROR_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " CmdReadRange " << r + // << " status " << NKikimrProto::EReplyStatus_Name(outStatus) + // << " message " << read.Message + // << " key " << EscapeC(read.Key)); + if (outStatus == NKikimrProto::NODATA) { - for (ui32 itemIdx = 0; itemIdx < read.ReadItems.size(); ++itemIdx) { - TIntermediate::TRead::TReadItem &item = read.ReadItems[itemIdx]; + for (ui32 itemIdx = 0; itemIdx < read.ReadItems.size(); ++itemIdx) { + TIntermediate::TRead::TReadItem &item = read.ReadItems[itemIdx]; // Make sure the blob is not referenced anymore auto refCountIt = RefCounts.find(item.LogoBlobId); if (refCountIt != RefCounts.end()) { TStringStream str; str << "KeyValue# " << TabletId - << " CmdReadRange " - // << " RangeReadIdx# " << i - << " ReadIdx# " << r + << " CmdReadRange " + // << " RangeReadIdx# " << i + << " ReadIdx# " << r << " ItemIdx# " << itemIdx - << " key# " << EscapeC(read.Key) + << " key# " << EscapeC(read.Key) << " BlobId# " << item.LogoBlobId.ToString() << " Status# " << NKikimrProto::EReplyStatus_Name(item.Status) << " outStatus# " << NKikimrProto::EReplyStatus_Name(outStatus) << " but blob has RefCount# " << refCountIt->second << " ! KEYVALUE CONSISTENCY ERROR!" - << " Message# " << read.Message - << " Marker# KV47"; + << " Message# " << read.Message + << " Marker# KV47"; Y_VERIFY(false, "%s", str.Str().c_str()); } } } } - - resultKv->SetStatus(outStatus); - resultKv->SetKey(read.Key); - if (request.IncludeData && (outStatus == NKikimrProto::OK || outStatus == NKikimrProto::OVERRUN)) { - resultKv->SetValue(read.Value); - Y_VERIFY(read.Value.size() == read.ValueSize); - } - resultKv->SetValueSize(read.ValueSize); - resultKv->SetCreationUnixTime(read.CreationUnixTime); - resultKv->SetStorageChannel(read.StorageChannel); - } - - legacyResponse->SetStatus(request.Status); -} - - -void SetStatusFlags(NKikimrKeyValue::Flags *flags, const TStorageStatusFlags &statusFlags) { - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceCyan)) { - flags->set_disk_space_cyan(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) { - flags->set_disk_space_light_yellow_move(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) { - flags->set_disk_space_yellow_stop(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightOrange)) { - flags->set_disk_space_light_orange(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceOrange)) { - flags->set_disk_space_orange(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceRed)) { - flags->set_disk_space_red(true); - } - if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceBlack)) { - flags->set_disk_space_black(true); - } -} - -void TKeyValueState::ProcessCmd(TIntermediate::TWrite &request, - NKikimrClient::TKeyValueResponse::TWriteResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) -{ - TIndexRecord& record = Index[request.Key]; - Dereference(record, db, ctx); - - record.Chain = {}; - ui32 storage_channel = 0; - if (request.Status == NKikimrProto::SCHEDULED) { - TString inlineData = request.Data; - record.Chain.push_back(TIndexRecord::TChainItem(inlineData, 0)); - CountWriteRecord(0, inlineData.size()); - request.Status = NKikimrProto::OK; - storage_channel = InlineStorageChannelInPublicApi; - } else { - int channel = -1; - - ui64 offset = 0; - for (const TLogoBlobID& logoBlobId : request.LogoBlobIds) { - record.Chain.push_back(TIndexRecord::TChainItem(logoBlobId, offset)); - offset += logoBlobId.BlobSize(); - CountWriteRecord(logoBlobId.Channel(), logoBlobId.BlobSize()); - if (channel == -1) { - channel = logoBlobId.Channel(); - } else { - // all blobs from the same write must be within the same channel - Y_VERIFY(channel == (int)logoBlobId.Channel()); + + resultKv->SetStatus(outStatus); + resultKv->SetKey(read.Key); + if (request.IncludeData && (outStatus == NKikimrProto::OK || outStatus == NKikimrProto::OVERRUN)) { + resultKv->SetValue(read.Value); + Y_VERIFY(read.Value.size() == read.ValueSize); + } + resultKv->SetValueSize(read.ValueSize); + resultKv->SetCreationUnixTime(read.CreationUnixTime); + resultKv->SetStorageChannel(read.StorageChannel); + } + + legacyResponse->SetStatus(request.Status); +} + + +void SetStatusFlags(NKikimrKeyValue::Flags *flags, const TStorageStatusFlags &statusFlags) { + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceCyan)) { + flags->set_disk_space_cyan(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) { + flags->set_disk_space_light_yellow_move(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) { + flags->set_disk_space_yellow_stop(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightOrange)) { + flags->set_disk_space_light_orange(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceOrange)) { + flags->set_disk_space_orange(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceRed)) { + flags->set_disk_space_red(true); + } + if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceBlack)) { + flags->set_disk_space_black(true); + } +} + +void TKeyValueState::ProcessCmd(TIntermediate::TWrite &request, + NKikimrClient::TKeyValueResponse::TWriteResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) +{ + TIndexRecord& record = Index[request.Key]; + Dereference(record, db, ctx); + + record.Chain = {}; + ui32 storage_channel = 0; + if (request.Status == NKikimrProto::SCHEDULED) { + TString inlineData = request.Data; + record.Chain.push_back(TIndexRecord::TChainItem(inlineData, 0)); + CountWriteRecord(0, inlineData.size()); + request.Status = NKikimrProto::OK; + storage_channel = InlineStorageChannelInPublicApi; + } else { + int channel = -1; + + ui64 offset = 0; + for (const TLogoBlobID& logoBlobId : request.LogoBlobIds) { + record.Chain.push_back(TIndexRecord::TChainItem(logoBlobId, offset)); + offset += logoBlobId.BlobSize(); + CountWriteRecord(logoBlobId.Channel(), logoBlobId.BlobSize()); + if (channel == -1) { + channel = logoBlobId.Channel(); + } else { + // all blobs from the same write must be within the same channel + Y_VERIFY(channel == (int)logoBlobId.Channel()); } - } - storage_channel = channel + MainStorageChannelInPublicApi; - - ctx.Send(ChannelBalancerActorId, new TChannelBalancer::TEvReportWriteLatency(channel, request.Latency)); - } - - record.CreationUnixTime = unixTime; - UpdateKeyValue(request.Key, record, db, ctx); - - if (legacyResponse) { - legacyResponse->SetStatus(NKikimrProto::OK); - legacyResponse->SetStatusFlags(request.StatusFlags.Raw); - } - if (response) { - response->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); - auto *flags = response->mutable_status_flags(); - SetStatusFlags(flags, request.StatusFlags); - response->set_storage_channel(storage_channel); - } -} - -void TKeyValueState::ProcessCmd(const TIntermediate::TDelete &request, - NKikimrClient::TKeyValueResponse::TDeleteRangeResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 /*unixTime*/) -{ - TraverseRange(request.Range, [&](TIndex::iterator it) { - stat.Deletes++; - stat.DeleteBytes += it->second.GetFullValueSize(); - Dereference(it->second, db, ctx); - THelpers::DbEraseUserKey(it->first, db, ctx); - Index.erase(it); - }); - - if (legacyResponse) { - legacyResponse->SetStatus(NKikimrProto::OK); - } -} - -void TKeyValueState::ProcessCmd(const TIntermediate::TRename &request, - NKikimrClient::TKeyValueResponse::TRenameResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) -{ - auto oldIter = Index.find(request.OldKey); - Y_VERIFY(oldIter != Index.end()); - TIndexRecord& source = oldIter->second; - - TIndexRecord& dest = Index[request.NewKey]; - Dereference(dest, db, ctx); - dest.Chain = std::move(source.Chain); - dest.CreationUnixTime = unixTime; - - THelpers::DbEraseUserKey(oldIter->first, db, ctx); - Index.erase(oldIter); - - UpdateKeyValue(request.NewKey, dest, db, ctx); - - if (legacyResponse) { - legacyResponse->SetStatus(NKikimrProto::OK); - } -} - -void TKeyValueState::ProcessCmd(const TIntermediate::TCopyRange &request, - NKikimrClient::TKeyValueResponse::TCopyRangeResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 /*unixTime*/) -{ - TVector<TIndex::iterator> itemsToClone; - - TraverseRange(request.Range, [&](TIndex::iterator it) { - if (it->first.StartsWith(request.PrefixToRemove)) { - itemsToClone.push_back(it); - } - }); - - for (TIndex::iterator it : itemsToClone) { - const TIndexRecord& sourceRecord = it->second; - for (const TIndexRecord::TChainItem& item : sourceRecord.Chain) { - if (!item.IsInline()) { - ++RefCounts[item.LogoBlobId]; - } - } - - TString newKey = request.PrefixToAdd + it->first.substr(request.PrefixToRemove.size()); - TIndexRecord& record = Index[newKey]; - Dereference(record, db, ctx); - record.Chain = sourceRecord.Chain; - record.CreationUnixTime = sourceRecord.CreationUnixTime; - UpdateKeyValue(newKey, record, db, ctx); - } - - if (legacyResponse) { - legacyResponse->SetStatus(NKikimrProto::OK); - } -} - -void TKeyValueState::ProcessCmd(const TIntermediate::TConcat &request, - NKikimrClient::TKeyValueResponse::TConcatResult *legacyResponse, - NKikimrKeyValue::Channel */*response*/, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) -{ - TVector<TIndexRecord::TChainItem> chain; - ui64 offset = 0; - - for (const TString& key : request.InputKeys) { - auto it = Index.find(key); - Y_VERIFY(it != Index.end()); - TIndexRecord& input = it->second; - - for (TIndexRecord::TChainItem& chainItem : input.Chain) { - if (chainItem.IsInline()) { - chain.push_back(TIndexRecord::TChainItem(chainItem.InlineData, offset)); - } else { - const TLogoBlobID& id = chainItem.LogoBlobId; - chain.push_back(TIndexRecord::TChainItem(id, offset)); - ++RefCounts[id]; + } + storage_channel = channel + MainStorageChannelInPublicApi; + + ctx.Send(ChannelBalancerActorId, new TChannelBalancer::TEvReportWriteLatency(channel, request.Latency)); + } + + record.CreationUnixTime = unixTime; + UpdateKeyValue(request.Key, record, db, ctx); + + if (legacyResponse) { + legacyResponse->SetStatus(NKikimrProto::OK); + legacyResponse->SetStatusFlags(request.StatusFlags.Raw); + } + if (response) { + response->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); + auto *flags = response->mutable_status_flags(); + SetStatusFlags(flags, request.StatusFlags); + response->set_storage_channel(storage_channel); + } +} + +void TKeyValueState::ProcessCmd(const TIntermediate::TDelete &request, + NKikimrClient::TKeyValueResponse::TDeleteRangeResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 /*unixTime*/) +{ + TraverseRange(request.Range, [&](TIndex::iterator it) { + stat.Deletes++; + stat.DeleteBytes += it->second.GetFullValueSize(); + Dereference(it->second, db, ctx); + THelpers::DbEraseUserKey(it->first, db, ctx); + Index.erase(it); + }); + + if (legacyResponse) { + legacyResponse->SetStatus(NKikimrProto::OK); + } +} + +void TKeyValueState::ProcessCmd(const TIntermediate::TRename &request, + NKikimrClient::TKeyValueResponse::TRenameResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) +{ + auto oldIter = Index.find(request.OldKey); + Y_VERIFY(oldIter != Index.end()); + TIndexRecord& source = oldIter->second; + + TIndexRecord& dest = Index[request.NewKey]; + Dereference(dest, db, ctx); + dest.Chain = std::move(source.Chain); + dest.CreationUnixTime = unixTime; + + THelpers::DbEraseUserKey(oldIter->first, db, ctx); + Index.erase(oldIter); + + UpdateKeyValue(request.NewKey, dest, db, ctx); + + if (legacyResponse) { + legacyResponse->SetStatus(NKikimrProto::OK); + } +} + +void TKeyValueState::ProcessCmd(const TIntermediate::TCopyRange &request, + NKikimrClient::TKeyValueResponse::TCopyRangeResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 /*unixTime*/) +{ + TVector<TIndex::iterator> itemsToClone; + + TraverseRange(request.Range, [&](TIndex::iterator it) { + if (it->first.StartsWith(request.PrefixToRemove)) { + itemsToClone.push_back(it); + } + }); + + for (TIndex::iterator it : itemsToClone) { + const TIndexRecord& sourceRecord = it->second; + for (const TIndexRecord::TChainItem& item : sourceRecord.Chain) { + if (!item.IsInline()) { + ++RefCounts[item.LogoBlobId]; } - offset += chainItem.GetSize(); - } - - if (!request.KeepInputs) { - Dereference(input, db, ctx); - THelpers::DbEraseUserKey(it->first, db, ctx); - Index.erase(it); } - } - - TIndexRecord& record = Index[request.OutputKey]; - Dereference(record, db, ctx); - record.Chain = std::move(chain); - record.CreationUnixTime = unixTime; - UpdateKeyValue(request.OutputKey, record, db, ctx); - - if (legacyResponse) { - legacyResponse->SetStatus(NKikimrProto::OK); - } -} - -void TKeyValueState::CmdRead(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { - for (ui64 i = 0; i < intermediate->Reads.size(); ++i) { - auto &request = intermediate->Reads[i]; - auto *response = intermediate->Response.AddReadResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); - } - if (intermediate->ReadCommand && std::holds_alternative<TIntermediate::TRead>(*intermediate->ReadCommand)) { - auto &request = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); - auto *response = intermediate->Response.AddReadResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); - } -} - -void TKeyValueState::CmdReadRange(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { - Y_UNUSED(ctx); - Y_UNUSED(db); - for (ui64 i = 0; i < intermediate->RangeReads.size(); ++i) { - auto &rangeRead = intermediate->RangeReads[i]; - auto *rangeReadResult = intermediate->Response.AddReadRangeResult(); - ProcessCmd(rangeRead, rangeReadResult, nullptr, db, ctx, intermediate->Stat, 0); - } - if (intermediate->ReadCommand && std::holds_alternative<TIntermediate::TRangeRead>(*intermediate->ReadCommand)) { - auto &request = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); - auto *response = intermediate->Response.AddReadRangeResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); - } -} - -void TKeyValueState::CmdRename(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { - ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); - for (ui32 i = 0; i < intermediate->Renames.size(); ++i) { - auto& request = intermediate->Renames[i]; - auto *response = intermediate->Response.AddRenameResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); - } -} + TString newKey = request.PrefixToAdd + it->first.substr(request.PrefixToRemove.size()); + TIndexRecord& record = Index[newKey]; + Dereference(record, db, ctx); + record.Chain = sourceRecord.Chain; + record.CreationUnixTime = sourceRecord.CreationUnixTime; + UpdateKeyValue(newKey, record, db, ctx); + } + + if (legacyResponse) { + legacyResponse->SetStatus(NKikimrProto::OK); + } +} + +void TKeyValueState::ProcessCmd(const TIntermediate::TConcat &request, + NKikimrClient::TKeyValueResponse::TConcatResult *legacyResponse, + NKikimrKeyValue::Channel */*response*/, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &/*stat*/, ui64 unixTime) +{ + TVector<TIndexRecord::TChainItem> chain; + ui64 offset = 0; + + for (const TString& key : request.InputKeys) { + auto it = Index.find(key); + Y_VERIFY(it != Index.end()); + TIndexRecord& input = it->second; + + for (TIndexRecord::TChainItem& chainItem : input.Chain) { + if (chainItem.IsInline()) { + chain.push_back(TIndexRecord::TChainItem(chainItem.InlineData, offset)); + } else { + const TLogoBlobID& id = chainItem.LogoBlobId; + chain.push_back(TIndexRecord::TChainItem(id, offset)); + ++RefCounts[id]; + } + offset += chainItem.GetSize(); + } + + if (!request.KeepInputs) { + Dereference(input, db, ctx); + THelpers::DbEraseUserKey(it->first, db, ctx); + Index.erase(it); + } + } + + TIndexRecord& record = Index[request.OutputKey]; + Dereference(record, db, ctx); + record.Chain = std::move(chain); + record.CreationUnixTime = unixTime; + UpdateKeyValue(request.OutputKey, record, db, ctx); + + if (legacyResponse) { + legacyResponse->SetStatus(NKikimrProto::OK); + } +} + +void TKeyValueState::CmdRead(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { + for (ui64 i = 0; i < intermediate->Reads.size(); ++i) { + auto &request = intermediate->Reads[i]; + auto *response = intermediate->Response.AddReadResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); + } + if (intermediate->ReadCommand && std::holds_alternative<TIntermediate::TRead>(*intermediate->ReadCommand)) { + auto &request = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); + auto *response = intermediate->Response.AddReadResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); + } +} + +void TKeyValueState::CmdReadRange(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { + Y_UNUSED(ctx); + Y_UNUSED(db); + for (ui64 i = 0; i < intermediate->RangeReads.size(); ++i) { + auto &rangeRead = intermediate->RangeReads[i]; + auto *rangeReadResult = intermediate->Response.AddReadRangeResult(); + ProcessCmd(rangeRead, rangeReadResult, nullptr, db, ctx, intermediate->Stat, 0); + } + if (intermediate->ReadCommand && std::holds_alternative<TIntermediate::TRangeRead>(*intermediate->ReadCommand)) { + auto &request = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); + auto *response = intermediate->Response.AddReadRangeResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); + } +} + +void TKeyValueState::CmdRename(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { + ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); + for (ui32 i = 0; i < intermediate->Renames.size(); ++i) { + auto& request = intermediate->Renames[i]; + auto *response = intermediate->Response.AddRenameResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); + } +} + void TKeyValueState::CmdDelete(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { for (ui32 i = 0; i < intermediate->Deletes.size(); ++i) { auto& request = intermediate->Deletes[i]; - auto *response = intermediate->Response.AddDeleteRangeResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); + auto *response = intermediate->Response.AddDeleteRangeResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); } } @@ -1230,8 +1230,8 @@ void TKeyValueState::CmdWrite(THolder<TIntermediate> &intermediate, ISimpleDb &d ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); for (ui32 i = 0; i < intermediate->Writes.size(); ++i) { auto& request = intermediate->Writes[i]; - auto *response = intermediate->Response.AddWriteResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); + auto *response = intermediate->Response.AddWriteResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); } ResourceMetrics->TryUpdate(ctx); } @@ -1241,47 +1241,47 @@ void TKeyValueState::CmdGetStatus(THolder<TIntermediate> &intermediate, ISimpleD Y_UNUSED(ctx); for (ui32 i = 0; i < intermediate->GetStatuses.size(); ++i) { auto& request = intermediate->GetStatuses[i]; - if (intermediate->EvType == TEvKeyValue::TEvRequest::EventType) { - auto& response = *intermediate->Response.AddGetStatusResult(); - - response.SetStatus(request.Status); - response.SetStorageChannel(request.StorageChannel); - response.SetStatusFlags(request.StatusFlags.Raw); - } else if ((intermediate->EvType == TEvKeyValue::TEvGetStatus::EventType)) { - auto response = intermediate->GetStatusResponse.add_channel(); - - if (request.Status == NKikimrProto::OK) { - response->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); - } else if (request.Status == NKikimrProto::TIMEOUT) { - response->set_status(NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT); - } else { - response->set_status(NKikimrKeyValue::Statuses::RSTATUS_ERROR); - } - - if (request.StorageChannel == NKikimrClient::TKeyValueRequest::INLINE) { - response->set_storage_channel(1); - } else { - response->set_storage_channel(request.StorageChannel - BLOB_CHANNEL + MainStorageChannelInPublicApi); - } - - SetStatusFlags(response->mutable_status_flags(), request.StatusFlags); - } - } - intermediate->GetStatusResponse.set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); + if (intermediate->EvType == TEvKeyValue::TEvRequest::EventType) { + auto& response = *intermediate->Response.AddGetStatusResult(); + + response.SetStatus(request.Status); + response.SetStorageChannel(request.StorageChannel); + response.SetStatusFlags(request.StatusFlags.Raw); + } else if ((intermediate->EvType == TEvKeyValue::TEvGetStatus::EventType)) { + auto response = intermediate->GetStatusResponse.add_channel(); + + if (request.Status == NKikimrProto::OK) { + response->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); + } else if (request.Status == NKikimrProto::TIMEOUT) { + response->set_status(NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT); + } else { + response->set_status(NKikimrKeyValue::Statuses::RSTATUS_ERROR); + } + + if (request.StorageChannel == NKikimrClient::TKeyValueRequest::INLINE) { + response->set_storage_channel(1); + } else { + response->set_storage_channel(request.StorageChannel - BLOB_CHANNEL + MainStorageChannelInPublicApi); + } + + SetStatusFlags(response->mutable_status_flags(), request.StatusFlags); + } + } + intermediate->GetStatusResponse.set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); } void TKeyValueState::CmdCopyRange(THolder<TIntermediate>& intermediate, ISimpleDb& db, const TActorContext& ctx) { for (const auto& request : intermediate->CopyRanges) { - auto *response = intermediate->Response.AddCopyRangeResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); + auto *response = intermediate->Response.AddCopyRangeResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, 0); } } void TKeyValueState::CmdConcat(THolder<TIntermediate>& intermediate, ISimpleDb& db, const TActorContext& ctx) { ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); for (const auto& request : intermediate->Concats) { - auto *response = intermediate->Response.AddConcatResult(); - ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); + auto *response = intermediate->Response.AddConcatResult(); + ProcessCmd(request, response, nullptr, db, ctx, intermediate->Stat, unixTime); } } @@ -1320,220 +1320,220 @@ void TKeyValueState::CmdSetExecutorFastLogPolicy(THolder<TIntermediate> &interme } } -void TKeyValueState::CmdCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { - ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); - bool wasWrite = false; - auto getChannel = [&](auto &cmd) -> NKikimrKeyValue::Channel* { - using Type = std::decay_t<decltype(cmd)>; - if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { - if (intermediate->EvType != TEvKeyValue::TEvExecuteTransaction::EventType) { - return nullptr; - } - ui32 storageChannel = MainStorageChannelInPublicApi; - if (cmd.Status == NKikimrProto::SCHEDULED) { - storageChannel = InlineStorageChannelInPublicApi; - } - if (cmd.LogoBlobIds.size()) { - storageChannel = cmd.LogoBlobIds.front().Channel() - BLOB_CHANNEL + MainStorageChannelInPublicApi; - } - auto it = intermediate->Channels.find(storageChannel); - if (it == intermediate->Channels.end()) { - auto channel = intermediate->ExecuteTransactionResponse.add_channel(); - intermediate->Channels.emplace(storageChannel, channel); - return channel; - } - return it->second; - } - return nullptr; - }; - auto getLegacyResponse = [&](auto &cmd) { - using Type = std::decay_t<decltype(cmd)>; - if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { - wasWrite = true; - return intermediate->Response.AddWriteResult(); - } - if constexpr (std::is_same_v<Type, TIntermediate::TDelete>) { - return intermediate->Response.AddDeleteRangeResult(); - } - if constexpr (std::is_same_v<Type, TIntermediate::TRename>) { - return intermediate->Response.AddRenameResult(); - } - if constexpr (std::is_same_v<Type, TIntermediate::TCopyRange>) { - return intermediate->Response.AddCopyRangeResult(); - } - if constexpr (std::is_same_v<Type, TIntermediate::TConcat>) { - return intermediate->Response.AddConcatResult(); - } - }; - auto process = [&](auto &cmd) { - ProcessCmd(cmd, getLegacyResponse(cmd), getChannel(cmd), db, ctx, intermediate->Stat, unixTime); - }; - for (auto &cmd : intermediate->Commands) { - std::visit(process, cmd); - } - if (wasWrite) { - ResourceMetrics->TryUpdate(ctx); - } -} - -TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TCopyRange &cmd, TKeySet& keys, - ui32 /*index*/) const -{ - TVector<TString> nkeys; - auto range = GetRange(cmd.Range, keys); - for (auto it = range.first; it != range.second; ++it) { - if (it->StartsWith(cmd.PrefixToRemove)) { - nkeys.push_back(cmd.PrefixToAdd + it->substr(cmd.PrefixToRemove.size())); - } - } - keys.insert(nkeys.begin(), nkeys.end()); - return {}; -} - -TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TRename &cmd, TKeySet& keys, - ui32 index) const -{ - auto it = keys.find(cmd.OldKey); - if (it == keys.end()) { - TStringStream str; - str << "KeyValue# " << TabletId - << " OldKey# " << EscapeC(cmd.OldKey) << " does not exist in CmdRename(" << index << ")" - << " Marker# KV18"; - return {false, str.Str()}; - } - keys.erase(it); - keys.insert(cmd.NewKey); - return {}; -} - -TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TConcat &cmd, TKeySet& keys, - ui32 index) const -{ - for (const TString& key : cmd.InputKeys) { - auto it = keys.find(key); - if (it == keys.end()) { - TStringStream str; - str << "KeyValue# " << TabletId - << " InputKey# " << EscapeC(key) << " does not exist in CmdConcat(" << index << ")" - << " Marker# KV19"; - return {false, str.Str()}; - } - if (!cmd.KeepInputs) { - keys.erase(it); - } - } - - keys.insert(cmd.OutputKey); - return {}; -} - -TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TDelete &cmd, TKeySet& keys, - ui32 /*index*/) const -{ - auto r = GetRange(cmd.Range, keys); - keys.erase(r.first, r.second); - return {}; -} - -TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TWrite &cmd, TKeySet& keys, - ui32 /*index*/) const -{ - keys.insert(cmd.Key); - return {}; -} - -bool TKeyValueState::CheckCmdCopyRanges(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, - TKeySet& keys, const TTabletStorageInfo* /*info*/) -{ - for (const auto& cmd : intermediate->CopyRanges) { - CheckCmd(cmd, keys, 0); - } +void TKeyValueState::CmdCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx) { + ui64 unixTime = TAppData::TimeProvider->Now().Seconds(); + bool wasWrite = false; + auto getChannel = [&](auto &cmd) -> NKikimrKeyValue::Channel* { + using Type = std::decay_t<decltype(cmd)>; + if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { + if (intermediate->EvType != TEvKeyValue::TEvExecuteTransaction::EventType) { + return nullptr; + } + ui32 storageChannel = MainStorageChannelInPublicApi; + if (cmd.Status == NKikimrProto::SCHEDULED) { + storageChannel = InlineStorageChannelInPublicApi; + } + if (cmd.LogoBlobIds.size()) { + storageChannel = cmd.LogoBlobIds.front().Channel() - BLOB_CHANNEL + MainStorageChannelInPublicApi; + } + auto it = intermediate->Channels.find(storageChannel); + if (it == intermediate->Channels.end()) { + auto channel = intermediate->ExecuteTransactionResponse.add_channel(); + intermediate->Channels.emplace(storageChannel, channel); + return channel; + } + return it->second; + } + return nullptr; + }; + auto getLegacyResponse = [&](auto &cmd) { + using Type = std::decay_t<decltype(cmd)>; + if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { + wasWrite = true; + return intermediate->Response.AddWriteResult(); + } + if constexpr (std::is_same_v<Type, TIntermediate::TDelete>) { + return intermediate->Response.AddDeleteRangeResult(); + } + if constexpr (std::is_same_v<Type, TIntermediate::TRename>) { + return intermediate->Response.AddRenameResult(); + } + if constexpr (std::is_same_v<Type, TIntermediate::TCopyRange>) { + return intermediate->Response.AddCopyRangeResult(); + } + if constexpr (std::is_same_v<Type, TIntermediate::TConcat>) { + return intermediate->Response.AddConcatResult(); + } + }; + auto process = [&](auto &cmd) { + ProcessCmd(cmd, getLegacyResponse(cmd), getChannel(cmd), db, ctx, intermediate->Stat, unixTime); + }; + for (auto &cmd : intermediate->Commands) { + std::visit(process, cmd); + } + if (wasWrite) { + ResourceMetrics->TryUpdate(ctx); + } +} + +TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TCopyRange &cmd, TKeySet& keys, + ui32 /*index*/) const +{ + TVector<TString> nkeys; + auto range = GetRange(cmd.Range, keys); + for (auto it = range.first; it != range.second; ++it) { + if (it->StartsWith(cmd.PrefixToRemove)) { + nkeys.push_back(cmd.PrefixToAdd + it->substr(cmd.PrefixToRemove.size())); + } + } + keys.insert(nkeys.begin(), nkeys.end()); + return {}; +} + +TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TRename &cmd, TKeySet& keys, + ui32 index) const +{ + auto it = keys.find(cmd.OldKey); + if (it == keys.end()) { + TStringStream str; + str << "KeyValue# " << TabletId + << " OldKey# " << EscapeC(cmd.OldKey) << " does not exist in CmdRename(" << index << ")" + << " Marker# KV18"; + return {false, str.Str()}; + } + keys.erase(it); + keys.insert(cmd.NewKey); + return {}; +} + +TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TConcat &cmd, TKeySet& keys, + ui32 index) const +{ + for (const TString& key : cmd.InputKeys) { + auto it = keys.find(key); + if (it == keys.end()) { + TStringStream str; + str << "KeyValue# " << TabletId + << " InputKey# " << EscapeC(key) << " does not exist in CmdConcat(" << index << ")" + << " Marker# KV19"; + return {false, str.Str()}; + } + if (!cmd.KeepInputs) { + keys.erase(it); + } + } + + keys.insert(cmd.OutputKey); + return {}; +} + +TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TDelete &cmd, TKeySet& keys, + ui32 /*index*/) const +{ + auto r = GetRange(cmd.Range, keys); + keys.erase(r.first, r.second); + return {}; +} + +TKeyValueState::TCheckResult TKeyValueState::CheckCmd(const TIntermediate::TWrite &cmd, TKeySet& keys, + ui32 /*index*/) const +{ + keys.insert(cmd.Key); + return {}; +} + +bool TKeyValueState::CheckCmdCopyRanges(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, + TKeySet& keys, const TTabletStorageInfo* /*info*/) +{ + for (const auto& cmd : intermediate->CopyRanges) { + CheckCmd(cmd, keys, 0); + } return true; } -bool TKeyValueState::CheckCmdRenames(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, - const TTabletStorageInfo *info) -{ +bool TKeyValueState::CheckCmdRenames(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + const TTabletStorageInfo *info) +{ ui32 index = 0; for (const auto& cmd : intermediate->Renames) { - const auto &[ok, msg] = CheckCmd(cmd, keys, index++); - if (!ok) { - ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); + const auto &[ok, msg] = CheckCmd(cmd, keys, index++); + if (!ok) { + ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); return false; } } return true; } -bool TKeyValueState::CheckCmdConcats(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, - const TTabletStorageInfo *info) -{ +bool TKeyValueState::CheckCmdConcats(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + const TTabletStorageInfo *info) +{ ui32 index = 0; for (const auto& cmd : intermediate->Concats) { - const auto &[ok, msg] = CheckCmd(cmd, keys, index++); - if (!ok) { - ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); - return false; + const auto &[ok, msg] = CheckCmd(cmd, keys, index++); + if (!ok) { + ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); + return false; } } return true; } -bool TKeyValueState::CheckCmdDeletes(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, - const TTabletStorageInfo* /*info*/) -{ +bool TKeyValueState::CheckCmdDeletes(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, + const TTabletStorageInfo* /*info*/) +{ for (const auto& cmd : intermediate->Deletes) { - CheckCmd(cmd, keys, 0); + CheckCmd(cmd, keys, 0); } return true; } -bool TKeyValueState::CheckCmdWrites(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, - const TTabletStorageInfo* /*info*/) -{ +bool TKeyValueState::CheckCmdWrites(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, + const TTabletStorageInfo* /*info*/) +{ for (const auto& cmd : intermediate->Writes) { - CheckCmd(cmd, keys, 0); + CheckCmd(cmd, keys, 0); } return true; } bool TKeyValueState::CheckCmdGetStatus(THolder<TIntermediate>& /*intermediate*/, const TActorContext& /*ctx*/, - TKeySet& /*keys*/, const TTabletStorageInfo* /*info*/) -{ - return true; -} - -bool TKeyValueState::CheckCmds(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, - const TTabletStorageInfo* info) -{ - ui32 renameIndex = 0; - ui32 concatIndex = 0; - - auto nextIdx = [&](auto &cmd) -> ui32 { - using Type = std::decay_t<decltype(cmd)>; - if constexpr (std::is_same_v<Type, TIntermediate::TRename>) { - return renameIndex++; - } - if constexpr (std::is_same_v<Type, TIntermediate::TConcat>) { - return concatIndex++; - } - return 0; - }; - auto visitor = [&](auto &cmd) { - return CheckCmd(cmd, keys, nextIdx(cmd)); - }; - - for (const auto& cmd : intermediate->Commands) { - const auto &[ok, msg] = std::visit(visitor, cmd); - if (!ok) { - ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); - return false; - } - } + TKeySet& /*keys*/, const TTabletStorageInfo* /*info*/) +{ return true; } +bool TKeyValueState::CheckCmds(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + const TTabletStorageInfo* info) +{ + ui32 renameIndex = 0; + ui32 concatIndex = 0; + + auto nextIdx = [&](auto &cmd) -> ui32 { + using Type = std::decay_t<decltype(cmd)>; + if constexpr (std::is_same_v<Type, TIntermediate::TRename>) { + return renameIndex++; + } + if constexpr (std::is_same_v<Type, TIntermediate::TConcat>) { + return concatIndex++; + } + return 0; + }; + auto visitor = [&](auto &cmd) { + return CheckCmd(cmd, keys, nextIdx(cmd)); + }; + + for (const auto& cmd : intermediate->Commands) { + const auto &[ok, msg] = std::visit(visitor, cmd); + if (!ok) { + ReplyError(ctx, msg, NMsgBusProxy::MSTATUS_ERROR, intermediate, info); + return false; + } + } + return true; +} + void TKeyValueState::ProcessCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx, const TTabletStorageInfo *info) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " TTxRequest ProcessCmds"); @@ -1551,12 +1551,12 @@ void TKeyValueState::ProcessCmds(THolder<TIntermediate> &intermediate, ISimpleDb success = false; } - success = success && CheckCmdCopyRanges(intermediate, ctx, keys, info); - success = success && CheckCmdRenames(intermediate, ctx, keys, info); - success = success && CheckCmdConcats(intermediate, ctx, keys, info); - success = success && CheckCmdDeletes(intermediate, ctx, keys, info); - success = success && CheckCmdWrites(intermediate, ctx, keys, info); - success = success && CheckCmds(intermediate, ctx, keys, info); + success = success && CheckCmdCopyRanges(intermediate, ctx, keys, info); + success = success && CheckCmdRenames(intermediate, ctx, keys, info); + success = success && CheckCmdConcats(intermediate, ctx, keys, info); + success = success && CheckCmdDeletes(intermediate, ctx, keys, info); + success = success && CheckCmdWrites(intermediate, ctx, keys, info); + success = success && CheckCmds(intermediate, ctx, keys, info); success = success && CheckCmdGetStatus(intermediate, ctx, keys, info); if (!success) { for (const auto& cmd : intermediate->Writes) { @@ -1564,15 +1564,15 @@ void TKeyValueState::ProcessCmds(THolder<TIntermediate> &intermediate, ISimpleDb Dereference(logoBlobId, db, ctx, true); } } - for (const auto& cmd : intermediate->Commands) { - if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { - continue; - } - auto& write = std::get<TIntermediate::TWrite>(cmd); - for (const TLogoBlobID& logoBlobId : write.LogoBlobIds) { - Dereference(logoBlobId, db, ctx, true); - } - } + for (const auto& cmd : intermediate->Commands) { + if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { + continue; + } + auto& write = std::get<TIntermediate::TWrite>(cmd); + for (const TLogoBlobID& logoBlobId : write.LogoBlobIds) { + Dereference(logoBlobId, db, ctx, true); + } + } } else { // Read + validate CmdRead(intermediate, db, ctx); @@ -1585,7 +1585,7 @@ void TKeyValueState::ProcessCmds(THolder<TIntermediate> &intermediate, ISimpleDb CmdDelete(intermediate, db, ctx); CmdWrite(intermediate, db, ctx); CmdGetStatus(intermediate, db, ctx); - CmdCmds(intermediate, db, ctx); + CmdCmds(intermediate, db, ctx); // Blob trimming CmdTrimLeakedBlobs(intermediate, db, ctx); @@ -1688,16 +1688,16 @@ void TKeyValueState::OnRequestComplete(ui64 requestUid, ui64 generation, ui64 st CountRequestComplete(status, stat, ctx); ResourceMetrics->TryUpdate(ctx); - if (Queue.size() && IntermediatesInFlight < IntermediatesInFlightLimit) { - TRequestType::EType requestType = Queue.front()->Stat.RequestType; + if (Queue.size() && IntermediatesInFlight < IntermediatesInFlightLimit) { + TRequestType::EType requestType = Queue.front()->Stat.RequestType; - CountLatencyQueue(Queue.front()->Stat); + CountLatencyQueue(Queue.front()->Stat); - ProcessPostponedIntermediate(ctx, std::move(Queue.front()), info); - Queue.pop_front(); - ++IntermediatesInFlight; + ProcessPostponedIntermediate(ctx, std::move(Queue.front()), info); + Queue.pop_front(); + ++IntermediatesInFlight; - CountRequestTakeOffOrEnqueue(requestType); + CountRequestTakeOffOrEnqueue(requestType); } if (StoredState.GetChannelGeneration() == generation) { @@ -1752,209 +1752,209 @@ bool TKeyValueState::CheckGeneration(const TActorContext &ctx, NKikimrClient::TK return false; } - -template <typename TypeWithPriority> -void SetPriority(NKikimrBlobStorage::EGetHandleClass *outHandleClass, ui8 priority) { - *outHandleClass = NKikimrBlobStorage::FastRead; - if constexpr (std::is_same_v<TypeWithPriority, NKikimrKeyValue::Priorities>) { - switch (priority) { - case TypeWithPriority::PRIORITY_UNSPECIFIED: - case TypeWithPriority::PRIORITY_REALTIME: - *outHandleClass = NKikimrBlobStorage::FastRead; - break; - case TypeWithPriority::PRIORITY_BACKGROUND: - *outHandleClass = NKikimrBlobStorage::AsyncRead; - break; - } - } else { - switch (priority) { - case TypeWithPriority::REALTIME: - *outHandleClass = NKikimrBlobStorage::FastRead; - break; - case TypeWithPriority::BACKGROUND: - *outHandleClass = NKikimrBlobStorage::AsyncRead; - break; - } - } -} - -template <typename TypeWithPriority, bool WithOverrun = false, ui64 SpecificReadResultSizeEstimation=ReadResultSizeEstimation> -bool PrepareOneRead(const TString &key, TIndexRecord &indexRecord, ui64 offset, ui64 size, ui8 priority, - ui64 cmdLimitBytes, THolder<TIntermediate> &intermediate, TIntermediate::TRead &response, bool &outIsInlineOnly) -{ - for (ui64 idx = 0; idx < indexRecord.Chain.size(); ++idx) { - if (!indexRecord.Chain[idx].IsInline()) { - outIsInlineOnly = false; - break; - } - } - - if (!size) { - size = std::numeric_limits<decltype(size)>::max(); - } - ui64 fullValueSize = indexRecord.GetFullValueSize(); - offset = std::min(offset, fullValueSize); - size = std::min(size, fullValueSize - offset); - ui64 metaDataSize = key.size() + SpecificReadResultSizeEstimation; - ui64 recSize = std::max(size, ErrorMessageSizeEstimation) + metaDataSize; - - response.RequestedSize = size; - bool isOverRun = false; - - if (intermediate->IsTruncated - || intermediate->TotalSize + recSize > intermediate->TotalSizeLimit - || (cmdLimitBytes && intermediate->TotalSize + recSize > cmdLimitBytes)) { - response.Status = NKikimrProto::OVERRUN; - if (!WithOverrun - || std::min(intermediate->TotalSizeLimit, cmdLimitBytes) < intermediate->TotalSize + metaDataSize) - { - return true; - } - if (cmdLimitBytes) { - size = std::min(intermediate->TotalSizeLimit, cmdLimitBytes) - intermediate->TotalSize - metaDataSize; - } else { - size = intermediate->TotalSizeLimit - intermediate->TotalSize - metaDataSize; - } - isOverRun = true; - } - - response.ValueSize = size; - response.CreationUnixTime = indexRecord.CreationUnixTime; - response.Key = key; - - SetPriority<TypeWithPriority>(&response.HandleClass, priority); - - if (size) { - const ui32 numReads = indexRecord.GetReadItems(offset, size, response); - intermediate->TotalSize += recSize; - intermediate->TotalReadsScheduled += numReads; - } else if (response.Status != NKikimrProto::OVERRUN) { - response.Status = NKikimrProto::OK; - } - return isOverRun; -} - -template <typename TypeWithPriority, ui64 SpecificKeyValuePairSizeEstimation> -bool PrepareOneReadFromRangeReadWithoutData(const TString &key, TIndexRecord &indexRecord, ui8 priority, - THolder<TIntermediate> &intermediate, TIntermediate::TRangeRead &response, - ui64 &cmdSizeBytes, ui64 cmdLimitBytes, bool *outIsInlineOnly) -{ - if (intermediate->IsTruncated) { - return false; - } - - NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = - NKikimrClient::TKeyValueRequest::MAIN; - if (indexRecord.Chain.size()) { - if (indexRecord.Chain[0].IsInline()) { - storageChannel = NKikimrClient::TKeyValueRequest::INLINE; - } else { - *outIsInlineOnly = false; - ui32 storageChannelIdx = indexRecord.Chain[0].LogoBlobId.Channel(); - ui32 storageChannelOffset = storageChannelIdx - BLOB_CHANNEL; - storageChannel = (NKikimrClient::TKeyValueRequest::EStorageChannel)storageChannelOffset; - } - } - - ui64 metadataSize = key.size() + SpecificKeyValuePairSizeEstimation; - if (intermediate->TotalSize + metadataSize > intermediate->TotalSizeLimit - || cmdSizeBytes + metadataSize > cmdLimitBytes) { - STLOG(NLog::PRI_TRACE, NKikimrServices::KEYVALUE, KV330, "Went beyond limits", - (intermediate->TotalSize + metadataSize, intermediate->TotalSize + metadataSize), - (intermediate->TotalSizeLimit, intermediate->TotalSizeLimit), - (cmdSizeBytes + metadataSize, cmdSizeBytes + metadataSize), - (cmdLimitBytes, cmdLimitBytes)); - return true; - } - response.Reads.emplace_back(key, indexRecord.GetFullValueSize(), indexRecord.CreationUnixTime, - storageChannel); - intermediate->TotalSize += metadataSize; - SetPriority<TypeWithPriority>(&response.HandleClass, priority); - - cmdSizeBytes += metadataSize; - return false; -} - -struct TSeqInfo { - ui32 Reads = 0; - ui32 RunLen = 0; - ui32 Generation = 0; - ui32 Step = 0; - ui32 Cookie = 0; -}; - -template <typename TypeWithPriority, ui64 SpecificKeyValuePairSizeEstimation> -bool PrepareOneReadFromRangeReadWithData(const TString &key, TIndexRecord &indexRecord, ui8 priority, - THolder<TIntermediate> &intermediate, TIntermediate::TRangeRead &response, - ui64 &cmdSizeBytes, ui64 cmdLimitBytes, TSeqInfo &seq, bool *outIsInlineOnly) -{ - if (intermediate->IsTruncated) { - return false; - } - - NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = - NKikimrClient::TKeyValueRequest::MAIN; - if (indexRecord.Chain.size()) { - if (indexRecord.Chain[0].IsInline()) { - storageChannel = NKikimrClient::TKeyValueRequest::INLINE; - } else { - *outIsInlineOnly = false; - ui32 storageChannelIdx = indexRecord.Chain[0].LogoBlobId.Channel(); - ui32 storageChannelOffset = storageChannelIdx - BLOB_CHANNEL; - storageChannel = (NKikimrClient::TKeyValueRequest::EStorageChannel)storageChannelOffset; - } - } - - bool isSeq = false; - bool isInline = false; - if (indexRecord.Chain.size() == 1) { - if (indexRecord.Chain.front().IsInline()) { - isSeq = true; - isInline = true; - } else { - const TLogoBlobID& id = indexRecord.Chain.front().LogoBlobId; - isSeq = id.Generation() == seq.Generation - && id.Step() == seq.Step - && id.Cookie() == seq.Cookie; - seq.Generation = id.Generation(); - seq.Step = id.Step(); - seq.Cookie = id.Cookie() + 1; - } - } - if (isSeq) { - seq.Reads++; - if (seq.Reads > intermediate->SequentialReadLimit && !isInline) { - isSeq = false; - } else { - ++seq.RunLen; - } - } - if (!isSeq) { - seq.RunLen = 1; - } - - ui64 valueSize = indexRecord.GetFullValueSize(); - ui64 metadataSize = key.size() + SpecificKeyValuePairSizeEstimation; - if (intermediate->TotalSize + valueSize + metadataSize > intermediate->TotalSizeLimit - || cmdSizeBytes + valueSize + metadataSize > cmdLimitBytes - || (seq.RunLen == 1 && intermediate->TotalReadsScheduled >= intermediate->TotalReadsLimit)) { - return true; - } - - TIntermediate::TRead read(key, valueSize, indexRecord.CreationUnixTime, storageChannel); - const ui32 numReads = indexRecord.GetReadItems(0, valueSize, read); - SetPriority<TypeWithPriority>(&response.HandleClass, priority); - SetPriority<TypeWithPriority>(&read.HandleClass, priority); - - response.Reads.push_back(std::move(read)); - - intermediate->TotalSize += valueSize + metadataSize; - intermediate->TotalReadsScheduled += numReads; - - cmdSizeBytes += valueSize + metadataSize; - return false; -} - + +template <typename TypeWithPriority> +void SetPriority(NKikimrBlobStorage::EGetHandleClass *outHandleClass, ui8 priority) { + *outHandleClass = NKikimrBlobStorage::FastRead; + if constexpr (std::is_same_v<TypeWithPriority, NKikimrKeyValue::Priorities>) { + switch (priority) { + case TypeWithPriority::PRIORITY_UNSPECIFIED: + case TypeWithPriority::PRIORITY_REALTIME: + *outHandleClass = NKikimrBlobStorage::FastRead; + break; + case TypeWithPriority::PRIORITY_BACKGROUND: + *outHandleClass = NKikimrBlobStorage::AsyncRead; + break; + } + } else { + switch (priority) { + case TypeWithPriority::REALTIME: + *outHandleClass = NKikimrBlobStorage::FastRead; + break; + case TypeWithPriority::BACKGROUND: + *outHandleClass = NKikimrBlobStorage::AsyncRead; + break; + } + } +} + +template <typename TypeWithPriority, bool WithOverrun = false, ui64 SpecificReadResultSizeEstimation=ReadResultSizeEstimation> +bool PrepareOneRead(const TString &key, TIndexRecord &indexRecord, ui64 offset, ui64 size, ui8 priority, + ui64 cmdLimitBytes, THolder<TIntermediate> &intermediate, TIntermediate::TRead &response, bool &outIsInlineOnly) +{ + for (ui64 idx = 0; idx < indexRecord.Chain.size(); ++idx) { + if (!indexRecord.Chain[idx].IsInline()) { + outIsInlineOnly = false; + break; + } + } + + if (!size) { + size = std::numeric_limits<decltype(size)>::max(); + } + ui64 fullValueSize = indexRecord.GetFullValueSize(); + offset = std::min(offset, fullValueSize); + size = std::min(size, fullValueSize - offset); + ui64 metaDataSize = key.size() + SpecificReadResultSizeEstimation; + ui64 recSize = std::max(size, ErrorMessageSizeEstimation) + metaDataSize; + + response.RequestedSize = size; + bool isOverRun = false; + + if (intermediate->IsTruncated + || intermediate->TotalSize + recSize > intermediate->TotalSizeLimit + || (cmdLimitBytes && intermediate->TotalSize + recSize > cmdLimitBytes)) { + response.Status = NKikimrProto::OVERRUN; + if (!WithOverrun + || std::min(intermediate->TotalSizeLimit, cmdLimitBytes) < intermediate->TotalSize + metaDataSize) + { + return true; + } + if (cmdLimitBytes) { + size = std::min(intermediate->TotalSizeLimit, cmdLimitBytes) - intermediate->TotalSize - metaDataSize; + } else { + size = intermediate->TotalSizeLimit - intermediate->TotalSize - metaDataSize; + } + isOverRun = true; + } + + response.ValueSize = size; + response.CreationUnixTime = indexRecord.CreationUnixTime; + response.Key = key; + + SetPriority<TypeWithPriority>(&response.HandleClass, priority); + + if (size) { + const ui32 numReads = indexRecord.GetReadItems(offset, size, response); + intermediate->TotalSize += recSize; + intermediate->TotalReadsScheduled += numReads; + } else if (response.Status != NKikimrProto::OVERRUN) { + response.Status = NKikimrProto::OK; + } + return isOverRun; +} + +template <typename TypeWithPriority, ui64 SpecificKeyValuePairSizeEstimation> +bool PrepareOneReadFromRangeReadWithoutData(const TString &key, TIndexRecord &indexRecord, ui8 priority, + THolder<TIntermediate> &intermediate, TIntermediate::TRangeRead &response, + ui64 &cmdSizeBytes, ui64 cmdLimitBytes, bool *outIsInlineOnly) +{ + if (intermediate->IsTruncated) { + return false; + } + + NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = + NKikimrClient::TKeyValueRequest::MAIN; + if (indexRecord.Chain.size()) { + if (indexRecord.Chain[0].IsInline()) { + storageChannel = NKikimrClient::TKeyValueRequest::INLINE; + } else { + *outIsInlineOnly = false; + ui32 storageChannelIdx = indexRecord.Chain[0].LogoBlobId.Channel(); + ui32 storageChannelOffset = storageChannelIdx - BLOB_CHANNEL; + storageChannel = (NKikimrClient::TKeyValueRequest::EStorageChannel)storageChannelOffset; + } + } + + ui64 metadataSize = key.size() + SpecificKeyValuePairSizeEstimation; + if (intermediate->TotalSize + metadataSize > intermediate->TotalSizeLimit + || cmdSizeBytes + metadataSize > cmdLimitBytes) { + STLOG(NLog::PRI_TRACE, NKikimrServices::KEYVALUE, KV330, "Went beyond limits", + (intermediate->TotalSize + metadataSize, intermediate->TotalSize + metadataSize), + (intermediate->TotalSizeLimit, intermediate->TotalSizeLimit), + (cmdSizeBytes + metadataSize, cmdSizeBytes + metadataSize), + (cmdLimitBytes, cmdLimitBytes)); + return true; + } + response.Reads.emplace_back(key, indexRecord.GetFullValueSize(), indexRecord.CreationUnixTime, + storageChannel); + intermediate->TotalSize += metadataSize; + SetPriority<TypeWithPriority>(&response.HandleClass, priority); + + cmdSizeBytes += metadataSize; + return false; +} + +struct TSeqInfo { + ui32 Reads = 0; + ui32 RunLen = 0; + ui32 Generation = 0; + ui32 Step = 0; + ui32 Cookie = 0; +}; + +template <typename TypeWithPriority, ui64 SpecificKeyValuePairSizeEstimation> +bool PrepareOneReadFromRangeReadWithData(const TString &key, TIndexRecord &indexRecord, ui8 priority, + THolder<TIntermediate> &intermediate, TIntermediate::TRangeRead &response, + ui64 &cmdSizeBytes, ui64 cmdLimitBytes, TSeqInfo &seq, bool *outIsInlineOnly) +{ + if (intermediate->IsTruncated) { + return false; + } + + NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = + NKikimrClient::TKeyValueRequest::MAIN; + if (indexRecord.Chain.size()) { + if (indexRecord.Chain[0].IsInline()) { + storageChannel = NKikimrClient::TKeyValueRequest::INLINE; + } else { + *outIsInlineOnly = false; + ui32 storageChannelIdx = indexRecord.Chain[0].LogoBlobId.Channel(); + ui32 storageChannelOffset = storageChannelIdx - BLOB_CHANNEL; + storageChannel = (NKikimrClient::TKeyValueRequest::EStorageChannel)storageChannelOffset; + } + } + + bool isSeq = false; + bool isInline = false; + if (indexRecord.Chain.size() == 1) { + if (indexRecord.Chain.front().IsInline()) { + isSeq = true; + isInline = true; + } else { + const TLogoBlobID& id = indexRecord.Chain.front().LogoBlobId; + isSeq = id.Generation() == seq.Generation + && id.Step() == seq.Step + && id.Cookie() == seq.Cookie; + seq.Generation = id.Generation(); + seq.Step = id.Step(); + seq.Cookie = id.Cookie() + 1; + } + } + if (isSeq) { + seq.Reads++; + if (seq.Reads > intermediate->SequentialReadLimit && !isInline) { + isSeq = false; + } else { + ++seq.RunLen; + } + } + if (!isSeq) { + seq.RunLen = 1; + } + + ui64 valueSize = indexRecord.GetFullValueSize(); + ui64 metadataSize = key.size() + SpecificKeyValuePairSizeEstimation; + if (intermediate->TotalSize + valueSize + metadataSize > intermediate->TotalSizeLimit + || cmdSizeBytes + valueSize + metadataSize > cmdLimitBytes + || (seq.RunLen == 1 && intermediate->TotalReadsScheduled >= intermediate->TotalReadsLimit)) { + return true; + } + + TIntermediate::TRead read(key, valueSize, indexRecord.CreationUnixTime, storageChannel); + const ui32 numReads = indexRecord.GetReadItems(0, valueSize, read); + SetPriority<TypeWithPriority>(&response.HandleClass, priority); + SetPriority<TypeWithPriority>(&read.HandleClass, priority); + + response.Reads.push_back(std::move(read)); + + intermediate->TotalSize += valueSize + metadataSize; + intermediate->TotalReadsScheduled += numReads; + + cmdSizeBytes += valueSize + metadataSize; + return false; +} + bool TKeyValueState::PrepareCmdRead(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, bool &outIsInlineOnly) { outIsInlineOnly = true; @@ -1971,21 +1971,21 @@ bool TKeyValueState::PrepareCmdRead(const TActorContext &ctx, NKikimrClient::TKe return true; } - ui64 offset = request.HasOffset() ? request.GetOffset() : 0; - ui64 size = request.HasSize() ? request.GetSize() : 0; - NKikimrClient::TKeyValueRequest::EPriority priority = NKikimrClient::TKeyValueRequest::REALTIME; - if (request.HasPriority()) { - priority = request.GetPriority(); - } - + ui64 offset = request.HasOffset() ? request.GetOffset() : 0; + ui64 size = request.HasSize() ? request.GetSize() : 0; + NKikimrClient::TKeyValueRequest::EPriority priority = NKikimrClient::TKeyValueRequest::REALTIME; + if (request.HasPriority()) { + priority = request.GetPriority(); + } + auto it = Index.find(request.GetKey()); if (it == Index.end()) { response.Status = NKikimrProto::NODATA; response.Message = "No such key Marker# KV48"; } else { - bool isOverrun = PrepareOneRead<NKikimrClient::TKeyValueRequest>(it->first, it->second, offset, size, - priority, 0, intermediate, response, outIsInlineOnly); - if (isOverrun) { + bool isOverrun = PrepareOneRead<NKikimrClient::TKeyValueRequest>(it->first, it->second, offset, size, + priority, 0, intermediate, response, outIsInlineOnly); + if (isOverrun) { if (!intermediate->IsTruncated) { CountOverrun(); intermediate->IsTruncated = true; @@ -1996,56 +1996,56 @@ bool TKeyValueState::PrepareCmdRead(const TActorContext &ctx, NKikimrClient::TKe return false; } -template <typename TypeWithPriority, bool CheckUTF8 = false, - ui64 MetaDataSizeWithData = KeyValuePairSizeEstimation, - ui64 MetaDataSizeWithoutData = KeyValuePairSizeEstimation> -void ProcessOneCmdReadRange(TKeyValueState *self, const TKeyRange &range, ui64 cmdLimitBytes, bool includeData, - ui8 priority, TIntermediate::TRangeRead &response, THolder<TIntermediate> &intermediate, bool *outIsInlineOnly) -{ - ui64 cmdSizeBytes = 0; - TSeqInfo seq; - seq.RunLen = 1; - - self->TraverseRange(range, [&](TKeyValueState::TIndex::iterator it) { - if (intermediate->IsTruncated) { - return; - } - - auto &[key, indexRecord] = *it; - - if (CheckUTF8 && !IsUtf(key)) { - TIntermediate::TRead read; - read.CreationUnixTime = indexRecord.CreationUnixTime; - EscapeC(key, read.Key); - read.Status = NKikimrProto::ERROR; - read.Message = "Key isn't UTF8"; - response.Reads.push_back(std::move(read)); - return; - } - - bool isOverRun = false; - if (includeData) { - isOverRun = PrepareOneReadFromRangeReadWithData<NKikimrClient::TKeyValueRequest, MetaDataSizeWithData>( - key, indexRecord, priority, intermediate, response, - cmdSizeBytes, cmdLimitBytes, seq, outIsInlineOnly); - } else { - isOverRun = PrepareOneReadFromRangeReadWithoutData<NKikimrClient::TKeyValueRequest, MetaDataSizeWithoutData>( - key, indexRecord, priority, intermediate, response, cmdSizeBytes, - cmdLimitBytes, outIsInlineOnly); - } - if (isOverRun) { - self->CountOverrun(); - intermediate->IsTruncated = true; - } - }); - - if (intermediate->IsTruncated) { - response.Status = NKikimrProto::OVERRUN; - } else if (response.Reads.size() == 0) { - response.Status = NKikimrProto::NODATA; - } -} - +template <typename TypeWithPriority, bool CheckUTF8 = false, + ui64 MetaDataSizeWithData = KeyValuePairSizeEstimation, + ui64 MetaDataSizeWithoutData = KeyValuePairSizeEstimation> +void ProcessOneCmdReadRange(TKeyValueState *self, const TKeyRange &range, ui64 cmdLimitBytes, bool includeData, + ui8 priority, TIntermediate::TRangeRead &response, THolder<TIntermediate> &intermediate, bool *outIsInlineOnly) +{ + ui64 cmdSizeBytes = 0; + TSeqInfo seq; + seq.RunLen = 1; + + self->TraverseRange(range, [&](TKeyValueState::TIndex::iterator it) { + if (intermediate->IsTruncated) { + return; + } + + auto &[key, indexRecord] = *it; + + if (CheckUTF8 && !IsUtf(key)) { + TIntermediate::TRead read; + read.CreationUnixTime = indexRecord.CreationUnixTime; + EscapeC(key, read.Key); + read.Status = NKikimrProto::ERROR; + read.Message = "Key isn't UTF8"; + response.Reads.push_back(std::move(read)); + return; + } + + bool isOverRun = false; + if (includeData) { + isOverRun = PrepareOneReadFromRangeReadWithData<NKikimrClient::TKeyValueRequest, MetaDataSizeWithData>( + key, indexRecord, priority, intermediate, response, + cmdSizeBytes, cmdLimitBytes, seq, outIsInlineOnly); + } else { + isOverRun = PrepareOneReadFromRangeReadWithoutData<NKikimrClient::TKeyValueRequest, MetaDataSizeWithoutData>( + key, indexRecord, priority, intermediate, response, cmdSizeBytes, + cmdLimitBytes, outIsInlineOnly); + } + if (isOverRun) { + self->CountOverrun(); + intermediate->IsTruncated = true; + } + }); + + if (intermediate->IsTruncated) { + response.Status = NKikimrProto::OVERRUN; + } else if (response.Reads.size() == 0) { + response.Status = NKikimrProto::NODATA; + } +} + bool TKeyValueState::PrepareCmdReadRange(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, bool &inOutIsInlineOnly) { intermediate->RangeReads.resize(kvRequest.CmdReadRangeSize()); @@ -2068,12 +2068,12 @@ bool TKeyValueState::PrepareCmdReadRange(const TActorContext &ctx, NKikimrClient ui64 cmdLimitBytes = request.HasLimitBytes() ? request.GetLimitBytes() : Max<ui64>(); bool includeData = request.HasIncludeData() && request.GetIncludeData(); - ui8 priority = request.HasPriority() ? request.GetPriority() : Max<ui8>(); + ui8 priority = request.HasPriority() ? request.GetPriority() : Max<ui8>(); response.IncludeData = includeData; - response.LimitBytes = cmdLimitBytes; + response.LimitBytes = cmdLimitBytes; - ProcessOneCmdReadRange<NKikimrClient::TKeyValueRequest>(this, range, cmdLimitBytes, includeData, priority, - response, intermediate, &inOutIsInlineOnly); + ProcessOneCmdReadRange<NKikimrClient::TKeyValueRequest>(this, range, cmdLimitBytes, includeData, priority, + response, intermediate, &inOutIsInlineOnly); } return false; } @@ -2082,8 +2082,8 @@ bool TKeyValueState::PrepareCmdRename(const TActorContext &ctx, NKikimrClient::T THolder<TIntermediate> &intermediate) { for (ui32 i = 0; i < kvRequest.CmdRenameSize(); ++i) { auto& request = kvRequest.GetCmdRename(i); - intermediate->Commands.emplace_back(TIntermediate::TRename()); - auto& interm = std::get<TIntermediate::TRename>(intermediate->Commands.back()); + intermediate->Commands.emplace_back(TIntermediate::TRename()); + auto& interm = std::get<TIntermediate::TRename>(intermediate->Commands.back()); if (!request.HasOldKey()) { TStringStream str; @@ -2111,8 +2111,8 @@ bool TKeyValueState::PrepareCmdDelete(const TActorContext &ctx, NKikimrClient::T ui64 nToDelete = 0; for (ui32 i = 0; i < kvRequest.CmdDeleteRangeSize(); ++i) { auto& request = kvRequest.GetCmdDeleteRange(i); - intermediate->Commands.emplace_back(TIntermediate::TDelete()); - auto& interm = std::get<TIntermediate::TDelete>(intermediate->Commands.back()); + intermediate->Commands.emplace_back(TIntermediate::TDelete()); + auto& interm = std::get<TIntermediate::TDelete>(intermediate->Commands.back()); if (!request.HasRange()) { TStringStream str; @@ -2141,39 +2141,39 @@ bool TKeyValueState::PrepareCmdDelete(const TActorContext &ctx, NKikimrClient::T return false; } -void TKeyValueState::SplitIntoBlobs(TIntermediate::TWrite &cmd, bool isInline, ui32 storageChannelIdx) { - if (isInline) { - cmd.Status = NKikimrProto::SCHEDULED; - cmd.StatusFlags = TStorageStatusFlags(ui32(NKikimrBlobStorage::StatusIsValid)); - if (GetIsTabletYellowMove()) { - cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)); - } - if (GetIsTabletYellowStop()) { - cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceYellowStop)); - } - } else { - cmd.Status = NKikimrProto::UNKNOWN; - ui64 sizeRemain = cmd.Data.size(); - while (sizeRemain) { - ui32 blobSize = Min<ui64>(sizeRemain, 8 << 20); - cmd.LogoBlobIds.push_back(AllocateLogoBlobId(blobSize, storageChannelIdx)); - sizeRemain -= blobSize; - } - for (const TLogoBlobID& logoBlobId : cmd.LogoBlobIds) { - ui32 newRefCount = ++RefCounts[logoBlobId]; - Y_VERIFY(newRefCount == 1); - } - } -} - +void TKeyValueState::SplitIntoBlobs(TIntermediate::TWrite &cmd, bool isInline, ui32 storageChannelIdx) { + if (isInline) { + cmd.Status = NKikimrProto::SCHEDULED; + cmd.StatusFlags = TStorageStatusFlags(ui32(NKikimrBlobStorage::StatusIsValid)); + if (GetIsTabletYellowMove()) { + cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)); + } + if (GetIsTabletYellowStop()) { + cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceYellowStop)); + } + } else { + cmd.Status = NKikimrProto::UNKNOWN; + ui64 sizeRemain = cmd.Data.size(); + while (sizeRemain) { + ui32 blobSize = Min<ui64>(sizeRemain, 8 << 20); + cmd.LogoBlobIds.push_back(AllocateLogoBlobId(blobSize, storageChannelIdx)); + sizeRemain -= blobSize; + } + for (const TLogoBlobID& logoBlobId : cmd.LogoBlobIds) { + ui32 newRefCount = ++RefCounts[logoBlobId]; + Y_VERIFY(newRefCount == 1); + } + } +} + bool TKeyValueState::PrepareCmdWrite(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info) { - intermediate->WriteIndices.reserve(kvRequest.CmdWriteSize()); + intermediate->WriteIndices.reserve(kvRequest.CmdWriteSize()); for (ui32 i = 0; i < kvRequest.CmdWriteSize(); ++i) { auto& request = kvRequest.GetCmdWrite(i); - intermediate->WriteIndices.push_back(intermediate->Commands.size()); - auto& cmd = intermediate->Commands.emplace_back(TIntermediate::TWrite()); - auto& interm = std::get<TIntermediate::TWrite>(cmd); + intermediate->WriteIndices.push_back(intermediate->Commands.size()); + auto& cmd = intermediate->Commands.emplace_back(TIntermediate::TWrite()); + auto& interm = std::get<TIntermediate::TWrite>(cmd); if (!request.HasKey()) { TStringStream str; @@ -2249,45 +2249,45 @@ bool TKeyValueState::PrepareCmdWrite(const TActorContext &ctx, NKikimrClient::TK break; } } - SplitIntoBlobs(interm, isInline, storageChannelIdx); + SplitIntoBlobs(interm, isInline, storageChannelIdx); } return false; } - -TKeyValueState::TPrepareResult TKeyValueState::InitGetStatusCommand(TIntermediate::TGetStatus &cmd, - NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel, const TTabletStorageInfo *info) -{ - TString msg; - if (storageChannel == NKikimrClient::TKeyValueRequest::INLINE) { - cmd.StorageChannel = storageChannel; - cmd.LogoBlobId = TLogoBlobID(); - cmd.Status = NKikimrProto::OK; - cmd.StatusFlags = TStorageStatusFlags(ui32(NKikimrBlobStorage::StatusIsValid)); - if (GetIsTabletYellowMove()) { - cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)); - } - if (GetIsTabletYellowStop()) { - cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceYellowStop)); - } - } else { - ui32 storageChannelOffset = (ui32)storageChannel; - ui32 storageChannelIdx = storageChannelOffset + BLOB_CHANNEL; - ui32 endChannel = info->Channels.size(); - if (storageChannelIdx >= endChannel) { - storageChannelIdx = BLOB_CHANNEL; - msg = TStringBuilder() << "KeyValue# " << TabletId - << " CmdGetStatus StorageChannel# " << storageChannelOffset - << " does not exist, using MAIN"; - } - - cmd.StorageChannel = storageChannel; - cmd.LogoBlobId = AllocateLogoBlobId(1, storageChannelIdx); - cmd.Status = NKikimrProto::UNKNOWN; - } - return {false, msg}; -} - + +TKeyValueState::TPrepareResult TKeyValueState::InitGetStatusCommand(TIntermediate::TGetStatus &cmd, + NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel, const TTabletStorageInfo *info) +{ + TString msg; + if (storageChannel == NKikimrClient::TKeyValueRequest::INLINE) { + cmd.StorageChannel = storageChannel; + cmd.LogoBlobId = TLogoBlobID(); + cmd.Status = NKikimrProto::OK; + cmd.StatusFlags = TStorageStatusFlags(ui32(NKikimrBlobStorage::StatusIsValid)); + if (GetIsTabletYellowMove()) { + cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)); + } + if (GetIsTabletYellowStop()) { + cmd.StatusFlags.Merge(ui32(NKikimrBlobStorage::StatusDiskSpaceYellowStop)); + } + } else { + ui32 storageChannelOffset = (ui32)storageChannel; + ui32 storageChannelIdx = storageChannelOffset + BLOB_CHANNEL; + ui32 endChannel = info->Channels.size(); + if (storageChannelIdx >= endChannel) { + storageChannelIdx = BLOB_CHANNEL; + msg = TStringBuilder() << "KeyValue# " << TabletId + << " CmdGetStatus StorageChannel# " << storageChannelOffset + << " does not exist, using MAIN"; + } + + cmd.StorageChannel = storageChannel; + cmd.LogoBlobId = AllocateLogoBlobId(1, storageChannelIdx); + cmd.Status = NKikimrProto::UNKNOWN; + } + return {false, msg}; +} + bool TKeyValueState::PrepareCmdGetStatus(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info) { intermediate->GetStatuses.resize(kvRequest.CmdGetStatusSize()); @@ -2301,9 +2301,9 @@ bool TKeyValueState::PrepareCmdGetStatus(const TActorContext &ctx, NKikimrClient if (request.HasStorageChannel()) { storageChannel = request.GetStorageChannel(); } - TPrepareResult result = InitGetStatusCommand(interm, storageChannel, info); - if (result.ErrorMsg && !result.WithError) { - LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, result.ErrorMsg << " Marker# KV76"); + TPrepareResult result = InitGetStatusCommand(interm, storageChannel, info); + if (result.ErrorMsg && !result.WithError) { + LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, result.ErrorMsg << " Marker# KV76"); } } return false; @@ -2314,8 +2314,8 @@ bool TKeyValueState::PrepareCmdCopyRange(const TActorContext& ctx, NKikimrClient THolder<TIntermediate>& intermediate) { for (ui32 i = 0; i < kvRequest.CmdCopyRangeSize(); ++i) { auto& request = kvRequest.GetCmdCopyRange(i); - intermediate->Commands.emplace_back(TIntermediate::TCopyRange()); - auto& interm = std::get<TIntermediate::TCopyRange>(intermediate->Commands.back()); + intermediate->Commands.emplace_back(TIntermediate::TCopyRange()); + auto& interm = std::get<TIntermediate::TCopyRange>(intermediate->Commands.back()); if ((!request.HasPrefixToAdd() || request.GetPrefixToAdd().empty()) && (!request.HasPrefixToRemove() || request.GetPrefixToRemove().empty())) { @@ -2340,8 +2340,8 @@ bool TKeyValueState::PrepareCmdConcat(const TActorContext& ctx, NKikimrClient::T THolder<TIntermediate>& intermediate) { for (ui32 i = 0; i < kvRequest.CmdConcatSize(); ++i) { auto& request = kvRequest.GetCmdConcat(i); - intermediate->Commands.emplace_back(TIntermediate::TConcat()); - auto& interm = std::get<TIntermediate::TConcat>(intermediate->Commands.back()); + intermediate->Commands.emplace_back(TIntermediate::TConcat()); + auto& interm = std::get<TIntermediate::TConcat>(intermediate->Commands.back()); if (!request.HasOutputKey()) { TStringStream str; @@ -2389,150 +2389,150 @@ bool TKeyValueState::PrepareCmdSetExecutorFastLogPolicy(const TActorContext & /* return false; } -using TPrepareResult = TKeyValueState::TPrepareResult; - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Rename &request, THolder<TIntermediate> &intermediate) { - intermediate->Commands.emplace_back(TIntermediate::TRename()); - auto &cmd = std::get<TIntermediate::TRename>(intermediate->Commands.back()); - cmd.OldKey = request.old_key(); - cmd.NewKey = request.new_key(); - return {}; -} - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Concat &request, THolder<TIntermediate> &intermediate) { - intermediate->Commands.emplace_back(TIntermediate::TConcat()); - auto &cmd = std::get<TIntermediate::TConcat>(intermediate->Commands.back()); - auto inputKeys = request.input_keys(); - cmd.InputKeys.insert(cmd.InputKeys.end(), inputKeys.begin(), inputKeys.end()); - - cmd.OutputKey = request.output_key(); - cmd.KeepInputs = request.keep_inputs(); - return {}; -} - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::CopyRange &request, THolder<TIntermediate> &intermediate) { - intermediate->Commands.emplace_back(TIntermediate::TCopyRange()); - auto &cmd = std::get<TIntermediate::TCopyRange>(intermediate->Commands.back()); - auto convResult = ConvertRange(request.range(), &cmd.Range, "CopyRange"); - if (convResult.WithError) { - return {true, convResult.ErrorMsg}; - } - cmd.PrefixToAdd = request.prefix_to_add(); - cmd.PrefixToRemove = request.prefix_to_remove(); - return {}; -} - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Write &request, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info) -{ - intermediate->Commands.emplace_back(TIntermediate::TWrite()); - auto &cmd = std::get<TIntermediate::TWrite>(intermediate->Commands.back()); - cmd.Key = request.key(); - cmd.Data = request.value(); - switch (request.tactic()) { - case TCommand::Write::TACTIC_MIN_LATENCY: - cmd.Tactic = TEvBlobStorage::TEvPut::TacticMinLatency; - break; - case TCommand::Write::TACTIC_MAX_THROUGHPUT: - cmd.Tactic = TEvBlobStorage::TEvPut::TacticMaxThroughput; - break; - default: - cmd.Tactic = TEvBlobStorage::TEvPut::TacticDefault; - break; - } - - cmd.HandleClass = NKikimrBlobStorage::UserData; - if (request.priority() == NKikimrKeyValue::Priorities::PRIORITY_BACKGROUND) { - cmd.HandleClass = NKikimrBlobStorage::AsyncBlob; - } - - bool isInline = false; - ui32 storageChannelIdx = BLOB_CHANNEL; - ui32 storageChannel = request.storage_channel(); - if (!storageChannel) { - storageChannel = MainStorageChannelInPublicApi; - } - ui32 storageChannelOffset = storageChannel - MainStorageChannelInPublicApi; - - if (storageChannel == InlineStorageChannelInPublicApi) { - isInline = true; - } else { - storageChannelIdx = storageChannelOffset + BLOB_CHANNEL; - ui32 endChannel = info->Channels.size(); - if (storageChannelIdx >= endChannel) { - storageChannelIdx = BLOB_CHANNEL; - } - } - SplitIntoBlobs(cmd, isInline, storageChannelIdx); - return {}; -} - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::DeleteRange &request, THolder<TIntermediate> &intermediate, - const TActorContext &ctx) -{ - intermediate->Commands.emplace_back(TIntermediate::TDelete()); - auto &cmd = std::get<TIntermediate::TDelete>(intermediate->Commands.back()); - auto convResult = ConvertRange(request.range(), &cmd.Range, "DeleteRange"); - if (convResult.WithError) { - return {true, convResult.ErrorMsg}; - } - ui32 nToDelete = 0; - TraverseRange(cmd.Range, [&](TIndex::iterator it) { - Y_UNUSED(it); - nToDelete++; - }); - // The use of >, not >= is important here. - if (nToDelete > DeletesPerRequestLimit && !AppData(ctx)->AllowHugeKeyValueDeletes) { - TStringBuilder str; - str << "KeyValue# " << TabletId; - str << " Can't delete Range, in DeleteRange, total limit of deletions per request (" - << DeletesPerRequestLimit << ") reached, Marker# KV90"; - TString msg = str; - ReplyError<TEvKeyValue::TEvExecuteTransactionResponse>(ctx, msg, - NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, intermediate, nullptr); - return {true, msg}; - } - return {}; -} - -TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand &request, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info, const TActorContext &ctx) -{ - switch (request.action_case()) { - case NKikimrKeyValue::ExecuteTransactionRequest::Command::ACTION_NOT_SET: - return {true, "Command not specified Marker# KV68"}; - case NKikimrKeyValue::ExecuteTransactionRequest::Command::kDeleteRange: - return PrepareOneCmd(request.delete_range(), intermediate, ctx); - case NKikimrKeyValue::ExecuteTransactionRequest::Command::kRename: - return PrepareOneCmd(request.rename(), intermediate); - case NKikimrKeyValue::ExecuteTransactionRequest::Command::kCopyRange: - return PrepareOneCmd(request.copy_range(), intermediate); - case NKikimrKeyValue::ExecuteTransactionRequest::Command::kConcat: - return PrepareOneCmd(request.concat(), intermediate); - case NKikimrKeyValue::ExecuteTransactionRequest::Command::kWrite: - return PrepareOneCmd(request.write(), intermediate, info); - } -} - -TPrepareResult TKeyValueState::PrepareCommands(NKikimrKeyValue::ExecuteTransactionRequest &kvRequest, - THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info, const TActorContext &ctx) -{ - for (i32 idx = 0; idx < kvRequest.commands_size(); ++idx) { - auto &cmd = kvRequest.commands(idx); - TPrepareResult result = PrepareOneCmd(cmd, intermediate, info, ctx); - if (cmd.has_write()) { - intermediate->WriteIndices.push_back(idx); - } - if (result.WithError) { - return result; - } - } - if (intermediate->EvType == TEvKeyValue::TEvExecuteTransaction::EventType) { - intermediate->ExecuteTransactionResponse.set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); - } - return {}; -} - +using TPrepareResult = TKeyValueState::TPrepareResult; + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Rename &request, THolder<TIntermediate> &intermediate) { + intermediate->Commands.emplace_back(TIntermediate::TRename()); + auto &cmd = std::get<TIntermediate::TRename>(intermediate->Commands.back()); + cmd.OldKey = request.old_key(); + cmd.NewKey = request.new_key(); + return {}; +} + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Concat &request, THolder<TIntermediate> &intermediate) { + intermediate->Commands.emplace_back(TIntermediate::TConcat()); + auto &cmd = std::get<TIntermediate::TConcat>(intermediate->Commands.back()); + auto inputKeys = request.input_keys(); + cmd.InputKeys.insert(cmd.InputKeys.end(), inputKeys.begin(), inputKeys.end()); + + cmd.OutputKey = request.output_key(); + cmd.KeepInputs = request.keep_inputs(); + return {}; +} + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::CopyRange &request, THolder<TIntermediate> &intermediate) { + intermediate->Commands.emplace_back(TIntermediate::TCopyRange()); + auto &cmd = std::get<TIntermediate::TCopyRange>(intermediate->Commands.back()); + auto convResult = ConvertRange(request.range(), &cmd.Range, "CopyRange"); + if (convResult.WithError) { + return {true, convResult.ErrorMsg}; + } + cmd.PrefixToAdd = request.prefix_to_add(); + cmd.PrefixToRemove = request.prefix_to_remove(); + return {}; +} + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::Write &request, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info) +{ + intermediate->Commands.emplace_back(TIntermediate::TWrite()); + auto &cmd = std::get<TIntermediate::TWrite>(intermediate->Commands.back()); + cmd.Key = request.key(); + cmd.Data = request.value(); + switch (request.tactic()) { + case TCommand::Write::TACTIC_MIN_LATENCY: + cmd.Tactic = TEvBlobStorage::TEvPut::TacticMinLatency; + break; + case TCommand::Write::TACTIC_MAX_THROUGHPUT: + cmd.Tactic = TEvBlobStorage::TEvPut::TacticMaxThroughput; + break; + default: + cmd.Tactic = TEvBlobStorage::TEvPut::TacticDefault; + break; + } + + cmd.HandleClass = NKikimrBlobStorage::UserData; + if (request.priority() == NKikimrKeyValue::Priorities::PRIORITY_BACKGROUND) { + cmd.HandleClass = NKikimrBlobStorage::AsyncBlob; + } + + bool isInline = false; + ui32 storageChannelIdx = BLOB_CHANNEL; + ui32 storageChannel = request.storage_channel(); + if (!storageChannel) { + storageChannel = MainStorageChannelInPublicApi; + } + ui32 storageChannelOffset = storageChannel - MainStorageChannelInPublicApi; + + if (storageChannel == InlineStorageChannelInPublicApi) { + isInline = true; + } else { + storageChannelIdx = storageChannelOffset + BLOB_CHANNEL; + ui32 endChannel = info->Channels.size(); + if (storageChannelIdx >= endChannel) { + storageChannelIdx = BLOB_CHANNEL; + } + } + SplitIntoBlobs(cmd, isInline, storageChannelIdx); + return {}; +} + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand::DeleteRange &request, THolder<TIntermediate> &intermediate, + const TActorContext &ctx) +{ + intermediate->Commands.emplace_back(TIntermediate::TDelete()); + auto &cmd = std::get<TIntermediate::TDelete>(intermediate->Commands.back()); + auto convResult = ConvertRange(request.range(), &cmd.Range, "DeleteRange"); + if (convResult.WithError) { + return {true, convResult.ErrorMsg}; + } + ui32 nToDelete = 0; + TraverseRange(cmd.Range, [&](TIndex::iterator it) { + Y_UNUSED(it); + nToDelete++; + }); + // The use of >, not >= is important here. + if (nToDelete > DeletesPerRequestLimit && !AppData(ctx)->AllowHugeKeyValueDeletes) { + TStringBuilder str; + str << "KeyValue# " << TabletId; + str << " Can't delete Range, in DeleteRange, total limit of deletions per request (" + << DeletesPerRequestLimit << ") reached, Marker# KV90"; + TString msg = str; + ReplyError<TEvKeyValue::TEvExecuteTransactionResponse>(ctx, msg, + NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, intermediate, nullptr); + return {true, msg}; + } + return {}; +} + +TPrepareResult TKeyValueState::PrepareOneCmd(const TCommand &request, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info, const TActorContext &ctx) +{ + switch (request.action_case()) { + case NKikimrKeyValue::ExecuteTransactionRequest::Command::ACTION_NOT_SET: + return {true, "Command not specified Marker# KV68"}; + case NKikimrKeyValue::ExecuteTransactionRequest::Command::kDeleteRange: + return PrepareOneCmd(request.delete_range(), intermediate, ctx); + case NKikimrKeyValue::ExecuteTransactionRequest::Command::kRename: + return PrepareOneCmd(request.rename(), intermediate); + case NKikimrKeyValue::ExecuteTransactionRequest::Command::kCopyRange: + return PrepareOneCmd(request.copy_range(), intermediate); + case NKikimrKeyValue::ExecuteTransactionRequest::Command::kConcat: + return PrepareOneCmd(request.concat(), intermediate); + case NKikimrKeyValue::ExecuteTransactionRequest::Command::kWrite: + return PrepareOneCmd(request.write(), intermediate, info); + } +} + +TPrepareResult TKeyValueState::PrepareCommands(NKikimrKeyValue::ExecuteTransactionRequest &kvRequest, + THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info, const TActorContext &ctx) +{ + for (i32 idx = 0; idx < kvRequest.commands_size(); ++idx) { + auto &cmd = kvRequest.commands(idx); + TPrepareResult result = PrepareOneCmd(cmd, intermediate, info, ctx); + if (cmd.has_write()) { + intermediate->WriteIndices.push_back(idx); + } + if (result.WithError) { + return result; + } + } + if (intermediate->EvType == TEvKeyValue::TEvExecuteTransaction::EventType) { + intermediate->ExecuteTransactionResponse.set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); + } + return {}; +} + void TKeyValueState::ReplyError(const TActorContext &ctx, TString errorDescription, NMsgBusProxy::EResponseStatus status, THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info) { @@ -2559,423 +2559,423 @@ void TKeyValueState::ReplyError(const TActorContext &ctx, TString errorDescripti } } -bool TKeyValueState::PrepareReadRequest(const TActorContext &ctx, TEvKeyValue::TEvRead::TPtr &ev, - THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType) -{ - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareReadRequest Marker# KV53"); - - NKikimrKeyValue::ReadRequest &request = ev->Get()->Record; - StoredState.SetChannelGeneration(ExecutorGeneration); - StoredState.SetChannelStep(NextLogoBlobStep - 1); - - intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, - StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), TRequestType::ReadOnly)); - - intermediate->HasCookie = true; - intermediate->Cookie = request.cookie(); - - intermediate->RequestUid = NextRequestUid; - ++NextRequestUid; - RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvRead::EventType; - - bool isInlineOnly = true; - intermediate->ReadCommand = TIntermediate::TRead(); - auto &response = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); - response.Key = request.key(); - - if (CheckDeadline(ctx, ev->Get(), intermediate)) { - return false; - } - - if (CheckGeneration(ctx, ev->Get(), intermediate)) { - return false; - } - - auto it = Index.find(request.key()); - if (it == Index.end()) { - response.Status = NKikimrProto::NODATA; - response.Message = "No such key Marker# KV55"; - ReplyError<TEvKeyValue::TEvReadResponse>(ctx, response.Message, - NKikimrKeyValue::Statuses::RSTATUS_NO_DATA, intermediate); - return false; - } - bool isOverRun = PrepareOneRead<NKikimrKeyValue::Priorities, true, ReadResultSizeEstimationNewApi>( - it->first, it->second, request.offset(), request.size(), request.priority(), request.limit_bytes(), - intermediate, response, isInlineOnly); - - if (isInlineOnly) { - *outRequestType = TRequestType::ReadOnlyInline; - intermediate->Stat.RequestType = *outRequestType; - } - intermediate->IsTruncated = isOverRun; - return true; -} - -bool TKeyValueState::PrepareReadRangeRequest(const TActorContext &ctx, TEvKeyValue::TEvReadRange::TPtr &ev, - THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType) -{ - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareReadRangeRequest Marker# KV57"); - - NKikimrKeyValue::ReadRangeRequest &request = ev->Get()->Record; - StoredState.SetChannelGeneration(ExecutorGeneration); - StoredState.SetChannelStep(NextLogoBlobStep - 1); - - TRequestType::EType requestType = TRequestType::ReadOnly; - intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, - StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); - - intermediate->HasCookie = true; - intermediate->Cookie = request.cookie(); - - intermediate->RequestUid = NextRequestUid; - ++NextRequestUid; - RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvReadRange::EventType; - intermediate->TotalSize = ReadRangeRequestMetaDataSizeEstimation; - - intermediate->ReadCommand = TIntermediate::TRangeRead(); - auto &response = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); - - if (CheckDeadline(ctx, ev->Get(), intermediate)) { - response.Status = NKikimrProto::ERROR; - return false; - } - - if (CheckGeneration(ctx, ev->Get(), intermediate)) { - response.Status = NKikimrProto::ERROR; - return false; - } - - TKeyRange range; - auto convResult = ConvertRange(request.range(), &range, "ReadRange"); - if (convResult.WithError) { - response.Status = NKikimrProto::ERROR; - return false; - } - response.Status = NKikimrProto::OK; - - ui64 cmdLimitBytes = request.limit_bytes(); - if (!cmdLimitBytes) { - cmdLimitBytes = Max<ui64>(); - } - bool includeData = request.include_data(); - ui8 priority = request.priority(); - response.LimitBytes = cmdLimitBytes; - response.IncludeData = includeData; - - bool isInlineOnly = true; - auto processOneCmdReadRange = ProcessOneCmdReadRange<NKikimrKeyValue::Priorities, true, - KeyValuePairSizeEstimationNewApi, KeyInfoSizeEstimation>; - processOneCmdReadRange(this, range, cmdLimitBytes, includeData, priority, response, intermediate, &isInlineOnly); - - if (isInlineOnly) { - *outRequestType = TRequestType::ReadOnlyInline; - intermediate->Stat.RequestType = *outRequestType; - } - return true; -} - - -bool TKeyValueState::PrepareExecuteTransactionRequest(const TActorContext &ctx, - TEvKeyValue::TEvExecuteTransaction::TPtr &ev, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info) -{ - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " PrepareExecuteTransactionRequest Marker# KV72"); - - NKikimrKeyValue::ExecuteTransactionRequest &request = ev->Get()->Record; - StoredState.SetChannelGeneration(ExecutorGeneration); - StoredState.SetChannelStep(NextLogoBlobStep - 1); - - TRequestType::EType requestType = TRequestType::WriteOnly; - intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, - StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); - - intermediate->HasCookie = true; - intermediate->Cookie = request.cookie(); - intermediate->ExecuteTransactionResponse.set_cookie(request.cookie()); - - intermediate->RequestUid = NextRequestUid; - ++NextRequestUid; - RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvExecuteTransaction::EventType; - - if (CheckDeadline(ctx, ev->Get(), intermediate)) { - return false; - } - - if (CheckGeneration(ctx, ev->Get(), intermediate)) { - return false; - } - - TPrepareResult result = PrepareCommands(request, intermediate, info, ctx); - - if (result.WithError) { - // discard allocated LogoBlobIds from the Write commands - for (ui32 idx : intermediate->WriteIndices) { - const auto &cmd = intermediate->Commands[idx]; - Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); - auto &write = std::get<TIntermediate::TWrite>(cmd); - for (const TLogoBlobID &id : write.LogoBlobIds) { - const ui32 count = RefCounts.erase(id); - Y_VERIFY(count == 1); - } - } - - LOG_ERROR_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " PrepareExecuteTransactionRequest return flase, Marker# KV73" - << " Submsg# " << result.ErrorMsg); - return false; - } - - return true; -} - - -TKeyValueState::TPrepareResult TKeyValueState::PrepareOneGetStatus(TIntermediate::TGetStatus &cmd, - ui64 publicStorageChannel, const TTabletStorageInfo *info) -{ - NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = NKikimrClient::TKeyValueRequest::MAIN; - if (publicStorageChannel == 1) { - storageChannel = NKikimrClient::TKeyValueRequest::INLINE; - } else if (publicStorageChannel) { - ui32 storageChannelIdx = BLOB_CHANNEL + publicStorageChannel - MainStorageChannelInPublicApi; - storageChannel = NKikimrClient::TKeyValueRequest::EStorageChannel(storageChannelIdx); - } - return InitGetStatusCommand(cmd, storageChannel, info);; -} - - -bool TKeyValueState::PrepareGetStatusRequest(const TActorContext &ctx, TEvKeyValue::TEvGetStatus::TPtr &ev, - THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info) -{ - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareGetStatusRequest Marker# KV78"); - - NKikimrKeyValue::GetStatusRequest &request = ev->Get()->Record; - StoredState.SetChannelGeneration(ExecutorGeneration); - StoredState.SetChannelStep(NextLogoBlobStep - 1); - - TRequestType::EType requestType = TRequestType::ReadOnly; - intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, - StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); - - intermediate->RequestUid = NextRequestUid; - ++NextRequestUid; - RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvGetStatus::EventType; - - if (CheckDeadline(ctx, ev->Get(), intermediate)) { - return false; - } - - if (CheckGeneration(ctx, ev->Get(), intermediate)) { - return false; - } - - intermediate->GetStatuses.resize(request.storage_channel_size()); - for (i32 idx = 0; idx < request.storage_channel_size(); ++idx) { - TPrepareResult result = PrepareOneGetStatus(intermediate->GetStatuses[idx], request.storage_channel(idx), info); - if (result.ErrorMsg && !result.WithError) { - LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, result.ErrorMsg << " Marker# KV77"); - } - } - return true; -} - -bool TKeyValueState::PrepareObtainLockRequest(const TActorContext &ctx, TEvKeyValue::TEvObtainLock::TPtr &ev, - THolder<TIntermediate> &intermediate) -{ - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareObtainLockRequest Marker# KV79"); - - StoredState.SetChannelGeneration(ExecutorGeneration); - StoredState.SetChannelStep(NextLogoBlobStep - 1); - - TRequestType::EType requestType = TRequestType::ReadOnly; - intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, - StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); - - intermediate->RequestUid = NextRequestUid; - ++NextRequestUid; - RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvObtainLock::EventType; - intermediate->HasIncrementGeneration = true; - return true; -} - -void RegisterReadRequestActor(const TActorContext &ctx, THolder<TIntermediate> &&intermediate, - const TTabletStorageInfo *info) -{ - ctx.RegisterWithSameMailbox(CreateKeyValueStorageReadRequest(std::move(intermediate), info)); -} - -void RegisterRequestActor(const TActorContext &ctx, THolder<TIntermediate> &&intermediate, - const TTabletStorageInfo *info) -{ - ctx.RegisterWithSameMailbox(CreateKeyValueStorageRequest(std::move(intermediate), info)); -} - -void TKeyValueState::ProcessPostponedIntermediate(const TActorContext& ctx, THolder<TIntermediate> &&intermediate, - const TTabletStorageInfo *info) -{ - switch(intermediate->EvType) { - case TEvKeyValue::TEvRequest::EventType: - return RegisterRequestActor(ctx, std::move(intermediate), info); - case TEvKeyValue::TEvRead::EventType: - case TEvKeyValue::TEvReadRange::EventType: - return RegisterReadRequestActor(ctx, std::move(intermediate), info); - default: - Y_FAIL_S("Unexpected event type# " << intermediate->EvType); - } -} - -void TKeyValueState::OnEvReadRequest(TEvKeyValue::TEvRead::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info) -{ - THolder<TIntermediate> intermediate; - - ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); - ResourceMetrics->TryUpdate(ctx); - - TRequestType::EType requestType = TRequestType::ReadOnly; - CountRequestIncoming(requestType); - - if (PrepareReadRequest(ctx, ev, intermediate, &requestType)) { - ++InFlightForStep[StoredState.GetChannelStep()]; - if (requestType == TRequestType::ReadOnlyInline) { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create storage inline read request, Marker# KV49"); - RegisterReadRequestActor(ctx, std::move(intermediate), info); - ++RoInlineIntermediatesInFlight; - } else { - if (IntermediatesInFlight < IntermediatesInFlightLimit) { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create storage read request, Marker# KV54"); - RegisterReadRequestActor(ctx, std::move(intermediate), info); - ++IntermediatesInFlight; - } else { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Enqueue storage read request, Marker# KV56"); - PostponeIntermediate<TEvKeyValue::TEvRead>(std::move(intermediate)); - } - } - CountRequestTakeOffOrEnqueue(requestType); - } else { - intermediate->UpdateStat(); - CountRequestOtherError(requestType); - } -} - -void TKeyValueState::OnEvReadRangeRequest(TEvKeyValue::TEvReadRange::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info) -{ - THolder<TIntermediate> intermediate; - - ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); - ResourceMetrics->TryUpdate(ctx); - - TRequestType::EType requestType = TRequestType::ReadOnly; - CountRequestIncoming(requestType); - - if (PrepareReadRangeRequest(ctx, ev, intermediate, &requestType)) { - ++InFlightForStep[StoredState.GetChannelStep()]; - if (requestType == TRequestType::ReadOnlyInline) { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create storage inline read range request, Marker# KV58"); - RegisterReadRequestActor(ctx, std::move(intermediate), info); - ++RoInlineIntermediatesInFlight; - } else { - if (IntermediatesInFlight < IntermediatesInFlightLimit) { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create storage read range request, Marker# KV66"); - RegisterReadRequestActor(ctx, std::move(intermediate), info); - ++IntermediatesInFlight; - } else { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Enqueue storage read range request, Marker# KV59"); - PostponeIntermediate<TEvKeyValue::TEvReadRange>(std::move(intermediate)); - } - } - CountRequestTakeOffOrEnqueue(requestType); - } else { - intermediate->UpdateStat(); - CountRequestOtherError(requestType); - } - CountRequestTakeOffOrEnqueue(requestType); -} - -void TKeyValueState::OnEvExecuteTransaction(TEvKeyValue::TEvExecuteTransaction::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info) -{ - THolder<TIntermediate> intermediate; - - ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); - ResourceMetrics->TryUpdate(ctx); - - TRequestType::EType requestType = TRequestType::WriteOnly; - CountRequestIncoming(requestType); - - if (PrepareExecuteTransactionRequest(ctx, ev, intermediate, info)) { - ++InFlightForStep[StoredState.GetChannelStep()]; - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create storage request for WO, Marker# KV67"); - RegisterRequestActor(ctx, std::move(intermediate), info); - - CountRequestTakeOffOrEnqueue(requestType); - } else { - intermediate->UpdateStat(); - CountRequestOtherError(requestType); - } -} - -void TKeyValueState::OnEvGetStatus(TEvKeyValue::TEvGetStatus::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info) -{ - THolder<TIntermediate> intermediate; - - ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); - ResourceMetrics->TryUpdate(ctx); - - TRequestType::EType requestType = TRequestType::ReadOnlyInline; - CountRequestIncoming(requestType); - - if (PrepareGetStatusRequest(ctx, ev, intermediate, info)) { - ++InFlightForStep[StoredState.GetChannelStep()]; - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create GetStatus request, Marker# KV75"); - RegisterRequestActor(ctx, std::move(intermediate), info); - ++RoInlineIntermediatesInFlight; - CountRequestTakeOffOrEnqueue(requestType); - } else { - intermediate->UpdateStat(); - CountRequestOtherError(requestType); - } -} - -void TKeyValueState::OnEvObtainLock(TEvKeyValue::TEvObtainLock::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info) -{ - THolder<TIntermediate> intermediate; - - ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); - ResourceMetrics->TryUpdate(ctx); - - TRequestType::EType requestType = TRequestType::ReadOnlyInline; - - CountRequestIncoming(requestType); - if (PrepareObtainLockRequest(ctx, ev, intermediate)) { - ++InFlightForStep[StoredState.GetChannelStep()]; - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " Create ObtainLock request, Marker# KV80"); - RegisterRequestActor(ctx, std::move(intermediate), info); - ++RoInlineIntermediatesInFlight; - CountRequestTakeOffOrEnqueue(requestType); - } else { - intermediate->UpdateStat(); - CountRequestOtherError(requestType); - } -} - +bool TKeyValueState::PrepareReadRequest(const TActorContext &ctx, TEvKeyValue::TEvRead::TPtr &ev, + THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType) +{ + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareReadRequest Marker# KV53"); + + NKikimrKeyValue::ReadRequest &request = ev->Get()->Record; + StoredState.SetChannelGeneration(ExecutorGeneration); + StoredState.SetChannelStep(NextLogoBlobStep - 1); + + intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, + StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), TRequestType::ReadOnly)); + + intermediate->HasCookie = true; + intermediate->Cookie = request.cookie(); + + intermediate->RequestUid = NextRequestUid; + ++NextRequestUid; + RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); + intermediate->EvType = TEvKeyValue::TEvRead::EventType; + + bool isInlineOnly = true; + intermediate->ReadCommand = TIntermediate::TRead(); + auto &response = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); + response.Key = request.key(); + + if (CheckDeadline(ctx, ev->Get(), intermediate)) { + return false; + } + + if (CheckGeneration(ctx, ev->Get(), intermediate)) { + return false; + } + + auto it = Index.find(request.key()); + if (it == Index.end()) { + response.Status = NKikimrProto::NODATA; + response.Message = "No such key Marker# KV55"; + ReplyError<TEvKeyValue::TEvReadResponse>(ctx, response.Message, + NKikimrKeyValue::Statuses::RSTATUS_NO_DATA, intermediate); + return false; + } + bool isOverRun = PrepareOneRead<NKikimrKeyValue::Priorities, true, ReadResultSizeEstimationNewApi>( + it->first, it->second, request.offset(), request.size(), request.priority(), request.limit_bytes(), + intermediate, response, isInlineOnly); + + if (isInlineOnly) { + *outRequestType = TRequestType::ReadOnlyInline; + intermediate->Stat.RequestType = *outRequestType; + } + intermediate->IsTruncated = isOverRun; + return true; +} + +bool TKeyValueState::PrepareReadRangeRequest(const TActorContext &ctx, TEvKeyValue::TEvReadRange::TPtr &ev, + THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType) +{ + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareReadRangeRequest Marker# KV57"); + + NKikimrKeyValue::ReadRangeRequest &request = ev->Get()->Record; + StoredState.SetChannelGeneration(ExecutorGeneration); + StoredState.SetChannelStep(NextLogoBlobStep - 1); + + TRequestType::EType requestType = TRequestType::ReadOnly; + intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, + StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); + + intermediate->HasCookie = true; + intermediate->Cookie = request.cookie(); + + intermediate->RequestUid = NextRequestUid; + ++NextRequestUid; + RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); + intermediate->EvType = TEvKeyValue::TEvReadRange::EventType; + intermediate->TotalSize = ReadRangeRequestMetaDataSizeEstimation; + + intermediate->ReadCommand = TIntermediate::TRangeRead(); + auto &response = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); + + if (CheckDeadline(ctx, ev->Get(), intermediate)) { + response.Status = NKikimrProto::ERROR; + return false; + } + + if (CheckGeneration(ctx, ev->Get(), intermediate)) { + response.Status = NKikimrProto::ERROR; + return false; + } + + TKeyRange range; + auto convResult = ConvertRange(request.range(), &range, "ReadRange"); + if (convResult.WithError) { + response.Status = NKikimrProto::ERROR; + return false; + } + response.Status = NKikimrProto::OK; + + ui64 cmdLimitBytes = request.limit_bytes(); + if (!cmdLimitBytes) { + cmdLimitBytes = Max<ui64>(); + } + bool includeData = request.include_data(); + ui8 priority = request.priority(); + response.LimitBytes = cmdLimitBytes; + response.IncludeData = includeData; + + bool isInlineOnly = true; + auto processOneCmdReadRange = ProcessOneCmdReadRange<NKikimrKeyValue::Priorities, true, + KeyValuePairSizeEstimationNewApi, KeyInfoSizeEstimation>; + processOneCmdReadRange(this, range, cmdLimitBytes, includeData, priority, response, intermediate, &isInlineOnly); + + if (isInlineOnly) { + *outRequestType = TRequestType::ReadOnlyInline; + intermediate->Stat.RequestType = *outRequestType; + } + return true; +} + + +bool TKeyValueState::PrepareExecuteTransactionRequest(const TActorContext &ctx, + TEvKeyValue::TEvExecuteTransaction::TPtr &ev, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info) +{ + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " PrepareExecuteTransactionRequest Marker# KV72"); + + NKikimrKeyValue::ExecuteTransactionRequest &request = ev->Get()->Record; + StoredState.SetChannelGeneration(ExecutorGeneration); + StoredState.SetChannelStep(NextLogoBlobStep - 1); + + TRequestType::EType requestType = TRequestType::WriteOnly; + intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, + StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); + + intermediate->HasCookie = true; + intermediate->Cookie = request.cookie(); + intermediate->ExecuteTransactionResponse.set_cookie(request.cookie()); + + intermediate->RequestUid = NextRequestUid; + ++NextRequestUid; + RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); + intermediate->EvType = TEvKeyValue::TEvExecuteTransaction::EventType; + + if (CheckDeadline(ctx, ev->Get(), intermediate)) { + return false; + } + + if (CheckGeneration(ctx, ev->Get(), intermediate)) { + return false; + } + + TPrepareResult result = PrepareCommands(request, intermediate, info, ctx); + + if (result.WithError) { + // discard allocated LogoBlobIds from the Write commands + for (ui32 idx : intermediate->WriteIndices) { + const auto &cmd = intermediate->Commands[idx]; + Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); + auto &write = std::get<TIntermediate::TWrite>(cmd); + for (const TLogoBlobID &id : write.LogoBlobIds) { + const ui32 count = RefCounts.erase(id); + Y_VERIFY(count == 1); + } + } + + LOG_ERROR_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " PrepareExecuteTransactionRequest return flase, Marker# KV73" + << " Submsg# " << result.ErrorMsg); + return false; + } + + return true; +} + + +TKeyValueState::TPrepareResult TKeyValueState::PrepareOneGetStatus(TIntermediate::TGetStatus &cmd, + ui64 publicStorageChannel, const TTabletStorageInfo *info) +{ + NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel = NKikimrClient::TKeyValueRequest::MAIN; + if (publicStorageChannel == 1) { + storageChannel = NKikimrClient::TKeyValueRequest::INLINE; + } else if (publicStorageChannel) { + ui32 storageChannelIdx = BLOB_CHANNEL + publicStorageChannel - MainStorageChannelInPublicApi; + storageChannel = NKikimrClient::TKeyValueRequest::EStorageChannel(storageChannelIdx); + } + return InitGetStatusCommand(cmd, storageChannel, info);; +} + + +bool TKeyValueState::PrepareGetStatusRequest(const TActorContext &ctx, TEvKeyValue::TEvGetStatus::TPtr &ev, + THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info) +{ + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareGetStatusRequest Marker# KV78"); + + NKikimrKeyValue::GetStatusRequest &request = ev->Get()->Record; + StoredState.SetChannelGeneration(ExecutorGeneration); + StoredState.SetChannelStep(NextLogoBlobStep - 1); + + TRequestType::EType requestType = TRequestType::ReadOnly; + intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, + StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); + + intermediate->RequestUid = NextRequestUid; + ++NextRequestUid; + RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); + intermediate->EvType = TEvKeyValue::TEvGetStatus::EventType; + + if (CheckDeadline(ctx, ev->Get(), intermediate)) { + return false; + } + + if (CheckGeneration(ctx, ev->Get(), intermediate)) { + return false; + } + + intermediate->GetStatuses.resize(request.storage_channel_size()); + for (i32 idx = 0; idx < request.storage_channel_size(); ++idx) { + TPrepareResult result = PrepareOneGetStatus(intermediate->GetStatuses[idx], request.storage_channel(idx), info); + if (result.ErrorMsg && !result.WithError) { + LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, result.ErrorMsg << " Marker# KV77"); + } + } + return true; +} + +bool TKeyValueState::PrepareObtainLockRequest(const TActorContext &ctx, TEvKeyValue::TEvObtainLock::TPtr &ev, + THolder<TIntermediate> &intermediate) +{ + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareObtainLockRequest Marker# KV79"); + + StoredState.SetChannelGeneration(ExecutorGeneration); + StoredState.SetChannelStep(NextLogoBlobStep - 1); + + TRequestType::EType requestType = TRequestType::ReadOnly; + intermediate.Reset(new TIntermediate(ev->Sender, ctx.SelfID, + StoredState.GetChannelGeneration(), StoredState.GetChannelStep(), requestType)); + + intermediate->RequestUid = NextRequestUid; + ++NextRequestUid; + RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); + intermediate->EvType = TEvKeyValue::TEvObtainLock::EventType; + intermediate->HasIncrementGeneration = true; + return true; +} + +void RegisterReadRequestActor(const TActorContext &ctx, THolder<TIntermediate> &&intermediate, + const TTabletStorageInfo *info) +{ + ctx.RegisterWithSameMailbox(CreateKeyValueStorageReadRequest(std::move(intermediate), info)); +} + +void RegisterRequestActor(const TActorContext &ctx, THolder<TIntermediate> &&intermediate, + const TTabletStorageInfo *info) +{ + ctx.RegisterWithSameMailbox(CreateKeyValueStorageRequest(std::move(intermediate), info)); +} + +void TKeyValueState::ProcessPostponedIntermediate(const TActorContext& ctx, THolder<TIntermediate> &&intermediate, + const TTabletStorageInfo *info) +{ + switch(intermediate->EvType) { + case TEvKeyValue::TEvRequest::EventType: + return RegisterRequestActor(ctx, std::move(intermediate), info); + case TEvKeyValue::TEvRead::EventType: + case TEvKeyValue::TEvReadRange::EventType: + return RegisterReadRequestActor(ctx, std::move(intermediate), info); + default: + Y_FAIL_S("Unexpected event type# " << intermediate->EvType); + } +} + +void TKeyValueState::OnEvReadRequest(TEvKeyValue::TEvRead::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info) +{ + THolder<TIntermediate> intermediate; + + ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); + ResourceMetrics->TryUpdate(ctx); + + TRequestType::EType requestType = TRequestType::ReadOnly; + CountRequestIncoming(requestType); + + if (PrepareReadRequest(ctx, ev, intermediate, &requestType)) { + ++InFlightForStep[StoredState.GetChannelStep()]; + if (requestType == TRequestType::ReadOnlyInline) { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create storage inline read request, Marker# KV49"); + RegisterReadRequestActor(ctx, std::move(intermediate), info); + ++RoInlineIntermediatesInFlight; + } else { + if (IntermediatesInFlight < IntermediatesInFlightLimit) { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create storage read request, Marker# KV54"); + RegisterReadRequestActor(ctx, std::move(intermediate), info); + ++IntermediatesInFlight; + } else { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Enqueue storage read request, Marker# KV56"); + PostponeIntermediate<TEvKeyValue::TEvRead>(std::move(intermediate)); + } + } + CountRequestTakeOffOrEnqueue(requestType); + } else { + intermediate->UpdateStat(); + CountRequestOtherError(requestType); + } +} + +void TKeyValueState::OnEvReadRangeRequest(TEvKeyValue::TEvReadRange::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info) +{ + THolder<TIntermediate> intermediate; + + ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); + ResourceMetrics->TryUpdate(ctx); + + TRequestType::EType requestType = TRequestType::ReadOnly; + CountRequestIncoming(requestType); + + if (PrepareReadRangeRequest(ctx, ev, intermediate, &requestType)) { + ++InFlightForStep[StoredState.GetChannelStep()]; + if (requestType == TRequestType::ReadOnlyInline) { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create storage inline read range request, Marker# KV58"); + RegisterReadRequestActor(ctx, std::move(intermediate), info); + ++RoInlineIntermediatesInFlight; + } else { + if (IntermediatesInFlight < IntermediatesInFlightLimit) { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create storage read range request, Marker# KV66"); + RegisterReadRequestActor(ctx, std::move(intermediate), info); + ++IntermediatesInFlight; + } else { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Enqueue storage read range request, Marker# KV59"); + PostponeIntermediate<TEvKeyValue::TEvReadRange>(std::move(intermediate)); + } + } + CountRequestTakeOffOrEnqueue(requestType); + } else { + intermediate->UpdateStat(); + CountRequestOtherError(requestType); + } + CountRequestTakeOffOrEnqueue(requestType); +} + +void TKeyValueState::OnEvExecuteTransaction(TEvKeyValue::TEvExecuteTransaction::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info) +{ + THolder<TIntermediate> intermediate; + + ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); + ResourceMetrics->TryUpdate(ctx); + + TRequestType::EType requestType = TRequestType::WriteOnly; + CountRequestIncoming(requestType); + + if (PrepareExecuteTransactionRequest(ctx, ev, intermediate, info)) { + ++InFlightForStep[StoredState.GetChannelStep()]; + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create storage request for WO, Marker# KV67"); + RegisterRequestActor(ctx, std::move(intermediate), info); + + CountRequestTakeOffOrEnqueue(requestType); + } else { + intermediate->UpdateStat(); + CountRequestOtherError(requestType); + } +} + +void TKeyValueState::OnEvGetStatus(TEvKeyValue::TEvGetStatus::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info) +{ + THolder<TIntermediate> intermediate; + + ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); + ResourceMetrics->TryUpdate(ctx); + + TRequestType::EType requestType = TRequestType::ReadOnlyInline; + CountRequestIncoming(requestType); + + if (PrepareGetStatusRequest(ctx, ev, intermediate, info)) { + ++InFlightForStep[StoredState.GetChannelStep()]; + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create GetStatus request, Marker# KV75"); + RegisterRequestActor(ctx, std::move(intermediate), info); + ++RoInlineIntermediatesInFlight; + CountRequestTakeOffOrEnqueue(requestType); + } else { + intermediate->UpdateStat(); + CountRequestOtherError(requestType); + } +} + +void TKeyValueState::OnEvObtainLock(TEvKeyValue::TEvObtainLock::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info) +{ + THolder<TIntermediate> intermediate; + + ResourceMetrics->Network.Increment(ev->Get()->Record.ByteSize()); + ResourceMetrics->TryUpdate(ctx); + + TRequestType::EType requestType = TRequestType::ReadOnlyInline; + + CountRequestIncoming(requestType); + if (PrepareObtainLockRequest(ctx, ev, intermediate)) { + ++InFlightForStep[StoredState.GetChannelStep()]; + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " Create ObtainLock request, Marker# KV80"); + RegisterRequestActor(ctx, std::move(intermediate), info); + ++RoInlineIntermediatesInFlight; + CountRequestTakeOffOrEnqueue(requestType); + } else { + intermediate->UpdateStat(); + CountRequestOtherError(requestType); + } +} + void TKeyValueState::OnEvIntermediate(TIntermediate &intermediate, const TActorContext &ctx) { Y_UNUSED(ctx); CountLatencyBsOps(intermediate.Stat); @@ -3010,26 +3010,26 @@ void TKeyValueState::OnEvRequest(TEvKeyValue::TEvRequest::TPtr &ev, const TActor if (PrepareIntermediate(ev, intermediate, requestType, ctx, info)) { // Spawn KeyValueStorageRequest actor on the same thread - ++InFlightForStep[StoredState.GetChannelStep()]; + ++InFlightForStep[StoredState.GetChannelStep()]; if (requestType == TRequestType::WriteOnly) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " Create storage request for WO, Marker# KV42"); - RegisterRequestActor(ctx, std::move(intermediate), info); + RegisterRequestActor(ctx, std::move(intermediate), info); } else if (requestType == TRequestType::ReadOnlyInline) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " Create storage request for RO_INLINE, Marker# KV45"); - RegisterRequestActor(ctx, std::move(intermediate), info); + RegisterRequestActor(ctx, std::move(intermediate), info); ++RoInlineIntermediatesInFlight; } else { if (IntermediatesInFlight < IntermediatesInFlightLimit) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " Create storage request for RO/RW, Marker# KV43"); - RegisterRequestActor(ctx, std::move(intermediate), info); + RegisterRequestActor(ctx, std::move(intermediate), info); ++IntermediatesInFlight; } else { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " Enqueue storage request for RO/RW, Marker# KV44"); - PostponeIntermediate<TEvKeyValue::TEvRequest>(std::move(intermediate)); + PostponeIntermediate<TEvKeyValue::TEvRequest>(std::move(intermediate)); } } @@ -3055,7 +3055,7 @@ bool TKeyValueState::PrepareIntermediate(TEvKeyValue::TEvRequest::TPtr &ev, THol intermediate->RequestUid = NextRequestUid; ++NextRequestUid; RequestInputTime[intermediate->RequestUid] = TAppData::TimeProvider->Now(); - intermediate->EvType = TEvKeyValue::TEvRequest::EventType; + intermediate->EvType = TEvKeyValue::TEvRequest::EventType; intermediate->HasCookie = request.HasCookie(); if (request.HasCookie()) { @@ -3081,10 +3081,10 @@ bool TKeyValueState::PrepareIntermediate(TEvKeyValue::TEvRequest::TPtr &ev, THol intermediate->Stat.RequestType = inOutRequestType; } - ui32 cmdCount = request.CmdWriteSize() + request.CmdDeleteRangeSize() + request.CmdRenameSize() - + request.CmdCopyRangeSize() + request.CmdConcatSize(); - intermediate->Commands.reserve(cmdCount); - + ui32 cmdCount = request.CmdWriteSize() + request.CmdDeleteRangeSize() + request.CmdRenameSize() + + request.CmdCopyRangeSize() + request.CmdConcatSize(); + intermediate->Commands.reserve(cmdCount); + error = error || PrepareCmdCopyRange(ctx, request, intermediate); error = error || PrepareCmdRename(ctx, request, intermediate); error = error || PrepareCmdConcat(ctx, request, intermediate); @@ -3094,12 +3094,12 @@ bool TKeyValueState::PrepareIntermediate(TEvKeyValue::TEvRequest::TPtr &ev, THol error = error || PrepareCmdTrimLeakedBlobs(ctx, request, intermediate, info); error = error || PrepareCmdSetExecutorFastLogPolicy(ctx, request, intermediate, info); - intermediate->WriteCount = request.CmdWriteSize(); - intermediate->DeleteCount = request.CmdDeleteRangeSize(); - intermediate->RenameCount = request.CmdRenameSize(); - intermediate->CopyRangeCount = request.CmdCopyRangeSize(); - intermediate->ConcatCount = request.CmdConcatSize(); - + intermediate->WriteCount = request.CmdWriteSize(); + intermediate->DeleteCount = request.CmdDeleteRangeSize(); + intermediate->RenameCount = request.CmdRenameSize(); + intermediate->CopyRangeCount = request.CmdCopyRangeSize(); + intermediate->ConcatCount = request.CmdConcatSize(); + if (error) { // discard allocated LogoBlobIds from the Write commands for (const auto &write : intermediate->Writes) { @@ -3109,16 +3109,16 @@ bool TKeyValueState::PrepareIntermediate(TEvKeyValue::TEvRequest::TPtr &ev, THol } } - for (ui32 idx : intermediate->WriteIndices) { - const auto &cmd = intermediate->Commands[idx]; - Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); - auto &write = std::get<TIntermediate::TWrite>(cmd); - for (const TLogoBlobID &id : write.LogoBlobIds) { - const ui32 count = RefCounts.erase(id); - Y_VERIFY(count == 1); - } - } - + for (ui32 idx : intermediate->WriteIndices) { + const auto &cmd = intermediate->Commands[idx]; + Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); + auto &write = std::get<TIntermediate::TWrite>(cmd); + for (const TLogoBlobID &id : write.LogoBlobIds) { + const ui32 count = RefCounts.erase(id); + Y_VERIFY(count == 1); + } + } + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " PrepareIntermediate return flase, Marker# KV41"); return false; diff --git a/ydb/core/keyvalue/keyvalue_state.h b/ydb/core/keyvalue/keyvalue_state.h index f57a69d35d3..e13d7160e13 100644 --- a/ydb/core/keyvalue/keyvalue_state.h +++ b/ydb/core/keyvalue/keyvalue_state.h @@ -56,9 +56,9 @@ class TKeyValueState { }; public: - using TIndex = TMap<TString, TIndexRecord>; - using TCommand = NKikimrKeyValue::ExecuteTransactionRequest::Command; - + using TIndex = TMap<TString, TIndexRecord>; + using TCommand = NKikimrKeyValue::ExecuteTransactionRequest::Command; + class TIncrementalKeySet { TMap<TString, TIndexRecord>& Index; TSet<TString> AddedKeys; @@ -334,34 +334,34 @@ public: void OnEvEraseCollect(const TActorContext &ctx); void Reply(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info); - void ProcessCmd(TIntermediate::TRead &read, - NKikimrClient::TKeyValueResponse::TReadResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(TIntermediate::TRangeRead &request, - NKikimrClient::TKeyValueResponse::TReadRangeResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(TIntermediate::TWrite &request, - NKikimrClient::TKeyValueResponse::TWriteResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(const TIntermediate::TDelete &request, - NKikimrClient::TKeyValueResponse::TDeleteRangeResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(const TIntermediate::TRename &request, - NKikimrClient::TKeyValueResponse::TRenameResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(const TIntermediate::TCopyRange &request, - NKikimrClient::TKeyValueResponse::TCopyRangeResult *legacyResponse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); - void ProcessCmd(const TIntermediate::TConcat &request, - NKikimrClient::TKeyValueResponse::TConcatResult *resplegacyResponseonse, - NKikimrKeyValue::Channel *response, - ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(TIntermediate::TRead &read, + NKikimrClient::TKeyValueResponse::TReadResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(TIntermediate::TRangeRead &request, + NKikimrClient::TKeyValueResponse::TReadRangeResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(TIntermediate::TWrite &request, + NKikimrClient::TKeyValueResponse::TWriteResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(const TIntermediate::TDelete &request, + NKikimrClient::TKeyValueResponse::TDeleteRangeResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(const TIntermediate::TRename &request, + NKikimrClient::TKeyValueResponse::TRenameResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(const TIntermediate::TCopyRange &request, + NKikimrClient::TKeyValueResponse::TCopyRangeResult *legacyResponse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); + void ProcessCmd(const TIntermediate::TConcat &request, + NKikimrClient::TKeyValueResponse::TConcatResult *resplegacyResponseonse, + NKikimrKeyValue::Channel *response, + ISimpleDb &db, const TActorContext &ctx, TRequestStat &stat, ui64 unixTime); void CmdRead(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); void CmdReadRange(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); void CmdRename(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); @@ -372,36 +372,36 @@ public: void CmdConcat(THolder<TIntermediate>& intermediate, ISimpleDb& db, const TActorContext& ctx); void CmdTrimLeakedBlobs(THolder<TIntermediate>& intermediate, ISimpleDb& db, const TActorContext& ctx); void CmdSetExecutorFastLogPolicy(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); - void CmdCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); + void CmdCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); void ProcessCmds(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx, const TTabletStorageInfo *info); bool IncrementGeneration(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx); - struct TCheckResult { - bool Result = true; - TString ErrorMsg; - }; - - TCheckResult CheckCmd(const TIntermediate::TRename &cmd, TKeySet& keys, ui32 index) const; - TCheckResult CheckCmd(const TIntermediate::TDelete &cmd, TKeySet& keys, ui32 index) const; - TCheckResult CheckCmd(const TIntermediate::TWrite &cmd, TKeySet& keys, ui32 index) const; - TCheckResult CheckCmd(const TIntermediate::TCopyRange &cmd, TKeySet& keys, ui32 index) const; - TCheckResult CheckCmd(const TIntermediate::TConcat &cmd, TKeySet& keys, ui32 index) const; - - bool CheckCmdRenames(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + struct TCheckResult { + bool Result = true; + TString ErrorMsg; + }; + + TCheckResult CheckCmd(const TIntermediate::TRename &cmd, TKeySet& keys, ui32 index) const; + TCheckResult CheckCmd(const TIntermediate::TDelete &cmd, TKeySet& keys, ui32 index) const; + TCheckResult CheckCmd(const TIntermediate::TWrite &cmd, TKeySet& keys, ui32 index) const; + TCheckResult CheckCmd(const TIntermediate::TCopyRange &cmd, TKeySet& keys, ui32 index) const; + TCheckResult CheckCmd(const TIntermediate::TConcat &cmd, TKeySet& keys, ui32 index) const; + + bool CheckCmdRenames(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, const TTabletStorageInfo *info); - bool CheckCmdDeletes(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + bool CheckCmdDeletes(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, const TTabletStorageInfo *info); - bool CheckCmdWrites(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + bool CheckCmdWrites(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, const TTabletStorageInfo *info); - bool CheckCmdCopyRanges(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + bool CheckCmdCopyRanges(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, const TTabletStorageInfo *info); - bool CheckCmdConcats(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, + bool CheckCmdConcats(THolder<TIntermediate>& intermediate, const TActorContext& ctx, TKeySet& keys, const TTabletStorageInfo *info); bool CheckCmdGetStatus(THolder<TIntermediate>& /*intermediate*/, const TActorContext& /*ctx*/, TKeySet& /*keys*/, const TTabletStorageInfo* /*info*/); - bool CheckCmds(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, - const TTabletStorageInfo* /*info*/); + bool CheckCmds(THolder<TIntermediate>& intermediate, const TActorContext& /*ctx*/, TKeySet& keys, + const TTabletStorageInfo* /*info*/); void Step(); TLogoBlobID AllocateLogoBlobId(ui32 size, ui32 storageChannelIdx); @@ -417,23 +417,23 @@ public: return PerGenerationCounter; } - void OnEvReadRequest(TEvKeyValue::TEvRead::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info); - void OnEvReadRangeRequest(TEvKeyValue::TEvReadRange::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info); - void OnEvExecuteTransaction(TEvKeyValue::TEvExecuteTransaction::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info); - void OnEvGetStatus(TEvKeyValue::TEvGetStatus::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info); - void OnEvObtainLock(TEvKeyValue::TEvObtainLock::TPtr &ev, const TActorContext &ctx, - const TTabletStorageInfo *info); - + void OnEvReadRequest(TEvKeyValue::TEvRead::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info); + void OnEvReadRangeRequest(TEvKeyValue::TEvReadRange::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info); + void OnEvExecuteTransaction(TEvKeyValue::TEvExecuteTransaction::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info); + void OnEvGetStatus(TEvKeyValue::TEvGetStatus::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info); + void OnEvObtainLock(TEvKeyValue::TEvObtainLock::TPtr &ev, const TActorContext &ctx, + const TTabletStorageInfo *info); + void OnPeriodicRefresh(const TActorContext &ctx); void OnUpdateWeights(TChannelBalancer::TEvUpdateWeights::TPtr ev); - + void OnRequestComplete(ui64 requestUid, ui64 generation, ui64 step, const TActorContext &ctx, - const TTabletStorageInfo *info, NMsgBusProxy::EResponseStatus status, const TRequestStat &stat); - + const TTabletStorageInfo *info, NMsgBusProxy::EResponseStatus status, const TRequestStat &stat); + void OnEvIntermediate(TIntermediate &intermediate, const TActorContext &ctx); void OnEvRequest(TEvKeyValue::TEvRequest::TPtr &ev, const TActorContext &ctx, const TTabletStorageInfo *info); bool PrepareIntermediate(TEvKeyValue::TEvRequest::TPtr &ev, THolder<TIntermediate> &intermediate, @@ -443,57 +443,57 @@ public: bool CheckDeadline(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate); - - template <typename TRequest> - bool CheckDeadline(const TActorContext &ctx, TRequest *request, - THolder<TIntermediate> &intermediate) - { - ui64 deadlineInstantMs = request->Record.deadline_instant_ms(); - if (!deadlineInstantMs) { - return false; - } - intermediate->Deadline = TInstant::MicroSeconds(deadlineInstantMs * 1000ull); - - TInstant now = TAppData::TimeProvider->Now(); - if (intermediate->Deadline <= now) { - TStringStream str; - str << "KeyValue# " << TabletId; - str << " Deadline reached before processing the request!"; - str << " DeadlineInstantMs# " << deadlineInstantMs; - str << " < Now# " << (ui64)now.MilliSeconds(); - ReplyError<typename TRequest::TResponse>(ctx, str.Str(), - NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT, intermediate); - return true; - } - - return false; - } - + + template <typename TRequest> + bool CheckDeadline(const TActorContext &ctx, TRequest *request, + THolder<TIntermediate> &intermediate) + { + ui64 deadlineInstantMs = request->Record.deadline_instant_ms(); + if (!deadlineInstantMs) { + return false; + } + intermediate->Deadline = TInstant::MicroSeconds(deadlineInstantMs * 1000ull); + + TInstant now = TAppData::TimeProvider->Now(); + if (intermediate->Deadline <= now) { + TStringStream str; + str << "KeyValue# " << TabletId; + str << " Deadline reached before processing the request!"; + str << " DeadlineInstantMs# " << deadlineInstantMs; + str << " < Now# " << (ui64)now.MilliSeconds(); + ReplyError<typename TRequest::TResponse>(ctx, str.Str(), + NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT, intermediate); + return true; + } + + return false; + } + bool CheckGeneration(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate); - - template <typename TGrpcRequestWithLockGeneration> - bool CheckGeneration(const TActorContext &ctx, TGrpcRequestWithLockGeneration *kvRequest, - THolder<TIntermediate> &intermediate) - { - auto &record = kvRequest->Record; - intermediate->HasGeneration = true; - intermediate->Generation = record.lock_generation(); - if (record.lock_generation() != StoredState.GetUserGeneration()) { - TStringStream str; - str << "KeyValue# " << TabletId; - str << " Generation mismatch! Requested# " << record.lock_generation(); - str << " Actual# " << StoredState.GetUserGeneration(); - str << " Marker# KV05"; - ReplyError<typename TGrpcRequestWithLockGeneration::TResponse>(ctx, str.Str(), - NKikimrKeyValue::Statuses::RSTATUS_ERROR, intermediate); - return true; - } - return false; - } - - void SplitIntoBlobs(TIntermediate::TWrite &cmd, bool isInline, ui32 storageChannelIdx); - + + template <typename TGrpcRequestWithLockGeneration> + bool CheckGeneration(const TActorContext &ctx, TGrpcRequestWithLockGeneration *kvRequest, + THolder<TIntermediate> &intermediate) + { + auto &record = kvRequest->Record; + intermediate->HasGeneration = true; + intermediate->Generation = record.lock_generation(); + if (record.lock_generation() != StoredState.GetUserGeneration()) { + TStringStream str; + str << "KeyValue# " << TabletId; + str << " Generation mismatch! Requested# " << record.lock_generation(); + str << " Actual# " << StoredState.GetUserGeneration(); + str << " Marker# KV05"; + ReplyError<typename TGrpcRequestWithLockGeneration::TResponse>(ctx, str.Str(), + NKikimrKeyValue::Statuses::RSTATUS_ERROR, intermediate); + return true; + } + return false; + } + + void SplitIntoBlobs(TIntermediate::TWrite &cmd, bool isInline, ui32 storageChannelIdx); + bool PrepareCmdRead(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, bool &outIsInlineOnly); bool PrepareCmdReadRange(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, @@ -514,144 +514,144 @@ public: THolder<TIntermediate>& intermediate, const TTabletStorageInfo *info); bool PrepareCmdSetExecutorFastLogPolicy(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info); - - - struct TPrepareResult { - bool WithError = false; - TString ErrorMsg; - }; - TPrepareResult PrepareOneCmd(const TCommand::Rename &request, THolder<TIntermediate> &intermediate); - TPrepareResult PrepareOneCmd(const TCommand::Concat &request, THolder<TIntermediate> &intermediate); - TPrepareResult PrepareOneCmd(const TCommand::CopyRange &request, THolder<TIntermediate> &intermediate); - TPrepareResult PrepareOneCmd(const TCommand::Write &request, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info); - TPrepareResult PrepareOneCmd(const TCommand::DeleteRange &request, THolder<TIntermediate> &intermediate, - const TActorContext &ctx); - TPrepareResult PrepareOneCmd(const TCommand &request, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info, const TActorContext &ctx); - TPrepareResult PrepareCommands(NKikimrKeyValue::ExecuteTransactionRequest &kvRequest, - THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info, const TActorContext &ctx); - TPrepareResult InitGetStatusCommand(TIntermediate::TGetStatus &cmd, - NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel, const TTabletStorageInfo *info); + + + struct TPrepareResult { + bool WithError = false; + TString ErrorMsg; + }; + TPrepareResult PrepareOneCmd(const TCommand::Rename &request, THolder<TIntermediate> &intermediate); + TPrepareResult PrepareOneCmd(const TCommand::Concat &request, THolder<TIntermediate> &intermediate); + TPrepareResult PrepareOneCmd(const TCommand::CopyRange &request, THolder<TIntermediate> &intermediate); + TPrepareResult PrepareOneCmd(const TCommand::Write &request, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info); + TPrepareResult PrepareOneCmd(const TCommand::DeleteRange &request, THolder<TIntermediate> &intermediate, + const TActorContext &ctx); + TPrepareResult PrepareOneCmd(const TCommand &request, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info, const TActorContext &ctx); + TPrepareResult PrepareCommands(NKikimrKeyValue::ExecuteTransactionRequest &kvRequest, + THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info, const TActorContext &ctx); + TPrepareResult InitGetStatusCommand(TIntermediate::TGetStatus &cmd, + NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel, const TTabletStorageInfo *info); void ReplyError(const TActorContext &ctx, TString errorDescription, NMsgBusProxy::EResponseStatus status, THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info = nullptr); - template <typename TResponse> - void ReplyError(const TActorContext &ctx, TString errorDescription, - NKikimrKeyValue::Statuses::ReplyStatus status, THolder<TIntermediate> &intermediate, - const TTabletStorageInfo *info = nullptr) - { - LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, errorDescription); - Y_VERIFY(!intermediate->IsReplied); - std::unique_ptr<TResponse> response = std::make_unique<TResponse>(); - response->Record.set_status(status); - - if constexpr (std::is_same_v<TResponse, TEvKeyValue::TEvReadResponse>) { - auto &cmd = *intermediate->ReadCommand; - Y_VERIFY(std::holds_alternative<TIntermediate::TRead>(cmd)); - TIntermediate::TRead &interRead = std::get<TIntermediate::TRead>(cmd); - response->Record.set_requested_offset(interRead.Offset); - response->Record.set_requested_size(interRead.ValueSize); - response->Record.set_requested_key(interRead.Key); - } - if constexpr (!std::is_same_v<TResponse, TEvKeyValue::TEvGetStatusResponse>) { - if (intermediate->HasCookie) { - response->Record.set_cookie(intermediate->Cookie); - } - } - - if (errorDescription) { - response->Record.set_msg(errorDescription); - } - - ResourceMetrics->Network.Increment(response->Record.ByteSize()); - - intermediate->IsReplied = true; - ctx.Send(intermediate->RespondTo, response.release()); - if (info) { - intermediate->UpdateStat(); - OnRequestComplete(intermediate->RequestUid, intermediate->CreatedAtGeneration, intermediate->CreatedAtStep, - ctx, info, TEvKeyValue::TEvNotify::ConvertStatus(status), intermediate->Stat); - } else { //metrics change report in OnRequestComplete is not done - ResourceMetrics->TryUpdate(ctx); - RequestInputTime.erase(intermediate->RequestUid); - } - } - - bool PrepareReadRequest(const TActorContext &ctx, TEvKeyValue::TEvRead::TPtr &ev, - THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestTyp); - bool PrepareReadRangeRequest(const TActorContext &ctx, TEvKeyValue::TEvReadRange::TPtr &ev, - THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType); - bool PrepareExecuteTransactionRequest(const TActorContext &ctx, TEvKeyValue::TEvExecuteTransaction::TPtr &ev, - THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info); - TPrepareResult PrepareOneGetStatus(TIntermediate::TGetStatus &cmd, ui64 publicStorageChannel, - const TTabletStorageInfo *info); - bool PrepareGetStatusRequest(const TActorContext &ctx, TEvKeyValue::TEvGetStatus::TPtr &ev, - THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info); - bool PrepareObtainLockRequest(const TActorContext &ctx, TEvKeyValue::TEvObtainLock::TPtr &ev, - THolder<TIntermediate> &intermediate); - - template <typename TRequestType> - void PostponeIntermediate(THolder<TIntermediate> &&intermediate) { - intermediate->Stat.EnqueuedAs = Queue.size() + 1; - Queue.push_back(std::move(intermediate)); - } - void ProcessPostponedIntermediate(const TActorContext& ctx, THolder<TIntermediate> &&intermediate, - const TTabletStorageInfo *info); - + template <typename TResponse> + void ReplyError(const TActorContext &ctx, TString errorDescription, + NKikimrKeyValue::Statuses::ReplyStatus status, THolder<TIntermediate> &intermediate, + const TTabletStorageInfo *info = nullptr) + { + LOG_INFO_S(ctx, NKikimrServices::KEYVALUE, errorDescription); + Y_VERIFY(!intermediate->IsReplied); + std::unique_ptr<TResponse> response = std::make_unique<TResponse>(); + response->Record.set_status(status); + + if constexpr (std::is_same_v<TResponse, TEvKeyValue::TEvReadResponse>) { + auto &cmd = *intermediate->ReadCommand; + Y_VERIFY(std::holds_alternative<TIntermediate::TRead>(cmd)); + TIntermediate::TRead &interRead = std::get<TIntermediate::TRead>(cmd); + response->Record.set_requested_offset(interRead.Offset); + response->Record.set_requested_size(interRead.ValueSize); + response->Record.set_requested_key(interRead.Key); + } + if constexpr (!std::is_same_v<TResponse, TEvKeyValue::TEvGetStatusResponse>) { + if (intermediate->HasCookie) { + response->Record.set_cookie(intermediate->Cookie); + } + } + + if (errorDescription) { + response->Record.set_msg(errorDescription); + } + + ResourceMetrics->Network.Increment(response->Record.ByteSize()); + + intermediate->IsReplied = true; + ctx.Send(intermediate->RespondTo, response.release()); + if (info) { + intermediate->UpdateStat(); + OnRequestComplete(intermediate->RequestUid, intermediate->CreatedAtGeneration, intermediate->CreatedAtStep, + ctx, info, TEvKeyValue::TEvNotify::ConvertStatus(status), intermediate->Stat); + } else { //metrics change report in OnRequestComplete is not done + ResourceMetrics->TryUpdate(ctx); + RequestInputTime.erase(intermediate->RequestUid); + } + } + + bool PrepareReadRequest(const TActorContext &ctx, TEvKeyValue::TEvRead::TPtr &ev, + THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestTyp); + bool PrepareReadRangeRequest(const TActorContext &ctx, TEvKeyValue::TEvReadRange::TPtr &ev, + THolder<TIntermediate> &intermediate, TRequestType::EType *outRequestType); + bool PrepareExecuteTransactionRequest(const TActorContext &ctx, TEvKeyValue::TEvExecuteTransaction::TPtr &ev, + THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info); + TPrepareResult PrepareOneGetStatus(TIntermediate::TGetStatus &cmd, ui64 publicStorageChannel, + const TTabletStorageInfo *info); + bool PrepareGetStatusRequest(const TActorContext &ctx, TEvKeyValue::TEvGetStatus::TPtr &ev, + THolder<TIntermediate> &intermediate, const TTabletStorageInfo *info); + bool PrepareObtainLockRequest(const TActorContext &ctx, TEvKeyValue::TEvObtainLock::TPtr &ev, + THolder<TIntermediate> &intermediate); + + template <typename TRequestType> + void PostponeIntermediate(THolder<TIntermediate> &&intermediate) { + intermediate->Stat.EnqueuedAs = Queue.size() + 1; + Queue.push_back(std::move(intermediate)); + } + void ProcessPostponedIntermediate(const TActorContext& ctx, THolder<TIntermediate> &&intermediate, + const TTabletStorageInfo *info); + bool ConvertRange(const NKikimrClient::TKeyValueRequest::TKeyRange& from, TKeyRange *to, const TActorContext& ctx, THolder<TIntermediate>& intermediate, const char *cmd, ui32 index); - struct TConvertRangeResult { - TString ErrorMsg; - bool WithError = false; - }; - - TConvertRangeResult ConvertRange(const NKikimrKeyValue::KVRange& range, TKeyRange *to, const char *cmd) - { - if (range.has_from_key_inclusive()) { - to->HasFrom = true; - to->KeyFrom = range.from_key_inclusive(); - to->IncludeFrom = true; - } else if (range.has_from_key_exclusive()) { - to->HasFrom = true; - to->KeyFrom = range.from_key_exclusive(); - to->IncludeFrom = false; - } else { - to->HasFrom = false; - } - - if (range.has_to_key_inclusive()) { - to->HasTo = true; - to->KeyTo = range.to_key_inclusive(); - to->IncludeTo = true; - } else if (range.has_to_key_exclusive()) { - to->HasTo = true; - to->KeyTo = range.to_key_exclusive(); - to->IncludeTo = false; - } else { - to->HasTo = false; - } - - if (to->HasFrom && to->HasTo) { - if (!to->IncludeFrom && !to->IncludeTo && to->KeyFrom >= to->KeyTo) { - TString msg = TStringBuilder() << "KeyValue# " << TabletId - << " Range.KeyFrom >= Range.KeyTo and both exclusive in " << cmd - << " Marker# KV31"; - return TConvertRangeResult{msg, true}; - } - if (to->KeyFrom > to->KeyTo) { - TString msg = TStringBuilder() << "KeyValue# " << TabletId - << " Range.KeyFrom > Range.KeyTo and both exclusive in " << cmd - << " Marker# KV33"; - return TConvertRangeResult{msg, true}; - } - } - - return {}; - } - + struct TConvertRangeResult { + TString ErrorMsg; + bool WithError = false; + }; + + TConvertRangeResult ConvertRange(const NKikimrKeyValue::KVRange& range, TKeyRange *to, const char *cmd) + { + if (range.has_from_key_inclusive()) { + to->HasFrom = true; + to->KeyFrom = range.from_key_inclusive(); + to->IncludeFrom = true; + } else if (range.has_from_key_exclusive()) { + to->HasFrom = true; + to->KeyFrom = range.from_key_exclusive(); + to->IncludeFrom = false; + } else { + to->HasFrom = false; + } + + if (range.has_to_key_inclusive()) { + to->HasTo = true; + to->KeyTo = range.to_key_inclusive(); + to->IncludeTo = true; + } else if (range.has_to_key_exclusive()) { + to->HasTo = true; + to->KeyTo = range.to_key_exclusive(); + to->IncludeTo = false; + } else { + to->HasTo = false; + } + + if (to->HasFrom && to->HasTo) { + if (!to->IncludeFrom && !to->IncludeTo && to->KeyFrom >= to->KeyTo) { + TString msg = TStringBuilder() << "KeyValue# " << TabletId + << " Range.KeyFrom >= Range.KeyTo and both exclusive in " << cmd + << " Marker# KV31"; + return TConvertRangeResult{msg, true}; + } + if (to->KeyFrom > to->KeyTo) { + TString msg = TStringBuilder() << "KeyValue# " << TabletId + << " Range.KeyFrom > Range.KeyTo and both exclusive in " << cmd + << " Marker# KV33"; + return TConvertRangeResult{msg, true}; + } + } + + return {}; + } + template<typename Container, typename Iterator = typename Container::iterator> static std::pair<Iterator, Iterator> GetRange(const TKeyRange& range, Container& container) { auto first = !range.HasFrom ? container.begin() diff --git a/ydb/core/keyvalue/keyvalue_storage_read_request.cpp b/ydb/core/keyvalue/keyvalue_storage_read_request.cpp index 3adab0a8631..d88ca3f1d13 100644 --- a/ydb/core/keyvalue/keyvalue_storage_read_request.cpp +++ b/ydb/core/keyvalue/keyvalue_storage_read_request.cpp @@ -1,482 +1,482 @@ -#include "keyvalue_storage_read_request.h" -#include "keyvalue_const.h" - +#include "keyvalue_storage_read_request.h" +#include "keyvalue_const.h" + #include <ydb/core/util/stlog.h> -#include <library/cpp/actors/protos/services_common.pb.h> - - -namespace NKikimr { -namespace NKeyValue { - -#define STLOG_WITH_ERROR_DESCRIPTION(VARIABLE, PRIO, COMP, MARKER, TEXT, ...) \ - do { \ - struct MARKER {}; \ - VARIABLE = (TStringBuilder() << VARIABLE << Endl \ - << ::NKikimr::NStLog::TMessage<MARKER>(__FILE__, __LINE__, #MARKER, TStringBuilder() << TEXT) \ - STLOG_PARAMS(__VA_ARGS__)); \ - STLOG(PRIO, COMP, MARKER, TEXT, __VA_ARGS__); \ - } while(false) \ -// STLOG_WITH_ERROR_DESCRIPTION - - -class TKeyValueStorageReadRequest : public TActorBootstrapped<TKeyValueStorageReadRequest> { - struct TGetBatch { - TStackVec<ui32, 1> ReadItemIndecies; - ui32 GroupId; - ui32 Cookie; - TInstant SentTime; - - TGetBatch(ui32 groupId, ui32 cookie) - : GroupId(groupId) - , Cookie(cookie) - {} - }; - - struct TReadItemInfo { - TIntermediate::TRead *Read; - TIntermediate::TRead::TReadItem *ReadItem; - }; - - THolder<TIntermediate> IntermediateResult; - const TTabletStorageInfo *TabletInfo; - TStackVec<TGetBatch, 1> Batches; - - ui32 ReceivedGetResults = 0; - TString ErrorDescription; - - TStackVec<TReadItemInfo, 1> ReadItems; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::KEYVALUE_ACTOR; - } - - std::variant<TIntermediate::TRead, TIntermediate::TRangeRead>& GetCommand() const { - return *IntermediateResult->ReadCommand; - } - - bool IsRead() const { - return std::holds_alternative<TIntermediate::TRead>(GetCommand()); - } - - bool IsRangeRead() const { - return std::holds_alternative<TIntermediate::TRangeRead>(GetCommand()); - } - - void AddRead(TIntermediate::TRead &read) { - for (auto &readItem : read.ReadItems) { - ReadItems.push_back({&read, &readItem}); - } - } - - NKikimrBlobStorage::EGetHandleClass GetHandleClass() const { - auto visitor = [&] (auto &request) { - return request.HandleClass; - }; - return std::visit(visitor, GetCommand()); - } - - void Bootstrap() { - if (IntermediateResult->Deadline != TInstant::Max()) { - TInstant now = TActivationContext::Now(); - if (IntermediateResult->Deadline <= now) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV313, - "Deadline reached before processing request.", - (KeyValue, TabletInfo->TabletID), - (Deadline, IntermediateResult->Deadline.MilliSeconds()), - (Now, now.MilliSeconds()), - (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), - (EnqueuedAs, IntermediateResult->Stat.EnqueuedAs)); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT); - return; - } - - const TDuration timeout = IntermediateResult->Deadline - now; - Schedule(timeout, new TEvents::TEvWakeup()); - } - - ui32 readCount = 0; - auto addReadItems = [&](auto &request) { - using Type = std::decay_t<decltype(request)>; - if constexpr (std::is_same_v<Type, TIntermediate::TRead>) { - AddRead(request); - readCount++; - } else { - for (auto &read : request.Reads) { - AddRead(read); - readCount++; - } - } - }; - std::visit(addReadItems, GetCommand()); - - if (ReadItems.empty()) { - auto getStatus = [&](auto &request) { - return request.Status; - }; - NKikimrProto::EReplyStatus status = std::visit(getStatus, GetCommand()); - - STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV320, "Inline read request", - (KeyValue, TabletInfo->TabletID), - (Status, status)); - bool isError = status != NKikimrProto::OK - && status != NKikimrProto::UNKNOWN - && status != NKikimrProto::NODATA - && status != NKikimrProto::OVERRUN; - if (isError) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV321, - "Expected OK, UNKNOWN, NODATA or OVERRUN but given " << NKikimrProto::EReplyStatus_Name(status)); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - } else { - STLOG(NLog::PRI_DEBUG, NKikimrServices::KEYVALUE, KV322, - "Expected OK or UNKNOWN and given " << NKikimrProto::EReplyStatus_Name(status) - << " readCount# " << readCount); - NKikimrKeyValue::Statuses::ReplyStatus replyStatus = ConvertStatus(status); - if (!readCount) { - replyStatus = NKikimrKeyValue::Statuses::RSTATUS_NO_DATA; - } else if (status == NKikimrProto::UNKNOWN) { - replyStatus = NKikimrKeyValue::Statuses::RSTATUS_OK; - } - - SendResponseAndPassAway(replyStatus); - } - } - - Become(&TThis::StateWait); - SendGets(); - } - - void SendGets() { - THashMap<ui32, ui32> mapFromGroupToBatch; - - for (ui32 readItemIdx = 0; readItemIdx < ReadItems.size(); ++readItemIdx) { - TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; - TLogoBlobID &id = readItem.LogoBlobId; - ui32 group = TabletInfo->GroupFor(id.Channel(), id.Generation()); - - // INVALID GROUP - if (group == Max<ui32>()) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV315, - "InternalError can't find correct group", - (KeyValue, TabletInfo->TabletID), - (Channel, id.Channel()), - (Generation, id.Generation())); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - return; - } - - auto it = mapFromGroupToBatch.find(group); - if (it == mapFromGroupToBatch.end()) { - it = mapFromGroupToBatch.emplace(group, Batches.size()).first; - Batches.emplace_back(group, Batches.size()); - } - TGetBatch &batch = Batches[it->second]; - batch.ReadItemIndecies.push_back(readItemIdx); - } - - NKikimrBlobStorage::EGetHandleClass handleClass = GetHandleClass(); - - for (TGetBatch &batch : Batches) { - TArrayHolder<TEvBlobStorage::TEvGet::TQuery> readQueries( - new TEvBlobStorage::TEvGet::TQuery[batch.ReadItemIndecies.size()]); - for (ui32 readQueryIdx = 0; readQueryIdx < batch.ReadItemIndecies.size(); ++readQueryIdx) { - ui32 readItemIdx = batch.ReadItemIndecies[readQueryIdx]; - TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; - readQueries[readQueryIdx].Set(readItem.LogoBlobId, readItem.BlobOffset, readItem.BlobSize); - readItem.InFlight = true; - } - - std::unique_ptr<TEvBlobStorage::TEvGet> get = std::make_unique<TEvBlobStorage::TEvGet>( - readQueries, batch.ReadItemIndecies.size(), IntermediateResult->Deadline, handleClass, false); - - SendToBSProxy(TActivationContext::AsActorContext(), batch.GroupId, get.release(), - batch.Cookie); - batch.SentTime = TActivationContext::Now(); - } - } - - void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev) { - TEvBlobStorage::TEvGetResult *result = ev->Get(); - STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV20, "Received GetResult", - (KeyValue, TabletInfo->TabletID), - (GroupId, result->GroupId), - (Status, result->Status), - (ResponseSz, result->ResponseSz), - (ErrorReason, result->ErrorReason), - (ReadRequestCookie, IntermediateResult->Cookie)); - - if (ev->Cookie >= Batches.size()) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV319, - "Received EvGetResult with an unexpected cookie.", - (KeyValue, TabletInfo->TabletID), - (Cookie, ev->Cookie), - (SentGets, Batches.size()), - (GroupId, result->GroupId), - (Status, result->Status), - (Deadline, IntermediateResult->Deadline.MilliSeconds()), - (Now, TActivationContext::Now().MilliSeconds()), - (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), - (ErrorReason, result->ErrorReason)); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - return; - } - - TGetBatch &batch = Batches[ev->Cookie]; - - if (result->GroupId != batch.GroupId) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV318, - "Received EvGetResult from an unexpected storage group.", - (KeyValue, TabletInfo->TabletID), - (GroupId, result->GroupId), - (ExpecetedGroupId, batch.GroupId), - (Status, result->Status), - (Deadline, IntermediateResult->Deadline.MilliSeconds()), - (Now, TActivationContext::Now().MilliSeconds()), - (SentAt, batch.SentTime), - (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), - (ErrorReason, result->ErrorReason)); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - return; - } - - if (result->Status != NKikimrProto::OK) { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV316, - "Unexpected EvGetResult.", - (KeyValue, TabletInfo->TabletID), - (Status, result->Status), - (Deadline, IntermediateResult->Deadline.MilliSeconds()), - (Now, TActivationContext::Now().MilliSeconds()), - (SentAt, batch.SentTime), - (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), - (ErrorReason, result->ErrorReason)); - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - return; - } - - - bool hasErrorResponses = false; - for (ui32 readQueryIdx = 0; readQueryIdx < batch.ReadItemIndecies.size(); ++readQueryIdx) { - ui32 readItemIdx = batch.ReadItemIndecies[readQueryIdx]; - TEvBlobStorage::TEvGetResult::TResponse &response = ev->Get()->Responses[readQueryIdx]; - TIntermediate::TRead &read = *ReadItems[readItemIdx].Read; - TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; - read.Status = response.Status; - - if (response.Status == NKikimrProto::OK) { - if (read.Value.size() != read.ValueSize) { - read.Value.resize(read.ValueSize); - } - Y_VERIFY_S(response.Buffer.size() == readItem.BlobSize, - "response.Buffer.size()# " << response.Buffer.size() - << " readItem.BlobSize# " << readItem.BlobSize); - Y_VERIFY_S(readItem.ValueOffset + readItem.BlobSize <= read.ValueSize, - "readItem.ValueOffset# " << readItem.ValueOffset - << " readItem.BlobSize# " << readItem.BlobSize - << " read.ValueSize# " << read.ValueSize); - memcpy(const_cast<char *>(read.Value.data()) + readItem.ValueOffset, response.Buffer.data(), response.Buffer.size()); - IntermediateResult->Stat.GroupReadBytes[std::make_pair(response.Id.Channel(), batch.GroupId)] += response.Buffer.size(); - // FIXME: count distinct blobs?" keyvalue_storage_request.cpp:279 - IntermediateResult->Stat.GroupReadIops[std::make_pair(response.Id.Channel(), batch.GroupId)] += 1; - } else { - STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV317, - "Unexpected EvGetResult.", - (KeyValue, TabletInfo->TabletID), - (Status, result->Status), - (ResponseStatus, response.Status), - (Deadline, IntermediateResult->Deadline.MilliSeconds()), - (Now, TActivationContext::Now().MilliSeconds()), - (SentAt, batch.SentTime), - (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), - (ErrorReason, result->ErrorReason)); - hasErrorResponses = true; - } - - Y_VERIFY(response.Status != NKikimrProto::UNKNOWN); - readItem.Status = response.Status; - readItem.InFlight = false; - } - if (hasErrorResponses) { - ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); - return; - } - - ReceivedGetResults++; - if (ReceivedGetResults == Batches.size()) { - SendResponseAndPassAway(IntermediateResult->IsTruncated ? - NKikimrKeyValue::Statuses::RSTATUS_OVERRUN : - NKikimrKeyValue::Statuses::RSTATUS_OK); - } - } - - void SendNotify(NKikimrKeyValue::Statuses::ReplyStatus status) { - IntermediateResult->UpdateStat(); - Send(IntermediateResult->KeyValueActorId, new TEvKeyValue::TEvNotify( - IntermediateResult->RequestUid, - IntermediateResult->CreatedAtGeneration, IntermediateResult->CreatedAtStep, - IntermediateResult->Stat, status)); - } - - std::unique_ptr<TEvKeyValue::TEvReadResponse> CreateReadResponse(NKikimrKeyValue::Statuses::ReplyStatus status, - const TString &errorDescription) - { - auto response = std::make_unique<TEvKeyValue::TEvReadResponse>(); - response->Record.set_status(status); - if (errorDescription) { - response->Record.set_msg(errorDescription); - } - if (IntermediateResult->HasCookie) { - response->Record.set_cookie(IntermediateResult->Cookie); - } - return response; - } - - std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> CreateReadRangeResponse( - NKikimrKeyValue::Statuses::ReplyStatus status, const TString &errorDescription) - { - auto response = std::make_unique<TEvKeyValue::TEvReadRangeResponse>(); - response->Record.set_status(status); - if (errorDescription) { - response->Record.set_msg(errorDescription); - } - return response; - } - - std::unique_ptr<IEventBase> MakeErrorResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { - if (IsRead()) { - return CreateReadResponse(status, ErrorDescription); - } else { - return CreateReadRangeResponse(status, ErrorDescription); - } - } - - void ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::ReplyStatus status) { - std::unique_ptr<IEventBase> response = MakeErrorResponse(status); - Send(IntermediateResult->RespondTo, response.release()); - IntermediateResult->IsReplied = true; - SendNotify(status); - PassAway(); - } - - TString MakeErrorMsg(const TString &msg) const { - TStringBuilder builder; - if (ErrorDescription) { - builder << ErrorDescription << ';'; - } - if (msg) { - builder << "Message# " << msg << ';'; - } - return builder; - } - - std::unique_ptr<TEvKeyValue::TEvReadResponse> MakeReadResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { - auto &cmd = GetCommand(); - Y_VERIFY(std::holds_alternative<TIntermediate::TRead>(cmd)); - TIntermediate::TRead &interRead = std::get<TIntermediate::TRead>(cmd); - - TString errorMsg = MakeErrorMsg(interRead.Message); - std::unique_ptr<TEvKeyValue::TEvReadResponse> response = CreateReadResponse(status, errorMsg); - - response->Record.set_requested_key(interRead.Key); - response->Record.set_requested_offset(interRead.Offset); - response->Record.set_requested_size(interRead.RequestedSize); - response->Record.set_value(interRead.Value); - - return response; - } - - NKikimrKeyValue::Statuses::ReplyStatus ConvertStatus(NKikimrProto::EReplyStatus status) { - if (status == NKikimrProto::OK) { - return NKikimrKeyValue::Statuses::RSTATUS_OK; - } else if (status == NKikimrProto::OVERRUN) { - return NKikimrKeyValue::Statuses::RSTATUS_OVERRUN; - } else if (status == NKikimrProto::NODATA) { - return NKikimrKeyValue::Statuses::RSTATUS_NO_DATA; - } else { - return NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR; - } - } - - std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> MakeReadRangeResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { - auto &cmd = GetCommand(); - Y_VERIFY(std::holds_alternative<TIntermediate::TRangeRead>(cmd)); - TIntermediate::TRangeRead &interRange = std::get<TIntermediate::TRangeRead>(cmd); - - TStringBuilder msgBuilder; - if (ErrorDescription) { - msgBuilder << ErrorDescription << ';'; - } - for (ui32 idx = 0; idx < interRange.Reads.size(); ++idx) { - auto &interRead = interRange.Reads[idx]; - if (interRead.Message) { - msgBuilder << "Messages[" << idx << "]# " << interRead.Message << ';'; - } - } - - std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> response = CreateReadRangeResponse(status, msgBuilder); - NKikimrKeyValue::ReadRangeResult &readRangeResult = response->Record; - - for (auto &interRead : interRange.Reads) { - auto *kvp = readRangeResult.add_pair(); - kvp->set_key(interRead.Key); - kvp->set_value(interRead.Value); - kvp->set_value_size(interRead.ValueSize); - kvp->set_creation_unix_time(interRead.CreationUnixTime); - ui32 storageChannel = MainStorageChannelInPublicApi; - if (interRead.StorageChannel == NKikimrClient::TKeyValueRequest::INLINE) { - storageChannel = InlineStorageChannelInPublicApi; - } else { - storageChannel = interRead.StorageChannel + MainStorageChannelInPublicApi; - } - kvp->set_storage_channel(storageChannel); - kvp->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); - } - readRangeResult.set_status(status); - - return response; - } - - std::unique_ptr<IEventBase> MakeResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { - if (IsRead()) { - return MakeReadResponse(status); - } else { - return MakeReadRangeResponse(status); - } - } - - void SendResponseAndPassAway(NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) { - STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV34, "Send respose", - (KeyValue, TabletInfo->TabletID), - (Status, NKikimrKeyValue::Statuses_ReplyStatus_Name(status)), - (ReadRequestCookie, IntermediateResult->Cookie)); - std::unique_ptr<IEventBase> response = MakeResponse(status); - Send(IntermediateResult->RespondTo, response.release()); - IntermediateResult->IsReplied = true; - SendNotify(status); - PassAway(); - } - - STATEFN(StateWait) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvGetResult, Handle); - default: - Y_FAIL(); - } - } - - TKeyValueStorageReadRequest(THolder<TIntermediate> &&intermediate, - const TTabletStorageInfo *tabletInfo) - : IntermediateResult(std::move(intermediate)) - , TabletInfo(tabletInfo) - {} -}; - - -IActor* CreateKeyValueStorageReadRequest(THolder<TIntermediate>&& intermediate, - const TTabletStorageInfo *tabletInfo) -{ - return new TKeyValueStorageReadRequest(std::move(intermediate), tabletInfo); -} - -} // NKeyValue - -} // NKikimr +#include <library/cpp/actors/protos/services_common.pb.h> + + +namespace NKikimr { +namespace NKeyValue { + +#define STLOG_WITH_ERROR_DESCRIPTION(VARIABLE, PRIO, COMP, MARKER, TEXT, ...) \ + do { \ + struct MARKER {}; \ + VARIABLE = (TStringBuilder() << VARIABLE << Endl \ + << ::NKikimr::NStLog::TMessage<MARKER>(__FILE__, __LINE__, #MARKER, TStringBuilder() << TEXT) \ + STLOG_PARAMS(__VA_ARGS__)); \ + STLOG(PRIO, COMP, MARKER, TEXT, __VA_ARGS__); \ + } while(false) \ +// STLOG_WITH_ERROR_DESCRIPTION + + +class TKeyValueStorageReadRequest : public TActorBootstrapped<TKeyValueStorageReadRequest> { + struct TGetBatch { + TStackVec<ui32, 1> ReadItemIndecies; + ui32 GroupId; + ui32 Cookie; + TInstant SentTime; + + TGetBatch(ui32 groupId, ui32 cookie) + : GroupId(groupId) + , Cookie(cookie) + {} + }; + + struct TReadItemInfo { + TIntermediate::TRead *Read; + TIntermediate::TRead::TReadItem *ReadItem; + }; + + THolder<TIntermediate> IntermediateResult; + const TTabletStorageInfo *TabletInfo; + TStackVec<TGetBatch, 1> Batches; + + ui32 ReceivedGetResults = 0; + TString ErrorDescription; + + TStackVec<TReadItemInfo, 1> ReadItems; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::KEYVALUE_ACTOR; + } + + std::variant<TIntermediate::TRead, TIntermediate::TRangeRead>& GetCommand() const { + return *IntermediateResult->ReadCommand; + } + + bool IsRead() const { + return std::holds_alternative<TIntermediate::TRead>(GetCommand()); + } + + bool IsRangeRead() const { + return std::holds_alternative<TIntermediate::TRangeRead>(GetCommand()); + } + + void AddRead(TIntermediate::TRead &read) { + for (auto &readItem : read.ReadItems) { + ReadItems.push_back({&read, &readItem}); + } + } + + NKikimrBlobStorage::EGetHandleClass GetHandleClass() const { + auto visitor = [&] (auto &request) { + return request.HandleClass; + }; + return std::visit(visitor, GetCommand()); + } + + void Bootstrap() { + if (IntermediateResult->Deadline != TInstant::Max()) { + TInstant now = TActivationContext::Now(); + if (IntermediateResult->Deadline <= now) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV313, + "Deadline reached before processing request.", + (KeyValue, TabletInfo->TabletID), + (Deadline, IntermediateResult->Deadline.MilliSeconds()), + (Now, now.MilliSeconds()), + (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), + (EnqueuedAs, IntermediateResult->Stat.EnqueuedAs)); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT); + return; + } + + const TDuration timeout = IntermediateResult->Deadline - now; + Schedule(timeout, new TEvents::TEvWakeup()); + } + + ui32 readCount = 0; + auto addReadItems = [&](auto &request) { + using Type = std::decay_t<decltype(request)>; + if constexpr (std::is_same_v<Type, TIntermediate::TRead>) { + AddRead(request); + readCount++; + } else { + for (auto &read : request.Reads) { + AddRead(read); + readCount++; + } + } + }; + std::visit(addReadItems, GetCommand()); + + if (ReadItems.empty()) { + auto getStatus = [&](auto &request) { + return request.Status; + }; + NKikimrProto::EReplyStatus status = std::visit(getStatus, GetCommand()); + + STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV320, "Inline read request", + (KeyValue, TabletInfo->TabletID), + (Status, status)); + bool isError = status != NKikimrProto::OK + && status != NKikimrProto::UNKNOWN + && status != NKikimrProto::NODATA + && status != NKikimrProto::OVERRUN; + if (isError) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV321, + "Expected OK, UNKNOWN, NODATA or OVERRUN but given " << NKikimrProto::EReplyStatus_Name(status)); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + } else { + STLOG(NLog::PRI_DEBUG, NKikimrServices::KEYVALUE, KV322, + "Expected OK or UNKNOWN and given " << NKikimrProto::EReplyStatus_Name(status) + << " readCount# " << readCount); + NKikimrKeyValue::Statuses::ReplyStatus replyStatus = ConvertStatus(status); + if (!readCount) { + replyStatus = NKikimrKeyValue::Statuses::RSTATUS_NO_DATA; + } else if (status == NKikimrProto::UNKNOWN) { + replyStatus = NKikimrKeyValue::Statuses::RSTATUS_OK; + } + + SendResponseAndPassAway(replyStatus); + } + } + + Become(&TThis::StateWait); + SendGets(); + } + + void SendGets() { + THashMap<ui32, ui32> mapFromGroupToBatch; + + for (ui32 readItemIdx = 0; readItemIdx < ReadItems.size(); ++readItemIdx) { + TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; + TLogoBlobID &id = readItem.LogoBlobId; + ui32 group = TabletInfo->GroupFor(id.Channel(), id.Generation()); + + // INVALID GROUP + if (group == Max<ui32>()) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV315, + "InternalError can't find correct group", + (KeyValue, TabletInfo->TabletID), + (Channel, id.Channel()), + (Generation, id.Generation())); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + return; + } + + auto it = mapFromGroupToBatch.find(group); + if (it == mapFromGroupToBatch.end()) { + it = mapFromGroupToBatch.emplace(group, Batches.size()).first; + Batches.emplace_back(group, Batches.size()); + } + TGetBatch &batch = Batches[it->second]; + batch.ReadItemIndecies.push_back(readItemIdx); + } + + NKikimrBlobStorage::EGetHandleClass handleClass = GetHandleClass(); + + for (TGetBatch &batch : Batches) { + TArrayHolder<TEvBlobStorage::TEvGet::TQuery> readQueries( + new TEvBlobStorage::TEvGet::TQuery[batch.ReadItemIndecies.size()]); + for (ui32 readQueryIdx = 0; readQueryIdx < batch.ReadItemIndecies.size(); ++readQueryIdx) { + ui32 readItemIdx = batch.ReadItemIndecies[readQueryIdx]; + TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; + readQueries[readQueryIdx].Set(readItem.LogoBlobId, readItem.BlobOffset, readItem.BlobSize); + readItem.InFlight = true; + } + + std::unique_ptr<TEvBlobStorage::TEvGet> get = std::make_unique<TEvBlobStorage::TEvGet>( + readQueries, batch.ReadItemIndecies.size(), IntermediateResult->Deadline, handleClass, false); + + SendToBSProxy(TActivationContext::AsActorContext(), batch.GroupId, get.release(), + batch.Cookie); + batch.SentTime = TActivationContext::Now(); + } + } + + void Handle(TEvBlobStorage::TEvGetResult::TPtr &ev) { + TEvBlobStorage::TEvGetResult *result = ev->Get(); + STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV20, "Received GetResult", + (KeyValue, TabletInfo->TabletID), + (GroupId, result->GroupId), + (Status, result->Status), + (ResponseSz, result->ResponseSz), + (ErrorReason, result->ErrorReason), + (ReadRequestCookie, IntermediateResult->Cookie)); + + if (ev->Cookie >= Batches.size()) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV319, + "Received EvGetResult with an unexpected cookie.", + (KeyValue, TabletInfo->TabletID), + (Cookie, ev->Cookie), + (SentGets, Batches.size()), + (GroupId, result->GroupId), + (Status, result->Status), + (Deadline, IntermediateResult->Deadline.MilliSeconds()), + (Now, TActivationContext::Now().MilliSeconds()), + (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), + (ErrorReason, result->ErrorReason)); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + return; + } + + TGetBatch &batch = Batches[ev->Cookie]; + + if (result->GroupId != batch.GroupId) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV318, + "Received EvGetResult from an unexpected storage group.", + (KeyValue, TabletInfo->TabletID), + (GroupId, result->GroupId), + (ExpecetedGroupId, batch.GroupId), + (Status, result->Status), + (Deadline, IntermediateResult->Deadline.MilliSeconds()), + (Now, TActivationContext::Now().MilliSeconds()), + (SentAt, batch.SentTime), + (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), + (ErrorReason, result->ErrorReason)); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + return; + } + + if (result->Status != NKikimrProto::OK) { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV316, + "Unexpected EvGetResult.", + (KeyValue, TabletInfo->TabletID), + (Status, result->Status), + (Deadline, IntermediateResult->Deadline.MilliSeconds()), + (Now, TActivationContext::Now().MilliSeconds()), + (SentAt, batch.SentTime), + (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), + (ErrorReason, result->ErrorReason)); + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + return; + } + + + bool hasErrorResponses = false; + for (ui32 readQueryIdx = 0; readQueryIdx < batch.ReadItemIndecies.size(); ++readQueryIdx) { + ui32 readItemIdx = batch.ReadItemIndecies[readQueryIdx]; + TEvBlobStorage::TEvGetResult::TResponse &response = ev->Get()->Responses[readQueryIdx]; + TIntermediate::TRead &read = *ReadItems[readItemIdx].Read; + TIntermediate::TRead::TReadItem &readItem = *ReadItems[readItemIdx].ReadItem; + read.Status = response.Status; + + if (response.Status == NKikimrProto::OK) { + if (read.Value.size() != read.ValueSize) { + read.Value.resize(read.ValueSize); + } + Y_VERIFY_S(response.Buffer.size() == readItem.BlobSize, + "response.Buffer.size()# " << response.Buffer.size() + << " readItem.BlobSize# " << readItem.BlobSize); + Y_VERIFY_S(readItem.ValueOffset + readItem.BlobSize <= read.ValueSize, + "readItem.ValueOffset# " << readItem.ValueOffset + << " readItem.BlobSize# " << readItem.BlobSize + << " read.ValueSize# " << read.ValueSize); + memcpy(const_cast<char *>(read.Value.data()) + readItem.ValueOffset, response.Buffer.data(), response.Buffer.size()); + IntermediateResult->Stat.GroupReadBytes[std::make_pair(response.Id.Channel(), batch.GroupId)] += response.Buffer.size(); + // FIXME: count distinct blobs?" keyvalue_storage_request.cpp:279 + IntermediateResult->Stat.GroupReadIops[std::make_pair(response.Id.Channel(), batch.GroupId)] += 1; + } else { + STLOG_WITH_ERROR_DESCRIPTION(ErrorDescription, NLog::PRI_ERROR, NKikimrServices::KEYVALUE, KV317, + "Unexpected EvGetResult.", + (KeyValue, TabletInfo->TabletID), + (Status, result->Status), + (ResponseStatus, response.Status), + (Deadline, IntermediateResult->Deadline.MilliSeconds()), + (Now, TActivationContext::Now().MilliSeconds()), + (SentAt, batch.SentTime), + (GotAt, IntermediateResult->Stat.IntermediateCreatedAt.MilliSeconds()), + (ErrorReason, result->ErrorReason)); + hasErrorResponses = true; + } + + Y_VERIFY(response.Status != NKikimrProto::UNKNOWN); + readItem.Status = response.Status; + readItem.InFlight = false; + } + if (hasErrorResponses) { + ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); + return; + } + + ReceivedGetResults++; + if (ReceivedGetResults == Batches.size()) { + SendResponseAndPassAway(IntermediateResult->IsTruncated ? + NKikimrKeyValue::Statuses::RSTATUS_OVERRUN : + NKikimrKeyValue::Statuses::RSTATUS_OK); + } + } + + void SendNotify(NKikimrKeyValue::Statuses::ReplyStatus status) { + IntermediateResult->UpdateStat(); + Send(IntermediateResult->KeyValueActorId, new TEvKeyValue::TEvNotify( + IntermediateResult->RequestUid, + IntermediateResult->CreatedAtGeneration, IntermediateResult->CreatedAtStep, + IntermediateResult->Stat, status)); + } + + std::unique_ptr<TEvKeyValue::TEvReadResponse> CreateReadResponse(NKikimrKeyValue::Statuses::ReplyStatus status, + const TString &errorDescription) + { + auto response = std::make_unique<TEvKeyValue::TEvReadResponse>(); + response->Record.set_status(status); + if (errorDescription) { + response->Record.set_msg(errorDescription); + } + if (IntermediateResult->HasCookie) { + response->Record.set_cookie(IntermediateResult->Cookie); + } + return response; + } + + std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> CreateReadRangeResponse( + NKikimrKeyValue::Statuses::ReplyStatus status, const TString &errorDescription) + { + auto response = std::make_unique<TEvKeyValue::TEvReadRangeResponse>(); + response->Record.set_status(status); + if (errorDescription) { + response->Record.set_msg(errorDescription); + } + return response; + } + + std::unique_ptr<IEventBase> MakeErrorResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { + if (IsRead()) { + return CreateReadResponse(status, ErrorDescription); + } else { + return CreateReadRangeResponse(status, ErrorDescription); + } + } + + void ReplyErrorAndPassAway(NKikimrKeyValue::Statuses::ReplyStatus status) { + std::unique_ptr<IEventBase> response = MakeErrorResponse(status); + Send(IntermediateResult->RespondTo, response.release()); + IntermediateResult->IsReplied = true; + SendNotify(status); + PassAway(); + } + + TString MakeErrorMsg(const TString &msg) const { + TStringBuilder builder; + if (ErrorDescription) { + builder << ErrorDescription << ';'; + } + if (msg) { + builder << "Message# " << msg << ';'; + } + return builder; + } + + std::unique_ptr<TEvKeyValue::TEvReadResponse> MakeReadResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { + auto &cmd = GetCommand(); + Y_VERIFY(std::holds_alternative<TIntermediate::TRead>(cmd)); + TIntermediate::TRead &interRead = std::get<TIntermediate::TRead>(cmd); + + TString errorMsg = MakeErrorMsg(interRead.Message); + std::unique_ptr<TEvKeyValue::TEvReadResponse> response = CreateReadResponse(status, errorMsg); + + response->Record.set_requested_key(interRead.Key); + response->Record.set_requested_offset(interRead.Offset); + response->Record.set_requested_size(interRead.RequestedSize); + response->Record.set_value(interRead.Value); + + return response; + } + + NKikimrKeyValue::Statuses::ReplyStatus ConvertStatus(NKikimrProto::EReplyStatus status) { + if (status == NKikimrProto::OK) { + return NKikimrKeyValue::Statuses::RSTATUS_OK; + } else if (status == NKikimrProto::OVERRUN) { + return NKikimrKeyValue::Statuses::RSTATUS_OVERRUN; + } else if (status == NKikimrProto::NODATA) { + return NKikimrKeyValue::Statuses::RSTATUS_NO_DATA; + } else { + return NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR; + } + } + + std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> MakeReadRangeResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { + auto &cmd = GetCommand(); + Y_VERIFY(std::holds_alternative<TIntermediate::TRangeRead>(cmd)); + TIntermediate::TRangeRead &interRange = std::get<TIntermediate::TRangeRead>(cmd); + + TStringBuilder msgBuilder; + if (ErrorDescription) { + msgBuilder << ErrorDescription << ';'; + } + for (ui32 idx = 0; idx < interRange.Reads.size(); ++idx) { + auto &interRead = interRange.Reads[idx]; + if (interRead.Message) { + msgBuilder << "Messages[" << idx << "]# " << interRead.Message << ';'; + } + } + + std::unique_ptr<TEvKeyValue::TEvReadRangeResponse> response = CreateReadRangeResponse(status, msgBuilder); + NKikimrKeyValue::ReadRangeResult &readRangeResult = response->Record; + + for (auto &interRead : interRange.Reads) { + auto *kvp = readRangeResult.add_pair(); + kvp->set_key(interRead.Key); + kvp->set_value(interRead.Value); + kvp->set_value_size(interRead.ValueSize); + kvp->set_creation_unix_time(interRead.CreationUnixTime); + ui32 storageChannel = MainStorageChannelInPublicApi; + if (interRead.StorageChannel == NKikimrClient::TKeyValueRequest::INLINE) { + storageChannel = InlineStorageChannelInPublicApi; + } else { + storageChannel = interRead.StorageChannel + MainStorageChannelInPublicApi; + } + kvp->set_storage_channel(storageChannel); + kvp->set_status(NKikimrKeyValue::Statuses::RSTATUS_OK); + } + readRangeResult.set_status(status); + + return response; + } + + std::unique_ptr<IEventBase> MakeResponse(NKikimrKeyValue::Statuses::ReplyStatus status) { + if (IsRead()) { + return MakeReadResponse(status); + } else { + return MakeReadRangeResponse(status); + } + } + + void SendResponseAndPassAway(NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) { + STLOG(NLog::PRI_INFO, NKikimrServices::KEYVALUE, KV34, "Send respose", + (KeyValue, TabletInfo->TabletID), + (Status, NKikimrKeyValue::Statuses_ReplyStatus_Name(status)), + (ReadRequestCookie, IntermediateResult->Cookie)); + std::unique_ptr<IEventBase> response = MakeResponse(status); + Send(IntermediateResult->RespondTo, response.release()); + IntermediateResult->IsReplied = true; + SendNotify(status); + PassAway(); + } + + STATEFN(StateWait) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvGetResult, Handle); + default: + Y_FAIL(); + } + } + + TKeyValueStorageReadRequest(THolder<TIntermediate> &&intermediate, + const TTabletStorageInfo *tabletInfo) + : IntermediateResult(std::move(intermediate)) + , TabletInfo(tabletInfo) + {} +}; + + +IActor* CreateKeyValueStorageReadRequest(THolder<TIntermediate>&& intermediate, + const TTabletStorageInfo *tabletInfo) +{ + return new TKeyValueStorageReadRequest(std::move(intermediate), tabletInfo); +} + +} // NKeyValue + +} // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_storage_read_request.h b/ydb/core/keyvalue/keyvalue_storage_read_request.h index 20e01dc5e95..2a4d2256055 100644 --- a/ydb/core/keyvalue/keyvalue_storage_read_request.h +++ b/ydb/core/keyvalue/keyvalue_storage_read_request.h @@ -1,12 +1,12 @@ -#pragma once -#include "defs.h" -#include "keyvalue_events.h" - -namespace NKikimr { -namespace NKeyValue { - -IActor* CreateKeyValueStorageReadRequest(THolder<TIntermediate>&& intermediate, - const TTabletStorageInfo *tabletInfo); - -} // NKeyValue -} // NKikimr +#pragma once +#include "defs.h" +#include "keyvalue_events.h" + +namespace NKikimr { +namespace NKeyValue { + +IActor* CreateKeyValueStorageReadRequest(THolder<TIntermediate>&& intermediate, + const TTabletStorageInfo *tabletInfo); + +} // NKeyValue +} // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_storage_read_request_ut.cpp b/ydb/core/keyvalue/keyvalue_storage_read_request_ut.cpp index 3d33155ad56..15ee4b0c9bb 100644 --- a/ydb/core/keyvalue/keyvalue_storage_read_request_ut.cpp +++ b/ydb/core/keyvalue/keyvalue_storage_read_request_ut.cpp @@ -1,459 +1,459 @@ -#include "keyvalue_storage_read_request.h" - +#include "keyvalue_storage_read_request.h" + #include <ydb/core/util/testactorsys.h> - -#include <library/cpp/testing/unittest/registar.h> - - -namespace NKikimr { - -namespace NKeyValue { - -struct TBlobStorageMockState { - struct TBlob { - NKikimrProto::EReplyStatus Status = NKikimrProto::NODATA; - TString Buffer; - }; - - struct TGroup { - std::unordered_map<TLogoBlobID, TBlob, THash<TLogoBlobID>> Blobs; - NKikimrProto::EReplyStatus Status = NKikimrProto::OK; - std::optional<ui32> GroupId; - std::optional<ui32> Cookie; - }; - - std::unordered_map<ui32, TGroup> Groups; - - void Put(ui32 groupId, TLogoBlobID blobId, NKikimrProto::EReplyStatus status, const TString &buffer) { - TBlob &blob = Groups[groupId].Blobs[blobId]; - blob.Status = status; - blob.Buffer = buffer; - } - - std::unique_ptr<TEvBlobStorage::TEvGetResult> MakeGetResult(ui32 groupId, TEvBlobStorage::TEvGet *get, - std::function<const std::pair<NKikimrProto::EReplyStatus, const TString&>&(TLogoBlobID blobId)> getBlob) - { - TGroup &group = Groups[groupId]; - if (group.GroupId) { - groupId = *group.GroupId; - } - std::unique_ptr<TEvBlobStorage::TEvGetResult> getResult = std::make_unique<TEvBlobStorage::TEvGetResult>( - group.Status, get->QuerySize, groupId); - getResult->Responses.Reset(new TEvBlobStorage::TEvGetResult::TResponse[get->QuerySize]); - for (ui32 queryIdx = 0; queryIdx < get->QuerySize; ++queryIdx) { - auto &query = get->Queries[queryIdx]; - auto &response = getResult->Responses[queryIdx]; - response.Id = query.Id; - response.Shift = query.Shift; - response.RequestedSize = query.Size; - std::tie(response.Status, response.Buffer) = getBlob(query.Id); - if (response.Status == NKikimrProto::OK) { - TString buffer = TString::Uninitialized(query.Size); - memcpy(const_cast<char *>(buffer.data()), response.Buffer.data() + query.Shift, response.Buffer.size()); - response.Buffer = buffer; - } - } - return getResult; - } - - std::unique_ptr<TEvBlobStorage::TEvGetResult> OnGet(ui32 groupId, TEvBlobStorage::TEvGet *get) { - TGroup &group = Groups[groupId]; - - if (group.Status != NKikimrProto::OK) { - TString str(""); - return MakeGetResult(groupId, get, [&](...){ return std::make_pair(group.Status, str); }); - } - - return MakeGetResult(groupId, get, [&group](TLogoBlobID blobId) { - TBlob &blob = group.Blobs[blobId]; - return std::make_pair(blob.Status, blob.Buffer); - }); - } -}; - -struct TBlobStorageMock : TActorBootstrapped<TBlobStorageMock> { - ui32 GroupId; - TBlobStorageMockState *State; - - TBlobStorageMock(ui32 groupId, TBlobStorageMockState *state) - : GroupId(groupId) - , State(state) - {} - - void Bootstrap() { - Become(&TThis::Waiting); - } - - void Handle(TEvBlobStorage::TEvGet::TPtr &ev) { - std::unique_ptr<TEvBlobStorage::TEvGetResult> result = State->OnGet(GroupId, ev->Get()); - auto &group = State->Groups[GroupId]; - ui64 cookie = ev->Cookie; - if (group.Cookie) { - cookie = *group.Cookie; - } - Send(ev->Sender, result.release(), ev->Flags, cookie, std::move(ev->TraceId)); - } - - STATEFN(Waiting) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBlobStorage::TEvGet, Handle); - default: - Y_FAIL(); - } - } -}; - - -struct TTestEnv { - TBlobStorageMockState BlobStorageState; - - std::unique_ptr<TTabletStorageInfo> TabletInfo; - - std::unordered_map<ui64, TActorId> GroupActors; // [groupId, actorId] - - TTestEnv() - : TabletInfo(std::make_unique<TTabletStorageInfo>(1, TTabletTypes::KeyValue)) - { - } - - void AddStorageGroup(TTestActorSystem &runtime, ui64 groupId) { - if (GroupActors.count(groupId)) { - return; - } - - TActorId groupActor = runtime.Register(new TBlobStorageMock(groupId, &BlobStorageState), 1); - TActorId proxyId = MakeBlobStorageProxyID(groupId); - runtime.RegisterService(proxyId, groupActor); - GroupActors[groupId] = groupActor; - } - - void AddStorageGroups(TTestActorSystem &runtime, const std::vector<ui32> &groupIds) { - for (ui32 groupId : groupIds) { - AddStorageGroup(runtime, groupId); - } - } - - void BindGroupsToChannel(TTestActorSystem &runtime, const std::vector<ui32> &groupIds) - { - AddStorageGroups(runtime, groupIds); - for (ui32 channelIdx = 0; channelIdx < groupIds.size(); ++channelIdx) { - TabletInfo->Channels.emplace_back(channelIdx, TErasureType::Erasure4Plus2Block); - TabletInfo->Channels.back().History.emplace_back(1, groupIds[channelIdx]); - } - } -}; - - -struct TBuilderResult { - THolder<TIntermediate> Intermediate; - std::unordered_map<std::string, std::string> Values; -}; - - -struct TReadItem { - std::string Value; - TLogoBlobID BlobId; - ui32 Offset; - ui32 Size; - - TReadItem(const std::string &value, TLogoBlobID blobId, ui32 offset, ui32 size) - : Value(value) - , BlobId(blobId) - , Offset(offset) - , Size(size) - {} -}; - - -struct TReadRequestBuilder { - std::string Key; - std::vector<TReadItem> Items; - - TReadRequestBuilder(const std::string &key) - : Key(key) - {} - - TReadRequestBuilder& AddToEnd(const std::string &partOfValue, TLogoBlobID blobId, ui32 offset, ui32 size) { - Items.emplace_back(partOfValue, blobId, offset, size); - return *this; - } - - TBuilderResult Build(TActorId respondTo, TActorId keyValueActorId, ui32 channelGeneration = 1, ui32 channelStep = 1) - { - std::unique_ptr<TIntermediate> intermediate = std::make_unique<TIntermediate>(respondTo, keyValueActorId, - channelGeneration, channelStep, TRequestType::ReadOnly); - TStringBuilder valueBuilder; - for (auto &[value, blobId, offset, size] : Items) { - valueBuilder << value; - } - std::string value = valueBuilder; - - - intermediate->ReadCommand = TIntermediate::TRead(TString(Key), value.size(), 0, - NKikimrClient::TKeyValueRequest::MAIN); - TIntermediate::TRead &read = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); - ui32 valueOffset = 0; - for (auto &[value, blobId, offset, size] : Items) { - read.ReadItems.emplace_back(blobId, offset, size, valueOffset); - valueOffset += size; - } - TBuilderResult res; - res.Values[Key] = value; - res.Intermediate.Reset(intermediate.release()); - return res; - } -}; - - -struct TRangeReadRequestBuilder { - std::map<std::string, std::vector<TReadItem>> Reads; - ui64 CmdLimitBytes = Max<ui64>(); - bool IncludeData = true; - - TRangeReadRequestBuilder() - {} - - TRangeReadRequestBuilder& AddRead(const std::string &key, std::vector<TReadItem> &&items) { - Reads[key] = std::move(items); - return *this; - } - - TRangeReadRequestBuilder& AddRead(const std::string &key, const std::string &value, TLogoBlobID blobId) { - Reads[key] = {TReadItem(value, blobId, 0, value.size())}; - return *this; - } - - TBuilderResult Build(TActorId respondTo, TActorId keyValueActorId, ui32 channelGeneration = 1, ui32 channelStep = 1) - { - std::unique_ptr<TIntermediate> intermediate = std::make_unique<TIntermediate>(respondTo, keyValueActorId, - channelGeneration, channelStep, TRequestType::ReadOnly); - - TBuilderResult res; - intermediate->ReadCommand = TIntermediate::TRangeRead(); - auto &range = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); - - for (auto &[key, items] : Reads) { - TStringBuilder valueBuilder; - for (auto &[value, blobId, offset, size] : items) { - valueBuilder << value; - } - std::string value = valueBuilder; - - range.Reads.emplace_back(TString(key), value.size(), 0, NKikimrClient::TKeyValueRequest::MAIN); - TIntermediate::TRead &read = range.Reads.back(); - ui32 valueOffset = 0; - for (auto &[value, blobId, offset, size] : items) { - read.ReadItems.emplace_back(blobId, offset, size, valueOffset); - valueOffset += size; - } - res.Values[key] = value; - } - - res.Intermediate.Reset(intermediate.release()); - return res; - } -}; - - -Y_UNIT_TEST_SUITE(KeyValueReadStorage) { - -void RunTest(TTestEnv &env, TReadRequestBuilder &builder, - const std::vector<ui32> &groupIds, NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) { - TTestActorSystem runtime(1); - runtime.Start(); - runtime.SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); - env.BindGroupsToChannel(runtime, groupIds); - - TActorId edgeActor = runtime.AllocateEdgeActor(1); - auto [intermediate, expectedValues] = builder.Build(edgeActor, edgeActor, 1, 1); - - runtime.Register(CreateKeyValueStorageReadRequest(std::move(intermediate), env.TabletInfo.get()), 1); - - std::unique_ptr<IEventHandle> ev = runtime.WaitForEdgeActorEvent({edgeActor}); - UNIT_ASSERT(ev->Type == TEvKeyValue::EvReadResponse); - TEvKeyValue::TEvReadResponse *response = ev->Get<TEvKeyValue::TEvReadResponse>(); - NKikimrKeyValue::ReadResult &record = response->Record; - - if (status == NKikimrKeyValue::Statuses::RSTATUS_OK) { - UNIT_ASSERT_C(record.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) - << " Received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) - << " Message# " << record.msg()); - UNIT_ASSERT_VALUES_EQUAL(record.value(), expectedValues[record.requested_key()]); - } else { - UNIT_ASSERT_C(record.status() == status, "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) - << " received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) - << " Message# " << record.msg()); - } - - runtime.Stop(); -} - -Y_UNIT_TEST(ReadOk) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); - builder.AddToEnd("b", id, 0, 1); - - RunTest(env, builder, groupIds); -} - -Y_UNIT_TEST(ReadWithTwoPartsOk) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id1(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id1, NKikimrProto::OK, "b"); - builder.AddToEnd("b", id1, 0, 1); - - TLogoBlobID id2(1, 2, 3, 2, 1, 1); - env.BlobStorageState.Put(groupIds[2], id2, NKikimrProto::OK, "c"); - builder.AddToEnd("c", id2, 0, 1); - - RunTest(env, builder, groupIds); -} - -Y_UNIT_TEST(ReadNotWholeBlobOk) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 3, 0); - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "abc"); - builder.AddToEnd("b", id, 1, 1); - - RunTest(env, builder, groupIds); -} - -Y_UNIT_TEST(ReadError) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Groups[groupIds[2]].Status = NKikimrProto::ERROR; - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); - builder.AddToEnd("b", id, 0, 1); - - RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); -} - -Y_UNIT_TEST(ReadOneItemError) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::ERROR, "b"); - builder.AddToEnd("b", id, 0, 1); - - RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); -} - -Y_UNIT_TEST(ReadErrorWithWrongGroupId) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Groups[groupIds[2]].GroupId = groupIds[1]; - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); - builder.AddToEnd("b", id, 0, 1); - - RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); -} - -Y_UNIT_TEST(ReadErrorWithUncorrectCookie) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TReadRequestBuilder builder("a"); - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Groups[groupIds[2]].Cookie = 1000; - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); - builder.AddToEnd("b", id, 0, 1); - - RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); -} - - -void RunTest(TTestEnv &env, TRangeReadRequestBuilder &builder, const std::vector<ui32> &groupIds, - NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) -{ - TTestActorSystem runtime(1); - runtime.Start(); - runtime.SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); - env.BindGroupsToChannel(runtime, groupIds); - - TActorId edgeActor = runtime.AllocateEdgeActor(1); - auto [intermediate, expectedValues] = builder.Build(edgeActor, edgeActor, 1, 1); - - runtime.Register(CreateKeyValueStorageReadRequest(std::move(intermediate), env.TabletInfo.get()), 1); - - std::unique_ptr<IEventHandle> ev = runtime.WaitForEdgeActorEvent({edgeActor}); - UNIT_ASSERT(ev->Type == TEvKeyValue::EvReadRangeResponse); - TEvKeyValue::TEvReadRangeResponse *response = ev->Get<TEvKeyValue::TEvReadRangeResponse>(); - NKikimrKeyValue::ReadRangeResult &record = response->Record; - - if (status == NKikimrKeyValue::Statuses::RSTATUS_OK) { - UNIT_ASSERT_C(record.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, - "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) - << " Received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) - << " Message# " << record.msg()); - for (auto &kvp : record.pair()) { - UNIT_ASSERT_VALUES_EQUAL(kvp.value(), expectedValues[kvp.key()]); - } - } else { - UNIT_ASSERT_C(record.status() == status, - "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) - << " received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) - << " Message# " << record.msg()); - } - - runtime.Stop(); -} - -Y_UNIT_TEST(ReadRangeOk1Key) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TRangeReadRequestBuilder builder; - TLogoBlobID id(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); - builder.AddRead("key", "b", id); - - RunTest(env, builder, groupIds); -} - -Y_UNIT_TEST(ReadRangeOk) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TRangeReadRequestBuilder builder; - TLogoBlobID id1(1, 2, 3, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id1, NKikimrProto::OK, "b"); - builder.AddRead("key", "b", id1); - TLogoBlobID id2(1, 2, 4, 2, 1, 0); - env.BlobStorageState.Put(groupIds[2], id2, NKikimrProto::OK, "c"); - builder.AddRead("key2", "c", id2); - - RunTest(env, builder, groupIds); -} - - -Y_UNIT_TEST(ReadRangeNoData) { - TTestEnv env; - std::vector<ui32> groupIds = {1, 2, 3}; - - TRangeReadRequestBuilder builder; - - RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_NO_DATA); -} - -} // Y_UNIT_TEST_SUITE(KeyValueReadStorage) - -} // NKeyValue -} // NKikimr + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr { + +namespace NKeyValue { + +struct TBlobStorageMockState { + struct TBlob { + NKikimrProto::EReplyStatus Status = NKikimrProto::NODATA; + TString Buffer; + }; + + struct TGroup { + std::unordered_map<TLogoBlobID, TBlob, THash<TLogoBlobID>> Blobs; + NKikimrProto::EReplyStatus Status = NKikimrProto::OK; + std::optional<ui32> GroupId; + std::optional<ui32> Cookie; + }; + + std::unordered_map<ui32, TGroup> Groups; + + void Put(ui32 groupId, TLogoBlobID blobId, NKikimrProto::EReplyStatus status, const TString &buffer) { + TBlob &blob = Groups[groupId].Blobs[blobId]; + blob.Status = status; + blob.Buffer = buffer; + } + + std::unique_ptr<TEvBlobStorage::TEvGetResult> MakeGetResult(ui32 groupId, TEvBlobStorage::TEvGet *get, + std::function<const std::pair<NKikimrProto::EReplyStatus, const TString&>&(TLogoBlobID blobId)> getBlob) + { + TGroup &group = Groups[groupId]; + if (group.GroupId) { + groupId = *group.GroupId; + } + std::unique_ptr<TEvBlobStorage::TEvGetResult> getResult = std::make_unique<TEvBlobStorage::TEvGetResult>( + group.Status, get->QuerySize, groupId); + getResult->Responses.Reset(new TEvBlobStorage::TEvGetResult::TResponse[get->QuerySize]); + for (ui32 queryIdx = 0; queryIdx < get->QuerySize; ++queryIdx) { + auto &query = get->Queries[queryIdx]; + auto &response = getResult->Responses[queryIdx]; + response.Id = query.Id; + response.Shift = query.Shift; + response.RequestedSize = query.Size; + std::tie(response.Status, response.Buffer) = getBlob(query.Id); + if (response.Status == NKikimrProto::OK) { + TString buffer = TString::Uninitialized(query.Size); + memcpy(const_cast<char *>(buffer.data()), response.Buffer.data() + query.Shift, response.Buffer.size()); + response.Buffer = buffer; + } + } + return getResult; + } + + std::unique_ptr<TEvBlobStorage::TEvGetResult> OnGet(ui32 groupId, TEvBlobStorage::TEvGet *get) { + TGroup &group = Groups[groupId]; + + if (group.Status != NKikimrProto::OK) { + TString str(""); + return MakeGetResult(groupId, get, [&](...){ return std::make_pair(group.Status, str); }); + } + + return MakeGetResult(groupId, get, [&group](TLogoBlobID blobId) { + TBlob &blob = group.Blobs[blobId]; + return std::make_pair(blob.Status, blob.Buffer); + }); + } +}; + +struct TBlobStorageMock : TActorBootstrapped<TBlobStorageMock> { + ui32 GroupId; + TBlobStorageMockState *State; + + TBlobStorageMock(ui32 groupId, TBlobStorageMockState *state) + : GroupId(groupId) + , State(state) + {} + + void Bootstrap() { + Become(&TThis::Waiting); + } + + void Handle(TEvBlobStorage::TEvGet::TPtr &ev) { + std::unique_ptr<TEvBlobStorage::TEvGetResult> result = State->OnGet(GroupId, ev->Get()); + auto &group = State->Groups[GroupId]; + ui64 cookie = ev->Cookie; + if (group.Cookie) { + cookie = *group.Cookie; + } + Send(ev->Sender, result.release(), ev->Flags, cookie, std::move(ev->TraceId)); + } + + STATEFN(Waiting) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBlobStorage::TEvGet, Handle); + default: + Y_FAIL(); + } + } +}; + + +struct TTestEnv { + TBlobStorageMockState BlobStorageState; + + std::unique_ptr<TTabletStorageInfo> TabletInfo; + + std::unordered_map<ui64, TActorId> GroupActors; // [groupId, actorId] + + TTestEnv() + : TabletInfo(std::make_unique<TTabletStorageInfo>(1, TTabletTypes::KeyValue)) + { + } + + void AddStorageGroup(TTestActorSystem &runtime, ui64 groupId) { + if (GroupActors.count(groupId)) { + return; + } + + TActorId groupActor = runtime.Register(new TBlobStorageMock(groupId, &BlobStorageState), 1); + TActorId proxyId = MakeBlobStorageProxyID(groupId); + runtime.RegisterService(proxyId, groupActor); + GroupActors[groupId] = groupActor; + } + + void AddStorageGroups(TTestActorSystem &runtime, const std::vector<ui32> &groupIds) { + for (ui32 groupId : groupIds) { + AddStorageGroup(runtime, groupId); + } + } + + void BindGroupsToChannel(TTestActorSystem &runtime, const std::vector<ui32> &groupIds) + { + AddStorageGroups(runtime, groupIds); + for (ui32 channelIdx = 0; channelIdx < groupIds.size(); ++channelIdx) { + TabletInfo->Channels.emplace_back(channelIdx, TErasureType::Erasure4Plus2Block); + TabletInfo->Channels.back().History.emplace_back(1, groupIds[channelIdx]); + } + } +}; + + +struct TBuilderResult { + THolder<TIntermediate> Intermediate; + std::unordered_map<std::string, std::string> Values; +}; + + +struct TReadItem { + std::string Value; + TLogoBlobID BlobId; + ui32 Offset; + ui32 Size; + + TReadItem(const std::string &value, TLogoBlobID blobId, ui32 offset, ui32 size) + : Value(value) + , BlobId(blobId) + , Offset(offset) + , Size(size) + {} +}; + + +struct TReadRequestBuilder { + std::string Key; + std::vector<TReadItem> Items; + + TReadRequestBuilder(const std::string &key) + : Key(key) + {} + + TReadRequestBuilder& AddToEnd(const std::string &partOfValue, TLogoBlobID blobId, ui32 offset, ui32 size) { + Items.emplace_back(partOfValue, blobId, offset, size); + return *this; + } + + TBuilderResult Build(TActorId respondTo, TActorId keyValueActorId, ui32 channelGeneration = 1, ui32 channelStep = 1) + { + std::unique_ptr<TIntermediate> intermediate = std::make_unique<TIntermediate>(respondTo, keyValueActorId, + channelGeneration, channelStep, TRequestType::ReadOnly); + TStringBuilder valueBuilder; + for (auto &[value, blobId, offset, size] : Items) { + valueBuilder << value; + } + std::string value = valueBuilder; + + + intermediate->ReadCommand = TIntermediate::TRead(TString(Key), value.size(), 0, + NKikimrClient::TKeyValueRequest::MAIN); + TIntermediate::TRead &read = std::get<TIntermediate::TRead>(*intermediate->ReadCommand); + ui32 valueOffset = 0; + for (auto &[value, blobId, offset, size] : Items) { + read.ReadItems.emplace_back(blobId, offset, size, valueOffset); + valueOffset += size; + } + TBuilderResult res; + res.Values[Key] = value; + res.Intermediate.Reset(intermediate.release()); + return res; + } +}; + + +struct TRangeReadRequestBuilder { + std::map<std::string, std::vector<TReadItem>> Reads; + ui64 CmdLimitBytes = Max<ui64>(); + bool IncludeData = true; + + TRangeReadRequestBuilder() + {} + + TRangeReadRequestBuilder& AddRead(const std::string &key, std::vector<TReadItem> &&items) { + Reads[key] = std::move(items); + return *this; + } + + TRangeReadRequestBuilder& AddRead(const std::string &key, const std::string &value, TLogoBlobID blobId) { + Reads[key] = {TReadItem(value, blobId, 0, value.size())}; + return *this; + } + + TBuilderResult Build(TActorId respondTo, TActorId keyValueActorId, ui32 channelGeneration = 1, ui32 channelStep = 1) + { + std::unique_ptr<TIntermediate> intermediate = std::make_unique<TIntermediate>(respondTo, keyValueActorId, + channelGeneration, channelStep, TRequestType::ReadOnly); + + TBuilderResult res; + intermediate->ReadCommand = TIntermediate::TRangeRead(); + auto &range = std::get<TIntermediate::TRangeRead>(*intermediate->ReadCommand); + + for (auto &[key, items] : Reads) { + TStringBuilder valueBuilder; + for (auto &[value, blobId, offset, size] : items) { + valueBuilder << value; + } + std::string value = valueBuilder; + + range.Reads.emplace_back(TString(key), value.size(), 0, NKikimrClient::TKeyValueRequest::MAIN); + TIntermediate::TRead &read = range.Reads.back(); + ui32 valueOffset = 0; + for (auto &[value, blobId, offset, size] : items) { + read.ReadItems.emplace_back(blobId, offset, size, valueOffset); + valueOffset += size; + } + res.Values[key] = value; + } + + res.Intermediate.Reset(intermediate.release()); + return res; + } +}; + + +Y_UNIT_TEST_SUITE(KeyValueReadStorage) { + +void RunTest(TTestEnv &env, TReadRequestBuilder &builder, + const std::vector<ui32> &groupIds, NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) { + TTestActorSystem runtime(1); + runtime.Start(); + runtime.SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); + env.BindGroupsToChannel(runtime, groupIds); + + TActorId edgeActor = runtime.AllocateEdgeActor(1); + auto [intermediate, expectedValues] = builder.Build(edgeActor, edgeActor, 1, 1); + + runtime.Register(CreateKeyValueStorageReadRequest(std::move(intermediate), env.TabletInfo.get()), 1); + + std::unique_ptr<IEventHandle> ev = runtime.WaitForEdgeActorEvent({edgeActor}); + UNIT_ASSERT(ev->Type == TEvKeyValue::EvReadResponse); + TEvKeyValue::TEvReadResponse *response = ev->Get<TEvKeyValue::TEvReadResponse>(); + NKikimrKeyValue::ReadResult &record = response->Record; + + if (status == NKikimrKeyValue::Statuses::RSTATUS_OK) { + UNIT_ASSERT_C(record.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) + << " Received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) + << " Message# " << record.msg()); + UNIT_ASSERT_VALUES_EQUAL(record.value(), expectedValues[record.requested_key()]); + } else { + UNIT_ASSERT_C(record.status() == status, "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) + << " received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) + << " Message# " << record.msg()); + } + + runtime.Stop(); +} + +Y_UNIT_TEST(ReadOk) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); + builder.AddToEnd("b", id, 0, 1); + + RunTest(env, builder, groupIds); +} + +Y_UNIT_TEST(ReadWithTwoPartsOk) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id1(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id1, NKikimrProto::OK, "b"); + builder.AddToEnd("b", id1, 0, 1); + + TLogoBlobID id2(1, 2, 3, 2, 1, 1); + env.BlobStorageState.Put(groupIds[2], id2, NKikimrProto::OK, "c"); + builder.AddToEnd("c", id2, 0, 1); + + RunTest(env, builder, groupIds); +} + +Y_UNIT_TEST(ReadNotWholeBlobOk) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 3, 0); + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "abc"); + builder.AddToEnd("b", id, 1, 1); + + RunTest(env, builder, groupIds); +} + +Y_UNIT_TEST(ReadError) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Groups[groupIds[2]].Status = NKikimrProto::ERROR; + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); + builder.AddToEnd("b", id, 0, 1); + + RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); +} + +Y_UNIT_TEST(ReadOneItemError) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::ERROR, "b"); + builder.AddToEnd("b", id, 0, 1); + + RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); +} + +Y_UNIT_TEST(ReadErrorWithWrongGroupId) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Groups[groupIds[2]].GroupId = groupIds[1]; + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); + builder.AddToEnd("b", id, 0, 1); + + RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); +} + +Y_UNIT_TEST(ReadErrorWithUncorrectCookie) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TReadRequestBuilder builder("a"); + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Groups[groupIds[2]].Cookie = 1000; + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); + builder.AddToEnd("b", id, 0, 1); + + RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR); +} + + +void RunTest(TTestEnv &env, TRangeReadRequestBuilder &builder, const std::vector<ui32> &groupIds, + NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_OK) +{ + TTestActorSystem runtime(1); + runtime.Start(); + runtime.SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); + env.BindGroupsToChannel(runtime, groupIds); + + TActorId edgeActor = runtime.AllocateEdgeActor(1); + auto [intermediate, expectedValues] = builder.Build(edgeActor, edgeActor, 1, 1); + + runtime.Register(CreateKeyValueStorageReadRequest(std::move(intermediate), env.TabletInfo.get()), 1); + + std::unique_ptr<IEventHandle> ev = runtime.WaitForEdgeActorEvent({edgeActor}); + UNIT_ASSERT(ev->Type == TEvKeyValue::EvReadRangeResponse); + TEvKeyValue::TEvReadRangeResponse *response = ev->Get<TEvKeyValue::TEvReadRangeResponse>(); + NKikimrKeyValue::ReadRangeResult &record = response->Record; + + if (status == NKikimrKeyValue::Statuses::RSTATUS_OK) { + UNIT_ASSERT_C(record.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, + "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) + << " Received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) + << " Message# " << record.msg()); + for (auto &kvp : record.pair()) { + UNIT_ASSERT_VALUES_EQUAL(kvp.value(), expectedValues[kvp.key()]); + } + } else { + UNIT_ASSERT_C(record.status() == status, + "Expected# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(status) + << " received# " << NKikimrKeyValue::Statuses::ReplyStatus_Name(record.status()) + << " Message# " << record.msg()); + } + + runtime.Stop(); +} + +Y_UNIT_TEST(ReadRangeOk1Key) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TRangeReadRequestBuilder builder; + TLogoBlobID id(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id, NKikimrProto::OK, "b"); + builder.AddRead("key", "b", id); + + RunTest(env, builder, groupIds); +} + +Y_UNIT_TEST(ReadRangeOk) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TRangeReadRequestBuilder builder; + TLogoBlobID id1(1, 2, 3, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id1, NKikimrProto::OK, "b"); + builder.AddRead("key", "b", id1); + TLogoBlobID id2(1, 2, 4, 2, 1, 0); + env.BlobStorageState.Put(groupIds[2], id2, NKikimrProto::OK, "c"); + builder.AddRead("key2", "c", id2); + + RunTest(env, builder, groupIds); +} + + +Y_UNIT_TEST(ReadRangeNoData) { + TTestEnv env; + std::vector<ui32> groupIds = {1, 2, 3}; + + TRangeReadRequestBuilder builder; + + RunTest(env, builder, groupIds, NKikimrKeyValue::Statuses::RSTATUS_NO_DATA); +} + +} // Y_UNIT_TEST_SUITE(KeyValueReadStorage) + +} // NKeyValue +} // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_storage_request.cpp b/ydb/core/keyvalue/keyvalue_storage_request.cpp index e9032d2ed3d..2fad1bfae83 100644 --- a/ydb/core/keyvalue/keyvalue_storage_request.cpp +++ b/ydb/core/keyvalue/keyvalue_storage_request.cpp @@ -13,8 +13,8 @@ namespace NKeyValue { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class TKeyValueStorageRequest : public TActorBootstrapped<TKeyValueStorageRequest> { - using TBase = TActorBootstrapped<TKeyValueStorageRequest>; - + using TBase = TActorBootstrapped<TKeyValueStorageRequest>; + ui64 ReadRequestsSent; ui64 ReadRequestsReplied; ui64 WriteRequestsSent; @@ -39,8 +39,8 @@ class TKeyValueStorageRequest : public TActorBootstrapped<TKeyValueStorageReques const TDuration MuteDuration = TDuration::Seconds(5); TLogPriorityMuteChecker<NLog::PRI_DEBUG, NLog::PRI_ERROR> ErrorStateMuteChecker; - THashMap<ui32, TInstant> TimeForNextSend; - + THashMap<ui32, TInstant> TimeForNextSend; + struct TReadQueueItem { TIntermediate::TRead *Read; TIntermediate::TRead::TReadItem *ReadItem; @@ -60,7 +60,7 @@ class TKeyValueStorageRequest : public TActorBootstrapped<TKeyValueStorageReques TStackVec<ui32, 16> YellowMoveChannels; TStackVec<ui32, 16> YellowStopChannels; - + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::KEYVALUE_ACTOR; @@ -86,7 +86,7 @@ public: IntermediateResults->Stat.KeyvalueStorageRequestSentAt = TAppData::TimeProvider->Now(); } - void CheckYellow(const TStorageStatusFlags &statusFlags, ui32 currentGroup) { + void CheckYellow(const TStorageStatusFlags &statusFlags, ui32 currentGroup) { if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) { for (ui32 channel : xrange(TabletInfo->Channels.size())) { const ui32 group = TabletInfo->ChannelInfo(channel)->LatestEntry()->GroupID; @@ -95,25 +95,25 @@ public: } } SortUnique(YellowMoveChannels); - } + } if (statusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) { for (ui32 channel : xrange(TabletInfo->Channels.size())) { const ui32 group = TabletInfo->ChannelInfo(channel)->LatestEntry()->GroupID; if (currentGroup == group) { YellowStopChannels.push_back(channel); } - } + } SortUnique(YellowStopChannels); - } - } - + } + } + void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev, const TActorContext &ctx) { const TDuration duration = TDuration::Seconds(PutTimer.Passed()); IntermediateResults->Stat.PutLatencies.push_back(duration.MilliSeconds()); - auto groupId = ev->Get()->GroupId; - CheckYellow(ev->Get()->StatusFlags, groupId); - + auto groupId = ev->Get()->GroupId; + CheckYellow(ev->Get()->StatusFlags, groupId); + NKikimrProto::EReplyStatus status = ev->Get()->Status; if (status != NKikimrProto::OK) { TInstant now = TAppData::TimeProvider->Now(); @@ -140,7 +140,7 @@ public: } ui64 cookie = ev->Cookie; ui64 writeIdx = cookie; - if (writeIdx >= IntermediateResults->Writes.size() && writeIdx >= IntermediateResults->Commands.size()) { + if (writeIdx >= IntermediateResults->Writes.size() && writeIdx >= IntermediateResults->Commands.size()) { TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " EvPut cookie# " << (ui64)cookie; @@ -149,15 +149,15 @@ public: ReplyErrorAndDie(ctx, str.Str()); return; } - - TIntermediate::TWrite *wr = nullptr; - if (IntermediateResults->Writes.size()) { - wr = &IntermediateResults->Writes[writeIdx]; - } else { - wr = &std::get<TIntermediate::TWrite>(IntermediateResults->Commands[writeIdx]); - } - wr->StatusFlags.Merge(ev->Get()->StatusFlags.Raw); - wr->Latency = duration; + + TIntermediate::TWrite *wr = nullptr; + if (IntermediateResults->Writes.size()) { + wr = &IntermediateResults->Writes[writeIdx]; + } else { + wr = &std::get<TIntermediate::TWrite>(IntermediateResults->Commands[writeIdx]); + } + wr->StatusFlags.Merge(ev->Get()->StatusFlags.Raw); + wr->Latency = duration; ++WriteRequestsReplied; IntermediateResults->Stat.GroupWrittenBytes[std::make_pair(ev->Get()->Id.Channel(), groupId)] += ev->Get()->Id.BlobSize(); IntermediateResults->Stat.GroupWrittenIops[std::make_pair(ev->Get()->Id.Channel(), groupId)] += 1; // FIXME: count distinct blobs? @@ -169,9 +169,9 @@ public: ui64 durationMs = (now - GetStatusSentAt).MilliSeconds(); IntermediateResults->Stat.GetStatusLatencies.push_back(durationMs); - ui64 cookie = ev->Cookie; // groupId - CheckYellow(ev->Get()->StatusFlags, cookie); - + ui64 cookie = ev->Cookie; // groupId + CheckYellow(ev->Get()->StatusFlags, cookie); + NKikimrProto::EReplyStatus status = ev->Get()->Status; if (status != NKikimrProto::OK) { TStringStream str; @@ -361,15 +361,15 @@ public: getStatus.Status = NKikimrProto::OK; } } - for (auto& cmd : IntermediateResults->Commands) { - if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { - continue; - } - auto& write = std::get<TIntermediate::TWrite>(cmd); - if (write.Status == NKikimrProto::UNKNOWN) { - write.Status = NKikimrProto::OK; - } - } + for (auto& cmd : IntermediateResults->Commands) { + if (!std::holds_alternative<TIntermediate::TWrite>(cmd)) { + continue; + } + auto& write = std::get<TIntermediate::TWrite>(cmd); + if (write.Status == NKikimrProto::UNKNOWN) { + write.Status = NKikimrProto::OK; + } + } IntermediateResults->Stat.YellowStopChannels.reserve(YellowStopChannels.size()); IntermediateResults->Stat.YellowStopChannels.insert(IntermediateResults->Stat.YellowStopChannels.end(), YellowStopChannels.begin(), YellowStopChannels.end()); @@ -477,7 +477,7 @@ public: IntermediateResults->Stat.YellowMoveChannels.reserve(YellowMoveChannels.size()); IntermediateResults->Stat.YellowMoveChannels.insert(IntermediateResults->Stat.YellowMoveChannels.end(), YellowMoveChannels.begin(), YellowMoveChannels.end()); - + IntermediateResults->UpdateStat(); TActorId keyValueActorId = IntermediateResults->KeyValueActorId; ctx.Send(keyValueActorId, new TEvKeyValue::TEvNotify( @@ -629,45 +629,45 @@ public: } void SendWriteRequests(const TActorContext &ctx) { - auto sendWrite = [&](ui32 i, auto &request) -> void { - using Type = std::decay_t<decltype(request)>; - if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { - if (request.Status != NKikimrProto::SCHEDULED) { - Y_VERIFY(request.Status == NKikimrProto::UNKNOWN); - - ui64 offset = 0; - for (const TLogoBlobID& logoBlobId : request.LogoBlobIds) { - THolder<TEvBlobStorage::TEvPut> put( - new TEvBlobStorage::TEvPut( - logoBlobId, request.Data.substr(offset, logoBlobId.BlobSize()), - IntermediateResults->Deadline, request.HandleClass, - request.Tactic)); - const ui32 groupId = TabletInfo->GroupFor(logoBlobId.Channel(), logoBlobId.Generation()); - Y_VERIFY(groupId != Max<ui32>(), "Put Blob# %s is mapped to an invalid group (-1)!", - logoBlobId.ToString().c_str()); - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletInfo->TabletID - << " Send TEvPut# " << put->ToString() << " to groupId# " << groupId - << " now# " << TAppData::TimeProvider->Now().MilliSeconds() << " Marker# KV60"); - - SendPutToGroup(ctx, groupId, TabletInfo.Get(), std::move(put), i); - - ++WriteRequestsSent; - offset += logoBlobId.BlobSize(); - } + auto sendWrite = [&](ui32 i, auto &request) -> void { + using Type = std::decay_t<decltype(request)>; + if constexpr (std::is_same_v<Type, TIntermediate::TWrite>) { + if (request.Status != NKikimrProto::SCHEDULED) { + Y_VERIFY(request.Status == NKikimrProto::UNKNOWN); + + ui64 offset = 0; + for (const TLogoBlobID& logoBlobId : request.LogoBlobIds) { + THolder<TEvBlobStorage::TEvPut> put( + new TEvBlobStorage::TEvPut( + logoBlobId, request.Data.substr(offset, logoBlobId.BlobSize()), + IntermediateResults->Deadline, request.HandleClass, + request.Tactic)); + const ui32 groupId = TabletInfo->GroupFor(logoBlobId.Channel(), logoBlobId.Generation()); + Y_VERIFY(groupId != Max<ui32>(), "Put Blob# %s is mapped to an invalid group (-1)!", + logoBlobId.ToString().c_str()); + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletInfo->TabletID + << " Send TEvPut# " << put->ToString() << " to groupId# " << groupId + << " now# " << TAppData::TimeProvider->Now().MilliSeconds() << " Marker# KV60"); + + SendPutToGroup(ctx, groupId, TabletInfo.Get(), std::move(put), i); + + ++WriteRequestsSent; + offset += logoBlobId.BlobSize(); + } } } - }; - - for (ui64 i : IntermediateResults->WriteIndices) { - auto &cmd = IntermediateResults->Commands[i]; - Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); - auto& write = std::get<TIntermediate::TWrite>(cmd); - sendWrite(i, write); - } - - for (ui64 i = 0; i < IntermediateResults->Writes.size(); ++i) { - sendWrite(i, IntermediateResults->Writes[i]); + }; + + for (ui64 i : IntermediateResults->WriteIndices) { + auto &cmd = IntermediateResults->Commands[i]; + Y_VERIFY(std::holds_alternative<TIntermediate::TWrite>(cmd)); + auto& write = std::get<TIntermediate::TWrite>(cmd); + sendWrite(i, write); } + + for (ui64 i = 0; i < IntermediateResults->Writes.size(); ++i) { + sendWrite(i, IntermediateResults->Writes[i]); + } PutTimer.Reset(); } diff --git a/ydb/core/keyvalue/keyvalue_ut.cpp b/ydb/core/keyvalue/keyvalue_ut.cpp index 4ae1fcf56a6..b35d51507d1 100644 --- a/ydb/core/keyvalue/keyvalue_ut.cpp +++ b/ydb/core/keyvalue/keyvalue_ut.cpp @@ -15,7 +15,7 @@ void SetupLogging(TTestActorRuntime& runtime) { NActors::NLog::EPriority priority = ENABLE_DETAILED_KV_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR; NActors::NLog::EPriority otherPriority = NLog::PRI_ERROR; - runtime.SetLogPriority(NKikimrServices::KEYVALUE, priority); + runtime.SetLogPriority(NKikimrServices::KEYVALUE, priority); runtime.SetLogPriority(NKikimrServices::BOOTSTRAPPER, priority); runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, priority); runtime.SetLogPriority(NKikimrServices::TABLET_EXECUTOR, priority); @@ -78,7 +78,7 @@ struct TTestContext { outActiveZone = false; Runtime.Reset(new TTestBasicRuntime); Runtime->SetScheduledLimit(100); - Runtime->SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); + Runtime->SetLogPriority(NKikimrServices::KEYVALUE, NLog::PRI_DEBUG); SetupLogging(*Runtime); SetupTabletServices(*Runtime); setup(*Runtime); @@ -115,18 +115,18 @@ struct TFinalizer { // SINGLE COMMAND TEST FUNCTIONS //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void DoWithRetry(std::function<bool(void)> action, i32 retryCount = 2) { - bool isEnd = false; - for (i32 retriesLeft = retryCount; !isEnd && retriesLeft > 0; --retriesLeft) { - try { - isEnd = action(); - } catch (NActors::TSchedulingLimitReachedException) { - UNIT_ASSERT(retriesLeft != 1); - } - } - UNIT_ASSERT(isEnd); -} - +void DoWithRetry(std::function<bool(void)> action, i32 retryCount = 2) { + bool isEnd = false; + for (i32 retriesLeft = retryCount; !isEnd && retriesLeft > 0; --retriesLeft) { + try { + isEnd = action(); + } catch (NActors::TSchedulingLimitReachedException) { + UNIT_ASSERT(retriesLeft != 1); + } + } + UNIT_ASSERT(isEnd); +} + void CmdWrite(const TDeque<TString> &keys, const TDeque<TString> &values, const NKikimrClient::TKeyValueRequest::EStorageChannel storageChannel, const NKikimrClient::TKeyValueRequest::EPriority priority, TTestContext &tc) { @@ -134,33 +134,33 @@ void CmdWrite(const TDeque<TString> &keys, const TDeque<TString> &values, TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - for (ui64 idx = 0; idx < keys.size(); ++idx) { - auto write = request->Record.AddCmdWrite(); - write->SetKey(keys[idx]); - write->SetValue(values[idx]); - write->SetStorageChannel(storageChannel); - write->SetPriority(priority); - } - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.WriteResultSize(), values.size()); - for (ui64 idx = 0; idx < values.size(); ++idx) { - const auto &writeResult = result->Record.GetWriteResult(idx); - UNIT_ASSERT(writeResult.HasStatus()); - UNIT_ASSERT_EQUAL(writeResult.GetStatus(), NKikimrProto::OK); - UNIT_ASSERT(writeResult.HasStatusFlags()); - if (values[idx].size()) { - UNIT_ASSERT(writeResult.GetStatusFlags() & ui32(NKikimrBlobStorage::StatusIsValid)); + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + for (ui64 idx = 0; idx < keys.size(); ++idx) { + auto write = request->Record.AddCmdWrite(); + write->SetKey(keys[idx]); + write->SetValue(values[idx]); + write->SetStorageChannel(storageChannel); + write->SetPriority(priority); + } + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.WriteResultSize(), values.size()); + for (ui64 idx = 0; idx < values.size(); ++idx) { + const auto &writeResult = result->Record.GetWriteResult(idx); + UNIT_ASSERT(writeResult.HasStatus()); + UNIT_ASSERT_EQUAL(writeResult.GetStatus(), NKikimrProto::OK); + UNIT_ASSERT(writeResult.HasStatusFlags()); + if (values[idx].size()) { + UNIT_ASSERT(writeResult.GetStatusFlags() & ui32(NKikimrBlobStorage::StatusIsValid)); } } - return true; - }); + return true; + }); } void CmdWrite(const TString &key, const TString &value, @@ -179,34 +179,34 @@ void CmdRead(const TDeque<TString> &keys, TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - for (const auto &key: keys) { - auto read = request->Record.AddCmdRead(); - read->SetKey(key); - read->SetPriority(priority); - } - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.ReadResultSize(), keys.size()); - for (ui64 idx = 0; idx < expectedValues.size(); ++idx) { - const auto &readResult = result->Record.GetReadResult(idx); - UNIT_ASSERT(readResult.HasStatus()); - if (expectedNodatas.size() == 0 || !expectedNodatas[idx]) { - UNIT_ASSERT_EQUAL(readResult.GetStatus(), NKikimrProto::OK); - UNIT_ASSERT(readResult.HasValue()); - UNIT_ASSERT_VALUES_EQUAL(readResult.GetValue(), expectedValues[idx]); - } else { - UNIT_ASSERT_EQUAL(readResult.GetStatus(), NKikimrProto::NODATA); + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + for (const auto &key: keys) { + auto read = request->Record.AddCmdRead(); + read->SetKey(key); + read->SetPriority(priority); + } + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.ReadResultSize(), keys.size()); + for (ui64 idx = 0; idx < expectedValues.size(); ++idx) { + const auto &readResult = result->Record.GetReadResult(idx); + UNIT_ASSERT(readResult.HasStatus()); + if (expectedNodatas.size() == 0 || !expectedNodatas[idx]) { + UNIT_ASSERT_EQUAL(readResult.GetStatus(), NKikimrProto::OK); + UNIT_ASSERT(readResult.HasValue()); + UNIT_ASSERT_VALUES_EQUAL(readResult.GetValue(), expectedValues[idx]); + } else { + UNIT_ASSERT_EQUAL(readResult.GetStatus(), NKikimrProto::NODATA); } } - return true; - }); + return true; + }); } void CmdRename(const TDeque<TString> &oldKeys, const TDeque<TString> &newKeys, TTestContext &tc, @@ -215,33 +215,33 @@ void CmdRename(const TDeque<TString> &oldKeys, const TDeque<TString> &newKeys, T TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - for (ui64 idx = 0; idx < oldKeys.size(); ++idx) { - auto cmd = request->Record.AddCmdRename(); - cmd->SetOldKey(oldKeys[idx]); - cmd->SetNewKey(newKeys[idx]); - } - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - if (expectOk) { - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.RenameResultSize(), oldKeys.size()); + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + for (ui64 idx = 0; idx < oldKeys.size(); ++idx) { + auto cmd = request->Record.AddCmdRename(); + cmd->SetOldKey(oldKeys[idx]); + cmd->SetNewKey(newKeys[idx]); + } + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + if (expectOk) { + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.RenameResultSize(), oldKeys.size()); for (ui64 idx = 0; idx < oldKeys.size(); ++idx) { - const auto &renameResult = result->Record.GetRenameResult(idx); - UNIT_ASSERT(renameResult.HasStatus()); - UNIT_ASSERT_EQUAL(renameResult.GetStatus(), NKikimrProto::OK); + const auto &renameResult = result->Record.GetRenameResult(idx); + UNIT_ASSERT(renameResult.HasStatus()); + UNIT_ASSERT_EQUAL(renameResult.GetStatus(), NKikimrProto::OK); } - } else { - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_ERROR); + } else { + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_ERROR); } - - return true; - }); + + return true; + }); } void CmdRename(const TString &oldKey, const TString &newKey, TTestContext &tc, bool expectOk = true) { @@ -254,27 +254,27 @@ void CmdConcat(const TDeque<TString> &inputKeys, const TString &outputKey, const TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - auto cmd = request->Record.AddCmdConcat(); - for (ui64 idx = 0; idx < inputKeys.size(); ++idx) { - cmd->AddInputKeys(inputKeys[idx]); - } - cmd->SetOutputKey(outputKey); - cmd->SetKeepInputs(keepInputs); - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.ConcatResultSize(), 1); - UNIT_ASSERT(result->Record.GetConcatResult(0).HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetConcatResult(0).GetStatus(), NKikimrProto::OK); - - return true; - }); + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + auto cmd = request->Record.AddCmdConcat(); + for (ui64 idx = 0; idx < inputKeys.size(); ++idx) { + cmd->AddInputKeys(inputKeys[idx]); + } + cmd->SetOutputKey(outputKey); + cmd->SetKeepInputs(keepInputs); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.ConcatResultSize(), 1); + UNIT_ASSERT(result->Record.GetConcatResult(0).HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetConcatResult(0).GetStatus(), NKikimrProto::OK); + + return true; + }); } void CmdDeleteRange(const TString &from, const bool includeFrom, const TString &to, const bool includeTo, @@ -282,33 +282,33 @@ void CmdDeleteRange(const TString &from, const bool includeFrom, const TString & TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - auto deleteRange = request->Record.AddCmdDeleteRange(); - deleteRange->MutableRange()->SetFrom(from); - deleteRange->MutableRange()->SetIncludeFrom(includeFrom); - deleteRange->MutableRange()->SetTo(to); - deleteRange->MutableRange()->SetIncludeTo(includeTo); - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - if (expectedStatus == NMsgBusProxy::MSTATUS_OK) { - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.DeleteRangeResultSize(), 1); - UNIT_ASSERT(result->Record.GetDeleteRangeResult(0).HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetDeleteRangeResult(0).GetStatus(), NKikimrProto::OK); - } else { - UNIT_ASSERT_EQUAL_C(result->Record.GetStatus(), expectedStatus, - "Expected# " << (ui32)expectedStatus - << " Got# " << (ui32)result->Record.GetStatus() - << " ErrorReason# \"" << result->Record.GetErrorReason() << "\""); - } - - return true; - }); + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + auto deleteRange = request->Record.AddCmdDeleteRange(); + deleteRange->MutableRange()->SetFrom(from); + deleteRange->MutableRange()->SetIncludeFrom(includeFrom); + deleteRange->MutableRange()->SetTo(to); + deleteRange->MutableRange()->SetIncludeTo(includeTo); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + if (expectedStatus == NMsgBusProxy::MSTATUS_OK) { + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.DeleteRangeResultSize(), 1); + UNIT_ASSERT(result->Record.GetDeleteRangeResult(0).HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetDeleteRangeResult(0).GetStatus(), NKikimrProto::OK); + } else { + UNIT_ASSERT_EQUAL_C(result->Record.GetStatus(), expectedStatus, + "Expected# " << (ui32)expectedStatus + << " Got# " << (ui32)result->Record.GetStatus() + << " ErrorReason# \"" << result->Record.GetErrorReason() << "\""); + } + + return true; + }); } void CmdCopyRange(const TString &from, const bool includeFrom, const TString &to, const bool includeTo, @@ -316,38 +316,38 @@ void CmdCopyRange(const TString &from, const bool includeFrom, const TString &to TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - auto copyRange = request->Record.AddCmdCopyRange(); - copyRange->MutableRange()->SetFrom(from); - copyRange->MutableRange()->SetIncludeFrom(includeFrom); - copyRange->MutableRange()->SetTo(to); - copyRange->MutableRange()->SetIncludeTo(includeTo); - copyRange->SetPrefixToAdd(prefixToAdd); - copyRange->SetPrefixToRemove(prefixToRemove); - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.CopyRangeResultSize(), 1); - UNIT_ASSERT(result->Record.GetCopyRangeResult(0).HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetCopyRangeResult(0).GetStatus(), NKikimrProto::OK); - - return true; - }); -} - -template <typename TRequest> + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + auto copyRange = request->Record.AddCmdCopyRange(); + copyRange->MutableRange()->SetFrom(from); + copyRange->MutableRange()->SetIncludeFrom(includeFrom); + copyRange->MutableRange()->SetTo(to); + copyRange->MutableRange()->SetIncludeTo(includeTo); + copyRange->SetPrefixToAdd(prefixToAdd); + copyRange->SetPrefixToRemove(prefixToRemove); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.CopyRangeResultSize(), 1); + UNIT_ASSERT(result->Record.GetCopyRangeResult(0).HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetCopyRangeResult(0).GetStatus(), NKikimrProto::OK); + + return true; + }); +} + +template <typename TRequest> struct TDesiredPair { - typename TRequest::ProtoRecordType Request; - typename TRequest::TResponse::ProtoRecordType Response; -}; - -template <> -struct TDesiredPair<TEvKeyValue::TEvRequest> { + typename TRequest::ProtoRecordType Request; + typename TRequest::TResponse::ProtoRecordType Response; +}; + +template <> +struct TDesiredPair<TEvKeyValue::TEvRequest> { NKikimrClient::TKeyValueRequest Request; NKikimrClient::TKeyValueResponse Response; }; @@ -383,34 +383,34 @@ void CheckResponse(NKikimrClient::TResponse &ar, NKikimrClient::TKeyValueRespons } } -void RunRequest(TDesiredPair<TEvKeyValue::TEvRequest> &dp, TTestContext &tc, ui64 line) { +void RunRequest(TDesiredPair<TEvKeyValue::TEvRequest> &dp, TTestContext &tc, ui64 line) { TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - request->Record = dp.Request; + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + request->Record = dp.Request; - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT_C(result, "Line# " << line); - UNIT_ASSERT_C(result->Record.HasStatus(), "Line# " << line); - UNIT_ASSERT_EQUAL_C(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK, "Line# " << line); - - CheckResponse(result->Record, dp.Response, line); - - return true; - }); + UNIT_ASSERT_C(result, "Line# " << line); + UNIT_ASSERT_C(result->Record.HasStatus(), "Line# " << line); + UNIT_ASSERT_EQUAL_C(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK, "Line# " << line); + + CheckResponse(result->Record, dp.Response, line); + + return true; + }); } void AddCmdReadRange(const TString &from, const bool includeFrom, const TString &to, const bool includeTo, const bool includeData, const ui64 limitBytes, const NKikimrClient::TKeyValueRequest::EPriority priority, const TDeque<TString> &expectedKeys, const TDeque<TString> &expectedValues, - const NKikimrProto::EReplyStatus expectedStatus, TTestContext &tc, TDesiredPair<TEvKeyValue::TEvRequest> &dp) { + const NKikimrProto::EReplyStatus expectedStatus, TTestContext &tc, TDesiredPair<TEvKeyValue::TEvRequest> &dp) { Y_UNUSED(tc); Y_VERIFY(!includeData || expectedKeys.size() == expectedValues.size()); @@ -446,411 +446,411 @@ void CmdGetStatus(const NKikimrClient::TKeyValueRequest::EStorageChannel storage TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - auto getStatus = request->Record.AddCmdGetStatus(); - getStatus->SetStorageChannel(storageChannel); - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT_VALUES_EQUAL(result->Record.GetStatusResultSize(), 1); - UNIT_ASSERT(result->Record.GetGetStatusResult(0).HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetGetStatusResult(0).GetStatus(), NKikimrProto::OK); - UNIT_ASSERT(result->Record.GetGetStatusResult(0).HasStatusFlags()); - UNIT_ASSERT_EQUAL(result->Record.GetGetStatusResult(0).GetStatusFlags(), expectedStatusFlags); - - return true; - }); + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + auto getStatus = request->Record.AddCmdGetStatus(); + getStatus->SetStorageChannel(storageChannel); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT_VALUES_EQUAL(result->Record.GetStatusResultSize(), 1); + UNIT_ASSERT(result->Record.GetGetStatusResult(0).HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetGetStatusResult(0).GetStatus(), NKikimrProto::OK); + UNIT_ASSERT(result->Record.GetGetStatusResult(0).HasStatusFlags()); + UNIT_ASSERT_EQUAL(result->Record.GetGetStatusResult(0).GetStatusFlags(), expectedStatusFlags); + + return true; + }); } void CmdSetExecutorFastLogPolicy(bool isAllowed, TTestContext &tc) { TAutoPtr<IEventHandle> handle; TEvKeyValue::TEvResponse *result; THolder<TEvKeyValue::TEvRequest> request; - - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request.Reset(new TEvKeyValue::TEvRequest); - auto cmd = request->Record.MutableCmdSetExecutorFastLogPolicy(); - cmd->SetIsAllowed(isAllowed); - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); - result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); - UNIT_ASSERT(result); - UNIT_ASSERT(result->Record.HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); - UNIT_ASSERT(result->Record.HasSetExecutorFastLogPolicyResult()); - UNIT_ASSERT(result->Record.GetSetExecutorFastLogPolicyResult().HasStatus()); - UNIT_ASSERT_EQUAL(result->Record.GetSetExecutorFastLogPolicyResult().GetStatus(), NKikimrProto::OK); - - return true; - }); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// NEW SINGLE COMMAND TEST FUNCTIONS -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -struct TKeyValuePair { - TString Key; - TString Value; -}; - -template <typename TRequestEvent> -void ExecuteEvent(TDesiredPair<TRequestEvent> &dp, TTestContext &tc) { - TAutoPtr<IEventHandle> handle; - std::unique_ptr<TRequestEvent> request; - typename TRequestEvent::TResponse *response; - DoWithRetry([&] { - tc.Runtime->ResetScheduledCount(); - request = std::make_unique<TRequestEvent>(); - request->Record = dp.Request; - tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.release(), 0, GetPipeConfigWithRetries()); - response = tc.Runtime->GrabEdgeEvent<typename TRequestEvent::TResponse>(handle); - dp.Response = response->Record; - return true; - }); -} - -template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> -void ExecuteWrite(TTestContext &tc, const TDeque<TKeyValuePair> &pairs, ui64 lockedGeneration, ui64 storageChannel, - NKikimrKeyValue::Priorities::Priority priority) -{ - TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; - - for (auto &[key, value] : pairs) { - NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); - NKikimrKeyValue::ExecuteTransactionRequest::Command::Write *write = cmd->mutable_write(); - - write->set_key(key); - write->set_value(value); - write->set_storage_channel(storageChannel); - write->set_priority(priority); - } - - dp.Request.set_tablet_id(tc.TabletId); - dp.Request.set_lock_generation(lockedGeneration); - - ExecuteEvent(dp, tc); - UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) - << " msg# " << dp.Response.msg()); -} - -enum class EBorderKind { - Include, - Exclude, - Without -}; - -template <bool IsSuccess = true> -void ExecuteDeleteRange(TTestContext &tc, - const TString &from, EBorderKind fromKind, - const TString &to, EBorderKind toKind, - ui64 lock_generation) -{ - TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; - dp.Request.set_lock_generation(lock_generation); - dp.Request.set_tablet_id(tc.TabletId); - - NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); - NKikimrKeyValue::ExecuteTransactionRequest::Command::DeleteRange *deleteRange = cmd->mutable_delete_range(); - - auto *r = deleteRange->mutable_range(); - - switch (fromKind) { - case EBorderKind::Include: - r->set_from_key_inclusive(from); - break; - case EBorderKind::Exclude: - r->set_from_key_exclusive(from); - break; - case EBorderKind::Without: - break; - } - - switch (toKind) { - case EBorderKind::Include: - r->set_to_key_inclusive(to); - break; - case EBorderKind::Exclude: - r->set_to_key_exclusive(to); - break; - case EBorderKind::Without: - break; - } - - ExecuteEvent(dp, tc); - if constexpr (IsSuccess) { - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); - } else { - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); - } -} - - -template <bool IsSuccess = true> -void ExecuteCopyRange(TTestContext &tc, - const TString &from, EBorderKind fromKind, - const TString &to, EBorderKind toKind, - ui64 lock_generation, const TString &prefixToAdd, const TString &prefixToRemove) -{ - TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; - dp.Request.set_lock_generation(lock_generation); - dp.Request.set_tablet_id(tc.TabletId); - - NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); - NKikimrKeyValue::ExecuteTransactionRequest::Command::CopyRange *copyRange = cmd->mutable_copy_range(); - - auto *r = copyRange->mutable_range(); - switch (fromKind) { - case EBorderKind::Include: - r->set_from_key_inclusive(from); - break; - case EBorderKind::Exclude: - r->set_from_key_exclusive(from); - break; - case EBorderKind::Without: - break; - } - - switch (toKind) { - case EBorderKind::Include: - r->set_to_key_inclusive(to); - break; - case EBorderKind::Exclude: - r->set_to_key_exclusive(to); - break; - case EBorderKind::Without: - break; - } - - copyRange->set_prefix_to_remove(prefixToRemove); - copyRange->set_prefix_to_add(prefixToAdd); - - ExecuteEvent(dp, tc); - if constexpr (IsSuccess) { - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); - } else { - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); - } -} - - -struct TKeyRenamePair { - TString OldKey; - TString NewKey; -}; - - -void ExecuteRename(TTestContext &tc, const TDeque<TKeyRenamePair> &pairs, ui64 lockedGeneration) -{ - TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; - - for (auto &[oldKey, newKey] : pairs) { - NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); - NKikimrKeyValue::ExecuteTransactionRequest::Command::Rename *rename = cmd->mutable_rename(); - - rename->set_old_key(oldKey); - rename->set_new_key(newKey); - } - - dp.Request.set_tablet_id(tc.TabletId); - dp.Request.set_lock_generation(lockedGeneration); - - ExecuteEvent(dp, tc); - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); -} - -void ExecuteConcat(TTestContext &tc, const TString &newKey, const TDeque<TString> &inputKeys, ui64 lockedGeneration, - bool keepKeys) -{ - TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; - - NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); - NKikimrKeyValue::ExecuteTransactionRequest::Command::Concat *concat = cmd->mutable_concat(); - - concat->set_output_key(newKey); - concat->set_keep_inputs(keepKeys); - for (auto &key : inputKeys) { - concat->add_input_keys(key); - } - - dp.Request.set_tablet_id(tc.TabletId); - dp.Request.set_lock_generation(lockedGeneration); - - ExecuteEvent(dp, tc); - UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " msg# " << dp.Response.msg()); -} - - -template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> -void ExecuteRead(TTestContext &tc, const TString &key, const TString &expectedValue, ui32 offset, ui32 size, - ui64 lock_generation, ui64 limit_bytes=0) -{ - TDesiredPair<TEvKeyValue::TEvRead> dp; - dp.Request.set_key(key); - dp.Request.set_offset(offset); - dp.Request.set_size(size); - dp.Request.set_lock_generation(lock_generation); - dp.Request.set_tablet_id(tc.TabletId); - dp.Request.set_limit_bytes(limit_bytes); - ExecuteEvent(dp, tc); - - UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) - << " msg# " << dp.Response.msg()); - if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK - || ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OVERRUN) - { - UNIT_ASSERT(dp.Response.value() == expectedValue); - if (size) { - UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_size(), size); - } - } else { - UNIT_ASSERT(dp.Response.value() == ""); - } - UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_key(), key); - UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_offset(), offset); -} - - -template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> -void ExecuteReadRange(TTestContext &tc, - const TString &from, EBorderKind fromKind, - const TString &to, EBorderKind toKind, - const TDeque<TKeyValuePair> &expectedPairs, - ui64 lock_generation, bool includeData, ui64 limitBytes) -{ - TDesiredPair<TEvKeyValue::TEvReadRange> dp; - - dp.Request.set_lock_generation(lock_generation); - dp.Request.set_include_data(includeData); - dp.Request.set_limit_bytes(limitBytes); - dp.Request.set_tablet_id(tc.TabletId); - - auto *r = dp.Request.mutable_range(); - switch (fromKind) { - case EBorderKind::Include: - r->set_from_key_inclusive(from); - break; - case EBorderKind::Exclude: - r->set_from_key_exclusive(from); - break; - case EBorderKind::Without: - break; + + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request.Reset(new TEvKeyValue::TEvRequest); + auto cmd = request->Record.MutableCmdSetExecutorFastLogPolicy(); + cmd->SetIsAllowed(isAllowed); + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); + result = tc.Runtime->GrabEdgeEvent<TEvKeyValue::TEvResponse>(handle); + UNIT_ASSERT(result); + UNIT_ASSERT(result->Record.HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetStatus(), NMsgBusProxy::MSTATUS_OK); + UNIT_ASSERT(result->Record.HasSetExecutorFastLogPolicyResult()); + UNIT_ASSERT(result->Record.GetSetExecutorFastLogPolicyResult().HasStatus()); + UNIT_ASSERT_EQUAL(result->Record.GetSetExecutorFastLogPolicyResult().GetStatus(), NKikimrProto::OK); + + return true; + }); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// NEW SINGLE COMMAND TEST FUNCTIONS +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct TKeyValuePair { + TString Key; + TString Value; +}; + +template <typename TRequestEvent> +void ExecuteEvent(TDesiredPair<TRequestEvent> &dp, TTestContext &tc) { + TAutoPtr<IEventHandle> handle; + std::unique_ptr<TRequestEvent> request; + typename TRequestEvent::TResponse *response; + DoWithRetry([&] { + tc.Runtime->ResetScheduledCount(); + request = std::make_unique<TRequestEvent>(); + request->Record = dp.Request; + tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.release(), 0, GetPipeConfigWithRetries()); + response = tc.Runtime->GrabEdgeEvent<typename TRequestEvent::TResponse>(handle); + dp.Response = response->Record; + return true; + }); +} + +template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> +void ExecuteWrite(TTestContext &tc, const TDeque<TKeyValuePair> &pairs, ui64 lockedGeneration, ui64 storageChannel, + NKikimrKeyValue::Priorities::Priority priority) +{ + TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; + + for (auto &[key, value] : pairs) { + NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); + NKikimrKeyValue::ExecuteTransactionRequest::Command::Write *write = cmd->mutable_write(); + + write->set_key(key); + write->set_value(value); + write->set_storage_channel(storageChannel); + write->set_priority(priority); + } + + dp.Request.set_tablet_id(tc.TabletId); + dp.Request.set_lock_generation(lockedGeneration); + + ExecuteEvent(dp, tc); + UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) + << " msg# " << dp.Response.msg()); +} + +enum class EBorderKind { + Include, + Exclude, + Without +}; + +template <bool IsSuccess = true> +void ExecuteDeleteRange(TTestContext &tc, + const TString &from, EBorderKind fromKind, + const TString &to, EBorderKind toKind, + ui64 lock_generation) +{ + TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; + dp.Request.set_lock_generation(lock_generation); + dp.Request.set_tablet_id(tc.TabletId); + + NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); + NKikimrKeyValue::ExecuteTransactionRequest::Command::DeleteRange *deleteRange = cmd->mutable_delete_range(); + + auto *r = deleteRange->mutable_range(); + + switch (fromKind) { + case EBorderKind::Include: + r->set_from_key_inclusive(from); + break; + case EBorderKind::Exclude: + r->set_from_key_exclusive(from); + break; + case EBorderKind::Without: + break; + } + + switch (toKind) { + case EBorderKind::Include: + r->set_to_key_inclusive(to); + break; + case EBorderKind::Exclude: + r->set_to_key_exclusive(to); + break; + case EBorderKind::Without: + break; + } + + ExecuteEvent(dp, tc); + if constexpr (IsSuccess) { + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); + } else { + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); + } +} + + +template <bool IsSuccess = true> +void ExecuteCopyRange(TTestContext &tc, + const TString &from, EBorderKind fromKind, + const TString &to, EBorderKind toKind, + ui64 lock_generation, const TString &prefixToAdd, const TString &prefixToRemove) +{ + TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; + dp.Request.set_lock_generation(lock_generation); + dp.Request.set_tablet_id(tc.TabletId); + + NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); + NKikimrKeyValue::ExecuteTransactionRequest::Command::CopyRange *copyRange = cmd->mutable_copy_range(); + + auto *r = copyRange->mutable_range(); + switch (fromKind) { + case EBorderKind::Include: + r->set_from_key_inclusive(from); + break; + case EBorderKind::Exclude: + r->set_from_key_exclusive(from); + break; + case EBorderKind::Without: + break; + } + + switch (toKind) { + case EBorderKind::Include: + r->set_to_key_inclusive(to); + break; + case EBorderKind::Exclude: + r->set_to_key_exclusive(to); + break; + case EBorderKind::Without: + break; + } + + copyRange->set_prefix_to_remove(prefixToRemove); + copyRange->set_prefix_to_add(prefixToAdd); + + ExecuteEvent(dp, tc); + if constexpr (IsSuccess) { + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); + } else { + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); + } +} + + +struct TKeyRenamePair { + TString OldKey; + TString NewKey; +}; + + +void ExecuteRename(TTestContext &tc, const TDeque<TKeyRenamePair> &pairs, ui64 lockedGeneration) +{ + TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; + + for (auto &[oldKey, newKey] : pairs) { + NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); + NKikimrKeyValue::ExecuteTransactionRequest::Command::Rename *rename = cmd->mutable_rename(); + + rename->set_old_key(oldKey); + rename->set_new_key(newKey); + } + + dp.Request.set_tablet_id(tc.TabletId); + dp.Request.set_lock_generation(lockedGeneration); + + ExecuteEvent(dp, tc); + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); +} + +void ExecuteConcat(TTestContext &tc, const TString &newKey, const TDeque<TString> &inputKeys, ui64 lockedGeneration, + bool keepKeys) +{ + TDesiredPair<TEvKeyValue::TEvExecuteTransaction> dp; + + NKikimrKeyValue::ExecuteTransactionRequest::Command *cmd = dp.Request.add_commands(); + NKikimrKeyValue::ExecuteTransactionRequest::Command::Concat *concat = cmd->mutable_concat(); + + concat->set_output_key(newKey); + concat->set_keep_inputs(keepKeys); + for (auto &key : inputKeys) { + concat->add_input_keys(key); + } + + dp.Request.set_tablet_id(tc.TabletId); + dp.Request.set_lock_generation(lockedGeneration); + + ExecuteEvent(dp, tc); + UNIT_ASSERT_C(dp.Response.status() == NKikimrKeyValue::Statuses::RSTATUS_OK, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " msg# " << dp.Response.msg()); +} + + +template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> +void ExecuteRead(TTestContext &tc, const TString &key, const TString &expectedValue, ui32 offset, ui32 size, + ui64 lock_generation, ui64 limit_bytes=0) +{ + TDesiredPair<TEvKeyValue::TEvRead> dp; + dp.Request.set_key(key); + dp.Request.set_offset(offset); + dp.Request.set_size(size); + dp.Request.set_lock_generation(lock_generation); + dp.Request.set_tablet_id(tc.TabletId); + dp.Request.set_limit_bytes(limit_bytes); + ExecuteEvent(dp, tc); + + UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) + << " msg# " << dp.Response.msg()); + if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK + || ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OVERRUN) + { + UNIT_ASSERT(dp.Response.value() == expectedValue); + if (size) { + UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_size(), size); + } + } else { + UNIT_ASSERT(dp.Response.value() == ""); } - - switch (toKind) { - case EBorderKind::Include: - r->set_to_key_inclusive(to); - break; - case EBorderKind::Exclude: - r->set_to_key_exclusive(to); - break; - case EBorderKind::Without: - break; - } - - ExecuteEvent(dp, tc); - UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) - << " msg# " << dp.Response.msg()); - if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK) { - UNIT_ASSERT_VALUES_EQUAL(dp.Response.pair_size(), expectedPairs.size()); - for (ui32 idx = 0; idx < expectedPairs.size(); ++idx) { - auto &pair = dp.Response.pair(idx); - UNIT_ASSERT_C(pair.status() == NKikimrKeyValue::Statuses::RSTATUS_OK || - pair.status() == NKikimrKeyValue::Statuses::RSTATUS_ERROR, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(pair.status()) - << " msg# " << dp.Response.msg()); - UNIT_ASSERT_VALUES_EQUAL_C(pair.key(), expectedPairs[idx].Key, "msg# " << dp.Response.msg()); - UNIT_ASSERT_VALUES_EQUAL_C(pair.value(), expectedPairs[idx].Value, "msg# " << dp.Response.msg()); - } - } -} - - -template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> -void ExecuteGetStatus(TTestContext &tc, const TDeque<ui32> &channels, ui64 lock_generation) { - TDesiredPair<TEvKeyValue::TEvGetStatus> dp; - dp.Request.set_lock_generation(lock_generation); - dp.Request.set_tablet_id(tc.TabletId); - for (ui32 channel : channels) { - dp.Request.add_storage_channel(channel); - } - - ExecuteEvent(dp, tc); - UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, - "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) - << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) - << " msg# " << dp.Response.msg()); - if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK) { - for (auto &channel : dp.Response.channel()) { - UNIT_ASSERT(channel.status() == NKikimrKeyValue::Statuses::RSTATUS_OK); - } - } -} - -void ExecuteObtainLock(TTestContext &tc, ui64 expectedLockGeneration) { - TDesiredPair<TEvKeyValue::TEvObtainLock> dp; - dp.Request.set_tablet_id(tc.TabletId); - ExecuteEvent(dp, tc); - UNIT_ASSERT_VALUES_EQUAL(dp.Response.lock_generation(), expectedLockGeneration); -} - + UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_key(), key); + UNIT_ASSERT_VALUES_EQUAL(dp.Response.requested_offset(), offset); +} + + +template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> +void ExecuteReadRange(TTestContext &tc, + const TString &from, EBorderKind fromKind, + const TString &to, EBorderKind toKind, + const TDeque<TKeyValuePair> &expectedPairs, + ui64 lock_generation, bool includeData, ui64 limitBytes) +{ + TDesiredPair<TEvKeyValue::TEvReadRange> dp; + + dp.Request.set_lock_generation(lock_generation); + dp.Request.set_include_data(includeData); + dp.Request.set_limit_bytes(limitBytes); + dp.Request.set_tablet_id(tc.TabletId); + + auto *r = dp.Request.mutable_range(); + switch (fromKind) { + case EBorderKind::Include: + r->set_from_key_inclusive(from); + break; + case EBorderKind::Exclude: + r->set_from_key_exclusive(from); + break; + case EBorderKind::Without: + break; + } + + switch (toKind) { + case EBorderKind::Include: + r->set_to_key_inclusive(to); + break; + case EBorderKind::Exclude: + r->set_to_key_exclusive(to); + break; + case EBorderKind::Without: + break; + } + + ExecuteEvent(dp, tc); + UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) + << " msg# " << dp.Response.msg()); + if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK) { + UNIT_ASSERT_VALUES_EQUAL(dp.Response.pair_size(), expectedPairs.size()); + for (ui32 idx = 0; idx < expectedPairs.size(); ++idx) { + auto &pair = dp.Response.pair(idx); + UNIT_ASSERT_C(pair.status() == NKikimrKeyValue::Statuses::RSTATUS_OK || + pair.status() == NKikimrKeyValue::Statuses::RSTATUS_ERROR, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(pair.status()) + << " msg# " << dp.Response.msg()); + UNIT_ASSERT_VALUES_EQUAL_C(pair.key(), expectedPairs[idx].Key, "msg# " << dp.Response.msg()); + UNIT_ASSERT_VALUES_EQUAL_C(pair.value(), expectedPairs[idx].Value, "msg# " << dp.Response.msg()); + } + } +} + + +template <NKikimrKeyValue::Statuses::ReplyStatus ExpectedStatus = NKikimrKeyValue::Statuses::RSTATUS_OK> +void ExecuteGetStatus(TTestContext &tc, const TDeque<ui32> &channels, ui64 lock_generation) { + TDesiredPair<TEvKeyValue::TEvGetStatus> dp; + dp.Request.set_lock_generation(lock_generation); + dp.Request.set_tablet_id(tc.TabletId); + for (ui32 channel : channels) { + dp.Request.add_storage_channel(channel); + } + + ExecuteEvent(dp, tc); + UNIT_ASSERT_C(dp.Response.status() == ExpectedStatus, + "got# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(dp.Response.status()) + << " exp# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(ExpectedStatus) + << " msg# " << dp.Response.msg()); + if constexpr (ExpectedStatus == NKikimrKeyValue::Statuses::RSTATUS_OK) { + for (auto &channel : dp.Response.channel()) { + UNIT_ASSERT(channel.status() == NKikimrKeyValue::Statuses::RSTATUS_OK); + } + } +} + +void ExecuteObtainLock(TTestContext &tc, ui64 expectedLockGeneration) { + TDesiredPair<TEvKeyValue::TEvObtainLock> dp; + dp.Request.set_tablet_id(tc.TabletId); + ExecuteEvent(dp, tc); + UNIT_ASSERT_VALUES_EQUAL(dp.Response.lock_generation(), expectedLockGeneration); +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TEST CASES //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -Y_UNIT_TEST(TestBasicWriteRead) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "value", 0, 0, 0); - }); -} - -Y_UNIT_TEST(TestBasicWriteReadOverrun) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ui64 limitBytes = 1 + 5 + 3 // Key id, length - + 1 + 5 + 1 // Value id, length, value - + 1 + 8 // Offset id, value - + 1 + 8 // Size id, value - + 1 + 1 // Status id, value - ; - ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, "key", "v", 0, 0, 0, limitBytes); - }); -} - +Y_UNIT_TEST(TestBasicWriteRead) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "value", 0, 0, 0); + }); +} + +Y_UNIT_TEST(TestBasicWriteReadOverrun) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ui64 limitBytes = 1 + 5 + 3 // Key id, length + + 1 + 5 + 1 // Value id, length, value + + 1 + 8 // Offset id, value + + 1 + 8 // Size id, value + + 1 + 1 // Status id, value + ; + ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, "key", "v", 0, 0, 0, limitBytes); + }); +} + Y_UNIT_TEST(TestWriteReadDeleteWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -868,22 +868,22 @@ Y_UNIT_TEST(TestWriteReadDeleteWithRestartsThenResponseOk) { }); } - -Y_UNIT_TEST(TestWriteReadDeleteWithRestartsThenResponseOkWithNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "value", 0, 0, 0); - ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); - ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); - }); -} - - + +Y_UNIT_TEST(TestWriteReadDeleteWithRestartsThenResponseOkWithNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + ExecuteWrite(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "value", 0, 0, 0); + ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); + ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); + }); +} + + Y_UNIT_TEST(TestRewriteThenLastValue) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -908,27 +908,27 @@ Y_UNIT_TEST(TestRewriteThenLastValue) { }); } - -Y_UNIT_TEST(TestRewriteThenLastValueNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TVector<ui32> channels = {0, 1}; - for (auto &ch1 : channels) { - for (auto &ch2 : channels) { - ExecuteWrite(tc, {{"key", "value"}}, 0, ch1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "value", 0, 0, 0); - ExecuteWrite(tc, {{"key", "updated"}}, 0, ch2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "updated", 0, 0, 0); - } - } - }); -} - - + +Y_UNIT_TEST(TestRewriteThenLastValueNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TVector<ui32> channels = {0, 1}; + for (auto &ch1 : channels) { + for (auto &ch2 : channels) { + ExecuteWrite(tc, {{"key", "value"}}, 0, ch1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "value", 0, 0, 0); + ExecuteWrite(tc, {{"key", "updated"}}, 0, ch2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "updated", 0, 0, 0); + } + } + }); +} + + Y_UNIT_TEST(TestWriteTrimWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1023,22 +1023,22 @@ Y_UNIT_TEST(TestInlineWriteReadDeleteWithRestartsThenResponseOk) { }); } - -Y_UNIT_TEST(TestInlineWriteReadDeleteWithRestartsThenResponseOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - ExecuteWrite(tc, {{"key", "value"}}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "value", 0, 0, 0); - ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); - ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); - }); -} - - + +Y_UNIT_TEST(TestInlineWriteReadDeleteWithRestartsThenResponseOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + ExecuteWrite(tc, {{"key", "value"}}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "value", 0, 0, 0); + ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); + ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); + }); +} + + Y_UNIT_TEST(TestWrite200KDeleteThenResponseError) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1067,32 +1067,32 @@ Y_UNIT_TEST(TestWrite200KDeleteThenResponseError) { }); } - -Y_UNIT_TEST(TestWrite200KDeleteThenResponseErrorNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - tc.Runtime->GetAppData(0).AllowHugeKeyValueDeletes = false; - tc.Runtime->SetDispatchedEventsLimit(10'000'0000); - activeZone = false; - const ui32 BatchCount = 60; - const ui32 ItemsPerBatch = 2048; - for (ui32 i = 0; i < BatchCount; ++i) { - TDeque<TKeyValuePair> pairs; - for (ui32 j = 0; j < ItemsPerBatch; ++j) { - TString key = Sprintf("k%08" PRIu32, i * ItemsPerBatch + j); - pairs.push_back({key, TString("v")}); - } - ExecuteWrite(tc, pairs, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_BACKGROUND); - } - ExecuteDeleteRange<false>(tc, "a", EBorderKind::Include, "z", EBorderKind::Include, 0); - }); -} - - + +Y_UNIT_TEST(TestWrite200KDeleteThenResponseErrorNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + tc.Runtime->GetAppData(0).AllowHugeKeyValueDeletes = false; + tc.Runtime->SetDispatchedEventsLimit(10'000'0000); + activeZone = false; + const ui32 BatchCount = 60; + const ui32 ItemsPerBatch = 2048; + for (ui32 i = 0; i < BatchCount; ++i) { + TDeque<TKeyValuePair> pairs; + for (ui32 j = 0; j < ItemsPerBatch; ++j) { + TString key = Sprintf("k%08" PRIu32, i * ItemsPerBatch + j); + pairs.push_back({key, TString("v")}); + } + ExecuteWrite(tc, pairs, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_BACKGROUND); + } + ExecuteDeleteRange<false>(tc, "a", EBorderKind::Include, "z", EBorderKind::Include, 0); + }); +} + + /*Y_UNIT_TEST(TestWrite16MillionReadDeleteWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1164,19 +1164,19 @@ Y_UNIT_TEST(TestWriteReadWithRestartsThenResponseOk) { expectedValues.push_back(values[itemIdx]); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key1", true, "key4", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key0", false, "key4", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key0", false, "key3", true, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1184,38 +1184,38 @@ Y_UNIT_TEST(TestWriteReadWithRestartsThenResponseOk) { }); } - -Y_UNIT_TEST(TestWriteReadWithRestartsThenResponseOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TDeque<TKeyValuePair> pairs; - { - TStringBuilder value; - for (ui32 itemIdx = 0; itemIdx <= 4; ++itemIdx) { - value << "x"; - pairs.push_back({Sprintf("key%" PRIu32, itemIdx), TString(value)}); - } - } - - ExecuteWrite(tc, pairs, 0, 11, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - TDeque<TKeyValuePair> expectedPairs; - for (ui32 itemIdx = 1; itemIdx < 4; ++itemIdx) { - expectedPairs.push_back(pairs[itemIdx]); - } - - ExecuteReadRange(tc, "key1", EBorderKind::Include, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); - }); -} - - + +Y_UNIT_TEST(TestWriteReadWithRestartsThenResponseOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TDeque<TKeyValuePair> pairs; + { + TStringBuilder value; + for (ui32 itemIdx = 0; itemIdx <= 4; ++itemIdx) { + value << "x"; + pairs.push_back({Sprintf("key%" PRIu32, itemIdx), TString(value)}); + } + } + + ExecuteWrite(tc, pairs, 0, 11, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + TDeque<TKeyValuePair> expectedPairs; + for (ui32 itemIdx = 1; itemIdx < 4; ++itemIdx) { + expectedPairs.push_back(pairs[itemIdx]); + } + + ExecuteReadRange(tc, "key1", EBorderKind::Include, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); + }); +} + + Y_UNIT_TEST(TestInlineWriteReadWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1250,19 +1250,19 @@ Y_UNIT_TEST(TestInlineWriteReadWithRestartsThenResponseOk) { expectedValues.push_back(values[itemIdx]); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key1", true, "key4", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key0", false, "key4", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key0", false, "key3", true, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1271,52 +1271,52 @@ Y_UNIT_TEST(TestInlineWriteReadWithRestartsThenResponseOk) { } -Y_UNIT_TEST(TestInlineWriteReadWithRestartsThenResponseOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TDeque<TKeyValuePair> expectedPairs; - { - TStringBuilder value; - for (ui32 itemIdx = 0; itemIdx <= 4; ++itemIdx) { - value << "x"; - TKeyValuePair pair{Sprintf("key%" PRIu32, itemIdx), TString(value)}; - if (itemIdx && itemIdx < 4) { - expectedPairs.push_back(pair); - } - ExecuteWrite(tc, {pair}, 0, itemIdx % 2 == 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - } - } - - ExecuteReadRange(tc, "key1", EBorderKind::Include, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); - ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); - }); -} - -Y_UNIT_TEST(TestInlineWriteReadWithRestartsWithNotCorrectUTF8NewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - - TDeque<TKeyValuePair> expectedPairs; - TKeyValuePair pair{TString("key1\0"), TString("value")}; - expectedPairs.push_back({TString("key1\\0"), TString()}); - ExecuteWrite(tc, {pair}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key0", EBorderKind::Include, - "key1", EBorderKind::Exclude, expectedPairs, 0, true, 0); - }); -} - - +Y_UNIT_TEST(TestInlineWriteReadWithRestartsThenResponseOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TDeque<TKeyValuePair> expectedPairs; + { + TStringBuilder value; + for (ui32 itemIdx = 0; itemIdx <= 4; ++itemIdx) { + value << "x"; + TKeyValuePair pair{Sprintf("key%" PRIu32, itemIdx), TString(value)}; + if (itemIdx && itemIdx < 4) { + expectedPairs.push_back(pair); + } + ExecuteWrite(tc, {pair}, 0, itemIdx % 2 == 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + } + } + + ExecuteReadRange(tc, "key1", EBorderKind::Include, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key4", EBorderKind::Exclude, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key0", EBorderKind::Exclude, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); + ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, expectedPairs, 0, true, 0); + }); +} + +Y_UNIT_TEST(TestInlineWriteReadWithRestartsWithNotCorrectUTF8NewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + + TDeque<TKeyValuePair> expectedPairs; + TKeyValuePair pair{TString("key1\0"), TString("value")}; + expectedPairs.push_back({TString("key1\\0"), TString()}); + ExecuteWrite(tc, {pair}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key0", EBorderKind::Include, + "key1", EBorderKind::Exclude, expectedPairs, 0, true, 0); + }); +} + + Y_UNIT_TEST(TestEmptyWriteReadDeleteWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1334,7 +1334,7 @@ Y_UNIT_TEST(TestEmptyWriteReadDeleteWithRestartsThenResponseOk) { expectedKeys.push_back("key"); expectedValues.push_back(""); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1346,26 +1346,26 @@ Y_UNIT_TEST(TestEmptyWriteReadDeleteWithRestartsThenResponseOk) { }); } - -Y_UNIT_TEST(TestEmptyWriteReadDeleteWithRestartsThenResponseOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TDeque<TKeyValuePair> pairs; - pairs.push_back({"key", ""}); - ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "", 0, 0, 0); - - ExecuteReadRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, pairs, 0, true, 0); - ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); - ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); - }); -} - - + +Y_UNIT_TEST(TestEmptyWriteReadDeleteWithRestartsThenResponseOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TDeque<TKeyValuePair> pairs; + pairs.push_back({"key", ""}); + ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "", 0, 0, 0); + + ExecuteReadRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, pairs, 0, true, 0); + ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); + ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); + }); +} + + Y_UNIT_TEST(TestInlineEmptyWriteReadDeleteWithRestartsThenResponseOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1383,7 +1383,7 @@ Y_UNIT_TEST(TestInlineEmptyWriteReadDeleteWithRestartsThenResponseOk) { expectedKeys.push_back("key"); expectedValues.push_back(""); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1395,26 +1395,26 @@ Y_UNIT_TEST(TestInlineEmptyWriteReadDeleteWithRestartsThenResponseOk) { }); } - -Y_UNIT_TEST(TestInlineEmptyWriteReadDeleteWithRestartsThenResponseOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TDeque<TKeyValuePair> pairs; - pairs.push_back({"key", ""}); - ExecuteWrite(tc, pairs, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead(tc, "key", "", 0, 0, 0); - - ExecuteReadRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, pairs, 0, true, 0); - ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); - ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); - }); -} - - + +Y_UNIT_TEST(TestInlineEmptyWriteReadDeleteWithRestartsThenResponseOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TDeque<TKeyValuePair> pairs; + pairs.push_back({"key", ""}); + ExecuteWrite(tc, pairs, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead(tc, "key", "", 0, 0, 0); + + ExecuteReadRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, pairs, 0, true, 0); + ExecuteDeleteRange(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, 0); + ExecuteRead<NKikimrKeyValue::Statuses::RSTATUS_NO_DATA>(tc, "key", "", 0, 0, 0); + }); +} + + TString PrepareData(ui32 size, ui32 flavor) { TString data = TString::Uninitialized(size); for (ui32 i = 0; i < size; ++i) { @@ -1455,13 +1455,13 @@ Y_UNIT_TEST(TestWriteReadRangeLimitThenLimitWorks) { Y_VERIFY(expectedKeys.size()); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, false, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, false, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); AddCmdReadRange("k0000000", true, "k9999999", true, false, sizeLimit, @@ -1469,7 +1469,7 @@ Y_UNIT_TEST(TestWriteReadRangeLimitThenLimitWorks) { RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, expectedKeys[expectedKeys.size() - 1], true, false, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); AddCmdReadRange("k0000000", true, "k9999999", true, false, sizeLimit, @@ -1481,43 +1481,43 @@ Y_UNIT_TEST(TestWriteReadRangeLimitThenLimitWorks) { }); } - -Y_UNIT_TEST(TestWriteReadRangeLimitThenLimitWorksNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - const ui32 itemCount = 1000; - TDeque<TKeyValuePair> pairs; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), PrepareData(10000, itemIdx)}); - } - ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - TDeque<TKeyValuePair> expectedPairs; - ui64 totalSize = 0; - ui64 sizeLimit = 200; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - totalSize += pairs[itemIdx].Key.size() + 32; - if (totalSize > sizeLimit) { - break; - } - expectedPairs.push_back({pairs[itemIdx].Key, ""}); - } - Y_VERIFY(expectedPairs.size()); - - ExecuteReadRange(tc, "", EBorderKind::Without, - expectedPairs[expectedPairs.size() - 1].Key, EBorderKind::Include, - expectedPairs, 0, false, sizeLimit); - ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, - "", EBorderKind::Without, "", EBorderKind::Without, - expectedPairs, 0, false, sizeLimit); - }); -} - - + +Y_UNIT_TEST(TestWriteReadRangeLimitThenLimitWorksNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + const ui32 itemCount = 1000; + TDeque<TKeyValuePair> pairs; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), PrepareData(10000, itemIdx)}); + } + ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + TDeque<TKeyValuePair> expectedPairs; + ui64 totalSize = 0; + ui64 sizeLimit = 200; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + totalSize += pairs[itemIdx].Key.size() + 32; + if (totalSize > sizeLimit) { + break; + } + expectedPairs.push_back({pairs[itemIdx].Key, ""}); + } + Y_VERIFY(expectedPairs.size()); + + ExecuteReadRange(tc, "", EBorderKind::Without, + expectedPairs[expectedPairs.size() - 1].Key, EBorderKind::Include, + expectedPairs, 0, false, sizeLimit); + ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, + "", EBorderKind::Without, "", EBorderKind::Without, + expectedPairs, 0, false, sizeLimit); + }); +} + + Y_UNIT_TEST(TestWriteReadRangeDataLimitThenLimitWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1550,13 +1550,13 @@ Y_UNIT_TEST(TestWriteReadRangeDataLimitThenLimitWorks) { Y_VERIFY(expectedKeys.size()); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, @@ -1566,46 +1566,46 @@ Y_UNIT_TEST(TestWriteReadRangeDataLimitThenLimitWorks) { }); } - -void MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(ui32 storageChannel) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - const ui32 itemCount = 1000; - TDeque<TKeyValuePair> pairs; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), PrepareData(8, itemIdx)}); - } - ExecuteWrite(tc, pairs, 0, storageChannel, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - TDeque<TKeyValuePair> expectedPairs; - ui64 totalSize = 0; - ui64 sizeLimit = 200; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - totalSize += pairs[itemIdx].Key.size() + pairs[itemIdx].Value.size() + 32; - if (totalSize > sizeLimit) { - break; - } - expectedPairs.push_back(pairs[itemIdx]); - } - - ExecuteReadRange(tc, "", EBorderKind::Without, - expectedPairs[expectedPairs.size() - 1].Key, EBorderKind::Include, - expectedPairs, 0, true, sizeLimit); - ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, - "", EBorderKind::Without, "", EBorderKind::Without, - expectedPairs, 0, true, sizeLimit); - }); -} - -Y_UNIT_TEST(TestWriteReadRangeDataLimitThenLimitWorksNewApi) { - MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(0); -} - - + +void MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(ui32 storageChannel) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + const ui32 itemCount = 1000; + TDeque<TKeyValuePair> pairs; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), PrepareData(8, itemIdx)}); + } + ExecuteWrite(tc, pairs, 0, storageChannel, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + TDeque<TKeyValuePair> expectedPairs; + ui64 totalSize = 0; + ui64 sizeLimit = 200; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + totalSize += pairs[itemIdx].Key.size() + pairs[itemIdx].Value.size() + 32; + if (totalSize > sizeLimit) { + break; + } + expectedPairs.push_back(pairs[itemIdx]); + } + + ExecuteReadRange(tc, "", EBorderKind::Without, + expectedPairs[expectedPairs.size() - 1].Key, EBorderKind::Include, + expectedPairs, 0, true, sizeLimit); + ExecuteReadRange<NKikimrKeyValue::Statuses::RSTATUS_OVERRUN>(tc, + "", EBorderKind::Without, "", EBorderKind::Without, + expectedPairs, 0, true, sizeLimit); + }); +} + +Y_UNIT_TEST(TestWriteReadRangeDataLimitThenLimitWorksNewApi) { + MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(0); +} + + Y_UNIT_TEST(TestInlineWriteReadRangeLimitThenLimitWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1638,13 +1638,13 @@ Y_UNIT_TEST(TestInlineWriteReadRangeLimitThenLimitWorks) { Y_VERIFY(expectedKeys.size()); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); RunRequest(dp, tc, __LINE__); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OVERRUN, tc, dp); AddCmdReadRange("k0000000", true, "k9999999", true, true, sizeLimit, @@ -1654,12 +1654,12 @@ Y_UNIT_TEST(TestInlineWriteReadRangeLimitThenLimitWorks) { }); } - -Y_UNIT_TEST(TestInlineWriteReadRangeLimitThenLimitWorksNewApi) { - MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(1); -} - - + +Y_UNIT_TEST(TestInlineWriteReadRangeLimitThenLimitWorksNewApi) { + MakeTestWriteReadRangeDataLimitThenLimitWorksNewApi(1); +} + + Y_UNIT_TEST(TestCopyRangeWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1687,49 +1687,49 @@ Y_UNIT_TEST(TestCopyRangeWorks) { expectedKeys.push_back(Sprintf("pk%07" PRIu32, itemIdx)); expectedValues.push_back(Sprintf("v%07" PRIu32, (ui32)itemIdx)); } - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("p", false, "q", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); }); } - -void MakeTestCopyRangeWorksNewApi(ui64 storageChannel) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - const ui32 itemCount = 100; - TDeque<TKeyValuePair> pairs; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), Sprintf("v%07" PRIu32, (ui32)itemIdx)}); - } - ExecuteWrite(tc, pairs, 0, storageChannel, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - ExecuteCopyRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, 0, "p", ""); - ExecuteWrite(tc, {{"0", "0"}}, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - - TDeque<TString> expectedKeys; - TDeque<TString> expectedValues; - - TDeque<TKeyValuePair> expectedPairs; - for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { - expectedPairs.push_back({Sprintf("pk%07" PRIu32, itemIdx), Sprintf("v%07" PRIu32, (ui32)itemIdx)}); - } - ExecuteReadRange(tc, "p", EBorderKind::Exclude, "q", EBorderKind::Exclude, - expectedPairs, 0, true, 0); - }); -} - - -Y_UNIT_TEST(TestCopyRangeWorksNewApi) { - MakeTestCopyRangeWorksNewApi(0); -} - - + +void MakeTestCopyRangeWorksNewApi(ui64 storageChannel) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + const ui32 itemCount = 100; + TDeque<TKeyValuePair> pairs; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + pairs.push_back({Sprintf("k%07" PRIu32, itemIdx), Sprintf("v%07" PRIu32, (ui32)itemIdx)}); + } + ExecuteWrite(tc, pairs, 0, storageChannel, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + ExecuteCopyRange(tc, "", EBorderKind::Without, "", EBorderKind::Without, 0, "p", ""); + ExecuteWrite(tc, {{"0", "0"}}, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + + TDeque<TString> expectedKeys; + TDeque<TString> expectedValues; + + TDeque<TKeyValuePair> expectedPairs; + for (ui32 itemIdx = 0; itemIdx < itemCount; ++itemIdx) { + expectedPairs.push_back({Sprintf("pk%07" PRIu32, itemIdx), Sprintf("v%07" PRIu32, (ui32)itemIdx)}); + } + ExecuteReadRange(tc, "p", EBorderKind::Exclude, "q", EBorderKind::Exclude, + expectedPairs, 0, true, 0); + }); +} + + +Y_UNIT_TEST(TestCopyRangeWorksNewApi) { + MakeTestCopyRangeWorksNewApi(0); +} + + Y_UNIT_TEST(TestInlineCopyRangeWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1763,7 +1763,7 @@ Y_UNIT_TEST(TestInlineCopyRangeWorks) { expectedKeys.push_back(Sprintf("pk%07" PRIu32, itemIdx)); expectedValues.push_back(Sprintf("v%07" PRIu32, (ui32)itemIdx)); } - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("p", false, "q", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, expectedKeys, expectedValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1771,11 +1771,11 @@ Y_UNIT_TEST(TestInlineCopyRangeWorks) { } -Y_UNIT_TEST(TestInlineCopyRangeWorksNewApi) { - MakeTestCopyRangeWorksNewApi(1); -} - +Y_UNIT_TEST(TestInlineCopyRangeWorksNewApi) { + MakeTestCopyRangeWorksNewApi(1); +} + Y_UNIT_TEST(TestConcatWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1787,14 +1787,14 @@ Y_UNIT_TEST(TestConcatWorks) { NKikimrClient::TKeyValueRequest::REALTIME, tc); CmdConcat({"1", "2", "3", "4"}, "5", false, tc); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("1", true, "5", true, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, {"5"}, {"hello, world!"}, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); } CmdConcat({"5", "5"}, "5", true, tc); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("1", true, "5", true, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, {"5"}, {"hello, world!hello, world!"}, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1803,29 +1803,29 @@ Y_UNIT_TEST(TestConcatWorks) { } -Y_UNIT_TEST(TestConcatWorksNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TDeque<TKeyValuePair> pairs = { - {"1", "hello"}, - {"2", ", "}, - {"3", "world"}, - {"4", "!"}, - }; - ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteConcat(tc, "5", {"1", "2", "3", "4"}, 0, false); - ExecuteReadRange(tc, "1", EBorderKind::Include, "5", EBorderKind::Include, {{"5", "hello, world!"}}, 0, true, 0); - ExecuteConcat(tc, "5", {"5", "5"}, 0, true); - ExecuteReadRange(tc, "1", EBorderKind::Include, "5", EBorderKind::Include, - {{"5", "hello, world!hello, world!"}}, 0, true, 0); - }); -} - - +Y_UNIT_TEST(TestConcatWorksNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TDeque<TKeyValuePair> pairs = { + {"1", "hello"}, + {"2", ", "}, + {"3", "world"}, + {"4", "!"}, + }; + ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteConcat(tc, "5", {"1", "2", "3", "4"}, 0, false); + ExecuteReadRange(tc, "1", EBorderKind::Include, "5", EBorderKind::Include, {{"5", "hello, world!"}}, 0, true, 0); + ExecuteConcat(tc, "5", {"5", "5"}, 0, true); + ExecuteReadRange(tc, "1", EBorderKind::Include, "5", EBorderKind::Include, + {{"5", "hello, world!hello, world!"}}, 0, true, 0); + }); +} + + Y_UNIT_TEST(TestRenameWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1837,7 +1837,7 @@ Y_UNIT_TEST(TestRenameWorks) { NKikimrClient::TKeyValueRequest::REALTIME, tc); CmdRename("2", "3", tc); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("1", true, "3", true, true, 1000, NKikimrClient::TKeyValueRequest::REALTIME, {"1", "3"}, {"123", "456"}, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1846,28 +1846,28 @@ Y_UNIT_TEST(TestRenameWorks) { } -Y_UNIT_TEST(TestRenameWorksewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - - TDeque<TKeyValuePair> pairs = { - {"1", "123"}, - {"2", "456"}, - {"3", "789"}, - {"4", "012"}, - }; - ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRename(tc, {{"2", "3"}}, 0); - ExecuteReadRange(tc, "1", EBorderKind::Include, "3", EBorderKind::Include, - {{"1", "123"}, {"3", "456"}}, 0, true, 0); - }); -} - - +Y_UNIT_TEST(TestRenameWorksewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + + TDeque<TKeyValuePair> pairs = { + {"1", "123"}, + {"2", "456"}, + {"3", "789"}, + {"4", "012"}, + }; + ExecuteWrite(tc, pairs, 0, 0, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRename(tc, {{"2", "3"}}, 0); + ExecuteReadRange(tc, "1", EBorderKind::Include, "3", EBorderKind::Include, + {{"1", "123"}, {"3", "456"}}, 0, true, 0); + }); +} + + Y_UNIT_TEST(TestWriteToExtraChannelThenReadMixedChannelsReturnsOk) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1884,7 +1884,7 @@ Y_UNIT_TEST(TestWriteToExtraChannelThenReadMixedChannelsReturnsOk) { CmdRead({"key1", "key2", "key3"}, NKikimrClient::TKeyValueRequest::REALTIME, {"value1", "value2", "value3"}, {}, tc); { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("key1", true, "key3", true, true, 1000, NKikimrClient::TKeyValueRequest::REALTIME, {"key1", "key2", "key3"}, {"value1", "value2", "value3"}, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -1892,24 +1892,24 @@ Y_UNIT_TEST(TestWriteToExtraChannelThenReadMixedChannelsReturnsOk) { }); } - -Y_UNIT_TEST(TestWriteToExtraChannelThenReadMixedChannelsReturnsOkNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - - ExecuteWrite(tc, {{"key1", "value1"}}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteWrite(tc, {{"key2", "value2"}}, 0, 3, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteWrite(tc, {{"key3", "value3"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, - {{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}, 0, true, 0); - }); -} - - + +Y_UNIT_TEST(TestWriteToExtraChannelThenReadMixedChannelsReturnsOkNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + + ExecuteWrite(tc, {{"key1", "value1"}}, 0, 1, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteWrite(tc, {{"key2", "value2"}}, 0, 3, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteWrite(tc, {{"key3", "value3"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteReadRange(tc, "key1", EBorderKind::Include, "key3", EBorderKind::Include, + {{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}, 0, true, 0); + }); +} + + Y_UNIT_TEST(TestIncrementalKeySet) { // generate initial key set TSet<TString> keys; @@ -2003,17 +2003,17 @@ Y_UNIT_TEST(TestGetStatusWorks) { }); } -Y_UNIT_TEST(TestGetStatusWorksNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - ExecuteGetStatus(tc, {1, 2}, 0); - }); -} - +Y_UNIT_TEST(TestGetStatusWorksNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + ExecuteGetStatus(tc, {1, 2}, 0); + }); +} + Y_UNIT_TEST(TestWriteReadWhileWriteWorks) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -2092,7 +2092,7 @@ Y_UNIT_TEST(TestWriteDeleteThenReadRemaining) { NKikimrClient::TKeyValueRequest::REALTIME, tc); } { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, allKeys, allValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -2120,7 +2120,7 @@ Y_UNIT_TEST(TestWriteDeleteThenReadRemaining) { // Cerr << "total keys: " << BatchCount * ItemsPerBatch << ", after deletes: " << aliveKeys.size() << Endl; activeZone = true; { - TDesiredPair<TEvKeyValue::TEvRequest> dp; + TDesiredPair<TEvKeyValue::TEvRequest> dp; AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, aliveKeys, aliveValues, NKikimrProto::OK, tc, dp); RunRequest(dp, tc, __LINE__); @@ -2128,27 +2128,27 @@ Y_UNIT_TEST(TestWriteDeleteThenReadRemaining) { }); } - -Y_UNIT_TEST(TestObtainLockNewApi) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - constexpr NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_ERROR; - ExecuteObtainLock(tc, 1); - - ExecuteGetStatus<status>(tc, {1, 2}, 0); - ExecuteWrite<status>(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - ExecuteRead<status>(tc, "key", "value", 0, 0, 0); - ExecuteReadRange<status>(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, - {{"key", "value"}}, 0, true, 0); - - ExecuteGetStatus(tc, {1, 2}, 1); - ExecuteWrite(tc, {{"key", "value"}}, 1, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); - }); -} - + +Y_UNIT_TEST(TestObtainLockNewApi) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + constexpr NKikimrKeyValue::Statuses::ReplyStatus status = NKikimrKeyValue::Statuses::RSTATUS_ERROR; + ExecuteObtainLock(tc, 1); + + ExecuteGetStatus<status>(tc, {1, 2}, 0); + ExecuteWrite<status>(tc, {{"key", "value"}}, 0, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + ExecuteRead<status>(tc, "key", "value", 0, 0, 0); + ExecuteReadRange<status>(tc, "key", EBorderKind::Include, "key", EBorderKind::Include, + {{"key", "value"}}, 0, true, 0); + + ExecuteGetStatus(tc, {1, 2}, 1); + ExecuteWrite(tc, {{"key", "value"}}, 1, 2, NKikimrKeyValue::Priorities::PRIORITY_REALTIME); + }); +} + } // TKeyValueTest } // NKikimr diff --git a/ydb/core/keyvalue/protos/events.proto b/ydb/core/keyvalue/protos/events.proto index fa365981ae9..52ee16a4d69 100644 --- a/ydb/core/keyvalue/protos/events.proto +++ b/ydb/core/keyvalue/protos/events.proto @@ -1,190 +1,190 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NKikimrKeyValue; -option java_package = "ru.yandex.kikimr.proto"; -option java_outer_classname = "KeyValueProtos"; -option java_multiple_files = true; - - -message Priorities { - enum Priority { - PRIORITY_UNSPECIFIED = 0; - - // High priority for user-initiated operations. - PRIORITY_REALTIME = 1; - - // Low prioroty for background system activity. - PRIORITY_BACKGROUND = 2; - } -} - -message Statuses { - enum ReplyStatus { - RSTATUS_UNSPECIFIED = 0; - RSTATUS_OK = 1; - RSTATUS_ERROR = 2; - RSTATUS_TIMEOUT = 3; - RSTATUS_INTERNAL_ERROR = 4; - RSTATUS_NO_DATA = 5; - RSTATUS_OVERRUN = 6; - } -} - -message Flags { - bool disk_space_cyan = 1; - bool disk_space_light_yellow_move = 2; - bool disk_space_yellow_stop = 3; - bool disk_space_light_orange = 4; - bool disk_space_orange = 5; - bool disk_space_red = 6; - bool disk_space_black = 7; -} - -message Channel { - Statuses.ReplyStatus status = 1; - uint32 storage_channel = 2; - optional Flags status_flags = 3; -} - -message KVRange { - oneof from_bound { - bytes from_key_inclusive = 1; - bytes from_key_exclusive = 2; - } - - oneof to_bound { - bytes to_key_inclusive = 3; - bytes to_key_exclusive = 4; - } -} - -message ReadRequest { - uint64 tablet_id = 1; - uint64 lock_generation = 2; - bytes key = 3; - uint64 offset = 4; - uint64 size = 5; - uint64 cookie = 6; - Priorities.Priority priority = 7; - uint64 deadline_instant_ms = 8; - uint64 limit_bytes = 9; -} - -message ReadResult { - bytes requested_key = 1; - uint64 requested_offset = 2; - uint64 requested_size = 3; - bytes value = 4; - string msg = 5; - Statuses.ReplyStatus status = 6; - uint64 cookie = 7; -} - -message ReadRangeRequest { - uint64 tablet_id = 1; - uint64 lock_generation = 2; - - KVRange range = 3; - - bool include_data = 4; - uint64 limit_bytes = 5; - Priorities.Priority priority = 6; - uint64 cookie = 7; - uint64 deadline_instant_ms = 8; -} - -message ReadRangeResult { - message KeyValuePair { - bytes key = 1; - bytes value = 2; - uint32 value_size = 3; - - // Unix time of the creation of the key-value pair (in ms). - uint64 creation_unix_time = 4; - uint32 storage_channel = 5; // Returns the _actual_ storage channel - Statuses.ReplyStatus status = 6; - } - Statuses.ReplyStatus status = 1; - string msg = 2; - repeated KeyValuePair pair = 3; - uint64 cookie = 4; -} - -message ExecuteTransactionRequest { - message Command { - message Rename { - bytes old_key = 1; - bytes new_key = 2; - } - message Concat { - repeated bytes input_keys = 1; - bytes output_key = 2; - bool keep_inputs = 3; - } - message CopyRange { - KVRange range = 1; - bytes prefix_to_remove = 2; - bytes prefix_to_add = 3; - } - message Write { - enum Tactic { - TACTIC_UNSPECIFIED = 0; - TACTIC_MAX_THROUGHPUT = 1; - TACTIC_MIN_LATENCY = 2; - } - bytes key = 1; - bytes value = 2; - uint32 storage_channel = 3; - Priorities.Priority priority = 4; - Tactic tactic = 5; - } - message DeleteRange { - KVRange range = 1; - } - - oneof action { - DeleteRange delete_range = 1; - Rename rename = 2; - CopyRange copy_range = 3; - Concat concat = 4; - Write write = 5; - } - } - - uint64 tablet_id = 1; - uint64 lock_generation = 2; - repeated Command commands = 3; - uint64 cookie = 4; - uint64 deadline_instant_ms = 5; -} - -message ExecuteTransactionResult { - repeated Channel channel = 1; - Statuses.ReplyStatus status = 2; - string msg = 3; - uint64 cookie = 4; -} - -message GetStatusRequest { - uint64 tablet_id = 1; - uint64 lock_generation = 2; - repeated uint32 storage_channel = 3; - uint64 deadline_instant_ms = 4; -} - -message GetStatusResult { - repeated Channel channel = 1; - Statuses.ReplyStatus status = 2; - string msg = 3; -} - -message ObtainLockRequest { - uint64 tablet_id = 1; - uint64 cookie = 2; -} - -message ObtainLockResult { - uint64 lock_generation = 1; - uint64 cookie = 2; -} +syntax = "proto3"; +option cc_enable_arenas = true; + +package NKikimrKeyValue; +option java_package = "ru.yandex.kikimr.proto"; +option java_outer_classname = "KeyValueProtos"; +option java_multiple_files = true; + + +message Priorities { + enum Priority { + PRIORITY_UNSPECIFIED = 0; + + // High priority for user-initiated operations. + PRIORITY_REALTIME = 1; + + // Low prioroty for background system activity. + PRIORITY_BACKGROUND = 2; + } +} + +message Statuses { + enum ReplyStatus { + RSTATUS_UNSPECIFIED = 0; + RSTATUS_OK = 1; + RSTATUS_ERROR = 2; + RSTATUS_TIMEOUT = 3; + RSTATUS_INTERNAL_ERROR = 4; + RSTATUS_NO_DATA = 5; + RSTATUS_OVERRUN = 6; + } +} + +message Flags { + bool disk_space_cyan = 1; + bool disk_space_light_yellow_move = 2; + bool disk_space_yellow_stop = 3; + bool disk_space_light_orange = 4; + bool disk_space_orange = 5; + bool disk_space_red = 6; + bool disk_space_black = 7; +} + +message Channel { + Statuses.ReplyStatus status = 1; + uint32 storage_channel = 2; + optional Flags status_flags = 3; +} + +message KVRange { + oneof from_bound { + bytes from_key_inclusive = 1; + bytes from_key_exclusive = 2; + } + + oneof to_bound { + bytes to_key_inclusive = 3; + bytes to_key_exclusive = 4; + } +} + +message ReadRequest { + uint64 tablet_id = 1; + uint64 lock_generation = 2; + bytes key = 3; + uint64 offset = 4; + uint64 size = 5; + uint64 cookie = 6; + Priorities.Priority priority = 7; + uint64 deadline_instant_ms = 8; + uint64 limit_bytes = 9; +} + +message ReadResult { + bytes requested_key = 1; + uint64 requested_offset = 2; + uint64 requested_size = 3; + bytes value = 4; + string msg = 5; + Statuses.ReplyStatus status = 6; + uint64 cookie = 7; +} + +message ReadRangeRequest { + uint64 tablet_id = 1; + uint64 lock_generation = 2; + + KVRange range = 3; + + bool include_data = 4; + uint64 limit_bytes = 5; + Priorities.Priority priority = 6; + uint64 cookie = 7; + uint64 deadline_instant_ms = 8; +} + +message ReadRangeResult { + message KeyValuePair { + bytes key = 1; + bytes value = 2; + uint32 value_size = 3; + + // Unix time of the creation of the key-value pair (in ms). + uint64 creation_unix_time = 4; + uint32 storage_channel = 5; // Returns the _actual_ storage channel + Statuses.ReplyStatus status = 6; + } + Statuses.ReplyStatus status = 1; + string msg = 2; + repeated KeyValuePair pair = 3; + uint64 cookie = 4; +} + +message ExecuteTransactionRequest { + message Command { + message Rename { + bytes old_key = 1; + bytes new_key = 2; + } + message Concat { + repeated bytes input_keys = 1; + bytes output_key = 2; + bool keep_inputs = 3; + } + message CopyRange { + KVRange range = 1; + bytes prefix_to_remove = 2; + bytes prefix_to_add = 3; + } + message Write { + enum Tactic { + TACTIC_UNSPECIFIED = 0; + TACTIC_MAX_THROUGHPUT = 1; + TACTIC_MIN_LATENCY = 2; + } + bytes key = 1; + bytes value = 2; + uint32 storage_channel = 3; + Priorities.Priority priority = 4; + Tactic tactic = 5; + } + message DeleteRange { + KVRange range = 1; + } + + oneof action { + DeleteRange delete_range = 1; + Rename rename = 2; + CopyRange copy_range = 3; + Concat concat = 4; + Write write = 5; + } + } + + uint64 tablet_id = 1; + uint64 lock_generation = 2; + repeated Command commands = 3; + uint64 cookie = 4; + uint64 deadline_instant_ms = 5; +} + +message ExecuteTransactionResult { + repeated Channel channel = 1; + Statuses.ReplyStatus status = 2; + string msg = 3; + uint64 cookie = 4; +} + +message GetStatusRequest { + uint64 tablet_id = 1; + uint64 lock_generation = 2; + repeated uint32 storage_channel = 3; + uint64 deadline_instant_ms = 4; +} + +message GetStatusResult { + repeated Channel channel = 1; + Statuses.ReplyStatus status = 2; + string msg = 3; +} + +message ObtainLockRequest { + uint64 tablet_id = 1; + uint64 cookie = 2; +} + +message ObtainLockResult { + uint64 lock_generation = 1; + uint64 cookie = 2; +} diff --git a/ydb/core/keyvalue/protos/ya.make b/ydb/core/keyvalue/protos/ya.make index 464bec3c220..9631e46975a 100644 --- a/ydb/core/keyvalue/protos/ya.make +++ b/ydb/core/keyvalue/protos/ya.make @@ -1,18 +1,18 @@ -PROTO_LIBRARY() - +PROTO_LIBRARY() + OWNER( kruall g:kikimr ) - -IF (OS_WINDOWS) - NO_OPTIMIZE_PY_PROTOS() -ENDIF() - -SRCS( - events.proto -) - -EXCLUDE_TAGS(GO_PROTO) - -END() + +IF (OS_WINDOWS) + NO_OPTIMIZE_PY_PROTOS() +ENDIF() + +SRCS( + events.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/core/keyvalue/ut/ya.make b/ydb/core/keyvalue/ut/ya.make index 7124f2ccf87..7eb389580db 100644 --- a/ydb/core/keyvalue/ut/ya.make +++ b/ydb/core/keyvalue/ut/ya.make @@ -15,7 +15,7 @@ IF (WITH_VALGRIND OR SANITIZER_TYPE) ELSE() TIMEOUT(600) SIZE(MEDIUM) - SPLIT_FACTOR(10) + SPLIT_FACTOR(10) ENDIF() PEERDIR( @@ -30,7 +30,7 @@ YQL_LAST_ABI_VERSION() SRCS( keyvalue_ut.cpp keyvalue_collector_ut.cpp - keyvalue_storage_read_request_ut.cpp + keyvalue_storage_read_request_ut.cpp ) END() diff --git a/ydb/core/keyvalue/ya.make b/ydb/core/keyvalue/ya.make index 17bb1853c05..e924e8df91c 100644 --- a/ydb/core/keyvalue/ya.make +++ b/ydb/core/keyvalue/ya.make @@ -38,7 +38,7 @@ SRCS( keyvalue_state.cpp keyvalue_state.h keyvalue_state_collect.cpp - keyvalue_storage_read_request.cpp + keyvalue_storage_read_request.cpp keyvalue_storage_request.cpp keyvalue_storage_request.h keyvalue_stored_state_data.cpp @@ -48,7 +48,7 @@ SRCS( PEERDIR( library/cpp/actors/core - library/cpp/actors/protos + library/cpp/actors/protos ydb/core/base ydb/core/blobstorage/base ydb/core/blobstorage/dsproxy @@ -65,7 +65,7 @@ END() RECURSE( protos ) - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index a08ccbf865c..30363915c5c 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -122,7 +122,7 @@ message TMsgQoS { uint32 ProxyNodeId = 10; // set when client is DS Proxy from specific node uint32 ReplVDiskId = 11; // set when client is replication actor from specific vdisk uint64 VDiskLoadId = 13; // set when client is load test with specific tag - uint32 VPatchVDiskId = 14; // set when client is vpatch actor from specific vdisk + uint32 VPatchVDiskId = 14; // set when client is vpatch actor from specific vdisk } optional TExecTimeStats ExecTimeStats = 12; optional NActorsProto.TActorId SenderActorId = 15; @@ -188,172 +188,172 @@ message TGroupInfo { optional EPDiskType DeviceType = 14; } -message TEvVPatchStart { - optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; - optional NKikimrBlobStorage.TVDiskID VDiskID = 3; - - optional bool NotifyIfNotReady = 4; - optional uint64 Cookie = 5; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TEvVPatchFoundParts { - optional NKikimrProto.EReplyStatus Status = 1; - optional string ErrorReason = 200; // textual description of error - optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; - optional NKikimrBlobStorage.TVDiskID VDiskID = 4; - - optional uint64 Cookie = 5; - repeated uint32 OriginalParts = 6; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; - - optional fixed64 IncarnationGuid = 30; - - optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation -} - -message TDiffBlock { - optional uint64 Offset = 1; - optional bytes Buffer = 2; -} - -message TXorDiffReceiver { - optional NKikimrBlobStorage.TVDiskID VDiskID = 1; - optional uint32 PartId = 2; -} - -message TEvVPatchDiff { - optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 1; - optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 2; - optional NKikimrBlobStorage.TVDiskID VDiskID = 3; - repeated TDiffBlock Diffs = 4; - optional uint64 Cookie = 5; - - repeated TXorDiffReceiver XorReceivers= 6; - optional uint32 ExpectedXorDiffs = 7; - optional bool ForceEnd = 8; - optional bool NotifyIfNotReady = 9; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TEvVPatchXorDiff { - optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 1; - optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 2; - optional NKikimrBlobStorage.TVDiskID VDiskID = 3; - repeated TDiffBlock Diffs = 4; - optional uint64 Cookie = 5; - optional uint32 FromPartId = 6; - optional bool NotifyIfNotReady = 7; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TEvVPatchXorDiffResult { - optional NKikimrProto.EReplyStatus Status = 1; - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -}; - -message TEvVPatchResult { - optional NKikimrProto.EReplyStatus Status = 1; - optional string ErrorReason = 200; // textual description of error - optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 2; - optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 3; - optional NKikimrBlobStorage.TVDiskID VDiskID = 4; - - optional uint64 Cookie = 5; - optional uint32 StatusFlags = 6; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; - - optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown - optional fixed64 IncarnationGuid = 30; - - optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation -} - -message TEvVMovedPatch { - optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; - optional uint32 OriginalGroupId = 3; - optional uint32 PatchedGroupId = 4; - repeated TDiffBlock Diffs = 5; - - optional NKikimrBlobStorage.TVDiskID VDiskID = 6; - - optional bool IgnoreBlock = 7; - optional bool NotifyIfNotReady = 8; - optional uint64 Cookie = 9; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TEvVMovedPatchResult { - optional NKikimrProto.EReplyStatus Status = 1; - optional string ErrorReason = 200; // textual description of error - optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; - - optional NKikimrBlobStorage.TVDiskID VDiskID = 4; - optional uint64 Cookie = 5; - optional uint32 StatusFlags = 6; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; - - optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' - optional fixed64 IncarnationGuid = 30; - - optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation -} - -message TEvVInplacePatch { - optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; - optional uint32 OriginalGroupId = 3; - optional uint32 PatchedGroupId = 4; - repeated TDiffBlock Diffs = 5; - - optional NKikimrBlobStorage.TVDiskID VDiskID = 6; - - optional bool IgnoreBlock = 7; - optional bool NotifyIfNotReady = 8; - optional uint64 Cookie = 9; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TEvVInplacePatchResult { - optional NKikimrProto.EReplyStatus Status = 1; - optional string ErrorReason = 200; // textual description of error - optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; - optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; - - optional NKikimrBlobStorage.TVDiskID VDiskID = 4; - optional uint64 Cookie = 5; - optional uint32 StatusFlags = 6; - - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; - - optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' - optional fixed64 IncarnationGuid = 30; - - optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation -} - +message TEvVPatchStart { + optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; + optional NKikimrBlobStorage.TVDiskID VDiskID = 3; + + optional bool NotifyIfNotReady = 4; + optional uint64 Cookie = 5; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TEvVPatchFoundParts { + optional NKikimrProto.EReplyStatus Status = 1; + optional string ErrorReason = 200; // textual description of error + optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; + optional NKikimrBlobStorage.TVDiskID VDiskID = 4; + + optional uint64 Cookie = 5; + repeated uint32 OriginalParts = 6; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; + + optional fixed64 IncarnationGuid = 30; + + optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation +} + +message TDiffBlock { + optional uint64 Offset = 1; + optional bytes Buffer = 2; +} + +message TXorDiffReceiver { + optional NKikimrBlobStorage.TVDiskID VDiskID = 1; + optional uint32 PartId = 2; +} + +message TEvVPatchDiff { + optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 1; + optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 2; + optional NKikimrBlobStorage.TVDiskID VDiskID = 3; + repeated TDiffBlock Diffs = 4; + optional uint64 Cookie = 5; + + repeated TXorDiffReceiver XorReceivers= 6; + optional uint32 ExpectedXorDiffs = 7; + optional bool ForceEnd = 8; + optional bool NotifyIfNotReady = 9; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TEvVPatchXorDiff { + optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 1; + optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 2; + optional NKikimrBlobStorage.TVDiskID VDiskID = 3; + repeated TDiffBlock Diffs = 4; + optional uint64 Cookie = 5; + optional uint32 FromPartId = 6; + optional bool NotifyIfNotReady = 7; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TEvVPatchXorDiffResult { + optional NKikimrProto.EReplyStatus Status = 1; + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +}; + +message TEvVPatchResult { + optional NKikimrProto.EReplyStatus Status = 1; + optional string ErrorReason = 200; // textual description of error + optional NKikimrProto.TLogoBlobID OriginalPartBlobId = 2; + optional NKikimrProto.TLogoBlobID PatchedPartBlobId = 3; + optional NKikimrBlobStorage.TVDiskID VDiskID = 4; + + optional uint64 Cookie = 5; + optional uint32 StatusFlags = 6; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; + + optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown + optional fixed64 IncarnationGuid = 30; + + optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation +} + +message TEvVMovedPatch { + optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; + optional uint32 OriginalGroupId = 3; + optional uint32 PatchedGroupId = 4; + repeated TDiffBlock Diffs = 5; + + optional NKikimrBlobStorage.TVDiskID VDiskID = 6; + + optional bool IgnoreBlock = 7; + optional bool NotifyIfNotReady = 8; + optional uint64 Cookie = 9; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TEvVMovedPatchResult { + optional NKikimrProto.EReplyStatus Status = 1; + optional string ErrorReason = 200; // textual description of error + optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; + + optional NKikimrBlobStorage.TVDiskID VDiskID = 4; + optional uint64 Cookie = 5; + optional uint32 StatusFlags = 6; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; + + optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' + optional fixed64 IncarnationGuid = 30; + + optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation +} + +message TEvVInplacePatch { + optional NKikimrProto.TLogoBlobID OriginalBlobId = 1; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 2; + optional uint32 OriginalGroupId = 3; + optional uint32 PatchedGroupId = 4; + repeated TDiffBlock Diffs = 5; + + optional NKikimrBlobStorage.TVDiskID VDiskID = 6; + + optional bool IgnoreBlock = 7; + optional bool NotifyIfNotReady = 8; + optional uint64 Cookie = 9; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TEvVInplacePatchResult { + optional NKikimrProto.EReplyStatus Status = 1; + optional string ErrorReason = 200; // textual description of error + optional NKikimrProto.TLogoBlobID OriginalBlobId = 2; + optional NKikimrProto.TLogoBlobID PatchedBlobId = 3; + + optional NKikimrBlobStorage.TVDiskID VDiskID = 4; + optional uint64 Cookie = 5; + optional uint32 StatusFlags = 6; + + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; + + optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' + optional fixed64 IncarnationGuid = 30; + + optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation +} + message TEvVPut { optional NKikimrProto.TLogoBlobID BlobID = 1; optional bytes Buffer = 2; @@ -386,53 +386,53 @@ message TEvVPutResult { optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation } -message TVMultiPutItem { - optional NKikimrProto.TLogoBlobID BlobID = 1; - optional bytes Buffer = 2; - - optional uint64 FullDataSize = 3; - optional uint64 Cookie = 4; -} - -message TEvVMultiPut { - repeated TVMultiPutItem Items = 1; - - optional TVDiskID VDiskID = 2; - +message TVMultiPutItem { + optional NKikimrProto.TLogoBlobID BlobID = 1; + optional bytes Buffer = 2; + + optional uint64 FullDataSize = 3; + optional uint64 Cookie = 4; +} + +message TEvVMultiPut { + repeated TVMultiPutItem Items = 1; + + optional TVDiskID VDiskID = 2; + optional bool IgnoreBlock = 3 [default = false]; - optional bool NotifyIfNotReady = 4; - optional uint64 Cookie = 5; - optional EPutHandleClass HandleClass = 6; - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; -} - -message TVMultiPutResultItem { - optional NKikimrProto.EReplyStatus Status = 1; + optional bool NotifyIfNotReady = 4; + optional uint64 Cookie = 5; + optional EPutHandleClass HandleClass = 6; + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; +} + +message TVMultiPutResultItem { + optional NKikimrProto.EReplyStatus Status = 1; optional string ErrorReason = 200; - optional NKikimrProto.TLogoBlobID BlobID = 2; - - optional uint64 Cookie = 3; - optional uint32 StatusFlags = 4 [default = 0]; -} - -message TEvVMultiPutResult { - optional NKikimrProto.EReplyStatus Status = 1; + optional NKikimrProto.TLogoBlobID BlobID = 2; + + optional uint64 Cookie = 3; + optional uint32 StatusFlags = 4 [default = 0]; +} + +message TEvVMultiPutResult { + optional NKikimrProto.EReplyStatus Status = 1; optional string ErrorReason = 200; - repeated TVMultiPutResultItem Items = 2; - - optional TVDiskID VDiskID = 3; - optional uint64 Cookie = 4; - optional uint32 StatusFlags = 5 [default = 0]; - optional TMsgQoS MsgQoS = 10; - optional TTimestamps Timestamps = 23; - - optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' + repeated TVMultiPutResultItem Items = 2; + + optional TVDiskID VDiskID = 3; + optional uint64 Cookie = 4; + optional uint32 StatusFlags = 5 [default = 0]; + optional TMsgQoS MsgQoS = 10; + optional TTimestamps Timestamps = 23; + + optional float ApproximateFreeSpaceShare = 25 [default = 0]; // 0 is a special value for 'unknown' optional fixed64 IncarnationGuid = 30; optional TGroupInfo RecentGroup = 100; // filled in by VDisk on RACE when VDisk has newer generation -} - +} + message TRangeQuery { optional NKikimrProto.TLogoBlobID From = 1; optional NKikimrProto.TLogoBlobID To = 2; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d64169d4fc0..e6be416cdd7 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -639,7 +639,7 @@ message TFeatureFlags { optional bool AllowOnlineIndexBuild = 24 [default = true]; // deprecated: always true optional bool EnablePersistentQueryStats = 25 [default = false]; optional bool DisableDataShardBarrier = 26 [default = false]; - optional bool EnablePutBatchingForBlobStorage = 27 [default = true]; + optional bool EnablePutBatchingForBlobStorage = 27 [default = true]; optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true optional bool EnableKqpScanQueries = 29 [default = true]; // deprecated: always true optional bool EnablePersistentPartitionStats = 30 [default = false]; @@ -657,12 +657,12 @@ message TFeatureFlags { optional bool AllowStreamExecuteYqlScript = 42 [default = true]; optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true optional bool EnableOlapSchemaOperations = 44 [default = false]; - optional bool EnableVPatch = 45 [default = false]; + optional bool EnableVPatch = 45 [default = false]; optional bool EnableMvccSnapshotReads = 46 [default = false]; optional Tribool EnableMvcc = 47 [default = VALUE_TRUE]; optional bool EnableSchemeTransactionsAtSchemeShard = 48 [default = false]; optional bool EnableArrowFormatAtDatashard = 49 [default = false]; - optional bool Enable3x3RequestsForMirror3DCMinLatencyPut = 50 [default = false]; + optional bool Enable3x3RequestsForMirror3DCMinLatencyPut = 50 [default = false]; optional bool EnableBackgroundCompaction = 51 [default = true]; optional bool EnableArrowFormatInChannels = 52 [default = false]; optional bool EnableBackgroundCompactionServerless = 53 [default = false]; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index c17c8a7dc37..ea1e35d1920 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -46,9 +46,9 @@ enum EServiceKikimr { BS_PROXY_MULTICOLLECT = 289; BS_CONTROLLER_AUDIT = 304; BS_SELFHEAL = 342; - BS_PROXY_PATCH = 343; + BS_PROXY_PATCH = 343; BS_VDISK_SCRUB = 344; - BS_VDISK_PATCH = 345; + BS_VDISK_PATCH = 345; // DATASHARD section // TX_DATASHARD = 290; // diff --git a/ydb/core/test_tablet/load_actor_impl.h b/ydb/core/test_tablet/load_actor_impl.h index 71bd0e4f41e..3cfaf138cf6 100644 --- a/ydb/core/test_tablet/load_actor_impl.h +++ b/ydb/core/test_tablet/load_actor_impl.h @@ -14,8 +14,8 @@ namespace NKikimr::NTestShard { TActorId TabletActorId; const NKikimrClient::TTestShardControlRequest::TCmdInitialize Settings; - ui64 ValidationRunningCount = 0; - + ui64 ValidationRunningCount = 0; + struct TKeyInfo { const ui32 Len = 0; ::NTestShard::TStateServer::EEntityState ConfirmedState = ::NTestShard::TStateServer::ABSENT; diff --git a/ydb/core/test_tablet/load_actor_mon.cpp b/ydb/core/test_tablet/load_actor_mon.cpp index 24dfb0bc3c2..d8b9aa1887d 100644 --- a/ydb/core/test_tablet/load_actor_mon.cpp +++ b/ydb/core/test_tablet/load_actor_mon.cpp @@ -116,11 +116,11 @@ namespace NKikimr::NTestShard { TABLED() { str << num; } } - TABLER() { - TABLED() { str << "Count of validation runnings"; } - TABLED() { str << self->ValidationRunningCount; } - } - + TABLER() { + TABLED() { str << "Count of validation runnings"; } + TABLED() { str << self->ValidationRunningCount; } + } + const std::vector<double> ps{0.0, 0.5, 0.9, 0.99, 1.0}; auto output = [&](auto& r, const char *name) { diff --git a/ydb/core/test_tablet/load_actor_read_validate.cpp b/ydb/core/test_tablet/load_actor_read_validate.cpp index f6fc171f1ec..459897c586d 100644 --- a/ydb/core/test_tablet/load_actor_read_validate.cpp +++ b/ydb/core/test_tablet/load_actor_read_validate.cpp @@ -19,14 +19,14 @@ namespace NKikimr::NTestShard { std::unordered_map<ui64, TString> QueriesInFlight; ui64 LastCookie = 0; - bool IssueReadMode = false; // true - via EvRequest, false - via EvReadRequest - bool IssueReadRangeMode = false; // true - via EvRequest, false - via EvReadRequest - - ui32 WaitedReadsViaEvResponse = 0; - ui32 WaitedReadsViaEvReadResponse = 0; - ui32 WaitedReadRangesViaEvResponse = 0; - ui32 WaitedReadRangesViaEvReadRangeResponse = 0; - + bool IssueReadMode = false; // true - via EvRequest, false - via EvReadRequest + bool IssueReadRangeMode = false; // true - via EvRequest, false - via EvReadRequest + + ui32 WaitedReadsViaEvResponse = 0; + ui32 WaitedReadsViaEvReadResponse = 0; + ui32 WaitedReadRangesViaEvResponse = 0; + ui32 WaitedReadRangesViaEvReadRangeResponse = 0; + // read retry logic std::unordered_set<TString> KeyReadsWaitingForRetry; ui32 RetryCount = 0; @@ -77,7 +77,7 @@ namespace NKikimr::NTestShard { } } - void SendReadRangeViaEvRequest() { + void SendReadRangeViaEvRequest() { auto request = std::make_unique<TEvKeyValue::TEvRequest>(); auto& record = request->Record; record.SetTabletId(TabletId); @@ -91,40 +91,40 @@ namespace NKikimr::NTestShard { Send(TabletActorId, request.release()); } - void SendReadRangeViaEvReadRange() { - auto request = std::make_unique<TEvKeyValue::TEvReadRange>(); - auto& record = request->Record; - record.set_tablet_id(TabletId); - record.set_cookie(0); - auto *range = record.mutable_range(); - range->set_from_key_exclusive(*LastKey); - Send(TabletActorId, request.release()); - } - - void IssueNextReadRangeQuery() { - IssueReadRangeMode = !IssueReadRangeMode; - if (IssueReadRangeMode) { - WaitedReadRangesViaEvResponse++; - SendReadRangeViaEvRequest(); - } else { - WaitedReadRangesViaEvReadRangeResponse++; - SendReadRangeViaEvReadRange(); - } - } - - TString PopQueryByCookie(ui64 cookie) { - auto it = QueriesInFlight.find(cookie); - Y_VERIFY(it != QueriesInFlight.end()); - TString key = std::move(it->second); - QueriesInFlight.erase(it); - return key; - } - + void SendReadRangeViaEvReadRange() { + auto request = std::make_unique<TEvKeyValue::TEvReadRange>(); + auto& record = request->Record; + record.set_tablet_id(TabletId); + record.set_cookie(0); + auto *range = record.mutable_range(); + range->set_from_key_exclusive(*LastKey); + Send(TabletActorId, request.release()); + } + + void IssueNextReadRangeQuery() { + IssueReadRangeMode = !IssueReadRangeMode; + if (IssueReadRangeMode) { + WaitedReadRangesViaEvResponse++; + SendReadRangeViaEvRequest(); + } else { + WaitedReadRangesViaEvReadRangeResponse++; + SendReadRangeViaEvReadRange(); + } + } + + TString PopQueryByCookie(ui64 cookie) { + auto it = QueriesInFlight.find(cookie); + Y_VERIFY(it != QueriesInFlight.end()); + TString key = std::move(it->second); + QueriesInFlight.erase(it); + return key; + } + void Handle(TEvKeyValue::TEvResponse::TPtr ev) { auto& r = ev->Get()->Record; if (r.GetCookie()) { - WaitedReadsViaEvResponse--; - TString key = PopQueryByCookie(r.GetCookie()); + WaitedReadsViaEvResponse--; + TString key = PopQueryByCookie(r.GetCookie()); if (r.GetStatus() != NMsgBusProxy::MSTATUS_OK) { STLOG(PRI_ERROR, TEST_SHARD, TS18, "CmdRead failed", (TabletId, TabletId), (Status, r.GetStatus()), @@ -153,7 +153,7 @@ namespace NKikimr::NTestShard { " actual# " << MD5::Calc(value) << " len# " << value.size()); STLOG(PRI_DEBUG, TEST_SHARD, TS16, "read key", (TabletId, TabletId), (Key, key)); } else { - WaitedReadRangesViaEvResponse--; + WaitedReadRangesViaEvResponse--; if (r.GetStatus() != NMsgBusProxy::MSTATUS_OK) { STLOG(PRI_ERROR, TEST_SHARD, TS17, "CmdRangeRead failed", (TabletId, TabletId), (Status, r.GetStatus()), (ErrorReason, r.GetErrorReason())); @@ -180,73 +180,73 @@ namespace NKikimr::NTestShard { FinishIfPossible(); } - void Handle(TEvKeyValue::TEvReadResponse::TPtr ev) { - auto& record = ev->Get()->Record; - WaitedReadsViaEvReadResponse--; - - const TString key = PopQueryByCookie(record.cookie()); - const NKikimrKeyValue::Statuses::ReplyStatus status = record.status(); - - if (status == NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT) { - STLOG(PRI_ERROR, TEST_SHARD, TS23, "CmdRead failed", (TabletId, TabletId), (Status, status), - (ErrorReason, record.msg())); - return IssueRead(key); - } - - if (status == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR && RetryCount < 10) { - const bool inserted = KeyReadsWaitingForRetry.insert(key).second; - Y_VERIFY(inserted); - STLOG(PRI_ERROR, TEST_SHARD, TS24, "read key failed -- going to retry", (TabletId, TabletId), - (Key, key), (ErrorReason, record.msg())); - return; - } - - Y_VERIFY_S(status == NKikimrKeyValue::Statuses::RSTATUS_OK, - "Status# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(status) - << " Cookie# " << record.cookie() - << " Message# " << record.msg()); - - const TString& value = record.value(); - const bool inserted = Keys.try_emplace(key, value.size()).second; - Y_VERIFY(inserted); - Y_VERIFY_S(MD5::Calc(value) == key, "TabletId# " << TabletId << " Key# " << key << " digest mismatch" - " actual# " << MD5::Calc(value) << " len# " << value.size()); - STLOG(PRI_DEBUG, TEST_SHARD, TS16, "read key", (TabletId, TabletId), (Key, key)); - FinishIfPossible(); - } - - void Handle(TEvKeyValue::TEvReadRangeResponse::TPtr ev) { - WaitedReadRangesViaEvReadRangeResponse--; - auto& record = ev->Get()->Record; - const NKikimrKeyValue::Statuses::ReplyStatus status = record.status(); - - if (status != NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT) { - STLOG(PRI_ERROR, TEST_SHARD, TS19, "CmdRangeRead failed", (TabletId, TabletId), (Status, status), - (ErrorReason, record.msg())); - return IssueNextReadRangeQuery(); - } - - Y_VERIFY_S(status == NKikimrKeyValue::Statuses::RSTATUS_OK - || status == NKikimrKeyValue::Statuses::RSTATUS_NO_DATA - || status == NKikimrKeyValue::Statuses::RSTATUS_OVERRUN, - "TabletId# " << TabletId << " CmdReadRange failed" - << " Status# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(status)); - - for (const auto& pair : record.pair()) { - const TString& key = pair.key(); - LastKey = key; - IssueRead(key); - } - if (!record.pair_size()) { - STLOG(PRI_INFO, TEST_SHARD, TS20, "finished reading from KeyValue tablet", (TabletId, TabletId)); - KeyValueReadComplete = true; - } else { - IssueNextReadRangeQuery(); - } - FinishIfPossible(); - } - - ui64 SendReadViaEvRequest(const TString& key) { + void Handle(TEvKeyValue::TEvReadResponse::TPtr ev) { + auto& record = ev->Get()->Record; + WaitedReadsViaEvReadResponse--; + + const TString key = PopQueryByCookie(record.cookie()); + const NKikimrKeyValue::Statuses::ReplyStatus status = record.status(); + + if (status == NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT) { + STLOG(PRI_ERROR, TEST_SHARD, TS23, "CmdRead failed", (TabletId, TabletId), (Status, status), + (ErrorReason, record.msg())); + return IssueRead(key); + } + + if (status == NKikimrKeyValue::Statuses::RSTATUS_INTERNAL_ERROR && RetryCount < 10) { + const bool inserted = KeyReadsWaitingForRetry.insert(key).second; + Y_VERIFY(inserted); + STLOG(PRI_ERROR, TEST_SHARD, TS24, "read key failed -- going to retry", (TabletId, TabletId), + (Key, key), (ErrorReason, record.msg())); + return; + } + + Y_VERIFY_S(status == NKikimrKeyValue::Statuses::RSTATUS_OK, + "Status# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(status) + << " Cookie# " << record.cookie() + << " Message# " << record.msg()); + + const TString& value = record.value(); + const bool inserted = Keys.try_emplace(key, value.size()).second; + Y_VERIFY(inserted); + Y_VERIFY_S(MD5::Calc(value) == key, "TabletId# " << TabletId << " Key# " << key << " digest mismatch" + " actual# " << MD5::Calc(value) << " len# " << value.size()); + STLOG(PRI_DEBUG, TEST_SHARD, TS16, "read key", (TabletId, TabletId), (Key, key)); + FinishIfPossible(); + } + + void Handle(TEvKeyValue::TEvReadRangeResponse::TPtr ev) { + WaitedReadRangesViaEvReadRangeResponse--; + auto& record = ev->Get()->Record; + const NKikimrKeyValue::Statuses::ReplyStatus status = record.status(); + + if (status != NKikimrKeyValue::Statuses::RSTATUS_TIMEOUT) { + STLOG(PRI_ERROR, TEST_SHARD, TS19, "CmdRangeRead failed", (TabletId, TabletId), (Status, status), + (ErrorReason, record.msg())); + return IssueNextReadRangeQuery(); + } + + Y_VERIFY_S(status == NKikimrKeyValue::Statuses::RSTATUS_OK + || status == NKikimrKeyValue::Statuses::RSTATUS_NO_DATA + || status == NKikimrKeyValue::Statuses::RSTATUS_OVERRUN, + "TabletId# " << TabletId << " CmdReadRange failed" + << " Status# " << NKikimrKeyValue::Statuses_ReplyStatus_Name(status)); + + for (const auto& pair : record.pair()) { + const TString& key = pair.key(); + LastKey = key; + IssueRead(key); + } + if (!record.pair_size()) { + STLOG(PRI_INFO, TEST_SHARD, TS20, "finished reading from KeyValue tablet", (TabletId, TabletId)); + KeyValueReadComplete = true; + } else { + IssueNextReadRangeQuery(); + } + FinishIfPossible(); + } + + ui64 SendReadViaEvRequest(const TString& key) { auto request = std::make_unique<TEvKeyValue::TEvRequest>(); auto& record = request->Record; record.SetTabletId(TabletId); @@ -255,33 +255,33 @@ namespace NKikimr::NTestShard { auto *read = record.AddCmdRead(); read->SetKey(key); Send(TabletActorId, request.release()); - return cookie; - } - - ui64 SendReadViaEvReadRequest(const TString& key) { - auto request = std::make_unique<TEvKeyValue::TEvRead>(); - auto& record = request->Record; - record.set_tablet_id(TabletId); - const ui64 cookie = ++LastCookie; - record.set_cookie(cookie); - record.set_key(key); - Send(TabletActorId, request.release()); - return cookie; - } - - ui64 SendRead(const TString& key) { - IssueReadMode = !IssueReadMode; - if (IssueReadMode) { - WaitedReadsViaEvResponse++; - return SendReadViaEvRequest(key); - } else { - WaitedReadsViaEvReadResponse++; - return SendReadViaEvReadRequest(key); - } - } - - void IssueRead(const TString& key) { - const ui64 cookie = SendRead(key); + return cookie; + } + + ui64 SendReadViaEvReadRequest(const TString& key) { + auto request = std::make_unique<TEvKeyValue::TEvRead>(); + auto& record = request->Record; + record.set_tablet_id(TabletId); + const ui64 cookie = ++LastCookie; + record.set_cookie(cookie); + record.set_key(key); + Send(TabletActorId, request.release()); + return cookie; + } + + ui64 SendRead(const TString& key) { + IssueReadMode = !IssueReadMode; + if (IssueReadMode) { + WaitedReadsViaEvResponse++; + return SendReadViaEvRequest(key); + } else { + WaitedReadsViaEvReadResponse++; + return SendReadViaEvReadRequest(key); + } + } + + void IssueRead(const TString& key) { + const ui64 cookie = SendRead(key); const bool inserted = QueriesInFlight.emplace(cookie, key).second; Y_VERIFY(inserted); } @@ -530,30 +530,30 @@ namespace NKikimr::NTestShard { TABLED() { str << QueriesInFlight.size(); } } } - TABLEBODY() { - TABLER() { - TABLED() { str << "WaitedReadsViaEvResponse"; } - TABLED() { str << WaitedReadsViaEvResponse; } - } - } - TABLEBODY() { - TABLER() { - TABLED() { str << "WaitedReadsViaEvReadResponse"; } - TABLED() { str << WaitedReadsViaEvReadResponse; } - } - } - TABLEBODY() { - TABLER() { - TABLED() { str << "WaitedReadRangesViaEvResponse"; } - TABLED() { str << WaitedReadRangesViaEvResponse; } - } - } - TABLEBODY() { - TABLER() { - TABLED() { str << "WaitedReadRangesViaEvReadRangeResponse"; } - TABLED() { str << WaitedReadRangesViaEvReadRangeResponse; } - } - } + TABLEBODY() { + TABLER() { + TABLED() { str << "WaitedReadsViaEvResponse"; } + TABLED() { str << WaitedReadsViaEvResponse; } + } + } + TABLEBODY() { + TABLER() { + TABLED() { str << "WaitedReadsViaEvReadResponse"; } + TABLED() { str << WaitedReadsViaEvReadResponse; } + } + } + TABLEBODY() { + TABLER() { + TABLED() { str << "WaitedReadRangesViaEvResponse"; } + TABLED() { str << WaitedReadRangesViaEvResponse; } + } + } + TABLEBODY() { + TABLER() { + TABLED() { str << "WaitedReadRangesViaEvReadRangeResponse"; } + TABLED() { str << WaitedReadRangesViaEvReadRangeResponse; } + } + } } } } @@ -566,8 +566,8 @@ namespace NKikimr::NTestShard { } STRICT_STFUNC(StateFunc, - hFunc(TEvKeyValue::TEvReadResponse, Handle); - hFunc(TEvKeyValue::TEvReadRangeResponse, Handle); + hFunc(TEvKeyValue::TEvReadResponse, Handle); + hFunc(TEvKeyValue::TEvReadRangeResponse, Handle); hFunc(TEvKeyValue::TEvResponse, Handle); hFunc(TEvStateServerStatus, Handle); hFunc(TEvStateServerReadResult, Handle); @@ -582,7 +582,7 @@ namespace NKikimr::NTestShard { Send(TabletActorId, new TTestShard::TEvSwitchMode(TTestShard::EMode::READ_VALIDATE)); Y_VERIFY(!ValidationActorId); ValidationActorId = RegisterWithSameMailbox(new TValidationActor(*this, initialCheck)); - ValidationRunningCount++; + ValidationRunningCount++; } void TLoadActor::Handle(TEvValidationFinished::TPtr ev) { diff --git a/ydb/core/testlib/basics/helpers.cpp b/ydb/core/testlib/basics/helpers.cpp index 259e4f9da94..00b9a697cea 100644 --- a/ydb/core/testlib/basics/helpers.cpp +++ b/ydb/core/testlib/basics/helpers.cpp @@ -38,34 +38,34 @@ namespace NKikimr { return pipeConfig; } - struct TPDiskReplyChecker : IReplyChecker { - ~TPDiskReplyChecker() - { - } - - void OnRequest(IEventHandle *request) override { - if (request->Type == TEvBlobStorage::EvMultiLog) { - NPDisk::TEvMultiLog *evLogs = request->Get<NPDisk::TEvMultiLog>(); - LastLsn = evLogs->LsnSeg.Last; - } else { - LastLsn = {}; - } - } - - bool IsWaitingForMoreResponses(IEventHandle *response) override { - if (!LastLsn) { - return false; - } - Y_VERIFY_S(response->Type == TEvBlobStorage::EvLogResult, "expected EvLogResult " - << (ui64)TEvBlobStorage::EvLogResult << ", but given " << response->Type); - NPDisk::TEvLogResult *evResult = response->Get<NPDisk::TEvLogResult>(); - ui64 responseLastLsn = evResult->Results.back().Lsn; - return *LastLsn > responseLastLsn; - } - - TMaybe<ui64> LastLsn; - }; - + struct TPDiskReplyChecker : IReplyChecker { + ~TPDiskReplyChecker() + { + } + + void OnRequest(IEventHandle *request) override { + if (request->Type == TEvBlobStorage::EvMultiLog) { + NPDisk::TEvMultiLog *evLogs = request->Get<NPDisk::TEvMultiLog>(); + LastLsn = evLogs->LsnSeg.Last; + } else { + LastLsn = {}; + } + } + + bool IsWaitingForMoreResponses(IEventHandle *response) override { + if (!LastLsn) { + return false; + } + Y_VERIFY_S(response->Type == TEvBlobStorage::EvLogResult, "expected EvLogResult " + << (ui64)TEvBlobStorage::EvLogResult << ", but given " << response->Type); + NPDisk::TEvLogResult *evResult = response->Get<NPDisk::TEvLogResult>(); + ui64 responseLastLsn = evResult->Results.back().Lsn; + return *LastLsn > responseLastLsn; + } + + TMaybe<ui64> LastLsn; + }; + void TStrandedPDiskServiceFactory::Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) { diff --git a/ydb/core/util/log_priority_mute_checker.h b/ydb/core/util/log_priority_mute_checker.h index 6baac545c81..f10d71acf44 100644 --- a/ydb/core/util/log_priority_mute_checker.h +++ b/ydb/core/util/log_priority_mute_checker.h @@ -1,92 +1,92 @@ -#pragma once - -#include "defs.h" - -#include <library/cpp/actors/core/log_settings.h> -#include <util/system/atomic.h> - -namespace NKikimr { - - template <NActors::NLog::EPriority MainPrio, NActors::NLog::EPriority MutePrio> - struct TLogPriorityMuteChecker { - using EPriority = NActors::NLog::EPriority; - static constexpr EPriority MainPriority = MainPrio; - static constexpr EPriority MutePriority = MutePrio; - - TInstant MuteDeadline; - - void MuteUntil(const TInstant &muteDeadline) { - MuteDeadline = muteDeadline; - } - - void Unmute() { - MuteDeadline = TInstant::Zero(); - } - - bool IsMuted(const TInstant &now) const { - return now < MuteDeadline; - } - - EPriority CheckPriority(const TInstant &now) const { - return IsMuted(now) ? MutePriority : MainPriority; - } - - EPriority Register(const TInstant &now, const TInstant &muteDeadline) { - if (IsMuted(now)) { - return MutePriority; - } - MuteUntil(muteDeadline); - return MainPriority; - } - - EPriority Register(const TInstant &now, const TDuration &muteDuration) { - if (IsMuted(now)) { - return MutePriority; - } - MuteUntil(now + muteDuration); - return MainPriority; - } - }; - - template <NActors::NLog::EPriority MainPrio, NActors::NLog::EPriority MutePrio> - struct TAtomicLogPriorityMuteChecker { - using EPriority = NActors::NLog::EPriority; - static constexpr EPriority MainPriority = MainPrio; - static constexpr EPriority MutePriority = MutePrio; - - TAtomic MuteDeadlineUs = 0; - - void MuteUntil(const TInstant &muteDeadline) { - AtomicSet(MuteDeadlineUs, muteDeadline.MicroSeconds()); - } - - void Unmute() { - AtomicSet(MuteDeadlineUs, 0); - } - - bool IsMuted(const TInstant &now) const { - return now < TInstant::MicroSeconds(AtomicGet(MuteDeadlineUs)); - } - - EPriority CheckPriority(const TInstant &now) const { - return IsMuted(now) ? MutePriority : MainPriority; - } - - EPriority Register(const TInstant &now, const TInstant &muteDeadline) { - if (IsMuted(now)) { - return MutePriority; - } - MuteUntil(muteDeadline); - return MainPriority; - } - - EPriority Register(const TInstant &now, const TDuration &muteDuration) { - if (IsMuted(now)) { - return MutePriority; - } - MuteUntil(now + muteDuration); - return MainPriority; - } - }; - -} +#pragma once + +#include "defs.h" + +#include <library/cpp/actors/core/log_settings.h> +#include <util/system/atomic.h> + +namespace NKikimr { + + template <NActors::NLog::EPriority MainPrio, NActors::NLog::EPriority MutePrio> + struct TLogPriorityMuteChecker { + using EPriority = NActors::NLog::EPriority; + static constexpr EPriority MainPriority = MainPrio; + static constexpr EPriority MutePriority = MutePrio; + + TInstant MuteDeadline; + + void MuteUntil(const TInstant &muteDeadline) { + MuteDeadline = muteDeadline; + } + + void Unmute() { + MuteDeadline = TInstant::Zero(); + } + + bool IsMuted(const TInstant &now) const { + return now < MuteDeadline; + } + + EPriority CheckPriority(const TInstant &now) const { + return IsMuted(now) ? MutePriority : MainPriority; + } + + EPriority Register(const TInstant &now, const TInstant &muteDeadline) { + if (IsMuted(now)) { + return MutePriority; + } + MuteUntil(muteDeadline); + return MainPriority; + } + + EPriority Register(const TInstant &now, const TDuration &muteDuration) { + if (IsMuted(now)) { + return MutePriority; + } + MuteUntil(now + muteDuration); + return MainPriority; + } + }; + + template <NActors::NLog::EPriority MainPrio, NActors::NLog::EPriority MutePrio> + struct TAtomicLogPriorityMuteChecker { + using EPriority = NActors::NLog::EPriority; + static constexpr EPriority MainPriority = MainPrio; + static constexpr EPriority MutePriority = MutePrio; + + TAtomic MuteDeadlineUs = 0; + + void MuteUntil(const TInstant &muteDeadline) { + AtomicSet(MuteDeadlineUs, muteDeadline.MicroSeconds()); + } + + void Unmute() { + AtomicSet(MuteDeadlineUs, 0); + } + + bool IsMuted(const TInstant &now) const { + return now < TInstant::MicroSeconds(AtomicGet(MuteDeadlineUs)); + } + + EPriority CheckPriority(const TInstant &now) const { + return IsMuted(now) ? MutePriority : MainPriority; + } + + EPriority Register(const TInstant &now, const TInstant &muteDeadline) { + if (IsMuted(now)) { + return MutePriority; + } + MuteUntil(muteDeadline); + return MainPriority; + } + + EPriority Register(const TInstant &now, const TDuration &muteDuration) { + if (IsMuted(now)) { + return MutePriority; + } + MuteUntil(now + muteDuration); + return MainPriority; + } + }; + +} diff --git a/ydb/core/util/log_priority_mute_checker_ut.cpp b/ydb/core/util/log_priority_mute_checker_ut.cpp index c09bb14d300..e8df7dafc58 100644 --- a/ydb/core/util/log_priority_mute_checker_ut.cpp +++ b/ydb/core/util/log_priority_mute_checker_ut.cpp @@ -1,110 +1,110 @@ -#include <library/cpp/testing/unittest/registar.h> - -#include "log_priority_mute_checker.h" - -using namespace NKikimr; -using namespace NActors::NLog; - -TInstant GetTime(ui64 seconds) { - return TInstant::Zero() + TDuration::Seconds(seconds); -} - -void CheckPriority(EPriority expected, TMaybe<EPriority> prio) { - UNIT_ASSERT(prio); - UNIT_ASSERT_EQUAL(*prio, expected); -} - -void CheckPriority(EPriority expected, EPriority prio) { - UNIT_ASSERT_EQUAL(prio, expected); -} - -Y_UNIT_TEST_SUITE(TLogPriorityMuteTests) { - - template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> - void MakeMuteUntilTest(){ - TMuteChecker<PRI_ERROR, PRI_DEBUG> checker; - TInstant muteDeadline = GetTime(3); - checker.MuteUntil(muteDeadline); - for (ui64 i = 0; i < 3; ++i) { - CheckPriority(PRI_DEBUG, checker.CheckPriority(GetTime(i))); - } - for (ui64 i = 3; i < 6; ++i) { - CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(i))); - } - } - Y_UNIT_TEST(MuteUntilTest) { - MakeMuteUntilTest<TLogPriorityMuteChecker>(); - } - Y_UNIT_TEST(AtomicMuteUntilTest) { - MakeMuteUntilTest<TAtomicLogPriorityMuteChecker>(); - } - - template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> - void MakeUnmuteTest() { - TMuteChecker<PRI_ERROR, PRI_DEBUG> checker; - TInstant muteDeadline = GetTime(3); - checker.MuteUntil(muteDeadline); - for (ui64 i = 0; i < 3; ++i) { - CheckPriority(PRI_DEBUG, checker.CheckPriority(GetTime(i))); - } - checker.Unmute(); - for (ui64 i = 0; i < 6; ++i) { - CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(i))); - } - } - Y_UNIT_TEST(UnmuteTest) { - MakeUnmuteTest<TLogPriorityMuteChecker>(); - } - Y_UNIT_TEST(AtomicUnmuteTest) { - MakeUnmuteTest<TAtomicLogPriorityMuteChecker>(); - } - - template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> - void MakeCheckPriorityWithSetMuteTest() { - TLogPriorityMuteChecker<PRI_ERROR, PRI_DEBUG> checker; - ui64 secondsForDeadlines[3] = {3, 6, 12}; - - TInstant fakeDeadline = TInstant::Max(); - ui64 currentTime = 0; - for (ui64 deadlineIdx = 0; deadlineIdx < 3; ++deadlineIdx) { - TInstant realDeadline = GetTime(secondsForDeadlines[deadlineIdx]); - CheckPriority(PRI_ERROR, checker.Register(GetTime(currentTime++), realDeadline)); - while (currentTime < secondsForDeadlines[deadlineIdx]) { - CheckPriority(PRI_DEBUG, checker.Register(GetTime(currentTime++), fakeDeadline)); - } - } - CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(currentTime))); - } - Y_UNIT_TEST(CheckPriorityWithSetMuteTest) { - MakeCheckPriorityWithSetMuteTest<TLogPriorityMuteChecker>(); - } - Y_UNIT_TEST(AtomicCheckPriorityWithSetMuteTest) { - MakeCheckPriorityWithSetMuteTest<TAtomicLogPriorityMuteChecker>(); - } - - template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> - void MakeCheckPriorityWithSetMuteDurationTest() { - TLogPriorityMuteChecker<PRI_ERROR, PRI_DEBUG> checker; - ui64 secondsForDurations[3] = {3, 6, 12}; - - TDuration fakeDurations = TDuration::Seconds(600); - ui64 currentTime = 0; - ui64 deadline = 0; - for (ui64 durationIdx = 0; durationIdx < 3; ++durationIdx) { - TDuration realDuration = TDuration::Seconds(secondsForDurations[durationIdx]); - deadline += secondsForDurations[durationIdx]; - CheckPriority(PRI_ERROR, checker.Register(GetTime(currentTime++), realDuration)); - while (currentTime < deadline) { - CheckPriority(PRI_DEBUG, checker.Register(GetTime(currentTime++), fakeDurations)); - } - } - CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(currentTime))); - } - Y_UNIT_TEST(CheckPriorityWithSetMuteDurationTest) { - MakeCheckPriorityWithSetMuteDurationTest<TLogPriorityMuteChecker>(); - } - Y_UNIT_TEST(AtomicCheckPriorityWithSetMuteDurationTest) { - MakeCheckPriorityWithSetMuteDurationTest<TAtomicLogPriorityMuteChecker>(); - } - -} +#include <library/cpp/testing/unittest/registar.h> + +#include "log_priority_mute_checker.h" + +using namespace NKikimr; +using namespace NActors::NLog; + +TInstant GetTime(ui64 seconds) { + return TInstant::Zero() + TDuration::Seconds(seconds); +} + +void CheckPriority(EPriority expected, TMaybe<EPriority> prio) { + UNIT_ASSERT(prio); + UNIT_ASSERT_EQUAL(*prio, expected); +} + +void CheckPriority(EPriority expected, EPriority prio) { + UNIT_ASSERT_EQUAL(prio, expected); +} + +Y_UNIT_TEST_SUITE(TLogPriorityMuteTests) { + + template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> + void MakeMuteUntilTest(){ + TMuteChecker<PRI_ERROR, PRI_DEBUG> checker; + TInstant muteDeadline = GetTime(3); + checker.MuteUntil(muteDeadline); + for (ui64 i = 0; i < 3; ++i) { + CheckPriority(PRI_DEBUG, checker.CheckPriority(GetTime(i))); + } + for (ui64 i = 3; i < 6; ++i) { + CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(i))); + } + } + Y_UNIT_TEST(MuteUntilTest) { + MakeMuteUntilTest<TLogPriorityMuteChecker>(); + } + Y_UNIT_TEST(AtomicMuteUntilTest) { + MakeMuteUntilTest<TAtomicLogPriorityMuteChecker>(); + } + + template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> + void MakeUnmuteTest() { + TMuteChecker<PRI_ERROR, PRI_DEBUG> checker; + TInstant muteDeadline = GetTime(3); + checker.MuteUntil(muteDeadline); + for (ui64 i = 0; i < 3; ++i) { + CheckPriority(PRI_DEBUG, checker.CheckPriority(GetTime(i))); + } + checker.Unmute(); + for (ui64 i = 0; i < 6; ++i) { + CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(i))); + } + } + Y_UNIT_TEST(UnmuteTest) { + MakeUnmuteTest<TLogPriorityMuteChecker>(); + } + Y_UNIT_TEST(AtomicUnmuteTest) { + MakeUnmuteTest<TAtomicLogPriorityMuteChecker>(); + } + + template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> + void MakeCheckPriorityWithSetMuteTest() { + TLogPriorityMuteChecker<PRI_ERROR, PRI_DEBUG> checker; + ui64 secondsForDeadlines[3] = {3, 6, 12}; + + TInstant fakeDeadline = TInstant::Max(); + ui64 currentTime = 0; + for (ui64 deadlineIdx = 0; deadlineIdx < 3; ++deadlineIdx) { + TInstant realDeadline = GetTime(secondsForDeadlines[deadlineIdx]); + CheckPriority(PRI_ERROR, checker.Register(GetTime(currentTime++), realDeadline)); + while (currentTime < secondsForDeadlines[deadlineIdx]) { + CheckPriority(PRI_DEBUG, checker.Register(GetTime(currentTime++), fakeDeadline)); + } + } + CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(currentTime))); + } + Y_UNIT_TEST(CheckPriorityWithSetMuteTest) { + MakeCheckPriorityWithSetMuteTest<TLogPriorityMuteChecker>(); + } + Y_UNIT_TEST(AtomicCheckPriorityWithSetMuteTest) { + MakeCheckPriorityWithSetMuteTest<TAtomicLogPriorityMuteChecker>(); + } + + template <template <NActors::NLog::EPriority, NActors::NLog::EPriority> typename TMuteChecker> + void MakeCheckPriorityWithSetMuteDurationTest() { + TLogPriorityMuteChecker<PRI_ERROR, PRI_DEBUG> checker; + ui64 secondsForDurations[3] = {3, 6, 12}; + + TDuration fakeDurations = TDuration::Seconds(600); + ui64 currentTime = 0; + ui64 deadline = 0; + for (ui64 durationIdx = 0; durationIdx < 3; ++durationIdx) { + TDuration realDuration = TDuration::Seconds(secondsForDurations[durationIdx]); + deadline += secondsForDurations[durationIdx]; + CheckPriority(PRI_ERROR, checker.Register(GetTime(currentTime++), realDuration)); + while (currentTime < deadline) { + CheckPriority(PRI_DEBUG, checker.Register(GetTime(currentTime++), fakeDurations)); + } + } + CheckPriority(PRI_ERROR, checker.CheckPriority(GetTime(currentTime))); + } + Y_UNIT_TEST(CheckPriorityWithSetMuteDurationTest) { + MakeCheckPriorityWithSetMuteDurationTest<TLogPriorityMuteChecker>(); + } + Y_UNIT_TEST(AtomicCheckPriorityWithSetMuteDurationTest) { + MakeCheckPriorityWithSetMuteDurationTest<TAtomicLogPriorityMuteChecker>(); + } + +} diff --git a/ydb/core/util/testactorsys.cpp b/ydb/core/util/testactorsys.cpp index 73d27bc9d92..34c7ead1219 100644 --- a/ydb/core/util/testactorsys.cpp +++ b/ydb/core/util/testactorsys.cpp @@ -40,14 +40,14 @@ public: } bool Send(TAutoPtr<IEventHandle>& ev) override { - if (TlsActivationContext) { - const TActorContext& ctx = TActivationContext::AsActorContext(); - IActor* sender = Context->GetActor(ctx.SelfID); - TTestDecorator* decorator = dynamic_cast<TTestDecorator*>(sender); - if (decorator && !decorator->BeforeSending(ev)) { - ev = nullptr; - } - } + if (TlsActivationContext) { + const TActorContext& ctx = TActivationContext::AsActorContext(); + IActor* sender = Context->GetActor(ctx.SelfID); + TTestDecorator* decorator = dynamic_cast<TTestDecorator*>(sender); + if (decorator && !decorator->BeforeSending(ev)) { + ev = nullptr; + } + } return Context->Send(ev, NodeId); } @@ -134,10 +134,10 @@ TActorId TTestActorSystem::CreateTestBootstrapper(TTabletStorageInfo *info, std: void TTestActorSystem::SetupTabletRuntime(bool isMirror3dc, ui32 stateStorageNodeId, ui32 targetNodeId) { auto setup = MakeIntrusive<TTableNameserverSetup>(); - ui32 nodeCountInDC = (MaxNodeId + 2) / 3; + ui32 nodeCountInDC = (MaxNodeId + 2) / 3; for (ui32 nodeId : GetNodes()) { const TString name = Sprintf("127.0.0.%u", nodeId); - ui32 dcNum = isMirror3dc ? ((nodeId + nodeCountInDC - 1) / nodeCountInDC) : 1; + ui32 dcNum = isMirror3dc ? ((nodeId + nodeCountInDC - 1) / nodeCountInDC) : 1; NActorsInterconnect::TNodeLocation location; location.SetDataCenter(ToString(dcNum)); location.SetRack(ToString(nodeId)); diff --git a/ydb/core/util/ut/ya.make b/ydb/core/util/ut/ya.make index 315713cad86..03c8d392503 100644 --- a/ydb/core/util/ut/ya.make +++ b/ydb/core/util/ut/ya.make @@ -36,7 +36,7 @@ SRCS( intrusive_heap_ut.cpp intrusive_stack_ut.cpp lf_stack_ut.cpp - log_priority_mute_checker_ut.cpp + log_priority_mute_checker_ut.cpp lz4_data_generator_ut.cpp operation_queue_ut.cpp operation_queue_priority_ut.cpp diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 4f325d8fcbd..e4444ec3ccb 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -32,7 +32,7 @@ SRCS( intrusive_heap.cpp intrusive_heap.h intrusive_stack.h - log_priority_mute_checker.h + log_priority_mute_checker.h memory_tracker.cpp memory_tracker.h operation_queue.h |