From dd44e94fe3f82dcfc500e854fb85cd93cdfef613 Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Thu, 13 Jun 2024 17:40:34 +0300 Subject: Reset replication counters correctly to avoid data race on VDisk restart (#5486) --- ydb/core/blobstorage/nodewarden/node_warden_impl.h | 7 ++++++ .../blobstorage/nodewarden/node_warden_vdisk.cpp | 29 ++++++++++++++++++++-- ydb/core/blobstorage/ut_group/main.cpp | 1 + .../vdisk/skeleton/blobstorage_skeletonfront.cpp | 2 ++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 844f3cc0b5d..18f904006b5 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -335,6 +335,9 @@ namespace NKikimr::NStorage { bool ReadOnly; }; std::optional RuntimeData; + bool ShutdownPending = false; + bool RestartAfterShutdown = false; + TDuration YardInitDelay; // Last VDiskId reported to Node Whiteboard. std::optional WhiteboardVDiskId; @@ -386,6 +389,7 @@ namespace NKikimr::NStorage { }; std::map LocalVDisks; + THashMap VDiskIdByActor; std::map SlayInFlight; std::set PDiskRestartInFlight; TIntrusiveList VDisksWithUnreportedMetrics; @@ -393,6 +397,7 @@ namespace NKikimr::NStorage { void DestroyLocalVDisk(TVDiskRecord& vdisk); void PoisonLocalVDisk(TVDiskRecord& vdisk); void StartLocalVDiskActor(TVDiskRecord& vdisk, TDuration yardInitDelay); + void HandleGone(STATEFN_SIG); void ApplyServiceSetVDisks(const NKikimrBlobStorage::TNodeWardenServiceSet& serviceSet); // process VDisk configuration @@ -646,6 +651,8 @@ namespace NKikimr::NStorage { hFunc(TEvNodeWardenQueryBaseConfig, Handle); hFunc(TEvNodeConfigInvokeOnRootResult, Handle); + fFunc(TEvents::TSystem::Gone, HandleGone); + default: EnqueuePendingMessage(ev); break; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp index dbd0da73a8e..13552c050d1 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp @@ -48,6 +48,7 @@ namespace NKikimr::NStorage { vdisk.ScrubCookie = 0; // disable reception of Scrub messages from this disk vdisk.ScrubCookieForController = 0; // and from controller too vdisk.Status = NKikimrBlobStorage::EVDiskStatus::ERROR; + vdisk.ShutdownPending = true; SendDiskMetrics(false); } @@ -70,6 +71,12 @@ namespace NKikimr::NStorage { return; } + if (vdisk.ShutdownPending) { + vdisk.RestartAfterShutdown = true; + vdisk.YardInitDelay = Max(vdisk.YardInitDelay, yardInitDelay); + return; + } + // find underlying PDisk and determine its media type auto pdiskIt = LocalPDisks.find({vslotId.NodeId, vslotId.PDiskId}); Y_VERIFY_S(pdiskIt != LocalPDisks.end(), "PDiskId# " << vslotId.NodeId << ":" << vslotId.PDiskId << " not found"); @@ -206,8 +213,10 @@ namespace NKikimr::NStorage { // create an actor auto *as = TActivationContext::ActorSystem(); - as->RegisterLocalService(vdiskServiceId, as->Register(CreateVDisk(vdiskConfig, groupInfo, AppData()->Counters), - TMailboxType::Revolving, AppData()->SystemPoolId)); + TActorId actorId = as->Register(CreateVDisk(vdiskConfig, groupInfo, AppData()->Counters), + TMailboxType::Revolving, AppData()->SystemPoolId); + as->RegisterLocalService(vdiskServiceId, actorId); + VDiskIdByActor.try_emplace(actorId, vslotId); STLOG(PRI_DEBUG, BS_NODE, NW24, "StartLocalVDiskActor done", (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId), (PDiskGuid, pdiskGuid)); @@ -233,6 +242,22 @@ namespace NKikimr::NStorage { vdisk.ScrubCookie = scrubCookie; } + void TNodeWarden::HandleGone(STATEFN_SIG) { + if (const auto it = VDiskIdByActor.find(ev->Sender); it != VDiskIdByActor.end()) { + if (const auto jt = LocalVDisks.find(it->second); jt != LocalVDisks.end()) { + TVDiskRecord& vdisk = jt->second; + Y_ABORT_UNLESS(vdisk.ShutdownPending); + vdisk.ShutdownPending = false; + if (vdisk.RestartAfterShutdown) { + StartLocalVDiskActor(vdisk, vdisk.YardInitDelay); + vdisk.RestartAfterShutdown = false; + vdisk.YardInitDelay = TDuration::Zero(); + } + } + VDiskIdByActor.erase(it); + } + } + void TNodeWarden::ApplyServiceSetVDisks(const NKikimrBlobStorage::TNodeWardenServiceSet& serviceSet) { for (const auto& vdisk : serviceSet.GetVDisks()) { ApplyLocalVDiskInfo(vdisk); diff --git a/ydb/core/blobstorage/ut_group/main.cpp b/ydb/core/blobstorage/ut_group/main.cpp index d1b2010a8da..4f246c3fbfd 100644 --- a/ydb/core/blobstorage/ut_group/main.cpp +++ b/ydb/core/blobstorage/ut_group/main.cpp @@ -123,6 +123,7 @@ public: cFunc(TEvBlobStorage::EvDropDonor, Ignore); cFunc(TEvBlobStorage::EvGroupStatReport, Ignore); cFunc(TEvBlobStorage::EvNotifyVDiskGenerationChange, Ignore); + cFunc(TEvents::TSystem::Gone, Ignore); fFunc(TEvBlobStorage::EvPut, ForwardToProxy); fFunc(TEvBlobStorage::EvGet, ForwardToProxy); diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp index 2ad10639072..ee1aae20719 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp @@ -2209,6 +2209,8 @@ namespace NKikimr { void PassAway() override { VDiskCountersBase->RemoveSubgroupChain(CountersChain); + TActivationContext::Send(new IEventHandle(TEvents::TSystem::Gone, 0, + MakeBlobStorageNodeWardenID(SelfId().NodeId()), SelfId(), nullptr, 0)); TActorBootstrapped::PassAway(); } }; -- cgit v1.3