diff options
author | Alexey Efimov <xeno@prnwatch.com> | 2022-04-13 19:11:28 +0300 |
---|---|---|
committer | Alexey Efimov <xeno@prnwatch.com> | 2022-04-13 19:11:28 +0300 |
commit | 69ca387575eef928c4e3d4d164a12ae8c13c05cc (patch) | |
tree | 69c8c619cbedf3cc4890727df12c58816a9e8462 | |
parent | e794c6d1d117804bd581a0b9d038d24597af7666 (diff) | |
download | ydb-69ca387575eef928c4e3d4d164a12ae8c13c05cc.tar.gz |
refactoring of Hive's transactions KIKIMR-13552
ref:e6f7b80db641f4b405f780871ad5b3bef1db12fd
36 files changed, 970 insertions, 816 deletions
diff --git a/ydb/core/base/hive.h b/ydb/core/base/hive.h index fe62ef3954..7d4ae4cb51 100644 --- a/ydb/core/base/hive.h +++ b/ydb/core/base/hive.h @@ -437,6 +437,16 @@ namespace NKikimr { Record.AddShardLocalIdx(idx); } } + + TEvDeleteTabletReply(NKikimrProto::EReplyStatus status, ui64 hiveID, const NKikimrHive::TEvDeleteTablet& request) { + Record.SetStatus(status); + Record.SetOrigin(hiveID); + Record.SetTxId_Deprecated(request.GetTxId_Deprecated()); + Record.SetShardOwnerId(request.GetShardOwnerId()); + for (auto idx : request.GetShardLocalIdx()) { + Record.AddShardLocalIdx(idx); + } + } }; struct TEvDeleteOwnerTablets : public TEventPB<TEvDeleteOwnerTablets, @@ -543,6 +553,20 @@ namespace NKikimr { }; struct TEvRequestHiveInfo : TEventPB<TEvRequestHiveInfo, NKikimrHive::TEvRequestHiveInfo, EvRequestHiveInfo> { + struct TRequestHiveInfoInitializer { + ui64 TabletId = 0; + bool ReturnFollowers = false; + bool ReturnMetrics = false; + bool ReturnChannelHistory = false; + }; + + TEvRequestHiveInfo(TRequestHiveInfoInitializer initializer) { + Record.SetTabletID(initializer.TabletId); + Record.SetReturnFollowers(initializer.ReturnFollowers); + Record.SetReturnMetrics(initializer.ReturnMetrics); + Record.SetReturnChannelHistory(initializer.ReturnChannelHistory); + } + TEvRequestHiveInfo() = default; TEvRequestHiveInfo(bool returnFollowers) { diff --git a/ydb/core/mind/hive/hive.h b/ydb/core/mind/hive/hive.h index c51cb1df93..9ff1446624 100644 --- a/ydb/core/mind/hive/hive.h +++ b/ydb/core/mind/hive/hive.h @@ -78,6 +78,83 @@ struct ISubActor { virtual void Cleanup() = 0; }; + +struct TCompleteNotifications { + TVector<std::pair<THolder<IEventHandle>, TDuration>> Notifications; + TActorId SelfID; + + void Reset(const TActorId& selfId) { + Notifications.clear(); + SelfID = selfId; + } + + void Send(const TActorId& recipient, IEventBase* ev, ui32 flags = 0, ui64 cookie = 0) { + Y_VERIFY(!!SelfID); + Notifications.emplace_back(new IEventHandle(recipient, SelfID, ev, flags, cookie), TDuration()); + } + + void Schedule(TDuration duration, IEventBase* ev, ui32 flags = 0, ui64 cookie = 0) { + Y_VERIFY(!!SelfID); + Notifications.emplace_back(new IEventHandle(SelfID, {}, ev, flags, cookie), duration); + } + + size_t size() const { + return Notifications.size(); + } + + void Send(const TActorContext& ctx) { + for (auto& [notification, duration] : Notifications) { + if (duration) { + ctx.ExecutorThread.Schedule(duration, notification.Release()); + } else { + ctx.ExecutorThread.Send(notification.Release()); + } + } + Notifications.clear(); + } +}; + +struct TCompleteActions { + std::vector<std::unique_ptr<IActor>> Actors; + std::vector<std::function<void()>> Callbacks; + + void Reset() { + Actors.clear(); + Callbacks.clear(); + } + + void Register(IActor* actor) { + Actors.emplace_back(actor); + } + + void Callback(std::function<void()> callback) { + Callbacks.emplace_back(std::move(callback)); + } + + void Run(const TActorContext& ctx) { + for (auto& callback : Callbacks) { + callback(); + } + Callbacks.clear(); + for (auto& actor : Actors) { + ctx.Register(actor.release()); + } + Actors.clear(); + } +}; + +struct TSideEffects : TCompleteNotifications, TCompleteActions { + void Reset(const TActorId& selfId) { + TCompleteActions::Reset(); + TCompleteNotifications::Reset(selfId); + } + + void Complete(const TActorContext& ctx) { + TCompleteActions::Run(ctx); + TCompleteNotifications::Send(ctx); + } +}; + TResourceNormalizedValues NormalizeRawValues(const TResourceRawValues& values, const TResourceRawValues& maximum); NMetrics::EResource GetDominantResourceType(const TResourceRawValues& values, const TResourceRawValues& maximum); @@ -146,3 +223,46 @@ struct TDrainSettings { } // NHive } // NKikimr + +template <> +inline void Out<NKikimr::NHive::TCompleteNotifications>(IOutputStream& o, const NKikimr::NHive::TCompleteNotifications& n) { + if (!n.Notifications.empty()) { + o << "Notifications: "; + for (auto it = n.Notifications.begin(); it != n.Notifications.end(); ++it) { + if (it != n.Notifications.begin()) { + o << ','; + } + o << Hex(it->first->Type) << " " << it->first.Get()->Recipient; + } + } +} + +template <> +inline void Out<NKikimr::NHive::TCompleteActions>(IOutputStream& o, const NKikimr::NHive::TCompleteActions& n) { + if (!n.Callbacks.empty()) { + o << "Callbacks: " << n.Callbacks.size(); + } + if (!n.Actors.empty()) { + if (!n.Callbacks.empty()) { + o << ' '; + } + o << "Actions: "; + for (auto it = n.Actors.begin(); it != n.Actors.end(); ++it) { + if (it != n.Actors.begin()) { + o << '.'; + } + o << TypeName(*(it->get())); + } + } +} + +template <> +inline void Out<NKikimr::NHive::TSideEffects>(IOutputStream& o, const NKikimr::NHive::TSideEffects& e) { + o << '{'; + o << static_cast<const NKikimr::NHive::TCompleteNotifications&>(e); + if (!e.Notifications.empty() && !e.Actors.empty()) { + o << ' '; + } + o << static_cast<const NKikimr::NHive::TCompleteActions&>(e); + o << '}'; +} diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index e7fd0e1d7c..1940d405cb 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -168,7 +168,16 @@ void THive::DeleteTabletWithoutStorage(TLeaderTabletInfo* tablet) { Send(SelfId(), new TEvTabletBase::TEvDeleteTabletResult(NKikimrProto::OK, tablet->Id)); } -void THive::RunProcessBootQueue() { +void THive::DeleteTabletWithoutStorage(TLeaderTabletInfo* tablet, TSideEffects& sideEffects) { + Y_ENSURE_LOG(tablet->IsDeleting(), "tablet " << tablet->Id); + Y_ENSURE_LOG(tablet->TabletStorageInfo->Channels.empty() || tablet->TabletStorageInfo->Channels[0].History.empty(), "tablet " << tablet->Id); + + // Tablet has no storage, so there's nothing to block or delete + // Simulate a response from CreateTabletReqDelete as if all steps have been completed + sideEffects.Send(SelfId(), new TEvTabletBase::TEvDeleteTabletResult(NKikimrProto::OK, tablet->Id)); +} + +void THive::ExecuteProcessBootQueue(TCompleteNotifications& notifications) { TInstant now = TActivationContext::Now(); BLOG_D("Handle ProcessBootQueue (size: " << BootQueue.BootQueue.size() << ")"); THPTimer bootQueueProcessingTimer; @@ -199,7 +208,7 @@ void THive::RunProcessBootQueue() { } } else { for (const TActorId actorToNotify : tablet->ActorsToNotifyOnRestart) { - Send(actorToNotify, new TEvPrivate::TEvRestartComplete(tablet->GetFullTabletId(), "boot delay")); + notifications.Send(actorToNotify, new TEvPrivate::TEvRestartComplete(tablet->GetFullTabletId(), "boot delay")); } tablet->ActorsToNotifyOnRestart.clear(); if (!bestNodeResult.TryToContinue) { @@ -412,6 +421,8 @@ void THive::Handle(TEvPrivate::TEvBootTablets::TPtr&) { } } TVector<TTabletId> tabletsToReleaseFromParent; + TSideEffects sideEffects; + sideEffects.Reset(SelfId()); for (auto& tab : Tablets) { TLeaderTabletInfo& tablet = tab.second; if (tablet.NeedToReleaseFromParent) { @@ -422,9 +433,9 @@ void THive::Handle(TEvPrivate::TEvBootTablets::TPtr&) { } else if (tablet.IsReadyToAssignGroups()) { tablet.InitiateAssignTabletGroups(); } else if (tablet.IsReadyToBlockStorage()) { - tablet.InitiateBlockStorage(); + tablet.InitiateBlockStorage(sideEffects); } else if (tablet.IsDeleting()) { - if (!tablet.InitiateBlockStorage(std::numeric_limits<ui32>::max())) { + if (!tablet.InitiateBlockStorage(sideEffects, std::numeric_limits<ui32>::max())) { DeleteTabletWithoutStorage(&tablet); } } else if (tablet.IsStopped() && tablet.State == ETabletState::Stopped) { @@ -445,6 +456,7 @@ void THive::Handle(TEvPrivate::TEvBootTablets::TPtr&) { SeenDomain(tablet.ObjectDomain); } } + sideEffects.Complete(DEPRECATED_CTX); SignalTabletActive(DEPRECATED_CTX); ReadyForConnections = true; if (AreWeRootHive()) { @@ -615,12 +627,18 @@ void THive::Handle(TEvHive::TEvTabletMetrics::TPtr& ev) { void THive::Handle(TEvInterconnect::TEvNodeConnected::TPtr &ev) { TNodeId nodeId = ev->Get()->NodeId; - BLOG_W("Handle TEvInterconnect::TEvNodeConnected, NodeId " << nodeId); - Send(GetNameserviceActorId(), new TEvInterconnect::TEvGetNode(nodeId)); + if (ConnectedNodes.insert(nodeId).second) { + BLOG_W("Handle TEvInterconnect::TEvNodeConnected, NodeId " << nodeId << " Cookie " << ev->Cookie); + Send(GetNameserviceActorId(), new TEvInterconnect::TEvGetNode(nodeId)); + } else { + BLOG_TRACE("Handle TEvInterconnect::TEvNodeConnected (duplicate), NodeId " << nodeId << " Cookie " << ev->Cookie); + } } void THive::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr &ev) { - BLOG_W("Handle TEvInterconnect::TEvNodeDisconnected, NodeId " << ev->Get()->NodeId); + TNodeId nodeId = ev->Get()->NodeId; + BLOG_W("Handle TEvInterconnect::TEvNodeDisconnected, NodeId " << nodeId); + ConnectedNodes.erase(nodeId); Execute(CreateDisconnectNode(THolder<TEvInterconnect::TEvNodeDisconnected>(ev->Release().Release()))); } @@ -704,26 +722,32 @@ void THive::Handle(TEvPrivate::TEvKickTablet::TPtr &ev) { void THive::Handle(TEvHive::TEvInitiateBlockStorage::TPtr& ev) { TTabletId tabletId = ev->Get()->TabletId; BLOG_D("THive::Handle::TEvInitiateBlockStorage TabletId=" << tabletId); + TSideEffects sideEffects; + sideEffects.Reset(SelfId()); TLeaderTabletInfo* tablet = FindTabletEvenInDeleting(tabletId); if (tablet != nullptr) { if (tablet->IsDeleting()) { - if (!tablet->InitiateBlockStorage(std::numeric_limits<ui32>::max())) { + if (!tablet->InitiateBlockStorage(sideEffects, std::numeric_limits<ui32>::max())) { DeleteTabletWithoutStorage(tablet); } } else if (tablet->IsReadyToBlockStorage()) { - tablet->InitiateBlockStorage(); + tablet->InitiateBlockStorage(sideEffects); } } + sideEffects.Complete(DEPRECATED_CTX); } void THive::Handle(TEvHive::TEvInitiateDeleteStorage::TPtr &ev) { TTabletId tabletId = ev->Get()->TabletId; BLOG_D("THive::Handle::TEvInitiateDeleteStorage TabletId=" << tabletId); + TSideEffects sideEffects; + sideEffects.Reset(SelfId()); TLeaderTabletInfo* tablet = FindTabletEvenInDeleting(tabletId); if (tablet != nullptr) { - tablet->InitiateDeleteStorage(); + tablet->InitiateDeleteStorage(sideEffects); } + sideEffects.Complete(DEPRECATED_CTX); } void THive::Handle(TEvHive::TEvGetTabletStorageInfo::TPtr& ev) { @@ -1601,6 +1625,17 @@ void THive::FillTabletInfo(NKikimrHive::TEvResponseHiveInfo& response, ui64 tabl if (req.GetReturnMetrics()) { tabletInfo.MutableMetrics()->CopyFrom(info->GetResourceValues()); } + if (req.GetReturnChannelHistory()) { + for (const auto& channel : info->TabletStorageInfo->Channels) { + auto& tabletChannel = *tabletInfo.AddTabletChannels(); + for (const auto& history : channel.History) { + auto& tabletHistory = *tabletChannel.AddHistory(); + tabletHistory.SetGroup(history.GroupID); + tabletHistory.SetGeneration(history.FromGeneration); + tabletHistory.SetTimestamp(history.Timestamp.MilliSeconds()); + } + } + } if (req.GetReturnFollowers()) { for (const auto& follower : info->Followers) { if (req.HasFollowerID() && req.GetFollowerID() != follower.Id) @@ -2354,16 +2389,6 @@ void THive::KickTablet(const TTabletInfo& tablet) { Send(SelfId(), new TEvPrivate::TEvKickTablet(tablet)); } -void THive::StopTablet(const TActorId& local, const TTabletInfo& tablet) { - BLOG_D("Sending TEvStopTablet(" << tablet.ToString() << ") to node " << local.NodeId()); - Send(local, new TEvLocal::TEvStopTablet(tablet.GetFullTabletId())); -} - -void THive::StopTablet(const TActorId& local, TFullTabletId tabletId) { - BLOG_D("Sending TEvStopTablet(" << tabletId << ") to node " << local.NodeId()); - Send(local, new TEvLocal::TEvStopTablet(tabletId)); -} - void THive::Handle(TEvHive::TEvRequestTabletIdSequence::TPtr& ev) { Execute(CreateRequestTabletSequence(std::move(ev))); } diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index a9034f9d17..f8b884adfb 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -116,30 +116,6 @@ namespace std { namespace NKikimr { namespace NHive { -struct TCompleteNotifications { - TVector<THolder<IEventHandle>> Notifications; - TActorId SelfID; - - void Reset(const TActorId &selfId) { - Notifications.clear(); - SelfID = selfId; - } - - void Send(const TActorId &recipient, IEventBase *ev, ui32 flags = 0, ui64 cookie = 0) { - Notifications.emplace_back(new IEventHandle(recipient, SelfID, ev, flags, cookie)); - } - - void Send(const TActorContext& ctx) { - for (auto& notification : Notifications) { - ctx.ExecutorThread.Send(notification.Release()); - } - } - - size_t size() const { - return Notifications.size(); - } -}; - TResourceRawValues ResourceRawValuesFromMetrics(const NKikimrTabletBase::TMetrics& metrics); NKikimrTabletBase::TMetrics MetricsFromResourceRawValues(const TResourceRawValues& values); TResourceRawValues ResourceRawValuesFromMetrics(const NKikimrHive::TTabletMetrics& tabletMetrics); @@ -183,6 +159,7 @@ protected: friend class TDrainNodeWaitActor; friend class TTxInitScheme; + friend class TTxDeleteBase; friend class TTxDeleteTablet; friend class TTxDeleteOwnerTablets; friend class TTxReassignGroups; @@ -402,6 +379,7 @@ protected: }; std::unordered_map<std::pair<ui64, ui64>, TPendingCreateTablet> PendingCreateTablets; + std::deque<THolder<IEventHandle>> PendingOperations; ui64 UpdateTabletMetricsInProgress = 0; static constexpr ui64 MAX_UPDATE_TABLET_METRICS_IN_PROGRESS = 10000; // 10K @@ -424,6 +402,7 @@ protected: std::unordered_map<TTabletTypes::EType, NKikimrConfig::THiveTabletLimit> TabletLimit; // built from CurrentConfig std::unordered_map<TTabletTypes::EType, NKikimrHive::TDataCentersPreference> DefaultDataCentersPreference; std::unordered_set<TDataCenterId> RegisteredDataCenterIds; + std::unordered_set<TNodeId> ConnectedNodes; // to be removed later bool TabletOwnersSynced = false; @@ -622,7 +601,7 @@ public: void KickTablet(const TTabletInfo& tablet); void StopTablet(const TActorId& local, const TTabletInfo& tablet); void StopTablet(const TActorId& local, TFullTabletId tabletId); - void RunProcessBootQueue(); + void ExecuteProcessBootQueue(TCompleteNotifications& notifications); TTabletMetricsAggregates DefaultResourceMetricsAggregates; ui64 MetricsWindowSize = TDuration::Minutes(1).MilliSeconds(); @@ -772,6 +751,7 @@ public: protected: void ScheduleDisconnectNode(THolder<TEvPrivate::TEvProcessDisconnectNode> event); void DeleteTabletWithoutStorage(TLeaderTabletInfo* tablet); + void DeleteTabletWithoutStorage(TLeaderTabletInfo* tablet, TSideEffects& sideEffects); void ScheduleUnlockTabletExecution(TNodeInfo& node); TString DebugDomainsActiveNodes() const; TResourceNormalizedValues GetStDevResourceValues() const; diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index 988eb6e801..1bf3502a96 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -61,6 +61,7 @@ namespace { runtime.SetLogPriority(NKikimrServices::HIVE, priority); runtime.SetLogPriority(NKikimrServices::BS_CONTROLLER, priority); } + runtime.SetLogPriority(NKikimrServices::BS_CONTROLLER, NLog::PRI_ERROR); runtime.SetLogPriority(NKikimrServices::LOCAL, priority); runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, otherPriority); runtime.SetLogPriority(NKikimrServices::TABLET_EXECUTOR, otherPriority); @@ -1935,7 +1936,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { UNIT_ASSERT_VALUES_UNEQUAL(tablet.GetTabletID(), tabletId); } - SendDeleteTestOwner(runtime, hiveTablet, MakeHolder<TEvHive::TEvDeleteOwnerTablets>(testerTablet, 124), 0, NKikimrProto::ALREADY); + SendDeleteTestOwner(runtime, hiveTablet, MakeHolder<TEvHive::TEvDeleteOwnerTablets>(testerTablet, 124), 0, NKikimrProto::OK); } Y_UNIT_TEST(TestDeleteOwnerTabletsMany) { @@ -1962,7 +1963,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { UNIT_ASSERT(!tabletIds.contains(tablet.GetTabletID())); } - SendDeleteTestOwner(runtime, hiveTablet, MakeHolder<TEvHive::TEvDeleteOwnerTablets>(testerTablet, 124), 0, NKikimrProto::ALREADY); + SendDeleteTestOwner(runtime, hiveTablet, MakeHolder<TEvHive::TEvDeleteOwnerTablets>(testerTablet, 124), 0, NKikimrProto::OK); } Y_UNIT_TEST(TestDeleteTabletWithFollowers) { @@ -2095,8 +2096,8 @@ Y_UNIT_TEST_SUITE(THiveTest) { TAutoPtr<IEventHandle> handle; auto createTabletReply = runtime.GrabEdgeEventRethrow<TEvHive::TEvCreateTabletReply>(handle); UNIT_ASSERT(createTabletReply); - UNIT_ASSERT_EQUAL_C(createTabletReply->Record.GetStatus(), NKikimrProto::ALREADY, - (ui32)createTabletReply->Record.GetStatus() << " != " << (ui32)NKikimrProto::ALREADY); + UNIT_ASSERT_EQUAL_C(createTabletReply->Record.GetStatus(), NKikimrProto::OK, + (ui32)createTabletReply->Record.GetStatus() << " != " << (ui32)NKikimrProto::OK); UNIT_ASSERT_EQUAL_C(createTabletReply->Record.GetOwner(), testerTablet, createTabletReply->Record.GetOwner() << " != " << testerTablet); ui64 tabletId = createTabletReply->Record.GetTabletID(); @@ -2418,19 +2419,37 @@ Y_UNIT_TEST_SUITE(THiveTest) { ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, MakeHolder<TEvHive::TEvCreateTablet>(testerTablet, 0, tabletType, BINDED_CHANNELS), 0, true); MakeSureTabletIsUp(runtime, tabletId, 0); + TActorId sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(hiveTablet, sender, new TEvHive::TEvRequestHiveInfo({ + .TabletId = tabletId, + .ReturnChannelHistory = true, + })); + TAutoPtr<IEventHandle> handle; + TEvHive::TEvResponseHiveInfo* response = runtime.GrabEdgeEventRethrow<TEvHive::TEvResponseHiveInfo>(handle); + std::unordered_set<ui32> tabletGroups; + for (const NKikimrHive::TTabletInfo& tablet : response->Record.GetTablets()) { + for (const NKikimrHive::TTabletChannelInfo& channel : tablet.GetTabletChannels()) { + for (const NKikimrHive::TTabletChannelGenInfo& history : channel.GetHistory()) { + tabletGroups.insert(history.GetGroup()); + } + } + } auto updateDiskStatus = MakeHolder<TEvBlobStorage::TEvControllerUpdateDiskStatus>(); - NKikimrBlobStorage::TVDiskMetrics* vdiskMetrics = updateDiskStatus->Record.AddVDisksMetrics(); + for (ui32 groupId = 0x80000000; groupId < 0x8000000a; ++groupId) { + if (tabletGroups.count(groupId) == 0) { + NKikimrBlobStorage::TVDiskMetrics* vdiskMetrics = updateDiskStatus->Record.AddVDisksMetrics(); - vdiskMetrics->MutableVDiskId()->SetGroupID(2147483650); - vdiskMetrics->MutableVDiskId()->SetGroupGeneration(1); - vdiskMetrics->MutableVDiskId()->SetRing(0); - vdiskMetrics->MutableVDiskId()->SetDomain(0); - vdiskMetrics->MutableVDiskId()->SetVDisk(0); - vdiskMetrics->SetAvailableSize(100000); + vdiskMetrics->MutableVDiskId()->SetGroupID(groupId); + vdiskMetrics->MutableVDiskId()->SetGroupGeneration(1); + vdiskMetrics->MutableVDiskId()->SetRing(0); + vdiskMetrics->MutableVDiskId()->SetDomain(0); + vdiskMetrics->MutableVDiskId()->SetVDisk(0); + vdiskMetrics->SetAvailableSize(100000); + } + } - TActorId sender = runtime.AllocateEdgeActor(); runtime.SendToPipe(MakeBSControllerID(0), sender, updateDiskStatus.Release(), 0, GetPipeConfigWithRetries()); SendReassignTabletSpace(runtime, hiveTablet, tabletId, {}, 0); @@ -2448,11 +2467,6 @@ Y_UNIT_TEST_SUITE(THiveTest) { SendCreateTestTablet(runtime, hiveTablet, testerTablet, MakeHolder<TEvHive::TEvCreateTablet>(testerTablet, 0, tabletType, newBindings), 0, true); - { - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); - runtime.DispatchEvents(options); - } MakeSureTabletIsUp(runtime, tabletId, 0); } @@ -2495,7 +2509,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { MakeHolder<TEvHive::TEvCreateTablet>(testerTablet, 0, tabletType, BINDED_CHANNELS), 0, true, - NKikimrProto::ALREADY); + NKikimrProto::OK); MakeSureTabletIsUp(runtime, tabletId, 0); @@ -2838,12 +2852,12 @@ Y_UNIT_TEST_SUITE(THiveTest) { auto* followerGroup = ev->Record.AddFollowerGroups(); followerGroup->SetFollowerCount(3); followerGroup->SetAllowLeaderPromotion(true); - SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - { + SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); + /*{ TDispatchOptions options; options.FinalEvents.emplace_back(TEvLocal::EvTabletStatus, 4); runtime.DispatchEvents(options); - } + }*/ // checking distribution, should be equal number of tablets on every node { std::array<int, 2> nodeTablets = {}; @@ -3404,9 +3418,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500, tabletType, BINDED_CHANNELS)); ev->Record.SetAllowFollowerPromotion(false); ev->Record.SetFollowerCount(2); - ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - - WaitForTabletsBecomeActive(runtime, 3); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries(); @@ -3466,9 +3478,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { TTabletTypes::EType tabletType = TTabletTypes::Dummy; THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500, tabletType, BINDED_CHANNELS)); ev->Record.SetCrossDataCenterFollowerCount(2); - ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - - WaitForTabletsBecomeActive(runtime, 7); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); NTabletPipe::TClientConfig pipeConfig; pipeConfig.ForceLocal = true; @@ -3521,9 +3531,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { followerGroup->SetFollowerCount(1); followerGroup->SetLocalNodeOnly(true); followerGroup->SetAllowClientRead(true); - ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - - WaitForTabletsBecomeActive(runtime, 2); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); ui32 leaderNode = 999; { @@ -3618,9 +3626,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { TTabletTypes::EType tabletType = TTabletTypes::Dummy; THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500, tabletType, BINDED_CHANNELS)); ev->Record.SetCrossDataCenterFollowerCount(FOLLOWERS); - ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - - WaitForTabletsBecomeActive(runtime, NODES + 1); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); NTabletPipe::TClientConfig pipeConfig; pipeConfig.ForceLocal = true; @@ -3774,9 +3780,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500, tabletType, BINDED_CHANNELS)); ev->Record.SetObjectId(1337); ev->Record.SetCrossDataCenterFollowerCount(FOLLOWERS); - ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, false); - - WaitForTabletsBecomeActive(runtime, FOLLOWERS * DCS + 1); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true); ui32 leaderNode = 0; ui32 followersNode = 0; @@ -3886,7 +3890,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { MakeSureTabletIsUp(runtime, tabletId, 0); THolder<TEvHive::TEvCreateTablet> ev2(new TEvHive::TEvCreateTablet(testerTablet, 0, tabletType, BINDED_CHANNELS)); ev2->Record.SetTabletBootMode(NKikimrHive::TABLET_BOOT_MODE_EXTERNAL); - ui64 tabletId2 = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev2), 0, false, NKikimrProto::ALREADY); + ui64 tabletId2 = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev2), 0, false, NKikimrProto::OK); UNIT_ASSERT_VALUES_EQUAL(tabletId, tabletId2); MakeSureTabletIsDown(runtime, tabletId2, 0); } diff --git a/ydb/core/mind/hive/leader_tablet_info.cpp b/ydb/core/mind/hive/leader_tablet_info.cpp index 83bef57a31..87205b6cfe 100644 --- a/ydb/core/mind/hive/leader_tablet_info.cpp +++ b/ydb/core/mind/hive/leader_tablet_info.cpp @@ -84,18 +84,18 @@ bool TLeaderTabletInfo::InitiateAssignTabletGroups() { return true; } -bool TLeaderTabletInfo::InitiateBlockStorage() { +bool TLeaderTabletInfo::InitiateBlockStorage(TSideEffects& sideEffects) { // attempt to kill tablet before blocking the storage group - Kill(); + Kill(sideEffects); // blocks PREVIOUS entry of tablet history IActor* x = CreateTabletReqBlockBlobStorage(Hive.SelfId(), TabletStorageInfo.Get(), KnownGeneration, true); - Hive.Register(x); + sideEffects.Register(x); return true; } -bool TLeaderTabletInfo::InitiateBlockStorage(ui32 generation) { +bool TLeaderTabletInfo::InitiateBlockStorage(TSideEffects& sideEffects, ui32 generation) { // attempt to kill tablet before blocking the storage group - Kill(); + Kill(sideEffects); // blocks LATEST entry of tablet history const TTabletChannelInfo* channel = TabletStorageInfo->ChannelInfo(0); if (IsDeleting() && channel == nullptr) { @@ -103,13 +103,13 @@ bool TLeaderTabletInfo::InitiateBlockStorage(ui32 generation) { } Y_VERIFY(channel != nullptr && !channel->History.empty()); IActor* x = CreateTabletReqBlockBlobStorage(Hive.SelfId(), TabletStorageInfo.Get(), generation, false); - Hive.Register(x); + sideEffects.Register(x); return true; } -bool TLeaderTabletInfo::InitiateDeleteStorage() { +bool TLeaderTabletInfo::InitiateDeleteStorage(TSideEffects& sideEffects) { IActor* x = CreateTabletReqDelete(Hive.SelfId(), TabletStorageInfo); - Hive.Register(x); + sideEffects.Register(x); return true; } diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index 88425b3cae..f5e833efb0 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -110,7 +110,7 @@ public: } bool IsSomeoneAliveOnNode(TNodeId nodeId) const; - + bool IsLockedToActor() const { return !!LockedToActor; } @@ -194,16 +194,16 @@ public: return result; } - void Kill() { + void Kill(TSideEffects& sideEffects) { for (TFollowerTabletInfo& follower : Followers) { - follower.Kill(); + follower.SendStopTablet(sideEffects); } - TTabletInfo::Kill(); + TTabletInfo::SendStopTablet(sideEffects); } - bool InitiateBlockStorage(); - bool InitiateBlockStorage(ui32 generation); - bool InitiateDeleteStorage(); + bool InitiateBlockStorage(TSideEffects& sideEffects); + bool InitiateBlockStorage(TSideEffects& sideEffects, ui32 generation); + bool InitiateDeleteStorage(TSideEffects& sideEffects); void IncreaseGeneration() { Y_VERIFY(KnownGeneration < Max<ui32>()); @@ -279,11 +279,11 @@ public: return *it; } - void NotifyStorageInfo(const TActorContext& ctx) { + void NotifyStorageInfo(TCompleteNotifications& notifications) { TVector<TActorId> targets; targets.swap(StorageInfoSubscribers); for (TActorId target : targets) { - ctx.Send(target, new TEvHive::TEvGetTabletStorageInfoResult(Id, *TabletStorageInfo)); + notifications.Send(target, new TEvHive::TEvGetTabletStorageInfoResult(Id, *TabletStorageInfo)); } } @@ -294,7 +294,7 @@ public: } void ActualizeTabletStatistics(TInstant now); - + void ResetTabletGroupsRequests() { ChannelProfileNewGroup.reset(); } diff --git a/ydb/core/mind/hive/storage_pool_info_ut.cpp b/ydb/core/mind/hive/storage_pool_info_ut.cpp index d0e21abc3f..fe0b4248f0 100644 --- a/ydb/core/mind/hive/storage_pool_info_ut.cpp +++ b/ydb/core/mind/hive/storage_pool_info_ut.cpp @@ -1,8 +1,15 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> +#include <util/stream/null.h> #include <ydb/core/protos/blobstorage.pb.h> #include "storage_pool_info.h" +#ifdef NDEBUG +#define Ctest Cnull +#else +#define Ctest Cerr +#endif + using namespace NKikimr; using namespace NHive; using namespace NKikimrBlobStorage; @@ -115,29 +122,29 @@ Y_UNIT_TEST_SUITE(StoragePool) { #ifndef _NDEBUG auto avg = GetAvg(groupUnits); - Cerr << "avg = " << avg << Endl; + Ctest << "avg = " << avg << Endl; auto min = GetMin(groupUnits); - Cerr << "min = " << min << Endl; + Ctest << "min = " << min << Endl; auto max = GetMax(groupUnits); - Cerr << "max = " << max << Endl; + Ctest << "max = " << max << Endl; auto stdDev = GetStdDev(groupUnits); - Cerr << "std-dev = " << stdDev << Endl; + Ctest << "std-dev = " << stdDev << Endl; for (int i = 0; i < CHANNELS; ++i) { auto avg = GetAvg(groupChannels[i]); - Cerr << "ch." << i << " avg = " << avg << Endl; + Ctest << "ch." << i << " avg = " << avg << Endl; auto min = GetMin(groupChannels[i]); - Cerr << "ch." << i << " min = " << min << Endl; + Ctest << "ch." << i << " min = " << min << Endl; auto max = GetMax(groupChannels[i]); - Cerr << "ch." << i << " max = " << max << Endl; + Ctest << "ch." << i << " max = " << max << Endl; auto stdDev = GetStdDev(groupChannels[i]); - Cerr << "ch." << i << " std-dev = " << stdDev << Endl; + Ctest << "ch." << i << " std-dev = " << stdDev << Endl; } #endif @@ -183,11 +190,11 @@ Y_UNIT_TEST_SUITE(StoragePool) { #ifndef _NDEBUG auto avg = GetAvg(groupUnits); - Cerr << "avg = " << avg << Endl; + Ctest << "avg = " << avg << Endl; auto stdDev = GetStdDev(groupUnits); - Cerr << "std-dev = " << stdDev << Endl; + Ctest << "std-dev = " << stdDev << Endl; #endif UNIT_ASSERT_VALUES_EQUAL(round(GetAvg(groupUnits)), 1250); diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 9fddd10801..81af51c3a9 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -195,12 +195,26 @@ bool TTabletInfo::InitiateBoot() { } } -bool TTabletInfo::InitiateStop() { +TActorId TTabletInfo::GetLocal() const { TActorId local; + TNodeInfo* node = GetNode(); + if (node != nullptr) { + local = node->Local; + } + return local; +} + +TNodeInfo* TTabletInfo::GetNode() const { TNodeInfo* node = Node; if (node == nullptr && NodeId != 0) { node = Hive.FindNode(NodeId); } + return node; +} + +bool TTabletInfo::InitiateStop(TSideEffects& sideEffects) { + TNodeInfo* node = GetNode(); + TActorId local; if (node != nullptr) { local = node->Local; } @@ -209,13 +223,13 @@ bool TTabletInfo::InitiateStop() { // we only do it when we have PreferredNodeId, which means that we are moving from one node to another LastNodeId = node->Id; } else { - SendStopTablet(local, GetFullTabletId()); + SendStopTablet(local, sideEffects); LastNodeId = 0; } if (IsLeader()) { for (TFollowerTabletInfo& follower : AsLeader().Followers) { if (follower.FollowerGroup.LocalNodeOnly) { - follower.InitiateStop(); + follower.InitiateStop(sideEffects); } } } @@ -292,20 +306,6 @@ bool TTabletInfo::Kick() { } } -void TTabletInfo::Kill() { - TActorId local; - TNodeInfo* node = Node; - if (node == nullptr && NodeId != 0) { - node = Hive.FindNode(NodeId); - } - if (node != nullptr) { - local = node->Local; - } - if (local) { - Hive.StopTablet(local, *this); - } -} - const TVector<i64>& TTabletInfo::GetTabletAllowedMetricIds() const { return Hive.GetTabletTypeAllowedMetricIds(GetLeader().Type); } @@ -458,9 +458,22 @@ bool TTabletInfo::InitiateStart(TNodeInfo* node) { return false; } -void TTabletInfo::SendStopTablet(const TActorId& local, TFullTabletId tabletId) { +void TTabletInfo::SendStopTablet(TSideEffects& sideEffects) { + TActorId local = GetLocal(); if (local) { - Hive.StopTablet(local, tabletId); + SendStopTablet(local, sideEffects); + } +} + +void TTabletInfo::SendStopTablet(const TActorId& local, TSideEffects& sideEffects) { + if (local) { + TFullTabletId tabletId = GetFullTabletId(); + ui32 gen = 0; + if (IsLeader()) { + gen = AsLeader().KnownGeneration; + } + BLOG_D("Sending TEvStopTablet(" << ToString() << " gen " << gen << ") to node " << local.NodeId()); + sideEffects.Send(local, new TEvLocal::TEvStopTablet(tabletId, gen)); } } diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index dc360a30e4..aa03ecd357 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -214,13 +214,14 @@ public: bool BecomeRunning(TNodeId nodeId); bool BecomeStopped(); - void SendStopTablet(const TActorId& local, TFullTabletId tabletId); - - bool InitiateStop(); + TNodeInfo* GetNode() const; + TActorId GetLocal() const; + void SendStopTablet(TSideEffects& sideEffects); + void SendStopTablet(const TActorId& local, TSideEffects& sideEffects); + bool InitiateStop(TSideEffects& sideEffects); void BecomeUnknown(TNodeInfo* node); bool Kick(); - void Kill(); const TVector<i64>& GetTabletAllowedMetricIds() const; void UpdateResourceUsage(const NKikimrTabletBase::TMetrics& metrics); diff --git a/ydb/core/mind/hive/tx__block_storage_result.cpp b/ydb/core/mind/hive/tx__block_storage_result.cpp index 28315ec93b..7ca9b1c417 100644 --- a/ydb/core/mind/hive/tx__block_storage_result.cpp +++ b/ydb/core/mind/hive/tx__block_storage_result.cpp @@ -7,6 +7,7 @@ namespace NHive { class TTxBlockStorageResult : public TTransactionBase<THive> { TEvTabletBase::TEvBlockBlobStorageResult::TPtr Result; TTabletId TabletId; + TSideEffects SideEffects; public: TTxBlockStorageResult(TEvTabletBase::TEvBlockBlobStorageResult::TPtr& ev, THive* hive) : TBase(hive) @@ -17,6 +18,7 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_BLOCK_STORAGE_RESULT; } bool Execute(TTransactionContext& txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); TEvTabletBase::TEvBlockBlobStorageResult* msg = Result->Get(); BLOG_D("THive::TTxBlockStorageResult::Execute(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")"); TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(TabletId); @@ -27,19 +29,10 @@ public: db.Table<Schema::Tablet>().Key(tablet->Id).Update(NIceDb::TUpdate<Schema::Tablet::State>(ETabletState::ReadyToWork)); } else if (tablet->State == ETabletState::Deleting) { for (TFollowerTabletInfo& follower : tablet->Followers) { - follower.InitiateStop(); + follower.InitiateStop(SideEffects); } } } - } - return true; - } - - void Complete(const TActorContext& ctx) override { - TEvTabletBase::TEvBlockBlobStorageResult* msg = Result->Get(); - BLOG_D("THive::TTxBlockStorageResult::Complete(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")"); - TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(TabletId); - if (tablet != nullptr) { if (msg->Status == NKikimrProto::OK || msg->Status == NKikimrProto::RACE || msg->Status == NKikimrProto::BLOCKED @@ -48,21 +41,28 @@ public: if (msg->Status != NKikimrProto::EReplyStatus::OK) { BLOG_W("THive::TTxBlockStorageResult Complete status was " << NKikimrProto::EReplyStatus_Name(msg->Status) << " for TabletId " << tablet->Id); } - ctx.Send(Self->SelfId(), new TEvHive::TEvInitiateDeleteStorage(tablet->Id)); + SideEffects.Send(Self->SelfId(), new TEvHive::TEvInitiateDeleteStorage(tablet->Id)); } else { tablet->State = ETabletState::ReadyToWork; if (tablet->IsBootingSuppressed()) { // Use best effort to kill currently running tablet - ctx.Register(CreateTabletKiller(TabletId, /* nodeId */ 0, tablet->KnownGeneration)); + SideEffects.Register(CreateTabletKiller(TabletId, /* nodeId */ 0, tablet->KnownGeneration)); } else { Self->Execute(Self->CreateRestartTablet(tablet->GetFullTabletId())); } } } else { BLOG_W("THive::TTxBlockStorageResult retrying for " << TabletId << " because of " << NKikimrProto::EReplyStatus_Name(msg->Status)); - ctx.Schedule(TDuration::MilliSeconds(1000), new TEvHive::TEvInitiateBlockStorage(tablet->Id)); + SideEffects.Schedule(TDuration::MilliSeconds(1000), new TEvHive::TEvInitiateBlockStorage(tablet->Id)); } } + return true; + } + + void Complete(const TActorContext& ctx) override { + TEvTabletBase::TEvBlockBlobStorageResult* msg = Result->Get(); + BLOG_D("THive::TTxBlockStorageResult::Complete(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")"); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__create_tablet.cpp b/ydb/core/mind/hive/tx__create_tablet.cpp index f3b27bfd9a..3bccb63888 100644 --- a/ydb/core/mind/hive/tx__create_tablet.cpp +++ b/ydb/core/mind/hive/tx__create_tablet.cpp @@ -14,7 +14,6 @@ class TTxCreateTablet : public TTransactionBase<THive> { const TActorId Sender; const ui64 Cookie; - NKikimrProto::EReplyStatus Status; NKikimrHive::EErrorReason ErrorReason; ui64 TabletId; TObjectId ObjectId; @@ -25,12 +24,13 @@ class TTxCreateTablet : public TTransactionBase<THive> { TVector<TDataCenterId> AllowedDataCenterIds; NKikimrHive::TDataCentersPreference DataCentersPreference; TVector<TSubDomainKey> AllowedDomains; - ETabletState State; NKikimrHive::TTabletCategory TabletCategory; TVector<NKikimrHive::TFollowerGroup> FollowerGroups; NKikimrHive::ETabletBootMode BootMode; NKikimrHive::TForwardRequest ForwardRequest; + TSideEffects SideEffects; + public: TTxCreateTablet(NKikimrHive::TEvCreateTablet record, const TActorId& sender, const ui64 cookie, THive* hive) : TBase(hive) @@ -42,13 +42,11 @@ public: StateStorageGroupFromTabletID(hive->TabletID())) , Sender(sender) , Cookie(cookie) - , Status(NKikimrProto::UNKNOWN) , TabletId(0) , ObjectId(0) , ObjectDomain(RequestData.GetObjectDomain()) , BoundChannels(RequestData.GetBindedChannels().begin(), RequestData.GetBindedChannels().end()) , AllowedDomains(RequestData.GetAllowedDomains().begin(), RequestData.GetAllowedDomains().end()) - , State(ETabletState::Unknown) , BootMode(RequestData.GetTabletBootMode()) { const ui32 allowedNodeIdsSize = RequestData.AllowedNodeIDsSize(); @@ -99,7 +97,7 @@ public: ObjectId = RequestData.GetObjectId(); } - void UpdateChannelsBinding(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db) { + bool UpdateChannelsBinding(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db) { Y_VERIFY(tablet.BoundChannels.size() <= BoundChannels.size(), "only expansion channels number is allowed in Binded Channels"); std::bitset<MAX_TABLET_CHANNELS> newChannels; @@ -129,7 +127,7 @@ public: if (newChannels.any()) { tablet.ChannelProfileNewGroup |= newChannels; - tablet.State = State = ETabletState::GroupAssignment; + tablet.State = ETabletState::GroupAssignment; tablet.ChannelProfileReassignReason = NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_NO; tablet.BoundChannels = BoundChannels; for (auto& bind : tablet.BoundChannels) { @@ -137,10 +135,11 @@ public: } db.Table<Schema::Tablet>().Key(TabletId) .Update<Schema::Tablet::State, Schema::Tablet::ReassignReason, Schema::Tablet::ActorsToNotify>( - State, NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_NO, {Sender} + tablet.State, NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_NO, {Sender} ); - Status = NKikimrProto::OK; // otherwise it would be ALREADY + return true; } + return false; } bool ValidateChannelsBinding(TLeaderTabletInfo& tablet) { @@ -151,11 +150,52 @@ public: return true; } + void PostponeCreateTablet() { + THive::TPendingCreateTablet& pendingCreateTablet(Self->PendingCreateTablets[{OwnerId, OwnerIdx}]); + pendingCreateTablet.CreateTablet = RequestData; // TODO: consider std::move + pendingCreateTablet.Sender = Sender; + pendingCreateTablet.Cookie = Cookie; + } + + void RequestFreeSequence() { + if (Self->AreWeSubDomainHive()) { + if (!Self->RequestingSequenceNow) { + Self->RequestFreeSequence(); + } + } + } + + void ReplyToSender(NKikimrProto::EReplyStatus status) { + BLOG_D("THive::TTxCreateTablet::Execute TabletId: " << TabletId << + " Status: " << NKikimrProto::EReplyStatus_Name(status)); + Y_VERIFY(!!Sender); + THolder<TEvHive::TEvCreateTabletReply> reply = MakeHolder<TEvHive::TEvCreateTabletReply>(status, OwnerId, OwnerIdx, TabletId, Self->TabletID(), ErrorReason); + if (ForwardRequest.HasHiveTabletId()) { + reply->Record.MutableForwardRequest()->CopyFrom(ForwardRequest); + } + SideEffects.Send(Sender, reply.Release(), 0, Cookie); + } + + void ProcessTablet(TLeaderTabletInfo& tablet) { + if (tablet.IsReadyToAssignGroups()) { + tablet.InitiateAssignTabletGroups(); + } else if (tablet.IsBootingSuppressed()) { + // Tablet will never boot, so notify about creation right now + for (const TActorId& actor : tablet.ActorsToNotify) { + SideEffects.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); + } + tablet.ActorsToNotify.clear(); + } else { + tablet.TryToBoot(); + } + } + TTxType GetTxType() const override { return NHive::TXTYPE_CREATE_TABLET; } bool Execute(TTransactionContext &txc, const TActorContext&) override { - BLOG_D("THive::TTxCreateTablet::Execute"); - State = ETabletState::Unknown; + const TOwnerIdxType::TValueType ownerIdx(OwnerId, OwnerIdx); + BLOG_D("THive::TTxCreateTablet::Execute " << RequestData.ShortDebugString()); + SideEffects.Reset(Self->SelfId()); ErrorReason = NKikimrHive::ERROR_REASON_UNKNOWN; for (const auto& domain : AllowedDomains) { if (!Self->SeenDomain(domain)) { @@ -168,13 +208,13 @@ public: } } if (Self->BlockedOwners.count(OwnerId) != 0) { - Status = NKikimrProto::BLOCKED; BLOG_W("THive::TTxCreateTablet::Execute Owner " << OwnerId << " is blocked"); + ReplyToSender(NKikimrProto::BLOCKED); return true; } NIceDb::TNiceDb db(txc.DB); - // check if tablet is already created - const TOwnerIdxType::TValueType ownerIdx(OwnerId, OwnerIdx); + // check if tablet has already been created + { auto itOwner = Self->OwnerToTablet.find(ownerIdx); if (itOwner != Self->OwnerToTablet.end()) { // tablet is already created @@ -186,60 +226,37 @@ public: TabletId = tabletId; if (existingTabletType != TabletType || tablet->SeizedByChild) { if (tablet->SeizedByChild) { - BLOG_D("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " seized by child - operation postponed"); - Status = NKikimrProto::UNKNOWN; // retry later + BLOG_W("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " seized by child - operation postponed"); + PostponeCreateTablet(); } else { - Status = NKikimrProto::ERROR; + BLOG_ERROR("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " has different type"); + ReplyToSender(NKikimrProto::ERROR); } - } else { - Status = NKikimrProto::ALREADY; - } - if (Status == NKikimrProto::ALREADY) { - if (BootMode == NKikimrHive::TABLET_BOOT_MODE_EXTERNAL) { - // Make sure any running tablets are stopped - for (TFollowerTabletInfo& follower : tablet->Followers) { - follower.InitiateStop(); - } - tablet->InitiateStop(); - } - - State = tablet->State; - if (State == ETabletState::StoppingInGroupAssignment) { - BLOG_D("THive::TTxCreateTablet::Execute TabletId: " << TabletId << - " Status: " << (ui32)Status << " Stopping in group assignment"); - tablet->ActorsToNotify.push_back(Sender); - tablet->BootMode = BootMode; - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::ActorsToNotify>(tablet->ActorsToNotify); - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::BootMode>(tablet->BootMode); - if (tablet->State != ETabletState::GroupAssignment) { - tablet->State = State = ETabletState::GroupAssignment; - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(State); - } - return true; - } else if (State == ETabletState::Stopping || State == ETabletState::Stopped) { - BLOG_D("THive::TTxCreateTablet::Execute TabletId: " << TabletId << - " Status: " << (ui32)Status << " Stopping or Stopped"); - tablet->ActorsToNotify.push_back(Sender); - tablet->BootMode = BootMode; - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::ActorsToNotify>(tablet->ActorsToNotify); - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::BootMode>(tablet->BootMode); - if (tablet->State != ETabletState::ReadyToWork) { - tablet->State = State = ETabletState::ReadyToWork; - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(State); - } - return true; - } - } else { - BLOG_D("THive::TTxCreateTablet::Execute TabletId: " << TabletId << " Status: " << Status); return true; } + BLOG_D("THive::TTxCreateTablet::Execute TabletId: " << TabletId << " State: " << ETabletStateName(tablet->State)); if (!ValidateChannelsBinding(*tablet)) { - Status = NKikimrProto::ERROR; + BLOG_ERROR("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " has invalid channel bindings"); + ReplyToSender(NKikimrProto::ERROR); return true; } - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(State); + if (BootMode == NKikimrHive::TABLET_BOOT_MODE_EXTERNAL) { + // Make sure any running tablets are stopped + for (TFollowerTabletInfo& follower : tablet->Followers) { + follower.InitiateStop(SideEffects); + } + tablet->InitiateStop(SideEffects); + } + + if (tablet->State == ETabletState::StoppingInGroupAssignment) { + tablet->State = ETabletState::GroupAssignment; + } else if (tablet->State == ETabletState::Stopping || tablet->State == ETabletState::Stopped) { + tablet->State = ETabletState::ReadyToWork; + } + + db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(tablet->State); tablet->ActorsToNotify.push_back(Sender); db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::ActorsToNotify>(tablet->ActorsToNotify); tablet->AllowedNodes = AllowedNodeIds; @@ -287,19 +304,23 @@ public: for (ui32 i = followerGroup.GetComputedFollowerCount(Self->GetDataCenters()); i < oldFollowerCount; ++i) { TFollowerTabletInfo& follower = tablet->Followers.back(); db.Table<Schema::TabletFollowerTablet>().Key(TabletId, follower.Id).Delete(); - follower.InitiateStop(); + follower.InitiateStop(SideEffects); tablet->Followers.pop_back(); } ++itFollowerGroup; } + ProcessTablet(*tablet); + + BLOG_D("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " has been successfully updated"); + ReplyToSender(NKikimrProto::OK); return true; } } else if (RequestData.HasTabletID()) { TTabletId tabletId = RequestData.GetTabletID(); if (Self->CheckForForwardTabletRequest(tabletId, ForwardRequest)) { TabletId = tabletId; - Status = NKikimrProto::INVALID_OWNER; // actually this status from blob storage, but I think it fits this situation perfectly + ReplyToSender(NKikimrProto::INVALID_OWNER); // actually this status from blob storage, but I think it fits this situation perfectly return true; // abort transaction } } @@ -307,7 +328,8 @@ public: switch((TTabletTypes::EType)TabletType) { case TTabletTypes::BSController: - Status = NKikimrProto::ERROR; + BLOG_ERROR("THive::TTxCreateTablet::Execute Cannot create such tablet"); + ReplyToSender(NKikimrProto::ERROR); return true; default: break; @@ -316,7 +338,9 @@ public: std::vector<TSequencer::TOwnerType> modified; auto tabletIdIndex = Self->Sequencer.AllocateElement(modified); if (tabletIdIndex == TSequencer::NO_ELEMENT) { - Status = NKikimrProto::UNKNOWN; + BLOG_W("THive::TTxCreateTablet::Execute CreateTablet Postponed"); + PostponeCreateTablet(); + RequestFreeSequence(); return true; } else { TabletId = MakeTabletID(AssignStateStorage, Self->HiveUid, tabletIdIndex); @@ -338,13 +362,11 @@ public: TInstant now = TlsActivationContext->Now(); // insert entry for new tablet - State = ETabletState::GroupAssignment; - TLeaderTabletInfo& tablet = Self->GetTablet(TabletId); tablet.NodeId = 0; tablet.Type = (TTabletTypes::EType)TabletType; tablet.KnownGeneration = 0; // because we will increase it on start - tablet.State = State; + tablet.State = ETabletState::GroupAssignment; tablet.ActorsToNotify.push_back(Sender); tablet.AllowedNodes = AllowedNodeIds; tablet.Owner = ownerIdx; @@ -456,56 +478,28 @@ public: Self->OwnerToTablet.emplace(ownerIdx, TabletId); Self->ObjectToTabletMetrics[tablet.ObjectId].IncreaseCount(); Self->TabletTypeToTabletMetrics[tablet.Type].IncreaseCount(); - Status = NKikimrProto::OK; - return true; - } - void Complete(const TActorContext& ctx) override { - if (Status != NKikimrProto::UNKNOWN) { - BLOG_D("THive::TTxCreateTablet::Complete TabletId: " << TabletId << - " Status: " << NKikimrProto::EReplyStatus_Name(Status) << " State: " << ETabletStateName(State)); - Y_VERIFY(!!Sender); - THolder<TEvHive::TEvCreateTabletReply> reply = MakeHolder<TEvHive::TEvCreateTabletReply>(Status, OwnerId, OwnerIdx, TabletId, Self->TabletID(), ErrorReason); - if (ForwardRequest.HasHiveTabletId()) { - reply->Record.MutableForwardRequest()->CopyFrom(ForwardRequest); - } - ctx.Send(Sender, reply.Release(), 0, Cookie); - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - if (Status == NKikimrProto::OK && tablet->Type == TTabletTypes::Hive) { - auto itSubDomain = Self->Domains.find(tablet->ObjectDomain); - if (itSubDomain != Self->Domains.end()) { - if (itSubDomain->second.HiveId == 0) { - itSubDomain->second.HiveId = tablet->Id; - } - Self->Execute(Self->CreateUpdateDomain(tablet->ObjectDomain)); - } - } - if (Status == NKikimrProto::OK && tablet->IsReadyToAssignGroups()) { - tablet->InitiateAssignTabletGroups(); - } else if (Status == NKikimrProto::OK && tablet->IsBootingSuppressed()) { - // Tablet will never boot, so notify about creation right now - for (const TActorId& actor : tablet->ActorsToNotify) { - ctx.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); - } - tablet->ActorsToNotify.clear(); - } else { - tablet->TryToBoot(); - } - } - Self->ProcessBootQueue(); - } else { - BLOG_D("THive::TTxCreateTablet::Complete CreateTablet Postponed"); - THive::TPendingCreateTablet& pendingCreateTablet(Self->PendingCreateTablets[{OwnerId, OwnerIdx}]); - pendingCreateTablet.CreateTablet = RequestData; // TODO: consider std::move - pendingCreateTablet.Sender = Sender; - pendingCreateTablet.Cookie = Cookie; - if (Self->AreWeSubDomainHive()) { - if (!Self->RequestingSequenceNow) { - Self->RequestFreeSequence(); + ReplyToSender(NKikimrProto::OK); + + if (tablet.Type == TTabletTypes::Hive) { + auto itSubDomain = Self->Domains.find(tablet.ObjectDomain); + if (itSubDomain != Self->Domains.end()) { + if (itSubDomain->second.HiveId == 0) { + itSubDomain->second.HiveId = tablet.Id; } + Self->Execute(Self->CreateUpdateDomain(tablet.ObjectDomain)); } } + + ProcessTablet(tablet); + + return true; + } + + void Complete(const TActorContext& ctx) override { + const TOwnerIdxType::TValueType ownerIdx(OwnerId, OwnerIdx); + BLOG_D("THive::TTxCreateTablet::Complete " << ownerIdx << " TabletId: " << TabletId << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__delete_tablet.cpp b/ydb/core/mind/hive/tx__delete_tablet.cpp index 6d481e695f..b94204287e 100644 --- a/ydb/core/mind/hive/tx__delete_tablet.cpp +++ b/ydb/core/mind/hive/tx__delete_tablet.cpp @@ -4,207 +4,209 @@ namespace NKikimr { namespace NHive { -class TTxDeleteTablet : public TTransactionBase<THive> { - TEvHive::TEvDeleteTablet::TPtr Event; - ui64 OwnerId = 0; - TVector<ui64> LocalIdxs; - NKikimrProto::EReplyStatus Status = NKikimrProto::ERROR; - TVector<TTabletId> TabletIds; - TCompleteNotifications Notifications; - NKikimrHive::TForwardRequest ForwardRequest; +class TTxDeleteBase : public TTransactionBase<THive> { +protected: + TSideEffects SideEffects; public: - TTxDeleteTablet(TEvHive::TEvDeleteTablet::TPtr& ev, THive* hive) - : TBase(hive) - , Event(ev) - {} - TTxType GetTxType() const override { return NHive::TXTYPE_DELETE_TABLET; } - TTabletId DoDeleteTablet(ui64 owner, ui64 idx, NIceDb::TNiceDb& db) { - TTabletId deletedTablet = 0; + TTxDeleteBase(THive* hive) + : TBase(hive) + {} - TOwnerIdxType::TValueType ownerIdx(owner, idx); - auto it = Self->OwnerToTablet.find(ownerIdx); - if (it != Self->OwnerToTablet.end()) { - deletedTablet = it->second; - BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << it->second); - TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(it->second); - Y_VERIFY(tablet != nullptr, "%s", (TStringBuilder() << "Tablet " << it->second << " OwnerIdx " << ownerIdx).data()); + void DeleteTablet(TTabletId tabletId, NIceDb::TNiceDb& db) { + BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << tabletId); + TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId); + if (tablet != nullptr) { if (tablet->SeizedByChild) { - BLOG_W("THive::TTxDeleteTablet tablet " << it->second << " seized by child"); - return 0; + BLOG_W("THive::TTxDeleteTablet tablet " << tabletId << " seized by child"); + return; } if (tablet->State != ETabletState::Deleting) { tablet->State = ETabletState::Deleting; - tablet->InitiateStop(); db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State, Schema::Tablet::LeaderNode>(ETabletState::Deleting, 0); + for (const TActorId& actor : tablet->ActorsToNotifyOnRestart) { + SideEffects.Send(actor, new TEvPrivate::TEvRestartComplete(tablet->GetFullTabletId(), "delete")); + } + tablet->ActorsToNotifyOnRestart.clear(); + tablet->InitiateStop(SideEffects); for (TTabletInfo& follower : tablet->Followers) { - follower.InitiateStop(); + for (const TActorId& actor : follower.ActorsToNotifyOnRestart) { + SideEffects.Send(actor, new TEvPrivate::TEvRestartComplete(follower.GetFullTabletId(), "delete")); + } + follower.ActorsToNotifyOnRestart.clear(); + follower.InitiateStop(SideEffects); db.Table<Schema::TabletFollowerTablet>().Key(follower.GetFullTabletId()).Update<Schema::TabletFollowerTablet::FollowerNode>(0); } - if (!tablet->InitiateBlockStorage(std::numeric_limits<ui32>::max())) { - Self->DeleteTabletWithoutStorage(tablet); + if (!tablet->InitiateBlockStorage(SideEffects, std::numeric_limits<ui32>::max())) { + Self->DeleteTabletWithoutStorage(tablet, SideEffects); } } else { - BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << it->second << " already in ETabletState::Deleting"); + BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << tabletId << " already in ETabletState::Deleting"); } } else { - BLOG_W("THive::TTxDeleteTablet tablet " << ownerIdx << " wasn't found"); - Self->PendingCreateTablets.erase({owner, idx}); + BLOG_W("THive::TTxDeleteTablet tablet " << tabletId << " wasn't found"); } + } +}; - return deletedTablet; +class TTxDeleteTablet : public TTxDeleteBase { + TEvHive::TEvDeleteTablet::TPtr Event; + +public: + TTxDeleteTablet(TEvHive::TEvDeleteTablet::TPtr& ev, THive* hive) + : TTxDeleteBase(hive) + , Event(ev) + {} + + void RespondToSender(NKikimrProto::EReplyStatus status, const NKikimrHive::TForwardRequest& forwardRequest = {}) { + const NKikimrHive::TEvDeleteTablet& rec = Event->Get()->Record; + auto response = MakeHolder<TEvHive::TEvDeleteTabletReply>(status, Self->TabletID(), rec); + if (forwardRequest.GetHiveTabletId() != 0) { + response->Record.MutableForwardRequest()->CopyFrom(forwardRequest); + } + BLOG_D("THive::TTxDeleteTablet::Execute() result " << response->Record.ShortDebugString()); + SideEffects.Send(Event->Sender, response.Release(), 0, Event->Cookie); } bool Execute(TTransactionContext& txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); const NKikimrHive::TEvDeleteTablet& rec = Event->Get()->Record; - NIceDb::TNiceDb db(txc.DB); - OwnerId = rec.GetShardOwnerId(); - for (ui64 idx : rec.GetShardLocalIdx()) { - LocalIdxs.push_back(idx); + BLOG_D("THive::TTxDeleteTablet::Execute() " << rec.ShortDebugString()); + // resolving ownerid:owneridx to tabletids + std::vector<TTabletId> tablets; + tablets.reserve(rec.ShardLocalIdxSize()); + ui64 owner = rec.GetShardOwnerId(); + for (size_t pos = 0; pos < rec.ShardLocalIdxSize(); ++pos) { + ui64 idx = rec.GetShardLocalIdx(pos); + TOwnerIdxType::TValueType ownerIdx(owner, idx); + auto it = Self->OwnerToTablet.find(ownerIdx); + if (it != Self->OwnerToTablet.end()) { + tablets.push_back(it->second); + } else { + if (pos < rec.TabletIDSize()) { + TTabletId tabletId = rec.GetTabletID(pos); + BLOG_W("THive::TTxDeleteTablet tablet " << ownerIdx << " wasn't found - using supplied " << tabletId); + tablets.push_back(tabletId); + } else { + BLOG_W("THive::TTxDeleteTablet tablet " << ownerIdx << " wasn't found"); + } + } + if (Self->PendingCreateTablets.erase({owner, idx}) != 0) { + BLOG_NOTICE("THive::TTxDeleteTablet tablet " << ownerIdx << " was cleared from pending creates"); + } } - for (TTabletId tabletId : rec.GetTabletID()) { - TTabletId prevForwardHiveTabletId = ForwardRequest.GetHiveTabletId(); - if (Self->CheckForForwardTabletRequest(tabletId, ForwardRequest)) { - if (prevForwardHiveTabletId != 0 && prevForwardHiveTabletId != ForwardRequest.GetHiveTabletId()) { + // checking for possible forwards + NKikimrHive::TForwardRequest forwardRequest; + for (TTabletId tabletId : tablets) { + TTabletId prevForwardHiveTabletId = forwardRequest.GetHiveTabletId(); + if (Self->CheckForForwardTabletRequest(tabletId, forwardRequest)) { + if (prevForwardHiveTabletId != 0 && prevForwardHiveTabletId != forwardRequest.GetHiveTabletId()) { BLOG_ERROR("Forward of DeleteTablet is not possible - different owners of tablets"); - Status = NKikimrProto::ERROR; // actually this status from blob storage, but I think it fits this situation perfectly + RespondToSender(NKikimrProto::ERROR); return true; // abort transaction } } } - if (ForwardRequest.GetHiveTabletId() != 0) { - Status = NKikimrProto::INVALID_OWNER; // actually this status from blob storage, but I think it fits this situation perfectly + // respond with forward + if (forwardRequest.GetHiveTabletId() != 0) { + // actually this status from blob storage, but I think it fits this situation perfectly + RespondToSender(NKikimrProto::INVALID_OWNER, forwardRequest); return true; // abort transaction } - for (ui64 idx : rec.GetShardLocalIdx()) { - Status = NKikimrProto::OK; - if (TTabletId deletedTablet = DoDeleteTablet(OwnerId, idx, db)) { - TabletIds.push_back(deletedTablet); - } - } - for (ui64 tabletId : TabletIds) { - TLeaderTabletInfo* tablet = Self->FindTablet(tabletId); + // checking for possible migration + for (TTabletId tabletId : tablets) { + TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId); if (tablet != nullptr) { - for (const TActorId& actor : tablet->ActorsToNotifyOnRestart) { - Notifications.Send(actor, new TEvPrivate::TEvRestartComplete(tablet->GetFullTabletId(), "delete")); - } - tablet->ActorsToNotifyOnRestart.clear(); - for (TTabletInfo& follower : tablet->Followers) { - for (const TActorId& actor : follower.ActorsToNotifyOnRestart) { - Notifications.Send(actor, new TEvPrivate::TEvRestartComplete(follower.GetFullTabletId(), "delete")); - } - follower.ActorsToNotifyOnRestart.clear(); + if (tablet->SeizedByChild) { + BLOG_W("THive::TTxDeleteTablet tablet " << tabletId << " seized by child"); + RespondToSender(NKikimrProto::ERROR); + return true; // abort transaction } } } + NIceDb::TNiceDb db(txc.DB); + for (TTabletId tabletId : tablets) { + DeleteTablet(tabletId, db); + } + RespondToSender(NKikimrProto::OK); return true; } void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxDeleteTablet::Complete(" << TabletIds << ")"); - THolder<TEvHive::TEvDeleteTabletReply> response = MakeHolder<TEvHive::TEvDeleteTabletReply>(Status, Self->TabletID(), Event->Get()->Record.GetTxId_Deprecated(), OwnerId, LocalIdxs); - if (ForwardRequest.HasHiveTabletId()) { - response->Record.MutableForwardRequest()->CopyFrom(ForwardRequest); - } - ctx.Send(Event->Sender, response.Release(), 0, Event->Cookie); - Notifications.Send(ctx); + BLOG_D("THive::TTxDeleteTablet::Complete() SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; -ITransaction* THive::CreateDeleteTablet(TEvHive::TEvDeleteTablet::TPtr& ev) { - return new TTxDeleteTablet(ev, this); -} - - -// TODO: split -class TTxDeleteOwnerTablets : public TTransactionBase<THive> { +class TTxDeleteOwnerTablets : public TTxDeleteBase { TEvHive::TEvDeleteOwnerTablets::TPtr Event; - NKikimrProto::EReplyStatus Status = NKikimrProto::OK; - TVector<ui64> ToDelete; + public: TTxDeleteOwnerTablets(TEvHive::TEvDeleteOwnerTablets::TPtr& ev, THive* hive) - : TBase(hive) + : TTxDeleteBase(hive) , Event(ev) {} - TTabletId DoDeleteTablet(ui64 owner, ui64 idx, NIceDb::TNiceDb& db) { - TTabletId deletedTablet = 0; - - TOwnerIdxType::TValueType ownerIdx(owner, idx); - auto it = Self->OwnerToTablet.find(ownerIdx); - if (it != Self->OwnerToTablet.end()) { - deletedTablet = it->second; - BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << it->second); - TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(it->second); - Y_VERIFY(tablet != nullptr, "%s", (TStringBuilder() << "Tablet " << it->second << " OwnerIdx " << ownerIdx).data()); - if (tablet->SeizedByChild) { - BLOG_W("THive::TTxDeleteTablet tablet " << it->second << " seized by child"); - return 0; - } - if (tablet->State != ETabletState::Deleting) { - tablet->State = ETabletState::Deleting; - tablet->InitiateStop(); - db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State, Schema::Tablet::LeaderNode>(ETabletState::Deleting, 0); - for (TTabletInfo& follower : tablet->Followers) { - follower.InitiateStop(); - db.Table<Schema::TabletFollowerTablet>().Key(follower.GetFullTabletId()).Update<Schema::TabletFollowerTablet::FollowerNode>(0); - } - if (!tablet->InitiateBlockStorage(std::numeric_limits<ui32>::max())) { - Self->DeleteTabletWithoutStorage(tablet); - } - } else { - BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << it->second << " already in ETabletState::Deleting"); - } - } else { - BLOG_W("THive::TTxDeleteTablet tablet " << ownerIdx << " wasn't found"); - Self->PendingCreateTablets.erase({owner, idx}); - } - - return deletedTablet; + void RespondToSender(NKikimrProto::EReplyStatus status) { + const NKikimrHive::TEvDeleteOwnerTablets& rec = Event->Get()->Record; + auto response = MakeHolder<TEvHive::TEvDeleteOwnerTabletsReply>(status, Self->TabletID(), rec.GetOwner(), rec.GetTxId()); + BLOG_D("THive::TTxDeleteOwnerTablets::Execute() result " << response->Record.ShortDebugString()); + SideEffects.Send(Event->Sender, response.Release(), 0, Event->Cookie); } bool Execute(TTransactionContext& txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); const NKikimrHive::TEvDeleteOwnerTablets& rec = Event->Get()->Record; - BLOG_D("THive::TEvDeleteOwnerTablets::Execute Owner: " << rec.GetOwner()); - for (auto item : Self->OwnerToTablet) { - if (item.first.first != rec.GetOwner()) { + BLOG_D("THive::TEvDeleteOwnerTablets::Execute() " << rec.ShortDebugString()); + // resolving owner to tabletids + std::vector<TTabletId> tablets; + ui64 owner = rec.GetOwner(); + for (const auto& [ownerIdx, tabletId] : Self->OwnerToTablet) { + if (ownerIdx.first != owner) { continue; } - const TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(item.second); - if (tablet) { - if (!tablet->IsDeleting()) { - ToDelete.push_back(item.first.second); - } + tablets.push_back(tabletId); + if (Self->PendingCreateTablets.erase(ownerIdx) != 0) { + auto id = ownerIdx; + BLOG_NOTICE("THive::TTxDeleteOwnerTablets tablet " << id << " was cleared from pending creates"); } } - - if (ToDelete.empty()) { - Status = NKikimrProto::ALREADY; - return true; + // checking for possible migration + for (TTabletId tabletId : tablets) { + TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId); + if (tablet != nullptr) { + if (tablet->SeizedByChild) { + BLOG_W("THive::TTxDeleteTablet tablet " << tabletId << " seized by child"); + RespondToSender(NKikimrProto::ERROR); + return true; // abort transaction + } + } } - NIceDb::TNiceDb db(txc.DB); - for (auto idx : ToDelete) { - DoDeleteTablet(rec.GetOwner(), idx, db); + for (TTabletId tabletId : tablets) { + DeleteTablet(tabletId, db); } db.Table<Schema::BlockedOwner>().Key(rec.GetOwner()).Update(); - + Self->BlockedOwners.emplace(Event->Get()->Record.GetOwner()); + RespondToSender(NKikimrProto::OK); return true; } void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TEvDeleteOwnerTablets::Complete(" << Event->Get()->Record.GetOwner() << "), " << ToDelete.size() << " tablet has been deleted"); - Self->BlockedOwners.emplace(Event->Get()->Record.GetOwner()); - ctx.Send(Event->Sender, new TEvHive::TEvDeleteOwnerTabletsReply(Status, Self->TabletID(), Event->Get()->Record.GetOwner(), Event->Get()->Record.GetTxId())); + BLOG_D("THive::TEvDeleteOwnerTablets::Complete(" << Event->Get()->Record.GetOwner() << ") SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; +ITransaction* THive::CreateDeleteTablet(TEvHive::TEvDeleteTablet::TPtr& ev) { + return new TTxDeleteTablet(ev, this); +} + ITransaction* THive::CreateDeleteOwnerTablets(TEvHive::TEvDeleteOwnerTablets::TPtr& ev) { return new TTxDeleteOwnerTablets(ev, this); } - } // NHive } // NKikimr diff --git a/ydb/core/mind/hive/tx__delete_tablet_result.cpp b/ydb/core/mind/hive/tx__delete_tablet_result.cpp index bbe89acae4..3a1bd7df54 100644 --- a/ydb/core/mind/hive/tx__delete_tablet_result.cpp +++ b/ydb/core/mind/hive/tx__delete_tablet_result.cpp @@ -7,9 +7,7 @@ namespace NHive { class TTxDeleteTabletResult : public TTransactionBase<THive> { TEvTabletBase::TEvDeleteTabletResult::TPtr Result; TTabletId TabletId; - TLeaderTabletInfo* Tablet = nullptr; - TVector<TActorId> StorageInfoSubscribers; - TActorId UnlockedFromActor; + TSideEffects SideEffects; public: TTxDeleteTabletResult(TEvTabletBase::TEvDeleteTabletResult::TPtr& ev, THive* hive) @@ -21,59 +19,56 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_DELETE_TABLET_RESULT; } bool Execute(TTransactionContext& txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); TEvTabletBase::TEvDeleteTabletResult* msg = Result->Get(); BLOG_D("THive::TTxDeleteTabletResult::Execute(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")"); - Tablet = Self->FindTabletEvenInDeleting(TabletId); - if (Tablet != nullptr) { + TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(TabletId); + if (tablet != nullptr) { if (msg->Status == NKikimrProto::OK) { NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Metrics>().Key(Tablet->Id, 0).Delete(); - for (const TTabletChannelInfo& channelInfo : Tablet->TabletStorageInfo->Channels) { + db.Table<Schema::Metrics>().Key(tablet->Id, 0).Delete(); + for (const TTabletChannelInfo& channelInfo : tablet->TabletStorageInfo->Channels) { for (const TTabletChannelInfo::THistoryEntry& historyInfo : channelInfo.History) { - db.Table<Schema::TabletChannelGen>().Key(Tablet->Id, channelInfo.Channel, historyInfo.FromGeneration).Delete(); + db.Table<Schema::TabletChannelGen>().Key(tablet->Id, channelInfo.Channel, historyInfo.FromGeneration).Delete(); } - db.Table<Schema::TabletChannel>().Key(Tablet->Id, channelInfo.Channel).Delete(); + db.Table<Schema::TabletChannel>().Key(tablet->Id, channelInfo.Channel).Delete(); } - for (TFollowerTabletInfo& follower : Tablet->Followers) { + for (TFollowerTabletInfo& follower : tablet->Followers) { auto fullTabletId = follower.GetFullTabletId(); db.Table<Schema::TabletFollowerTablet>().Key(fullTabletId).Delete(); db.Table<Schema::Metrics>().Key(fullTabletId).Delete(); } - for (TFollowerGroup& group : Tablet->FollowerGroups) { - db.Table<Schema::TabletFollowerGroup>().Key(Tablet->Id, group.Id).Delete(); + for (TFollowerGroup& group : tablet->FollowerGroups) { + db.Table<Schema::TabletFollowerGroup>().Key(tablet->Id, group.Id).Delete(); } - db.Table<Schema::Tablet>().Key(Tablet->Id).Delete(); - StorageInfoSubscribers.swap(Tablet->StorageInfoSubscribers); - UnlockedFromActor = Tablet->ClearLockedToActor(); - Self->PendingCreateTablets.erase({Tablet->Owner.first, Tablet->Owner.second}); - Self->DeleteTablet(Tablet->Id); + db.Table<Schema::Tablet>().Key(tablet->Id).Delete(); + TVector<TActorId> storageInfoSubscribers; + storageInfoSubscribers.swap(tablet->StorageInfoSubscribers); + for (const TActorId& subscriber : storageInfoSubscribers) { + SideEffects.Send( + subscriber, + new TEvHive::TEvGetTabletStorageInfoResult(TabletId, NKikimrProto::ERROR, "Tablet deleted")); + } + TActorId unlockedFromActor = tablet->ClearLockedToActor(); + if (unlockedFromActor) { + // Notify lock owner that lock has been lost + SideEffects.Send(unlockedFromActor, new TEvHive::TEvLockTabletExecutionLost(TabletId)); + } + Self->PendingCreateTablets.erase({tablet->Owner.first, tablet->Owner.second}); + Self->DeleteTablet(tablet->Id); + } else { + BLOG_W("THive::TTxDeleteTabletResult retrying for " << TabletId << " because of " << NKikimrProto::EReplyStatus_Name(msg->Status)); + Y_ENSURE_LOG(tablet->IsDeleting(), " tablet " << tablet->Id); + SideEffects.Schedule(TDuration::MilliSeconds(1000), new TEvHive::TEvInitiateDeleteStorage(tablet->Id)); } } return true; } void Complete(const TActorContext& ctx) override { - TEvTabletBase::TEvDeleteTabletResult* msg = Result->Get(); - BLOG_D("THive::TTxDeleteTabletResult(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")::Complete"); - Tablet = Self->FindTabletEvenInDeleting(TabletId); - if (Tablet) { - if (msg->Status == NKikimrProto::OK) { - //ctx.Send(Tablet.ActorToNotify, new TEvHive::TEvDeleteTabletReply(NKikimrProto::OK, Self->TabletID(), rec.GetTxId())); - } else { - BLOG_W("THive::TTxDeleteTabletResult retrying for " << TabletId << " because of " << NKikimrProto::EReplyStatus_Name(msg->Status)); - Y_ENSURE_LOG(Tablet->IsDeleting(), " tablet " << Tablet->Id); - ctx.Schedule(TDuration::MilliSeconds(1000), new TEvHive::TEvInitiateDeleteStorage(Tablet->Id)); - } - } - for (const TActorId& subscriber : StorageInfoSubscribers) { - ctx.Send( - subscriber, - new TEvHive::TEvGetTabletStorageInfoResult(TabletId, NKikimrProto::ERROR, "Tablet deleted")); - } - if (UnlockedFromActor) { - // Notify lock owner that lock has been lost - ctx.Send(UnlockedFromActor, new TEvHive::TEvLockTabletExecutionLost(TabletId)); - } + BLOG_D("THive::TTxDeleteTabletResult(" << TabletId << ")::Complete SideEffects " << SideEffects); + SideEffects.Complete(ctx); + } }; diff --git a/ydb/core/mind/hive/tx__kill_node.cpp b/ydb/core/mind/hive/tx__kill_node.cpp index 268327a0dd..9e21d6cddf 100644 --- a/ydb/core/mind/hive/tx__kill_node.cpp +++ b/ydb/core/mind/hive/tx__kill_node.cpp @@ -8,6 +8,7 @@ class TTxKillNode : public TTransactionBase<THive> { protected: TNodeId NodeId; TActorId Local; + TSideEffects SideEffects; public: TTxKillNode(TNodeId nodeId, const TActorId& local, THive *hive) : TBase(hive) @@ -19,6 +20,7 @@ public: bool Execute(TTransactionContext &txc, const TActorContext&) override { BLOG_D("THive::TTxKillNode(" << NodeId << ")::Execute"); + SideEffects.Reset(Self->SelfId()); TInstant now = TInstant::Now(); TNodeInfo* node = Self->FindNode(NodeId); if (node != nullptr) { @@ -43,7 +45,7 @@ public: node->BecomeDisconnected(); for (const TActorId& pipeServer : node->PipeServers) { BLOG_TRACE("THive::TTxKillNode - killing pipe server " << pipeServer); - Self->Send(pipeServer, new TEvents::TEvPoisonPill()); + SideEffects.Send(pipeServer, new TEvents::TEvPoisonPill()); } node->PipeServers.clear(); if (node->CanBeDeleted()) { @@ -56,8 +58,9 @@ public: return true; } - void Complete(const TActorContext&) override { + void Complete(const TActorContext& ctx) override { BLOG_D("THive::TTxKillNode(" << NodeId << ")::Complete"); + SideEffects.Complete(ctx); if (Local) { TNodeInfo* node = Self->FindNode(Local.NodeId()); if (node == nullptr || node->IsDisconnected()) { diff --git a/ydb/core/mind/hive/tx__lock_tablet.cpp b/ydb/core/mind/hive/tx__lock_tablet.cpp index 6365dd0988..e6b782f6d2 100644 --- a/ydb/core/mind/hive/tx__lock_tablet.cpp +++ b/ydb/core/mind/hive/tx__lock_tablet.cpp @@ -14,8 +14,7 @@ private: const TActorId Sender; const ui64 Cookie; - NKikimrProto::EReplyStatus Status; - TString StatusMessage; + TSideEffects SideEffects; TActorId PreviousOwner; public: @@ -34,34 +33,44 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_LOCK_TABLET_EXECUTION; } bool Execute(TTransactionContext& txc, const TActorContext&) override { - BLOG_D("THive::TTxLockTabletExecution::Execute"); + BLOG_D("THive::TTxLockTabletExecution::Execute TabletId: " << TabletId); + + SideEffects.Reset(Self->SelfId()); if (!OwnerActor) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to lock tablet " << TabletId - << " to an invalid owner actor"; + SideEffects.Send(Sender, new TEvHive::TEvLockTabletExecutionResult( + TabletId, + NKikimrProto::ERROR, + TStringBuilder() << "Trying to lock tablet " << TabletId << " to an invalid owner actor" + ), 0, Cookie); return true; } TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); if (tablet == nullptr) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to lock tablet " << TabletId - << ", which doesn't exist"; + SideEffects.Send(Sender, new TEvHive::TEvLockTabletExecutionResult( + TabletId, + NKikimrProto::ERROR, + TStringBuilder() << "Trying to lock tablet " << TabletId << ", which doesn't exist" + ), 0, Cookie); return true; } if (OwnerActor.NodeId() != Sender.NodeId()) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to lock tablet " << TabletId - << " to " << OwnerActor << ", which is on a different node"; + SideEffects.Send(Sender, new TEvHive::TEvLockTabletExecutionResult( + TabletId, + NKikimrProto::ERROR, + TStringBuilder() << "Trying to lock tablet " << TabletId << " to " << OwnerActor << ", which is on a different node" + ), 0, Cookie); return true; } if (IsReconnect && tablet->LockedToActor != OwnerActor) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to restore lock to tablet " << TabletId - << ", which has expired"; + SideEffects.Send(Sender, new TEvHive::TEvLockTabletExecutionResult( + TabletId, + NKikimrProto::ERROR, + TStringBuilder() << "Trying to restore lock to tablet " << TabletId << ", which has expired" + ), 0, Cookie); return true; } @@ -74,38 +83,31 @@ public: NIceDb::TUpdate<Schema::Tablet::LockedToActor>(tablet->LockedToActor), NIceDb::TUpdate<Schema::Tablet::LockedReconnectTimeout>(tablet->LockedReconnectTimeout.MilliSeconds())); - Status = NKikimrProto::OK; - return true; - } - - void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxLockTabletExecution::Complete TabletId: " << TabletId - << " Owner: " << OwnerActor << " Status: " << Status << " " << StatusMessage); - ui32 flags = 0; - if (Status == NKikimrProto::OK) { - if (PreviousOwner && PreviousOwner != OwnerActor) { - // Notify previous owner that its lock ownership has been lost - ctx.Send(PreviousOwner, new TEvHive::TEvLockTabletExecutionLost(TabletId)); - } + if (PreviousOwner && PreviousOwner != OwnerActor) { + // Notify previous owner that its lock ownership has been lost + SideEffects.Send(PreviousOwner, new TEvHive::TEvLockTabletExecutionLost(TabletId)); + } - if (TLeaderTabletInfo* tablet = Self->FindTablet(TabletId)) { - // Tablet still exists by the time transaction finished - if (tablet->IsLockedToActor()) { - // Make sure running tablets will be stopped - for (auto& follower : tablet->Followers) { - follower.InitiateStop(); - } - tablet->InitiateStop(); - } - if (tablet->LockedToActor == OwnerActor && tablet->PendingUnlockSeqNo == 0) { - // Lock is still valid, watch for node disconnections - flags |= IEventHandle::FlagSubscribeOnSession; - } + if (tablet->IsLockedToActor()) { + // Make sure running tablets will be stopped + for (auto& follower : tablet->Followers) { + follower.InitiateStop(SideEffects); } + tablet->InitiateStop(SideEffects); + } + if (tablet->LockedToActor == OwnerActor && tablet->PendingUnlockSeqNo == 0) { + // Lock is still valid, watch for node disconnections + flags |= IEventHandle::FlagSubscribeOnSession; } - ctx.Send(Sender, new TEvHive::TEvLockTabletExecutionResult(TabletId, Status, StatusMessage), flags, Cookie); + SideEffects.Send(Sender, new TEvHive::TEvLockTabletExecutionResult(TabletId, NKikimrProto::OK, {}), flags, Cookie); + return true; + } + + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxLockTabletExecution::Complete TabletId: " << TabletId << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } private: diff --git a/ydb/core/mind/hive/tx__process_boot_queue.cpp b/ydb/core/mind/hive/tx__process_boot_queue.cpp index 8ce6459862..0ca8a6d23a 100644 --- a/ydb/core/mind/hive/tx__process_boot_queue.cpp +++ b/ydb/core/mind/hive/tx__process_boot_queue.cpp @@ -5,6 +5,8 @@ namespace NKikimr { namespace NHive { class TTxProcessBootQueue : public TTransactionBase<THive> { + TSideEffects SideEffects; + public: TTxProcessBootQueue(THive *hive) : TBase(hive) @@ -14,12 +16,14 @@ public: bool Execute(TTransactionContext&, const TActorContext&) override { BLOG_D("THive::TTxProcessBootQueue()::Execute"); - Self->RunProcessBootQueue(); + SideEffects.Reset(Self->SelfId()); + Self->ExecuteProcessBootQueue(SideEffects); return true; } - void Complete(const TActorContext&) override { + void Complete(const TActorContext& ctx) override { BLOG_D("THive::TTxProcessBootQueue()::Complete"); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__process_pending_operations.cpp b/ydb/core/mind/hive/tx__process_pending_operations.cpp index afaf7203d4..e817b450f5 100644 --- a/ydb/core/mind/hive/tx__process_pending_operations.cpp +++ b/ydb/core/mind/hive/tx__process_pending_operations.cpp @@ -5,9 +5,6 @@ namespace NKikimr { namespace NHive { class TTxProcessPendingOperations : public TTransactionBase<THive> { -protected: - std::deque<THolder<IEventHandle>> Events; - public: TTxProcessPendingOperations(THive *hive) : TBase(hive) @@ -20,17 +17,18 @@ public: for (auto& [owner, pendingCreateTablet] : Self->PendingCreateTablets) { THolder<TEvHive::TEvCreateTablet> evCreateTablet(new TEvHive::TEvCreateTablet()); evCreateTablet->Record = pendingCreateTablet.CreateTablet; - Events.emplace_back(new IEventHandle(Self->SelfId(), pendingCreateTablet.Sender, evCreateTablet.Release(), 0, pendingCreateTablet.Cookie)); + BLOG_D("THive::TTxProcessPendingOperations(): retry CreateTablet"); + TlsActivationContext->Send(new IEventHandle(Self->SelfId(), pendingCreateTablet.Sender, evCreateTablet.Release(), 0, pendingCreateTablet.Cookie)); + } + for (auto& handle : Self->PendingOperations) { + TlsActivationContext->Send(handle); } + Self->PendingOperations.clear(); return true; } void Complete(const TActorContext&) override { BLOG_D("THive::TTxProcessPendingOperations()::Complete"); - for (THolder<IEventHandle>& event : Events) { - BLOG_D("THive::TTxProcessPendingOperations(): retry event " << event->Type); - TlsActivationContext->Send(event.Release()); - } } }; diff --git a/ydb/core/mind/hive/tx__reassign_groups.cpp b/ydb/core/mind/hive/tx__reassign_groups.cpp index ec9f4bfdd2..6c59f39b74 100644 --- a/ydb/core/mind/hive/tx__reassign_groups.cpp +++ b/ydb/core/mind/hive/tx__reassign_groups.cpp @@ -9,7 +9,6 @@ protected: TTabletId TabletId; TActorId Sender; std::bitset<MAX_TABLET_CHANNELS> ChannelProfileNewGroup; - bool InitiatedReassignTablet = false; public: TTxReassignGroups(TTabletId tabletId, @@ -45,7 +44,7 @@ public: db.Table<Schema::TabletChannel>().Key(TabletId, channelId).Update(NIceDb::TUpdate<Schema::TabletChannel::NeedNewGroup>(true)); } } - InitiatedReassignTablet = true; + tablet->InitiateAssignTabletGroups(); } else { BLOG_W("THive::TTxReassignGroups(" << tablet->Id << ")::Execute - tablet is not ready for group reassignment"); } @@ -55,16 +54,6 @@ public: void Complete(const TActorContext&) override { BLOG_D("THive::TTxReassignGroups(" << TabletId << ")::Complete"); - if (InitiatedReassignTablet) { - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - if (tablet->IsReadyToAssignGroups()) { - tablet->InitiateAssignTabletGroups(); - } else { - BLOG_W("THive::TTxReassignGroups(" << tablet->Id << ")::Complete - tablet is not ready for group reassignment"); - } - } - } } }; diff --git a/ydb/core/mind/hive/tx__release_tablets.cpp b/ydb/core/mind/hive/tx__release_tablets.cpp index cdfd82ce06..2cf1916d4d 100644 --- a/ydb/core/mind/hive/tx__release_tablets.cpp +++ b/ydb/core/mind/hive/tx__release_tablets.cpp @@ -9,6 +9,7 @@ class TTxReleaseTablets : public TTransactionBase<THive> { THolder<TEvHive::TEvReleaseTabletsReply> Response = MakeHolder<TEvHive::TEvReleaseTabletsReply>(); TVector<std::pair<TTabletId, TActorId>> UnlockedFromActor; bool NeedToProcessPendingOperations = false; + TSideEffects SideEffects; public: TTxReleaseTablets(THolder<TEvHive::TEvReleaseTablets::THandle> event, THive *hive) @@ -21,6 +22,7 @@ public: bool Execute(TTransactionContext& txc, const TActorContext&) override { const NKikimrHive::TEvReleaseTablets& request(Request->Get()->Record); NKikimrHive::TEvReleaseTabletsReply& response(Response->Record); + SideEffects.Reset(Self->SelfId()); BLOG_D("THive::TTxReleaseTablets::Execute " << request); NIceDb::TNiceDb db(txc.DB); for (TTabletId tabletId : request.GetTabletIDs()) { @@ -29,10 +31,10 @@ public: Y_VERIFY(tablet->SeizedByChild); if (tablet->IsAlive() && tablet->Node != nullptr) { - tablet->SendStopTablet(tablet->Node->Local, tablet->GetFullTabletId()); + tablet->SendStopTablet(tablet->Node->Local, SideEffects); for (TFollowerTabletInfo& follower : tablet->Followers) { if (follower.IsAlive() && follower.Node != nullptr) { - follower.SendStopTablet(follower.Node->Local, follower.GetFullTabletId()); + follower.SendStopTablet(follower.Node->Local, SideEffects); } } } @@ -76,7 +78,8 @@ public: } void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxReleaseTablets::Complete " << Request->Get()->Record); + BLOG_D("THive::TTxReleaseTablets::Complete " << Request->Get()->Record << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); for (const auto& unlockedFromActor : UnlockedFromActor) { // Notify lock owner that lock has been lost ctx.Send(unlockedFromActor.second, new TEvHive::TEvLockTabletExecutionLost(unlockedFromActor.first)); diff --git a/ydb/core/mind/hive/tx__restart_tablet.cpp b/ydb/core/mind/hive/tx__restart_tablet.cpp index 3c070ed706..730b793602 100644 --- a/ydb/core/mind/hive/tx__restart_tablet.cpp +++ b/ydb/core/mind/hive/tx__restart_tablet.cpp @@ -8,6 +8,7 @@ class TTxRestartTablet : public TTransactionBase<THive> { protected: TFullTabletId TabletId; TNodeId PreferredNodeId; + TSideEffects SideEffects; public: TTxRestartTablet(TFullTabletId tabletId, THive *hive) : TBase(hive) @@ -24,9 +25,10 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_RESTART_TABLET; } bool Execute(TTransactionContext &txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); TTabletInfo* tablet = Self->FindTablet(TabletId); if (tablet != nullptr) { - if (PreferredNodeId != 0) { + if (PreferredNodeId == 0) { BLOG_D("THive::TTxRestartTablet(" << tablet->ToString() << ")::Execute"); } else { BLOG_D("THive::TTxRestartTablet(" << tablet->ToString() << " to node " << PreferredNodeId << ")::Execute"); @@ -40,7 +42,7 @@ public: db.Table<Schema::TabletFollowerTablet>().Key(tablet->GetFullTabletId()).Update<Schema::TabletFollowerTablet::FollowerNode>(0); } } - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); } tablet->PreferredNodeId = PreferredNodeId; tablet->GetLeader().TryToBoot(); @@ -48,8 +50,9 @@ public: return true; } - void Complete(const TActorContext&) override { - BLOG_D("THive::TTxRestartTablet(" << TabletId << ")::Complete"); + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxRestartTablet(" << TabletId << ")::Complete SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__resume_tablet.cpp b/ydb/core/mind/hive/tx__resume_tablet.cpp index a7252bff72..f034f0a5bd 100644 --- a/ydb/core/mind/hive/tx__resume_tablet.cpp +++ b/ydb/core/mind/hive/tx__resume_tablet.cpp @@ -7,31 +7,32 @@ namespace NHive { class TTxResumeTablet : public TTransactionBase<THive> { const TTabletId TabletId; const TActorId ActorToNotify; - NKikimrProto::EReplyStatus Status; + TSideEffects SideEffects; public: TTxResumeTablet(ui64 tabletId, const TActorId &actorToNotify, THive *hive) : TBase(hive) , TabletId(tabletId) , ActorToNotify(actorToNotify) - , Status(NKikimrProto::UNKNOWN) {} TTxType GetTxType() const override { return NHive::TXTYPE_RESUME_TABLET; } bool Execute(TTransactionContext &txc, const TActorContext&) override { BLOG_D("THive::TTxResumeTablet::Execute Tablet: " << TabletId); + SideEffects.Reset(Self->SelfId()); TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); if (tablet != nullptr) { + NKikimrProto::EReplyStatus status = NKikimrProto::UNKNOWN; ETabletState State = tablet->State; ETabletState NewState = State; NIceDb::TNiceDb db(txc.DB); switch (State) { case ETabletState::GroupAssignment: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; case ETabletState::ReadyToWork: - Status = NKikimrProto::ALREADY; + status = NKikimrProto::ALREADY; break; case ETabletState::Stopped: // Switch to ReadyToWork @@ -40,44 +41,42 @@ public: } else { NewState = ETabletState::ReadyToWork; } - Status = NKikimrProto::OK; + status = NKikimrProto::OK; break; case ETabletState::Deleting: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; case ETabletState::BlockStorage: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; case ETabletState::Stopping: case ETabletState::StoppingInGroupAssignment: case ETabletState::Unknown: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; } - if (Status == NKikimrProto::OK && NewState != State) { - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(NewState); - tablet->State = NewState; + if (status == NKikimrProto::OK) { + if (NewState != State) { + db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(NewState); + tablet->State = NewState; + } + if (tablet->IsReadyToBoot()) { + tablet->InitiateBoot(); + } else if (tablet->IsReadyToAssignGroups()) { + tablet->InitiateAssignTabletGroups(); + } + } + if (status != NKikimrProto::UNKNOWN) { + SideEffects.Send(ActorToNotify, new TEvHive::TEvResumeTabletResult(status, TabletId), 0, 0); } + Self->ProcessBootQueue(); } return true; } void Complete(const TActorContext& ctx) override { BLOG_D("THive::TTxResumeTablet::Complete TabletId: " << TabletId); - if (Status != NKikimrProto::UNKNOWN) { - ctx.Send(ActorToNotify, new TEvHive::TEvResumeTabletResult(Status, TabletId), 0, 0); - if (Status == NKikimrProto::OK) { - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - if (tablet->IsReadyToBoot()) { - tablet->InitiateBoot(); - } else if (tablet->IsReadyToAssignGroups()) { - tablet->InitiateAssignTabletGroups(); - } - } - } - } - Self->ProcessBootQueue(); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__start_tablet.cpp b/ydb/core/mind/hive/tx__start_tablet.cpp index 4cef3a0e06..12a7ec6a70 100644 --- a/ydb/core/mind/hive/tx__start_tablet.cpp +++ b/ydb/core/mind/hive/tx__start_tablet.cpp @@ -9,7 +9,7 @@ class TTxStartTablet : public TTransactionBase<THive> { TActorId Local; ui64 Cookie; bool External; - ui32 KnownGeneration = -1; + TSideEffects SideEffects; public: TTxStartTablet(TFullTabletId tabletId, const TActorId& local, ui64 cookie, bool external, THive *hive) @@ -23,86 +23,83 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_START_TABLET; } bool Execute(TTransactionContext& txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); BLOG_D("THive::TTxStartTablet::Execute Tablet " << TabletId); TTabletInfo* tablet = Self->FindTablet(TabletId); if (tablet != nullptr) { + // finish fast-move operation + if (tablet->LastNodeId != 0 && tablet->LastNodeId != Local.NodeId()) { + TNodeInfo* lastNode = Self->FindNode(tablet->LastNodeId); + if (lastNode != nullptr && lastNode->Local) { + tablet->SendStopTablet(lastNode->Local, SideEffects); + } + tablet->LastNodeId = 0; + } + // increase generation if (tablet->IsLeader()) { TLeaderTabletInfo& leader = tablet->AsLeader(); if (leader.IsStarting() || leader.IsBootingSuppressed() && External) { NIceDb::TNiceDb db(txc.DB); leader.IncreaseGeneration(); - KnownGeneration = leader.KnownGeneration; db.Table<Schema::Tablet>().Key(leader.Id).Update<Schema::Tablet::KnownGeneration>(leader.KnownGeneration); } else { - BLOG_W("THive::TTxStartTablet::Execute Tablet " << TabletId << " skipped generation increment"); + BLOG_W("THive::TTxStartTablet::Execute Tablet " << leader.ToString() << " (" << leader.StateString() << ") skipped generation increment " << (ui64)leader.State); } } - } else { - BLOG_W("THive::TTxStartTablet::Execute Tablet " << TabletId << " wasn't found"); - } - return true; - } - - void Complete(const TActorContext& ctx) override { - TTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - if (tablet->LastNodeId != 0 && tablet->LastNodeId != Local.NodeId()) { - TNodeInfo* lastNode = Self->FindNode(tablet->LastNodeId); - if (lastNode != nullptr && lastNode->Local) { - Self->StopTablet(lastNode->Local, tablet->GetFullTabletId()); - } - tablet->LastNodeId = 0; - } if (tablet->IsLeader()) { TLeaderTabletInfo& leader = tablet->AsLeader(); if (leader.IsStartingOnNode(Local.NodeId()) || leader.IsBootingSuppressed() && External) { - if (KnownGeneration == leader.KnownGeneration) { - BLOG_D("THive::TTxStartTablet::Complete, Sending TEvBootTablet(" << leader.ToString() << ")" - << " to node " << Local.NodeId() - << " storage " << leader.TabletStorageInfo->ToString()); - TFollowerId promotableFollowerId = leader.GetFollowerPromotableOnNode(Local.NodeId()); - ctx.Send(Local, - new TEvLocal::TEvBootTablet(*leader.TabletStorageInfo, promotableFollowerId, KnownGeneration), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - Cookie); - return; - } else { - BLOG_W("THive::TTxStartTablet::Complete, ignoring outstanding TEvBootTablet(" << leader.ToString() << ") - wrong generation"); - } + BLOG_D("THive::TTxStartTablet::Execute, Sending TEvBootTablet(" << leader.ToString() << ")" + << " to node " << Local.NodeId() + << " storage " << leader.TabletStorageInfo->ToString()); + TFollowerId promotableFollowerId = leader.GetFollowerPromotableOnNode(Local.NodeId()); + SideEffects.Send(Local, + new TEvLocal::TEvBootTablet(*leader.TabletStorageInfo, promotableFollowerId, leader.KnownGeneration), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + Cookie); + return true; } else { - BLOG_W("THive::TTxStartTablet::Complete, ignoring outstanding TEvBootTablet(" << leader.ToString() << ") - wrong state or node"); + BLOG_W("THive::TTxStartTablet::Execute, ignoring TEvBootTablet(" << leader.ToString() << ") - wrong state or node"); } } else { TFollowerTabletInfo& follower = tablet->AsFollower(); if (follower.IsStartingOnNode(Local.NodeId())) { - BLOG_D("THive::TTxStartTablet::Complete, Sending TEvBootTablet(" << follower.ToString() << ")" + BLOG_D("THive::TTxStartTablet::Execute, Sending TEvBootTablet(" << follower.ToString() << ")" << " to node " << Local.NodeId() << " storage " << follower.LeaderTablet.TabletStorageInfo->ToString()); - ctx.Send(Local, + SideEffects.Send(Local, new TEvLocal::TEvBootTablet(*follower.LeaderTablet.TabletStorageInfo, follower.Id), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, Cookie); - return; + return true; } else { - BLOG_W("THive::TTxStartTablet::Complete, ignoring outstanding TEvBootTablet(" << follower.ToString() << ") - wrong state or node"); + BLOG_W("THive::TTxStartTablet::Execute, ignoring TEvBootTablet(" << follower.ToString() << ") - wrong state or node"); } } // if anything wrong - attempt to restart the tablet - if (tablet->InitiateStop()) { + if (tablet->InitiateStop(SideEffects)) { if (tablet->IsLeader()) { - BLOG_NOTICE("THive::TTxStartTablet::Complete, jump-starting tablet " << tablet->ToString()); + BLOG_NOTICE("THive::TTxStartTablet::Execute, jump-starting tablet " << tablet->ToString()); tablet->AsLeader().TryToBoot(); } } + } else { + BLOG_W("THive::TTxStartTablet::Execute Tablet " << TabletId << " wasn't found"); } if (External) { // Always send some reply for external start requests - BLOG_W("THive::TTxStartTablet::Complete, Aborting external boot of " << TabletId.first << "." << TabletId.second); - ctx.Send(Local, + BLOG_W("THive::TTxStartTablet::Execute, Aborting external boot of " << TabletId.first << "." << TabletId.second); + SideEffects.Send(Local, new TEvHive::TEvBootTabletReply(NKikimrProto::EReplyStatus::ERROR), 0, Cookie); } + return true; + } + + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxStartTablet::Complete Tablet " << TabletId << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__status.cpp b/ydb/core/mind/hive/tx__status.cpp index 361b9b080e..ae940878a7 100644 --- a/ydb/core/mind/hive/tx__status.cpp +++ b/ydb/core/mind/hive/tx__status.cpp @@ -36,6 +36,10 @@ public: Self->UpdateRegisteredDataCenters(node.Location.GetDataCenterId()); } Self->ProcessWaitQueue(); // new node connected + if (node.Drain && Self->BalancerNodes.count(nodeId) == 0) { + BLOG_D("THive::TTxStatus(" << nodeId << ")::Complete - continuing node drain"); + Self->StartHiveDrain(nodeId, {.Persist = true, .KeepDown = node.Down}); + } } else { BLOG_W("THive::TTxStatus(status=" << static_cast<int>(status) << " node=" << TNodeInfo::EVolatileStateName(node.GetVolatileState()) << ") - killing node " << node.Id); @@ -47,13 +51,6 @@ public: void Complete(const TActorContext&) override { TNodeId nodeId = Local.NodeId(); BLOG_D("THive::TTxStatus(" << nodeId << ")::Complete"); - TNodeInfo* node = Self->FindNode(nodeId); - if (node != nullptr && node->IsAlive()) { - if (node->Drain && Self->BalancerNodes.count(nodeId) == 0) { - BLOG_D("THive::TTxStatus(" << nodeId << ")::Complete - continuing node drain"); - Self->StartHiveDrain(nodeId, {.Persist = true, .KeepDown = node->Down}); - } - } } }; diff --git a/ydb/core/mind/hive/tx__stop_tablet.cpp b/ydb/core/mind/hive/tx__stop_tablet.cpp index c05103b258..0b1ee994cf 100644 --- a/ydb/core/mind/hive/tx__stop_tablet.cpp +++ b/ydb/core/mind/hive/tx__stop_tablet.cpp @@ -5,85 +5,84 @@ namespace NKikimr { namespace NHive { class TTxStopTablet : public TTransactionBase<THive> { - const TTabletId TabletId; - const TActorId ActorToNotify; - NKikimrProto::EReplyStatus Status; + TTabletId TabletId; + TActorId ActorToNotify; + TSideEffects SideEffects; public: TTxStopTablet(ui64 tabletId, const TActorId &actorToNotify, THive *hive) : TBase(hive) , TabletId(tabletId) , ActorToNotify(actorToNotify) - , Status(NKikimrProto::UNKNOWN) {} TTxType GetTxType() const override { return NHive::TXTYPE_STOP_TABLET; } bool Execute(TTransactionContext &txc, const TActorContext&) override { BLOG_D("THive::TTxStopTablet::Execute Tablet: " << TabletId); + SideEffects.Reset(Self->SelfId()); + NKikimrProto::EReplyStatus status = NKikimrProto::UNKNOWN; TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); if (tablet != nullptr) { - ETabletState State = tablet->State; - ETabletState NewState = State; + ETabletState state = tablet->State; + ETabletState newState = state; NIceDb::TNiceDb db(txc.DB); - switch (State) { + switch (state) { case ETabletState::GroupAssignment: // switch to StoppingInGroupAssignment - NewState = ETabletState::StoppingInGroupAssignment; - Status = NKikimrProto::OK; + newState = ETabletState::StoppingInGroupAssignment; + status = NKikimrProto::OK; // TODO: Notify of previous request failure // TODO: Set new notification receiver break; case ETabletState::Stopped: // notify with OK - Status = NKikimrProto::ALREADY; + status = NKikimrProto::ALREADY; break; case ETabletState::ReadyToWork: // Switch to Stopping - NewState = ETabletState::Stopped; + newState = ETabletState::Stopped; for (TTabletInfo& follower : tablet->Followers) { if (follower.IsAlive()) { - follower.InitiateStop(); + follower.InitiateStop(SideEffects); db.Table<Schema::TabletFollowerTablet>().Key(follower.GetFullTabletId()).Update<Schema::TabletFollowerTablet::FollowerNode>(0); } } if (tablet->IsAlive()) { - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::LeaderNode>(0); } - Status = NKikimrProto::OK; + status = NKikimrProto::OK; break; case ETabletState::Deleting: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; case ETabletState::BlockStorage: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; case ETabletState::Stopping: case ETabletState::StoppingInGroupAssignment: case ETabletState::Unknown: - Status = NKikimrProto::ERROR; + status = NKikimrProto::ERROR; break; } - if (Status == NKikimrProto::OK && NewState != State) { - db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(NewState); - tablet->State = NewState; + if (status == NKikimrProto::OK && newState != state) { + db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::State>(newState); + tablet->State = newState; } + if (status != NKikimrProto::UNKNOWN) { + SideEffects.Send(ActorToNotify, new TEvHive::TEvStopTabletResult(status, TabletId), 0, 0); + Self->ReportStoppedToWhiteboard(*tablet); + BLOG_D("Report tablet " << tablet->ToString() << " as stopped to Whiteboard"); + } + Self->ProcessBootQueue(); } return true; } void Complete(const TActorContext& ctx) override { BLOG_D("THive::TTxStopTablet::Complete TabletId: " << TabletId); - if (Status != NKikimrProto::UNKNOWN) { - ctx.Send(ActorToNotify, new TEvHive::TEvStopTabletResult(Status, TabletId), 0, 0); - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - Self->ReportStoppedToWhiteboard(*tablet); - BLOG_D("Report tablet " << tablet->ToString() << " as stopped to Whiteboard"); - } - } - Self->ProcessBootQueue(); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__sync_tablets.cpp b/ydb/core/mind/hive/tx__sync_tablets.cpp index 29eb4cfba1..33c84e313f 100644 --- a/ydb/core/mind/hive/tx__sync_tablets.cpp +++ b/ydb/core/mind/hive/tx__sync_tablets.cpp @@ -7,8 +7,8 @@ namespace NHive { class TTxSyncTablets : public TTransactionBase<THive> { TActorId Local; NKikimrLocal::TEvSyncTablets SyncTablets; - THashSet<std::pair<TTabletId, TFollowerId>> TabletsToStop; - THashSet<std::pair<TTabletId, TFollowerId>> TabletsToBoot; + TSideEffects SideEffects; + public: TTxSyncTablets(const TActorId &local, NKikimrLocal::TEvSyncTablets& rec, THive* hive) : TBase(hive) @@ -37,15 +37,16 @@ public: return false; } - bool Execute(TTransactionContext &txc, const TActorContext&) override { + bool Execute(TTransactionContext &txc, const TActorContext& ctx) override { BLOG_D("THive::TTxSyncTablets(" << Local << ")::Execute"); + SideEffects.Reset(Self->SelfId()); NIceDb::TNiceDb db(txc.DB); TNodeInfo& node = Self->GetNode(Local.NodeId()); - TabletsToStop.clear(); - TabletsToBoot.clear(); + THashSet<std::pair<TTabletId, TFollowerId>> tabletsToStop; + THashSet<std::pair<TTabletId, TFollowerId>> tabletsToBoot; for (const auto& t : node.Tablets) { for (TTabletInfo* tablet : t.second) { - TabletsToStop.insert(tablet->GetFullTabletId()); + tabletsToStop.insert(tablet->GetFullTabletId()); } } for (const NKikimrLocal::TEvSyncTablets_TTabletInfo& ti : SyncTablets.GetInbootTablets()) { @@ -58,16 +59,16 @@ public: } tablet->BecomeStarting(node.Id); BLOG_TRACE("THive::TTxSyncTablets(" << Local << ") confirmed starting tablet " << tabletId); - TabletsToStop.erase(tabletId); + tabletsToStop.erase(tabletId); if (tablet->GetLeader().IsBootingSuppressed()) { - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); } continue; } } else { - Self->StopTablet(Local, tabletId); + SideEffects.Send(Local, new TEvLocal::TEvStopTablet(tabletId)); BLOG_TRACE("THive::TTxSyncTablets(" << Local << ") rejected unknown starting tablet " << tabletId); - TabletsToStop.erase(tabletId); + tabletsToStop.erase(tabletId); } } for (const NKikimrLocal::TEvSyncTablets_TTabletInfo& ti : SyncTablets.GetOnlineTablets()) { @@ -82,7 +83,7 @@ public: if (tablet->IsLeader()) { db.Table<Schema::Tablet>().Key(tablet->GetLeader().Id).Update(NIceDb::TUpdate<Schema::Tablet::LeaderNode>(tablet->NodeId), NIceDb::TUpdate<Schema::Tablet::KnownGeneration>(tablet->GetLeader().KnownGeneration)); - TabletsToBoot.insert(tabletId); + tabletsToBoot.insert(tabletId); } else { db.Table<Schema::TabletFollowerTablet>().Key(tablet->GetFullTabletId()).Update( NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(tablet->AsFollower().FollowerGroup.Id), @@ -90,39 +91,40 @@ public: } } BLOG_TRACE("THive::TTxSyncTablets(" << Local << ") confirmed running tablet " << tabletId); - TabletsToStop.erase(tabletId); + tabletsToStop.erase(tabletId); if (tablet->GetLeader().IsBootingSuppressed()) { - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); } continue; } else if (ti.GetBootMode() == NKikimrLocal::EBootMode::BOOT_MODE_FOLLOWER) { - Self->StopTablet(Local, tabletId); // the tablet is running somewhere else + SideEffects.Send(Local, new TEvLocal::TEvStopTablet(tabletId)); // the tablet is running somewhere else BLOG_TRACE("THive::TTxSyncTablets(" << Local << ") confirmed and stopped running tablet " << tabletId); - TabletsToBoot.insert(tabletId); - TabletsToStop.erase(tabletId); + tabletsToBoot.insert(tabletId); + tabletsToStop.erase(tabletId); continue; } } else { - Self->StopTablet(Local, tabletId); + SideEffects.Send(Local, new TEvLocal::TEvStopTablet(tabletId)); BLOG_TRACE("THive::TTxSyncTablets(" << Local << ") rejected unknown running tablet " << tabletId); - TabletsToStop.erase(tabletId); + tabletsToStop.erase(tabletId); } } - return true; - } - - void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxSyncTablets(" << Local << ")::Complete"); - for (std::pair<TTabletId, TFollowerId> tabletId : TabletsToStop) { + for (std::pair<TTabletId, TFollowerId> tabletId : tabletsToStop) { Self->Execute(Self->CreateRestartTablet(tabletId), ctx); } - for (std::pair<TTabletId, TFollowerId> tabletId : TabletsToBoot) { + for (std::pair<TTabletId, TFollowerId> tabletId : tabletsToBoot) { TTabletInfo* tablet = Self->FindTablet(tabletId.first, tabletId.second); if (tablet != nullptr) { tablet->GetLeader().TryToBoot(); // for followers } } Self->ProcessBootQueue(); + return true; + } + + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxSyncTablets(" << Local << ")::Complete"); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__unlock_tablet.cpp b/ydb/core/mind/hive/tx__unlock_tablet.cpp index 66786b43b2..cf3f7a54bd 100644 --- a/ydb/core/mind/hive/tx__unlock_tablet.cpp +++ b/ydb/core/mind/hive/tx__unlock_tablet.cpp @@ -12,8 +12,7 @@ class TTxUnlockTabletExecution : public TTransactionBase<THive> { const TActorId Sender; const ui64 Cookie; - NKikimrProto::EReplyStatus Status; - TString StatusMessage; + TSideEffects SideEffects; TActorId PreviousOwner; public: @@ -38,27 +37,25 @@ public: {} bool Execute(TTransactionContext& txc, const TActorContext&) override { - BLOG_D("THive::TTxUnlockTabletExecution::Execute"); - + BLOG_D("THive::TTxUnlockTabletExecution::Execute TabletId: " << TabletId); + SideEffects.Reset(Self->SelfId()); TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(TabletId); if (tablet == nullptr) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to unlock tablet " << TabletId - << ", which doesn't exist"; + SideEffects.Send(Sender, new TEvHive::TEvUnlockTabletExecutionResult(TabletId, NKikimrProto::ERROR, + TStringBuilder() << "Trying to unlock tablet " << TabletId << ", which doesn't exist"), 0, Cookie); return true; } if (OwnerActor && tablet->LockedToActor != OwnerActor) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to unlock tablet " << TabletId - << ", which is locked to " << tablet->LockedToActor << ", not " << OwnerActor; + SideEffects.Send(Sender, new TEvHive::TEvUnlockTabletExecutionResult(TabletId, NKikimrProto::ERROR, + TStringBuilder() << "Trying to unlock tablet " << TabletId + << ", which is locked to " << tablet->LockedToActor << ", not " << OwnerActor), 0, Cookie); return true; } if (SeqNo && tablet->PendingUnlockSeqNo != SeqNo) { - Status = NKikimrProto::ERROR; - StatusMessage = TStringBuilder() << "Trying to unlock tablet " << TabletId - << ", which is out of sequence"; + SideEffects.Send(Sender, new TEvHive::TEvUnlockTabletExecutionResult(TabletId, NKikimrProto::ERROR, + TStringBuilder() << "Trying to unlock tablet " << TabletId << ", which is out of sequence"), 0, Cookie); return true; } @@ -71,32 +68,22 @@ public: NIceDb::TUpdate<Schema::Tablet::LockedToActor>(tablet->LockedToActor), NIceDb::TUpdate<Schema::Tablet::LockedReconnectTimeout>(tablet->LockedReconnectTimeout.MilliSeconds())); - Status = NKikimrProto::OK; + if (PreviousOwner) { + // Notify previous owner that its lock ownership has been lost + SideEffects.Send(PreviousOwner, new TEvHive::TEvLockTabletExecutionLost(TabletId)); + } + + if (!tablet->IsLockedToActor()) { + // Try to boot it if possible + tablet->TryToBoot(); + } + SideEffects.Send(Sender, new TEvHive::TEvUnlockTabletExecutionResult(TabletId, NKikimrProto::OK, {}), 0, Cookie); return true; } void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxUnlockTabletExecution::Complete TabletId: " << TabletId - << " Status: " << Status << " " << StatusMessage); - - if (Status == NKikimrProto::OK) { - if (PreviousOwner) { - // Notify previous owner that its lock ownership has been lost - ctx.Send(PreviousOwner, new TEvHive::TEvLockTabletExecutionLost(TabletId)); - } - - if (TLeaderTabletInfo* tablet = Self->FindTablet(TabletId)) { - // Tablet still exists by the time transaction finished - if (!tablet->IsLockedToActor()) { - // Try to boot it if possible - tablet->TryToBoot(); - } - } - } - - if (Sender) { - ctx.Send(Sender, new TEvHive::TEvUnlockTabletExecutionResult(TabletId, Status, StatusMessage), 0, Cookie); - } + BLOG_D("THive::TTxUnlockTabletExecution::Complete TabletId: " << TabletId << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } private: diff --git a/ydb/core/mind/hive/tx__update_tablet_groups.cpp b/ydb/core/mind/hive/tx__update_tablet_groups.cpp index 2d9a11bc3b..1fec5d8e90 100644 --- a/ydb/core/mind/hive/tx__update_tablet_groups.cpp +++ b/ydb/core/mind/hive/tx__update_tablet_groups.cpp @@ -12,11 +12,7 @@ namespace NHive { class TTxUpdateTabletGroups : public TTransactionBase<THive> { TTabletId TabletId; TVector<NKikimrBlobStorage::TEvControllerSelectGroupsResult::TGroupParameters> Groups; - ETabletState NewTabletState = ETabletState::GroupAssignment; - bool NeedToBlockStorage = false; - bool Changed = false; - bool Ignored = false; - TCompleteNotifications Notifications; + TSideEffects SideEffects; public: TTxUpdateTabletGroups(TTabletId tabletId, TVector<NKikimrBlobStorage::TEvControllerSelectGroupsResult::TGroupParameters> groups, THive *hive) @@ -28,24 +24,33 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_UPDATE_TABLET_GROUPS; } bool Execute(TTransactionContext &txc, const TActorContext& ctx) override { + SideEffects.Reset(Self->SelfId()); + + ETabletState newTabletState = ETabletState::GroupAssignment; + bool needToBlockStorage = false; + bool changed = false; TStringBuilder tabletBootState; TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); if (!tablet) { BLOG_W("THive::TTxUpdateTabletGroups:: tablet " << TabletId << " wasn't found"); - Ignored = true; return true; } - BLOG_D("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}(" + BLOG_D("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}(" << tablet->Id << "," << tablet->ChannelProfileReassignReason << "," << Groups << ")"); Y_VERIFY(tablet->TabletStorageInfo); + TIntrusivePtr<TTabletStorageInfo>& tabletStorageInfo(tablet->TabletStorageInfo); + ui32 channels = tablet->GetChannelCount(); + NIceDb::TNiceDb db(txc.DB); if (tablet->ChannelProfileNewGroup.count() != Groups.size() && !Groups.empty()) { - BLOG_W("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " + BLOG_ERROR("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " << tablet->Id << " ChannelProfileNewGroup has incorrect size"); - Ignored = true; + db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State>(ETabletState::ReadyToWork); + tablet->State = ETabletState::ReadyToWork; + tablet->TryToBoot(); return true; } @@ -53,13 +58,12 @@ public: BLOG_W("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " << tablet->Id << " ChannelProfileNewGroup is empty"); - Ignored = true; + db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State>(ETabletState::ReadyToWork); + tablet->State = ETabletState::ReadyToWork; + tablet->TryToBoot(); return true; } - TIntrusivePtr<TTabletStorageInfo>& tabletStorageInfo(tablet->TabletStorageInfo); - ui32 channels = tablet->GetChannelCount(); - if (tablet->ChannelProfileReassignReason == NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_SPACE) { TInstant lastChangeTimestamp; @@ -77,15 +81,15 @@ public: TDuration timeSinceLastReassign = ctx.Now() - lastChangeTimestamp; if (lastChangeTimestamp && Self->GetMinPeriodBetweenReassign() && timeSinceLastReassign < Self->GetMinPeriodBetweenReassign()) { BLOG_W("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " - << tablet->Id - << " SpaceReassign was too soon - ignored"); - NewTabletState = ETabletState::ReadyToWork; + << tablet->Id + << " SpaceReassign was too soon - ignored"); + db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State>(ETabletState::ReadyToWork); + tablet->State = ETabletState::ReadyToWork; + tablet->TryToBoot(); return true; } } - NIceDb::TNiceDb db(txc.DB); - // updating tablet channels TVector<TTabletChannelInfo>& tabletChannels = tablet->TabletStorageInfo->Channels; ui32 orderNumber = 0; @@ -103,9 +107,9 @@ public: group = tablet->FindFreeAllocationUnit(channelId); if (group == nullptr) { BLOG_ERROR("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " - << tablet->Id - << " could not find a group for channel " << channelId - << " pool " << tablet->GetChannelStoragePoolName(channelId)); + << tablet->Id + << " could not find a group for channel " << channelId + << " pool " << tablet->GetChannelStoragePoolName(channelId)); if (tabletBootState.empty()) { tabletBootState << "Couldn't find a group for channel: "; tabletBootState << channelId; @@ -117,11 +121,11 @@ public: continue; } else { BLOG_D("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " - << tablet->Id - << " channel " - << channelId - << " assigned to group " - << group->GetGroupID()); + << tablet->Id + << " channel " + << channelId + << " assigned to group " + << group->GetGroupID()); } } @@ -166,9 +170,9 @@ public: } if (channel->History.size() > 1) { // now we block storage for every change of a group's history - NeedToBlockStorage = true; + needToBlockStorage = true; } - Changed = true; + changed = true; if (!tablet->AcquireAllocationUnit(channelId)) { BLOG_ERROR("Failed to aquire AU for tablet " << tablet->Id << " channel " << channelId); @@ -186,7 +190,7 @@ public: } } - if (Changed && (tablet->ChannelProfileNewGroup.none() || !hasEmptyChannel)) { + if (changed && (tablet->ChannelProfileNewGroup.none() || !hasEmptyChannel)) { ++tabletStorageInfo->Version; db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::TabletStorageVersion>(tabletStorageInfo->Version); @@ -205,16 +209,16 @@ public: } } - if (NeedToBlockStorage) { - NewTabletState = ETabletState::BlockStorage; + if (needToBlockStorage) { + newTabletState = ETabletState::BlockStorage; } else { - NewTabletState = ETabletState::ReadyToWork; + newTabletState = ETabletState::ReadyToWork; } if (tablet->IsBootingSuppressed()) { // Tablet will never boot, so will notify about creation right after commit for (const TActorId& actor : tablet->ActorsToNotify) { - Notifications.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); + SideEffects.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); } tablet->ActorsToNotify.clear(); db.Table<Schema::Tablet>().Key(TabletId).UpdateToNull<Schema::Tablet::ActorsToNotify>(); @@ -222,10 +226,10 @@ public: } else { BLOG_W("THive::TTxUpdateTabletGroups::Execute{" << (ui64)this << "}: tablet " << tablet->Id - << " wasn't changed anything"); + << " wasn't changed"); if (hasEmptyChannel) { // we can't continue with partial/unsuccessfull reassign on 0 generation - NewTabletState = ETabletState::GroupAssignment; + newTabletState = ETabletState::GroupAssignment; } else { // we will continue to boot tablet even with unsuccessfull reassign for (ui32 channelId = 0; channelId < channels; ++channelId) { @@ -236,53 +240,46 @@ public: tablet->ChannelProfileNewGroup.reset(channelId); } } - NewTabletState = ETabletState::ReadyToWork; + newTabletState = ETabletState::ReadyToWork; } } - db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State>(NewTabletState); + db.Table<Schema::Tablet>().Key(tablet->Id).Update<Schema::Tablet::State>(newTabletState); + tablet->State = newTabletState; if (!tabletBootState.empty()) { tablet->BootState = tabletBootState; + } else { + tablet->BootState = {}; } - return true; - } - - void Complete(const TActorContext& ctx) override { - if (Ignored) { - BLOG_NOTICE("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete" - " - Ignored transaction"); - } else { - BLOG_D("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete"); - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - tablet->State = NewTabletState; - if (Changed) { - tablet->NotifyStorageInfo(ctx); - Notifications.Send(ctx); - if (tablet->IsReadyToBlockStorage()) { - if (!tablet->InitiateBlockStorage()) { - BLOG_W("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete" - " - InitiateBlockStorage was not successfull"); - } - } else if (tablet->IsReadyToWork()) { - if (!tablet->InitiateStop()) { - BLOG_W("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete" - " - InitiateStop was not successfull"); - } - } else if (tablet->IsBootingSuppressed()) { - // Use best effort to kill currently running tablet - ctx.Register(CreateTabletKiller(TabletId, /* nodeId */ 0, tablet->KnownGeneration)); - } + if (changed) { + tablet->NotifyStorageInfo(SideEffects); + if (tablet->IsReadyToBlockStorage()) { + if (!tablet->InitiateBlockStorage(SideEffects)) { + BLOG_W("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Execute" + " - InitiateBlockStorage was not successfull"); } - if (!tablet->TryToBoot()) { - BLOG_NOTICE("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete" - " - TryToBoot was not successfull"); + } else if (tablet->IsReadyToWork()) { + if (!tablet->InitiateStop(SideEffects)) { + BLOG_W("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Execute" + " - InitiateStop was not successfull"); } + } else if (tablet->IsBootingSuppressed()) { + // Use best effort to kill currently running tablet + SideEffects.Register(CreateTabletKiller(TabletId, /* nodeId */ 0, tablet->KnownGeneration)); } - Self->ProcessBootQueue(); } + if (!tablet->TryToBoot()) { + BLOG_NOTICE("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Execute" + " - TryToBoot was not successfull"); + } + return true; + } + + void Complete(const TActorContext& ctx) override { + BLOG_D("THive::TTxUpdateTabletGroups{" << (ui64)this << "}(" << TabletId << ")::Complete SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; diff --git a/ydb/core/mind/hive/tx__update_tablet_status.cpp b/ydb/core/mind/hive/tx__update_tablet_status.cpp index 7e682e857d..3aae018458 100644 --- a/ydb/core/mind/hive/tx__update_tablet_status.cpp +++ b/ydb/core/mind/hive/tx__update_tablet_status.cpp @@ -5,13 +5,13 @@ namespace NKikimr { namespace NHive { class TTxUpdateTabletStatus : public TTransactionBase<THive> { - const TTabletId TabletId; + TTabletId TabletId; + TFollowerId FollowerId; const TActorId Local; const TEvLocal::TEvTabletStatus::EStatus Status; const TEvTablet::TEvTabletDead::EReason Reason; ui32 Generation; - TFollowerId FollowerId; - TCompleteNotifications Notifications; + TSideEffects SideEffects; public: TTxUpdateTabletStatus( @@ -24,11 +24,11 @@ public: THive *hive) : TBase(hive) , TabletId(tabletId) + , FollowerId(followerId) , Local(local) , Status(status) , Reason(reason) , Generation(generation) - , FollowerId(followerId) {} TTxType GetTxType() const override { return NHive::TXTYPE_UPDATE_TABLET_STATUS; } @@ -62,6 +62,7 @@ public: } bool Execute(TTransactionContext &txc, const TActorContext&) override { + SideEffects.Reset(Self->SelfId()); TTabletInfo* tablet = Self->FindTablet(TabletId, FollowerId); if (tablet != nullptr) { BLOG_D("THive::TTxUpdateTabletStatus::Execute for tablet " @@ -76,7 +77,6 @@ public: << Local); NIceDb::TNiceDb db(txc.DB); TInstant now = TActivationContext::Now(); - Notifications.Reset(Self->SelfId()); if (Status == TEvLocal::TEvTabletStatus::StatusOk) { tablet->Statistics.AddRestartTimestamp(now.MilliSeconds()); tablet->ActualizeTabletStatistics(now); @@ -90,18 +90,18 @@ public: return true; } for (const TActorId& actor : tablet->ActorsToNotifyOnRestart) { - Notifications.Send(actor, new TEvPrivate::TEvRestartComplete({TabletId, FollowerId}, "OK")); + SideEffects.Send(actor, new TEvPrivate::TEvRestartComplete({TabletId, FollowerId}, "OK")); } tablet->ActorsToNotifyOnRestart.clear(); if (tablet->GetLeader().IsDeleting()) { - tablet->SendStopTablet(Local, {TabletId, FollowerId}); + tablet->SendStopTablet(SideEffects); return true; } tablet->BecomeRunning(Local.NodeId()); if (tablet->GetLeader().IsLockedToActor()) { // Tablet is locked and shouldn't be running, but we just found out it's running on this node // Ask it to stop using InitiateStop (which uses data saved by BecomeRunning call above) - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); } tablet->BootState = Self->BootStateRunning; tablet->Statistics.SetLastAliveTimestamp(now.MilliSeconds()); @@ -118,13 +118,13 @@ public: NIceDb::TUpdate<Schema::TabletFollowerTablet::Statistics>(tablet->Statistics)); } for (const TActorId& actor : tablet->ActorsToNotify) { - Notifications.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); + SideEffects.Send(actor, new TEvHive::TEvTabletCreationResult(NKikimrProto::OK, TabletId)); } tablet->ActorsToNotify.clear(); db.Table<Schema::Tablet>().Key(TabletId).UpdateToNull<Schema::Tablet::ActorsToNotify>(); } else { if (Local) { - Notifications.Send(Local, new TEvLocal::TEvDeadTabletAck(std::make_pair(TabletId, FollowerId), Generation)); + SideEffects.Send(Local, new TEvLocal::TEvDeadTabletAck(std::make_pair(TabletId, FollowerId), Generation)); } if (tablet->IsLeader()) { TLeaderTabletInfo& leader(tablet->AsLeader()); @@ -152,7 +152,7 @@ public: NIceDb::TUpdate<Schema::TabletFollowerTablet::FollowerNode>(0), NIceDb::TUpdate<Schema::TabletFollowerTablet::Statistics>(tablet->Statistics)); } - tablet->InitiateStop(); + tablet->InitiateStop(SideEffects); } } switch (tablet->GetLeader().State) { @@ -164,7 +164,7 @@ public: case ETabletState::Stopping: if (tablet->IsLeader()) { for (const TActorId& actor : tablet->GetLeader().ActorsToNotify) { - Notifications.Send(actor, new TEvHive::TEvStopTabletResult(NKikimrProto::OK, TabletId)); + SideEffects.Send(actor, new TEvHive::TEvStopTabletResult(NKikimrProto::OK, TabletId)); } tablet->GetLeader().ActorsToNotify.clear(); } @@ -181,18 +181,15 @@ public: break; }; } + tablet->GetLeader().TryToBoot(); } + Self->ProcessBootQueue(); // it's required to start followers on successful leader start return true; } void Complete(const TActorContext& ctx) override { - BLOG_D("THive::TTxUpdateTabletStatus::Complete TabletId: " << TabletId << " Notifications: " << Notifications); - Notifications.Send(ctx); - TLeaderTabletInfo* tablet = Self->FindTablet(TabletId); - if (tablet != nullptr) { - tablet->TryToBoot(); - } - Self->ProcessBootQueue(); + BLOG_D("THive::TTxUpdateTabletStatus::Complete TabletId: " << TabletId << " SideEffects: " << SideEffects); + SideEffects.Complete(ctx); } }; @@ -208,15 +205,3 @@ ITransaction* THive::CreateUpdateTabletStatus( } // NHive } // NKikimr - -template <> -inline void Out<NKikimr::NHive::TCompleteNotifications>(IOutputStream& o, const NKikimr::NHive::TCompleteNotifications& n) { - o << n.SelfID << " -> ["; - for (auto it = n.Notifications.begin(); it != n.Notifications.end(); ++it) { - if (it != n.Notifications.begin()) { - o << ','; - } - o << it->Get()->Recipient; - } - o << "]"; -} diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp index 6761669124..e9d6a24723 100644 --- a/ydb/core/mind/local.cpp +++ b/ydb/core/mind/local.cpp @@ -481,17 +481,20 @@ class TLocalNodeRegistrar : public TActorBootstrapped<TLocalNodeRegistrar> { const NKikimrLocal::TEvStopTablet &record = ev->Get()->Record; Y_VERIFY(record.HasTabletId()); TTabletId tabletId(record.GetTabletId(), record.GetFollowerId()); - LOG_DEBUG_S(ctx, NKikimrServices::LOCAL, "TLocalNodeRegistrar: Handle TEvStopTablet TabletId:" << tabletId); + ui32 generation(record.GetGeneration()); + LOG_DEBUG_S(ctx, NKikimrServices::LOCAL, "TLocalNodeRegistrar: Handle TEvStopTablet TabletId:" << tabletId << " Generation:" << generation); auto onlineTabletIt = OnlineTablets.find(tabletId); if (onlineTabletIt != OnlineTablets.end()) { - // Provide/check generation here - ctx.Send(onlineTabletIt->second.Tablet, new TEvTablet::TEvTabletStop(tabletId.first, TEvTablet::TEvTabletStop::ReasonStop)); + if (generation == 0 || onlineTabletIt->second.Generation <= generation) { + ctx.Send(onlineTabletIt->second.Tablet, new TEvTablet::TEvTabletStop(tabletId.first, TEvTablet::TEvTabletStop::ReasonStop)); + } } else { auto inbootTabletIt = InbootTablets.find(tabletId); if (inbootTabletIt != InbootTablets.end()) { - // Provide/check generation here - ctx.Send(inbootTabletIt->second.Tablet, new TEvTablet::TEvTabletStop(tabletId.first, TEvTablet::TEvTabletStop::ReasonStop)); + if (generation == 0 || onlineTabletIt->second.Generation <= generation) { + ctx.Send(inbootTabletIt->second.Tablet, new TEvTablet::TEvTabletStop(tabletId.first, TEvTablet::TEvTabletStop::ReasonStop)); + } } } } diff --git a/ydb/core/mind/local.h b/ydb/core/mind/local.h index 05c9bdc48e..0738bc20f6 100644 --- a/ydb/core/mind/local.h +++ b/ydb/core/mind/local.h @@ -173,10 +173,11 @@ struct TEvLocal { TEvStopTablet() {} - TEvStopTablet(std::pair<ui64, ui32> tabletId) + TEvStopTablet(std::pair<ui64, ui32> tabletId, ui32 generation = 0) { Record.SetTabletId(tabletId.first); Record.SetFollowerId(tabletId.second); + Record.SetGeneration(generation); } }; diff --git a/ydb/core/protos/hive.proto b/ydb/core/protos/hive.proto index 6b430b95e5..0b6bac0755 100644 --- a/ydb/core/protos/hive.proto +++ b/ydb/core/protos/hive.proto @@ -258,6 +258,7 @@ message TEvRequestHiveInfo { optional bool ReturnFollowers = 3; optional uint32 FollowerID = 4; optional bool ReturnMetrics = 5; + optional bool ReturnChannelHistory = 6; } message TEvResponseHiveInfo { diff --git a/ydb/core/protos/local.proto b/ydb/core/protos/local.proto index a385a3cf3e..8178c4f10a 100644 --- a/ydb/core/protos/local.proto +++ b/ydb/core/protos/local.proto @@ -69,6 +69,7 @@ message TEvBootTablet { message TEvStopTablet { optional fixed64 TabletId = 1; optional uint32 FollowerId = 2; + optional uint32 Generation = 3; } message TEvTabletStatus { diff --git a/ydb/core/tablet/tablet_req_blockbs.cpp b/ydb/core/tablet/tablet_req_blockbs.cpp index 7e2e94ec95..ac5914fb59 100644 --- a/ydb/core/tablet/tablet_req_blockbs.cpp +++ b/ydb/core/tablet/tablet_req_blockbs.cpp @@ -8,10 +8,11 @@ namespace NKikimr { constexpr ui32 MAX_ATTEMPTS = 3; class TTabletReqBlockBlobStorageGroup : public TActorBootstrapped<TTabletReqBlockBlobStorageGroup> { - const TActorId Owner; - const ui64 TabletId; - const ui32 GroupId; - const ui32 Generation; +public: + TActorId Owner; + ui64 TabletId; + ui32 GroupId; + ui32 Generation; ui32 ErrorCount; void ReplyAndDie(NKikimrProto::EReplyStatus status, const TString &reason = { }) { @@ -69,9 +70,8 @@ public: Become(&TThis::StateWait); } - TTabletReqBlockBlobStorageGroup(const TActorId &owner, ui64 tabletId, ui32 groupId, ui32 gen) - : Owner(owner) - , TabletId(tabletId) + TTabletReqBlockBlobStorageGroup(ui64 tabletId, ui32 groupId, ui32 gen) + : TabletId(tabletId) , GroupId(groupId) , Generation(gen) , ErrorCount(0) @@ -79,12 +79,11 @@ public: }; class TTabletReqBlockBlobStorage : public TActorBootstrapped<TTabletReqBlockBlobStorage> { - const TActorId Owner; - TIntrusiveConstPtr<TTabletStorageInfo> Info; - const ui32 Generation; - const bool BlockPrevEntry; - + TActorId Owner; + ui64 TabletId; + ui32 Generation; ui32 Replied = 0; + TVector<THolder<TTabletReqBlockBlobStorageGroup>> Requests; TVector<TActorId> ReqActors; void PassAway() override { @@ -96,7 +95,7 @@ class TTabletReqBlockBlobStorage : public TActorBootstrapped<TTabletReqBlockBlob } void ReplyAndDie(NKikimrProto::EReplyStatus status, const TString &reason = { }) { - Send(Owner, new TEvTabletBase::TEvBlockBlobStorageResult(status, Info->TabletID, reason)); + Send(Owner, new TEvTabletBase::TEvBlockBlobStorageResult(status, TabletId, reason)); PassAway(); } @@ -116,35 +115,50 @@ class TTabletReqBlockBlobStorage : public TActorBootstrapped<TTabletReqBlockBlob } } public: - TTabletReqBlockBlobStorage(TActorId owner, TTabletStorageInfo *info, ui32 generation, bool blockPrevEntry) + TTabletReqBlockBlobStorage(TActorId owner, TTabletStorageInfo* info, ui32 generation, bool blockPrevEntry) : Owner(owner) - , Info(info) + , TabletId(info->TabletID) , Generation(generation) - , BlockPrevEntry(blockPrevEntry) - {} + { + std::unordered_set<ui32> blocked; + Requests.reserve(blockPrevEntry ? info->Channels.size() * 2 : info->Channels.size()); + for (auto& channel : info->Channels) { + if (channel.History.empty()) { + continue; + } + auto itEntry = channel.History.rbegin(); + while (itEntry != channel.History.rend() && itEntry->FromGeneration > generation) { + ++itEntry; + } + if (itEntry == channel.History.rend()) { + continue; + } + if (blocked.insert(itEntry->GroupID).second) { + Requests.emplace_back(new TTabletReqBlockBlobStorageGroup(TabletId, itEntry->GroupID, Generation)); + } + + if (blockPrevEntry) { + ++itEntry; + if (itEntry == channel.History.rend()) { + continue; + } + if (blocked.insert(itEntry->GroupID).second) { + Requests.emplace_back(new TTabletReqBlockBlobStorageGroup(TabletId, itEntry->GroupID, Generation)); + } + } + } + } static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TABLET_REQ_BLOCK_BS; } void Bootstrap() { - TSet<ui32> blocked; - - const ui64 tabletId = Info->TabletID; - ReqActors.reserve(Info->Channels.size()); - for (auto &x : Info->Channels) { - if (auto *g = x.LatestEntry()) { - if (blocked.insert(g->GroupID).second) - ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, g->GroupID, Generation))); - } - - if (BlockPrevEntry) { - if (auto *pg = x.PreviousEntry()) - if (blocked.insert(pg->GroupID).second) - ReqActors.push_back(RegisterWithSameMailbox(new TTabletReqBlockBlobStorageGroup(SelfId(), tabletId, pg->GroupID, Generation))); - } + ReqActors.reserve(Requests.size()); + for (auto& req : Requests) { + req->Owner = SelfId(); + ReqActors.push_back(RegisterWithSameMailbox(req.Release())); } - Become(&TThis::StateWait); } @@ -157,7 +171,7 @@ public: } }; -IActor* CreateTabletReqBlockBlobStorage(const TActorId &owner, TTabletStorageInfo *info, ui32 generation, bool blockPrevEntry) { +IActor* CreateTabletReqBlockBlobStorage(const TActorId& owner, TTabletStorageInfo* info, ui32 generation, bool blockPrevEntry) { return new TTabletReqBlockBlobStorage(owner, info, generation, blockPrevEntry); } diff --git a/ydb/core/tablet/tablet_req_delete.cpp b/ydb/core/tablet/tablet_req_delete.cpp index 754c79bb32..5a3b12f4c7 100644 --- a/ydb/core/tablet/tablet_req_delete.cpp +++ b/ydb/core/tablet/tablet_req_delete.cpp @@ -20,7 +20,7 @@ class TTabletReqDelete : public TActorBootstrapped<TTabletReqDelete> { }; const TActorId Owner; - TIntrusivePtr<TTabletStorageInfo> TabletStorageInfo; + ui64 TabletId; TVector<TRequestInfo> Requests; ui32 FinishedRequests; ui32 ErrorCount; @@ -30,23 +30,25 @@ class TTabletReqDelete : public TActorBootstrapped<TTabletReqDelete> { if (status == NKikimrProto::OK) { const TActorId tabletStateServiceId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()); ctx.Send(tabletStateServiceId, new NNodeWhiteboard::TEvWhiteboard::TEvTabletStateUpdate( - TabletStorageInfo->TabletID, + TabletId, 0, NKikimrWhiteboard::TTabletStateInfo::Deleted, std::numeric_limits<ui32>::max()), true); // TODO(xenoxeno): broadcast message to more/all nodes ... maybe? } - ctx.Send(Owner, new TEvTabletBase::TEvDeleteTabletResult(status, TabletStorageInfo->TabletID)); + ctx.Send(Owner, new TEvTabletBase::TEvDeleteTabletResult(status, TabletId)); Die(ctx); } - void GenerateRequests() { + void GenerateRequests(const TIntrusivePtr<TTabletStorageInfo>& tabletStorageInfo) { THashSet<std::pair<ui32, ui32>> groupChannels; - for (const TTabletChannelInfo& channelInfo : TabletStorageInfo->Channels) { + for (const TTabletChannelInfo& channelInfo : tabletStorageInfo->Channels) { for (const TTabletChannelInfo::THistoryEntry& historyInfo : channelInfo.History) { - if (groupChannels.emplace(historyInfo.GroupID, channelInfo.Channel).second) { - Requests.emplace_back(historyInfo.GroupID, channelInfo.Channel); + if (historyInfo.FromGeneration <= Generation) { + if (groupChannels.emplace(historyInfo.GroupID, channelInfo.Channel).second) { + Requests.emplace_back(historyInfo.GroupID, channelInfo.Channel); + } } } } @@ -58,7 +60,7 @@ class TTabletReqDelete : public TActorBootstrapped<TTabletReqDelete> { const ui32 recordGeneration = total ? Generation : Generation + 1; const ui32 perGenerationCounter = total ? Max<ui32>() : 0; auto event = TEvBlobStorage::TEvCollectGarbage::CreateHardBarrier( - TabletStorageInfo->TabletID, // tabletId + TabletId, // tabletId recordGeneration, // recordGeneration perGenerationCounter, // perGenerationCounter info.Channel, // channel @@ -82,9 +84,9 @@ class TTabletReqDelete : public TActorBootstrapped<TTabletReqDelete> { ++FinishedRequests; if (FinishedRequests >= Requests.size()) { if (Generation == std::numeric_limits<ui32>::max()) { - ui64 StateStorageId = StateStorageGroupFromTabletID(TabletStorageInfo->TabletID); + ui64 StateStorageId = StateStorageGroupFromTabletID(TabletId); const TActorId proxyActorID = MakeStateStorageProxyID(StateStorageId); - ctx.Send(proxyActorID, new TEvStateStorage::TEvDelete(TabletStorageInfo->TabletID)); + ctx.Send(proxyActorID, new TEvStateStorage::TEvDelete(TabletId)); } ReplyAndDie(NKikimrProto::OK, ctx); @@ -119,14 +121,15 @@ public: TTabletReqDelete(const TActorId &owner, const TIntrusivePtr<TTabletStorageInfo>& tabletStorageInfo, ui32 generation = std::numeric_limits<ui32>::max()) : Owner(owner) - , TabletStorageInfo(tabletStorageInfo) + , TabletId(tabletStorageInfo->TabletID) , FinishedRequests(0) , ErrorCount(0) , Generation(generation) - {} + { + GenerateRequests(tabletStorageInfo); + } void Bootstrap(const TActorContext& ctx) { - GenerateRequests(); for (std::size_t i = 0; i < Requests.size(); ++i) { SendRequest(i, ctx); } diff --git a/ydb/core/tx/tx_proxy/proxy_ut_helpers.cpp b/ydb/core/tx/tx_proxy/proxy_ut_helpers.cpp index a9becc80d7..b87837acc8 100644 --- a/ydb/core/tx/tx_proxy/proxy_ut_helpers.cpp +++ b/ydb/core/tx/tx_proxy/proxy_ut_helpers.cpp @@ -403,8 +403,9 @@ void CheckTableBecomeOfline(TBaseTestEnv &env, ui64 tablet_id) { UNIT_ASSERT( env.GetClient().WaitForTabletDown(&env.GetRuntime(), tablet_id, true, WaitTimeOut)); //ensure that tablet do not wake up + TDuration negativeTimeout = TDuration::Seconds(1); UNIT_ASSERT( - !env.GetClient().WaitForTabletAlive(&env.GetRuntime(), tablet_id, true, WaitTimeOut)); + !env.GetClient().WaitForTabletAlive(&env.GetRuntime(), tablet_id, true, negativeTimeout)); } void CheckTableRunOnProperTenantNode(TBaseTestEnv &env, const TString &tenant, ui64 tablet_id) { |