diff options
| author | Ilia Shakhov <[email protected]> | 2026-07-10 14:09:08 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-07-10 14:09:08 +0300 |
| commit | 34cf7ca54f934e552dd49cd09bea3b69a36f9262 (patch) | |
| tree | 9a770caed3cdb9cc15765b26fddfc466ece1162c | |
| parent | 264ec759347b38d53fadb3a5e0934e55c21346f8 (diff) | |
Add long leases logic to NodeBroker & clients (#45855)
Co-authored-by: Ilia Shakhov <[email protected]>
| -rw-r--r-- | ydb/core/mind/dynamic_nameserver.cpp | 120 | ||||
| -rw-r--r-- | ydb/core/mind/dynamic_nameserver_impl.h | 34 | ||||
| -rw-r--r-- | ydb/core/mind/dynamic_nameserver_mon.cpp | 10 | ||||
| -rw-r--r-- | ydb/core/mind/lease_holder.cpp | 94 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker.cpp | 91 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker.h | 5 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker__extend_lease.cpp | 3 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker__register_node.cpp | 14 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker__scheme.h | 9 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker_impl.h | 20 | ||||
| -rw-r--r-- | ydb/core/mind/node_broker_ut.cpp | 409 | ||||
| -rw-r--r-- | ydb/core/protos/feature_flags.proto | 1 | ||||
| -rw-r--r-- | ydb/core/protos/node_broker.proto | 7 | ||||
| -rw-r--r-- | ydb/library/actors/core/interconnect.h | 4 |
14 files changed, 718 insertions, 103 deletions
diff --git a/ydb/core/mind/dynamic_nameserver.cpp b/ydb/core/mind/dynamic_nameserver.cpp index a8d4f068786..c4172501bc9 100644 --- a/ydb/core/mind/dynamic_nameserver.cpp +++ b/ydb/core/mind/dynamic_nameserver.cpp @@ -30,12 +30,10 @@ public: } TActorCacheMiss(TDynamicNameserver* owner, ui32 nodeId, TDynamicConfigPtr config, - TIntrusivePtr<TListNodesCache> listNodesCache, TAutoPtr<IEventHandle> origRequest, TMonotonic deadline) : TActorBase(&TThis::StateWork) , TCacheMiss(nodeId, config, origRequest, deadline, 0) , Owner(owner) - , ListNodesCache(listNodesCache) { } @@ -62,7 +60,7 @@ public: TActorBase::PassAway(); } - void ConvertToActor(TDynamicNameserver*, TIntrusivePtr<TListNodesCache>, const TActorContext &) override {} + void ConvertToActor(TDynamicNameserver*, const TActorContext &) override {} private: using TCacheMiss::Config; @@ -91,7 +89,7 @@ private: // Reset proxy if node expired. if (exists) { ResetInterconnectProxyConfig(NodeId, ctx); - ListNodesCache->Invalidate(); // node was erased + Owner->InvalidateListNodesCache(); // node was erased } Owner->UpdateCounters(); OnError(rec.GetStatus().GetReason(), ctx); @@ -99,12 +97,12 @@ private: } TDynamicConfig::TDynamicNodeInfo node(rec.GetNode()); - if (!exists || !oldNode.EqualExceptExpire(node)) { - ListNodesCache->Invalidate(); + if (!exists || !oldNode.EqualExceptExpireAndLiveness(node) || oldNode.Liveness != node.Liveness) { + Owner->InvalidateListNodesCache(); } // If ID is re-used by another node then proxy has to be reset. - if (exists && !oldNode.EqualExceptExpire(node)) + if (exists && !oldNode.EqualExceptExpireAndLiveness(node)) ResetInterconnectProxyConfig(NodeId, ctx); Config->DynamicNodes.emplace(NodeId, node); @@ -113,7 +111,6 @@ private: } TDynamicNameserver* Owner; - TIntrusivePtr<TListNodesCache> ListNodesCache; }; TCacheMiss::TCacheMiss(ui32 nodeId, TDynamicConfigPtr config, TAutoPtr<IEventHandle> origRequest, @@ -175,11 +172,10 @@ public: ctx.Send(OrigRequest->Sender, reply.Release()); } - void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache, - const TActorContext &ctx) override { + void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) override { Config->PendingCacheMisses.Remove(this); - auto* actor = new TActorCacheMiss<TCacheMissGet>(owner, NodeId, Config, listNodesCache, OrigRequest, Deadline); + auto* actor = new TActorCacheMiss<TCacheMissGet>(owner, NodeId, Config, OrigRequest, Deadline); actor->NeedScheduleDeadline = NeedScheduleDeadline; ctx.RegisterWithSameMailbox(actor); actor->SendRequest(); @@ -212,11 +208,10 @@ public: ctx.Send(OrigRequest->Sender, reply); } - void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache, - const TActorContext &ctx) override { + void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) override { Config->PendingCacheMisses.Remove(this); - auto* actor = new TActorCacheMiss<TCacheMissResolve>(owner, NodeId, Config, listNodesCache, OrigRequest, Deadline); + auto* actor = new TActorCacheMiss<TCacheMissResolve>(owner, NodeId, Config, OrigRequest, Deadline); actor->NeedScheduleDeadline = NeedScheduleDeadline; ctx.RegisterWithSameMailbox(actor); actor->SendRequest(); @@ -235,6 +230,8 @@ void TDynamicNameserver::Bootstrap(const TActorContext &ctx) } EnableDeltaProtocol = AppData()->FeatureFlags.GetEnableNodeBrokerDeltaProtocol(); + EnableLongLease = AppData()->FeatureFlags.GetEnableNodeBrokerLongLease(); + ui32 featureFlagsItem = NKikimrConsole::TConfigItem::FeatureFlagsItem; Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(featureFlagsItem)); @@ -261,7 +258,7 @@ void TDynamicNameserver::Handle(TEvNodeWardenStorageConfig::TPtr ev) { const auto& config = *ev->Get()->Config; BridgeInfo = std::move(ev->Get()->BridgeInfo); - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); if (ev->Get()->SelfManagementEnabled) { // self-management through distconf is enabled and we are operating based on their tables, so apply them now @@ -290,7 +287,7 @@ void TDynamicNameserver::Handle(TEvNodeWardenStorageConfig::TPtr ev) { void TDynamicNameserver::ReplaceNameserverSetup(TIntrusivePtr<TTableNameserverSetup> newStaticConfig) { if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) { StaticConfig = std::move(newStaticConfig); - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); for (const auto& subscriber : StaticNodeChangeSubscribers) { TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes)); } @@ -369,7 +366,7 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId, auto it = config->DynamicNodes.find(nodeId); if (it != config->DynamicNodes.end() - && it->second.Expire > ctx.Now()) + && it->second.EffectiveExpire(EnableLongLease) > ctx.Now()) { RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, ev->Sender, SelfId(), deadline)); } else if (config->ExpiredNodes.contains(nodeId) @@ -380,7 +377,7 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId, } else { TCacheMiss* cacheMiss; if (ProtocolState == EProtocolState::UseEpochProtocol) { - auto* actor = new TActorCacheMiss<TCacheMissResolve>(this, nodeId, config, ListNodesCache, ev, deadline); + auto* actor = new TActorCacheMiss<TCacheMissResolve>(this, nodeId, config, ev, deadline); cacheMiss = actor; RegisterWithSameMailbox(actor); actor->SendRequest(); @@ -397,10 +394,18 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId, } } -void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext &ctx) +void TDynamicNameserver::InvalidateListNodesCache() +{ + ListNodesCacheAll->Invalidate(); + ListNodesCacheAlive->Invalidate(); +} + +void TDynamicNameserver::SendNodesList(TActorId recipient, bool onlyAliveNodes, const TActorContext &ctx) { auto now = ctx.Now(); - if (ListNodesCache->NeedUpdate(now)) { + auto& cache = onlyAliveNodes ? ListNodesCacheAlive : ListNodesCacheAll; + + if (cache->NeedUpdate(now)) { auto newNodes = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>(); auto newExpire = TInstant::Max(); const bool bridgeModeEnabled = AppData()->BridgeModeEnabled; @@ -431,17 +436,23 @@ void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext & for (auto &config : DynamicConfigs) { for (auto &pr : config->DynamicNodes) { - if (pr.second.Expire > now) { - newNodes->emplace_back(pr.first, pr.second.Address, - pr.second.Host, pr.second.ResolveHost, - pr.second.Port, pr.second.Location, false); - newExpire = std::min(newExpire, pr.second.Expire); - if (newPileMap) { - TNodeLocation location(pr.second.Location); - const auto& bridgePileName = location.GetBridgePileName(); - if (bridgePileName && pileNameMap.contains(*bridgePileName)) { - newPileMap->at(pileNameMap[*bridgePileName]).push_back(pr.first); - } + if (EnableLongLease) { + if (onlyAliveNodes && pr.second.Liveness != ENodeLiveness::Alive) { + continue; // dead nodes are not included in alive-only list + } + } else if (pr.second.Expire <= now) { + continue; // expired nodes are not included + } + + newNodes->emplace_back(pr.first, pr.second.Address, + pr.second.Host, pr.second.ResolveHost, + pr.second.Port, pr.second.Location, false); + newExpire = std::min(newExpire, pr.second.EffectiveExpire(EnableLongLease)); + if (newPileMap) { + TNodeLocation location(pr.second.Location); + const auto& bridgePileName = location.GetBridgePileName(); + if (bridgePileName && pileNameMap.contains(*bridgePileName)) { + newPileMap->at(pileNameMap[*bridgePileName]).push_back(pr.first); } } } @@ -453,16 +464,16 @@ void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext & } } - ListNodesCache->Update(std::move(newNodes), newExpire, std::move(newPileMap)); + cache->Update(std::move(newNodes), newExpire, std::move(newPileMap)); } - ctx.Send(recipient, new TEvInterconnect::TEvNodesInfo(ListNodesCache->GetNodes(), ListNodesCache->GetPileMap())); + ctx.Send(recipient, new TEvInterconnect::TEvNodesInfo(cache->GetNodes(), cache->GetPileMap())); } void TDynamicNameserver::SendNodesList(const TActorContext &ctx) { - for (auto &sender : ListNodesQueue) { - SendNodesList(sender, ctx); + for (auto &[sender, onlyAliveNodes] : ListNodesQueue) { + SendNodesList(sender, onlyAliveNodes, ctx); } ListNodesQueue.clear(); } @@ -500,8 +511,10 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec, if (it == config->DynamicNodes.end()) { config->DynamicNodes.emplace(nodeId, info); } else { - if (it->second.EqualExceptExpire(info)) { + if (it->second.EqualExceptExpireAndLiveness(info)) { it->second.Expire = info.Expire; + it->second.ExpireV2 = info.ExpireV2; + it->second.Liveness = info.Liveness; } else { ResetInterconnectProxyConfig(nodeId, ctx); it->second = info; @@ -517,7 +530,7 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec, config->ExpiredNodes.emplace(node.GetNodeId()); } - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); config->Epoch = rec.GetEpoch(); ctx.Schedule(config->Epoch.End - ctx.Now(), new TEvPrivate::TEvUpdateEpoch(domain, config->Epoch.Id + 1)); @@ -531,14 +544,16 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec, if (it == config->DynamicNodes.end()) { config->DynamicNodes.emplace(nodeId, info); } else { - if (it->second.EqualExceptExpire(info)) { + if (it->second.EqualExceptExpireAndLiveness(info)) { it->second.Expire = info.Expire; + it->second.ExpireV2 = info.ExpireV2; + it->second.Liveness = info.Liveness; } else { ResetInterconnectProxyConfig(nodeId, ctx); it->second = info; } } - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); } config->Epoch = rec.GetEpoch(); } @@ -607,8 +622,10 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvListNodes::TPtr &ev, { LOG_D("Handle " << ev->Get()->ToString()); + const bool onlyAliveNodes = ev->Get()->OnlyAliveNodes; + if (ProtocolState == EProtocolState::Connecting) { - SendNodesList(ev->Sender, ctx); + SendNodesList(ev->Sender, onlyAliveNodes, ctx); } else { if (ListNodesQueue.empty()) { auto dinfo = AppData(ctx)->DomainsInfo; @@ -624,7 +641,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvListNodes::TPtr &ev, PendingRequests.Set(domain); } } - ListNodesQueue.push_back(ev->Sender); + ListNodesQueue.emplace_back(ev->Sender, onlyAliveNodes); } if (ev->Get()->SubscribeToStaticNodeChanges) { @@ -651,7 +668,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc ui32 domain = AppData()->DomainsInfo->GetDomain()->DomainUid; const auto& config = DynamicConfigs[domain]; auto it = config->DynamicNodes.find(nodeId); - if (it != config->DynamicNodes.end() && it->second.Expire > ctx.Now()) { + if (it != config->DynamicNodes.end() && it->second.EffectiveExpire(EnableLongLease) > ctx.Now()) { reply->Node = MakeHolder<TEvInterconnect::TNodeInfo>(it->first, it->second.Address, it->second.Host, it->second.ResolveHost, it->second.Port, it->second.Location); @@ -663,7 +680,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc TCacheMiss* cacheMiss; const TMonotonic deadline = ev->Get()->Deadline; if (ProtocolState == EProtocolState::UseEpochProtocol) { - auto* actor = new TActorCacheMiss<TCacheMissGet>(this, nodeId, config, ListNodesCache, ev.Release(), deadline); + auto* actor = new TActorCacheMiss<TCacheMissGet>(this, nodeId, config, ev.Release(), deadline); cacheMiss = actor; RegisterWithSameMailbox(actor); actor->SendRequest(); @@ -753,7 +770,7 @@ void TDynamicNameserver::Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, con } for (const auto &[cacheMiss, _] : DynamicConfigs[domain]->CacheMissHolders) { - cacheMiss->ConvertToActor(this, ListNodesCache, ctx); + cacheMiss->ConvertToActor(this, ctx); } // cache misses are managed by actorsystem in epoch protocol DynamicConfigs[domain]->CacheMissHolders.clear(); @@ -810,14 +827,16 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T if (it == config->DynamicNodes.end()) { config->DynamicNodes.emplace(nodeId, info); } else { - if (it->second.EqualExceptExpire(info)) { + if (it->second.EqualExceptExpireAndLiveness(info)) { it->second.Expire = info.Expire; + it->second.ExpireV2 = info.ExpireV2; + it->second.Liveness = info.Liveness; } else { ResetInterconnectProxyConfig(nodeId, ctx); it->second = info; } } - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); config->ExpiredNodes.erase(nodeId); break; } @@ -825,7 +844,7 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T ui32 nodeId = u.GetExpiredNode(); if (config->DynamicNodes.erase(nodeId) > 0) { ResetInterconnectProxyConfig(nodeId, ctx); - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); } config->ExpiredNodes.insert(nodeId); break; @@ -834,7 +853,7 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T ui32 nodeId = u.GetRemovedNode(); if (config->DynamicNodes.erase(nodeId) > 0) { ResetInterconnectProxyConfig(nodeId, ctx); - ListNodesCache->Invalidate(); + InvalidateListNodesCache(); } config->ExpiredNodes.erase(nodeId); break; @@ -923,6 +942,11 @@ void TDynamicNameserver::Handle(NConsole::TEvConsole::TEvConfigNotificationReque ui32 domain = AppData()->DomainsInfo->GetDomain()->DomainUid; NTabletPipe::CloseClient(ctx, DynamicConfigs[domain]->NodeBrokerPipe); } + + if (EnableLongLease != featureFlags.GetEnableNodeBrokerLongLease()) { + EnableLongLease = featureFlags.GetEnableNodeBrokerLongLease(); + InvalidateListNodesCache(); + } } Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie); } diff --git a/ydb/core/mind/dynamic_nameserver_impl.h b/ydb/core/mind/dynamic_nameserver_impl.h index 49df9238cea..b58bf7bf303 100644 --- a/ydb/core/mind/dynamic_nameserver_impl.h +++ b/ydb/core/mind/dynamic_nameserver_impl.h @@ -49,8 +49,7 @@ public: virtual void OnSuccess(const TActorContext &); virtual void OnError(const TString &error, const TActorContext &); - virtual void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache, - const TActorContext &ctx) = 0; + virtual void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) = 0; struct THeapIndexByDeadline { size_t& operator()(TCacheMiss& cacheMiss) const; @@ -83,9 +82,13 @@ struct TDynamicConfig : public TThrRefBase { const TString &resolveHost, ui16 port, const TNodeLocation &location, - TInstant expire) + TInstant expire, + TInstant expireV2, + ENodeLiveness liveness) : TNodeInfo(address, host, resolveHost, port, location) , Expire(expire) + , ExpireV2(expireV2) + , Liveness(liveness) { } @@ -95,14 +98,16 @@ struct TDynamicConfig : public TThrRefBase { info.GetResolveHost(), (ui16)info.GetPort(), TNodeLocation(info.GetLocation()), - TInstant::MicroSeconds(info.GetExpire())) + TInstant::MicroSeconds(info.GetExpire()), + TInstant::MicroSeconds(info.HasExpireV2() ? info.GetExpireV2() : info.GetExpire()), + static_cast<ENodeLiveness>(info.GetLiveness())) { } TDynamicNodeInfo(const TDynamicNodeInfo &other) = default; TDynamicNodeInfo &operator=(const TDynamicNodeInfo &other) = default; - bool EqualExceptExpire(const TDynamicNodeInfo &other) const + bool EqualExceptExpireAndLiveness(const TDynamicNodeInfo &other) const { return Host == other.Host && Address == other.Address @@ -111,7 +116,13 @@ struct TDynamicConfig : public TThrRefBase { && Location == other.Location; } + TInstant EffectiveExpire(bool enableLongLease) const { + return enableLongLease ? ExpireV2 : Expire; + } + TInstant Expire; + TInstant ExpireV2; + ENodeLiveness Liveness = ENodeLiveness::Alive; }; THashMap<ui32, TDynamicNodeInfo> DynamicNodes; @@ -186,7 +197,8 @@ public: TDynamicNameserver(const TIntrusivePtr<TTableNameserverSetup> &setup, ui32 resolvePoolId) : StaticConfig(setup) - , ListNodesCache(MakeIntrusive<TListNodesCache>()) + , ListNodesCacheAll(MakeIntrusive<TListNodesCache>()) + , ListNodesCacheAlive(MakeIntrusive<TListNodesCache>()) , ResolvePoolId(resolvePoolId) { Y_ABORT_UNLESS(StaticConfig->IsEntriesUnique()); @@ -251,8 +263,9 @@ private: const TActorContext &ctx); void ResolveStaticNode(ui32 nodeId, TActorId sender, TMonotonic deadline, const TActorContext &ctx); void ResolveDynamicNode(ui32 nodeId, TAutoPtr<IEventHandle> ev, TMonotonic deadline, const TActorContext &ctx); - void SendNodesList(TActorId recipient, const TActorContext &ctx); + void SendNodesList(TActorId recipient, bool onlyAliveNodes, const TActorContext &ctx); void SendNodesList(const TActorContext &ctx); + void InvalidateListNodesCache(); void PendingRequestAnswered(ui32 domain, const TActorContext &ctx); void UpdateState(const NKikimrNodeBroker::TNodesInfo &rec, const TActorContext &ctx); @@ -288,8 +301,9 @@ private: private: TIntrusivePtr<TTableNameserverSetup> StaticConfig; std::array<TDynamicConfigPtr, DOMAINS_COUNT> DynamicConfigs; - TVector<TActorId> ListNodesQueue; - TIntrusivePtr<TListNodesCache> ListNodesCache; + TVector<std::pair<TActorId, bool>> ListNodesQueue; + TIntrusivePtr<TListNodesCache> ListNodesCacheAll; + TIntrusivePtr<TListNodesCache> ListNodesCacheAlive; TBridgeInfo::TPtr BridgeInfo; // When ListNodes requests are sent to NodeBroker tablets this @@ -301,7 +315,9 @@ private: THashSet<TActorId> StaticNodeChangeSubscribers; bool SubscribedToConsoleNSConfig = false; + bool EnableLongLease = false; bool EnableDeltaProtocol = false; + EProtocolState ProtocolState = EProtocolState::Connecting; bool SyncInProgress = false; ui64 SyncCookie = 0; diff --git a/ydb/core/mind/dynamic_nameserver_mon.cpp b/ydb/core/mind/dynamic_nameserver_mon.cpp index 8b19303d0f0..36267c13d88 100644 --- a/ydb/core/mind/dynamic_nameserver_mon.cpp +++ b/ydb/core/mind/dynamic_nameserver_mon.cpp @@ -91,6 +91,7 @@ void OutputNodeInfo(ui32 nodeId, const TTableNameserverSetup::TNodeInfo &info, IOutputStream &str, TInstant expire = TInstant::Zero(), + ENodeLiveness liveness = ENodeLiveness::Alive, const TString &cl = "") { str << "<tr class='" << cl << "'>" << Endl @@ -102,6 +103,7 @@ void OutputNodeInfo(ui32 nodeId, << " <td>" << info.Location.ToString() << "</td>" << Endl; if (expire) str << "<td>" << ToString(expire) << "</td>" << Endl; + str << "<td>" << (liveness == ENodeLiveness::Alive ? "Alive" : "Dead") << "</td>" << Endl; str << "</tr>" << Endl; } @@ -142,7 +144,8 @@ void OutputStaticNodes(const TTableNameserverSetup &setup, void OutputDynamicNodes(const TString &domain, TDynamicConfigPtr config, - IOutputStream &str) + IOutputStream &str, + bool enableLongLease) { str << "<div><table class='nodes'>" << Endl << " <caption>Dynamic nodes in " << domain @@ -159,6 +162,7 @@ void OutputDynamicNodes(const TString &domain, << " <th>Rack</th>" << Endl << " <th>Body</th>" << Endl << " <th>Expire</th>" << Endl + << " <th>Liveness</th>" << Endl << " </tr>" << Endl << " </thead>" << Endl << " <tbody class='center-align'>" << Endl; @@ -168,7 +172,7 @@ void OutputDynamicNodes(const TString &domain, ids.insert(pr.first); for (auto id : ids) { auto &node = config->DynamicNodes.at(id); - OutputNodeInfo(id, node, str, node.Expire); + OutputNodeInfo(id, node, str, node.EffectiveExpire(enableLongLease), node.Liveness); } ids.clear(); @@ -217,7 +221,7 @@ void TDynamicNameserver::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext OutputStaticNodes(*StaticConfig, str); if (const auto& domain = AppData(ctx)->DomainsInfo->Domain) { - OutputDynamicNodes(domain->Name, DynamicConfigs[domain->DomainUid], str); + OutputDynamicNodes(domain->Name, DynamicConfigs[domain->DomainUid], str, EnableLongLease); } } ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); diff --git a/ydb/core/mind/lease_holder.cpp b/ydb/core/mind/lease_holder.cpp index eae4144b723..d6c4edf1050 100644 --- a/ydb/core/mind/lease_holder.cpp +++ b/ydb/core/mind/lease_holder.cpp @@ -1,13 +1,17 @@ #include "lease_holder.h" #include "node_broker.h" +#include <ydb/core/base/appdata.h> +#include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/configs_dispatcher.h> +#include <ydb/core/cms/console/console.h> +#include <ydb/core/mon/mon.h> +#include <ydb/core/protos/feature_flags.pb.h> + #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> -#include <ydb/core/mon/mon.h> #include <ydb/library/actors/core/mon.h> #include <ydb/library/actors/util/should_continue.h> -#include <ydb/core/base/appdata.h> -#include <ydb/core/base/tablet_pipe.h> #include <library/cpp/monlib/service/pages/templates.h> @@ -41,6 +45,7 @@ public: TLeaseHolder(TInstant expire) : LastPingEpoch(0) , Expire(expire) + , ExpireV2(expire) { } @@ -54,6 +59,12 @@ public: false, ctx.ActorSystem(), ctx.SelfID); } + EnableLongLease = AppData()->FeatureFlags.GetEnableNodeBrokerLongLease(); + + const ui32 featureFlagsItem = NKikimrConsole::TConfigItem::FeatureFlagsItem; + Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), + new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(featureFlagsItem)); + Become(&TThis::StatePing); ScheduleExpire(ctx); @@ -69,6 +80,8 @@ private: HFunc(TEvPrivate::TEvExpire, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, HandleIdle); HFunc(TEvTabletPipe::TEvClientConnected, HandleIdle); + HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); IgnoreFunc(TEvNodeBroker::TEvExtendLeaseResponse); default: @@ -85,6 +98,8 @@ private: HFunc(TEvPrivate::TEvExpire, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); + HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); default: Y_ABORT("TLeaseHolder::StatePing unexpected event type: %" PRIx32 " event: %s", @@ -138,6 +153,18 @@ private: OnPipeDestroyed(ctx); } + void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev, const TActorContext &ctx) { + const auto& record = ev->Get()->Record; + if (record.HasConfig() && record.GetConfig().HasFeatureFlags()) { + const auto& featureFlags = record.GetConfig().GetFeatureFlags(); + if (EnableLongLease != featureFlags.GetEnableNodeBrokerLongLease()) { + EnableLongLease = featureFlags.GetEnableNodeBrokerLongLease(); + ScheduleExpire(ctx); + } + } + Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie); + } + void Connect(const TActorContext &ctx) { NTabletPipe::TClientConfig config; @@ -160,31 +187,27 @@ private: } Expire = TInstant::MicroSeconds(rec.GetExpire()); + ExpireV2 = TInstant::MicroSeconds(rec.HasExpireV2() ? rec.GetExpireV2() : rec.GetExpire()); + LastResponse = ctx.Now(); LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, - "Node has now extended lease expiring " << ToString(Expire)); - - if (rec.HasEpoch()) { - LastPingEpoch = rec.GetEpoch().GetId(); - EpochEnd = TInstant::FromValue(rec.GetEpoch().GetEnd()); + "Node has now extended lease expiring " << ToString(EffectiveExpire())); - if (Expire != TInstant::Max()) { - Y_ABORT_UNLESS(Expire > EpochEnd); - Y_ABORT_UNLESS(rec.GetExpire() == rec.GetEpoch().GetNextEnd()); + Y_ABORT_UNLESS(rec.HasEpoch()); + LastPingEpoch = rec.GetEpoch().GetId(); + EpochEnd = TInstant::FromValue(rec.GetEpoch().GetEnd()); + NextEpochEnd = TInstant::FromValue(rec.GetEpoch().GetNextEnd()); - ui64 window = (Expire - EpochEnd).GetValue() / 2; - Y_ABORT_UNLESS(window); + if (EffectiveExpire() != TInstant::Max()) { + Y_ABORT_UNLESS(EffectiveExpire() >= NextEpochEnd); + ui64 window = (NextEpochEnd - EpochEnd).GetValue() / 2; + Y_ABORT_UNLESS(window); - NextPing = EpochEnd + TDuration::FromValue(RandomNumber<ui64>(window)); - } - } else { - NextPing = ctx.Now() + (Expire - ctx.Now()) / 2; - } - - if (Expire != TInstant::Max()) + NextPing = EpochEnd + TDuration::FromValue(RandomNumber<ui64>(window)); ctx.Schedule(NextPing - ctx.Now(), new TEvents::TEvWakeup()); - else + } else { NextPing = TInstant::Max(); + } Become(&TThis::StateIdle); } @@ -203,11 +226,15 @@ private: TStringStream str; HTML(str) { PRE() { - str << "Lease expires: " << ToString(Expire) << Endl + str << "Lease expires: " << ToString(EffectiveExpire()) << Endl << "Last lease extension: " << ToString(LastResponse) << Endl << "Last ping epoch: " << LastPingEpoch << Endl << "Epoch end: " << ToString(EpochEnd) << Endl - << "Next ping at: " << ToString(NextPing) << Endl; + << "Next epoch end: " << ToString(NextEpochEnd) << Endl + << "Next ping at: " << ToString(NextPing) << Endl + << "Enable long lease: " << EnableLongLease << Endl + << "Expire: " << ToString(Expire) << Endl + << "ExpireV2: " << ToString(ExpireV2) << Endl; } } ctx.Send(ev->Sender, new TEvHttpInfoRes(str.Str())); @@ -215,13 +242,15 @@ private: void ScheduleExpire(const TActorContext &ctx) { - if (Expire != TInstant::Max()) - ctx.Schedule(Expire - ctx.Now(), new TEvPrivate::TEvExpire); + ExpireCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); + if (EffectiveExpire() != TInstant::Max()) { + ctx.Schedule(EffectiveExpire() - ctx.Now(), new TEvPrivate::TEvExpire, ExpireCookieHolder.Get()); + } } void Handle(TEvPrivate::TEvExpire::TPtr &,const TActorContext &ctx) { - if (Expire <= ctx.Now()) + if (EffectiveExpire() <= ctx.Now()) StopNode(ctx); else ScheduleExpire(ctx); @@ -240,13 +269,24 @@ private: return t.ToRfc822StringLocal(); } + TInstant EffectiveExpire() const + { + return EnableLongLease ? ExpireV2 : Expire; + } + private: TActorId NodeBrokerPipe; ui64 LastPingEpoch; TInstant EpochEnd; - TInstant Expire; + TInstant NextEpochEnd; + TInstant LastResponse; TInstant NextPing; + + bool EnableLongLease = false; + TInstant Expire; + TInstant ExpireV2; + NActors::TSchedulerCookieHolder ExpireCookieHolder; }; IActor *CreateLeaseHolder(TInstant expire) diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp index 494cef4c4de..8d67ac47e16 100644 --- a/ydb/core/mind/node_broker.cpp +++ b/ydb/core/mind/node_broker.cpp @@ -104,6 +104,7 @@ void TNodeBroker::OnActivateExecutor(const TActorContext &ctx) MaxDynamicId = Max(MinDynamicId, (ui64)Min(appData->DynamicNameserviceConfig->MaxDynamicNodeId, TActorId::MaxNodeId)); EnableStableNodeNames = appData->FeatureFlags.GetEnableStableNodeNames(); + EnableLongLease = appData->FeatureFlags.GetEnableNodeBrokerLongLease(); Executor()->RegisterExternalTabletCounters(TabletCountersPtr); Committed.ClearState(); @@ -168,6 +169,9 @@ bool TNodeBroker::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, << " Location: " << node.Location.ToString() << Endl << " Lease: " << node.Lease << Endl << " Expire: " << node.ExpirationString() << Endl + << " ExpireV2: " << node.ExpirationV2String() << Endl + << " AliveUntil: " << node.AliveUntilString() << Endl + << " Liveness: " << (node.Liveness == ENodeLiveness::Dead ? "Dead" : "Alive") << Endl << " AuthorizedByCertificate: " << (node.AuthorizedByCertificate ? "true" : "false") << Endl << " ServicedSubDomain: " << node.ServicedSubDomain << Endl << " SlotIndex: " << node.SlotIndex << Endl; @@ -303,22 +307,38 @@ void TNodeBroker::TState::AddNode(const TNodeInfo &info) } } +bool TNodeBroker::TState::IsLeaseExtendable(const TNodeInfo &node) const +{ + return node.Expire < Epoch.NextEnd || node.ExpireV2 < Epoch.NextEnd + LeaseDuration; +} + void TNodeBroker::TState::ExtendLease(TNodeInfo &node) { node.Version = Epoch.Version + 1; + ++node.Lease; node.Expire = Epoch.NextEnd; + node.ExpireV2 = Epoch.NextEnd + LeaseDuration; + + node.AliveUntil = Epoch.NextEnd; + node.Liveness = ENodeLiveness::Alive; LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, - LogPrefix() << " Extended lease of " << node.IdString() << " up to " - << node.ExpirationString() << " (lease " << node.Lease << ")"); + LogPrefix() << " Extended lease of " << node.IdString() << " up to v1: " + << node.ExpirationString() << " and v2: " << node.ExpirationV2String() + << " (lease " << node.Lease << ")"); } void TNodeBroker::TState::FixNodeId(TNodeInfo &node) { node.Version = Epoch.Version + 1; + ++node.Lease; node.Expire = TInstant::Max(); + node.ExpireV2 = TInstant::Max(); + + node.AliveUntil = TInstant::Max(); + node.Liveness = ENodeLiveness::Alive; LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, LogPrefix() << " Fix ID for node " << node.IdString()); @@ -474,6 +494,8 @@ void TNodeBroker::FillNodeInfo(const TNodeInfo &node, info.SetResolveHost(node.ResolveHost); info.SetAddress(node.Address); info.SetExpire(node.Expire.GetValue()); + info.SetExpireV2(node.ExpireV2.GetValue()); + info.SetLiveness(static_cast<ui32>(node.Liveness)); node.Location.Serialize(info.MutableLocation(), false); FillNodeName(node.SlotIndex, info); } @@ -489,8 +511,17 @@ void TNodeBroker::FillNodeName(const std::optional<ui32> &slotIndex, void TNodeBroker::TState::ComputeNextEpochDiff(TStateDiff &diff) { + if (Self->EnableLongLease) { + for (auto &pr : Nodes) { + if (pr.second.AliveUntil <= Epoch.End && pr.second.Liveness == ENodeLiveness::Alive) { + diff.NodesToMakeDead.push_back(pr.first); + } + } + } + for (auto &pr : Nodes) { - if (pr.second.Expire <= Epoch.End) + auto expire = Self->EnableLongLease ? pr.second.ExpireV2 : pr.second.Expire; + if (expire <= Epoch.End) diff.NodesToExpire.push_back(pr.first); } @@ -522,6 +553,17 @@ void TNodeBroker::TState::ApplyStateDiff(const TStateDiff &diff) Nodes.erase(it); } + for (auto id : diff.NodesToMakeDead) { + auto it = Nodes.find(id); + Y_ABORT_UNLESS(it != Nodes.end()); + + LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, + LogPrefix() << " Node " << it->second.IdString() << " is marked as dead"); + + it->second.Liveness = ENodeLiveness::Dead; + it->second.Version = diff.NewEpoch.Version; + } + for (auto id : diff.NodesToRemove) { auto it = ExpiredNodes.find(id); Y_ABORT_UNLESS(it != ExpiredNodes.end()); @@ -794,11 +836,19 @@ void TNodeBroker::TState::LoadConfigFromProto(const NKikimrNodeBroker::TConfig & EpochDuration = TDuration::MicroSeconds(config.GetEpochDuration()); if (EpochDuration < MIN_LEASE_DURATION) { LOG_ERROR_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, - LogPrefix() << " Configured lease duration (" << EpochDuration << ") is too" + LogPrefix() << " Configured epoch duration (" << EpochDuration << ") is too" " small. Using min. value: " << MIN_LEASE_DURATION); EpochDuration = MIN_LEASE_DURATION; } + LeaseDuration = TDuration::MicroSeconds(config.GetLeaseDuration()); + if (LeaseDuration < MIN_LEASE_DURATION) { + LOG_ERROR_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER, + LogPrefix() << " Configured lease duration (" << LeaseDuration << ") is too" + " small. Using min. value: " << MIN_LEASE_DURATION); + LeaseDuration = MIN_LEASE_DURATION; + } + StableNodeNamePrefix = config.GetStableNodeNamePrefix(); BannedIds.clear(); @@ -858,6 +908,7 @@ void TNodeBroker::TDirtyState::DbAddNode(const TNodeInfo &node, << " location=" << node.Location.ToString() << " lease=" << node.Lease << " expire=" << node.ExpirationString() + << " expirev2=" << node.ExpirationV2String() << " servicedsubdomain=" << node.ServicedSubDomain << " slotindex=" << node.SlotIndex << " authorizedbycertificate=" << (node.AuthorizedByCertificate ? "true" : "false")); @@ -878,6 +929,9 @@ void TNodeBroker::TDirtyState::DbAddNode(const TNodeInfo &node, .Update<T::Address>(node.Address) .Update<T::Lease>(node.Lease) .Update<T::Expire>(node.Expire.GetValue()) + .Update<T::ExpireV2>(node.ExpireV2.GetValue()) + .Update<T::AliveUntil>(node.AliveUntil.GetValue()) + .Update<T::Liveness>(node.Liveness) .Update<T::Location>(node.Location.GetSerializedLocation()) .Update<T::ServicedSubDomain>(node.ServicedSubDomain) .Update<T::AuthorizedByCertificate>(node.AuthorizedByCertificate); @@ -896,6 +950,7 @@ void TNodeBroker::TDirtyState::DbApplyStateDiff(const TStateDiff &diff, TTransactionContext &txc) { DbUpdateNodes(diff.NodesToExpire, txc); + DbUpdateNodes(diff.NodesToMakeDead, txc); DbUpdateNodes(diff.NodesToRemove, txc); DbUpdateEpoch(diff.NewEpoch, txc); DbUpdateApproxEpochStart(diff.NewApproxEpochStart, txc); @@ -1088,6 +1143,8 @@ TNodeBroker::TDbChanges TNodeBroker::TDirtyState::DbLoadNodes(auto &nodesRowset, AddNode(info); } else { auto expire = TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::Expire>()); + auto expireV2 = Max(expire, TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::ExpireV2>())); + std::optional<TNodeLocation> modernLocation; if (nodesRowset.template HaveValue<Schema::Nodes::Location>()) { modernLocation.emplace(TNodeLocation::FromSerialized, nodesRowset.template GetValue<Schema::Nodes::Location>()); @@ -1108,12 +1165,22 @@ TNodeBroker::TDbChanges TNodeBroker::TDirtyState::DbLoadNodes(auto &nodesRowset, info.Lease = nodesRowset.template GetValue<Schema::Nodes::Lease>(); info.Expire = expire; + info.ExpireV2 = expireV2; + info.AliveUntil = Max(expire, TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::AliveUntil>())); + info.Liveness = nodesRowset.template GetValue<Schema::Nodes::Liveness>(); + info.ServicedSubDomain = TSubDomainKey(nodesRowset.template GetValueOrDefault<Schema::Nodes::ServicedSubDomain>()); if (nodesRowset.template HaveValue<Schema::Nodes::SlotIndex>()) { info.SlotIndex = nodesRowset.template GetValue<Schema::Nodes::SlotIndex>(); } info.AuthorizedByCertificate = nodesRowset.template GetValue<Schema::Nodes::AuthorizedByCertificate>(); - info.State = expire > Epoch.Start ? ENodeState::Active : ENodeState::Expired; + + if (Self->EnableLongLease) { + info.State = expireV2 > Epoch.Start ? ENodeState::Active : ENodeState::Expired; + } else { + // Dead nodes stay active until epoch end + info.State = info.Liveness == ENodeLiveness::Dead || expire > Epoch.Start ? ENodeState::Active : ENodeState::Expired; + } AddNode(info); LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, @@ -1424,6 +1491,7 @@ void TNodeBroker::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const auto& appConfig = ev->Get()->Record.GetConfig(); if (appConfig.HasFeatureFlags()) { EnableStableNodeNames = appConfig.GetFeatureFlags().GetEnableStableNodeNames(); + EnableLongLease = appConfig.GetFeatureFlags().GetEnableNodeBrokerLongLease(); } if (ev->Get()->Record.HasLocal() && ev->Get()->Record.GetLocal()) { @@ -1757,11 +1825,14 @@ TNodeBroker::TNodeInfo::TNodeInfo(ui32 nodeId, ENodeState state, ui64 version, c TNodeLocation(schema.GetLocation())) , Lease(schema.GetLease()) , Expire(TInstant::MicroSeconds(schema.GetExpire())) + , ExpireV2(TInstant::MicroSeconds(schema.GetExpireV2())) , AuthorizedByCertificate(schema.GetAuthorizedByCertificate()) , SlotIndex(schema.GetSlotIndex()) , ServicedSubDomain(schema.GetServicedSubDomain()) , State(state) , Version(version) + , AliveUntil(TInstant::MicroSeconds(schema.GetAliveUntil())) + , Liveness(static_cast<ENodeLiveness>(schema.GetLiveness())) {} TNodeBroker::TNodeInfo::TNodeInfo(ui32 nodeId, ENodeState state, ui64 version) @@ -1775,7 +1846,9 @@ bool TNodeBroker::TNodeInfo::EqualCachedData(const TNodeInfo &other) const && ResolveHost == other.ResolveHost && Address == other.Address && Location == other.Location - && Expire == other.Expire; + && Expire == other.Expire + && ExpireV2 == other.ExpireV2 + && Liveness == other.Liveness; } bool TNodeBroker::TNodeInfo::EqualExceptVersion(const TNodeInfo &other) const @@ -1811,6 +1884,9 @@ TString TNodeBroker::TNodeInfo::ToString() const << ", Address: " << Address << ", Lease: " << Lease << ", Expire: " << ExpirationString() + << ", ExpireV2: " << ExpirationV2String() + << ", AliveUntil: " << AliveUntilString() + << ", Liveness: " << static_cast<ui32>(Liveness) << ", Location: " << Location.ToString() << ", AuthorizedByCertificate: " << AuthorizedByCertificate << ", SlotIndex: " << SlotIndex @@ -1827,6 +1903,9 @@ TNodeInfoSchema TNodeBroker::TNodeInfo::SerializeToSchema() const { serialized.SetAddress(Address); serialized.SetLease(Lease); serialized.SetExpire(Expire.MicroSeconds()); + serialized.SetExpireV2(ExpireV2.MicroSeconds()); + serialized.SetAliveUntil(AliveUntil.MicroSeconds()); + serialized.SetLiveness(static_cast<ui32>(Liveness)); Location.Serialize(serialized.MutableLocation(), false); serialized.MutableServicedSubDomain()->CopyFrom(ServicedSubDomain); if (SlotIndex.has_value()) { diff --git a/ydb/core/mind/node_broker.h b/ydb/core/mind/node_broker.h index 0e3856c6bcb..9ab24f2911e 100644 --- a/ydb/core/mind/node_broker.h +++ b/ydb/core/mind/node_broker.h @@ -25,6 +25,11 @@ namespace NKikimr { namespace NNodeBroker { +enum class ENodeLiveness : ui8 { + Alive = 0, + Dead = 1, +}; + struct TEpochInfo { ui64 Id = 0; ui64 Version = 0; diff --git a/ydb/core/mind/node_broker__extend_lease.cpp b/ydb/core/mind/node_broker__extend_lease.cpp index 030a99f3324..2a23c40adb7 100644 --- a/ydb/core/mind/node_broker__extend_lease.cpp +++ b/ydb/core/mind/node_broker__extend_lease.cpp @@ -58,7 +58,7 @@ public: return Error(TStatus::WRONG_REQUEST, "Node ID is banned", ctx); auto &node = it->second; - if (node.Expire < Self->Dirty.Epoch.NextEnd) { + if (Self->Dirty.IsLeaseExtendable(node)) { Self->Dirty.ExtendLease(node); Self->Dirty.DbAddNode(node, txc); Self->Dirty.UpdateEpochVersion(); @@ -67,6 +67,7 @@ public: } Response->Record.SetExpire(node.Expire.GetValue()); + Response->Record.SetExpireV2(node.ExpireV2.GetValue()); Response->Record.MutableStatus()->SetCode(TStatus::OK); Self->Dirty.Epoch.Serialize(*Response->Record.MutableEpoch()); diff --git a/ydb/core/mind/node_broker__register_node.cpp b/ydb/core/mind/node_broker__register_node.cpp index 64266d34695..e53d3df77cb 100644 --- a/ydb/core/mind/node_broker__register_node.cpp +++ b/ydb/core/mind/node_broker__register_node.cpp @@ -73,7 +73,6 @@ public: auto host = rec.GetHost(); ui16 port = (ui16)rec.GetPort(); TString addr = rec.GetAddress(); - auto expire = rec.GetFixedNodeId() ? TInstant::Max() : Self->Dirty.Epoch.NextEnd; LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxRegisterNode Execute"); LOG_INFO_S(ctx, NKikimrServices::NODE_BROKER, @@ -135,7 +134,7 @@ public: Self->Dirty.FixNodeId(node); Self->Dirty.DbAddNode(node, txc); FixNodeId = true; - } else if (!node.IsFixed() && node.Expire < expire) { + } else if (Self->Dirty.IsLeaseExtendable(node)) { Self->Dirty.ExtendLease(node); Self->Dirty.DbAddNode(node, txc); ExtendLease = true; @@ -170,7 +169,16 @@ public: Node = MakeHolder<TNodeInfo>(NodeId, rec.GetAddress(), host, rec.GetResolveHost(), port, loc); Node->AuthorizedByCertificate = rec.GetAuthorizedByCertificate(); Node->Lease = 1; - Node->Expire = expire; + if (rec.GetFixedNodeId()) { + Node->Expire = TInstant::Max(); + Node->ExpireV2 = TInstant::Max(); + Node->AliveUntil = TInstant::Max(); + } else { + Node->Expire = Self->Dirty.Epoch.NextEnd; + Node->ExpireV2 = Self->Dirty.Epoch.NextEnd + Self->Dirty.LeaseDuration; + Node->AliveUntil = Self->Dirty.Epoch.NextEnd; + } + Node->Liveness = ENodeLiveness::Alive; Node->Version = Self->Dirty.Epoch.Version + 1; Node->State = ENodeState::Active; diff --git a/ydb/core/mind/node_broker__scheme.h b/ydb/core/mind/node_broker__scheme.h index a740a7ac7e7..46dffe5451f 100644 --- a/ydb/core/mind/node_broker__scheme.h +++ b/ydb/core/mind/node_broker__scheme.h @@ -14,6 +14,7 @@ namespace NKikimr { namespace NNodeBroker { enum class ENodeState : ui8; +enum class ENodeLiveness : ui8; struct Schema : NIceDb::Schema { enum class EMainNodesTable : ui64 { @@ -33,11 +34,14 @@ struct Schema : NIceDb::Schema { // struct Body : Column<9, NScheme::NTypeIds::Uint64> {}; struct Lease : Column<10, NScheme::NTypeIds::Uint32> {}; struct Expire : Column<11, NScheme::NTypeIds::Uint64> {}; + struct ExpireV2 : Column<17, NScheme::NTypeIds::Uint64> {}; struct Location : Column<12, NScheme::NTypeIds::String> {}; struct ServicedSubDomain : Column<13, NScheme::NTypeIds::String> { using Type = NKikimrSubDomains::TDomainKey; }; struct SlotIndex : Column<14, NScheme::NTypeIds::Uint32> {}; struct AuthorizedByCertificate : Column<15, NScheme::NTypeIds::Bool> {}; //struct BridgePileId : Column<16, NScheme::NTypeIds::Uint32> {}; + struct AliveUntil : Column<18, NScheme::NTypeIds::Uint64> {}; + struct Liveness : Column<19, NScheme::NTypeIds::Uint8> { using Type = ENodeLiveness; }; using TKey = TableKey<ID>; using TColumns = TableColumns< @@ -51,7 +55,10 @@ struct Schema : NIceDb::Schema { Location, ServicedSubDomain, SlotIndex, - AuthorizedByCertificate + AuthorizedByCertificate, + ExpireV2, + AliveUntil, + Liveness >; }; diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h index 177be517b72..4d30c86f4df 100644 --- a/ydb/core/mind/node_broker_impl.h +++ b/ydb/core/mind/node_broker_impl.h @@ -114,7 +114,7 @@ private: bool IsFixed() const { - return Expire == TInstant::Max(); + return Expire == TInstant::Max() || ExpireV2 == TInstant::Max(); } static TString ExpirationString(TInstant expire) @@ -133,20 +133,34 @@ private: return ExpirationString(Expire); } + TString ExpirationV2String() const + { + return ExpirationString(ExpireV2); + } + + TString AliveUntilString() const + { + return AliveUntil.ToRfc822StringLocal(); + } + // Lease is incremented each time node extends its lifetime. ui32 Lease = 0; TInstant Expire; + TInstant ExpireV2; bool AuthorizedByCertificate = false; std::optional<ui32> SlotIndex; TSubDomainKey ServicedSubDomain; ENodeState State = ENodeState::Removed; ui64 Version = 0; + TInstant AliveUntil; + ENodeLiveness Liveness = ENodeLiveness::Alive; }; // State changes to apply while moving to the next epoch. struct TStateDiff { TVector<ui32> NodesToExpire; TVector<ui32> NodesToRemove; + TVector<ui32> NodesToMakeDead; TEpochInfo NewEpoch; TApproximateEpochStartInfo NewApproxEpochStart; }; @@ -337,6 +351,8 @@ private: const TActorContext &ctx); bool EnableStableNodeNames = false; + bool EnableLongLease = false; + ui64 MaxStaticId; ui64 MinDynamicId; ui64 MaxDynamicId; @@ -367,6 +383,7 @@ private: // Internal state modifiers. Don't affect DB. void RegisterNewNode(const TNodeInfo &info); void AddNode(const TNodeInfo &info); + bool IsLeaseExtendable(const TNodeInfo &node) const; void ExtendLease(TNodeInfo &node); void FixNodeId(TNodeInfo &node); void RecomputeFreeIds(); @@ -397,6 +414,7 @@ private: // Current config. NKikimrNodeBroker::TConfig Config; TDuration EpochDuration = TDuration::Hours(1); + TDuration LeaseDuration = TDuration::Hours(72); TVector<std::pair<ui32, ui32>> BannedIds; ui64 ConfigSubscriptionId = 0; TString StableNodeNamePrefix = "slot-"; diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp index 017f3b517af..41c7db5a91b 100644 --- a/ydb/core/mind/node_broker_ut.cpp +++ b/ydb/core/mind/node_broker_ut.cpp @@ -2,6 +2,8 @@ #include "node_broker__scheme.h" #include "dynamic_nameserver_impl.h" +#include <ydb/core/cms/console/console.h> + #include <ydb/core/testlib/basics/appdata.h> #include <ydb/core/testlib/basics/storage.h> #include <ydb/core/testlib/basics/helpers.h> @@ -76,7 +78,8 @@ THashMap<ui32, TIntrusivePtr<TNodeWardenConfig>> NodeWardenConfigs; void SetupServices(TTestActorRuntime &runtime, ui32 maxDynNodes, - bool enableNodeBrokerDeltaProtocol) + bool enableNodeBrokerDeltaProtocol, + bool enableNodeBrokerLongLease = false) { const ui32 domainsNum = 1; const ui32 disksInDomain = 1; @@ -191,6 +194,7 @@ void SetupServices(TTestActorRuntime &runtime, app.FeatureFlags.SetEnableNodeBrokerSingleDomainMode(true); app.FeatureFlags.SetEnableStableNodeNames(true); app.FeatureFlags.SetEnableNodeBrokerDeltaProtocol(enableNodeBrokerDeltaProtocol); + app.FeatureFlags.SetEnableNodeBrokerLongLease(enableNodeBrokerLongLease); runtime.Initialize(app.Unwrap()); @@ -261,6 +265,50 @@ void SetEpochDuration(TTestActorRuntime& runtime, SetConfig(runtime, sender, config); } +void SetLeaseDuration(TTestActorRuntime& runtime, + TActorId sender, + TDuration lease) +{ + NKikimrNodeBroker::TConfig config; + config.SetLeaseDuration(lease.GetValue()); + SetConfig(runtime, sender, config); +} + +// Toggle the EnableNodeBrokerLongLease feature flag on the running NodeBroker +// tablet at runtime by delivering a config notification (as the configs +// dispatcher would in production) and waiting for the acknowledgement. +void SetNodeBrokerLongLease(TTestActorRuntime& runtime, + TActorId sender, + bool enable) +{ + auto event = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationRequest>(); + auto& featureFlags = *event->Record.MutableConfig()->MutableFeatureFlags(); + featureFlags.SetEnableStableNodeNames(true); + featureFlags.SetEnableNodeBrokerLongLease(enable); + runtime.SendToPipe(MakeNodeBrokerID(), sender, event.Release(), 0, GetPipeConfigWithRetries()); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(handle); +} + +// Toggle the EnableNodeBrokerLongLease feature flag on the node-0 dynamic +// nameserver at runtime via a config notification, and wait for the ack. +void SetNameserverLongLease(TTestActorRuntime& runtime, + TActorId sender, + bool enable) +{ + auto event = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationRequest>(); + auto& featureFlags = *event->Record.MutableConfig()->MutableFeatureFlags(); + // Keep the delta protocol flag unchanged (these tests use the epoch + // protocol); a differing value would make the nameserver reset its pipe. + featureFlags.SetEnableNodeBrokerDeltaProtocol(false); + featureFlags.SetEnableNodeBrokerLongLease(enable); + runtime.Send(new IEventHandle(GetNameserviceActorId(), sender, event.Release())); + + TAutoPtr<IEventHandle> handle; + runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(handle); +} + void AsyncSetBannedIds(TTestActorRuntime& runtime, TActorId sender, const TVector<std::pair<ui32, ui32>> ids) @@ -285,7 +333,8 @@ void SetBannedIds(TTestActorRuntime& runtime, void Setup(TTestActorRuntime& runtime, ui32 maxDynNodes = 3, const TVector<TString>& databases = {}, - bool enableNodeBrokerDeltaProtocol = false) + bool enableNodeBrokerDeltaProtocol = false, + bool enableNodeBrokerLongLease = false) { using namespace NMalloc; TMallocInfo mallocInfo = MallocInfo(); @@ -301,7 +350,7 @@ void Setup(TTestActorRuntime& runtime, runtime.SetScheduledEventFilter(scheduledFilter); SetupLogging(runtime); - SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol); + SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol, enableNodeBrokerLongLease); TActorId sender = runtime.AllocateEdgeActor(); ui32 txId = 100; @@ -670,6 +719,27 @@ NKikimrNodeBroker::TEpoch CheckNodesList(TTestActorRuntime &runtime, return rec.GetEpoch(); } +void CheckNodeLiveness(TTestActorRuntime &runtime, + TActorId sender, + ui32 nodeId, + ENodeLiveness liveness) +{ + const auto &rec = ListNodes(runtime, sender); + // The nodes list is a concatenation of the epoch snapshot and per-node + // deltas, so a node may appear more than once; the last occurrence wins. + bool found = false; + ui32 actual = 0; + for (const auto &node : rec.GetNodes()) { + if (node.GetNodeId() == nodeId) { + found = true; + actual = node.GetLiveness(); + } + } + UNIT_ASSERT_C(found, "node " << nodeId << " is not in the alive nodes list"); + UNIT_ASSERT_VALUES_EQUAL_C(actual, static_cast<ui32>(liveness), + "unexpected liveness for node " << nodeId); +} + void CheckNodeInfo(TTestActorRuntime &runtime, TActorId sender, ui32 nodeId, @@ -956,6 +1026,33 @@ THolder<TEvInterconnect::TEvNodesInfo> GetNameserverNodesListEv(TTestActorRuntim return IEventHandle::Release<TEvInterconnect::TEvNodesInfo>(handle); } +THolder<TEvInterconnect::TEvNodesInfo> GetNameserverNodesListEv(TTestActorRuntime &runtime, TActorId sender, + bool onlyAliveNodes) { + runtime.Send(new IEventHandle(GetNameserviceActorId(), sender, + new TEvInterconnect::TEvListNodes(false, onlyAliveNodes))); + + TAutoPtr<IEventHandle> handle; + auto reply = runtime.GrabEdgeEventRethrow<TEvInterconnect::TEvNodesInfo>(handle); + UNIT_ASSERT(reply); + + return IEventHandle::Release<TEvInterconnect::TEvNodesInfo>(handle); +} + +// Returns the set of dynamic (non-static) node ids the nameservice reports. +// With onlyAliveNodes the request asks the nameservice to omit nodes the +// NodeBroker has marked dead (only meaningful when the long lease feature is on). +THashSet<ui32> GetNameserverDynamicNodeIds(TTestActorRuntime &runtime, TActorId sender, + bool onlyAliveNodes = false) +{ + ui32 maxStaticNodeId = runtime.GetAppData().DynamicNameserviceConfig->MaxStaticNodeId; + auto ev = GetNameserverNodesListEv(runtime, sender, onlyAliveNodes); + THashSet<ui32> ids; + for (auto &node : ev->Nodes) + if (node.NodeId > maxStaticNodeId) + ids.insert(node.NodeId); + return ids; +} + void GetNameserverNodesList(TTestActorRuntime &runtime, TActorId sender, THashMap<ui32, TEvInterconnect::TNodeInfo> &nodes, @@ -1395,6 +1492,204 @@ Y_UNIT_TEST_SUITE(TNodeBrokerTest) { CheckLeaseExtension(runtime, sender, NODE2, TStatus::OK, epoch, true); } + Y_UNIT_TEST(LongLeaseNodeBecomesDeadThenExpires) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + // Lease duration equal to a single (default) epoch: a node without pings + // survives one extra epoch (AliveUntil) before becoming Dead and another + // epoch (ExpireV2) before it finally expires. + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + auto epoch = GetEpoch(runtime, sender); + // Register node NODE1, it is Alive. + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + // After the first epoch the long lease keeps the node Alive. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + // After the second epoch AliveUntil has passed: the node becomes Dead, + // but is not expired yet (ExpireV2 is still in the future). + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // After the third epoch ExpireV2 has passed: the node expires. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId()); + } + + Y_UNIT_TEST(LongLeasePingKeepsNodeAlive) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + auto epoch = GetEpoch(runtime, sender); + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + + // As long as the node keeps extending its lease each epoch, it never + // becomes Dead nor expires, even with the long lease feature enabled. + for (ui32 i = 0; i < 5; ++i) { + epoch = WaitForEpochUpdate(runtime, sender); + CheckLeaseExtension(runtime, sender, NODE1, TStatus::OK, epoch); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + } + } + + Y_UNIT_TEST(LongLeaseDeadNodeRevivedByRegistration) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + auto epoch = GetEpoch(runtime, sender); + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + + // Let the node become Dead. + WaitForEpochUpdate(runtime, sender); + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // Re-registration of a Dead node revives it (extends lease, back to Alive). + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + // The revived node survives the following epoch as Alive. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + } + + Y_UNIT_TEST(LongLeaseDeadStatePersistedAfterRestart) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + auto epoch = GetEpoch(runtime, sender); + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + + // Let the node become Dead. + WaitForEpochUpdate(runtime, sender); + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // Dead state (AliveUntil/Liveness) must survive a tablet restart. + RestartNodeBroker(runtime); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // The node still expires by ExpireV2 in the next epoch. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId()); + } + + Y_UNIT_TEST(LongLeaseDisabledNodeExpiresByExpire) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ false); + TActorId sender = runtime.AllocateEdgeActor(); + + // Even with a lease duration configured, when the feature is disabled the + // node expires by Expire (a single epoch after its last ping) and never + // transitions to the Dead state. + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + auto epoch = GetEpoch(runtime, sender); + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + // Expires by Expire, without ever becoming Dead. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId()); + } + + Y_UNIT_TEST(LongLeaseEnabledAtRuntimeExtendsLease) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ false); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + // Register while the feature is disabled. Under the old rules this node + // would expire one epoch after its last ping (see + // LongLeaseDisabledNodeExpiresByExpire). + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + // Enable the feature at runtime, before the node would have expired. + SetNodeBrokerLongLease(runtime, sender, true); + + // With the feature now on, the node follows the long-lease lifecycle based + // on the ExpireV2/AliveUntil stored at registration: it stays Alive one + // epoch, then becomes Dead instead of expiring. + auto epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive); + + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // It only expires (by ExpireV2) the epoch after becoming Dead. + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId()); + } + + Y_UNIT_TEST(LongLeaseDisabledAtRuntimeExpiresDeadNode) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1); + + // Let the node become Dead (but not expired) under the long lease rules. + WaitForEpochUpdate(runtime, sender); + auto epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId()); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // Disable the feature while a Dead-but-not-expired node exists. The broker + // switches back to the Expire-based rule; the stale node (whose Expire is + // already in the past) must expire at the next epoch. It must not be + // processed as both "make dead" and "expire" in the same diff, which would + // crash ApplyStateDiff. + SetNodeBrokerLongLease(runtime, sender, false); + + epoch = WaitForEpochUpdate(runtime, sender); + CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId()); + } + Y_UNIT_TEST(TestListNodes) { TTestBasicRuntime runtime(8, false); @@ -4992,6 +5287,114 @@ Y_UNIT_TEST_SUITE(TDynamicNameserverTest) { // No pending cache miss requests are left CheckNoPendingCacheMissesLeft(runtime, 0); } + + Y_UNIT_TEST(ListNodesEpochDeltaLongLeaseOnlyAlive) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + // Lease duration equal to a single (default) epoch: a node without pings + // stays Alive for one epoch, becomes Dead the next and expires the epoch + // after that (see TNodeBrokerTest::LongLeaseNodeBecomesDeadThenExpires). + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1); + + // While the node is Alive it is served in both the full and the + // alive-only nodes lists. + UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + + // Advance until the node becomes Dead, but has not expired yet. + WaitForEpochUpdate(runtime, sender); + WaitForEpochUpdate(runtime, sender); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // The full list still contains the dead-but-not-expired node, while the + // alive-only list omits it. + UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + + // After the node finally expires it disappears from both lists. + WaitForEpochUpdate(runtime, sender); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + } + + Y_UNIT_TEST(LongLeaseDeadNodeStillResolvable) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1); + CheckResolveNode(runtime, sender, NODE1, "1.2.3.4"); + + // The node stops pinging and becomes Dead, but its long-lease expiration + // (ExpireV2) is still in the future, so the nameservice keeps resolving it + // (ResolveDynamicNode/GetNode use GetExpire(EnableLongLease) == ExpireV2). + WaitForEpochUpdate(runtime, sender); + WaitForEpochUpdate(runtime, sender); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // Refresh the nameservice view from the NodeBroker, then resolve. + GetNameserverDynamicNodeIds(runtime, sender); + CheckResolveNode(runtime, sender, NODE1, "1.2.3.4"); + CheckGetNode(runtime, sender, NODE1, true); + + // Once ExpireV2 passes the node expires and is no longer resolvable. + WaitForEpochUpdate(runtime, sender); + GetNameserverDynamicNodeIds(runtime, sender); + CheckResolveUnknownNode(runtime, sender, NODE1); + CheckGetNode(runtime, sender, NODE1, false); + + // No pending cache miss requests are left + CheckNoPendingCacheMissesLeft(runtime, 0); + } + + Y_UNIT_TEST(LongLeaseFeatureFlagToggledAtRuntime) + { + TTestBasicRuntime runtime(8, false); + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + TActorId sender = runtime.AllocateEdgeActor(); + + SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); + + CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4", + 1, 2, 3, 4, TStatus::OK, NODE1); + + // Drive the node into the Dead-but-not-expired window. The NodeBroker + // keeps the long lease feature enabled the whole time, so the node stays + // registered (its ExpireV2 is still in the future) while we toggle the + // feature on the nameserver only. + WaitForEpochUpdate(runtime, sender); + WaitForEpochUpdate(runtime, sender); + CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead); + + // Nameserver has long lease ON: the dead node is served from ExpireV2 in + // the full list and hidden from the alive-only list. + UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + + // Turn the feature OFF at runtime. The nameserver invalidates its caches + // and switches to the shorter Expire; the node (whose Expire is already in + // the past) drops out of both lists, and the alive-only filter is disabled. + SetNameserverLongLease(runtime, sender, false); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + + // Turn it back ON: the node reappears in the full list (served from + // ExpireV2 again) and is again hidden from the alive-only list, proving the + // flag switch and cache invalidation took effect. + SetNameserverLongLease(runtime, sender, true); + UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1)); + UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1)); + } } Y_UNIT_TEST_SUITE(TSlotIndexesPoolTest) { diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 7f1d5b1374a..2321aa9e7eb 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -335,4 +335,5 @@ message TFeatureFlags { optional bool EnableFulltextIndexRowId = 289 [default = true]; optional bool EnableForcedColumnCompactions = 290 [default = false]; optional bool EnableExportFiltering = 291 [default = false]; + optional bool EnableNodeBrokerLongLease = 292 [default = false]; } diff --git a/ydb/core/protos/node_broker.proto b/ydb/core/protos/node_broker.proto index deb544c3823..4240a396d9b 100644 --- a/ydb/core/protos/node_broker.proto +++ b/ydb/core/protos/node_broker.proto @@ -21,7 +21,9 @@ message TNodeInfo { optional string Address = 5; optional NActorsInterconnect.TNodeLocation Location = 6; optional uint64 Expire = 7; + optional uint64 ExpireV2 = 9; optional string Name = 8; + optional uint32 Liveness = 10; } message TNodeInfoSchema { @@ -35,6 +37,9 @@ message TNodeInfoSchema { optional NKikimrSubDomains.TDomainKey ServicedSubDomain = 8; optional uint32 SlotIndex = 9; optional bool AuthorizedByCertificate = 10; + optional uint64 ExpireV2 = 11; + optional uint64 AliveUntil = 12; + optional uint32 Liveness = 13; } message TVersionInfo { @@ -113,6 +118,7 @@ message TExtendLeaseResponse { optional TStatus Status = 1; optional uint32 NodeId = 2; optional uint64 Expire = 3; + optional uint64 ExpireV2 = 5; // Epoch at which ping happened. Lease is extended // until the end of the next epoch. optional TEpoch Epoch = 4; @@ -129,6 +135,7 @@ message TConfig { // Don't allocate and extend lease for IDs from banned intervals. repeated TNodeIds BannedNodeIds = 2; optional string StableNodeNamePrefix = 3 [default = "slot-"]; + optional uint64 LeaseDuration = 4 [default = 259200000000]; } message TGetConfigRequest { diff --git a/ydb/library/actors/core/interconnect.h b/ydb/library/actors/core/interconnect.h index 9125bb7a54b..70b51a85d53 100644 --- a/ydb/library/actors/core/interconnect.h +++ b/ydb/library/actors/core/interconnect.h @@ -209,11 +209,13 @@ namespace NActors { struct TEvListNodes: public TEventLocal<TEvListNodes, EvListNodes> { const bool SubscribeToStaticNodeChanges = false; + const bool OnlyAliveNodes = true; TEvListNodes() = default; - TEvListNodes(bool subscribeToStaticNodeChanges) + TEvListNodes(bool subscribeToStaticNodeChanges, bool onlyAliveNodes = true) : SubscribeToStaticNodeChanges(subscribeToStaticNodeChanges) + , OnlyAliveNodes(onlyAliveNodes) {} }; |
