summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIlia Shakhov <[email protected]>2026-07-10 14:09:08 +0300
committerGitHub <[email protected]>2026-07-10 14:09:08 +0300
commit34cf7ca54f934e552dd49cd09bea3b69a36f9262 (patch)
tree9a770caed3cdb9cc15765b26fddfc466ece1162c
parent264ec759347b38d53fadb3a5e0934e55c21346f8 (diff)
Add long leases logic to NodeBroker & clients (#45855)
Co-authored-by: Ilia Shakhov <[email protected]>
-rw-r--r--ydb/core/mind/dynamic_nameserver.cpp120
-rw-r--r--ydb/core/mind/dynamic_nameserver_impl.h34
-rw-r--r--ydb/core/mind/dynamic_nameserver_mon.cpp10
-rw-r--r--ydb/core/mind/lease_holder.cpp94
-rw-r--r--ydb/core/mind/node_broker.cpp91
-rw-r--r--ydb/core/mind/node_broker.h5
-rw-r--r--ydb/core/mind/node_broker__extend_lease.cpp3
-rw-r--r--ydb/core/mind/node_broker__register_node.cpp14
-rw-r--r--ydb/core/mind/node_broker__scheme.h9
-rw-r--r--ydb/core/mind/node_broker_impl.h20
-rw-r--r--ydb/core/mind/node_broker_ut.cpp409
-rw-r--r--ydb/core/protos/feature_flags.proto1
-rw-r--r--ydb/core/protos/node_broker.proto7
-rw-r--r--ydb/library/actors/core/interconnect.h4
14 files changed, 718 insertions, 103 deletions
diff --git a/ydb/core/mind/dynamic_nameserver.cpp b/ydb/core/mind/dynamic_nameserver.cpp
index a8d4f068786..c4172501bc9 100644
--- a/ydb/core/mind/dynamic_nameserver.cpp
+++ b/ydb/core/mind/dynamic_nameserver.cpp
@@ -30,12 +30,10 @@ public:
}
TActorCacheMiss(TDynamicNameserver* owner, ui32 nodeId, TDynamicConfigPtr config,
- TIntrusivePtr<TListNodesCache> listNodesCache,
TAutoPtr<IEventHandle> origRequest, TMonotonic deadline)
: TActorBase(&TThis::StateWork)
, TCacheMiss(nodeId, config, origRequest, deadline, 0)
, Owner(owner)
- , ListNodesCache(listNodesCache)
{
}
@@ -62,7 +60,7 @@ public:
TActorBase::PassAway();
}
- void ConvertToActor(TDynamicNameserver*, TIntrusivePtr<TListNodesCache>, const TActorContext &) override {}
+ void ConvertToActor(TDynamicNameserver*, const TActorContext &) override {}
private:
using TCacheMiss::Config;
@@ -91,7 +89,7 @@ private:
// Reset proxy if node expired.
if (exists) {
ResetInterconnectProxyConfig(NodeId, ctx);
- ListNodesCache->Invalidate(); // node was erased
+ Owner->InvalidateListNodesCache(); // node was erased
}
Owner->UpdateCounters();
OnError(rec.GetStatus().GetReason(), ctx);
@@ -99,12 +97,12 @@ private:
}
TDynamicConfig::TDynamicNodeInfo node(rec.GetNode());
- if (!exists || !oldNode.EqualExceptExpire(node)) {
- ListNodesCache->Invalidate();
+ if (!exists || !oldNode.EqualExceptExpireAndLiveness(node) || oldNode.Liveness != node.Liveness) {
+ Owner->InvalidateListNodesCache();
}
// If ID is re-used by another node then proxy has to be reset.
- if (exists && !oldNode.EqualExceptExpire(node))
+ if (exists && !oldNode.EqualExceptExpireAndLiveness(node))
ResetInterconnectProxyConfig(NodeId, ctx);
Config->DynamicNodes.emplace(NodeId, node);
@@ -113,7 +111,6 @@ private:
}
TDynamicNameserver* Owner;
- TIntrusivePtr<TListNodesCache> ListNodesCache;
};
TCacheMiss::TCacheMiss(ui32 nodeId, TDynamicConfigPtr config, TAutoPtr<IEventHandle> origRequest,
@@ -175,11 +172,10 @@ public:
ctx.Send(OrigRequest->Sender, reply.Release());
}
- void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache,
- const TActorContext &ctx) override {
+ void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) override {
Config->PendingCacheMisses.Remove(this);
- auto* actor = new TActorCacheMiss<TCacheMissGet>(owner, NodeId, Config, listNodesCache, OrigRequest, Deadline);
+ auto* actor = new TActorCacheMiss<TCacheMissGet>(owner, NodeId, Config, OrigRequest, Deadline);
actor->NeedScheduleDeadline = NeedScheduleDeadline;
ctx.RegisterWithSameMailbox(actor);
actor->SendRequest();
@@ -212,11 +208,10 @@ public:
ctx.Send(OrigRequest->Sender, reply);
}
- void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache,
- const TActorContext &ctx) override {
+ void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) override {
Config->PendingCacheMisses.Remove(this);
- auto* actor = new TActorCacheMiss<TCacheMissResolve>(owner, NodeId, Config, listNodesCache, OrigRequest, Deadline);
+ auto* actor = new TActorCacheMiss<TCacheMissResolve>(owner, NodeId, Config, OrigRequest, Deadline);
actor->NeedScheduleDeadline = NeedScheduleDeadline;
ctx.RegisterWithSameMailbox(actor);
actor->SendRequest();
@@ -235,6 +230,8 @@ void TDynamicNameserver::Bootstrap(const TActorContext &ctx)
}
EnableDeltaProtocol = AppData()->FeatureFlags.GetEnableNodeBrokerDeltaProtocol();
+ EnableLongLease = AppData()->FeatureFlags.GetEnableNodeBrokerLongLease();
+
ui32 featureFlagsItem = NKikimrConsole::TConfigItem::FeatureFlagsItem;
Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(featureFlagsItem));
@@ -261,7 +258,7 @@ void TDynamicNameserver::Handle(TEvNodeWardenStorageConfig::TPtr ev) {
const auto& config = *ev->Get()->Config;
BridgeInfo = std::move(ev->Get()->BridgeInfo);
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
if (ev->Get()->SelfManagementEnabled) {
// self-management through distconf is enabled and we are operating based on their tables, so apply them now
@@ -290,7 +287,7 @@ void TDynamicNameserver::Handle(TEvNodeWardenStorageConfig::TPtr ev) {
void TDynamicNameserver::ReplaceNameserverSetup(TIntrusivePtr<TTableNameserverSetup> newStaticConfig) {
if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) {
StaticConfig = std::move(newStaticConfig);
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
for (const auto& subscriber : StaticNodeChangeSubscribers) {
TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes));
}
@@ -369,7 +366,7 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId,
auto it = config->DynamicNodes.find(nodeId);
if (it != config->DynamicNodes.end()
- && it->second.Expire > ctx.Now())
+ && it->second.EffectiveExpire(EnableLongLease) > ctx.Now())
{
RegisterWithSameMailbox(CreateResolveActor(it->second.ResolveHost, it->second.Port, nodeId, it->second.Address, ev->Sender, SelfId(), deadline));
} else if (config->ExpiredNodes.contains(nodeId)
@@ -380,7 +377,7 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId,
} else {
TCacheMiss* cacheMiss;
if (ProtocolState == EProtocolState::UseEpochProtocol) {
- auto* actor = new TActorCacheMiss<TCacheMissResolve>(this, nodeId, config, ListNodesCache, ev, deadline);
+ auto* actor = new TActorCacheMiss<TCacheMissResolve>(this, nodeId, config, ev, deadline);
cacheMiss = actor;
RegisterWithSameMailbox(actor);
actor->SendRequest();
@@ -397,10 +394,18 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId,
}
}
-void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext &ctx)
+void TDynamicNameserver::InvalidateListNodesCache()
+{
+ ListNodesCacheAll->Invalidate();
+ ListNodesCacheAlive->Invalidate();
+}
+
+void TDynamicNameserver::SendNodesList(TActorId recipient, bool onlyAliveNodes, const TActorContext &ctx)
{
auto now = ctx.Now();
- if (ListNodesCache->NeedUpdate(now)) {
+ auto& cache = onlyAliveNodes ? ListNodesCacheAlive : ListNodesCacheAll;
+
+ if (cache->NeedUpdate(now)) {
auto newNodes = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>();
auto newExpire = TInstant::Max();
const bool bridgeModeEnabled = AppData()->BridgeModeEnabled;
@@ -431,17 +436,23 @@ void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext &
for (auto &config : DynamicConfigs) {
for (auto &pr : config->DynamicNodes) {
- if (pr.second.Expire > now) {
- newNodes->emplace_back(pr.first, pr.second.Address,
- pr.second.Host, pr.second.ResolveHost,
- pr.second.Port, pr.second.Location, false);
- newExpire = std::min(newExpire, pr.second.Expire);
- if (newPileMap) {
- TNodeLocation location(pr.second.Location);
- const auto& bridgePileName = location.GetBridgePileName();
- if (bridgePileName && pileNameMap.contains(*bridgePileName)) {
- newPileMap->at(pileNameMap[*bridgePileName]).push_back(pr.first);
- }
+ if (EnableLongLease) {
+ if (onlyAliveNodes && pr.second.Liveness != ENodeLiveness::Alive) {
+ continue; // dead nodes are not included in alive-only list
+ }
+ } else if (pr.second.Expire <= now) {
+ continue; // expired nodes are not included
+ }
+
+ newNodes->emplace_back(pr.first, pr.second.Address,
+ pr.second.Host, pr.second.ResolveHost,
+ pr.second.Port, pr.second.Location, false);
+ newExpire = std::min(newExpire, pr.second.EffectiveExpire(EnableLongLease));
+ if (newPileMap) {
+ TNodeLocation location(pr.second.Location);
+ const auto& bridgePileName = location.GetBridgePileName();
+ if (bridgePileName && pileNameMap.contains(*bridgePileName)) {
+ newPileMap->at(pileNameMap[*bridgePileName]).push_back(pr.first);
}
}
}
@@ -453,16 +464,16 @@ void TDynamicNameserver::SendNodesList(TActorId recipient, const TActorContext &
}
}
- ListNodesCache->Update(std::move(newNodes), newExpire, std::move(newPileMap));
+ cache->Update(std::move(newNodes), newExpire, std::move(newPileMap));
}
- ctx.Send(recipient, new TEvInterconnect::TEvNodesInfo(ListNodesCache->GetNodes(), ListNodesCache->GetPileMap()));
+ ctx.Send(recipient, new TEvInterconnect::TEvNodesInfo(cache->GetNodes(), cache->GetPileMap()));
}
void TDynamicNameserver::SendNodesList(const TActorContext &ctx)
{
- for (auto &sender : ListNodesQueue) {
- SendNodesList(sender, ctx);
+ for (auto &[sender, onlyAliveNodes] : ListNodesQueue) {
+ SendNodesList(sender, onlyAliveNodes, ctx);
}
ListNodesQueue.clear();
}
@@ -500,8 +511,10 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec,
if (it == config->DynamicNodes.end()) {
config->DynamicNodes.emplace(nodeId, info);
} else {
- if (it->second.EqualExceptExpire(info)) {
+ if (it->second.EqualExceptExpireAndLiveness(info)) {
it->second.Expire = info.Expire;
+ it->second.ExpireV2 = info.ExpireV2;
+ it->second.Liveness = info.Liveness;
} else {
ResetInterconnectProxyConfig(nodeId, ctx);
it->second = info;
@@ -517,7 +530,7 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec,
config->ExpiredNodes.emplace(node.GetNodeId());
}
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
config->Epoch = rec.GetEpoch();
ctx.Schedule(config->Epoch.End - ctx.Now(),
new TEvPrivate::TEvUpdateEpoch(domain, config->Epoch.Id + 1));
@@ -531,14 +544,16 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec,
if (it == config->DynamicNodes.end()) {
config->DynamicNodes.emplace(nodeId, info);
} else {
- if (it->second.EqualExceptExpire(info)) {
+ if (it->second.EqualExceptExpireAndLiveness(info)) {
it->second.Expire = info.Expire;
+ it->second.ExpireV2 = info.ExpireV2;
+ it->second.Liveness = info.Liveness;
} else {
ResetInterconnectProxyConfig(nodeId, ctx);
it->second = info;
}
}
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
}
config->Epoch = rec.GetEpoch();
}
@@ -607,8 +622,10 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvListNodes::TPtr &ev,
{
LOG_D("Handle " << ev->Get()->ToString());
+ const bool onlyAliveNodes = ev->Get()->OnlyAliveNodes;
+
if (ProtocolState == EProtocolState::Connecting) {
- SendNodesList(ev->Sender, ctx);
+ SendNodesList(ev->Sender, onlyAliveNodes, ctx);
} else {
if (ListNodesQueue.empty()) {
auto dinfo = AppData(ctx)->DomainsInfo;
@@ -624,7 +641,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvListNodes::TPtr &ev,
PendingRequests.Set(domain);
}
}
- ListNodesQueue.push_back(ev->Sender);
+ ListNodesQueue.emplace_back(ev->Sender, onlyAliveNodes);
}
if (ev->Get()->SubscribeToStaticNodeChanges) {
@@ -651,7 +668,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc
ui32 domain = AppData()->DomainsInfo->GetDomain()->DomainUid;
const auto& config = DynamicConfigs[domain];
auto it = config->DynamicNodes.find(nodeId);
- if (it != config->DynamicNodes.end() && it->second.Expire > ctx.Now()) {
+ if (it != config->DynamicNodes.end() && it->second.EffectiveExpire(EnableLongLease) > ctx.Now()) {
reply->Node = MakeHolder<TEvInterconnect::TNodeInfo>(it->first, it->second.Address,
it->second.Host, it->second.ResolveHost,
it->second.Port, it->second.Location);
@@ -663,7 +680,7 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc
TCacheMiss* cacheMiss;
const TMonotonic deadline = ev->Get()->Deadline;
if (ProtocolState == EProtocolState::UseEpochProtocol) {
- auto* actor = new TActorCacheMiss<TCacheMissGet>(this, nodeId, config, ListNodesCache, ev.Release(), deadline);
+ auto* actor = new TActorCacheMiss<TCacheMissGet>(this, nodeId, config, ev.Release(), deadline);
cacheMiss = actor;
RegisterWithSameMailbox(actor);
actor->SendRequest();
@@ -753,7 +770,7 @@ void TDynamicNameserver::Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, con
}
for (const auto &[cacheMiss, _] : DynamicConfigs[domain]->CacheMissHolders) {
- cacheMiss->ConvertToActor(this, ListNodesCache, ctx);
+ cacheMiss->ConvertToActor(this, ctx);
}
// cache misses are managed by actorsystem in epoch protocol
DynamicConfigs[domain]->CacheMissHolders.clear();
@@ -810,14 +827,16 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T
if (it == config->DynamicNodes.end()) {
config->DynamicNodes.emplace(nodeId, info);
} else {
- if (it->second.EqualExceptExpire(info)) {
+ if (it->second.EqualExceptExpireAndLiveness(info)) {
it->second.Expire = info.Expire;
+ it->second.ExpireV2 = info.ExpireV2;
+ it->second.Liveness = info.Liveness;
} else {
ResetInterconnectProxyConfig(nodeId, ctx);
it->second = info;
}
}
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
config->ExpiredNodes.erase(nodeId);
break;
}
@@ -825,7 +844,7 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T
ui32 nodeId = u.GetExpiredNode();
if (config->DynamicNodes.erase(nodeId) > 0) {
ResetInterconnectProxyConfig(nodeId, ctx);
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
}
config->ExpiredNodes.insert(nodeId);
break;
@@ -834,7 +853,7 @@ void TDynamicNameserver::Handle(TEvNodeBroker::TEvUpdateNodes::TPtr &ev, const T
ui32 nodeId = u.GetRemovedNode();
if (config->DynamicNodes.erase(nodeId) > 0) {
ResetInterconnectProxyConfig(nodeId, ctx);
- ListNodesCache->Invalidate();
+ InvalidateListNodesCache();
}
config->ExpiredNodes.erase(nodeId);
break;
@@ -923,6 +942,11 @@ void TDynamicNameserver::Handle(NConsole::TEvConsole::TEvConfigNotificationReque
ui32 domain = AppData()->DomainsInfo->GetDomain()->DomainUid;
NTabletPipe::CloseClient(ctx, DynamicConfigs[domain]->NodeBrokerPipe);
}
+
+ if (EnableLongLease != featureFlags.GetEnableNodeBrokerLongLease()) {
+ EnableLongLease = featureFlags.GetEnableNodeBrokerLongLease();
+ InvalidateListNodesCache();
+ }
}
Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie);
}
diff --git a/ydb/core/mind/dynamic_nameserver_impl.h b/ydb/core/mind/dynamic_nameserver_impl.h
index 49df9238cea..b58bf7bf303 100644
--- a/ydb/core/mind/dynamic_nameserver_impl.h
+++ b/ydb/core/mind/dynamic_nameserver_impl.h
@@ -49,8 +49,7 @@ public:
virtual void OnSuccess(const TActorContext &);
virtual void OnError(const TString &error, const TActorContext &);
- virtual void ConvertToActor(TDynamicNameserver* owner, TIntrusivePtr<TListNodesCache> listNodesCache,
- const TActorContext &ctx) = 0;
+ virtual void ConvertToActor(TDynamicNameserver* owner, const TActorContext &ctx) = 0;
struct THeapIndexByDeadline {
size_t& operator()(TCacheMiss& cacheMiss) const;
@@ -83,9 +82,13 @@ struct TDynamicConfig : public TThrRefBase {
const TString &resolveHost,
ui16 port,
const TNodeLocation &location,
- TInstant expire)
+ TInstant expire,
+ TInstant expireV2,
+ ENodeLiveness liveness)
: TNodeInfo(address, host, resolveHost, port, location)
, Expire(expire)
+ , ExpireV2(expireV2)
+ , Liveness(liveness)
{
}
@@ -95,14 +98,16 @@ struct TDynamicConfig : public TThrRefBase {
info.GetResolveHost(),
(ui16)info.GetPort(),
TNodeLocation(info.GetLocation()),
- TInstant::MicroSeconds(info.GetExpire()))
+ TInstant::MicroSeconds(info.GetExpire()),
+ TInstant::MicroSeconds(info.HasExpireV2() ? info.GetExpireV2() : info.GetExpire()),
+ static_cast<ENodeLiveness>(info.GetLiveness()))
{
}
TDynamicNodeInfo(const TDynamicNodeInfo &other) = default;
TDynamicNodeInfo &operator=(const TDynamicNodeInfo &other) = default;
- bool EqualExceptExpire(const TDynamicNodeInfo &other) const
+ bool EqualExceptExpireAndLiveness(const TDynamicNodeInfo &other) const
{
return Host == other.Host
&& Address == other.Address
@@ -111,7 +116,13 @@ struct TDynamicConfig : public TThrRefBase {
&& Location == other.Location;
}
+ TInstant EffectiveExpire(bool enableLongLease) const {
+ return enableLongLease ? ExpireV2 : Expire;
+ }
+
TInstant Expire;
+ TInstant ExpireV2;
+ ENodeLiveness Liveness = ENodeLiveness::Alive;
};
THashMap<ui32, TDynamicNodeInfo> DynamicNodes;
@@ -186,7 +197,8 @@ public:
TDynamicNameserver(const TIntrusivePtr<TTableNameserverSetup> &setup, ui32 resolvePoolId)
: StaticConfig(setup)
- , ListNodesCache(MakeIntrusive<TListNodesCache>())
+ , ListNodesCacheAll(MakeIntrusive<TListNodesCache>())
+ , ListNodesCacheAlive(MakeIntrusive<TListNodesCache>())
, ResolvePoolId(resolvePoolId)
{
Y_ABORT_UNLESS(StaticConfig->IsEntriesUnique());
@@ -251,8 +263,9 @@ private:
const TActorContext &ctx);
void ResolveStaticNode(ui32 nodeId, TActorId sender, TMonotonic deadline, const TActorContext &ctx);
void ResolveDynamicNode(ui32 nodeId, TAutoPtr<IEventHandle> ev, TMonotonic deadline, const TActorContext &ctx);
- void SendNodesList(TActorId recipient, const TActorContext &ctx);
+ void SendNodesList(TActorId recipient, bool onlyAliveNodes, const TActorContext &ctx);
void SendNodesList(const TActorContext &ctx);
+ void InvalidateListNodesCache();
void PendingRequestAnswered(ui32 domain, const TActorContext &ctx);
void UpdateState(const NKikimrNodeBroker::TNodesInfo &rec,
const TActorContext &ctx);
@@ -288,8 +301,9 @@ private:
private:
TIntrusivePtr<TTableNameserverSetup> StaticConfig;
std::array<TDynamicConfigPtr, DOMAINS_COUNT> DynamicConfigs;
- TVector<TActorId> ListNodesQueue;
- TIntrusivePtr<TListNodesCache> ListNodesCache;
+ TVector<std::pair<TActorId, bool>> ListNodesQueue;
+ TIntrusivePtr<TListNodesCache> ListNodesCacheAll;
+ TIntrusivePtr<TListNodesCache> ListNodesCacheAlive;
TBridgeInfo::TPtr BridgeInfo;
// When ListNodes requests are sent to NodeBroker tablets this
@@ -301,7 +315,9 @@ private:
THashSet<TActorId> StaticNodeChangeSubscribers;
bool SubscribedToConsoleNSConfig = false;
+ bool EnableLongLease = false;
bool EnableDeltaProtocol = false;
+
EProtocolState ProtocolState = EProtocolState::Connecting;
bool SyncInProgress = false;
ui64 SyncCookie = 0;
diff --git a/ydb/core/mind/dynamic_nameserver_mon.cpp b/ydb/core/mind/dynamic_nameserver_mon.cpp
index 8b19303d0f0..36267c13d88 100644
--- a/ydb/core/mind/dynamic_nameserver_mon.cpp
+++ b/ydb/core/mind/dynamic_nameserver_mon.cpp
@@ -91,6 +91,7 @@ void OutputNodeInfo(ui32 nodeId,
const TTableNameserverSetup::TNodeInfo &info,
IOutputStream &str,
TInstant expire = TInstant::Zero(),
+ ENodeLiveness liveness = ENodeLiveness::Alive,
const TString &cl = "")
{
str << "<tr class='" << cl << "'>" << Endl
@@ -102,6 +103,7 @@ void OutputNodeInfo(ui32 nodeId,
<< " <td>" << info.Location.ToString() << "</td>" << Endl;
if (expire)
str << "<td>" << ToString(expire) << "</td>" << Endl;
+ str << "<td>" << (liveness == ENodeLiveness::Alive ? "Alive" : "Dead") << "</td>" << Endl;
str << "</tr>" << Endl;
}
@@ -142,7 +144,8 @@ void OutputStaticNodes(const TTableNameserverSetup &setup,
void OutputDynamicNodes(const TString &domain,
TDynamicConfigPtr config,
- IOutputStream &str)
+ IOutputStream &str,
+ bool enableLongLease)
{
str << "<div><table class='nodes'>" << Endl
<< " <caption>Dynamic nodes in " << domain
@@ -159,6 +162,7 @@ void OutputDynamicNodes(const TString &domain,
<< " <th>Rack</th>" << Endl
<< " <th>Body</th>" << Endl
<< " <th>Expire</th>" << Endl
+ << " <th>Liveness</th>" << Endl
<< " </tr>" << Endl
<< " </thead>" << Endl
<< " <tbody class='center-align'>" << Endl;
@@ -168,7 +172,7 @@ void OutputDynamicNodes(const TString &domain,
ids.insert(pr.first);
for (auto id : ids) {
auto &node = config->DynamicNodes.at(id);
- OutputNodeInfo(id, node, str, node.Expire);
+ OutputNodeInfo(id, node, str, node.EffectiveExpire(enableLongLease), node.Liveness);
}
ids.clear();
@@ -217,7 +221,7 @@ void TDynamicNameserver::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext
OutputStaticNodes(*StaticConfig, str);
if (const auto& domain = AppData(ctx)->DomainsInfo->Domain) {
- OutputDynamicNodes(domain->Name, DynamicConfigs[domain->DomainUid], str);
+ OutputDynamicNodes(domain->Name, DynamicConfigs[domain->DomainUid], str, EnableLongLease);
}
}
ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
diff --git a/ydb/core/mind/lease_holder.cpp b/ydb/core/mind/lease_holder.cpp
index eae4144b723..d6c4edf1050 100644
--- a/ydb/core/mind/lease_holder.cpp
+++ b/ydb/core/mind/lease_holder.cpp
@@ -1,13 +1,17 @@
#include "lease_holder.h"
#include "node_broker.h"
+#include <ydb/core/base/appdata.h>
+#include <ydb/core/base/tablet_pipe.h>
+#include <ydb/core/cms/console/configs_dispatcher.h>
+#include <ydb/core/cms/console/console.h>
+#include <ydb/core/mon/mon.h>
+#include <ydb/core/protos/feature_flags.pb.h>
+
#include <ydb/library/actors/core/actor_bootstrapped.h>
#include <ydb/library/actors/core/hfunc.h>
-#include <ydb/core/mon/mon.h>
#include <ydb/library/actors/core/mon.h>
#include <ydb/library/actors/util/should_continue.h>
-#include <ydb/core/base/appdata.h>
-#include <ydb/core/base/tablet_pipe.h>
#include <library/cpp/monlib/service/pages/templates.h>
@@ -41,6 +45,7 @@ public:
TLeaseHolder(TInstant expire)
: LastPingEpoch(0)
, Expire(expire)
+ , ExpireV2(expire)
{
}
@@ -54,6 +59,12 @@ public:
false, ctx.ActorSystem(), ctx.SelfID);
}
+ EnableLongLease = AppData()->FeatureFlags.GetEnableNodeBrokerLongLease();
+
+ const ui32 featureFlagsItem = NKikimrConsole::TConfigItem::FeatureFlagsItem;
+ Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
+ new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(featureFlagsItem));
+
Become(&TThis::StatePing);
ScheduleExpire(ctx);
@@ -69,6 +80,8 @@ private:
HFunc(TEvPrivate::TEvExpire, Handle);
HFunc(TEvTabletPipe::TEvClientDestroyed, HandleIdle);
HFunc(TEvTabletPipe::TEvClientConnected, HandleIdle);
+ HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle);
+ IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse);
IgnoreFunc(TEvNodeBroker::TEvExtendLeaseResponse);
default:
@@ -85,6 +98,8 @@ private:
HFunc(TEvPrivate::TEvExpire, Handle);
HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
HFunc(TEvTabletPipe::TEvClientConnected, Handle);
+ HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle);
+ IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse);
default:
Y_ABORT("TLeaseHolder::StatePing unexpected event type: %" PRIx32 " event: %s",
@@ -138,6 +153,18 @@ private:
OnPipeDestroyed(ctx);
}
+ void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev, const TActorContext &ctx) {
+ const auto& record = ev->Get()->Record;
+ if (record.HasConfig() && record.GetConfig().HasFeatureFlags()) {
+ const auto& featureFlags = record.GetConfig().GetFeatureFlags();
+ if (EnableLongLease != featureFlags.GetEnableNodeBrokerLongLease()) {
+ EnableLongLease = featureFlags.GetEnableNodeBrokerLongLease();
+ ScheduleExpire(ctx);
+ }
+ }
+ Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie);
+ }
+
void Connect(const TActorContext &ctx)
{
NTabletPipe::TClientConfig config;
@@ -160,31 +187,27 @@ private:
}
Expire = TInstant::MicroSeconds(rec.GetExpire());
+ ExpireV2 = TInstant::MicroSeconds(rec.HasExpireV2() ? rec.GetExpireV2() : rec.GetExpire());
+
LastResponse = ctx.Now();
LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER,
- "Node has now extended lease expiring " << ToString(Expire));
-
- if (rec.HasEpoch()) {
- LastPingEpoch = rec.GetEpoch().GetId();
- EpochEnd = TInstant::FromValue(rec.GetEpoch().GetEnd());
+ "Node has now extended lease expiring " << ToString(EffectiveExpire()));
- if (Expire != TInstant::Max()) {
- Y_ABORT_UNLESS(Expire > EpochEnd);
- Y_ABORT_UNLESS(rec.GetExpire() == rec.GetEpoch().GetNextEnd());
+ Y_ABORT_UNLESS(rec.HasEpoch());
+ LastPingEpoch = rec.GetEpoch().GetId();
+ EpochEnd = TInstant::FromValue(rec.GetEpoch().GetEnd());
+ NextEpochEnd = TInstant::FromValue(rec.GetEpoch().GetNextEnd());
- ui64 window = (Expire - EpochEnd).GetValue() / 2;
- Y_ABORT_UNLESS(window);
+ if (EffectiveExpire() != TInstant::Max()) {
+ Y_ABORT_UNLESS(EffectiveExpire() >= NextEpochEnd);
+ ui64 window = (NextEpochEnd - EpochEnd).GetValue() / 2;
+ Y_ABORT_UNLESS(window);
- NextPing = EpochEnd + TDuration::FromValue(RandomNumber<ui64>(window));
- }
- } else {
- NextPing = ctx.Now() + (Expire - ctx.Now()) / 2;
- }
-
- if (Expire != TInstant::Max())
+ NextPing = EpochEnd + TDuration::FromValue(RandomNumber<ui64>(window));
ctx.Schedule(NextPing - ctx.Now(), new TEvents::TEvWakeup());
- else
+ } else {
NextPing = TInstant::Max();
+ }
Become(&TThis::StateIdle);
}
@@ -203,11 +226,15 @@ private:
TStringStream str;
HTML(str) {
PRE() {
- str << "Lease expires: " << ToString(Expire) << Endl
+ str << "Lease expires: " << ToString(EffectiveExpire()) << Endl
<< "Last lease extension: " << ToString(LastResponse) << Endl
<< "Last ping epoch: " << LastPingEpoch << Endl
<< "Epoch end: " << ToString(EpochEnd) << Endl
- << "Next ping at: " << ToString(NextPing) << Endl;
+ << "Next epoch end: " << ToString(NextEpochEnd) << Endl
+ << "Next ping at: " << ToString(NextPing) << Endl
+ << "Enable long lease: " << EnableLongLease << Endl
+ << "Expire: " << ToString(Expire) << Endl
+ << "ExpireV2: " << ToString(ExpireV2) << Endl;
}
}
ctx.Send(ev->Sender, new TEvHttpInfoRes(str.Str()));
@@ -215,13 +242,15 @@ private:
void ScheduleExpire(const TActorContext &ctx)
{
- if (Expire != TInstant::Max())
- ctx.Schedule(Expire - ctx.Now(), new TEvPrivate::TEvExpire);
+ ExpireCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+ if (EffectiveExpire() != TInstant::Max()) {
+ ctx.Schedule(EffectiveExpire() - ctx.Now(), new TEvPrivate::TEvExpire, ExpireCookieHolder.Get());
+ }
}
void Handle(TEvPrivate::TEvExpire::TPtr &,const TActorContext &ctx)
{
- if (Expire <= ctx.Now())
+ if (EffectiveExpire() <= ctx.Now())
StopNode(ctx);
else
ScheduleExpire(ctx);
@@ -240,13 +269,24 @@ private:
return t.ToRfc822StringLocal();
}
+ TInstant EffectiveExpire() const
+ {
+ return EnableLongLease ? ExpireV2 : Expire;
+ }
+
private:
TActorId NodeBrokerPipe;
ui64 LastPingEpoch;
TInstant EpochEnd;
- TInstant Expire;
+ TInstant NextEpochEnd;
+
TInstant LastResponse;
TInstant NextPing;
+
+ bool EnableLongLease = false;
+ TInstant Expire;
+ TInstant ExpireV2;
+ NActors::TSchedulerCookieHolder ExpireCookieHolder;
};
IActor *CreateLeaseHolder(TInstant expire)
diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp
index 494cef4c4de..8d67ac47e16 100644
--- a/ydb/core/mind/node_broker.cpp
+++ b/ydb/core/mind/node_broker.cpp
@@ -104,6 +104,7 @@ void TNodeBroker::OnActivateExecutor(const TActorContext &ctx)
MaxDynamicId = Max(MinDynamicId, (ui64)Min(appData->DynamicNameserviceConfig->MaxDynamicNodeId, TActorId::MaxNodeId));
EnableStableNodeNames = appData->FeatureFlags.GetEnableStableNodeNames();
+ EnableLongLease = appData->FeatureFlags.GetEnableNodeBrokerLongLease();
Executor()->RegisterExternalTabletCounters(TabletCountersPtr);
Committed.ClearState();
@@ -168,6 +169,9 @@ bool TNodeBroker::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev,
<< " Location: " << node.Location.ToString() << Endl
<< " Lease: " << node.Lease << Endl
<< " Expire: " << node.ExpirationString() << Endl
+ << " ExpireV2: " << node.ExpirationV2String() << Endl
+ << " AliveUntil: " << node.AliveUntilString() << Endl
+ << " Liveness: " << (node.Liveness == ENodeLiveness::Dead ? "Dead" : "Alive") << Endl
<< " AuthorizedByCertificate: " << (node.AuthorizedByCertificate ? "true" : "false") << Endl
<< " ServicedSubDomain: " << node.ServicedSubDomain << Endl
<< " SlotIndex: " << node.SlotIndex << Endl;
@@ -303,22 +307,38 @@ void TNodeBroker::TState::AddNode(const TNodeInfo &info)
}
}
+bool TNodeBroker::TState::IsLeaseExtendable(const TNodeInfo &node) const
+{
+ return node.Expire < Epoch.NextEnd || node.ExpireV2 < Epoch.NextEnd + LeaseDuration;
+}
+
void TNodeBroker::TState::ExtendLease(TNodeInfo &node)
{
node.Version = Epoch.Version + 1;
+
++node.Lease;
node.Expire = Epoch.NextEnd;
+ node.ExpireV2 = Epoch.NextEnd + LeaseDuration;
+
+ node.AliveUntil = Epoch.NextEnd;
+ node.Liveness = ENodeLiveness::Alive;
LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
- LogPrefix() << " Extended lease of " << node.IdString() << " up to "
- << node.ExpirationString() << " (lease " << node.Lease << ")");
+ LogPrefix() << " Extended lease of " << node.IdString() << " up to v1: "
+ << node.ExpirationString() << " and v2: " << node.ExpirationV2String()
+ << " (lease " << node.Lease << ")");
}
void TNodeBroker::TState::FixNodeId(TNodeInfo &node)
{
node.Version = Epoch.Version + 1;
+
++node.Lease;
node.Expire = TInstant::Max();
+ node.ExpireV2 = TInstant::Max();
+
+ node.AliveUntil = TInstant::Max();
+ node.Liveness = ENodeLiveness::Alive;
LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
LogPrefix() << " Fix ID for node " << node.IdString());
@@ -474,6 +494,8 @@ void TNodeBroker::FillNodeInfo(const TNodeInfo &node,
info.SetResolveHost(node.ResolveHost);
info.SetAddress(node.Address);
info.SetExpire(node.Expire.GetValue());
+ info.SetExpireV2(node.ExpireV2.GetValue());
+ info.SetLiveness(static_cast<ui32>(node.Liveness));
node.Location.Serialize(info.MutableLocation(), false);
FillNodeName(node.SlotIndex, info);
}
@@ -489,8 +511,17 @@ void TNodeBroker::FillNodeName(const std::optional<ui32> &slotIndex,
void TNodeBroker::TState::ComputeNextEpochDiff(TStateDiff &diff)
{
+ if (Self->EnableLongLease) {
+ for (auto &pr : Nodes) {
+ if (pr.second.AliveUntil <= Epoch.End && pr.second.Liveness == ENodeLiveness::Alive) {
+ diff.NodesToMakeDead.push_back(pr.first);
+ }
+ }
+ }
+
for (auto &pr : Nodes) {
- if (pr.second.Expire <= Epoch.End)
+ auto expire = Self->EnableLongLease ? pr.second.ExpireV2 : pr.second.Expire;
+ if (expire <= Epoch.End)
diff.NodesToExpire.push_back(pr.first);
}
@@ -522,6 +553,17 @@ void TNodeBroker::TState::ApplyStateDiff(const TStateDiff &diff)
Nodes.erase(it);
}
+ for (auto id : diff.NodesToMakeDead) {
+ auto it = Nodes.find(id);
+ Y_ABORT_UNLESS(it != Nodes.end());
+
+ LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
+ LogPrefix() << " Node " << it->second.IdString() << " is marked as dead");
+
+ it->second.Liveness = ENodeLiveness::Dead;
+ it->second.Version = diff.NewEpoch.Version;
+ }
+
for (auto id : diff.NodesToRemove) {
auto it = ExpiredNodes.find(id);
Y_ABORT_UNLESS(it != ExpiredNodes.end());
@@ -794,11 +836,19 @@ void TNodeBroker::TState::LoadConfigFromProto(const NKikimrNodeBroker::TConfig &
EpochDuration = TDuration::MicroSeconds(config.GetEpochDuration());
if (EpochDuration < MIN_LEASE_DURATION) {
LOG_ERROR_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
- LogPrefix() << " Configured lease duration (" << EpochDuration << ") is too"
+ LogPrefix() << " Configured epoch duration (" << EpochDuration << ") is too"
" small. Using min. value: " << MIN_LEASE_DURATION);
EpochDuration = MIN_LEASE_DURATION;
}
+ LeaseDuration = TDuration::MicroSeconds(config.GetLeaseDuration());
+ if (LeaseDuration < MIN_LEASE_DURATION) {
+ LOG_ERROR_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
+ LogPrefix() << " Configured lease duration (" << LeaseDuration << ") is too"
+ " small. Using min. value: " << MIN_LEASE_DURATION);
+ LeaseDuration = MIN_LEASE_DURATION;
+ }
+
StableNodeNamePrefix = config.GetStableNodeNamePrefix();
BannedIds.clear();
@@ -858,6 +908,7 @@ void TNodeBroker::TDirtyState::DbAddNode(const TNodeInfo &node,
<< " location=" << node.Location.ToString()
<< " lease=" << node.Lease
<< " expire=" << node.ExpirationString()
+ << " expirev2=" << node.ExpirationV2String()
<< " servicedsubdomain=" << node.ServicedSubDomain
<< " slotindex=" << node.SlotIndex
<< " authorizedbycertificate=" << (node.AuthorizedByCertificate ? "true" : "false"));
@@ -878,6 +929,9 @@ void TNodeBroker::TDirtyState::DbAddNode(const TNodeInfo &node,
.Update<T::Address>(node.Address)
.Update<T::Lease>(node.Lease)
.Update<T::Expire>(node.Expire.GetValue())
+ .Update<T::ExpireV2>(node.ExpireV2.GetValue())
+ .Update<T::AliveUntil>(node.AliveUntil.GetValue())
+ .Update<T::Liveness>(node.Liveness)
.Update<T::Location>(node.Location.GetSerializedLocation())
.Update<T::ServicedSubDomain>(node.ServicedSubDomain)
.Update<T::AuthorizedByCertificate>(node.AuthorizedByCertificate);
@@ -896,6 +950,7 @@ void TNodeBroker::TDirtyState::DbApplyStateDiff(const TStateDiff &diff,
TTransactionContext &txc)
{
DbUpdateNodes(diff.NodesToExpire, txc);
+ DbUpdateNodes(diff.NodesToMakeDead, txc);
DbUpdateNodes(diff.NodesToRemove, txc);
DbUpdateEpoch(diff.NewEpoch, txc);
DbUpdateApproxEpochStart(diff.NewApproxEpochStart, txc);
@@ -1088,6 +1143,8 @@ TNodeBroker::TDbChanges TNodeBroker::TDirtyState::DbLoadNodes(auto &nodesRowset,
AddNode(info);
} else {
auto expire = TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::Expire>());
+ auto expireV2 = Max(expire, TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::ExpireV2>()));
+
std::optional<TNodeLocation> modernLocation;
if (nodesRowset.template HaveValue<Schema::Nodes::Location>()) {
modernLocation.emplace(TNodeLocation::FromSerialized, nodesRowset.template GetValue<Schema::Nodes::Location>());
@@ -1108,12 +1165,22 @@ TNodeBroker::TDbChanges TNodeBroker::TDirtyState::DbLoadNodes(auto &nodesRowset,
info.Lease = nodesRowset.template GetValue<Schema::Nodes::Lease>();
info.Expire = expire;
+ info.ExpireV2 = expireV2;
+ info.AliveUntil = Max(expire, TInstant::FromValue(nodesRowset.template GetValue<Schema::Nodes::AliveUntil>()));
+ info.Liveness = nodesRowset.template GetValue<Schema::Nodes::Liveness>();
+
info.ServicedSubDomain = TSubDomainKey(nodesRowset.template GetValueOrDefault<Schema::Nodes::ServicedSubDomain>());
if (nodesRowset.template HaveValue<Schema::Nodes::SlotIndex>()) {
info.SlotIndex = nodesRowset.template GetValue<Schema::Nodes::SlotIndex>();
}
info.AuthorizedByCertificate = nodesRowset.template GetValue<Schema::Nodes::AuthorizedByCertificate>();
- info.State = expire > Epoch.Start ? ENodeState::Active : ENodeState::Expired;
+
+ if (Self->EnableLongLease) {
+ info.State = expireV2 > Epoch.Start ? ENodeState::Active : ENodeState::Expired;
+ } else {
+ // Dead nodes stay active until epoch end
+ info.State = info.Liveness == ENodeLiveness::Dead || expire > Epoch.Start ? ENodeState::Active : ENodeState::Expired;
+ }
AddNode(info);
LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER,
@@ -1424,6 +1491,7 @@ void TNodeBroker::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev,
const auto& appConfig = ev->Get()->Record.GetConfig();
if (appConfig.HasFeatureFlags()) {
EnableStableNodeNames = appConfig.GetFeatureFlags().GetEnableStableNodeNames();
+ EnableLongLease = appConfig.GetFeatureFlags().GetEnableNodeBrokerLongLease();
}
if (ev->Get()->Record.HasLocal() && ev->Get()->Record.GetLocal()) {
@@ -1757,11 +1825,14 @@ TNodeBroker::TNodeInfo::TNodeInfo(ui32 nodeId, ENodeState state, ui64 version, c
TNodeLocation(schema.GetLocation()))
, Lease(schema.GetLease())
, Expire(TInstant::MicroSeconds(schema.GetExpire()))
+ , ExpireV2(TInstant::MicroSeconds(schema.GetExpireV2()))
, AuthorizedByCertificate(schema.GetAuthorizedByCertificate())
, SlotIndex(schema.GetSlotIndex())
, ServicedSubDomain(schema.GetServicedSubDomain())
, State(state)
, Version(version)
+ , AliveUntil(TInstant::MicroSeconds(schema.GetAliveUntil()))
+ , Liveness(static_cast<ENodeLiveness>(schema.GetLiveness()))
{}
TNodeBroker::TNodeInfo::TNodeInfo(ui32 nodeId, ENodeState state, ui64 version)
@@ -1775,7 +1846,9 @@ bool TNodeBroker::TNodeInfo::EqualCachedData(const TNodeInfo &other) const
&& ResolveHost == other.ResolveHost
&& Address == other.Address
&& Location == other.Location
- && Expire == other.Expire;
+ && Expire == other.Expire
+ && ExpireV2 == other.ExpireV2
+ && Liveness == other.Liveness;
}
bool TNodeBroker::TNodeInfo::EqualExceptVersion(const TNodeInfo &other) const
@@ -1811,6 +1884,9 @@ TString TNodeBroker::TNodeInfo::ToString() const
<< ", Address: " << Address
<< ", Lease: " << Lease
<< ", Expire: " << ExpirationString()
+ << ", ExpireV2: " << ExpirationV2String()
+ << ", AliveUntil: " << AliveUntilString()
+ << ", Liveness: " << static_cast<ui32>(Liveness)
<< ", Location: " << Location.ToString()
<< ", AuthorizedByCertificate: " << AuthorizedByCertificate
<< ", SlotIndex: " << SlotIndex
@@ -1827,6 +1903,9 @@ TNodeInfoSchema TNodeBroker::TNodeInfo::SerializeToSchema() const {
serialized.SetAddress(Address);
serialized.SetLease(Lease);
serialized.SetExpire(Expire.MicroSeconds());
+ serialized.SetExpireV2(ExpireV2.MicroSeconds());
+ serialized.SetAliveUntil(AliveUntil.MicroSeconds());
+ serialized.SetLiveness(static_cast<ui32>(Liveness));
Location.Serialize(serialized.MutableLocation(), false);
serialized.MutableServicedSubDomain()->CopyFrom(ServicedSubDomain);
if (SlotIndex.has_value()) {
diff --git a/ydb/core/mind/node_broker.h b/ydb/core/mind/node_broker.h
index 0e3856c6bcb..9ab24f2911e 100644
--- a/ydb/core/mind/node_broker.h
+++ b/ydb/core/mind/node_broker.h
@@ -25,6 +25,11 @@
namespace NKikimr {
namespace NNodeBroker {
+enum class ENodeLiveness : ui8 {
+ Alive = 0,
+ Dead = 1,
+};
+
struct TEpochInfo {
ui64 Id = 0;
ui64 Version = 0;
diff --git a/ydb/core/mind/node_broker__extend_lease.cpp b/ydb/core/mind/node_broker__extend_lease.cpp
index 030a99f3324..2a23c40adb7 100644
--- a/ydb/core/mind/node_broker__extend_lease.cpp
+++ b/ydb/core/mind/node_broker__extend_lease.cpp
@@ -58,7 +58,7 @@ public:
return Error(TStatus::WRONG_REQUEST, "Node ID is banned", ctx);
auto &node = it->second;
- if (node.Expire < Self->Dirty.Epoch.NextEnd) {
+ if (Self->Dirty.IsLeaseExtendable(node)) {
Self->Dirty.ExtendLease(node);
Self->Dirty.DbAddNode(node, txc);
Self->Dirty.UpdateEpochVersion();
@@ -67,6 +67,7 @@ public:
}
Response->Record.SetExpire(node.Expire.GetValue());
+ Response->Record.SetExpireV2(node.ExpireV2.GetValue());
Response->Record.MutableStatus()->SetCode(TStatus::OK);
Self->Dirty.Epoch.Serialize(*Response->Record.MutableEpoch());
diff --git a/ydb/core/mind/node_broker__register_node.cpp b/ydb/core/mind/node_broker__register_node.cpp
index 64266d34695..e53d3df77cb 100644
--- a/ydb/core/mind/node_broker__register_node.cpp
+++ b/ydb/core/mind/node_broker__register_node.cpp
@@ -73,7 +73,6 @@ public:
auto host = rec.GetHost();
ui16 port = (ui16)rec.GetPort();
TString addr = rec.GetAddress();
- auto expire = rec.GetFixedNodeId() ? TInstant::Max() : Self->Dirty.Epoch.NextEnd;
LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxRegisterNode Execute");
LOG_INFO_S(ctx, NKikimrServices::NODE_BROKER,
@@ -135,7 +134,7 @@ public:
Self->Dirty.FixNodeId(node);
Self->Dirty.DbAddNode(node, txc);
FixNodeId = true;
- } else if (!node.IsFixed() && node.Expire < expire) {
+ } else if (Self->Dirty.IsLeaseExtendable(node)) {
Self->Dirty.ExtendLease(node);
Self->Dirty.DbAddNode(node, txc);
ExtendLease = true;
@@ -170,7 +169,16 @@ public:
Node = MakeHolder<TNodeInfo>(NodeId, rec.GetAddress(), host, rec.GetResolveHost(), port, loc);
Node->AuthorizedByCertificate = rec.GetAuthorizedByCertificate();
Node->Lease = 1;
- Node->Expire = expire;
+ if (rec.GetFixedNodeId()) {
+ Node->Expire = TInstant::Max();
+ Node->ExpireV2 = TInstant::Max();
+ Node->AliveUntil = TInstant::Max();
+ } else {
+ Node->Expire = Self->Dirty.Epoch.NextEnd;
+ Node->ExpireV2 = Self->Dirty.Epoch.NextEnd + Self->Dirty.LeaseDuration;
+ Node->AliveUntil = Self->Dirty.Epoch.NextEnd;
+ }
+ Node->Liveness = ENodeLiveness::Alive;
Node->Version = Self->Dirty.Epoch.Version + 1;
Node->State = ENodeState::Active;
diff --git a/ydb/core/mind/node_broker__scheme.h b/ydb/core/mind/node_broker__scheme.h
index a740a7ac7e7..46dffe5451f 100644
--- a/ydb/core/mind/node_broker__scheme.h
+++ b/ydb/core/mind/node_broker__scheme.h
@@ -14,6 +14,7 @@ namespace NKikimr {
namespace NNodeBroker {
enum class ENodeState : ui8;
+enum class ENodeLiveness : ui8;
struct Schema : NIceDb::Schema {
enum class EMainNodesTable : ui64 {
@@ -33,11 +34,14 @@ struct Schema : NIceDb::Schema {
// struct Body : Column<9, NScheme::NTypeIds::Uint64> {};
struct Lease : Column<10, NScheme::NTypeIds::Uint32> {};
struct Expire : Column<11, NScheme::NTypeIds::Uint64> {};
+ struct ExpireV2 : Column<17, NScheme::NTypeIds::Uint64> {};
struct Location : Column<12, NScheme::NTypeIds::String> {};
struct ServicedSubDomain : Column<13, NScheme::NTypeIds::String> { using Type = NKikimrSubDomains::TDomainKey; };
struct SlotIndex : Column<14, NScheme::NTypeIds::Uint32> {};
struct AuthorizedByCertificate : Column<15, NScheme::NTypeIds::Bool> {};
//struct BridgePileId : Column<16, NScheme::NTypeIds::Uint32> {};
+ struct AliveUntil : Column<18, NScheme::NTypeIds::Uint64> {};
+ struct Liveness : Column<19, NScheme::NTypeIds::Uint8> { using Type = ENodeLiveness; };
using TKey = TableKey<ID>;
using TColumns = TableColumns<
@@ -51,7 +55,10 @@ struct Schema : NIceDb::Schema {
Location,
ServicedSubDomain,
SlotIndex,
- AuthorizedByCertificate
+ AuthorizedByCertificate,
+ ExpireV2,
+ AliveUntil,
+ Liveness
>;
};
diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h
index 177be517b72..4d30c86f4df 100644
--- a/ydb/core/mind/node_broker_impl.h
+++ b/ydb/core/mind/node_broker_impl.h
@@ -114,7 +114,7 @@ private:
bool IsFixed() const
{
- return Expire == TInstant::Max();
+ return Expire == TInstant::Max() || ExpireV2 == TInstant::Max();
}
static TString ExpirationString(TInstant expire)
@@ -133,20 +133,34 @@ private:
return ExpirationString(Expire);
}
+ TString ExpirationV2String() const
+ {
+ return ExpirationString(ExpireV2);
+ }
+
+ TString AliveUntilString() const
+ {
+ return AliveUntil.ToRfc822StringLocal();
+ }
+
// Lease is incremented each time node extends its lifetime.
ui32 Lease = 0;
TInstant Expire;
+ TInstant ExpireV2;
bool AuthorizedByCertificate = false;
std::optional<ui32> SlotIndex;
TSubDomainKey ServicedSubDomain;
ENodeState State = ENodeState::Removed;
ui64 Version = 0;
+ TInstant AliveUntil;
+ ENodeLiveness Liveness = ENodeLiveness::Alive;
};
// State changes to apply while moving to the next epoch.
struct TStateDiff {
TVector<ui32> NodesToExpire;
TVector<ui32> NodesToRemove;
+ TVector<ui32> NodesToMakeDead;
TEpochInfo NewEpoch;
TApproximateEpochStartInfo NewApproxEpochStart;
};
@@ -337,6 +351,8 @@ private:
const TActorContext &ctx);
bool EnableStableNodeNames = false;
+ bool EnableLongLease = false;
+
ui64 MaxStaticId;
ui64 MinDynamicId;
ui64 MaxDynamicId;
@@ -367,6 +383,7 @@ private:
// Internal state modifiers. Don't affect DB.
void RegisterNewNode(const TNodeInfo &info);
void AddNode(const TNodeInfo &info);
+ bool IsLeaseExtendable(const TNodeInfo &node) const;
void ExtendLease(TNodeInfo &node);
void FixNodeId(TNodeInfo &node);
void RecomputeFreeIds();
@@ -397,6 +414,7 @@ private:
// Current config.
NKikimrNodeBroker::TConfig Config;
TDuration EpochDuration = TDuration::Hours(1);
+ TDuration LeaseDuration = TDuration::Hours(72);
TVector<std::pair<ui32, ui32>> BannedIds;
ui64 ConfigSubscriptionId = 0;
TString StableNodeNamePrefix = "slot-";
diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp
index 017f3b517af..41c7db5a91b 100644
--- a/ydb/core/mind/node_broker_ut.cpp
+++ b/ydb/core/mind/node_broker_ut.cpp
@@ -2,6 +2,8 @@
#include "node_broker__scheme.h"
#include "dynamic_nameserver_impl.h"
+#include <ydb/core/cms/console/console.h>
+
#include <ydb/core/testlib/basics/appdata.h>
#include <ydb/core/testlib/basics/storage.h>
#include <ydb/core/testlib/basics/helpers.h>
@@ -76,7 +78,8 @@ THashMap<ui32, TIntrusivePtr<TNodeWardenConfig>> NodeWardenConfigs;
void SetupServices(TTestActorRuntime &runtime,
ui32 maxDynNodes,
- bool enableNodeBrokerDeltaProtocol)
+ bool enableNodeBrokerDeltaProtocol,
+ bool enableNodeBrokerLongLease = false)
{
const ui32 domainsNum = 1;
const ui32 disksInDomain = 1;
@@ -191,6 +194,7 @@ void SetupServices(TTestActorRuntime &runtime,
app.FeatureFlags.SetEnableNodeBrokerSingleDomainMode(true);
app.FeatureFlags.SetEnableStableNodeNames(true);
app.FeatureFlags.SetEnableNodeBrokerDeltaProtocol(enableNodeBrokerDeltaProtocol);
+ app.FeatureFlags.SetEnableNodeBrokerLongLease(enableNodeBrokerLongLease);
runtime.Initialize(app.Unwrap());
@@ -261,6 +265,50 @@ void SetEpochDuration(TTestActorRuntime& runtime,
SetConfig(runtime, sender, config);
}
+void SetLeaseDuration(TTestActorRuntime& runtime,
+ TActorId sender,
+ TDuration lease)
+{
+ NKikimrNodeBroker::TConfig config;
+ config.SetLeaseDuration(lease.GetValue());
+ SetConfig(runtime, sender, config);
+}
+
+// Toggle the EnableNodeBrokerLongLease feature flag on the running NodeBroker
+// tablet at runtime by delivering a config notification (as the configs
+// dispatcher would in production) and waiting for the acknowledgement.
+void SetNodeBrokerLongLease(TTestActorRuntime& runtime,
+ TActorId sender,
+ bool enable)
+{
+ auto event = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationRequest>();
+ auto& featureFlags = *event->Record.MutableConfig()->MutableFeatureFlags();
+ featureFlags.SetEnableStableNodeNames(true);
+ featureFlags.SetEnableNodeBrokerLongLease(enable);
+ runtime.SendToPipe(MakeNodeBrokerID(), sender, event.Release(), 0, GetPipeConfigWithRetries());
+
+ TAutoPtr<IEventHandle> handle;
+ runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(handle);
+}
+
+// Toggle the EnableNodeBrokerLongLease feature flag on the node-0 dynamic
+// nameserver at runtime via a config notification, and wait for the ack.
+void SetNameserverLongLease(TTestActorRuntime& runtime,
+ TActorId sender,
+ bool enable)
+{
+ auto event = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationRequest>();
+ auto& featureFlags = *event->Record.MutableConfig()->MutableFeatureFlags();
+ // Keep the delta protocol flag unchanged (these tests use the epoch
+ // protocol); a differing value would make the nameserver reset its pipe.
+ featureFlags.SetEnableNodeBrokerDeltaProtocol(false);
+ featureFlags.SetEnableNodeBrokerLongLease(enable);
+ runtime.Send(new IEventHandle(GetNameserviceActorId(), sender, event.Release()));
+
+ TAutoPtr<IEventHandle> handle;
+ runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(handle);
+}
+
void AsyncSetBannedIds(TTestActorRuntime& runtime,
TActorId sender,
const TVector<std::pair<ui32, ui32>> ids)
@@ -285,7 +333,8 @@ void SetBannedIds(TTestActorRuntime& runtime,
void Setup(TTestActorRuntime& runtime,
ui32 maxDynNodes = 3,
const TVector<TString>& databases = {},
- bool enableNodeBrokerDeltaProtocol = false)
+ bool enableNodeBrokerDeltaProtocol = false,
+ bool enableNodeBrokerLongLease = false)
{
using namespace NMalloc;
TMallocInfo mallocInfo = MallocInfo();
@@ -301,7 +350,7 @@ void Setup(TTestActorRuntime& runtime,
runtime.SetScheduledEventFilter(scheduledFilter);
SetupLogging(runtime);
- SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol);
+ SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol, enableNodeBrokerLongLease);
TActorId sender = runtime.AllocateEdgeActor();
ui32 txId = 100;
@@ -670,6 +719,27 @@ NKikimrNodeBroker::TEpoch CheckNodesList(TTestActorRuntime &runtime,
return rec.GetEpoch();
}
+void CheckNodeLiveness(TTestActorRuntime &runtime,
+ TActorId sender,
+ ui32 nodeId,
+ ENodeLiveness liveness)
+{
+ const auto &rec = ListNodes(runtime, sender);
+ // The nodes list is a concatenation of the epoch snapshot and per-node
+ // deltas, so a node may appear more than once; the last occurrence wins.
+ bool found = false;
+ ui32 actual = 0;
+ for (const auto &node : rec.GetNodes()) {
+ if (node.GetNodeId() == nodeId) {
+ found = true;
+ actual = node.GetLiveness();
+ }
+ }
+ UNIT_ASSERT_C(found, "node " << nodeId << " is not in the alive nodes list");
+ UNIT_ASSERT_VALUES_EQUAL_C(actual, static_cast<ui32>(liveness),
+ "unexpected liveness for node " << nodeId);
+}
+
void CheckNodeInfo(TTestActorRuntime &runtime,
TActorId sender,
ui32 nodeId,
@@ -956,6 +1026,33 @@ THolder<TEvInterconnect::TEvNodesInfo> GetNameserverNodesListEv(TTestActorRuntim
return IEventHandle::Release<TEvInterconnect::TEvNodesInfo>(handle);
}
+THolder<TEvInterconnect::TEvNodesInfo> GetNameserverNodesListEv(TTestActorRuntime &runtime, TActorId sender,
+ bool onlyAliveNodes) {
+ runtime.Send(new IEventHandle(GetNameserviceActorId(), sender,
+ new TEvInterconnect::TEvListNodes(false, onlyAliveNodes)));
+
+ TAutoPtr<IEventHandle> handle;
+ auto reply = runtime.GrabEdgeEventRethrow<TEvInterconnect::TEvNodesInfo>(handle);
+ UNIT_ASSERT(reply);
+
+ return IEventHandle::Release<TEvInterconnect::TEvNodesInfo>(handle);
+}
+
+// Returns the set of dynamic (non-static) node ids the nameservice reports.
+// With onlyAliveNodes the request asks the nameservice to omit nodes the
+// NodeBroker has marked dead (only meaningful when the long lease feature is on).
+THashSet<ui32> GetNameserverDynamicNodeIds(TTestActorRuntime &runtime, TActorId sender,
+ bool onlyAliveNodes = false)
+{
+ ui32 maxStaticNodeId = runtime.GetAppData().DynamicNameserviceConfig->MaxStaticNodeId;
+ auto ev = GetNameserverNodesListEv(runtime, sender, onlyAliveNodes);
+ THashSet<ui32> ids;
+ for (auto &node : ev->Nodes)
+ if (node.NodeId > maxStaticNodeId)
+ ids.insert(node.NodeId);
+ return ids;
+}
+
void GetNameserverNodesList(TTestActorRuntime &runtime,
TActorId sender,
THashMap<ui32, TEvInterconnect::TNodeInfo> &nodes,
@@ -1395,6 +1492,204 @@ Y_UNIT_TEST_SUITE(TNodeBrokerTest) {
CheckLeaseExtension(runtime, sender, NODE2, TStatus::OK, epoch, true);
}
+ Y_UNIT_TEST(LongLeaseNodeBecomesDeadThenExpires)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ // Lease duration equal to a single (default) epoch: a node without pings
+ // survives one extra epoch (AliveUntil) before becoming Dead and another
+ // epoch (ExpireV2) before it finally expires.
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ auto epoch = GetEpoch(runtime, sender);
+ // Register node NODE1, it is Alive.
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ // After the first epoch the long lease keeps the node Alive.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ // After the second epoch AliveUntil has passed: the node becomes Dead,
+ // but is not expired yet (ExpireV2 is still in the future).
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // After the third epoch ExpireV2 has passed: the node expires.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId());
+ }
+
+ Y_UNIT_TEST(LongLeasePingKeepsNodeAlive)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ auto epoch = GetEpoch(runtime, sender);
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+
+ // As long as the node keeps extending its lease each epoch, it never
+ // becomes Dead nor expires, even with the long lease feature enabled.
+ for (ui32 i = 0; i < 5; ++i) {
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckLeaseExtension(runtime, sender, NODE1, TStatus::OK, epoch);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+ }
+ }
+
+ Y_UNIT_TEST(LongLeaseDeadNodeRevivedByRegistration)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ auto epoch = GetEpoch(runtime, sender);
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+
+ // Let the node become Dead.
+ WaitForEpochUpdate(runtime, sender);
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // Re-registration of a Dead node revives it (extends lease, back to Alive).
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ // The revived node survives the following epoch as Alive.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+ }
+
+ Y_UNIT_TEST(LongLeaseDeadStatePersistedAfterRestart)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ auto epoch = GetEpoch(runtime, sender);
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+
+ // Let the node become Dead.
+ WaitForEpochUpdate(runtime, sender);
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // Dead state (AliveUntil/Liveness) must survive a tablet restart.
+ RestartNodeBroker(runtime);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // The node still expires by ExpireV2 in the next epoch.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId());
+ }
+
+ Y_UNIT_TEST(LongLeaseDisabledNodeExpiresByExpire)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ false);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ // Even with a lease duration configured, when the feature is disabled the
+ // node expires by Expire (a single epoch after its last ping) and never
+ // transitions to the Dead state.
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ auto epoch = GetEpoch(runtime, sender);
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ // Expires by Expire, without ever becoming Dead.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId());
+ }
+
+ Y_UNIT_TEST(LongLeaseEnabledAtRuntimeExtendsLease)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ false);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ // Register while the feature is disabled. Under the old rules this node
+ // would expire one epoch after its last ping (see
+ // LongLeaseDisabledNodeExpiresByExpire).
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1);
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ // Enable the feature at runtime, before the node would have expired.
+ SetNodeBrokerLongLease(runtime, sender, true);
+
+ // With the feature now on, the node follows the long-lease lifecycle based
+ // on the ExpireV2/AliveUntil stored at registration: it stays Alive one
+ // epoch, then becomes Dead instead of expiring.
+ auto epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Alive);
+
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // It only expires (by ExpireV2) the epoch after becoming Dead.
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId());
+ }
+
+ Y_UNIT_TEST(LongLeaseDisabledAtRuntimeExpiresDeadNode)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1);
+
+ // Let the node become Dead (but not expired) under the long lease rules.
+ WaitForEpochUpdate(runtime, sender);
+ auto epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {NODE1}, {}, epoch.GetId());
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // Disable the feature while a Dead-but-not-expired node exists. The broker
+ // switches back to the Expire-based rule; the stale node (whose Expire is
+ // already in the past) must expire at the next epoch. It must not be
+ // processed as both "make dead" and "expire" in the same diff, which would
+ // crash ApplyStateDiff.
+ SetNodeBrokerLongLease(runtime, sender, false);
+
+ epoch = WaitForEpochUpdate(runtime, sender);
+ CheckNodesList(runtime, sender, {}, {NODE1}, epoch.GetId());
+ }
+
Y_UNIT_TEST(TestListNodes)
{
TTestBasicRuntime runtime(8, false);
@@ -4992,6 +5287,114 @@ Y_UNIT_TEST_SUITE(TDynamicNameserverTest) {
// No pending cache miss requests are left
CheckNoPendingCacheMissesLeft(runtime, 0);
}
+
+ Y_UNIT_TEST(ListNodesEpochDeltaLongLeaseOnlyAlive)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ // Lease duration equal to a single (default) epoch: a node without pings
+ // stays Alive for one epoch, becomes Dead the next and expires the epoch
+ // after that (see TNodeBrokerTest::LongLeaseNodeBecomesDeadThenExpires).
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1);
+
+ // While the node is Alive it is served in both the full and the
+ // alive-only nodes lists.
+ UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+
+ // Advance until the node becomes Dead, but has not expired yet.
+ WaitForEpochUpdate(runtime, sender);
+ WaitForEpochUpdate(runtime, sender);
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // The full list still contains the dead-but-not-expired node, while the
+ // alive-only list omits it.
+ UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+
+ // After the node finally expires it disappears from both lists.
+ WaitForEpochUpdate(runtime, sender);
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+ }
+
+ Y_UNIT_TEST(LongLeaseDeadNodeStillResolvable)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1);
+ CheckResolveNode(runtime, sender, NODE1, "1.2.3.4");
+
+ // The node stops pinging and becomes Dead, but its long-lease expiration
+ // (ExpireV2) is still in the future, so the nameservice keeps resolving it
+ // (ResolveDynamicNode/GetNode use GetExpire(EnableLongLease) == ExpireV2).
+ WaitForEpochUpdate(runtime, sender);
+ WaitForEpochUpdate(runtime, sender);
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // Refresh the nameservice view from the NodeBroker, then resolve.
+ GetNameserverDynamicNodeIds(runtime, sender);
+ CheckResolveNode(runtime, sender, NODE1, "1.2.3.4");
+ CheckGetNode(runtime, sender, NODE1, true);
+
+ // Once ExpireV2 passes the node expires and is no longer resolvable.
+ WaitForEpochUpdate(runtime, sender);
+ GetNameserverDynamicNodeIds(runtime, sender);
+ CheckResolveUnknownNode(runtime, sender, NODE1);
+ CheckGetNode(runtime, sender, NODE1, false);
+
+ // No pending cache miss requests are left
+ CheckNoPendingCacheMissesLeft(runtime, 0);
+ }
+
+ Y_UNIT_TEST(LongLeaseFeatureFlagToggledAtRuntime)
+ {
+ TTestBasicRuntime runtime(8, false);
+ Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true);
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ SetLeaseDuration(runtime, sender, TDuration::Minutes(5));
+
+ CheckRegistration(runtime, sender, "host1", 1001, "host1.host1.host1", "1.2.3.4",
+ 1, 2, 3, 4, TStatus::OK, NODE1);
+
+ // Drive the node into the Dead-but-not-expired window. The NodeBroker
+ // keeps the long lease feature enabled the whole time, so the node stays
+ // registered (its ExpireV2 is still in the future) while we toggle the
+ // feature on the nameserver only.
+ WaitForEpochUpdate(runtime, sender);
+ WaitForEpochUpdate(runtime, sender);
+ CheckNodeLiveness(runtime, sender, NODE1, ENodeLiveness::Dead);
+
+ // Nameserver has long lease ON: the dead node is served from ExpireV2 in
+ // the full list and hidden from the alive-only list.
+ UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+
+ // Turn the feature OFF at runtime. The nameserver invalidates its caches
+ // and switches to the shorter Expire; the node (whose Expire is already in
+ // the past) drops out of both lists, and the alive-only filter is disabled.
+ SetNameserverLongLease(runtime, sender, false);
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+
+ // Turn it back ON: the node reappears in the full list (served from
+ // ExpireV2 again) and is again hidden from the alive-only list, proving the
+ // flag switch and cache invalidation took effect.
+ SetNameserverLongLease(runtime, sender, true);
+ UNIT_ASSERT(GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ false).contains(NODE1));
+ UNIT_ASSERT(!GetNameserverDynamicNodeIds(runtime, sender, /* onlyAlive */ true).contains(NODE1));
+ }
}
Y_UNIT_TEST_SUITE(TSlotIndexesPoolTest) {
diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto
index 7f1d5b1374a..2321aa9e7eb 100644
--- a/ydb/core/protos/feature_flags.proto
+++ b/ydb/core/protos/feature_flags.proto
@@ -335,4 +335,5 @@ message TFeatureFlags {
optional bool EnableFulltextIndexRowId = 289 [default = true];
optional bool EnableForcedColumnCompactions = 290 [default = false];
optional bool EnableExportFiltering = 291 [default = false];
+ optional bool EnableNodeBrokerLongLease = 292 [default = false];
}
diff --git a/ydb/core/protos/node_broker.proto b/ydb/core/protos/node_broker.proto
index deb544c3823..4240a396d9b 100644
--- a/ydb/core/protos/node_broker.proto
+++ b/ydb/core/protos/node_broker.proto
@@ -21,7 +21,9 @@ message TNodeInfo {
optional string Address = 5;
optional NActorsInterconnect.TNodeLocation Location = 6;
optional uint64 Expire = 7;
+ optional uint64 ExpireV2 = 9;
optional string Name = 8;
+ optional uint32 Liveness = 10;
}
message TNodeInfoSchema {
@@ -35,6 +37,9 @@ message TNodeInfoSchema {
optional NKikimrSubDomains.TDomainKey ServicedSubDomain = 8;
optional uint32 SlotIndex = 9;
optional bool AuthorizedByCertificate = 10;
+ optional uint64 ExpireV2 = 11;
+ optional uint64 AliveUntil = 12;
+ optional uint32 Liveness = 13;
}
message TVersionInfo {
@@ -113,6 +118,7 @@ message TExtendLeaseResponse {
optional TStatus Status = 1;
optional uint32 NodeId = 2;
optional uint64 Expire = 3;
+ optional uint64 ExpireV2 = 5;
// Epoch at which ping happened. Lease is extended
// until the end of the next epoch.
optional TEpoch Epoch = 4;
@@ -129,6 +135,7 @@ message TConfig {
// Don't allocate and extend lease for IDs from banned intervals.
repeated TNodeIds BannedNodeIds = 2;
optional string StableNodeNamePrefix = 3 [default = "slot-"];
+ optional uint64 LeaseDuration = 4 [default = 259200000000];
}
message TGetConfigRequest {
diff --git a/ydb/library/actors/core/interconnect.h b/ydb/library/actors/core/interconnect.h
index 9125bb7a54b..70b51a85d53 100644
--- a/ydb/library/actors/core/interconnect.h
+++ b/ydb/library/actors/core/interconnect.h
@@ -209,11 +209,13 @@ namespace NActors {
struct TEvListNodes: public TEventLocal<TEvListNodes, EvListNodes> {
const bool SubscribeToStaticNodeChanges = false;
+ const bool OnlyAliveNodes = true;
TEvListNodes() = default;
- TEvListNodes(bool subscribeToStaticNodeChanges)
+ TEvListNodes(bool subscribeToStaticNodeChanges, bool onlyAliveNodes = true)
: SubscribeToStaticNodeChanges(subscribeToStaticNodeChanges)
+ , OnlyAliveNodes(onlyAliveNodes)
{}
};