Add soft MinDynamicNodeId barrier, KIKIMR-16159

Add UT

Add soft MinDynamicNodeId limit

9 files changed, 48 insertions, 3 deletions

diff --git a/ydb/core/base/nameservice.h b/ydb/core/base/nameservice.h
index dc023bc8ac1..9066abffaaa 100644
--- a/ydb/core/base/nameservice.h
+++ b/ydb/core/base/nameservice.h
@@ -7,6 +7,7 @@ namespace NKikimr {
 struct TDynamicNameserviceConfig : public TThrRefBase {
     ui32 MaxStaticNodeId;
     ui32 MaxDynamicNodeId;
+    ui32 MinDynamicNodeId;
 };
 
 } // NKikimr
diff --git a/ydb/core/cms/cms_ut_common.cpp b/ydb/core/cms/cms_ut_common.cpp
index e30891e2f5a..e8e0771192e 100644
--- a/ydb/core/cms/cms_ut_common.cpp
+++ b/ydb/core/cms/cms_ut_common.cpp
@@ -505,6 +505,7 @@ static void SetupServices(TTestActorRuntime &runtime,
     runtime.Initialize(app.Unwrap());
     auto dnsConfig = new TDynamicNameserviceConfig();
     dnsConfig->MaxStaticNodeId = 1000;
+    dnsConfig->MinDynamicNodeId = 1001;
     dnsConfig->MaxDynamicNodeId = 2000;
     runtime.GetAppData().DynamicNameserviceConfig = dnsConfig;
     runtime.GetAppData().DisableCheckingSysNodesCms = true;
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index 50e6fa15ce4..2c7249d5e2d 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -309,6 +309,7 @@ public:
         auto &dnConfig = Config.GetDynamicNameserviceConfig();
         TIntrusivePtr<TDynamicNameserviceConfig> config = new TDynamicNameserviceConfig;
         config->MaxStaticNodeId = dnConfig.GetMaxStaticNodeId();
+        config->MinDynamicNodeId = dnConfig.GetMinDynamicNodeId();
         config->MaxDynamicNodeId = dnConfig.GetMaxDynamicNodeId();
         appData->DynamicNameserviceConfig = config;
     }
diff --git a/ydb/core/mind/dynamic_nameserver_mon.cpp b/ydb/core/mind/dynamic_nameserver_mon.cpp
index 65a3b9664e4..782f7661303 100644
--- a/ydb/core/mind/dynamic_nameserver_mon.cpp
+++ b/ydb/core/mind/dynamic_nameserver_mon.cpp
@@ -173,6 +173,10 @@ void TDynamicNameserver::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext
                 << "      <td>" << config->MaxStaticNodeId << "</td>" << Endl
                 << "    </tr>" << Endl
                 << "    <tr>" << Endl
+                << "      <td class='right-align'>Min dynamic node ID:</td>" << Endl
+                << "      <td>" << config->MinDynamicNodeId << "</td>" << Endl
+                << "    </tr>" << Endl
+                << "    <tr>" << Endl
                 << "      <td class='right-align'>Max dynamic node ID:</td>" << Endl
                 << "      <td>" << config->MaxDynamicNodeId << "</td>" << Endl
                 << "    </tr>" << Endl
diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp
index 7d23621c61e..a65bbd33a3d 100644
--- a/ydb/core/mind/node_broker.cpp
+++ b/ydb/core/mind/node_broker.cpp
@@ -62,6 +62,7 @@ void TNodeBroker::OnActivateExecutor(const TActorContext &ctx)
     SingleDomainAlloc = SingleDomain && appData->FeatureFlags.GetEnableNodeBrokerSingleDomainMode();
 
     MaxStaticId = Min(appData->DynamicNameserviceConfig->MaxStaticNodeId, TActorId::MaxNodeId);
+    MinDynamicId = Min(appData->DynamicNameserviceConfig->MinDynamicNodeId, TActorId::MaxNodeId);
     MaxDynamicId = Min(appData->DynamicNameserviceConfig->MaxDynamicNodeId, TActorId::MaxNodeId);
 
     ClearState();
@@ -203,10 +204,10 @@ void TNodeBroker::RecomputeFreeIds()
     FreeIds.Clear();
 
     if (SingleDomainAlloc) {
-        FreeIds.Set(MaxStaticId + 1, MaxDynamicId + 1);
+        FreeIds.Set(MinDynamicId, MaxDynamicId + 1);
     } else {
-        auto firstId = RewriteNodeId(MaxStaticId + 1);
-        if (firstId <= MaxStaticId)
+        auto firstId = RewriteNodeId(MinDynamicId);
+        if (firstId < MinDynamicId)
             firstId += NodeIdStep();
 
         auto lastId = RewriteNodeId(MaxDynamicId);
diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h
index 3a592006537..3b46c172f7e 100644
--- a/ydb/core/mind/node_broker_impl.h
+++ b/ydb/core/mind/node_broker_impl.h
@@ -311,6 +311,7 @@ private:
     // Current config.
     NKikimrNodeBroker::TConfig Config;
     ui64 MaxStaticId;
+    ui64 MinDynamicId;
     ui64 MaxDynamicId;
     TDuration EpochDuration;
     TVector<std::pair<ui32, ui32>> BannedIds;
diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp
index 49f2acca650..890b3b5f0dc 100644
--- a/ydb/core/mind/node_broker_ut.cpp
+++ b/ydb/core/mind/node_broker_ut.cpp
@@ -153,6 +153,7 @@ void SetupServices(TTestActorRuntime &runtime,
     runtime.GetAppData().DynamicNameserviceConfig = new TDynamicNameserviceConfig;
     auto dnConfig = runtime.GetAppData().DynamicNameserviceConfig;
     dnConfig->MaxStaticNodeId = 1023;
+    dnConfig->MinDynamicNodeId = 1024;
     dnConfig->MaxDynamicNodeId = 1024 + (singleDomainMode ? (maxDynNodes - 1) : 32 * (maxDynNodes - 1));
     runtime.GetAppData().FeatureFlags.SetEnableNodeBrokerSingleDomainMode(singleDomainMode);
 
@@ -1206,6 +1207,37 @@ Y_UNIT_TEST_SUITE(TNodeBrokerTest) {
         WaitForEpochUpdate(runtime, sender);
         epoch = CheckNodesList(runtime, sender, {1024}, {}, 3);
     }
+
+    Y_UNIT_TEST(MinDynamicNodeIdShifted)
+    {
+        TTestBasicRuntime runtime(8, false);
+        Setup(runtime);
+        TActorId sender = runtime.AllocateEdgeActor();
+
+        // There should be no dynamic nodes initially.
+        auto epoch = GetEpoch(runtime, sender);
+        // Register node 1024.
+        CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+                          1, 2, 3, 4, TStatus::OK, 1024, epoch.GetNextEnd());
+        
+        // Update config and restart NodeBroker
+        auto dnConfig = runtime.GetAppData().DynamicNameserviceConfig;
+        dnConfig->MinDynamicNodeId += 64;
+        dnConfig->MaxDynamicNodeId += 64;
+        RestartNodeBroker(runtime);
+
+        // Register node 1088.
+        CheckRegistration(runtime, sender, "host2", 1001, "host2.yandex.net", "1.2.3.5",
+                          1, 2, 3, 5, TStatus::OK, 1088, epoch.GetNextEnd());
+
+        // Wait until epoch expiration.
+        WaitForEpochUpdate(runtime, sender);
+        epoch = GetEpoch(runtime, sender);
+
+        // Check lease extension for both nodes.
+        CheckLeaseExtension(runtime, sender, 1024, TStatus::OK, epoch);
+        CheckLeaseExtension(runtime, sender, 1088, TStatus::OK, epoch);
+    }
 }
 
 Y_UNIT_TEST_SUITE(TDynamicNameserverTest) {
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index 4a6b94c04db..228efda1d1b 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -172,6 +172,9 @@ message TDynamicNameserviceConfig {
     optional uint32 MaxStaticNodeId = 1 [default = 1000];
     optional uint32 MaxDynamicNodeId = 2 [default = 200000];
     optional uint64 LeaseDuration = 3 [default = 3600000000]; // DEPRECATED
+    // Soft limit, new dynamic nodes will be registered in range [MinDynamicNodeId, MaxDynamicNodeId],
+    // but already existing dynamic nodes in range (MaxStaticNodeId, MaxDynamicNodeId) will be handled properly
+    optional uint32 MinDynamicNodeId = 4 [default = 50000];
 }
 
 message TDomainsConfig {
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index c1041f5c0a1..bbea4fbb2e7 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -238,6 +238,7 @@ namespace Tests {
             appData.DynamicNameserviceConfig = new TDynamicNameserviceConfig;
             auto dnConfig = appData.DynamicNameserviceConfig;
             dnConfig->MaxStaticNodeId = 1023;
+            dnConfig->MinDynamicNodeId = 1024;
             dnConfig->MaxDynamicNodeId = 1024 + 100;
         });