Fix app data initialization races in TServer for tests

7 files changed, 55 insertions, 25 deletions

diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp
index 03688fa00d..482b7b97e6 100644
--- a/ydb/core/testlib/actors/test_runtime.cpp
+++ b/ydb/core/testlib/actors/test_runtime.cpp
@@ -84,6 +84,11 @@ namespace NActors {
         CleanupNodes();
     }
 
+    void TTestActorRuntime::AddAppDataInit(std::function<void(ui32, NKikimr::TAppData&)> callback) {
+        Y_VERIFY(!IsInitialized, "Actor system is already initialized");
+        AppDataInit_.push_back(std::move(callback));
+    }
+
     void TTestActorRuntime::Initialize(TEgg egg) {
         IsInitialized = true;
 
@@ -147,6 +152,10 @@ namespace NActors {
                 nodeAppData->KeyConfig.CopyFrom(app0->KeyConfig);
             }
 
+            for (auto& callback : AppDataInit_) {
+                callback(nodeIndex, *nodeAppData);
+            }
+
             if (NeedMonitoring && !SingleSysEnv) {
                 ui16 port = GetPortManager().GetPort();
                 node->Mon.Reset(new NActors::TSyncHttpMon({
@@ -168,6 +177,8 @@ namespace NActors {
                 nodeAppData->Mon->Start();
             }
         }
+
+        AppDataInit_.clear();
     }
 
     ui16 TTestActorRuntime::GetMonPort(ui32 nodeIndex) const {
diff --git a/ydb/core/testlib/actors/test_runtime.h b/ydb/core/testlib/actors/test_runtime.h
index c9bb99b006..407970ff25 100644
--- a/ydb/core/testlib/actors/test_runtime.h
+++ b/ydb/core/testlib/actors/test_runtime.h
@@ -52,6 +52,7 @@ namespace NActors {
 
         ~TTestActorRuntime();
 
+        void AddAppDataInit(std::function<void(ui32, NKikimr::TAppData&)> callback);
         virtual void Initialize(TEgg);
 
         ui16 GetMonPort(ui32 nodeIndex = 0) const;
@@ -72,6 +73,7 @@ namespace NActors {
         }
 
         static bool DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline);
+
     private:
         void Initialize() override;
         TIntrusivePtr<::NMonitoring::TDynamicCounters> GetCountersForComponent(TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, const char* component) override;
@@ -92,5 +94,6 @@ namespace NActors {
         THolder<IDestructable> Opaque;
         TVector<ui16> MonPorts;
         TActorId SleepEdgeActor;
+        TVector<std::function<void(ui32, NKikimr::TAppData&)>> AppDataInit_;
     };
 } // namespace NActors
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index ff072904ee..3a2150e536 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -212,27 +212,33 @@ namespace Tests {
         Runtime->SetupMonitoring();
         Runtime->SetLogBackend(Settings->LogBackend);
 
+        Runtime->AddAppDataInit([this](ui32 nodeIdx, NKikimr::TAppData& appData) {
+            Y_UNUSED(nodeIdx);
+
+            appData.AuthConfig.MergeFrom(Settings->AuthConfig);
+            appData.PQConfig.MergeFrom(Settings->PQConfig);
+            appData.PQClusterDiscoveryConfig.MergeFrom(Settings->PQClusterDiscoveryConfig);
+            appData.NetClassifierConfig.MergeFrom(Settings->NetClassifierConfig);
+            appData.StreamingConfig.MergeFrom(Settings->AppConfig.GetGRpcConfig().GetStreamingConfig());
+            appData.EnforceUserTokenRequirement = Settings->AppConfig.GetDomainsConfig().GetSecurityConfig().GetEnforceUserTokenRequirement();
+            appData.DomainsConfig.MergeFrom(Settings->AppConfig.GetDomainsConfig());
+            appData.PersQueueGetReadSessionsInfoWorkerFactory = Settings->PersQueueGetReadSessionsInfoWorkerFactory.get();
+            appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
+            appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
+
+            appData.DynamicNameserviceConfig = new TDynamicNameserviceConfig;
+            auto dnConfig = appData.DynamicNameserviceConfig;
+            dnConfig->MaxStaticNodeId = 1023;
+            dnConfig->MaxDynamicNodeId = 1024 + 100;
+        });
+
         const bool mockDisk = (StaticNodes() + DynamicNodes()) == 1 && Settings->EnableMockOnSingleNode;
         SetupTabletServices(*Runtime, &app, mockDisk, Settings->CustomDiskParams, Settings->CacheParams);
 
+        // WARNING: must be careful about modifying app data after actor system starts
+
         for (ui32 nodeIdx = 0; nodeIdx < StaticNodes() + DynamicNodes(); ++nodeIdx) {
             SetupDomainLocalService(nodeIdx);
-            Runtime->GetAppData(nodeIdx).AuthConfig.MergeFrom(Settings->AuthConfig);
-            Runtime->GetAppData(nodeIdx).PQConfig.MergeFrom(Settings->PQConfig);
-            Runtime->GetAppData(nodeIdx).PQClusterDiscoveryConfig.MergeFrom(Settings->PQClusterDiscoveryConfig);
-            Runtime->GetAppData(nodeIdx).NetClassifierConfig.MergeFrom(Settings->NetClassifierConfig);
-            Runtime->GetAppData(nodeIdx).StreamingConfig.MergeFrom(Settings->AppConfig.GetGRpcConfig().GetStreamingConfig());
-            Runtime->GetAppData(nodeIdx).EnforceUserTokenRequirement = Settings->AppConfig.GetDomainsConfig().GetSecurityConfig().GetEnforceUserTokenRequirement();
-            Runtime->GetAppData(nodeIdx).DomainsConfig.MergeFrom(Settings->AppConfig.GetDomainsConfig());
-            Runtime->GetAppData(nodeIdx).PersQueueGetReadSessionsInfoWorkerFactory = Settings->PersQueueGetReadSessionsInfoWorkerFactory.get();
-            Runtime->GetAppData(nodeIdx).DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
-            Runtime->GetAppData(nodeIdx).PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
-
-            Runtime->GetAppData(nodeIdx).DynamicNameserviceConfig = new TDynamicNameserviceConfig;
-            auto dnConfig = Runtime->GetAppData(nodeIdx).DynamicNameserviceConfig;
-            dnConfig->MaxStaticNodeId = 1023;
-            dnConfig->MaxDynamicNodeId = 1024 + 100;
-
             SetupConfigurators(nodeIdx);
             SetupProxies(nodeIdx);
         }
diff --git a/ydb/services/bg_tasks/service.cpp b/ydb/services/bg_tasks/service.cpp
index 9fbd6fcec2..488884fb2d 100644
--- a/ydb/services/bg_tasks/service.cpp
+++ b/ydb/services/bg_tasks/service.cpp
@@ -7,11 +7,13 @@ NActors::TActorId MakeServiceId(const ui32 nodeId) {
 }
 
 void TServiceOperator::Register() {
-    Singleton<TServiceOperator>()->EnabledFlag = true;
+    auto* service = Singleton<TServiceOperator>();
+    service->EnabledFlag.store(true);
 }
 
 bool TServiceOperator::IsEnabled() {
-    return Singleton<TServiceOperator>()->EnabledFlag;
+    auto* service = Singleton<TServiceOperator>();
+    return service->EnabledFlag.load();
 }
 
 }
diff --git a/ydb/services/bg_tasks/service.h b/ydb/services/bg_tasks/service.h
index 308836037f..58844f1f07 100644
--- a/ydb/services/bg_tasks/service.h
+++ b/ydb/services/bg_tasks/service.h
@@ -73,7 +73,7 @@ public:
 class TServiceOperator {
 private:
     friend class TExecutor;
-    bool EnabledFlag = false;
+    std::atomic<bool> EnabledFlag{ false };
     static void Register();
 public:
     static bool IsEnabled();
diff --git a/ydb/services/metadata/service.cpp b/ydb/services/metadata/service.cpp
index 1f8bf69bba..617e8fb40b 100644
--- a/ydb/services/metadata/service.cpp
+++ b/ydb/services/metadata/service.cpp
@@ -9,16 +9,22 @@ NActors::TActorId MakeServiceId(const ui32 nodeId) {
 }
 
 void TServiceOperator::Register(const TConfig& config) {
-    Singleton<TServiceOperator>()->EnabledFlag = true;
-    Singleton<TServiceOperator>()->Path = config.GetPath();
+    auto* service = Singleton<TServiceOperator>();
+    std::unique_lock<std::shared_mutex> lock(service->Lock);
+    service->EnabledFlag = true;
+    service->Path = config.GetPath();
 }
 
 bool TServiceOperator::IsEnabled() {
-    return Singleton<TServiceOperator>()->EnabledFlag;
+    auto* service = Singleton<TServiceOperator>();
+    std::shared_lock<std::shared_mutex> lock(service->Lock);
+    return service->EnabledFlag;
 }
 
-const TString& TServiceOperator::GetPath() {
-    return Singleton<TServiceOperator>()->Path;
+TString TServiceOperator::GetPath() {
+    auto* service = Singleton<TServiceOperator>();
+    std::shared_lock<std::shared_mutex> lock(service->Lock);
+    return service->Path;
 }
 
 }
diff --git a/ydb/services/metadata/service.h b/ydb/services/metadata/service.h
index a144249101..9cdb3c255a 100644
--- a/ydb/services/metadata/service.h
+++ b/ydb/services/metadata/service.h
@@ -1,6 +1,7 @@
 #pragma once
 #include <ydb/services/metadata/abstract/common.h>
 #include <library/cpp/actors/core/event_local.h>
+#include <shared_mutex>
 
 namespace NKikimr::NMetadata::NProvider {
 
@@ -72,12 +73,13 @@ class TConfig;
 class TServiceOperator {
 private:
     friend class TService;
+    std::shared_mutex Lock;
     bool EnabledFlag = false;
     TString Path = ".metadata";
     static void Register(const TConfig& config);
 public:
     static bool IsEnabled();
-    static const TString& GetPath();
+    static TString GetPath();
 };
 
 }