Add config transformation to console config pipeline

add config transformation to console config pipeline

8 files changed, 70 insertions, 29 deletions

diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt
index 33e4b61983..6bab24b1d6 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt
@@ -19,6 +19,7 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   cpp-yaml-fyamlcpp
   cms-console-util
   ydb-core-protos
+  ydb-library-yaml_config
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
index ad0cd42cc8..d4946fdcfe 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
@@ -20,6 +20,7 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   cpp-yaml-fyamlcpp
   cms-console-util
   ydb-core-protos
+  ydb-library-yaml_config
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt
index ad0cd42cc8..d4946fdcfe 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt
@@ -20,6 +20,7 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   cpp-yaml-fyamlcpp
   cms-console-util
   ydb-core-protos
+  ydb-library-yaml_config
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt
index 33e4b61983..6bab24b1d6 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt
@@ -19,6 +19,7 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   cpp-yaml-fyamlcpp
   cms-console-util
   ydb-core-protos
+  ydb-library-yaml_config
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp
diff --git a/ydb/core/cms/console/yaml_config/yaml_config.cpp b/ydb/core/cms/console/yaml_config/yaml_config.cpp
index 8518d15381..3ec4d72730 100644
--- a/ydb/core/cms/console/yaml_config/yaml_config.cpp
+++ b/ydb/core/cms/console/yaml_config/yaml_config.cpp
@@ -2,6 +2,7 @@
 #include "yaml_config_impl.h"
 
 #include <ydb/core/base/appdata.h>
+#include <ydb/library/yaml_config/yaml_config_parser.h>
 
 #include <library/cpp/protobuf/json/json2proto.h>
 
@@ -438,6 +439,8 @@ NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUn
 
     NJson::ReadJsonTree(resolvedJsonConfig, &json);
 
+    NKikimr::NYaml::TransformConfig(json, true);
+
     NKikimrConfig::TAppConfig yamlProtoConfig;
 
     NProtobufJson::TJson2ProtoConfig c;
@@ -688,6 +691,8 @@ void ResolveAndParseYamlConfig(
     NJson::TJsonValue json;
     Y_VERIFY(NJson::ReadJsonTree(resolvedJsonConfigStream.Str(), &json), "Got invalid config from Console");
 
+    NKikimr::NYaml::TransformConfig(json, true);
+
     NProtobufJson::MergeJson2Proto(json, appConfig, NYamlConfig::GetJsonToProtoConfig());
 }
 
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
index 0b7139288c..7667fad001 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
@@ -383,6 +383,23 @@ protected:
             ythrow yexception() << "wrong '--node-kind' value '" << NodeKind << "', only '" << NODE_KIND_YDB << "' or '" << NODE_KIND_YQ << "' is allowed";
         }
 
+        RunConfig.Labels["node_id"] = ToString(NodeId);
+        RunConfig.Labels["node_host"] = FQDNHostName();
+        RunConfig.Labels["tenant"] = RunConfig.TenantName;
+        RunConfig.Labels["node_type"] = NodeType;
+        // will be replaced with proper version info
+        RunConfig.Labels["branch"] = GetBranch();
+        RunConfig.Labels["rev"] = ToString(GetProgramSvnRevision());
+        RunConfig.Labels["dynamic"] = ToString(NodeBrokerAddresses.empty() ? "false" : "true");
+
+        for (const auto& [name, value] : RunConfig.Labels) {
+            auto *label = RunConfig.AppConfig.AddLabels();
+            label->SetName(name);
+            label->SetValue(value);
+        }
+
+        RunConfig.ClusterName = ClusterName;
+
         MaybeRegisterAndLoadConfigs();
 
         LoadYamlConfig();
@@ -689,21 +706,6 @@ protected:
             messageBusConfig->SetStartTracingBusProxy(!!TracePath);
             messageBusConfig->SetTracePath(TracePath);
         }
-
-        RunConfig.Labels["node_id"] = ToString(NodeId);
-        RunConfig.Labels["node_host"] = FQDNHostName();
-        RunConfig.Labels["tenant"] = RunConfig.TenantName;
-        // will be replaced with proper version info
-        RunConfig.Labels["branch"] = GetBranch();
-        RunConfig.Labels["rev"] = ToString(GetProgramSvnRevision());
-
-        for (const auto& [name, value] : RunConfig.Labels) {
-            auto *label = RunConfig.AppConfig.AddLabels();
-            label->SetName(name);
-            label->SetValue(value);
-        }
-
-        RunConfig.ClusterName = ClusterName;
     }
 
     inline bool LoadConfigFromCMS() {
diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp
index f7e3262d0a..afa87a200c 100644
--- a/ydb/library/yaml_config/yaml_config_parser.cpp
+++ b/ydb/library/yaml_config/yaml_config_parser.cpp
@@ -271,6 +271,11 @@ namespace NKikimr::NYaml {
         }
 
         auto& config = json["actor_system_config"];
+
+        if (config.Has("use_auto_config") && config["use_auto_config"].GetBooleanSafe()) {
+            return; // do nothing for auto config
+        }
+
         auto& executors = config["executor"];
 
         const std::vector<std::pair<TString, TString>> defaultExecutors = {{"io_executor", "IO"}, {"sys_executor", "SYSTEM"}, {"user_executor", "USER"}, {"batch_executor", "BATCH"}};
@@ -487,7 +492,11 @@ namespace NKikimr::NYaml {
         }
     }
 
-    void PrepareSystemTabletsInfo(NJson::TJsonValue& json)  {
+    void PrepareSystemTabletsInfo(NJson::TJsonValue& json, bool relaxed)  {
+        if (relaxed && !json.Has("system_tablets") && (!json.Has("nameservice_config") || !json["nameservice_config"].Has("node"))) {
+            return;
+        }
+
         if (!json.Has("system_tablets")) {
             auto& config = json["system_tablets"];
             config.SetType(NJson::EJsonValueType::JSON_MAP);
@@ -507,11 +516,15 @@ namespace NKikimr::NYaml {
 
     }
 
-    void PrepareBootstrapConfig(NJson::TJsonValue& json) {
+    void PrepareBootstrapConfig(NJson::TJsonValue& json, bool relaxed) {
         if (json.Has("bootstrap_config") && json["bootstrap_config"].Has("tablet")) {
             return;
         }
 
+        if (relaxed && !json.Has("system_tablets")) {
+            return;
+        }
+
         if (!json.Has("bootstrap_config")) {
             auto& bootstrapConfig = json["bootstrap_config"];
             bootstrapConfig.SetType(NJson::EJsonValueType::JSON_MAP);
@@ -527,7 +540,10 @@ namespace NKikimr::NYaml {
         }
     }
 
-    void PrepareDomainsConfig(NJson::TJsonValue& json) {
+    void PrepareDomainsConfig(NJson::TJsonValue& json, bool relaxed) {
+        if (relaxed && !json.Has("domains_config")) {
+            return;
+        }
 
         Y_ENSURE_BT(json.Has("domains_config"));
         Y_ENSURE_BT(json["domains_config"].IsMap());
@@ -591,7 +607,11 @@ namespace NKikimr::NYaml {
         }
     }
 
-    void PrepareSecurityConfig(NJson::TJsonValue& json) {
+    void PrepareSecurityConfig(NJson::TJsonValue& json, bool relaxed) {
+        if (relaxed && !json.Has("domains_config")) {
+            return;
+        }
+
         Y_ENSURE_BT(json.Has("domains_config"));
         Y_ENSURE_BT(json["domains_config"].IsMap());
 
@@ -756,7 +776,11 @@ namespace NKikimr::NYaml {
         json.EraseValue("storage_config_generation");
     }
 
-    void PrepareLogConfig(NJson::TJsonValue& json) {
+    void PrepareLogConfig(NJson::TJsonValue& json, bool relaxed) {
+        if (relaxed) {
+            return;
+        }
+
         if (!json.Has("log_config")) {
             json["log_config"].SetType(NJson::EJsonValueType::JSON_MAP);
         }
@@ -766,7 +790,11 @@ namespace NKikimr::NYaml {
         }
     }
 
-    void PrepareIcConfig(NJson::TJsonValue& json) {
+    void PrepareIcConfig(NJson::TJsonValue& json, bool relaxed) {
+        if (relaxed) {
+            return;
+        }
+
         if (!json.Has("interconnect_config")) {
             auto& config = json["interconnect_config"];
             config.SetType(NJson::EJsonValueType::JSON_MAP);
@@ -778,16 +806,16 @@ namespace NKikimr::NYaml {
         }
     }
 
-    void PrepareJson(NJson::TJsonValue& json){
+    void TransformConfig(NJson::TJsonValue& json, bool relaxed) {
         PrepareNameserviceConfig(json);
         PrepareActorSystemConfig(json);
         PrepareStaticGroup(json);
-        PrepareIcConfig(json);
-        PrepareLogConfig(json);
-        PrepareSystemTabletsInfo(json);
-        PrepareDomainsConfig(json);
-        PrepareSecurityConfig(json);
-        PrepareBootstrapConfig(json);
+        PrepareIcConfig(json, relaxed);
+        PrepareLogConfig(json, relaxed);
+        PrepareSystemTabletsInfo(json, relaxed);
+        PrepareDomainsConfig(json, relaxed);
+        PrepareSecurityConfig(json, relaxed);
+        PrepareBootstrapConfig(json, relaxed);
         ClearFields(json);
     }
 
@@ -831,7 +859,7 @@ namespace NKikimr::NYaml {
     void Parse(const TString& data, NKikimrConfig::TAppConfig& config) {
         auto yamlNode = YAML::Load(data);
         NJson::TJsonValue jsonNode = Yaml2Json(yamlNode, true);
-        PrepareJson(jsonNode);
+        TransformConfig(jsonNode);
         NProtobufJson::MergeJson2Proto(jsonNode, config, GetJsonToProtoConfig());
     }
 }
diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h
index 52dd59350d..ac6a293b1a 100644
--- a/ydb/library/yaml_config/yaml_config_parser.h
+++ b/ydb/library/yaml_config/yaml_config_parser.h
@@ -17,5 +17,7 @@ namespace NKikimr::NYaml {
 
     NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data);
 
+    void TransformConfig(NJson::TJsonValue& config, bool relaxed = false);
+
     void Parse(const TString& data, NKikimrConfig::TAppConfig& config);
 }