Add yaml config api versioning

add yaml config api versioning

14 files changed, 54 insertions, 22 deletions

diff --git a/ydb/core/cms/console/configs_cache.cpp b/ydb/core/cms/console/configs_cache.cpp
index 49fedcba920..abe0ac93850 100644
--- a/ydb/core/cms/console/configs_cache.cpp
+++ b/ydb/core/cms/console/configs_cache.cpp
@@ -96,7 +96,7 @@ void TConfigsCache::Bootstrap(const TActorContext &ctx) {
         kinds.push_back(kind);
     }
 
-    auto client = CreateConfigsSubscriber(SelfId(), kinds, CurrentConfig);
+    auto client = CreateConfigsSubscriber(SelfId(), kinds, CurrentConfig, 1);
     SubscriptionClient = ctx.RegisterWithSameMailbox(client);
 
     Become(&TThis::StateWork);
diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp
index ffb3affe67a..1f2840ac5ae 100644
--- a/ydb/core/cms/console/configs_dispatcher.cpp
+++ b/ydb/core/cms/console/configs_dispatcher.cpp
@@ -265,7 +265,8 @@ void TConfigsDispatcher::Bootstrap()
         TVector<ui32>(DYNAMIC_KINDS.begin(), DYNAMIC_KINDS.end()),
         CurrentConfig,
         0,
-        true);
+        true,
+        1);
     CommonSubscriptionClient = RegisterWithSameMailbox(commonClient);
 
     Become(&TThis::StateInit);
@@ -330,13 +331,17 @@ NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig()
 {
     NKikimrConfig::TAppConfig newYamlProtoConfig = {};
 
-    NYamlConfig::ResolveAndParseYamlConfig(
-        YamlConfig,
-        VolatileYamlConfigs,
-        Labels,
-        newYamlProtoConfig,
-        &ResolvedYamlConfig,
-        &ResolvedJsonConfig);
+    try {
+        NYamlConfig::ResolveAndParseYamlConfig(
+            YamlConfig,
+            VolatileYamlConfigs,
+            Labels,
+            newYamlProtoConfig,
+            &ResolvedYamlConfig,
+            &ResolvedJsonConfig);
+    } catch (const yexception& ex) {
+        BLOG_ERROR("Got invalid config from console error# " << ex.what());
+    }
 
     return newYamlProtoConfig;
 }
diff --git a/ydb/core/cms/console/console__apply_yaml_config.cpp b/ydb/core/cms/console/console__apply_yaml_config.cpp
index cd8a24ff1b6..71b1afff2a8 100644
--- a/ydb/core/cms/console/console__apply_yaml_config.cpp
+++ b/ydb/core/cms/console/console__apply_yaml_config.cpp
@@ -53,7 +53,10 @@ public:
 
                 db.Table<Schema::YamlConfig>().Key(Version + 1)
                     .Update<Schema::YamlConfig::Config>(UpdatedConfig)
-                    .Update<Schema::YamlConfig::Dropped>(false);
+                    // set config dropped by default to support rollback to previous versions
+                    // where new config layout is not supported
+                    // it will lead to ignoring config from new versions
+                    .Update<Schema::YamlConfig::Dropped>(true);
 
                 /* Later we shift this boundary to support rollback and history */
                 db.Table<Schema::YamlConfig>().Key(Version)
diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp
index 3782e063365..47dac4b76bb 100644
--- a/ydb/core/cms/console/console_configs_manager.cpp
+++ b/ydb/core/cms/console/console_configs_manager.cpp
@@ -362,7 +362,9 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc,
     if (!yamlConfigRowset.EndOfSet()) {
         YamlVersion = yamlConfigRowset.template GetValue<Schema::YamlConfig::Version>();
         YamlConfig = yamlConfigRowset.template GetValue<Schema::YamlConfig::Config>();
-        YamlDropped = yamlConfigRowset.template GetValue<Schema::YamlConfig::Dropped>();
+        // ignore this as deprecated
+        // now used only for disabling new config layout for older console
+        YamlDropped = false;
     }
 
     while (!configItemRowset.EndOfSet()) {
diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp
index 5554a7f08d5..d834093e768 100644
--- a/ydb/core/cms/console/console_configs_provider.cpp
+++ b/ydb/core/cms/console/console_configs_provider.cpp
@@ -747,6 +747,7 @@ void TConfigsProvider::DumpStateHTML(IOutputStream &os) const {
                            << "  ItemKinds: " << KindsToString(s->ItemKinds) << Endl
                            << "  ConfigVersions: " << s->LastProvided.ShortDebugString() << Endl
                            << "  ServeYaml: " << s->ServeYaml << Endl
+                           << "  YamlApiVersion: " << s->YamlApiVersion << Endl
                            << "  YamlVersion: " << s->YamlConfigVersion << ".[";
                         bool first = true;
                         for (auto &[id, hash] : s->VolatileYamlConfigHashes) {
@@ -804,8 +805,9 @@ void TConfigsProvider::Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev
     subscription->ItemKinds.insert(rec.GetConfigItemKinds().begin(), rec.GetConfigItemKinds().end());
     subscription->LastProvided.CopyFrom(rec.GetKnownVersion());
 
-    if (rec.HasServeYaml()) {
+    if (rec.HasServeYaml() && rec.GetServeYaml() && rec.HasYamlApiVersion() && rec.GetYamlApiVersion() == 1) {
         subscription->ServeYaml = rec.GetServeYaml();
+        subscription->YamlApiVersion = rec.GetYamlApiVersion();
         subscription->YamlConfigVersion = rec.GetYamlVersion();
         for (auto &volatileConfigVersion : rec.GetVolatileYamlVersion()) {
             subscription->VolatileYamlConfigHashes[volatileConfigVersion.GetId()] = volatileConfigVersion.GetHash();
@@ -1088,7 +1090,7 @@ void TConfigsProvider::Handle(TEvConsole::TEvGetNodeConfigRequest::TPtr &ev, con
     LOG_TRACE_S(ctx, NKikimrServices::CMS_CONFIGS,
                 "Send TEvGetNodeConfigResponse: " << response->Record.ShortDebugString());
 
-    if (rec.HasServeYaml() && rec.GetServeYaml()) {
+    if (rec.HasServeYaml() && rec.GetServeYaml() && rec.HasYamlApiVersion() && rec.GetYamlApiVersion() == 1) {
         response->Record.SetYamlConfig(YamlConfig);
 
         for (auto &[id, config] : VolatileYamlConfigs) {
diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp
index 635eca31b12..ec031997170 100644
--- a/ydb/core/cms/console/console_configs_subscriber.cpp
+++ b/ydb/core/cms/console/console_configs_subscriber.cpp
@@ -45,6 +45,7 @@ public:
             const TVector<ui32> &kinds,
             const NKikimrConfig::TAppConfig &currentConfig,
             bool processYaml,
+            ui64 version,
             const TString &yamlConfig,
             const TMap<ui64, TString> &volatileYamlConfigs)
         : OwnerId(ownerId)
@@ -55,6 +56,7 @@ public:
         , LastOrder(0)
         , CurrentConfig(currentConfig)
         , ServeYaml(processYaml)
+        , Version(version)
         , YamlConfig(yamlConfig)
         , VolatileYamlConfigs(volatileYamlConfigs)
     {
@@ -337,6 +339,7 @@ private:
         request->Record.MutableOptions()->SetNodeType(NodeType);
         request->Record.MutableOptions()->SetHost(FQDNHostName());
         request->Record.SetServeYaml(ServeYaml);
+        request->Record.SetYamlApiVersion(Version);
 
         for (auto &kind : Kinds)
             request->Record.AddConfigItemKinds(kind);
@@ -386,6 +389,7 @@ private:
     NKikimrConfig::TAppConfig CurrentConfig;
 
     bool ServeYaml = false;
+    ui64 Version;
     TString YamlConfig;
     TMap<ui64, TString> VolatileYamlConfigs;
     ui64 YamlConfigVersion = 0;
@@ -406,10 +410,11 @@ IActor *CreateConfigsSubscriber(
     const NKikimrConfig::TAppConfig &currentConfig,
     ui64 cookie,
     bool processYaml,
+    ui64 version,
     const TString &yamlConfig,
     const TMap<ui64, TString> &volatileYamlConfigs)
 {
-    return new TConfigsSubscriber(ownerId, cookie, kinds, currentConfig, processYaml, yamlConfig, volatileYamlConfigs);
+    return new TConfigsSubscriber(ownerId, cookie, kinds, currentConfig, processYaml, version, yamlConfig, volatileYamlConfigs);
 }
 
 } // namespace NKikimr::NConsole
diff --git a/ydb/core/cms/console/console_configs_subscriber.h b/ydb/core/cms/console/console_configs_subscriber.h
index d10abea137a..52d0feb81cd 100644
--- a/ydb/core/cms/console/console_configs_subscriber.h
+++ b/ydb/core/cms/console/console_configs_subscriber.h
@@ -14,6 +14,7 @@ IActor *CreateConfigsSubscriber(
     const NKikimrConfig::TAppConfig &currentConfig,
     ui64 cookie = 0,
     bool processYaml = false,
+    ui64 version = 0,
     const TString &yamlConfig = {},
     const TMap<ui64, TString> &volatileYamlConfigs = {});
 
diff --git a/ydb/core/cms/console/console_ut_configs.cpp b/ydb/core/cms/console/console_ut_configs.cpp
index 984f68bf7f3..534d8ce3f06 100644
--- a/ydb/core/cms/console/console_ut_configs.cpp
+++ b/ydb/core/cms/console/console_ut_configs.cpp
@@ -3917,7 +3917,8 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) {
             TVector<ui32>({(ui32)NKikimrConsole::TConfigItem::LogConfigItem}),
             NKikimrConfig::TAppConfig(),
             0,
-            true);
+            true,
+            1);
         runtime.Register(subscriber, 1);
 
         TDispatchOptions options1;
@@ -4018,6 +4019,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) {
         event->Record.MutableOptions()->SetTenant("tenant-1");
         event->Record.MutableOptions()->SetNodeType("type1");
         event->Record.SetServeYaml(true);
+        event->Record.SetYamlApiVersion(1);
         event->Record.AddConfigItemKinds(NKikimrConsole::TConfigItem::LogConfigItem);
         runtime.SendToPipe(MakeConsoleID(0), edgeId, event, 0, GetPipeConfigWithRetries());
 
@@ -4092,6 +4094,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) {
         event->Record.MutableOptions()->SetTenant("tenant-1");
         event->Record.MutableOptions()->SetNodeType("type1");
         event->Record.SetServeYaml(true);
+        event->Record.SetYamlApiVersion(1);
         event->Record.AddConfigItemKinds(NKikimrConsole::TConfigItem::LogConfigItem);
         runtime.SendToPipe(MakeConsoleID(0), edgeId, event, 0, GetPipeConfigWithRetries());
 
@@ -4140,6 +4143,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) {
             event->Record.MutableOptions()->SetTenant("tenant-1");
             event->Record.MutableOptions()->SetNodeType("type1");
             event->Record.SetServeYaml(true);
+            event->Record.SetYamlApiVersion(1);
             event->Record.SetYamlVersion(1);
             event->Record.MutableKnownVersion()->CopyFrom(config1.GetVersion());
             return event;
diff --git a/ydb/core/cms/console/util/config_index.h b/ydb/core/cms/console/util/config_index.h
index 6af912e0ef7..d2c6edc0b12 100644
--- a/ydb/core/cms/console/util/config_index.h
+++ b/ydb/core/cms/console/util/config_index.h
@@ -665,6 +665,7 @@ struct TInMemorySubscription : public TThrRefBase {
     NKikimrConfig::TConfigVersion LastProvided;
 
     bool ServeYaml = false;
+    ui64 YamlApiVersion = 0;
     ui64 YamlConfigVersion = 0;
     TMap<ui64, ui64> VolatileYamlConfigHashes;
 
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
index 1153b5c6c1b..6226468d514 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
@@ -771,7 +771,8 @@ protected:
                                                              NodeType,
                                                              DeduceNodeDomain(),
                                                              AppConfig.GetAuthConfig().GetStaffApiUserToken(),
-                                                             true);
+                                                             true,
+                                                             1);
 
                 if (result.IsSuccess()) {
                     auto appConfig = result.GetConfig();
@@ -1307,7 +1308,8 @@ protected:
                                                      NodeType,
                                                      DeduceNodeDomain(),
                                                      AppConfig.GetAuthConfig().GetStaffApiUserToken(),
-                                                     true);
+                                                     true,
+                                                     1);
 
         if (!result.IsSuccess()) {
             error = result.GetErrorMessage();
diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto
index f711a4534eb..5293ff2afd6 100644
--- a/ydb/core/protos/console_config.proto
+++ b/ydb/core/protos/console_config.proto
@@ -362,6 +362,7 @@ message TGetNodeConfigRequest {
     // Filter items by specified kinds (return full config by default).
     repeated uint32 ItemKinds = 2;
     optional bool ServeYaml = 3;
+    optional uint32 YamlApiVersion = 4;
 }
 
 message TGetNodeConfigResponse {
@@ -398,6 +399,7 @@ message TConfigSubscriptionRequest {
     optional bool ServeYaml = 5;
     optional uint64 YamlVersion = 6;
     repeated TVolatileYamlConfigVersion VolatileYamlVersion = 7;
+    optional uint32 YamlApiVersion = 8;
 }
 
 message TConfigSubscriptionResponse {
diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp
index f256c230b80..f3a28b4a3fd 100644
--- a/ydb/public/lib/deprecated/kicli/configurator.cpp
+++ b/ydb/public/lib/deprecated/kicli/configurator.cpp
@@ -63,9 +63,10 @@ TConfigurationResult TNodeConfigurator::SyncGetNodeConfig(ui32 nodeId,
                                                           const TString &nodeType,
                                                           const TString& domain,
                                                           const TString& token,
-                                                          bool serveYaml) const
+                                                          bool serveYaml,
+                                                          ui64 version) const
 {
-    auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token, serveYaml);
+    auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token, serveYaml, version);
     auto result = future.GetValue(TDuration::Max());
     return TConfigurationResult(result);
 }
diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h
index 410fe458608..b61cc72f762 100644
--- a/ydb/public/lib/deprecated/kicli/kicli.h
+++ b/ydb/public/lib/deprecated/kicli/kicli.h
@@ -701,7 +701,8 @@ public:
                                            const TString &nodeType,
                                            const TString& domain = "",
                                            const TString& token = "",
-                                           bool serveYaml = false) const;
+                                           bool serveYaml = false,
+                                           ui64 version = 0) const;
 
 private:
     TNodeConfigurator(TKikimr& kikimr);
@@ -807,7 +808,8 @@ protected:
                                                const TString &nodeType,
                                                const TString& domain,
                                                const TString& token = TString(),
-                                               bool serveYaml = false);
+                                               bool serveYaml = false,
+                                               ui64 version = 0);
 
     template <typename T> static void DumpRequest(const T& pb) {
         if (DUMP_REQUESTS) {
diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp
index 052ae193d44..2c2761538f6 100644
--- a/ydb/public/lib/deprecated/kicli/kikimr.cpp
+++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp
@@ -599,11 +599,13 @@ NThreading::TFuture<TResult> TKikimr::GetNodeConfig(ui32 nodeId,
                                                     const TString &nodeType,
                                                     const TString& domain,
                                                     const TString& token,
-                                                    bool serveYaml)
+                                                    bool serveYaml,
+                                                    ui64 version)
 {
     TAutoPtr<NMsgBusProxy::TBusConsoleRequest> request = new NMsgBusProxy::TBusConsoleRequest;
     auto &rec = request->Record;
     rec.MutableGetNodeConfigRequest()->SetServeYaml(serveYaml);
+    rec.MutableGetNodeConfigRequest()->SetYamlApiVersion(version);
     auto &node = *rec.MutableGetNodeConfigRequest()->MutableNode();
     node.SetNodeId(nodeId);
     node.SetHost(host);