Add unknown configs support under flag

add unknown config support under flag

10 files changed, 107 insertions, 13 deletions

diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp
index 8cf3a4ab13..a1c2126dcd 100644
--- a/ydb/core/cms/console/console__replace_yaml_config.cpp
+++ b/ydb/core/cms/console/console__replace_yaml_config.cpp
@@ -47,8 +47,12 @@ public:
                     ythrow yexception() << "Version mismatch";
                 }
 
+                if (req.GetRequest().allow_unknown_fields()) {
+                    UnknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector;
+                }
+
                 for (auto& [_, config] : resolved.Configs) {
-                    auto cfg = NYamlConfig::YamlToProto(config.second);
+                    auto cfg = NYamlConfig::YamlToProto(config.second, req.GetRequest().allow_unknown_fields(), true, UnknownFieldsCollector);
                 }
 
                 if (!req.GetRequest().dry_run()) {
@@ -65,7 +69,17 @@ public:
                 }
             }
 
-            Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, new TEvConsole::TEvReplaceYamlConfigResponse());
+            auto ev = MakeHolder<TEvConsole::TEvReplaceYamlConfigResponse>();
+
+            if (UnknownFieldsCollector) {
+                for (auto& [path, info] : UnknownFieldsCollector->GetUnknownKeys()) {
+                    auto *issue = ev->Record.AddIssues();
+                        issue->set_severity(NYql::TSeverityIds::S_WARNING);
+                        issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path);
+                }
+            }
+
+            Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, ev.Release());
         } catch (const yexception& ex) {
             Error = true;
 
@@ -107,6 +121,7 @@ private:
     THolder<NActors::IEventHandle> Response;
     bool Error = false;
     bool Modify = false;
+    TSimpleSharedPtr<NYamlConfig::TBasicUnknownFieldsCollector> UnknownFieldsCollector = nullptr;
     ui32 Version;
     TString Cluster;
     TString UpdatedConfig;
diff --git a/ydb/core/cms/console/console__set_yaml_config.cpp b/ydb/core/cms/console/console__set_yaml_config.cpp
index f219c8af99..9a56466458 100644
--- a/ydb/core/cms/console/console__set_yaml_config.cpp
+++ b/ydb/core/cms/console/console__set_yaml_config.cpp
@@ -36,8 +36,12 @@ public:
                 auto tree = NFyaml::TDocument::Parse(UpdatedConfig);
                 auto resolved = NYamlConfig::ResolveAll(tree);
 
+                if (req.GetRequest().allow_unknown_fields()) {
+                    UnknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector;
+                }
+
                 for (auto& [_, config] : resolved.Configs) {
-                    auto cfg = NYamlConfig::YamlToProto(config.second);
+                    auto cfg = NYamlConfig::YamlToProto(config.second, req.GetRequest().allow_unknown_fields(), true, UnknownFieldsCollector);
                 }
 
                 if (!req.GetRequest().dry_run()) {
@@ -54,7 +58,17 @@ public:
                 }
             }
 
-            Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, new TEvConsole::TEvSetYamlConfigResponse());
+            auto ev = MakeHolder<TEvConsole::TEvSetYamlConfigResponse>();
+
+            if (UnknownFieldsCollector) {
+                for (auto& [path, info] : UnknownFieldsCollector->GetUnknownKeys()) {
+                    auto *issue = ev->Record.AddIssues();
+                        issue->set_severity(NYql::TSeverityIds::S_WARNING);
+                        issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path);
+                }
+            }
+
+            Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, ev.Release());
         } catch (const yexception& ex) {
             Error = true;
 
@@ -96,6 +110,7 @@ private:
     THolder<NActors::IEventHandle> Response;
     bool Error = false;
     bool Modify = false;
+    TSimpleSharedPtr<NYamlConfig::TBasicUnknownFieldsCollector> UnknownFieldsCollector = nullptr;
     TString UpdatedConfig;
 };
 
diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp
index 26c1058d80..3ceed6a7b3 100644
--- a/ydb/core/cms/console/console_configs_manager.cpp
+++ b/ydb/core/cms/console/console_configs_manager.cpp
@@ -821,7 +821,7 @@ void TConfigsManager::Handle(TEvConsole::TEvAddVolatileConfigRequest::TPtr &ev,
             auto resolved = NYamlConfig::ResolveAll(tree);
 
             for (auto &[_, config] : resolved.Configs) {
-                auto cfg = NYamlConfig::YamlToProto(config.second);
+                auto cfg = NYamlConfig::YamlToProto(config.second, true);
             }
 
             if (ClusterName != clusterName) {
diff --git a/ydb/core/grpc_services/rpc_dynamic_config.cpp b/ydb/core/grpc_services/rpc_dynamic_config.cpp
index 38d0e137c7..94193fb7ee 100644
--- a/ydb/core/grpc_services/rpc_dynamic_config.cpp
+++ b/ydb/core/grpc_services/rpc_dynamic_config.cpp
@@ -147,6 +147,14 @@ private:
         return TBase::Reply(ev->Get()->Record.GetYdbStatus(), ev->Get()->Record.GetIssues(), TActivationContext::AsActorContext());
     }
 
+    void Handle(TEvConsole::TEvSetYamlConfigResponse::TPtr& ev) {
+        return TBase::Reply(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetIssues(), TActivationContext::AsActorContext());
+    }
+
+    void Handle(TEvConsole::TEvReplaceYamlConfigResponse::TPtr& ev) {
+        return TBase::Reply(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetIssues(), TActivationContext::AsActorContext());
+    }
+
     template<typename T>
     void Handle(T& ev)
     {
diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto
index 76c52ce8dc..5c2ae8a49e 100644
--- a/ydb/core/protos/console_config.proto
+++ b/ydb/core/protos/console_config.proto
@@ -247,6 +247,7 @@ message TSetYamlConfigRequest {
 }
 
 message TSetYamlConfigResponse {
+    repeated Ydb.Issue.IssueMessage Issues = 1;
 }
 
 message TReplaceYamlConfigRequest {
@@ -255,6 +256,7 @@ message TReplaceYamlConfigRequest {
 }
 
 message TReplaceYamlConfigResponse {
+    repeated Ydb.Issue.IssueMessage Issues = 1;
 }
 
 message TDropConfigRequest {
diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp
index a21aeed17e..4ca4607098 100644
--- a/ydb/library/yaml_config/yaml_config.cpp
+++ b/ydb/library/yaml_config/yaml_config.cpp
@@ -8,7 +8,12 @@
 
 namespace NYamlConfig {
 
-NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown, bool preTransform) {
+NKikimrConfig::TAppConfig YamlToProto(
+    const NFyaml::TNodeRef& node,
+    bool allowUnknown,
+    bool preTransform,
+    TSimpleSharedPtr<NProtobufJson::IUnknownFieldsCollector> unknownFieldsCollector) {
+
     TStringStream sstr;
 
     sstr << NFyaml::TJsonEmitter(node);
@@ -31,6 +36,7 @@ NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUn
     c.CastRobust = true;
     c.MapAsObject = true;
     c.AllowUnknownFields = allowUnknown;
+    c.UnknownFieldsCollector = std::move(unknownFieldsCollector);
 
     NProtobufJson::MergeJson2Proto(json, yamlProtoConfig, c);
 
diff --git a/ydb/library/yaml_config/yaml_config.h b/ydb/library/yaml_config/yaml_config.h
index 6a5ddd4ba3..c11ed96873 100644
--- a/ydb/library/yaml_config/yaml_config.h
+++ b/ydb/library/yaml_config/yaml_config.h
@@ -2,6 +2,7 @@
 
 #include <library/cpp/yaml/fyamlcpp/fyamlcpp.h>
 #include <library/cpp/actors/core/actor.h>
+#include <library/cpp/protobuf/json/json2proto.h>
 
 #include <ydb/core/protos/config.pb.h>
 #include <ydb/core/protos/console_config.pb.h>
@@ -21,10 +22,51 @@
 
 namespace NYamlConfig {
 
+struct TBasicUnknownFieldsCollector : public NProtobufJson::IUnknownFieldsCollector {
+    void OnEnterMapItem(const TString& key) override {
+        CurrentPath.push_back(key);
+    }
+
+    void OnEnterArrayItem(ui64 id) override {
+        CurrentPath.push_back(ToString(id));
+    }
+
+    void OnLeaveMapItem() override {
+        CurrentPath.pop_back();
+    }
+
+    void OnLeaveArrayItem() override {
+        CurrentPath.pop_back();
+    }
+
+    void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) override {
+        TString path;
+        for (auto& piece : CurrentPath) {
+            path.append("/");
+            path.append(piece);
+        }
+        path.append("/");
+        path.append(key);
+        UnknownKeys[std::move(path)] = {key, value.full_name()};
+    }
+
+    const TMap<TString, std::pair<TString, TString>>& GetUnknownKeys() const {
+        return UnknownKeys;
+    }
+
+private:
+    TVector<TString> CurrentPath;
+    TMap<TString, std::pair<TString, TString>> UnknownKeys;
+};
+
 /**
  * Converts YAML representation to ProtoBuf
  */
-NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown = false, bool preTransform = true);
+NKikimrConfig::TAppConfig YamlToProto(
+    const NFyaml::TNodeRef& node,
+    bool allowUnknown = false,
+    bool preTransform = true,
+    TSimpleSharedPtr<NProtobufJson::IUnknownFieldsCollector> unknownFieldsCollector = nullptr);
 
 /**
  * Resolves config for given labels and stores result to appConfig
diff --git a/ydb/public/api/protos/draft/ydb_dynamic_config.proto b/ydb/public/api/protos/draft/ydb_dynamic_config.proto
index 23f00c68de..9ba65d706d 100644
--- a/ydb/public/api/protos/draft/ydb_dynamic_config.proto
+++ b/ydb/public/api/protos/draft/ydb_dynamic_config.proto
@@ -18,6 +18,7 @@ message SetConfigRequest {
     // Config in YAML format. metadata will be ignored
     string config = 2;
     bool dry_run = 3;
+    bool allow_unknown_fields = 4;
 }
 
 message SetConfigResponse {
@@ -29,6 +30,7 @@ message ReplaceConfigRequest {
     // Config in YAML format
     string config = 2;
     bool dry_run = 3;
+    bool allow_unknown_fields = 4;
 }
 
 message ReplaceConfigResponse {
diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp
index 76d5cbc9b5..09c07eebbf 100644
--- a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp
+++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp
@@ -12,10 +12,11 @@ public:
     {
     }
 
-    TAsyncStatus SetConfig(const TString& config, bool dryRun, const TClusterConfigSettings& settings = {}) {
+    TAsyncStatus SetConfig(const TString& config, bool dryRun, bool allowUnknownFields, const TClusterConfigSettings& settings = {}) {
         auto request = MakeOperationRequest<Ydb::DynamicConfig::SetConfigRequest>(settings);
         request.set_config(config);
         request.set_dry_run(dryRun);
+        request.set_allow_unknown_fields(allowUnknownFields);
 
         return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::SetConfigRequest, Ydb::DynamicConfig::SetConfigResponse>(
             std::move(request),
@@ -23,10 +24,11 @@ public:
             TRpcRequestSettings::Make(settings));
     }
 
-    TAsyncStatus ReplaceConfig(const TString& config, bool dryRun, const TClusterConfigSettings& settings = {}) {
+    TAsyncStatus ReplaceConfig(const TString& config, bool dryRun, bool allowUnknownFields, const TClusterConfigSettings& settings = {}) {
         auto request = MakeOperationRequest<Ydb::DynamicConfig::ReplaceConfigRequest>(settings);
         request.set_config(config);
         request.set_dry_run(dryRun);
+        request.set_allow_unknown_fields(allowUnknownFields);
 
         return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::ReplaceConfigRequest, Ydb::DynamicConfig::ReplaceConfigResponse>(
             std::move(request),
@@ -342,15 +344,17 @@ TDynamicConfigClient::TDynamicConfigClient(const TDriver& driver)
 TAsyncStatus TDynamicConfigClient::SetConfig(
     const TString& config,
     bool dryRun,
+    bool allowUnknownFields,
     const TClusterConfigSettings& settings) {
-    return Impl_->SetConfig(config, dryRun, settings);
+    return Impl_->SetConfig(config, dryRun, allowUnknownFields, settings);
 }
 
 TAsyncStatus TDynamicConfigClient::ReplaceConfig(
     const TString& config,
     bool dryRun,
+    bool allowUnknownFields,
     const TClusterConfigSettings& settings) {
-    return Impl_->ReplaceConfig(config, dryRun, settings);
+    return Impl_->ReplaceConfig(config, dryRun, allowUnknownFields, settings);
 }
 
 TAsyncStatus TDynamicConfigClient::DropConfig(
diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h
index 1925ae2f5d..1c02d54274 100644
--- a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h
+++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h
@@ -180,10 +180,10 @@ public:
     explicit TDynamicConfigClient(const TDriver& driver);
 
     // Set config
-    TAsyncStatus SetConfig(const TString& config, bool dryRun = false, const TClusterConfigSettings& settings = {});
+    TAsyncStatus SetConfig(const TString& config, bool dryRun = false, bool allowUnknownFields = false, const TClusterConfigSettings& settings = {});
 
     // Replace config
-    TAsyncStatus ReplaceConfig(const TString& config, bool dryRun = false, const TClusterConfigSettings& settings = {});
+    TAsyncStatus ReplaceConfig(const TString& config, bool dryRun = false, bool allowUnknownFields = false, const TClusterConfigSettings& settings = {});
 
     // Drop config
     TAsyncStatus DropConfig(