Add yaml config utils

add yaml_config utils

9 files changed, 237 insertions, 13 deletions

diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp
index fc804128e4..9ab1465dfe 100644
--- a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp
+++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp
@@ -2,11 +2,23 @@
 
 #include <contrib/libs/libfyaml/include/libfyaml.h>
 
+#include <util/digest/murmur.h>
+
 namespace NFyaml {
 
 #define ENSURE_NODE_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Node")
 #define ENSURE_DOCUMENT_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Document")
 
+const char* zstr = "";
+
+struct TStringHashT {
+    size_t operator()(const TString& str) const {
+        auto* ptr = str.empty() ? zstr : &str[0];
+        return MurmurHash<size_t>(reinterpret_cast<char*>(&ptr), sizeof(ptr));
+    }
+};
+
+
 enum class EErrorType {
     Debug = FYET_DEBUG,
     Info = FYET_INFO,
@@ -280,8 +292,12 @@ TNode TNodeRef::Copy() const {
     return TNode(fy_node_copy(fy_node_document(Node_), Node_));
 }
 
-TNode TNodeRef::Copy(const TDocument& to) const {
+TNode TNodeRef::Copy(TDocument& to) const {
     ENSURE_NODE_NOT_EMPTY(Node_);
+    auto* fromDoc = fy_node_document(Node_);
+    auto& fromUserdata = *reinterpret_cast<THashSet<TString, TStringHashT>*>(fy_document_get_userdata(fromDoc));
+    auto& toUserdata = *reinterpret_cast<THashSet<TString, TStringHashT>*>(fy_document_get_userdata(to.Document_.get()));
+    toUserdata.insert(fromUserdata.begin(), fromUserdata.end());
     return TNode(fy_node_copy(to.Document_.get(), Node_));
 }
 
@@ -694,6 +710,16 @@ TDocumentNodeIterator& TDocumentNodeIterator::operator++() {
     return *this;
 }
 
+TDocument::TDocument(TString str, fy_document* doc, fy_diag* diag)
+    : Document_(doc, fy_document_destroy)
+    , Diag_(diag, fy_diag_destroy)
+{
+    auto* userdata = new THashSet<TString, TStringHashT>({str});
+    fy_document_set_userdata(doc, userdata);
+    fy_document_register_on_destroy(doc, &DestroyDocumentStrings);
+    RegisterUserDataCleanup();
+}
+
 TDocument::TDocument(fy_document* doc, fy_diag* diag)
     : Document_(doc, fy_document_destroy)
     , Diag_(diag, fy_diag_destroy)
@@ -701,7 +727,9 @@ TDocument::TDocument(fy_document* doc, fy_diag* diag)
     RegisterUserDataCleanup();
 }
 
-TDocument TDocument::Parse(const char* cstr) {
+
+TDocument TDocument::Parse(TString str) {
+    auto* cstr = str.empty() ? zstr : &str[0];
     fy_diag_cfg dcfg;
     fy_diag_cfg_default(&dcfg);
     std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_diag_create(&dcfg), fy_diag_destroy);
@@ -721,12 +749,19 @@ TDocument TDocument::Parse(const char* cstr) {
             ythrow yexception() << err->file << ":" << err->line << ":" << err->column << " " << err->msg;
         }
     }
-    return TDocument(doc, diag.release());
+    return TDocument(std::move(str), doc, diag.release());
 }
 
 TDocument TDocument::Clone() const {
     ENSURE_DOCUMENT_NOT_EMPTY(Document_);
     fy_document* doc = fy_document_clone(Document_.get());
+    fy_document_set_userdata(
+        doc,
+        new THashSet<TString, TStringHashT>(
+            *reinterpret_cast<THashSet<TString, TStringHashT>*>(fy_document_get_userdata(Document_.get()))
+        )
+    );
+    fy_document_register_on_destroy(doc, &DestroyDocumentStrings);
     return TDocument(doc, fy_document_get_diag(doc));
 }
 
@@ -836,13 +871,15 @@ std::unique_ptr<char, void(*)(char*)> TJsonEmitter::EmitToCharArray() const {
     return res;
 }
 
-TParser::TParser(fy_parser* parser, fy_diag* diag)
-    : Parser_(parser, fy_parser_destroy)
+TParser::TParser(TString rawStream, fy_parser* parser, fy_diag* diag)
+    : RawDocumentStream_(std::move(rawStream))
+    , Parser_(parser, fy_parser_destroy)
     , Diag_(diag, fy_diag_destroy)
 {}
 
-TParser TParser::Create(const char* stream)
+TParser TParser::Create(TString str)
 {
+    auto* stream = str.empty() ? zstr : &str[0];
     fy_diag_cfg dcfg;
     fy_diag_cfg_default(&dcfg);
     std::unique_ptr<fy_diag, void(*)(fy_diag*)> diag(fy_diag_create(&dcfg), fy_diag_destroy);
@@ -865,7 +902,7 @@ TParser TParser::Create(const char* stream)
 
     fy_parser_set_string(parser, stream, -1);
 
-    return TParser(parser, diag.release());
+    return TParser(std::move(str), parser, diag.release());
 }
 
 std::optional<TDocument> TParser::NextDocument() {
@@ -874,7 +911,7 @@ std::optional<TDocument> TParser::NextDocument() {
         return std::nullopt;
     }
 
-    return TDocument(doc, fy_document_get_diag(doc));
+    return TDocument(RawDocumentStream_, doc, fy_document_get_diag(doc));
 }
 
 namespace NDetail {
diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.h b/library/cpp/yaml/fyamlcpp/fyamlcpp.h
index 5b645a557c..c1ec03d25f 100644
--- a/library/cpp/yaml/fyamlcpp/fyamlcpp.h
+++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.h
@@ -4,6 +4,7 @@
 #include <util/system/compiler.h>
 #include <util/system/yassert.h>
 #include <util/stream/str.h>
+#include <util/generic/hash_set.h>
 
 #include <memory>
 #include <optional>
@@ -116,7 +117,7 @@ public:
 
     TNode Copy() const;
 
-    TNode Copy(const TDocument& to) const;
+    TNode Copy(TDocument& to) const;
 
     bool IsAlias() const;
 
@@ -509,7 +510,9 @@ class TDocument {
     friend class TNodeRef;
     friend class TJsonEmitter;
     friend class TParser;
+    friend class TMapping;
 
+    TDocument(TString str, fy_document* doc = nullptr, fy_diag* diag = nullptr);
     TDocument(fy_document* doc = nullptr, fy_diag* diag = nullptr);
 
 public:
@@ -518,7 +521,7 @@ public:
         , Diag_(std::move(other.Diag_))
     {}
 
-    static TDocument Parse(const char* cstr);
+    static TDocument Parse(TString cstr);
 
     TDocument Clone() const;
 
@@ -583,6 +586,14 @@ private:
         }
     }
 
+    static void DestroyDocumentStrings(fy_document *fyd, void *user) {
+        Y_UNUSED(fyd);
+        if (user) {
+            auto* data = reinterpret_cast<THashSet<TString>*>(user);
+            delete data;
+        }
+    }
+
     bool RegisterUserDataCleanup();
     void UnregisterUserDataCleanup();
 };
@@ -599,12 +610,13 @@ private:
 };
 
 class TParser {
-    TParser(fy_parser* doc, fy_diag* diag);
+    TParser(TString rawStream, fy_parser* doc, fy_diag* diag);
 public:
-    static TParser Create(const char* cstr);
+    static TParser Create(TString str);
 
     std::optional<TDocument> NextDocument();
 private:
+    TString RawDocumentStream_;
     std::unique_ptr<fy_parser, void(*)(fy_parser*)> Parser_;
     std::unique_ptr<fy_diag, void(*)(fy_diag*)> Diag_;
 };
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt b/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt
index 7f2bea933d..4d09e79f3a 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt
@@ -15,6 +15,9 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   yutil
   cpp-yaml-fyamlcpp
   OpenSSL::OpenSSL
+  ydb-core-protos
+  cpp-actors-core
+  cpp-protobuf-json
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
index 4064d3d798..b78dac11e7 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt
@@ -16,6 +16,9 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   yutil
   cpp-yaml-fyamlcpp
   OpenSSL::OpenSSL
+  ydb-core-protos
+  cpp-actors-core
+  cpp-protobuf-json
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp
diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt
index 4064d3d798..b78dac11e7 100644
--- a/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt
+++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt
@@ -16,6 +16,9 @@ target_link_libraries(cms-console-yaml_config PUBLIC
   yutil
   cpp-yaml-fyamlcpp
   OpenSSL::OpenSSL
+  ydb-core-protos
+  cpp-actors-core
+  cpp-protobuf-json
 )
 target_sources(cms-console-yaml_config PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp
diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt
index 8c32eb5d0b..44a8808b89 100644
--- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt
+++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt
@@ -24,6 +24,8 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE
   -Wl,-sdk_version,10.15
   -fPIC
   -fPIC
+  -framework
+  CoreFoundation
 )
 target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp
diff --git a/ydb/core/cms/console/yaml_config/yaml_config.cpp b/ydb/core/cms/console/yaml_config/yaml_config.cpp
index 3c43828476..ae4a461526 100644
--- a/ydb/core/cms/console/yaml_config/yaml_config.cpp
+++ b/ydb/core/cms/console/yaml_config/yaml_config.cpp
@@ -1,6 +1,9 @@
 #include "yaml_config.h"
 #include "yaml_config_impl.h"
 
+#include <library/cpp/protobuf/json/json2proto.h>
+#include <ydb/core/base/appdata.h>
+
 namespace NYamlConfig {
 
 inline const TMap<TString, EYamlConfigLabelTypeClass> ClassMapping{
@@ -378,6 +381,31 @@ bool Fit(
     return true;
 }
 
+NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown) {
+    TStringStream sstr;
+
+    sstr << NFyaml::TJsonEmitter(node);
+
+    TString resolvedJsonConfig = sstr.Str();
+
+    NJson::TJsonValue json;
+
+    NJson::ReadJsonTree(resolvedJsonConfig, &json);
+
+    NKikimrConfig::TAppConfig yamlProtoConfig;
+
+    NProtobufJson::TJson2ProtoConfig c;
+    c.SetFieldNameMode(NProtobufJson::TJson2ProtoConfig::FieldNameSnakeCaseDense);
+    c.SetEnumValueMode(NProtobufJson::TJson2ProtoConfig::EnumCaseInsensetive);
+    c.CastRobust = true;
+    c.MapAsObject = true;
+    c.AllowUnknownFields = allowUnknown;
+
+    NProtobufJson::MergeJson2Proto(json, yamlProtoConfig, c);
+
+    return yamlProtoConfig;
+}
+
 TResolvedConfig ResolveAll(NFyaml::TDocument& doc)
 {
     TVector<TString> labelNames;
@@ -492,6 +520,44 @@ TResolvedConfig ResolveAll(NFyaml::TDocument& doc)
     return {labelNames, std::move(configs)};
 }
 
+void ValidateVolatileConfig(NFyaml::TDocument& doc) {
+    auto root = doc.Root();
+    auto seq = root.Sequence();
+    if (seq.size() == 0) {
+        ythrow yexception() << "Empty volatile config";
+    }
+    for (auto& elem : seq) {
+        auto map = elem.Map();
+        if (map.size() != 3) {
+            ythrow yexception() << "Invalid volatile config element: " << elem.Path();
+        }
+        for (auto& mapElem : map) {
+            auto key = mapElem.Key().Scalar();
+            if (key == "description") {
+                mapElem.Value().Scalar();
+            } else if (key == "selector") {
+                mapElem.Value().Map();
+            } else if (key == "config") {
+                mapElem.Value().Map();
+            } else {
+                ythrow yexception() << "Unknown element in volatile config: " << elem.Path();
+            }
+        }
+    }
+}
+
+void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig) {
+    auto configRoot = config.Root();
+    auto volatileConfigRoot = volatileConfig.Root();
+
+    auto seq = volatileConfigRoot.Sequence();
+    auto selectors = configRoot.Map().at("selector_config").Sequence();
+    for (auto& elem : seq) {
+        auto node = elem.Copy(config);
+        selectors.Append(node.Ref());
+    }
+}
+
 } // namespace NYamlConfig
 
 template <>
diff --git a/ydb/core/cms/console/yaml_config/yaml_config.h b/ydb/core/cms/console/yaml_config/yaml_config.h
index 7c4c5e7d40..85f8007170 100644
--- a/ydb/core/cms/console/yaml_config/yaml_config.h
+++ b/ydb/core/cms/console/yaml_config/yaml_config.h
@@ -1,6 +1,9 @@
 #pragma once
 
 #include <library/cpp/yaml/fyamlcpp/fyamlcpp.h>
+#include <library/cpp/actors/core/actor.h>
+
+#include <ydb/core/protos/config.pb.h>
 
 #include <openssl/sha.h>
 
@@ -98,6 +101,11 @@ TDocumentConfig Resolve(
     const TSet<TNamedLabel>& labels);
 
 /**
+ * Converts YAML representation to ProtoBuf
+ */
+NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown = false);
+
+/**
  * TLabel is a representation of label for config resolution
  *
  * It can be in three states:
@@ -142,4 +150,14 @@ struct TResolvedConfig {
  */
 TResolvedConfig ResolveAll(NFyaml::TDocument& doc);
 
-}// namespace NYamlConfig
+/**
+ * Validates single YAML volatile config schema
+ */
+void ValidateVolatileConfig(NFyaml::TDocument& doc);
+
+/**
+ * Appends volatile configs to the end of selectors list
+ */
+void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig);
+
+} // namespace NYamlConfig
diff --git a/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp b/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp
index 5099bb3e0f..1738cb08a2 100644
--- a/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp
+++ b/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp
@@ -703,6 +703,77 @@ const char *UnresolvedAllConfig14 = R"(---
       ? 14
 )";
 
+const char *SimpleConfig = R"(---
+cluster: test
+version: 12.1
+config:
+  num: 1
+allowed_labels:
+  tenant:
+    type: string
+
+selector_config: []
+)";
+
+const char *VolatilePart = R"(
+- description: test 4
+  selector:
+    tenant: /dev_global
+  config:
+    actor_system_config: {}
+    cms_config:
+      sentinel_config:
+        enable: false
+- description: test 5
+  selector:
+    canary: true
+  config:
+    actor_system_config: {}
+    cms_config:
+      sentinel_config:
+        enable: true
+)";
+
+const char *Concatenated = R"(---
+cluster: test
+version: 12.1
+config:
+  num: 1
+allowed_labels:
+  tenant:
+    type: string
+selector_config: [
+  {
+    description: test 4,
+    selector: {
+      tenant: /dev_global
+    },
+    config: {
+      actor_system_config: {},
+      cms_config: {
+        sentinel_config: {
+          enable: false
+        }
+      }
+    }
+  },
+  {
+    description: test 5,
+    selector: {
+      canary: true
+    },
+    config: {
+      actor_system_config: {},
+      cms_config: {
+        sentinel_config: {
+          enable: true
+        }
+      }
+    }
+  }
+  ]
+)";
+
 using EType = NYamlConfig::TLabel::EType;
 
 TMap<TSet<TVector<NYamlConfig::TLabel>>, TVector<int>> ExpectedResolved =
@@ -1325,4 +1396,13 @@ Y_UNIT_TEST_SUITE(YamlConfig) {
             // TODO extend ExpectedResolved manually and compare
         }
     }
+
+    Y_UNIT_TEST(AppendVolatileConfig) {
+        auto cfg = NFyaml::TDocument::Parse(SimpleConfig);
+        auto volatilePart = NFyaml::TDocument::Parse(VolatilePart);
+        NYamlConfig::AppendVolatileConfigs(cfg, volatilePart);
+        TStringStream stream;
+        stream << cfg;
+        UNIT_ASSERT_VALUES_EQUAL(stream.Str(), TString(Concatenated));
+    }
 }