diff options
author | innokentii <innokentii@yandex-team.com> | 2023-07-27 13:38:09 +0300 |
---|---|---|
committer | innokentii <innokentii@yandex-team.com> | 2023-07-27 13:38:09 +0300 |
commit | 114da2ea902184524cac007509fbe899d8154b05 (patch) | |
tree | 25ab62bface17b4f1dcc1fbbf1dadd35a86a9b9b | |
parent | 479f755935f129820b3f33ff07381351c8a42830 (diff) | |
download | ydb-114da2ea902184524cac007509fbe899d8154b05.tar.gz |
Split dynconfig tools to private and public parts
split dynconfig to public and private part
16 files changed, 1132 insertions, 959 deletions
diff --git a/ydb/library/yaml_config/CMakeLists.darwin-x86_64.txt b/ydb/library/yaml_config/CMakeLists.darwin-x86_64.txt index a46c3ed7d3..6925807f48 100644 --- a/ydb/library/yaml_config/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yaml_config/CMakeLists.darwin-x86_64.txt @@ -7,6 +7,7 @@ find_package(OpenSSL REQUIRED) +add_subdirectory(public) add_subdirectory(ut) add_library(ydb-library-yaml_config) @@ -23,6 +24,7 @@ target_link_libraries(ydb-library-yaml_config PUBLIC cms-console-util ydb-core-erasure ydb-core-protos + library-yaml_config-public ) target_sources(ydb-library-yaml_config PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/console_dumper.cpp diff --git a/ydb/library/yaml_config/CMakeLists.linux-aarch64.txt b/ydb/library/yaml_config/CMakeLists.linux-aarch64.txt index ef9a80b77d..7e23908a1a 100644 --- a/ydb/library/yaml_config/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yaml_config/CMakeLists.linux-aarch64.txt @@ -7,6 +7,7 @@ find_package(OpenSSL REQUIRED) +add_subdirectory(public) add_subdirectory(ut) add_library(ydb-library-yaml_config) @@ -24,6 +25,7 @@ target_link_libraries(ydb-library-yaml_config PUBLIC cms-console-util ydb-core-erasure ydb-core-protos + library-yaml_config-public ) target_sources(ydb-library-yaml_config PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/console_dumper.cpp diff --git a/ydb/library/yaml_config/CMakeLists.linux-x86_64.txt b/ydb/library/yaml_config/CMakeLists.linux-x86_64.txt index ef9a80b77d..7e23908a1a 100644 --- a/ydb/library/yaml_config/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yaml_config/CMakeLists.linux-x86_64.txt @@ -7,6 +7,7 @@ find_package(OpenSSL REQUIRED) +add_subdirectory(public) add_subdirectory(ut) add_library(ydb-library-yaml_config) @@ -24,6 +25,7 @@ target_link_libraries(ydb-library-yaml_config PUBLIC cms-console-util ydb-core-erasure ydb-core-protos + library-yaml_config-public ) target_sources(ydb-library-yaml_config PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/console_dumper.cpp diff --git a/ydb/library/yaml_config/CMakeLists.windows-x86_64.txt b/ydb/library/yaml_config/CMakeLists.windows-x86_64.txt index a46c3ed7d3..6925807f48 100644 --- a/ydb/library/yaml_config/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yaml_config/CMakeLists.windows-x86_64.txt @@ -7,6 +7,7 @@ find_package(OpenSSL REQUIRED) +add_subdirectory(public) add_subdirectory(ut) add_library(ydb-library-yaml_config) @@ -23,6 +24,7 @@ target_link_libraries(ydb-library-yaml_config PUBLIC cms-console-util ydb-core-erasure ydb-core-protos + library-yaml_config-public ) target_sources(ydb-library-yaml_config PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/console_dumper.cpp diff --git a/ydb/library/yaml_config/public/CMakeLists.darwin-x86_64.txt b/ydb/library/yaml_config/public/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..d531309764 --- /dev/null +++ b/ydb/library/yaml_config/public/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(library-yaml_config-public) +target_link_libraries(library-yaml_config-public PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + contrib-libs-protobuf + contrib-libs-yaml-cpp + cpp-actors-core + cpp-protobuf-json + cpp-yaml-fyamlcpp +) +target_sources(library-yaml_config-public PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/public/yaml_config.cpp +) diff --git a/ydb/library/yaml_config/public/CMakeLists.linux-aarch64.txt b/ydb/library/yaml_config/public/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..b41be6717e --- /dev/null +++ b/ydb/library/yaml_config/public/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(library-yaml_config-public) +target_link_libraries(library-yaml_config-public PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + contrib-libs-protobuf + contrib-libs-yaml-cpp + cpp-actors-core + cpp-protobuf-json + cpp-yaml-fyamlcpp +) +target_sources(library-yaml_config-public PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/public/yaml_config.cpp +) diff --git a/ydb/library/yaml_config/public/CMakeLists.linux-x86_64.txt b/ydb/library/yaml_config/public/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..b41be6717e --- /dev/null +++ b/ydb/library/yaml_config/public/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(library-yaml_config-public) +target_link_libraries(library-yaml_config-public PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + contrib-libs-protobuf + contrib-libs-yaml-cpp + cpp-actors-core + cpp-protobuf-json + cpp-yaml-fyamlcpp +) +target_sources(library-yaml_config-public PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/public/yaml_config.cpp +) diff --git a/ydb/library/yaml_config/public/CMakeLists.txt b/ydb/library/yaml_config/public/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yaml_config/public/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yaml_config/public/CMakeLists.windows-x86_64.txt b/ydb/library/yaml_config/public/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..d531309764 --- /dev/null +++ b/ydb/library/yaml_config/public/CMakeLists.windows-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(library-yaml_config-public) +target_link_libraries(library-yaml_config-public PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + contrib-libs-protobuf + contrib-libs-yaml-cpp + cpp-actors-core + cpp-protobuf-json + cpp-yaml-fyamlcpp +) +target_sources(library-yaml_config-public PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/public/yaml_config.cpp +) diff --git a/ydb/library/yaml_config/public/ya.make b/ydb/library/yaml_config/public/ya.make new file mode 100644 index 0000000000..e99b4674b5 --- /dev/null +++ b/ydb/library/yaml_config/public/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCS( + yaml_config.cpp + yaml_config.h +) + +PEERDIR( + contrib/libs/openssl + contrib/libs/protobuf + contrib/libs/yaml-cpp + library/cpp/actors/core + library/cpp/protobuf/json + library/cpp/yaml/fyamlcpp +) + +END() diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp new file mode 100644 index 0000000000..de3e1221a3 --- /dev/null +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -0,0 +1,759 @@ +#include "yaml_config.h" +#include "yaml_config_impl.h" + +#include <util/digest/sequence.h> + +template <> +struct THash<NYamlConfig::TLabel> { + inline size_t operator()(const NYamlConfig::TLabel& value) const { + return CombineHashes(THash<TString>{}(value.Value), (size_t)value.Type); + } +}; + +template <> +struct THash<TVector<TString>> { + inline size_t operator()(const TVector<TString>& value) const { + size_t result = 0; + for (auto& str : value) { + result = CombineHashes(result, THash<TString>{}(str)); + } + return result; + } +}; + +template <> +struct THash<TVector<NYamlConfig::TLabel>> : public TSimpleRangeHash {}; + +namespace NYamlConfig { + +inline const TMap<TString, EYamlConfigLabelTypeClass> ClassMapping{ + std::pair{TString("enum"), EYamlConfigLabelTypeClass::Closed}, + std::pair{TString("string"), EYamlConfigLabelTypeClass::Open}, +}; + +inline const TStringBuf inheritMapTag{"!inherit"}; +inline const TStringBuf inheritSeqTag{"!inherit:"}; +inline const TStringBuf inheritMapInSeqTag{"!inherit"}; +inline const TStringBuf removeTag{"!remove"}; +inline const TStringBuf appendTag{"!append"}; + +TString GetKey(const NFyaml::TNodeRef& node, TString key) { + auto map = node.Map(); + auto k = map.at(key).Scalar(); + return k; +} + +bool Fit(const TSelector& selector, const TSet<TNamedLabel>& labels) { + bool result = true; + size_t matched = 0; + for (auto& label : labels) { + if (auto it = selector.NotIn.find(label.Name); it != selector.NotIn.end() + && it->second.Values.contains(label.Value)) { + + return false; + } + + if (auto it = selector.In.find(label.Name); it != selector.In.end()) { + if (!it->second.Values.contains(label.Value)) { + result = false; + } else { + ++matched; + } + } + } + return (matched == selector.In.size()) && result; +} + +TSelector ParseSelector(const NFyaml::TNodeRef& selectors) { + TSelector result; + if (selectors) { + auto selectorsMap = selectors.Map(); + + for (auto it = selectorsMap.begin(); it != selectorsMap.end(); ++it) { + switch (it->Value().Type()) { + case NFyaml::ENodeType::Scalar: + { + auto [label, _] = result.In.try_emplace( + it->Key().Scalar(), + TLabelValueSet{}); + label->second.Values.insert(it->Value().Scalar()); + } + break; + case NFyaml::ENodeType::Mapping: + { + auto in = it->Value().Map()["in"]; + auto notIn = it->Value().Map()["not_in"]; + if (in && notIn) { + ythrow TYamlConfigEx() << "Using both in and not_in for same label: " + << it->Value().Path(); + } + + if (in) { + auto inSeq = in.Sequence(); + auto [label, _] = result.In.try_emplace( + it->Key().Scalar(), + TLabelValueSet{}); + for(auto it3 = inSeq.begin(); it3 != inSeq.end(); ++it3) { + label->second.Values.insert(it3->Scalar()); + } + } + + if (notIn) { + auto notInSeq = notIn.Sequence(); + auto [label, _] = result.NotIn.try_emplace( + it->Key().Scalar(), + TLabelValueSet{}); + for(auto it3 = notInSeq.begin(); it3 != notInSeq.end(); ++it3) { + label->second.Values.insert(it3->Scalar()); + } + } + } + break; + default: + { + ythrow TYamlConfigEx() << "Selector should be scalar, \"in\" or \"not_in\": " + << it->Value().Path(); + } + break; + } + } + } else { + ythrow TYamlConfigEx() << "Selector shouldn't be empty"; + } + return result; +} + +TYamlConfigModel ParseConfig(NFyaml::TDocument& doc) { + TYamlConfigModel res{.Doc = doc}; + auto root = doc.Root().Map(); + res.Config = root.at("config"); + + auto allowedLabels = root.at("allowed_labels").Map(); + + for (auto it = allowedLabels.begin(); it != allowedLabels.end(); ++it) { + auto type = it->Value().Map().at("type"); + if (!type || type.Type() != NFyaml::ENodeType::Scalar) { + ythrow TYamlConfigEx() << "Label type should be Scalar"; + } + + EYamlConfigLabelTypeClass classType; + + if (auto classIt = ClassMapping.find(type.Scalar()); classIt != ClassMapping.end()) { + classType = classIt->second; + } else { + ythrow TYamlConfigEx() << "Unsupported label type: " << type.Scalar(); + } + + auto label = res.AllowedLabels.try_emplace( + it->Key().Scalar(), + TLabelType{classType, TSet<TString>{""}}); + + if (auto labelDesc = it->Value().Map()["values"]; labelDesc) { + auto values = labelDesc.Map(); + for(auto it2 = values.begin(); it2 != values.end(); ++it2) { + label.first->second.Values.insert(it2->Key().Scalar()); + } + } + } + + auto selectorConfig = root.at("selector_config").Sequence(); + + for (auto it = selectorConfig.begin(); it != selectorConfig.end(); ++it) { + TYamlConfigModel::TSelectorModel selector; + + auto selectorRoot = it->Map(); + selector.Description = selectorRoot.at("description").Scalar(); + selector.Config = selectorRoot.at("config"); + selector.Selector = ParseSelector(selectorRoot.at("selector")); + + res.Selectors.push_back(selector); + } + + return res; +} + +TMap<TString, TLabelType> CollectLabels(NFyaml::TDocument& doc) { + + auto config = ParseConfig(doc); + + TMap<TString, TLabelType> result = config.AllowedLabels; + + for (auto& selector : config.Selectors) { + for (auto& [name, valueSet] : selector.Selector.In) { + result[name].Values.insert(valueSet.Values.begin(), valueSet.Values.end()); + } + + for (auto& [name, valueSet] : selector.Selector.NotIn) { + result[name].Values.insert(valueSet.Values.begin(), valueSet.Values.end()); + } + } + + return result; +} + +bool IsMapInherit(const NFyaml::TNodeRef& node) { + if (auto tag = node.Tag(); tag) { + switch (node.Type()) { + case NFyaml::ENodeType::Mapping: + return *tag == inheritMapTag; + case NFyaml::ENodeType::Sequence: + return tag->StartsWith(inheritSeqTag); + case NFyaml::ENodeType::Scalar: + return false; + } + } + return false; +} + +bool IsSeqInherit(const NFyaml::TNodeRef& node) { + if (auto tag = node.Tag(); tag) { + switch (node.Type()) { + case NFyaml::ENodeType::Mapping: + return *tag == inheritMapInSeqTag; + case NFyaml::ENodeType::Sequence: + return false; + case NFyaml::ENodeType::Scalar: + return false; + } + } + return false; +} + +void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from); + +bool IsSeqAppend(const NFyaml::TNodeRef& node) { + if (auto tag = node.Tag(); tag) { + switch (node.Type()) { + case NFyaml::ENodeType::Mapping: + return false; + case NFyaml::ENodeType::Sequence: + return *tag == appendTag; + case NFyaml::ENodeType::Scalar: + return false; + } + } + return false; +} + +bool IsRemove(const NFyaml::TNodeRef& node) { + if (auto tag = node.Tag(); tag) { + return *tag == removeTag; + } + return false; +} + +void Inherit(NFyaml::TMapping& toMap, const NFyaml::TMapping& fromMap) { + for (auto it = fromMap.begin(); it != fromMap.end(); ++it) { + if (auto toEntry = toMap.pair_at_opt(it->Key().Scalar()); toEntry) { + auto fromNode = it->Value(); + auto toNode = toEntry.Value(); + + if (IsMapInherit(fromNode)) { + Apply(toNode, fromNode); + } else if (IsSeqAppend(fromNode)) { + Append(toNode, fromNode); + } else { + toMap.Remove(toEntry.Key()); + toMap.Append(it->Key().Copy().Ref(), it->Value().Copy().Ref()); + } + } else { + toMap.Append(it->Key().Copy().Ref(), it->Value().Copy().Ref()); + } + } +} + +void Inherit(NFyaml::TSequence& toSeq, const NFyaml::TSequence& fromSeq, const TString& key) { + TMap<TString, NFyaml::TNodeRef> nodes; + + for (auto it = toSeq.begin(); it != toSeq.end(); ++it) { + nodes[GetKey(*it, key)] = *it; + } + + for (auto it = fromSeq.begin(); it != fromSeq.end(); ++it) { + auto fromKey = GetKey(*it, key); + + if (nodes.contains(fromKey)) { + if (IsSeqInherit(*it)) { + Apply(nodes[fromKey], *it); + } else if (IsSeqAppend(*it)) { + Append(nodes[fromKey], *it); + } else if (IsRemove(*it)) { + toSeq.Remove(nodes[fromKey]); + nodes.erase(fromKey); + } else { + auto newNode = it->Copy(); + toSeq.InsertAfter(nodes[fromKey], newNode.Ref()); + toSeq.Remove(nodes[fromKey]); + nodes[fromKey] = newNode.Ref(); + } + } else { + auto newNode = it->Copy(); + toSeq.Append(newNode.Ref()); + nodes[fromKey] = newNode.Ref(); + } + } +} + +void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { + Y_ENSURE_EX(to, TYamlConfigEx() << "Appending to empty value: " + << to.Path() << " <- " << from.Path()); + Y_ENSURE_EX(to.Type() == NFyaml::ENodeType::Sequence && from.Type() == NFyaml::ENodeType::Sequence, TYamlConfigEx() << "Appending to wrong type" + << to.Path() << " <- " << from.Path()); + + auto fromSeq = from.Sequence(); + auto toSeq = to.Sequence(); + + for (auto it = fromSeq.begin(); it != fromSeq.end(); ++it) { + auto newNode = it->Copy(); + toSeq.Append(newNode.Ref()); + } +} + +void Apply(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { + Y_ENSURE_EX(to, TYamlConfigEx() << "Overriding empty value: " + << to.Path() << " <- " << from.Path()); + Y_ENSURE_EX(to.Type() == from.Type(), TYamlConfigEx() << "Overriding value with different types: " + << to.Path() << " <- " << from.Path()); + + switch (from.Type()) { + case NFyaml::ENodeType::Mapping: + { + auto toMap = to.Map(); + auto fromMap = from.Map(); + Inherit(toMap, fromMap); + } + break; + case NFyaml::ENodeType::Sequence: + { + auto tag = from.Tag(); + auto key = tag->substr(inheritSeqTag.length()); + auto toSeq = to.Sequence(); + auto fromSeq = from.Sequence(); + Inherit(toSeq, fromSeq, key); + } + break; + case NFyaml::ENodeType::Scalar: + { + ythrow TYamlConfigEx() << "Override with scalar: " + << to.Path() << " <- " << from.Path(); + } + break; + } +} + +void RemoveTags(NFyaml::TDocument& doc) { + for (auto it = doc.begin(); it != doc.end(); ++it) { + it->RemoveTag(); + } +} + +TDocumentConfig Resolve( + const NFyaml::TDocument& doc, + const TSet<TNamedLabel>& labels) +{ + TDocumentConfig res{doc.Clone(), NFyaml::TNodeRef{}}; + res.first.Resolve(); + + auto rootMap = res.first.Root().Map(); + auto config = rootMap.at("config"); + auto selectorConfig = rootMap.at("selector_config").Sequence(); + + for (auto it = selectorConfig.begin(); it != selectorConfig.end(); ++it) { + auto selectorMap = it->Map(); + auto desc = selectorMap.at("description").Scalar(); + auto selectorNode = selectorMap.at("selector"); + auto selector = ParseSelector(selectorNode); + if (Fit(selector, labels)) { + Apply(config, selectorMap.at("config")); + } + } + + RemoveTags(res.first); + + res.second = config; + + return res; +} + +void Combine( + TVector<TVector<TLabel>>& labelCombinations, + TVector<TLabel>& combination, + const TVector<std::pair<TString, TSet<TLabel>>>& labels, + size_t offset) +{ + if (offset == labels.size()) { + labelCombinations.push_back(combination); + return; + } + + for (auto& label : labels[offset].second) { + combination[offset] = label; + Combine(labelCombinations, combination, labels, offset + 1); + } +} + +bool Fit( + const TSelector& selector, + const TVector<TLabel>& labels, + const TVector<std::pair<TString, TSet<TLabel>>>& names) +{ + for (size_t i = 0; i < labels.size(); ++ i) { + auto& label = labels[i]; + auto& name = names[i].first; + switch(label.Type) { + case TLabel::EType::Negative: + if (selector.In.contains(name)) { + return false; + } + break; + case TLabel::EType::Empty: [[fallthrough]]; + case TLabel::EType::Common: + if (auto it = selector.In.find(name); it != selector.In.end() + && !it->second.Values.contains(label.Value)) { + + return false; + } + if (auto it = selector.NotIn.find(name); it != selector.NotIn.end() + && it->second.Values.contains(label.Value)) { + + return false; + } + break; + } + } + + return true; +} + +TResolvedConfig ResolveAll(NFyaml::TDocument& doc) +{ + TVector<TString> labelNames; + TVector<std::pair<TString, TSet<TLabel>>> labels; + + auto config = ParseConfig(doc); + auto namedLabels = CollectLabels(doc); + + for (auto& [name, values]: namedLabels) { + TSet<TLabel> set; + if (values.Class == EYamlConfigLabelTypeClass::Open) { + set.insert(TLabel{TLabel::EType::Negative, {}}); + } + for (auto& value: values.Values) { + if (value != "") { + set.insert(TLabel{TLabel::EType::Common, value}); + } else { + set.insert(TLabel{TLabel::EType::Empty, {}}); + } + } + labels.push_back({name, set}); + labelNames.push_back(name); + } + + TVector<TVector<TLabel>> labelCombinations; + + TVector<TLabel> combination; + combination.resize(labels.size()); + + Combine(labelCombinations, combination, labels, 0); + + auto cmp = [](const TVector<int>& lhs, const TVector<int>& rhs) { + auto lhsIt = lhs.begin(); + auto rhsIt = rhs.begin(); + + while (lhsIt != lhs.end() && rhsIt != rhs.end() && (*lhsIt == *rhsIt)) { + lhsIt++; + rhsIt++; + } + + if (lhsIt == lhs.end()) { + return false; + } else if (rhsIt == rhs.end()) { + return true; + } + + return *lhsIt < *rhsIt; + }; + + using TTriePath = TVector<int>; + + struct TTrieNode { + TSimpleSharedPtr<TDocumentConfig> ResolvedConfig; + TVector<TVector<TLabel>> LabelCombinations; + }; + + std::map<TTriePath, TSimpleSharedPtr<TDocumentConfig>, decltype(cmp)> selectorsTrie(cmp); + std::map<TTriePath, TTrieNode, decltype(cmp)> appliedSelectors(cmp); + + auto rootConfig = TTrieNode { + MakeSimpleShared<TDocumentConfig>(std::move(doc), config.Config), + {}, + }; + + selectorsTrie[{0}] = rootConfig.ResolvedConfig; + + for (size_t j = 0; j < labelCombinations.size(); ++j) { + TSimpleSharedPtr<TDocumentConfig> cur = rootConfig.ResolvedConfig; + TTriePath triePath({0}); + + for (size_t i = 0; i < config.Selectors.size(); ++i) { + if (Fit(config.Selectors[i].Selector, labelCombinations[j], labels)) { + triePath.push_back(i + 1); + if (auto it = selectorsTrie.find(triePath); it != selectorsTrie.end()) { + cur = it->second; + } else { + auto clone = cur->first.Clone(); + auto cloneConfig = ParseConfig(clone); + + Apply(cloneConfig.Config, cloneConfig.Selectors[i].Config); + + cur = MakeSimpleShared<std::pair<NFyaml::TDocument, NFyaml::TNodeRef>>( + std::move(clone), + cloneConfig.Config), + selectorsTrie[triePath] = cur; + } + } + } + + if (auto it = appliedSelectors.find(triePath); it != appliedSelectors.end()) { + it->second.LabelCombinations.push_back(labelCombinations[j]); + } else { + appliedSelectors.try_emplace(triePath, TTrieNode{ + cur, + {labelCombinations[j]} + }); + } + } + + selectorsTrie.clear(); + + TMap<TSet<TVector<TLabel>>, TDocumentConfig> configs; + + for (auto& [_, value]: appliedSelectors) { + configs.try_emplace( + TSet<TVector<TLabel>>( + value.LabelCombinations.begin(), + value.LabelCombinations.end()), + std::make_pair(std::move(value.ResolvedConfig->first), value.ResolvedConfig->second)); + } + + return {labelNames, std::move(configs)}; +} + +size_t Hash(const NFyaml::TNodeRef& resolved) { + TStringStream ss; + ss << resolved; + TString s = ss.Str(); + return THash<TString>{}(s); +} + +size_t Hash(const TResolvedConfig& config) +{ + size_t configsHash = 0; + for (auto& [labelSet, docConfig] : config.Configs) { + for (auto labels : labelSet) { + auto labelsHash = THash<TVector<TLabel>>{}(labels); + configsHash = CombineHashes(labelsHash, configsHash); + } + configsHash = CombineHashes(Hash(docConfig.second), configsHash); + } + + return CombineHashes(THash<TVector<TString>>{}(config.Labels), configsHash); +} + +void ValidateVolatileConfig(NFyaml::TDocument& doc) { + auto root = doc.Root(); + auto seq = root.Map().at("selector_config").Sequence(); + if (seq.size() == 0) { + ythrow yexception() << "Empty volatile config"; + } + for (auto& elem : seq) { + auto map = elem.Map(); + if (map.size() != 3) { + ythrow yexception() << "Invalid volatile config element: " << elem.Path(); + } + for (auto& mapElem : map) { + auto key = mapElem.Key().Scalar(); + if (key == "description") { + mapElem.Value().Scalar(); + } else if (key == "selector") { + mapElem.Value().Map(); + } else if (key == "config") { + mapElem.Value().Map(); + } else { + ythrow yexception() << "Unknown element in volatile config: " << elem.Path(); + } + } + } +} + +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig) { + auto configRoot = config.Root(); + auto volatileConfigRoot = volatileConfig.Root(); + + auto seq = volatileConfigRoot.Sequence(); + auto selectors = configRoot.Map().at("selector_config").Sequence(); + for (auto& elem : seq) { + auto node = elem.Copy(config); + selectors.Append(node.Ref()); + } +} + +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig) { + auto configRoot = config.Root(); + + auto seq = volatileConfig.Sequence(); + auto selectors = configRoot.Map().at("selector_config").Sequence(); + for (auto& elem : seq) { + auto node = elem.Copy(config); + selectors.Append(node.Ref()); + } +} + +ui64 GetVersion(const TString& config) { + auto metadata = GetMetadata(config); + return metadata.Version.value_or(0); +} + +TMetadata GetMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map()["metadata"]; node) { + auto versionNode = node.Map()["version"]; + auto clusterNode = node.Map()["cluster"]; + return TMetadata{ + .Version = versionNode ? std::optional{FromString<ui64>(versionNode.Scalar())} : std::nullopt, + .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + +TVolatileMetadata GetVolatileMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map().at("metadata"); node) { + auto versionNode = node.Map().at("version"); + auto clusterNode = node.Map().at("cluster"); + auto idNode = node.Map().at("id"); + return TVolatileMetadata{ + .Version = versionNode ? std::make_optional(FromString<ui64>(versionNode.Scalar())) : std::nullopt, + .Cluster = clusterNode ? std::make_optional(clusterNode.Scalar()) : std::nullopt, + .Id = idNode ? std::make_optional(FromString<ui64>(idNode.Scalar())) : std::nullopt, + }; + } + + return {}; +} + +TString ReplaceMetadata(const TString& config, const std::function<void(TStringStream&)>& serializeMetadata) { + TStringStream sstr; + auto doc = NFyaml::TDocument::Parse(config); + if (doc.Root().Style() == NFyaml::ENodeStyle::Flow) { + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + doc.Root().Map().Remove(pair); + } + serializeMetadata(sstr); + sstr << "\n" << doc; + } else { + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + auto begin = pair.Key().BeginMark().InputPos; + auto end = pair.Value().EndMark().InputPos; + sstr << config.substr(0, begin); + serializeMetadata(sstr); + if (end < config.length() && config[end] == ':') { + end = end + 1; + } + sstr << config.substr(end, TString::npos); + } else { + if (doc.HasExplicitDocumentStart()) { + auto docStart = doc.BeginMark().InputPos + 4; + sstr << config.substr(0, docStart); + serializeMetadata(sstr); + sstr << "\n" << config.substr(docStart, TString::npos); + } else { + serializeMetadata(sstr); + sstr << "\n" << config; + } + } + } + return sstr.Str(); + +} + +TString ReplaceMetadata(const TString& config, const TMetadata& metadata) { + auto serializeMetadata = [&](TStringStream& sstr) { + sstr + << "metadata:" + << "\n kind: MainConfig" + << "\n cluster: \"" << *metadata.Cluster << "\"" + << "\n version: " << *metadata.Version; + }; + return ReplaceMetadata(config, serializeMetadata); +} + +TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata) { + auto serializeMetadata = [&](TStringStream& sstr) { + sstr + << "metadata:" + << "\n kind: VolatileConfig" + << "\n cluster: \"" << *metadata.Cluster << "\"" + << "\n version: " << *metadata.Version + << "\n id: " << *metadata.Id; + }; + return ReplaceMetadata(config, serializeMetadata); +} + +bool IsConfigKindEquals(const TString& config, const TString& kind) { + try { + auto doc = NFyaml::TDocument::Parse(config); + return doc.Root().Map().at("metadata").Map().at("kind").Scalar() == kind; + } catch (yexception& e) { + return false; + } +} + +bool IsVolatileConfig(const TString& config) { + return IsConfigKindEquals(config, "VolatileConfig"); +} + +bool IsMainConfig(const TString& config) { + return IsConfigKindEquals(config, "MainConfig"); +} + +TString StripMetadata(const TString& config) { + auto doc = NFyaml::TDocument::Parse(config); + + TStringStream sstr; + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + auto begin = pair.Key().BeginMark().InputPos; + sstr << config.substr(0, begin); + auto end = pair.Value().EndMark().InputPos; + sstr << config.substr(end, TString::npos); + } else { + if (doc.HasExplicitDocumentStart()) { + auto docStart = doc.BeginMark().InputPos + 4; + sstr << config.substr(0, docStart); + sstr << "\n" << config.substr(docStart, TString::npos); + } else { + sstr << config; + } + } + + return sstr.Str(); +} + +} // namespace NYamlConfig + +template <> +void Out<NYamlConfig::TLabel>(IOutputStream& out, const NYamlConfig::TLabel& value) { + out << (int)value.Type << ":" << value.Value; +} diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h new file mode 100644 index 0000000000..7ac7081c99 --- /dev/null +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -0,0 +1,226 @@ +#pragma once + +#include <library/cpp/yaml/fyamlcpp/fyamlcpp.h> +#include <library/cpp/actors/core/actor.h> + +#include <openssl/sha.h> + +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/generic/set.h> +#include <util/generic/map.h> +#include <util/stream/str.h> + +#include <unordered_map> +#include <map> +#include <string> + +namespace NYamlConfig { + +struct TYamlConfigEx : public yexception {}; + +using TDocumentConfig = std::pair<NFyaml::TDocument, NFyaml::TNodeRef>; + +/** + * Open - labels like tenant, where we don't know final set of values + * Closed - labels with predefined set of values, e.g. size + */ +enum class EYamlConfigLabelTypeClass { + Open, + Closed, +}; + +/** + * TNamedLabel - representation of label used for selector + */ +class TNamedLabel { +public: + TString Name; + TString Value; + bool Inv = false; + + bool operator<(const TNamedLabel& other) const { return Name < other.Name; } +}; + +/** + * TLabelType - represents known set of values for a label with its type + */ +class TLabelType { +public: + EYamlConfigLabelTypeClass Class; + TSet<TString> Values; + + bool operator==(const TLabelType& other) const { return Values == other.Values; } +}; + +class TLabelValueSet { +public: + TSet<TString> Values; + + bool operator==(const TLabelValueSet& other) const { return Values == other.Values; } +}; + +class TSelector { +public: + TMap<TString, TLabelValueSet> In; + TMap<TString, TLabelValueSet> NotIn; +}; + +struct TYamlConfigModel { + struct TSelectorModel { + TString Description; + TSelector Selector; + NFyaml::TNodeRef Config; + }; + + const NFyaml::TDocument& Doc; + NFyaml::TNodeRef Config; + TMap<TString, TLabelType> AllowedLabels; + TVector<TSelectorModel> Selectors; +}; + +/** + * Collects all labels present in document + * For Open labels gathers all labels from all selectors + * For Closed labels additionally validates that there is no additional labels + */ +TMap<TString, TLabelType> CollectLabels(NFyaml::TDocument& doc); + +/** + * Parses config and fills corresponding struct + */ +TYamlConfigModel ParseConfig(NFyaml::TDocument& doc); + +/** + * Generates config for specific set of labels applying all matching selectors + */ +TDocumentConfig Resolve( + const NFyaml::TDocument& doc, + const TSet<TNamedLabel>& labels); + +/** + * TLabel is a representation of label for config resolution + * + * It can be in three states: + * - Empty with Type == EType::Empty and Value == "" + * it equals empty or undefined label + * - Common with Type == EType::Common and Value == arbitrary string + * it equals defined label with corresponding value + * - Negative with Type == EType::Negative and Value == "" + * it is used for Open labels and equals any label not present in labels + * discovered by CollectLabels (and also not equals Empty label) + */ +struct TLabel { + enum class EType { + Negative = 0, + Empty, + Common, + }; + + EType Type; + TString Value; + + bool operator<(const TLabel& other) const { + int lhs = static_cast<int>(Type); + int rhs = static_cast<int>(other.Type); + return std::tie(lhs, Value) < std::tie(rhs, other.Value); + } + + bool operator==(const TLabel& other) const { + int lhs = static_cast<int>(Type); + int rhs = static_cast<int>(other.Type); + return std::tie(lhs, Value) == std::tie(rhs, other.Value); + } +}; + +struct TResolvedConfig { + TVector<TString> Labels; + TMap<TSet<TVector<TLabel>>, TDocumentConfig> Configs; +}; + +/** + * Generates configs for all label combinations + */ +TResolvedConfig ResolveAll(NFyaml::TDocument& doc); + +/** + * Calculates hash of resolved config + * Used to ensure that cli resolves config the same as a server + */ +size_t Hash(const TResolvedConfig& config); + +/** + * Validates single YAML volatile config schema + */ +void ValidateVolatileConfig(NFyaml::TDocument& doc); + +/** + * Appends volatile configs to the end of selectors list + * **Important**: Document should be a list with selectors + */ +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig); + +/** + * Appends volatile configs to the end of selectors list + * **Important**: Node should be a list with selectors + */ +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig); + +/** + * Parses config version + */ +ui64 GetVersion(const TString& config); + +/** + * Represents config metadata + */ +struct TMetadata { + std::optional<ui64> Version; + std::optional<TString> Cluster; +}; + +/** + * Parses config metadata + */ +TMetadata GetMetadata(const TString& config); + +/** + * Represents volatile config metadata + */ +struct TVolatileMetadata { + std::optional<ui64> Version; + std::optional<TString> Cluster; + std::optional<ui64> Id; +}; + +/** + * Parses volatile config metadata + */ +TVolatileMetadata GetVolatileMetadata(const TString& config); + +/** + * Replaces metadata in config + */ +TString ReplaceMetadata(const TString& config, const TMetadata& metadata); + +/** + * Replaces volatile metadata in config + */ +TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata); + +/** + * Checks whether string is volatile config or not + */ +bool IsVolatileConfig(const TString& config); + +/** + * Checks whether string is main config or not + */ +bool IsMainConfig(const TString& config); + +/** + * Strips metadata from config + */ +TString StripMetadata(const TString& config); + +} // namespace NYamlConfig diff --git a/ydb/library/yaml_config/yaml_config_impl.h b/ydb/library/yaml_config/public/yaml_config_impl.h index 2f9a6f7646..2f9a6f7646 100644 --- a/ydb/library/yaml_config/yaml_config_impl.h +++ b/ydb/library/yaml_config/public/yaml_config_impl.h diff --git a/ydb/library/yaml_config/ya.make b/ydb/library/yaml_config/ya.make index 0fa4761e80..3e9602ba3e 100644 --- a/ydb/library/yaml_config/ya.make +++ b/ydb/library/yaml_config/ya.make @@ -5,7 +5,6 @@ SRCS( console_dumper.h yaml_config.cpp yaml_config.h - yaml_config_impl.h yaml_config_parser.cpp yaml_config_parser.h ) @@ -21,10 +20,15 @@ PEERDIR( ydb/core/cms/console/util ydb/core/erasure ydb/core/protos + ydb/library/yaml_config/public ) END() +RECURSE( + public +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp index 4a61f10fe7..acd1a4114b 100644 --- a/ydb/library/yaml_config/yaml_config.cpp +++ b/ydb/library/yaml_config/yaml_config.cpp @@ -1,433 +1,13 @@ #include "yaml_config.h" -#include "yaml_config_impl.h" + #include "yaml_config_parser.h" #include <ydb/core/base/appdata.h> #include <library/cpp/protobuf/json/json2proto.h> -template <> -struct THash<NYamlConfig::TLabel> { - inline size_t operator()(const NYamlConfig::TLabel& value) const { - return CombineHashes(THash<TString>{}(value.Value), (size_t)value.Type); - } -}; - -template <> -struct THash<TVector<TString>> { - inline size_t operator()(const TVector<TString>& value) const { - size_t result = 0; - for (auto& str : value) { - result = CombineHashes(result, THash<TString>{}(str)); - } - return result; - } -}; - -template <> -struct THash<TVector<NYamlConfig::TLabel>> : public TSimpleRangeHash {}; - namespace NYamlConfig { -inline const TMap<TString, EYamlConfigLabelTypeClass> ClassMapping{ - std::pair{TString("enum"), EYamlConfigLabelTypeClass::Closed}, - std::pair{TString("string"), EYamlConfigLabelTypeClass::Open}, -}; - -inline const TStringBuf inheritMapTag{"!inherit"}; -inline const TStringBuf inheritSeqTag{"!inherit:"}; -inline const TStringBuf inheritMapInSeqTag{"!inherit"}; -inline const TStringBuf removeTag{"!remove"}; -inline const TStringBuf appendTag{"!append"}; - -TString GetKey(const NFyaml::TNodeRef& node, TString key) { - auto map = node.Map(); - auto k = map.at(key).Scalar(); - return k; -} - -bool Fit(const TSelector& selector, const TSet<TNamedLabel>& labels) { - bool result = true; - size_t matched = 0; - for (auto& label : labels) { - if (auto it = selector.NotIn.find(label.Name); it != selector.NotIn.end() - && it->second.Values.contains(label.Value)) { - - return false; - } - - if (auto it = selector.In.find(label.Name); it != selector.In.end()) { - if (!it->second.Values.contains(label.Value)) { - result = false; - } else { - ++matched; - } - } - } - return (matched == selector.In.size()) && result; -} - -TSelector ParseSelector(const NFyaml::TNodeRef& selectors) { - TSelector result; - if (selectors) { - auto selectorsMap = selectors.Map(); - - for (auto it = selectorsMap.begin(); it != selectorsMap.end(); ++it) { - switch (it->Value().Type()) { - case NFyaml::ENodeType::Scalar: - { - auto [label, _] = result.In.try_emplace( - it->Key().Scalar(), - TLabelValueSet{}); - label->second.Values.insert(it->Value().Scalar()); - } - break; - case NFyaml::ENodeType::Mapping: - { - auto in = it->Value().Map()["in"]; - auto notIn = it->Value().Map()["not_in"]; - if (in && notIn) { - ythrow TYamlConfigEx() << "Using both in and not_in for same label: " - << it->Value().Path(); - } - - if (in) { - auto inSeq = in.Sequence(); - auto [label, _] = result.In.try_emplace( - it->Key().Scalar(), - TLabelValueSet{}); - for(auto it3 = inSeq.begin(); it3 != inSeq.end(); ++it3) { - label->second.Values.insert(it3->Scalar()); - } - } - - if (notIn) { - auto notInSeq = notIn.Sequence(); - auto [label, _] = result.NotIn.try_emplace( - it->Key().Scalar(), - TLabelValueSet{}); - for(auto it3 = notInSeq.begin(); it3 != notInSeq.end(); ++it3) { - label->second.Values.insert(it3->Scalar()); - } - } - } - break; - default: - { - ythrow TYamlConfigEx() << "Selector should be scalar, \"in\" or \"not_in\": " - << it->Value().Path(); - } - break; - } - } - } else { - ythrow TYamlConfigEx() << "Selector shouldn't be empty"; - } - return result; -} - -TYamlConfigModel ParseConfig(NFyaml::TDocument& doc) { - TYamlConfigModel res{.Doc = doc}; - auto root = doc.Root().Map(); - res.Config = root.at("config"); - - auto allowedLabels = root.at("allowed_labels").Map(); - - for (auto it = allowedLabels.begin(); it != allowedLabels.end(); ++it) { - auto type = it->Value().Map().at("type"); - if (!type || type.Type() != NFyaml::ENodeType::Scalar) { - ythrow TYamlConfigEx() << "Label type should be Scalar"; - } - - EYamlConfigLabelTypeClass classType; - - if (auto classIt = ClassMapping.find(type.Scalar()); classIt != ClassMapping.end()) { - classType = classIt->second; - } else { - ythrow TYamlConfigEx() << "Unsupported label type: " << type.Scalar(); - } - - auto label = res.AllowedLabels.try_emplace( - it->Key().Scalar(), - TLabelType{classType, TSet<TString>{""}}); - - if (auto labelDesc = it->Value().Map()["values"]; labelDesc) { - auto values = labelDesc.Map(); - for(auto it2 = values.begin(); it2 != values.end(); ++it2) { - label.first->second.Values.insert(it2->Key().Scalar()); - } - } - } - - auto selectorConfig = root.at("selector_config").Sequence(); - - for (auto it = selectorConfig.begin(); it != selectorConfig.end(); ++it) { - TYamlConfigModel::TSelectorModel selector; - - auto selectorRoot = it->Map(); - selector.Description = selectorRoot.at("description").Scalar(); - selector.Config = selectorRoot.at("config"); - selector.Selector = ParseSelector(selectorRoot.at("selector")); - - res.Selectors.push_back(selector); - } - - return res; -} - -TMap<TString, TLabelType> CollectLabels(NFyaml::TDocument& doc) { - - auto config = ParseConfig(doc); - - TMap<TString, TLabelType> result = config.AllowedLabels; - - for (auto& selector : config.Selectors) { - for (auto& [name, valueSet] : selector.Selector.In) { - result[name].Values.insert(valueSet.Values.begin(), valueSet.Values.end()); - } - - for (auto& [name, valueSet] : selector.Selector.NotIn) { - result[name].Values.insert(valueSet.Values.begin(), valueSet.Values.end()); - } - } - - return result; -} - -bool IsMapInherit(const NFyaml::TNodeRef& node) { - if (auto tag = node.Tag(); tag) { - switch (node.Type()) { - case NFyaml::ENodeType::Mapping: - return *tag == inheritMapTag; - case NFyaml::ENodeType::Sequence: - return tag->StartsWith(inheritSeqTag); - case NFyaml::ENodeType::Scalar: - return false; - } - } - return false; -} - -bool IsSeqInherit(const NFyaml::TNodeRef& node) { - if (auto tag = node.Tag(); tag) { - switch (node.Type()) { - case NFyaml::ENodeType::Mapping: - return *tag == inheritMapInSeqTag; - case NFyaml::ENodeType::Sequence: - return false; - case NFyaml::ENodeType::Scalar: - return false; - } - } - return false; -} - -void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from); - -bool IsSeqAppend(const NFyaml::TNodeRef& node) { - if (auto tag = node.Tag(); tag) { - switch (node.Type()) { - case NFyaml::ENodeType::Mapping: - return false; - case NFyaml::ENodeType::Sequence: - return *tag == appendTag; - case NFyaml::ENodeType::Scalar: - return false; - } - } - return false; -} - -bool IsRemove(const NFyaml::TNodeRef& node) { - if (auto tag = node.Tag(); tag) { - return *tag == removeTag; - } - return false; -} - -void Inherit(NFyaml::TMapping& toMap, const NFyaml::TMapping& fromMap) { - for (auto it = fromMap.begin(); it != fromMap.end(); ++it) { - if (auto toEntry = toMap.pair_at_opt(it->Key().Scalar()); toEntry) { - auto fromNode = it->Value(); - auto toNode = toEntry.Value(); - - if (IsMapInherit(fromNode)) { - Apply(toNode, fromNode); - } else if (IsSeqAppend(fromNode)) { - Append(toNode, fromNode); - } else { - toMap.Remove(toEntry.Key()); - toMap.Append(it->Key().Copy().Ref(), it->Value().Copy().Ref()); - } - } else { - toMap.Append(it->Key().Copy().Ref(), it->Value().Copy().Ref()); - } - } -} - -void Inherit(NFyaml::TSequence& toSeq, const NFyaml::TSequence& fromSeq, const TString& key) { - TMap<TString, NFyaml::TNodeRef> nodes; - - for (auto it = toSeq.begin(); it != toSeq.end(); ++it) { - nodes[GetKey(*it, key)] = *it; - } - - for (auto it = fromSeq.begin(); it != fromSeq.end(); ++it) { - auto fromKey = GetKey(*it, key); - - if (nodes.contains(fromKey)) { - if (IsSeqInherit(*it)) { - Apply(nodes[fromKey], *it); - } else if (IsSeqAppend(*it)) { - Append(nodes[fromKey], *it); - } else if (IsRemove(*it)) { - toSeq.Remove(nodes[fromKey]); - nodes.erase(fromKey); - } else { - auto newNode = it->Copy(); - toSeq.InsertAfter(nodes[fromKey], newNode.Ref()); - toSeq.Remove(nodes[fromKey]); - nodes[fromKey] = newNode.Ref(); - } - } else { - auto newNode = it->Copy(); - toSeq.Append(newNode.Ref()); - nodes[fromKey] = newNode.Ref(); - } - } -} - -void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { - Y_ENSURE_EX(to, TYamlConfigEx() << "Appending to empty value: " - << to.Path() << " <- " << from.Path()); - Y_ENSURE_EX(to.Type() == NFyaml::ENodeType::Sequence && from.Type() == NFyaml::ENodeType::Sequence, TYamlConfigEx() << "Appending to wrong type" - << to.Path() << " <- " << from.Path()); - - auto fromSeq = from.Sequence(); - auto toSeq = to.Sequence(); - - for (auto it = fromSeq.begin(); it != fromSeq.end(); ++it) { - auto newNode = it->Copy(); - toSeq.Append(newNode.Ref()); - } -} - -void Apply(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { - Y_ENSURE_EX(to, TYamlConfigEx() << "Overriding empty value: " - << to.Path() << " <- " << from.Path()); - Y_ENSURE_EX(to.Type() == from.Type(), TYamlConfigEx() << "Overriding value with different types: " - << to.Path() << " <- " << from.Path()); - - switch (from.Type()) { - case NFyaml::ENodeType::Mapping: - { - auto toMap = to.Map(); - auto fromMap = from.Map(); - Inherit(toMap, fromMap); - } - break; - case NFyaml::ENodeType::Sequence: - { - auto tag = from.Tag(); - auto key = tag->substr(inheritSeqTag.length()); - auto toSeq = to.Sequence(); - auto fromSeq = from.Sequence(); - Inherit(toSeq, fromSeq, key); - } - break; - case NFyaml::ENodeType::Scalar: - { - ythrow TYamlConfigEx() << "Override with scalar: " - << to.Path() << " <- " << from.Path(); - } - break; - } -} - -void RemoveTags(NFyaml::TDocument& doc) { - for (auto it = doc.begin(); it != doc.end(); ++it) { - it->RemoveTag(); - } -} - -TDocumentConfig Resolve( - const NFyaml::TDocument& doc, - const TSet<TNamedLabel>& labels) -{ - TDocumentConfig res{doc.Clone(), NFyaml::TNodeRef{}}; - res.first.Resolve(); - - auto rootMap = res.first.Root().Map(); - auto config = rootMap.at("config"); - auto selectorConfig = rootMap.at("selector_config").Sequence(); - - for (auto it = selectorConfig.begin(); it != selectorConfig.end(); ++it) { - auto selectorMap = it->Map(); - auto desc = selectorMap.at("description").Scalar(); - auto selectorNode = selectorMap.at("selector"); - auto selector = ParseSelector(selectorNode); - if (Fit(selector, labels)) { - Apply(config, selectorMap.at("config")); - } - } - - RemoveTags(res.first); - - res.second = config; - - return res; -} - -void Combine( - TVector<TVector<TLabel>>& labelCombinations, - TVector<TLabel>& combination, - const TVector<std::pair<TString, TSet<TLabel>>>& labels, - size_t offset) -{ - if (offset == labels.size()) { - labelCombinations.push_back(combination); - return; - } - - for (auto& label : labels[offset].second) { - combination[offset] = label; - Combine(labelCombinations, combination, labels, offset + 1); - } -} - -bool Fit( - const TSelector& selector, - const TVector<TLabel>& labels, - const TVector<std::pair<TString, TSet<TLabel>>>& names) -{ - for (size_t i = 0; i < labels.size(); ++ i) { - auto& label = labels[i]; - auto& name = names[i].first; - switch(label.Type) { - case TLabel::EType::Negative: - if (selector.In.contains(name)) { - return false; - } - break; - case TLabel::EType::Empty: [[fallthrough]]; - case TLabel::EType::Common: - if (auto it = selector.In.find(name); it != selector.In.end() - && !it->second.Values.contains(label.Value)) { - - return false; - } - if (auto it = selector.NotIn.find(name); it != selector.NotIn.end() - && it->second.Values.contains(label.Value)) { - - return false; - } - break; - } - } - - return true; -} - NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown, bool preTransform) { TStringStream sstr; @@ -457,195 +37,6 @@ NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUn return yamlProtoConfig; } -TResolvedConfig ResolveAll(NFyaml::TDocument& doc) -{ - TVector<TString> labelNames; - TVector<std::pair<TString, TSet<TLabel>>> labels; - - auto config = ParseConfig(doc); - auto namedLabels = CollectLabels(doc); - - for (auto& [name, values]: namedLabels) { - TSet<TLabel> set; - if (values.Class == EYamlConfigLabelTypeClass::Open) { - set.insert(TLabel{TLabel::EType::Negative, {}}); - } - for (auto& value: values.Values) { - if (value != "") { - set.insert(TLabel{TLabel::EType::Common, value}); - } else { - set.insert(TLabel{TLabel::EType::Empty, {}}); - } - } - labels.push_back({name, set}); - labelNames.push_back(name); - } - - TVector<TVector<TLabel>> labelCombinations; - - TVector<TLabel> combination; - combination.resize(labels.size()); - - Combine(labelCombinations, combination, labels, 0); - - auto cmp = [](const TVector<int>& lhs, const TVector<int>& rhs) { - auto lhsIt = lhs.begin(); - auto rhsIt = rhs.begin(); - - while (lhsIt != lhs.end() && rhsIt != rhs.end() && (*lhsIt == *rhsIt)) { - lhsIt++; - rhsIt++; - } - - if (lhsIt == lhs.end()) { - return false; - } else if (rhsIt == rhs.end()) { - return true; - } - - return *lhsIt < *rhsIt; - }; - - using TTriePath = TVector<int>; - - struct TTrieNode { - TSimpleSharedPtr<TDocumentConfig> ResolvedConfig; - TVector<TVector<TLabel>> LabelCombinations; - }; - - std::map<TTriePath, TSimpleSharedPtr<TDocumentConfig>, decltype(cmp)> selectorsTrie(cmp); - std::map<TTriePath, TTrieNode, decltype(cmp)> appliedSelectors(cmp); - - auto rootConfig = TTrieNode { - MakeSimpleShared<TDocumentConfig>(std::move(doc), config.Config), - {}, - }; - - selectorsTrie[{0}] = rootConfig.ResolvedConfig; - - for (size_t j = 0; j < labelCombinations.size(); ++j) { - TSimpleSharedPtr<TDocumentConfig> cur = rootConfig.ResolvedConfig; - TTriePath triePath({0}); - - for (size_t i = 0; i < config.Selectors.size(); ++i) { - if (Fit(config.Selectors[i].Selector, labelCombinations[j], labels)) { - triePath.push_back(i + 1); - if (auto it = selectorsTrie.find(triePath); it != selectorsTrie.end()) { - cur = it->second; - } else { - auto clone = cur->first.Clone(); - auto cloneConfig = ParseConfig(clone); - - Apply(cloneConfig.Config, cloneConfig.Selectors[i].Config); - - cur = MakeSimpleShared<std::pair<NFyaml::TDocument, NFyaml::TNodeRef>>( - std::move(clone), - cloneConfig.Config), - selectorsTrie[triePath] = cur; - } - } - } - - if (auto it = appliedSelectors.find(triePath); it != appliedSelectors.end()) { - it->second.LabelCombinations.push_back(labelCombinations[j]); - } else { - appliedSelectors.try_emplace(triePath, TTrieNode{ - cur, - {labelCombinations[j]} - }); - } - } - - selectorsTrie.clear(); - - TMap<TSet<TVector<TLabel>>, TDocumentConfig> configs; - - for (auto& [_, value]: appliedSelectors) { - configs.try_emplace( - TSet<TVector<TLabel>>( - value.LabelCombinations.begin(), - value.LabelCombinations.end()), - std::make_pair(std::move(value.ResolvedConfig->first), value.ResolvedConfig->second)); - } - - return {labelNames, std::move(configs)}; -} - -size_t Hash(const NFyaml::TNodeRef& resolved) { - TStringStream ss; - ss << resolved; - TString s = ss.Str(); - return THash<TString>{}(s); -} - -size_t Hash(const TResolvedConfig& config) -{ - size_t configsHash = 0; - for (auto& [labelSet, docConfig] : config.Configs) { - for (auto labels : labelSet) { - auto labelsHash = THash<TVector<TLabel>>{}(labels); - configsHash = CombineHashes(labelsHash, configsHash); - } - configsHash = CombineHashes(Hash(docConfig.second), configsHash); - } - - return CombineHashes(THash<TVector<TString>>{}(config.Labels), configsHash); -} - -void ValidateVolatileConfig(NFyaml::TDocument& doc) { - auto root = doc.Root(); - auto seq = root.Map().at("selector_config").Sequence(); - if (seq.size() == 0) { - ythrow yexception() << "Empty volatile config"; - } - for (auto& elem : seq) { - auto map = elem.Map(); - if (map.size() != 3) { - ythrow yexception() << "Invalid volatile config element: " << elem.Path(); - } - for (auto& mapElem : map) { - auto key = mapElem.Key().Scalar(); - if (key == "description") { - mapElem.Value().Scalar(); - } else if (key == "selector") { - mapElem.Value().Map(); - } else if (key == "config") { - mapElem.Value().Map(); - } else { - ythrow yexception() << "Unknown element in volatile config: " << elem.Path(); - } - } - } -} - -void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig) { - auto configRoot = config.Root(); - auto volatileConfigRoot = volatileConfig.Root(); - - auto seq = volatileConfigRoot.Sequence(); - auto selectors = configRoot.Map().at("selector_config").Sequence(); - for (auto& elem : seq) { - auto node = elem.Copy(config); - selectors.Append(node.Ref()); - } -} - -void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig) { - auto configRoot = config.Root(); - - auto seq = volatileConfig.Sequence(); - auto selectors = configRoot.Map().at("selector_config").Sequence(); - for (auto& elem : seq) { - auto node = elem.Copy(config); - selectors.Append(node.Ref()); - } -} - -ui64 GetVersion(const TString& config) { - auto metadata = GetMetadata(config); - return metadata.Version.value_or(0); -} - /** * Config used to convert protobuf from/to json * changes how names are translated e.g. PDiskInfo -> pdisk_info instead of p_disk_info @@ -717,146 +108,4 @@ void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig: } } -TMetadata GetMetadata(const TString& config) { - if (config.empty()) { - return {}; - } - - auto doc = NFyaml::TDocument::Parse(config); - - if (auto node = doc.Root().Map()["metadata"]; node) { - auto versionNode = node.Map()["version"]; - auto clusterNode = node.Map()["cluster"]; - return TMetadata{ - .Version = versionNode ? std::optional{FromString<ui64>(versionNode.Scalar())} : std::nullopt, - .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, - }; - } - - return {}; -} - -TVolatileMetadata GetVolatileMetadata(const TString& config) { - if (config.empty()) { - return {}; - } - - auto doc = NFyaml::TDocument::Parse(config); - - if (auto node = doc.Root().Map().at("metadata"); node) { - auto versionNode = node.Map().at("version"); - auto clusterNode = node.Map().at("cluster"); - auto idNode = node.Map().at("id"); - return TVolatileMetadata{ - .Version = versionNode ? std::make_optional(FromString<ui64>(versionNode.Scalar())) : std::nullopt, - .Cluster = clusterNode ? std::make_optional(clusterNode.Scalar()) : std::nullopt, - .Id = idNode ? std::make_optional(FromString<ui64>(idNode.Scalar())) : std::nullopt, - }; - } - - return {}; -} - -TString ReplaceMetadata(const TString& config, const std::function<void(TStringStream&)>& serializeMetadata) { - TStringStream sstr; - auto doc = NFyaml::TDocument::Parse(config); - if (doc.Root().Style() == NFyaml::ENodeStyle::Flow) { - if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { - doc.Root().Map().Remove(pair); - } - serializeMetadata(sstr); - sstr << "\n" << doc; - } else { - if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { - auto begin = pair.Key().BeginMark().InputPos; - auto end = pair.Value().EndMark().InputPos; - sstr << config.substr(0, begin); - serializeMetadata(sstr); - if (end < config.length() && config[end] == ':') { - end = end + 1; - } - sstr << config.substr(end, TString::npos); - } else { - if (doc.HasExplicitDocumentStart()) { - auto docStart = doc.BeginMark().InputPos + 4; - sstr << config.substr(0, docStart); - serializeMetadata(sstr); - sstr << "\n" << config.substr(docStart, TString::npos); - } else { - serializeMetadata(sstr); - sstr << "\n" << config; - } - } - } - return sstr.Str(); - -} - -TString ReplaceMetadata(const TString& config, const TMetadata& metadata) { - auto serializeMetadata = [&](TStringStream& sstr) { - sstr - << "metadata:" - << "\n kind: MainConfig" - << "\n cluster: \"" << *metadata.Cluster << "\"" - << "\n version: " << *metadata.Version; - }; - return ReplaceMetadata(config, serializeMetadata); -} - -TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata) { - auto serializeMetadata = [&](TStringStream& sstr) { - sstr - << "metadata:" - << "\n kind: VolatileConfig" - << "\n cluster: \"" << *metadata.Cluster << "\"" - << "\n version: " << *metadata.Version - << "\n id: " << *metadata.Id; - }; - return ReplaceMetadata(config, serializeMetadata); -} - -bool IsConfigKindEquals(const TString& config, const TString& kind) { - try { - auto doc = NFyaml::TDocument::Parse(config); - return doc.Root().Map().at("metadata").Map().at("kind").Scalar() == kind; - } catch (yexception& e) { - return false; - } -} - -bool IsVolatileConfig(const TString& config) { - return IsConfigKindEquals(config, "VolatileConfig"); -} - -bool IsMainConfig(const TString& config) { - return IsConfigKindEquals(config, "MainConfig"); -} - -TString StripMetadata(const TString& config) { - auto doc = NFyaml::TDocument::Parse(config); - - TStringStream sstr; - if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { - auto begin = pair.Key().BeginMark().InputPos; - sstr << config.substr(0, begin); - auto end = pair.Value().EndMark().InputPos; - sstr << config.substr(end, TString::npos); - } else { - if (doc.HasExplicitDocumentStart()) { - auto docStart = doc.BeginMark().InputPos + 4; - sstr << config.substr(0, docStart); - sstr << "\n" << config.substr(docStart, TString::npos); - } else { - sstr << config; - } - } - - return sstr.Str(); -} - } // namespace NYamlConfig - -template <> -void Out<NYamlConfig::TLabel>(IOutputStream& out, const NYamlConfig::TLabel& value) { - out << (int)value.Type << ":" << value.Value; -} diff --git a/ydb/library/yaml_config/yaml_config.h b/ydb/library/yaml_config/yaml_config.h index be34e712c9..6a5ddd4ba3 100644 --- a/ydb/library/yaml_config/yaml_config.h +++ b/ydb/library/yaml_config/yaml_config.h @@ -5,6 +5,7 @@ #include <ydb/core/protos/config.pb.h> #include <ydb/core/protos/console_config.pb.h> +#include <ydb/library/yaml_config/public/yaml_config.h> #include <openssl/sha.h> @@ -20,166 +21,12 @@ namespace NYamlConfig { -struct TYamlConfigEx : public yexception {}; - -using TDocumentConfig = std::pair<NFyaml::TDocument, NFyaml::TNodeRef>; - -/** - * Open - labels like tenant, where we don't know final set of values - * Closed - labels with predefined set of values, e.g. size - */ -enum class EYamlConfigLabelTypeClass { - Open, - Closed, -}; - -/** - * TNamedLabel - representation of label used for selector - */ -class TNamedLabel { -public: - TString Name; - TString Value; - bool Inv = false; - - bool operator<(const TNamedLabel& other) const { return Name < other.Name; } -}; - -/** - * TLabelType - represents known set of values for a label with its type - */ -class TLabelType { -public: - EYamlConfigLabelTypeClass Class; - TSet<TString> Values; - - bool operator==(const TLabelType& other) const { return Values == other.Values; } -}; - -class TLabelValueSet { -public: - TSet<TString> Values; - - bool operator==(const TLabelValueSet& other) const { return Values == other.Values; } -}; - -class TSelector { -public: - TMap<TString, TLabelValueSet> In; - TMap<TString, TLabelValueSet> NotIn; -}; - -struct TYamlConfigModel { - struct TSelectorModel { - TString Description; - TSelector Selector; - NFyaml::TNodeRef Config; - }; - - const NFyaml::TDocument& Doc; - NFyaml::TNodeRef Config; - TMap<TString, TLabelType> AllowedLabels; - TVector<TSelectorModel> Selectors; -}; - -/** - * Collects all labels present in document - * For Open labels gathers all labels from all selectors - * For Closed labels additionally validates that there is no additional labels - */ -TMap<TString, TLabelType> CollectLabels(NFyaml::TDocument& doc); - -/** - * Parses config and fills corresponding struct - */ -TYamlConfigModel ParseConfig(NFyaml::TDocument& doc); - -/** - * Generates config for specific set of labels applying all matching selectors - */ -TDocumentConfig Resolve( - const NFyaml::TDocument& doc, - const TSet<TNamedLabel>& labels); - /** * Converts YAML representation to ProtoBuf */ NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUnknown = false, bool preTransform = true); /** - * TLabel is a representation of label for config resolution - * - * It can be in three states: - * - Empty with Type == EType::Empty and Value == "" - * it equals empty or undefined label - * - Common with Type == EType::Common and Value == arbitrary string - * it equals defined label with corresponding value - * - Negative with Type == EType::Negative and Value == "" - * it is used for Open labels and equals any label not present in labels - * discovered by CollectLabels (and also not equals Empty label) - */ -struct TLabel { - enum class EType { - Negative = 0, - Empty, - Common, - }; - - EType Type; - TString Value; - - bool operator<(const TLabel& other) const { - int lhs = static_cast<int>(Type); - int rhs = static_cast<int>(other.Type); - return std::tie(lhs, Value) < std::tie(rhs, other.Value); - } - - bool operator==(const TLabel& other) const { - int lhs = static_cast<int>(Type); - int rhs = static_cast<int>(other.Type); - return std::tie(lhs, Value) == std::tie(rhs, other.Value); - } -}; - -struct TResolvedConfig { - TVector<TString> Labels; - TMap<TSet<TVector<TLabel>>, TDocumentConfig> Configs; -}; - -/** - * Generates configs for all label combinations - */ -TResolvedConfig ResolveAll(NFyaml::TDocument& doc); - -/** - * Calculates hash of resolved config - * Used to ensure that cli resolves config the same as a server - */ -size_t Hash(const TResolvedConfig& config); - -/** - * Validates single YAML volatile config schema - */ -void ValidateVolatileConfig(NFyaml::TDocument& doc); - -/** - * Appends volatile configs to the end of selectors list - * **Important**: Document should be a list with selectors - */ -void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig); - -/** - * Appends volatile configs to the end of selectors list - * **Important**: Node should be a list with selectors - */ -void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig); - -/** - * Parses config version - */ -ui64 GetVersion(const TString& config); - -/** * Resolves config for given labels and stores result to appConfig * Stores intermediate resolve data in resolvedYamlConfig and resolvedJsonConfig if given */ @@ -197,56 +44,4 @@ void ResolveAndParseYamlConfig( */ void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to); -/** - * Represents config metadata - */ -struct TMetadata { - std::optional<ui64> Version; - std::optional<TString> Cluster; -}; - -/** - * Parses config metadata - */ -TMetadata GetMetadata(const TString& config); - -/** - * Represents volatile config metadata - */ -struct TVolatileMetadata { - std::optional<ui64> Version; - std::optional<TString> Cluster; - std::optional<ui64> Id; -}; - -/** - * Parses volatile config metadata - */ -TVolatileMetadata GetVolatileMetadata(const TString& config); - -/** - * Replaces metadata in config - */ -TString ReplaceMetadata(const TString& config, const TMetadata& metadata); - -/** - * Replaces volatile metadata in config - */ -TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata); - -/** - * Checks whether string is volatile config or not - */ -bool IsVolatileConfig(const TString& config); - -/** - * Checks whether string is main config or not - */ -bool IsMainConfig(const TString& config); - -/** - * Strips metadata from config - */ -TString StripMetadata(const TString& config); - } // namespace NYamlConfig |