diff options
author | innokentii <innokentii@yandex-team.com> | 2023-04-14 17:49:13 +0300 |
---|---|---|
committer | innokentii <innokentii@yandex-team.com> | 2023-04-14 17:49:13 +0300 |
commit | 23f5096a596f04f844880e32fa6c3c7bffc48d9a (patch) | |
tree | 83394c98004dfe56eb79c7bd460c4ce7ec8bb826 | |
parent | 89641a451ba880b8b290b9e8dc8c00d236620eb3 (diff) | |
download | ydb-23f5096a596f04f844880e32fa6c3c7bffc48d9a.tar.gz |
Add yaml-dump command
add console dumper
33 files changed, 587 insertions, 22 deletions
diff --git a/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt index 83483cc6f6d..0a471cee3fe 100644 --- a/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -37,7 +38,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp diff --git a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt index 3e67e94ecf9..5036d73fd19 100644 --- a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -38,7 +39,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp diff --git a/ydb/core/cms/console/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/CMakeLists.linux-x86_64.txt index 3e67e94ecf9..5036d73fd19 100644 --- a/ydb/core/cms/console/CMakeLists.linux-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.linux-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -38,7 +39,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp diff --git a/ydb/core/cms/console/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/CMakeLists.windows-x86_64.txt index 83483cc6f6d..0a471cee3fe 100644 --- a/ydb/core/cms/console/CMakeLists.windows-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.windows-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -37,7 +38,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 324b4c44569..2a0078c059f 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -1,11 +1,11 @@ #include "config_helpers.h" -#include "config_index.h" #include "configs_dispatcher.h" #include "console_configs_subscriber.h" #include "console.h" #include "http.h" #include "util.h" +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/cms/console/yaml_config/yaml_config.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/mon/mon.h> diff --git a/ydb/core/cms/console/console__cleanup_subscriptions.cpp b/ydb/core/cms/console/console__cleanup_subscriptions.cpp index 4a44271a9ff..f56c0a74f2c 100644 --- a/ydb/core/cms/console/console__cleanup_subscriptions.cpp +++ b/ydb/core/cms/console/console__cleanup_subscriptions.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/cms/console/validators/registry.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console__set_config.cpp b/ydb/core/cms/console/console__set_config.cpp index cd062e1cff2..77604fa38dd 100644 --- a/ydb/core/cms/console/console__set_config.cpp +++ b/ydb/core/cms/console/console__set_config.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console__toggle_config_validator.cpp b/ydb/core/cms/console/console__toggle_config_validator.cpp index fb2dfb20e70..8731b6806ce 100644 --- a/ydb/core/cms/console/console__toggle_config_validator.cpp +++ b/ydb/core/cms/console/console__toggle_config_validator.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/cms/console/validators/registry.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index 3ccf7586fa9..09233dfd7ff 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -2,7 +2,6 @@ #include "defs.h" -#include "config_index.h" #include "configs_config.h" #include "console.h" #include "logger.h" @@ -11,6 +10,7 @@ #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <library/cpp/actors/core/hfunc.h> diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index f19afcac05a..e247ef4b90d 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -3,10 +3,10 @@ #include "defs.h" #include "configs_config.h" -#include "config_index.h" #include "console.h" #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <library/cpp/actors/core/hfunc.h> diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index 3602df18534..635eca31b12 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -1,10 +1,10 @@ #include "console_configs_subscriber.h" #include "console.h" -#include "config_index.h" #include "util.h" #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/mind/tenant_pool.h> #include <library/cpp/actors/core/actor_bootstrapped.h> diff --git a/ydb/core/cms/console/modifications_validator.h b/ydb/core/cms/console/modifications_validator.h index 17a53eaee79..a9486a9950d 100644 --- a/ydb/core/cms/console/modifications_validator.h +++ b/ydb/core/cms/console/modifications_validator.h @@ -1,8 +1,9 @@ #pragma once -#include "config_index.h" #include "configs_config.h" +#include <ydb/core/cms/console/util/config_index.h> + namespace NKikimr::NConsole { class TModificationsValidatorTests; diff --git a/ydb/core/cms/console/ut_helpers.h b/ydb/core/cms/console/ut_helpers.h index 617aa2dfe9f..23077863cea 100644 --- a/ydb/core/cms/console/ut_helpers.h +++ b/ydb/core/cms/console/ut_helpers.h @@ -1,11 +1,11 @@ #pragma once #include "config_helpers.h" -#include "config_index.h" #include "console_configs_provider.h" #include "console_impl.h" #include "console_tenants_manager.h" +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/testlib/tenant_runtime.h> #include <ydb/core/testlib/tenant_helpers.h> diff --git a/ydb/core/cms/console/util/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/util/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..2a8aa60113a --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..4793dbe9ea1 --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/util/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/util/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..4793dbe9ea1 --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.linux-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/util/CMakeLists.txt b/ydb/core/cms/console/util/CMakeLists.txt new file mode 100644 index 00000000000..a692f90f36e --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/cms/console/util/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/util/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..2a8aa60113a --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/config_index.cpp b/ydb/core/cms/console/util/config_index.cpp index c338689f773..c338689f773 100644 --- a/ydb/core/cms/console/config_index.cpp +++ b/ydb/core/cms/console/util/config_index.cpp diff --git a/ydb/core/cms/console/config_index.h b/ydb/core/cms/console/util/config_index.h index 063a3313ec9..6af912e0ef7 100644 --- a/ydb/core/cms/console/config_index.h +++ b/ydb/core/cms/console/util/config_index.h @@ -156,6 +156,8 @@ struct TUsageScope { // 1 - lhs is more prioritized static int ComparePriority(const TUsageScope &lhs, const TUsageScope &rhs); + + bool operator<(const TUsageScope& other) const { return ComparePriority(*this, other) < 0; }; }; /** diff --git a/ydb/core/cms/console/util/defs.h b/ydb/core/cms/console/util/defs.h new file mode 100644 index 00000000000..9e2937dac77 --- /dev/null +++ b/ydb/core/cms/console/util/defs.h @@ -0,0 +1,5 @@ +#pragma once +// unique tag: ./ydb/core/cms/console/util/defs.h +#include <ydb/core/base/defs.h> +#include <ydb/core/base/events.h> +#include <ydb/core/util/yverify_stream.h> diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt index 4d09e79f3ac..33e4b61983a 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.darwin-x86_64.txt @@ -13,12 +13,14 @@ add_library(cms-console-yaml_config) target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt index b78dac11e71..ad0cd42cc8b 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt @@ -14,12 +14,14 @@ target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt index b78dac11e71..ad0cd42cc8b 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-x86_64.txt @@ -14,12 +14,14 @@ target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt index 4d09e79f3ac..33e4b61983a 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.windows-x86_64.txt @@ -13,12 +13,14 @@ add_library(cms-console-yaml_config) target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/console_dumper.cpp b/ydb/core/cms/console/yaml_config/console_dumper.cpp new file mode 100644 index 00000000000..313b6179d8f --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper.cpp @@ -0,0 +1,372 @@ +#include "console_dumper.h" + +#include <ydb/core/cms/console/yaml_config/yaml_config.h> +#include <ydb/core/cms/console/util/config_index.h> +#include <library/cpp/protobuf/json/proto2json.h> +#include <library/cpp/yaml/fyamlcpp/fyamlcpp.h> + +namespace NYamlConfig { + +using namespace NKikimr; + +using TUsageScope = NConsole::TUsageScope; + +using TId = ui64; +using TGeneration = ui64; +using TDomainKey = std::tuple<ui32, TId, TGeneration>; +using TDomainItemsContainer = TMap<TDomainKey, NKikimrConsole::TConfigItem>; + +// Use config hash to clue items with same body +using TConfigHash = ui64; +using TSelectorKey = std::tuple<TUsageScope, TConfigHash>; +using TSelectorItemsContainer = TMap<TSelectorKey, TVector<NKikimrConsole::TConfigItem>>; + +struct TSelectorData { + ui32 MergeStrategy; + NYamlConfig::TSelector Rules; + NKikimrConfig::TAppConfig Config; + TString Description; +}; + +void MarkYamlForMergeOverwriteRepeated(NFyaml::TNodeRef &node) { + auto rootMap = node.Map(); + for (auto &child : rootMap) { + auto value = child.Value(); + if (value.Type() == NFyaml::ENodeType::Mapping) { + value.SetTag("!inherit"); + MarkYamlForMergeOverwriteRepeated(value); + } + } +} + +void MarkYamlForMerge(NFyaml::TNodeRef &node) { + auto rootMap = node.Map(); + for (auto &child : rootMap) { + auto value = child.Value(); + if (value.Type() == NFyaml::ENodeType::Mapping) { + value.SetTag("!inherit"); + MarkYamlForMerge(value); + } else if (value.Type() == NFyaml::ENodeType::Sequence) { + value.SetTag("!append"); + } + } +} + +void Beautify(NFyaml::TDocument &doc) { + for (auto &node : doc) { + if (node.Style() == NFyaml::ENodeStyle::DoubleQuoted) { + node.SetStyle(NFyaml::ENodeStyle::Any); + } + } +} + +void ClearOverwrittenRepeated(::google::protobuf::Message &to, + const ::google::protobuf::Message &from) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + if (field->is_repeated()) { + if (reflection->FieldSize(from, field)) { + reflection->ClearField(&to, field); + } + } else if (field->type() == ::google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + if (reflection->HasField(to, field) && reflection->HasField(from, field)) { + ClearOverwrittenRepeated(*reflection->MutableMessage(&to, field), + reflection->GetMessage(from, field)); + } + } + } +} + +void MergeMessageOverwriteRepeated(::google::protobuf::Message &to, + const ::google::protobuf::Message &from, + ui32 kind) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + auto tag = field->number(); + + if (tag != (int)kind) { + continue; + } + + auto &toMsg = *reflection->MutableMessage(&to, field); + auto &fromMsg = reflection->GetMessage(from, field); + + + ClearOverwrittenRepeated(toMsg, fromMsg); + toMsg.MergeFrom(fromMsg); + } +} + +void CopyFrom(::google::protobuf::Message &to, + const ::google::protobuf::Message &from, + ui32 kind) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + auto tag = field->number(); + + if (tag != (int)kind) { + continue; + } + + reflection->MutableMessage(&to, field)->CopyFrom(reflection->GetMessage(from, field)); + } +} + +std::pair<TDomainItemsContainer, TSelectorItemsContainer> ExtractSuitableItems( + const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems) { + + TDomainItemsContainer domainItemsByOrder; + TSelectorItemsContainer selectorItemsByOrder; + + for (auto &item : configItems) { + if (item.GetKind() == NKikimrConsole::TConfigItem::NameserviceConfigItem || + item.GetKind() == NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem || + item.GetKind() == NKikimrConsole::TConfigItem::NamedConfigsItem || + item.GetCookie().StartsWith("ydbcp")) { + continue; + } + + if (item.GetUsageScope().GetFilterCase() == NKikimrConsole::TUsageScope::FILTER_NOT_SET) { + domainItemsByOrder.emplace( + std::tuple<ui32, ui64, ui64>{item.GetOrder(), item.GetId().GetId(), item.GetId().GetGeneration()}, + item); + } else { + TUsageScope scope(item.GetUsageScope(), item.GetOrder()); + TSelectorKey key{scope, THash<TString>{}(item.GetConfig().ShortDebugString())}; + if (auto it = selectorItemsByOrder.find(key); it != selectorItemsByOrder.end()) { + Y_VERIFY(it->second.back().GetMergeStrategy() == item.GetMergeStrategy()); + it->second.emplace_back(item); + } else { + selectorItemsByOrder.emplace(key, TVector<NKikimrConsole::TConfigItem>{item}); + } + } + } + + return {domainItemsByOrder, selectorItemsByOrder}; +} + +NKikimrConfig::TAppConfig BundleDomainConfig(const TDomainItemsContainer &items) { + NKikimrConfig::TAppConfig config; + + for (auto &[_, item] : items) { + if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::MERGE) { + config.MergeFrom(item.GetConfig()); + } else if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::OVERWRITE) { + CopyFrom(config, item.GetConfig(), item.GetKind()); + } else if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::MERGE_OVERWRITE_REPEATED) { + MergeMessageOverwriteRepeated(config, item.GetConfig(), item.GetKind()); + } + } + + return config; +} + +NProtobufJson::TProto2JsonConfig GetProto2JsonConfig() { + return NProtobufJson::TProto2JsonConfig() + .SetFormatOutput(false) + .SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName) + .SetFieldNameMode(NProtobufJson::TProto2JsonConfig::FieldNameSnakeCaseDense) + .SetStringifyNumbers(NProtobufJson::TProto2JsonConfig::StringifyLongNumbersForDouble); +} + +TVector<TSelectorData> FillSelectorsData(const TSelectorItemsContainer &items) { + TVector<TSelectorData> selectors; + + for (auto &[tuple, items] : items) { + auto &item = items.back(); + + NYamlConfig::TSelector rules; + + auto &scope = item.GetUsageScope(); + switch (scope.GetFilterCase()) { + case NKikimrConsole::TUsageScope::kNodeFilter: + { + TSet<TString> nodeIds; + for (auto &it : items) { + for (auto &id : it.GetUsageScope().GetNodeFilter().GetNodes()) { + nodeIds.insert(ToString(id)); + } + } + + rules.In.emplace("node_id", NYamlConfig::TLabelValueSet{nodeIds}); + } + break; + case NKikimrConsole::TUsageScope::kHostFilter: + { + TSet<TString> hosts; + for (auto &it : items) { + for (auto &host : it.GetUsageScope().GetHostFilter().GetHosts()) { + hosts.insert(host); + } + } + + rules.In.emplace("host", NYamlConfig::TLabelValueSet{hosts}); + } + break; + case NKikimrConsole::TUsageScope::kTenantAndNodeTypeFilter: + if (!scope.GetTenantAndNodeTypeFilter().GetTenant().empty() && + !scope.GetTenantAndNodeTypeFilter().GetNodeType().empty()) { + for (auto &it : items) { + rules.In.emplace( + "tenant", + NYamlConfig::TLabelValueSet{ + TSet<TString>{it.GetUsageScope().GetTenantAndNodeTypeFilter().GetTenant()}}); + rules.In.emplace( + "node_type", + NYamlConfig::TLabelValueSet{ + TSet<TString>{it.GetUsageScope().GetTenantAndNodeTypeFilter().GetNodeType()}}); + + TStringStream desc; + desc << "cookie=" << item.GetCookie() + << " merge_strategy=" << NConsole::TConfigItem::MergeStrategyName(item.GetMergeStrategy()) + << " id=" << it.GetId().GetId() << "." << it.GetId().GetGeneration();; + + selectors.emplace_back(TSelectorData{ + item.GetMergeStrategy(), + rules, + item.GetConfig(), + desc.Str()}); + rules.In.clear(); + } + + break; + } + if (!scope.GetTenantAndNodeTypeFilter().GetTenant().empty()) { + TSet<TString> tenants; + for (auto &it : items) { + tenants.insert(it.GetUsageScope().GetTenantAndNodeTypeFilter().GetTenant()); + } + + rules.In.emplace("tenant", NYamlConfig::TLabelValueSet{tenants}); + } + if (!scope.GetTenantAndNodeTypeFilter().GetNodeType().empty()) { + TSet<TString> nodeTypes; + for (auto &it : items) { + nodeTypes.insert(it.GetUsageScope().GetTenantAndNodeTypeFilter().GetNodeType()); + } + + rules.In.emplace("node_type", NYamlConfig::TLabelValueSet{nodeTypes}); + } + break; + default: break; + } + if (!rules.In.empty()) { + TStringStream desc; + desc << "cookie=" << item.GetCookie() + << " merge_strategy=" << NConsole::TConfigItem::MergeStrategyName(item.GetMergeStrategy()) + << " id="; + bool first = true; + for (auto &it : items) { + desc << (first ? "" : ",") << it.GetId().GetId() << "." << it.GetId().GetGeneration(); + first = false; + } + + selectors.emplace_back(TSelectorData{ + item.GetMergeStrategy(), + rules, + item.GetConfig(), + desc.Str()}); + } + } + + return selectors; +} + +void SerializeSelectorsToYaml( + const TVector<TSelectorData> &selectors, + NFyaml::TDocument &doc, + NFyaml::TSequence &seq) { + + const TString selectorTemplate = R"( +description: "" +selector: {} +config: {} +)"; + + auto selectorTemplateYaml = NFyaml::TDocument::Parse(selectorTemplate); + auto selectorConfigRoot = selectorTemplateYaml.Root(); + + for (auto &selector : selectors) { + auto config = NFyaml::TDocument::Parse(NProtobufJson::Proto2Json(selector.Config, GetProto2JsonConfig())); + auto configNode = config.Root().Copy(doc); + auto configNodeRef = configNode.Ref(); + + switch (selector.MergeStrategy) { + case NKikimrConsole::TConfigItem::MERGE_OVERWRITE_REPEATED: + MarkYamlForMergeOverwriteRepeated(configNodeRef); + break; + case NKikimrConsole::TConfigItem::MERGE: + MarkYamlForMerge(configNodeRef); + break; + default: break; + } + + auto node = selectorConfigRoot.Copy(doc); + seq.Append(node.Ref()); + node.Ref().Map().pair_at("config").SetValue(configNode.Ref()); + + node.Ref().Map().pair_at("description").SetValue(doc.Buildf("%s", selector.Description.c_str())); + + auto selectorNode = node.Ref().Map().at("selector").Map(); + for (auto &[label, values] : selector.Rules.In) { + if (values.Values.size() == 1) { + auto labelNode = doc.Buildf("%s", label.c_str()); + auto valueNode = doc.Buildf("%s", values.Values.begin()->c_str()); + selectorNode.Append(labelNode, valueNode); + } else { + auto labelNode = doc.Buildf("%s", label.c_str()); + auto inNode = doc.Buildf("{ in: [] }"); + auto inSeq = inNode.Map().at("in").Sequence(); + for (auto &value : values.Values) { + auto valueNode = doc.Buildf("%s", value.c_str()); + inSeq.Append(valueNode); + } + selectorNode.Append(labelNode, inNode); + } + } + } +} + +TString DumpConsoleConfigs(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems) { + const auto [domainItemsByOrder, selectorItemsByOrder] = ExtractSuitableItems(configItems); + + const NKikimrConfig::TAppConfig configProto = BundleDomainConfig(domainItemsByOrder); + auto mainConfigYaml = NFyaml::TDocument::Parse( + NProtobufJson::Proto2Json(configProto, GetProto2JsonConfig())); + + const TString configTemplate = R"( +config: {} + +allowed_labels: + node_id: {type: string} + host: {type: string} + tenant: {type: string} + +selector_config: [] +)"; + + auto outDoc = NFyaml::TDocument::Parse(configTemplate); + + auto configTemplateConfig = outDoc.Root().Map().pair_at("config"); + auto mainConfigRoot = mainConfigYaml.Root().Copy(outDoc); + configTemplateConfig.SetValue(mainConfigRoot.Ref()); + + const auto selectors = FillSelectorsData(selectorItemsByOrder); + auto selectorsSeq = outDoc.Root().Map().at("selector_config").Sequence(); + SerializeSelectorsToYaml(selectors, outDoc, selectorsSeq); + + Beautify(outDoc); + + TStringStream res; + res << outDoc; + + return res.Str(); +} + +} // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/console_dumper.h b/ydb/core/cms/console/yaml_config/console_dumper.h new file mode 100644 index 00000000000..599a022a28b --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper.h @@ -0,0 +1,11 @@ +#pragma once + +#include <ydb/core/protos/console_config.pb.h> + +#include <util/generic/string.h> + +namespace NYamlConfig { + +TString DumpConsoleConfigs(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems); + +} // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp b/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp new file mode 100644 index 00000000000..ee9d6da028b --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp @@ -0,0 +1,33 @@ +#include "console_dumper.h" + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(ConsoleDumper) { + Y_UNIT_TEST(Basic) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem &configItem = *items.Add(); + configItem.SetMergeStrategy(2); + configItem.SetOrder(21); + configItem.SetCookie("test"); + auto &entry = *configItem.MutableConfig()->MutableLogConfig()->AddEntry(); + entry.SetComponent("BG_TASKS"); + entry.SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } +} diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin-x86_64.txt index 2d7066199bc..deeb889bb78 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin-x86_64.txt @@ -26,6 +26,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE CoreFoundation ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) set_property( diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt index 3194ef14402..867c9020e61 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt @@ -29,6 +29,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE -ldl ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) set_property( diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-x86_64.txt index 18808a169e9..91bbfb58c5c 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-x86_64.txt @@ -30,6 +30,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE -ldl ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) set_property( diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.windows-x86_64.txt index e6160f26e75..1b729c84678 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.windows-x86_64.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.windows-x86_64.txt @@ -19,6 +19,7 @@ target_link_libraries(ydb-core-cms-console-yaml_config-ut PUBLIC cms-console-yaml_config ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) set_property( diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp index 1091cad3272..3e75e90895f 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp @@ -1,7 +1,9 @@ -#include <util/string/type.h> #include "cli.h" #include "cli_cmds.h" +#include <ydb/core/cms/console/yaml_config/console_dumper.h> + +#include <util/string/type.h> #include <util/string/split.h> #include <util/system/fs.h> @@ -313,12 +315,44 @@ public: } }; +class TClientCommandConsoleConfigsDumpYaml : public TConsoleClientCommand { +public: + TClientCommandConsoleConfigsDumpYaml() + : TConsoleClientCommand("dump-yaml", {}, "Dump config in yaml format") + { + } + + virtual void Config(TConfig& config) override { + TConsoleClientCommand::Config(config); + config.SetFreeArgsNum(0); + } + + virtual void Parse(TConfig& config) override { + TConsoleClientCommand::Parse(config); + Request.MutableGetConfigItemsRequest(); + } + + virtual void PrintResponse(const NKikimrClient::TConsoleResponse &response) override + { + if (response.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { + Cout << "ERROR: " << response.GetStatus().GetCode() + << " (" << response.GetStatus().GetReason() << ")" << Endl; + return; + } + + auto &items = response.GetGetConfigItemsResponse().GetConfigItems(); + + Cout << NYamlConfig::DumpConsoleConfigs(items); + } +}; + class TClientCommandConsoleConfigs : public TClientCommandTree { public: TClientCommandConsoleConfigs() : TClientCommandTree("configs", {}, "") { AddCommand(std::make_unique<TClientCommandConsoleConfigsLoad>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigsDumpYaml>()); AddCommand(std::make_unique<TClientCommandConsoleConfigsUpdate>()); } }; |