diff options
author | innokentii <innokentii@ydb.tech> | 2023-12-06 08:51:27 +0300 |
---|---|---|
committer | innokentii <innokentii@ydb.tech> | 2023-12-06 09:11:35 +0300 |
commit | a27ba7422c788f1d0d3e1c76ecaad794b40d7993 (patch) | |
tree | 0a51fb14761ea91a5432b23593ea60a8d06cb142 | |
parent | 277df39749f132aff6ec10f78e89605a453817f0 (diff) | |
download | ydb-a27ba7422c788f1d0d3e1c76ecaad794b40d7993.tar.gz |
Improve configs introspection
fix
fix
improve configs introspection
-rw-r--r-- | ydb/core/cms/console/CMakeLists.darwin-arm64.txt | 12 | ||||
-rw-r--r-- | ydb/core/cms/console/CMakeLists.darwin-x86_64.txt | 12 | ||||
-rw-r--r-- | ydb/core/cms/console/CMakeLists.linux-aarch64.txt | 12 | ||||
-rw-r--r-- | ydb/core/cms/console/CMakeLists.linux-x86_64.txt | 12 | ||||
-rw-r--r-- | ydb/core/cms/console/CMakeLists.windows-x86_64.txt | 12 | ||||
-rw-r--r-- | ydb/core/cms/console/config_item_info.h | 26 | ||||
-rw-r--r-- | ydb/core/cms/console/configs_dispatcher.cpp | 276 | ||||
-rw-r--r-- | ydb/core/cms/console/configs_dispatcher.h | 4 | ||||
-rw-r--r-- | ydb/core/cms/console/http.cpp | 230 | ||||
-rw-r--r-- | ydb/core/cms/console/http.h | 31 | ||||
-rw-r--r-- | ydb/core/cms/console/util.cpp | 16 | ||||
-rw-r--r-- | ydb/core/cms/console/util.h | 5 | ||||
-rw-r--r-- | ydb/core/cms/console/ya.make | 2 | ||||
-rw-r--r-- | ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp | 133 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/config.h | 3 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/kikimr_services_initializers.cpp | 3 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/kikimr_services_initializers.h | 1 |
17 files changed, 603 insertions, 187 deletions
diff --git a/ydb/core/cms/console/CMakeLists.darwin-arm64.txt b/ydb/core/cms/console/CMakeLists.darwin-arm64.txt index 97f0a4db67..589b75be3d 100644 --- a/ydb/core/cms/console/CMakeLists.darwin-arm64.txt +++ b/ydb/core/cms/console/CMakeLists.darwin-arm64.txt @@ -9,11 +9,18 @@ add_subdirectory(ut) add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -85,3 +92,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + ydb/core/cms/console/config_item_info.h +) diff --git a/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt b/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt index 97f0a4db67..589b75be3d 100644 --- a/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt @@ -9,11 +9,18 @@ add_subdirectory(ut) add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -85,3 +92,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + ydb/core/cms/console/config_item_info.h +) diff --git a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt index 013fad4050..6d6b88064f 100644 --- a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt @@ -9,12 +9,19 @@ add_subdirectory(ut) add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -86,3 +93,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + ydb/core/cms/console/config_item_info.h +) diff --git a/ydb/core/cms/console/CMakeLists.linux-x86_64.txt b/ydb/core/cms/console/CMakeLists.linux-x86_64.txt index 013fad4050..6d6b88064f 100644 --- a/ydb/core/cms/console/CMakeLists.linux-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.linux-x86_64.txt @@ -9,12 +9,19 @@ add_subdirectory(ut) add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -86,3 +93,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + ydb/core/cms/console/config_item_info.h +) diff --git a/ydb/core/cms/console/CMakeLists.windows-x86_64.txt b/ydb/core/cms/console/CMakeLists.windows-x86_64.txt index 97f0a4db67..589b75be3d 100644 --- a/ydb/core/cms/console/CMakeLists.windows-x86_64.txt +++ b/ydb/core/cms/console/CMakeLists.windows-x86_64.txt @@ -9,11 +9,18 @@ add_subdirectory(ut) add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -85,3 +92,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + ydb/core/cms/console/config_item_info.h +) diff --git a/ydb/core/cms/console/config_item_info.h b/ydb/core/cms/console/config_item_info.h new file mode 100644 index 0000000000..a34f64a130 --- /dev/null +++ b/ydb/core/cms/console/config_item_info.h @@ -0,0 +1,26 @@ +#pragma once + +#include <util/generic/vector.h> +#include <util/system/types.h> + +struct TConfigItemInfo { + enum class EUpdateKind { + MutableConfigPartFromFile, + MutableConfigPartFromBaseConfig, + MutableConfigPartMergeFromFile, + ReplaceConfigWithConsoleYaml, + ReplaceConfigWithConsoleProto, + ReplaceConfigWithBase, + LoadYamlConfigFromFile, + SetExplicitly, + UpdateExplicitly, + }; + + struct TUpdate { + const char* File; + ui32 Line; + EUpdateKind Kind; + }; + + TVector<TUpdate> Updates; +}; diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index db85bbe440..9879c65ba8 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -136,7 +136,8 @@ public: const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig); + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap<ui32, TConfigItemInfo> &configInitInfo); void Bootstrap(); @@ -233,6 +234,7 @@ private: NKikimrConfig::TAppConfig CurrentConfig; const NKikimrConfig::TAppConfig InitialCmsConfig; const NKikimrConfig::TAppConfig InitialCmsYamlConfig; + const THashMap<ui32, TConfigItemInfo> ConfigInitInfo; ui64 NextRequestCookie; TVector<TActorId> HttpRequests; TActorId CommonSubscriptionClient; @@ -256,12 +258,14 @@ TConfigsDispatcher::TConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig) + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap<ui32, TConfigItemInfo> &configInitInfo) : Labels(labels) , InitialConfig(config) , CurrentConfig(config) , InitialCmsConfig(initialCmsConfig) , InitialCmsYamlConfig(initialCmsYamlConfig) + , ConfigInitInfo(configInitInfo) , NextRequestCookie(Now().GetValue()) { } @@ -480,143 +484,184 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) } } str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("state", "State") { - PRE() { - str << "SelfId: " << SelfId() << Endl; - auto s = CurrentStateFunc(); - str << "State: " << ( s == &TThis::StateWork ? "StateWork" - : s == &TThis::StateInit ? "StateInit" - : "Unknown" ) << Endl; - str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; - str << "Subscriptions: " << Endl; - for (auto &[kinds, subscription] : SubscriptionsByKinds) { - str << "- Kinds: " << KindsToString(kinds) << Endl - << " Subscription: " << Endl - << " Yaml: " << subscription->Yaml << Endl - << " Subscribers: " << Endl; - for (auto &[id, updates] : subscription->Subscribers) { - str << " - Actor: " << id << Endl; - str << " UpdatesSent: " << updates << Endl; - } - if (subscription->YamlVersion) { - str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; - bool first = true; - for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { - str << (first ? "" : ",") << id << "." << hash; - first = false; - } - str << "]" << Endl; - } else { - str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; - } - str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; - if (subscription->UpdateInProcess) { - str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl - << " SubscribersToUpdate:"; - for (auto &id : subscription->SubscribersToUpdate) { - str << " " << id; - } - str << Endl; - str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl - << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; - if (subscription->UpdateInProcessYamlVersion) { - str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; - } - } - } - str << "Subscribers:" << Endl; - for (auto &[subscriber, _] : SubscriptionsBySubscriber) { - str << "- " << subscriber << Endl; - } + COLLAPSED_REF_CONTENT("effective-config", "Effective config") { + str << "<div class=\"alert alert-primary tab-left\" role=\"alert\">" << Endl; + str << "It is assumed that all config items marked dynamic are consumed by corresponding components at runtime." << Endl; + str << "<br />" << Endl; + str << "If any component is unable to update some config at runtime check \"Effective startup config\" below to see actual config for this component." << Endl; + str << "<br />" << Endl; + str << "Coloring: \"<font color=\"red\">config not set</font>\"," + << " \"<font color=\"green\">config set in dynamic config</font>\", \"<font color=\"#007bff\">config set in static config</font>\"" << Endl; + str << "</div>" << Endl; + NHttp::OutputRichConfigHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("effective-startup-config", "Effective startup config") { + str << "<div class=\"alert alert-primary tab-left\" role=\"alert\">" << Endl; + str << "Some of these configs may be overwritten by dynamic ones." << Endl; + str << "</div>" << Endl; + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("effective-dynamic-config", "Effective dynamic config") { + str << "<div class=\"alert alert-primary tab-left\" role=\"alert\">" << Endl; + str << "Some subscribers may get static configs if they exist even if dynamic one of corresponding kind is not present." << Endl; + str << "</div>" << Endl; + NKikimrConfig::TAppConfig trunc; + if (YamlConfigEnabled) { + ReplaceConfigItems(YamlProtoConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig); + ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(NON_YAML_KINDS), trunc, false); + } else { + ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig); } + NHttp::OutputConfigHTML(str, trunc); } str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { - DIV() { - TAG(TH5) { - str << "Persistent Config" << Endl; + COLLAPSED_REF_CONTENT("debug-info", "Debug info") { + DIV_CLASS("tab-left") { + COLLAPSED_REF_CONTENT("effective-config-debug-info", "Effective config debug info") { + NHttp::OutputConfigDebugInfoHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, ConfigInitInfo, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled); } - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { - str << YamlConfig; + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("state", "State") { + PRE() { + str << "SelfId: " << SelfId() << Endl; + auto s = CurrentStateFunc(); + str << "State: " << ( s == &TThis::StateWork ? "StateWork" + : s == &TThis::StateInit ? "StateInit" + : "Unknown" ) << Endl; + str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; + str << "Subscriptions: " << Endl; + for (auto &[kinds, subscription] : SubscriptionsByKinds) { + str << "- Kinds: " << KindsToString(kinds) << Endl + << " Subscription: " << Endl + << " Yaml: " << subscription->Yaml << Endl + << " Subscribers: " << Endl; + for (auto &[id, updates] : subscription->Subscribers) { + str << " - Actor: " << id << Endl; + str << " UpdatesSent: " << updates << Endl; + } + if (subscription->YamlVersion) { + str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; + bool first = true; + for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { + str << (first ? "" : ",") << id << "." << hash; + first = false; + } + str << "]" << Endl; + } else { + str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; + } + str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; + if (subscription->UpdateInProcess) { + str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl + << " SubscribersToUpdate:"; + for (auto &id : subscription->SubscribersToUpdate) { + str << " " << id; + } + str << Endl; + str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl + << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; + if (subscription->UpdateInProcessYamlVersion) { + str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; + } + } + } + str << "Subscribers:" << Endl; + for (auto &[subscriber, _] : SubscriptionsBySubscriber) { + str << "- " << subscriber << Endl; + } } } - str << "<hr/>" << Endl; - for (auto &[id, config] : VolatileYamlConfigs) { + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { DIV() { TAG(TH5) { - str << "Volatile Config Id: " << id << Endl; + str << "Persistent Config" << Endl; } TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { DIV_CLASS("yaml-sticky-btn") { } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { DIV_CLASS("yaml-sticky-btn") { } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { DIV_CLASS("yaml-sticky-btn") { } } - DIV_CLASS("yaml-config-item") { - str << config; + TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { + str << YamlConfig; + } + } + str << "<hr/>" << Endl; + for (auto &[id, config] : VolatileYamlConfigs) { + DIV() { + TAG(TH5) { + str << "Volatile Config Id: " << id << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + DIV_CLASS("yaml-config-item") { + str << config; + } + } } } } } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { + str << ResolvedYamlConfig; + } + } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { + PRE() { + str << ResolvedJsonConfig << Endl; + } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { + NHttp::OutputConfigHTML(str, YamlProtoConfig); } - TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { - str << ResolvedYamlConfig; + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("current-config", "Current config") { + NHttp::OutputConfigHTML(str, CurrentConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-config", "Initial config") { + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") { + NHttp::OutputConfigHTML(str, InitialCmsConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") { + NHttp::OutputConfigHTML(str, InitialCmsYamlConfig); } } } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { - PRE() { - str << ResolvedJsonConfig << Endl; - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { - NHttp::OutputConfigHTML(str, YamlProtoConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("current-config", "Current config") { - NHttp::OutputConfigHTML(str, CurrentConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("initial-config", "Initial config") { - NHttp::OutputConfigHTML(str, InitialConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") { - NHttp::OutputConfigHTML(str, InitialCmsConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") { - NHttp::OutputConfigHTML(str, InitialCmsYamlConfig); - } } } } @@ -955,9 +1000,10 @@ IActor *CreateConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig) + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap<ui32, TConfigItemInfo> &configInitInfo) { - return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig); + return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig, configInitInfo); } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/configs_dispatcher.h b/ydb/core/cms/console/configs_dispatcher.h index 879e8bf58f..b771a8a725 100644 --- a/ydb/core/cms/console/configs_dispatcher.h +++ b/ydb/core/cms/console/configs_dispatcher.h @@ -2,6 +2,7 @@ #include "defs.h" #include <ydb/core/protos/config.pb.h> +#include <ydb/core/cms/console/config_item_info.h> #include <util/generic/vector.h> @@ -115,7 +116,8 @@ IActor *CreateConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels, const NKikimrConfig::TAppConfig &initialCmsConfig = {}, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {}); + const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {}, + const THashMap<ui32, TConfigItemInfo> &configInitInfo = {}); inline TActorId MakeConfigsDispatcherID(ui32 node = 0) { char x[12] = { 'c', 'o', 'n', 'f', 'i', 'g', 's', 'd', 'i', 's', 'p' }; diff --git a/ydb/core/cms/console/http.cpp b/ydb/core/cms/console/http.cpp index 43419c6372..b8960053b5 100644 --- a/ydb/core/cms/console/http.cpp +++ b/ydb/core/cms/console/http.cpp @@ -14,6 +14,12 @@ void OutputStyles(IOutputStream &os) << ".tab-left {" << Endl << " margin-left: 20px" << Endl << "}" << Endl + << ".collapse-ref.empty {" << Endl + << " color: red !important" << Endl + << "}" << Endl + << ".collapse-ref.dynamic {" << Endl + << " color: green !important" << Endl + << "}" << Endl << "</style>" << Endl; } @@ -29,6 +35,64 @@ void OutputStaticPart(IOutputStream &os) } } +void PrintField(IOutputStream &os, const NKikimrConfig::TAppConfig &config, const google::protobuf::FieldDescriptor *field) { + auto *reflection = config.GetReflection(); + HTML(os) { + PRE() { + switch (field->cpp_type()) { + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: + os << reflection->GetInt32(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: + os << reflection->GetInt64(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + os << reflection->GetUInt32(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + os << reflection->GetUInt64(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + os << reflection->GetDouble(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + os << reflection->GetFloat(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + os << reflection->GetBool(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + os << reflection->GetEnum(config, field)->name(); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: + if (field->is_repeated()) { + int count = reflection->FieldSize(config, field); + for (int index = 0; index < count; ++index) { + os << "[" << index << "]: " + << reflection->GetRepeatedString(config, field, index) << '\n'; + } + } else { + os << reflection->GetString(config, field); + } + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + if (field->is_repeated()) { + int count = reflection->FieldSize(config, field); + for (int index = 0; index < count; ++index) { + os << "[" << index << "]:" << '\n' + << reflection->GetRepeatedMessage(config, field, index).DebugString(); + } + } else { + os << reflection->GetMessage(config, field).DebugString(); + } + break; + default: + os << "<unsupported value type>"; + } + } + } +}; + void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config) { HTML(os) { @@ -41,57 +105,119 @@ void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config TString id = TStringBuilder() << field->name() << "-" << RandomNumber<ui64>(); COLLAPSED_REF_CONTENT(id, field->name()) { DIV_CLASS("tab-left") { - PRE() { - switch (field->cpp_type()) { - case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: - os << reflection->GetInt32(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: - os << reflection->GetInt64(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: - os << reflection->GetUInt32(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: - os << reflection->GetUInt64(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: - os << reflection->GetDouble(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: - os << reflection->GetFloat(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: - os << reflection->GetBool(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: - os << reflection->GetEnum(config, field)->name(); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: - if (field->is_repeated()) { - int count = reflection->FieldSize(config, field); - for (int index = 0; index < count; ++index) { - os << "[" << index << "]: " - << reflection->GetRepeatedString(config, field, index) << '\n'; + PrintField(os, config, field); + } + } + os << "<br/>" << Endl; + } + } + } +} + +void OutputConfigDebugInfoHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashMap<ui32, TConfigItemInfo>& configInitInfo, + const THashSet<ui32> &dynamicKinds, + const THashSet<ui32> &nonYamlKinds, + bool yamlEnabled) +{ + HTML(os) { + DIV_CLASS("tab-left") { + auto *desc = initialConfig.GetDescriptor(); + auto *reflection = initialConfig.GetReflection(); + + for (int i = 0; i < desc->field_count(); i++) { + auto *field = desc->field(i); + auto tag = field->number(); + + bool yaml = false; + bool empty = false; + bool fallback = false; + + const NKikimrConfig::TAppConfig *config = nullptr; + + auto chooseSource = [&](auto& primaryConfig, auto& fallbackConfig) { + if (field->is_repeated()) { + if (reflection->FieldSize(primaryConfig, field)) { + config = &primaryConfig; + } else if (reflection->FieldSize(fallbackConfig, field)) { + config = &fallbackConfig; + fallback = true; + } else { + empty = true; + } + } else { + if (reflection->HasField(primaryConfig, field)) { + config = &primaryConfig; + } else if (reflection->HasField(fallbackConfig, field)) { + config = &fallbackConfig; + fallback = true; + } else { + empty = true; + } + } + }; + + if (dynamicKinds.contains(tag)) { + if (yamlEnabled && !nonYamlKinds.contains(tag)) { + chooseSource(yamlConfig, initialConfig); + yaml = true; + } else { + chooseSource(protoConfig, initialConfig); + } + } else { + fallback = true; + if (field->is_repeated()) { + if (reflection->FieldSize(initialConfig, field)) { + config = &initialConfig; + } else { + empty = true; + } + } else { + if (reflection->HasField(initialConfig, field)) { + config = &initialConfig; + } else { + empty = true; + } + } + } + + TString id = TStringBuilder() << field->name() << "-" << RandomNumber<ui64>(); + COLLAPSED_REF_CONTENT(id, field->name(), (empty ? "empty" : ""), (yaml ? "yaml" : ""), (fallback || empty ? "" : "dynamic")) { + DIV_CLASS("tab-left") { + if (empty) { + os << "<div class=\"alert alert-primary\" role=\"alert\">" << Endl; + os << "This config item isn't set" << Endl; + os << "</div>" << Endl; + } else { + if (!configInitInfo.empty()) { + os << "<div class=\"alert alert-primary\" role=\"alert\"><ol type=\"1\">" << Endl; + if (auto it = configInitInfo.find(tag); it != configInitInfo.end()) { + for (const auto& update : it->second.Updates) { + os << "<li><b>" << update.Kind << "</b> at <b>" << update.File << "</b>:" << update.Line << "<br/></li>"; } - } else { - os << reflection->GetString(config, field); } - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - if (field->is_repeated()) { - int count = reflection->FieldSize(config, field); - for (int index = 0; index < count; ++index) { - os << "[" << index << "]:" << '\n' - << reflection->GetRepeatedMessage(config, field, index).DebugString(); - } + if (yaml) { + os << "<li>Overwritten by dynamic yaml config</li>"; + } else if (!fallback) { + os << "<li>Overwritten by dynamic proto config</li>"; + } + os << "</ol></div>" << Endl; + } else { + os << "<div class=\"alert alert-primary\" role=\"alert\">" << Endl; + if (fallback) { + os << "This config item is set in static config"; + } else if (yaml) { + os << "This config item is set in dynamic yaml config"; } else { - os << reflection->GetMessage(config, field).DebugString(); + os << "This config item is set in dynamic proto config"; } - break; - default: - os << "<unsupported value type>"; + os << "</div>" << Endl; } + PrintField(os, *config, field); } } } @@ -101,4 +227,16 @@ void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config } } +void OutputRichConfigHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashSet<ui32> &dynamicKinds, + const THashSet<ui32> &nonYamlKinds, + bool yamlEnabled) +{ + OutputConfigDebugInfoHTML(os, initialConfig, yamlConfig, protoConfig, {}, dynamicKinds, nonYamlKinds, yamlEnabled); +} + } // namespace NKikimr::NConsole::NHttp diff --git a/ydb/core/cms/console/http.h b/ydb/core/cms/console/http.h index f6850aa527..d9e0f7a45f 100644 --- a/ydb/core/cms/console/http.h +++ b/ydb/core/cms/console/http.h @@ -2,21 +2,26 @@ #include "defs.h" #include <ydb/core/protos/config.pb.h> +#include <ydb/core/cms/console/config_item_info.h> #include <library/cpp/monlib/service/pages/templates.h> #include <util/stream/output.h> -#define COLLAPSED_REF_CONTENT(target, text) \ - WITH_SCOPED(tmp, NKikimr::NConsole::NHttp::TCollapsedRef(__stream, target, text)) +#define COLLAPSED_REF_CONTENT(target, text, ...) \ + WITH_SCOPED(tmp, NKikimr::NConsole::NHttp::TCollapsedRef(__stream, target, text __VA_OPT__(,) __VA_ARGS__)) namespace NKikimr::NConsole::NHttp { struct TCollapsedRef { - TCollapsedRef(IOutputStream& str, const TString& target, const TString& text) + template <class... TClasses> + TCollapsedRef(IOutputStream& str, const TString& target, const TString& text, TClasses... classes) : Str(str) { - Str << "<a class='collapse-ref' data-toggle='collapse' data-target='#" << target << "'>" + Str << "<a class='collapse-ref"; + void *dummy[sizeof...(TClasses)] = {(void*)&(Str << " " << classes)...}; + Y_UNUSED(dummy); + Str << "' data-toggle='collapse' data-target='#" << target << "'>" << text << "</a>"; Str << "<div id='" << target << "' class='collapse'>"; } @@ -40,5 +45,23 @@ struct TCollapsedRef { void OutputStaticPart(IOutputStream &os); void OutputStyles(IOutputStream &os); void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config); +void OutputRichConfigHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashSet<ui32> &dynamicKinds, + const THashSet<ui32> &nonYamlKinds, + bool yamlEnabled); +void OutputConfigDebugInfoHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashMap<ui32, TConfigItemInfo>& configInitInfo, + const THashSet<ui32> &dynamicKinds, + const THashSet<ui32> &nonYamlKinds, + bool yamlEnabled); + } // namespace NKikimr::NConsole::NHttp diff --git a/ydb/core/cms/console/util.cpp b/ydb/core/cms/console/util.cpp index d898275ab8..d2c1c6dccb 100644 --- a/ydb/core/cms/console/util.cpp +++ b/ydb/core/cms/console/util.cpp @@ -51,11 +51,21 @@ TDynBitMap KindsToBitMap(const TVector<ui32> &kinds) return result; } +TDynBitMap KindsToBitMap(const THashSet<ui32> &kinds) +{ + TDynBitMap result; + for (auto &kind : kinds) + result.Set(kind); + + return result; +} + void ReplaceConfigItems( const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds, - const NKikimrConfig::TAppConfig &fallback) + const NKikimrConfig::TAppConfig &fallback, + bool cleanupFallback) { NKikimrConfig::TAppConfig fromCopy = from; NKikimrConfig::TAppConfig fallbackCopy = fallback; @@ -78,7 +88,9 @@ void ReplaceConfigItems( if (reflection->HasField(fromCopy, field)) { reflection->ClearField(&fromCopy, field); } - reflection->ClearField(&fallbackCopy, field); + if (cleanupFallback) { + reflection->ClearField(&fallbackCopy, field); + } } } else { reflection->ClearField(&to, field); diff --git a/ydb/core/cms/console/util.h b/ydb/core/cms/console/util.h index 8ec1ef5eb8..12553a2f2f 100644 --- a/ydb/core/cms/console/util.h +++ b/ydb/core/cms/console/util.h @@ -17,6 +17,8 @@ TString KindsToString(const TVector<ui32> &kinds); TDynBitMap KindsToBitMap(const TVector<ui32> &kinds); +TDynBitMap KindsToBitMap(const THashSet<ui32> &kinds); + /** * Replace 'kinds' in 'to' from 'from' * repeated items are removed @@ -25,7 +27,8 @@ void ReplaceConfigItems( const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds, - const NKikimrConfig::TAppConfig &fallback = {}); + const NKikimrConfig::TAppConfig &fallback = {}, + bool cleanupFallback = true); bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); diff --git a/ydb/core/cms/console/ya.make b/ydb/core/cms/console/ya.make index 0361a88c6b..1f786284e4 100644 --- a/ydb/core/cms/console/ya.make +++ b/ydb/core/cms/console/ya.make @@ -71,6 +71,8 @@ SRCS( util.h ) +GENERATE_ENUM_SERIALIZATION(config_item_info.h) + PEERDIR( ydb/library/actors/core ydb/library/actors/http diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 7496bf45f3..0f2ad3efa2 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -23,6 +23,26 @@ extern TAutoPtr<NKikimrConfig::TAllocatorConfig> DummyAllocatorConfig(); namespace NKikimr { namespace NDriverClient { +struct TCallContext { + const char* File; + int Line; +}; + +#define TRACE_CONFIG_CHANGE(CHANGE_CONTEXT, KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[KIND].Updates.emplace_back( \ + CHANGE_CONTEXT.File, CHANGE_CONTEXT.Line, TConfigItemInfo::EUpdateKind:: CHANGE_KIND) + +#define TRACE_CONFIG_CHANGE_INPLACE(KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[KIND].Updates.emplace_back( \ + __FILE__, __LINE__, TConfigItemInfo::EUpdateKind:: CHANGE_KIND) + +#define TRACE_CONFIG_CHANGE_INPLACE_T(KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[NKikimrConsole::TConfigItem:: KIND ## Item].Updates.emplace_back( \ + __FILE__, __LINE__, TConfigItemInfo::EUpdateKind:: CHANGE_KIND) + +#define CALL_CTX() TCallContext{__FILE__, __LINE__} + + constexpr auto NODE_KIND_YDB = "ydb"; constexpr auto NODE_KIND_YQ = "yq"; @@ -275,7 +295,9 @@ protected: TProto *MutableConfigPart(TConfig& config, const char *optname, bool (NKikimrConfig::TAppConfig::*hasConfig)() const, const TProto& (NKikimrConfig::TAppConfig::*getConfig)() const, - TProto* (NKikimrConfig::TAppConfig::*mutableConfig)()) { + TProto* (NKikimrConfig::TAppConfig::*mutableConfig)(), + ui32 kind, + TCallContext callCtx) { TProto *res = nullptr; if (!HierarchicalCfg && (AppConfig.*hasConfig)()) { return nullptr; // this field is already provided in AppConfig, so we don't overwrite it @@ -284,9 +306,11 @@ protected: if (optname && config.ParseResult->Has(optname)) { const bool success = ParsePBFromFile(config.ParseResult->Get(optname), res = (AppConfig.*mutableConfig)()); Y_ABORT_UNLESS(success); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartFromFile); } else if ((BaseConfig.*hasConfig)()) { res = (AppConfig.*mutableConfig)(); res->CopyFrom((BaseConfig.*getConfig)()); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartFromBaseConfig); } return res; @@ -294,7 +318,9 @@ protected: template<typename TProto> TProto *MutableConfigPartMerge(TConfig& config, const char *optname, - TProto* (NKikimrConfig::TAppConfig::*mutableConfig)()) { + TProto* (NKikimrConfig::TAppConfig::*mutableConfig)(), + ui32 kind, + TCallContext callCtx) { TProto *res = nullptr; if (config.ParseResult->Has(optname)) { @@ -303,6 +329,7 @@ protected: Y_ABORT_UNLESS(success); res = (AppConfig.*mutableConfig)(); res->MergeFrom(cfg); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartMergeFromFile); } return res; @@ -341,12 +368,14 @@ protected: TClientCommand::Parse(config); #define OPTION(NAME, FIELD) MutableConfigPart(config, NAME, &NKikimrConfig::TAppConfig::Has##FIELD, \ - &NKikimrConfig::TAppConfig::Get##FIELD, &NKikimrConfig::TAppConfig::Mutable##FIELD) -#define OPTION_MERGE(NAME, FIELD) MutableConfigPartMerge(config, NAME, &NKikimrConfig::TAppConfig::Mutable##FIELD) + &NKikimrConfig::TAppConfig::Get##FIELD, &NKikimrConfig::TAppConfig::Mutable##FIELD, \ + (ui32)NKikimrConsole::TConfigItem:: FIELD ## Item, TCallContext{__FILE__, __LINE__}) +#define OPTION_MERGE(NAME, FIELD) MutableConfigPartMerge(config, NAME, &NKikimrConfig::TAppConfig::Mutable##FIELD, \ + (ui32)NKikimrConsole::TConfigItem:: FIELD ## Item, TCallContext{__FILE__, __LINE__}) OPTION("auth-file", AuthConfig); LoadBaseConfig(config); - LoadYamlConfig(); + LoadYamlConfig(CALL_CTX()); OPTION_MERGE("auth-token-file", AuthConfig); // start memorylog as soon as possible @@ -437,11 +466,12 @@ protected: LoadConfigForDynamicNode(); } - LoadYamlConfig(); + LoadYamlConfig(CALL_CTX()); OPTION("sys-file", ActorSystemConfig); if (!AppConfig.HasActorSystemConfig()) { AppConfig.MutableActorSystemConfig()->CopyFrom(*DummyActorSystemConfig()); + TRACE_CONFIG_CHANGE_INPLACE_T(ActorSystemConfig, SetExplicitly); } OPTION("domains-file", DomainsConfig); @@ -463,15 +493,20 @@ protected: } // This flag is set per node and we prefer flag over CMS. if (config.ParseResult->Has("syslog-service-tag") - && !AppConfig.GetLogConfig().GetSysLogService()) + && !AppConfig.GetLogConfig().GetSysLogService()) { AppConfig.MutableLogConfig()->SetSysLogService(SysLogServiceTag); + TRACE_CONFIG_CHANGE_INPLACE_T(LogConfig, UpdateExplicitly); + } - if (config.ParseResult->Has("log-file-name")) + if (config.ParseResult->Has("log-file-name")) { AppConfig.MutableLogConfig()->SetBackendFileName(LogFileName); + TRACE_CONFIG_CHANGE_INPLACE_T(LogConfig, UpdateExplicitly); + } if (auto interconnectConfig = OPTION("ic-file", InterconnectConfig)) { if (config.ParseResult->Has("tcp")) { interconnectConfig->SetStartTcp(true); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } } @@ -479,6 +514,7 @@ protected: if (auto bootstrapConfig = OPTION("bootstrap-file", BootstrapConfig)) { bootstrapConfig->MutableCompileServiceConfig()->SetInflightLimit(CompileInflightLimit); + TRACE_CONFIG_CHANGE_INPLACE_T(BootstrapConfig, UpdateExplicitly); } OPTION("vdisk-file", VDiskConfig); @@ -509,43 +545,53 @@ protected: if (!AppConfig.HasAllocatorConfig()) { AppConfig.MutableAllocatorConfig()->CopyFrom(*DummyAllocatorConfig()); + TRACE_CONFIG_CHANGE_INPLACE_T(AllocatorConfig, UpdateExplicitly); } // apply certificates, if any if (!PathToInterconnectCertFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToCertificateFile(PathToInterconnectCertFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (!PathToInterconnectPrivateKeyFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToPrivateKeyFile(PathToInterconnectPrivateKeyFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (!PathToInterconnectCaFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToCaFile(PathToInterconnectCaFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasCert()) { AppConfig.MutableGRpcConfig()->SetPathToCertificateFile(AppConfig.GetGRpcConfig().GetCert()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcCertFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToCertificateFile(PathToGrpcCertFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasKey()) { AppConfig.MutableGRpcConfig()->SetPathToPrivateKeyFile(AppConfig.GetGRpcConfig().GetKey()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcPrivateKeyFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToPrivateKeyFile(PathToGrpcPrivateKeyFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasCA()) { AppConfig.MutableGRpcConfig()->SetPathToCaFile(AppConfig.GetGRpcConfig().GetCA()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcCaFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToCaFile(PathToGrpcCaFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!AppConfig.HasDomainsConfig()) @@ -575,6 +621,7 @@ protected: if (config.ParseResult->Has("suppress-version-check")) { if (AppConfig.HasNameserviceConfig()) { AppConfig.MutableNameserviceConfig()->SetSuppressVersionCheck(true); + TRACE_CONFIG_CHANGE_INPLACE_T(NameserviceConfig, UpdateExplicitly); } else { ythrow yexception() << "--suppress-version-check option is provided without static nameservice config"; } @@ -589,35 +636,48 @@ protected: } } - if (!AppConfig.HasMonitoringConfig()) + if (!AppConfig.HasMonitoringConfig()) { AppConfig.MutableMonitoringConfig()->SetMonitoringThreads(MonitoringThreads); - if (!AppConfig.HasRestartsCountConfig() && RestartsCountFile) + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } + if (!AppConfig.HasRestartsCountConfig() && RestartsCountFile) { AppConfig.MutableRestartsCountConfig()->SetRestartsCountFile(RestartsCountFile); + TRACE_CONFIG_CHANGE_INPLACE_T(RestartsCountConfig, UpdateExplicitly); + } // Ports and node type are always applied (even if config was loaded from CMS). - if (MonitoringPort) + if (MonitoringPort) { AppConfig.MutableMonitoringConfig()->SetMonitoringPort(MonitoringPort); - if (MonitoringAddress) + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } + if (MonitoringAddress) { AppConfig.MutableMonitoringConfig()->SetMonitoringAddress(MonitoringAddress); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } if (MonitoringCertificateFile) { TString sslCertificate = TUnbufferedFileInput(MonitoringCertificateFile).ReadAll(); if (!sslCertificate.empty()) { AppConfig.MutableMonitoringConfig()->SetMonitoringCertificate(sslCertificate); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } else { ythrow yexception() << "invalid ssl certificate file"; } } - if (SqsHttpPort) + if (SqsHttpPort) { AppConfig.MutableSqsConfig()->MutableHttpServerConfig()->SetPort(SqsHttpPort); + TRACE_CONFIG_CHANGE_INPLACE_T(SqsConfig, UpdateExplicitly); + } if (GRpcPort) { auto& conf = *AppConfig.MutableGRpcConfig(); conf.SetStartGRpcProxy(true); conf.SetPort(GRpcPort); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcsPort) { auto& conf = *AppConfig.MutableGRpcConfig(); conf.SetStartGRpcProxy(true); conf.SetSslPort(GRpcsPort); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcPublicHost) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -627,6 +687,7 @@ protected: ext.SetPublicHost(GRpcPublicHost); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcPublicPort) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -636,6 +697,7 @@ protected: ext.SetPublicPort(GRpcPublicPort); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcsPublicPort) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -645,25 +707,37 @@ protected: ext.SetPublicSslPort(GRpcsPublicPort); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } for (const auto& addr : GRpcPublicAddressesV4) { AppConfig.MutableGRpcConfig()->AddPublicAddressesV4(addr); } + if (GRpcPublicAddressesV4.size()) { + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); + } for (const auto& addr : GRpcPublicAddressesV6) { AppConfig.MutableGRpcConfig()->AddPublicAddressesV6(addr); } + if (GRpcPublicAddressesV6.size()) { + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); + } if (GRpcPublicTargetNameOverride) { AppConfig.MutableGRpcConfig()->SetPublicTargetNameOverride(GRpcPublicTargetNameOverride); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } - if (config.ParseResult->Has("node-type")) + if (config.ParseResult->Has("node-type")) { AppConfig.MutableTenantPoolConfig()->SetNodeType(NodeType); + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); + } if (config.ParseResult->Has("tenant") && InterconnectPort != DefaultInterconnectPort) { AppConfig.MutableMonitoringConfig()->SetHostLabelOverride(HostAndICPort()); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } if (config.ParseResult->Has("data-center")) { AppConfig.MutableMonitoringConfig()->SetDataCenter(to_lower(DataCenter)); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } if (config.ParseResult->Has("tenant")) { @@ -672,17 +746,20 @@ protected: slot.SetTenantName(TenantName); slot.SetIsDynamic(false); RunConfig.TenantName = TenantName; + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); } else { auto &slot = *AppConfig.MutableTenantPoolConfig()->AddSlots(); slot.SetId("static-slot"); slot.SetTenantName(CanonizePath(DeduceNodeDomain())); slot.SetIsDynamic(false); RunConfig.TenantName = CanonizePath(DeduceNodeDomain()); + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); } if (config.ParseResult->Has("data-center")) { if (AppConfig.HasFederatedQueryConfig()) { AppConfig.MutableFederatedQueryConfig()->MutableNodesManager()->SetDataCenter(to_lower(DataCenter)); + TRACE_CONFIG_CHANGE_INPLACE_T(FederatedQueryConfig, UpdateExplicitly); } } @@ -740,6 +817,8 @@ protected: } messageBusConfig->SetStartTracingBusProxy(!!TracePath); messageBusConfig->SetTracePath(TracePath); + + TRACE_CONFIG_CHANGE_INPLACE_T(MessageBusConfig, UpdateExplicitly); } if (AppConfig.HasDynamicNameserviceConfig()) { @@ -841,7 +920,7 @@ protected: return false; } - inline void LoadYamlConfig() { + inline void LoadYamlConfig(TCallContext callCtx) { for(const TString& yamlConfigFile: YamlConfigFiles) { auto yamlConfig = TFileInput(yamlConfigFile); NKikimrConfig::TAppConfig parsedConfig; @@ -864,6 +943,7 @@ protected: if (reflection->HasField(parsedConfig, fieldDescriptor)) { reflection->SwapFields(&AppConfig, &parsedConfig, {fieldDescriptor}); + TRACE_CONFIG_CHANGE(callCtx, fieldIdx, ReplaceConfigWithConsoleProto); } } } @@ -1258,8 +1338,10 @@ protected: // By now naming config should be loaded and probably replaced with // info from registration response. Don't lose it in case CMS has no // config for naming service. - if (!AppConfig.HasNameserviceConfig()) + if (!AppConfig.HasNameserviceConfig()) { AppConfig.MutableNameserviceConfig()->Swap(appConfig.MutableNameserviceConfig()); + RunConfig.ConfigInitInfo[NKikimrConsole::TConfigItem::NameserviceConfigItem].Updates.pop_back(); + } } bool SaveConfigForNodeToCache(const NKikimrConfig::TAppConfig &appConfig) { @@ -1329,8 +1411,27 @@ protected: if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { appConfig = yamlConfig; NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), appConfig); + + for (ui32 kind = NKikimrConsole::TConfigItem::EKind_MIN; kind <= NKikimrConsole::TConfigItem::EKind_MAX; kind++) { + if (kind == NKikimrConsole::TConfigItem::Auto || !NKikimrConsole::TConfigItem::EKind_IsValid(kind)) { + continue; + } + if ((kind == NKikimrConsole::TConfigItem::NameserviceConfigItem && appConfig.HasNameserviceConfig()) + || (kind == NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem && appConfig.HasNetClassifierDistributableConfig()) + || (kind == NKikimrConsole::TConfigItem::NamedConfigsItem && appConfig.NamedConfigsSize())) { + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleProto); + } else { + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleYaml); + } + } } else { appConfig = result.GetConfig(); + for (ui32 kind = NKikimrConsole::TConfigItem::EKind_MIN; kind <= NKikimrConsole::TConfigItem::EKind_MAX; kind++) { + if (kind == NKikimrConsole::TConfigItem::Auto || !NKikimrConsole::TConfigItem::EKind_IsValid(kind)) { + continue; + } + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleProto); + } } if (RunConfig.PathToConfigCacheFile) { diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index 00dcdc5d4a..85fca864c7 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -2,6 +2,7 @@ #include <ydb/core/protos/config.pb.h> #include <ydb/core/base/event_filter.h> +#include <ydb/core/cms/console/config_item_info.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> #include <util/generic/hash.h> @@ -127,7 +128,6 @@ union TBasicKikimrServicesMask { static_assert(sizeof(TBasicKikimrServicesMask) == 16, "expected sizeof(TBasicKikimrServicesMask) == 16"); - struct TKikimrRunConfig { NKikimrConfig::TAppConfig& AppConfig; ui32 NodeId; @@ -144,6 +144,7 @@ struct TKikimrRunConfig { NKikimrConfig::TAppConfig InitialCmsConfig; NKikimrConfig::TAppConfig InitialCmsYamlConfig; + THashMap<ui32, TConfigItemInfo> ConfigInitInfo; TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, ui32 nodeId = 0, const TKikimrScopeId& scopeId = {}); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 0abd416b84..c09b59d292 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2440,11 +2440,12 @@ TConfigsDispatcherInitializer::TConfigsDispatcherInitializer(const TKikimrRunCon , Labels(runConfig.Labels) , InitialCmsConfig(runConfig.InitialCmsConfig) , InitialCmsYamlConfig(runConfig.InitialCmsYamlConfig) + , ConfigInitInfo(runConfig.ConfigInitInfo) { } void TConfigsDispatcherInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig); + IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig, ConfigInitInfo); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( NConsole::MakeConfigsDispatcherID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 079936ca35..4f3b622d8b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -479,6 +479,7 @@ private: TMap<TString, TString> Labels; NKikimrConfig::TAppConfig InitialCmsConfig; NKikimrConfig::TAppConfig InitialCmsYamlConfig; + THashMap<ui32, TConfigItemInfo> ConfigInitInfo; }; class TConfigsCacheInitializer : public IKikimrServicesInitializer { |