diff options
| author | Innokentii Mokin <[email protected]> | 2026-06-08 22:35:46 +0400 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-06-08 21:35:46 +0300 |
| commit | 1e8cdec56900ee856c25a66a49014e3014529d44 (patch) | |
| tree | f5a9ffbdf1589b30c6cde3b6a1a08a775690305c | |
| parent | 37da33fd7f782c86883ec7469f284819dbb8b68d (diff) | |
Highlight unknown config fields in console + dispatcher dev UI (#42504)
| -rw-r--r-- | ydb/core/cms/console/configs_dispatcher.cpp | 57 | ||||
| -rw-r--r-- | ydb/core/cms/console/configs_dispatcher_ut.cpp | 78 | ||||
| -rw-r--r-- | ydb/core/cms/console/console__get_yaml_config.cpp | 3 | ||||
| -rw-r--r-- | ydb/core/cms/console/console__replace_yaml_config.cpp | 9 | ||||
| -rw-r--r-- | ydb/core/cms/console/console__scheme.h | 4 | ||||
| -rw-r--r-- | ydb/core/cms/console/console_configs_manager.cpp | 93 | ||||
| -rw-r--r-- | ydb/core/cms/console/console_configs_manager.h | 10 | ||||
| -rw-r--r-- | ydb/core/cms/console/console_ut_configs.cpp | 136 | ||||
| -rw-r--r-- | ydb/core/cms/ui/cms.css | 93 | ||||
| -rw-r--r-- | ydb/core/cms/ui/common.js | 236 | ||||
| -rw-r--r-- | ydb/core/cms/ui/configs_dispatcher_main.js | 6 | ||||
| -rw-r--r-- | ydb/core/cms/ui/index.html | 1 | ||||
| -rw-r--r-- | ydb/core/cms/ui/yaml_config.js | 7 | ||||
| -rw-r--r-- | ydb/core/protos/console_config.proto | 17 | ||||
| -rw-r--r-- | ydb/library/yaml_config/yaml_config.cpp | 5 | ||||
| -rw-r--r-- | ydb/library/yaml_config/yaml_config.h | 3 |
16 files changed, 733 insertions, 25 deletions
diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index b4f535eb263..de10d7c8d20 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -22,10 +22,12 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_writer.h> +#include <library/cpp/protobuf/json/proto2json.h> #include <util/generic/bitmap.h> #include <util/generic/ptr.h> #include <util/string/join.h> +#include <util/string/subst.h> #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR || defined BLOG_TRACE #error log macro definition clash @@ -325,6 +327,8 @@ private: TString ResolvedJsonConfig; NKikimrConfig::TAppConfig YamlProtoConfig; bool YamlConfigEnabled = false; + // Unknown/deprecated fields detected in the resolved config (recomputed on each change). + NKikimrConsole::TYamlConfigUnknownFields ResolvedConfigUnknownFields; }; @@ -439,11 +443,28 @@ TConfigsDispatcher::TSubscriber::TPtr TConfigsDispatcher::FindSubscriber(TActorI return nullptr; } +static NJson::TJsonValue UnknownFieldsToJsonArray(const NKikimrConsole::TYamlConfigUnknownFields& fields) { + NProtobufJson::TProto2JsonConfig cfg; + cfg.SetFieldNameMode(NProtobufJson::TProto2JsonConfig::FieldNameSnakeCaseDense); + + NJson::TJsonValue array(NJson::JSON_ARRAY); + for (const auto& f : fields.GetFields()) { + NJson::TJsonValue item; + NProtobufJson::Proto2Json(f, item, cfg); + array.AppendValue(std::move(item)); + } + return array; +} + NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() { NKikimrConfig::TAppConfig newYamlProtoConfig = {}; + ResolvedConfigUnknownFields.Clear(); + try { + auto unknownFieldsCollector = MakeSimpleShared<NYamlConfig::TBasicUnknownFieldsCollector>(); + NYamlConfig::ResolveAndParseYamlConfig( MainYamlConfig, VolatileYamlConfigs, @@ -451,7 +472,17 @@ NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() newYamlProtoConfig, DatabaseYamlConfig, &ResolvedYamlConfig, - &ResolvedJsonConfig); + &ResolvedJsonConfig, + unknownFieldsCollector); + + const auto& deprecatedPaths = NKikimrConfig::TAppConfig::GetReservedChildrenPaths(); + for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { + auto *f = ResolvedConfigUnknownFields.AddFields(); + f->SetPath(path); + f->SetName(info.first); + f->SetProto(info.second); + f->SetDeprecated(deprecatedPaths.contains(path)); + } } catch (const yexception& ex) { BLOG_ERROR("Got invalid config from console error# " << ex.what()); } @@ -491,6 +522,8 @@ void TConfigsDispatcher::ReplyMonJson(TActorId mailbox) { response.InsertValue("yaml_config", MainYamlConfig); response.InsertValue("resolved_json_config", NJson::ReadJsonFastTree(ResolvedJsonConfig, true)); + + response.InsertValue("unknown_fields", UnknownFieldsToJsonArray(ResolvedConfigUnknownFields)); response.InsertValue("current_json_config", NJson::ReadJsonFastTree(SecureProto2JsonString(CurrentConfig, NYamlConfig::GetProto2JsonConfig()), true)); auto state = GetState(); @@ -568,8 +601,22 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) str << "{'nodeName':'" << node.Host << "'}, "; } - str << "];" << Endl - << "</script>" << Endl + str << "];" << Endl; + + // path/name/proto come from user-uploaded YAML keys, so emitting them raw into a JS + // string literal would allow breaking out of the string or injecting script. JSON + // encoding escapes that; we additionally escape "</" so a value cannot terminate the + // surrounding <script> block prematurely. + { + NJson::TJsonValue unknownFieldsJson = UnknownFieldsToJsonArray(ResolvedConfigUnknownFields); + TStringStream unknownFieldsStream; + NJson::WriteJson(&unknownFieldsStream, &unknownFieldsJson, {}); + TString unknownFieldsStr = unknownFieldsStream.Str(); + SubstGlobal(unknownFieldsStr, "</", "<\\/"); + str << "var unknownFields = " << unknownFieldsStr << ";" << Endl; + } + + str << "</script>" << Endl << "<script src='../cms/ext/fuse.min.js'></script>" << Endl << "<script src='../cms/common.js'></script>" << Endl << "<script src='../cms/ext/fuzzycomplete.min.js'></script>" << Endl @@ -859,6 +906,10 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) } } str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("resolved-unknown-fields", "Unknown fields") { + TAG_ATTRS(TDiv, {{"id", "resolved-unknown-fields-list"}, {"class", "unknown-fields-list"}}) { } + } + str << "<br />" << Endl; COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { diff --git a/ydb/core/cms/console/configs_dispatcher_ut.cpp b/ydb/core/cms/console/configs_dispatcher_ut.cpp index a8347031a6d..fb4a166c4b2 100644 --- a/ydb/core/cms/console/configs_dispatcher_ut.cpp +++ b/ydb/core/cms/console/configs_dispatcher_ut.cpp @@ -1556,6 +1556,83 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherObservabilityTests) { UNIT_ASSERT(!state.HasStorageYaml); } + // The dispatcher computes unknown/deprecated fields for the *resolved* config it + // receives from the console and exposes them in its mon JSON ("unknown_fields"). + Y_UNIT_TEST(TestMonJsonReportsResolvedUnknownFields) { + // The dispatcher (re)computes unknown/deprecated fields in ParseYamlProtoConfig, + // which only runs when it receives a config-subscription notification carrying a + // changed YAML body. That notification is only produced once the dispatcher has a + // subscriber, so we must: replace the YAML on the console (allowing the unknown + // field), then add a subscriber and wait for it to be notified -- by which point + // the dispatcher has resolved the config and filled ResolvedConfigUnknownFields. + NKikimrConfig::TAppConfig bootstrapConfig; + auto *label = bootstrapConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + + const TString yamlWithUnknown = R"( +--- +metadata: + cluster: "" + version: 0 + +config: + log_config: + cluster_name: cluster1 + yaml_config_enabled: true + unknown_field_for_test: 42 + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), bootstrapConfig); + TAutoPtr<IEventHandle> handle; + const TActorId dispatcherId = InitConfigsDispatcher(runtime); + + { + auto *event = new TEvConsole::TEvReplaceYamlConfigRequest; + event->Record.MutableRequest()->set_config(yamlWithUnknown); + event->Record.MutableRequest()->set_allow_unknown_fields(true); + runtime.SendToConsole(event); + + runtime.GrabEdgeEventRethrow<TEvConsole::TEvReplaceYamlConfigResponse>(handle); + } + + // Subscribing drives the dispatcher to fetch & resolve the YAML config; grabbing the + // subscriber's notification guarantees ParseYamlProtoConfig has already run. + AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); + runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + + const TActorId edge = runtime.AllocateEdgeActor(); + auto request = MakeHolder<THttpRequest>(HTTP_METHOD_GET); + request->HttpHeaders.AddHeader("Content-Type", "application/json"); + NMonitoring::TMonService2HttpRequest monReq(nullptr, request.Get(), nullptr, nullptr, "", nullptr); + runtime.Send(new IEventHandle(dispatcherId, edge, new NMon::TEvHttpInfo(monReq))); + + const auto* response = runtime.GrabEdgeEventRethrow<NMon::TEvHttpInfoRes>(handle); + const TString& answer = response->Answer; + + const size_t jsonBegin = answer.find('{'); + UNIT_ASSERT_UNEQUAL(jsonBegin, TString::npos); + const NJson::TJsonValue json = ReadJsonFromString(answer.substr(jsonBegin)); + + UNIT_ASSERT_C(json.Has("unknown_fields"), "no unknown_fields in mon json: " << answer); + bool found = false; + for (const auto& f : json["unknown_fields"].GetArray()) { + if (f["name"].GetString() == "unknown_field_for_test") { + found = true; + UNIT_ASSERT_VALUES_EQUAL(f["deprecated"].GetBoolean(), false); + } + } + UNIT_ASSERT_C(found, "unknown_field_for_test not reported: " << answer); + } + Y_UNIT_TEST(TestMonJsonMasksSensitiveFieldsInDebugInfo) { NKikimrConfig::TAppConfig config; @@ -1609,6 +1686,7 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherObservabilityTests) { "current_json_config": {}, "has_storage_yaml": false, "yaml_config": "", + "unknown_fields": [], "initial_json_config": { "grpc_config": { "key": "***" diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index bcde997152a..5fc9376ebfa 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -39,6 +39,9 @@ public: identity.set_version(Self->YamlVersion); Response->Record.MutableResponse()->add_config(Self->MainYamlConfig); + // Unknown/deprecated fields of the main config, cached at upload time. + *Response->Record.MutableMainConfigUnknownFields() = Self->MainYamlConfigUnknownFields.GetFields(); + for (const auto& [database, config] : Self->DatabaseYamlConfigs) { auto& dbIdentity = *Response->Record.MutableResponse()->add_identity(); dbIdentity.set_database(database); diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 311da23c1d7..2a9c3d049ba 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -134,6 +134,10 @@ public: Cluster = opCtx.Cluster; Modify = opCtx.UpdatedConfig != Self->MainYamlConfig || Self->YamlDropped; + // Snapshot unknown/deprecated fields detected at upload time so the UI can + // highlight them later without re-resolving the (expensive) config on each view. + BuildYamlConfigUnknownFields(opCtx.UnknownFields, opCtx.DeprecatedFields, UpdatedUnknownFields); + if (IngressDatabase) { WarnDatabaseBypass = true; } @@ -146,7 +150,8 @@ public: // set config dropped by default to support rollback to previous versions // where new config layout is not supported // it will lead to ignoring config from new versions - .Update<Schema::YamlConfig::Dropped>(true); + .Update<Schema::YamlConfig::Dropped>(true) + .Update<Schema::YamlConfig::UnknownFields>(UpdatedUnknownFields.SerializeAsString()); /* Later we shift this boundary to support rollback and history */ db.Table<Schema::YamlConfig>().Key(Version) @@ -194,6 +199,7 @@ public: Self->YamlVersion = Version + 1; Self->MainYamlConfig = UpdatedMainConfig; + Self->MainYamlConfigUnknownFields = std::move(UpdatedUnknownFields); Self->YamlDropped = false; Self->VolatileYamlConfigs.clear(); @@ -240,6 +246,7 @@ private: ui32 Version; TString Cluster; TString UpdatedMainConfig; + NKikimrConsole::TYamlConfigUnknownFields UpdatedUnknownFields; }; class TConfigsManager::TTxReplaceDatabaseYamlConfig diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index cd528844ece..85624700ee4 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -160,9 +160,11 @@ struct Schema : NIceDb::Schema { struct Version : Column<1, NScheme::NTypeIds::Uint64> {}; struct Config : Column<2, NScheme::NTypeIds::String> {}; struct Dropped : Column<3, NScheme::NTypeIds::Bool> {}; + // serialized NKikimrConsole::TYamlConfigUnknownFields snapshot taken at upload time + struct UnknownFields : Column<4, NScheme::NTypeIds::String> {}; using TKey = TableKey<Version>; - using TColumns = TableColumns<Version, Config, Dropped>; + using TColumns = TableColumns<Version, Config, Dropped, UnknownFields>; }; struct DatabaseYamlConfigs : Table<104> { diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 03fc6ded833..3f26d4cb65e 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -75,6 +75,28 @@ void TConfigsManager::ReplaceMainConfigMetadata(const TString &config, bool forc } } +void BuildYamlConfigUnknownFields( + const TMap<TString, std::pair<TString, TString>>& unknownFields, + const TMap<TString, std::pair<TString, TString>>& deprecatedFields, + NKikimrConsole::TYamlConfigUnknownFields& out) +{ + out.Clear(); + for (const auto& [path, info] : unknownFields) { + auto *f = out.AddFields(); + f->SetPath(path); + f->SetName(info.first); + f->SetProto(info.second); + f->SetDeprecated(false); + } + for (const auto& [path, info] : deprecatedFields) { + auto *f = out.AddFields(); + f->SetPath(path); + f->SetName(info.first); + f->SetProto(info.second); + f->SetDeprecated(true); + } +} + void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { try { // Re-applying an unchanged body with the same version is silently accepted @@ -93,32 +115,63 @@ void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { } auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); - TSimpleSharedPtr<NYamlConfig::TBasicUnknownFieldsCollector> unknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector; + // Collect unknown/deprecated fields per editable location so the UI can point at + // (and tint the parents of) the exact place a field lives in the document, including + // fields nested inside selector_config entries. Paths mirror the editable YAML: + // "/config/..." for the base config and "/selector_config/<i>/config/..." for the + // i-th selector. This is best-effort and must never reject a config -- acceptance is + // decided solely by the resolved-doc validation below. + const auto& deprecatedPaths = NKikimrConfig::TAppConfig::GetReservedChildrenPaths(); + auto collectBlock = [&](const NFyaml::TNodeRef& configNode, const TString& prefix) { + auto collector = MakeSimpleShared<NYamlConfig::TBasicUnknownFieldsCollector>(prefix); + try { + NYamlConfig::YamlToProto(configNode, true, true, collector); + } catch (const std::exception&) { + // A partial selector fragment may not transform standalone; ignore. + } + for (const auto& [path, info] : collector->GetUnknownKeys()) { + // Reserved (deprecated) paths are config-content-relative; strip the + // location prefix ("/<prefix>") before matching. + const TString leafPath = path.substr(prefix.size() + 1); + if (deprecatedPaths.contains(leafPath)) { + opCtx.DeprecatedFields[path] = info; + } else { + opCtx.UnknownFields[path] = info; + } + } + }; + + try { + auto root = tree.Root().Map(); + if (root.Has("config")) { + collectBlock(root.at("config"), "config"); + } + if (root.Has("selector_config")) { + auto selectors = root.at("selector_config").Sequence(); + for (size_t i = 0; i < selectors.size(); ++i) { + auto item = selectors.at(static_cast<int>(i)).Map(); + if (item.Has("config")) { + collectBlock(item.at("config"), + TStringBuilder() << "selector_config/" << i << "/config"); + } + } + } + } catch (const std::exception&) { + // Best-effort field collection; never blocks config acceptance. + } + + // Validate the fully resolved configuration. This decides accept/reject. std::vector<TString> errors; NYamlConfig::ResolveUniqueDocs( tree, [&](NYamlConfig::TDocumentConfig&& config) { - auto cfg = NYamlConfig::YamlToProto( - config.second, - true, - true, - unknownFieldsCollector); + auto cfg = NYamlConfig::YamlToProto(config.second, true, true); NKikimr::NConfig::EValidationResult result = NKikimr::NConfig::ValidateConfig(cfg, errors); if (result == NKikimr::NConfig::EValidationResult::Error) { ythrow yexception() << errors.front(); } }); - - const auto& deprecatedPaths = NKikimrConfig::TAppConfig::GetReservedChildrenPaths(); - - for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { - if (deprecatedPaths.contains(path)) { - opCtx.DeprecatedFields[path] = info; - } else { - opCtx.UnknownFields[path] = info; - } - } } } catch (const yexception &e) { opCtx.Error = e.what(); @@ -549,6 +602,14 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, // ignore this as deprecated // now used only for disabling new config layout for older console YamlDropped = false; + + // Restore the unknown-fields snapshot cached at upload time (no re-validation). + MainYamlConfigUnknownFields.Clear(); + const TString serializedUnknownFields = + yamlConfigRowset.template GetValueOrDefault<Schema::YamlConfig::UnknownFields>(TString()); + if (serializedUnknownFields) { + Y_PROTOBUF_SUPPRESS_NODISCARD MainYamlConfigUnknownFields.ParseFromString(serializedUnknownFields); + } } while (!databaseYamlConfigRowset.EndOfSet()) { diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index 3827804896f..86aa9be6d4f 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -29,6 +29,13 @@ using NTabletFlatExecutor::TTransactionContext; class TConsole; +// Converts the unknown/deprecated field maps collected during YAML config validation +// into the persisted/served proto snapshot. Deprecated fields get Deprecated=true. +void BuildYamlConfigUnknownFields( + const TMap<TString, std::pair<TString, TString>>& unknownFields, + const TMap<TString, std::pair<TString, TString>>& deprecatedFields, + NKikimrConsole::TYamlConfigUnknownFields& out); + class TConfigsManager : public TActorBootstrapped<TConfigsManager> { private: @@ -350,6 +357,9 @@ private: TString ClusterName; ui32 YamlVersion = 0; TString MainYamlConfig; + // Unknown/deprecated fields of MainYamlConfig, cached from the moment the config was uploaded + // (computed during ValidateMainConfig, persisted in Schema::YamlConfig::UnknownFields). + NKikimrConsole::TYamlConfigUnknownFields MainYamlConfigUnknownFields; THashMap<TString, TDatabaseYamlConfig> DatabaseYamlConfigs; bool YamlDropped = false; bool YamlReadOnly = true; diff --git a/ydb/core/cms/console/console_ut_configs.cpp b/ydb/core/cms/console/console_ut_configs.cpp index 6da2cd933b9..ecd5b99fcc2 100644 --- a/ydb/core/cms/console/console_ut_configs.cpp +++ b/ydb/core/cms/console/console_ut_configs.cpp @@ -708,6 +708,38 @@ config: cluster_name: cluster1 )"; +const TString YAML_CONFIG_WITH_UNKNOWN = R"( +--- +metadata: + cluster: "" + version: 0 +config: + log_config: + cluster_name: cluster1 + unknown_field_for_test: 42 +)"; + +const TString YAML_CONFIG_WITH_UNKNOWN_IN_SELECTOR = R"( +--- +metadata: + cluster: "" + version: 0 +config: + log_config: + cluster_name: cluster1 +allowed_labels: + test: + type: enum + values: + ? true +selector_config: +- description: selector with unknown + selector: + test: true + config: + unknown_in_selector: 7 +)"; + const TString YAML_CONFIG_2 = R"( --- metadata: @@ -1227,13 +1259,17 @@ void DoReplaceYamlConfig(TTenantTestRuntime &runtime, const TString &yaml, Ydb::StatusIds::StatusCode expectedCode, const TString &errorSubstring = {}, - bool allowAbsentDatabase = false) + bool allowAbsentDatabase = false, + bool allowUnknownFields = false) { auto *event = new TEvConsole::TEvReplaceYamlConfigRequest; event->Record.MutableRequest()->set_config(yaml); if (allowAbsentDatabase) { event->Record.MutableRequest()->set_allow_absent_database(true); } + if (allowUnknownFields) { + event->Record.MutableRequest()->set_allow_unknown_fields(true); + } runtime.SendToConsole(event); TAutoPtr<IEventHandle> handle; @@ -1281,6 +1317,104 @@ void DoEnsureDatabaseConfigReplacedWith(TTenantTestRuntime &runtime, const TStri } // anonymous namespace Y_UNIT_TEST_SUITE(TConsoleConfigTests) { + // Unit test for the unknown/deprecated -> proto conversion helper. + Y_UNIT_TEST(BuildYamlConfigUnknownFieldsHelper) { + TMap<TString, std::pair<TString, TString>> unknown{ + {"/config/foo", {"foo", "NKikimrConfig.TAppConfig"}}, + }; + TMap<TString, std::pair<TString, TString>> deprecated{ + {"/config/old", {"old", "NKikimrConfig.TAppConfig"}}, + }; + + NKikimrConsole::TYamlConfigUnknownFields out; + BuildYamlConfigUnknownFields(unknown, deprecated, out); + + UNIT_ASSERT_VALUES_EQUAL(out.FieldsSize(), 2u); + THashMap<TString, NKikimrConsole::TYamlConfigUnknownField> byName; + for (const auto& f : out.GetFields()) { + byName[f.GetName()] = f; + } + UNIT_ASSERT_C(byName.contains("foo"), "missing unknown field 'foo'"); + UNIT_ASSERT_VALUES_EQUAL(byName.at("foo").GetPath(), "/config/foo"); + UNIT_ASSERT_VALUES_EQUAL(byName.at("foo").GetProto(), "NKikimrConfig.TAppConfig"); + UNIT_ASSERT_VALUES_EQUAL(byName.at("foo").GetDeprecated(), false); + UNIT_ASSERT_C(byName.contains("old"), "missing deprecated field 'old'"); + UNIT_ASSERT_VALUES_EQUAL(byName.at("old").GetDeprecated(), true); + } + + // Uploading a config with an unknown field (allow_unknown_fields) caches it and + // GetAllConfigs returns it. + Y_UNIT_TEST(YamlConfigUnknownFieldsRoundTrip) { + TTenantTestRuntime runtime(MultipleNodesConsoleTestConfig()); + + DoReplaceYamlConfig(runtime, YAML_CONFIG_WITH_UNKNOWN, Ydb::StatusIds::SUCCESS, + /* errorSubstring = */ {}, /* allowAbsentDatabase = */ false, /* allowUnknownFields = */ true); + + auto *event = new TEvConsole::TEvGetAllConfigsRequest; + runtime.SendToConsole(event); + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvConsole::TEvGetAllConfigsResponse>(handle); + + const auto& unknown = response->Record.GetMainConfigUnknownFields(); + UNIT_ASSERT_VALUES_EQUAL(unknown.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetName(), "unknown_field_for_test"); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetDeprecated(), false); + } + + // The cached unknown fields are persisted and survive a console tablet restart + // (no re-validation needed on load). + Y_UNIT_TEST(YamlConfigUnknownFieldsSurviveReboot) { + TTenantTestRuntime runtime(MultipleNodesConsoleTestConfig()); + + DoReplaceYamlConfig(runtime, YAML_CONFIG_WITH_UNKNOWN, Ydb::StatusIds::SUCCESS, + /* errorSubstring = */ {}, /* allowAbsentDatabase = */ false, /* allowUnknownFields = */ true); + + GracefulRestartTablet(runtime, MakeConsoleID(), runtime.AllocateEdgeActor(0)); + + auto *event = new TEvConsole::TEvGetAllConfigsRequest; + runtime.SendToConsole(event); + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvConsole::TEvGetAllConfigsResponse>(handle); + + const auto& unknown = response->Record.GetMainConfigUnknownFields(); + UNIT_ASSERT_VALUES_EQUAL(unknown.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetName(), "unknown_field_for_test"); + } + + // A clean config yields no unknown fields. + Y_UNIT_TEST(YamlConfigNoUnknownFieldsWhenClean) { + TTenantTestRuntime runtime(MultipleNodesConsoleTestConfig()); + + DoReplaceYamlConfig(runtime, YAML_CONFIG_1, Ydb::StatusIds::SUCCESS); + + auto *event = new TEvConsole::TEvGetAllConfigsRequest; + runtime.SendToConsole(event); + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvConsole::TEvGetAllConfigsResponse>(handle); + + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetMainConfigUnknownFields().size(), 0); + } + + // An unknown field nested inside a selector_config entry is reported with a path that + // pinpoints the selector, so the UI can highlight it and its parents in the editable YAML. + Y_UNIT_TEST(YamlConfigUnknownFieldsInSelectorHaveSelectorPath) { + TTenantTestRuntime runtime(MultipleNodesConsoleTestConfig()); + + DoReplaceYamlConfig(runtime, YAML_CONFIG_WITH_UNKNOWN_IN_SELECTOR, Ydb::StatusIds::SUCCESS, + /* errorSubstring = */ {}, /* allowAbsentDatabase = */ false, /* allowUnknownFields = */ true); + + auto *event = new TEvConsole::TEvGetAllConfigsRequest; + runtime.SendToConsole(event); + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvConsole::TEvGetAllConfigsResponse>(handle); + + const auto& unknown = response->Record.GetMainConfigUnknownFields(); + UNIT_ASSERT_VALUES_EQUAL(unknown.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetName(), "unknown_in_selector"); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetPath(), "/selector_config/0/config/unknown_in_selector"); + UNIT_ASSERT_VALUES_EQUAL(unknown[0].GetDeprecated(), false); + } + Y_UNIT_TEST(TestAddConfigItem) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); InitializeTestConfigItems(); diff --git a/ydb/core/cms/ui/cms.css b/ydb/core/cms/ui/cms.css index 0877344afce..74601cd2b0f 100644 --- a/ydb/core/cms/ui/cms.css +++ b/ydb/core/cms/ui/cms.css @@ -549,3 +549,96 @@ td { border-color: rgb(172, 41, 37); color: white; } + +/* Unknown / deprecated config field highlighting (monaco inline decorations). */ +.unknown-field-red { + background: rgba(255, 0, 0, 0.18); + border-bottom: 2px solid #d33; +} + +.deprecated-field-amber { + background: rgba(255, 191, 0, 0.20); + border-bottom: 2px solid #d8a200; +} + +/* Consequence lines: parents and children of an unknown/deprecated field. A light whole-line + tint so a collapsed parent or a nested child still signals there is something invalid in the + block; these are only fallout from the real issue. */ +.unknown-consequence { + background: rgba(255, 0, 0, 0.06); +} + +.deprecated-consequence { + background: rgba(255, 191, 0, 0.07); +} + +/* Full-strength left gutter bar on the offending field itself. */ +.unknown-field-red-gutter { + background: #d33; + width: 3px !important; + margin-left: 3px; +} + +.deprecated-field-amber-gutter { + background: #d8a200; + width: 3px !important; + margin-left: 3px; +} + +/* Dimmed (50%) gutter bar for consequence lines (parents/children) -- only a consequence of + the original issue, so it is half as bright as the field's own bar. */ +.unknown-consequence-gutter { + background: rgba(221, 51, 51, 0.5); + width: 3px !important; + margin-left: 3px; +} + +.deprecated-consequence-gutter { + background: rgba(216, 162, 0, 0.5); + width: 3px !important; + margin-left: 3px; +} + +/* Unknown-fields list widget. */ +.unknown-fields-list { + padding: 4px 12px; +} + +.unknown-fields-ul { + list-style: none; + padding-left: 0; + margin: 0; +} + +.unknown-fields-ul li { + padding: 2px 0; + font-family: monospace; +} + +.unknown-fields-clickable { + cursor: pointer; +} + +.unknown-fields-clickable:hover { + text-decoration: underline; +} + +.unknown-fields-badge { + display: inline-block; + min-width: 78px; + text-align: center; + border-radius: 4px; + font-size: 11px; + line-height: 16px; + padding: 0 6px; + margin-right: 6px; + color: #fff; +} + +.unknown-field-item .unknown-fields-badge { + background: #d33; +} + +.deprecated-field-item .unknown-fields-badge { + background: #d8a200; +} diff --git a/ydb/core/cms/ui/common.js b/ydb/core/cms/ui/common.js index 0b5c20f63be..7c27ce1cb20 100644 --- a/ydb/core/cms/ui/common.js +++ b/ydb/core/cms/ui/common.js @@ -107,3 +107,239 @@ function copyToClipboard(textToCopy) { }); } } + +// Locate the editor ranges for a collector path like +// "/selector_config/2/config/blob_storage_config/foo": the leaf key range plus the range of +// each ancestor (keys *and* sequence items, e.g. the selector entry) found along the path. +// Numeric segments are array indices and are descended into the matching sequence item, so a +// field nested inside a selector_config entry is located in the right entry and its parents +// (selector_config, that selector, config, ...) can be tinted. Returns +// {leaf: monaco range or null, ancestors: [monaco range, ...]}. +function findUnknownFieldRanges(model, path, name) { + if (!model) { + return {leaf: null, ancestors: []}; + } + var segments = (path || '').split('/').filter(function(s) { + return s.length > 0; // keep array indices so selector entries can be located + }); + if (segments.length === 0) { + segments = [name]; + } + var leaf = segments[segments.length - 1]; + + // Match a YAML key at the start of a line, optionally right after a "- " list marker + // (e.g. " foo:" or "- foo:"), at or after fromLine. + function keyMatch(key, fromLine) { + var escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + var matches = model.findMatches('^\\s*(-\\s+)?' + escaped + '\\s*:', false, true, false, null, true); + for (var i = 0; i < matches.length; ++i) { + if (matches[i].range.startLineNumber >= fromLine) { + return matches[i].range; + } + } + return null; + } + + function indentOf(lineNumber) { + return model.getLineContent(lineNumber).match(/^\s*/)[0].length; + } + + // Locate the (index)-th item of the YAML sequence whose items start at/after fromLine. + // Sequence items begin with "- "; lock onto the indentation of the first such item and + // count only siblings at that indentation, stopping when the block dedents. + function arrayItemMatch(index, fromLine) { + var matches = model.findMatches('^\\s*-\\s', false, true, false, null, true); + var siblings = []; + var indent = null; + for (var i = 0; i < matches.length; ++i) { + var r = matches[i].range; + if (r.startLineNumber < fromLine) { + continue; + } + var ind = indentOf(r.startLineNumber); + if (indent === null) { + indent = ind; + } + if (ind < indent) { + break; // sequence ended (dedent) + } + if (ind === indent) { + siblings.push(r); // deeper markers belong to a nested sequence -> skip + } + } + return siblings[index] || null; + } + + var fromLine = 1; + var ancestors = []; + // Descend ancestors to scope the search to the correct subtree, collecting each. + for (var i = 0; i < segments.length - 1; ++i) { + var seg = segments[i]; + var anc = /^\d+$/.test(seg) + ? arrayItemMatch(parseInt(seg, 10), fromLine) + : keyMatch(seg, fromLine); + if (anc) { + ancestors.push(anc); + // Continue from this ancestor's line (inclusive) so a key sharing the line with a + // "- " marker (e.g. "- config:") is still matched on the next descent step. + fromLine = anc.startLineNumber; + } + } + return {leaf: keyMatch(leaf, fromLine), ancestors: ancestors}; +} + +// Back-compat thin wrapper: just the leaf range (or null). +function findUnknownFieldRange(model, path, name) { + return findUnknownFieldRanges(model, path, name).leaf; +} + +function lineIndent(model, lineNumber) { + return model.getLineContent(lineNumber).match(/^\s*/)[0].length; +} + +// Whole-line ranges for every descendant of the field whose key is at leafRange: the indented +// block beneath it (blanks skipped) until the YAML dedents back to the field's own level. Lets +// us mark the field's whole subtree as a consequence of the same issue. +function childLineRanges(model, leafRange) { + var startLn = leafRange.startLineNumber; + var baseIndent = lineIndent(model, startLn); + var total = model.getLineCount(); + var ranges = []; + for (var ln = startLn + 1; ln <= total; ++ln) { + if (/^\s*$/.test(model.getLineContent(ln))) { + continue; // blank line inside the block + } + if (lineIndent(model, ln) <= baseIndent) { + break; // dedent -> end of this field's subtree + } + ranges.push({startLineNumber: ln, startColumn: 1, endLineNumber: ln, endColumn: 1}); + } + return ranges; +} + +// Highlights unknown (red) / deprecated (amber) fields in a monaco editor and +// renders a clickable list into listContainer. Clicking a list item scrolls the +// editor to the field occurrence. +// editor - monaco editor instance +// listContainer - DOM element (or null) to render the list into +// fields - [{path, name, proto, deprecated}] +function highlightUnknownFields(editor, listContainer, fields) { + if (!editor || typeof monaco === 'undefined') { + return; + } + var model = editor.getModel(); + fields = fields || []; + + var decorations = []; + var located = []; + var leafLines = {}; // line numbers that carry a strong (leaf) highlight + var consequenceByLine = {}; // line -> {range, unknown}: parents + children of a field + + // A consequence line (ancestor or descendant of a flagged field) gets a light tint and a + // dimmed gutter -- it is only fallout from the real issue. Unknown (red) outranks + // deprecated (amber) when a line is fallout from both kinds. + function markConsequence(range, unknown) { + var ln = range.startLineNumber; + var prev = consequenceByLine[ln]; + consequenceByLine[ln] = { + range: range, + unknown: !!unknown || !!(prev && prev.unknown) + }; + } + + fields.forEach(function(f) { + var cls = f.deprecated ? 'deprecated-field-amber' : 'unknown-field-red'; + var found = findUnknownFieldRanges(model, f.path, f.name); + var range = found.leaf; + if (range) { + leafLines[range.startLineNumber] = true; + decorations.push({ + range: range, + options: { + inlineClassName: cls, + className: cls, + linesDecorationsClassName: cls + '-gutter', + isWholeLine: false, + overviewRuler: { + color: f.deprecated ? '#d8a200' : '#d33', + position: monaco.editor.OverviewRulerLane.Right + }, + hoverMessage: { + value: (f.deprecated ? 'Deprecated' : 'Unknown') + + ' field `' + (f.name || '') + '` in `' + (f.proto || '') + '`' + } + } + }); + // The field's own subtree is a consequence of the same issue. + childLineRanges(model, range).forEach(function(childRange) { + markConsequence(childRange, !f.deprecated); + }); + } + // Ancestors are consequences too: a collapsed parent still hints at trouble inside. + found.ancestors.forEach(function(anc) { + markConsequence(anc, !f.deprecated); + }); + located.push({field: f, range: range}); + }); + + // Light tint + dimmed gutter for consequence lines (parents and children), skipping lines + // that already carry a strong leaf highlight. + Object.keys(consequenceByLine).forEach(function(ln) { + if (leafLines[ln]) { + return; + } + var c = consequenceByLine[ln]; + var ccls = c.unknown ? 'unknown-consequence' : 'deprecated-consequence'; + decorations.push({ + range: c.range, + options: { + isWholeLine: true, + className: ccls, + linesDecorationsClassName: ccls + '-gutter', + overviewRuler: { + color: c.unknown ? 'rgba(221, 51, 51, 0.5)' : 'rgba(216, 162, 0, 0.5)', + position: monaco.editor.OverviewRulerLane.Left + }, + hoverMessage: { + value: 'Part of ' + (c.unknown ? 'an unknown' : 'a deprecated') + ' field' + } + } + }); + }); + + editor.__unknownFieldDecorations = editor.deltaDecorations( + editor.__unknownFieldDecorations || [], decorations); + + if (listContainer) { + listContainer.innerHTML = ''; + if (located.length === 0) { + listContainer.textContent = 'No unknown fields.'; + return; + } + var ul = document.createElement('ul'); + ul.className = 'unknown-fields-ul'; + located.forEach(function(item) { + var f = item.field; + var li = document.createElement('li'); + li.className = f.deprecated ? 'deprecated-field-item' : 'unknown-field-item'; + var badge = document.createElement('span'); + badge.className = 'unknown-fields-badge'; + badge.textContent = f.deprecated ? 'deprecated' : 'unknown'; + li.appendChild(badge); + li.appendChild(document.createTextNode(' ' + (f.path || f.name))); + if (item.range) { + li.classList.add('unknown-fields-clickable'); + li.onclick = function() { + editor.revealLineInCenter(item.range.startLineNumber); + editor.setPosition({ + lineNumber: item.range.startLineNumber, + column: item.range.startColumn + }); + editor.focus(); + }; + } + ul.appendChild(li); + }); + listContainer.appendChild(ul); + } +} diff --git a/ydb/core/cms/ui/configs_dispatcher_main.js b/ydb/core/cms/ui/configs_dispatcher_main.js index 28e57583b53..882476596d1 100644 --- a/ydb/core/cms/ui/configs_dispatcher_main.js +++ b/ydb/core/cms/ui/configs_dispatcher_main.js @@ -55,6 +55,12 @@ function main() { $("#copy-resolved-yaml-config").click(function() { copyToClipboard(codeMirrorResolved.getValue()); }); + + // Highlight unknown/deprecated fields of the resolved config (inlined by the server). + if (typeof unknownFields !== 'undefined') { + highlightUnknownFields(codeMirrorResolved, + document.getElementById('resolved-unknown-fields-list'), unknownFields); + } }); $("#host-ref").text("YDB Developer UI - " + window.location.hostname); diff --git a/ydb/core/cms/ui/index.html b/ydb/core/cms/ui/index.html index 7d42a9a7ff0..14e41a9148f 100644 --- a/ydb/core/cms/ui/index.html +++ b/ydb/core/cms/ui/index.html @@ -318,6 +318,7 @@ </div> <div id="main-editor-container"></div> </form> + <div id="yaml-unknown-fields" class="unknown-fields-list"></div> <button type="button" class="btn btn-success float-right disabled" id="yaml-apply-button" title="Set config field 'allow_edit_yaml_in_ui' to 'true' to enable this button">Apply</button> </div> diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index 69a05ce616f..0870f3b3580 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -247,6 +247,13 @@ class YamlConfigState { this.config = data.Response.config[0]; } + // Highlight unknown/deprecated fields (cached by console at upload time). + // GetAllConfigs serializes proto field names as-is (PascalCase) -> normalize. + this.unknownFields = (data.MainConfigUnknownFields || []).map(function(f) { + return { path: f.Path, name: f.Name, proto: f.Proto, deprecated: !!f.Deprecated }; + }); + highlightUnknownFields(this.codeMirror, document.getElementById('yaml-unknown-fields'), this.unknownFields); + if (data.Response.volatile_configs === undefined) { data.Response.volatile_configs = []; } diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 2c3ec39adf2..21e4e48af8b 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -326,9 +326,26 @@ message TGetAllConfigsRequest { optional bool BypassAuth = 5; } +// A single config field that the config parser did not recognize. +// Deprecated=false: genuinely unknown field (rendered red in the UI). +// Deprecated=true: reserved/removed proto slot (rendered amber in the UI). +message TYamlConfigUnknownField { + optional string Path = 1; // full path to the field, e.g. /config/blob_storage_config/foo + optional string Name = 2; // leaf key name, e.g. foo + optional string Proto = 3; // full name of the proto message that owns the slot + optional bool Deprecated = 4; +} + +// Persisted/cached snapshot of unknown fields detected when a config was uploaded. +message TYamlConfigUnknownFields { + repeated TYamlConfigUnknownField Fields = 1; +} + message TGetAllConfigsResponse { reserved 1; optional Ydb.DynamicConfig.GetConfigResult Response = 2; + // Unknown/deprecated fields of the main (cluster) config, cached from upload time. + repeated TYamlConfigUnknownField MainConfigUnknownFields = 3; } message TIsYamlReadOnlyRequest { diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp index bca2d583b58..027d55c391d 100644 --- a/ydb/library/yaml_config/yaml_config.cpp +++ b/ydb/library/yaml_config/yaml_config.cpp @@ -40,7 +40,8 @@ void ResolveAndParseYamlConfig( NKikimrConfig::TAppConfig& appConfig, std::optional<TString> databaseYamlConfig, TString* resolvedYamlConfig, - TString* resolvedJsonConfig) + TString* resolvedJsonConfig, + TSimpleSharedPtr<NProtobufJson::IUnknownFieldsCollector> unknownFieldsCollector) { TStringStream resolvedJsonConfigStream; bool hasMetadata = false; @@ -90,7 +91,7 @@ void ResolveAndParseYamlConfig( appConfig.SetYamlConfigEnabled(true); } - NYaml::Parse(json, NYaml::GetJsonToProtoConfig(true), appConfig, true, /*phase=*/ nullptr, /*relaxed=*/ true); + NYaml::Parse(json, NYaml::GetJsonToProtoConfig(true, std::move(unknownFieldsCollector)), appConfig, true, /*phase=*/ nullptr, /*relaxed=*/ true); } void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to) { diff --git a/ydb/library/yaml_config/yaml_config.h b/ydb/library/yaml_config/yaml_config.h index 5367d1ad4a8..b52de576b89 100644 --- a/ydb/library/yaml_config/yaml_config.h +++ b/ydb/library/yaml_config/yaml_config.h @@ -102,7 +102,8 @@ void ResolveAndParseYamlConfig( NKikimrConfig::TAppConfig& appConfig, std::optional<TString> databaseYamlConfig = std::nullopt, TString* resolvedYamlConfig = nullptr, - TString* resolvedJsonConfig = nullptr); + TString* resolvedJsonConfig = nullptr, + TSimpleSharedPtr<NProtobufJson::IUnknownFieldsCollector> unknownFieldsCollector = nullptr); enum class EValidationResult { Ok, |
