diff options
author | Alexander Rutkovsky <alexvru@ydb.tech> | 2025-02-20 15:28:40 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-02-20 15:28:40 +0300 |
commit | fe62452fd88a2c291517b6cf6a4a2ba7dfc54b65 (patch) | |
tree | aecf155b9f764e0c26d05257aadebfc925c3c28f | |
parent | 2c7f23f1d8285f50d0d488cfd365a209e395c374 (diff) | |
download | ydb-fe62452fd88a2c291517b6cf6a4a2ba7dfc54b65.tar.gz |
Implement dual-config mode for distconf (#14825)
19 files changed, 423 insertions, 192 deletions
diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp index 8a47b3d1a0..810ecc662a 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp @@ -43,29 +43,44 @@ namespace NKikimr::NStorage { // initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while // deriving it from NodeWarden configuration - if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) { + if (!Cfg->StartupConfigYaml) { return "missing initial config YAML"; } - TStringStream ss; - TString yaml = Cfg->SelfManagementConfig->GetInitialConfigYaml(); - ui32 version = 0; + + ui64 version = 0; try { - auto json = NYaml::Yaml2Json(YAML::Load(yaml), true); - if (json.Has("metadata")) { - if (auto& metadata = json["metadata"]; metadata.Has("version")) { - version = metadata["version"].GetUIntegerRobust(); - } - } + version = NYamlConfig::GetMainMetadata(Cfg->StartupConfigYaml).Version.value_or(0); } catch (const std::exception& ex) { - return TStringBuilder() << "failed to parse initial config YAML: " << ex.what(); + return TStringBuilder() << "failed to parse initial main YAML: " << ex.what(); } if (version) { - return TStringBuilder() << "initial config version must be zero"; + return TStringBuilder() << "initial main config version must be zero"; } - if (const auto& error = UpdateConfigComposite(*config, yaml, std::nullopt)) { + + if (const auto& error = UpdateConfigComposite(*config, Cfg->StartupConfigYaml, std::nullopt)) { return TStringBuilder() << "failed to update config yaml: " << *error; } + if (Cfg->StartupStorageYaml) { + ui64 storageVersion = 0; + try { + storageVersion = NYamlConfig::GetStorageMetadata(*Cfg->StartupStorageYaml).Version.value_or(0); + } catch (const std::exception& ex) { + return TStringBuilder() << "failed to parse initial storage YAML: " << ex.what(); + } + if (storageVersion) { + return TStringBuilder() << "initial storage config version must be zero"; + } + + TString s; + if (TStringOutput output(s); true) { + TZstdCompress zstd(&output); + zstd << *Cfg->StartupStorageYaml; + } + config->SetCompressedStorageYaml(s); + config->SetExpectedStorageYamlVersion(storageVersion + 1); + } + if (!Cfg->DomainsConfig) { // no automatic configuration required } else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it const auto& ss = Cfg->DomainsConfig->GetStateStorage(0); diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp index 8587895012..3de9d07123 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp @@ -665,11 +665,19 @@ namespace NKikimr::NStorage { ? std::make_optional(request.GetSwitchDedicatedStorageSection()) : std::nullopt; - const TString *storageYamlPtr = newStorageYaml ? &newStorageYaml.value() : - Self->StorageConfigYaml ? &Self->StorageConfigYaml.value() : nullptr; - const bool targetDedicatedStorageSection = switchDedicatedStorageSection.value_or(Self->StorageConfigYaml.has_value()); + if (switchDedicatedStorageSection) { + // check that configs are explicitly defined when we are switching dual-config mode + if (!NewYaml) { + return FinishWithError(TResult::ERROR, "main config must be specified when switching dedicated" + " storage section mode"); + } else if (*switchDedicatedStorageSection && !newStorageYaml) { + return FinishWithError(TResult::ERROR, "storage config must be specified when turning on dedicated" + " storage section mode"); + } + } + if (request.GetDedicatedStorageSectionConfigMode() != targetDedicatedStorageSection) { return FinishWithError(TResult::ERROR, "DedicatedStorageSectionConfigMode does not match target state"); } else if (newStorageYaml && !targetDedicatedStorageSection) { @@ -678,71 +686,97 @@ namespace NKikimr::NStorage { } else if (switchDedicatedStorageSection && *switchDedicatedStorageSection == Self->StorageConfigYaml.has_value()) { // this enable/disable command does not change the state return FinishWithError(TResult::ERROR, "dedicated storage config section is already in requested state"); - } else if (targetDedicatedStorageSection && !storageYamlPtr) { - // we are going to turn on dual-config mode, but no storage config provided - return FinishWithError(TResult::ERROR, "no dedicated storage config section provided"); } - const TString *mainYamlPtr = NewYaml ? &NewYaml.value() : &Self->MainConfigYaml; - - std::optional<ui64> newYamlVersion; - std::optional<ui64> newStorageYamlVersion; + TString state; + NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig); + std::optional<ui64> newExpectedStorageYamlVersion; - NKikimrConfig::TAppConfig appConfig; - const char *state = ""; + if (config.HasExpectedStorageYamlVersion()) { + newExpectedStorageYamlVersion.emplace(config.GetExpectedStorageYamlVersion()); + } try { - if (storageYamlPtr) { // parse the storage yaml first - state = "loading storage YAML"; - auto json = NYaml::Yaml2Json(YAML::Load(*storageYamlPtr), true); - state = "parsing storage YAML"; - NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true); - state = "extracting storage YAML metadata"; - if (json.Has("metadata")) { - if (auto& metadata = json["metadata"]; metadata.Has("version")) { - newStorageYamlVersion = metadata["version"].GetUIntegerRobust(); - } + auto load = [&](const TString& yaml, ui64& version, const char *expectedKind) { + state = TStringBuilder() << "loading " << expectedKind << " YAML"; + NJson::TJsonValue json = NYaml::Yaml2Json(YAML::Load(yaml), true); + + state = TStringBuilder() << "extracting " << expectedKind << " metadata"; + if (!json.Has("metadata") || !json["metadata"].IsMap()) { + throw yexception() << "no metadata section"; } + auto& metadata = json["metadata"]; + NYaml::ValidateMetadata(metadata); + if (!metadata.Has("kind") || metadata["kind"] != expectedKind) { + throw yexception() << "missing or invalid kind provided"; + } + version = metadata["version"].GetUIntegerRobust(); + + state = TStringBuilder() << "validating " << expectedKind << " config section"; + if (!json.Has("config") || !json["config"].IsMap()) { + throw yexception() << "missing config section"; + } + + return json; + }; + + NJson::TJsonValue main; + NJson::TJsonValue storage; + const NJson::TJsonValue *effective = nullptr; + + if (newStorageYaml) { + ui64 version = 0; + storage = load(*newStorageYaml, version, "StorageConfig"); + if (const ui64 expected = Self->StorageConfig->GetExpectedStorageYamlVersion(); version != expected) { + return FinishWithError(TResult::ERROR, TStringBuilder() + << "storage config version must be increasing by one" + << " new version# " << version + << " expected version# " << expected); + } + + newExpectedStorageYamlVersion = version + 1; + effective = &storage; } - state = "loading main YAML"; - auto json = NYaml::Yaml2Json(YAML::Load(*mainYamlPtr), true); - state = "parsing main YAML"; - NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true); - state = "extracting main YAML metadata"; - if (json.Has("metadata")) { - if (auto& metadata = json["metadata"]; metadata.Has("version")) { - newYamlVersion = metadata["version"].GetUIntegerRobust(); + if (NewYaml) { + ui64 version = 0; + main = load(*NewYaml, version, "MainConfig"); + if (const ui64 expected = *Self->MainConfigYamlVersion + 1; version != expected) { + return FinishWithError(TResult::ERROR, TStringBuilder() + << "main config version must be increasing by one" + << " new version# " << version + << " expected version# " << expected); + } + + if (!effective && !Self->StorageConfigYaml) { + effective = &main; } } - } catch (const std::exception& ex) { - return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state - << ": " << ex.what()); - } - if (newYamlVersion && *newYamlVersion != *Self->MainConfigYamlVersion + 1) { - return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one" - << " new version# " << *newYamlVersion << " expected version# " << *Self->MainConfigYamlVersion + 1); - } else if (newStorageYamlVersion && *newStorageYamlVersion != Self->StorageConfig->GetExpectedStorageYamlVersion()) { - return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one" - << " new version# " << *newStorageYamlVersion - << " expected version# " << Self->StorageConfig->GetExpectedStorageYamlVersion()); - } + if (effective) { + state = "parsing final config"; - TString errorReason; - NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig); - const bool success = DeriveStorageConfig(appConfig, &config, &errorReason); - if (!success) { - return FinishWithError(TResult::ERROR, TStringBuilder() << "error while deriving StorageConfig: " - << errorReason); + NKikimrConfig::TAppConfig appConfig; + NYaml::Parse(*effective, NYaml::GetJsonToProtoConfig(), appConfig, true); + + if (TString errorReason; !DeriveStorageConfig(appConfig, &config, &errorReason)) { + return FinishWithError(TResult::ERROR, TStringBuilder() + << "error while deriving StorageConfig: " << errorReason); + } + } + } catch (const std::exception& ex) { + return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state + << ": " << ex.what()); } if (NewYaml) { if (const auto& error = UpdateConfigComposite(config, *NewYaml, std::nullopt)) { return FinishWithError(TResult::ERROR, TStringBuilder() << "failed to update config yaml: " << *error); } - } else { - config.SetConfigComposite(Self->StorageConfig->GetConfigComposite()); + } + + if (newExpectedStorageYamlVersion) { + config.SetExpectedStorageYamlVersion(*newExpectedStorageYamlVersion); } if (newStorageYaml) { @@ -750,17 +784,11 @@ namespace NKikimr::NStorage { TString s; if (TStringOutput output(s); true) { TZstdCompress zstd(&output); - ::Save(&zstd, *newStorageYaml); - ::Save(&zstd, *newStorageYamlVersion); + zstd << *newStorageYaml; } config.SetCompressedStorageYaml(s); - config.SetExpectedStorageYamlVersion(*newStorageYamlVersion + 1); - } else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) { - // delete compressed storage yaml section as this request turns off dedicated storage yaml - } else if (Self->StorageConfig->HasCompressedStorageYaml()) { - // retain current storage yaml - config.SetCompressedStorageYaml(Self->StorageConfig->GetCompressedStorageYaml()); - config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion()); + } else if (!targetDedicatedStorageSection) { + config.ClearCompressedStorageYaml(); } // advance the config generation @@ -774,9 +802,7 @@ namespace NKikimr::NStorage { << "ReplaceStorageConfig config validation failed: " << *error); } - const bool pushToConsole = true; - - if (!pushToConsole || !request.GetSkipConsoleValidation()) { + if (request.GetSkipConsoleValidation() || !NewYaml) { return StartProposition(&config); } @@ -785,7 +811,7 @@ namespace NKikimr::NStorage { !Self->SelfManagementEnabled && config.GetSelfManagementConfig().GetEnabled(); - if (NewYaml && !Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) { + if (!Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) { FinishWithError(TResult::ERROR, "console pipe is not available"); } else { ProposedStorageConfig = std::move(config); diff --git a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp index 1a93f44538..a9e9b2e5d4 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp @@ -1,4 +1,5 @@ #include "distconf.h" +#include "node_warden_impl.h" #include <google/protobuf/util/json_util.h> @@ -171,7 +172,9 @@ namespace NKikimr::NStorage { if (config) { TString s; NProtoBuf::TextFormat::PrintToString(*config, &s); - out << "<pre>" << s << "</pre>"; + out << "<pre>"; + EscapeHtmlString(out, s); + out << "</pre>"; } else { out << "not defined"; } diff --git a/ydb/core/blobstorage/nodewarden/node_warden.h b/ydb/core/blobstorage/nodewarden/node_warden.h index 5d00dc3586..3744003c01 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden.h +++ b/ydb/core/blobstorage/nodewarden/node_warden.h @@ -24,6 +24,8 @@ namespace NKikimr { std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig; TString ConfigStorePath; std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig; + TString StartupConfigYaml; + std::optional<TString> StartupStorageYaml; TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory; TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds; TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index fc5e766259..01aac81294 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -441,11 +441,7 @@ void TNodeWarden::Bootstrap() { const bool success = DeriveStorageConfig(appConfig, &StorageConfig, &errorReason); Y_VERIFY_S(success, "failed to generate initial TStorageConfig: " << errorReason); - //LoadConfigVersion(); - if (Cfg->YamlConfig) { - YamlConfig.emplace(); - YamlConfig->CopyFrom(*Cfg->YamlConfig); - } + YamlConfig = std::move(Cfg->YamlConfig); // Start a statically configured set if (Cfg->BlobStorageConfig.HasServiceSet()) { @@ -635,31 +631,53 @@ void TNodeWarden::ProcessShredStatus(ui64 cookie, ui64 generation, std::optional } } -void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::optional<TString> storageYaml) { +void TNodeWarden::PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml, + std::optional<ui64> storageYamlVersion) { if (!Cfg->ConfigStorePath) { + // no storage directory specified + return; + } else if (auto *appData = AppData(); appData->DynamicNameserviceConfig && + appData->DynamicNameserviceConfig->MaxStaticNodeId < LocalNodeId) { + // this is a dynamic node return; } - if (YamlConfig && version <= YamlConfig->GetConfigVersion()) { - return; // some kind of a race + STLOG(PRI_DEBUG, BS_NODE, NW51, "persisting new configurations", + (MainYaml, mainYaml), (MainYamlVersion, mainYamlVersion), (StorageYaml, storageYaml), + (StorageYamlVersion, storageYamlVersion), (YamlConfig, YamlConfig)); + + const bool updateMain = mainYaml && (!YamlConfig || !YamlConfig->HasMainConfigVersion() || + YamlConfig->GetMainConfigVersion() < mainYamlVersion); + + const bool updateStorage = !storageYamlVersion || // delete storage config file in single-config mode + storageYaml && (!YamlConfig || !YamlConfig->HasStorageConfigVersion() || + YamlConfig->GetStorageConfigVersion() < storageYamlVersion); + + if (!updateMain && !updateStorage) { + return; // nothing to do } struct TSaveContext { TString ConfigStorePath; - TString ConfigYaml; - ui64 Version; + std::optional<TString> MainYaml; + ui64 MainYamlVersion; std::optional<TString> StorageYaml; - bool Success = true; - TString ErrorMessage; - TActorId SelfId; + std::optional<ui64> StorageYamlVersion; + bool UpdateMain; + bool UpdateStorage; + bool DeleteStorage; }; - auto saveCtx = std::make_shared<TSaveContext>(); - saveCtx->ConfigStorePath = Cfg->ConfigStorePath; - saveCtx->ConfigYaml = std::move(configYaml); - saveCtx->StorageYaml = std::move(storageYaml); - saveCtx->Version = std::move(version); - saveCtx->SelfId = SelfId(); + auto saveCtx = std::make_shared<TSaveContext>(TSaveContext{ + .ConfigStorePath = Cfg->ConfigStorePath, + .MainYaml = std::move(mainYaml), + .MainYamlVersion = mainYamlVersion, + .StorageYaml = std::move(storageYaml), + .StorageYamlVersion = storageYamlVersion, + .UpdateMain = updateMain, + .UpdateStorage = updateStorage, + .DeleteStorage = !storageYamlVersion, + }); EnqueueSyncOp([this, saveCtx](const TActorContext&) { bool success = true; @@ -694,14 +712,16 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op } }; - if (success) { - success = saveConfig(saveCtx->ConfigYaml, YamlConfigFileName); + if (success && saveCtx->UpdateMain) { + success = saveConfig(*saveCtx->MainYaml, YamlConfigFileName); if (success) { STLOG(PRI_INFO, BS_NODE, NW94, "Yaml config saved"); } } - if (success && saveCtx->StorageYaml) { + if (saveCtx->DeleteStorage) { + std::filesystem::remove(std::filesystem::path(saveCtx->ConfigStorePath.c_str()) / StorageConfigFileName); + } else if (success && saveCtx->UpdateStorage) { success = saveConfig(*saveCtx->StorageYaml, StorageConfigFileName); if (success) { STLOG(PRI_INFO, BS_NODE, NW95, "Storage config saved"); @@ -713,16 +733,22 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op if (!YamlConfig) { YamlConfig.emplace(); } - YamlConfig->SetYAML(saveCtx->ConfigYaml); - YamlConfig->SetConfigVersion(saveCtx->Version); + if (saveCtx->UpdateMain) { + YamlConfig->SetMainConfig(*saveCtx->MainYaml); + YamlConfig->SetMainConfigVersion(saveCtx->MainYamlVersion); + } + if (saveCtx->DeleteStorage) { + YamlConfig->ClearStorageConfig(); + YamlConfig->ClearStorageConfigVersion(); + } else if (saveCtx->UpdateStorage) { + YamlConfig->SetStorageConfig(*saveCtx->StorageYaml); + YamlConfig->SetStorageConfigVersion(*saveCtx->StorageYamlVersion); + } ConfigSaveTimer.Reset(); } else { - NKikimrBlobStorage::TYamlConfig yamlConfig; - yamlConfig.SetYAML(saveCtx->ConfigYaml); - yamlConfig.SetConfigVersion(saveCtx->Version); - TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()), - new IEventHandle(SelfId(), SelfId(), - new TEvPrivate::TEvRetrySaveConfig(yamlConfig), 0, ExpectedSaveConfigCookie)); + TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()), new IEventHandle( + SelfId(), {}, new TEvPrivate::TEvRetrySaveConfig(std::move(saveCtx->MainYaml), saveCtx->MainYamlVersion, + std::move(saveCtx->StorageYaml), saveCtx->StorageYamlVersion), 0, ExpectedSaveConfigCookie)); } }; }); @@ -733,7 +759,7 @@ void TNodeWarden::Handle(TEvRegisterPDiskLoadActor::TPtr ev) { } void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr ev) { - const auto& record = ev->Get()->Record; + auto& record = ev->Get()->Record; if (record.HasAvailDomain() && record.GetAvailDomain() != AvailDomainId) { // AvailDomain may arrive unset @@ -814,13 +840,26 @@ void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr } if (record.HasYamlConfig()) { - const auto& request = record.GetYamlConfig(); - if (request.HasYAML()) { - TString yaml = NYamlConfig::DecompressYamlString(request.GetYAML()); - ui64 version = request.GetConfigVersion(); - PersistConfig(yaml, version); - ExpectedSaveConfigCookie++; + auto& yaml = *record.MutableYamlConfig(); + + if (yaml.HasCompressedMainConfig()) { + Y_DEBUG_ABORT_UNLESS(!yaml.HasMainConfig()); + yaml.SetMainConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedMainConfig())); + yaml.ClearCompressedMainConfig(); + } + + if (yaml.HasCompressedStorageConfig()) { + Y_DEBUG_ABORT_UNLESS(!yaml.HasStorageConfig()); + yaml.SetStorageConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedStorageConfig())); + yaml.ClearCompressedStorageConfig(); } + + PersistConfig(yaml.HasMainConfig() ? std::make_optional(yaml.GetMainConfig()) : std::nullopt, + yaml.GetMainConfigVersion(), + yaml.HasStorageConfig() ? std::make_optional(yaml.GetStorageConfig()) : std::nullopt, + yaml.HasStorageConfigVersion() ? std::make_optional(yaml.GetStorageConfigVersion()) : std::nullopt); + + ExpectedSaveConfigCookie++; } } @@ -1048,8 +1087,8 @@ void TNodeWarden::Handle(TEvPrivate::TEvUpdateNodeDrives::TPtr&) { void TNodeWarden::Handle(TEvPrivate::TEvRetrySaveConfig::TPtr& ev) { STLOG(PRI_TRACE, BS_NODE, NW97, "Handle(TEvRetrySaveConfig)"); if (ev->Cookie == ExpectedSaveConfigCookie) { - const auto& yamlConfig = ev->Get()->YamlConfig; - PersistConfig(yamlConfig.GetYAML(), yamlConfig.GetConfigVersion()); + auto *msg = ev->Get(); + PersistConfig(std::move(msg->MainYaml), msg->MainYamlVersion, std::move(msg->StorageYaml), msg->StorageYamlVersion); ExpectedSaveConfigCookie++; } } @@ -1270,7 +1309,6 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app return false; } smTo->CopyFrom(smFrom); - smTo->ClearInitialConfigYaml(); // do not let this section into final StorageConfig } else { config->ClearSelfManagementConfig(); } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 541fe11f7f..dbf9f7423c 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -177,9 +177,17 @@ namespace NKikimr::NStorage { }; struct TEvRetrySaveConfig : TEventLocal<TEvRetrySaveConfig, EvRetrySaveConfig> { - NKikimrBlobStorage::TYamlConfig YamlConfig; - TEvRetrySaveConfig(const NKikimrBlobStorage::TYamlConfig& yamlConfig) - : YamlConfig(yamlConfig) + std::optional<TString> MainYaml; + ui64 MainYamlVersion; + std::optional<TString> StorageYaml; + std::optional<ui64> StorageYamlVersion; + + TEvRetrySaveConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml, + std::optional<ui64> storageYamlVersion) + : MainYaml(std::move(mainYaml)) + , MainYamlVersion(mainYamlVersion) + , StorageYaml(std::move(storageYaml)) + , StorageYamlVersion(storageYamlVersion) {} }; }; @@ -567,7 +575,8 @@ namespace NKikimr::NStorage { void Handle(NPDisk::TEvShredPDisk::TPtr ev); void ProcessShredStatus(ui64 cookie, ui64 generation, std::optional<TString> error); - void PersistConfig(const TString& yaml, ui64 version, std::optional<TString> storageYaml = std::nullopt); + void PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml, + std::optional<ui64> storageYamlVersion); void LoadConfigVersion(); void Handle(TEvRegisterPDiskLoadActor::TPtr ev); @@ -675,6 +684,8 @@ namespace NKikimr::NStorage { bool DeriveStorageConfig(const NKikimrConfig::TAppConfig& appConfig, NKikimrBlobStorage::TStorageConfig *config, TString *errorReason); + void EscapeHtmlString(IOutputStream& out, const TString& s); + } template<> diff --git a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp index fbff25eb58..8aa7d36c8b 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp @@ -108,21 +108,27 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) { out << "<p>Self-management enabled: " << (SelfManagementEnabled ? "yes" : "no") << "</p>"; TString s; NProtoBuf::TextFormat::PrintToString(StorageConfig, &s); - out << "<pre>" << s << "</pre>"; + out << "<pre>"; + EscapeHtmlString(out, s); + out << "</pre>"; } TAG(TH3) { out << "Static service set"; } DIV() { TString s; NProtoBuf::TextFormat::PrintToString(StaticServices, &s); - out << "<pre>" << s << "</pre>"; + out << "<pre>"; + EscapeHtmlString(out, s); + out << "</pre>"; } TAG(TH3) { out << "Dynamic service set"; } DIV() { TString s; NProtoBuf::TextFormat::PrintToString(DynamicServices, &s); - out << "<pre>" << s << "</pre>"; + out << "<pre>"; + EscapeHtmlString(out, s); + out << "</pre>"; } RenderLocalDrives(out); @@ -362,3 +368,41 @@ void TNodeWarden::RenderLocalDrives(IOutputStream& out) { } } } + +void NKikimr::NStorage::EscapeHtmlString(IOutputStream& out, const TString& s) { + size_t begin = 0; + auto dump = [&](size_t end) { + out << TStringBuf(s.data() + begin, end - begin); + begin = end + 1; + }; + for (size_t i = 0, len = s.size(); i < len; ++i) { + char ch = s[i]; + switch (ch) { + case '&': + dump(i); + out << "&"; + break; + + case '<': + dump(i); + out << "<"; + break; + + case '>': + dump(i); + out << ">"; + break; + + case '\'': + dump(i); + out << "'"; + break; + + case '"': + dump(i); + out << """; + break; + } + } + dump(s.size()); +} diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp index 74d06def0d..fb32bc381b 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp @@ -103,8 +103,12 @@ void TNodeWarden::SendRegisterNode() { } if (!Cfg->ConfigStorePath.empty() && YamlConfig) { - ev->Record.SetConfigVersion(YamlConfig->GetConfigVersion()); - ev->Record.SetConfigHash(NKikimr::NYaml::GetConfigHash(YamlConfig->GetYAML())); + ev->Record.SetMainConfigVersion(YamlConfig->GetMainConfigVersion()); + ev->Record.SetMainConfigHash(NYaml::GetConfigHash(YamlConfig->GetMainConfig())); + if (YamlConfig->HasStorageConfigVersion()) { + ev->Record.SetStorageConfigVersion(YamlConfig->GetStorageConfigVersion()); + ev->Record.SetStorageConfigHash(NYaml::GetConfigHash(YamlConfig->GetStorageConfig())); + } } SendToController(std::move(ev)); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp index d5c4d8099e..42af938068 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp @@ -122,17 +122,20 @@ void TNodeWarden::Handle(TEvNodeWardenStorageConfig::TPtr ev) { STLOG_DEBUG_FAIL(BS_NODE, NW49, "failed to decompose yaml configuration", (Error, error)); } else if (mainConfigYaml) { std::optional<TString> storageConfigYaml; + std::optional<ui64> storageConfigYamlVersion; if (StorageConfig.HasCompressedStorageYaml()) { try { TStringInput s(StorageConfig.GetCompressedStorageYaml()); storageConfigYaml.emplace(TZstdDecompress(&s).ReadAll()); + storageConfigYamlVersion.emplace(NYamlConfig::GetStorageMetadata(*storageConfigYaml).Version.value_or(0)); } catch (const std::exception& ex) { Y_ABORT("CompressedStorageYaml format incorrect: %s", ex.what()); } } // TODO(alexvru): make this blocker for confirmation? - PersistConfig(std::move(mainConfigYaml), mainConfigYamlVersion, std::move(storageConfigYaml)); + PersistConfig(std::move(mainConfigYaml), mainConfigYamlVersion, std::move(storageConfigYaml), + storageConfigYamlVersion); } } else { Y_DEBUG_ABORT_UNLESS(!StorageConfig.HasCompressedStorageYaml()); diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp index 80004b6bda..066b7d9e0a 100644 --- a/ydb/core/config/init/init.cpp +++ b/ydb/core/config/init/init.cpp @@ -1,5 +1,6 @@ #include "init_impl.h" #include "mock.h" +#include <ydb/library/yaml_json/yaml_to_json.h> namespace NKikimr::NConfig { @@ -67,7 +68,7 @@ public: TString GetProtoFromFile(const TString& path, IErrorCollector& errorCollector) const override { fs::path filePath(path.c_str()); - if (!IsFileExists(filePath)) { + if (!fs::is_regular_file(filePath)) { errorCollector.Fatal(Sprintf("File %s doesn't exists", path.c_str())); return {}; } @@ -581,7 +582,8 @@ void LoadBootstrapConfig(IProtoConfigFileProvider& protoConfigFileProvider, IErr } } -void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location) { +void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, const TString& storageYamlConfigFile, + bool loadedFromStore, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location) { if (!mainYamlConfigFile) { return; } @@ -590,24 +592,45 @@ void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKi IErrorCollector& errorCollector = refs.ErrorCollector; IProtoConfigFileProvider& protoConfigFileProvider = refs.ProtoConfigFileProvider; + std::optional<TString> storageYamlConfigString; + if (storageYamlConfigFile) { + storageYamlConfigString.emplace(protoConfigFileProvider.GetProtoFromFile(storageYamlConfigFile, errorCollector)); + } + const TString mainYamlConfigString = protoConfigFileProvider.GetProtoFromFile(mainYamlConfigFile, errorCollector); appConfig.SetStartupConfigYaml(mainYamlConfigString); - - if (appConfig.GetSelfManagementConfig().GetEnabled()) { - // fill in InitialConfigYaml only when self-management through distconf is enabled - appConfig.MutableSelfManagementConfig()->SetInitialConfigYaml(mainYamlConfigString); + if (storageYamlConfigString) { + appConfig.SetStartupStorageYaml(*storageYamlConfigString); } - if (appConfig.GetConfigLoadedFromStore()) { - auto* yamlConfig = appConfig.MutableStoredConfigYaml(); - yamlConfig->SetYAML(mainYamlConfigString); - yamlConfig->SetConfigVersion(NYamlConfig::GetVersion(mainYamlConfigString)); + if (loadedFromStore) { + auto *yamlConfig = appConfig.MutableStoredConfigYaml(); + yamlConfig->SetMainConfig(mainYamlConfigString); + yamlConfig->SetMainConfigVersion(NYamlConfig::GetVersion(mainYamlConfigString)); + if (storageYamlConfigString) { + yamlConfig->SetStorageConfig(*storageYamlConfigString); + yamlConfig->SetStorageConfigVersion(NYamlConfig::GetVersion(*storageYamlConfigString)); + } } /* * FIXME: if (ErrorCollector.HasFatal()) { return; } */ - NKikimrConfig::TAppConfig parsedConfig = NKikimr::NYaml::Parse(mainYamlConfigString); // FIXME + + NKikimrConfig::TAppConfig parsedConfig; + + if (storageYamlConfigString) { + auto storage = NKikimr::NYaml::Yaml2Json(YAML::Load(*storageYamlConfigString), true); + auto main = NKikimr::NYaml::Yaml2Json(YAML::Load(mainYamlConfigString), true); + auto& target = main["config"].GetMapSafe(); + for (auto&& [key, value] : std::move(storage["config"].GetMapSafe())) { + target.emplace(std::move(key), std::move(value)); + } + NKikimr::NYaml::Parse(main, NKikimr::NYaml::GetJsonToProtoConfig(), parsedConfig, true); + } else { + parsedConfig = NKikimr::NYaml::Parse(mainYamlConfigString); // FIXME + } + /* * FIXME: if (ErrorCollector.HasFatal()) { return; } */ diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index a0a907b096..283bcccd10 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -54,17 +54,13 @@ namespace NKikimr::NConfig { constexpr TStringBuf NODE_KIND_YDB = "ydb"; constexpr TStringBuf NODE_KIND_YQ = "yq"; -constexpr TStringBuf CONFIG_NAME = "config.yaml"; +constexpr const char *CONFIG_NAME = "config.yaml"; +constexpr const char *STORAGE_CONFIG_NAME = "storage.yaml"; constexpr static ui32 DefaultLogLevel = NActors::NLog::PRI_WARN; // log settings constexpr static ui32 DefaultLogSamplingLevel = NActors::NLog::PRI_DEBUG; // log settings constexpr static ui32 DefaultLogSamplingRate = 0; // log settings -inline bool IsFileExists(const fs::path& path) { - std::error_code ec; - return fs::exists(path, ec) && !ec; -} - template<typename T> bool ParsePBFromString(const TString &content, T *pb, bool allowUnknown = false) { if (!allowUnknown) { @@ -181,7 +177,9 @@ auto MutableConfigPartMerge( void AddProtoConfigOptions(IProtoConfigFileProvider& out); void LoadBootstrapConfig(IProtoConfigFileProvider& protoConfigFileProvider, IErrorCollector& errorCollector, TVector<TString> configFiles, NKikimrConfig::TAppConfig& out); -void LoadMainYamlConfig(TConfigRefs refs, const TString& yamlConfigFile, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location = NCompat::TSourceLocation::current()); +void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, const TString& storageYamlConfigFile, + bool loadedFromStore, NKikimrConfig::TAppConfig& appConfig, + const NCompat::TSourceLocation location = NCompat::TSourceLocation::current()); void CopyNodeLocation(NActorsInterconnect::TNodeLocation* dst, const NYdb::NDiscovery::TNodeLocation& src); void CopyNodeLocation(NYdb::NDiscovery::TNodeLocation* dst, const NActorsInterconnect::TNodeLocation& src); @@ -1068,17 +1066,27 @@ public: LoadBootstrapConfig(ProtoConfigFileProvider, ErrorCollector, freeArgs, BaseConfig); TString yamlConfigFile = CommonAppOptions.YamlConfigFile; - if (!CommonAppOptions.ConfigStorePath.empty()) { + TString storageYamlConfigFile; + bool loadedFromStore = false; + + if (CommonAppOptions.ConfigStorePath) { AppConfig.SetConfigStorePath(CommonAppOptions.ConfigStorePath); - const TString autoConfigPath = TStringBuilder() << CommonAppOptions.ConfigStorePath << "/" << CONFIG_NAME; - fs::path path(autoConfigPath.c_str()); - if (IsFileExists(path)) { - AppConfig.SetConfigLoadedFromStore(true); - yamlConfigFile = autoConfigPath; + auto dir = fs::path(CommonAppOptions.ConfigStorePath.c_str()); + + if (auto path = dir / STORAGE_CONFIG_NAME; fs::is_regular_file(path)) { + storageYamlConfigFile = path.c_str(); + } + + if (auto path = dir / CONFIG_NAME; fs::is_regular_file(path)) { + yamlConfigFile = path.c_str(); + loadedFromStore = true; + } else { + storageYamlConfigFile.clear(); } } - LoadMainYamlConfig(refs, yamlConfigFile, AppConfig); + + LoadMainYamlConfig(refs, yamlConfigFile, storageYamlConfigFile, loadedFromStore, AppConfig); OptionMerge("auth-token-file", TCfg::TAuthConfigFieldTag{}); // start memorylog as soon as possible @@ -1100,7 +1108,7 @@ public: InitDynamicNode(); } - LoadMainYamlConfig(refs, yamlConfigFile, AppConfig); + LoadMainYamlConfig(refs, yamlConfigFile, storageYamlConfigFile, loadedFromStore, AppConfig); Option("sys-file", TCfg::TActorSystemConfigFieldTag{}); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 9fce4fe760..713cbda309 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -956,6 +956,11 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se nodeWardenConfig->YamlConfig.emplace(Config.GetStoredConfigYaml()); } + nodeWardenConfig->StartupConfigYaml = Config.GetStartupConfigYaml(); + nodeWardenConfig->StartupStorageYaml = Config.HasStartupStorageYaml() + ? std::make_optional(Config.GetStartupStorageYaml()) + : std::nullopt; + ObtainTenantKey(&nodeWardenConfig->TenantKey, Config.GetKeyConfig()); ObtainStaticKey(&nodeWardenConfig->StaticKey); ObtainPDiskKey(&nodeWardenConfig->PDiskKey, Config.GetPDiskKeyConfig()); diff --git a/ydb/core/mind/bscontroller/commit_config.cpp b/ydb/core/mind/bscontroller/commit_config.cpp index 8bf84acca1..24303faf60 100644 --- a/ydb/core/mind/bscontroller/commit_config.cpp +++ b/ydb/core/mind/bscontroller/commit_config.cpp @@ -58,24 +58,50 @@ namespace NKikimr::NBsController { Self->StorageConfig = std::move(*StorageConfig); Self->ApplyStorageConfig(true); } + + std::optional<NKikimrBlobStorage::TYamlConfig> update; + if (YamlConfig) { Self->YamlConfig = std::move(YamlConfig); - const auto& configVersion = GetVersion(*Self->YamlConfig); - const auto& compressedConfig = CompressSingleConfig(*Self->YamlConfig); + Self->YamlConfigHash = GetSingleConfigHash(*Self->YamlConfig); + + if (!update) { + update.emplace(); + } + update->SetCompressedMainConfig(CompressSingleConfig(*Self->YamlConfig)); + update->SetMainConfigVersion(GetVersion(*Self->YamlConfig)); + } + if (StorageYamlConfig) { + const bool hadStorageConfigBefore = Self->StorageYamlConfig.has_value(); + + Self->StorageYamlConfig = std::move(*StorageYamlConfig); + Self->StorageYamlConfigVersion = NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0); + Self->StorageYamlConfigHash = NYaml::GetConfigHash(*Self->StorageYamlConfig); + + if (Self->StorageYamlConfig) { + if (!update) { + update.emplace(); + } + update->SetCompressedStorageConfig(CompressStorageYamlConfig(*Self->StorageYamlConfig)); + } else if (hadStorageConfigBefore && !update) { + update.emplace(); // issue an update without storage yaml version meaning we are in single-config mode + } + } + if (update && Self->StorageYamlConfig) { + update->SetStorageConfigVersion(NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0)); + } + + Self->ConsoleInteraction->OnConfigCommit(); + + if (update) { for (auto& node: Self->Nodes) { if (node.second.ConnectedServerId) { auto configPersistEv = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(); - auto* yamlConfig = configPersistEv->Record.MutableYamlConfig(); - yamlConfig->SetYAML(compressedConfig); - yamlConfig->SetConfigVersion(configVersion); + configPersistEv->Record.MutableYamlConfig()->CopyFrom(*update); Self->SendToWarden(node.first, std::move(configPersistEv), 0); } } } - if (StorageYamlConfig) { - Self->StorageYamlConfig = std::move(*StorageYamlConfig); - } - Self->ConsoleInteraction->OnConfigCommit(); } }; diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index 66df79db2c..20d38c32ad 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -1547,7 +1547,10 @@ private: NKikimrBlobStorage::TStorageConfig StorageConfig; bool SelfManagementEnabled = false; std::optional<TYamlConfig> YamlConfig; + ui64 YamlConfigHash = 0; std::optional<TString> StorageYamlConfig; // if separate config is in effect + ui64 StorageYamlConfigVersion = 0; + ui64 StorageYamlConfigHash = 0; TBackoffTimer GetBlockBackoff{1, 1000}; THashMap<TPDiskId, std::reference_wrapper<const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk>> StaticPDiskMap; diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp index 39836414f2..22b7074c0d 100644 --- a/ydb/core/mind/bscontroller/load_everything.cpp +++ b/ydb/core/mind/bscontroller/load_everything.cpp @@ -95,9 +95,12 @@ public: Self->TryToRelocateBrokenDisksLocallyFirst = state.GetValue<T::TryToRelocateBrokenDisksLocallyFirst>(); if (state.HaveValue<T::YamlConfig>()) { Self->YamlConfig = DecompressYamlConfig(state.GetValue<T::YamlConfig>()); + Self->YamlConfigHash = GetSingleConfigHash(*Self->YamlConfig); } if (state.HaveValue<T::StorageYamlConfig>()) { Self->StorageYamlConfig = DecompressStorageYamlConfig(state.GetValue<T::StorageYamlConfig>()); + Self->StorageYamlConfigVersion = NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0); + Self->StorageYamlConfigHash = NYaml::GetConfigHash(*Self->StorageYamlConfig); } if (state.HaveValue<T::ShredState>()) { Self->ShredState.OnLoad(state.GetValue<T::ShredState>()); diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index b45eb78b1b..ee6e970268 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -354,25 +354,44 @@ public: } } - // Check config version - if (Self->YamlConfig && !Self->StorageYamlConfig) { - const auto& configVersion = GetVersion(*Self->YamlConfig); - const auto& nodeId = record.GetNodeID(); - if (record.GetConfigVersion() != configVersion) { - if (record.GetConfigVersion() > configVersion) { - STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "Version on node greater than BSC", (NodeId, nodeId), (NewVersion, record.GetConfigVersion()), (OldVersion, configVersion)); - } - STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXRN10, "Send update config", (NodeId, nodeId), (NewVersion, record.GetConfigVersion()), (OldVersion, configVersion)); - auto *yamlConfig = Response->Record.MutableYamlConfig(); - yamlConfig->SetYAML(CompressSingleConfig(*Self->YamlConfig)); - yamlConfig->SetConfigVersion(record.GetConfigVersion()); + if (Self->YamlConfig) { + const ui64 configVersion = GetVersion(*Self->YamlConfig); + auto *yamlConfig = Response->Record.MutableYamlConfig(); + yamlConfig->SetMainConfigVersion(configVersion); + + if (record.GetMainConfigVersion() < configVersion) { + yamlConfig->SetCompressedMainConfig(CompressSingleConfig(*Self->YamlConfig)); + } else if (configVersion < record.GetMainConfigVersion()) { + STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "main config version on node is greater than one known to BSC", + (NodeId, record.GetNodeID()), + (NodeVersion, record.GetMainConfigVersion()), + (StoredVersion, configVersion)); + } else if (record.GetMainConfigHash() != Self->YamlConfigHash) { + STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "node main config hash mismatch", + (NodeId, record.GetNodeID())); } - else if (record.GetConfigHash() != GetSingleConfigHash(*Self->YamlConfig)) { - STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "Config hash on node mismatch", (NodeId, nodeId)); + } else if (record.HasMainConfigVersion()) { + // TODO(alexvru): report? + } + + if (Self->StorageYamlConfig) { + const ui64 configVersion = Self->StorageYamlConfigVersion; + auto *yamlConfig = Response->Record.MutableYamlConfig(); + yamlConfig->SetStorageConfigVersion(Self->StorageYamlConfigVersion); + Y_DEBUG_ABORT_UNLESS(yamlConfig->HasMainConfigVersion()); // no storage config without main one + + if (!record.HasStorageConfigVersion() || record.GetStorageConfigVersion() < configVersion) { + yamlConfig->SetCompressedStorageConfig(CompressStorageYamlConfig(*Self->StorageYamlConfig)); + } else if (configVersion < record.GetStorageConfigVersion()) { + STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "storage config version on node is greater than one known to BSC", + (NodeId, record.GetNodeID()), + (NodeVersion, record.GetMainConfigVersion()), + (StoredVersion, configVersion)); + } else if (record.GetStorageConfigHash() != Self->StorageYamlConfigHash) { + STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "node storage config hash mismatch", + (NodeId, record.GetNodeID())); } - } else { - // TODO(mregrock): Implement for double config mode - } + } return true; } diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 87552ea39b..df0815c571 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1101,8 +1101,10 @@ message TEvControllerRegisterNode { } repeated TShredStatus ShredStatus = 9; - optional uint64 ConfigVersion = 10; - optional uint64 ConfigHash = 11; + optional uint64 MainConfigVersion = 10; + optional uint64 MainConfigHash = 11; + optional uint64 StorageConfigVersion = 12; + optional uint64 StorageConfigHash = 13; } message TEvControllerUpdateNodeDrives { @@ -1111,12 +1113,12 @@ message TEvControllerUpdateNodeDrives { } message TYamlConfig { - optional string YAML = 1; - optional string DedicatedStorageYAML = 2; - optional string DedicatedClusterYAML = 3; - optional uint64 ConfigVersion = 4; - optional uint64 StorageConfigVersion = 5; - optional uint64 ClusterConfigVersion = 6; + optional string MainConfig = 1; + optional uint64 MainConfigVersion = 2; + optional string StorageConfig = 3; + optional uint64 StorageConfigVersion = 4; + optional bytes CompressedMainConfig = 5; + optional bytes CompressedStorageConfig = 6; } message TEvControllerNodeServiceSetUpdate { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 6a93c32ad3..278dba5183 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -332,9 +332,6 @@ message TSelfManagementConfig { // generation of the config; when set, one can automatically apply in-filesystem config optional uint64 Generation = 2; - // internally-used field with initial config.yaml loaded on the start (for the bootstrap process); not filled by the user - optional bytes InitialConfigYaml = 3; - // subset of DefineStoragePool command optional string ErasureSpecies = 11; optional NKikimrBlobStorage.TGroupGeometry Geometry = 12; @@ -2244,8 +2241,8 @@ message TAppConfig { optional string ConfigStorePath = 104; optional NKikimrBlobStorage.TYamlConfig StoredConfigYaml = 105; - optional bool ConfigLoadedFromStore = 106; optional string StartupConfigYaml = 107; + optional string StartupStorageYaml = 108; } message TYdbVersion { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 6cf63bab21..ac5e5e2428 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1121,7 +1121,6 @@ namespace NKikimr::NYaml { if (config.HasSelfManagementConfig()) { auto *smConfig = config.MutableSelfManagementConfig(); Y_ENSURE_BT(smConfig->HasEnabled(), "Enabled field is mandatory"); - Y_ENSURE_BT(!smConfig->HasInitialConfigYaml(), "InitialConfigYaml is not intended to be filled by user"); if (smConfig->GetEnabled()) { if (!smConfig->HasErasureSpecies()) { smConfig->SetErasureSpecies(ephemeralConfig.GetStaticErasure()); |