aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Rutkovsky <alexvru@ydb.tech>2025-02-20 15:28:40 +0300
committerGitHub <noreply@github.com>2025-02-20 15:28:40 +0300
commitfe62452fd88a2c291517b6cf6a4a2ba7dfc54b65 (patch)
treeaecf155b9f764e0c26d05257aadebfc925c3c28f
parent2c7f23f1d8285f50d0d488cfd365a209e395c374 (diff)
downloadydb-fe62452fd88a2c291517b6cf6a4a2ba7dfc54b65.tar.gz
Implement dual-config mode for distconf (#14825)
-rw-r--r--ydb/core/blobstorage/nodewarden/distconf_generate.cpp41
-rw-r--r--ydb/core/blobstorage/nodewarden/distconf_invoke.cpp152
-rw-r--r--ydb/core/blobstorage/nodewarden/distconf_mon.cpp5
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden.h2
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_impl.cpp118
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_impl.h19
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_mon.cpp50
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp8
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_resource.cpp5
-rw-r--r--ydb/core/config/init/init.cpp45
-rw-r--r--ydb/core/config/init/init_impl.h38
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp5
-rw-r--r--ydb/core/mind/bscontroller/commit_config.cpp44
-rw-r--r--ydb/core/mind/bscontroller/impl.h3
-rw-r--r--ydb/core/mind/bscontroller/load_everything.cpp3
-rw-r--r--ydb/core/mind/bscontroller/register_node.cpp53
-rw-r--r--ydb/core/protos/blobstorage.proto18
-rw-r--r--ydb/core/protos/config.proto5
-rw-r--r--ydb/library/yaml_config/yaml_config_parser.cpp1
19 files changed, 423 insertions, 192 deletions
diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp
index 8a47b3d1a0..810ecc662a 100644
--- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp
+++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp
@@ -43,29 +43,44 @@ namespace NKikimr::NStorage {
// initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while
// deriving it from NodeWarden configuration
- if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) {
+ if (!Cfg->StartupConfigYaml) {
return "missing initial config YAML";
}
- TStringStream ss;
- TString yaml = Cfg->SelfManagementConfig->GetInitialConfigYaml();
- ui32 version = 0;
+
+ ui64 version = 0;
try {
- auto json = NYaml::Yaml2Json(YAML::Load(yaml), true);
- if (json.Has("metadata")) {
- if (auto& metadata = json["metadata"]; metadata.Has("version")) {
- version = metadata["version"].GetUIntegerRobust();
- }
- }
+ version = NYamlConfig::GetMainMetadata(Cfg->StartupConfigYaml).Version.value_or(0);
} catch (const std::exception& ex) {
- return TStringBuilder() << "failed to parse initial config YAML: " << ex.what();
+ return TStringBuilder() << "failed to parse initial main YAML: " << ex.what();
}
if (version) {
- return TStringBuilder() << "initial config version must be zero";
+ return TStringBuilder() << "initial main config version must be zero";
}
- if (const auto& error = UpdateConfigComposite(*config, yaml, std::nullopt)) {
+
+ if (const auto& error = UpdateConfigComposite(*config, Cfg->StartupConfigYaml, std::nullopt)) {
return TStringBuilder() << "failed to update config yaml: " << *error;
}
+ if (Cfg->StartupStorageYaml) {
+ ui64 storageVersion = 0;
+ try {
+ storageVersion = NYamlConfig::GetStorageMetadata(*Cfg->StartupStorageYaml).Version.value_or(0);
+ } catch (const std::exception& ex) {
+ return TStringBuilder() << "failed to parse initial storage YAML: " << ex.what();
+ }
+ if (storageVersion) {
+ return TStringBuilder() << "initial storage config version must be zero";
+ }
+
+ TString s;
+ if (TStringOutput output(s); true) {
+ TZstdCompress zstd(&output);
+ zstd << *Cfg->StartupStorageYaml;
+ }
+ config->SetCompressedStorageYaml(s);
+ config->SetExpectedStorageYamlVersion(storageVersion + 1);
+ }
+
if (!Cfg->DomainsConfig) { // no automatic configuration required
} else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it
const auto& ss = Cfg->DomainsConfig->GetStateStorage(0);
diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp
index 8587895012..3de9d07123 100644
--- a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp
+++ b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp
@@ -665,11 +665,19 @@ namespace NKikimr::NStorage {
? std::make_optional(request.GetSwitchDedicatedStorageSection())
: std::nullopt;
- const TString *storageYamlPtr = newStorageYaml ? &newStorageYaml.value() :
- Self->StorageConfigYaml ? &Self->StorageConfigYaml.value() : nullptr;
-
const bool targetDedicatedStorageSection = switchDedicatedStorageSection.value_or(Self->StorageConfigYaml.has_value());
+ if (switchDedicatedStorageSection) {
+ // check that configs are explicitly defined when we are switching dual-config mode
+ if (!NewYaml) {
+ return FinishWithError(TResult::ERROR, "main config must be specified when switching dedicated"
+ " storage section mode");
+ } else if (*switchDedicatedStorageSection && !newStorageYaml) {
+ return FinishWithError(TResult::ERROR, "storage config must be specified when turning on dedicated"
+ " storage section mode");
+ }
+ }
+
if (request.GetDedicatedStorageSectionConfigMode() != targetDedicatedStorageSection) {
return FinishWithError(TResult::ERROR, "DedicatedStorageSectionConfigMode does not match target state");
} else if (newStorageYaml && !targetDedicatedStorageSection) {
@@ -678,71 +686,97 @@ namespace NKikimr::NStorage {
} else if (switchDedicatedStorageSection && *switchDedicatedStorageSection == Self->StorageConfigYaml.has_value()) {
// this enable/disable command does not change the state
return FinishWithError(TResult::ERROR, "dedicated storage config section is already in requested state");
- } else if (targetDedicatedStorageSection && !storageYamlPtr) {
- // we are going to turn on dual-config mode, but no storage config provided
- return FinishWithError(TResult::ERROR, "no dedicated storage config section provided");
}
- const TString *mainYamlPtr = NewYaml ? &NewYaml.value() : &Self->MainConfigYaml;
-
- std::optional<ui64> newYamlVersion;
- std::optional<ui64> newStorageYamlVersion;
+ TString state;
+ NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
+ std::optional<ui64> newExpectedStorageYamlVersion;
- NKikimrConfig::TAppConfig appConfig;
- const char *state = "";
+ if (config.HasExpectedStorageYamlVersion()) {
+ newExpectedStorageYamlVersion.emplace(config.GetExpectedStorageYamlVersion());
+ }
try {
- if (storageYamlPtr) { // parse the storage yaml first
- state = "loading storage YAML";
- auto json = NYaml::Yaml2Json(YAML::Load(*storageYamlPtr), true);
- state = "parsing storage YAML";
- NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
- state = "extracting storage YAML metadata";
- if (json.Has("metadata")) {
- if (auto& metadata = json["metadata"]; metadata.Has("version")) {
- newStorageYamlVersion = metadata["version"].GetUIntegerRobust();
- }
+ auto load = [&](const TString& yaml, ui64& version, const char *expectedKind) {
+ state = TStringBuilder() << "loading " << expectedKind << " YAML";
+ NJson::TJsonValue json = NYaml::Yaml2Json(YAML::Load(yaml), true);
+
+ state = TStringBuilder() << "extracting " << expectedKind << " metadata";
+ if (!json.Has("metadata") || !json["metadata"].IsMap()) {
+ throw yexception() << "no metadata section";
}
+ auto& metadata = json["metadata"];
+ NYaml::ValidateMetadata(metadata);
+ if (!metadata.Has("kind") || metadata["kind"] != expectedKind) {
+ throw yexception() << "missing or invalid kind provided";
+ }
+ version = metadata["version"].GetUIntegerRobust();
+
+ state = TStringBuilder() << "validating " << expectedKind << " config section";
+ if (!json.Has("config") || !json["config"].IsMap()) {
+ throw yexception() << "missing config section";
+ }
+
+ return json;
+ };
+
+ NJson::TJsonValue main;
+ NJson::TJsonValue storage;
+ const NJson::TJsonValue *effective = nullptr;
+
+ if (newStorageYaml) {
+ ui64 version = 0;
+ storage = load(*newStorageYaml, version, "StorageConfig");
+ if (const ui64 expected = Self->StorageConfig->GetExpectedStorageYamlVersion(); version != expected) {
+ return FinishWithError(TResult::ERROR, TStringBuilder()
+ << "storage config version must be increasing by one"
+ << " new version# " << version
+ << " expected version# " << expected);
+ }
+
+ newExpectedStorageYamlVersion = version + 1;
+ effective = &storage;
}
- state = "loading main YAML";
- auto json = NYaml::Yaml2Json(YAML::Load(*mainYamlPtr), true);
- state = "parsing main YAML";
- NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
- state = "extracting main YAML metadata";
- if (json.Has("metadata")) {
- if (auto& metadata = json["metadata"]; metadata.Has("version")) {
- newYamlVersion = metadata["version"].GetUIntegerRobust();
+ if (NewYaml) {
+ ui64 version = 0;
+ main = load(*NewYaml, version, "MainConfig");
+ if (const ui64 expected = *Self->MainConfigYamlVersion + 1; version != expected) {
+ return FinishWithError(TResult::ERROR, TStringBuilder()
+ << "main config version must be increasing by one"
+ << " new version# " << version
+ << " expected version# " << expected);
+ }
+
+ if (!effective && !Self->StorageConfigYaml) {
+ effective = &main;
}
}
- } catch (const std::exception& ex) {
- return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
- << ": " << ex.what());
- }
- if (newYamlVersion && *newYamlVersion != *Self->MainConfigYamlVersion + 1) {
- return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
- << " new version# " << *newYamlVersion << " expected version# " << *Self->MainConfigYamlVersion + 1);
- } else if (newStorageYamlVersion && *newStorageYamlVersion != Self->StorageConfig->GetExpectedStorageYamlVersion()) {
- return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
- << " new version# " << *newStorageYamlVersion
- << " expected version# " << Self->StorageConfig->GetExpectedStorageYamlVersion());
- }
+ if (effective) {
+ state = "parsing final config";
- TString errorReason;
- NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
- const bool success = DeriveStorageConfig(appConfig, &config, &errorReason);
- if (!success) {
- return FinishWithError(TResult::ERROR, TStringBuilder() << "error while deriving StorageConfig: "
- << errorReason);
+ NKikimrConfig::TAppConfig appConfig;
+ NYaml::Parse(*effective, NYaml::GetJsonToProtoConfig(), appConfig, true);
+
+ if (TString errorReason; !DeriveStorageConfig(appConfig, &config, &errorReason)) {
+ return FinishWithError(TResult::ERROR, TStringBuilder()
+ << "error while deriving StorageConfig: " << errorReason);
+ }
+ }
+ } catch (const std::exception& ex) {
+ return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
+ << ": " << ex.what());
}
if (NewYaml) {
if (const auto& error = UpdateConfigComposite(config, *NewYaml, std::nullopt)) {
return FinishWithError(TResult::ERROR, TStringBuilder() << "failed to update config yaml: " << *error);
}
- } else {
- config.SetConfigComposite(Self->StorageConfig->GetConfigComposite());
+ }
+
+ if (newExpectedStorageYamlVersion) {
+ config.SetExpectedStorageYamlVersion(*newExpectedStorageYamlVersion);
}
if (newStorageYaml) {
@@ -750,17 +784,11 @@ namespace NKikimr::NStorage {
TString s;
if (TStringOutput output(s); true) {
TZstdCompress zstd(&output);
- ::Save(&zstd, *newStorageYaml);
- ::Save(&zstd, *newStorageYamlVersion);
+ zstd << *newStorageYaml;
}
config.SetCompressedStorageYaml(s);
- config.SetExpectedStorageYamlVersion(*newStorageYamlVersion + 1);
- } else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) {
- // delete compressed storage yaml section as this request turns off dedicated storage yaml
- } else if (Self->StorageConfig->HasCompressedStorageYaml()) {
- // retain current storage yaml
- config.SetCompressedStorageYaml(Self->StorageConfig->GetCompressedStorageYaml());
- config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion());
+ } else if (!targetDedicatedStorageSection) {
+ config.ClearCompressedStorageYaml();
}
// advance the config generation
@@ -774,9 +802,7 @@ namespace NKikimr::NStorage {
<< "ReplaceStorageConfig config validation failed: " << *error);
}
- const bool pushToConsole = true;
-
- if (!pushToConsole || !request.GetSkipConsoleValidation()) {
+ if (request.GetSkipConsoleValidation() || !NewYaml) {
return StartProposition(&config);
}
@@ -785,7 +811,7 @@ namespace NKikimr::NStorage {
!Self->SelfManagementEnabled &&
config.GetSelfManagementConfig().GetEnabled();
- if (NewYaml && !Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
+ if (!Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
FinishWithError(TResult::ERROR, "console pipe is not available");
} else {
ProposedStorageConfig = std::move(config);
diff --git a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp
index 1a93f44538..a9e9b2e5d4 100644
--- a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp
+++ b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp
@@ -1,4 +1,5 @@
#include "distconf.h"
+#include "node_warden_impl.h"
#include <google/protobuf/util/json_util.h>
@@ -171,7 +172,9 @@ namespace NKikimr::NStorage {
if (config) {
TString s;
NProtoBuf::TextFormat::PrintToString(*config, &s);
- out << "<pre>" << s << "</pre>";
+ out << "<pre>";
+ EscapeHtmlString(out, s);
+ out << "</pre>";
} else {
out << "not defined";
}
diff --git a/ydb/core/blobstorage/nodewarden/node_warden.h b/ydb/core/blobstorage/nodewarden/node_warden.h
index 5d00dc3586..3744003c01 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden.h
+++ b/ydb/core/blobstorage/nodewarden/node_warden.h
@@ -24,6 +24,8 @@ namespace NKikimr {
std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
TString ConfigStorePath;
std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
+ TString StartupConfigYaml;
+ std::optional<TString> StartupStorageYaml;
TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
index fc5e766259..01aac81294 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
@@ -441,11 +441,7 @@ void TNodeWarden::Bootstrap() {
const bool success = DeriveStorageConfig(appConfig, &StorageConfig, &errorReason);
Y_VERIFY_S(success, "failed to generate initial TStorageConfig: " << errorReason);
- //LoadConfigVersion();
- if (Cfg->YamlConfig) {
- YamlConfig.emplace();
- YamlConfig->CopyFrom(*Cfg->YamlConfig);
- }
+ YamlConfig = std::move(Cfg->YamlConfig);
// Start a statically configured set
if (Cfg->BlobStorageConfig.HasServiceSet()) {
@@ -635,31 +631,53 @@ void TNodeWarden::ProcessShredStatus(ui64 cookie, ui64 generation, std::optional
}
}
-void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::optional<TString> storageYaml) {
+void TNodeWarden::PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
+ std::optional<ui64> storageYamlVersion) {
if (!Cfg->ConfigStorePath) {
+ // no storage directory specified
+ return;
+ } else if (auto *appData = AppData(); appData->DynamicNameserviceConfig &&
+ appData->DynamicNameserviceConfig->MaxStaticNodeId < LocalNodeId) {
+ // this is a dynamic node
return;
}
- if (YamlConfig && version <= YamlConfig->GetConfigVersion()) {
- return; // some kind of a race
+ STLOG(PRI_DEBUG, BS_NODE, NW51, "persisting new configurations",
+ (MainYaml, mainYaml), (MainYamlVersion, mainYamlVersion), (StorageYaml, storageYaml),
+ (StorageYamlVersion, storageYamlVersion), (YamlConfig, YamlConfig));
+
+ const bool updateMain = mainYaml && (!YamlConfig || !YamlConfig->HasMainConfigVersion() ||
+ YamlConfig->GetMainConfigVersion() < mainYamlVersion);
+
+ const bool updateStorage = !storageYamlVersion || // delete storage config file in single-config mode
+ storageYaml && (!YamlConfig || !YamlConfig->HasStorageConfigVersion() ||
+ YamlConfig->GetStorageConfigVersion() < storageYamlVersion);
+
+ if (!updateMain && !updateStorage) {
+ return; // nothing to do
}
struct TSaveContext {
TString ConfigStorePath;
- TString ConfigYaml;
- ui64 Version;
+ std::optional<TString> MainYaml;
+ ui64 MainYamlVersion;
std::optional<TString> StorageYaml;
- bool Success = true;
- TString ErrorMessage;
- TActorId SelfId;
+ std::optional<ui64> StorageYamlVersion;
+ bool UpdateMain;
+ bool UpdateStorage;
+ bool DeleteStorage;
};
- auto saveCtx = std::make_shared<TSaveContext>();
- saveCtx->ConfigStorePath = Cfg->ConfigStorePath;
- saveCtx->ConfigYaml = std::move(configYaml);
- saveCtx->StorageYaml = std::move(storageYaml);
- saveCtx->Version = std::move(version);
- saveCtx->SelfId = SelfId();
+ auto saveCtx = std::make_shared<TSaveContext>(TSaveContext{
+ .ConfigStorePath = Cfg->ConfigStorePath,
+ .MainYaml = std::move(mainYaml),
+ .MainYamlVersion = mainYamlVersion,
+ .StorageYaml = std::move(storageYaml),
+ .StorageYamlVersion = storageYamlVersion,
+ .UpdateMain = updateMain,
+ .UpdateStorage = updateStorage,
+ .DeleteStorage = !storageYamlVersion,
+ });
EnqueueSyncOp([this, saveCtx](const TActorContext&) {
bool success = true;
@@ -694,14 +712,16 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op
}
};
- if (success) {
- success = saveConfig(saveCtx->ConfigYaml, YamlConfigFileName);
+ if (success && saveCtx->UpdateMain) {
+ success = saveConfig(*saveCtx->MainYaml, YamlConfigFileName);
if (success) {
STLOG(PRI_INFO, BS_NODE, NW94, "Yaml config saved");
}
}
- if (success && saveCtx->StorageYaml) {
+ if (saveCtx->DeleteStorage) {
+ std::filesystem::remove(std::filesystem::path(saveCtx->ConfigStorePath.c_str()) / StorageConfigFileName);
+ } else if (success && saveCtx->UpdateStorage) {
success = saveConfig(*saveCtx->StorageYaml, StorageConfigFileName);
if (success) {
STLOG(PRI_INFO, BS_NODE, NW95, "Storage config saved");
@@ -713,16 +733,22 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op
if (!YamlConfig) {
YamlConfig.emplace();
}
- YamlConfig->SetYAML(saveCtx->ConfigYaml);
- YamlConfig->SetConfigVersion(saveCtx->Version);
+ if (saveCtx->UpdateMain) {
+ YamlConfig->SetMainConfig(*saveCtx->MainYaml);
+ YamlConfig->SetMainConfigVersion(saveCtx->MainYamlVersion);
+ }
+ if (saveCtx->DeleteStorage) {
+ YamlConfig->ClearStorageConfig();
+ YamlConfig->ClearStorageConfigVersion();
+ } else if (saveCtx->UpdateStorage) {
+ YamlConfig->SetStorageConfig(*saveCtx->StorageYaml);
+ YamlConfig->SetStorageConfigVersion(*saveCtx->StorageYamlVersion);
+ }
ConfigSaveTimer.Reset();
} else {
- NKikimrBlobStorage::TYamlConfig yamlConfig;
- yamlConfig.SetYAML(saveCtx->ConfigYaml);
- yamlConfig.SetConfigVersion(saveCtx->Version);
- TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()),
- new IEventHandle(SelfId(), SelfId(),
- new TEvPrivate::TEvRetrySaveConfig(yamlConfig), 0, ExpectedSaveConfigCookie));
+ TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()), new IEventHandle(
+ SelfId(), {}, new TEvPrivate::TEvRetrySaveConfig(std::move(saveCtx->MainYaml), saveCtx->MainYamlVersion,
+ std::move(saveCtx->StorageYaml), saveCtx->StorageYamlVersion), 0, ExpectedSaveConfigCookie));
}
};
});
@@ -733,7 +759,7 @@ void TNodeWarden::Handle(TEvRegisterPDiskLoadActor::TPtr ev) {
}
void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr ev) {
- const auto& record = ev->Get()->Record;
+ auto& record = ev->Get()->Record;
if (record.HasAvailDomain() && record.GetAvailDomain() != AvailDomainId) {
// AvailDomain may arrive unset
@@ -814,13 +840,26 @@ void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr
}
if (record.HasYamlConfig()) {
- const auto& request = record.GetYamlConfig();
- if (request.HasYAML()) {
- TString yaml = NYamlConfig::DecompressYamlString(request.GetYAML());
- ui64 version = request.GetConfigVersion();
- PersistConfig(yaml, version);
- ExpectedSaveConfigCookie++;
+ auto& yaml = *record.MutableYamlConfig();
+
+ if (yaml.HasCompressedMainConfig()) {
+ Y_DEBUG_ABORT_UNLESS(!yaml.HasMainConfig());
+ yaml.SetMainConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedMainConfig()));
+ yaml.ClearCompressedMainConfig();
+ }
+
+ if (yaml.HasCompressedStorageConfig()) {
+ Y_DEBUG_ABORT_UNLESS(!yaml.HasStorageConfig());
+ yaml.SetStorageConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedStorageConfig()));
+ yaml.ClearCompressedStorageConfig();
}
+
+ PersistConfig(yaml.HasMainConfig() ? std::make_optional(yaml.GetMainConfig()) : std::nullopt,
+ yaml.GetMainConfigVersion(),
+ yaml.HasStorageConfig() ? std::make_optional(yaml.GetStorageConfig()) : std::nullopt,
+ yaml.HasStorageConfigVersion() ? std::make_optional(yaml.GetStorageConfigVersion()) : std::nullopt);
+
+ ExpectedSaveConfigCookie++;
}
}
@@ -1048,8 +1087,8 @@ void TNodeWarden::Handle(TEvPrivate::TEvUpdateNodeDrives::TPtr&) {
void TNodeWarden::Handle(TEvPrivate::TEvRetrySaveConfig::TPtr& ev) {
STLOG(PRI_TRACE, BS_NODE, NW97, "Handle(TEvRetrySaveConfig)");
if (ev->Cookie == ExpectedSaveConfigCookie) {
- const auto& yamlConfig = ev->Get()->YamlConfig;
- PersistConfig(yamlConfig.GetYAML(), yamlConfig.GetConfigVersion());
+ auto *msg = ev->Get();
+ PersistConfig(std::move(msg->MainYaml), msg->MainYamlVersion, std::move(msg->StorageYaml), msg->StorageYamlVersion);
ExpectedSaveConfigCookie++;
}
}
@@ -1270,7 +1309,6 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
return false;
}
smTo->CopyFrom(smFrom);
- smTo->ClearInitialConfigYaml(); // do not let this section into final StorageConfig
} else {
config->ClearSelfManagementConfig();
}
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
index 541fe11f7f..dbf9f7423c 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
@@ -177,9 +177,17 @@ namespace NKikimr::NStorage {
};
struct TEvRetrySaveConfig : TEventLocal<TEvRetrySaveConfig, EvRetrySaveConfig> {
- NKikimrBlobStorage::TYamlConfig YamlConfig;
- TEvRetrySaveConfig(const NKikimrBlobStorage::TYamlConfig& yamlConfig)
- : YamlConfig(yamlConfig)
+ std::optional<TString> MainYaml;
+ ui64 MainYamlVersion;
+ std::optional<TString> StorageYaml;
+ std::optional<ui64> StorageYamlVersion;
+
+ TEvRetrySaveConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
+ std::optional<ui64> storageYamlVersion)
+ : MainYaml(std::move(mainYaml))
+ , MainYamlVersion(mainYamlVersion)
+ , StorageYaml(std::move(storageYaml))
+ , StorageYamlVersion(storageYamlVersion)
{}
};
};
@@ -567,7 +575,8 @@ namespace NKikimr::NStorage {
void Handle(NPDisk::TEvShredPDisk::TPtr ev);
void ProcessShredStatus(ui64 cookie, ui64 generation, std::optional<TString> error);
- void PersistConfig(const TString& yaml, ui64 version, std::optional<TString> storageYaml = std::nullopt);
+ void PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
+ std::optional<ui64> storageYamlVersion);
void LoadConfigVersion();
void Handle(TEvRegisterPDiskLoadActor::TPtr ev);
@@ -675,6 +684,8 @@ namespace NKikimr::NStorage {
bool DeriveStorageConfig(const NKikimrConfig::TAppConfig& appConfig, NKikimrBlobStorage::TStorageConfig *config,
TString *errorReason);
+ void EscapeHtmlString(IOutputStream& out, const TString& s);
+
}
template<>
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp
index fbff25eb58..8aa7d36c8b 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp
@@ -108,21 +108,27 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) {
out << "<p>Self-management enabled: " << (SelfManagementEnabled ? "yes" : "no") << "</p>";
TString s;
NProtoBuf::TextFormat::PrintToString(StorageConfig, &s);
- out << "<pre>" << s << "</pre>";
+ out << "<pre>";
+ EscapeHtmlString(out, s);
+ out << "</pre>";
}
TAG(TH3) { out << "Static service set"; }
DIV() {
TString s;
NProtoBuf::TextFormat::PrintToString(StaticServices, &s);
- out << "<pre>" << s << "</pre>";
+ out << "<pre>";
+ EscapeHtmlString(out, s);
+ out << "</pre>";
}
TAG(TH3) { out << "Dynamic service set"; }
DIV() {
TString s;
NProtoBuf::TextFormat::PrintToString(DynamicServices, &s);
- out << "<pre>" << s << "</pre>";
+ out << "<pre>";
+ EscapeHtmlString(out, s);
+ out << "</pre>";
}
RenderLocalDrives(out);
@@ -362,3 +368,41 @@ void TNodeWarden::RenderLocalDrives(IOutputStream& out) {
}
}
}
+
+void NKikimr::NStorage::EscapeHtmlString(IOutputStream& out, const TString& s) {
+ size_t begin = 0;
+ auto dump = [&](size_t end) {
+ out << TStringBuf(s.data() + begin, end - begin);
+ begin = end + 1;
+ };
+ for (size_t i = 0, len = s.size(); i < len; ++i) {
+ char ch = s[i];
+ switch (ch) {
+ case '&':
+ dump(i);
+ out << "&amp;";
+ break;
+
+ case '<':
+ dump(i);
+ out << "&lt;";
+ break;
+
+ case '>':
+ dump(i);
+ out << "&gt;";
+ break;
+
+ case '\'':
+ dump(i);
+ out << "&#39;";
+ break;
+
+ case '"':
+ dump(i);
+ out << "&quot;";
+ break;
+ }
+ }
+ dump(s.size());
+}
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp
index 74d06def0d..fb32bc381b 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp
@@ -103,8 +103,12 @@ void TNodeWarden::SendRegisterNode() {
}
if (!Cfg->ConfigStorePath.empty() && YamlConfig) {
- ev->Record.SetConfigVersion(YamlConfig->GetConfigVersion());
- ev->Record.SetConfigHash(NKikimr::NYaml::GetConfigHash(YamlConfig->GetYAML()));
+ ev->Record.SetMainConfigVersion(YamlConfig->GetMainConfigVersion());
+ ev->Record.SetMainConfigHash(NYaml::GetConfigHash(YamlConfig->GetMainConfig()));
+ if (YamlConfig->HasStorageConfigVersion()) {
+ ev->Record.SetStorageConfigVersion(YamlConfig->GetStorageConfigVersion());
+ ev->Record.SetStorageConfigHash(NYaml::GetConfigHash(YamlConfig->GetStorageConfig()));
+ }
}
SendToController(std::move(ev));
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp
index d5c4d8099e..42af938068 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp
@@ -122,17 +122,20 @@ void TNodeWarden::Handle(TEvNodeWardenStorageConfig::TPtr ev) {
STLOG_DEBUG_FAIL(BS_NODE, NW49, "failed to decompose yaml configuration", (Error, error));
} else if (mainConfigYaml) {
std::optional<TString> storageConfigYaml;
+ std::optional<ui64> storageConfigYamlVersion;
if (StorageConfig.HasCompressedStorageYaml()) {
try {
TStringInput s(StorageConfig.GetCompressedStorageYaml());
storageConfigYaml.emplace(TZstdDecompress(&s).ReadAll());
+ storageConfigYamlVersion.emplace(NYamlConfig::GetStorageMetadata(*storageConfigYaml).Version.value_or(0));
} catch (const std::exception& ex) {
Y_ABORT("CompressedStorageYaml format incorrect: %s", ex.what());
}
}
// TODO(alexvru): make this blocker for confirmation?
- PersistConfig(std::move(mainConfigYaml), mainConfigYamlVersion, std::move(storageConfigYaml));
+ PersistConfig(std::move(mainConfigYaml), mainConfigYamlVersion, std::move(storageConfigYaml),
+ storageConfigYamlVersion);
}
} else {
Y_DEBUG_ABORT_UNLESS(!StorageConfig.HasCompressedStorageYaml());
diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp
index 80004b6bda..066b7d9e0a 100644
--- a/ydb/core/config/init/init.cpp
+++ b/ydb/core/config/init/init.cpp
@@ -1,5 +1,6 @@
#include "init_impl.h"
#include "mock.h"
+#include <ydb/library/yaml_json/yaml_to_json.h>
namespace NKikimr::NConfig {
@@ -67,7 +68,7 @@ public:
TString GetProtoFromFile(const TString& path, IErrorCollector& errorCollector) const override {
fs::path filePath(path.c_str());
- if (!IsFileExists(filePath)) {
+ if (!fs::is_regular_file(filePath)) {
errorCollector.Fatal(Sprintf("File %s doesn't exists", path.c_str()));
return {};
}
@@ -581,7 +582,8 @@ void LoadBootstrapConfig(IProtoConfigFileProvider& protoConfigFileProvider, IErr
}
}
-void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location) {
+void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, const TString& storageYamlConfigFile,
+ bool loadedFromStore, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location) {
if (!mainYamlConfigFile) {
return;
}
@@ -590,24 +592,45 @@ void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKi
IErrorCollector& errorCollector = refs.ErrorCollector;
IProtoConfigFileProvider& protoConfigFileProvider = refs.ProtoConfigFileProvider;
+ std::optional<TString> storageYamlConfigString;
+ if (storageYamlConfigFile) {
+ storageYamlConfigString.emplace(protoConfigFileProvider.GetProtoFromFile(storageYamlConfigFile, errorCollector));
+ }
+
const TString mainYamlConfigString = protoConfigFileProvider.GetProtoFromFile(mainYamlConfigFile, errorCollector);
appConfig.SetStartupConfigYaml(mainYamlConfigString);
-
- if (appConfig.GetSelfManagementConfig().GetEnabled()) {
- // fill in InitialConfigYaml only when self-management through distconf is enabled
- appConfig.MutableSelfManagementConfig()->SetInitialConfigYaml(mainYamlConfigString);
+ if (storageYamlConfigString) {
+ appConfig.SetStartupStorageYaml(*storageYamlConfigString);
}
- if (appConfig.GetConfigLoadedFromStore()) {
- auto* yamlConfig = appConfig.MutableStoredConfigYaml();
- yamlConfig->SetYAML(mainYamlConfigString);
- yamlConfig->SetConfigVersion(NYamlConfig::GetVersion(mainYamlConfigString));
+ if (loadedFromStore) {
+ auto *yamlConfig = appConfig.MutableStoredConfigYaml();
+ yamlConfig->SetMainConfig(mainYamlConfigString);
+ yamlConfig->SetMainConfigVersion(NYamlConfig::GetVersion(mainYamlConfigString));
+ if (storageYamlConfigString) {
+ yamlConfig->SetStorageConfig(*storageYamlConfigString);
+ yamlConfig->SetStorageConfigVersion(NYamlConfig::GetVersion(*storageYamlConfigString));
+ }
}
/*
* FIXME: if (ErrorCollector.HasFatal()) { return; }
*/
- NKikimrConfig::TAppConfig parsedConfig = NKikimr::NYaml::Parse(mainYamlConfigString); // FIXME
+
+ NKikimrConfig::TAppConfig parsedConfig;
+
+ if (storageYamlConfigString) {
+ auto storage = NKikimr::NYaml::Yaml2Json(YAML::Load(*storageYamlConfigString), true);
+ auto main = NKikimr::NYaml::Yaml2Json(YAML::Load(mainYamlConfigString), true);
+ auto& target = main["config"].GetMapSafe();
+ for (auto&& [key, value] : std::move(storage["config"].GetMapSafe())) {
+ target.emplace(std::move(key), std::move(value));
+ }
+ NKikimr::NYaml::Parse(main, NKikimr::NYaml::GetJsonToProtoConfig(), parsedConfig, true);
+ } else {
+ parsedConfig = NKikimr::NYaml::Parse(mainYamlConfigString); // FIXME
+ }
+
/*
* FIXME: if (ErrorCollector.HasFatal()) { return; }
*/
diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h
index a0a907b096..283bcccd10 100644
--- a/ydb/core/config/init/init_impl.h
+++ b/ydb/core/config/init/init_impl.h
@@ -54,17 +54,13 @@ namespace NKikimr::NConfig {
constexpr TStringBuf NODE_KIND_YDB = "ydb";
constexpr TStringBuf NODE_KIND_YQ = "yq";
-constexpr TStringBuf CONFIG_NAME = "config.yaml";
+constexpr const char *CONFIG_NAME = "config.yaml";
+constexpr const char *STORAGE_CONFIG_NAME = "storage.yaml";
constexpr static ui32 DefaultLogLevel = NActors::NLog::PRI_WARN; // log settings
constexpr static ui32 DefaultLogSamplingLevel = NActors::NLog::PRI_DEBUG; // log settings
constexpr static ui32 DefaultLogSamplingRate = 0; // log settings
-inline bool IsFileExists(const fs::path& path) {
- std::error_code ec;
- return fs::exists(path, ec) && !ec;
-}
-
template<typename T>
bool ParsePBFromString(const TString &content, T *pb, bool allowUnknown = false) {
if (!allowUnknown) {
@@ -181,7 +177,9 @@ auto MutableConfigPartMerge(
void AddProtoConfigOptions(IProtoConfigFileProvider& out);
void LoadBootstrapConfig(IProtoConfigFileProvider& protoConfigFileProvider, IErrorCollector& errorCollector, TVector<TString> configFiles, NKikimrConfig::TAppConfig& out);
-void LoadMainYamlConfig(TConfigRefs refs, const TString& yamlConfigFile, NKikimrConfig::TAppConfig& appConfig, const NCompat::TSourceLocation location = NCompat::TSourceLocation::current());
+void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, const TString& storageYamlConfigFile,
+ bool loadedFromStore, NKikimrConfig::TAppConfig& appConfig,
+ const NCompat::TSourceLocation location = NCompat::TSourceLocation::current());
void CopyNodeLocation(NActorsInterconnect::TNodeLocation* dst, const NYdb::NDiscovery::TNodeLocation& src);
void CopyNodeLocation(NYdb::NDiscovery::TNodeLocation* dst, const NActorsInterconnect::TNodeLocation& src);
@@ -1068,17 +1066,27 @@ public:
LoadBootstrapConfig(ProtoConfigFileProvider, ErrorCollector, freeArgs, BaseConfig);
TString yamlConfigFile = CommonAppOptions.YamlConfigFile;
- if (!CommonAppOptions.ConfigStorePath.empty()) {
+ TString storageYamlConfigFile;
+ bool loadedFromStore = false;
+
+ if (CommonAppOptions.ConfigStorePath) {
AppConfig.SetConfigStorePath(CommonAppOptions.ConfigStorePath);
- const TString autoConfigPath = TStringBuilder() << CommonAppOptions.ConfigStorePath << "/" << CONFIG_NAME;
- fs::path path(autoConfigPath.c_str());
- if (IsFileExists(path)) {
- AppConfig.SetConfigLoadedFromStore(true);
- yamlConfigFile = autoConfigPath;
+ auto dir = fs::path(CommonAppOptions.ConfigStorePath.c_str());
+
+ if (auto path = dir / STORAGE_CONFIG_NAME; fs::is_regular_file(path)) {
+ storageYamlConfigFile = path.c_str();
+ }
+
+ if (auto path = dir / CONFIG_NAME; fs::is_regular_file(path)) {
+ yamlConfigFile = path.c_str();
+ loadedFromStore = true;
+ } else {
+ storageYamlConfigFile.clear();
}
}
- LoadMainYamlConfig(refs, yamlConfigFile, AppConfig);
+
+ LoadMainYamlConfig(refs, yamlConfigFile, storageYamlConfigFile, loadedFromStore, AppConfig);
OptionMerge("auth-token-file", TCfg::TAuthConfigFieldTag{});
// start memorylog as soon as possible
@@ -1100,7 +1108,7 @@ public:
InitDynamicNode();
}
- LoadMainYamlConfig(refs, yamlConfigFile, AppConfig);
+ LoadMainYamlConfig(refs, yamlConfigFile, storageYamlConfigFile, loadedFromStore, AppConfig);
Option("sys-file", TCfg::TActorSystemConfigFieldTag{});
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 9fce4fe760..713cbda309 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -956,6 +956,11 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se
nodeWardenConfig->YamlConfig.emplace(Config.GetStoredConfigYaml());
}
+ nodeWardenConfig->StartupConfigYaml = Config.GetStartupConfigYaml();
+ nodeWardenConfig->StartupStorageYaml = Config.HasStartupStorageYaml()
+ ? std::make_optional(Config.GetStartupStorageYaml())
+ : std::nullopt;
+
ObtainTenantKey(&nodeWardenConfig->TenantKey, Config.GetKeyConfig());
ObtainStaticKey(&nodeWardenConfig->StaticKey);
ObtainPDiskKey(&nodeWardenConfig->PDiskKey, Config.GetPDiskKeyConfig());
diff --git a/ydb/core/mind/bscontroller/commit_config.cpp b/ydb/core/mind/bscontroller/commit_config.cpp
index 8bf84acca1..24303faf60 100644
--- a/ydb/core/mind/bscontroller/commit_config.cpp
+++ b/ydb/core/mind/bscontroller/commit_config.cpp
@@ -58,24 +58,50 @@ namespace NKikimr::NBsController {
Self->StorageConfig = std::move(*StorageConfig);
Self->ApplyStorageConfig(true);
}
+
+ std::optional<NKikimrBlobStorage::TYamlConfig> update;
+
if (YamlConfig) {
Self->YamlConfig = std::move(YamlConfig);
- const auto& configVersion = GetVersion(*Self->YamlConfig);
- const auto& compressedConfig = CompressSingleConfig(*Self->YamlConfig);
+ Self->YamlConfigHash = GetSingleConfigHash(*Self->YamlConfig);
+
+ if (!update) {
+ update.emplace();
+ }
+ update->SetCompressedMainConfig(CompressSingleConfig(*Self->YamlConfig));
+ update->SetMainConfigVersion(GetVersion(*Self->YamlConfig));
+ }
+ if (StorageYamlConfig) {
+ const bool hadStorageConfigBefore = Self->StorageYamlConfig.has_value();
+
+ Self->StorageYamlConfig = std::move(*StorageYamlConfig);
+ Self->StorageYamlConfigVersion = NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0);
+ Self->StorageYamlConfigHash = NYaml::GetConfigHash(*Self->StorageYamlConfig);
+
+ if (Self->StorageYamlConfig) {
+ if (!update) {
+ update.emplace();
+ }
+ update->SetCompressedStorageConfig(CompressStorageYamlConfig(*Self->StorageYamlConfig));
+ } else if (hadStorageConfigBefore && !update) {
+ update.emplace(); // issue an update without storage yaml version meaning we are in single-config mode
+ }
+ }
+ if (update && Self->StorageYamlConfig) {
+ update->SetStorageConfigVersion(NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0));
+ }
+
+ Self->ConsoleInteraction->OnConfigCommit();
+
+ if (update) {
for (auto& node: Self->Nodes) {
if (node.second.ConnectedServerId) {
auto configPersistEv = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
- auto* yamlConfig = configPersistEv->Record.MutableYamlConfig();
- yamlConfig->SetYAML(compressedConfig);
- yamlConfig->SetConfigVersion(configVersion);
+ configPersistEv->Record.MutableYamlConfig()->CopyFrom(*update);
Self->SendToWarden(node.first, std::move(configPersistEv), 0);
}
}
}
- if (StorageYamlConfig) {
- Self->StorageYamlConfig = std::move(*StorageYamlConfig);
- }
- Self->ConsoleInteraction->OnConfigCommit();
}
};
diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
index 66df79db2c..20d38c32ad 100644
--- a/ydb/core/mind/bscontroller/impl.h
+++ b/ydb/core/mind/bscontroller/impl.h
@@ -1547,7 +1547,10 @@ private:
NKikimrBlobStorage::TStorageConfig StorageConfig;
bool SelfManagementEnabled = false;
std::optional<TYamlConfig> YamlConfig;
+ ui64 YamlConfigHash = 0;
std::optional<TString> StorageYamlConfig; // if separate config is in effect
+ ui64 StorageYamlConfigVersion = 0;
+ ui64 StorageYamlConfigHash = 0;
TBackoffTimer GetBlockBackoff{1, 1000};
THashMap<TPDiskId, std::reference_wrapper<const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk>> StaticPDiskMap;
diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp
index 39836414f2..22b7074c0d 100644
--- a/ydb/core/mind/bscontroller/load_everything.cpp
+++ b/ydb/core/mind/bscontroller/load_everything.cpp
@@ -95,9 +95,12 @@ public:
Self->TryToRelocateBrokenDisksLocallyFirst = state.GetValue<T::TryToRelocateBrokenDisksLocallyFirst>();
if (state.HaveValue<T::YamlConfig>()) {
Self->YamlConfig = DecompressYamlConfig(state.GetValue<T::YamlConfig>());
+ Self->YamlConfigHash = GetSingleConfigHash(*Self->YamlConfig);
}
if (state.HaveValue<T::StorageYamlConfig>()) {
Self->StorageYamlConfig = DecompressStorageYamlConfig(state.GetValue<T::StorageYamlConfig>());
+ Self->StorageYamlConfigVersion = NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0);
+ Self->StorageYamlConfigHash = NYaml::GetConfigHash(*Self->StorageYamlConfig);
}
if (state.HaveValue<T::ShredState>()) {
Self->ShredState.OnLoad(state.GetValue<T::ShredState>());
diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp
index b45eb78b1b..ee6e970268 100644
--- a/ydb/core/mind/bscontroller/register_node.cpp
+++ b/ydb/core/mind/bscontroller/register_node.cpp
@@ -354,25 +354,44 @@ public:
}
}
- // Check config version
- if (Self->YamlConfig && !Self->StorageYamlConfig) {
- const auto& configVersion = GetVersion(*Self->YamlConfig);
- const auto& nodeId = record.GetNodeID();
- if (record.GetConfigVersion() != configVersion) {
- if (record.GetConfigVersion() > configVersion) {
- STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "Version on node greater than BSC", (NodeId, nodeId), (NewVersion, record.GetConfigVersion()), (OldVersion, configVersion));
- }
- STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXRN10, "Send update config", (NodeId, nodeId), (NewVersion, record.GetConfigVersion()), (OldVersion, configVersion));
- auto *yamlConfig = Response->Record.MutableYamlConfig();
- yamlConfig->SetYAML(CompressSingleConfig(*Self->YamlConfig));
- yamlConfig->SetConfigVersion(record.GetConfigVersion());
+ if (Self->YamlConfig) {
+ const ui64 configVersion = GetVersion(*Self->YamlConfig);
+ auto *yamlConfig = Response->Record.MutableYamlConfig();
+ yamlConfig->SetMainConfigVersion(configVersion);
+
+ if (record.GetMainConfigVersion() < configVersion) {
+ yamlConfig->SetCompressedMainConfig(CompressSingleConfig(*Self->YamlConfig));
+ } else if (configVersion < record.GetMainConfigVersion()) {
+ STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "main config version on node is greater than one known to BSC",
+ (NodeId, record.GetNodeID()),
+ (NodeVersion, record.GetMainConfigVersion()),
+ (StoredVersion, configVersion));
+ } else if (record.GetMainConfigHash() != Self->YamlConfigHash) {
+ STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "node main config hash mismatch",
+ (NodeId, record.GetNodeID()));
}
- else if (record.GetConfigHash() != GetSingleConfigHash(*Self->YamlConfig)) {
- STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "Config hash on node mismatch", (NodeId, nodeId));
+ } else if (record.HasMainConfigVersion()) {
+ // TODO(alexvru): report?
+ }
+
+ if (Self->StorageYamlConfig) {
+ const ui64 configVersion = Self->StorageYamlConfigVersion;
+ auto *yamlConfig = Response->Record.MutableYamlConfig();
+ yamlConfig->SetStorageConfigVersion(Self->StorageYamlConfigVersion);
+ Y_DEBUG_ABORT_UNLESS(yamlConfig->HasMainConfigVersion()); // no storage config without main one
+
+ if (!record.HasStorageConfigVersion() || record.GetStorageConfigVersion() < configVersion) {
+ yamlConfig->SetCompressedStorageConfig(CompressStorageYamlConfig(*Self->StorageYamlConfig));
+ } else if (configVersion < record.GetStorageConfigVersion()) {
+ STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN09, "storage config version on node is greater than one known to BSC",
+ (NodeId, record.GetNodeID()),
+ (NodeVersion, record.GetMainConfigVersion()),
+ (StoredVersion, configVersion));
+ } else if (record.GetStorageConfigHash() != Self->StorageYamlConfigHash) {
+ STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXRN11, "node storage config hash mismatch",
+ (NodeId, record.GetNodeID()));
}
- } else {
- // TODO(mregrock): Implement for double config mode
- }
+ }
return true;
}
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index 87552ea39b..df0815c571 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -1101,8 +1101,10 @@ message TEvControllerRegisterNode {
}
repeated TShredStatus ShredStatus = 9;
- optional uint64 ConfigVersion = 10;
- optional uint64 ConfigHash = 11;
+ optional uint64 MainConfigVersion = 10;
+ optional uint64 MainConfigHash = 11;
+ optional uint64 StorageConfigVersion = 12;
+ optional uint64 StorageConfigHash = 13;
}
message TEvControllerUpdateNodeDrives {
@@ -1111,12 +1113,12 @@ message TEvControllerUpdateNodeDrives {
}
message TYamlConfig {
- optional string YAML = 1;
- optional string DedicatedStorageYAML = 2;
- optional string DedicatedClusterYAML = 3;
- optional uint64 ConfigVersion = 4;
- optional uint64 StorageConfigVersion = 5;
- optional uint64 ClusterConfigVersion = 6;
+ optional string MainConfig = 1;
+ optional uint64 MainConfigVersion = 2;
+ optional string StorageConfig = 3;
+ optional uint64 StorageConfigVersion = 4;
+ optional bytes CompressedMainConfig = 5;
+ optional bytes CompressedStorageConfig = 6;
}
message TEvControllerNodeServiceSetUpdate {
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index 6a93c32ad3..278dba5183 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -332,9 +332,6 @@ message TSelfManagementConfig {
// generation of the config; when set, one can automatically apply in-filesystem config
optional uint64 Generation = 2;
- // internally-used field with initial config.yaml loaded on the start (for the bootstrap process); not filled by the user
- optional bytes InitialConfigYaml = 3;
-
// subset of DefineStoragePool command
optional string ErasureSpecies = 11;
optional NKikimrBlobStorage.TGroupGeometry Geometry = 12;
@@ -2244,8 +2241,8 @@ message TAppConfig {
optional string ConfigStorePath = 104;
optional NKikimrBlobStorage.TYamlConfig StoredConfigYaml = 105;
- optional bool ConfigLoadedFromStore = 106;
optional string StartupConfigYaml = 107;
+ optional string StartupStorageYaml = 108;
}
message TYdbVersion {
diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp
index 6cf63bab21..ac5e5e2428 100644
--- a/ydb/library/yaml_config/yaml_config_parser.cpp
+++ b/ydb/library/yaml_config/yaml_config_parser.cpp
@@ -1121,7 +1121,6 @@ namespace NKikimr::NYaml {
if (config.HasSelfManagementConfig()) {
auto *smConfig = config.MutableSelfManagementConfig();
Y_ENSURE_BT(smConfig->HasEnabled(), "Enabled field is mandatory");
- Y_ENSURE_BT(!smConfig->HasInitialConfigYaml(), "InitialConfigYaml is not intended to be filled by user");
if (smConfig->GetEnabled()) {
if (!smConfig->HasErasureSpecies()) {
smConfig->SetErasureSpecies(ephemeralConfig.GetStaticErasure());