aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoryuryalekseev <yuryalekseev@yandex-team.com>2023-02-02 14:35:49 +0300
committeryuryalekseev <yuryalekseev@yandex-team.com>2023-02-02 14:35:49 +0300
commitea84a5fee092f387e3e16209ce3773a9edad3602 (patch)
tree5ca5e3b43b077f45483f29b8dd67c43bb5569ddd
parent5eaac88e381d86190ba39f09c7864ffc5075fbce (diff)
downloadydb-ea84a5fee092f387e3e16209ce3773a9edad3602.tar.gz
Streamline manual addition/removal of disks to BSC.
-rw-r--r--ydb/core/mind/bscontroller/cmds_drive_status.cpp197
-rw-r--r--ydb/core/mind/bscontroller/config.h4
-rw-r--r--ydb/core/mind/bscontroller/config_fit_pdisks.cpp481
-rw-r--r--ydb/core/mind/bscontroller/error.h1
-rw-r--r--ydb/core/mind/bscontroller/impl.h2
-rw-r--r--ydb/core/mind/bscontroller/register_node.cpp87
-rw-r--r--ydb/core/mind/bscontroller/ut_bscontroller/main.cpp25
-rw-r--r--ydb/core/protos/blobstorage_config.proto5
8 files changed, 352 insertions, 450 deletions
diff --git a/ydb/core/mind/bscontroller/cmds_drive_status.cpp b/ydb/core/mind/bscontroller/cmds_drive_status.cpp
index 31b1696ea8..be9b6b6cb5 100644
--- a/ydb/core/mind/bscontroller/cmds_drive_status.cpp
+++ b/ydb/core/mind/bscontroller/cmds_drive_status.cpp
@@ -95,196 +95,87 @@ namespace NKikimr::NBsController {
void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TAddDriveSerial& cmd,
TStatus& /*status*/) {
- const TString& newSerial = cmd.GetSerial();
-
- Schema::DriveSerial::BoxId::Type boxId = cmd.GetBoxId();
- const TDriveSerialInfo *driveInfo = DrivesSerials.Find(newSerial);
+ const auto& serial = cmd.GetSerial();
+ auto boxId = cmd.GetBoxId();
+ auto driveInfo = DrivesSerials.Find(serial);
if (driveInfo && driveInfo->LifeStage != NKikimrBlobStorage::TDriveLifeStage::REMOVED) {
- throw TExAlready() << "Device with such serial already exists in BSC database and not in lifeStage REMOVED";
+ throw TExAlready() << "Device with such serial already exists in BSC database in lifeStage" << driveInfo->LifeStage;
}
- if (auto it = NodeForSerial.find(newSerial); it != NodeForSerial.end()) {
- // Serial of drive is known, but drive not present in DrivesSerial
- // Check is it defined in HostConfigs
- TNodeId nodeId = it->second;
- const TNodeInfo& nodeInfo = Nodes.Get().at(nodeId);
- TString path = nodeInfo.KnownDrives.at(newSerial).Path;
-
- TPDiskId from = TPDiskId::MinForNode(nodeId);
- TPDiskId to = TPDiskId::MaxForNode(nodeId);
- std::optional<TPDiskId> updatePDiskId;
- PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) {
- if (pdiskInfo.Path == path) {
- updatePDiskId = pdiskId;
- return false;
- }
- return true;
- });
- if (updatePDiskId) {
- // PDisk is defined through HostConfigs, but there may be fictional row in DrivesSerials
- // if row is present - delete it
- if (driveInfo) {
- DrivesSerials.DeleteExistingEntry(newSerial);
- driveInfo = nullptr;
- }
- TPDiskInfo *pdiskInfo = PDisks.FindForUpdate(*updatePDiskId);
- if (pdiskInfo->ExpectedSerial == newSerial) {
- throw TExAlready() << "Device with such serial already exists in BSC database and is defined through "
- << "HostConfigs";
- }
- pdiskInfo->ExpectedSerial = newSerial;
- if (pdiskInfo->BoxId != boxId) {
- throw TExError() << "Drive is defind in host configs, but placed in another box# " << pdiskInfo->BoxId;
- }
- STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA06, "Set new ExpectedSerial for HostConfigs drive",
- (UniqueId, UniqueId), (Serial, newSerial), (BoxId, boxId), (PDiskId, *updatePDiskId), (Path, path));
- Fit.Boxes.insert(boxId);
- return;
- }
+ auto it = NodeIdByDiskSerialNumber.find(serial);
+ if (it == NodeIdByDiskSerialNumber.end()) {
+ throw TExError() << "Couldn't find node id for disk with serial number: " << serial;
}
+ auto nodeId = it->second;
- {
- // Additional check, may give false negative if ExpectedSerial for pdisk is unknown
- TMaybe<TPDiskId> from;
- TMaybe<TPDiskId> to;
- if (auto it = NodeForSerial.find(newSerial); it != NodeForSerial.end()) {
- from = TPDiskId::MinForNode(it->second);
- to = TPDiskId::MaxForNode(it->second);
- }
+ const auto& nodes = Nodes.Get();
+ auto nodeIt = nodes.find(nodeId);
+ if (nodeIt == nodes.end()) {
+ throw TExError() << "Couldn't find node with node id: " << nodeId << " for disk with serial number: " << serial;
+ }
- std::optional<TPDiskId> existingPDisk;
- PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) {
- if (newSerial == pdiskInfo.ExpectedSerial) {
- existingPDisk = pdiskId;
- return false;
- }
- return true;
- });
- if (existingPDisk) {
- throw TExAlready() << "Device with such serial already exists in BSC database and is defined in HostConfigs"
- << " pdiskId# " << *existingPDisk;
- }
+ const auto& nodeInfo = nodeIt->second;
+ auto driveIt = nodeInfo.KnownDrives.find(serial);
+ if (driveIt == nodeInfo.KnownDrives.end()) {
+ throw TExError() << "Couldn't find disk on node: " << nodeId << " by serial number: " << serial;
}
- // delete existing entry, if any, but keep its GUID
- std::optional<TMaybe<Schema::DriveSerial::Guid::Type>> guid = driveInfo ? std::make_optional(driveInfo->Guid) : std::nullopt;
+ // delete REMOVED entry, if any, but keep its GUID
+ auto guid = driveInfo ? std::make_optional(driveInfo->Guid) : std::nullopt;
if (driveInfo) {
- DrivesSerials.DeleteExistingEntry(newSerial);
+ DrivesSerials.DeleteExistingEntry(serial);
}
- TDriveSerialInfo *driveInfoNew = DrivesSerials.ConstructInplaceNewEntry(newSerial, boxId);
+ auto driveInfoMutable = DrivesSerials.ConstructInplaceNewEntry(serial, boxId);
if (guid) {
- driveInfoNew->Guid = *guid;
+ driveInfoMutable->Guid = *guid;
}
- driveInfoNew->Kind = cmd.GetKind();
- driveInfoNew->PDiskType = cmd.GetPDiskType();
+ driveInfoMutable->Kind = cmd.GetKind();
+ driveInfoMutable->PDiskType = cmd.GetPDiskType();
TString config;
- const bool success = cmd.GetPDiskConfig().SerializeToString(&config);
+ auto success = cmd.GetPDiskConfig().SerializeToString(&config);
Y_VERIFY(success);
- driveInfoNew->PDiskConfig = config;
+ driveInfoMutable->PDiskConfig = config;
+ driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ADDED;
Fit.Boxes.insert(boxId);
- STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA00, "AddDriveSerial", (UniqueId, UniqueId), (Serial, newSerial),
+ STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA00, "AddDriveSerial", (UniqueId, UniqueId), (Serial, serial),
(BoxId, boxId));
}
void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TRemoveDriveSerial& cmd,
TStatus& /*status*/) {
- const TString& serial = cmd.GetSerial();
-
- if (const TDriveSerialInfo *driveInfo = DrivesSerials.Find(serial); !driveInfo) {
- // Drive is defined in HostConfigs
- //
+ const auto& serial = cmd.GetSerial();
- // Fast search (works only for online nodes)
- std::optional<TNodeId> nodeId;
- if (auto it = NodeForSerial.find(serial); it != NodeForSerial.end()) {
- nodeId = it->second;
- } else {
- // Slow PDisks fullscan
- PDisks.ForEach([&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) {
- if (pdiskInfo.ExpectedSerial == serial) {
- nodeId = pdiskId.NodeId;
- return false;
- }
- return true;
- });
- }
- if (!nodeId) {
- throw TExError() << "Device with such serial is unknown for BSC";
- }
-
- TPDiskId from = TPDiskId::MinForNode(*nodeId);
- TPDiskId to = TPDiskId::MaxForNode(*nodeId);
- std::optional<TPDiskId> removePDiskId;
- PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) {
- if (pdiskInfo.ExpectedSerial == serial) {
- if (pdiskInfo.NumActiveSlots) {
- throw TExError() << "There are active vdisks on that drive";
- }
- if (removePDiskId) {
- throw TExError() << "has two pdisks defined in HostConfigs with same serial number";
- }
- removePDiskId = pdiskId;
- }
- return true;
- });
- if (!removePDiskId) {
- throw TExError() << "The serial was seen in cluster on node# " << *nodeId
- << " but now there are no pdisks with the serial";
- }
- auto* pdiskUpdate = PDisks.FindForUpdate(*removePDiskId);
- pdiskUpdate->ExpectedSerial = {};
- STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA08, "Reset ExpectedSerial for HostConfig drive",
- (UniqueId, UniqueId), (Serial, serial), (PDiskId, *removePDiskId));
-
- // create fictional row in DrivesSerials to be able to reply kAlready for already removed disk
- // even if they are defined through HostConfig
- TDriveSerialInfo *driveInfoNew = DrivesSerials.ConstructInplaceNewEntry(serial, pdiskUpdate->BoxId);
- driveInfoNew->Guid = pdiskUpdate->Guid;
- driveInfoNew->Kind = pdiskUpdate->Kind.Kind();
- driveInfoNew->PDiskType = PDiskTypeToPDiskType(pdiskUpdate->Kind.Type());
- driveInfoNew->PDiskConfig = pdiskUpdate->PDiskConfig;
- driveInfoNew->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED;
-
- Fit.Boxes.insert(pdiskUpdate->BoxId);
- } else {
- if (driveInfo->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) {
- throw TExAlready() << "Drive is already removed";
- }
+ auto driveInfo = DrivesSerials.Find(serial);
+ if (!driveInfo) {
+ throw TExError() << "Couldn't find disk with serial number: " << serial;
+ }
- if (driveInfo->NodeId && driveInfo->PDiskId) {
- TPDiskId pdiskId(*driveInfo->NodeId, *driveInfo->PDiskId);
- if (auto* pdiskInfo = PDisks.Find(pdiskId)) {
- if (pdiskInfo->NumActiveSlots) {
- throw TExError() << "There are active vdisks on that drive";
- } else {
- // PDisk will be deleted automatically in FitPDisks
- }
- }
- }
+ if (driveInfo->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) {
+ throw TExError() << "Disk with serial number: " << serial << " has already been removed";
+ }
- TDriveSerialInfo *driveInfoMutable = DrivesSerials.FindForUpdate(serial);
- driveInfoMutable->NodeId.Clear();
- driveInfoMutable->PDiskId.Clear();
- driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED;
+ auto driveInfoMutable = DrivesSerials.FindForUpdate(serial);
+ driveInfoMutable->NodeId.Clear();
+ driveInfoMutable->PDiskId.Clear();
+ driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED;
- Fit.Boxes.insert(driveInfoMutable->BoxId);
+ Fit.Boxes.insert(driveInfo->BoxId);
- STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA07, "RemoveDriveSerial", (UniqueId, UniqueId), (Serial, serial));
- }
+ STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA07, "RemoveDriveSerial", (UniqueId, UniqueId), (Serial, serial));
}
void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TForgetDriveSerial& cmd,
TStatus& /*status*/) {
- const TString& serial = cmd.GetSerial();
+ const auto& serial = cmd.GetSerial();
- if (const TDriveSerialInfo *driveInfo = DrivesSerials.Find(serial)) {
+ if (auto driveInfo = DrivesSerials.Find(serial)) {
switch (driveInfo->LifeStage) {
case NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN:
[[fallthrough]];
diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h
index 23cc993dac..01d8f4fcef 100644
--- a/ydb/core/mind/bscontroller/config.h
+++ b/ydb/core/mind/bscontroller/config.h
@@ -105,7 +105,7 @@ namespace NKikimr {
// static pdisk/vdisk states
std::map<TVSlotId, TStaticVSlotInfo>& StaticVSlots;
std::map<TPDiskId, TStaticPDiskInfo>& StaticPDisks;
- const std::map<TString, TNodeId>& NodeForSerial;
+ const std::unordered_map<TString, TNodeId>& NodeIdByDiskSerialNumber;
TCowHolder<Schema::State::SerialManagementStage::Type> SerialManagementStage;
@@ -135,7 +135,7 @@ namespace NKikimr {
, DefaultMaxSlots(controller.DefaultMaxSlots)
, StaticVSlots(controller.StaticVSlots)
, StaticPDisks(controller.StaticPDisks)
- , NodeForSerial(controller.NodeForSerial)
+ , NodeIdByDiskSerialNumber(controller.NodeIdByDiskSerialNumber)
, SerialManagementStage(&controller.SerialManagementStage)
, StoragePoolStat(*controller.StoragePoolStat)
{
diff --git a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp
index d8934d4b40..1c34987e89 100644
--- a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp
+++ b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp
@@ -1,9 +1,54 @@
#include "config.h"
+#include <util/generic/string.h>
+#include <util/system/types.h>
+
+namespace NKikimr {
+ namespace NBsController {
+
+ struct TDiskId {
+ ui32 NodeId = 0;
+ TString Path;
+
+ bool operator==(const TDiskId& other) const {
+ return NodeId == other.NodeId && Path == other.Path;
+ }
+ };
+
+ struct TDiskInfo {
+ ui32 NodeId = 0;
+ TBlobStorageController::THostId HostId = {};
+ TBoxId BoxId = 0;
+ TString Path;
+ TString LastSeenPath;
+ TString Serial;
+ TString LastSeenSerial;
+ bool SharedWithOs = false;
+ bool ReadCentric = false;
+ TPDiskCategory PDiskCategory = {};
+ TString PDiskConfig;
+
+ TDiskId GetId() const {
+ return {NodeId, Path};
+ }
+ };
+
+ } // NBsController
+} // NKikimr
+
+namespace std {
+ template <>
+ struct hash<NKikimr::NBsController::TDiskId> {
+ size_t operator()(const NKikimr::NBsController::TDiskId& diskId) const {
+ return hash<ui32>()(diskId.NodeId) ^ hash<TString>()(diskId.Path);
+ }
+ };
+}
+
namespace NKikimr {
namespace NBsController {
- TPDiskId FindFirstEmptyPDiskId(const TOverlayMap<TPDiskId, TBlobStorageController::TPDiskInfo>& pdisks,
+ static TPDiskId FindFirstEmptyPDiskId(const TOverlayMap<TPDiskId, TBlobStorageController::TPDiskInfo>& pdisks,
TNodeId nodeId) {
Schema::PDisk::PDiskID::Type nextPDiskID = 1000; // start allocation from this number
// generate PDisk id; skip generated one if it already exists (e.g. user has added
@@ -21,6 +66,183 @@ namespace NKikimr {
return proto.ParseFromString(s) ? SingleLineProto(proto) : "<error>";
}
+ static void UpdatePDiskIfNeeded(const TPDiskId& pdiskId, const TDiskInfo& disk, ui32 defaultMaxSlots, TBlobStorageController::TConfigState& state) {
+ auto pdiskInfo = state.PDisks.Find(pdiskId);
+ Y_VERIFY(pdiskInfo != nullptr);
+ if (pdiskInfo->Kind != disk.PDiskCategory ||
+ pdiskInfo->SharedWithOs != disk.SharedWithOs ||
+ pdiskInfo->ReadCentric != disk.ReadCentric ||
+ pdiskInfo->BoxId != disk.BoxId ||
+ pdiskInfo->PDiskConfig != disk.PDiskConfig)
+ {
+ // update PDisk configuration
+ auto pdiskInfo = state.PDisks.FindForUpdate(pdiskId);
+ Y_VERIFY(pdiskInfo != nullptr);
+ pdiskInfo->Kind = disk.PDiskCategory;
+ pdiskInfo->SharedWithOs = disk.SharedWithOs;
+ pdiskInfo->ReadCentric = disk.ReadCentric;
+ pdiskInfo->BoxId = disk.BoxId;
+ pdiskInfo->PDiskConfig = disk.PDiskConfig;
+ pdiskInfo->ExtractConfig(defaultMaxSlots);
+ }
+ }
+
+ // return TString not const TString& to make sure we never use dangling reference
+ static TString GetDiskPathFromNode(ui32 nodeId, const TString& serialNumber, const TBlobStorageController::TConfigState& state, bool throwOnError = false) {
+ if (auto nodeIt = state.Nodes.Get().find(nodeId); nodeIt != state.Nodes.Get().end()) {
+ for (const auto& [_, driveData] : nodeIt->second.KnownDrives) {
+ if (serialNumber == driveData.SerialNumber) {
+ return driveData.Path;
+ }
+ }
+ if (throwOnError) {
+ throw TExError() << "Couldn't find disk's path by serial number " << TErrorParams::DiskSerialNumber(serialNumber);
+ }
+ } else {
+ if (throwOnError) {
+ throw TExError() << "Unknown node id " << TErrorParams::NodeId(nodeId);
+ }
+ }
+
+ return TString();
+ }
+
+ // return TString not const TString& to make sure we never use dangling reference
+ static TString GetDiskSerialNumberFromNode(ui32 nodeId, const TString& path, const TBlobStorageController::TConfigState& state, bool throwOnError = false) {
+ if (auto nodeIt = state.Nodes.Get().find(nodeId); nodeIt != state.Nodes.Get().end()) {
+ for (const auto& [_, driveData] : nodeIt->second.KnownDrives) {
+ if (path == driveData.Path) {
+ return driveData.SerialNumber;
+ }
+ }
+ if (throwOnError) {
+ throw TExError() << "Couldn't find disk's serial number by path " << TErrorParams::Path(path);
+ }
+ } else {
+ if (throwOnError) {
+ throw TExError() << "Unknown node id " << TErrorParams::NodeId(nodeId);
+ }
+ }
+
+ return TString();
+ }
+
+ static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromHostConfig(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) {
+ std::unordered_map<TDiskId, TDiskInfo> disks;
+
+ const auto& hostConfigs = state.HostConfigs.Get();
+ const auto& boxes = state.Boxes.Get();
+ for (const TBoxId& boxId : relevantBoxes) {
+ const auto boxIt = boxes.find(boxId);
+ if (boxIt == boxes.end()) {
+ continue; // box was deleted
+ }
+ const auto& box = boxIt->second;
+
+ THashSet<TNodeId> usedNodes;
+ for (const auto& [hostKey, hostValue] : box.Hosts) {
+ const auto& hostConfigId = hostValue.HostConfigId;
+ auto it = hostConfigs.find(hostConfigId);
+ if (it == hostConfigs.end()) {
+ throw TExHostConfigNotFound(hostConfigId);
+ }
+ const auto& hostConfig = it->second;
+
+ const TBlobStorageController::THostId hostId(hostKey.Fqdn, hostKey.IcPort);
+ const auto& nodeId = state.HostRecords->ResolveNodeId(hostKey, hostValue);
+ if (!nodeId) {
+ throw TExHostNotFound(hostKey) << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId);
+ } else if (!usedNodes.insert(*nodeId).second) {
+ throw TExError() << "duplicate NodeId" << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId)
+ << TErrorParams::Fqdn(hostKey.Fqdn) << TErrorParams::IcPort(hostKey.IcPort);
+ }
+
+ for (const auto& [drive, driveInfo] : hostConfig.Drives) {
+ auto serial = GetDiskSerialNumberFromNode(*nodeId, drive.Path, state, /* throwOnError */ false);
+
+ TDiskInfo disk;
+ disk.BoxId = boxId;
+ disk.HostId = hostId;
+ disk.LastSeenPath = TString();
+ disk.LastSeenSerial = serial;
+ disk.NodeId = *nodeId;
+ disk.Path = drive.Path;
+ disk.PDiskCategory = TPDiskCategory(PDiskTypeToPDiskType(driveInfo.Type), driveInfo.Kind);
+ disk.PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString());
+ disk.ReadCentric = driveInfo.ReadCentric;
+ disk.Serial = serial;
+ disk.SharedWithOs = driveInfo.SharedWithOs;
+
+ auto diskId = disk.GetId();
+ auto [_, inserted] = disks.try_emplace(diskId, std::move(disk));
+ if (!inserted) {
+ throw TExError() << "Came across duplicate disk on node: " << TErrorParams::NodeId(diskId.NodeId) << " with path: " << TErrorParams::Path(diskId.Path);
+ }
+ }
+ }
+ }
+
+ return disks;
+ }
+
+ static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromDrivesSerials(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) {
+ std::unordered_map<TDiskId, TDiskInfo> disks;
+
+ state.DrivesSerials.ScanRange({}, {}, [&](const auto& serial, const auto& driveInfo, const auto&) {
+ if (!relevantBoxes.contains(driveInfo.BoxId)) {
+ return true;
+ }
+
+ if (serial.Serial.empty()) {
+ STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP04, "Missing disks's serial number");
+ return true;
+ }
+
+ auto nodeId = driveInfo.NodeId;
+ if (!nodeId) {
+ STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP05, "Empty node id for disk with serial number.", (SerialNumber, serial.Serial));
+ return true;
+ }
+ auto hostId = state.HostRecords->GetHostId(*nodeId);
+ if (!hostId) {
+ STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP06, "Couldn't find host id for node.", (NodeId, *nodeId));
+ return true;
+ }
+
+ auto path = GetDiskPathFromNode(*nodeId, serial, state, /* throwOnError */ true);
+
+ TDiskInfo disk;
+ disk.BoxId = driveInfo.BoxId;
+ disk.HostId = *hostId;
+ disk.LastSeenPath = path;
+ disk.LastSeenSerial = TString();
+ disk.NodeId = *nodeId;
+ disk.Path = path;
+ disk.PDiskCategory = TPDiskCategory(PDiskTypeToPDiskType(driveInfo.PDiskType), driveInfo.Kind);
+ disk.PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString());
+ disk.ReadCentric = false;
+ disk.Serial = serial;
+ disk.SharedWithOs = false;
+
+ auto diskId = disk.GetId();
+ auto [_, inserted] = disks.try_emplace(diskId, std::move(disk));
+ if (!inserted) {
+ throw TExError() << "Came across duplicate disk on node: " << TErrorParams::NodeId(diskId.NodeId) << " with path: " << TErrorParams::Path(diskId.Path);
+ }
+
+ return true;
+ });
+
+ return disks;
+ }
+
+ static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromDrivesSerialsAndHostConfig(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) {
+ auto disksFromDrivesSerials = GetDisksFromDrivesSerials(state, relevantBoxes);
+ auto disksFromHostConfig = GetDisksFromHostConfig(state, relevantBoxes);
+ disksFromHostConfig.merge(disksFromDrivesSerials);
+ return disksFromHostConfig;
+ }
+
Schema::PDisk::Guid::Type TBlobStorageController::CheckStaticPDisk(TConfigState &state, TPDiskId pdiskId,
const TPDiskCategory& category, const TMaybe<Schema::PDisk::PDiskConfig::Type>& pdiskConfig,
ui32 *staticSlotUsage) {
@@ -46,251 +268,64 @@ namespace NKikimr {
return info.Guid;
}
- void TBlobStorageController::AllocatePDiskWithSerial(TConfigState& state, ui32 nodeId, const TSerial& serial,
- TDriveSerialInfo *driveInfo) {
- TPDiskId pdiskId = FindFirstEmptyPDiskId(state.PDisks, nodeId);
-
- const TNodeInfo& nodeInfo = state.Nodes.Get().at(nodeId);
- const NPDisk::TDriveData& driveData = nodeInfo.KnownDrives.at(serial.Serial);
- TString fsPath = driveData.Path;
-
- NPDisk::EDeviceType type = PDiskTypeToPDiskType(driveInfo->PDiskType);
- if (type == NPDisk::DEVICE_TYPE_UNKNOWN) {
- type = driveData.DeviceType;
- }
- const TPDiskCategory category(type, driveInfo->Kind);
-
- if (const auto pdiskId = state.FindPDiskByLocation(nodeId, fsPath)) {
- throw TExError() << "PDisk found in PDisks by specific path, fsPath# " << fsPath.Quote()
- << " pdiskId# " << *pdiskId;
- }
-
- ui32 staticSlotUsage = 0;
- auto staticPDiskId = state.FindStaticPDiskByLocation(nodeId, fsPath);
- if (!staticPDiskId) {
- staticPDiskId = state.FindStaticPDiskByLocation(nodeId, serial.Serial);
- }
- if (staticPDiskId) {
- // PDisk is static one, so take it's pdiskId and guid
- // and check that parameters match
- pdiskId = *staticPDiskId;
- driveInfo->Guid = CheckStaticPDisk(state, pdiskId, category, driveInfo->PDiskConfig, &staticSlotUsage);
- }
- // Update FK in DriveSerial table and check for guid
- driveInfo->NodeId = pdiskId.NodeId;
- driveInfo->PDiskId = pdiskId.PDiskId;
- driveInfo->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ALLOCATED;
-
- // if guid is known, reuse it, else generate new
- if (!driveInfo->Guid) {
- driveInfo->Guid = RandomNumber<Schema::PDisk::Guid::Type>();
- }
-
- const auto hostId = state.HostRecords->GetHostId(nodeId);
- if (!hostId) {
- throw TExError() << "Unable to find hostId by nodeId# " << nodeId;
- }
- state.PDisks.ConstructInplaceNewEntry(pdiskId, *hostId, TString(), category.GetRaw(),
- *driveInfo->Guid, false, false, 1000, driveInfo->PDiskConfig.GetOrElse(TString()),
- driveInfo->BoxId, DefaultMaxSlots, NKikimrBlobStorage::EDriveStatus::ACTIVE,
- TInstant::Zero(), NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, serial.Serial,
- TString(), fsPath, staticSlotUsage);
-
- STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP01, "Create new pdisk", (PDiskId, pdiskId), (Path, fsPath));
- }
-
- void TBlobStorageController::ValidatePDiskWithSerial(TConfigState& state, ui32 nodeId, const TSerial& serial,
- const TDriveSerialInfo& driveInfo, std::function<TDriveSerialInfo*()> getMutableItem) {
- // check existing pdisk
- if (!driveInfo.NodeId || !driveInfo.PDiskId) {
- throw TExError() << "Drive is in ALLOCATED stage but has "
- << " NodeId# " << driveInfo.NodeId
- << " PDiskId# " << driveInfo.PDiskId;
- }
-
- const TPDiskId pdiskId(*driveInfo.NodeId, *driveInfo.PDiskId);
- const TPDiskInfo *pdiskInfo = state.PDisks.Find(pdiskId);
- if (!pdiskInfo) {
- throw TExError() << "Unable to find pdisk# " << pdiskId << " from DriveSerial table";
- } else if (pdiskInfo->Path) {
- throw TExError() << "Going to replace existing pdisk with non-empty path# " << pdiskInfo->Path;
- } else if (pdiskInfo->ExpectedSerial != serial.Serial) {
- throw TExError() << "Going to replace existing pdisk with different serial number"
- << " new sn# " << serial.Serial << " existing serial# " << pdiskInfo->ExpectedSerial;
- } else if (pdiskInfo->Guid != *driveInfo.Guid) {
- throw TExError() << "Going to replace existring pdisk with different guid"
- << " guid from DrivesSerials# " << *driveInfo.Guid
- << " guid from PDisks# " << pdiskInfo->Guid;
- }
-
- const TString& path = serial.Serial;
- if (!state.FindStaticPDiskByLocation(*driveInfo.NodeId, path) && !state.FindPDiskByLocation(*driveInfo.NodeId, path)) {
- throw TExError() << "Drive is in ALLOCATED state and PDisk is created,"
- " but is not found neither in PDisks, nor in StaticPDisks by specific path";
- }
-
- if (nodeId && nodeId != *driveInfo.NodeId) {
- // Drive was moved from previous node to new
- TDriveSerialInfo *info = getMutableItem();
- info->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ERROR;
- }
-
- state.PDisksToRemove.erase(pdiskId);
- }
-
void TBlobStorageController::FitPDisksForUserConfig(TConfigState &state) {
- auto pdisksForBoxes = std::exchange(state.Fit.Boxes, {});
- if (pdisksForBoxes.empty()) {
+ auto relevantBoxes = std::exchange(state.Fit.Boxes, {});
+ if (relevantBoxes.empty()) {
return;
}
// re-fill PDisksToRemove set with all PDisks, we will erase remaining ones from this set a bit later
state.PDisksToRemove.clear();
state.PDisks.ForEach([&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) {
- if (pdisksForBoxes.contains(pdiskInfo.BoxId)) {
+ if (relevantBoxes.contains(pdiskInfo.BoxId)) {
state.PDisksToRemove.insert(pdiskId);
}
return true;
});
- // Create new pdisks from DriveSerial table
-
- // Iterate over initial DrivesSerials map since every call to Unshare will invalidate iterators
- state.DrivesSerials.ScanRange({}, {}, [&](const auto& serial, const auto& driveInfo, const auto& getMutableItem) {
- if (!pdisksForBoxes.contains(driveInfo.BoxId)) {
- return true;
- }
- if (driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN) {
- // Try to find drive in currently online nodes and create new PDisk
- if (auto nodeIt = NodeForSerial.find(serial.Serial); nodeIt != NodeForSerial.end()) {
- AllocatePDiskWithSerial(state, nodeIt->second, serial, getMutableItem());
- }
- } else if (driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::ALLOCATED
- || driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::ERROR) {
- const auto it = NodeForSerial.find(serial.Serial);
- const ui32 nodeId = it != NodeForSerial.end() ? it->second : 0;
- // TODO(alexvru): check where no entry in NodeForSerial is a valid case
- ValidatePDiskWithSerial(state, nodeId, serial, driveInfo, getMutableItem);
- }
- return true;
- });
-
- const auto& hostConfigs = state.HostConfigs.Get();
- const auto& boxes = state.Boxes.Get();
- for (const TBoxId& boxId : pdisksForBoxes) {
- const auto boxIt = boxes.find(boxId);
- if (boxIt == boxes.end()) {
- continue; // box was deleted
- }
- const TBoxInfo& box = boxIt->second;
-
- THashSet<TNodeId> usedNodes;
- for (const auto& [hostKey, hostValue] : box.Hosts) {
- const THostConfigId &hostConfigId = hostValue.HostConfigId;
- auto it = hostConfigs.find(hostConfigId);
- if (it == hostConfigs.end()) {
- throw TExHostConfigNotFound(hostConfigId);
- }
- const THostConfigInfo &hostConfig = it->second;
-
- const THostId hostId(hostKey.Fqdn, hostKey.IcPort);
- const auto& nodeId = state.HostRecords->ResolveNodeId(hostKey, hostValue);
- if (!nodeId) {
- throw TExHostNotFound(hostKey) << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId);
- } else if (!usedNodes.insert(*nodeId).second) {
- throw TExError() << "duplicate NodeId" << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId)
- << TErrorParams::Fqdn(hostKey.Fqdn) << TErrorParams::IcPort(hostKey.IcPort);
+ auto disks = GetDisksFromDrivesSerialsAndHostConfig(state, relevantBoxes);
+ for (const auto& [diskId, disk] : disks) {
+ TPDiskId pdiskId;
+ // check if we already have spawned some PDisk at this location
+ if (auto pdiskIdOptional = state.FindPDiskByLocation(diskId.NodeId, diskId.Path)) {
+ pdiskId = *pdiskIdOptional;
+ // yes, we have; find it by id and update some characteristics (that we can update)
+ UpdatePDiskIfNeeded(pdiskId, disk, DefaultMaxSlots, state);
+ } else {
+ // no, this disk is not in map yet; see if it is mentioned in static configuration
+ Schema::PDisk::Guid::Type guid;
+
+ ui32 staticSlotUsage = 0;
+ if (auto pdiskIdOptional = state.FindStaticPDiskByLocation(disk.NodeId, disk.Path)) {
+ // yes, take some data from static configuration
+ pdiskId = *pdiskIdOptional;
+ guid = CheckStaticPDisk(state, pdiskId, disk.PDiskCategory, disk.PDiskConfig, &staticSlotUsage);
+ } else {
+ pdiskId = FindFirstEmptyPDiskId(state.PDisks, disk.NodeId);
+ guid = RandomNumber<Schema::PDisk::Guid::Type>();
}
- for (const auto& [drive, driveInfo] : hostConfig.Drives) {
- TPDiskId pdiskId;
- const TPDiskCategory category(PDiskTypeToPDiskType(driveInfo.Type), driveInfo.Kind);
-
- // check if we already have spawned some PDisk at this location
- if (const auto found = state.FindPDiskByLocation(*nodeId, drive.Path)) {
- // yes, we do; find it by id and update some characteristics (that we can update)
- pdiskId = *found;
- const TPDiskInfo *pdisk = state.PDisks.Find(pdiskId);
- Y_VERIFY(pdisk);
- // update PDisk configuration if needed
- if (pdisk->Kind != category || pdisk->SharedWithOs != driveInfo.SharedWithOs ||
- pdisk->ReadCentric != driveInfo.ReadCentric || pdisk->BoxId != boxId ||
- pdisk->PDiskConfig != driveInfo.PDiskConfig.GetOrElse(TString())) {
- TPDiskInfo *pdisk = state.PDisks.FindForUpdate(pdiskId);
- pdisk->Kind = category;
- pdisk->SharedWithOs = driveInfo.SharedWithOs;
- pdisk->ReadCentric = driveInfo.ReadCentric;
- pdisk->BoxId = boxId;
- pdisk->PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString());
- pdisk->ExtractConfig(DefaultMaxSlots);
- }
- } else {
- Schema::PDisk::Guid::Type guid;
-
- // no, this disk is not in map yet; see if it is mentioned in static configuration
- ui32 staticSlotUsage = 0;
- if (const auto found = state.FindStaticPDiskByLocation(*nodeId, drive.Path)) {
- // yes, take some data from static configuration
- pdiskId = *found;
- guid = CheckStaticPDisk(state, pdiskId, category, driveInfo.PDiskConfig, &staticSlotUsage);
- } else {
- pdiskId = FindFirstEmptyPDiskId(state.PDisks, *nodeId);
- guid = RandomNumber<Schema::PDisk::Guid::Type>();
- }
- TString path = drive.Path;
- // try find current serial number for device
- TString currentSerial;
- if (auto nodeIt = state.Nodes.Get().find(*nodeId); nodeIt != state.Nodes.Get().end()) {
- for (const auto& [serial, driveData] : nodeIt->second.KnownDrives) {
- if (driveData.Path == path) {
- currentSerial = serial;
- break;
- }
- }
- }
-
- // emplace PDisk into set
- state.PDisks.ConstructInplaceNewEntry(pdiskId, hostId, path, category.GetRaw(),
- guid, driveInfo.SharedWithOs, driveInfo.ReadCentric, 1000,
- driveInfo.PDiskConfig.GetOrElse(TString()), boxId, DefaultMaxSlots,
- NKikimrBlobStorage::EDriveStatus::ACTIVE, TInstant::Zero(),
- NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE,
- currentSerial, currentSerial, TString(), staticSlotUsage);
-
- // insert PDisk into location map
- STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP02, "Create new pdisk", (PDiskId, pdiskId),
- (Path, path));
- }
+ // create PDisk
+ state.PDisks.ConstructInplaceNewEntry(pdiskId, disk.HostId, disk.Path,
+ disk.PDiskCategory.GetRaw(), guid, disk.SharedWithOs, disk.ReadCentric,
+ /* nextVslotId */ 1000, disk.PDiskConfig, disk.BoxId, DefaultMaxSlots,
+ NKikimrBlobStorage::EDriveStatus::ACTIVE, /* statusTimestamp */ TInstant::Zero(),
+ NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE,
+ disk.Serial, disk.LastSeenSerial, disk.LastSeenPath, staticSlotUsage);
- state.PDisksToRemove.erase(pdiskId);
- }
+ STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP02, "Create new pdisk", (PDiskId, pdiskId), (Path, disk.Path));
}
+
+ state.PDisksToRemove.erase(pdiskId);
}
+
for (const auto& pdiskId : state.PDisksToRemove) {
STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP03, "PDisk to remove:", (PDiskId, pdiskId));
}
state.CheckConsistency();
}
- void TBlobStorageController::FitPDisksForNode(TConfigState& state, ui32 nodeId, const std::vector<TSerial>& serials) {
- for (const auto& serial : serials) {
- if (const TDriveSerialInfo *driveInfo = state.DrivesSerials.Find(serial)) {
- switch (driveInfo->LifeStage) {
- case NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN:
- AllocatePDiskWithSerial(state, nodeId, serial, state.DrivesSerials.FindForUpdate(serial));
- break;
-
- case NKikimrBlobStorage::TDriveLifeStage::ALLOCATED:
- case NKikimrBlobStorage::TDriveLifeStage::ERROR:
- ValidatePDiskWithSerial(state, nodeId, serial, *driveInfo,
- [&] { return state.DrivesSerials.FindForUpdate(serial); });
- break;
-
- default:
- break;
- }
- }
- }
+ void TBlobStorageController::FitPDisksForNode(TConfigState&, ui32, const std::vector<TSerial>&) {
}
} // NBsController
diff --git a/ydb/core/mind/bscontroller/error.h b/ydb/core/mind/bscontroller/error.h
index 9676a5074c..6effcb3fef 100644
--- a/ydb/core/mind/bscontroller/error.h
+++ b/ydb/core/mind/bscontroller/error.h
@@ -36,6 +36,7 @@ namespace NKikimr::NBsController {
P(ItemConfigGenerationExpected, ui64)
P(GroupId, ui32)
P(StoragePoolName, TString)
+ P(DiskSerialNumber, TString)
struct TVDiskIdTraits {
using Type = TVDiskID;
diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
index e848848d3d..50cfa19e6d 100644
--- a/ydb/core/mind/bscontroller/impl.h
+++ b/ydb/core/mind/bscontroller/impl.h
@@ -848,7 +848,7 @@ public:
};
- std::map<TString, TNodeId> NodeForSerial;
+ std::unordered_map<TString, TNodeId> NodeIdByDiskSerialNumber;
TMap<ui32, TSet<ui32>> NodesAwaitingKeysForGroup;
struct THostConfigInfo {
diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp
index 44ca2510a3..633d3fc5bb 100644
--- a/ydb/core/mind/bscontroller/register_node.cpp
+++ b/ydb/core/mind/bscontroller/register_node.cpp
@@ -36,74 +36,52 @@ class TBlobStorageController::TTxUpdateNodeDrives
STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXRN05, "Add devicesData from NodeWarden",
(NodeId, nodeId), (Devices, createLog()));
- std::map<TString, TString> serialForPath;
- for (const auto& data : Record.GetDrivesData()) {
- serialForPath[data.GetPath()] = data.GetSerialNumber();
+ std::unordered_map<TString, TString> diskSerialNumberByPath;
+ for (const auto& disk : Record.GetDrivesData()) {
+ diskSerialNumberByPath[disk.GetPath()] = disk.GetSerialNumber();
}
NIceDb::TNiceDb db(txc.DB);
using T = Schema::PDisk;
- TPDiskId minPDiskId = TPDiskId::MinForNode(nodeId);
- for (auto it = Self->PDisks.lower_bound(minPDiskId); it != Self->PDisks.end() && it->first.NodeId == nodeId; ++it) {
- Y_VERIFY(it->second);
- TPDiskInfo& info = *it->second;
- TPDiskId pdiskId = it->first;
+ auto minPDiskId = TPDiskId::MinForNode(nodeId);
+ for (auto pdiskIt = Self->PDisks.lower_bound(minPDiskId); pdiskIt != Self->PDisks.end() && pdiskIt->first.NodeId == nodeId; ++pdiskIt) {
+ Y_VERIFY(pdiskIt->second);
+ auto& pdiskInfo = *pdiskIt->second;
+ auto pdiskId = pdiskIt->first;
- const T::TKey::Type key(pdiskId.GetKey());
+ auto key(pdiskId.GetKey());
TString serial;
- if (auto serialIt = serialForPath.find(info.Path); serialIt != serialForPath.end()) {
+ // update pdisk's ExpectedSerial if necessary
+ if (auto serialIt = diskSerialNumberByPath.find(pdiskInfo.Path); serialIt != diskSerialNumberByPath.end()) {
serial = serialIt->second;
- if (info.ExpectedSerial != serial) {
+ if (pdiskInfo.ExpectedSerial != serial) {
+ // the disk on the node with the same label (path) as pdisk has a different serial number
TStringStream log;
- auto prio = NLog::PRI_NOTICE;
-
- if (!info.ExpectedSerial) {
- if (auto driveIt = Self->DrivesSerials.find(TSerial{serial}); driveIt != Self->DrivesSerials.end()) {
- log << "device is managed by HostConfigs and was removed.";
- if (driveIt->second->LifeStage == NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN) {
- log << " Drive was added while node was offline, so update ExpectedSerial and"
- << " remove fictional row from DriveSerial table";
- info.ExpectedSerial = serial;
- Self->DrivesSerials.erase(driveIt);
- db.Table<Schema::DriveSerial>().Key(TSerial{serial}.GetKey()).Delete();
- } else if (driveIt->second->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) {
- log << " Drive is still marked as REMOVED, so do not update ExpectedSerial";
- }
- } else {
- // disk has not seen yet
- info.ExpectedSerial = serial;
- }
- } else if (Self->SerialManagementStage == NKikimrBlobStorage::TSerialManagementStage::CHECK_SERIAL) {
- prio = NLog::PRI_ERROR;
- log << "new serial mismatched stored pdisk's serial";
- } else {
- log << "Set new ExpectedSerial for pdisk";
-
- auto [it, emplaced] = Self->DrivesSerials.emplace(serial, MakeHolder<TDriveSerialInfo>(info.BoxId));
- it->second->Guid = info.Guid;
- it->second->Kind = info.Kind.Kind();
- it->second->PDiskType = PDiskTypeToPDiskType(info.Kind.Type());
- it->second->PDiskConfig = info.PDiskConfig;
- it->second->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED;
+ auto prio = NLog::PRI_ERROR;
- TDriveSerialInfo::Apply(Self, [&, it = it] (auto* adapter) {
- adapter->IssueUpdateRow(txc, TSerial{serial}, *it->second);
- });
-
- info.ExpectedSerial = serial;
+ if (Self->SerialManagementStage == NKikimrBlobStorage::TSerialManagementStage::CHECK_SERIAL) {
+ // don't update ExpectedSerial so that the corresponding PDisk wouldn't be able to start next time
+ log << "disk's serial reported by the node doesn't match pdisk's serial, don't update anything";
+ } else {
+ log << "disk's serial reported by the node doesn't match pdisk's serial, update the later";
+ // update ExpectedSerial
+ pdiskInfo.ExpectedSerial = serial;
db.Table<T>().Key(key).Update<T::ExpectedSerial>(serial);
}
- STLOG(prio, BS_CONTROLLER, BSCTXRN06, log.Str(), (PDiskId, pdiskId), (Path, info.Path),
- (OldSerial, info.ExpectedSerial), (NewSerial, serial));
+
+ STLOG(prio, BS_CONTROLLER, BSCTXRN06, log.Str(), (PDiskId, pdiskId), (Path, pdiskInfo.Path),
+ (OldSerial, pdiskInfo.ExpectedSerial), (NewSerial, serial));
}
}
- if (info.LastSeenSerial != serial) {
- info.LastSeenSerial = serial;
+
+ // update pdisk's LastSeenSerial if necessary
+ if (pdiskInfo.LastSeenSerial != serial) {
+ pdiskInfo.LastSeenSerial = serial;
db.Table<T>().Key(key).Update<T::LastSeenSerial>(serial);
if (serial) {
- Self->ReadPDisk(pdiskId, info, result, NKikimrBlobStorage::RESTART);
+ Self->ReadPDisk(pdiskId, pdiskInfo, result, NKikimrBlobStorage::RESTART);
}
}
}
@@ -111,14 +89,15 @@ class TBlobStorageController::TTxUpdateNodeDrives
TNodeInfo& nodeInfo = Self->GetNode(nodeId);
Self->EraseKnownDrivesOnDisconnected(&nodeInfo);
+ // Update NodeIdByDiskSerialNumber and KnownDrives
for (const auto& data : Record.GetDrivesData()) {
const auto& serial = data.GetSerialNumber();
- if (auto it = Self->NodeForSerial.find(serial); it != Self->NodeForSerial.end() && it->second != nodeId) {
+ if (auto it = Self->NodeIdByDiskSerialNumber.find(serial); it != Self->NodeIdByDiskSerialNumber.end() && it->second != nodeId) {
STLOG(PRI_ERROR, BS_CONTROLLER, BSCTXRN03,
"Received drive from NewNodeId, but drive is reported as placed in OldNodeId",
(NewNodeId, nodeId), (OldNodeId, it->second), (Serial, serial));
} else {
- Self->NodeForSerial[serial] = nodeId;
+ Self->NodeIdByDiskSerialNumber[serial] = nodeId;
}
NPDisk::TDriveData driveData;
DriveDataToDriveData(data, driveData);
@@ -517,7 +496,7 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId) {
void TBlobStorageController::EraseKnownDrivesOnDisconnected(TNodeInfo *nodeInfo) {
for (const auto& [serial, driveData] : nodeInfo->KnownDrives) {
- NodeForSerial.erase(serial);
+ NodeIdByDiskSerialNumber.erase(serial);
}
nodeInfo->KnownDrives.clear();
}
diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp
index 277efd7b66..cee71cfa67 100644
--- a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp
+++ b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp
@@ -838,17 +838,10 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) {
pb->SetSerial("SN_123");
pb->SetBoxId(1);
NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
- if (i == 0) {
- UNIT_ASSERT(response.GetSuccess());
- UNIT_ASSERT(response.StatusSize() == 1);
- UNIT_ASSERT(response.GetStatus(0).GetSuccess());
- } else {
- UNIT_ASSERT(!response.GetSuccess());
- UNIT_ASSERT(response.StatusSize() == 1);
- UNIT_ASSERT(!response.GetStatus(0).GetSuccess());
- UNIT_ASSERT(response.GetStatus(0).GetFailReason()
- == NKikimrBlobStorage::TConfigResponse::TStatus::kAlready);
- }
+ UNIT_ASSERT(!response.GetSuccess());
+ UNIT_ASSERT(response.StatusSize() == 1);
+ UNIT_ASSERT(!response.GetStatus(0).GetSuccess());
+ UNIT_ASSERT(response.GetStatus(0).GetErrorDescription());
}
};
RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, test);
@@ -867,18 +860,20 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) {
pb->SetSerial(TStringBuilder() << "SN_" << i);
pb->SetBoxId(1);
NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
- UNIT_ASSERT(response.GetSuccess());
+ UNIT_ASSERT(!response.GetSuccess());
UNIT_ASSERT(response.StatusSize() == 1);
- UNIT_ASSERT(response.GetStatus(0).GetSuccess());
+ UNIT_ASSERT(!response.GetStatus(0).GetSuccess());
+ UNIT_ASSERT(response.GetStatus(0).GetErrorDescription());
}
for (size_t i = 0; i < disksCount; ++i) {
NKikimrBlobStorage::TConfigRequest request;
auto pb = request.AddCommand()->MutableRemoveDriveSerial();
pb->SetSerial(TStringBuilder() << "SN_" << i);
NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
- UNIT_ASSERT(response.GetSuccess());
+ UNIT_ASSERT(!response.GetSuccess());
UNIT_ASSERT(response.StatusSize() == 1);
- UNIT_ASSERT(response.GetStatus(0).GetSuccess());
+ UNIT_ASSERT(!response.GetStatus(0).GetSuccess());
+ UNIT_ASSERT(response.GetStatus(0).GetErrorDescription());
}
};
RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, test);
diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto
index 438ba15828..b0316a0f29 100644
--- a/ydb/core/protos/blobstorage_config.proto
+++ b/ydb/core/protos/blobstorage_config.proto
@@ -221,8 +221,8 @@ message TDriveLifeStage {
enum E {
UNKNOWN = 0; // life stage is unknown (default)
NOT_SEEN = 1; // info about drive is located in BSC db, but drive is not seen in any node
- ALLOCATED = 2; // PDisk is created
- REMOVED = 3; // drive marked as removed
+ ADDED = 2; // PDisk has been added to the DrivesSerials table
+ REMOVED = 3; // PDisk has been removed from the DrivesSerials table
ERROR = 4; // drive was moved between nodes with allocated VDisks
}
}
@@ -706,6 +706,7 @@ message TConfigResponse {
NKikimrBlobStorage.TVDiskID VDiskId = 12;
NKikimrBlobStorage.TVSlotId VSlotId = 13;
string StoragePoolName = 14;
+ string DiskSerialNumber = 15;
}
}