diff options
author | yuryalekseev <yuryalekseev@yandex-team.com> | 2023-02-02 14:35:49 +0300 |
---|---|---|
committer | yuryalekseev <yuryalekseev@yandex-team.com> | 2023-02-02 14:35:49 +0300 |
commit | ea84a5fee092f387e3e16209ce3773a9edad3602 (patch) | |
tree | 5ca5e3b43b077f45483f29b8dd67c43bb5569ddd | |
parent | 5eaac88e381d86190ba39f09c7864ffc5075fbce (diff) | |
download | ydb-ea84a5fee092f387e3e16209ce3773a9edad3602.tar.gz |
Streamline manual addition/removal of disks to BSC.
-rw-r--r-- | ydb/core/mind/bscontroller/cmds_drive_status.cpp | 197 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/config.h | 4 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/config_fit_pdisks.cpp | 481 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/error.h | 1 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/impl.h | 2 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/register_node.cpp | 87 | ||||
-rw-r--r-- | ydb/core/mind/bscontroller/ut_bscontroller/main.cpp | 25 | ||||
-rw-r--r-- | ydb/core/protos/blobstorage_config.proto | 5 |
8 files changed, 352 insertions, 450 deletions
diff --git a/ydb/core/mind/bscontroller/cmds_drive_status.cpp b/ydb/core/mind/bscontroller/cmds_drive_status.cpp index 31b1696ea8..be9b6b6cb5 100644 --- a/ydb/core/mind/bscontroller/cmds_drive_status.cpp +++ b/ydb/core/mind/bscontroller/cmds_drive_status.cpp @@ -95,196 +95,87 @@ namespace NKikimr::NBsController { void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TAddDriveSerial& cmd, TStatus& /*status*/) { - const TString& newSerial = cmd.GetSerial(); - - Schema::DriveSerial::BoxId::Type boxId = cmd.GetBoxId(); - const TDriveSerialInfo *driveInfo = DrivesSerials.Find(newSerial); + const auto& serial = cmd.GetSerial(); + auto boxId = cmd.GetBoxId(); + auto driveInfo = DrivesSerials.Find(serial); if (driveInfo && driveInfo->LifeStage != NKikimrBlobStorage::TDriveLifeStage::REMOVED) { - throw TExAlready() << "Device with such serial already exists in BSC database and not in lifeStage REMOVED"; + throw TExAlready() << "Device with such serial already exists in BSC database in lifeStage" << driveInfo->LifeStage; } - if (auto it = NodeForSerial.find(newSerial); it != NodeForSerial.end()) { - // Serial of drive is known, but drive not present in DrivesSerial - // Check is it defined in HostConfigs - TNodeId nodeId = it->second; - const TNodeInfo& nodeInfo = Nodes.Get().at(nodeId); - TString path = nodeInfo.KnownDrives.at(newSerial).Path; - - TPDiskId from = TPDiskId::MinForNode(nodeId); - TPDiskId to = TPDiskId::MaxForNode(nodeId); - std::optional<TPDiskId> updatePDiskId; - PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) { - if (pdiskInfo.Path == path) { - updatePDiskId = pdiskId; - return false; - } - return true; - }); - if (updatePDiskId) { - // PDisk is defined through HostConfigs, but there may be fictional row in DrivesSerials - // if row is present - delete it - if (driveInfo) { - DrivesSerials.DeleteExistingEntry(newSerial); - driveInfo = nullptr; - } - TPDiskInfo *pdiskInfo = PDisks.FindForUpdate(*updatePDiskId); - if (pdiskInfo->ExpectedSerial == newSerial) { - throw TExAlready() << "Device with such serial already exists in BSC database and is defined through " - << "HostConfigs"; - } - pdiskInfo->ExpectedSerial = newSerial; - if (pdiskInfo->BoxId != boxId) { - throw TExError() << "Drive is defind in host configs, but placed in another box# " << pdiskInfo->BoxId; - } - STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA06, "Set new ExpectedSerial for HostConfigs drive", - (UniqueId, UniqueId), (Serial, newSerial), (BoxId, boxId), (PDiskId, *updatePDiskId), (Path, path)); - Fit.Boxes.insert(boxId); - return; - } + auto it = NodeIdByDiskSerialNumber.find(serial); + if (it == NodeIdByDiskSerialNumber.end()) { + throw TExError() << "Couldn't find node id for disk with serial number: " << serial; } + auto nodeId = it->second; - { - // Additional check, may give false negative if ExpectedSerial for pdisk is unknown - TMaybe<TPDiskId> from; - TMaybe<TPDiskId> to; - if (auto it = NodeForSerial.find(newSerial); it != NodeForSerial.end()) { - from = TPDiskId::MinForNode(it->second); - to = TPDiskId::MaxForNode(it->second); - } + const auto& nodes = Nodes.Get(); + auto nodeIt = nodes.find(nodeId); + if (nodeIt == nodes.end()) { + throw TExError() << "Couldn't find node with node id: " << nodeId << " for disk with serial number: " << serial; + } - std::optional<TPDiskId> existingPDisk; - PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) { - if (newSerial == pdiskInfo.ExpectedSerial) { - existingPDisk = pdiskId; - return false; - } - return true; - }); - if (existingPDisk) { - throw TExAlready() << "Device with such serial already exists in BSC database and is defined in HostConfigs" - << " pdiskId# " << *existingPDisk; - } + const auto& nodeInfo = nodeIt->second; + auto driveIt = nodeInfo.KnownDrives.find(serial); + if (driveIt == nodeInfo.KnownDrives.end()) { + throw TExError() << "Couldn't find disk on node: " << nodeId << " by serial number: " << serial; } - // delete existing entry, if any, but keep its GUID - std::optional<TMaybe<Schema::DriveSerial::Guid::Type>> guid = driveInfo ? std::make_optional(driveInfo->Guid) : std::nullopt; + // delete REMOVED entry, if any, but keep its GUID + auto guid = driveInfo ? std::make_optional(driveInfo->Guid) : std::nullopt; if (driveInfo) { - DrivesSerials.DeleteExistingEntry(newSerial); + DrivesSerials.DeleteExistingEntry(serial); } - TDriveSerialInfo *driveInfoNew = DrivesSerials.ConstructInplaceNewEntry(newSerial, boxId); + auto driveInfoMutable = DrivesSerials.ConstructInplaceNewEntry(serial, boxId); if (guid) { - driveInfoNew->Guid = *guid; + driveInfoMutable->Guid = *guid; } - driveInfoNew->Kind = cmd.GetKind(); - driveInfoNew->PDiskType = cmd.GetPDiskType(); + driveInfoMutable->Kind = cmd.GetKind(); + driveInfoMutable->PDiskType = cmd.GetPDiskType(); TString config; - const bool success = cmd.GetPDiskConfig().SerializeToString(&config); + auto success = cmd.GetPDiskConfig().SerializeToString(&config); Y_VERIFY(success); - driveInfoNew->PDiskConfig = config; + driveInfoMutable->PDiskConfig = config; + driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ADDED; Fit.Boxes.insert(boxId); - STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA00, "AddDriveSerial", (UniqueId, UniqueId), (Serial, newSerial), + STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA00, "AddDriveSerial", (UniqueId, UniqueId), (Serial, serial), (BoxId, boxId)); } void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TRemoveDriveSerial& cmd, TStatus& /*status*/) { - const TString& serial = cmd.GetSerial(); - - if (const TDriveSerialInfo *driveInfo = DrivesSerials.Find(serial); !driveInfo) { - // Drive is defined in HostConfigs - // + const auto& serial = cmd.GetSerial(); - // Fast search (works only for online nodes) - std::optional<TNodeId> nodeId; - if (auto it = NodeForSerial.find(serial); it != NodeForSerial.end()) { - nodeId = it->second; - } else { - // Slow PDisks fullscan - PDisks.ForEach([&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) { - if (pdiskInfo.ExpectedSerial == serial) { - nodeId = pdiskId.NodeId; - return false; - } - return true; - }); - } - if (!nodeId) { - throw TExError() << "Device with such serial is unknown for BSC"; - } - - TPDiskId from = TPDiskId::MinForNode(*nodeId); - TPDiskId to = TPDiskId::MaxForNode(*nodeId); - std::optional<TPDiskId> removePDiskId; - PDisks.ForEachInRange(from, to, [&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) { - if (pdiskInfo.ExpectedSerial == serial) { - if (pdiskInfo.NumActiveSlots) { - throw TExError() << "There are active vdisks on that drive"; - } - if (removePDiskId) { - throw TExError() << "has two pdisks defined in HostConfigs with same serial number"; - } - removePDiskId = pdiskId; - } - return true; - }); - if (!removePDiskId) { - throw TExError() << "The serial was seen in cluster on node# " << *nodeId - << " but now there are no pdisks with the serial"; - } - auto* pdiskUpdate = PDisks.FindForUpdate(*removePDiskId); - pdiskUpdate->ExpectedSerial = {}; - STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA08, "Reset ExpectedSerial for HostConfig drive", - (UniqueId, UniqueId), (Serial, serial), (PDiskId, *removePDiskId)); - - // create fictional row in DrivesSerials to be able to reply kAlready for already removed disk - // even if they are defined through HostConfig - TDriveSerialInfo *driveInfoNew = DrivesSerials.ConstructInplaceNewEntry(serial, pdiskUpdate->BoxId); - driveInfoNew->Guid = pdiskUpdate->Guid; - driveInfoNew->Kind = pdiskUpdate->Kind.Kind(); - driveInfoNew->PDiskType = PDiskTypeToPDiskType(pdiskUpdate->Kind.Type()); - driveInfoNew->PDiskConfig = pdiskUpdate->PDiskConfig; - driveInfoNew->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED; - - Fit.Boxes.insert(pdiskUpdate->BoxId); - } else { - if (driveInfo->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) { - throw TExAlready() << "Drive is already removed"; - } + auto driveInfo = DrivesSerials.Find(serial); + if (!driveInfo) { + throw TExError() << "Couldn't find disk with serial number: " << serial; + } - if (driveInfo->NodeId && driveInfo->PDiskId) { - TPDiskId pdiskId(*driveInfo->NodeId, *driveInfo->PDiskId); - if (auto* pdiskInfo = PDisks.Find(pdiskId)) { - if (pdiskInfo->NumActiveSlots) { - throw TExError() << "There are active vdisks on that drive"; - } else { - // PDisk will be deleted automatically in FitPDisks - } - } - } + if (driveInfo->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) { + throw TExError() << "Disk with serial number: " << serial << " has already been removed"; + } - TDriveSerialInfo *driveInfoMutable = DrivesSerials.FindForUpdate(serial); - driveInfoMutable->NodeId.Clear(); - driveInfoMutable->PDiskId.Clear(); - driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED; + auto driveInfoMutable = DrivesSerials.FindForUpdate(serial); + driveInfoMutable->NodeId.Clear(); + driveInfoMutable->PDiskId.Clear(); + driveInfoMutable->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED; - Fit.Boxes.insert(driveInfoMutable->BoxId); + Fit.Boxes.insert(driveInfo->BoxId); - STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA07, "RemoveDriveSerial", (UniqueId, UniqueId), (Serial, serial)); - } + STLOG(PRI_INFO, BS_CONTROLLER_AUDIT, BSCA07, "RemoveDriveSerial", (UniqueId, UniqueId), (Serial, serial)); } void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TForgetDriveSerial& cmd, TStatus& /*status*/) { - const TString& serial = cmd.GetSerial(); + const auto& serial = cmd.GetSerial(); - if (const TDriveSerialInfo *driveInfo = DrivesSerials.Find(serial)) { + if (auto driveInfo = DrivesSerials.Find(serial)) { switch (driveInfo->LifeStage) { case NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN: [[fallthrough]]; diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h index 23cc993dac..01d8f4fcef 100644 --- a/ydb/core/mind/bscontroller/config.h +++ b/ydb/core/mind/bscontroller/config.h @@ -105,7 +105,7 @@ namespace NKikimr { // static pdisk/vdisk states std::map<TVSlotId, TStaticVSlotInfo>& StaticVSlots; std::map<TPDiskId, TStaticPDiskInfo>& StaticPDisks; - const std::map<TString, TNodeId>& NodeForSerial; + const std::unordered_map<TString, TNodeId>& NodeIdByDiskSerialNumber; TCowHolder<Schema::State::SerialManagementStage::Type> SerialManagementStage; @@ -135,7 +135,7 @@ namespace NKikimr { , DefaultMaxSlots(controller.DefaultMaxSlots) , StaticVSlots(controller.StaticVSlots) , StaticPDisks(controller.StaticPDisks) - , NodeForSerial(controller.NodeForSerial) + , NodeIdByDiskSerialNumber(controller.NodeIdByDiskSerialNumber) , SerialManagementStage(&controller.SerialManagementStage) , StoragePoolStat(*controller.StoragePoolStat) { diff --git a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp index d8934d4b40..1c34987e89 100644 --- a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp +++ b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp @@ -1,9 +1,54 @@ #include "config.h" +#include <util/generic/string.h> +#include <util/system/types.h> + +namespace NKikimr { + namespace NBsController { + + struct TDiskId { + ui32 NodeId = 0; + TString Path; + + bool operator==(const TDiskId& other) const { + return NodeId == other.NodeId && Path == other.Path; + } + }; + + struct TDiskInfo { + ui32 NodeId = 0; + TBlobStorageController::THostId HostId = {}; + TBoxId BoxId = 0; + TString Path; + TString LastSeenPath; + TString Serial; + TString LastSeenSerial; + bool SharedWithOs = false; + bool ReadCentric = false; + TPDiskCategory PDiskCategory = {}; + TString PDiskConfig; + + TDiskId GetId() const { + return {NodeId, Path}; + } + }; + + } // NBsController +} // NKikimr + +namespace std { + template <> + struct hash<NKikimr::NBsController::TDiskId> { + size_t operator()(const NKikimr::NBsController::TDiskId& diskId) const { + return hash<ui32>()(diskId.NodeId) ^ hash<TString>()(diskId.Path); + } + }; +} + namespace NKikimr { namespace NBsController { - TPDiskId FindFirstEmptyPDiskId(const TOverlayMap<TPDiskId, TBlobStorageController::TPDiskInfo>& pdisks, + static TPDiskId FindFirstEmptyPDiskId(const TOverlayMap<TPDiskId, TBlobStorageController::TPDiskInfo>& pdisks, TNodeId nodeId) { Schema::PDisk::PDiskID::Type nextPDiskID = 1000; // start allocation from this number // generate PDisk id; skip generated one if it already exists (e.g. user has added @@ -21,6 +66,183 @@ namespace NKikimr { return proto.ParseFromString(s) ? SingleLineProto(proto) : "<error>"; } + static void UpdatePDiskIfNeeded(const TPDiskId& pdiskId, const TDiskInfo& disk, ui32 defaultMaxSlots, TBlobStorageController::TConfigState& state) { + auto pdiskInfo = state.PDisks.Find(pdiskId); + Y_VERIFY(pdiskInfo != nullptr); + if (pdiskInfo->Kind != disk.PDiskCategory || + pdiskInfo->SharedWithOs != disk.SharedWithOs || + pdiskInfo->ReadCentric != disk.ReadCentric || + pdiskInfo->BoxId != disk.BoxId || + pdiskInfo->PDiskConfig != disk.PDiskConfig) + { + // update PDisk configuration + auto pdiskInfo = state.PDisks.FindForUpdate(pdiskId); + Y_VERIFY(pdiskInfo != nullptr); + pdiskInfo->Kind = disk.PDiskCategory; + pdiskInfo->SharedWithOs = disk.SharedWithOs; + pdiskInfo->ReadCentric = disk.ReadCentric; + pdiskInfo->BoxId = disk.BoxId; + pdiskInfo->PDiskConfig = disk.PDiskConfig; + pdiskInfo->ExtractConfig(defaultMaxSlots); + } + } + + // return TString not const TString& to make sure we never use dangling reference + static TString GetDiskPathFromNode(ui32 nodeId, const TString& serialNumber, const TBlobStorageController::TConfigState& state, bool throwOnError = false) { + if (auto nodeIt = state.Nodes.Get().find(nodeId); nodeIt != state.Nodes.Get().end()) { + for (const auto& [_, driveData] : nodeIt->second.KnownDrives) { + if (serialNumber == driveData.SerialNumber) { + return driveData.Path; + } + } + if (throwOnError) { + throw TExError() << "Couldn't find disk's path by serial number " << TErrorParams::DiskSerialNumber(serialNumber); + } + } else { + if (throwOnError) { + throw TExError() << "Unknown node id " << TErrorParams::NodeId(nodeId); + } + } + + return TString(); + } + + // return TString not const TString& to make sure we never use dangling reference + static TString GetDiskSerialNumberFromNode(ui32 nodeId, const TString& path, const TBlobStorageController::TConfigState& state, bool throwOnError = false) { + if (auto nodeIt = state.Nodes.Get().find(nodeId); nodeIt != state.Nodes.Get().end()) { + for (const auto& [_, driveData] : nodeIt->second.KnownDrives) { + if (path == driveData.Path) { + return driveData.SerialNumber; + } + } + if (throwOnError) { + throw TExError() << "Couldn't find disk's serial number by path " << TErrorParams::Path(path); + } + } else { + if (throwOnError) { + throw TExError() << "Unknown node id " << TErrorParams::NodeId(nodeId); + } + } + + return TString(); + } + + static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromHostConfig(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) { + std::unordered_map<TDiskId, TDiskInfo> disks; + + const auto& hostConfigs = state.HostConfigs.Get(); + const auto& boxes = state.Boxes.Get(); + for (const TBoxId& boxId : relevantBoxes) { + const auto boxIt = boxes.find(boxId); + if (boxIt == boxes.end()) { + continue; // box was deleted + } + const auto& box = boxIt->second; + + THashSet<TNodeId> usedNodes; + for (const auto& [hostKey, hostValue] : box.Hosts) { + const auto& hostConfigId = hostValue.HostConfigId; + auto it = hostConfigs.find(hostConfigId); + if (it == hostConfigs.end()) { + throw TExHostConfigNotFound(hostConfigId); + } + const auto& hostConfig = it->second; + + const TBlobStorageController::THostId hostId(hostKey.Fqdn, hostKey.IcPort); + const auto& nodeId = state.HostRecords->ResolveNodeId(hostKey, hostValue); + if (!nodeId) { + throw TExHostNotFound(hostKey) << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId); + } else if (!usedNodes.insert(*nodeId).second) { + throw TExError() << "duplicate NodeId" << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId) + << TErrorParams::Fqdn(hostKey.Fqdn) << TErrorParams::IcPort(hostKey.IcPort); + } + + for (const auto& [drive, driveInfo] : hostConfig.Drives) { + auto serial = GetDiskSerialNumberFromNode(*nodeId, drive.Path, state, /* throwOnError */ false); + + TDiskInfo disk; + disk.BoxId = boxId; + disk.HostId = hostId; + disk.LastSeenPath = TString(); + disk.LastSeenSerial = serial; + disk.NodeId = *nodeId; + disk.Path = drive.Path; + disk.PDiskCategory = TPDiskCategory(PDiskTypeToPDiskType(driveInfo.Type), driveInfo.Kind); + disk.PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString()); + disk.ReadCentric = driveInfo.ReadCentric; + disk.Serial = serial; + disk.SharedWithOs = driveInfo.SharedWithOs; + + auto diskId = disk.GetId(); + auto [_, inserted] = disks.try_emplace(diskId, std::move(disk)); + if (!inserted) { + throw TExError() << "Came across duplicate disk on node: " << TErrorParams::NodeId(diskId.NodeId) << " with path: " << TErrorParams::Path(diskId.Path); + } + } + } + } + + return disks; + } + + static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromDrivesSerials(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) { + std::unordered_map<TDiskId, TDiskInfo> disks; + + state.DrivesSerials.ScanRange({}, {}, [&](const auto& serial, const auto& driveInfo, const auto&) { + if (!relevantBoxes.contains(driveInfo.BoxId)) { + return true; + } + + if (serial.Serial.empty()) { + STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP04, "Missing disks's serial number"); + return true; + } + + auto nodeId = driveInfo.NodeId; + if (!nodeId) { + STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP05, "Empty node id for disk with serial number.", (SerialNumber, serial.Serial)); + return true; + } + auto hostId = state.HostRecords->GetHostId(*nodeId); + if (!hostId) { + STLOG(PRI_ERROR, BS_CONTROLLER, BSCFP06, "Couldn't find host id for node.", (NodeId, *nodeId)); + return true; + } + + auto path = GetDiskPathFromNode(*nodeId, serial, state, /* throwOnError */ true); + + TDiskInfo disk; + disk.BoxId = driveInfo.BoxId; + disk.HostId = *hostId; + disk.LastSeenPath = path; + disk.LastSeenSerial = TString(); + disk.NodeId = *nodeId; + disk.Path = path; + disk.PDiskCategory = TPDiskCategory(PDiskTypeToPDiskType(driveInfo.PDiskType), driveInfo.Kind); + disk.PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString()); + disk.ReadCentric = false; + disk.Serial = serial; + disk.SharedWithOs = false; + + auto diskId = disk.GetId(); + auto [_, inserted] = disks.try_emplace(diskId, std::move(disk)); + if (!inserted) { + throw TExError() << "Came across duplicate disk on node: " << TErrorParams::NodeId(diskId.NodeId) << " with path: " << TErrorParams::Path(diskId.Path); + } + + return true; + }); + + return disks; + } + + static std::unordered_map<TDiskId, TDiskInfo> GetDisksFromDrivesSerialsAndHostConfig(TBlobStorageController::TConfigState& state, std::set<TBoxId>& relevantBoxes) { + auto disksFromDrivesSerials = GetDisksFromDrivesSerials(state, relevantBoxes); + auto disksFromHostConfig = GetDisksFromHostConfig(state, relevantBoxes); + disksFromHostConfig.merge(disksFromDrivesSerials); + return disksFromHostConfig; + } + Schema::PDisk::Guid::Type TBlobStorageController::CheckStaticPDisk(TConfigState &state, TPDiskId pdiskId, const TPDiskCategory& category, const TMaybe<Schema::PDisk::PDiskConfig::Type>& pdiskConfig, ui32 *staticSlotUsage) { @@ -46,251 +268,64 @@ namespace NKikimr { return info.Guid; } - void TBlobStorageController::AllocatePDiskWithSerial(TConfigState& state, ui32 nodeId, const TSerial& serial, - TDriveSerialInfo *driveInfo) { - TPDiskId pdiskId = FindFirstEmptyPDiskId(state.PDisks, nodeId); - - const TNodeInfo& nodeInfo = state.Nodes.Get().at(nodeId); - const NPDisk::TDriveData& driveData = nodeInfo.KnownDrives.at(serial.Serial); - TString fsPath = driveData.Path; - - NPDisk::EDeviceType type = PDiskTypeToPDiskType(driveInfo->PDiskType); - if (type == NPDisk::DEVICE_TYPE_UNKNOWN) { - type = driveData.DeviceType; - } - const TPDiskCategory category(type, driveInfo->Kind); - - if (const auto pdiskId = state.FindPDiskByLocation(nodeId, fsPath)) { - throw TExError() << "PDisk found in PDisks by specific path, fsPath# " << fsPath.Quote() - << " pdiskId# " << *pdiskId; - } - - ui32 staticSlotUsage = 0; - auto staticPDiskId = state.FindStaticPDiskByLocation(nodeId, fsPath); - if (!staticPDiskId) { - staticPDiskId = state.FindStaticPDiskByLocation(nodeId, serial.Serial); - } - if (staticPDiskId) { - // PDisk is static one, so take it's pdiskId and guid - // and check that parameters match - pdiskId = *staticPDiskId; - driveInfo->Guid = CheckStaticPDisk(state, pdiskId, category, driveInfo->PDiskConfig, &staticSlotUsage); - } - // Update FK in DriveSerial table and check for guid - driveInfo->NodeId = pdiskId.NodeId; - driveInfo->PDiskId = pdiskId.PDiskId; - driveInfo->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ALLOCATED; - - // if guid is known, reuse it, else generate new - if (!driveInfo->Guid) { - driveInfo->Guid = RandomNumber<Schema::PDisk::Guid::Type>(); - } - - const auto hostId = state.HostRecords->GetHostId(nodeId); - if (!hostId) { - throw TExError() << "Unable to find hostId by nodeId# " << nodeId; - } - state.PDisks.ConstructInplaceNewEntry(pdiskId, *hostId, TString(), category.GetRaw(), - *driveInfo->Guid, false, false, 1000, driveInfo->PDiskConfig.GetOrElse(TString()), - driveInfo->BoxId, DefaultMaxSlots, NKikimrBlobStorage::EDriveStatus::ACTIVE, - TInstant::Zero(), NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, serial.Serial, - TString(), fsPath, staticSlotUsage); - - STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP01, "Create new pdisk", (PDiskId, pdiskId), (Path, fsPath)); - } - - void TBlobStorageController::ValidatePDiskWithSerial(TConfigState& state, ui32 nodeId, const TSerial& serial, - const TDriveSerialInfo& driveInfo, std::function<TDriveSerialInfo*()> getMutableItem) { - // check existing pdisk - if (!driveInfo.NodeId || !driveInfo.PDiskId) { - throw TExError() << "Drive is in ALLOCATED stage but has " - << " NodeId# " << driveInfo.NodeId - << " PDiskId# " << driveInfo.PDiskId; - } - - const TPDiskId pdiskId(*driveInfo.NodeId, *driveInfo.PDiskId); - const TPDiskInfo *pdiskInfo = state.PDisks.Find(pdiskId); - if (!pdiskInfo) { - throw TExError() << "Unable to find pdisk# " << pdiskId << " from DriveSerial table"; - } else if (pdiskInfo->Path) { - throw TExError() << "Going to replace existing pdisk with non-empty path# " << pdiskInfo->Path; - } else if (pdiskInfo->ExpectedSerial != serial.Serial) { - throw TExError() << "Going to replace existing pdisk with different serial number" - << " new sn# " << serial.Serial << " existing serial# " << pdiskInfo->ExpectedSerial; - } else if (pdiskInfo->Guid != *driveInfo.Guid) { - throw TExError() << "Going to replace existring pdisk with different guid" - << " guid from DrivesSerials# " << *driveInfo.Guid - << " guid from PDisks# " << pdiskInfo->Guid; - } - - const TString& path = serial.Serial; - if (!state.FindStaticPDiskByLocation(*driveInfo.NodeId, path) && !state.FindPDiskByLocation(*driveInfo.NodeId, path)) { - throw TExError() << "Drive is in ALLOCATED state and PDisk is created," - " but is not found neither in PDisks, nor in StaticPDisks by specific path"; - } - - if (nodeId && nodeId != *driveInfo.NodeId) { - // Drive was moved from previous node to new - TDriveSerialInfo *info = getMutableItem(); - info->LifeStage = NKikimrBlobStorage::TDriveLifeStage::ERROR; - } - - state.PDisksToRemove.erase(pdiskId); - } - void TBlobStorageController::FitPDisksForUserConfig(TConfigState &state) { - auto pdisksForBoxes = std::exchange(state.Fit.Boxes, {}); - if (pdisksForBoxes.empty()) { + auto relevantBoxes = std::exchange(state.Fit.Boxes, {}); + if (relevantBoxes.empty()) { return; } // re-fill PDisksToRemove set with all PDisks, we will erase remaining ones from this set a bit later state.PDisksToRemove.clear(); state.PDisks.ForEach([&](const TPDiskId& pdiskId, const TPDiskInfo& pdiskInfo) { - if (pdisksForBoxes.contains(pdiskInfo.BoxId)) { + if (relevantBoxes.contains(pdiskInfo.BoxId)) { state.PDisksToRemove.insert(pdiskId); } return true; }); - // Create new pdisks from DriveSerial table - - // Iterate over initial DrivesSerials map since every call to Unshare will invalidate iterators - state.DrivesSerials.ScanRange({}, {}, [&](const auto& serial, const auto& driveInfo, const auto& getMutableItem) { - if (!pdisksForBoxes.contains(driveInfo.BoxId)) { - return true; - } - if (driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN) { - // Try to find drive in currently online nodes and create new PDisk - if (auto nodeIt = NodeForSerial.find(serial.Serial); nodeIt != NodeForSerial.end()) { - AllocatePDiskWithSerial(state, nodeIt->second, serial, getMutableItem()); - } - } else if (driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::ALLOCATED - || driveInfo.LifeStage == NKikimrBlobStorage::TDriveLifeStage::ERROR) { - const auto it = NodeForSerial.find(serial.Serial); - const ui32 nodeId = it != NodeForSerial.end() ? it->second : 0; - // TODO(alexvru): check where no entry in NodeForSerial is a valid case - ValidatePDiskWithSerial(state, nodeId, serial, driveInfo, getMutableItem); - } - return true; - }); - - const auto& hostConfigs = state.HostConfigs.Get(); - const auto& boxes = state.Boxes.Get(); - for (const TBoxId& boxId : pdisksForBoxes) { - const auto boxIt = boxes.find(boxId); - if (boxIt == boxes.end()) { - continue; // box was deleted - } - const TBoxInfo& box = boxIt->second; - - THashSet<TNodeId> usedNodes; - for (const auto& [hostKey, hostValue] : box.Hosts) { - const THostConfigId &hostConfigId = hostValue.HostConfigId; - auto it = hostConfigs.find(hostConfigId); - if (it == hostConfigs.end()) { - throw TExHostConfigNotFound(hostConfigId); - } - const THostConfigInfo &hostConfig = it->second; - - const THostId hostId(hostKey.Fqdn, hostKey.IcPort); - const auto& nodeId = state.HostRecords->ResolveNodeId(hostKey, hostValue); - if (!nodeId) { - throw TExHostNotFound(hostKey) << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId); - } else if (!usedNodes.insert(*nodeId).second) { - throw TExError() << "duplicate NodeId" << TErrorParams::BoxId(boxId) << TErrorParams::NodeId(*nodeId) - << TErrorParams::Fqdn(hostKey.Fqdn) << TErrorParams::IcPort(hostKey.IcPort); + auto disks = GetDisksFromDrivesSerialsAndHostConfig(state, relevantBoxes); + for (const auto& [diskId, disk] : disks) { + TPDiskId pdiskId; + // check if we already have spawned some PDisk at this location + if (auto pdiskIdOptional = state.FindPDiskByLocation(diskId.NodeId, diskId.Path)) { + pdiskId = *pdiskIdOptional; + // yes, we have; find it by id and update some characteristics (that we can update) + UpdatePDiskIfNeeded(pdiskId, disk, DefaultMaxSlots, state); + } else { + // no, this disk is not in map yet; see if it is mentioned in static configuration + Schema::PDisk::Guid::Type guid; + + ui32 staticSlotUsage = 0; + if (auto pdiskIdOptional = state.FindStaticPDiskByLocation(disk.NodeId, disk.Path)) { + // yes, take some data from static configuration + pdiskId = *pdiskIdOptional; + guid = CheckStaticPDisk(state, pdiskId, disk.PDiskCategory, disk.PDiskConfig, &staticSlotUsage); + } else { + pdiskId = FindFirstEmptyPDiskId(state.PDisks, disk.NodeId); + guid = RandomNumber<Schema::PDisk::Guid::Type>(); } - for (const auto& [drive, driveInfo] : hostConfig.Drives) { - TPDiskId pdiskId; - const TPDiskCategory category(PDiskTypeToPDiskType(driveInfo.Type), driveInfo.Kind); - - // check if we already have spawned some PDisk at this location - if (const auto found = state.FindPDiskByLocation(*nodeId, drive.Path)) { - // yes, we do; find it by id and update some characteristics (that we can update) - pdiskId = *found; - const TPDiskInfo *pdisk = state.PDisks.Find(pdiskId); - Y_VERIFY(pdisk); - // update PDisk configuration if needed - if (pdisk->Kind != category || pdisk->SharedWithOs != driveInfo.SharedWithOs || - pdisk->ReadCentric != driveInfo.ReadCentric || pdisk->BoxId != boxId || - pdisk->PDiskConfig != driveInfo.PDiskConfig.GetOrElse(TString())) { - TPDiskInfo *pdisk = state.PDisks.FindForUpdate(pdiskId); - pdisk->Kind = category; - pdisk->SharedWithOs = driveInfo.SharedWithOs; - pdisk->ReadCentric = driveInfo.ReadCentric; - pdisk->BoxId = boxId; - pdisk->PDiskConfig = driveInfo.PDiskConfig.GetOrElse(TString()); - pdisk->ExtractConfig(DefaultMaxSlots); - } - } else { - Schema::PDisk::Guid::Type guid; - - // no, this disk is not in map yet; see if it is mentioned in static configuration - ui32 staticSlotUsage = 0; - if (const auto found = state.FindStaticPDiskByLocation(*nodeId, drive.Path)) { - // yes, take some data from static configuration - pdiskId = *found; - guid = CheckStaticPDisk(state, pdiskId, category, driveInfo.PDiskConfig, &staticSlotUsage); - } else { - pdiskId = FindFirstEmptyPDiskId(state.PDisks, *nodeId); - guid = RandomNumber<Schema::PDisk::Guid::Type>(); - } - TString path = drive.Path; - // try find current serial number for device - TString currentSerial; - if (auto nodeIt = state.Nodes.Get().find(*nodeId); nodeIt != state.Nodes.Get().end()) { - for (const auto& [serial, driveData] : nodeIt->second.KnownDrives) { - if (driveData.Path == path) { - currentSerial = serial; - break; - } - } - } - - // emplace PDisk into set - state.PDisks.ConstructInplaceNewEntry(pdiskId, hostId, path, category.GetRaw(), - guid, driveInfo.SharedWithOs, driveInfo.ReadCentric, 1000, - driveInfo.PDiskConfig.GetOrElse(TString()), boxId, DefaultMaxSlots, - NKikimrBlobStorage::EDriveStatus::ACTIVE, TInstant::Zero(), - NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, - currentSerial, currentSerial, TString(), staticSlotUsage); - - // insert PDisk into location map - STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP02, "Create new pdisk", (PDiskId, pdiskId), - (Path, path)); - } + // create PDisk + state.PDisks.ConstructInplaceNewEntry(pdiskId, disk.HostId, disk.Path, + disk.PDiskCategory.GetRaw(), guid, disk.SharedWithOs, disk.ReadCentric, + /* nextVslotId */ 1000, disk.PDiskConfig, disk.BoxId, DefaultMaxSlots, + NKikimrBlobStorage::EDriveStatus::ACTIVE, /* statusTimestamp */ TInstant::Zero(), + NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, + disk.Serial, disk.LastSeenSerial, disk.LastSeenPath, staticSlotUsage); - state.PDisksToRemove.erase(pdiskId); - } + STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP02, "Create new pdisk", (PDiskId, pdiskId), (Path, disk.Path)); } + + state.PDisksToRemove.erase(pdiskId); } + for (const auto& pdiskId : state.PDisksToRemove) { STLOG(PRI_NOTICE, BS_CONTROLLER, BSCFP03, "PDisk to remove:", (PDiskId, pdiskId)); } state.CheckConsistency(); } - void TBlobStorageController::FitPDisksForNode(TConfigState& state, ui32 nodeId, const std::vector<TSerial>& serials) { - for (const auto& serial : serials) { - if (const TDriveSerialInfo *driveInfo = state.DrivesSerials.Find(serial)) { - switch (driveInfo->LifeStage) { - case NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN: - AllocatePDiskWithSerial(state, nodeId, serial, state.DrivesSerials.FindForUpdate(serial)); - break; - - case NKikimrBlobStorage::TDriveLifeStage::ALLOCATED: - case NKikimrBlobStorage::TDriveLifeStage::ERROR: - ValidatePDiskWithSerial(state, nodeId, serial, *driveInfo, - [&] { return state.DrivesSerials.FindForUpdate(serial); }); - break; - - default: - break; - } - } - } + void TBlobStorageController::FitPDisksForNode(TConfigState&, ui32, const std::vector<TSerial>&) { } } // NBsController diff --git a/ydb/core/mind/bscontroller/error.h b/ydb/core/mind/bscontroller/error.h index 9676a5074c..6effcb3fef 100644 --- a/ydb/core/mind/bscontroller/error.h +++ b/ydb/core/mind/bscontroller/error.h @@ -36,6 +36,7 @@ namespace NKikimr::NBsController { P(ItemConfigGenerationExpected, ui64) P(GroupId, ui32) P(StoragePoolName, TString) + P(DiskSerialNumber, TString) struct TVDiskIdTraits { using Type = TVDiskID; diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index e848848d3d..50cfa19e6d 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -848,7 +848,7 @@ public: }; - std::map<TString, TNodeId> NodeForSerial; + std::unordered_map<TString, TNodeId> NodeIdByDiskSerialNumber; TMap<ui32, TSet<ui32>> NodesAwaitingKeysForGroup; struct THostConfigInfo { diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index 44ca2510a3..633d3fc5bb 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -36,74 +36,52 @@ class TBlobStorageController::TTxUpdateNodeDrives STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXRN05, "Add devicesData from NodeWarden", (NodeId, nodeId), (Devices, createLog())); - std::map<TString, TString> serialForPath; - for (const auto& data : Record.GetDrivesData()) { - serialForPath[data.GetPath()] = data.GetSerialNumber(); + std::unordered_map<TString, TString> diskSerialNumberByPath; + for (const auto& disk : Record.GetDrivesData()) { + diskSerialNumberByPath[disk.GetPath()] = disk.GetSerialNumber(); } NIceDb::TNiceDb db(txc.DB); using T = Schema::PDisk; - TPDiskId minPDiskId = TPDiskId::MinForNode(nodeId); - for (auto it = Self->PDisks.lower_bound(minPDiskId); it != Self->PDisks.end() && it->first.NodeId == nodeId; ++it) { - Y_VERIFY(it->second); - TPDiskInfo& info = *it->second; - TPDiskId pdiskId = it->first; + auto minPDiskId = TPDiskId::MinForNode(nodeId); + for (auto pdiskIt = Self->PDisks.lower_bound(minPDiskId); pdiskIt != Self->PDisks.end() && pdiskIt->first.NodeId == nodeId; ++pdiskIt) { + Y_VERIFY(pdiskIt->second); + auto& pdiskInfo = *pdiskIt->second; + auto pdiskId = pdiskIt->first; - const T::TKey::Type key(pdiskId.GetKey()); + auto key(pdiskId.GetKey()); TString serial; - if (auto serialIt = serialForPath.find(info.Path); serialIt != serialForPath.end()) { + // update pdisk's ExpectedSerial if necessary + if (auto serialIt = diskSerialNumberByPath.find(pdiskInfo.Path); serialIt != diskSerialNumberByPath.end()) { serial = serialIt->second; - if (info.ExpectedSerial != serial) { + if (pdiskInfo.ExpectedSerial != serial) { + // the disk on the node with the same label (path) as pdisk has a different serial number TStringStream log; - auto prio = NLog::PRI_NOTICE; - - if (!info.ExpectedSerial) { - if (auto driveIt = Self->DrivesSerials.find(TSerial{serial}); driveIt != Self->DrivesSerials.end()) { - log << "device is managed by HostConfigs and was removed."; - if (driveIt->second->LifeStage == NKikimrBlobStorage::TDriveLifeStage::NOT_SEEN) { - log << " Drive was added while node was offline, so update ExpectedSerial and" - << " remove fictional row from DriveSerial table"; - info.ExpectedSerial = serial; - Self->DrivesSerials.erase(driveIt); - db.Table<Schema::DriveSerial>().Key(TSerial{serial}.GetKey()).Delete(); - } else if (driveIt->second->LifeStage == NKikimrBlobStorage::TDriveLifeStage::REMOVED) { - log << " Drive is still marked as REMOVED, so do not update ExpectedSerial"; - } - } else { - // disk has not seen yet - info.ExpectedSerial = serial; - } - } else if (Self->SerialManagementStage == NKikimrBlobStorage::TSerialManagementStage::CHECK_SERIAL) { - prio = NLog::PRI_ERROR; - log << "new serial mismatched stored pdisk's serial"; - } else { - log << "Set new ExpectedSerial for pdisk"; - - auto [it, emplaced] = Self->DrivesSerials.emplace(serial, MakeHolder<TDriveSerialInfo>(info.BoxId)); - it->second->Guid = info.Guid; - it->second->Kind = info.Kind.Kind(); - it->second->PDiskType = PDiskTypeToPDiskType(info.Kind.Type()); - it->second->PDiskConfig = info.PDiskConfig; - it->second->LifeStage = NKikimrBlobStorage::TDriveLifeStage::REMOVED; + auto prio = NLog::PRI_ERROR; - TDriveSerialInfo::Apply(Self, [&, it = it] (auto* adapter) { - adapter->IssueUpdateRow(txc, TSerial{serial}, *it->second); - }); - - info.ExpectedSerial = serial; + if (Self->SerialManagementStage == NKikimrBlobStorage::TSerialManagementStage::CHECK_SERIAL) { + // don't update ExpectedSerial so that the corresponding PDisk wouldn't be able to start next time + log << "disk's serial reported by the node doesn't match pdisk's serial, don't update anything"; + } else { + log << "disk's serial reported by the node doesn't match pdisk's serial, update the later"; + // update ExpectedSerial + pdiskInfo.ExpectedSerial = serial; db.Table<T>().Key(key).Update<T::ExpectedSerial>(serial); } - STLOG(prio, BS_CONTROLLER, BSCTXRN06, log.Str(), (PDiskId, pdiskId), (Path, info.Path), - (OldSerial, info.ExpectedSerial), (NewSerial, serial)); + + STLOG(prio, BS_CONTROLLER, BSCTXRN06, log.Str(), (PDiskId, pdiskId), (Path, pdiskInfo.Path), + (OldSerial, pdiskInfo.ExpectedSerial), (NewSerial, serial)); } } - if (info.LastSeenSerial != serial) { - info.LastSeenSerial = serial; + + // update pdisk's LastSeenSerial if necessary + if (pdiskInfo.LastSeenSerial != serial) { + pdiskInfo.LastSeenSerial = serial; db.Table<T>().Key(key).Update<T::LastSeenSerial>(serial); if (serial) { - Self->ReadPDisk(pdiskId, info, result, NKikimrBlobStorage::RESTART); + Self->ReadPDisk(pdiskId, pdiskInfo, result, NKikimrBlobStorage::RESTART); } } } @@ -111,14 +89,15 @@ class TBlobStorageController::TTxUpdateNodeDrives TNodeInfo& nodeInfo = Self->GetNode(nodeId); Self->EraseKnownDrivesOnDisconnected(&nodeInfo); + // Update NodeIdByDiskSerialNumber and KnownDrives for (const auto& data : Record.GetDrivesData()) { const auto& serial = data.GetSerialNumber(); - if (auto it = Self->NodeForSerial.find(serial); it != Self->NodeForSerial.end() && it->second != nodeId) { + if (auto it = Self->NodeIdByDiskSerialNumber.find(serial); it != Self->NodeIdByDiskSerialNumber.end() && it->second != nodeId) { STLOG(PRI_ERROR, BS_CONTROLLER, BSCTXRN03, "Received drive from NewNodeId, but drive is reported as placed in OldNodeId", (NewNodeId, nodeId), (OldNodeId, it->second), (Serial, serial)); } else { - Self->NodeForSerial[serial] = nodeId; + Self->NodeIdByDiskSerialNumber[serial] = nodeId; } NPDisk::TDriveData driveData; DriveDataToDriveData(data, driveData); @@ -517,7 +496,7 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId) { void TBlobStorageController::EraseKnownDrivesOnDisconnected(TNodeInfo *nodeInfo) { for (const auto& [serial, driveData] : nodeInfo->KnownDrives) { - NodeForSerial.erase(serial); + NodeIdByDiskSerialNumber.erase(serial); } nodeInfo->KnownDrives.clear(); } diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp index 277efd7b66..cee71cfa67 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp +++ b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp @@ -838,17 +838,10 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) { pb->SetSerial("SN_123"); pb->SetBoxId(1); NKikimrBlobStorage::TConfigResponse response = env.Invoke(request); - if (i == 0) { - UNIT_ASSERT(response.GetSuccess()); - UNIT_ASSERT(response.StatusSize() == 1); - UNIT_ASSERT(response.GetStatus(0).GetSuccess()); - } else { - UNIT_ASSERT(!response.GetSuccess()); - UNIT_ASSERT(response.StatusSize() == 1); - UNIT_ASSERT(!response.GetStatus(0).GetSuccess()); - UNIT_ASSERT(response.GetStatus(0).GetFailReason() - == NKikimrBlobStorage::TConfigResponse::TStatus::kAlready); - } + UNIT_ASSERT(!response.GetSuccess()); + UNIT_ASSERT(response.StatusSize() == 1); + UNIT_ASSERT(!response.GetStatus(0).GetSuccess()); + UNIT_ASSERT(response.GetStatus(0).GetErrorDescription()); } }; RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, test); @@ -867,18 +860,20 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) { pb->SetSerial(TStringBuilder() << "SN_" << i); pb->SetBoxId(1); NKikimrBlobStorage::TConfigResponse response = env.Invoke(request); - UNIT_ASSERT(response.GetSuccess()); + UNIT_ASSERT(!response.GetSuccess()); UNIT_ASSERT(response.StatusSize() == 1); - UNIT_ASSERT(response.GetStatus(0).GetSuccess()); + UNIT_ASSERT(!response.GetStatus(0).GetSuccess()); + UNIT_ASSERT(response.GetStatus(0).GetErrorDescription()); } for (size_t i = 0; i < disksCount; ++i) { NKikimrBlobStorage::TConfigRequest request; auto pb = request.AddCommand()->MutableRemoveDriveSerial(); pb->SetSerial(TStringBuilder() << "SN_" << i); NKikimrBlobStorage::TConfigResponse response = env.Invoke(request); - UNIT_ASSERT(response.GetSuccess()); + UNIT_ASSERT(!response.GetSuccess()); UNIT_ASSERT(response.StatusSize() == 1); - UNIT_ASSERT(response.GetStatus(0).GetSuccess()); + UNIT_ASSERT(!response.GetStatus(0).GetSuccess()); + UNIT_ASSERT(response.GetStatus(0).GetErrorDescription()); } }; RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, test); diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto index 438ba15828..b0316a0f29 100644 --- a/ydb/core/protos/blobstorage_config.proto +++ b/ydb/core/protos/blobstorage_config.proto @@ -221,8 +221,8 @@ message TDriveLifeStage { enum E { UNKNOWN = 0; // life stage is unknown (default) NOT_SEEN = 1; // info about drive is located in BSC db, but drive is not seen in any node - ALLOCATED = 2; // PDisk is created - REMOVED = 3; // drive marked as removed + ADDED = 2; // PDisk has been added to the DrivesSerials table + REMOVED = 3; // PDisk has been removed from the DrivesSerials table ERROR = 4; // drive was moved between nodes with allocated VDisks } } @@ -706,6 +706,7 @@ message TConfigResponse { NKikimrBlobStorage.TVDiskID VDiskId = 12; NKikimrBlobStorage.TVSlotId VSlotId = 13; string StoragePoolName = 14; + string DiskSerialNumber = 15; } } |