KIKIMR-17274: Add config request to return VDisk back to normal from read-only

7 files changed, 105 insertions, 22 deletions

diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
index 8eb4afab65..3afd44ff8c 100644
--- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h
+++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
@@ -5,6 +5,8 @@
 #include "node_warden_mock.h"
 
 #include <ydb/core/driver_lib/version/version.h>
+#include <ydb/core/mind/bscontroller/mood.h>
+
 #include <library/cpp/testing/unittest/registar.h>
 
 struct TEnvironmentSetup {
@@ -679,18 +681,45 @@ struct TEnvironmentSetup {
         UNIT_ASSERT(response.GetSuccess());
     }
 
-    void PutVDiskToReadOnly(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
-        NKikimrBlobStorage::TConfigRequest request;
-        auto *roCmd = request.AddCommand()->MutablePutVDiskToReadOnly();
-        auto *vslot = roCmd->MutableVSlotId();
+    void FillVSlotId(ui32 nodeId, ui32 pdiskId, ui32 vslotId, NKikimrBlobStorage::TVSlotId* vslot) {
         vslot->SetNodeId(nodeId);
         vslot->SetPDiskId(pdiskId);
         vslot->SetVSlotId(vslotId);
+    }
+
+    void PutVDiskToNormal(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
+        NKikimrBlobStorage::TConfigRequest request;
+        auto *roCmd = request.AddCommand()->MutablePutVDiskToNormal();
+        FillVSlotId(nodeId, pdiskId, vslotId, roCmd->MutableVSlotId());
+        VDiskIDFromVDiskID(vdiskId, roCmd->MutableVDiskId());
+        Cerr << "Invoking PutVDiskToNormal for vdisk " << vdiskId.ToString() << Endl;
+        auto response = Invoke(request);
+        UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
+    }
+
+    void PutVDiskToReadOnly(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
+        NKikimrBlobStorage::TConfigRequest request;
+        auto *roCmd = request.AddCommand()->MutablePutVDiskToReadOnly();
+        FillVSlotId(nodeId, pdiskId, vslotId, roCmd->MutableVSlotId());
         VDiskIDFromVDiskID(vdiskId, roCmd->MutableVDiskId());
+        Cerr << "Invoking PutVDiskToReadOnly for vdisk " << vdiskId.ToString() << Endl;
         auto response = Invoke(request);
         UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
     }
 
+    void PutVDiskToMood(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId, NKikimr::NBsController::TMood::EValue mood) {
+        switch (mood) {
+        case NKikimr::NBsController::TMood::Normal:
+            PutVDiskToNormal(nodeId, pdiskId, vslotId, vdiskId);
+            break;
+        case NKikimr::NBsController::TMood::ReadOnly:
+            PutVDiskToReadOnly(nodeId, pdiskId, vslotId, vdiskId);
+            break;
+        default:
+            ythrow yexception() << "unsupported mood: " << NKikimr::NBsController::TMood::Name(mood);
+        }
+    }
+
     void UpdateDriveStatus(ui32 nodeId, ui32 pdiskId, NKikimrBlobStorage::EDriveStatus status,
             NKikimrBlobStorage::EDecommitStatus decommitStatus) {
         NKikimrBlobStorage::TConfigRequest request;
diff --git a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
index 1013e3cb0a..9417f6fea5 100644
--- a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+++ b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
@@ -61,56 +61,70 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
             });
             TInstant getDeadline = env.Now() + TDuration::Seconds(30);
             auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender, /* termOnCapture */ false, getDeadline);
-            UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK);
             Cerr << "TEvGetResult: " << res->Get()->ToString() << Endl;
+            UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK);
+            UNIT_ASSERT_VALUES_EQUAL(res->Get()->ResponseSz, 1);
+            UNIT_ASSERT_VALUES_EQUAL(res->Get()->Responses[0].Buffer.size(), data.size());
             UNIT_ASSERT_VALUES_EQUAL(res->Get()->Responses[0].Buffer.ConvertToString(), data);
         };
 
+        auto readAllBlobs = [&] (ui32 steps) {
+            Cerr << "=== Read all " << steps << " blob(s) ===" << Endl;
+            for (ui32 step = 0; step < steps; ++step) {
+                sendGet(step);
+            }
+        };
+
         Cerr << "=== Trying to put and get a blob ===" << Endl;
         ui32 step = 0;
         sendPut(step, NKikimrProto::OK);
-        sendGet(step);
         ++step;
+        readAllBlobs(step);
 
-        auto putVDiskToRo = [&] (ui32 position) {
+        using NKikimr::NBsController::TMood;
+        auto putVDiskToMood = [&] (ui32 position, TMood::EValue mood) {
             const TVDiskID& someVDisk = info->GetVDiskId(position);
-
             auto baseConfig = env.FetchBaseConfig();
 
             const auto& somePDisk = baseConfig.GetPDisk(position);
             const auto& someVSlot = baseConfig.GetVSlot(position);
-            Cerr << "Issuing PutVDiskToReadOnly for position " << position << Endl;
-            env.PutVDiskToReadOnly(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId(), someVDisk);
+            Cerr << "Putting VDisk to mood " << TMood::Name(mood) << " for position " << position << Endl;
+            env.PutVDiskToMood(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId(), someVDisk, mood);
             env.Sim(TDuration::Seconds(30));
         };
 
         Cerr << "=== Putting VDisk #0 to read-only ===" << Endl;
-        putVDiskToRo(0);
+        putVDiskToMood(0, TMood::ReadOnly);
 
-        Cerr << "=== Write 10 blobs ===" << Endl;
+        Cerr << "=== Write 10 blobs, expect some VDisks refuse parts but writes go through ===" << Endl;
         for (ui32 i = 0; i < 10; ++i) {
             sendPut(step, NKikimrProto::OK);
             ++step;
         }
 
-        Cerr << "=== Read all blobs ===" << Endl;
-        for (ui32 i = 0; i < step; ++i) {
-            sendGet(i);
-        }
+        readAllBlobs(step);
 
         Cerr << "=== Put 2 more VDisks to read-only ===" << Endl;
-        putVDiskToRo(1);
-        putVDiskToRo(2);
+        putVDiskToMood(1, TMood::ReadOnly);
+        putVDiskToMood(2, TMood::ReadOnly);
 
         Cerr << "=== Write 10 more blobs, expect errors ===" << Endl;
         for (ui32 i = 0; i < 10; ++i) {
             sendPut(step, NKikimrProto::ERROR);
             ++step;
         }
+        // Even though previous writes were not successfull, some parts were written which is enough to read the blobs back, at least before GC happens.
+        readAllBlobs(step);
+
+        Cerr << "=== Restoring to normal VDisk #0 ===" << Endl;
+        putVDiskToMood(0, TMood::Normal);
 
-        Cerr << "=== Read all blobs again, expect it to work ===" << Endl;
-        for (ui32 i = 0; i < step; ++i) {
-            sendGet(i);
+        Cerr << "=== Write 10 blobs, expect some VDisks refuse parts but the writes still go through ===" << Endl;
+        for (ui32 i = 0; i < 10; ++i) {
+            sendPut(step, NKikimrProto::OK);
+            ++step;
         }
+
+        readAllBlobs(step);
     }
 }
diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp
index 4d4ac187f1..61c0a67447 100644
--- a/ydb/core/mind/bscontroller/bsc.cpp
+++ b/ydb/core/mind/bscontroller/bsc.cpp
@@ -329,6 +329,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kWipeVDisk:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kSanitizeGroup:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kCancelVirtualGroup:
+                    case NKikimrBlobStorage::TConfigRequest::TCommand::kPutVDiskToNormal:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kPutVDiskToReadOnly:
                         return 2; // read-write commands go with higher priority as they are needed to keep cluster intact
 
diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
index 071971b1bd..80c9b3b5e1 100644
--- a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
+++ b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
@@ -676,6 +676,37 @@ namespace NKikimr::NBsController {
         }
     }
 
+    void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TPutVDiskToNormal& cmd, TStatus& /*status*/) {
+        // first, find matching vslot
+        const TVSlotId& vslotId = cmd.GetVSlotId();
+        TVSlotInfo *vslot = VSlots.FindForUpdate(vslotId);
+        if (!vslot) {
+            throw TExVSlotNotFound(vslotId);
+        }
+
+        // second, validate vdisk id
+        const TVDiskID& vdiskId = VDiskIDFromVDiskID(cmd.GetVDiskId());
+        if (vslot->GetVDiskId() != vdiskId) {
+            throw TExVDiskIdIncorrect(vdiskId, vslotId);
+        }
+
+        bool allowedTransition = (
+            vslot->Mood == TMood::Normal ||
+            vslot->Mood == TMood::ReadOnly
+        );
+
+        if (!allowedTransition) {
+            throw TExError() << "unable to transition VDisk to normal from " << TMood::Name(static_cast<TMood::EValue>(vslot->Mood));
+        }
+
+        TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId);
+        vslot->Mood = TMood::Normal;
+        vslot->Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING;
+        vslot->IsReady = false;
+        GroupFailureModelChanged.insert(group->ID);
+        group->CalculateGroupStatus();
+    }
+
     void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& /*status*/) {
         // first, find matching vslot
         const TVSlotId& vslotId = cmd.GetVSlotId();
@@ -693,7 +724,6 @@ namespace NKikimr::NBsController {
         TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId);
         vslot->Mood = TMood::ReadOnly;
         vslot->Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING;
-        vslot->DropFromVSlotReadyTimestampQ();
         vslot->IsReady = false;
         GroupFailureModelChanged.insert(group->ID);
         group->CalculateGroupStatus();
diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h
index 3caa46df3a..936bd47120 100644
--- a/ydb/core/mind/bscontroller/config.h
+++ b/ydb/core/mind/bscontroller/config.h
@@ -286,6 +286,7 @@ namespace NKikimr {
             void ExecuteStep(const NKikimrBlobStorage::TWipeVDisk& cmd, TStatus& status);
             void ExecuteStep(const NKikimrBlobStorage::TSanitizeGroup& cmd, TStatus& status);
             void ExecuteStep(const NKikimrBlobStorage::TCancelVirtualGroup& cmd, TStatus& status);
+            void ExecuteStep(const NKikimrBlobStorage::TPutVDiskToNormal& cmd, TStatus& status);
             void ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& status);
         };
 
diff --git a/ydb/core/mind/bscontroller/config_cmd.cpp b/ydb/core/mind/bscontroller/config_cmd.cpp
index f7197c8866..4485026661 100644
--- a/ydb/core/mind/bscontroller/config_cmd.cpp
+++ b/ydb/core/mind/bscontroller/config_cmd.cpp
@@ -330,6 +330,7 @@ namespace NKikimr::NBsController {
                     HANDLE_COMMAND(WipeVDisk)
                     HANDLE_COMMAND(SanitizeGroup)
                     HANDLE_COMMAND(CancelVirtualGroup)
+                    HANDLE_COMMAND(PutVDiskToNormal)
                     HANDLE_COMMAND(PutVDiskToReadOnly)
 
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kAddMigrationPlan:
diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto
index 9b2580b68f..37ef5dc941 100644
--- a/ydb/core/protos/blobstorage_config.proto
+++ b/ydb/core/protos/blobstorage_config.proto
@@ -480,6 +480,12 @@ message TCancelVirtualGroup {
     uint32 GroupId = 1; // id of a group we are going to cancel
 }
 
+// An attempt to transition to the normal mode, e.g. from the read-only mode.
+message TPutVDiskToNormal {
+    NKikimrBlobStorage.TVSlotId VSlotId = 1;
+    NKikimrBlobStorage.TVDiskID VDiskId = 2;
+}
+
 message TPutVDiskToReadOnly {
     NKikimrBlobStorage.TVSlotId VSlotId = 1;
     NKikimrBlobStorage.TVDiskID VDiskId = 2;
@@ -528,6 +534,7 @@ message TConfigRequest {
             TReadSettings ReadSettings = 43;
             TCancelVirtualGroup CancelVirtualGroup = 44;
             TPutVDiskToReadOnly PutVDiskToReadOnly = 45;
+            TPutVDiskToNormal PutVDiskToNormal = 46;
 
             // commands intended for internal use
             TReassignGroupDisk ReassignGroupDisk = 19;