KIKIMR-17274: Fix readonly vdisk log writes

10 files changed, 90 insertions, 19 deletions

diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
index 604e62bcba9..48ddca94f1d 100644
--- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
+++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
@@ -44,7 +44,7 @@ struct TPDiskMockState::TImpl {
     std::unordered_map<TString, ui32> Blocks;
     TIntervalSet<ui64> Corrupted;
     NPDisk::TStatusFlags StatusFlags;
-    THashSet<TVDiskID> ReadOnlyVDisks;
+    THashSet<ui32> ReadOnlyVDisks;
     TString StateErrorReason;
 
     TImpl(ui32 nodeId, ui32 pdiskId, ui64 pdiskGuid, ui64 size, ui32 chunkSize)
@@ -264,14 +264,14 @@ struct TPDiskMockState::TImpl {
 
     void SetReadOnly(const TVDiskID& vDiskId, bool isReadOnly) {
         if (isReadOnly) {
-            ReadOnlyVDisks.insert(vDiskId);
+            ReadOnlyVDisks.insert(vDiskId.GroupID);
         } else {
-            ReadOnlyVDisks.erase(vDiskId);
+            ReadOnlyVDisks.erase(vDiskId.GroupID);
         }
     }
 
     bool IsReadOnly(const TVDiskID& vDiskId) const {
-        return ReadOnlyVDisks.contains(vDiskId);
+        return ReadOnlyVDisks.contains(vDiskId.GroupID);
     }
 };
 
diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
index fb6d48497d8..711e8003faa 100644
--- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h
+++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
@@ -702,7 +702,6 @@ struct TEnvironmentSetup {
         Cerr << "Invoking SetVDiskReadOnly for vdisk " << vdiskId.ToString() << Endl;
         auto response = Invoke(request);
         UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
-        PDiskMockStates[{nodeId, pdiskId}]->SetReadOnly(vdiskId, value);
     }
 
     void UpdateDriveStatus(ui32 nodeId, ui32 pdiskId, NKikimrBlobStorage::EDriveStatus status,
diff --git a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
index 8b37c0d1d77..d7ff263a8ea 100644
--- a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+++ b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
@@ -97,8 +97,24 @@ struct TTetsEnv {
         const auto& somePDisk = baseConfig.GetPDisk(position);
         const auto& someVSlot = baseConfig.GetVSlot(position);
         Cerr << "Setting VDisk read-only to " << value << " for position " << position << Endl;
+        if (!value) {
+            Env.PDiskMockStates[{somePDisk.GetNodeId(), somePDisk.GetPDiskId()}]->SetReadOnly(someVDisk, value);
+        }
         Env.SetVDiskReadOnly(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId(), someVDisk, value);
         Env.Sim(TDuration::Seconds(30));
+        if (value) {
+            Env.PDiskMockStates[{somePDisk.GetNodeId(), somePDisk.GetPDiskId()}]->SetReadOnly(someVDisk, value);
+        }
+    }
+
+    void SendCutLog(ui32 position) {
+        auto baseConfig = Env.FetchBaseConfig();
+        const auto& somePDisk = baseConfig.GetPDisk(position);
+        const TActorId sender = Env.Runtime->AllocateEdgeActor(somePDisk.GetNodeId(), __FILE__, __LINE__);
+        Env.Runtime->WrapInActorContext(sender, [&] () {
+            Env.PDiskMockStates[{somePDisk.GetNodeId(), somePDisk.GetPDiskId()}]->TrimQuery();
+        });
+        Env.Sim(TDuration::Minutes(1));
     }
 
     auto SendCollectGarbage(ui32 step) {
@@ -140,12 +156,12 @@ struct TTetsEnv {
         auto ev = std::make_unique<TEvLoad::TEvLoadTestRequest>();
         
         TString conf("StorageLoad: {\n"
-            "DurationSeconds: 1\n"
+            "DurationSeconds: 8\n"
             "Tablets: {\n"
                 "Tablets: { TabletId: 1 Channel: 0 GroupId: " + ToString(GroupInfo->GroupID) + " Generation: 1 }\n"
-                "WriteSizes: { Weight: 1.0 Min: 128 Max: 128 }\n"
+                "WriteSizes: { Weight: 1.0 Min: 1000000 Max: 4000000 }\n"
                 "WriteIntervals: { Weight: 1.0 Uniform: { MinUs: 100000 MaxUs: 100000 } }\n"
-                "MaxInFlightWriteRequests: 4\n"
+                "MaxInFlightWriteRequests: 10\n"
                 "FlushIntervals: { Weight: 1.0 Uniform: { MinUs: 1000000 MaxUs: 1000000 } }\n"
                 "PutHandleClass: TabletLog\n"
             "}\n"
@@ -163,7 +179,7 @@ struct TTetsEnv {
             auto res = Env.WaitForEdgeActorEvent<TEvLoad::TEvNodeFinishResponse>(sender, false);
             UNIT_ASSERT(res->Get()->Record.GetSuccess());
         }
-        Env.Sim(TDuration::Seconds(6));
+        Env.Sim(TDuration::Seconds(60));
     }
 
     TEnvironmentSetup Env;
@@ -192,6 +208,8 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
             Cerr << "=== Putting VDisk #" << i << " to read-only ===" << Endl;
             env.SetVDiskReadOnly(i, true);
             env.ReadAllBlobs(step);
+            env.SendCutLog(i);
+            env.SendCutLog((i + 1) % 7);
         }
 
         for (ui32 i = 0; i < 7; ++i) {
@@ -219,6 +237,7 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
             ++step;
         }
 
+        env.SendCutLog(0);
         env.ReadAllBlobs(step);
 
         Cerr << "=== Put 2 more VDisks to read-only ===" << Endl;
@@ -230,6 +249,10 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
             env.SendPut(step, NKikimrProto::ERROR);
             ++step;
         }
+        env.SendCutLog(0);
+        env.SendCutLog(1);
+        env.SendCutLog(2);
+        env.SendCutLog(3);
         // Even though previous writes were not successfull, some parts were written which is enough to read the blobs back, at least before GC happens.
         env.ReadAllBlobs(step);
 
@@ -269,6 +292,10 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
         env.SetVDiskReadOnly(1, true);
         env.SetVDiskReadOnly(2, true);
 
+        env.SendCutLog(1);
+        env.SendCutLog(2);
+        env.SendCutLog(3);
+
         ui32 stoppedStep = step;
 
         Cerr << "=== Write 10 more blobs, expect errors ===" << Endl;
@@ -304,12 +331,16 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
         env.SetVDiskReadOnly(0, true);
         env.SendPut(step++, NKikimrProto::OK);
         checkGarbageCollectOk();
+        env.SendCutLog(0);
         env.SetVDiskReadOnly(1, true);
         env.SendPut(step++, NKikimrProto::OK);
+        env.SendCutLog(1);
         checkGarbageCollectOk();
         env.SetVDiskReadOnly(2, true);
         env.SendPut(step, NKikimrProto::ERROR);
+        env.SendCutLog(2);
         checkGarbageCollectErr();
+        env.SendCutLog(2);
 
         for (ui32 i = 3; i < 7; ++i) {
             Cerr << "=== Putting VDisk #" << i << " to read-only ===" << Endl;
@@ -324,9 +355,11 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
         }
 
         env.SetVDiskReadOnly(4, false);
+        env.SendCutLog(4);
         checkGarbageCollectOk();
         env.SetVDiskReadOnly(5, false);
         checkGarbageCollectOk();
+        env.SendCutLog(5);
         env.SetVDiskReadOnly(6, false);
         checkGarbageCollectOk();
 
@@ -386,6 +419,8 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
             env.SetVDiskReadOnly(i, true);
             env.SetVDiskReadOnly((i + 1) % 7, true);
             env.SendPut(step++, NKikimrProto::OK);
+            env.SendCutLog(i);
+            env.SendCutLog((i + 1) % 7);
             env.SetVDiskReadOnly(i, false);
             env.SetVDiskReadOnly((i + 1) % 7, false);
         }
@@ -398,14 +433,28 @@ Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
 
         env.RunStorageLoad();
 
-        for (ui32 i = 0; i < 8; ++i) {
+        env.SetVDiskReadOnly(0, true);
+        env.RunStorageLoad();
+        env.SendCutLog(0);
+
+        env.SetVDiskReadOnly(1, true);
+        env.RunStorageLoad();
+        env.SendCutLog(1);
+
+        for (ui32 i = 2; i < 8; ++i) {
             env.SetVDiskReadOnly(i, true);
         }
         env.RunStorageLoad();
+        for (ui32 i = 2; i < 8; ++i) {
+            env.SendCutLog(i);
+        }
 
         for (ui32 i = 0; i < 7; ++i) {
             env.SetVDiskReadOnly(i, false);
         }
         env.RunStorageLoad();
+        for (ui32 i = 0; i < 8; ++i) {
+            env.SendCutLog(i);
+        }
     }
 }
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
index 04edc67ccb5..2affc1e91a6 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
@@ -229,7 +229,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
         HmCtx->LogCutterID = ctx.ExecutorThread.RegisterActor(CreateRecoveryLogCutter(std::move(logCutterCtx)));
         RepairedHuge->FinishRecovery(ctx);
         auto hugeKeeperCtx = std::make_shared<THugeKeeperCtx>(HmCtx->VCtx, HmCtx->PDiskCtx, HmCtx->LsnMngr,
-                HmCtx->MainID, HmCtx->LoggerID, HmCtx->LogCutterID, "{}");
+                HmCtx->MainID, HmCtx->LoggerID, HmCtx->LogCutterID, "{}", false);
         TAutoPtr<IActor> hugeKeeperActor(CreateHullHugeBlobKeeper(hugeKeeperCtx, RepairedHuge));
         HmCtx->HugeKeeperID = ctx.ExecutorThread.RegisterActor(hugeKeeperActor.Release());
 
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_synclog.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_synclog.cpp
index 3fa4a892be9..0f50801ad4f 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/test_synclog.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/test_synclog.cpp
@@ -226,7 +226,8 @@ class TSyncLogTestWriteActor : public TActorBootstrapped<TSyncLogTestWriteActor>
                 VDiskConfig->SyncLogMaxEntryPointSize,
                 VDiskConfig->SyncLogMaxMemAmount,
                 VDiskConfig->MaxResponseSize,
-                Db->SyncLogFirstLsnToKeep);
+                Db->SyncLogFirstLsnToKeep,
+                false);
         TestCtx->SyncLogId = ctx.Register(CreateSyncLogActor(slCtx, Conf->GroupInfo, TestCtx->SelfVDiskId, std::move(repaired)));
         // Send Db birth lsn
         ui64 dbBirthLsn = 0;
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp
index 7ee8ed87a98..46fe0b3bd0c 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp
@@ -605,7 +605,8 @@ namespace NKikimr {
             TActorId skeletonId,
             TActorId loggerId,
             TActorId logCutterId,
-            const TString &localRecoveryInfoDbg)
+            const TString &localRecoveryInfoDbg,
+            bool isReadOnlyVDisk)
         : VCtx(std::move(vctx))
         , PDiskCtx(std::move(pdiskCtx))
         , LsnMngr(std::move(lsnMngr))
@@ -615,6 +616,7 @@ namespace NKikimr {
         , LocalRecoveryInfoDbg(localRecoveryInfoDbg)
         , LsmHullGroup(VCtx->VDiskCounters, "subsystem", "lsmhull")
         , DskOutOfSpaceGroup(VCtx->VDiskCounters, "subsystem", "outofspace")
+        , IsReadOnlyVDisk(isReadOnlyVDisk)
     {}
 
     THugeKeeperCtx::~THugeKeeperCtx() = default;
@@ -683,6 +685,12 @@ namespace NKikimr {
 
         //////////// Cut Log Handler ///////////////////////////////////
         void TryToCutLog(const TActorContext &ctx) {
+            if (HugeKeeperCtx->IsReadOnlyVDisk) {
+                LOG_DEBUG(ctx, BS_LOGCUTTER,
+                    VDISKP(HugeKeeperCtx->VCtx->VDiskLogPrefix,
+                        "THullHugeKeeper: TryToCutLog: terminate; readonly vdisk"));
+                return;
+            }
             const ui64 firstLsnToKeep = State.FirstLsnToKeep();
             LOG_DEBUG(ctx, BS_LOGCUTTER,
                 VDISKP(HugeKeeperCtx->VCtx->VDiskLogPrefix,
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.h b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.h
index 0eaaca14f56..284c405763f 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.h
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.h
@@ -222,6 +222,7 @@ namespace NKikimr {
         const TString LocalRecoveryInfoDbg;
         NMonGroup::TLsmHullGroup LsmHullGroup;
         NMonGroup::TDskOutOfSpaceGroup DskOutOfSpaceGroup;
+        const bool IsReadOnlyVDisk;
 
         THugeKeeperCtx(
                 TIntrusivePtr<TVDiskContext> vctx,
@@ -230,7 +231,8 @@ namespace NKikimr {
                 TActorId skeletonId,
                 TActorId loggerId,
                 TActorId logCutterId,
-                const TString &localRecoveryInfoDbg);
+                const TString &localRecoveryInfoDbg,
+                bool isReadOnlyVDisk);
         ~THugeKeeperCtx();
     };
 
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index bf0b265411d..33b486f412c 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -127,6 +127,8 @@ namespace NKikimr {
                 ReplyError(NKikimrProto::ERROR, "disk is in donor mode", ev, ctx, TAppData::TimeProvider->Now());
             } else if (BlockWrites(GInfo->DecommitStatus)) {
                 ReplyError(NKikimrProto::ERROR, "group is being decommitted", ev, ctx, TAppData::TimeProvider->Now());
+            } else if (Config->BaseInfo.ReadOnly) {
+                ReplyError(NKikimrProto::ERROR, "disk is in readonly mode", ev, ctx, TAppData::TimeProvider->Now());
             } else {
                 return true;
             }
@@ -1821,7 +1823,7 @@ namespace NKikimr {
                 TString localRecovInfoStr = Db->LocalRecoveryInfo ? Db->LocalRecoveryInfo->ToString() : TString("{}");
                 auto hugeKeeperCtx = std::make_shared<THugeKeeperCtx>(VCtx, PDiskCtx, Db->LsnMngr,
                         ctx.SelfID, (TActorId)(Db->LoggerID), (TActorId)(Db->LogCutterID),
-                        localRecovInfoStr);
+                        localRecovInfoStr, Config->BaseInfo.ReadOnly);
                 auto hugeKeeper = CreateHullHugeBlobKeeper(hugeKeeperCtx, ev->Get()->RepairedHuge);
                 Db->HugeKeeperID.Set(ctx.Register(hugeKeeper));
                 ActiveActors.Insert(Db->HugeKeeperID); // keep forever
@@ -1839,7 +1841,8 @@ namespace NKikimr {
                         Config->SyncLogMaxEntryPointSize,
                         Config->SyncLogMaxMemAmount,
                         Config->MaxResponseSize,
-                        Db->SyncLogFirstLsnToKeep);
+                        Db->SyncLogFirstLsnToKeep,
+                        Config->BaseInfo.ReadOnly);
                 Db->SyncLogID.Set(ctx.Register(CreateSyncLogActor(slCtx, GInfo, SelfVDiskId, std::move(repairedSyncLog))));
                 ActiveActors.Insert(Db->SyncLogID); // keep forever
 
@@ -2164,6 +2167,9 @@ namespace NKikimr {
         // CUT LOG FORWARDER SECTOR
         ////////////////////////////////////////////////////////////////////////
         void Handle(NPDisk::TEvCutLog::TPtr &ev, const TActorContext &ctx) {
+            if (Config->BaseInfo.ReadOnly) {
+                return;
+            }
             std::unique_ptr<NPDisk::TEvCutLog> msg(ev->Release().Release());
 
             if (LocalDbInitialized) {
diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog.cpp
index 3547fec9541..f8474b19947 100644
--- a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog.cpp
+++ b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog.cpp
@@ -184,8 +184,10 @@ namespace NKikimr {
                     return;
                 }
 
-                // cut the log (according to confirmed old synced state)
-                CutLog(ctx, sourceVDisk, oldSyncState.SyncedLsn);
+                if (!SlCtx->IsReadOnlyVDisk) {
+                    // cut the log (according to confirmed old synced state)
+                    CutLog(ctx, sourceVDisk, oldSyncState.SyncedLsn);
+                }
                 // process the request further asyncronously
                 NeighborsPtr->Lock(sourceVDisk, oldSyncState.SyncedLsn);
                 auto aid = ctx.Register(CreateSyncLogReaderActor(SlCtx, VDiskIncarnationGuid, ev, ctx.SelfID, KeeperId,
diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog_public_events.h b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog_public_events.h
index 48616643a85..ac2bfb876d3 100644
--- a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog_public_events.h
+++ b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclog_public_events.h
@@ -152,6 +152,8 @@ namespace NKikimr {
             NMonGroup::TSyncLogIFaceGroup IFaceMonGroup;
             NMonGroup::TSyncLogCountersGroup CountersMonGroup;
 
+            const bool IsReadOnlyVDisk;
+
             TSyncLogCtx(TIntrusivePtr<TVDiskContext> vctx,
                     TIntrusivePtr<TLsnMngr> lsnMngr,
                     TPDiskCtxPtr pdiskCtx,
@@ -161,7 +163,8 @@ namespace NKikimr {
                     ui64 syncLogMaxEntryPointSize,
                     ui64 syncLogMaxMemAmount,
                     ui32 maxResponseSize,
-                    std::shared_ptr<TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep)
+                    std::shared_ptr<TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep,
+                    bool isReadOnlyVDisk)
                 : VCtx(std::move(vctx))
                 , LsnMngr(std::move(lsnMngr))
                 , PDiskCtx(std::move(pdiskCtx))
@@ -174,6 +177,7 @@ namespace NKikimr {
                 , SyncLogFirstLsnToKeep(std::move(syncLogFirstLsnToKeep))
                 , IFaceMonGroup(VCtx->VDiskCounters, "subsystem", "synclog")
                 , CountersMonGroup(VCtx->VDiskCounters, "subsystem", "synclogcounters")
+                , IsReadOnlyVDisk(isReadOnlyVDisk)
             {}
         };