diff options
author | tarum <tarum@yandex-team.com> | 2023-08-11 21:09:09 +0300 |
---|---|---|
committer | tarum <tarum@yandex-team.com> | 2023-08-11 21:56:32 +0300 |
commit | 0484ec70edb1e3f921df3836762b032ed304b64a (patch) | |
tree | fba3113a27dbdd19f35834da0c1106f44b0d1674 | |
parent | 656f1abe06dca744d46a0c0459a44cc7c5de919c (diff) | |
download | ydb-0484ec70edb1e3f921df3836762b032ed304b64a.tar.gz |
KIKIMR-17274: Support read-only mode in VDisk
33 files changed, 655 insertions, 47 deletions
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index d087de7720..3844d48c7c 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -543,6 +543,7 @@ struct TEvBlobStorage { EvRegisterPDiskLoadActor, EvStatusUpdate, EvDropDonor, + EvPutVDiskToReadOnly, EvCntReply = EvPut + 7 * 512, /// 268 635 648 EvVGenerationChangeResult, @@ -2329,6 +2330,7 @@ struct TEvBlobStorage { struct TEvControllerGroupDecommittedNotify; struct TEvControllerGroupDecommittedResponse; struct TEvControllerGroupMetricsExchange; + struct TEvPutVDiskToReadOnly; struct TEvMonStreamQuery; struct TEvMonStreamActorDeathNote; diff --git a/ydb/core/blobstorage/base/blobstorage_events.h b/ydb/core/blobstorage/base/blobstorage_events.h index 14b07bf363..905d343163 100644 --- a/ydb/core/blobstorage/base/blobstorage_events.h +++ b/ydb/core/blobstorage/base/blobstorage_events.h @@ -537,6 +537,14 @@ namespace NKikimr { NKikimrBlobStorage::TEvControllerGroupMetricsExchange, EvControllerGroupMetricsExchange> {}; + struct TEvBlobStorage::TEvPutVDiskToReadOnly : TEventLocal<TEvPutVDiskToReadOnly, EvPutVDiskToReadOnly> { + const TVDiskID VDiskId; + + TEvPutVDiskToReadOnly(TVDiskID vDiskId) + : VDiskId(std::move(vDiskId)) + {} + }; + struct TEvNodeWardenQueryGroupInfo : TEventPB<TEvNodeWardenQueryGroupInfo, NKikimrBlobStorage::TEvNodeWardenQueryGroupInfo, TEvBlobStorage::EvNodeWardenQueryGroupInfo> { TEvNodeWardenQueryGroupInfo() = default; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index acda31d319..c76bec20fa 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -254,6 +254,7 @@ namespace NKikimr::NStorage { TIntrusivePtr<TBlobStorageGroupInfo> GroupInfo; ui32 OrderNumber; bool DonorMode; + bool ReadOnly; }; std::optional<TRuntimeData> RuntimeData; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp index 3933308efc..8d8162fd11 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp @@ -57,6 +57,8 @@ namespace NKikimr::NStorage { const ui64 pdiskGuid = vdisk.Config.GetVDiskLocation().GetPDiskGuid(); const bool restartInFlight = InFlightRestartedPDisks.count({vslotId.NodeId, vslotId.PDiskId}); const bool donorMode = vdisk.Config.HasDonorMode(); + const bool readOnly = vdisk.Config.GetReadOnly(); + Y_VERIFY_S(!donorMode || !readOnly, "Only one of modes should be enabled: donorMode " << donorMode << ", readOnly " << readOnly); STLOG(PRI_DEBUG, BS_NODE, NW23, "StartLocalVDiskActor", (RestartInFlight, restartInFlight), (SlayInFlight, SlayInFlight.contains(vslotId)), (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId), @@ -158,7 +160,7 @@ namespace NKikimr::NStorage { TVDiskConfig::TBaseInfo baseInfo(vdiskId, pdiskServiceId, pdiskGuid, vslotId.PDiskId, deviceType, vslotId.VDiskSlotId, kind, NextLocalPDiskInitOwnerRound(), groupInfo->GetStoragePoolName(), donorMode, - donorDiskIds, scrubCookie, whiteboardInstanceGuid); + donorDiskIds, scrubCookie, whiteboardInstanceGuid, readOnly); baseInfo.ReplPDiskReadQuoter = pdiskIt->second.ReplPDiskReadQuoter; baseInfo.ReplPDiskWriteQuoter = pdiskIt->second.ReplPDiskWriteQuoter; @@ -197,7 +199,8 @@ namespace NKikimr::NStorage { vdisk.RuntimeData.emplace(TVDiskRecord::TRuntimeData{ .GroupInfo = groupInfo, .OrderNumber = groupInfo->GetOrderNumber(TVDiskIdShort(vdiskId)), - .DonorMode = donorMode + .DonorMode = donorMode, + .ReadOnly = readOnly, }); vdisk.Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING; @@ -221,9 +224,10 @@ namespace NKikimr::NStorage { // 4. Deleting VSlot during group reconfiguration or donor termination. // 5. Making VDisk a donor one. // 6. Updating VDisk generation when modifying group. + // 7. Putting VDisk into or out of read-only // // The main idea of this command is when VDisk is created, it does not change its configuration. It may be - // wiped out several times, it may become a donor and then it may be destroyed. That is a possible life cycle + // wiped out several times, it may become a donor or read-only and then it may be destroyed. That is a possible life cycle // of a VDisk in the occupied slot. if (!vdisk.HasVDiskID() || !vdisk.HasVDiskLocation()) { @@ -261,7 +265,7 @@ namespace NKikimr::NStorage { Slay(record); } else if (!record.RuntimeData) { StartLocalVDiskActor(record, TDuration::Zero()); - } else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode()) { + } else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode() || record.RuntimeData->ReadOnly != record.Config.GetReadOnly()) { PoisonLocalVDisk(record); StartLocalVDiskActor(record, TDuration::Seconds(15) /* PDisk confidence delay */); } diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt index 9e2d368616..0ae1ccfd5a 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat) add_subdirectory(ut_donor) add_subdirectory(ut_group_reconfiguration) add_subdirectory(ut_osiris) +add_subdirectory(ut_read_only_vdisk) add_subdirectory(ut_replication) add_subdirectory(ut_scrub) diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt index d85b0fed5e..0507f73ff2 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt @@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat) add_subdirectory(ut_donor) add_subdirectory(ut_group_reconfiguration) add_subdirectory(ut_osiris) +add_subdirectory(ut_read_only_vdisk) add_subdirectory(ut_replication) add_subdirectory(ut_scrub) diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt index f5ab8377bb..d49a3b9279 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat) add_subdirectory(ut_donor) add_subdirectory(ut_group_reconfiguration) add_subdirectory(ut_osiris) +add_subdirectory(ut_read_only_vdisk) add_subdirectory(ut_replication) add_subdirectory(ut_scrub) diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt index 4690192d32..d35c83a40f 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat) add_subdirectory(ut_donor) add_subdirectory(ut_group_reconfiguration) add_subdirectory(ut_osiris) +add_subdirectory(ut_read_only_vdisk) add_subdirectory(ut_replication) add_subdirectory(ut_scrub) diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 247411a1a2..74c3d2a372 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -98,6 +98,10 @@ struct TEnvironmentSetup { Cerr << "RandomSeed# " << seed << Endl; } + TInstant Now() { + return TAppData::TimeProvider->Now(); + } + TString GenerateRandomString(ui32 len) { TString res = TString::Uninitialized(len); char *p = res.Detach(); @@ -675,6 +679,18 @@ struct TEnvironmentSetup { UNIT_ASSERT(response.GetSuccess()); } + void PutVDiskToReadOnly(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) { + NKikimrBlobStorage::TConfigRequest request; + auto *roCmd = request.AddCommand()->MutablePutVDiskToReadOnly(); + auto *vslot = roCmd->MutableVSlotId(); + vslot->SetNodeId(nodeId); + vslot->SetPDiskId(pdiskId); + vslot->SetVSlotId(vslotId); + VDiskIDFromVDiskID(vdiskId, roCmd->MutableVDiskId()); + auto response = Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + void UpdateDriveStatus(ui32 nodeId, ui32 pdiskId, NKikimrBlobStorage::EDriveStatus status, NKikimrBlobStorage::EDecommitStatus decommitStatus) { NKikimrBlobStorage::TConfigRequest request; diff --git a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp new file mode 100644 index 0000000000..1013e3cb0a --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp @@ -0,0 +1,116 @@ +#include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> + +Y_UNIT_TEST_SUITE(ReadOnlyVDisk) { + + Y_UNIT_TEST(Basic) { + TEnvironmentSetup env{{ + .NodeCount = 8, + .VDiskReplPausedAtStart = false, + .Erasure = TBlobStorageGroupType::Erasure4Plus2Block, + }}; + env.CreateBoxAndPool(1, 1); + env.Sim(TDuration::Minutes(1)); + + auto groups = env.GetGroups(); + UNIT_ASSERT_VALUES_EQUAL(groups.size(), 1); + const TIntrusivePtr<TBlobStorageGroupInfo> info = env.GetGroupInfo(groups.front()); + + const TActorId vdiskActorId = info->GetActorId(0); + + auto prepData = [&] (const ui32 dataLen, const ui32 start) { + TString data(Reserve(dataLen)); + for (ui32 i = 0; i < dataLen; ++i) { + data.push_back('a' + (start + i) % 26); + } + return data; + }; + + TVector<TString> dataArr = { + prepData(128 * 1024, 0), + prepData(32 * 1024, 3), + }; + + auto sendPut = [&] (ui32 step, NKikimrProto::EReplyStatus expectedStatus) { + const TString& data = dataArr[step % 2]; + const TLogoBlobID id(1, 1, step, 0, data.size(), 0); + Cerr << "SEND TEvPut with key " << id.ToString() << Endl; + const TActorId sender = env.Runtime->AllocateEdgeActor(vdiskActorId.NodeId()); + auto ev = std::make_unique<TEvBlobStorage::TEvPut>(id, data, TInstant::Max()); + env.Runtime->WrapInActorContext(sender, [&] { + SendToBSProxy(sender, info->GroupID, ev.release()); + }); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvPutResult>(sender, false); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, expectedStatus); + Cerr << "TEvPutResult: " << res->Get()->ToString() << Endl; + }; + + auto sendGet = [&] (ui32 step) { + const TString& data = dataArr[step % 2]; + const TLogoBlobID blobId(1, 1, step, 0, data.size(), 0); + Cerr << "SEND TEvGet with key " << blobId.ToString() << Endl; + const TActorId sender = env.Runtime->AllocateEdgeActor(vdiskActorId.NodeId()); + auto ev = std::make_unique<TEvBlobStorage::TEvGet>( + blobId, + /* shift */ 0, + /* size */ data.size(), + TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead + ); + env.Runtime->WrapInActorContext(sender, [&] () { + SendToBSProxy(sender, info->GroupID, ev.release()); + }); + TInstant getDeadline = env.Now() + TDuration::Seconds(30); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender, /* termOnCapture */ false, getDeadline); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK); + Cerr << "TEvGetResult: " << res->Get()->ToString() << Endl; + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Responses[0].Buffer.ConvertToString(), data); + }; + + Cerr << "=== Trying to put and get a blob ===" << Endl; + ui32 step = 0; + sendPut(step, NKikimrProto::OK); + sendGet(step); + ++step; + + auto putVDiskToRo = [&] (ui32 position) { + const TVDiskID& someVDisk = info->GetVDiskId(position); + + auto baseConfig = env.FetchBaseConfig(); + + const auto& somePDisk = baseConfig.GetPDisk(position); + const auto& someVSlot = baseConfig.GetVSlot(position); + Cerr << "Issuing PutVDiskToReadOnly for position " << position << Endl; + env.PutVDiskToReadOnly(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId(), someVDisk); + env.Sim(TDuration::Seconds(30)); + }; + + Cerr << "=== Putting VDisk #0 to read-only ===" << Endl; + putVDiskToRo(0); + + Cerr << "=== Write 10 blobs ===" << Endl; + for (ui32 i = 0; i < 10; ++i) { + sendPut(step, NKikimrProto::OK); + ++step; + } + + Cerr << "=== Read all blobs ===" << Endl; + for (ui32 i = 0; i < step; ++i) { + sendGet(i); + } + + Cerr << "=== Put 2 more VDisks to read-only ===" << Endl; + putVDiskToRo(1); + putVDiskToRo(2); + + Cerr << "=== Write 10 more blobs, expect errors ===" << Endl; + for (ui32 i = 0; i < 10; ++i) { + sendPut(step, NKikimrProto::ERROR); + ++step; + } + + Cerr << "=== Read all blobs again, expect it to work ===" << Endl; + for (ui32 i = 0; i < step; ++i) { + sendGet(i); + } + } +} diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..2f34dba19e --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,74 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ut_blobstorage-ut_read_only_vdisk) +target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp +) +set_property( + TARGET + ut_blobstorage-ut_read_only_vdisk + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ut_blobstorage-ut_read_only_vdisk + TEST_TARGET + ut_blobstorage-ut_read_only_vdisk + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + TIMEOUT + 600 +) +target_allocator(ut_blobstorage-ut_read_only_vdisk + system_allocator +) +vcs_info(ut_blobstorage-ut_read_only_vdisk) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..202ee275e8 --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ut_blobstorage-ut_read_only_vdisk) +target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp +) +set_property( + TARGET + ut_blobstorage-ut_read_only_vdisk + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ut_blobstorage-ut_read_only_vdisk + TEST_TARGET + ut_blobstorage-ut_read_only_vdisk + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + TIMEOUT + 600 +) +target_allocator(ut_blobstorage-ut_read_only_vdisk + cpp-malloc-jemalloc +) +vcs_info(ut_blobstorage-ut_read_only_vdisk) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..4f579789d9 --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt @@ -0,0 +1,79 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ut_blobstorage-ut_read_only_vdisk) +target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp +) +set_property( + TARGET + ut_blobstorage-ut_read_only_vdisk + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ut_blobstorage-ut_read_only_vdisk + TEST_TARGET + ut_blobstorage-ut_read_only_vdisk + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + TIMEOUT + 600 +) +target_allocator(ut_blobstorage-ut_read_only_vdisk + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ut_blobstorage-ut_read_only_vdisk) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..fb688a40eb --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt @@ -0,0 +1,67 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ut_blobstorage-ut_read_only_vdisk) +target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp +) +set_property( + TARGET + ut_blobstorage-ut_read_only_vdisk + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ut_blobstorage-ut_read_only_vdisk + TEST_TARGET + ut_blobstorage-ut_read_only_vdisk + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ut_blobstorage-ut_read_only_vdisk + PROPERTY + TIMEOUT + 600 +) +target_allocator(ut_blobstorage-ut_read_only_vdisk + system_allocator +) +vcs_info(ut_blobstorage-ut_read_only_vdisk) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make new file mode 100644 index 0000000000..c390ceb28c --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(ydb/core/blobstorage/ut_blobstorage) + + FORK_SUBTESTS() + + SIZE(MEDIUM) + + TIMEOUT(600) + + SRCS( + read_only_vdisk.cpp + ) + + PEERDIR( + ydb/core/blobstorage/ut_blobstorage/lib + ) + +END() diff --git a/ydb/core/blobstorage/ut_blobstorage/ya.make b/ydb/core/blobstorage/ut_blobstorage/ya.make index 28b2a94903..a8b6ccd8e4 100644 --- a/ydb/core/blobstorage/ut_blobstorage/ya.make +++ b/ydb/core/blobstorage/ut_blobstorage/ya.make @@ -62,6 +62,7 @@ RECURSE_FOR_TESTS( ut_blob_depot_fat ut_donor ut_group_reconfiguration + ut_read_only_vdisk ut_osiris ut_replication ut_scrub diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp index b62819d417..51ea162e41 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp @@ -54,7 +54,8 @@ namespace NKikimr { AnubisTimeout = TDuration::Minutes(60); RunSyncer = true; RunAnubis = false; // FIXME: turn on by default - RunDefrag = true; + RunDefrag = !baseInfo.ReadOnly; + RunScrubber = !baseInfo.ReadOnly; SyncLogMaxDiskAmount = 0; //ui64(2) << ui64(30); // 2 GB SyncLogMaxEntryPointSize = ui64(128) << ui64(10); // 128 KB @@ -76,7 +77,7 @@ namespace NKikimr { HandoffMaxInFlightSize = 1000; HandoffMaxInFlightByteSize = 16u << 20u; HandoffTimeout = TDuration::Seconds(10); - RunRepl = true; + RunRepl = !baseInfo.ReadOnly; RunHandoff = false; SkeletonFrontGets_MaxInFlightCount = 24; diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h index efe272f3dd..e38278d33c 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h @@ -39,6 +39,8 @@ namespace NKikimr { TDuration YardInitDelay = TDuration::Zero(); const ui64 ScrubCookie = 0; const ui64 WhiteboardInstanceGuid = 0; + // handle only read requests: needed when VDisk can't write, e.g. no disk space, but still has the data + const bool ReadOnly = false; TBaseInfo( const TVDiskIdShort &vDiskIdShort, @@ -53,7 +55,9 @@ namespace NKikimr { const bool donorMode = false, std::vector<std::pair<TVDiskID, TActorId>> donorDiskIds = {}, ui64 scrubCookie = 0, - ui64 whiteboardInstanceGuid = 0) + ui64 whiteboardInstanceGuid = 0, + const bool readOnly = false + ) : VDiskIdShort(vDiskIdShort) , PDiskActorID(pDiskActorId) , InitOwnerRound(initOwnerRound) @@ -67,6 +71,7 @@ namespace NKikimr { , DonorDiskIds(std::move(donorDiskIds)) , ScrubCookie(scrubCookie) , WhiteboardInstanceGuid(whiteboardInstanceGuid) + , ReadOnly(readOnly) {} TBaseInfo(const TBaseInfo &) = default; @@ -139,6 +144,7 @@ namespace NKikimr { bool RunSyncer; bool RunAnubis; bool RunDefrag; + bool RunScrubber; ///////////// SYNCLOG SETTINGS ////////////////////// ui64 SyncLogMaxDiskAmount; diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp index f8b2c85ef5..b1b8daa11e 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp @@ -901,7 +901,7 @@ namespace NKikimr { if (record.HasCookie()) cookie = record.GetCookie(); auto handleClass = ev->Get()->Record.GetHandleClass(); - + std::unique_ptr<TEvBlobStorage::TEvVGetResult> result; if (ev->Get()->IsInternal) { result.reset(new TEvBlobStorage::TEvVGetResult(NKikimrProto::OK, SelfVDiskId, now, @@ -1725,6 +1725,32 @@ namespace NKikimr { UpdateVDiskStatus(NKikimrBlobStorage::ERROR); } + void StartDefrag(const TActorContext &ctx) { + auto defragCtx = std::make_shared<TDefragCtx>(VCtx, HugeBlobCtx, PDiskCtx, ctx.SelfID, + Db->HugeKeeperID, true); + DefragId = ctx.Register(CreateDefragActor(defragCtx, GInfo)); + ActiveActors.Insert(DefragId); // keep forever + } + + void StartScrubberActor(const TActorContext &ctx, NKikimrVDiskData::TScrubEntrypoint scrubEntrypoint, ui64 scrubEntrypointLsn) { + auto scrubCtx = MakeIntrusive<TScrubContext>( + VCtx, + PDiskCtx, + GInfo, + SelfId(), + Hull->GetHullDs()->LogoBlobs->LIActor, + SelfId().NodeId(), + Config->BaseInfo.PDiskId, + Config->BaseInfo.VDiskSlotId, + Config->BaseInfo.ScrubCookie, + Db->GetVDiskIncarnationGuid(), + Db->LsnMngr, + Db->LoggerID, + Db->LogCutterID); + ScrubId = ctx.Register(CreateScrubActor(std::move(scrubCtx), std::move(scrubEntrypoint), scrubEntrypointLsn)); + ActiveActors.Insert(ScrubId); + } + void Handle(TEvBlobStorage::TEvLocalRecoveryDone::TPtr &ev, const TActorContext &ctx) { LocalRecovInfo = ev->Get()->RecovInfo; LocalDbRecoveryID = TActorId(); @@ -1875,31 +1901,15 @@ namespace NKikimr { } if (Config->RunDefrag && AppData()->FeatureFlags.GetAllowVDiskDefrag()) { - auto defragCtx = std::make_shared<TDefragCtx>(VCtx, HugeBlobCtx, PDiskCtx, ctx.SelfID, - Db->HugeKeeperID, true); - DefragId = ctx.Register(CreateDefragActor(defragCtx, GInfo)); - ActiveActors.Insert(DefragId); // keep forever + StartDefrag(ctx); } // create scrubber actor - auto scrubCtx = MakeIntrusive<TScrubContext>( - VCtx, - PDiskCtx, - GInfo, - SelfId(), - Hull->GetHullDs()->LogoBlobs->LIActor, - SelfId().NodeId(), - Config->BaseInfo.PDiskId, - Config->BaseInfo.VDiskSlotId, - Config->BaseInfo.ScrubCookie, - Db->GetVDiskIncarnationGuid(), - Db->LsnMngr, - Db->LoggerID, - Db->LogCutterID); - ScrubId = ctx.Register(CreateScrubActor(std::move(scrubCtx), std::move(ev->Get()->ScrubEntrypoint), - ev->Get()->ScrubEntrypointLsn)); - ActiveActors.Insert(ScrubId); + if (Config->RunScrubber) { + StartScrubberActor(ctx, std::move(ev->Get()->ScrubEntrypoint), ev->Get()->ScrubEntrypointLsn); + } + // create syncer actor if (Config->RunSyncer && !Config->BaseInfo.DonorMode) { // switch to syncronization step Become(&TThis::StateSyncGuidRecovery); diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp index f47a76a4f0..7076f5b16b 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp @@ -1114,6 +1114,15 @@ namespace NKikimr { ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("NOT_READY")); } + template <class TEventPtr> + void DatabaseReadOnlyHandle(TEventPtr &ev, const TActorContext &ctx) { + LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix + << "Unavailable in read-only" + << " Sender# " << ev->Sender.ToString()); + TInstant now = TAppData::TimeProvider->Now(); + Reply(ev, ctx, NKikimrProto::ERROR, "VDisk is in read-only mode", now); + } + //////////////////////////////////////////////////////////////////////////// // HANDLE SECTOR //////////////////////////////////////////////////////////////////////////// @@ -1864,6 +1873,20 @@ namespace NKikimr { || std::is_same_v<TEv, TEvBlobStorage::TEvVGet> || std::is_same_v<TEv, TEvBlobStorage::TEvVPut>; + template <typename TEv> + static constexpr bool IsReadOnlyCompatible = ( + std::is_same_v<TEv, TEvBlobStorage::TEvVCheckReadiness> || + std::is_same_v<TEv, TEvBlobStorage::TEvVDbStat> || + std::is_same_v<TEv, TEvBlobStorage::TEvVGet> || + std::is_same_v<TEv, TEvBlobStorage::TEvVGetBarrier> || + std::is_same_v<TEv, TEvBlobStorage::TEvVGetBlock> || + std::is_same_v<TEv, TEvGetLogoBlobIndexStatRequest> || + std::is_same_v<TEv, TEvBlobStorage::TEvVStatus> || + std::is_same_v<TEv, TEvBlobStorage::TEvVAssimilate> || + std::is_same_v<TEv, TEvBlobStorage::TEvVSync> || + std::is_same_v<TEv, TEvBlobStorage::TEvVSyncFull> + ); + template<typename TEventType> void CheckExecute(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) { if constexpr (IsPatchEvent<TEventType>) { @@ -1897,6 +1920,9 @@ namespace NKikimr { return Reply(ev, ctx, NKikimrProto::RACE, "group generation mismatch", TAppData::TimeProvider->Now()); } else if (!GInfo->CheckScope(TKikimrScopeId(ev->OriginScopeId), ctx, true)) { DatabaseAccessDeniedHandle(ev, ctx); + } else if (Config->BaseInfo.ReadOnly && !IsReadOnlyCompatible<TEventType>) { + LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "Blocking request incompatible with read-only: " << TypeName<TEventType>()); + DatabaseReadOnlyHandle(ev, ctx); } else { SetReceivedTime(ev); CheckExecute(ev, ctx); diff --git a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp index 18f36a166d..285fdfb697 100644 --- a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp +++ b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp @@ -221,7 +221,7 @@ namespace NKikimr { void SyncGuid(const TActorContext &ctx) { Become(&TThis::SyncGuidStateFunc); GuidRecoveryId = ctx.Register(CreateVDiskGuidRecoveryActor(SyncerCtx->VCtx, GInfo, CommitterId, SelfId(), - LocalSyncerState)); + LocalSyncerState, SyncerCtx->Config->BaseInfo.ReadOnly)); ActiveActors.Insert(GuidRecoveryId); Phase = TPhaseVal::PhaseSyncGuid; } @@ -289,6 +289,14 @@ namespace NKikimr { // Recover Lost Data //////////////////////////////////////////////////////////////////////// void RecoverLostData(const TActorContext &ctx) { + if (SyncerCtx->Config->BaseInfo.ReadOnly) { + LOG_WARN(ctx, BS_SYNCER, + VDISKP(SyncerCtx->VCtx->VDiskLogPrefix, + "Unable to recover lost data in read-only mode. Transitioning to inconsistent state.")); + InconsistentState(ctx); + return; + } + Become(&TThis::RecoverLostDataStateFunc); const TVDiskEternalGuid guid = GuidRecovOutcome->Guid; RecoverLostDataId = ctx.Register(CreateSyncerRecoverLostDataActor(SyncerCtx, GInfo, CommitterId, ctx.SelfID, guid)); @@ -330,8 +338,17 @@ namespace NKikimr { new TEvSyncGuidRecoveryDone(NKikimrProto::OK, LocalSyncerState.DbBirthLsn)); SyncerData->Neighbors->DbBirthLsn = LocalSyncerState.DbBirthLsn; Become(&TThis::StandardModeStateFunc); - SchedulerId = ctx.Register(CreateSyncerSchedulerActor(SyncerCtx, GInfo, SyncerData, CommitterId)); - ActiveActors.Insert(SchedulerId); + if (!SyncerCtx->Config->BaseInfo.ReadOnly) { + LOG_DEBUG(ctx, BS_SYNCER, + VDISKP(SyncerCtx->VCtx->VDiskLogPrefix, + "%s: Creating syncer scheduler on node %d", __PRETTY_FUNCTION__, SelfId().NodeId())); + SchedulerId = ctx.Register(CreateSyncerSchedulerActor(SyncerCtx, GInfo, SyncerData, CommitterId)); + ActiveActors.Insert(SchedulerId); + } else { + LOG_WARN(ctx, BS_SYNCER, + VDISKP(SyncerCtx->VCtx->VDiskLogPrefix, + "%s: Skipping scheduler start due to read-only", __PRETTY_FUNCTION__)); + } Phase = TPhaseVal::PhaseStandardMode; } @@ -363,17 +380,25 @@ namespace NKikimr { // FIXME: check that this CACHE works correctly. It can be a good // idea to forward all this messages directly to Committer, because // is knows exact values + const ui64 prevGuidInMemory = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo.Info.GetGuid(); auto &data = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo; data.Info = info; // create reply auto result = std::make_unique<TEvBlobStorage::TEvVSyncGuidResult>(NKikimrProto::OK, selfVDisk, TAppData::TimeProvider->Now(), nullptr, nullptr, ev->GetChannel()); - // put reply into the queue and wait until it would be committed - ui64 seqNum = DelayedQueue.WriteRequest(ev->Sender, std::move(result)); - // commit - void *cookie = reinterpret_cast<void*>(intptr_t(seqNum)); - auto msg = TEvSyncerCommit::Remote(vdisk, state, guid, cookie); - ctx.Send(CommitterId, msg.release()); + if (!SyncerCtx->Config->BaseInfo.ReadOnly) { + // put reply into the queue and wait until it would be committed + ui64 seqNum = DelayedQueue.WriteRequest(ev->Sender, std::move(result)); + // commit + void *cookie = reinterpret_cast<void*>(intptr_t(seqNum)); + auto msg = TEvSyncerCommit::Remote(vdisk, state, guid, cookie); + ctx.Send(CommitterId, msg.release()); + } else { + LOG_WARN(ctx, BS_SYNCER, + VDISKP(SyncerCtx->VCtx->VDiskLogPrefix, + "%s: Skipping commit of incoming EvVSyncGuid: saved guid %s", + __PRETTY_FUNCTION__, prevGuidInMemory != guid ? "differs" : "matches")); + } } else { // handle READ request auto &data = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo.Info; @@ -465,7 +490,7 @@ namespace NKikimr { // Other handlers //////////////////////////////////////////////////////////////////////// void Handle(TEvLocalStatus::TPtr &ev, const TActorContext &ctx) { - if (Phase == TPhaseVal::PhaseStandardMode) { + if (Phase == TPhaseVal::PhaseStandardMode && SchedulerId) { ctx.Send(ev->Forward(SchedulerId)); } else { auto result = std::make_unique<TEvLocalStatusResult>(); @@ -502,7 +527,9 @@ namespace NKikimr { GInfo = msg->NewInfo; // reconfigure scheduler - ctx.Send(SchedulerId, msg->Clone()); + if (SchedulerId) { + ctx.Send(SchedulerId, msg->Clone()); + } // reconfigure propagators for (const auto &aid : PropagatorIds) { ctx.Send(aid, msg->Clone()); diff --git a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp index d4dc97f9ab..eb9983d2d7 100644 --- a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp +++ b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp @@ -638,6 +638,7 @@ namespace NKikimr { std::unique_ptr<TDecision> Decision; EPhase Phase = PhaseNotSet; TActorId FirstRunActorId; + const bool ReadOnly; //////////////////////////////////////////////////////////////////////// @@ -777,6 +778,14 @@ namespace NKikimr { // FirstRun Phase //////////////////////////////////////////////////////////////////////// void FirstRunPhase(const TActorContext &ctx, EFirstRunStep f) { + if (ReadOnly) { + const TString explanation = "unable to establish new GUID while in read-only"; + LOG_WARN(ctx, BS_SYNCER, + VDISKP(VCtx->VDiskLogPrefix, "TVDiskGuidRecoveryActor: %s", explanation.data())); + *Decision = TDecision::Inconsistency(explanation); + Finish(ctx, *Decision); + return; + } auto guid = Decision->GetGuid(); Become(&TThis::WaitForFirstRunStateFunc); FirstRunActorId = ctx.Register(CreateVDiskGuidFirstRunActor(VCtx, GInfo, CommitterId, ctx.SelfID, f, guid)); @@ -886,7 +895,8 @@ namespace NKikimr { TIntrusivePtr<TBlobStorageGroupInfo> info, const TActorId &committerId, const TActorId ¬ifyId, - const TLocalSyncerState &locallyRecoveredState) + const TLocalSyncerState &locallyRecoveredState, + bool readOnly) : TActorBootstrapped<TVDiskGuidRecoveryActor>() , VCtx(std::move(vctx)) , GInfo(std::move(info)) @@ -896,6 +906,7 @@ namespace NKikimr { GInfo->PickTopology(), locallyRecoveredState) , Decision() + , ReadOnly(readOnly) {} }; @@ -905,9 +916,10 @@ namespace NKikimr { TIntrusivePtr<TBlobStorageGroupInfo> info, const TActorId &committerId, const TActorId ¬ifyId, - const NSyncer::TLocalSyncerState &localState) { + const NSyncer::TLocalSyncerState &localState, + bool readOnly) { return new NSyncer::TVDiskGuidRecoveryActor(std::move(vctx), std::move(info), committerId, - notifyId, localState); + notifyId, localState, readOnly); } } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h index 1b3be5a03a..823055d2c7 100644 --- a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h +++ b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h @@ -144,6 +144,7 @@ namespace NKikimr { TIntrusivePtr<TBlobStorageGroupInfo> info, const TActorId &committerId, const TActorId ¬ifyId, - const NSyncer::TLocalSyncerState &localState); + const NSyncer::TLocalSyncerState &localState, + bool readOnly); } // NKikimr diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 4fc8b06918..4d4ac187f1 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -329,6 +329,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) { case NKikimrBlobStorage::TConfigRequest::TCommand::kWipeVDisk: case NKikimrBlobStorage::TConfigRequest::TCommand::kSanitizeGroup: case NKikimrBlobStorage::TConfigRequest::TCommand::kCancelVirtualGroup: + case NKikimrBlobStorage::TConfigRequest::TCommand::kPutVDiskToReadOnly: return 2; // read-write commands go with higher priority as they are needed to keep cluster intact case NKikimrBlobStorage::TConfigRequest::TCommand::kReadHostConfig: diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp index 5433b599d9..071971b1bd 100644 --- a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp +++ b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp @@ -676,4 +676,27 @@ namespace NKikimr::NBsController { } } + void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& /*status*/) { + // first, find matching vslot + const TVSlotId& vslotId = cmd.GetVSlotId(); + TVSlotInfo *vslot = VSlots.FindForUpdate(vslotId); + if (!vslot) { + throw TExVSlotNotFound(vslotId); + } + + // second, validate vdisk id + const TVDiskID& vdiskId = VDiskIDFromVDiskID(cmd.GetVDiskId()); + if (vslot->GetVDiskId() != vdiskId) { + throw TExVDiskIdIncorrect(vdiskId, vslotId); + } + + TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId); + vslot->Mood = TMood::ReadOnly; + vslot->Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING; + vslot->DropFromVSlotReadyTimestampQ(); + vslot->IsReady = false; + GroupFailureModelChanged.insert(group->ID); + group->CalculateGroupStatus(); + } + } // NKikimr::NBsController diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp index f7fb07b54c..6500f6ed70 100644 --- a/ydb/core/mind/bscontroller/config.cpp +++ b/ydb/core/mind/bscontroller/config.cpp @@ -149,6 +149,10 @@ namespace NKikimr::NBsController { item.SetDoWipe(true); break; + case TMood::ReadOnly: + item.SetReadOnly(true); + break; + default: Y_FAIL(); } diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h index 5ca88f2cb6..3caa46df3a 100644 --- a/ydb/core/mind/bscontroller/config.h +++ b/ydb/core/mind/bscontroller/config.h @@ -286,6 +286,7 @@ namespace NKikimr { void ExecuteStep(const NKikimrBlobStorage::TWipeVDisk& cmd, TStatus& status); void ExecuteStep(const NKikimrBlobStorage::TSanitizeGroup& cmd, TStatus& status); void ExecuteStep(const NKikimrBlobStorage::TCancelVirtualGroup& cmd, TStatus& status); + void ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& status); }; } // NBsController diff --git a/ydb/core/mind/bscontroller/config_cmd.cpp b/ydb/core/mind/bscontroller/config_cmd.cpp index 5aa3f1c17e..f7197c8866 100644 --- a/ydb/core/mind/bscontroller/config_cmd.cpp +++ b/ydb/core/mind/bscontroller/config_cmd.cpp @@ -330,6 +330,7 @@ namespace NKikimr::NBsController { HANDLE_COMMAND(WipeVDisk) HANDLE_COMMAND(SanitizeGroup) HANDLE_COMMAND(CancelVirtualGroup) + HANDLE_COMMAND(PutVDiskToReadOnly) case NKikimrBlobStorage::TConfigRequest::TCommand::kAddMigrationPlan: case NKikimrBlobStorage::TConfigRequest::TCommand::kDeleteMigrationPlan: diff --git a/ydb/core/mind/bscontroller/mood.h b/ydb/core/mind/bscontroller/mood.h index 9e2452f78c..68488ff603 100644 --- a/ydb/core/mind/bscontroller/mood.h +++ b/ydb/core/mind/bscontroller/mood.h @@ -10,6 +10,7 @@ struct TMood { Wipe = 1, Delete = 2, Donor = 3, + ReadOnly = 4, }; static TString Name(const EValue value) { @@ -22,6 +23,8 @@ struct TMood { return "Delete"; case Donor: return "Donor"; + case ReadOnly: + return "ReadOnly"; } return Sprintf("Unknown%" PRIu64, (ui64)value); } diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index e043c91fdd..7425f9ff7a 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -427,8 +427,10 @@ void TBlobStorageController::ReadVSlot(const TVSlotInfo& vslot, TEvBlobStorage:: if (vslot.IsBeingDeleted()) { vDisk->SetDoDestroy(true); vDisk->SetEntityStatus(NKikimrBlobStorage::DESTROY); - } else { - vDisk->SetDoWipe(vslot.Mood == TMood::Wipe); + } else if (vslot.Mood == TMood::Wipe) { + vDisk->SetDoWipe(true); + } else if (vslot.Mood == TMood::ReadOnly) { + vDisk->SetReadOnly(true); } if (TGroupInfo *group = FindGroup(vslot.GroupId)) { diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index c962ede23f..0f6cfc8292 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -179,7 +179,7 @@ message TEvVPatchDiff { optional uint32 ExpectedXorDiffs = 7; optional bool ForceEnd = 8; optional bool NotifyIfNotReady = 9; - + optional NKikimrBlobStorage.EPutHandleClass HandleClass = 11; optional TMsgQoS MsgQoS = 10; @@ -1027,6 +1027,7 @@ message TNodeWardenServiceSet { optional string StoragePoolName = 8; optional TDonorMode DonorMode = 9; // vdisk runs in read-only limited mode without interaction with other disks of group repeated TDonor Donors = 10; // a set of donor disks used to accelerate replication + optional bool ReadOnly = 11; } message TReplBrokerConfig { diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto index 30fda97db6..9b2580b68f 100644 --- a/ydb/core/protos/blobstorage_config.proto +++ b/ydb/core/protos/blobstorage_config.proto @@ -480,6 +480,11 @@ message TCancelVirtualGroup { uint32 GroupId = 1; // id of a group we are going to cancel } +message TPutVDiskToReadOnly { + NKikimrBlobStorage.TVSlotId VSlotId = 1; + NKikimrBlobStorage.TVDiskID VDiskId = 2; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // INTERFACE PART //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -522,6 +527,7 @@ message TConfigRequest { TSanitizeGroup SanitizeGroup = 42; TReadSettings ReadSettings = 43; TCancelVirtualGroup CancelVirtualGroup = 44; + TPutVDiskToReadOnly PutVDiskToReadOnly = 45; // commands intended for internal use TReassignGroupDisk ReassignGroupDisk = 19; |