aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authortarum <tarum@yandex-team.com>2023-08-11 21:09:09 +0300
committertarum <tarum@yandex-team.com>2023-08-11 21:56:32 +0300
commit0484ec70edb1e3f921df3836762b032ed304b64a (patch)
treefba3113a27dbdd19f35834da0c1106f44b0d1674
parent656f1abe06dca744d46a0c0459a44cc7c5de919c (diff)
downloadydb-0484ec70edb1e3f921df3836762b032ed304b64a.tar.gz
KIKIMR-17274: Support read-only mode in VDisk
-rw-r--r--ydb/core/base/blobstorage.h2
-rw-r--r--ydb/core/blobstorage/base/blobstorage_events.h8
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_impl.h1
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp12
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/lib/env.h16
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp116
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt74
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt77
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt79
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt17
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt67
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make17
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/ya.make1
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.cpp5
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.h8
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp54
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp26
-rw-r--r--ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp49
-rw-r--r--ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp18
-rw-r--r--ydb/core/blobstorage/vdisk/syncer/guid_recovery.h3
-rw-r--r--ydb/core/mind/bscontroller/bsc.cpp1
-rw-r--r--ydb/core/mind/bscontroller/cmds_storage_pool.cpp23
-rw-r--r--ydb/core/mind/bscontroller/config.cpp4
-rw-r--r--ydb/core/mind/bscontroller/config.h1
-rw-r--r--ydb/core/mind/bscontroller/config_cmd.cpp1
-rw-r--r--ydb/core/mind/bscontroller/mood.h3
-rw-r--r--ydb/core/mind/bscontroller/register_node.cpp6
-rw-r--r--ydb/core/protos/blobstorage.proto3
-rw-r--r--ydb/core/protos/blobstorage_config.proto6
33 files changed, 655 insertions, 47 deletions
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
index d087de7720..3844d48c7c 100644
--- a/ydb/core/base/blobstorage.h
+++ b/ydb/core/base/blobstorage.h
@@ -543,6 +543,7 @@ struct TEvBlobStorage {
EvRegisterPDiskLoadActor,
EvStatusUpdate,
EvDropDonor,
+ EvPutVDiskToReadOnly,
EvCntReply = EvPut + 7 * 512, /// 268 635 648
EvVGenerationChangeResult,
@@ -2329,6 +2330,7 @@ struct TEvBlobStorage {
struct TEvControllerGroupDecommittedNotify;
struct TEvControllerGroupDecommittedResponse;
struct TEvControllerGroupMetricsExchange;
+ struct TEvPutVDiskToReadOnly;
struct TEvMonStreamQuery;
struct TEvMonStreamActorDeathNote;
diff --git a/ydb/core/blobstorage/base/blobstorage_events.h b/ydb/core/blobstorage/base/blobstorage_events.h
index 14b07bf363..905d343163 100644
--- a/ydb/core/blobstorage/base/blobstorage_events.h
+++ b/ydb/core/blobstorage/base/blobstorage_events.h
@@ -537,6 +537,14 @@ namespace NKikimr {
NKikimrBlobStorage::TEvControllerGroupMetricsExchange, EvControllerGroupMetricsExchange>
{};
+ struct TEvBlobStorage::TEvPutVDiskToReadOnly : TEventLocal<TEvPutVDiskToReadOnly, EvPutVDiskToReadOnly> {
+ const TVDiskID VDiskId;
+
+ TEvPutVDiskToReadOnly(TVDiskID vDiskId)
+ : VDiskId(std::move(vDiskId))
+ {}
+ };
+
struct TEvNodeWardenQueryGroupInfo : TEventPB<TEvNodeWardenQueryGroupInfo, NKikimrBlobStorage::TEvNodeWardenQueryGroupInfo,
TEvBlobStorage::EvNodeWardenQueryGroupInfo> {
TEvNodeWardenQueryGroupInfo() = default;
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
index acda31d319..c76bec20fa 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
@@ -254,6 +254,7 @@ namespace NKikimr::NStorage {
TIntrusivePtr<TBlobStorageGroupInfo> GroupInfo;
ui32 OrderNumber;
bool DonorMode;
+ bool ReadOnly;
};
std::optional<TRuntimeData> RuntimeData;
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
index 3933308efc..8d8162fd11 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
@@ -57,6 +57,8 @@ namespace NKikimr::NStorage {
const ui64 pdiskGuid = vdisk.Config.GetVDiskLocation().GetPDiskGuid();
const bool restartInFlight = InFlightRestartedPDisks.count({vslotId.NodeId, vslotId.PDiskId});
const bool donorMode = vdisk.Config.HasDonorMode();
+ const bool readOnly = vdisk.Config.GetReadOnly();
+ Y_VERIFY_S(!donorMode || !readOnly, "Only one of modes should be enabled: donorMode " << donorMode << ", readOnly " << readOnly);
STLOG(PRI_DEBUG, BS_NODE, NW23, "StartLocalVDiskActor", (RestartInFlight, restartInFlight),
(SlayInFlight, SlayInFlight.contains(vslotId)), (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId),
@@ -158,7 +160,7 @@ namespace NKikimr::NStorage {
TVDiskConfig::TBaseInfo baseInfo(vdiskId, pdiskServiceId, pdiskGuid, vslotId.PDiskId, deviceType,
vslotId.VDiskSlotId, kind, NextLocalPDiskInitOwnerRound(), groupInfo->GetStoragePoolName(), donorMode,
- donorDiskIds, scrubCookie, whiteboardInstanceGuid);
+ donorDiskIds, scrubCookie, whiteboardInstanceGuid, readOnly);
baseInfo.ReplPDiskReadQuoter = pdiskIt->second.ReplPDiskReadQuoter;
baseInfo.ReplPDiskWriteQuoter = pdiskIt->second.ReplPDiskWriteQuoter;
@@ -197,7 +199,8 @@ namespace NKikimr::NStorage {
vdisk.RuntimeData.emplace(TVDiskRecord::TRuntimeData{
.GroupInfo = groupInfo,
.OrderNumber = groupInfo->GetOrderNumber(TVDiskIdShort(vdiskId)),
- .DonorMode = donorMode
+ .DonorMode = donorMode,
+ .ReadOnly = readOnly,
});
vdisk.Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING;
@@ -221,9 +224,10 @@ namespace NKikimr::NStorage {
// 4. Deleting VSlot during group reconfiguration or donor termination.
// 5. Making VDisk a donor one.
// 6. Updating VDisk generation when modifying group.
+ // 7. Putting VDisk into or out of read-only
//
// The main idea of this command is when VDisk is created, it does not change its configuration. It may be
- // wiped out several times, it may become a donor and then it may be destroyed. That is a possible life cycle
+ // wiped out several times, it may become a donor or read-only and then it may be destroyed. That is a possible life cycle
// of a VDisk in the occupied slot.
if (!vdisk.HasVDiskID() || !vdisk.HasVDiskLocation()) {
@@ -261,7 +265,7 @@ namespace NKikimr::NStorage {
Slay(record);
} else if (!record.RuntimeData) {
StartLocalVDiskActor(record, TDuration::Zero());
- } else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode()) {
+ } else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode() || record.RuntimeData->ReadOnly != record.Config.GetReadOnly()) {
PoisonLocalVDisk(record);
StartLocalVDiskActor(record, TDuration::Seconds(15) /* PDisk confidence delay */);
}
diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt
index 9e2d368616..0ae1ccfd5a 100644
--- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin-x86_64.txt
@@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat)
add_subdirectory(ut_donor)
add_subdirectory(ut_group_reconfiguration)
add_subdirectory(ut_osiris)
+add_subdirectory(ut_read_only_vdisk)
add_subdirectory(ut_replication)
add_subdirectory(ut_scrub)
diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt
index d85b0fed5e..0507f73ff2 100644
--- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt
@@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat)
add_subdirectory(ut_donor)
add_subdirectory(ut_group_reconfiguration)
add_subdirectory(ut_osiris)
+add_subdirectory(ut_read_only_vdisk)
add_subdirectory(ut_replication)
add_subdirectory(ut_scrub)
diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt
index f5ab8377bb..d49a3b9279 100644
--- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-x86_64.txt
@@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat)
add_subdirectory(ut_donor)
add_subdirectory(ut_group_reconfiguration)
add_subdirectory(ut_osiris)
+add_subdirectory(ut_read_only_vdisk)
add_subdirectory(ut_replication)
add_subdirectory(ut_scrub)
diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt
index 4690192d32..d35c83a40f 100644
--- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.windows-x86_64.txt
@@ -12,6 +12,7 @@ add_subdirectory(ut_blob_depot_fat)
add_subdirectory(ut_donor)
add_subdirectory(ut_group_reconfiguration)
add_subdirectory(ut_osiris)
+add_subdirectory(ut_read_only_vdisk)
add_subdirectory(ut_replication)
add_subdirectory(ut_scrub)
diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
index 247411a1a2..74c3d2a372 100644
--- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h
+++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
@@ -98,6 +98,10 @@ struct TEnvironmentSetup {
Cerr << "RandomSeed# " << seed << Endl;
}
+ TInstant Now() {
+ return TAppData::TimeProvider->Now();
+ }
+
TString GenerateRandomString(ui32 len) {
TString res = TString::Uninitialized(len);
char *p = res.Detach();
@@ -675,6 +679,18 @@ struct TEnvironmentSetup {
UNIT_ASSERT(response.GetSuccess());
}
+ void PutVDiskToReadOnly(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
+ NKikimrBlobStorage::TConfigRequest request;
+ auto *roCmd = request.AddCommand()->MutablePutVDiskToReadOnly();
+ auto *vslot = roCmd->MutableVSlotId();
+ vslot->SetNodeId(nodeId);
+ vslot->SetPDiskId(pdiskId);
+ vslot->SetVSlotId(vslotId);
+ VDiskIDFromVDiskID(vdiskId, roCmd->MutableVDiskId());
+ auto response = Invoke(request);
+ UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
+ }
+
void UpdateDriveStatus(ui32 nodeId, ui32 pdiskId, NKikimrBlobStorage::EDriveStatus status,
NKikimrBlobStorage::EDecommitStatus decommitStatus) {
NKikimrBlobStorage::TConfigRequest request;
diff --git a/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
new file mode 100644
index 0000000000..1013e3cb0a
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
@@ -0,0 +1,116 @@
+#include <ydb/core/blobstorage/ut_blobstorage/lib/env.h>
+
+Y_UNIT_TEST_SUITE(ReadOnlyVDisk) {
+
+ Y_UNIT_TEST(Basic) {
+ TEnvironmentSetup env{{
+ .NodeCount = 8,
+ .VDiskReplPausedAtStart = false,
+ .Erasure = TBlobStorageGroupType::Erasure4Plus2Block,
+ }};
+ env.CreateBoxAndPool(1, 1);
+ env.Sim(TDuration::Minutes(1));
+
+ auto groups = env.GetGroups();
+ UNIT_ASSERT_VALUES_EQUAL(groups.size(), 1);
+ const TIntrusivePtr<TBlobStorageGroupInfo> info = env.GetGroupInfo(groups.front());
+
+ const TActorId vdiskActorId = info->GetActorId(0);
+
+ auto prepData = [&] (const ui32 dataLen, const ui32 start) {
+ TString data(Reserve(dataLen));
+ for (ui32 i = 0; i < dataLen; ++i) {
+ data.push_back('a' + (start + i) % 26);
+ }
+ return data;
+ };
+
+ TVector<TString> dataArr = {
+ prepData(128 * 1024, 0),
+ prepData(32 * 1024, 3),
+ };
+
+ auto sendPut = [&] (ui32 step, NKikimrProto::EReplyStatus expectedStatus) {
+ const TString& data = dataArr[step % 2];
+ const TLogoBlobID id(1, 1, step, 0, data.size(), 0);
+ Cerr << "SEND TEvPut with key " << id.ToString() << Endl;
+ const TActorId sender = env.Runtime->AllocateEdgeActor(vdiskActorId.NodeId());
+ auto ev = std::make_unique<TEvBlobStorage::TEvPut>(id, data, TInstant::Max());
+ env.Runtime->WrapInActorContext(sender, [&] {
+ SendToBSProxy(sender, info->GroupID, ev.release());
+ });
+ auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvPutResult>(sender, false);
+ UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, expectedStatus);
+ Cerr << "TEvPutResult: " << res->Get()->ToString() << Endl;
+ };
+
+ auto sendGet = [&] (ui32 step) {
+ const TString& data = dataArr[step % 2];
+ const TLogoBlobID blobId(1, 1, step, 0, data.size(), 0);
+ Cerr << "SEND TEvGet with key " << blobId.ToString() << Endl;
+ const TActorId sender = env.Runtime->AllocateEdgeActor(vdiskActorId.NodeId());
+ auto ev = std::make_unique<TEvBlobStorage::TEvGet>(
+ blobId,
+ /* shift */ 0,
+ /* size */ data.size(),
+ TInstant::Max(),
+ NKikimrBlobStorage::EGetHandleClass::FastRead
+ );
+ env.Runtime->WrapInActorContext(sender, [&] () {
+ SendToBSProxy(sender, info->GroupID, ev.release());
+ });
+ TInstant getDeadline = env.Now() + TDuration::Seconds(30);
+ auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender, /* termOnCapture */ false, getDeadline);
+ UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK);
+ Cerr << "TEvGetResult: " << res->Get()->ToString() << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(res->Get()->Responses[0].Buffer.ConvertToString(), data);
+ };
+
+ Cerr << "=== Trying to put and get a blob ===" << Endl;
+ ui32 step = 0;
+ sendPut(step, NKikimrProto::OK);
+ sendGet(step);
+ ++step;
+
+ auto putVDiskToRo = [&] (ui32 position) {
+ const TVDiskID& someVDisk = info->GetVDiskId(position);
+
+ auto baseConfig = env.FetchBaseConfig();
+
+ const auto& somePDisk = baseConfig.GetPDisk(position);
+ const auto& someVSlot = baseConfig.GetVSlot(position);
+ Cerr << "Issuing PutVDiskToReadOnly for position " << position << Endl;
+ env.PutVDiskToReadOnly(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId(), someVDisk);
+ env.Sim(TDuration::Seconds(30));
+ };
+
+ Cerr << "=== Putting VDisk #0 to read-only ===" << Endl;
+ putVDiskToRo(0);
+
+ Cerr << "=== Write 10 blobs ===" << Endl;
+ for (ui32 i = 0; i < 10; ++i) {
+ sendPut(step, NKikimrProto::OK);
+ ++step;
+ }
+
+ Cerr << "=== Read all blobs ===" << Endl;
+ for (ui32 i = 0; i < step; ++i) {
+ sendGet(i);
+ }
+
+ Cerr << "=== Put 2 more VDisks to read-only ===" << Endl;
+ putVDiskToRo(1);
+ putVDiskToRo(2);
+
+ Cerr << "=== Write 10 more blobs, expect errors ===" << Endl;
+ for (ui32 i = 0; i < 10; ++i) {
+ sendPut(step, NKikimrProto::ERROR);
+ ++step;
+ }
+
+ Cerr << "=== Read all blobs again, expect it to work ===" << Endl;
+ for (ui32 i = 0; i < step; ++i) {
+ sendGet(i);
+ }
+ }
+}
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..2f34dba19e
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,74 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ut_blobstorage-ut_read_only_vdisk)
+target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage
+)
+target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ blobstorage-ut_blobstorage-lib
+)
+target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+)
+set_property(
+ TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ut_blobstorage-ut_read_only_vdisk
+ system_allocator
+)
+vcs_info(ut_blobstorage-ut_read_only_vdisk)
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..202ee275e8
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ut_blobstorage-ut_read_only_vdisk)
+target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage
+)
+target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ blobstorage-ut_blobstorage-lib
+)
+target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+)
+set_property(
+ TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ut_blobstorage-ut_read_only_vdisk
+ cpp-malloc-jemalloc
+)
+vcs_info(ut_blobstorage-ut_read_only_vdisk)
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..4f579789d9
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,79 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ut_blobstorage-ut_read_only_vdisk)
+target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage
+)
+target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ blobstorage-ut_blobstorage-lib
+)
+target_link_options(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+)
+set_property(
+ TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ut_blobstorage-ut_read_only_vdisk
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ut_blobstorage-ut_read_only_vdisk)
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..fb688a40eb
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,67 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ut_blobstorage-ut_read_only_vdisk)
+target_include_directories(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage
+)
+target_link_libraries(ut_blobstorage-ut_read_only_vdisk PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ blobstorage-ut_blobstorage-lib
+)
+target_sources(ut_blobstorage-ut_read_only_vdisk PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/read_only_vdisk.cpp
+)
+set_property(
+ TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_TARGET
+ ut_blobstorage-ut_read_only_vdisk
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ut_blobstorage-ut_read_only_vdisk
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ut_blobstorage-ut_read_only_vdisk
+ system_allocator
+)
+vcs_info(ut_blobstorage-ut_read_only_vdisk)
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make
new file mode 100644
index 0000000000..c390ceb28c
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk/ya.make
@@ -0,0 +1,17 @@
+UNITTEST_FOR(ydb/core/blobstorage/ut_blobstorage)
+
+ FORK_SUBTESTS()
+
+ SIZE(MEDIUM)
+
+ TIMEOUT(600)
+
+ SRCS(
+ read_only_vdisk.cpp
+ )
+
+ PEERDIR(
+ ydb/core/blobstorage/ut_blobstorage/lib
+ )
+
+END()
diff --git a/ydb/core/blobstorage/ut_blobstorage/ya.make b/ydb/core/blobstorage/ut_blobstorage/ya.make
index 28b2a94903..a8b6ccd8e4 100644
--- a/ydb/core/blobstorage/ut_blobstorage/ya.make
+++ b/ydb/core/blobstorage/ut_blobstorage/ya.make
@@ -62,6 +62,7 @@ RECURSE_FOR_TESTS(
ut_blob_depot_fat
ut_donor
ut_group_reconfiguration
+ ut_read_only_vdisk
ut_osiris
ut_replication
ut_scrub
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
index b62819d417..51ea162e41 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
@@ -54,7 +54,8 @@ namespace NKikimr {
AnubisTimeout = TDuration::Minutes(60);
RunSyncer = true;
RunAnubis = false; // FIXME: turn on by default
- RunDefrag = true;
+ RunDefrag = !baseInfo.ReadOnly;
+ RunScrubber = !baseInfo.ReadOnly;
SyncLogMaxDiskAmount = 0; //ui64(2) << ui64(30); // 2 GB
SyncLogMaxEntryPointSize = ui64(128) << ui64(10); // 128 KB
@@ -76,7 +77,7 @@ namespace NKikimr {
HandoffMaxInFlightSize = 1000;
HandoffMaxInFlightByteSize = 16u << 20u;
HandoffTimeout = TDuration::Seconds(10);
- RunRepl = true;
+ RunRepl = !baseInfo.ReadOnly;
RunHandoff = false;
SkeletonFrontGets_MaxInFlightCount = 24;
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
index efe272f3dd..e38278d33c 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -39,6 +39,8 @@ namespace NKikimr {
TDuration YardInitDelay = TDuration::Zero();
const ui64 ScrubCookie = 0;
const ui64 WhiteboardInstanceGuid = 0;
+ // handle only read requests: needed when VDisk can't write, e.g. no disk space, but still has the data
+ const bool ReadOnly = false;
TBaseInfo(
const TVDiskIdShort &vDiskIdShort,
@@ -53,7 +55,9 @@ namespace NKikimr {
const bool donorMode = false,
std::vector<std::pair<TVDiskID, TActorId>> donorDiskIds = {},
ui64 scrubCookie = 0,
- ui64 whiteboardInstanceGuid = 0)
+ ui64 whiteboardInstanceGuid = 0,
+ const bool readOnly = false
+ )
: VDiskIdShort(vDiskIdShort)
, PDiskActorID(pDiskActorId)
, InitOwnerRound(initOwnerRound)
@@ -67,6 +71,7 @@ namespace NKikimr {
, DonorDiskIds(std::move(donorDiskIds))
, ScrubCookie(scrubCookie)
, WhiteboardInstanceGuid(whiteboardInstanceGuid)
+ , ReadOnly(readOnly)
{}
TBaseInfo(const TBaseInfo &) = default;
@@ -139,6 +144,7 @@ namespace NKikimr {
bool RunSyncer;
bool RunAnubis;
bool RunDefrag;
+ bool RunScrubber;
///////////// SYNCLOG SETTINGS //////////////////////
ui64 SyncLogMaxDiskAmount;
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index f8b2c85ef5..b1b8daa11e 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -901,7 +901,7 @@ namespace NKikimr {
if (record.HasCookie())
cookie = record.GetCookie();
auto handleClass = ev->Get()->Record.GetHandleClass();
-
+
std::unique_ptr<TEvBlobStorage::TEvVGetResult> result;
if (ev->Get()->IsInternal) {
result.reset(new TEvBlobStorage::TEvVGetResult(NKikimrProto::OK, SelfVDiskId, now,
@@ -1725,6 +1725,32 @@ namespace NKikimr {
UpdateVDiskStatus(NKikimrBlobStorage::ERROR);
}
+ void StartDefrag(const TActorContext &ctx) {
+ auto defragCtx = std::make_shared<TDefragCtx>(VCtx, HugeBlobCtx, PDiskCtx, ctx.SelfID,
+ Db->HugeKeeperID, true);
+ DefragId = ctx.Register(CreateDefragActor(defragCtx, GInfo));
+ ActiveActors.Insert(DefragId); // keep forever
+ }
+
+ void StartScrubberActor(const TActorContext &ctx, NKikimrVDiskData::TScrubEntrypoint scrubEntrypoint, ui64 scrubEntrypointLsn) {
+ auto scrubCtx = MakeIntrusive<TScrubContext>(
+ VCtx,
+ PDiskCtx,
+ GInfo,
+ SelfId(),
+ Hull->GetHullDs()->LogoBlobs->LIActor,
+ SelfId().NodeId(),
+ Config->BaseInfo.PDiskId,
+ Config->BaseInfo.VDiskSlotId,
+ Config->BaseInfo.ScrubCookie,
+ Db->GetVDiskIncarnationGuid(),
+ Db->LsnMngr,
+ Db->LoggerID,
+ Db->LogCutterID);
+ ScrubId = ctx.Register(CreateScrubActor(std::move(scrubCtx), std::move(scrubEntrypoint), scrubEntrypointLsn));
+ ActiveActors.Insert(ScrubId);
+ }
+
void Handle(TEvBlobStorage::TEvLocalRecoveryDone::TPtr &ev, const TActorContext &ctx) {
LocalRecovInfo = ev->Get()->RecovInfo;
LocalDbRecoveryID = TActorId();
@@ -1875,31 +1901,15 @@ namespace NKikimr {
}
if (Config->RunDefrag && AppData()->FeatureFlags.GetAllowVDiskDefrag()) {
- auto defragCtx = std::make_shared<TDefragCtx>(VCtx, HugeBlobCtx, PDiskCtx, ctx.SelfID,
- Db->HugeKeeperID, true);
- DefragId = ctx.Register(CreateDefragActor(defragCtx, GInfo));
- ActiveActors.Insert(DefragId); // keep forever
+ StartDefrag(ctx);
}
// create scrubber actor
- auto scrubCtx = MakeIntrusive<TScrubContext>(
- VCtx,
- PDiskCtx,
- GInfo,
- SelfId(),
- Hull->GetHullDs()->LogoBlobs->LIActor,
- SelfId().NodeId(),
- Config->BaseInfo.PDiskId,
- Config->BaseInfo.VDiskSlotId,
- Config->BaseInfo.ScrubCookie,
- Db->GetVDiskIncarnationGuid(),
- Db->LsnMngr,
- Db->LoggerID,
- Db->LogCutterID);
- ScrubId = ctx.Register(CreateScrubActor(std::move(scrubCtx), std::move(ev->Get()->ScrubEntrypoint),
- ev->Get()->ScrubEntrypointLsn));
- ActiveActors.Insert(ScrubId);
+ if (Config->RunScrubber) {
+ StartScrubberActor(ctx, std::move(ev->Get()->ScrubEntrypoint), ev->Get()->ScrubEntrypointLsn);
+ }
+ // create syncer actor
if (Config->RunSyncer && !Config->BaseInfo.DonorMode) {
// switch to syncronization step
Become(&TThis::StateSyncGuidRecovery);
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
index f47a76a4f0..7076f5b16b 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
@@ -1114,6 +1114,15 @@ namespace NKikimr {
ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("NOT_READY"));
}
+ template <class TEventPtr>
+ void DatabaseReadOnlyHandle(TEventPtr &ev, const TActorContext &ctx) {
+ LOG_ERROR_S(ctx, NKikimrServices::BS_SKELETON, VCtx->VDiskLogPrefix
+ << "Unavailable in read-only"
+ << " Sender# " << ev->Sender.ToString());
+ TInstant now = TAppData::TimeProvider->Now();
+ Reply(ev, ctx, NKikimrProto::ERROR, "VDisk is in read-only mode", now);
+ }
+
////////////////////////////////////////////////////////////////////////////
// HANDLE SECTOR
////////////////////////////////////////////////////////////////////////////
@@ -1864,6 +1873,20 @@ namespace NKikimr {
|| std::is_same_v<TEv, TEvBlobStorage::TEvVGet>
|| std::is_same_v<TEv, TEvBlobStorage::TEvVPut>;
+ template <typename TEv>
+ static constexpr bool IsReadOnlyCompatible = (
+ std::is_same_v<TEv, TEvBlobStorage::TEvVCheckReadiness> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVDbStat> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVGet> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVGetBarrier> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVGetBlock> ||
+ std::is_same_v<TEv, TEvGetLogoBlobIndexStatRequest> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVStatus> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVAssimilate> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVSync> ||
+ std::is_same_v<TEv, TEvBlobStorage::TEvVSyncFull>
+ );
+
template<typename TEventType>
void CheckExecute(TAutoPtr<TEventHandle<TEventType>>& ev, const TActorContext& ctx) {
if constexpr (IsPatchEvent<TEventType>) {
@@ -1897,6 +1920,9 @@ namespace NKikimr {
return Reply(ev, ctx, NKikimrProto::RACE, "group generation mismatch", TAppData::TimeProvider->Now());
} else if (!GInfo->CheckScope(TKikimrScopeId(ev->OriginScopeId), ctx, true)) {
DatabaseAccessDeniedHandle(ev, ctx);
+ } else if (Config->BaseInfo.ReadOnly && !IsReadOnlyCompatible<TEventType>) {
+ LOG_INFO_S(ctx, BS_SKELETON, VCtx->VDiskLogPrefix << "Blocking request incompatible with read-only: " << TypeName<TEventType>());
+ DatabaseReadOnlyHandle(ev, ctx);
} else {
SetReceivedTime(ev);
CheckExecute(ev, ctx);
diff --git a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp
index 18f36a166d..285fdfb697 100644
--- a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp
+++ b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp
@@ -221,7 +221,7 @@ namespace NKikimr {
void SyncGuid(const TActorContext &ctx) {
Become(&TThis::SyncGuidStateFunc);
GuidRecoveryId = ctx.Register(CreateVDiskGuidRecoveryActor(SyncerCtx->VCtx, GInfo, CommitterId, SelfId(),
- LocalSyncerState));
+ LocalSyncerState, SyncerCtx->Config->BaseInfo.ReadOnly));
ActiveActors.Insert(GuidRecoveryId);
Phase = TPhaseVal::PhaseSyncGuid;
}
@@ -289,6 +289,14 @@ namespace NKikimr {
// Recover Lost Data
////////////////////////////////////////////////////////////////////////
void RecoverLostData(const TActorContext &ctx) {
+ if (SyncerCtx->Config->BaseInfo.ReadOnly) {
+ LOG_WARN(ctx, BS_SYNCER,
+ VDISKP(SyncerCtx->VCtx->VDiskLogPrefix,
+ "Unable to recover lost data in read-only mode. Transitioning to inconsistent state."));
+ InconsistentState(ctx);
+ return;
+ }
+
Become(&TThis::RecoverLostDataStateFunc);
const TVDiskEternalGuid guid = GuidRecovOutcome->Guid;
RecoverLostDataId = ctx.Register(CreateSyncerRecoverLostDataActor(SyncerCtx, GInfo, CommitterId, ctx.SelfID, guid));
@@ -330,8 +338,17 @@ namespace NKikimr {
new TEvSyncGuidRecoveryDone(NKikimrProto::OK, LocalSyncerState.DbBirthLsn));
SyncerData->Neighbors->DbBirthLsn = LocalSyncerState.DbBirthLsn;
Become(&TThis::StandardModeStateFunc);
- SchedulerId = ctx.Register(CreateSyncerSchedulerActor(SyncerCtx, GInfo, SyncerData, CommitterId));
- ActiveActors.Insert(SchedulerId);
+ if (!SyncerCtx->Config->BaseInfo.ReadOnly) {
+ LOG_DEBUG(ctx, BS_SYNCER,
+ VDISKP(SyncerCtx->VCtx->VDiskLogPrefix,
+ "%s: Creating syncer scheduler on node %d", __PRETTY_FUNCTION__, SelfId().NodeId()));
+ SchedulerId = ctx.Register(CreateSyncerSchedulerActor(SyncerCtx, GInfo, SyncerData, CommitterId));
+ ActiveActors.Insert(SchedulerId);
+ } else {
+ LOG_WARN(ctx, BS_SYNCER,
+ VDISKP(SyncerCtx->VCtx->VDiskLogPrefix,
+ "%s: Skipping scheduler start due to read-only", __PRETTY_FUNCTION__));
+ }
Phase = TPhaseVal::PhaseStandardMode;
}
@@ -363,17 +380,25 @@ namespace NKikimr {
// FIXME: check that this CACHE works correctly. It can be a good
// idea to forward all this messages directly to Committer, because
// is knows exact values
+ const ui64 prevGuidInMemory = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo.Info.GetGuid();
auto &data = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo;
data.Info = info;
// create reply
auto result = std::make_unique<TEvBlobStorage::TEvVSyncGuidResult>(NKikimrProto::OK, selfVDisk,
TAppData::TimeProvider->Now(), nullptr, nullptr, ev->GetChannel());
- // put reply into the queue and wait until it would be committed
- ui64 seqNum = DelayedQueue.WriteRequest(ev->Sender, std::move(result));
- // commit
- void *cookie = reinterpret_cast<void*>(intptr_t(seqNum));
- auto msg = TEvSyncerCommit::Remote(vdisk, state, guid, cookie);
- ctx.Send(CommitterId, msg.release());
+ if (!SyncerCtx->Config->BaseInfo.ReadOnly) {
+ // put reply into the queue and wait until it would be committed
+ ui64 seqNum = DelayedQueue.WriteRequest(ev->Sender, std::move(result));
+ // commit
+ void *cookie = reinterpret_cast<void*>(intptr_t(seqNum));
+ auto msg = TEvSyncerCommit::Remote(vdisk, state, guid, cookie);
+ ctx.Send(CommitterId, msg.release());
+ } else {
+ LOG_WARN(ctx, BS_SYNCER,
+ VDISKP(SyncerCtx->VCtx->VDiskLogPrefix,
+ "%s: Skipping commit of incoming EvVSyncGuid: saved guid %s",
+ __PRETTY_FUNCTION__, prevGuidInMemory != guid ? "differs" : "matches"));
+ }
} else {
// handle READ request
auto &data = (*SyncerData->Neighbors)[vdisk].Get().PeerGuidInfo.Info;
@@ -465,7 +490,7 @@ namespace NKikimr {
// Other handlers
////////////////////////////////////////////////////////////////////////
void Handle(TEvLocalStatus::TPtr &ev, const TActorContext &ctx) {
- if (Phase == TPhaseVal::PhaseStandardMode) {
+ if (Phase == TPhaseVal::PhaseStandardMode && SchedulerId) {
ctx.Send(ev->Forward(SchedulerId));
} else {
auto result = std::make_unique<TEvLocalStatusResult>();
@@ -502,7 +527,9 @@ namespace NKikimr {
GInfo = msg->NewInfo;
// reconfigure scheduler
- ctx.Send(SchedulerId, msg->Clone());
+ if (SchedulerId) {
+ ctx.Send(SchedulerId, msg->Clone());
+ }
// reconfigure propagators
for (const auto &aid : PropagatorIds) {
ctx.Send(aid, msg->Clone());
diff --git a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp
index d4dc97f9ab..eb9983d2d7 100644
--- a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp
+++ b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.cpp
@@ -638,6 +638,7 @@ namespace NKikimr {
std::unique_ptr<TDecision> Decision;
EPhase Phase = PhaseNotSet;
TActorId FirstRunActorId;
+ const bool ReadOnly;
////////////////////////////////////////////////////////////////////////
@@ -777,6 +778,14 @@ namespace NKikimr {
// FirstRun Phase
////////////////////////////////////////////////////////////////////////
void FirstRunPhase(const TActorContext &ctx, EFirstRunStep f) {
+ if (ReadOnly) {
+ const TString explanation = "unable to establish new GUID while in read-only";
+ LOG_WARN(ctx, BS_SYNCER,
+ VDISKP(VCtx->VDiskLogPrefix, "TVDiskGuidRecoveryActor: %s", explanation.data()));
+ *Decision = TDecision::Inconsistency(explanation);
+ Finish(ctx, *Decision);
+ return;
+ }
auto guid = Decision->GetGuid();
Become(&TThis::WaitForFirstRunStateFunc);
FirstRunActorId = ctx.Register(CreateVDiskGuidFirstRunActor(VCtx, GInfo, CommitterId, ctx.SelfID, f, guid));
@@ -886,7 +895,8 @@ namespace NKikimr {
TIntrusivePtr<TBlobStorageGroupInfo> info,
const TActorId &committerId,
const TActorId &notifyId,
- const TLocalSyncerState &locallyRecoveredState)
+ const TLocalSyncerState &locallyRecoveredState,
+ bool readOnly)
: TActorBootstrapped<TVDiskGuidRecoveryActor>()
, VCtx(std::move(vctx))
, GInfo(std::move(info))
@@ -896,6 +906,7 @@ namespace NKikimr {
GInfo->PickTopology(),
locallyRecoveredState)
, Decision()
+ , ReadOnly(readOnly)
{}
};
@@ -905,9 +916,10 @@ namespace NKikimr {
TIntrusivePtr<TBlobStorageGroupInfo> info,
const TActorId &committerId,
const TActorId &notifyId,
- const NSyncer::TLocalSyncerState &localState) {
+ const NSyncer::TLocalSyncerState &localState,
+ bool readOnly) {
return new NSyncer::TVDiskGuidRecoveryActor(std::move(vctx), std::move(info), committerId,
- notifyId, localState);
+ notifyId, localState, readOnly);
}
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h
index 1b3be5a03a..823055d2c7 100644
--- a/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h
+++ b/ydb/core/blobstorage/vdisk/syncer/guid_recovery.h
@@ -144,6 +144,7 @@ namespace NKikimr {
TIntrusivePtr<TBlobStorageGroupInfo> info,
const TActorId &committerId,
const TActorId &notifyId,
- const NSyncer::TLocalSyncerState &localState);
+ const NSyncer::TLocalSyncerState &localState,
+ bool readOnly);
} // NKikimr
diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp
index 4fc8b06918..4d4ac187f1 100644
--- a/ydb/core/mind/bscontroller/bsc.cpp
+++ b/ydb/core/mind/bscontroller/bsc.cpp
@@ -329,6 +329,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
case NKikimrBlobStorage::TConfigRequest::TCommand::kWipeVDisk:
case NKikimrBlobStorage::TConfigRequest::TCommand::kSanitizeGroup:
case NKikimrBlobStorage::TConfigRequest::TCommand::kCancelVirtualGroup:
+ case NKikimrBlobStorage::TConfigRequest::TCommand::kPutVDiskToReadOnly:
return 2; // read-write commands go with higher priority as they are needed to keep cluster intact
case NKikimrBlobStorage::TConfigRequest::TCommand::kReadHostConfig:
diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
index 5433b599d9..071971b1bd 100644
--- a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
+++ b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
@@ -676,4 +676,27 @@ namespace NKikimr::NBsController {
}
}
+ void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& /*status*/) {
+ // first, find matching vslot
+ const TVSlotId& vslotId = cmd.GetVSlotId();
+ TVSlotInfo *vslot = VSlots.FindForUpdate(vslotId);
+ if (!vslot) {
+ throw TExVSlotNotFound(vslotId);
+ }
+
+ // second, validate vdisk id
+ const TVDiskID& vdiskId = VDiskIDFromVDiskID(cmd.GetVDiskId());
+ if (vslot->GetVDiskId() != vdiskId) {
+ throw TExVDiskIdIncorrect(vdiskId, vslotId);
+ }
+
+ TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId);
+ vslot->Mood = TMood::ReadOnly;
+ vslot->Status = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING;
+ vslot->DropFromVSlotReadyTimestampQ();
+ vslot->IsReady = false;
+ GroupFailureModelChanged.insert(group->ID);
+ group->CalculateGroupStatus();
+ }
+
} // NKikimr::NBsController
diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp
index f7fb07b54c..6500f6ed70 100644
--- a/ydb/core/mind/bscontroller/config.cpp
+++ b/ydb/core/mind/bscontroller/config.cpp
@@ -149,6 +149,10 @@ namespace NKikimr::NBsController {
item.SetDoWipe(true);
break;
+ case TMood::ReadOnly:
+ item.SetReadOnly(true);
+ break;
+
default:
Y_FAIL();
}
diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h
index 5ca88f2cb6..3caa46df3a 100644
--- a/ydb/core/mind/bscontroller/config.h
+++ b/ydb/core/mind/bscontroller/config.h
@@ -286,6 +286,7 @@ namespace NKikimr {
void ExecuteStep(const NKikimrBlobStorage::TWipeVDisk& cmd, TStatus& status);
void ExecuteStep(const NKikimrBlobStorage::TSanitizeGroup& cmd, TStatus& status);
void ExecuteStep(const NKikimrBlobStorage::TCancelVirtualGroup& cmd, TStatus& status);
+ void ExecuteStep(const NKikimrBlobStorage::TPutVDiskToReadOnly& cmd, TStatus& status);
};
} // NBsController
diff --git a/ydb/core/mind/bscontroller/config_cmd.cpp b/ydb/core/mind/bscontroller/config_cmd.cpp
index 5aa3f1c17e..f7197c8866 100644
--- a/ydb/core/mind/bscontroller/config_cmd.cpp
+++ b/ydb/core/mind/bscontroller/config_cmd.cpp
@@ -330,6 +330,7 @@ namespace NKikimr::NBsController {
HANDLE_COMMAND(WipeVDisk)
HANDLE_COMMAND(SanitizeGroup)
HANDLE_COMMAND(CancelVirtualGroup)
+ HANDLE_COMMAND(PutVDiskToReadOnly)
case NKikimrBlobStorage::TConfigRequest::TCommand::kAddMigrationPlan:
case NKikimrBlobStorage::TConfigRequest::TCommand::kDeleteMigrationPlan:
diff --git a/ydb/core/mind/bscontroller/mood.h b/ydb/core/mind/bscontroller/mood.h
index 9e2452f78c..68488ff603 100644
--- a/ydb/core/mind/bscontroller/mood.h
+++ b/ydb/core/mind/bscontroller/mood.h
@@ -10,6 +10,7 @@ struct TMood {
Wipe = 1,
Delete = 2,
Donor = 3,
+ ReadOnly = 4,
};
static TString Name(const EValue value) {
@@ -22,6 +23,8 @@ struct TMood {
return "Delete";
case Donor:
return "Donor";
+ case ReadOnly:
+ return "ReadOnly";
}
return Sprintf("Unknown%" PRIu64, (ui64)value);
}
diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp
index e043c91fdd..7425f9ff7a 100644
--- a/ydb/core/mind/bscontroller/register_node.cpp
+++ b/ydb/core/mind/bscontroller/register_node.cpp
@@ -427,8 +427,10 @@ void TBlobStorageController::ReadVSlot(const TVSlotInfo& vslot, TEvBlobStorage::
if (vslot.IsBeingDeleted()) {
vDisk->SetDoDestroy(true);
vDisk->SetEntityStatus(NKikimrBlobStorage::DESTROY);
- } else {
- vDisk->SetDoWipe(vslot.Mood == TMood::Wipe);
+ } else if (vslot.Mood == TMood::Wipe) {
+ vDisk->SetDoWipe(true);
+ } else if (vslot.Mood == TMood::ReadOnly) {
+ vDisk->SetReadOnly(true);
}
if (TGroupInfo *group = FindGroup(vslot.GroupId)) {
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index c962ede23f..0f6cfc8292 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -179,7 +179,7 @@ message TEvVPatchDiff {
optional uint32 ExpectedXorDiffs = 7;
optional bool ForceEnd = 8;
optional bool NotifyIfNotReady = 9;
-
+
optional NKikimrBlobStorage.EPutHandleClass HandleClass = 11;
optional TMsgQoS MsgQoS = 10;
@@ -1027,6 +1027,7 @@ message TNodeWardenServiceSet {
optional string StoragePoolName = 8;
optional TDonorMode DonorMode = 9; // vdisk runs in read-only limited mode without interaction with other disks of group
repeated TDonor Donors = 10; // a set of donor disks used to accelerate replication
+ optional bool ReadOnly = 11;
}
message TReplBrokerConfig {
diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto
index 30fda97db6..9b2580b68f 100644
--- a/ydb/core/protos/blobstorage_config.proto
+++ b/ydb/core/protos/blobstorage_config.proto
@@ -480,6 +480,11 @@ message TCancelVirtualGroup {
uint32 GroupId = 1; // id of a group we are going to cancel
}
+message TPutVDiskToReadOnly {
+ NKikimrBlobStorage.TVSlotId VSlotId = 1;
+ NKikimrBlobStorage.TVDiskID VDiskId = 2;
+}
+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// INTERFACE PART
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -522,6 +527,7 @@ message TConfigRequest {
TSanitizeGroup SanitizeGroup = 42;
TReadSettings ReadSettings = 43;
TCancelVirtualGroup CancelVirtualGroup = 44;
+ TPutVDiskToReadOnly PutVDiskToReadOnly = 45;
// commands intended for internal use
TReassignGroupDisk ReassignGroupDisk = 19;