diff options
author | Alexander Rutkovsky <alexander.rutkovsky@gmail.com> | 2022-06-28 11:52:51 +0300 |
---|---|---|
committer | Alexander Rutkovsky <alexander.rutkovsky@gmail.com> | 2022-06-28 11:52:51 +0300 |
commit | 3459dbb5a02f95837916a031c5357cb362f8a522 (patch) | |
tree | 732447819fbe1b1a90700c40d75e50a4b7b348f2 | |
parent | d0cf09e9ddedf9721115af488fb1cacd3e2df042 (diff) | |
download | ydb-3459dbb5a02f95837916a031c5357cb362f8a522.tar.gz |
Accept no writes in donor mode KIKIMR-14637
ref:432b6362cef45870d6bb55fdecaca0b68193f397
-rw-r--r-- | CMakeLists.darwin.txt | 1 | ||||
-rw-r--r-- | CMakeLists.linux.txt | 1 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt | 1 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt | 1 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/donor.cpp | 168 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.darwin.txt | 43 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.linux.txt | 47 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.txt | 13 | ||||
-rw-r--r-- | ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp | 43 |
9 files changed, 310 insertions, 8 deletions
diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt index 2806d3da7a..040aabe401 100644 --- a/CMakeLists.darwin.txt +++ b/CMakeLists.darwin.txt @@ -1019,6 +1019,7 @@ add_subdirectory(ydb/services/ydb/ut) add_subdirectory(ydb/public/sdk/cpp/client/ydb_extension) add_subdirectory(ydb/services/yq/ut_integration) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_blob_depot) +add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_donor) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_group_reconfiguration) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_osiris) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_replication) diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt index d3d40e4aee..71aa93dc49 100644 --- a/CMakeLists.linux.txt +++ b/CMakeLists.linux.txt @@ -1115,6 +1115,7 @@ add_subdirectory(ydb/services/ydb/ut) add_subdirectory(ydb/public/sdk/cpp/client/ydb_extension) add_subdirectory(ydb/services/yq/ut_integration) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_blob_depot) +add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_donor) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_group_reconfiguration) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_osiris) add_subdirectory(ydb/core/blobstorage/ut_blobstorage/ut_replication) diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt index 4af5291bff..20737cdd49 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt @@ -33,7 +33,6 @@ target_sources(ydb-core-blobstorage-ut_blobstorage PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/decommit_3dc.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/defrag.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/donor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/encryption.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/gc_quorum_3dc.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/incorrect_queries.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt index e0fec7255c..c29b009099 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt @@ -37,7 +37,6 @@ target_sources(ydb-core-blobstorage-ut_blobstorage PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/counting_events.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/decommit_3dc.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/defrag.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/donor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/encryption.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/gc_quorum_3dc.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/incorrect_queries.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/donor.cpp b/ydb/core/blobstorage/ut_blobstorage/donor.cpp index 4d0e2edb0f..40c02ace76 100644 --- a/ydb/core/blobstorage/ut_blobstorage/donor.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/donor.cpp @@ -83,4 +83,172 @@ Y_UNIT_TEST_SUITE(Donor) { UNIT_ASSERT(found); } + Y_UNIT_TEST(ConsistentWritesWhenSwitchingToDonorMode) { + TEnvironmentSetup env{{ + .NodeCount = 9, + .Erasure = TBlobStorageGroupType::Erasure4Plus2Block, + }}; + auto& runtime = *env.Runtime; + + env.EnableDonorMode(); + env.CreateBoxAndPool(1, 1); + env.Sim(TDuration::Seconds(20)); + auto groups = env.GetGroups(); + UNIT_ASSERT_VALUES_EQUAL(groups.size(), 1); + const ui32 groupId = groups.front(); + + class TWriterActor : public TActorBootstrapped<TWriterActor> { + public: + const ui32 GroupId; + bool *Stopped; + THashSet<TLogoBlobID> Data; + std::pair<ui32, ui32> CurrentBarrier; + ui64 TabletId = 1; + ui32 CurrentGeneration = 0; + ui32 CurrentStep = 0; + ui32 PutsInFlight = 0; + + public: + TWriterActor(ui32 groupId, bool *stopped) + : GroupId(groupId) + , Stopped(stopped) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + ++CurrentGeneration; + CurrentStep = 1; + Data.clear(); + SendToBSProxy(SelfId(), GroupId, new TEvBlobStorage::TEvCollectGarbage(TabletId, CurrentGeneration, 0, 0, + true, CurrentGeneration - 1, Max<ui32>(), nullptr, nullptr, TInstant::Max(), false)); + } + + void Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr ev) { + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Status, NKikimrProto::OK); + for (ui32 i = 0; i < 2; ++i) { + IssuePut(); + } + } + + void IssuePut() { + if (Data.size() == 10000) { + if (!PutsInFlight) { + *Stopped = true; + } + return; + } + + ui32 len = 1 + RandomNumber(100u); + TString data = TString::Uninitialized(len); + char *p = data.Detach(); + char *end = p + len; + TReallyFastRng32 rng(RandomNumber<ui64>()); + while (p + sizeof(ui32) <= end) { + *reinterpret_cast<ui32*>(p) = rng(); + p += sizeof(ui32); + } + for (; p != end; ++p) { + *p = rng(); + } + + const TLogoBlobID id(TabletId, CurrentGeneration, CurrentStep, 0, len, 0); + SendToBSProxy(SelfId(), GroupId, new TEvBlobStorage::TEvPut(id, data, TInstant::Max())); + Cerr << "Put# " << id << Endl; + Data.emplace(id); + + ++CurrentStep; + ++PutsInFlight; + } + + void Handle(TEvBlobStorage::TEvPutResult::TPtr ev) { + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Status, NKikimrProto::OK); + --PutsInFlight; + Schedule(TDuration::MicroSeconds(RandomNumber(1000u)), new TEvents::TEvWakeup); + } + + STRICT_STFUNC(StateFunc, + cFunc(TEvents::TSystem::Bootstrap, Bootstrap); + hFunc(TEvBlobStorage::TEvCollectGarbageResult, Handle); + hFunc(TEvBlobStorage::TEvPutResult, Handle); + cFunc(TEvents::TSystem::Wakeup, IssuePut); + ) + }; + + bool stopped = false; + bool resumePending = false; + TWriterActor *writer = new TWriterActor(groupId, &stopped); + const TActorId writerId = runtime.Register(writer, 1); + const TActorId edge = runtime.AllocateEdgeActor(1, __FILE__, __LINE__); + + for (THPTimer timer; TDuration::Seconds(timer.Passed()) <= TDuration::Minutes(3); ) { + NKikimrBlobStorage::TConfigRequest request; + request.AddCommand()->MutableQueryBaseConfig(); + auto response = env.Invoke(request); + UNIT_ASSERT(response.GetSuccess()); + UNIT_ASSERT_VALUES_EQUAL(response.StatusSize(), 1); + auto& config = response.GetStatus(0).GetBaseConfig(); + + bool allReady = true; + for (const auto& vslot : config.GetVSlot()) { + if (!vslot.GetReady()) { + allReady = false; + break; + } + } + + if (!allReady) { + env.Sim(TDuration::Seconds(1)); + continue; + } + + if (stopped) { + auto info = env.GetGroupInfo(groupId); + THashMap<TLogoBlobID, ui32> parts; + + for (ui32 i = 0; i < info->GetTotalVDisksNum(); ++i) { + TActorId queueId = env.CreateQueueActor(info->GetVDiskId(i), NKikimrBlobStorage::GetFastRead, 1000); + for (const auto& id : writer->Data) { + auto ev = TEvBlobStorage::TEvVGet::CreateExtremeDataQuery(info->GetVDiskId(i), TInstant::Max(), + NKikimrBlobStorage::FastRead, {}, {}, {id}); + runtime.Send(new IEventHandle(queueId, edge, ev.release()), queueId.NodeId()); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvVGetResult>(edge, false); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Record.GetStatus(), NKikimrProto::OK); + for (const auto& item : res->Get()->Record.GetResult()) { + if (item.GetStatus() == NKikimrProto::OK) { + ++parts[id]; + break; + } + } + } + runtime.Send(new IEventHandle(TEvents::TSystem::Poison, 0, queueId, {}, nullptr, 0), queueId.NodeId()); + } + for (const auto& id : writer->Data) { + UNIT_ASSERT(parts[id] >= 6); + } + + stopped = false; + resumePending = true; + } else { + if (resumePending) { + runtime.Send(new IEventHandle(TEvents::TSystem::Bootstrap, 0, writerId, {}, nullptr, 0), 1); + resumePending = false; + env.Sim(TDuration::MilliSeconds(RandomNumber(1000u))); + } + + const ui32 index = RandomNumber(config.VSlotSize()); + const auto& vslot = config.GetVSlot(index); + Cerr << "Reassign# " << index << " -- " << SingleLineProto(vslot) << Endl; + NKikimrBlobStorage::TConfigRequest request; + auto *cmd = request.AddCommand()->MutableReassignGroupDisk(); + cmd->SetGroupId(vslot.GetGroupId()); + cmd->SetGroupGeneration(vslot.GetGroupGeneration()); + cmd->SetFailRealmIdx(vslot.GetFailRealmIdx()); + cmd->SetFailDomainIdx(vslot.GetFailDomainIdx()); + cmd->SetVDiskIdx(vslot.GetVDiskIdx()); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + } + } + } diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.darwin.txt b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.darwin.txt new file mode 100644 index 0000000000..c018587a3b --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.darwin.txt @@ -0,0 +1,43 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-blobstorage-ut_blobstorage-ut_donor) +target_include_directories(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ydb-core-blobstorage-ut_blobstorage-ut_donor PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_link_options(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/donor.cpp +) +add_test( + NAME + ydb-core-blobstorage-ut_blobstorage-ut_donor + COMMAND + ydb-core-blobstorage-ut_blobstorage-ut_donor + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-blobstorage-ut_blobstorage-ut_donor) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.linux.txt b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.linux.txt new file mode 100644 index 0000000000..00ec75cbff --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.linux.txt @@ -0,0 +1,47 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-blobstorage-ut_blobstorage-ut_donor) +target_include_directories(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage +) +target_link_libraries(ydb-core-blobstorage-ut_blobstorage-ut_donor PUBLIC + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + blobstorage-ut_blobstorage-lib +) +target_link_options(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-blobstorage-ut_blobstorage-ut_donor PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/donor.cpp +) +add_test( + NAME + ydb-core-blobstorage-ut_blobstorage-ut_donor + COMMAND + ydb-core-blobstorage-ut_blobstorage-ut_donor + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-blobstorage-ut_blobstorage-ut_donor) diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.txt b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.txt new file mode 100644 index 0000000000..fc7b1ee73c --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/ut_donor/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (APPLE) + include(CMakeLists.darwin.txt) +elseif (UNIX AND NOT APPLE) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp index 54b755bf77..a5f5483bbd 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp @@ -96,6 +96,15 @@ namespace NKikimr { // Some stuff to handle a case when we can't accept TEvVPut requests // because of (fresh) compaction overload //////////////////////////////////////////////////////////////////////// + template<typename TEvent> + bool DonorCheck(TAutoPtr<TEventHandle<TEvent>>& ev, const TActorContext& ctx) { + if (Config->BaseInfo.DonorMode) { + ReplyError(NKikimrProto::ERROR, "disk is in donor mode", ev, ctx, TAppData::TimeProvider->Now()); + return false; + } + return true; + } + void ProcessPostponedEvents(const TActorContext &ctx, bool actualizeLevels) { if (OverloadHandler) { // we perform postponed events processing in batch to prioritize emergency @@ -146,6 +155,9 @@ namespace NKikimr { //////////////////////////////////////////////////////////////////////// void Handle(TEvBlobStorage::TEvVMovedPatch::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } const bool postpone = OverloadHandler->PostponeEvent(ev); if (!postpone) { PrivateHandle(ev, ctx); @@ -178,14 +190,18 @@ namespace NKikimr { } template <typename TEvPtr> - void ReplyVPatchError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvPtr &ev) { + void ReplyError(NKikimrProto::EReplyStatus status, const TString& errorReason, TEvPtr &ev, const TActorContext& ctx, + TInstant now) { using namespace NErrBuilder; - std::unique_ptr<IEventBase> res = ErroneousResult(VCtx, status, errorReason, ev, TActivationContext::Now(), + std::unique_ptr<IEventBase> res = ErroneousResult(VCtx, status, errorReason, ev, now, SkeletonFrontIDPtr, SelfVDiskId, Db->GetVDiskIncarnationGuid(), GInfo); - SendReply(TActivationContext::AsActorContext(), std::move(res), ev, BS_VDISK_PATCH); + SendReply(ctx, std::move(res), ev, BS_VDISK_PATCH); } void Handle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } const bool postpone = OverloadHandler->PostponeEvent(ev); if (!postpone) { PrivateHandle(ev, ctx); @@ -195,14 +211,14 @@ namespace NKikimr { void PrivateHandle(TEvBlobStorage::TEvVPatchStart::TPtr &ev, const TActorContext &ctx) { TInstant now = ctx.Now(); if (!EnableVPatch.Update(now)) { - ReplyVPatchError(NKikimrProto::ERROR, "VPatch is disabled", ev); + ReplyError(NKikimrProto::ERROR, "VPatch is disabled", ev, ctx, TAppData::TimeProvider->Now()); return; } TLogoBlobID patchedBlobId = LogoBlobIDFromLogoBlobID(ev->Get()->Record.GetPatchedBlobId()); if (VPatchActors.count(patchedBlobId)) { - ReplyVPatchError(NKikimrProto::ERROR, "The patching request already is running", ev); + ReplyError(NKikimrProto::ERROR, "The patching request already is running", ev, ctx, TAppData::TimeProvider->Now()); return; } @@ -219,6 +235,9 @@ namespace NKikimr { template <typename TEvDiffPtr> void HandleVPatchDiffResending(TEvDiffPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } if constexpr (std::is_same_v<TEvDiffPtr, TEvBlobStorage::TEvVPatchDiff::TPtr>) { LOG_DEBUG_S(ctx, BS_VDISK_PATCH, VCtx->VDiskLogPrefix << "TEvVPatch: recieve diff;" << " Event# " << ev->Get()->ToString()); @@ -234,7 +253,7 @@ namespace NKikimr { if (it != VPatchActors.end()) { TActivationContext::Send(ev->Forward(it->second)); } else { - ReplyVPatchError(NKikimrProto::ERROR, "VPatchActor doesn't exist", ev); + ReplyError(NKikimrProto::ERROR, "VPatchActor doesn't exist", ev, ctx, TAppData::TimeProvider->Now()); } } @@ -266,6 +285,9 @@ namespace NKikimr { } void Handle(TEvBlobStorage::TEvVMultiPut::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } const bool postpone = OverloadHandler->PostponeEvent(ev); if (!postpone) { PrivateHandle(ev, ctx); @@ -609,6 +631,9 @@ namespace NKikimr { } void Handle(TEvBlobStorage::TEvVPut::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } const bool postpone = OverloadHandler->PostponeEvent(ev); if (!postpone) { PrivateHandle(ev, ctx); @@ -883,6 +908,9 @@ namespace NKikimr { } void Handle(TEvBlobStorage::TEvVBlock::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } ++IFaceMonGroup->BlockMsgs(); TInstant now = TAppData::TimeProvider->Now(); NKikimrBlobStorage::TEvVBlock &record = ev->Get()->Record; @@ -997,6 +1025,9 @@ namespace NKikimr { } void Handle(TEvBlobStorage::TEvVCollectGarbage::TPtr &ev, const TActorContext &ctx) { + if (!DonorCheck(ev, ctx)) { + return; + } IFaceMonGroup->GCMsgs()++; TInstant now = TAppData::TimeProvider->Now(); NKikimrBlobStorage::TEvVCollectGarbage &record = ev->Get()->Record; |