rewrite check blob integrity logic; add tests (#18424)

10 files changed, 490 insertions, 36 deletions

diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
index 4fe233f98bd..598705b5a91 100644
--- a/ydb/core/base/blobstorage.h
+++ b/ydb/core/base/blobstorage.h
@@ -1463,10 +1463,12 @@ struct TEvBlobStorage {
         TString ErrorReason;
 
         enum EPlacementStatus {
-            PS_OK = 1,      // blob parts are placed according to fail model
-            PS_ERROR = 2,   // blob parts are definitely placed incorrectly or there are missing parts for sure
-            PS_UNKNOWN = 3, // status is unknown because of missing disks or network problems
-            PS_NOT_YET = 4, // there are missing parts but status may become OK after replication
+            PS_OK = 1,          // blob parts are placed according to fail model
+            PS_ERROR = 2,       // blob is lost/unrecoverable
+            PS_UNKNOWN = 3,     // status is unknown because of missing disks or network problems
+            PS_NOT_YET = 4,     // there are missing parts but status may become OK after replication
+            PS_RECOVERABLE = 5, // blob parts are definitely placed incorrectly or there are missing parts
+                                // but blob may be recovered
         };
         EPlacementStatus PlacementStatus;
 
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp
index ec6eb5268b3..19348bd1b26 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp
@@ -6,15 +6,17 @@
 
 namespace NKikimr {
 
-
 class TBlobStorageGroupCheckIntegrityRequest : public TBlobStorageGroupRequestActor {
     const TLogoBlobID Id;
     const TInstant Deadline;
     const NKikimrBlobStorage::EGetHandleClass GetHandleClass;
 
     TGroupQuorumTracker QuorumTracker;
-    std::unique_ptr<TBlobStatusTracker> BlobStatus; // treats NOT_YET as ERROR
-    std::unique_ptr<TBlobStatusTracker> BlobStatusOptimistic; // treats NOT_YET as currently replicating -> OK in the future
+
+    TSubgroupPartLayout PartLayout;
+    TSubgroupPartLayout PartLayoutWithNotYet;
+
+    bool HasErrorDisks = false;
 
     ui32 VGetsInFlight = 0;
 
@@ -33,26 +35,61 @@ class TBlobStorageGroupCheckIntegrityRequest : public TBlobStorageGroupRequestAc
         SendResponseAndDie(std::move(PendingResult));
     }
 
-    TString DumpBlobStatus() const {
+    TString DumpLayout() const {
         TStringStream str;
-        BlobStatus->Output(str, Info.Get());
+        PartLayout.Output(str, Info->Type);
         return str.Str();
     }
 
+    void UpdateFromResponseData(const NKikimrBlobStorage::TQueryResult& result, const TVDiskID& vDiskId) {
+        if (!result.HasBlobID()) {
+            return;
+        }
+        const TLogoBlobID id = LogoBlobIDFromLogoBlobID(result.GetBlobID());
+        if (id.FullID() != Id) {
+            return;
+        }
+        if (!result.HasStatus()) {
+            return;
+        }
+        const NKikimrProto::EReplyStatus status = result.GetStatus();
+
+        ui32 nodeId = Info->GetTopology().GetIdxInSubgroup(vDiskId, Id.Hash());
+        const ui32 partId = id.PartId();
+
+        if (!partId) {
+            return;
+        }
+
+        switch (status) {
+            case NKikimrProto::OK:
+                PartLayout.AddItem(nodeId, partId - 1, Info->Type);
+                PartLayoutWithNotYet.AddItem(nodeId, partId - 1, Info->Type);
+                break;
+
+            case NKikimrProto::NOT_YET:
+                PartLayoutWithNotYet.AddItem(nodeId, partId - 1, Info->Type);
+                break;
+
+            default:
+                break;
+        }
+    }
+
     void Handle(TEvBlobStorage::TEvVGetResult::TPtr &ev) {
         ProcessReplyFromQueue(ev->Get());
 
         const NKikimrBlobStorage::TEvVGetResult& record = ev->Get()->Record;
 
         if (!record.HasStatus()) {
-            ErrorReason = "erron in TEvVGetResult - no status";
+            ErrorReason = "error in TEvVGetResult - no status";
             ReplyAndDie(NKikimrProto::ERROR);
             return;
         }
         NKikimrProto::EReplyStatus status = record.GetStatus();
 
         if (!record.HasVDiskID()) {
-            ErrorReason = "erron in TEvVGetResult - no VDisk id";
+            ErrorReason = "error in TEvVGetResult - no VDisk id";
             ReplyAndDie(NKikimrProto::ERROR);
             return;
         }
@@ -75,17 +112,12 @@ class TBlobStorageGroupCheckIntegrityRequest : public TBlobStorageGroupRequestAc
                 Y_ABORT("unexpected newStatus# %s", NKikimrProto::EReplyStatus_Name(newStatus).data());
         }
 
-        for (size_t i = 0; i < record.ResultSize(); ++i) {
-            const auto& result = record.GetResult(i);
-            BlobStatus->UpdateFromResponseData(result, vDiskId, Info.Get());
-
-            if (result.GetStatus() == NKikimrProto::NOT_YET) {
-                NKikimrBlobStorage::TQueryResult okResult = result;
-                okResult.SetStatus(NKikimrProto::OK);
-                BlobStatusOptimistic->UpdateFromResponseData(okResult, vDiskId, Info.Get());
-            } else {
-                BlobStatusOptimistic->UpdateFromResponseData(result, vDiskId, Info.Get());
+        if (status == NKikimrProto::OK) {
+            for (size_t i = 0; i < record.ResultSize(); ++i) {
+                UpdateFromResponseData(record.GetResult(i), vDiskId);
             }
+        } else {
+            HasErrorDisks = true;
         }
 
         if (!VGetsInFlight) {
@@ -94,11 +126,15 @@ class TBlobStorageGroupCheckIntegrityRequest : public TBlobStorageGroupRequestAc
     }
 
     void Analyze() {
-        PendingResult.reset(new TEvBlobStorage::TEvCheckIntegrityResult(NKikimrProto::OK));
+        PendingResult.reset(new TEvCheckIntegrityResult(NKikimrProto::OK));
         PendingResult->Id = Id;
         PendingResult->DataStatus = TEvCheckIntegrityResult::DS_UNKNOWN; // TODO
 
-        TBlobStorageGroupInfo::EBlobState state = BlobStatus->GetBlobState(Info.Get(), nullptr);
+        TBlobStorageGroupInfo::TSubgroupVDisks faultyDisks(&Info->GetTopology()); // empty set
+
+        const auto& checker = Info->GetQuorumChecker();
+        TBlobStorageGroupInfo::EBlobState state = checker.GetBlobStateWithoutLayoutCheck(
+            PartLayout, faultyDisks);
 
         switch (state) {
             case TBlobStorageGroupInfo::EBS_DISINTEGRATED:
@@ -110,24 +146,23 @@ class TBlobStorageGroupCheckIntegrityRequest : public TBlobStorageGroupRequestAc
                 PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_OK;
                 break;
 
+            case TBlobStorageGroupInfo::EBS_RECOVERABLE_FRAGMENTARY:
             case TBlobStorageGroupInfo::EBS_UNRECOVERABLE_FRAGMENTARY:
-                PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_ERROR;
-                break;
+            case TBlobStorageGroupInfo::EBS_RECOVERABLE_DOUBTED: {
+                TBlobStorageGroupInfo::EBlobState stateNotYet = checker.GetBlobStateWithoutLayoutCheck(
+                    PartLayoutWithNotYet, faultyDisks);
 
-            case TBlobStorageGroupInfo::EBS_RECOVERABLE_FRAGMENTARY: {
-                TBlobStorageGroupInfo::EBlobState stateOptimistic =
-                    BlobStatusOptimistic->GetBlobState(Info.Get(), nullptr);
-
-                if (stateOptimistic == TBlobStorageGroupInfo::EBS_FULL) {
+                if (stateNotYet == TBlobStorageGroupInfo::EBS_FULL) {
                     PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_NOT_YET;
+                } else if (state == TBlobStorageGroupInfo::EBS_RECOVERABLE_FRAGMENTARY) {
+                    PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_RECOVERABLE;
+                } else if (HasErrorDisks) {
+                    PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_UNKNOWN;
                 } else {
                     PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_ERROR;
                 }
                 break;
             }
-            case TBlobStorageGroupInfo::EBS_RECOVERABLE_DOUBTED:
-                PendingResult->PlacementStatus = TEvCheckIntegrityResult::PS_UNKNOWN;
-                break;
         }
 
         ReplyAndDie(NKikimrProto::OK);
@@ -160,9 +195,6 @@ public:
     {}
 
     void Bootstrap() override {
-        BlobStatus.reset(new TBlobStatusTracker(Id, Info.Get()));
-        BlobStatusOptimistic.reset(new TBlobStatusTracker(Id, Info.Get()));
-
         for (const auto& vdisk : Info->GetVDisks()) {
             auto vDiskId = Info->GetVDiskId(vdisk.OrderNumber);
 
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
index 50101eacec5..10cd1b53c2d 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
@@ -879,6 +879,7 @@ namespace NKikimr {
             XX(Status)
             XX(Patch)
             XX(Assimilate)
+            XX(CheckIntegrity)
             default:
                 Y_ABORT();
 #undef XX
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp
index 76bf341ade8..861dcf53a18 100644
--- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp
+++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp
@@ -87,6 +87,27 @@ public:
         }
     }
 
+    TBlobStorageGroupInfo::EBlobState GetBlobStateWithoutLayoutCheck(const TSubgroupPartLayout& parts,
+            const TBlobStorageGroupInfo::TSubgroupVDisks& failedDisks) const override {
+        if (!CheckFailModelForSubgroup(failedDisks)) {
+            return TBlobStorageGroupInfo::EBS_DISINTEGRATED;
+        }
+
+        const TBlobStorageGroupType type = Top->GType;
+        ui32 effectiveReplicas = parts.CountEffectiveReplicas(type);
+        auto state = Top->BlobState(effectiveReplicas, failedDisks.GetNumSetItems());
+        if (state == TBlobStorageGroupInfo::EBS_FULL) {
+            return state;
+        }
+
+        ui32 distinctParts = parts.CountDistinctParts(type);
+        if (distinctParts < type.MinimalRestorablePartCount()) {
+            return TBlobStorageGroupInfo::EBS_UNRECOVERABLE_FRAGMENTARY;
+        }
+
+        return TBlobStorageGroupInfo::EBS_RECOVERABLE_FRAGMENTARY;
+    }
+
     ui32 GetPartsToResurrect(const TSubgroupPartLayout& parts, ui32 idxInSubgroup) const override {
         const TBlobStorageGroupType type = Top->GType;
         const ui32 effectiveReplicas = parts.CountEffectiveReplicas(Top->GType);
@@ -162,6 +183,11 @@ public:
         }
     }
 
+    TBlobStorageGroupInfo::EBlobState GetBlobStateWithoutLayoutCheck(const TSubgroupPartLayout& parts,
+            const TBlobStorageGroupInfo::TSubgroupVDisks& failedDisks) const override {
+        return GetBlobState(parts, failedDisks);
+    }
+
     ui32 GetPartsToResurrect(const TSubgroupPartLayout& parts, ui32 idxInSubgroup) const override {
         auto [data, any] = parts.GetMirror3of4State();
         if (!data) {
@@ -275,6 +301,11 @@ quitIter:   ;
         }
     }
 
+    TBlobStorageGroupInfo::EBlobState GetBlobStateWithoutLayoutCheck(const TSubgroupPartLayout& parts,
+            const TBlobStorageGroupInfo::TSubgroupVDisks& failedDisks) const override {
+        return GetBlobState(parts, failedDisks);
+    }
+
     ui32 GetPartsToResurrect(const TSubgroupPartLayout& parts, ui32 idxInSubgroup) const override {
         const TBlobStorageGroupInfo::TSubgroupVDisks& disksWithReplica = parts.GetInvolvedDisks(Top);
         const ui32 myRing = idxInSubgroup % 3;
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h
index d26ec1895f0..44069cc1c2c 100644
--- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h
+++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h
@@ -146,6 +146,10 @@ public:
 
         virtual EBlobState GetBlobState(const TSubgroupPartLayout& parts, const TSubgroupVDisks& failedDisks) const = 0;
 
+        // check recoverability of the blob based only on presense of different parts without checking the layout
+        virtual EBlobState GetBlobStateWithoutLayoutCheck(const TSubgroupPartLayout& parts,
+                const TSubgroupVDisks& failedDisks) const = 0;
+
         // check if we need to resurrect something; returns bit mask of parts needed for specified disk in group,
         // nth bit represents nth part; all returned parts are suitable for this particular disk
         virtual ui32 GetPartsToResurrect(const TSubgroupPartLayout& parts, ui32 idxInSubgroup) const = 0;
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h
index f92b7e789a6..2b4990bfe36 100644
--- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h
+++ b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout.h
@@ -51,6 +51,18 @@ namespace NKikimr {
         // Return a set of subgroup's disk contaning any replicas
         TBlobStorageGroupInfo::TSubgroupVDisks GetInvolvedDisks(const TBlobStorageGroupInfo::TTopology *top) const;
 
+        ui32 CountDistinctParts(const TBlobStorageGroupType& gtype) const {
+            const ui32 totalPartCount = gtype.TotalPartCount();
+            ui32 count = 0;
+            for (ui32 partIdx = 0; partIdx < totalPartCount; ++partIdx) {
+                ui32 mask = GetDisksWithPart(partIdx);
+                if (mask) {
+                    ++count;
+                }
+            }
+            return count;
+        }
+
         void Output(IOutputStream& str, const TBlobStorageGroupType &gtype) const {
             const ui32 totalPartCount = gtype.TotalPartCount();
             str << "{";
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
index acd8ec1d0aa..495526c91bf 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
@@ -93,6 +93,7 @@ STATEFN(TNodeWarden::StateOnline) {
         fFunc(TEvBlobStorage::TEvStatus::EventType, HandleForwarded);
         fFunc(TEvBlobStorage::TEvAssimilate::EventType, HandleForwarded);
         fFunc(TEvBlobStorage::TEvBunchOfEvents::EventType, HandleForwarded);
+        fFunc(TEvBlobStorage::TEvCheckIntegrity::EventType, HandleForwarded);
         fFunc(TEvRequestProxySessionsState::EventType, HandleForwarded);
 
         cFunc(TEvPrivate::EvGroupPendingQueueTick, HandleGroupPendingQueueTick);
diff --git a/ydb/core/blobstorage/ut_blobstorage/check_integrity.cpp b/ydb/core/blobstorage/ut_blobstorage/check_integrity.cpp
new file mode 100644
index 00000000000..705cd2b82ba
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/check_integrity.cpp
@@ -0,0 +1,354 @@
+#include <ydb/core/blobstorage/ut_blobstorage/lib/env.h>
+
+struct TCheckIntegrityEnvBase {
+    TEnvironmentSetup Env;
+    TIntrusivePtr<TBlobStorageGroupInfo> Info;
+    TLogoBlobID Id;
+    std::vector<TVDiskID> VDisks;
+
+    std::unique_ptr<IEventHandle> Result;
+
+    TCheckIntegrityEnvBase(TEnvironmentSetup::TSettings&& settings)
+        : Env(std::move(settings))
+    {
+        Env.CreateBoxAndPool(1, 1);
+        Env.Sim(TDuration::Minutes(1));
+
+        auto groups = Env.GetGroups();
+        UNIT_ASSERT(groups.size() == 1);
+        Info = Env.GetGroupInfo(groups.front());
+
+        TString error;
+        const bool success = TLogoBlobID::Parse(Id, "[72075186270680851:57:3905:6:786432:4194304:0]", error);
+        UNIT_ASSERT(success);
+    }
+
+    TEvBlobStorage::TEvCheckIntegrityResult* Request() {
+        const auto edge = Env.Runtime->AllocateEdgeActor(1, __FILE__, __LINE__);
+
+        Env.Runtime->WrapInActorContext(edge, [&] {
+            SendToBSProxy(edge, Info->GroupID, new TEvBlobStorage::TEvCheckIntegrity
+                (Id, TInstant::Max(), NKikimrBlobStorage::EGetHandleClass::FastRead));
+        });
+
+        Result.reset(Env.WaitForEdgeActorEvent<TEvBlobStorage::TEvCheckIntegrityResult>(edge).Release());
+        return Result->Get<TEvBlobStorage::TEvCheckIntegrityResult>();
+    }
+
+    bool InjectError(
+        NKikimrProto::EReplyStatus status,
+        const THashSet<TVDiskID>& errorDisks,
+        std::unique_ptr<IEventHandle>& ev)
+    {
+        if (ev->GetTypeRewrite() == TEvBlobStorage::EvVGet) {
+            auto* msg = ev->Get<TEvBlobStorage::TEvVGet>();
+            auto vDiskId = VDiskIDFromVDiskID(msg->Record.GetVDiskID());
+
+            if (!errorDisks.contains(vDiskId)) {
+                return true;
+            }
+
+            auto result = std::make_unique<TEvBlobStorage::TEvVGetResult>();
+            auto& record = result->Record;
+            record.SetStatus(status);
+
+            VDiskIDFromVDiskID(vDiskId, record.MutableVDiskID());
+
+            Env.Runtime->Send(
+                new IEventHandle(ev->Sender, ev->Recipient, result.release(), 0, ev->Cookie),
+                ev->Sender.NodeId());
+            return false;
+        }
+        return true;
+    }
+};
+
+struct TCheckIntegrityEnvBlock42 : public TCheckIntegrityEnvBase {
+    std::vector<TString> Parts;
+
+    TCheckIntegrityEnvBlock42()
+        : TCheckIntegrityEnvBase(TEnvironmentSetup::TSettings{
+            .NodeCount = 8,
+            .Erasure = TBlobStorageGroupType::Erasure4Plus2Block,
+        })
+    {
+        TString data = TString(Id.BlobSize(), 'X');
+
+        TDataPartSet partSet;
+        Info->Type.SplitData((TErasureType::ECrcMode)Id.CrcMode(), data, partSet);
+        for (ui32 i = 0; i < partSet.Parts.size(); ++i) {
+            Parts.push_back(partSet.Parts[i].OwnedString.ConvertToString());
+        }
+
+        for (ui32 i = 0; i < 8; ++i) {
+            auto vDiskIdShort = Info->GetTopology().GetVDiskInSubgroup(i, Id.Hash());
+            VDisks.push_back(Info->CreateVDiskID(vDiskIdShort));
+        }
+    }
+};
+
+struct TCheckIntegrityEnvMirror3dc : public TCheckIntegrityEnvBase {
+    TString Data;
+
+    TCheckIntegrityEnvMirror3dc()
+        : TCheckIntegrityEnvBase(TEnvironmentSetup::TSettings{
+            .NodeCount = 9,
+            .Erasure = TBlobStorageGroupType::ErasureMirror3dc,
+        })
+    {
+        Data = TString(Id.BlobSize(), 'X');
+
+        for (ui32 i = 0; i < 9; ++i) {
+            auto vDiskIdShort = Info->GetTopology().GetVDiskInSubgroup(i, Id.Hash());
+            VDisks.push_back(Info->CreateVDiskID(vDiskIdShort));
+        }
+    }
+};
+
+Y_UNIT_TEST_SUITE(CheckIntegrityBlock42) {
+
+    Y_UNIT_TEST(PlacementOk) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementOkHandoff) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 2; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+        check.Env.PutBlob(check.VDisks[6], TLogoBlobID(check.Id, 1), check.Parts[0]);
+        check.Env.PutBlob(check.VDisks[7], TLogoBlobID(check.Id, 2), check.Parts[1]);
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementMissingParts) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 2; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+
+    Y_UNIT_TEST(PlacementBlobIsLost) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_ERROR);
+    }
+
+    Y_UNIT_TEST(PlacementWrongDisks) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 2; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+        check.Env.PutBlob(check.VDisks[6], TLogoBlobID(check.Id, 1), check.Parts[0]);
+        check.Env.PutBlob(check.VDisks[6], TLogoBlobID(check.Id, 2), check.Parts[1]);
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+
+    Y_UNIT_TEST(PlacementAllOnHandoff) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[6], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+
+    Y_UNIT_TEST(PlacementDisintegrated) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        errorDisks.insert(check.VDisks[5]);
+        errorDisks.insert(check.VDisks[6]);
+        errorDisks.insert(check.VDisks[7]);
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::ERROR);
+        Cerr << result->ErrorReason << Endl;
+    }
+
+    Y_UNIT_TEST(PlacementOkWithErrors) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        errorDisks.insert(check.VDisks[6]);
+        errorDisks.insert(check.VDisks[7]);
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementWithErrorsOnBlobDisks) {
+        TCheckIntegrityEnvBlock42 check;
+
+        for (ui32 i = 0; i < 6; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Parts[i]);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        errorDisks.insert(check.VDisks[0]);
+        errorDisks.insert(check.VDisks[1]);
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+
+}
+
+Y_UNIT_TEST_SUITE(CheckIntegrityMirror3dc) {
+
+    Y_UNIT_TEST(PlacementOk) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Data);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementOkHandoff) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i + 3], TLogoBlobID(check.Id, i + 1), check.Data);
+        }
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementMissingParts) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        check.Env.PutBlob(check.VDisks[0], TLogoBlobID(check.Id, 1), check.Data);
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+
+    Y_UNIT_TEST(PlacementBlobIsLost) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_ERROR);
+    }
+
+    Y_UNIT_TEST(PlacementDisintegrated) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Data);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        for (ui32 i = 4; i < 9; ++i) {
+            errorDisks.insert(check.VDisks[i]);
+        }
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::ERROR);
+        Cerr << result->ErrorReason << Endl;
+    }
+
+    Y_UNIT_TEST(PlacementOkWithErrors) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Data);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        errorDisks.insert(check.VDisks[5]);
+        errorDisks.insert(check.VDisks[7]);
+        errorDisks.insert(check.VDisks[8]);
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_OK);
+    }
+
+    Y_UNIT_TEST(PlacementOkWithErrorsOnBlobDisks) {
+        TCheckIntegrityEnvMirror3dc check;
+
+        for (ui32 i = 0; i < 3; ++i) {
+            check.Env.PutBlob(check.VDisks[i], TLogoBlobID(check.Id, i + 1), check.Data);
+        }
+
+        THashSet<TVDiskID> errorDisks;
+        errorDisks.insert(check.VDisks[0]);
+        errorDisks.insert(check.VDisks[1]);
+
+        check.Env.Runtime->FilterFunction = [&](ui32, std::unique_ptr<IEventHandle>& ev) {
+            return check.InjectError(NKikimrProto::ERROR, errorDisks, ev);
+        };
+
+        auto result = check.Request();
+        UNIT_ASSERT(result->Status == NKikimrProto::OK);
+        UNIT_ASSERT(result->PlacementStatus == TEvBlobStorage::TEvCheckIntegrityResult::PS_RECOVERABLE);
+    }
+}
diff --git a/ydb/core/blobstorage/ut_blobstorage/ut_check_integrity/ya.make b/ydb/core/blobstorage/ut_blobstorage/ut_check_integrity/ya.make
new file mode 100644
index 00000000000..649027fbbb1
--- /dev/null
+++ b/ydb/core/blobstorage/ut_blobstorage/ut_check_integrity/ya.make
@@ -0,0 +1,16 @@
+UNITTEST_FOR(ydb/core/blobstorage/ut_blobstorage)
+
+    SIZE(MEDIUM)
+
+    FORK_SUBTESTS()
+
+    SRCS(
+        check_integrity.cpp
+    )
+
+    PEERDIR(
+        ydb/core/blobstorage/ut_blobstorage/lib
+    )
+
+END()
+
diff --git a/ydb/core/blobstorage/ut_blobstorage/ya.make b/ydb/core/blobstorage/ut_blobstorage/ya.make
index 7451f3c1296..2b72d663432 100644
--- a/ydb/core/blobstorage/ut_blobstorage/ya.make
+++ b/ydb/core/blobstorage/ut_blobstorage/ya.make
@@ -66,6 +66,7 @@ RECURSE_FOR_TESTS(
     ut_balancing
     ut_blob_depot
     ut_blob_depot_fat
+    ut_check_integrity
     ut_donor
     ut_group_reconfiguration
     ut_huge