Fix shredding stalling (#17374)

6 files changed, 64 insertions, 16 deletions

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h b/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h
index b443c38e151..986309c13be 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h
@@ -914,9 +914,11 @@ public:
                 COUNTER_INIT(BlobsSqueeze, true);
 
                 COUNTER_INIT(BlocksPromoteSsts, true);
+                COUNTER_INIT(BlocksExplicit, true);
                 COUNTER_INIT(BlocksBalance, true);
 
                 COUNTER_INIT(BarriersPromoteSsts, true);
+                COUNTER_INIT(BarriersExplicit, true);
                 COUNTER_INIT(BarriersBalance, true);
             }
 
@@ -928,9 +930,11 @@ public:
             COUNTER_DEF(BlobsSqueeze);
 
             COUNTER_DEF(BlocksPromoteSsts);
+            COUNTER_DEF(BlocksExplicit);
             COUNTER_DEF(BlocksBalance);
 
             COUNTER_DEF(BarriersPromoteSsts);
+            COUNTER_DEF(BarriersExplicit);
             COUNTER_DEF(BarriersBalance);
         };
 
diff --git a/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp b/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp
index 48418773646..291cc3084d9 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp
+++ b/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp
@@ -99,6 +99,13 @@ namespace NKikimr {
                 return action;
             }
 
+            // compact explicitly defined SST's, if set
+            action = TStrategyExplicit(HullCtx, Params, LevelSnap, Task).Select();
+            if (action != ActNothing) {
+                ++HullCtx->CompactionStrategyGroup.BlocksExplicit();
+                return action;
+            }
+
             // try to find what to compact based on levels balance
             action = TStrategyBalance(HullCtx, Params, LevelSnap, Task).Select();
             if (action != ActNothing) {
@@ -129,6 +136,13 @@ namespace NKikimr {
                 return action;
             }
 
+            // compact explicitly defined SST's, if set
+            action = TStrategyExplicit(HullCtx, Params, LevelSnap, Task).Select();
+            if (action != ActNothing) {
+                ++HullCtx->CompactionStrategyGroup.BarriersExplicit();
+                return action;
+            }
+
             // try to find what to compact based on levels balance
             action = TStrategyBalance(HullCtx, Params, LevelSnap, Task).Select();
             if (action != ActNothing) {
diff --git a/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.cpp b/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.cpp
index 470fc79090c..e7951b3490a 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.cpp
+++ b/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.cpp
@@ -22,6 +22,9 @@ namespace NKikimr {
                         TABLED() {SMALL() {str << StorageRatio.MonSummary();}}
                         TABLED() {SMALL() {str << (Info.IsCreatedByRepl() ? "REPL" : "COMP");}}
                         TABLED() {SMALL() {str << ToStringLocalTimeUpToSeconds(Info.CTime);}}
+                        TABLED() {SMALL() {
+                            str << AssignedSstId << '/' << VolatileOrderId << '@' << FormatList(AllChunks);
+                        }}
                         ++index;
                     }
                 }
diff --git a/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sstslice.cpp b/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sstslice.cpp
index 6ffe1a49021..b535b82f761 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sstslice.cpp
+++ b/ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sstslice.cpp
@@ -20,6 +20,7 @@ namespace NKikimr {
                         TABLEH() {str << NHullComp::TSstRatio::MonHeader();}
                         TABLEH() {str << "Origin";}
                         TABLEH() {str << "CTime";}
+                        TABLEH() {str << "Location";}
                     }
                 }
                 TABLEBODY() {
@@ -39,7 +40,7 @@ namespace NKikimr {
                     // total
                     TABLER() {
                         TABLED() {SMALL() {str << "index";}}
-                        TABLED() {SMALL() {str << "total";}}
+                        TABLED() {SMALL() {str << "level";}}
                         TABLED() {SMALL() {str << "lsns";}}
                         TABLED() {SMALL() {str << sum.IdxTotalSize << " / " << sum.InplaceDataTotalSize << " / "
                                            << sum.HugeDataTotalSize;}}
@@ -48,7 +49,9 @@ namespace NKikimr {
                                            << sum.ItemsWithHugeData;}}
                         TABLED() {SMALL() {str << "keys";}}
                         TABLED() {SMALL() {str << "ratio";}}
+                        TABLED() {SMALL() {str << "origin";}}
                         TABLED() {SMALL() {str << "time";}}
+                        TABLED() {SMALL() {str << "chunks";}}
                     }
                 }
             }
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index 23a715b2f53..9f31b51c07b 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -2486,6 +2486,9 @@ namespace NKikimr {
 
         void Handle(TEvents::TEvGone::TPtr &ev, const TActorContext &ctx) {
             Y_UNUSED(ctx);
+            if (ev->Sender == ShredActorId) {
+                ShredActorId = {};
+            }
             ActiveActors.Erase(ev->Sender);
         }
 
diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_shred.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_shred.cpp
index 62b94ba3df9..cece3117e21 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_shred.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_shred.cpp
@@ -24,11 +24,12 @@ namespace NKikimr {
 
         THashMap<TChunkIdx, EChunkType> ChunkTypes;
         THashSet<TChunkIdx> ChunksShredded;
-        THashSet<ui64> TablesToCompact;
+        THashSet<ui64> TablesToCompactLogoBlobs;
+        THashSet<ui64> TablesToCompactBlocks;
+        THashSet<ui64> TablesToCompactBarriers;
         ui32 RepliesPending = 0;
         bool SnapshotProcessed = false;
         bool DefragCompleted = false;
-        bool FoundAnyChunks = false;
 
     public:
         TSkeletonShredActor(NPDisk::TEvShredVDisk::TPtr ev, TShredCtxPtr shredCtx)
@@ -71,7 +72,6 @@ namespace NKikimr {
             RepliesPending = 2;
             SnapshotProcessed = false;
             DefragCompleted = false;
-            FoundAnyChunks = false;
         }
 
         void Handle(TEvListChunksResult::TPtr ev) {
@@ -84,7 +84,6 @@ namespace NKikimr {
                 for (const TChunkIdx chunkId : set) {
                     if (const auto it = ChunkTypes.find(chunkId); it != ChunkTypes.end()) {
                         it->second = type;
-                        FoundAnyChunks = true;
                         Y_VERIFY_DEBUG_S(ChunksToShred.contains(chunkId), ShredCtx->VCtx->VDiskLogPrefix);
                     } else {
                         Y_VERIFY_DEBUG_S(!ChunksToShred.contains(chunkId), ShredCtx->VCtx->VDiskLogPrefix);
@@ -104,15 +103,20 @@ namespace NKikimr {
                 (ActorId, SelfId()));
 
             auto& snap = ev->Get()->Snap;
-            TablesToCompact.clear();
-            Scan<true>(snap.HullCtx, snap.LogoBlobsSnap, TablesToCompact);
-            Scan<false>(snap.HullCtx, snap.BlocksSnap, TablesToCompact);
-            Scan<false>(snap.HullCtx, snap.BarriersSnap, TablesToCompact);
+            TablesToCompactLogoBlobs.clear();
+            TablesToCompactBlocks.clear();
+            TablesToCompactBarriers.clear();
+            Scan<true>(snap.HullCtx, snap.LogoBlobsSnap, TablesToCompactLogoBlobs);
+            Scan<false>(snap.HullCtx, snap.BlocksSnap, TablesToCompactBlocks);
+            Scan<false>(snap.HullCtx, snap.BarriersSnap, TablesToCompactBarriers);
             SnapshotProcessed = true;
+            DropUnknownChunks();
+            CheckIfDone();
             CheckDefragStage();
 
             STLOG(PRI_DEBUG, BS_SHRED, BSSV09, ShredCtx->VCtx->VDiskLogPrefix << "TEvTakeHullSnapshotResult processed",
-                (ActorId, SelfId()), (TablesToCompact, TablesToCompact));
+                (ActorId, SelfId()), (TablesToCompactLogoBlobs, TablesToCompactLogoBlobs),
+                (TablesToCompactBlocks, TablesToCompactBlocks), (TablesToCompactBarriers, TablesToCompactBarriers));
         }
 
         template<bool Blobs, typename TKey, typename TMemRec>
@@ -128,7 +132,6 @@ namespace NKikimr {
                         }
                         if (const auto it = ChunkTypes.find(p->ChunkIdx); it != ChunkTypes.end()) {
                             it->second = EChunkType::HUGE_CHUNK;
-                            FoundAnyChunks = true;
                         }
                     }
                 }
@@ -155,7 +158,6 @@ namespace NKikimr {
                         tablesToCompact.insert(seg.AssignedSstId);
                         STLOG(PRI_DEBUG, BS_SHRED, BSSV13, ShredCtx->VCtx->VDiskLogPrefix << "going to compact SST",
                             (SstId, seg.AssignedSstId), (AllChunks, seg.AllChunks));
-                        FoundAnyChunks = true;
                         Y_VERIFY_DEBUG_S(ChunksToShred.contains(chunkId), ShredCtx->VCtx->VDiskLogPrefix);
                     } else {
                         Y_VERIFY_DEBUG_S(!ChunksToShred.contains(chunkId), ShredCtx->VCtx->VDiskLogPrefix);
@@ -172,6 +174,19 @@ namespace NKikimr {
             }
         }
 
+        void DropUnknownChunks() {
+            for (auto it = ChunkTypes.begin(); it != ChunkTypes.end(); ) {
+                if (it->second == EChunkType::UNKNOWN) {
+                    const size_t num = ChunksToShred.erase(it->first);
+                    Y_VERIFY_DEBUG_S(num == 1, ShredCtx->VCtx->VDiskLogPrefix);
+                    ChunksShredded.insert(it->first);
+                    ChunkTypes.erase(it++);
+                } else {
+                    ++it;
+                }
+            }
+        }
+
         void HandleHullShredDefragResult() {
             STLOG(PRI_DEBUG, BS_SHRED, BSSV14, ShredCtx->VCtx->VDiskLogPrefix << "EvHullShredDefragResult received",
                 (ActorId, SelfId()));
@@ -184,8 +199,15 @@ namespace NKikimr {
                 return;
             }
 
-            if (!TablesToCompact.empty()) {
-                Send(ShredCtx->SkeletonId, TEvCompactVDisk::Create(EHullDbType::LogoBlobs, std::move(TablesToCompact)));
+            if (!TablesToCompactLogoBlobs.empty()) {
+                Send(ShredCtx->SkeletonId, TEvCompactVDisk::Create(EHullDbType::LogoBlobs,
+                    std::exchange(TablesToCompactLogoBlobs, {})));
+            } else if (!TablesToCompactBlocks.empty()) {
+                Send(ShredCtx->SkeletonId, TEvCompactVDisk::Create(EHullDbType::Blocks,
+                    std::exchange(TablesToCompactBlocks, {})));
+            } else if (!TablesToCompactBarriers.empty()) {
+                Send(ShredCtx->SkeletonId, TEvCompactVDisk::Create(EHullDbType::Barriers,
+                    std::exchange(TablesToCompactBarriers, {})));
             } else {
                 TActivationContext::Schedule(TDuration::Minutes(1), new IEventHandle(TEvents::TSystem::Wakeup, 0,
                     SelfId(), TActorId(), nullptr, 0));
@@ -195,8 +217,7 @@ namespace NKikimr {
         void Handle(TEvCompactVDiskResult::TPtr /*ev*/) {
             STLOG(PRI_DEBUG, BS_SHRED, BSSV11, ShredCtx->VCtx->VDiskLogPrefix << "TEvCompactVDiskResult received",
                 (ActorId, SelfId()));
-            TActivationContext::Schedule(TDuration::Minutes(1), new IEventHandle(TEvents::TSystem::Wakeup, 0, SelfId(),
-                TActorId(), nullptr, 0));
+            CheckDefragStage();
         }
 
         void Handle(TEvNotifyChunksDeleted::TPtr ev) {