YDB-7262 Add default permille ICB configuration for vdisk garbage compaction threshold (#6819)

Сloses #7262 

9 files changed, 26 insertions, 11 deletions

diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
index 754d0924d1..f2995a5b44 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
@@ -179,6 +179,7 @@ void TNodeWarden::Bootstrap() {
         icb->RegisterSharedControl(EnableSyncLogChunkCompressionSSD, "VDiskControls.EnableSyncLogChunkCompressionSSD");
         icb->RegisterSharedControl(MaxSyncLogChunksInFlightHDD, "VDiskControls.MaxSyncLogChunksInFlightHDD");
         icb->RegisterSharedControl(MaxSyncLogChunksInFlightSSD, "VDiskControls.MaxSyncLogChunksInFlightSSD");
+        icb->RegisterSharedControl(DefaultHugeGarbagePerMille, "VDiskControls.DefaultHugeGarbagePerMille");
 
         icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_ROT].BurstThresholdNs,
                 "VDiskControls.BurstThresholdNsHDD");
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
index 260bc82b22..51f4c2075b 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h
@@ -155,6 +155,7 @@ namespace NKikimr::NStorage {
         TControlWrapper EnableSyncLogChunkCompressionSSD;
         TControlWrapper MaxSyncLogChunksInFlightHDD;
         TControlWrapper MaxSyncLogChunksInFlightSSD;
+        TControlWrapper DefaultHugeGarbagePerMille;
 
         TReplQuoter::TPtr ReplNodeRequestQuoter;
         TReplQuoter::TPtr ReplNodeResponseQuoter;
@@ -180,6 +181,7 @@ namespace NKikimr::NStorage {
             , EnableSyncLogChunkCompressionSSD(0, 0, 1)
             , MaxSyncLogChunksInFlightHDD(10, 1, 1024)
             , MaxSyncLogChunksInFlightSSD(10, 1, 1024)
+            , DefaultHugeGarbagePerMille(300, 1, 1000)
             , CostMetricsParametersByMedia({
                 TCostMetricsParameters{200},
                 TCostMetricsParameters{50},
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
index fa8b812aa8..81395c427b 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
@@ -185,6 +185,7 @@ namespace NKikimr::NStorage {
         vdiskConfig->EnableVDiskCooldownTimeout = Cfg->EnableVDiskCooldownTimeout;
         vdiskConfig->ReplPausedAtStart = Cfg->VDiskReplPausedAtStart;
         vdiskConfig->EnableVPatch = EnableVPatch;
+        vdiskConfig->DefaultHugeGarbagePerMille = DefaultHugeGarbagePerMille;
 
         vdiskConfig->EnableLocalSyncLogDataCutting = EnableLocalSyncLogDataCutting;
         if (deviceType == NPDisk::EDeviceType::DEVICE_TYPE_ROT) {
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
index 31e30788ee..73b1d27daa 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -218,6 +218,7 @@ namespace NKikimr {
         TDuration WhiteboardUpdateInterval;
         bool EnableVDiskCooldownTimeout;
         TControlWrapper EnableVPatch = true;
+        TControlWrapper DefaultHugeGarbagePerMille;
 
         ///////////// COST METRICS SETTINGS ////////////////
         bool UseCostTracker = true;
diff --git a/ydb/core/blobstorage/vdisk/defrag/defrag_actor.cpp b/ydb/core/blobstorage/vdisk/defrag/defrag_actor.cpp
index fa7014b648..54a7d2cfaf 100644
--- a/ydb/core/blobstorage/vdisk/defrag/defrag_actor.cpp
+++ b/ydb/core/blobstorage/vdisk/defrag/defrag_actor.cpp
@@ -16,12 +16,14 @@ namespace NKikimr {
     ////////////////////////////////////////////////////////////////////////////
     TDefragCtx::TDefragCtx(
             const TIntrusivePtr<TVDiskContext> &vctx,
+            const TIntrusivePtr<TVDiskConfig> &vconfig,
             const std::shared_ptr<THugeBlobCtx> &hugeBlobCtx,
             const TPDiskCtxPtr &pdiskCtx,
             const TActorId &skeletonId,
             const TActorId &hugeKeeperId,
             bool runDefrageBySchedule)
         : VCtx(vctx)
+        , VCfg(vconfig)
         , HugeBlobCtx(hugeBlobCtx)
         , PDiskCtx(pdiskCtx)
         , SkeletonId(skeletonId)
@@ -48,7 +50,8 @@ namespace NKikimr {
     bool HugeHeapDefragmentationRequired(
             const TOutOfSpaceState& oos,
             ui32 hugeCanBeFreedChunks,
-            ui32 hugeTotalChunks) {
+            ui32 hugeTotalChunks,
+            double defaultPercent) {
 
         if (hugeCanBeFreedChunks < 10)
             return false;
@@ -56,11 +59,14 @@ namespace NKikimr {
         double percentOfGarbage = static_cast<double>(hugeCanBeFreedChunks) / hugeTotalChunks;
 
         if (oos.GetLocalColor() > TSpaceColor::CYAN) {
-            return percentOfGarbage >= 0.02;
+            // For anything worse than CYAN
+            return percentOfGarbage >= Min(0.02, defaultPercent);
         } else if (oos.GetLocalColor() > TSpaceColor::GREEN) {
-            return percentOfGarbage >= 0.15;
+            // For CYAN
+            return percentOfGarbage >= Min(0.15, defaultPercent);
         } else {
-            return percentOfGarbage >= 0.30;
+            // For GREEN
+            return percentOfGarbage >= Min(0.30, defaultPercent);
         }
     }
 
@@ -113,7 +119,8 @@ namespace NKikimr {
                     const auto& oos = DCtx->VCtx->GetOutOfSpaceState();
                     Y_ABORT_UNLESS(usefulChunks <= totalChunks);
                     const ui32 canBeFreedChunks = totalChunks - usefulChunks;
-                    if (HugeHeapDefragmentationRequired(oos, canBeFreedChunks, totalChunks)) {
+                    double defaultPercent = DCtx->VCfg->DefaultHugeGarbagePerMille / 1000.0;
+                    if (HugeHeapDefragmentationRequired(oos, canBeFreedChunks, totalChunks, defaultPercent)) {
                         TChunksToDefrag chunksToDefrag = calcStat.GetChunksToDefrag(DCtx->MaxChunksToDefrag);
                         Y_ABORT_UNLESS(chunksToDefrag);
                         STLOG(PRI_INFO, BS_VDISK_DEFRAG, BSVDD03, VDISKP(DCtx->VCtx->VDiskLogPrefix, "scan finished"),
diff --git a/ydb/core/blobstorage/vdisk/defrag/defrag_actor.h b/ydb/core/blobstorage/vdisk/defrag/defrag_actor.h
index f59ecee374..08c451b094 100644
--- a/ydb/core/blobstorage/vdisk/defrag/defrag_actor.h
+++ b/ydb/core/blobstorage/vdisk/defrag/defrag_actor.h
@@ -18,6 +18,7 @@ namespace NKikimr {
     ////////////////////////////////////////////////////////////////////////////
     struct TDefragCtx {
         const TIntrusivePtr<TVDiskContext> VCtx;
+        const TIntrusivePtr<TVDiskConfig> VCfg;
         const std::shared_ptr<THugeBlobCtx> HugeBlobCtx;
         const TPDiskCtxPtr PDiskCtx;
         const TActorId SkeletonId;
@@ -30,6 +31,7 @@ namespace NKikimr {
 
         TDefragCtx(
                 const TIntrusivePtr<TVDiskContext> &vctx,
+                const TIntrusivePtr<TVDiskConfig> &vconfig,
                 const std::shared_ptr<THugeBlobCtx> &hugeBlobCtx,
                 const TPDiskCtxPtr &pdiskCtx,
                 const TActorId &skeletonId,
@@ -45,7 +47,8 @@ namespace NKikimr {
     bool HugeHeapDefragmentationRequired(
             const TOutOfSpaceState& oos,
             ui32 hugeCanBeFreedChunks,
-            ui32 hugeTotalChunks);
+            ui32 hugeTotalChunks,
+            double defaultPercent);
 
     ////////////////////////////////////////////////////////////////////////////
     // VDISK DEFRAG ACTOR CREATOR
diff --git a/ydb/core/blobstorage/vdisk/defrag/defrag_actor_ut.cpp b/ydb/core/blobstorage/vdisk/defrag/defrag_actor_ut.cpp
index ebcfec5619..fa059a1d2c 100644
--- a/ydb/core/blobstorage/vdisk/defrag/defrag_actor_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/defrag/defrag_actor_ut.cpp
@@ -15,21 +15,21 @@ namespace NKikimr {
                 TOutOfSpaceState oos(1, 0);
                 ui32 hugeCanBeFreedChunks = 9;
                 ui32 hugeUsedChunks = 20;
-                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks);
+                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
                 UNIT_ASSERT(!defrag);
             }
             {
                 TOutOfSpaceState oos(1, 0);
                 ui32 hugeCanBeFreedChunks = 200;
                 ui32 hugeUsedChunks = 1000;
-                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks);
+                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
                 UNIT_ASSERT(!defrag);
             }
             {
                 TOutOfSpaceState oos(1, 0);
                 ui32 hugeCanBeFreedChunks = 301;
                 ui32 hugeUsedChunks = 1000;
-                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks);
+                bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
                 UNIT_ASSERT(defrag);
             }
         }
diff --git a/ydb/core/blobstorage/vdisk/defrag/defrag_quantum.cpp b/ydb/core/blobstorage/vdisk/defrag/defrag_quantum.cpp
index 1e1753bf6a..9f3cb4006e 100644
--- a/ydb/core/blobstorage/vdisk/defrag/defrag_quantum.cpp
+++ b/ydb/core/blobstorage/vdisk/defrag/defrag_quantum.cpp
@@ -95,7 +95,7 @@ namespace NKikimr {
                 Compact();
 
                 auto hugeStat = GetHugeStat();
-                Y_ABORT_UNLESS(hugeStat.LockedChunks.size() < 100);
+                Y_DEBUG_ABORT_UNLESS(hugeStat.LockedChunks.size() < 100);
             }
 
             Send(ParentActorId, new TEvDefragQuantumResult(std::move(stat)));
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index fbda80f13c..01c44f56a8 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -1809,7 +1809,7 @@ namespace NKikimr {
         }
 
         void StartDefrag(const TActorContext &ctx) {
-            auto defragCtx = std::make_shared<TDefragCtx>(VCtx, HugeBlobCtx, PDiskCtx, ctx.SelfID,
+            auto defragCtx = std::make_shared<TDefragCtx>(VCtx, Config, HugeBlobCtx, PDiskCtx, ctx.SelfID,
                 Db->HugeKeeperID, true);
             DefragId = ctx.Register(CreateDefragActor(defragCtx, GInfo));
             ActiveActors.Insert(DefragId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE); // keep forever