NBYDB-1732: uniform distribution of the recalculation ration of SSTs (#42317)

4 files changed, 74 insertions, 16 deletions

diff --git a/ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hullstorageratio.h b/ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hullstorageratio.h
index 839c12a5c57..97459779d6b 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hullstorageratio.h
+++ b/ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hullstorageratio.h
@@ -127,9 +127,12 @@ namespace NKikimr {
         ////////////////////////////////////////////////////////////////////////////
         class TSstRatioThreadSafeHolder {
         public:
-            void Set(TSstRatioPtr ratio) {
-                TGuard<TSpinLock> g(Lock);
-                Ratio = ratio;
+            void Set(TSstRatioPtr ratio, TInstant calculationTime) {
+                {
+                    TGuard<TSpinLock> g(Lock);
+                    Ratio = ratio;
+                }
+                CalculationTime = calculationTime;
             }
 
             TSstRatioPtr Get() const {
@@ -137,6 +140,14 @@ namespace NKikimr {
                 return Ratio;
             }
 
+            void SetCalculationTime(TInstant calculationTime) {
+                CalculationTime = calculationTime;
+            }
+
+            TInstant GetCalculationTime() const {
+                return CalculationTime;
+            }
+
             void OutputProto(NKikimrVDisk::StorageRatio* proto) const {
                 TSstRatioPtr ratio = Get();
                 if (ratio) {
@@ -157,6 +168,7 @@ namespace NKikimr {
         private:
             TSpinLock Lock;
             TIntrusivePtr<TSstRatio> Ratio;
+            TInstant CalculationTime = TInstant::Zero();
         };
 
 
diff --git a/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_ratio.h b/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_ratio.h
index 02806abc005..1f0a55bc07f 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_ratio.h
+++ b/ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_ratio.h
@@ -4,6 +4,7 @@
 #include "hulldb_compstrat_defs.h"
 #include <ydb/core/blobstorage/vdisk/hulldb/hull_ds_all_snap.h>
 #include <ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullmergeits.h>
+#include <util/digest/numeric.h>
 
 namespace NKikimr {
     namespace NHullComp {
@@ -67,27 +68,73 @@ namespace NKikimr {
             };
 
             struct TTimeSst {
-                ui64 Seconds;
+                TInstant NextCalculationTime;
                 TLevelSstPtr LevelSstPtr;
 
-                TTimeSst(ui64 seconds, const TLevelSstPtr &p)
-                    : Seconds(seconds)
+                TTimeSst(TInstant nextCalculationTime, const TLevelSstPtr &p)
+                    : NextCalculationTime(nextCalculationTime)
                     , LevelSstPtr(p)
                 {}
 
                 bool operator < (const TTimeSst &s) const {
-                    return Seconds < s.Seconds;
+                    return NextCalculationTime < s.NextCalculationTime;
                 }
             };
 
-            void OrderSstByStorageRatioTime(TVector<TTimeSst> &vec) {
+            static ui64 GetSstId(const TLevelSstPtr &p) {
+                return p.SstPtr->AssignedSstId ? p.SstPtr->AssignedSstId : p.SstPtr->VolatileOrderId;
+            }
+
+            static TDuration GetInitialRecalculationAge(const TLevelSstPtr &p, TDuration calcPeriod) {
+                const ui64 calcPeriodSeconds = calcPeriod.Seconds();
+                if (!calcPeriodSeconds) {
+                    return calcPeriod;
+                }
+
+                return TDuration::Seconds(IntHash(GetSstId(p)) % calcPeriodSeconds);
+            }
+
+            static TInstant GetInitialCalculationTime(const TLevelSstPtr &p, TInstant startTime, TDuration calcPeriod) {
+                const TInstant createTime = p.SstPtr->Info.CTime;
+                if (createTime != TInstant::Zero() && startTime < createTime + calcPeriod) {
+                    return createTime;
+                }
+
+                return startTime - GetInitialRecalculationAge(p, calcPeriod);
+            }
+
+            static TInstant GetCalculationTime(const TLevelSstPtr &p, TInstant startTime, TDuration calcPeriod) {
+                TInstant calculationTime = p.SstPtr->StorageRatio.GetCalculationTime();
+                if (calculationTime != TInstant::Zero()) {
+                    return calculationTime;
+                }
+
+                TSstRatioPtr ratio = p.SstPtr->StorageRatio.Get();
+                const TInstant initialCalculationTime = GetInitialCalculationTime(p, startTime, calcPeriod);
+                if (!ratio) {
+                    p.SstPtr->StorageRatio.SetCalculationTime(initialCalculationTime);
+                    return initialCalculationTime;
+                }
+
+                if (ratio->Time == TInstant::Zero()) {
+                    return TInstant::Zero();
+                }
+
+                p.SstPtr->StorageRatio.SetCalculationTime(ratio->Time);
+                return ratio->Time;
+            }
+
+            static TInstant GetNextCalculationTime(const TLevelSstPtr &p, TInstant startTime, TDuration calcPeriod) {
+                return GetCalculationTime(p, startTime, calcPeriod) + calcPeriod;
+            }
+
+            void OrderSstByStorageRatioTime(TVector<TTimeSst> &vec, TInstant startTime, TDuration calcPeriod) {
                 vec.clear();
                 TSstIterator it(&LevelSnap.SliceSnap);
                 it.SeekToFirst();
                 while (it.Valid()) {
                     TLevelSstPtr p = it.Get();
-                    ui64 seconds = p.SstPtr->StorageRatio.GetTime().Seconds();
-                    vec.push_back(TTimeSst(seconds, p));
+                    vec.push_back(TTimeSst(GetNextCalculationTime(p, startTime, calcPeriod), p));
                     it.Next();
                 }
                 Sort(vec.begin(), vec.end());
@@ -100,14 +147,13 @@ namespace NKikimr {
                 // order all ssts (including level 0) by storage ratio calculation time
                 TVector<TTimeSst> vec;
                 vec.reserve(1000u);
-                OrderSstByStorageRatioTime(vec);
+                OrderSstByStorageRatioTime(vec, startTime, calcPeriod);
 
                 // calculate storage ratio, don't spend much time on it, skip ssts that are actualized
                 for (const auto &x : vec) {
-                    const TInstant time = x.LevelSstPtr.SstPtr->StorageRatio.GetTime();
-                    if (startTime >= time + calcPeriod) {
+                    if (startTime >= x.NextCalculationTime) {
                         TSstRatioPtr newRatio = CalculateSstRatio(x.LevelSstPtr.SstPtr, startTime);
-                        x.LevelSstPtr.SstPtr->StorageRatio.Set(newRatio);
+                        x.LevelSstPtr.SstPtr->StorageRatio.Set(newRatio, newRatio->Time);
                         stat.SstsChecked++;
                     } else {
                         stat.BreakedActualRatio = true;
diff --git a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwriteindexsst.h b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwriteindexsst.h
index 95bc2d9f2f8..748466ddcd5 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwriteindexsst.h
+++ b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwriteindexsst.h
@@ -65,7 +65,7 @@ protected:
         ratio->IndexBytesTotal = ratio->IndexBytesKeep = info.IdxTotalSize;
         ratio->InplacedDataTotal = ratio->InplacedDataKeep = 0;
         ratio->HugeDataTotal = ratio->HugeDataKeep = 0;
-        LevelSegment->StorageRatio.Set(ratio);
+        LevelSegment->StorageRatio.Set(ratio, TInstant::Zero());
 
         TIdxDiskPlaceHolder placeHolder(SstId);
         placeHolder.Info = info;
diff --git a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h
index 9c28436618a..a6b83718960 100644
--- a/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h
+++ b/ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullwritesst.h
@@ -682,7 +682,7 @@ namespace NKikimr {
             ratio->IndexBytesTotal = ratio->IndexBytesKeep = info.IdxTotalSize;
             ratio->InplacedDataTotal = ratio->InplacedDataKeep = info.InplaceDataTotalSize;
             ratio->HugeDataTotal = ratio->HugeDataKeep = info.HugeDataTotalSize;
-            LevelSegment->StorageRatio.Set(ratio);
+            LevelSegment->StorageRatio.Set(ratio, TInstant::Zero());
 
             // write out place holder
             TIdxDiskPlaceHolder placeHolder(SstId);