Calculate VDisk occupancy metric

10 files changed, 96 insertions, 44 deletions

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
index 8a01a34752..d78c6859d4 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
@@ -1303,6 +1303,7 @@ struct TEvCheckSpaceResult : public TEventLocal<TEvCheckSpaceResult, TEvBlobStor
     ui32 TotalChunks; // contains common limit in shared free space mode, Total != Free + Used
     ui32 UsedChunks; // number of chunks allocated by requesting owner
     ui32 NumSlots; // number of VSlots over PDisk
+    double Occupancy = 0;
     TString ErrorReason;
 
     TEvCheckSpaceResult(NKikimrProto::EReplyStatus status, TStatusFlags statusFlags, ui32 freeChunks,
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h
index 434d643e84..4cf19a062f 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h
@@ -121,8 +121,8 @@ public:
     }
 
     // Tread-safe status flag getter
-    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(TOwner id, i64 allocationSize) const {
-        return QuotaForOwner[id].EstimateSpaceColor(allocationSize);
+    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(TOwner id, i64 allocationSize, double *occupancy) const {
+        return QuotaForOwner[id].EstimateSpaceColor(allocationSize, occupancy);
     }
 
     bool TryAllocate(TOwner id, i64 count, TString &outErrorReason) {
@@ -149,9 +149,11 @@ public:
         str << "<td>" << q.GetHardLimit() << "</td>";
         str << "<td>" << q.GetFree() << "</td>";
         str << "<td>" << q.GetUsed() << "</td>";
-        str << "<td>" << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(q.EstimateSpaceColor(0)) << "</td>";
-
+        double occupancy;
+        str << "<td>" << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(q.EstimateSpaceColor(0, &occupancy)) << "</td>";
+        str << "<td>" << occupancy << "</td>";
         str << "<td>" << q.Cyan << "</td>";
+        str << "<td>" << q.LightYellow << "</td>";
         str << "<td>" << q.Yellow << "</td>";
         str << "<td>" << q.LightOrange << "</td>";
         str << "<td>" << q.PreOrange << "</td>";
@@ -180,8 +182,10 @@ public:
                 <th>Free</th>
                 <th>Used</th>
                 <th>Color</th>
+                <th>Occupancy</th>
 
                 <th>Cyan</th>
+                <th>LightYellow</th>
                 <th>Yellow</th>
                 <th>LightOrange</th>
                 <th>PreOrange</th>
@@ -219,6 +223,7 @@ using TColor = NKikimrBlobStorage::TPDiskSpaceColor;
     TKeeperParams Params;
 
     TColor::E ColorBorder = NKikimrBlobStorage::TPDiskSpaceColor::GREEN;
+    double ColorBorderOccupancy = 0;
 
 public:
 
@@ -327,6 +332,7 @@ public:
         }
 
         ColorBorder = params.SpaceColorBorder;
+        ColorBorderOccupancy = chunkLimits.GetOccupancyForColor(ColorBorder, GlobalQuota->GetHardLimit(OwnerBeginUser));
         return true;
     }
 
@@ -408,19 +414,24 @@ public:
         }
     }
 
-    TStatusFlags GetSpaceStatusFlags(TOwner owner) const {
-        return SpaceColorToStatusFlag(GetSpaceColor(owner));
+    TStatusFlags GetSpaceStatusFlags(TOwner owner, double *occupancy) const {
+        return SpaceColorToStatusFlag(GetSpaceColor(owner, occupancy));
     }
 
-    TColor::E GetSpaceColor(TOwner owner) const {
-        return EstimateSpaceColor(owner, 0);
+    TColor::E GetSpaceColor(TOwner owner, double *occupancy) const {
+        return EstimateSpaceColor(owner, 0, occupancy);
     }
 
     // Estimate status flags after allocation of allocatinoSize
-    TColor::E EstimateSpaceColor(TOwner owner, i64 allocationSize) const {
+    TColor::E EstimateSpaceColor(TOwner owner, i64 allocationSize, double *occupancy) const {
         if (IsOwnerUser(owner)) {
-            TColor::E ret = Min(ColorBorder, OwnerQuota->EstimateSpaceColor(owner, allocationSize));
-            ret = Max(ret, SharedQuota->EstimateSpaceColor(allocationSize));
+            double ownerOccupancy, sharedOccupancy;
+            TColor::E ret = Min(ColorBorder, OwnerQuota->EstimateSpaceColor(owner, allocationSize, &ownerOccupancy));
+            ret = Max(ret, SharedQuota->EstimateSpaceColor(allocationSize, &sharedOccupancy));
+            *occupancy = Max(
+                Min(ColorBorderOccupancy, ownerOccupancy), // owner occupancy can't exceed its color border top value
+                sharedOccupancy
+            );
             return ret;
         } else {
             switch (owner) {
@@ -428,29 +439,26 @@ public:
                     if (Params.SeparateCommonLog) {
                         if (GlobalQuota->GetHardLimit(OwnerCommonStaticLog) == 0) {
                             // No static group bonus, use common quota for the request
-                            return GlobalQuota->EstimateSpaceColor(OwnerSystem, allocationSize);
+                            return GlobalQuota->EstimateSpaceColor(OwnerSystem, allocationSize, occupancy);
                         } else {
-                            return GlobalQuota->EstimateSpaceColor(OwnerCommonStaticLog, allocationSize);
+                            return GlobalQuota->EstimateSpaceColor(OwnerCommonStaticLog, allocationSize, occupancy);
                         }
                     } else {
                         if (GlobalQuota->GetHardLimit(OwnerCommonStaticLog) == 0) {
                             // No static group bonus, use common quota for the request
-                            return SharedQuota->EstimateSpaceColor(allocationSize);
+                            return SharedQuota->EstimateSpaceColor(allocationSize, occupancy);
                         } else {
-                            return GlobalQuota->EstimateSpaceColor(OwnerCommonStaticLog, allocationSize);
+                            return GlobalQuota->EstimateSpaceColor(OwnerCommonStaticLog, allocationSize, occupancy);
                         }
                     }
-                    break;
                 case OwnerSystem:
                     if (Params.SeparateCommonLog) {
-                        return GlobalQuota->EstimateSpaceColor(OwnerSystem, allocationSize);
+                        return GlobalQuota->EstimateSpaceColor(OwnerSystem, allocationSize, occupancy);
                     } else {
-                        return SharedQuota->EstimateSpaceColor(allocationSize);
+                        return SharedQuota->EstimateSpaceColor(allocationSize, occupancy);
                     }
-                    break;
                 default:
-                    return GlobalQuota->EstimateSpaceColor(owner, allocationSize);
-                    break;
+                    return GlobalQuota->EstimateSpaceColor(owner, allocationSize, occupancy);
             }
         }
     }
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_color_limits.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_color_limits.h
index fe9267de30..e227bd3869 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_color_limits.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_color_limits.h
@@ -22,6 +22,10 @@ struct TDiskColor {
     i64 CalculateQuota(i64 total) const {
         return total * Multiplier / Divisor + Addend;
     }
+
+    double CalculateOccupancy(i64 total) const {
+        return (double)CalculateQuota(total) / total;
+    }
 };
 
 struct TColorLimits {
@@ -70,6 +74,24 @@ struct TColorLimits {
             {982, 1000}, // Cyan: Ask to cut log
         };
     }
+
+    double GetOccupancyForColor(NKikimrBlobStorage::TPDiskSpaceColor::E color, i64 total) {
+        switch (color) {
+            case NKikimrBlobStorage::TPDiskSpaceColor::GREEN:          return Cyan.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::CYAN:           return LightYellow.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW:   return Yellow.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::YELLOW:         return LightOrange.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_ORANGE:   return PreOrange.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::PRE_ORANGE:     return Orange.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::ORANGE:         return Red.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::RED:            return Black.CalculateOccupancy(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::BLACK:          return 1.0;
+
+            case NKikimrBlobStorage::TPDiskSpaceColor_E_TPDiskSpaceColor_E_INT_MIN_SENTINEL_DO_NOT_USE_:
+            case NKikimrBlobStorage::TPDiskSpaceColor_E_TPDiskSpaceColor_E_INT_MAX_SENTINEL_DO_NOT_USE_:
+                Y_FAIL();
+        }
+    }
 };
 
 } // NPDisk
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
index 6ec5cd7b68..3403c678f9 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
@@ -585,12 +585,22 @@ ui32 TPDisk::GetUsedChunks(ui32 ownerId, const EOwnerGroupType ownerGroupType) c
     return ownedChunks;
 }
 
-NPDisk::TStatusFlags TPDisk::GetStatusFlags(TOwner ownerId, const EOwnerGroupType ownerGroupType) const {
+NPDisk::TStatusFlags TPDisk::GetStatusFlags(TOwner ownerId, const EOwnerGroupType ownerGroupType, double *occupancy) const {
+    double occupancy_;
+    NPDisk::TStatusFlags res;
+
     if (IsOwnerUser(ownerId)) {
-        return Keeper.GetSpaceStatusFlags(ownerId);
+        res = Keeper.GetSpaceStatusFlags(ownerId, &occupancy_);
+    } else {
+        TOwner keeperOwner = (ownerGroupType == EOwnerGroupType::Dynamic ? OwnerSystem : OwnerCommonStaticLog);
+        res = Keeper.GetSpaceStatusFlags(keeperOwner, &occupancy_);
     }
-    TOwner keeperOwner = (ownerGroupType == EOwnerGroupType::Dynamic ? OwnerSystem : OwnerCommonStaticLog);
-    return Keeper.GetSpaceStatusFlags(keeperOwner);
+
+    if (occupancy) {
+        *occupancy = occupancy_;
+    }
+
+    return res;
 }
 
 NPDisk::TStatusFlags TPDisk::NotEnoughDiskSpaceStatusFlags(ui32 ownerId, const EOwnerGroupType ownerGroupType) const {
@@ -1005,7 +1015,8 @@ TVector<TChunkIdx> TPDisk::LockChunksForOwner(TOwner owner, const ui32 count, TS
 
     const ui32 sharedFree = Keeper.GetFreeChunkCount() - 1;
     i64 ownerFree = Keeper.GetOwnerFree(owner);
-    auto color = Keeper.EstimateSpaceColor(owner, count);
+    double occupancy;
+    auto color = Keeper.EstimateSpaceColor(owner, count, &occupancy);
 
     auto makeError = [&](TString info) {
         guard.Release();
@@ -1016,6 +1027,7 @@ TVector<TChunkIdx> TPDisk::LockChunksForOwner(TOwner owner, const ui32 count, TS
             << " sharedFree# " << sharedFree
             << " ownerFree# " << ownerFree
             << " estimatedColor after lock# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(color)
+            << " occupancy after lock# " << occupancy
             << " " << info
             << " Marker# BPD21";
         errorReason = str.Str();
@@ -1246,7 +1258,8 @@ TVector<TChunkIdx> TPDisk::AllocateChunkForOwner(const TRequestBase *req, const
 
     const ui32 sharedFree = Keeper.GetFreeChunkCount() - 1;
     i64 ownerFree = Keeper.GetOwnerFree(req->Owner);
-    auto color = Keeper.EstimateSpaceColor(req->Owner, count);
+    double occupancy;
+    auto color = Keeper.EstimateSpaceColor(req->Owner, count, &occupancy);
 
     auto makeError = [&](TString info) {
         guard.Release();
@@ -1257,6 +1270,7 @@ TVector<TChunkIdx> TPDisk::AllocateChunkForOwner(const TRequestBase *req, const
             << " sharedFree# " << sharedFree
             << " ownerFree# " << ownerFree
             << " estimatedColor after allocation# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(color)
+            << " occupancy after allocation# " << occupancy
             << " " << info
             << " Marker# BPD20";
         errorReason = str.Str();
@@ -1479,7 +1493,9 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) {
             vdiskMetrics->MutableVDiskId()->ClearGroupGeneration();
             vdiskMetrics->SetAvailableSize(ownerFree);
             vdiskMetrics->SetAllocatedSize(ownerAllocated);
-            vdiskMetrics->SetStatusFlags(Keeper.GetSpaceStatusFlags(owner));
+            double occupancy;
+            vdiskMetrics->SetStatusFlags(Keeper.GetSpaceStatusFlags(owner, &occupancy));
+            vdiskMetrics->SetOccupancy(occupancy);
             auto *vslotId = vdiskMetrics->MutableVSlotId();
             vslotId->SetNodeId(ActorSystem->NodeId);
             vslotId->SetPDiskId(PDiskId);
@@ -1915,14 +1931,16 @@ void TPDisk::SchedulerConfigure(const TConfigureScheduler &reqCfg) {
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void TPDisk::CheckSpace(TCheckSpace &evCheckSpace) {
-    THolder<NPDisk::TEvCheckSpaceResult> result(new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK,
-                GetStatusFlags(evCheckSpace.Owner, evCheckSpace.OwnerGroupType),
+    double occupancy;
+    auto result = std::make_unique<NPDisk::TEvCheckSpaceResult>(NKikimrProto::OK,
+                GetStatusFlags(evCheckSpace.Owner, evCheckSpace.OwnerGroupType, &occupancy),
                 GetFreeChunks(evCheckSpace.Owner, evCheckSpace.OwnerGroupType),
                 GetTotalChunks(evCheckSpace.Owner, evCheckSpace.OwnerGroupType),
                 GetUsedChunks(evCheckSpace.Owner, evCheckSpace.OwnerGroupType),
                 AtomicGet(TotalOwners),
-                TString()));
-    ActorSystem->Send(evCheckSpace.Sender, result.Release());
+                TString());
+    result->Occupancy = occupancy;
+    ActorSystem->Send(evCheckSpace.Sender, result.release());
     Mon.CheckSpace.CountResponse();
     return;
 }
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
index 02108aa19b..e2c12017f3 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
@@ -230,7 +230,7 @@ public:
     ui32 GetTotalChunks(ui32 ownerId, const EOwnerGroupType ownerGroupType) const;
     ui32 GetFreeChunks(ui32 ownerId, const EOwnerGroupType ownerGroupType) const;
     ui32 GetUsedChunks(ui32 ownerId, const EOwnerGroupType ownerGroupType) const;
-    TStatusFlags GetStatusFlags(TOwner ownerId, const EOwnerGroupType ownerGroupType) const;
+    TStatusFlags GetStatusFlags(TOwner ownerId, const EOwnerGroupType ownerGroupType, double *occupancy = nullptr) const;
     TStatusFlags NotEnoughDiskSpaceStatusFlags(ui32 ownerId, const EOwnerGroupType ownerGroupType) const;
 
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
index dc40c06224..43e49b5592 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
@@ -659,7 +659,8 @@ bool TPDisk::AllocateLogChunks(ui32 chunksNeeded, ui32 chunksContainingPayload,
 
     // Check space and free it if needed
     using TColor = NKikimrBlobStorage::TPDiskSpaceColor;
-    TColor::E color = Keeper.EstimateSpaceColor(keeperOwner, chunksNeeded);
+    double occupancy;
+    TColor::E color = Keeper.EstimateSpaceColor(keeperOwner, chunksNeeded, &occupancy);
     if (color >= TColor::RED && !isAllowedForSpaceRed) {
         return false;
     }
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h
index a3fa2cc0bc..9bc898616d 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h
@@ -124,12 +124,12 @@ public:
         ChunkTracker.Release(owner, 1);
     }
 
-    TStatusFlags GetSpaceStatusFlags(TOwner owner) const {
-        return ChunkTracker.GetSpaceStatusFlags(owner);
+    TStatusFlags GetSpaceStatusFlags(TOwner owner, double *occupancy) const {
+        return ChunkTracker.GetSpaceStatusFlags(owner, occupancy);
     }
 
-    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(TOwner owner, i64 allocationSize) const {
-        return ChunkTracker.EstimateSpaceColor(owner, allocationSize);
+    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(TOwner owner, i64 allocationSize, double *occupancy) const {
+        return ChunkTracker.EstimateSpaceColor(owner, allocationSize, occupancy);
     }
 
     //
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_quota_record.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_quota_record.h
index c74ef52270..49037671ea 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_quota_record.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_quota_record.h
@@ -70,7 +70,9 @@ public:
         str << " HardLimit# " << HardLimit;
         str << " Free# " << Free;
         str << " Used# " << GetUsed();
-        str << " CurrentColor# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(EstimateSpaceColor(0)) << "\n";
+        double occupancy;
+        str << " CurrentColor# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(EstimateSpaceColor(0, &occupancy)) << "\n";
+        str << " Occupancy# " << occupancy << "\n";
 #define PRINT_DISK_SPACE_COLOR(NAME) str << " " #NAME "# " << NAME;
         DISK_SPACE_COLORS(PRINT_DISK_SPACE_COLOR)
 #undef PRINT_DISK_SPACE_COLOR
@@ -133,10 +135,12 @@ public:
 
     // Called from any thread
     // TODO(cthulhu): Profile and consider caching
-    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(i64 count) const {
+    NKikimrBlobStorage::TPDiskSpaceColor::E EstimateSpaceColor(i64 count, double *occupancy) const {
         using TColor = NKikimrBlobStorage::TPDiskSpaceColor;
         const i64 newFree = AtomicGet(Free) - count;
 
+        *occupancy = HardLimit ? (double)(HardLimit - newFree) / HardLimit : 1.0;
+
         if (newFree > AtomicGet(Cyan)) {
             return TColor::GREEN;
         } else if (newFree > AtomicGet(LightYellow)) {
diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
index 44bc4c48f6..173d3ff047 100644
--- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
+++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp
@@ -758,6 +758,7 @@ public:
         auto res = std::make_unique<NPDisk::TEvCheckSpaceResult>(NKikimrProto::OK, GetStatusFlags(),
             Impl.GetNumFreeChunks(), Impl.TotalChunks, Impl.TotalChunks - Impl.GetNumFreeChunks(),
             Impl.Owners.size(), TString());
+        res->Occupancy = (double)res->UsedChunks / res->TotalChunks;
         Impl.FindOwner(msg, res); // to ensure correct owner/round
         Send(ev->Sender, res.release());
     }
diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp
index 9a5b692796..0882d457aa 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp
@@ -85,11 +85,8 @@ namespace NKikimr {
 
             TotalChunks = msg->TotalChunks;
             FreeChunks = msg->FreeChunks;
-            const ui64 a = Min(msg->TotalChunks, msg->FreeChunks / Min(1u, msg->NumSlots));
-            const ui64 b = msg->UsedChunks;
-            const ui64 freeSpaceShareMult1000 = ui64(1 << 24) * a / (a + b);
             VCtx->OutOfSpaceState.UpdateLocal(msg->StatusFlags);
-            VCtx->OutOfSpaceState.UpdateLocalFreeSpaceShare(freeSpaceShareMult1000);
+            VCtx->OutOfSpaceState.UpdateLocalFreeSpaceShare(ui64(1 << 24) * (1.0 - msg->Occupancy));
             VCtx->OutOfSpaceState.UpdateLocalUsedChunks(msg->UsedChunks);
             MonGroup.DskTotalBytes() = msg->TotalChunks * PDiskCtx->Dsk->ChunkSize;
             MonGroup.DskFreeBytes() = msg->FreeChunks * PDiskCtx->Dsk->ChunkSize;