Add Immediate group stats (#39794)

6 files changed, 160 insertions, 97 deletions

diff --git a/ydb/core/mind/bscontroller/storage_stats_calculator.cpp b/ydb/core/mind/bscontroller/storage_stats_calculator.cpp
index bac7d8492f8..fb2af2ae938 100644
--- a/ydb/core/mind/bscontroller/storage_stats_calculator.cpp
+++ b/ydb/core/mind/bscontroller/storage_stats_calculator.cpp
@@ -30,6 +30,8 @@ private:
 
     struct TExPoison {};
 
+    using TPDiskEntry = std::decay_t<decltype(std::declval<TControllerSystemViewsState>().PDisks)>::value_type;
+
 public:
     TStorageStatsCoroCalculatorImpl(
         const TControllerSystemViewsState& systemViewsState,
@@ -112,8 +114,7 @@ public:
             }
         }
 
-        using T = std::decay_t<decltype(SystemViewsState.PDisks)>::value_type;
-        std::unordered_map<TBoxId, std::vector<const T*>> boxes;
+        std::unordered_map<TBoxId, std::vector<const TPDiskEntry*>> boxes;
         for (const auto& kv : SystemViewsState.PDisks) {
             if (kv.second.HasBoxId()) {
                 boxes[kv.second.GetBoxId()].push_back(&kv);
@@ -125,108 +126,144 @@ public:
             TStringInput s(entry.GetPDiskFilterData());
             Load(&s, filters);
 
+            const TBlobStorageGroupType erasureType(TBlobStorageGroupType::ErasureSpeciesByName(entry.GetErasureSpecies()));
+            const ui32 currentGroupsCreated = entry.GetCurrentGroupsCreated();
+
             for (const auto& [boxId, pdisks] : boxes) {
-                TBlobStorageGroupType type(TBlobStorageGroupType::ErasureSpeciesByName(entry.GetErasureSpecies()));
-                TGroupMapper mapper(TGroupGeometryInfo(type, NKikimrBlobStorage::TGroupGeometry())); // default geometry
+                TStats stats = CalculateStats(erasureType, currentGroupsCreated, filters, pdisks,
+                        /*considerDriveStatus=*/false,
+                        /*considerDecommitStatus=*/true,
+                        /*considerMaintenanceStatus=*/false);
+                TStats immediateStats = CalculateStats(erasureType, currentGroupsCreated, filters, pdisks,
+                        /*considerDriveStatus=*/true,
+                        /*considerDecommitStatus=*/true,
+                        /*considerMaintenanceStatus=*/true);
 
-                for (const auto& kv : pdisks) {
-                    const auto& [pdiskId, pdisk] = *kv;
-                    for (const auto& filter : filters) {
-                        const auto sharedWithOs = pdisk.HasSharedWithOs() ? MakeMaybe(pdisk.GetSharedWithOs()) : Nothing();
-                        const auto readCentric = pdisk.HasReadCentric() ? MakeMaybe(pdisk.GetReadCentric()) : Nothing();
-                        if (filter.MatchPDisk(pdisk.GetCategory(), sharedWithOs, readCentric)) {
-                            const TNodeLocation& location = HostRecordMap->GetLocation(pdiskId.NodeId);
-                            const bool usable = pdisk.GetDecommitStatus() == "DECOMMIT_NONE";
-                            const bool ok = mapper.RegisterPDisk({
-                                .PDiskId = pdiskId,
-                                .Location = location,
-                                .Usable = usable,
-                                .NumSlots = pdisk.GetNumActiveSlots(),
-                                .MaxSlots = pdisk.GetExpectedSlotCount(), // either inferred or user-defined
-                                .SlotSizeInUnits = pdisk.GetSlotSizeInUnits(), // either inferred or user-defined
-                                .Groups = {},
-                                .SpaceAvailable = 0,
-                                .Operational = true,
-                                .Decommitted = false, // this flag applies only to group reconfiguration
-                            });
-                            Y_ABORT_UNLESS(ok);
-                            break;
-                        }
-                    }
-                }
+                entry.SetAvailableGroupsToCreate(entry.GetAvailableGroupsToCreate() + stats.GroupsToCreate);
+                entry.SetAvailableSizeToCreate(entry.GetAvailableSizeToCreate() + stats.SizeToCreate);
 
-                // calculate number of groups we can create without accounting reserve
-                TGroupMapper::TGroupDefinition group;
-                TGroupMapperError error;
-                std::deque<ui64> groupSizes;
-                while (mapper.AllocateGroup(groupSizes.size(), group, {}, {}, 1u, 0, false, {}, error)) {
-                    std::vector<TGroupDiskInfo> disks;
-                    std::deque<NKikimrBlobStorage::TPDiskMetrics> pdiskMetrics;
-                    std::deque<NKikimrBlobStorage::TVDiskMetrics> vdiskMetrics;
+                entry.SetImmediateGroupsToCreate(entry.GetImmediateGroupsToCreate() + immediateStats.GroupsToCreate);
+                entry.SetImmediateSizeToCreate(entry.GetImmediateSizeToCreate() + immediateStats.SizeToCreate);
+            }
+        }
 
-                    for (const auto& realm : group) {
-                        for (const auto& domain : realm) {
-                            for (const auto& pdiskId : domain) {
-                                if (const auto it = SystemViewsState.PDisks.find(pdiskId); it != SystemViewsState.PDisks.end()) {
-                                    const NKikimrSysView::TPDiskInfo& pdisk = it->second;
-                                    auto& pm = *pdiskMetrics.emplace(pdiskMetrics.end());
-                                    auto& vm = *vdiskMetrics.emplace(vdiskMetrics.end());
-                                    if (pdisk.HasTotalSize()) {
-                                        pm.SetTotalSize(pdisk.GetTotalSize());
-                                    }
-                                    if (pdisk.HasEnforcedDynamicSlotSize()) {
-                                        pm.SetEnforcedDynamicSlotSize(pdisk.GetEnforcedDynamicSlotSize());
-                                    }
-                                    vm.SetAllocatedSize(0);
-                                    disks.push_back({&pm, &vm, pdisk.GetExpectedSlotCount()});
-                                }
-                            }
-                        }
-                    }
+        Send(ParentActorId, new TEvCalculateStorageStatsResponse(std::move(storageStats)));
+    }
 
-                    NKikimrSysView::TGroupInfo groupInfo;
-                    CalculateGroupUsageStats(&groupInfo, disks, type);
-                    groupSizes.push_back(groupInfo.GetAvailableSize());
+private:
+    struct TStats {
+        ui32 GroupsToCreate;
+        ui64 SizeToCreate;
+    };
+
+private:
+    void Yield() {
+        Send(new IEventHandle(EvResume, 0, SelfActorId, {}, nullptr, 0));
+        WaitForSpecificEvent([](IEventHandle& ev) { return ev.Type == EvResume; }, &TStorageStatsCoroCalculatorImpl::ProcessUnexpectedEvent);
+    }
 
-                    group.clear();
+    TStats CalculateStats(
+            TBlobStorageGroupType erasureType,
+            ui32 currentGroupsCreated,
+            const TSet<TBlobStorageController::TStoragePoolInfo::TPDiskFilter>& filters,
+            const std::vector<const TPDiskEntry*>& pdisks,
+            bool considerDriveStatus,
+            bool considerDecommitStatus,
+            bool considerMaintenanceStatus) {
+        TGroupMapper mapper(TGroupGeometryInfo(erasureType, NKikimrBlobStorage::TGroupGeometry())); // default geometry
 
-                    Yield();
+        for (const auto& kv : pdisks) {
+            const auto& [pdiskId, pdisk] = *kv;
+            for (const auto& filter : filters) {
+                const auto sharedWithOs = pdisk.HasSharedWithOs() ? MakeMaybe(pdisk.GetSharedWithOs()) : Nothing();
+                const auto readCentric = pdisk.HasReadCentric() ? MakeMaybe(pdisk.GetReadCentric()) : Nothing();
+                if (filter.MatchPDisk(pdisk.GetCategory(), sharedWithOs, readCentric)) {
+                    const TNodeLocation& location = HostRecordMap->GetLocation(pdiskId.NodeId);
+                    const bool usable = (!considerDriveStatus || !pdisk.HasStatusV2() || pdisk.GetStatusV2() == "ACTIVE") &&
+                            (!considerDecommitStatus || !pdisk.HasDecommitStatus() || pdisk.GetDecommitStatus() == "DECOMMIT_NONE") &&
+                            (!considerMaintenanceStatus || !pdisk.HasMaintenanceStatus() || pdisk.GetMaintenanceStatus() == "NO_REQUEST");
+                    const bool ok = mapper.RegisterPDisk({
+                        .PDiskId = pdiskId,
+                        .Location = location,
+                        .Usable = usable,
+                        .NumSlots = pdisk.GetNumActiveSlots(),
+                        .MaxSlots = pdisk.GetExpectedSlotCount(), // either inferred or user-defined
+                        .SlotSizeInUnits = pdisk.GetSlotSizeInUnits(), // either inferred or user-defined
+                        .Groups = {},
+                        .SpaceAvailable = 0,
+                        .Operational = true,
+                        .Decommitted = false, // this flag applies only to group reconfiguration
+                    });
+                    Y_ABORT_UNLESS(ok);
+                    break;
                 }
+            }
+        }
 
-                std::sort(groupSizes.begin(), groupSizes.end());
+        // calculate number of groups we can create without accounting reserve
+        TGroupMapper::TGroupDefinition group;
+        TGroupMapperError error;
+        std::deque<ui64> groupSizes;
+        while (mapper.AllocateGroup(groupSizes.size(), group, {}, {}, 1u, 0, false, {}, error)) {
+            std::vector<TGroupDiskInfo> disks;
+            std::deque<NKikimrBlobStorage::TPDiskMetrics> pdiskMetrics;
+            std::deque<NKikimrBlobStorage::TVDiskMetrics> vdiskMetrics;
 
-                // adjust it according to reserve
-                const ui32 total = groupSizes.size() + entry.GetCurrentGroupsCreated();
-                ui32 reserve = GroupReserveMin;
-                while (reserve < groupSizes.size() && (reserve - GroupReserveMin) * 1000000 / total < GroupReservePart) {
-                    ++reserve;
+            for (const auto& realm : group) {
+                for (const auto& domain : realm) {
+                    for (const auto& pdiskId : domain) {
+                        if (const auto it = SystemViewsState.PDisks.find(pdiskId); it != SystemViewsState.PDisks.end()) {
+                            const NKikimrSysView::TPDiskInfo& pdisk = it->second;
+                            auto& pm = *pdiskMetrics.emplace(pdiskMetrics.end());
+                            auto& vm = *vdiskMetrics.emplace(vdiskMetrics.end());
+                            if (pdisk.HasTotalSize()) {
+                                pm.SetTotalSize(pdisk.GetTotalSize());
+                            }
+                            if (pdisk.HasEnforcedDynamicSlotSize()) {
+                                pm.SetEnforcedDynamicSlotSize(pdisk.GetEnforcedDynamicSlotSize());
+                            }
+                            vm.SetAllocatedSize(0);
+                            disks.push_back({&pm, &vm, pdisk.GetExpectedSlotCount()});
+                        }
+                    }
                 }
-                reserve = Min<ui32>(reserve, groupSizes.size());
+            }
 
-                // cut sizes
-                while (reserve >= 2) {
-                    groupSizes.pop_front();
-                    groupSizes.pop_back();
-                    reserve -= 2;
-                }
+            NKikimrSysView::TGroupInfo groupInfo;
+            CalculateGroupUsageStats(&groupInfo, disks, erasureType);
+            groupSizes.push_back(groupInfo.GetAvailableSize());
 
-                if (reserve) {
-                    groupSizes.pop_front();
-                }
+            group.clear();
 
-                entry.SetAvailableGroupsToCreate(entry.GetAvailableGroupsToCreate() + groupSizes.size());
-                entry.SetAvailableSizeToCreate(entry.GetAvailableSizeToCreate() + std::accumulate(groupSizes.begin(),
-                    groupSizes.end(), ui64(0)));
-            }
+            Yield();
         }
 
-        Send(ParentActorId, new TEvCalculateStorageStatsResponse(std::move(storageStats)));
-    }
+        std::sort(groupSizes.begin(), groupSizes.end());
 
-private:
-    void Yield() {
-        Send(new IEventHandle(EvResume, 0, SelfActorId, {}, nullptr, 0));
-        WaitForSpecificEvent([](IEventHandle& ev) { return ev.Type == EvResume; }, &TStorageStatsCoroCalculatorImpl::ProcessUnexpectedEvent);
+        // adjust it according to reserve
+        const ui32 total = static_cast<ui32>(groupSizes.size()) + currentGroupsCreated;
+        ui32 reserve = GroupReserveMin;
+        while (reserve < groupSizes.size() && (reserve - GroupReserveMin) * 1000000 / total < GroupReservePart) {
+            ++reserve;
+        }
+        reserve = Min<ui32>(reserve, groupSizes.size());
+
+        // cut sizes
+        while (reserve >= 2) {
+            groupSizes.pop_front();
+            groupSizes.pop_back();
+            reserve -= 2;
+        }
+
+        if (reserve) {
+            groupSizes.pop_front();
+        }
+
+        return TStats{
+            .GroupsToCreate = static_cast<ui32>(groupSizes.size()),
+            .SizeToCreate = std::accumulate(groupSizes.begin(), groupSizes.end(),
+                    static_cast<ui64>(0))
+        };
     }
 
 private:
diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp
index 535d20e165a..616b7dcb168 100644
--- a/ydb/core/mind/bscontroller/sys_view.cpp
+++ b/ydb/core/mind/bscontroller/sys_view.cpp
@@ -267,6 +267,10 @@ public:
             erasureGroup->GetCounter("CurrentAvailableSize")->Set(entry.GetCurrentAvailableSize());
             erasureGroup->GetCounter("AvailableGroupsToCreate")->Set(entry.GetAvailableGroupsToCreate());
             erasureGroup->GetCounter("AvailableSizeToCreate")->Set(entry.GetAvailableSizeToCreate());
+            erasureGroup->GetCounter("ImmediateGroupsToCreate")->Set(entry.HasImmediateGroupsToCreate()
+                    ? entry.GetImmediateGroupsToCreate() : 0);
+            erasureGroup->GetCounter("ImmediateSizeToCreate")->Set(entry.HasImmediateSizeToCreate()
+                    ? entry.GetImmediateSizeToCreate() : 0);
         }
 
         // remove no longer present entries
diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto
index b415324e324..0b3cff0418d 100644
--- a/ydb/core/protos/sys_view.proto
+++ b/ydb/core/protos/sys_view.proto
@@ -364,8 +364,17 @@ message TStorageStatsEntry {
     optional uint32 CurrentGroupsCreated = 4;
     optional uint64 CurrentAllocatedSize = 5;
     optional uint64 CurrentAvailableSize = 6;
+
+    // estimated number of groups BSC is able to create if DriveStatus and MaintenanceStatus are ignored
     optional uint32 AvailableGroupsToCreate = 7;
-    optional uint64 AvailableSizeToCreate = 8; // total space of newly created groups, if they would've been created
+    // estimated total space of groups BSC is able to create if DriveStatus and MaintenanceStatus are ignored
+    optional uint64 AvailableSizeToCreate = 8;
+
+    // estimated number of groups BSC is able to create
+    optional uint32 ImmediateGroupsToCreate = 10;
+    // estimated total space of groups BSC is able to create
+    optional uint64 ImmediateSizeToCreate = 11;
+
     optional bytes PDiskFilterData = 9;
 }
 
diff --git a/ydb/core/sys_view/common/registry.h b/ydb/core/sys_view/common/registry.h
index 828c17cd22e..9aecddd3603 100644
--- a/ydb/core/sys_view/common/registry.h
+++ b/ydb/core/sys_view/common/registry.h
@@ -540,10 +540,13 @@ struct Schema : NIceDb::Schema {
         struct CurrentAvailableSize    : Column<6, NScheme::NTypeIds::Uint64> {};
         struct AvailableGroupsToCreate : Column<7, NScheme::NTypeIds::Uint32> {};
         struct AvailableSizeToCreate   : Column<8, NScheme::NTypeIds::Uint64> {};
+        struct ImmediateGroupsToCreate : Column<9, NScheme::NTypeIds::Uint32> {};
+        struct ImmediateSizeToCreate   : Column<10, NScheme::NTypeIds::Uint64> {};
 
         using TKey = TableKey<PDiskFilter, ErasureSpecies>;
         using TColumns = TableColumns<PDiskFilter, ErasureSpecies, CurrentGroupsCreated, CurrentAllocatedSize,
-                                      CurrentAvailableSize, AvailableGroupsToCreate, AvailableSizeToCreate>;
+                                      CurrentAvailableSize, AvailableGroupsToCreate, AvailableSizeToCreate,
+                                      ImmediateGroupsToCreate, ImmediateSizeToCreate>;
     };
 
     struct TopPartitions : Table<12> {
diff --git a/ydb/core/sys_view/storage/storage_stats.cpp b/ydb/core/sys_view/storage/storage_stats.cpp
index 046cbba1464..19b90533466 100644
--- a/ydb/core/sys_view/storage/storage_stats.cpp
+++ b/ydb/core/sys_view/storage/storage_stats.cpp
@@ -24,6 +24,8 @@ public:
             {T::CurrentAvailableSize::ColumnId, {E::kCurrentAvailableSizeFieldNumber}},
             {T::AvailableGroupsToCreate::ColumnId, {E::kAvailableGroupsToCreateFieldNumber}},
             {T::AvailableSizeToCreate::ColumnId, {E::kAvailableSizeToCreateFieldNumber}},
+            {T::ImmediateGroupsToCreate::ColumnId, {E::kImmediateGroupsToCreateFieldNumber}},
+            {T::ImmediateSizeToCreate::ColumnId, {E::kImmediateSizeToCreateFieldNumber}},
         };
         return fieldMap;
     }
diff --git a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json
index 49466b9d34d..ead49d966f8 100644
--- a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json
+++ b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json
@@ -440,14 +440,6 @@
   "ds_storage_stats": {
     "columns": [
       {
-        "name": "AvailableGroupsToCreate",
-        "type": "Uint32?"
-      },
-      {
-        "name": "AvailableSizeToCreate",
-        "type": "Uint64?"
-      },
-      {
         "name": "PDiskFilter",
         "type": "Utf8?"
       },
@@ -466,6 +458,22 @@
       {
         "name": "CurrentAvailableSize",
         "type": "Uint64?"
+      },
+      {
+        "name": "AvailableGroupsToCreate",
+        "type": "Uint32?"
+      },
+      {
+        "name": "AvailableSizeToCreate",
+        "type": "Uint64?"
+      },
+      {
+        "name": "ImmediateGroupsToCreate",
+        "type": "Uint32?"
+      },
+      {
+        "name": "ImmediateSizeToCreate",
+        "type": "Uint64?"
       }
     ],
     "primary_key": [