storage handler filters

added row initialization

added sort

sorting

added CheckGroupFilters method

2 files changed, 246 insertions, 16 deletions

diff --git a/ydb/core/viewer/json_storage.h b/ydb/core/viewer/json_storage.h
index 652fb9afd32..91c898c428e 100644
--- a/ydb/core/viewer/json_storage.h
+++ b/ydb/core/viewer/json_storage.h
@@ -69,6 +69,7 @@ class TJsonStorage : public TViewerPipeClient<TJsonStorage> {
     TString FilterTenant;
     THashSet<TString> FilterStoragePools;
     TVector<TString> FilterGroupIds;
+    TString Filter;
     std::unordered_set<TNodeId> FilterNodeIds;
     std::unordered_set<TNodeId> NodeIds;
     bool NeedGroups = true;
@@ -88,6 +89,48 @@ class TJsonStorage : public TViewerPipeClient<TJsonStorage> {
     };
 
     EWith With = EWith::Everything;
+    enum class EGroupSort {
+        PoolName,
+        Kind,
+        Erasure,
+        Degraded,
+        Usage,
+        GroupId,
+        Used,
+        Limit,
+        Read,
+        Write
+    };
+    enum class EVersion {
+        v1,
+        v2 // only this works with sorting
+    };
+    EVersion Version = EVersion::v1;
+    EGroupSort GroupSort = EGroupSort::PoolName;
+    bool ReverseSort = false;
+    std::optional<ui32> Offset;
+    std::optional<ui32> Limit;
+
+    struct TGroupRow {
+        TString PoolName;
+        TString GroupId;
+        TString Kind;
+        TString Erasure;
+        ui32 Degraded;
+        float Usage;
+        uint64 Used;
+        uint64 Limit;
+        uint64 Read;
+        uint64 Write;
+
+        TGroupRow() 
+            : Used(0)
+            , Limit(0)
+            , Read(0)
+            , Write(0) 
+        {}
+    };
+    THashMap<TString, TGroupRow> GroupRowsByGroupId;
 
 public:
     static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
@@ -113,6 +156,7 @@ public:
         NeedAdditionalNodesRequests = !FilterNodeIds.empty();
         SplitIds(params.Get("group_id"), ',', FilterGroupIds);
         Sort(FilterGroupIds);
+        Filter = params.Get("filter");
         NeedGroups = FromStringWithDefault<bool>(params.Get("need_groups"), true);
         NeedDisks = FromStringWithDefault<bool>(params.Get("need_disks"), NeedGroups);
         NeedDonors = FromStringWithDefault<bool>(params.Get("need_donors"), NeedDonors);
@@ -122,6 +166,43 @@ public:
         } if (params.Get("with") == "space") {
             With = EWith::SpaceProblems;
         }
+
+        TString version = params.Get("version");
+        if (version == "v1") {
+            Version = EVersion::v1;
+        } else if (version == "v2") {
+            Version = EVersion::v2;
+        }
+        Offset = FromStringWithDefault<ui32>(params.Get("offset"), 0);
+        Limit = FromStringWithDefault<ui32>(params.Get("limit"), std::numeric_limits<ui32>::max());
+        TStringBuf sort = params.Get("sort");
+        if (sort) {
+            if (sort.StartsWith("-") || sort.StartsWith("+")) {
+                ReverseSort = (sort[0] == '-');
+                sort.Skip(1);
+            }
+            if (sort == "PoolName") {
+                GroupSort = EGroupSort::PoolName;
+            } else if (sort == "Kind") {
+                GroupSort = EGroupSort::Kind;
+            } else if (sort == "Erasure") {
+                GroupSort = EGroupSort::Erasure;
+            } else if (sort == "Degraded") {
+                GroupSort = EGroupSort::Degraded;
+            } else if (sort == "Usage") {
+                GroupSort = EGroupSort::Usage;
+            } else if (sort == "GroupId") {
+                GroupSort = EGroupSort::GroupId;
+            } else if (sort == "Used") {
+                GroupSort = EGroupSort::Used;
+            } else if (sort == "Limit") {
+                GroupSort = EGroupSort::Limit;
+            } else if (sort == "Read") {
+                GroupSort = EGroupSort::Read;
+            } else if (sort == "Write") {
+                GroupSort = EGroupSort::Write;
+            }
+        }
     }
 
     void Bootstrap() {
@@ -385,6 +466,18 @@ public:
                     const TJsonSettings& jsonSettings) {
         const auto& info = static_cast<const NKikimrViewer::TStorageGroupInfo&>(protoFrom);
         TString groupId = info.GetGroupId();
+        if (Version == EVersion::v2) {
+            const auto& groupRow = GroupRowsByGroupId[groupId];
+            json << "\"PoolName\":\"" << groupRow.PoolName << "\",";
+            json << "\"Kind\":\"" << groupRow.Kind << "\",";
+            json << "\"Erasure\":\"" << groupRow.Erasure << "\",";
+            json << "\"Degraded\":\"" << groupRow.Degraded << "\",";
+            json << "\"Usage\":\"" << groupRow.Usage << "\",";
+            json << "\"Used\":\"" << groupRow.Used << "\",";
+            json << "\"Limit\":\"" << groupRow.Limit << "\",";
+            json << "\"Read\":\"" << groupRow.Read << "\",";
+            json << "\"Write\":\"" << groupRow.Write << "\",";
+        }
         auto ib = BSGroupIndex.find(groupId);
         if (ib != BSGroupIndex.end()) {
             TProtoToJson::ProtoToJsonInline(json, ib->second, jsonSettings);
@@ -479,6 +572,36 @@ public:
         }
     }
 
+    bool CheckGroupFilters(const TString& groupId, const TString& poolName) {
+        if (!FilterGroupIds.empty() && !BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) {
+            return false;
+        }
+        switch (With) {
+            case EWith::MissingDisks:
+                if (BSGroupWithMissingDisks.count(groupId) == 0) {
+                    return false;
+                }
+                break;
+            case EWith::SpaceProblems:
+                if (BSGroupWithSpaceProblems.count(groupId) == 0) {
+                    return false;
+                }
+                break;
+            case EWith::Everything:
+                break;
+        }
+        if (Filter) {
+            if (poolName.Contains(Filter)) {
+                return true;
+            }
+            if (groupId.Contains(Filter)) {
+                return true;
+            }
+            return false;
+        }
+        return true;
+    }
+
     void ReplyAndPassAway() {
         if (NeedAdditionalNodesRequests) {
             NeedAdditionalNodesRequests = false;
@@ -615,25 +738,13 @@ public:
             NKikimrViewer::TStoragePoolInfo* pool = StorageInfo.AddStoragePools();
             for (TString groupId : poolInfo.Groups) {
                 ++totalGroups;
-                if (!FilterGroupIds.empty() && !BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) {
+                if (!CheckGroupFilters(groupId, poolName)) {
                     continue;
                 }
-                switch (With) {
-                    case EWith::MissingDisks:
-                        if (BSGroupWithMissingDisks.count(groupId) == 0) {
-                            continue;
-                        }
-                        break;
-                    case EWith::SpaceProblems:
-                        if (BSGroupWithSpaceProblems.count(groupId) == 0) {
-                            continue;
-                        }
-                        break;
-                    case EWith::Everything:
-                        break;
-                }
                 ++foundGroups;
-                pool->AddGroups()->SetGroupId(groupId);
+                if (Version == EVersion::v1) {
+                    pool->AddGroups()->SetGroupId(groupId);
+                }
                 auto itHiveGroup = BSGroupHiveIndex.find(groupId);
                 if (itHiveGroup != BSGroupHiveIndex.end()) {
                     pool->SetAcquiredUnits(pool->GetAcquiredUnits() + itHiveGroup->second.GetAcquiredUnits());
@@ -658,6 +769,121 @@ public:
             pool->SetOverall(poolInfo.Overall);
         }
 
+        if (Version == EVersion::v2) {
+            TVector<TGroupRow> GroupRows;
+            for (const auto& [poolName, poolInfo] : StoragePoolInfo) {
+                if ((!FilterTenant.empty() || !FilterStoragePools.empty()) && FilterStoragePools.count(poolName) == 0) {
+                    continue;
+                }
+                for (TString groupId : poolInfo.Groups) {
+                    ++totalGroups;
+                    if (!CheckGroupFilters(groupId, poolName)) {
+                        continue;
+                    }
+                    ++foundGroups;
+
+                    TGroupRow row;
+                    row.PoolName = poolName;
+                    row.GroupId = groupId;
+                    row.Kind = poolInfo.Kind;
+
+                    bool degraded = false;
+                    auto ib = BSGroupIndex.find(groupId);
+                    if (ib != BSGroupIndex.end()) {
+                        row.Erasure = ib->second.GetErasureSpecies();
+                        const auto& vDiskIds = ib->second.GetVDiskIds();
+                        for (auto iv = vDiskIds.begin(); iv != vDiskIds.end(); ++iv) {
+                            const NKikimrBlobStorage::TVDiskID& vDiskId = *iv;
+                            auto ie = VDisksIndex.find(vDiskId);
+                            if (ie != VDisksIndex.end()) {
+                                ui32 nodeId = ie->second.GetNodeId();
+                                ui32 pDiskId = ie->second.GetPDiskId();
+                                degraded |= ie->second.GetReplicated() || ie->second.GetVDiskState() != NKikimrWhiteboard::EVDiskState::OK;
+                                row.Used += ie->second.GetAllocatedSize();
+                                row.Limit += ie->second.GetAllocatedSize() + ie->second.GetAvailableSize();
+                                row.Read += ie->second.GetReadThroughput();
+                                row.Write += ie->second.GetWriteThroughput();
+
+                                auto ip = PDisksIndex.find(std::make_pair(nodeId, pDiskId));
+                                if (ip != PDisksIndex.end()) {
+                                    degraded |= ip->second.GetState() != NKikimrBlobStorage::TPDiskState::Normal;
+                                    if (!ie->second.HasAvailableSize()) {
+                                        row.Limit += ip->second.GetAvailableSize();
+                                    }
+                                }
+                            }   
+                        }
+                    }
+
+                    if (degraded) {
+                        row.Degraded++;
+                    }
+                    row.Usage = (float)row.Used / row.Limit;
+                    GroupRows.emplace_back(row);
+                    GroupRowsByGroupId[groupId] = row;
+                }
+            }
+
+            bool reverse = ReverseSort;
+            switch (GroupSort) {
+                case EGroupSort::PoolName:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.PoolName < b.PoolName);
+                    });
+                    break;
+                case EGroupSort::GroupId:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.GroupId < b.GroupId);
+                    });
+                    break;
+                case EGroupSort::Kind:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Kind < b.Kind);
+                    });
+                    break;
+                case EGroupSort::Erasure:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Erasure < b.Erasure);
+                    });
+                    break;
+                case EGroupSort::Degraded:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Degraded < b.Degraded);
+                    });
+                    break;
+                case EGroupSort::Usage:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Usage < b.Usage);
+                    });
+                    break;
+                case EGroupSort::Used:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Used < b.Used);
+                    });
+                    break;
+                case EGroupSort::Limit:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Limit < b.Limit);
+                    });
+                    break;
+                case EGroupSort::Read:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Read < b.Read);
+                    });
+                    break;
+                case EGroupSort::Write:
+                    ::Sort(GroupRows, [reverse](const TGroupRow& a, const TGroupRow& b) {
+                        return reverse ^ (a.Write < b.Write);
+                    });
+                    break;
+            }
+
+            for (const auto& groupRow: GroupRows) {
+                NKikimrViewer::TStorageGroupInfo* group = StorageInfo.AddStorageGroups();
+                group->SetGroupId(groupRow.GroupId);
+            }
+        }
+
         const FieldDescriptor* field;
         if (NeedGroups) {
             field = NKikimrViewer::TStorageGroupInfo::descriptor()->FindFieldByName("GroupId");
@@ -732,6 +958,9 @@ struct TJsonRequestParameters<TJsonStorage> {
                       {"name":"need_groups","in":"query","description":"return groups information","required":false,"type":"boolean","default":true},
                       {"name":"need_disks","in":"query","description":"return disks information","required":false,"type":"boolean","default":true},
                       {"name":"with","in":"query","description":"filter groups by missing or space","required":false,"type":"string"},
+                      {"name":"sort","in":"query","description":"sort by (PoolName,Type,Erasure,Degraded,Usage,GroupId,Used,Limit,Read,Write)","required":false,"type":"string"},
+                      {"name":"offset","in":"query","description":"skip N nodes","required":false,"type":"integer"},
+                      {"name":"limit","in":"query","description":"limit to N nodes","required":false,"type":"integer"},
                       {"name":"timeout","in":"query","description":"timeout in ms","required":false,"type":"integer"}])___";
     }
 };
diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto
index c559cb03e56..23b3a18f301 100644
--- a/ydb/core/viewer/protos/viewer.proto
+++ b/ydb/core/viewer/protos/viewer.proto
@@ -391,6 +391,7 @@ message TStorageInfo {
     repeated TStoragePoolInfo StoragePools = 2;
     uint64 TotalGroups = 3;
     uint64 FoundGroups = 4;
+    repeated TStorageGroupInfo StorageGroups = 5;
 }
 
 message TNodeInfo {