YDB-2101 storage handler added check usage when out-of-space filter (#658)

* YDB-2101 storage handler added check usage when out-of-space filter

* remove base64 tests

3 files changed, 160 insertions, 88 deletions

diff --git a/ydb/core/viewer/json_storage.h b/ydb/core/viewer/json_storage.h
index a64fcf8db41..ce5cdb8bbf9 100644
--- a/ydb/core/viewer/json_storage.h
+++ b/ydb/core/viewer/json_storage.h
@@ -54,13 +54,13 @@ class TJsonStorage : public TJsonStorageBase {
         uint64 Read;
         uint64 Write;
 
-        TGroupRow() 
+        TGroupRow()
             : Degraded(0)
             , Usage(0)
             , Used(0)
             , Limit(0)
             , Read(0)
-            , Write(0) 
+            , Write(0)
         {}
     };
     THashMap<TString, TGroupRow> GroupRowsByGroupId;
@@ -245,6 +245,36 @@ public:
         }
     }
 
+    bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) {
+        if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) {
+            return false;
+        }
+        switch (With) {
+            case EWith::MissingDisks:
+                if (BSGroupWithMissingDisks.count(groupId) == 0) {
+                    return false;
+                }
+                break;
+            case EWith::SpaceProblems:
+                if (BSGroupWithSpaceProblems.count(groupId) == 0 && groupRow.Usage < 0.8) {
+                    return false;
+                }
+                break;
+            case EWith::Everything:
+                break;
+        }
+        if (Filter) {
+            if (poolName.Contains(Filter)) {
+                return true;
+            }
+            if (groupId.Contains(Filter)) {
+                return true;
+            }
+            return false;
+        }
+        return true;
+    }
+
     void ReplyAndPassAway() override {
         if (CheckAdditionalNodesInfoNeeded()) {
             return;
@@ -270,19 +300,62 @@ public:
         }
         ui64 foundGroups = 0;
         ui64 totalGroups = 0;
+        TVector<TGroupRow> GroupRows;
         for (const auto& [poolName, poolInfo] : StoragePoolInfo) {
             if ((!FilterTenant.empty() || !FilterStoragePools.empty()) && FilterStoragePools.count(poolName) == 0) {
                 continue;
             }
             NKikimrViewer::TStoragePoolInfo* pool = StorageInfo.AddStoragePools();
             for (TString groupId : poolInfo.Groups) {
+                TGroupRow row;
+                row.PoolName = poolName;
+                row.GroupId = groupId;
+                row.Kind = poolInfo.Kind;
+                auto ib = BSGroupIndex.find(groupId);
+                if (ib != BSGroupIndex.end()) {
+                    row.Erasure = ib->second.GetErasureSpecies();
+                    const auto& vDiskIds = ib->second.GetVDiskIds();
+                    for (auto iv = vDiskIds.begin(); iv != vDiskIds.end(); ++iv) {
+                        const NKikimrBlobStorage::TVDiskID& vDiskId = *iv;
+                        auto ie = VDisksIndex.find(vDiskId);
+                        bool degraded = false;
+                        if (ie != VDisksIndex.end()) {
+                            ui32 nodeId = ie->second.GetNodeId();
+                            ui32 pDiskId = ie->second.GetPDiskId();
+                            degraded |= !ie->second.GetReplicated() || ie->second.GetVDiskState() != NKikimrWhiteboard::EVDiskState::OK;
+                            row.Used += ie->second.GetAllocatedSize();
+                            row.Limit += ie->second.GetAllocatedSize() + ie->second.GetAvailableSize();
+                            row.Read += ie->second.GetReadThroughput();
+                            row.Write += ie->second.GetWriteThroughput();
+
+                            auto ip = PDisksIndex.find(std::make_pair(nodeId, pDiskId));
+                            if (ip != PDisksIndex.end()) {
+                                degraded |= ip->second.GetState() != NKikimrBlobStorage::TPDiskState::Normal;
+                                if (!ie->second.HasAvailableSize()) {
+                                    row.Limit += ip->second.GetAvailableSize();
+                                }
+                            }
+                        }
+                        if (degraded) {
+                            row.Degraded++;
+                        }
+                    }
+                }
+                row.Usage = row.Limit == 0 ? 100 : (float)row.Used / row.Limit;
+
                 ++totalGroups;
-                if (!CheckGroupFilters(groupId, poolName)) {
+                if (!CheckGroupFilters(groupId, poolName, row)) {
                     continue;
                 }
                 ++foundGroups;
                 if (Version == EVersion::v1) {
                     pool->AddGroups()->SetGroupId(groupId);
+                } else if (Version == EVersion::v2) {
+                    if (!UsageBuckets.empty() && !BinarySearch(UsageBuckets.begin(), UsageBuckets.end(), (ui32)(row.Usage * 100) / UsagePace)) {
+                        continue;
+                    }
+                    GroupRows.emplace_back(row);
+                    GroupRowsByGroupId[groupId] = row;
                 }
                 auto itHiveGroup = BSGroupHiveIndex.find(groupId);
                 if (itHiveGroup != BSGroupHiveIndex.end()) {
@@ -309,61 +382,6 @@ public:
         }
 
         if (Version == EVersion::v2) {
-            TVector<TGroupRow> GroupRows;
-            for (const auto& [poolName, poolInfo] : StoragePoolInfo) {
-                if ((!FilterTenant.empty() || !FilterStoragePools.empty()) && FilterStoragePools.count(poolName) == 0) {
-                    continue;
-                }
-                for (TString groupId : poolInfo.Groups) {
-                    if (!CheckGroupFilters(groupId, poolName)) {
-                        continue;
-                    }
-
-                    TGroupRow row;
-                    row.PoolName = poolName;
-                    row.GroupId = groupId;
-                    row.Kind = poolInfo.Kind;
-
-                    auto ib = BSGroupIndex.find(groupId);
-                    if (ib != BSGroupIndex.end()) {
-                        row.Erasure = ib->second.GetErasureSpecies();
-                        const auto& vDiskIds = ib->second.GetVDiskIds();
-                        for (auto iv = vDiskIds.begin(); iv != vDiskIds.end(); ++iv) {
-                            const NKikimrBlobStorage::TVDiskID& vDiskId = *iv;
-                            auto ie = VDisksIndex.find(vDiskId);
-                            bool degraded = false;
-                            if (ie != VDisksIndex.end()) {
-                                ui32 nodeId = ie->second.GetNodeId();
-                                ui32 pDiskId = ie->second.GetPDiskId();
-                                degraded |= !ie->second.GetReplicated() || ie->second.GetVDiskState() != NKikimrWhiteboard::EVDiskState::OK;
-                                row.Used += ie->second.GetAllocatedSize();
-                                row.Limit += ie->second.GetAllocatedSize() + ie->second.GetAvailableSize();
-                                row.Read += ie->second.GetReadThroughput();
-                                row.Write += ie->second.GetWriteThroughput();
-
-                                auto ip = PDisksIndex.find(std::make_pair(nodeId, pDiskId));
-                                if (ip != PDisksIndex.end()) {
-                                    degraded |= ip->second.GetState() != NKikimrBlobStorage::TPDiskState::Normal;
-                                    if (!ie->second.HasAvailableSize()) {
-                                        row.Limit += ip->second.GetAvailableSize();
-                                    }
-                                }
-                            }
-                            if (degraded) {
-                                row.Degraded++;
-                            }
-                        }
-                    }
-
-                    row.Usage = row.Limit == 0 ? 100 : (float)row.Used / row.Limit;
-                    if (!UsageBuckets.empty() && !BinarySearch(UsageBuckets.begin(), UsageBuckets.end(), (ui32)(row.Usage * 100) / UsagePace)) {
-                        continue;
-                    }
-                    GroupRows.emplace_back(row);
-                    GroupRowsByGroupId[groupId] = row;
-                }
-            }
-
             switch (GroupSort) {
                 case EGroupSort::PoolName:
                     SortCollection(GroupRows, [](const TGroupRow& node) { return node.PoolName;}, ReverseSort);
diff --git a/ydb/core/viewer/json_storage_base.h b/ydb/core/viewer/json_storage_base.h
index 47d97678fb9..26046561476 100644
--- a/ydb/core/viewer/json_storage_base.h
+++ b/ydb/core/viewer/json_storage_base.h
@@ -407,36 +407,6 @@ public:
     TList<NKikimrWhiteboard::TPDiskStateInfo> PDisksAppended;
     TList<NKikimrWhiteboard::TVDiskStateInfo> VDisksAppended;
 
-    bool CheckGroupFilters(const TString& groupId, const TString& poolName) {
-        if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) {
-            return false;
-        }
-        switch (With) {
-            case EWith::MissingDisks:
-                if (BSGroupWithMissingDisks.count(groupId) == 0) {
-                    return false;
-                }
-                break;
-            case EWith::SpaceProblems:
-                if (BSGroupWithSpaceProblems.count(groupId) == 0) {
-                    return false;
-                }
-                break;
-            case EWith::Everything:
-                break;
-        }
-        if (Filter) {
-            if (poolName.Contains(Filter)) {
-                return true;
-            }
-            if (groupId.Contains(Filter)) {
-                return true;
-            }
-            return false;
-        }
-        return true;
-    }
-
     bool CheckAdditionalNodesInfoNeeded() {
         if (NeedAdditionalNodesRequests) {
             NeedAdditionalNodesRequests = false;
diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp
index f88f1a6f0d1..40c989d9f65 100644
--- a/ydb/core/viewer/viewer_ut.cpp
+++ b/ydb/core/viewer/viewer_ut.cpp
@@ -2,6 +2,7 @@
 #include <library/cpp/testing/unittest/tests_data.h>
 #include <ydb/library/actors/interconnect/interconnect.h>
 #include <ydb/library/actors/helpers/selfping_actor.h>
+#include <library/cpp/json/json_value.h>
 #include <library/cpp/json/json_reader.h>
 #include <util/stream/null.h>
 #include <ydb/core/viewer/protos/viewer.pb.h>
@@ -291,6 +292,19 @@ Y_UNIT_TEST_SUITE(Viewer) {
         nodeId++;
     }
 
+    void ChangeVDiskStateResponse(TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev, NKikimrWhiteboard::EFlag diskSpace, ui64 used, ui64 limit) {
+        auto& pbRecord = (*ev)->Get()->Record;
+        auto state = pbRecord.add_vdiskstateinfo();
+        state->mutable_vdiskid()->set_vdisk(0);
+        state->mutable_vdiskid()->set_groupid(0);
+        state->mutable_vdiskid()->set_groupgeneration(1);
+        state->set_diskspace(diskSpace);
+        state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK);
+        state->set_nodeid(0);
+        state->set_allocatedsize(used);
+        state->set_availablesize(limit - used);
+    }
+
     void ChangeDescribeSchemeResult(TEvSchemeShard::TEvDescribeSchemeResult::TPtr* ev, int tabletsTotal) {
         auto record = (*ev)->Get()->MutableRecord();
         auto params = record->mutable_pathdescription()->mutable_domaindescription()->mutable_processingparams();
@@ -516,4 +530,74 @@ Y_UNIT_TEST_SUITE(Viewer) {
     {
         QueryTest("select \"Hello\"", false, "Hello");
     }
+
+    void StorageSpaceTest(const TString& withValue, const NKikimrWhiteboard::EFlag diskSpace, const ui64 used, const ui64 limit, const bool isExpectingGroup) {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port);
+        settings.InitKikimrRunConfig()
+                .SetNodeCount(1)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root");
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        THttpRequest httpReq(HTTP_METHOD_GET);
+        httpReq.CgiParameters.emplace("with", withValue);
+        httpReq.CgiParameters.emplace("version", "v2");
+        auto page = MakeHolder<TMonPage>("viewer", "title");
+        TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr);
+        auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
+
+        auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
+            Y_UNUSED(ev);
+            if (ev->GetTypeRewrite() == TEvWhiteboard::EvVDiskStateResponse) {
+                auto *x = reinterpret_cast<TEvWhiteboard::TEvVDiskStateResponse::TPtr*>(&ev);
+                ChangeVDiskStateResponse(x, diskSpace, used, limit);
+            }
+
+            return TTestActorRuntime::EEventAction::PROCESS;
+        };
+        runtime.SetObserverFunc(observerFunc);
+
+        runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
+        NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);
+
+        size_t pos = result->Answer.find('{');
+        TString jsonResult = result->Answer.substr(pos);
+        Ctest << "json result: " << jsonResult << Endl;
+        NJson::TJsonValue json;
+        try {
+            NJson::ReadJsonTree(jsonResult, &json, true);
+        }
+        catch (yexception ex) {
+            Ctest << ex.what() << Endl;
+        }
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().contains("StorageGroups"), isExpectingGroup);
+    }
+
+    Y_UNIT_TEST(StorageGroupOutputWithoutFilterNoDepends)
+    {
+        StorageSpaceTest("all", NKikimrWhiteboard::EFlag::Green, 10, 100, true);
+        StorageSpaceTest("all", NKikimrWhiteboard::EFlag::Red, 90, 100, true);
+    }
+
+    Y_UNIT_TEST(StorageGroupOutputWithSpaceCheckDependsOnVDiskSpaceStatus)
+    {
+        StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 10, 100, false);
+        StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Red, 10, 100, true);
+    }
+
+    Y_UNIT_TEST(StorageGroupOutputWithSpaceCheckDependsOnUsage)
+    {
+        StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 70, 100, false);
+        StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 80, 100, true);
+        StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 90, 100, true);
+    }
 }