several fixes for groups and nodes handlers (#9681)

4 files changed, 326 insertions, 239 deletions

diff --git a/ydb/core/viewer/json_handlers_storage.cpp b/ydb/core/viewer/json_handlers_storage.cpp
index ba8c9142c0..fa99923cea 100644
--- a/ydb/core/viewer/json_handlers_storage.cpp
+++ b/ydb/core/viewer/json_handlers_storage.cpp
@@ -4,7 +4,7 @@
 namespace NKikimr::NViewer {
 
 void InitStorageGroupsJsonHandler(TJsonHandlers& jsonHandlers) {
-    jsonHandlers.AddHandler("/storage/groups", new TJsonHandler<TStorageGroups>(TStorageGroups::GetSwagger()), 4);
+    jsonHandlers.AddHandler("/storage/groups", new TJsonHandler<TStorageGroups>(TStorageGroups::GetSwagger()), 5);
 }
 
 void InitStorageJsonHandlers(TJsonHandlers& jsonHandlers) {
diff --git a/ydb/core/viewer/json_handlers_viewer.cpp b/ydb/core/viewer/json_handlers_viewer.cpp
index 4e370e4b36..9e4ae55f21 100644
--- a/ydb/core/viewer/json_handlers_viewer.cpp
+++ b/ydb/core/viewer/json_handlers_viewer.cpp
@@ -243,7 +243,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) {
 }
 
 void InitViewerNodesJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 6);
+    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 7);
 }
 
 void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) {
diff --git a/ydb/core/viewer/storage_groups.h b/ydb/core/viewer/storage_groups.h
index 8b647f2c41..1071dfc4d7 100644
--- a/ydb/core/viewer/storage_groups.h
+++ b/ydb/core/viewer/storage_groups.h
@@ -123,6 +123,7 @@ public:
 
     // Common
     std::unordered_map<TPathId, TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> NavigateKeySetResult;
+    ui64 NavigateKeySetInFlight = 0;
     std::unordered_map<TPathId, TTabletId> PathId2HiveId;
     std::unordered_map<TTabletId, TRequestResponse<TEvHive::TEvResponseHiveStorageStats>> HiveStorageStats;
     ui64 HiveStorageStatsInFlight = 0;
@@ -588,6 +589,15 @@ public:
     ui64 FoundGroups = 0;
     std::vector<TString> Problems;
 
+    void AddProblem(const TString& problem) {
+        for (const auto& p : Problems) {
+            if (p == problem) {
+                return;
+            }
+        }
+        Problems.push_back(problem);
+    }
+
     static EGroupFields ParseEGroupFields(TStringBuf field) {
         EGroupFields result = EGroupFields::COUNT;
         if (field == "PoolName") {
@@ -759,6 +769,7 @@ public:
         if (Database) {
             if (!DatabaseNavigateResponse) {
                 DatabaseNavigateResponse = MakeRequestSchemeCacheNavigate(Database, 0);
+                ++NavigateKeySetInFlight;
             } else {
                 auto pathId = GetPathId(DatabaseNavigateResponse->GetRef());
                 auto result = NavigateKeySetResult.emplace(pathId, std::move(*DatabaseNavigateResponse));
@@ -824,65 +835,98 @@ public:
                 return;
             }
         }
-        if (NeedFilter) {
-            if (!FilterGroupIds.empty()) {
+        // group id pre-filter, affects TotalGroups count
+        if (!FilterGroupIds.empty()) {
+            TGroupView groupView;
+            for (TGroup* group : GroupView) {
+                if (FilterGroupIds.count(group->GroupId)) {
+                    groupView.push_back(group);
+                }
+            }
+            GroupView.swap(groupView);
+            FoundGroups = TotalGroups = GroupView.size();
+            FilterGroupIds.clear();
+            GroupsByGroupId.clear();
+        }
+        // storage pool pre-filter, affects TotalGroups count
+        if (!FilterStoragePools.empty()) {
+            if (FieldsAvailable.test(+EGroupFields::PoolName)) {
                 TGroupView groupView;
                 for (TGroup* group : GroupView) {
-                    if (FilterGroupIds.count(group->GroupId)) {
+                    if (FilterStoragePools.count(group->PoolName)) {
                         groupView.push_back(group);
                     }
                 }
                 GroupView.swap(groupView);
-                FilterGroupIds.clear();
+                FoundGroups = TotalGroups = GroupView.size();
+                FilterStoragePools.clear();
                 GroupsByGroupId.clear();
+            } else {
+                return;
             }
-            if (!FilterStoragePools.empty() && FieldsAvailable.test(+EGroupFields::PoolName)) {
+        }
+        // node_id + pdisk_id pre-filter, affects TotalGroups count
+        if (!FilterNodeIds.empty() && !FilterPDiskIds.empty()) {
+            if (FieldsAvailable.test(+EGroupFields::NodeId) && FieldsAvailable.test(+EGroupFields::PDiskId)) {
                 TGroupView groupView;
                 for (TGroup* group : GroupView) {
-                    if (FilterStoragePools.count(group->PoolName)) {
-                        groupView.push_back(group);
+                    for (const auto& vdisk : group->VDisks) {
+                        if (FilterNodeIds.count(vdisk.VSlotId.NodeId) && FilterPDiskIds.count(vdisk.VSlotId.PDiskId)) {
+                            groupView.push_back(group);
+                            break;
+                        }
                     }
                 }
                 GroupView.swap(groupView);
-                FilterStoragePools.clear();
+                FoundGroups = TotalGroups = GroupView.size();
+                FilterNodeIds.clear();
+                FilterPDiskIds.clear();
                 GroupsByGroupId.clear();
+            } else {
+                return;
             }
-            if (!FilterNodeIds.empty() && FieldsAvailable.test(+EGroupFields::NodeId)) {
+        }
+        // node_id pre-filter, affects TotalGroups count
+        if (!FilterNodeIds.empty()) {
+            if (FieldsAvailable.test(+EGroupFields::NodeId)) {
                 TGroupView groupView;
                 for (TGroup* group : GroupView) {
-                    bool found = false;
                     for (const auto& vdisk : group->VDisks) {
                         if (FilterNodeIds.count(vdisk.VSlotId.NodeId)) {
-                            found = true;
+                            groupView.push_back(group);
                             break;
                         }
                     }
-                    if (found) {
-                        groupView.push_back(group);
-                    }
                 }
                 GroupView.swap(groupView);
+                FoundGroups = TotalGroups = GroupView.size();
                 FilterNodeIds.clear();
                 GroupsByGroupId.clear();
+            } else {
+                return;
             }
-            if (!FilterPDiskIds.empty() && FieldsAvailable.test(+EGroupFields::PDiskId)) {
+        }
+        // pdisk_id pre-filter, affects TotalGroups count
+        if (!FilterPDiskIds.empty()) {
+            if (FieldsAvailable.test(+EGroupFields::PDiskId)) {
                 TGroupView groupView;
                 for (TGroup* group : GroupView) {
-                    bool found = false;
                     for (const auto& vdisk : group->VDisks) {
                         if (FilterPDiskIds.count(vdisk.VSlotId.PDiskId)) {
-                            found = true;
+                            groupView.push_back(group);
                             break;
                         }
                     }
-                    if (found) {
-                        groupView.push_back(group);
-                    }
                 }
                 GroupView.swap(groupView);
+                FoundGroups = TotalGroups = GroupView.size();
                 FilterPDiskIds.clear();
                 GroupsByGroupId.clear();
+            } else {
+                return;
             }
+        }
+        if (NeedFilter) {
             if (With == EWith::MissingDisks && FieldsAvailable.test(+EGroupFields::MissingDisks)) {
                 TGroupView groupView;
                 for (TGroup* group : GroupView) {
@@ -1102,6 +1146,7 @@ public:
                 if (NavigateKeySetResult.count(pathId) == 0) {
                     ui64 cookie = NavigateKeySetResult.size();
                     NavigateKeySetResult.emplace(pathId, MakeRequestSchemeCacheNavigate(pathId, cookie));
+                    ++NavigateKeySetInFlight;
                 }
             }
         }
@@ -1114,71 +1159,6 @@ public:
         }
     }
 
-    void ProcessNavigate(TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult>& navigateResult, bool firstNavigate) {
-        if (navigateResult.IsOk()) {
-            TString path = CanonizePath(navigateResult->Request->ResultSet.begin()->Path);
-            TIntrusiveConstPtr<TSchemeCacheNavigate::TDomainDescription> domainDescription = navigateResult->Request->ResultSet.begin()->DomainDescription;
-            TIntrusiveConstPtr<NSchemeCache::TDomainInfo> domainInfo = navigateResult->Request->ResultSet.begin()->DomainInfo;
-            if (domainInfo != nullptr && domainDescription != nullptr) {
-                if (FieldsNeeded(FieldsHive)) {
-                    TTabletId hiveId = domainInfo->Params.GetHive();
-                    if (hiveId != 0 && HiveStorageStats.count(hiveId) == 0) {
-                        HiveStorageStats.emplace(hiveId, MakeRequestHiveStorageStats(hiveId));
-                        ++HiveStorageStatsInFlight;
-                    }
-                }
-                if (Database && firstNavigate) { // filter by database
-                    for (const auto& storagePool : domainDescription->Description.GetStoragePools()) {
-                        TString storagePoolName = storagePool.GetName();
-                        DatabaseStoragePools.emplace(storagePoolName);
-                    }
-                    NeedFilter = true;
-                }
-            }
-        }
-    }
-
-    void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
-        bool firstNavigate = (ev->Cookie == 0);
-        TPathId pathId = GetPathId(ev);
-        if (firstNavigate && DatabaseNavigateResponse && pathId) {
-            NavigateKeySetResult.emplace(pathId, std::move(*DatabaseNavigateResponse));
-        }
-        auto itNavigateKeySetResult = NavigateKeySetResult.find(pathId);
-        if (itNavigateKeySetResult == NavigateKeySetResult.end()) {
-            BLOG_W("Invalid NavigateKeySetResult PathId: " << pathId << " Path: " << CanonizePath(ev->Get()->Request->ResultSet.begin()->Path));
-            return RequestDone();
-        }
-        auto& navigateResult(itNavigateKeySetResult->second);
-        navigateResult.Set(std::move(ev));
-        ProcessNavigate(navigateResult, firstNavigate);
-        RequestDone();
-    }
-
-    void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) {
-        auto itHiveStorageStats = HiveStorageStats.find(ev->Cookie);
-        if (itHiveStorageStats != HiveStorageStats.end()) {
-            itHiveStorageStats->second.Set(std::move(ev));
-            if (GroupsByGroupId.empty()) {
-                RebuildGroupsByGroupId();
-            }
-            for (const auto& pbPool : itHiveStorageStats->second->Record.GetPools()) {
-                for (const auto& pbGroup : pbPool.GetGroups()) {
-                    auto itGroup = GroupsByGroupId.find(pbGroup.GetGroupID());
-                    if (itGroup != GroupsByGroupId.end()) {
-                        itGroup->second->AllocationUnits += pbGroup.GetAcquiredUnits();
-                    }
-                }
-            }
-
-        }
-        if (--HiveStorageStatsInFlight == 0) {
-            FieldsAvailable |= FieldsHive;
-            ApplyEverything();
-        }
-        RequestDone();
-    }
-
     static TString GetMediaType(const TString& mediaType) {
         if (mediaType.StartsWith("Type:")) {
             return mediaType.substr(5);
@@ -1204,133 +1184,152 @@ public:
     }
 
     bool AreBSControllerRequestsDone() const {
-        return (!GetGroupsResponse || GetGroupsResponse->IsDone()) &&
-               (!GetStoragePoolsResponse || GetStoragePoolsResponse->IsDone()) &&
-               (!GetVSlotsResponse || GetVSlotsResponse->IsDone()) &&
-               (!GetPDisksResponse || GetPDisksResponse->IsDone());
+        return !GetGroupsResponse && !GetStoragePoolsResponse && !GetVSlotsResponse && !GetPDisksResponse;
     }
 
-    void ProcessBSControllerResponses() {
-        int requestsDone = 0;
-        if (GetGroupsResponse && GetGroupsResponse->IsOk() && FieldsNeeded(FieldsBsGroups)) {
-            GroupData.reserve(GetGroupsResponse->Get()->Record.EntriesSize());
-            for (const NKikimrSysView::TGroupEntry& entry : GetGroupsResponse->Get()->Record.GetEntries()) {
-                const NKikimrSysView::TGroupInfo& info = entry.GetInfo();
-                TGroup& group = GroupData.emplace_back();
-                group.GroupId = entry.GetKey().GetGroupId();
-                group.GroupGeneration = info.GetGeneration();
-                group.BoxId = info.GetBoxId();
-                group.PoolId = info.GetStoragePoolId();
-                group.Erasure = info.GetErasureSpeciesV2();
-                group.ErasureSpecies = TErasureType::ErasureSpeciesByName(group.Erasure);
-                //group.Used = info.GetAllocatedSize();
-                //group.Limit = info.GetAllocatedSize() + info.GetAvailableSize();
-                //group.Usage = group.Limit ? 100.0 * group.Used / group.Limit : 0;
-                group.PutTabletLogLatency = info.GetPutTabletLogLatency();
-                group.PutUserDataLatency = info.GetPutUserDataLatency();
-                group.GetFastLatency = info.GetGetFastLatency();
-            }
-            for (TGroup& group : GroupData) {
-                GroupView.emplace_back(&group);
+    bool TimeToAskWhiteboard() const {
+        return AreBSControllerRequestsDone() &&
+               NavigateKeySetInFlight == 0 &&
+               HiveStorageStatsInFlight == 0;
+    }
+
+    void ProcessResponses() {
+        AddEvent("ProcessResponses");
+        if (GetGroupsResponse && GetGroupsResponse->IsDone()) {
+            if (GetGroupsResponse->IsOk()) {
+                GroupData.reserve(GetGroupsResponse->Get()->Record.EntriesSize());
+                for (const NKikimrSysView::TGroupEntry& entry : GetGroupsResponse->Get()->Record.GetEntries()) {
+                    const NKikimrSysView::TGroupInfo& info = entry.GetInfo();
+                    TGroup& group = GroupData.emplace_back();
+                    group.GroupId = entry.GetKey().GetGroupId();
+                    group.GroupGeneration = info.GetGeneration();
+                    group.BoxId = info.GetBoxId();
+                    group.PoolId = info.GetStoragePoolId();
+                    group.Erasure = info.GetErasureSpeciesV2();
+                    group.ErasureSpecies = TErasureType::ErasureSpeciesByName(group.Erasure);
+                    //group.Used = info.GetAllocatedSize();
+                    //group.Limit = info.GetAllocatedSize() + info.GetAvailableSize();
+                    //group.Usage = group.Limit ? 100.0 * group.Used / group.Limit : 0;
+                    group.PutTabletLogLatency = info.GetPutTabletLogLatency();
+                    group.PutUserDataLatency = info.GetPutUserDataLatency();
+                    group.GetFastLatency = info.GetGetFastLatency();
+                }
+                for (TGroup& group : GroupData) {
+                    GroupView.emplace_back(&group);
+                }
+                GroupsByGroupId.clear();
+                FoundGroups = TotalGroups = GroupView.size();
+                FieldsAvailable |= FieldsBsGroups;
+                ApplyEverything();
+            } else {
+                AddProblem("bsc-storage-groups-no-data");
             }
-            GroupsByGroupId.clear();
-            FoundGroups = TotalGroups = GroupView.size();
-            FieldsAvailable |= FieldsBsGroups;
-            ApplyEverything();
-            ++requestsDone;
+            GetGroupsResponse.reset();
         }
-        if (FieldsAvailable.test(+EGroupFields::GroupId) && GetStoragePoolsResponse && GetStoragePoolsResponse->IsOk() && FieldsNeeded(FieldsBsPools)) {
-            std::unordered_map<std::pair<ui64, ui64>, const NKikimrSysView::TStoragePoolInfo*> indexStoragePool; // (box, id) -> pool
-            for (const NKikimrSysView::TStoragePoolEntry& entry : GetStoragePoolsResponse->Get()->Record.GetEntries()) {
-                const auto& key = entry.GetKey();
-                const NKikimrSysView::TStoragePoolInfo& pool = entry.GetInfo();
-                indexStoragePool.emplace(std::make_pair(key.GetBoxId(), key.GetStoragePoolId()), &pool);
-            }
-            ui64 rootSchemeshardId = AppData()->DomainsInfo->Domain->SchemeRoot;
-            for (TGroup* group : GroupView) {
-                if (group->BoxId == 0 && group->PoolId == 0) {
-                    group->PoolName = "static";
-                    group->Kind = ""; // TODO ?
-                    group->MediaType = ""; // TODO ?
-                    group->SchemeShardId = rootSchemeshardId;
-                    group->PathId = 1;
-                    group->EncryptionMode = 0; // TODO ?
-                } else {
-                    auto itStoragePool = indexStoragePool.find({group->BoxId, group->PoolId});
-                    if (itStoragePool != indexStoragePool.end()) {
-                        const NKikimrSysView::TStoragePoolInfo* pool = itStoragePool->second;
-                        group->PoolName = pool->GetName();
-                        group->Kind = pool->GetKind();
-                        group->SchemeShardId = pool->GetSchemeshardId();
-                        group->PathId = pool->GetPathId();
-                        group->MediaType = GetMediaType(pool->GetPDiskFilter());
-                        if (!group->Erasure) {
-                            group->Erasure = pool->GetErasureSpeciesV2();
-                            group->ErasureSpecies = TErasureType::ErasureSpeciesByName(group->Erasure);
-                        }
-                        group->EncryptionMode = pool->GetEncryptionMode();
+        if (FieldsAvailable.test(+EGroupFields::GroupId) && GetStoragePoolsResponse && GetStoragePoolsResponse->IsDone()) {
+            if (GetStoragePoolsResponse->IsOk()) {
+                std::unordered_map<std::pair<ui64, ui64>, const NKikimrSysView::TStoragePoolInfo*> indexStoragePool; // (box, id) -> pool
+                for (const NKikimrSysView::TStoragePoolEntry& entry : GetStoragePoolsResponse->Get()->Record.GetEntries()) {
+                    const auto& key = entry.GetKey();
+                    const NKikimrSysView::TStoragePoolInfo& pool = entry.GetInfo();
+                    indexStoragePool.emplace(std::make_pair(key.GetBoxId(), key.GetStoragePoolId()), &pool);
+                }
+                ui64 rootSchemeshardId = AppData()->DomainsInfo->Domain->SchemeRoot;
+                for (TGroup* group : GroupView) {
+                    if (group->BoxId == 0 && group->PoolId == 0) {
+                        group->PoolName = "static";
+                        group->Kind = ""; // TODO ?
+                        group->MediaType = ""; // TODO ?
+                        group->SchemeShardId = rootSchemeshardId;
+                        group->PathId = 1;
+                        group->EncryptionMode = 0; // TODO ?
                     } else {
-                        BLOG_W("Storage pool not found for group " << group->GroupId << " box " << group->BoxId << " pool " << group->PoolId);
+                        auto itStoragePool = indexStoragePool.find({group->BoxId, group->PoolId});
+                        if (itStoragePool != indexStoragePool.end()) {
+                            const NKikimrSysView::TStoragePoolInfo* pool = itStoragePool->second;
+                            group->PoolName = pool->GetName();
+                            group->Kind = pool->GetKind();
+                            group->SchemeShardId = pool->GetSchemeshardId();
+                            group->PathId = pool->GetPathId();
+                            group->MediaType = GetMediaType(pool->GetPDiskFilter());
+                            if (!group->Erasure) {
+                                group->Erasure = pool->GetErasureSpeciesV2();
+                                group->ErasureSpecies = TErasureType::ErasureSpeciesByName(group->Erasure);
+                            }
+                            group->EncryptionMode = pool->GetEncryptionMode();
+                        } else {
+                            BLOG_W("Storage pool not found for group " << group->GroupId << " box " << group->BoxId << " pool " << group->PoolId);
+                        }
                     }
                 }
+                FieldsAvailable |= FieldsBsPools;
+                ApplyEverything();
+                CollectHiveData();
+            } else {
+                AddProblem("bsc-storage-pools-no-data");
             }
-            FieldsAvailable |= FieldsBsPools;
-            ApplyEverything();
-            CollectHiveData();
-            ++requestsDone;
-        }
-        if (FieldsAvailable.test(+EGroupFields::GroupId) && GetVSlotsResponse && GetVSlotsResponse->IsOk() && FieldsNeeded(FieldsBsVSlots)) {
-            if (GroupsByGroupId.empty()) {
-                RebuildGroupsByGroupId();
-            }
-            size_t totalEntries = GetVSlotsResponse->Get()->Record.EntriesSize();
-            size_t errorEntries = 0;
-            for (const NKikimrSysView::TVSlotEntry& entry : GetVSlotsResponse->Get()->Record.GetEntries()) {
-                const NKikimrSysView::TVSlotKey& key = entry.GetKey();
-                const NKikimrSysView::TVSlotInfo& info = entry.GetInfo();
-                VSlotsByVSlotId[key] = &info;
-                if (info.GetStatusV2() == "ERROR") {
-                    ++errorEntries;
-                }
-                auto itGroup = GroupsByGroupId.find(info.GetGroupId());
-                if (itGroup != GroupsByGroupId.end() && itGroup->second->GroupGeneration == info.GetGroupGeneration()) {
-                    TGroup& group = *itGroup->second;
-                    TVDisk& vDisk = group.VDisks.emplace_back();
-                    FillVDiskFromVSlotInfo(vDisk, key, info);
-                    group.VDiskNodeIds.push_back(vDisk.VSlotId.NodeId);
+            GetStoragePoolsResponse.reset();
+        }
+        if (FieldsAvailable.test(+EGroupFields::GroupId) && GetVSlotsResponse && GetVSlotsResponse->IsDone()) {
+            if (GetVSlotsResponse->IsOk()) {
+                if (GroupsByGroupId.empty()) {
+                    RebuildGroupsByGroupId();
                 }
-            }
-            if (totalEntries > 0 && totalEntries > errorEntries) {
-                FieldsAvailable |= FieldsBsVSlots;
+                size_t totalEntries = GetVSlotsResponse->Get()->Record.EntriesSize();
+                size_t errorEntries = 0;
+                for (const NKikimrSysView::TVSlotEntry& entry : GetVSlotsResponse->Get()->Record.GetEntries()) {
+                    const NKikimrSysView::TVSlotKey& key = entry.GetKey();
+                    const NKikimrSysView::TVSlotInfo& info = entry.GetInfo();
+                    VSlotsByVSlotId[key] = &info;
+                    if (info.GetStatusV2() == "ERROR") {
+                        ++errorEntries;
+                    }
+                    auto itGroup = GroupsByGroupId.find(info.GetGroupId());
+                    if (itGroup != GroupsByGroupId.end() && itGroup->second->GroupGeneration == info.GetGroupGeneration()) {
+                        TGroup& group = *itGroup->second;
+                        TVDisk& vDisk = group.VDisks.emplace_back();
+                        FillVDiskFromVSlotInfo(vDisk, key, info);
+                        group.VDiskNodeIds.push_back(vDisk.VSlotId.NodeId);
+                    }
+                }
+                if (totalEntries > 0 && totalEntries > errorEntries) {
+                    FieldsAvailable |= FieldsBsVSlots;
+                    ApplyEverything();
+                } else {
+                    AddProblem("bsc-wrong-data");
+                }
+            } else {
+                AddProblem("bsc-vslots-no-data");
+            }
+            GetVSlotsResponse.reset();
+        }
+        if (GetPDisksResponse && GetPDisksResponse->IsDone()) {
+            if (GetPDisksResponse->IsOk()) {
+                for (const NKikimrSysView::TPDiskEntry& entry : GetPDisksResponse->Get()->Record.GetEntries()) {
+                    const NKikimrSysView::TPDiskKey& key = entry.GetKey();
+                    const NKikimrSysView::TPDiskInfo& info = entry.GetInfo();
+                    TPDisk& pDisk = PDisks[key];
+                    pDisk.PDiskId = key.GetPDiskId();
+                    pDisk.NodeId = key.GetNodeId();
+                    pDisk.SetCategory(info.GetCategory());
+                    pDisk.Path = info.GetPath();
+                    pDisk.Guid = info.GetGuid();
+                    pDisk.AvailableSize = info.GetAvailableSize();
+                    pDisk.TotalSize = info.GetTotalSize();
+                    pDisk.Status = info.GetStatusV2();
+                    pDisk.StatusChangeTimestamp = TInstant::MicroSeconds(info.GetStatusChangeTimestamp());
+                    pDisk.EnforcedDynamicSlotSize = info.GetEnforcedDynamicSlotSize();
+                    pDisk.ExpectedSlotCount = info.GetExpectedSlotCount();
+                    pDisk.NumActiveSlots = info.GetNumActiveSlots();
+                    pDisk.Category = info.GetCategory();
+                    pDisk.DecommitStatus = info.GetDecommitStatus();
+                }
+                FieldsAvailable |= FieldsBsPDisks;
                 ApplyEverything();
             } else {
-                Problems.emplace_back("bsc-wrong-data");
-            }
-            ++requestsDone;
-        }
-        if (GetPDisksResponse && GetPDisksResponse->IsOk() && FieldsNeeded(FieldsBsPDisks)) {
-            for (const NKikimrSysView::TPDiskEntry& entry : GetPDisksResponse->Get()->Record.GetEntries()) {
-                const NKikimrSysView::TPDiskKey& key = entry.GetKey();
-                const NKikimrSysView::TPDiskInfo& info = entry.GetInfo();
-                TPDisk& pDisk = PDisks[key];
-                pDisk.PDiskId = key.GetPDiskId();
-                pDisk.NodeId = key.GetNodeId();
-                pDisk.SetCategory(info.GetCategory());
-                pDisk.Path = info.GetPath();
-                pDisk.Guid = info.GetGuid();
-                pDisk.AvailableSize = info.GetAvailableSize();
-                pDisk.TotalSize = info.GetTotalSize();
-                pDisk.Status = info.GetStatusV2();
-                pDisk.StatusChangeTimestamp = TInstant::MicroSeconds(info.GetStatusChangeTimestamp());
-                pDisk.EnforcedDynamicSlotSize = info.GetEnforcedDynamicSlotSize();
-                pDisk.ExpectedSlotCount = info.GetExpectedSlotCount();
-                pDisk.NumActiveSlots = info.GetNumActiveSlots();
-                pDisk.Category = info.GetCategory();
-                pDisk.DecommitStatus = info.GetDecommitStatus();
-            }
-            FieldsAvailable |= FieldsBsPDisks;
-            ApplyEverything();
-            ++requestsDone;
+                AddProblem("bsc-pdisks-no-data");
+            }
+            GetPDisksResponse.reset();
         }
         if (FieldsAvailable.test(+EGroupFields::VDisk)) {
             if (FieldsNeeded(FieldsGroupState)) {
@@ -1350,17 +1349,88 @@ public:
                 }
             }
         }
-        if (AreBSControllerRequestsDone() && FieldsNeeded(FieldsWbDisks)) {
-            AddEvent("SendWhiteboardRequests");
-            for (TGroup* group : GroupView) {
-                for (TNodeId nodeId : group->VDiskNodeIds) {
-                    SendWhiteboardDisksRequest(nodeId);
+        if (AreBSControllerRequestsDone()) {
+            if (FieldsAvailable.test(+EGroupFields::GroupId) && FieldsNeeded(FieldsHive) && NavigateKeySetInFlight == 0 && HiveStorageStatsInFlight == 0) {
+                if (GroupsByGroupId.empty()) {
+                    RebuildGroupsByGroupId();
+                }
+                for (auto& hiveStorageStats : HiveStorageStats) {
+                    if (hiveStorageStats.second.IsOk()) {
+                        for (const auto& pbPool : hiveStorageStats.second->Record.GetPools()) {
+                            for (const auto& pbGroup : pbPool.GetGroups()) {
+                                auto itGroup = GroupsByGroupId.find(pbGroup.GetGroupID());
+                                if (itGroup != GroupsByGroupId.end()) {
+                                    itGroup->second->AllocationUnits += pbGroup.GetAcquiredUnits();
+                                }
+                            }
+                        }
+                    }
+                }
+                FieldsAvailable |= FieldsHive;
+                ApplyEverything();
+            }
+            if (TimeToAskWhiteboard() && FieldsNeeded(FieldsWbDisks)) {
+                AddEvent("TimeToAskWhiteboard");
+                for (TGroup* group : GroupView) {
+                    for (TNodeId nodeId : group->VDiskNodeIds) {
+                        SendWhiteboardDisksRequest(nodeId);
+                    }
                 }
             }
         }
-        if (requestsDone) {
-            RequestDone(requestsDone);
+    }
+
+    void ProcessNavigate(TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult>& navigateResult, bool firstNavigate) {
+        if (navigateResult.IsOk()) {
+            TString path = CanonizePath(navigateResult->Request->ResultSet.begin()->Path);
+            TIntrusiveConstPtr<TSchemeCacheNavigate::TDomainDescription> domainDescription = navigateResult->Request->ResultSet.begin()->DomainDescription;
+            TIntrusiveConstPtr<NSchemeCache::TDomainInfo> domainInfo = navigateResult->Request->ResultSet.begin()->DomainInfo;
+            if (domainInfo != nullptr && domainDescription != nullptr) {
+                if (FieldsNeeded(FieldsHive)) {
+                    TTabletId hiveId = domainInfo->Params.GetHive();
+                    if (hiveId != 0 && HiveStorageStats.count(hiveId) == 0) {
+                        HiveStorageStats.emplace(hiveId, MakeRequestHiveStorageStats(hiveId));
+                        ++HiveStorageStatsInFlight;
+                    }
+                }
+                if (Database && firstNavigate) { // filter by database
+                    for (const auto& storagePool : domainDescription->Description.GetStoragePools()) {
+                        TString storagePoolName = storagePool.GetName();
+                        DatabaseStoragePools.emplace(storagePoolName);
+                    }
+                    NeedFilter = true;
+                }
+            }
+        }
+    }
+
+    void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+        bool firstNavigate = (ev->Cookie == 0);
+        TPathId pathId = GetPathId(ev);
+        if (firstNavigate && DatabaseNavigateResponse && pathId) {
+            NavigateKeySetResult.emplace(pathId, std::move(*DatabaseNavigateResponse));
+        }
+        auto itNavigateKeySetResult = NavigateKeySetResult.find(pathId);
+        if (itNavigateKeySetResult == NavigateKeySetResult.end()) {
+            BLOG_W("Invalid NavigateKeySetResult PathId: " << pathId << " Path: " << CanonizePath(ev->Get()->Request->ResultSet.begin()->Path));
+            return RequestDone();
+        }
+        auto& navigateResult(itNavigateKeySetResult->second);
+        navigateResult.Set(std::move(ev));
+        ProcessNavigate(navigateResult, firstNavigate);
+        --NavigateKeySetInFlight;
+        ProcessResponses();
+        RequestDone();
+    }
+
+    void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) {
+        auto itHiveStorageStats = HiveStorageStats.find(ev->Cookie);
+        if (itHiveStorageStats != HiveStorageStats.end()) {
+            itHiveStorageStats->second.Set(std::move(ev));
         }
+        --HiveStorageStatsInFlight;
+        ProcessResponses();
+        RequestDone();
     }
 
     void Handle(NSysView::TEvSysView::TEvGetGroupsResponse::TPtr& ev) {
@@ -1369,7 +1439,8 @@ public:
             RequestDone();
             return;
         }
-        ProcessBSControllerResponses();
+        ProcessResponses();
+        RequestDone();
     }
 
     void Handle(NSysView::TEvSysView::TEvGetStoragePoolsResponse::TPtr& ev) {
@@ -1378,7 +1449,8 @@ public:
             RequestDone();
             return;
         }
-        ProcessBSControllerResponses();
+        ProcessResponses();
+        RequestDone();
     }
 
     void Handle(NSysView::TEvSysView::TEvGetVSlotsResponse::TPtr& ev) {
@@ -1387,7 +1459,8 @@ public:
             RequestDone();
             return;
         }
-        ProcessBSControllerResponses();
+        ProcessResponses();
+        RequestDone();
     }
 
     void Handle(NSysView::TEvSysView::TEvGetPDisksResponse::TPtr& ev) {
@@ -1396,7 +1469,8 @@ public:
             RequestDone();
             return;
         }
-        ProcessBSControllerResponses();
+        ProcessResponses();
+        RequestDone();
     }
 
     void RequestNodesList() {
@@ -1741,19 +1815,23 @@ public:
     }
 
     void OnBscError(const TString& error) {
-        if (GetGroupsResponse.has_value()) {
-            GetGroupsResponse->Error(error);
+        RequestWhiteboard();
+        if (GetGroupsResponse && GetGroupsResponse->Error(error)) {
+            ProcessResponses();
+            RequestDone();
         }
-        if (GetStoragePoolsResponse.has_value()) {
-            GetStoragePoolsResponse->Error(error);
+        if (GetStoragePoolsResponse && GetStoragePoolsResponse->Error(error)) {
+            ProcessResponses();
+            RequestDone();
         }
-        if (GetVSlotsResponse.has_value()) {
-            GetVSlotsResponse->Error(error);
+        if (GetVSlotsResponse && GetVSlotsResponse->Error(error)) {
+            ProcessResponses();
+            RequestDone();
         }
-        if (GetPDisksResponse.has_value()) {
-            GetPDisksResponse->Error(error);
+        if (GetPDisksResponse && GetPDisksResponse->Error(error)) {
+            ProcessResponses();
+            RequestDone();
         }
-        RequestWhiteboard();
     }
 
     void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) {
@@ -1761,17 +1839,17 @@ public:
             TString error = TStringBuilder() << "Failed to establish pipe: " << NKikimrProto::EReplyStatus_Name(ev->Get()->Status);
             auto it = HiveStorageStats.find(ev->Get()->TabletId);
             if (it != HiveStorageStats.end()) {
-                it->second.Error(error);
-                if (--HiveStorageStatsInFlight == 0) {
-                    FieldsAvailable |= FieldsHive;
+                if (it->second.Error(error)) {
+                    --HiveStorageStatsInFlight;
+                    ProcessResponses();
+                    RequestDone();
                 }
             }
             if (ev->Get()->TabletId == GetBSControllerId()) {
+                AddProblem("bsc-error");
                 OnBscError(error);
-                Problems.emplace_back("bsc-error");
             }
         }
-        TBase::Handle(ev); // all RequestDone() are handled by base handler
     }
 
     STATEFN(StateWork) {
@@ -1797,27 +1875,33 @@ public:
         switch (ev->Get()->Tag) {
             case TimeoutBSC:
                 if (!AreBSControllerRequestsDone()) {
+                    AddProblem("bsc-timeout");
                     OnBscError("timeout");
-                    Problems.emplace_back("bsc-timeout");
-                    RequestDone(FailPipeConnect(GetBSControllerId()));
                 }
                 break;
             case TimeoutFinal:
                 // bread crumbs
+                if (!AreBSControllerRequestsDone()) {
+                    AddProblem("bsc-incomplete");
+                }
+                if (HiveStorageStatsInFlight > 0) {
+                    AddProblem("hive-incomplete");
+                    for (auto& hiveStorageStats : HiveStorageStats) {
+                        if (hiveStorageStats.second.Error("timeout")) {
+                            --HiveStorageStatsInFlight;
+                            ProcessResponses();
+                            //RequestDone();
+                        }
+                    }
+                }
                 if (BSGroupStateRequestsInFlight > 0) {
-                    Problems.emplace_back("wb-incomplete-groups");
+                    AddProblem("wb-incomplete-groups");
                     ProcessWhiteboardGroups();
                 }
                 if (VDiskStateRequestsInFlight > 0 || PDiskStateRequestsInFlight > 0) {
-                    Problems.emplace_back("wb-incomplete-disks");
+                    AddProblem("wb-incomplete-disks");
                     ProcessWhiteboardDisks();
                 }
-                if (HiveStorageStatsInFlight > 0) {
-                    Problems.emplace_back("hive-incomplete");
-                }
-                if (!AreBSControllerRequestsDone()) {
-                    Problems.emplace_back("bsc-incomplete");
-                }
                 ReplyAndPassAway();
                 break;
         }
diff --git a/ydb/core/viewer/viewer_nodes.h b/ydb/core/viewer/viewer_nodes.h
index f5cd25173f..a98e9ef42e 100644
--- a/ydb/core/viewer/viewer_nodes.h
+++ b/ydb/core/viewer/viewer_nodes.h
@@ -1567,10 +1567,13 @@ public:
         }
         if (FilterStorageStage == EFilterStorageStage::VSlots && VSlotsResponse && VSlotsResponse->IsDone()) {
             if (VSlotsResponse->IsOk()) {
+                std::unordered_set<TNodeId> prevFilterNodeIds = std::move(FilterNodeIds);
                 std::unordered_map<std::pair<TNodeId, ui32>, std::size_t> slotsPerDisk;
                 for (const auto& slotEntry : VSlotsResponse->Get()->Record.GetEntries()) {
                     if (FilterGroupIds.count(slotEntry.GetInfo().GetGroupId()) > 0) {
-                        FilterNodeIds.insert(slotEntry.GetKey().GetNodeId());
+                        if (prevFilterNodeIds.empty() || prevFilterNodeIds.count(slotEntry.GetKey().GetNodeId()) > 0) {
+                            FilterNodeIds.insert(slotEntry.GetKey().GetNodeId());
+                        }
                         TNode* node = FindNode(slotEntry.GetKey().GetNodeId());
                         if (node) {
                             node->SysViewVDisks.emplace_back(slotEntry);