switch to scheme shard describe to get storage stats (#15290)

3 files changed, 107 insertions, 30 deletions

diff --git a/ydb/core/viewer/json_pipe_req.cpp b/ydb/core/viewer/json_pipe_req.cpp
index a6baa1f48e..79d03d5c44 100644
--- a/ydb/core/viewer/json_pipe_req.cpp
+++ b/ydb/core/viewer/json_pipe_req.cpp
@@ -227,6 +227,17 @@ TString TViewerPipeClient::GetError(const std::unique_ptr<TEvStateStorage::TEvBo
     }
 }
 
+bool TViewerPipeClient::IsSuccess(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev) {
+    return ev->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess;
+}
+
+TString TViewerPipeClient::GetError(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev) {
+    if (ev->GetRecord().HasReason()) {
+        return ev->GetRecord().GetReason();
+    }
+    return NKikimrScheme::EStatus_Name(ev->GetRecord().GetStatus());
+}
+
 void TViewerPipeClient::RequestHiveDomainStats(NNodeWhiteboard::TTabletId hiveId) {
     TActorId pipeClient = ConnectTabletPipe(hiveId);
     THolder<TEvHive::TEvRequestHiveDomainStats> request = MakeHolder<TEvHive::TEvRequestHiveDomainStats>();
@@ -594,6 +605,19 @@ TViewerPipeClient::TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResu
     return response;
 }
 
+TViewerPipeClient::TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>
+    TViewerPipeClient::MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path, const NKikimrSchemeOp::TDescribeOptions& options, ui64 cookie) {
+    auto request = std::make_unique<NSchemeShard::TEvSchemeShard::TEvDescribeScheme>();
+    request->Record.SetSchemeshardId(schemeShardId);
+    request->Record.SetPath(path);
+    request->Record.MutableOptions()->CopyFrom(options);
+    auto response = MakeRequestToTablet<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>(schemeShardId, request.release(), cookie);
+    if (response.Span) {
+        response.Span.Attribute("path", path);
+    }
+    return response;
+}
+
 void TViewerPipeClient::RequestTxProxyDescribe(const TString& path) {
     THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate());
     request->Record.MutableDescribePath()->SetPath(path);
diff --git a/ydb/core/viewer/json_pipe_req.h b/ydb/core/viewer/json_pipe_req.h
index c4d29c553f..cf182e1dc7 100644
--- a/ydb/core/viewer/json_pipe_req.h
+++ b/ydb/core/viewer/json_pipe_req.h
@@ -204,6 +204,17 @@ protected:
         return response;
     }
 
+    template<typename TResponse>
+    TRequestResponse<TResponse> MakeRequestToTablet(TTabletId tabletId, IEventBase* ev, ui64 cookie = 0) {
+        TActorId pipe = ConnectTabletPipe(tabletId);
+        TRequestResponse<TResponse> response(Span.CreateChild(TComponentTracingLevels::THttp::Detailed, TypeName(*ev)));
+        if (response.Span) {
+            response.Span.Attribute("tablet_id", "#" + ::ToString(tabletId));
+        }
+        SendRequestToPipe(pipe, ev, cookie, response.Span.GetTraceId());
+        return response;
+    }
+
     template<typename TRequest>
     TRequestResponse<typename NNodeWhiteboard::WhiteboardResponse<TRequest>::Type> MakeWhiteboardRequest(TNodeId nodeId, TRequest* ev, ui32 flags = IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession) {
         TActorId whiteboardServiceId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId);
@@ -241,6 +252,9 @@ protected:
     static bool IsSuccess(const std::unique_ptr<TEvStateStorage::TEvBoardInfo>& ev);
     static TString GetError(const std::unique_ptr<TEvStateStorage::TEvBoardInfo>& ev);
 
+    static bool IsSuccess(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev);
+    static TString GetError(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev);
+
     TRequestResponse<TEvHive::TEvResponseHiveDomainStats> MakeRequestHiveDomainStats(TTabletId hiveId);
     TRequestResponse<TEvHive::TEvResponseHiveStorageStats> MakeRequestHiveStorageStats(TTabletId hiveId);
     TRequestResponse<TEvHive::TEvResponseHiveNodeStats> MakeRequestHiveNodeStats(TTabletId hiveId, TEvHive::TEvRequestHiveNodeStats* request);
@@ -270,6 +284,7 @@ protected:
     void RequestSchemeCacheNavigate(const TPathId& pathId);
     TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate(const TString& path, ui64 cookie = 0);
     TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate(TPathId pathId, ui64 cookie = 0);
+    TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path, const NKikimrSchemeOp::TDescribeOptions& options = {}, ui64 cookie = 0);
     TRequestResponse<TEvViewer::TEvViewerResponse> MakeRequestViewer(TNodeId nodeId, TEvViewer::TEvViewerRequest* request, ui32 flags = 0);
     void RequestTxProxyDescribe(const TString& path);
     void RequestStateStorageEndpointsLookup(const TString& path);
diff --git a/ydb/core/viewer/viewer_tenantinfo.h b/ydb/core/viewer/viewer_tenantinfo.h
index ef16d07f47..6588995bb1 100644
--- a/ydb/core/viewer/viewer_tenantinfo.h
+++ b/ydb/core/viewer/viewer_tenantinfo.h
@@ -22,6 +22,8 @@ class TJsonTenantInfo : public TViewerPipeClient {
     std::optional<TRequestResponse<NConsole::TEvConsole::TEvListTenantsResponse>> ListTenantsResponse;
     std::unordered_map<TString, TRequestResponse<NConsole::TEvConsole::TEvGetTenantStatusResponse>> TenantStatusResponses;
     std::unordered_map<TString, TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> NavigateKeySetResult;
+    std::unordered_map<TString, TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>> DescribeSchemeResult;
+    std::unordered_map<TTabletId, std::vector<TString>> DescribesBySchemeShard;
     std::unordered_map<TTabletId, TRequestResponse<TEvHive::TEvResponseHiveDomainStats>> HiveDomainStats;
     std::unordered_map<TTabletId, TRequestResponse<TEvHive::TEvResponseHiveStorageStats>> HiveStorageStats;
     std::unordered_map<TNodeId, TRequestResponse<TEvWhiteboard::TEvSystemStateResponse>> SystemStateResponse;
@@ -52,6 +54,7 @@ class TJsonTenantInfo : public TViewerPipeClient {
     bool MetadataCache = true;
     THashMap<TString, std::vector<TNodeId>> TenantNodes;
     TTabletId RootHiveId = 0;
+    TTabletId RootSchemeShardId = 0;
     TString RootId; // id of root domain (tenant)
     NKikimrViewer::TTenantInfo Result;
 
@@ -97,6 +100,15 @@ public:
         }
     }
 
+    TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path) {
+        NKikimrSchemeOp::TDescribeOptions options;
+        options.SetReturnPartitioningInfo(false);
+        options.SetReturnPartitionConfig(false);
+        options.SetReturnChildren(false);
+        options.SetReturnRangeKey(false);
+        return TBase::MakeRequestSchemeShardDescribe(schemeShardId, path, options);
+    }
+
     void Bootstrap() override {
         if (NeedToRedirect()) {
             return;
@@ -120,6 +132,7 @@ public:
         TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo;
         auto* domain = domains->GetDomain();
         DomainPath = "/" + domain->Name;
+        RootSchemeShardId = domain->SchemeRoot;
         TPathId rootPathId(domain->SchemeRoot, 1);
         RootId = GetDomainId(rootPathId);
         RootHiveId = domains->GetHive();
@@ -146,6 +159,8 @@ public:
         HiveDomainStats[RootHiveId] = MakeRequestHiveDomainStats(RootHiveId);
         if (Storage) {
             HiveStorageStats[RootHiveId] = MakeRequestHiveStorageStats(RootHiveId);
+            DescribeSchemeResult[DomainPath] = MakeRequestSchemeShardDescribe(RootSchemeShardId, DomainPath);
+            DescribesBySchemeShard[RootSchemeShardId].emplace_back(DomainPath);
         }
 
         Become(&TThis::StateCollectingInfo, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup());
@@ -175,6 +190,7 @@ public:
             hFunc(TEvTabletPipe::TEvClientConnected, Handle);
             hFunc(TEvStateStorage::TEvBoardInfo, Handle);
             hFunc(NHealthCheck::TEvSelfCheckResultProto, Handle);
+            hFunc(TEvSchemeShard::TEvDescribeSchemeResult, Handle);
             cFunc(TEvents::TSystem::Wakeup, HandleTimeout);
         }
     }
@@ -202,6 +218,12 @@ public:
                     it->second.Error(error);
                 }
             }
+            auto itDescribes = DescribesBySchemeShard.find(ev->Get()->TabletId);
+            if (itDescribes != DescribesBySchemeShard.end()) {
+                for (const TString& path : itDescribes->second) {
+                    DescribeSchemeResult[path].Error(error);
+                }
+            }
         }
         TBase::Handle(ev); // all RequestDone() are handled by base handler
     }
@@ -397,14 +419,19 @@ public:
             if (result.IsOk()) {
                 auto domainInfo = result.Get()->Request->ResultSet.begin()->DomainInfo;
                 TTabletId hiveId = domainInfo->Params.GetHive();
+                TTabletId schemeShardId = domainInfo->Params.GetSchemeShard();
                 if (hiveId) {
                     if (HiveDomainStats.count(hiveId) == 0) {
                         HiveDomainStats[hiveId] = MakeRequestHiveDomainStats(hiveId);
                     }
-                    if (Storage) {
-                        if (HiveStorageStats.count(hiveId) == 0) {
-                            HiveStorageStats[hiveId] = MakeRequestHiveStorageStats(hiveId);
-                        }
+                }
+                if (Storage) {
+                    if (hiveId && HiveStorageStats.count(hiveId) == 0) {
+                        HiveStorageStats[hiveId] = MakeRequestHiveStorageStats(hiveId);
+                    }
+                    if (schemeShardId && DescribeSchemeResult.count(path) == 0) {
+                        DescribeSchemeResult[path] = MakeRequestSchemeShardDescribe(schemeShardId, path);
+                        DescribesBySchemeShard[schemeShardId].emplace_back(path);
                     }
                 }
                 NKikimrViewer::TTenant& tenant = TenantBySubDomainKey[domainInfo->DomainKey];
@@ -424,6 +451,13 @@ public:
         }
     }
 
+    void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) {
+        TString path = ev->Get()->GetRecord().GetPath();
+        if (DescribeSchemeResult[path].Set(std::move(ev))) {
+            RequestDone();
+        }
+    }
+
     void Handle(NNodeWhiteboard::TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) {
         ui32 nodeId = ev.Get()->Cookie;
         if (SystemStateResponse[nodeId].Set(std::move(ev))) {
@@ -745,7 +779,6 @@ public:
                         }
                     }
 
-                    THashMap<NKikimrViewer::TStorageUsage::EType, ui64> tablesStorageByType;
                     THashMap<NKikimrViewer::TStorageUsage::EType, TStorageQuota> storageQuotasByType;
 
                     for (const auto& quota : tenant.GetDatabaseQuotas().storage_quotas()) {
@@ -755,42 +788,47 @@ public:
                         usage.HardQuota += quota.data_size_hard_quota();
                     }
 
-                    if (entry.DomainDescription) {
-                        for (const auto& poolUsage : entry.DomainDescription->Description.GetDiskSpaceUsage().GetStoragePoolsUsage()) {
+                    auto itDescribeScheme = DescribeSchemeResult.find(name);
+                    if (itDescribeScheme != DescribeSchemeResult.end() && itDescribeScheme->second.IsOk()) {
+                        const auto& record = itDescribeScheme->second.Get()->GetRecord();
+                        const auto& domainDescription(record.GetPathDescription().GetDomainDescription());
+                        THashMap<NKikimrViewer::TStorageUsage::EType, ui64> tablesStorageByType;
+
+                        for (const auto& poolUsage : domainDescription.GetDiskSpaceUsage().GetStoragePoolsUsage()) {
                             auto type = GetStorageType(poolUsage.GetPoolKind());
                             tablesStorageByType[type] += poolUsage.GetTotalSize();
                         }
 
-                        if (tablesStorageByType.empty() && entry.DomainDescription->Description.HasDiskSpaceUsage()) {
-                            tablesStorageByType[GuessStorageType(entry.DomainDescription->Description)] =
-                                entry.DomainDescription->Description.GetDiskSpaceUsage().GetTables().GetTotalSize()
-                                + entry.DomainDescription->Description.GetDiskSpaceUsage().GetTopics().GetDataSize();
+                        if (tablesStorageByType.empty() && domainDescription.HasDiskSpaceUsage()) {
+                            tablesStorageByType[GuessStorageType(domainDescription)] =
+                            domainDescription.GetDiskSpaceUsage().GetTables().GetTotalSize()
+                                + domainDescription.GetDiskSpaceUsage().GetTopics().GetDataSize();
                         }
 
                         if (storageQuotasByType.empty()) {
-                            auto& quotas = storageQuotasByType[GuessStorageType(entry.DomainDescription->Description)];
-                            quotas.HardQuota = entry.DomainDescription->Description.GetDatabaseQuotas().data_size_hard_quota();
-                            quotas.SoftQuota = entry.DomainDescription->Description.GetDatabaseQuotas().data_size_soft_quota();
+                            auto& quotas = storageQuotasByType[GuessStorageType(domainDescription)];
+                            quotas.HardQuota = domainDescription.GetDatabaseQuotas().data_size_hard_quota();
+                            quotas.SoftQuota = domainDescription.GetDatabaseQuotas().data_size_soft_quota();
                         }
-                    }
 
-                    for (const auto& [type, size] : tablesStorageByType) {
-                        auto it = storageQuotasByType.find(type);
-                        auto& tablesStorage = *tenant.AddTablesStorage();
-                        tablesStorage.SetType(type);
-                        tablesStorage.SetSize(size);
-                        if (it != storageQuotasByType.end()) {
-                            tablesStorage.SetLimit(it->second.SoftQuota);
-                            tablesStorage.SetSoftQuota(it->second.SoftQuota);
-                            tablesStorage.SetHardQuota(it->second.HardQuota);
+                        for (const auto& [type, size] : tablesStorageByType) {
+                            auto it = storageQuotasByType.find(type);
+                            auto& tablesStorage = *tenant.AddTablesStorage();
+                            tablesStorage.SetType(type);
+                            tablesStorage.SetSize(size);
+                            if (it != storageQuotasByType.end()) {
+                                tablesStorage.SetLimit(it->second.SoftQuota);
+                                tablesStorage.SetSoftQuota(it->second.SoftQuota);
+                                tablesStorage.SetHardQuota(it->second.HardQuota);
+                            }
                         }
-                    }
 
-                    for (const auto& [type, pr] : databaseStorageByType) {
-                        auto& databaseStorage = *tenant.AddDatabaseStorage();
-                        databaseStorage.SetType(type);
-                        databaseStorage.SetSize(pr.first);
-                        databaseStorage.SetLimit(pr.first + pr.second);
+                        for (const auto& [type, pr] : databaseStorageByType) {
+                            auto& databaseStorage = *tenant.AddDatabaseStorage();
+                            databaseStorage.SetType(type);
+                            databaseStorage.SetSize(pr.first);
+                            databaseStorage.SetLimit(pr.first + pr.second);
+                        }
                     }
                 }