Improve BlobDepot monitoring (#3978)

17 files changed, 148 insertions, 29 deletions

diff --git a/ydb/core/blob_depot/assimilator.cpp b/ydb/core/blob_depot/assimilator.cpp
index 2869eb74f2..5acd790f67 100644
--- a/ydb/core/blob_depot/assimilator.cpp
+++ b/ydb/core/blob_depot/assimilator.cpp
@@ -89,6 +89,7 @@ namespace NKikimr::NBlobDepot {
             if (stateBits & TStateBits::Blobs) {
                 Load(&stream, SkipBlobsUpTo.emplace());
             }
+            UpdateAssimilatorPosition();
         }
 
         Become(&TThis::StateFunc);
@@ -255,6 +256,7 @@ namespace NKikimr::NBlobDepot {
 
             void Complete(const TActorContext&) override {
                 Self->Self->Data->CommitTrash(this);
+                Self->UpdateAssimilatorPosition();
 
                 if (MoreData) {
                     Self->Self->Execute(std::make_unique<TTxPutAssimilatedData>(*this));
@@ -363,6 +365,7 @@ namespace NKikimr::NBlobDepot {
 
     void TAssimilator::Handle(TEvBlobStorage::TEvGetResult::TPtr ev) {
         auto& msg = *ev->Get();
+        (msg.Status == NKikimrProto::OK ? Self->AssimilatorLatestOkGet : Self->AssimilatorLatestOkPut) = TInstant::Now();
         const auto it = GetIdToUnprocessedPuts.find(ev->Cookie);
         Y_ABORT_UNLESS(it != GetIdToUnprocessedPuts.end());
         ui32 getBytes = 0;
@@ -386,11 +389,17 @@ namespace NKikimr::NBlobDepot {
                     ++it->second;
                 }
                 getBytes += resp.Id.BlobSize();
+                ++Self->AssimilatorBlobsReadOk;
             } else if (resp.Status == NKikimrProto::NODATA) {
                 Self->Data->ExecuteTxCommitAssimilatedBlob(NKikimrProto::NODATA, TBlobSeqId(), TData::TKey(resp.Id),
                     TEvPrivate::EvTxComplete, SelfId(), it->first);
                 ++it->second;
+                ++Self->AssimilatorBlobsReadNoData;
+            } else {
+                ++Self->AssimilatorBlobsReadError;
+                continue;
             }
+            Self->AssimilatorLastReadBlobId = resp.Id;
         }
         if (getBytes) {
             Self->TabletCounters->Cumulative()[NKikimrBlobDepot::COUNTER_DECOMMIT_GET_BYTES] += getBytes;
@@ -416,8 +425,12 @@ namespace NKikimr::NBlobDepot {
 
     void TAssimilator::Handle(TEvBlobStorage::TEvPutResult::TPtr ev) {
         auto& msg = *ev->Get();
+        (msg.Status == NKikimrProto::OK ? Self->AssimilatorLatestOkPut : Self->AssimilatorLatestErrorPut) = TInstant::Now();
         if (msg.Status == NKikimrProto::OK) {
             Self->TabletCounters->Cumulative()[NKikimrBlobDepot::COUNTER_DECOMMIT_PUT_OK_BYTES] += msg.Id.BlobSize();
+            ++Self->AssimilatorBlobsPutOk;
+        } else {
+            ++Self->AssimilatorBlobsPutError;
         }
         const auto it = PutIdToKey.find(ev->Cookie);
         Y_ABORT_UNLESS(it != PutIdToKey.end());
@@ -540,6 +553,15 @@ namespace NKikimr::NBlobDepot {
         return stream.Str();
     }
 
+    void TAssimilator::UpdateAssimilatorPosition() const {
+        Self->AssimilatorPosition = TStringBuilder()
+            << "SkipBlocksUpTo# " << (SkipBlocksUpTo ? ToString(*SkipBlocksUpTo) : "<none>") << Endl
+            << "SkipBarriersUpTo# " << (SkipBarriersUpTo
+                ? TString(TStringBuilder() << std::get<0>(*SkipBarriersUpTo) << ':' << (int)std::get<1>(*SkipBarriersUpTo))
+                : "<none>") << Endl
+            << "SkipBlobsUpTo# " << (SkipBlobsUpTo ? SkipBlobsUpTo->ToString() : "<none>");
+    }
+
     void TBlobDepot::TData::ExecuteTxCommitAssimilatedBlob(NKikimrProto::EReplyStatus status, TBlobSeqId blobSeqId,
             TData::TKey key, ui32 notifyEventType, TActorId parentId, ui64 cookie, bool keep, bool doNotKeep) {
         Self->Execute(std::make_unique<TTxCommitAssimilatedBlob>(Self, status, blobSeqId, std::move(key),
diff --git a/ydb/core/blob_depot/assimilator.h b/ydb/core/blob_depot/assimilator.h
index 7e4b570bcb..bd23ea8f86 100644
--- a/ydb/core/blob_depot/assimilator.h
+++ b/ydb/core/blob_depot/assimilator.h
@@ -73,6 +73,7 @@ namespace NKikimr::NBlobDepot {
         void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr ev);
         void Handle(TEvBlobStorage::TEvControllerGroupDecommittedResponse::TPtr ev);
         TString SerializeAssimilatorState() const;
+        void UpdateAssimilatorPosition() const;
     };
 
 } // NKikimrBlobDepot::NBlobDepot
diff --git a/ydb/core/blob_depot/blob_depot_tablet.h b/ydb/core/blob_depot/blob_depot_tablet.h
index 583fed19d3..726ebafc59 100644
--- a/ydb/core/blob_depot/blob_depot_tablet.h
+++ b/ydb/core/blob_depot/blob_depot_tablet.h
@@ -290,6 +290,17 @@ namespace NKikimr::NBlobDepot {
         TActorId GroupAssimilatorId;
         EDecommitState DecommitState = EDecommitState::Default;
         std::optional<TString> AssimilatorState;
+        TString AssimilatorPosition;
+        TInstant AssimilatorLatestErrorGet;
+        TInstant AssimilatorLatestOkGet;
+        TInstant AssimilatorLatestErrorPut;
+        TInstant AssimilatorLatestOkPut;
+        TLogoBlobID AssimilatorLastReadBlobId;
+        ui64 AssimilatorBlobsReadOk = 0;
+        ui64 AssimilatorBlobsReadNoData = 0;
+        ui64 AssimilatorBlobsReadError = 0;
+        ui64 AssimilatorBlobsPutOk = 0;
+        ui64 AssimilatorBlobsPutError = 0;
 
         class TGroupAssimilator;
 
@@ -301,6 +312,8 @@ namespace NKikimr::NBlobDepot {
         ui64 BytesRead = 0;
         ui64 BytesWritten = 0;
         std::deque<std::tuple<TMonotonic, ui64, ui64>> MetricsQ;
+        ui64 ReadThroughput = 0;
+        ui64 WriteThroughput = 0;
 
         void DoGroupMetricsExchange();
         void Handle(TEvBlobStorage::TEvControllerGroupMetricsExchange::TPtr ev);
diff --git a/ydb/core/blob_depot/data_mon.cpp b/ydb/core/blob_depot/data_mon.cpp
index 9a286b6781..174ca8a53e 100644
--- a/ydb/core/blob_depot/data_mon.cpp
+++ b/ydb/core/blob_depot/data_mon.cpp
@@ -15,7 +15,6 @@ namespace NKikimr::NBlobDepot {
                 DIV_CLASS("panel-body") {
                     KEYVALUE_TABLE({
                         KEYVALUE_P("Loaded", Loaded ? "true" : "false");
-                        KEYVALUE_P("Last assimilated blob id", LastAssimilatedBlobId ? LastAssimilatedBlobId->ToString() : "<null>");
                         KEYVALUE_P("Data size, number of keys", Data.size());
                         KEYVALUE_P("RefCount size, number of blobs", RefCount.size());
                         KEYVALUE_P("Total stored data size, bytes", FormatByteSize(TotalStoredDataSize));
diff --git a/ydb/core/blob_depot/group_metrics_exchange.cpp b/ydb/core/blob_depot/group_metrics_exchange.cpp
index 6fbb75eee5..e9810bf008 100644
--- a/ydb/core/blob_depot/group_metrics_exchange.cpp
+++ b/ydb/core/blob_depot/group_metrics_exchange.cpp
@@ -81,12 +81,16 @@ namespace NKikimr::NBlobDepot {
                 }
             }
 
-            auto ev = std::make_unique<NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateUpdate>();
-            auto& wb = ev->Record;
-            wb.SetGroupID(Config.GetVirtualGroupId());
-            wb.SetAllocatedSize(Data->GetTotalStoredDataSize());
-            wb.SetAvailableSize(params->GetAvailableSize());
-            Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()), ev.release());
+            auto *wb = record.MutableWhiteboardUpdate();
+            wb->SetGroupID(Config.GetVirtualGroupId());
+            wb->SetAllocatedSize(Data->GetTotalStoredDataSize());
+            wb->SetAvailableSize(params->GetAvailableSize());
+            wb->SetReadThroughput(ReadThroughput);
+            wb->SetWriteThroughput(WriteThroughput);
+
+            if (ReadyForAgentQueries()) {
+                wb->SetBlobDepotOnlineTime(TInstant::Now().MilliSeconds());
+            }
 
             params->SetAllocatedSize(Data->GetTotalStoredDataSize());
             Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), response.release());
@@ -103,7 +107,9 @@ namespace NKikimr::NBlobDepot {
         const auto& record = ev->Get()->Record;
         BytesRead += record.GetBytesRead();
         BytesWritten += record.GetBytesWritten();
-        MetricsQ.emplace_back(TActivationContext::Monotonic(), BytesRead, BytesWritten);
+        if (Config.HasVirtualGroupId()) {
+            MetricsQ.emplace_back(TActivationContext::Monotonic(), BytesRead, BytesWritten);
+        }
         UpdateThroughputs(false);
     }
 
@@ -128,24 +134,17 @@ namespace NKikimr::NBlobDepot {
                 }
             }
 
-            ui64 readThroughput = 0;
-            ui64 writeThroughput = 0;
             const auto& [ts, read, written] = MetricsQ.front();
             if (ts + TDuration::Seconds(1) < now) {
-                readThroughput = (BytesRead - read) * 1'000'000 / (now - ts).MicroSeconds();
-                writeThroughput = (BytesWritten - written) * 1'000'000 / (now - ts).MicroSeconds();
+                ReadThroughput = (BytesRead - read) * 1'000'000 / (now - ts).MicroSeconds();
+                WriteThroughput = (BytesWritten - written) * 1'000'000 / (now - ts).MicroSeconds();
+            } else {
+                ReadThroughput = WriteThroughput = 0;
             }
-
-            auto ev = std::make_unique<NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateUpdate>();
-            auto& wb = ev->Record;
-            wb.SetGroupID(Config.GetVirtualGroupId());
-            wb.SetReadThroughput(readThroughput);
-            wb.SetWriteThroughput(writeThroughput);
-            Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()), ev.release());
         }
 
         if (reschedule) {
-            TActivationContext::Schedule(TDuration::Seconds(1), new IEventHandle(TEvPrivate::EvUpdateThroughputs, 0,
+            TActivationContext::Schedule(Window, new IEventHandle(TEvPrivate::EvUpdateThroughputs, 0,
                 SelfId(), {}, nullptr, 0));
         }
     }
diff --git a/ydb/core/blob_depot/mon_main.cpp b/ydb/core/blob_depot/mon_main.cpp
index 2953d187e4..051be4245b 100644
--- a/ydb/core/blob_depot/mon_main.cpp
+++ b/ydb/core/blob_depot/mon_main.cpp
@@ -445,6 +445,35 @@ namespace NKikimr::NBlobDepot {
                     })
                 }
             }
+
+            if (Configured && Config.GetIsDecommittingGroup()) {
+                DIV_CLASS("panel panel-info") {
+                    DIV_CLASS("panel-heading") {
+                        s << "Decommission";
+                    }
+                    DIV_CLASS("panel-body") {
+                        KEYVALUE_TABLE({
+                            KEYVALUE_P("Now", TInstant::Now());
+                            KEYVALUE_P("Decommit state", DecommitState);
+                            KEYVALUE_P("Assimilator state", GroupAssimilatorId ? "running" : "stopped");
+                            KEYVALUE_P("Assimilator position", TStringBuilder() << "<pre>" << AssimilatorPosition << "<pre/>");
+                            KEYVALUE_P("Last assimilated blob id", Data->LastAssimilatedBlobId ?
+                                Data->LastAssimilatedBlobId->ToString() : "<null>");
+                            KEYVALUE_P("Last read blob id", AssimilatorLastReadBlobId);
+                            KEYVALUE_P("Latest successful get", AssimilatorLatestOkGet);
+                            KEYVALUE_P("Latest erroneous get", AssimilatorLatestErrorGet);
+                            KEYVALUE_P("Latest successful put", AssimilatorLatestOkPut);
+                            KEYVALUE_P("Latest erroneous put", AssimilatorLatestErrorPut);
+                            KEYVALUE_P("Blobs read with OK", AssimilatorBlobsReadOk);
+                            KEYVALUE_P("Blobs read with NODATA", AssimilatorBlobsReadNoData);
+                            KEYVALUE_P("Blobs read with error", AssimilatorBlobsReadError);
+                            KEYVALUE_P("Blobs put with OK", AssimilatorBlobsPutOk);
+                            KEYVALUE_P("Blobs put with error", AssimilatorBlobsPutError);
+                        })
+                    }
+                }
+            }
+
             DIV_CLASS("panel panel-info") {
                 DIV_CLASS("panel-heading") {
                     s << "Data";
diff --git a/ydb/core/blob_depot/ya.make b/ydb/core/blob_depot/ya.make
index 9d3d3d9f69..3464266393 100644
--- a/ydb/core/blob_depot/ya.make
+++ b/ydb/core/blob_depot/ya.make
@@ -51,6 +51,8 @@ LIBRARY()
         ydb/core/protos
     )
 
+    GENERATE_ENUM_SERIALIZATION(schema.h)
+
 END()
 
 RECURSE(
diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp
index ef4a91e20d..d00a07e1b5 100644
--- a/ydb/core/mind/bscontroller/config.cpp
+++ b/ydb/core/mind/bscontroller/config.cpp
@@ -247,6 +247,16 @@ namespace NKikimr::NBsController {
                     NKikimrBlobStorage::TNodeWardenServiceSet *service = Services[nodeId].MutableServiceSet();
                     SerializeGroupInfo(service->AddGroups(), groupInfo, storagePoolName, scopeId);
                 }
+
+                // push group state notification to NodeWhiteboard
+                TBlobStorageGroupInfo::TDynamicInfo dynInfo(groupInfo.ID, groupInfo.Generation);
+                for (const auto& vdisk : groupInfo.VDisksInGroup) {
+                    const auto& id = vdisk->VSlotId;
+                    dynInfo.PushBackActorId(MakeBlobStorageVDiskID(id.NodeId, id.PDiskId, id.VSlotId));
+                }
+                State.NodeWhiteboardOutbox.emplace_back(new NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateUpdate(
+                    MakeIntrusive<TBlobStorageGroupInfo>(groupInfo.Topology, std::move(dynInfo), storagePoolName,
+                    scopeId, NPDisk::DEVICE_TYPE_UNKNOWN)));
             }
 
             void ApplyGroupDeleted(const TGroupId &groupId, const TGroupInfo& /*groupInfo*/) {
@@ -610,6 +620,9 @@ namespace NKikimr::NBsController {
             for (auto& ev : StatProcessorOutbox) {
                 Self.SelfId().Send(Self.StatProcessorActorId, ev.release());
             }
+            for (auto& ev : NodeWhiteboardOutbox) {
+                Self.SelfId().Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(Self.SelfId().NodeId()), ev.release());
+            }
 
             if (UpdateSelfHealInfoMsg) {
                 UpdateSelfHealInfoMsg->ConfigTxSeqNo = Self.NextConfigTxSeqNo;
diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h
index 8e84d3c72f..f11bcaa4ef 100644
--- a/ydb/core/mind/bscontroller/config.h
+++ b/ydb/core/mind/bscontroller/config.h
@@ -93,6 +93,7 @@ namespace NKikimr {
             // outgoing messages
             std::deque<std::tuple<TNodeId, std::unique_ptr<IEventBase>, ui64>> Outbox;
             std::deque<std::unique_ptr<IEventBase>> StatProcessorOutbox;
+            std::deque<std::unique_ptr<IEventBase>> NodeWhiteboardOutbox;
             THolder<TEvControllerUpdateSelfHealInfo> UpdateSelfHealInfoMsg;
 
             // deferred callbacks
diff --git a/ydb/core/mind/bscontroller/defs.h b/ydb/core/mind/bscontroller/defs.h
index 586c8eccdc..10222948d5 100644
--- a/ydb/core/mind/bscontroller/defs.h
+++ b/ydb/core/mind/bscontroller/defs.h
@@ -19,6 +19,7 @@
 #include <ydb/core/driver_lib/version/version.h>
 #include <ydb/core/engine/minikql/flat_local_tx_factory.h>
 #include <ydb/core/mind/table_adapter.h>
+#include <ydb/core/node_whiteboard/node_whiteboard.h>
 #include <ydb/core/protos/blobstorage_config.pb.h>
 #include <ydb/core/protos/blobstorage_distributed_config.pb.h>
 #include <ydb/core/protos/blobstorage.pb.h>
diff --git a/ydb/core/mind/bscontroller/group_metrics_exchange.cpp b/ydb/core/mind/bscontroller/group_metrics_exchange.cpp
index 5b18807f6c..1f24f59f30 100644
--- a/ydb/core/mind/bscontroller/group_metrics_exchange.cpp
+++ b/ydb/core/mind/bscontroller/group_metrics_exchange.cpp
@@ -59,6 +59,11 @@ namespace NKikimr::NBsController {
     };
 
     void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerGroupMetricsExchange::TPtr& ev) {
+        if (auto& record = ev->Get()->Record; record.HasWhiteboardUpdate()) {
+            auto ev = std::make_unique<NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateUpdate>();
+            ev->Record.Swap(record.MutableWhiteboardUpdate());
+            Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()), ev.release());
+        }
         Execute(new TTxGroupMetricsExchange(this, ev));
     }
 
diff --git a/ydb/core/node_whiteboard/node_whiteboard.h b/ydb/core/node_whiteboard/node_whiteboard.h
index ddab1853a3..ed20871470 100644
--- a/ydb/core/node_whiteboard/node_whiteboard.h
+++ b/ydb/core/node_whiteboard/node_whiteboard.h
@@ -305,10 +305,14 @@ struct TEvWhiteboard{
             Record.SetGroupID(groupInfo->GroupID);
             Record.SetGroupGeneration(groupInfo->GroupGeneration);
             Record.SetErasureSpecies(groupInfo->Type.ErasureSpeciesName(groupInfo->Type.GetErasure()));
-            for (ui32 i = 0; i < groupInfo->GetTotalVDisksNum(); ++i) {
-                VDiskIDFromVDiskID(groupInfo->GetVDiskId(i), Record.AddVDiskIds());
-                const TActorId& actorId = groupInfo->GetActorId(i);
-                Record.AddVDiskNodeIds(actorId.NodeId());
+            if (ui32 numVDisks = groupInfo->GetTotalVDisksNum()) {
+                for (ui32 i = 0; i < numVDisks; ++i) {
+                    VDiskIDFromVDiskID(groupInfo->GetVDiskId(i), Record.AddVDiskIds());
+                    const TActorId& actorId = groupInfo->GetActorId(i);
+                    Record.AddVDiskNodeIds(actorId.NodeId());
+                }
+            } else {
+                Record.SetNoVDisksInGroup(true);
             }
             Record.SetStoragePoolName(groupInfo->GetStoragePoolName());
             if (groupInfo->GetEncryptionMode() != TBlobStorageGroupInfo::EEM_NONE) {
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index 0aef3c10e4..dce7b976dc 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -8,6 +8,7 @@ import "ydb/core/protos/blobstorage_pdisk_config.proto";
 import "ydb/core/protos/blobstorage_config.proto";
 import "ydb/core/protos/blobstorage_disk.proto";
 import "ydb/core/protos/blobstorage_disk_color.proto";
+import "ydb/core/protos/node_whiteboard.proto";
 
 package NKikimrBlobStorage;
 option java_package = "ru.yandex.kikimr.proto";
@@ -1345,6 +1346,7 @@ message TGroupMetrics {
 message TEvControllerGroupMetricsExchange {
     repeated TGroupMetrics GroupMetrics = 1;
     repeated fixed32 GroupsToQuery = 2; // filled in by entities asking BSC to report some metrics in return
+    optional NKikimrWhiteboard.TBSGroupStateInfo WhiteboardUpdate = 3;
 }
 
 message TEvNodeWardenQueryGroupInfo {
diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto
index d5e2beafb7..3737bf3bde 100644
--- a/ydb/core/protos/node_whiteboard.proto
+++ b/ydb/core/protos/node_whiteboard.proto
@@ -247,6 +247,8 @@ message TBSGroupStateInfo {
     optional bool Encryption = 19;
     repeated uint32 VDiskNodeIds = 20;
     optional uint64 BlobDepotId = 21; // if set, then this is virtual group
+    optional bool NoVDisksInGroup = 22;
+    optional uint64 BlobDepotOnlineTime = 23;
 }
 
 message TEvBSGroupStateRequest {
diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp
index a1595729f6..f9ffcfb1fd 100644
--- a/ydb/core/tablet/node_whiteboard.cpp
+++ b/ydb/core/tablet/node_whiteboard.cpp
@@ -540,9 +540,17 @@ protected:
     }
 
     void Handle(TEvWhiteboard::TEvBSGroupStateUpdate::TPtr &ev, const TActorContext &ctx) {
-        auto& bSGroupStateInfo = BSGroupStateInfo[ev->Get()->Record.GetGroupID()];
-        if (CheckedMerge(bSGroupStateInfo, ev->Get()->Record) >= 100) {
-            bSGroupStateInfo.SetChangeTime(ctx.Now().MilliSeconds());
+        const auto& from = ev->Get()->Record;
+        auto& to = BSGroupStateInfo[from.GetGroupID()];
+        int modified = 0;
+        if (from.GetNoVDisksInGroup() && to.GetGroupGeneration() <= from.GetGroupGeneration()) {
+            modified += 100 * (2 - to.GetVDiskIds().empty() - to.GetVDiskNodeIds().empty());
+            to.ClearVDiskIds();
+            to.ClearVDiskNodeIds();
+        }
+        modified += CheckedMerge(to, from);
+        if (modified >= 100) {
+            to.SetChangeTime(ctx.Now().MilliSeconds());
         }
     }
 
diff --git a/ydb/core/viewer/content/v2/storage.js b/ydb/core/viewer/content/v2/storage.js
index f9c7170713..a527233cd2 100644
--- a/ydb/core/viewer/content/v2/storage.js
+++ b/ydb/core/viewer/content/v2/storage.js
@@ -140,10 +140,14 @@ Storage.prototype.appear = function() {
         } else if (!this.BlobDepotId) {
             erasureElem = $('<td>', {text: 'BlobDepot (error)'});
         } else {
+            label = 'BlobDepot';
+            if (this.BlobDepotOnlineTime === undefined || getTime() - this.BlobDepotOnlineTime >= 15000) {
+                label = 'BlobDepot (error)';
+            }
             erasureElem = $('<td>');
             var link = $('<a>', {
                 'href': '../../../tablets/app?TabletID=' + this.BlobDepotId,
-                'text': 'BlobDepot',
+                'text': label,
                 'title': this.BlobDepotId
             });
             erasureElem.html(link);
@@ -372,6 +376,14 @@ Storage.prototype.updateFromStorage = function(update) {
         //console.log(this);
         break;
     }
+
+    var blobDepotError = false;
+    if (this.BlobDepotId !== undefined && (!this.BlobDepotId || this.BlobDepotOnlineTime === undefined ||
+            getTime() - this.BlobDepotOnlineTime >= 15000)) {
+        this.color = red; // blob depot is ought to be working, but it does not
+        blobDepotError = true;
+    }
+
     var usage = 0;
     var missingDisks = 0;
     for (var numDisk in this.VDisks) {
@@ -412,6 +424,9 @@ Storage.prototype.updateFromStorage = function(update) {
         }
     }
     this.usage = usage;
+    if (blobDepotError) {
+        missingDisks = -1;
+    }
     this.missingDisks = missingDisks;
     if (this.visible) {
         this.update();
diff --git a/ydb/core/viewer/content/v2/viewer.js b/ydb/core/viewer/content/v2/viewer.js
index 74686cbdbf..eb96fddcb6 100644
--- a/ydb/core/viewer/content/v2/viewer.js
+++ b/ydb/core/viewer/content/v2/viewer.js
@@ -467,7 +467,10 @@ ViewerStorage.prototype.onStorageGroupChange = function(obj) {
         this.storageView.groupOrder = function(prev, next) { return Number(prev.substring(0, prev.length - 1)) > Number(next.substring(0, next.length - 1)); }
         break;
     case 'missing':
-        this.storageView.getStorageGroupName = function(storage) { var md = storage.getMissingDisks(); return md === 0 ? "Complete" : '-' + md; }
+        this.storageView.getStorageGroupName = function(storage) {
+            var md = storage.getMissingDisks();
+            return md === -1 ? "BlobDepot error" : md === 0 ? "Complete" : '-' + md;
+        }
         this.storageView.getStorageGroupHeader = function(storageGroup) { return storageGroup.storageTotal + ' groups'; }
         this.storageView.groupOrder = function(prev, next) { return prev < next; }
         break;