Show exclusive nodes in embedded ui KIKIMR-20675 (#1149)

* Show exclusive nodes in embedded ui KIKIMR-20675

4 files changed, 379 insertions, 4 deletions

diff --git a/ydb/core/viewer/json_compute.h b/ydb/core/viewer/json_compute.h
index ffab5483c1e..4c6f0463c90 100644
--- a/ydb/core/viewer/json_compute.h
+++ b/ydb/core/viewer/json_compute.h
@@ -284,6 +284,11 @@ public:
         for (const NKikimrHive::THiveNodeStats& nodeStat : nodeStats) {
             auto nodeId = nodeStat.GetNodeId();
             if (IsRequiredNode(nodeId)) {
+                const auto& nodeDomain = nodeStat.GetNodeDomain();
+                const TPathId subDomain(nodeDomain.GetSchemeShard(), nodeDomain.GetPathId());
+                if (FilterSubDomain && FilterSubDomain != subDomain) {
+                    continue;
+                }
                 NodeIds.emplace_back(nodeId); // order is important
                 TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId);
                 THolder<NNodeWhiteboard::TEvWhiteboard::TEvSystemStateRequest> request = MakeHolder<NNodeWhiteboard::TEvWhiteboard::TEvSystemStateRequest>();
diff --git a/ydb/core/viewer/json_nodes.h b/ydb/core/viewer/json_nodes.h
index 2db9b9a124a..f663e38a3ad 100644
--- a/ydb/core/viewer/json_nodes.h
+++ b/ydb/core/viewer/json_nodes.h
@@ -40,6 +40,7 @@ class TJsonNodes : public TViewerPipeClient<TJsonNodes> {
     TJsonSettings JsonSettings;
     ui32 Timeout = 0;
     TString FilterTenant;
+    TSubDomainKey FilterSubDomainKey;
     TString FilterPath;
     TString FilterStoragePool;
     std::unordered_set<TNodeId> FilterNodeIds;
@@ -405,6 +406,11 @@ public:
                 if (HiveId == 0) {
                     HiveId = entry.DomainInfo->Params.GetHive();
                 }
+                if (!FilterSubDomainKey) {
+                    const auto ownerId = entry.DomainInfo->DomainKey.OwnerId;
+                    const auto localPathId = entry.DomainInfo->DomainKey.LocalPathId;
+                    FilterSubDomainKey = TSubDomainKey(ownerId, localPathId);
+                }
                 if (entry.DomainInfo->ResourcesDomainKey && entry.DomainInfo->DomainKey != entry.DomainInfo->ResourcesDomainKey) {
                     TPathId resourceDomainKey(entry.DomainInfo->ResourcesDomainKey);
                     BLOG_TRACE("Requesting navigate for resource domain " << resourceDomainKey);
@@ -461,6 +467,10 @@ public:
     void Handle(TEvHive::TEvResponseHiveNodeStats::TPtr& ev) {
         BLOG_TRACE("ResponseHiveNodeStats()");
         for (const NKikimrHive::THiveNodeStats& nodeStats : ev->Get()->Record.GetNodeStats()) {
+            const TSubDomainKey nodeSubDomainKey = TSubDomainKey(nodeStats.GetNodeDomain());
+            if (FilterSubDomainKey && FilterSubDomainKey != nodeSubDomainKey) {
+                continue;
+            }
             ui32 nodeId = nodeStats.GetNodeId();
             auto& tabletInfo(TabletInfo[nodeId]);
             for (const NKikimrHive::THiveDomainStatsStateCount& stateStats : nodeStats.GetStateStats()) {
@@ -514,14 +524,18 @@ public:
     }
 
     void Handle(TEvStateStorage::TEvBoardInfo::TPtr& ev) {
-        BLOG_TRACE("Received TEvBoardInfo");
         if (ev->Get()->Status == TEvStateStorage::TEvBoardInfo::EStatus::Ok) {
+            BLOG_TRACE("Received TEvBoardInfo");
             for (const auto& [actorId, infoEntry] : ev->Get()->InfoEntries) {
                 auto nodeId(actorId.NodeId());
                 BLOG_TRACE("BoardInfo filter node by " << nodeId);
                 FilterNodeIds.insert(nodeId);
             }
+        } else {
+            BLOG_TRACE("Error receiving TEvBoardInfo response");
+            FilterNodeIds = { 0 };
         }
+
         if (--RequestsBeforeNodeList == 0) {
             ProcessNodeIds();
         }
diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h
index 36a4a448176..d507ff2a8b2 100644
--- a/ydb/core/viewer/json_tenantinfo.h
+++ b/ydb/core/viewer/json_tenantinfo.h
@@ -667,9 +667,12 @@ public:
 
                 if (tenant.GetType() == NKikimrViewer::Serverless) {
                     tenant.SetStorageAllocatedSize(tenant.GetMetrics().GetStorage());
-                    tenant.SetMemoryUsed(tenant.GetMetrics().GetMemory());
-                    tenant.ClearMemoryLimit();
-                    tenant.SetCoresUsed(static_cast<double>(tenant.GetMetrics().GetCPU()) / 1000000);
+                    const bool noExclusiveNodes = tenantNodes.empty();
+                    if (noExclusiveNodes) {
+                        tenant.SetMemoryUsed(tenant.GetMetrics().GetMemory());
+                        tenant.ClearMemoryLimit();
+                        tenant.SetCoresUsed(static_cast<double>(tenant.GetMetrics().GetCPU()) / 1000000);
+                    }
                 }
 
                 if (Tablets) {
diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp
index 40c989d9f65..6fbcfba9583 100644
--- a/ydb/core/viewer/viewer_ut.cpp
+++ b/ydb/core/viewer/viewer_ut.cpp
@@ -600,4 +600,357 @@ Y_UNIT_TEST_SUITE(Viewer) {
         StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 80, 100, true);
         StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 90, 100, true);
     }
+
+    Y_UNIT_TEST(ServerlessNodesPage)
+    {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port)
+                .SetNodeCount(1)
+                .SetDynamicNodeCount(1)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root")
+                .InitKikimrRunConfig();
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        THttpRequest httpReq(HTTP_METHOD_GET);
+        httpReq.CgiParameters.emplace("tenant", "/Root/serverless");
+        httpReq.CgiParameters.emplace("tablets", "true");
+        httpReq.CgiParameters.emplace("enums", "true");
+        httpReq.CgiParameters.emplace("sort", "");
+        httpReq.CgiParameters.emplace("type", "any");
+        auto page = MakeHolder<TMonPage>("viewer", "title");
+        TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/nodes", nullptr);
+        auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
+
+        runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
+        NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);
+
+        size_t pos = result->Answer.find('{');
+        TString jsonResult = result->Answer.substr(pos);
+        Ctest << "json result: " << jsonResult << Endl;
+        NJson::TJsonValue json;
+        try {
+            NJson::ReadJsonTree(jsonResult, &json, true);
+        }
+        catch (yexception ex) {
+            Ctest << ex.what() << Endl;
+        }
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalNodes"), "0");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundNodes"), "0");
+    }
+
+    Y_UNIT_TEST(ServerlessWithExclusiveNodes)
+    {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port)
+                .SetNodeCount(1)
+                .SetDynamicNodeCount(2)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root")
+                .InitKikimrRunConfig();
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        THttpRequest httpReq(HTTP_METHOD_GET);
+        httpReq.CgiParameters.emplace("tenant", "/Root/serverless");
+        httpReq.CgiParameters.emplace("tablets", "true");
+        httpReq.CgiParameters.emplace("enums", "true");
+        httpReq.CgiParameters.emplace("sort", "");
+        httpReq.CgiParameters.emplace("type", "any");
+        auto page = MakeHolder<TMonPage>("viewer", "title");
+        TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/nodes", nullptr);
+        auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
+
+        size_t staticNodeId = 0;
+        size_t sharedDynNodeId = 0;
+        size_t exclusiveDynNodeId = 0;    
+        auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
+            switch (ev->GetTypeRewrite()) {
+                case TEvInterconnect::EvNodesInfo: {
+                    auto *x = reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr*>(&ev);
+                    TVector<TEvInterconnect::TNodeInfo> &nodes = (*x)->Get()->Nodes;
+                    UNIT_ASSERT_EQUAL(nodes.size(), 3);
+                    staticNodeId = nodes[0];
+                    sharedDynNodeId = nodes[1];
+                    exclusiveDynNodeId = nodes[2];
+                    break;
+                }
+                case TEvStateStorage::EvBoardInfo: {
+                    auto *x = reinterpret_cast<TEvStateStorage::TEvBoardInfo::TPtr*>(&ev);
+                    auto *record = (*x)->Get();
+                    using EStatus = TEvStateStorage::TEvBoardInfo::EStatus;
+                    const_cast<EStatus&>(record->Status) = EStatus::Ok;
+                    TActorId actorOnExclusiveDynNode = TActorId(exclusiveDynNodeId, 0, 0, 0);
+                    record->InfoEntries[actorOnExclusiveDynNode] = {};
+                    break;
+                }
+            }
+
+            return TTestActorRuntime::EEventAction::PROCESS;
+        };
+        runtime.SetObserverFunc(observerFunc);
+
+        runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
+        NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);
+
+        size_t pos = result->Answer.find('{');
+        TString jsonResult = result->Answer.substr(pos);
+        Ctest << "json result: " << jsonResult << Endl;
+        NJson::TJsonValue json;
+        try {
+            NJson::ReadJsonTree(jsonResult, &json, true);
+        }
+        catch (yexception ex) {
+            Ctest << ex.what() << Endl;
+        }
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalNodes"), "1");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundNodes"), "1");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("Nodes").GetArray().size(), 1);
+        auto node = json.GetMap().at("Nodes").GetArray()[0].GetMap();
+        UNIT_ASSERT_VALUES_EQUAL(node.at("NodeId"), exclusiveDynNodeId);
+    }
+
+    Y_UNIT_TEST(SharedDoesntShowExclusiveNodes)
+    {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port)
+                .SetNodeCount(1)
+                .SetDynamicNodeCount(2)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root")
+                .InitKikimrRunConfig();
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        THttpRequest httpReq(HTTP_METHOD_GET);
+        httpReq.CgiParameters.emplace("tenant", "Root/shared");
+        httpReq.CgiParameters.emplace("tablets", "true");
+        httpReq.CgiParameters.emplace("enums", "true");
+        httpReq.CgiParameters.emplace("sort", "");
+        httpReq.CgiParameters.emplace("type", "any");
+        auto page = MakeHolder<TMonPage>("viewer", "title");
+        TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/nodes", nullptr);
+        auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
+
+        size_t staticNodeId = 0;
+        size_t sharedDynNodeId = 0;
+        size_t exclusiveDynNodeId = 0; 
+        auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
+            switch (ev->GetTypeRewrite()) {
+                case TEvInterconnect::EvNodesInfo: {
+                    auto *x = reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr*>(&ev);
+                    TVector<TEvInterconnect::TNodeInfo> &nodes = (*x)->Get()->Nodes;
+                    UNIT_ASSERT_EQUAL(nodes.size(), 3);
+                    staticNodeId = nodes[0];
+                    sharedDynNodeId = nodes[1];
+                    exclusiveDynNodeId = nodes[2];
+                    break;
+                }
+                case TEvStateStorage::EvBoardInfo: {
+                    auto *x = reinterpret_cast<TEvStateStorage::TEvBoardInfo::TPtr*>(&ev);
+                    auto *record = (*x)->Get();
+                    using EStatus = TEvStateStorage::TEvBoardInfo::EStatus;
+                    const_cast<EStatus&>(record->Status) = EStatus::Ok;
+                    TActorId actorOnSharedDynNode = TActorId(sharedDynNodeId, 0, 0, 0);
+                    record->InfoEntries[actorOnSharedDynNode] = {};
+                    break;
+                }
+            }
+
+            return TTestActorRuntime::EEventAction::PROCESS;
+        };
+        runtime.SetObserverFunc(observerFunc);
+
+        runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
+        NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);
+
+        size_t pos = result->Answer.find('{');
+        TString jsonResult = result->Answer.substr(pos);
+        Ctest << "json result: " << jsonResult << Endl;
+        NJson::TJsonValue json;
+        try {
+            NJson::ReadJsonTree(jsonResult, &json, true);
+        }
+        catch (yexception ex) {
+            Ctest << ex.what() << Endl;
+        }
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalNodes"), "1");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundNodes"), "1");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("Nodes").GetArray().size(), 1);
+        auto node = json.GetMap().at("Nodes").GetArray()[0].GetMap();
+        UNIT_ASSERT_VALUES_EQUAL(node.at("NodeId"), sharedDynNodeId);
+    }
+
+    Y_UNIT_TEST(ServerlessWithExclusiveNodesCheckTable)
+    {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port)
+                .SetNodeCount(1)
+                .SetDynamicNodeCount(3)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root")
+                .InitKikimrRunConfig();
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        THttpRequest httpReq(HTTP_METHOD_GET);
+        httpReq.CgiParameters.emplace("tenant", "/Root/serverless");
+        httpReq.CgiParameters.emplace("path", "/Root/serverless/users");
+        httpReq.CgiParameters.emplace("tablets", "true");
+        httpReq.CgiParameters.emplace("enums", "true");
+        httpReq.CgiParameters.emplace("sort", "");
+        httpReq.CgiParameters.emplace("type", "any");
+        auto page = MakeHolder<TMonPage>("viewer", "title");
+        TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/nodes", nullptr);
+        auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
+
+        const TPathId SERVERLESS_DOMAIN_KEY = {7000000000, 2};
+        const TPathId SHARED_DOMAIN_KEY = {7000000000, 1};
+        const TPathId SERVERLESS_TABLE = {7000000001, 2};
+
+        size_t staticNodeId = 0;
+        size_t sharedDynNodeId = 0;
+        size_t exclusiveDynNodeId = 0;
+        size_t secondExclusiveDynNodeId = 0;    
+        auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
+            switch (ev->GetTypeRewrite()) {
+                case TEvTxProxySchemeCache::EvNavigateKeySetResult: {
+                    auto *x = reinterpret_cast<TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr*>(&ev);
+                    TSchemeCacheNavigate::TEntry& entry((*x)->Get()->Request->ResultSet.front());
+                    TString path = CanonizePath(entry.Path);
+                    if (path == "/Root/serverless" || entry.TableId.PathId == SERVERLESS_DOMAIN_KEY) {
+                        entry.Status = TSchemeCacheNavigate::EStatus::Ok;
+                        entry.Kind = TSchemeCacheNavigate::EKind::KindExtSubdomain;
+                        entry.DomainInfo = MakeIntrusive<TDomainInfo>(SERVERLESS_DOMAIN_KEY, SHARED_DOMAIN_KEY);
+                    } else if (path == "/Root/shared" || entry.TableId.PathId == SHARED_DOMAIN_KEY) {
+                        entry.Status = TSchemeCacheNavigate::EStatus::Ok;
+                        entry.Kind = TSchemeCacheNavigate::EKind::KindExtSubdomain;
+                        entry.DomainInfo = MakeIntrusive<TDomainInfo>(SHARED_DOMAIN_KEY, SHARED_DOMAIN_KEY);
+                        auto domains = runtime.GetAppData().DomainsInfo;
+                        auto domain = domains->Domains.begin()->second;
+                        ui64 hiveId = domains->GetHive(domain->DefaultHiveUid);
+                        entry.DomainInfo->Params.SetHive(hiveId);
+                    } else if (path == "/Root/serverless/users" || entry.TableId.PathId == SERVERLESS_TABLE) {
+                        entry.Status = TSchemeCacheNavigate::EStatus::Ok;
+                        entry.Kind = TSchemeCacheNavigate::EKind::KindTable;
+                        entry.DomainInfo = MakeIntrusive<TDomainInfo>(SERVERLESS_DOMAIN_KEY, SHARED_DOMAIN_KEY);
+                        auto dirEntryInfo = MakeIntrusive<TSchemeCacheNavigate::TDirEntryInfo>();
+                        dirEntryInfo->Info.SetSchemeshardId(SERVERLESS_TABLE.OwnerId);
+                        dirEntryInfo->Info.SetPathId(SERVERLESS_TABLE.LocalPathId);
+                        entry.Self = dirEntryInfo;
+                    }
+                    break;
+                }
+                case TEvInterconnect::EvNodesInfo: {
+                    auto *x = reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr*>(&ev);
+                    TVector<TEvInterconnect::TNodeInfo> &nodes = (*x)->Get()->Nodes;
+                    UNIT_ASSERT_EQUAL(nodes.size(), 4);
+                    staticNodeId = nodes[0];
+                    sharedDynNodeId = nodes[1];
+                    exclusiveDynNodeId = nodes[2];
+                    secondExclusiveDynNodeId = nodes[3];
+                    break;
+                }
+                case TEvStateStorage::EvBoardInfo: {
+                    auto *x = reinterpret_cast<TEvStateStorage::TEvBoardInfo::TPtr*>(&ev);
+                    auto *record = (*x)->Get();
+                    using EStatus = TEvStateStorage::TEvBoardInfo::EStatus;
+                    const_cast<EStatus&>(record->Status) = EStatus::Ok;
+                    TActorId actorOnExclusiveDynNode = TActorId(exclusiveDynNodeId, 0, 0, 0);
+                    record->InfoEntries[actorOnExclusiveDynNode] = {};
+                    TActorId actorOnSecondExclusiveDynNode = TActorId(secondExclusiveDynNodeId, 0, 0, 0);
+                    record->InfoEntries[actorOnSecondExclusiveDynNode] = {};
+                    break;
+                }
+                case TEvHive::EvResponseHiveNodeStats: {
+                    auto *x = reinterpret_cast<TEvHive::TEvResponseHiveNodeStats::TPtr*>(&ev);
+                    auto &record = (*x)->Get()->Record;
+                    auto *sharedNodeStats = record.MutableNodeStats()->Add();
+                    sharedNodeStats->SetNodeId(sharedDynNodeId);
+                    sharedNodeStats->MutableNodeDomain()->SetSchemeShard(SHARED_DOMAIN_KEY.OwnerId);
+                    sharedNodeStats->MutableNodeDomain()->SetPathId(SHARED_DOMAIN_KEY.LocalPathId);
+
+                    auto *exclusiveNodeStats = record.MutableNodeStats()->Add();
+                    exclusiveNodeStats->SetNodeId(exclusiveDynNodeId);
+                    exclusiveNodeStats->MutableNodeDomain()->SetSchemeShard(SERVERLESS_DOMAIN_KEY.OwnerId);
+                    exclusiveNodeStats->MutableNodeDomain()->SetPathId(SERVERLESS_DOMAIN_KEY.LocalPathId);
+
+                    auto *secondExclusiveNodeStats = record.MutableNodeStats()->Add();
+                    secondExclusiveNodeStats->SetNodeId(secondExclusiveDynNodeId);
+                    secondExclusiveNodeStats->MutableNodeDomain()->SetSchemeShard(SERVERLESS_DOMAIN_KEY.OwnerId);
+                    secondExclusiveNodeStats->MutableNodeDomain()->SetPathId(SERVERLESS_DOMAIN_KEY.LocalPathId);
+
+                    // filtered one datashard from /Root/serverless/users
+                    auto *stateStats = secondExclusiveNodeStats->MutableStateStats()->Add();
+                    stateStats->SetTabletType(NKikimrTabletBase::TTabletTypes::DataShard);
+                    stateStats->SetVolatileState(NKikimrHive::TABLET_VOLATILE_STATE_RUNNING);
+                    stateStats->SetCount(1);
+                    break;
+                }
+            }
+
+            return TTestActorRuntime::EEventAction::PROCESS;
+        };
+        runtime.SetObserverFunc(observerFunc);
+
+        runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
+        NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);
+
+        size_t pos = result->Answer.find('{');
+        TString jsonResult = result->Answer.substr(pos);
+        Ctest << "json result: " << jsonResult << Endl;
+        NJson::TJsonValue json;
+        try {
+            NJson::ReadJsonTree(jsonResult, &json, true);
+        }
+        catch (yexception ex) {
+            Ctest << ex.what() << Endl;
+        }
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalNodes"), "2");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundNodes"), "2");
+        UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("Nodes").GetArray().size(), 2);
+        auto firstNode = json.GetMap().at("Nodes").GetArray()[0].GetMap();
+        UNIT_ASSERT_VALUES_EQUAL(firstNode.at("NodeId"), exclusiveDynNodeId);
+        UNIT_ASSERT(!firstNode.contains("Tablets"));
+        auto secondNode = json.GetMap().at("Nodes").GetArray()[1].GetMap();
+        UNIT_ASSERT_VALUES_EQUAL(secondNode.at("NodeId"), secondExclusiveDynNodeId);
+        UNIT_ASSERT_VALUES_EQUAL(secondNode.at("Tablets").GetArray().size(), 1);
+        auto tablet = secondNode.at("Tablets").GetArray()[0].GetMap();
+        UNIT_ASSERT_VALUES_EQUAL(tablet.at("Type"), "DataShard");
+        UNIT_ASSERT_VALUES_EQUAL(tablet.at("State"), "Green");
+        UNIT_ASSERT_VALUES_EQUAL(tablet.at("Count"), 1);
+    }
 }