diff options
author | zalyalov <zalyalov@yandex-team.com> | 2023-10-18 14:32:55 +0300 |
---|---|---|
committer | zalyalov <zalyalov@yandex-team.com> | 2023-10-18 15:12:54 +0300 |
commit | 2ae9c07e1dce24403b1c58785b9df0c5936564be (patch) | |
tree | e9d68ad1ecbf2d677ac2e379a39495e8ff5cf86a | |
parent | 115ed856c8c939faa597722a9e8b9c12a668c82b (diff) | |
download | ydb-2ae9c07e1dce24403b1c58785b9df0c5936564be.tar.gz |
add owner id to object id to make it unique KIKIMR-19697
-rw-r--r-- | ydb/core/mind/hive/hive.h | 3 | ||||
-rw-r--r-- | ydb/core/mind/hive/hive_impl.cpp | 10 | ||||
-rw-r--r-- | ydb/core/mind/hive/hive_impl.h | 9 | ||||
-rw-r--r-- | ydb/core/mind/hive/hive_statics.cpp | 4 | ||||
-rw-r--r-- | ydb/core/mind/hive/hive_ut.cpp | 70 | ||||
-rw-r--r-- | ydb/core/mind/hive/leader_tablet_info.h | 4 | ||||
-rw-r--r-- | ydb/core/mind/hive/monitoring.cpp | 6 | ||||
-rw-r--r-- | ydb/core/mind/hive/node_info.h | 2 | ||||
-rw-r--r-- | ydb/core/mind/hive/object_distribution.h | 16 | ||||
-rw-r--r-- | ydb/core/mind/hive/object_distribution_ut.cpp | 4 | ||||
-rw-r--r-- | ydb/core/mind/hive/tablet_info.cpp | 2 | ||||
-rw-r--r-- | ydb/core/mind/hive/tablet_info.h | 2 | ||||
-rw-r--r-- | ydb/core/mind/hive/tx__create_tablet.cpp | 8 | ||||
-rw-r--r-- | ydb/core/mind/hive/tx__load_everything.cpp | 12 | ||||
-rw-r--r-- | ydb/core/mind/hive/tx__seize_tablets.cpp | 2 | ||||
-rw-r--r-- | ydb/core/mind/hive/tx__seize_tablets_reply.cpp | 2 | ||||
-rw-r--r-- | ydb/core/mind/hive/tx__update_tablets_object.cpp | 22 | ||||
-rw-r--r-- | ydb/core/protos/hive.proto | 2 | ||||
-rw-r--r-- | ydb/core/viewer/json_compute.h | 6 | ||||
-rw-r--r-- | ydb/core/viewer/json_nodes.h | 10 | ||||
-rw-r--r-- | ydb/core/viewer/json_pipe_req.h | 7 |
21 files changed, 145 insertions, 58 deletions
diff --git a/ydb/core/mind/hive/hive.h b/ydb/core/mind/hive/hive.h index 1e0dad3b50..6e80f78264 100644 --- a/ydb/core/mind/hive/hive.h +++ b/ydb/core/mind/hive/hive.h @@ -46,6 +46,7 @@ using TStorageGroupId = ui32; using TFullTabletId = std::pair<TTabletId, TFollowerId>; using TObjectId = ui64; // schema object id, used to organize tablets of the same schema object using TOwnerId = ui64; +using TFullObjectId = std::pair<TOwnerId, TObjectId>; using TResourceRawValues = std::tuple<i64, i64, i64, i64>; // CPU, Memory, Network, Counter using TResourceNormalizedValues = std::tuple<double, double, double, double>; using TOwnerIdxType = NScheme::TPairUi64Ui64; @@ -255,7 +256,7 @@ struct TBalancerSettings { ui64 MaxInFlight = 1; const std::vector<TNodeId> FilterNodeIds = {}; EResourceToBalance ResourceToBalance = EResourceToBalance::Dominant; - std::optional<TObjectId> FilterObjectId; + std::optional<TFullObjectId> FilterObjectId; }; struct TBalancerStats { diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index b5497776c1..97cb577dfa 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -1715,7 +1715,7 @@ void THive::FillTabletInfo(NKikimrHive::TEvResponseHiveInfo& response, ui64 tabl auto& tabletInfo = *response.AddTablets(); tabletInfo.SetTabletID(tabletId); tabletInfo.SetTabletType(info->Type); - tabletInfo.SetObjectId(info->ObjectId); + tabletInfo.SetObjectId(info->ObjectId.second); tabletInfo.SetState(static_cast<ui32>(info->State)); tabletInfo.SetTabletBootMode(info->BootMode); tabletInfo.SetVolatileState(info->GetVolatileState()); @@ -1899,7 +1899,7 @@ void THive::Handle(TEvHive::TEvRequestHiveNodeStats::TPtr& ev) { } if (request.GetReturnExtendedTabletInfo()) { if (request.HasFilterTabletsByPathId()) { - auto itTabletsOfObject = node.TabletsOfObject.find(request.GetFilterTabletsByPathId()); + auto itTabletsOfObject = node.TabletsOfObject.find({request.GetFilterTabletsBySchemeShardId(), request.GetFilterTabletsByPathId()}); if (itTabletsOfObject != node.TabletsOfObject.end()) { std::vector<std::vector<ui32>> tabletStateToTypeToCount; tabletStateToTypeToCount.resize(NKikimrHive::ETabletVolatileState_ARRAYSIZE); @@ -2396,7 +2396,7 @@ void THive::DivideMetrics(NKikimrTabletBase::TMetrics& metrics, ui64 divider) { metrics.SetWriteThroughput(metrics.GetWriteThroughput() / divider); } -NKikimrTabletBase::TMetrics THive::GetDefaultResourceValuesForObject(TObjectId objectId) { +NKikimrTabletBase::TMetrics THive::GetDefaultResourceValuesForObject(TFullObjectId objectId) { NKikimrTabletBase::TMetrics metrics; auto itTablets = ObjectToTabletMetrics.find(objectId); if (itTablets != ObjectToTabletMetrics.end()) { @@ -2584,7 +2584,7 @@ TDuration THive::GetBalancerCooldown() const { } } -void THive::UpdateObjectCount(TObjectId object, TNodeId node, i64 diff) { +void THive::UpdateObjectCount(TFullObjectId object, TNodeId node, i64 diff) { if (!GetSpreadNeighbours()) { return; } @@ -2594,7 +2594,7 @@ void THive::UpdateObjectCount(TObjectId object, TNodeId node, i64 diff) { BLOG_TRACE("UpdateObjectCount " << "for " << object << " on " << node << " (" << diff << ") ~> Imbalance: " << ObjectDistributions.GetMaxImbalance()); } -ui64 THive::GetObjectImbalance(TObjectId object) { +ui64 THive::GetObjectImbalance(TFullObjectId object) { auto it = ObjectDistributions.Distributions.find(object); if (it == ObjectDistributions.Distributions.end()) { return 0; diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index 3858300571..b91c806454 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -148,6 +148,7 @@ TString LongToShortTabletName(const TString& longTabletName); TString GetLocationString(const NActors::TNodeLocation& location); void MakeTabletTypeSet(std::vector<TTabletTypes::EType>& list); bool IsValidTabletType(TTabletTypes::EType type); +bool IsValidObjectId(const TFullObjectId& objectId); TString GetRunningTabletsText(ui64 runningTablets, ui64 totalTablets, bool warmUp); class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveSharedSettings { @@ -351,7 +352,7 @@ protected: } }; - std::unordered_map<TObjectId, TAggregateMetrics> ObjectToTabletMetrics; + std::unordered_map<TFullObjectId, TAggregateMetrics> ObjectToTabletMetrics; std::unordered_map<TTabletTypes::EType, TAggregateMetrics> TabletTypeToTabletMetrics; TBootQueue BootQueue; @@ -658,8 +659,8 @@ public: void ExecuteProcessBootQueue(NIceDb::TNiceDb& db, TSideEffects& sideEffects); void UpdateTabletFollowersNumber(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db, TSideEffects& sideEffects); TDuration GetBalancerCooldown() const; - void UpdateObjectCount(TObjectId object, TNodeId node, i64 diff); - ui64 GetObjectImbalance(TObjectId object); + void UpdateObjectCount(TFullObjectId object, TNodeId node, i64 diff); + ui64 GetObjectImbalance(TFullObjectId object); ui32 GetEventPriority(IEventHandle* ev); void PushProcessIncomingEvent(); @@ -929,7 +930,7 @@ protected: THiveStats GetStats() const; void RemoveSubActor(ISubActor* subActor); const NKikimrLocal::TLocalConfig &GetLocalConfig() const { return LocalConfig; } - NKikimrTabletBase::TMetrics GetDefaultResourceValuesForObject(TObjectId objectId); + NKikimrTabletBase::TMetrics GetDefaultResourceValuesForObject(TFullObjectId objectId); NKikimrTabletBase::TMetrics GetDefaultResourceValuesForTabletType(TTabletTypes::EType type); NKikimrTabletBase::TMetrics GetDefaultResourceValuesForProfile(TTabletTypes::EType type, const TString& resourceProfile); static void AggregateMetricsMax(NKikimrTabletBase::TMetrics& aggregate, const NKikimrTabletBase::TMetrics& value); diff --git a/ydb/core/mind/hive/hive_statics.cpp b/ydb/core/mind/hive/hive_statics.cpp index c1302fc809..73abd086d2 100644 --- a/ydb/core/mind/hive/hive_statics.cpp +++ b/ydb/core/mind/hive/hive_statics.cpp @@ -388,6 +388,10 @@ bool IsValidTabletType(TTabletTypes::EType type) { ); } +bool IsValidObjectId(const TFullObjectId& objectId) { + return objectId.second != 0; +} + NJson::TJsonValue THive::GetBalancerProgressJson() { NJson::TJsonValue result; for (const auto& stats : BalancerStats) { diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index ca86bcf105..869d4e64f6 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -4265,6 +4265,76 @@ Y_UNIT_TEST_SUITE(THiveTest) { UNIT_ASSERT_VALUES_EQUAL(updatedOnFirstNode, 2); } + Y_UNIT_TEST(TestSpreadNeighboursDifferentOwners) { + static constexpr ui64 TABLETS_PER_OWNER = 6; + TTestBasicRuntime runtime(2, false); + Setup(runtime, true); + TActorId senderA = runtime.AllocateEdgeActor(); + const ui64 hiveTablet = MakeDefaultHiveID(0); + + CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive); + + // wait for creation of nodes + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvStatus, 2); + runtime.DispatchEvents(options); + } + + struct TTestOwner { + const ui64 Id; + ui64 Idx = 0; + + TTestOwner(ui64 id) : Id(id) {} + + ui64 CreateNewTablet(TTestBasicRuntime& runtime, ui64 hiveTablet) { + auto ev = MakeHolder<TEvHive::TEvCreateTablet>(Id, ++Idx, TTabletTypes::Dummy, BINDED_CHANNELS); + ev->Record.SetObjectId(1); + return SendCreateTestTablet(runtime, hiveTablet, Id, std::move(ev), 0, true); + } + }; + + TTestOwner owner1(MakeDefaultHiveID(1)); + TTestOwner owner2(MakeDefaultHiveID(2)); + + for (ui64 i = 0; i < TABLETS_PER_OWNER; ++i) { + ui64 tablet1; + ui64 tablet2; + if (i * 2 < TABLETS_PER_OWNER) { + tablet1 = owner1.CreateNewTablet(runtime, hiveTablet); + tablet2 = owner2.CreateNewTablet(runtime, hiveTablet); + } else { + tablet1 = owner2.CreateNewTablet(runtime, hiveTablet); + tablet2 = owner1.CreateNewTablet(runtime, hiveTablet); + } + MakeSureTabletIsUp(runtime, tablet1, 0); + MakeSureTabletIsUp(runtime, tablet2, 0); + } + + runtime.SendToPipe(hiveTablet, senderA, new TEvHive::TEvRequestHiveInfo()); + TAutoPtr<IEventHandle> handle; + TEvHive::TEvResponseHiveInfo* response = runtime.GrabEdgeEventRethrow<TEvHive::TEvResponseHiveInfo>(handle); + + struct TTestTabletInfo { + ui64 OwnerId; + ui64 NodeId; + + bool operator<(const TTestTabletInfo& other) const { + return std::tie(OwnerId, NodeId) < std::tie(other.OwnerId, other.NodeId); + } + }; + std::map<TTestTabletInfo, ui64> distribution; + + for (const auto& tablet : response->Record.GetTablets()) { + distribution[{tablet.GetTabletOwner().GetOwner(), tablet.GetNodeID()}]++; + } + + // Each node should have half tablet from each owner + for (const auto& p : distribution) { + UNIT_ASSERT_VALUES_EQUAL(p.second, TABLETS_PER_OWNER / 2); + } + } + Y_UNIT_TEST(TestHiveBalancerDifferentResources) { static constexpr ui64 TABLETS_PER_NODE = 4; TTestBasicRuntime runtime(2, false); diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index ee0e9a39b5..ec43cc2ddc 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -28,7 +28,7 @@ public: TTabletId Id; ETabletState State; TTabletTypes::EType Type; - TObjectId ObjectId; + TFullObjectId ObjectId; TSubDomainKey ObjectDomain; TVector<TNodeId> AllowedNodes; TVector<TDataCenterId> AllowedDataCenters; @@ -57,7 +57,7 @@ public: , Id(id) , State(ETabletState::Unknown) , Type(TTabletTypes::TypeInvalid) - , ObjectId(0) + , ObjectId(0, 0) , ChannelProfileReassignReason(NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_NO) , KnownGeneration(0) , Category(nullptr) diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index 751597a13c..fe7a8435cf 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -3355,7 +3355,7 @@ public: result["Id"] = TStringBuilder() << tablet.Id; result["State"] = ETabletStateName(tablet.State); result["Type"] = TTabletTypes::EType_Name(tablet.Type); - result["ObjectId"] = tablet.ObjectId; + result["ObjectId"] = TStringBuilder() << tablet.ObjectId; result["ObjectDomain"] = TStringBuilder() << tablet.ObjectDomain; result["AllowedNodes"] = MakeFrom(tablet.AllowedNodes); result["AllowedDataCenters"] = MakeFrom(tablet.AllowedDataCenters); @@ -3420,7 +3420,7 @@ public: TTxType GetTxType() const override { return NHive::TXTYPE_MON_OBJECT_STATS; } bool Execute(TTransactionContext&, const TActorContext&) override { - std::map<ui64, std::vector<std::pair<ui64, ui64>>> distributionOfObjects; + std::map<TFullObjectId, std::vector<std::pair<ui64, ui64>>> distributionOfObjects; for (const auto& node : Self->Nodes) { for (const auto& obj : node.second.TabletsOfObject) { distributionOfObjects[obj.first].emplace_back(node.first, obj.second.size()); @@ -3430,7 +3430,7 @@ public: Result.SetType(NJson::JSON_ARRAY); for (const auto& [key, val] : distributionOfObjects) { NJson::TJsonValue listItem; - listItem["objectId"] = key; + listItem["objectId"] = TStringBuilder() << key; NJson::TJsonValue& distribution = listItem["distribution"]; for (const auto& [node, cnt] : val) { distribution[TStringBuilder() << node] = cnt; diff --git a/ydb/core/mind/hive/node_info.h b/ydb/core/mind/hive/node_info.h index 7360475c09..603b4c1ab7 100644 --- a/ydb/core/mind/hive/node_info.h +++ b/ydb/core/mind/hive/node_info.h @@ -43,7 +43,7 @@ public: TDrainSettings DrainSettings; std::unordered_map<TTabletInfo::EVolatileState, std::unordered_set<TTabletInfo*>> Tablets; std::unordered_map<TTabletTypes::EType, std::unordered_set<TTabletInfo*>> TabletsRunningByType; - std::unordered_map<TObjectId, std::unordered_set<TTabletInfo*>> TabletsOfObject; + std::unordered_map<TFullObjectId, std::unordered_set<TTabletInfo*>> TabletsOfObject; TResourceRawValues ResourceValues; // accumulated resources from tablet metrics TResourceRawValues ResourceTotalValues; // actual used resources from the node (should be greater or equal one above) NMetrics::TAverageValue<TResourceRawValues, 20> AveragedResourceTotalValues; diff --git a/ydb/core/mind/hive/object_distribution.h b/ydb/core/mind/hive/object_distribution.h index e9509a8e97..f2814a8205 100644 --- a/ydb/core/mind/hive/object_distribution.h +++ b/ydb/core/mind/hive/object_distribution.h @@ -13,11 +13,11 @@ namespace NHive { struct TObjectDistribution { std::multiset<i64> SortedDistribution; std::unordered_map<TNodeId, i64> Distribution; - const TObjectId Id; + const TFullObjectId Id; double Mean = 0; double VarianceNumerator = 0; - TObjectDistribution(TObjectId id) : Id(id) {} + TObjectDistribution(TFullObjectId id) : Id(id) {} double GetImbalance() const { if (SortedDistribution.empty()) { @@ -84,7 +84,7 @@ struct TObjectDistribution { struct TObjectDistributions { std::multiset<TObjectDistribution> SortedDistributions; - std::unordered_map<TObjectId, std::multiset<TObjectDistribution>::iterator> Distributions; + std::unordered_map<TFullObjectId, std::multiset<TObjectDistribution>::iterator> Distributions; ui64 ImbalancedObjects = 0; std::unordered_set<TNodeId> Nodes; bool Enabled = true; @@ -97,16 +97,16 @@ struct TObjectDistributions { } struct TObjectToBalance { - TObjectId ObjectId; + TFullObjectId ObjectId; std::vector<TNodeId> Nodes; - TObjectToBalance(TObjectId objectId) : ObjectId(objectId) {} + TObjectToBalance(TFullObjectId objectId) : ObjectId(objectId) {} }; TObjectToBalance GetObjectToBalance() { Y_DEBUG_ABORT_UNLESS(!SortedDistributions.empty()); if (SortedDistributions.empty()) { - return TObjectToBalance(0); + return TObjectToBalance(TFullObjectId()); } const auto& dist = *SortedDistributions.rbegin(); i64 maxCnt = *dist.SortedDistribution.rbegin(); @@ -134,7 +134,7 @@ struct TObjectDistributions { } template <typename F> - bool UpdateDistribution(TObjectId object, F updateFunc) { + bool UpdateDistribution(TFullObjectId object, F updateFunc) { auto distIt = Distributions.find(object); if (distIt == Distributions.end()) { return false; @@ -162,7 +162,7 @@ struct TObjectDistributions { } - void UpdateCount(TObjectId object, TNodeId node, i64 diff) { + void UpdateCount(TFullObjectId object, TNodeId node, i64 diff) { if (!Enabled) { return; } diff --git a/ydb/core/mind/hive/object_distribution_ut.cpp b/ydb/core/mind/hive/object_distribution_ut.cpp index a695d0f03a..d33a58f33e 100644 --- a/ydb/core/mind/hive/object_distribution_ut.cpp +++ b/ydb/core/mind/hive/object_distribution_ut.cpp @@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(ObjectDistribuiton) { static constexpr size_t NUM_OBJECTS = 250; static constexpr size_t NUM_OPERATIONS = 10'000; - std::map<std::pair<TNodeId, TObjectId>, ui64> trueDistribution; + std::map<std::pair<TNodeId, TFullObjectId>, ui64> trueDistribution; std::mt19937 engine(42); std::uniform_int_distribution<TObjectId> pickObject(0, NUM_OBJECTS - 1); @@ -27,7 +27,7 @@ Y_UNIT_TEST_SUITE(ObjectDistribuiton) { } for (size_t i = 0; i < NUM_OPERATIONS; i++) { - TObjectId object = pickObject(engine); + TFullObjectId object = {0, pickObject(engine)}; TNodeId node = pickNode(engine); ui64& curCount = trueDistribution[{node, object}]; i64 diff = 1; diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 3c40dc6c8d..bf0950ef11 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -68,7 +68,7 @@ std::pair<TTabletId, TFollowerId> TTabletInfo::GetFullTabletId() const { } } -TObjectId TTabletInfo::GetObjectId() const { +TFullObjectId TTabletInfo::GetObjectId() const { return GetLeader().ObjectId; } diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index d77cc046f6..c32ae342eb 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -191,7 +191,7 @@ public: TFollowerTabletInfo& AsFollower(); const TFollowerTabletInfo& AsFollower() const; std::pair<TTabletId, TFollowerId> GetFullTabletId() const; - TObjectId GetObjectId() const; + TFullObjectId GetObjectId() const; TTabletTypes::EType GetTabletType() const; TString ToString() const; TString StateString() const; diff --git a/ydb/core/mind/hive/tx__create_tablet.cpp b/ydb/core/mind/hive/tx__create_tablet.cpp index ea30e91409..6e73801529 100644 --- a/ydb/core/mind/hive/tx__create_tablet.cpp +++ b/ydb/core/mind/hive/tx__create_tablet.cpp @@ -270,7 +270,7 @@ public: tablet->AssignDomains(ObjectDomain, AllowedDomains); db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::ObjectDomain>(ObjectDomain); db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::AllowedDomains>(AllowedDomains); - tablet->ObjectId = ObjectId; + tablet->ObjectId = {OwnerId, ObjectId}; db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::ObjectID>(ObjectId); tablet->BootMode = BootMode; db.Table<Schema::Tablet>().Key(TabletId).Update<Schema::Tablet::BootMode>(BootMode); @@ -371,7 +371,7 @@ public: tablet.AllowedDataCenters = AllowedDataCenterIds; tablet.DataCentersPreference = DataCentersPreference; tablet.BootMode = BootMode; - tablet.ObjectId = ObjectId; + tablet.ObjectId = {OwnerId, ObjectId}; tablet.AssignDomains(ObjectDomain, AllowedDomains); tablet.Statistics.SetLastAliveTimestamp(now.MilliSeconds()); tablet.BalancerPolicy = BalancerPolicy; @@ -392,7 +392,7 @@ public: NIceDb::TUpdate<Schema::Tablet::DataCentersPreference>(tablet.DataCentersPreference), NIceDb::TUpdate<Schema::Tablet::AllowedDomains>(AllowedDomains), NIceDb::TUpdate<Schema::Tablet::BootMode>(tablet.BootMode), - NIceDb::TUpdate<Schema::Tablet::ObjectID>(tablet.ObjectId), + NIceDb::TUpdate<Schema::Tablet::ObjectID>(tablet.ObjectId.second), NIceDb::TUpdate<Schema::Tablet::ObjectDomain>(ObjectDomain), NIceDb::TUpdate<Schema::Tablet::Statistics>(tablet.Statistics), NIceDb::TUpdate<Schema::Tablet::BalancerPolicy>(tablet.BalancerPolicy)); @@ -426,7 +426,7 @@ public: resourceValues.CopyFrom(Self->GetDefaultResourceValuesForTabletType(tablet.Type)); BLOG_D("THive::TTxCreateTablet::Execute; Default resources after merge for type " << tablet.Type << ": {" << resourceValues.ShortDebugString() << "}"); - if (tablet.ObjectId != 0) { + if (IsValidObjectId(tablet.ObjectId)) { resourceValues.MergeFrom(Self->GetDefaultResourceValuesForObject(tablet.ObjectId)); BLOG_D("THive::TTxCreateTablet::Execute; Default resources after merge for object " << tablet.ObjectId << ": {" << resourceValues.ShortDebugString() << "}"); } diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 90c78d3df2..b56a7df753 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -373,7 +373,13 @@ public: std::tuple<TTabletId, THive&>(tabletId, *Self)).first->second; tablet.State = tabletRowset.GetValue<Schema::Tablet::State>(); tablet.Type = tabletRowset.GetValue<Schema::Tablet::TabletType>(); - tablet.ObjectId = tabletRowset.GetValueOrDefault<Schema::Tablet::ObjectID>(); + + TObjectId objectId = tabletRowset.GetValueOrDefault<Schema::Tablet::ObjectID>(); + TOwnerIdxType::TValueType owner = tabletRowset.GetValue<Schema::Tablet::Owner>(); + Self->OwnerToTablet.emplace(owner, tabletId); + tablet.Owner = owner; + tablet.ObjectId = {owner.first, objectId}; + Self->ObjectToTabletMetrics[tablet.ObjectId].IncreaseCount(); Self->TabletTypeToTabletMetrics[tablet.Type].IncreaseCount(); tablet.AllowedNodes = tabletRowset.GetValue<Schema::Tablet::AllowedNodes>(); @@ -443,10 +449,6 @@ public: } tablet.InitTabletMetrics(); - TOwnerIdxType::TValueType owner = tabletRowset.GetValue<Schema::Tablet::Owner>(); - Self->OwnerToTablet.emplace(owner, tabletId); - tablet.Owner = owner; - tablet.TabletStorageInfo.Reset(new TTabletStorageInfo(tabletId, tablet.Type)); tablet.TabletStorageInfo->Version = tabletRowset.GetValueOrDefault<Schema::Tablet::TabletStorageVersion>(); tablet.TabletStorageInfo->TenantPathId = tablet.GetTenant(); diff --git a/ydb/core/mind/hive/tx__seize_tablets.cpp b/ydb/core/mind/hive/tx__seize_tablets.cpp index 09a93c9656..20a5947b9d 100644 --- a/ydb/core/mind/hive/tx__seize_tablets.cpp +++ b/ydb/core/mind/hive/tx__seize_tablets.cpp @@ -73,7 +73,7 @@ public: tabletInfo.SetTabletType(tablet.Type); tabletInfo.SetState(static_cast<ui32>(tablet.State)); tabletInfo.SetVolatileState(tablet.GetVolatileState()); - tabletInfo.SetObjectId(tablet.ObjectId); + tabletInfo.SetObjectId(tablet.ObjectId.second); tabletInfo.SetGeneration(tablet.KnownGeneration); ActorIdToProto(tablet.LockedToActor, tabletInfo.MutableLockedToActor()); tabletInfo.SetLockedReconnectTimeout(tablet.LockedReconnectTimeout.MilliSeconds()); diff --git a/ydb/core/mind/hive/tx__seize_tablets_reply.cpp b/ydb/core/mind/hive/tx__seize_tablets_reply.cpp index 8af4a5ca48..15114e8acf 100644 --- a/ydb/core/mind/hive/tx__seize_tablets_reply.cpp +++ b/ydb/core/mind/hive/tx__seize_tablets_reply.cpp @@ -32,7 +32,7 @@ public: tablet.State = static_cast<ETabletState>(protoTabletInfo.GetState()); tablet.Owner = owner; tablet.BootMode = protoTabletInfo.GetTabletBootMode(); - tablet.ObjectId = protoTabletInfo.GetObjectId(); + tablet.ObjectId = {owner.first, protoTabletInfo.GetObjectId()}; TVector<TSubDomainKey> allowedDomains; for (const auto& allowedDomain : protoTabletInfo.GetAllowedDomains()) { diff --git a/ydb/core/mind/hive/tx__update_tablets_object.cpp b/ydb/core/mind/hive/tx__update_tablets_object.cpp index 3cbca3ed80..83f45ab3ca 100644 --- a/ydb/core/mind/hive/tx__update_tablets_object.cpp +++ b/ydb/core/mind/hive/tx__update_tablets_object.cpp @@ -25,20 +25,24 @@ public: NIceDb::TNiceDb db(txc.DB); ui64 tabletsUpdated = 0; - auto& newObjectMetrics = Self->ObjectToTabletMetrics[objectId]; + THive::TAggregateMetrics* newObjectMetrics = nullptr; for (auto tabletId : msg->Record.GetTabletIds()) { auto tablet = Self->FindTablet(tabletId); if (tablet == nullptr) { continue; } - auto oldObject = tablet->ObjectId; - if (oldObject == objectId) { + auto oldObject = tablet->GetObjectId(); + tablet->ObjectId.second = objectId; + auto newObject = tablet->GetObjectId(); // It should be the same on every iteration + if (oldObject == newObject) { continue; } - tablet->ObjectId = objectId; ++tabletsUpdated; + if (!newObjectMetrics) { + newObjectMetrics = &Self->ObjectToTabletMetrics[newObject]; + } - newObjectMetrics.AggregateDiff({}, tablet->GetResourceValues(), tablet); + newObjectMetrics->AggregateDiff({}, tablet->GetResourceValues(), tablet); if (auto itObj = Self->ObjectToTabletMetrics.find(oldObject); itObj != Self->ObjectToTabletMetrics.end()) { auto& oldObjectMetrics = itObj->second; oldObjectMetrics.DecreaseCount(); @@ -51,16 +55,18 @@ public: if (auto node = tablet->GetNode(); node != nullptr) { node->TabletsOfObject[oldObject].erase(tablet); - node->TabletsOfObject[objectId].emplace(tablet); + node->TabletsOfObject[newObject].emplace(tablet); if (tablet->HasCounter()) { Self->UpdateObjectCount(oldObject, node->Id, -1); - Self->UpdateObjectCount(objectId, node->Id, +1); + Self->UpdateObjectCount(newObject, node->Id, +1); } } db.Table<Schema::Tablet>().Key(tabletId).Update<Schema::Tablet::ObjectID>(objectId); } - newObjectMetrics.IncreaseCount(tabletsUpdated); + if (newObjectMetrics) { + newObjectMetrics->IncreaseCount(tabletsUpdated); + } auto response = std::make_unique<TEvHive::TEvUpdateTabletsObjectReply>(NKikimrProto::OK); response->Record.SetTxId(Event->Get()->Record.GetTxId()); diff --git a/ydb/core/protos/hive.proto b/ydb/core/protos/hive.proto index f9944fa127..a84e37de05 100644 --- a/ydb/core/protos/hive.proto +++ b/ydb/core/protos/hive.proto @@ -301,7 +301,9 @@ message TEvResponseHiveDomainStats { message TEvRequestHiveNodeStats { optional bool ReturnMetrics = 5; optional bool ReturnExtendedTabletInfo = 6; + // The next 2 fields should always be used together optional uint64 FilterTabletsByPathId = 7; + optional uint64 FilterTabletsBySchemeShardId = 8; } message THiveNodeStats { diff --git a/ydb/core/viewer/json_compute.h b/ydb/core/viewer/json_compute.h index b898cfce63..2a72e000a7 100644 --- a/ydb/core/viewer/json_compute.h +++ b/ydb/core/viewer/json_compute.h @@ -301,15 +301,15 @@ public: void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { if (ev->Get()->Request->ResultSet.size() == 1 && ev->Get()->Request->ResultSet.begin()->Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { const NSchemeCache::TSchemeCacheNavigate::TEntry& result(ev->Get()->Request->ResultSet.front()); - ui64 pathId = 0; + TPathId pathId; if (!Path.empty() && result.Self) { switch (result.Self->Info.GetPathType()) { case NKikimrSchemeOp::EPathTypeSubDomain: case NKikimrSchemeOp::EPathTypeExtSubDomain: - pathId = 0; + pathId = TPathId(); break; default: - pathId = result.Self->Info.GetPathId(); + pathId = TPathId(result.Self->Info.GetSchemeshardId(), result.Self->Info.GetPathId()); break; } } diff --git a/ydb/core/viewer/json_nodes.h b/ydb/core/viewer/json_nodes.h index 3ba5117c20..cd3cb87bc9 100644 --- a/ydb/core/viewer/json_nodes.h +++ b/ydb/core/viewer/json_nodes.h @@ -83,7 +83,7 @@ class TJsonNodes : public TViewerPipeClient<TJsonNodes> { bool Storage = false; bool Tablets = false; - ui64 FilterPathId = 0; + TPathId FilterPathId; bool ResolveGroupsToNodes = false; TNodeId MinAllowedNodeId = std::numeric_limits<TNodeId>::min(); TNodeId MaxAllowedNodeId = std::numeric_limits<TNodeId>::max(); @@ -312,7 +312,7 @@ public: SendRequest(whiteboardServiceId, new TEvWhiteboard::TEvPDiskStateRequest(), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); SendRequest(whiteboardServiceId, new TEvWhiteboard::TEvVDiskStateRequest(), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); } - if (Tablets && FilterPathId == 0) { + if (Tablets && FilterPathId == TPathId()) { BLOG_TRACE("SendTabletStateRequest to " << nodeId); auto request = std::make_unique<TEvWhiteboard::TEvTabletStateRequest>(); request->Record.SetGroupBy("Type,State"); @@ -431,14 +431,14 @@ public: RequestSchemeCacheNavigate(domainKey); ++RequestsBeforeNodeList; - if (!FilterPath.empty() && Tablets && FilterPathId == 0) { - FilterPathId = entry.Self->Info.GetPathId(); + if (!FilterPath.empty() && Tablets && FilterPathId == TPathId()) { + FilterPathId = TPathId(entry.Self->Info.GetSchemeshardId(), entry.Self->Info.GetPathId()); } } } NavigateResult.emplace(path, std::move(entry)); - if (HiveId != 0 && FilterPathId != 0) { + if (HiveId != 0 && FilterPathId != TPathId()) { BLOG_TRACE("Requesting hive " << HiveId << " for path id " << FilterPathId); RequestHiveNodeStats(HiveId, FilterPathId); } diff --git a/ydb/core/viewer/json_pipe_req.h b/ydb/core/viewer/json_pipe_req.h index 705c0dd15b..49e4c6fb3d 100644 --- a/ydb/core/viewer/json_pipe_req.h +++ b/ydb/core/viewer/json_pipe_req.h @@ -96,13 +96,14 @@ protected: SendRequestToPipe(pipeClient, request.Release(), hiveId); } - void RequestHiveNodeStats(NNodeWhiteboard::TTabletId hiveId, ui64 pathId) { + void RequestHiveNodeStats(NNodeWhiteboard::TTabletId hiveId, TPathId pathId) { TActorId pipeClient = ConnectTabletPipe(hiveId); THolder<TEvHive::TEvRequestHiveNodeStats> request = MakeHolder<TEvHive::TEvRequestHiveNodeStats>(); request->Record.SetReturnMetrics(Metrics); - if (pathId) { + if (pathId != TPathId()) { request->Record.SetReturnExtendedTabletInfo(true); - request->Record.SetFilterTabletsByPathId(pathId); + request->Record.SetFilterTabletsBySchemeShardId(pathId.OwnerId); + request->Record.SetFilterTabletsByPathId(pathId.LocalPathId); } SendRequestToPipe(pipeClient, request.Release(), hiveId); } |