diff options
author | alexvru <alexvru@ydb.tech> | 2022-12-19 18:58:44 +0300 |
---|---|---|
committer | alexvru <alexvru@ydb.tech> | 2022-12-19 18:58:44 +0300 |
commit | 81a58118787d9507ae39aeecaf003285bc3da5f6 (patch) | |
tree | 47a16f5ba037581b4c850029e732e77df87ef657 | |
parent | 25a02bcefbd499da8e6be48f161e58354e63cc3c (diff) | |
download | ydb-81a58118787d9507ae39aeecaf003285bc3da5f6.tar.gz |
BlobDepot improvements and fixes
-rw-r--r-- | ydb/core/blob_depot/agent/agent_impl.h | 2 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/blocks.cpp | 15 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/blocks.h | 2 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/storage_collect_garbage.cpp | 31 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/storage_discover.cpp | 44 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/storage_get.cpp | 31 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/storage_put.cpp | 22 | ||||
-rw-r--r-- | ydb/core/blob_depot/agent/storage_range.cpp | 20 | ||||
-rw-r--r-- | ydb/core/blob_depot/data.cpp | 3 | ||||
-rw-r--r-- | ydb/core/blob_depot/data_trash.cpp | 45 | ||||
-rw-r--r-- | ydb/core/blob_depot/garbage_collection.cpp | 14 | ||||
-rw-r--r-- | ydb/core/blob_depot/types.h | 16 | ||||
-rw-r--r-- | ydb/core/util/stlog.h | 18 |
13 files changed, 166 insertions, 97 deletions
diff --git a/ydb/core/blob_depot/agent/agent_impl.h b/ydb/core/blob_depot/agent/agent_impl.h index 75c795b3da3..f707e350708 100644 --- a/ydb/core/blob_depot/agent/agent_impl.h +++ b/ydb/core/blob_depot/agent/agent_impl.h @@ -260,7 +260,7 @@ namespace NKikimr::NBlobDepot { virtual ui64 GetTabletId() const { return 0; } virtual void Initiate() = 0; - virtual void OnUpdateBlock(bool /*success*/) {} + virtual void OnUpdateBlock() {} virtual void OnRead(ui64 /*tag*/, NKikimrProto::EReplyStatus /*status*/, TString /*dataOrErrorReason*/) {} virtual void OnIdAllocated() {} virtual void OnDestroy(bool /*success*/) {} diff --git a/ydb/core/blob_depot/agent/blocks.cpp b/ydb/core/blob_depot/agent/blocks.cpp index 16ec06e9dbb..e4010d7a01a 100644 --- a/ydb/core/blob_depot/agent/blocks.cpp +++ b/ydb/core/blob_depot/agent/blocks.cpp @@ -35,10 +35,7 @@ namespace NKikimr::NBlobDepot { if (auto *p = std::get_if<TEvBlobDepot::TEvQueryBlocksResult*>(&response)) { Handle(std::move(context), (*p)->Record); } else if (std::holds_alternative<TTabletDisconnected>(response)) { - auto& queryBlockContext = context->Obtain<TQueryBlockContext>(); - if (const auto it = Blocks.find(queryBlockContext.TabletId); it != Blocks.end()) { - IssueOnUpdateBlock(it->second, false); - } + // do nothing, query executors will receive this notification as well } else { Y_FAIL("unexpected response type"); } @@ -56,15 +53,13 @@ namespace NKikimr::NBlobDepot { block.TimeToLive = TDuration::MilliSeconds(msg.GetTimeToLiveMs()); block.ExpirationTimestamp = queryBlockContext.Timestamp + block.TimeToLive; block.RefreshInFlight = false; - IssueOnUpdateBlock(block, true); + IssueOnUpdateBlock(block); } - void TBlobDepotAgent::TBlocksManager::IssueOnUpdateBlock(TBlock& block, bool success) { + void TBlobDepotAgent::TBlocksManager::IssueOnUpdateBlock(TBlock& block) { TIntrusiveList<TQuery, TPendingBlockChecks> pendingBlockChecks; pendingBlockChecks.Append(block.PendingBlockChecks); - pendingBlockChecks.ForEach([success](TQuery *query) { - query->OnUpdateBlock(success); - }); + pendingBlockChecks.ForEach([](TQuery *query) { query->OnUpdateBlock(); }); } std::tuple<ui32, ui64> TBlobDepotAgent::TBlocksManager::GetBlockForTablet(ui64 tabletId) { @@ -91,7 +86,7 @@ namespace NKikimr::NBlobDepot { block.BlockedGeneration = tablet.GetBlockedGeneration(); block.IssuerGuid = tablet.GetIssuerGuid(); block.ExpirationTimestamp = TMonotonic::Zero(); - IssueOnUpdateBlock(block, true); + IssueOnUpdateBlock(block); } } } diff --git a/ydb/core/blob_depot/agent/blocks.h b/ydb/core/blob_depot/agent/blocks.h index 551465fbdec..a505f0e01c3 100644 --- a/ydb/core/blob_depot/agent/blocks.h +++ b/ydb/core/blob_depot/agent/blocks.h @@ -39,7 +39,7 @@ namespace NKikimr::NBlobDepot { NKikimrProto::EReplyStatus CheckBlockForTablet(ui64 tabletId, ui32 generation, TQuery *query, ui32 *blockedGeneration); void ProcessResponse(ui64 /*id*/, TRequestContext::TPtr context, TResponse response) override; void Handle(TRequestContext::TPtr context, NKikimrBlobDepot::TEvQueryBlocksResult& msg); - void IssueOnUpdateBlock(TBlock& block, bool success); + void IssueOnUpdateBlock(TBlock& block); std::tuple<ui32, ui64> GetBlockForTablet(ui64 tabletId); void SetBlockForTablet(ui64 tabletId, ui32 blockedGeneration, TMonotonic timestamp, TDuration timeToLive); void OnBlockedTablets(const NProtoBuf::RepeatedPtrField<NKikimrBlobDepot::TEvPushNotify::TBlockedTablet>& tablets); diff --git a/ydb/core/blob_depot/agent/storage_collect_garbage.cpp b/ydb/core/blob_depot/agent/storage_collect_garbage.cpp index cb65548d3bb..9fee42907e0 100644 --- a/ydb/core/blob_depot/agent/storage_collect_garbage.cpp +++ b/ydb/core/blob_depot/agent/storage_collect_garbage.cpp @@ -19,22 +19,6 @@ namespace NKikimr::NBlobDepot { using TBlobStorageQuery::TBlobStorageQuery; void Initiate() override { - BDEV_QUERY(BDEV04, "TEvCollectGarbage_new", (U.TabletId, Request.TabletId), - (U.Generation, Request.RecordGeneration), (U.PerGenerationCounter, Request.PerGenerationCounter), - (U.Channel, Request.Channel), (U.Collect, Request.Collect), (U.Hard, Request.Hard), - (U.CollectGeneration, Request.CollectGeneration), (U.CollectStep, Request.CollectStep)); - - if (Request.Keep) { - for (const auto& id : *Request.Keep) { - BDEV_QUERY(BDEV05, "TEvCollectGarbage_keep", (U.BlobId, id)); - } - } - if (Request.DoNotKeep) { - for (const auto& id : *Request.DoNotKeep) { - BDEV_QUERY(BDEV06, "TEvCollectGarbage_doNotKeep", (U.BlobId, id)); - } - } - NumKeep = Request.Keep ? Request.Keep->size() : 0; NumDoNotKeep = Request.DoNotKeep ? Request.DoNotKeep->size() : 0; @@ -55,10 +39,12 @@ namespace NKikimr::NBlobDepot { for (; KeepIndex < NumKeep && numItemsIssued < MaxCollectGarbageFlagsPerMessage; ++KeepIndex) { LogoBlobIDFromLogoBlobID((*Request.Keep)[KeepIndex], record.AddKeep()); + BDEV_QUERY(BDEV05, "TEvCollectGarbage_keep", (U.BlobId, (*Request.Keep)[KeepIndex])); ++numItemsIssued; } for (; DoNotKeepIndex < NumDoNotKeep && numItemsIssued < MaxCollectGarbageFlagsPerMessage; ++DoNotKeepIndex) { LogoBlobIDFromLogoBlobID((*Request.DoNotKeep)[DoNotKeepIndex], record.AddDoNotKeep()); + BDEV_QUERY(BDEV06, "TEvCollectGarbage_doNotKeep", (U.BlobId, (*Request.DoNotKeep)[DoNotKeepIndex])); ++numItemsIssued; } @@ -73,6 +59,11 @@ namespace NKikimr::NBlobDepot { record.SetHard(Request.Hard); record.SetCollectGeneration(Request.CollectGeneration); record.SetCollectStep(Request.CollectStep); + + BDEV_QUERY(BDEV04, "TEvCollectGarbage_barrier", (U.TabletId, Request.TabletId), + (U.Generation, Request.RecordGeneration), (U.PerGenerationCounter, Request.PerGenerationCounter), + (U.Channel, Request.Channel), (U.Hard, Request.Hard), + (U.CollectGeneration, Request.CollectGeneration), (U.CollectStep, Request.CollectStep)); } Agent.Issue(std::move(record), this, nullptr); @@ -83,12 +74,8 @@ namespace NKikimr::NBlobDepot { ++CounterShift; } - void OnUpdateBlock(bool success) override { - if (success) { - Initiate(); - } else { - EndWithError(NKikimrProto::ERROR, "BlobDepot tablet disconnected"); - } + void OnUpdateBlock() override { + Initiate(); } void ProcessResponse(ui64 /*id*/, TRequestContext::TPtr context, TResponse response) override { diff --git a/ydb/core/blob_depot/agent/storage_discover.cpp b/ydb/core/blob_depot/agent/storage_discover.cpp index 7b52f1be963..9d36776235c 100644 --- a/ydb/core/blob_depot/agent/storage_discover.cpp +++ b/ydb/core/blob_depot/agent/storage_discover.cpp @@ -7,10 +7,6 @@ namespace NKikimr::NBlobDepot { template<> TBlobDepotAgent::TQuery *TBlobDepotAgent::CreateQuery<TEvBlobStorage::EvDiscover>(std::unique_ptr<IEventHandle> ev) { class TDiscoverQuery : public TBlobStorageQuery<TEvBlobStorage::TEvDiscover> { - ui64 TabletId = 0; - bool ReadBody; - ui32 MinGeneration = 0; - ui32 GetBlockedGenerationRetriesRemain = 10; bool DoneWithBlockedGeneration = false; @@ -24,14 +20,13 @@ namespace NKikimr::NBlobDepot { using TBlobStorageQuery::TBlobStorageQuery; void Initiate() override { - TabletId = Request.TabletId; - ReadBody = Request.ReadBody; - MinGeneration = Request.MinGeneration; + BDEV_QUERY(BDEV16, "TEvDiscover_begin", (U.TabletId, Request.TabletId), (U.ReadBody, Request.ReadBody), + (U.MinGeneration, Request.MinGeneration)); IssueResolve(); if (Request.DiscoverBlockedGeneration) { - const auto status = Agent.BlocksManager.CheckBlockForTablet(TabletId, Max<ui32>(), this, &BlockedGeneration); + const auto status = Agent.BlocksManager.CheckBlockForTablet(Request.TabletId, Max<ui32>(), this, &BlockedGeneration); if (status == NKikimrProto::OK) { DoneWithBlockedGeneration = true; } else if (status != NKikimrProto::UNKNOWN) { @@ -44,8 +39,8 @@ namespace NKikimr::NBlobDepot { void IssueResolve() { const ui8 channel = 0; - const TLogoBlobID from(TabletId, MinGeneration, 0, channel, 0, 0); - const TLogoBlobID to(TabletId, Max<ui32>(), Max<ui32>(), channel, TLogoBlobID::MaxBlobSize, TLogoBlobID::MaxCookie); + const TLogoBlobID from(Request.TabletId, Request.MinGeneration, 0, channel, 0, 0); + const TLogoBlobID to(Request.TabletId, Max<ui32>(), Max<ui32>(), channel, TLogoBlobID::MaxBlobSize, TLogoBlobID::MaxCookie); NKikimrBlobDepot::TEvResolve resolve; auto *item = resolve.AddItems(); @@ -56,7 +51,7 @@ namespace NKikimr::NBlobDepot { range->SetIncludeEnding(true); range->SetMaxKeys(1); range->SetReverse(true); - item->SetTabletId(TabletId); + item->SetTabletId(Request.TabletId); item->SetMustRestoreFirst(true); Agent.Issue(std::move(resolve), this, nullptr); @@ -74,15 +69,11 @@ namespace NKikimr::NBlobDepot { } } - void OnUpdateBlock(bool success) override { + void OnUpdateBlock() override { STLOG(PRI_DEBUG, BLOB_DEPOT_AGENT, BDA18, "OnUpdateBlock", (VirtualGroupId, Agent.VirtualGroupId), - (QueryId, GetQueryId()), (Success, success)); + (QueryId, GetQueryId())); - if (!success) { - return EndWithError(NKikimrProto::ERROR, "BlobDepot tablet disconnected"); - } - - const auto status = Agent.BlocksManager.CheckBlockForTablet(TabletId, Max<ui32>(), this, &BlockedGeneration); + const auto status = Agent.BlocksManager.CheckBlockForTablet(Request.TabletId, Max<ui32>(), this, &BlockedGeneration); if (status == NKikimrProto::OK) { DoneWithBlockedGeneration = true; CheckIfDone(); @@ -112,7 +103,7 @@ namespace NKikimr::NBlobDepot { << ": " << item.GetErrorReason()); } Y_VERIFY(item.ValueChainSize() == 1); - if (ReadBody) { + if (Request.ReadBody) { TReadArg arg{ item.GetValueChain(), NKikimrBlobStorage::Discover, @@ -130,7 +121,7 @@ namespace NKikimr::NBlobDepot { } } - if (!ReadBody || !Id) { + if (!Request.ReadBody || !Id) { DoneWithData = true; return CheckIfDone(); } @@ -162,13 +153,20 @@ namespace NKikimr::NBlobDepot { void CheckIfDone() { if (DoneWithBlockedGeneration && DoneWithData) { - Y_VERIFY_S(!Id || !ReadBody || Buffer.size() == Id.BlobSize(), "Id# " << Id << " Buffer.size# " << Buffer.size()); + Y_VERIFY_S(!Id || !Request.ReadBody || Buffer.size() == Id.BlobSize(), "Id# " << Id << " Buffer.size# " << Buffer.size()); + BDEV_QUERY(BDEV17, "TEvDiscover_end", (Status, NKikimrProto::OK), (ErrorReason, ""), (Id, Id), + (Buffer.size, Buffer.size()), (BlockedGeneration, BlockedGeneration)); EndWithSuccess(Id - ? std::make_unique<TEvBlobStorage::TEvDiscoverResult>(Id, MinGeneration, Buffer, BlockedGeneration) - : std::make_unique<TEvBlobStorage::TEvDiscoverResult>(NKikimrProto::NODATA, MinGeneration, BlockedGeneration)); + ? std::make_unique<TEvBlobStorage::TEvDiscoverResult>(Id, Request.MinGeneration, Buffer, BlockedGeneration) + : std::make_unique<TEvBlobStorage::TEvDiscoverResult>(NKikimrProto::NODATA, Request.MinGeneration, BlockedGeneration)); } } + void EndWithError(NKikimrProto::EReplyStatus status, const TString& errorReason) { + BDEV_QUERY(BDEV18, "TEvDiscover_end", (Status, status), (ErrorReason, errorReason)); + TBlobStorageQuery::EndWithError(status, errorReason); + } + ui64 GetTabletId() const override { return Request.TabletId; } diff --git a/ydb/core/blob_depot/agent/storage_get.cpp b/ydb/core/blob_depot/agent/storage_get.cpp index 7b011d2d52e..4c5be62edd9 100644 --- a/ydb/core/blob_depot/agent/storage_get.cpp +++ b/ydb/core/blob_depot/agent/storage_get.cpp @@ -31,6 +31,14 @@ namespace NKikimr::NBlobDepot { return; } + if (IS_LOG_PRIORITY_ENABLED(*TlsActivationContext, NLog::PRI_TRACE, NKikimrServices::BLOB_DEPOT_EVENTS)) { + for (ui32 i = 0; i < Request.QuerySize; ++i) { + const auto& q = Request.Queries[i]; + BDEV_QUERY(BDEV00, "TEvGet_new", (U.BlobId, q.Id), (U.Shift, q.Shift), (U.Size, q.Size), + (U.MustRestoreFirst, Request.MustRestoreFirst), (U.IsIndexOnly, Request.IsIndexOnly)); + } + } + Response = std::make_unique<TEvBlobStorage::TEvGetResult>(NKikimrProto::OK, Request.QuerySize, Agent.VirtualGroupId); AnswersRemain = Request.QuerySize; @@ -125,7 +133,28 @@ namespace NKikimr::NBlobDepot { } } } - EndWithSuccess(std::move(Response)); + EndWithSuccess(); + } + } + + void EndWithSuccess() { + TraceResponse(std::nullopt); + TBlobStorageQuery::EndWithSuccess(std::move(Response)); + } + + void EndWithError(NKikimrProto::EReplyStatus status, const TString& errorReason) { + TraceResponse(status); + TBlobStorageQuery::EndWithError(status, errorReason); + } + + void TraceResponse(std::optional<NKikimrProto::EReplyStatus> status) { + if (IS_LOG_PRIORITY_ENABLED(*TlsActivationContext, NLog::PRI_TRACE, NKikimrServices::BLOB_DEPOT_EVENTS)) { + for (ui32 i = 0; i < Response->ResponseSz; ++i) { + const auto& r = Response->Responses[i]; + BDEV_QUERY(BDEV01, "TEvGet_end", (BlobId, r.Id), (Shift, r.Shift), + (RequestedSize, r.RequestedSize), (Status, status.value_or(r.Status)), + (Buffer.size, r.Buffer.size())); + } } } diff --git a/ydb/core/blob_depot/agent/storage_put.cpp b/ydb/core/blob_depot/agent/storage_put.cpp index 27732744761..30d0ebc8b9e 100644 --- a/ydb/core/blob_depot/agent/storage_put.cpp +++ b/ydb/core/blob_depot/agent/storage_put.cpp @@ -33,9 +33,6 @@ namespace NKikimr::NBlobDepot { } void Initiate() override { - BDEV_QUERY(BDEV09, "TEvPut_new", (U.BlobId, Request.Id), (U.BufferSize, Request.Buffer.size()), - (U.HandleClass, Request.HandleClass)); - if (Request.Buffer.size() > MaxBlobSize) { return EndWithError(NKikimrProto::ERROR, "blob is way too big"); } else if (Request.Buffer.size() != Request.Id.BlobSize()) { @@ -97,6 +94,9 @@ namespace NKikimr::NBlobDepot { IsInFlight = true; } + BDEV_QUERY(BDEV09, "TEvPut_new", (U.BlobId, Request.Id), (U.BufferSize, Request.Buffer.size()), + (U.HandleClass, Request.HandleClass)); + Y_VERIFY(CommitBlobSeq.ItemsSize() == 0); auto *commitItem = CommitBlobSeq.AddItems(); commitItem->SetKey(Request.Id.AsBinaryString()); @@ -187,12 +187,8 @@ namespace NKikimr::NBlobDepot { Y_VERIFY(numErased || BlobSeqId.Generation < Agent.BlobDepotGeneration); } - void OnUpdateBlock(bool success) override { - if (success) { - CheckBlocks(); // just restart request - } else { - EndWithError(NKikimrProto::ERROR, "BlobDepot tablet disconnected"); - } + void OnUpdateBlock() override { + CheckBlocks(); // just restart request } void OnIdAllocated() override { @@ -261,12 +257,16 @@ namespace NKikimr::NBlobDepot { } void EndWithError(NKikimrProto::EReplyStatus status, const TString& errorReason) { - BDEV_QUERY(BDEV12, "TEvPut_end", (Status, status), (ErrorReason, errorReason)); + if (BlobSeqId) { + BDEV_QUERY(BDEV12, "TEvPut_end", (Status, status), (ErrorReason, errorReason)); + } TBlobStorageQuery::EndWithError(status, errorReason); } void EndWithSuccess() { - BDEV_QUERY(BDEV13, "TEvPut_end", (Status, NKikimrProto::OK)); + if (BlobSeqId) { + BDEV_QUERY(BDEV13, "TEvPut_end", (Status, NKikimrProto::OK)); + } if (IssueUncertainWrites) { // send a notification auto *item = CommitBlobSeq.MutableItems(0); diff --git a/ydb/core/blob_depot/agent/storage_range.cpp b/ydb/core/blob_depot/agent/storage_range.cpp index b65d36d77d0..81ee3e9e1b4 100644 --- a/ydb/core/blob_depot/agent/storage_range.cpp +++ b/ydb/core/blob_depot/agent/storage_range.cpp @@ -27,6 +27,9 @@ namespace NKikimr::NBlobDepot { return; } + BDEV_QUERY(BDEV02, "TEvRange_new", (U.TabletId, Request.TabletId), (U.From, Request.From), (U.To, Request.To), + (U.MustRestoreFirst, Request.MustRestoreFirst), (U.IndexOnly, Request.IsIndexOnly)); + Response = std::make_unique<TEvBlobStorage::TEvRangeResult>(NKikimrProto::OK, Request.From, Request.To, Agent.VirtualGroupId); @@ -159,11 +162,26 @@ namespace NKikimr::NBlobDepot { if (Reverse) { std::reverse(Response->Responses.begin(), Response->Responses.end()); } - EndWithSuccess(std::move(Response)); + EndWithSuccess(); Finished = true; } } + void EndWithSuccess() { + if (IS_LOG_PRIORITY_ENABLED(*TlsActivationContext, NLog::PRI_TRACE, NKikimrServices::BLOB_DEPOT_EVENTS)) { + for (const auto& r : Response->Responses) { + BDEV_QUERY(BDEV03, "TEvRange_item", (BlobId, r.Id), (Buffer.size, r.Buffer.size())); + } + BDEV_QUERY(BDEV14, "TEvRange_end", (Status, NKikimrProto::OK), (ErrorReason, "")); + } + TBlobStorageQuery::EndWithSuccess(std::move(Response)); + } + + void EndWithError(NKikimrProto::EReplyStatus status, const TString& errorReason) { + BDEV_QUERY(BDEV15, "TEvRange_end", (Status, status), (ErrorReason, errorReason)); + TBlobStorageQuery::EndWithError(status, errorReason); + } + ui64 GetTabletId() const override { return Request.TabletId; } diff --git a/ydb/core/blob_depot/data.cpp b/ydb/core/blob_depot/data.cpp index 6e558a324c9..cb2f3f7e28b 100644 --- a/ydb/core/blob_depot/data.cpp +++ b/ydb/core/blob_depot/data.cpp @@ -405,6 +405,9 @@ namespace NKikimr::NBlobDepot { bool TData::OnBarrierShift(ui64 tabletId, ui8 channel, bool hard, TGenStep previous, TGenStep current, ui32& maxItems, NTabletFlatExecutor::TTransactionContext& txc, void *cookie) { + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT18, "OnBarrierShift", (Id, Self->GetLogId()), (TabletId, tabletId), + (Channel, int(channel)), (Hard, hard), (Previous, previous), (Current, current), (MaxItems, maxItems)); + const TData::TKey first(TLogoBlobID(tabletId, previous.Generation(), previous.Step(), channel, 0, 0)); const TData::TKey last(TLogoBlobID(tabletId, current.Generation(), current.Step(), channel, TLogoBlobID::MaxBlobSize, TLogoBlobID::MaxCookie, TLogoBlobID::MaxPartId, TLogoBlobID::MaxCrcMode)); diff --git a/ydb/core/blob_depot/data_trash.cpp b/ydb/core/blob_depot/data_trash.cpp index 42907a9002e..0f1167db59e 100644 --- a/ydb/core/blob_depot/data_trash.cpp +++ b/ydb/core/blob_depot/data_trash.cpp @@ -130,22 +130,27 @@ namespace NKikimr::NBlobDepot { const ui64 id = ++LastCollectCmdId; CollectCmdToGroup.emplace(id, record.GroupId); - if (ev->Keep) { - for (const TLogoBlobID& blobId : *ev->Keep) { - BDEV(BDEV00, "TrashManager_issueKeep", (BDT, Self->TabletID()), (GroupId, record.GroupId), - (BlobId, blobId), (Cookie, id)); + if (IS_LOG_PRIORITY_ENABLED(*TlsActivationContext, NLog::PRI_TRACE, NKikimrServices::BLOB_DEPOT_EVENTS)) { + if (ev->Keep) { + for (const TLogoBlobID& blobId : *ev->Keep) { + Y_VERIFY(blobId.Channel() == record.Channel); + BDEV(BDEV00, "TrashManager_issueKeep", (BDT, Self->TabletID()), (GroupId, record.GroupId), + (Channel, int(record.Channel)), (BlobId, blobId), (Cookie, id)); + } } - } - if (ev->DoNotKeep) { - for (const TLogoBlobID& blobId : *ev->DoNotKeep) { - BDEV(BDEV01, "TrashManager_issueDoNotKeep", (BDT, Self->TabletID()), (GroupId, record.GroupId), - (BlobId, blobId), (Cookie, id)); + if (ev->DoNotKeep) { + for (const TLogoBlobID& blobId : *ev->DoNotKeep) { + Y_VERIFY(blobId.Channel() == record.Channel); + BDEV(BDEV01, "TrashManager_issueDoNotKeep", (BDT, Self->TabletID()), (GroupId, record.GroupId), + (Channel, int(record.Channel)), (BlobId, blobId), (Cookie, id)); + } + } + if (collect) { + BDEV(BDEV02, "TrashManager_issueCollect", (BDT, Self->TabletID()), (GroupId, record.GroupId), + (Channel, int(ev->Channel)), (RecordGeneration, ev->RecordGeneration), + (PerGenerationCounter, ev->PerGenerationCounter), (Hard, ev->Hard), + (CollectGeneration, ev->CollectGeneration), (CollectStep, ev->CollectStep), (Cookie, id)); } - } - if (collect) { - BDEV(BDEV02, "TrashManager_issueCollect", (BDT, Self->TabletID()), (GroupId, record.GroupId), - (Channel, int(ev->Channel)), (RecordGeneration, ev->RecordGeneration), (Hard, ev->Hard), - (CollectGeneration, ev->CollectGeneration), (CollectStep, ev->CollectStep), (Cookie, id)); } if (collect) { @@ -170,16 +175,16 @@ namespace NKikimr::NBlobDepot { } void TData::Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr ev) { - STLOG(PRI_DEBUG, BLOB_DEPOT, BDT12, "TEvCollectGarbageResult", (Id, Self->GetLogId()), - (Channel, ev->Get()->Channel), (GroupId, ev->Cookie), (Msg, ev->Get()->ToString())); - - BDEV(BDEV03, "TrashManager_collectResult", (BDT, Self->TabletID()), (Cookie, ev->Cookie), - (Status, ev->Get()->Status), (ErrorReason, ev->Get()->ErrorReason)); - auto cmd = CollectCmdToGroup.extract(ev->Cookie); Y_VERIFY(cmd); const ui32 groupId = cmd.mapped(); + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT12, "TEvCollectGarbageResult", (Id, Self->GetLogId()), + (Channel, ev->Get()->Channel), (GroupId, groupId), (Msg, ev->Get()->ToString())); + + BDEV(BDEV03, "TrashManager_collectResult", (BDT, Self->TabletID()), (GroupId, groupId), (Channel, ev->Get()->Channel), + (Cookie, ev->Cookie), (Status, ev->Get()->Status), (ErrorReason, ev->Get()->ErrorReason)); + const auto& key = std::make_tuple(ev->Get()->Channel, groupId); const auto it = RecordsPerChannelGroup.find(key); Y_VERIFY(it != RecordsPerChannelGroup.end()); diff --git a/ydb/core/blob_depot/garbage_collection.cpp b/ydb/core/blob_depot/garbage_collection.cpp index 62f7088b605..021bad4a10c 100644 --- a/ydb/core/blob_depot/garbage_collection.cpp +++ b/ydb/core/blob_depot/garbage_collection.cpp @@ -34,6 +34,9 @@ namespace NKikimr::NBlobDepot { bool Execute(TTransactionContext& txc, const TActorContext&) override { Y_VERIFY(Self->Data->IsLoaded()); + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT75, "TTxAdvanceBarrier::Execute", (Id, Self->GetLogId()), (TabletId, TabletId), + (Channel, int(Channel)), (Hard, Hard), (GenCtr, GenCtr), (CollectGenStep, CollectGenStep)); + const auto key = std::make_tuple(TabletId, Channel); auto& barriers = Self->BarrierServer->Barriers; const auto [it, inserted] = barriers.try_emplace(key); @@ -94,6 +97,9 @@ namespace NKikimr::NBlobDepot { } void Complete(const TActorContext&) override { + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT76, "TTxAdvanceBarrier::Complete", (Id, Self->GetLogId()), (TabletId, TabletId), + (Channel, int(Channel)), (MoreData, MoreData), (Success, Success)); + Self->Data->CommitTrash(this); if (MoreData) { @@ -132,6 +138,8 @@ namespace NKikimr::NBlobDepot { {} bool Execute(TTransactionContext& txc, const TActorContext&) override { + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT77, "TTxCollectGarbage::Execute", (Id, Self->GetLogId()), + (TabletId, TabletId), (Channel, int(Channel))); auto& record = Request->Get()->Record; MoreData = !ProcessFlags(txc, KeepIndex, record.GetKeep(), NKikimrBlobDepot::EKeepState::Keep) || !ProcessFlags(txc, DoNotKeepIndex, record.GetDoNotKeep(), NKikimrBlobDepot::EKeepState::DoNotKeep); @@ -139,6 +147,11 @@ namespace NKikimr::NBlobDepot { } void Complete(const TActorContext&) override { + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT78, "TTxCollectGarbage::Complete", (Id, Self->GetLogId()), + (TabletId, TabletId), (Channel, int(Channel)), (MoreData, MoreData)); + + Self->Data->CommitTrash(this); + if (MoreData) { Self->Execute(std::make_unique<TTxCollectGarbage>(Self, TabletId, Channel, KeepIndex, DoNotKeepIndex)); } else if (auto& record = Request->Get()->Record; record.HasCollectGeneration() && record.HasCollectStep()) { @@ -230,6 +243,7 @@ namespace NKikimr::NBlobDepot { void TBlobDepot::TBarrierServer::Handle(TEvBlobDepot::TEvCollectGarbage::TPtr ev) { const auto& record = ev->Get()->Record; + STLOG(PRI_DEBUG, BLOB_DEPOT, BDT74, "TBarrierServer::Handle(TEvCollectGarbage)", (Id, Self->GetLogId()), (Msg, record)); const auto key = std::make_tuple(record.GetTabletId(), record.GetChannel()); auto& barrier = Barriers[key]; barrier.ProcessingQ.emplace_back(ev.Release()); diff --git a/ydb/core/blob_depot/types.h b/ydb/core/blob_depot/types.h index 4330acaf2bd..069de64e6a4 100644 --- a/ydb/core/blob_depot/types.h +++ b/ydb/core/blob_depot/types.h @@ -207,6 +207,20 @@ namespace NKikimr::NBlobDepot { friend bool operator >=(const TGenStep& x, const TGenStep& y) { return x.Value >= y.Value; } }; -#define BDEV(...) STLOGJ(PRI_TRACE, BLOB_DEPOT_EVENTS, __VA_ARGS__) +#define BDEV(MARKER, TEXT, ...) \ + do { \ + auto& ctx = *TlsActivationContext; \ + const auto priority = NLog::PRI_TRACE; \ + const auto component = NKikimrServices::BLOB_DEPOT_EVENTS; \ + if (IS_LOG_PRIORITY_ENABLED(ctx, priority, component)) { \ + struct MARKER {}; \ + TStringStream __stream; \ + { \ + NJson::TJsonWriter __json(&__stream, false); \ + ::NKikimr::NStLog::TMessage<MARKER>("", 0, #MARKER)STLOG_PARAMS(__VA_ARGS__).WriteToJson(__json) << TEXT; \ + } \ + ::NActors::MemLogAdapter(ctx, priority, component, __stream.Str()); \ + }; \ + } while (false) } // NKikimr::NBlobDepot diff --git a/ydb/core/util/stlog.h b/ydb/core/util/stlog.h index 2f116ac664d..268db31990f 100644 --- a/ydb/core/util/stlog.h +++ b/ydb/core/util/stlog.h @@ -321,12 +321,14 @@ namespace NKikimr::NStLog { ~TJsonWriter() { Json.OpenMap(); - Json.WriteKey("Marker"); - Json.Write(Self->Marker); - Json.WriteKey("File"); - Json.Write(Self->GetFileName()); - Json.WriteKey("Line"); - Json.Write(Self->Line); + if (Self->Header()) { + Json.WriteKey("Marker"); + Json.Write(Self->Marker); + Json.WriteKey("File"); + Json.Write(Self->GetFileName()); + Json.WriteKey("Line"); + Json.Write(Self->Line); + } Json.WriteKey("Text"); Json.Write(Stream.Str()); Self->WriteParamsToJson(Json); @@ -339,6 +341,10 @@ namespace NKikimr::NStLog { } private: + bool Header() const { + return *File; + } + const char *GetFileName() const { const char *p = strrchr(File, '/'); return p ? p + 1 : File; |