diff options
author | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-10-14 21:06:05 +0300 |
---|---|---|
committer | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-10-14 21:19:41 +0300 |
commit | b8a327d16e258b2d459ec0c25fe08cc456b066b6 (patch) | |
tree | 05be26ac177be03e5069b40aa3afe014c261e2f8 | |
parent | 3a0ef9c562a551c13d20212dd8c94d630cf1dde0 (diff) | |
download | ydb-b8a327d16e258b2d459ec0c25fe08cc456b066b6.tar.gz |
KIKIMR-19215: additional signals
24 files changed, 198 insertions, 14 deletions
diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/read.h b/ydb/core/tx/columnshard/blobs_action/abstract/read.h index cbfa52b32d..70f5129a32 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/read.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/read.h @@ -21,6 +21,7 @@ private: THashMap<TBlobRange, TErrorStatus> Fails; std::shared_ptr<NBlobOperations::TReadCounters> Counters; bool Started = false; + YDB_ACCESSOR(bool, IsBackgroundProcess, true); protected: virtual void DoStartReading(const THashMap<TUnifiedBlobId, THashSet<TBlobRange>>& range) = 0; void StartReading(THashMap<TUnifiedBlobId, THashSet<TBlobRange>>&& ranges); diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp index 47c75900b3..72d058b14d 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp @@ -6,6 +6,7 @@ namespace NKikimr::NOlap { void IBlobsDeclareRemovingAction::DeclareRemove(const TUnifiedBlobId& blobId) { if (DeclaredBlobs.emplace(blobId).second) { ACFL_DEBUG("event", "DeclareRemove")("blob_id", blobId); + Counters->OnRequest(blobId.BlobSize()); return DoDeclareRemove(blobId); } } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/remove.h b/ydb/core/tx/columnshard/blobs_action/abstract/remove.h index 85b092f5de..6c4daffd12 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/remove.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/remove.h @@ -3,6 +3,7 @@ #include <util/generic/hash_set.h> #include <ydb/core/tx/columnshard/blob.h> #include <ydb/library/accessor/accessor.h> +#include <ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h> namespace NKikimr::NColumnShard { class TColumnShard; @@ -14,6 +15,7 @@ namespace NKikimr::NOlap { class IBlobsDeclareRemovingAction: public ICommonBlobsAction { private: using TBase = ICommonBlobsAction; + std::shared_ptr<NBlobOperations::TRemoveDeclareCounters> Counters; YDB_READONLY_DEF(THashSet<TUnifiedBlobId>, DeclaredBlobs); protected: virtual void DoDeclareRemove(const TUnifiedBlobId& blobId) = 0; @@ -26,6 +28,10 @@ public: } + void SetCounters(const std::shared_ptr<NBlobOperations::TRemoveDeclareCounters>& counters) { + Counters = counters; + } + void DeclareRemove(const TUnifiedBlobId& blobId); void OnExecuteTxAfterRemoving(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs, const bool success) { return DoOnExecuteTxAfterRemoving(self, dbBlobs, success); diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h index 8e9df493bf..b0a565ee55 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h @@ -6,6 +6,7 @@ #include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h> #include <ydb/core/tx/columnshard/blobs_action/counters/storage.h> +#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h> #include <ydb/library/accessor/accessor.h> namespace NKikimr::NColumnShard { @@ -47,9 +48,9 @@ protected: return ""; } - virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const = 0; - std::shared_ptr<IBlobsGCAction> StartGCAction() const { - return DoStartGCAction(); + virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<NBlobOperations::TRemoveGCCounters>& counters) const = 0; + std::shared_ptr<IBlobsGCAction> StartGCAction(const std::shared_ptr<NBlobOperations::TRemoveGCCounters>& counters) const { + return DoStartGCAction(counters); } public: @@ -76,8 +77,10 @@ public: return DoOnTieringModified(tiers); } - std::shared_ptr<IBlobsDeclareRemovingAction> StartDeclareRemovingAction(const TString& /*consumerId*/) { - return DoStartDeclareRemovingAction(); + std::shared_ptr<IBlobsDeclareRemovingAction> StartDeclareRemovingAction(const TString& consumerId) { + auto result = DoStartDeclareRemovingAction(); + result->SetCounters(Counters->GetConsumerCounter(consumerId)->GetRemoveDeclareCounters()); + return result; } std::shared_ptr<IBlobsWritingAction> StartWritingAction(const TString& consumerId) { auto result = DoStartWritingAction(); @@ -96,7 +99,7 @@ public: if (Stopped) { return false; } - auto task = StartGCAction(); + auto task = StartGCAction(Counters->GetConsumerCounter("GC")->GetRemoveGCCounters()); if (!task) { return false; } diff --git a/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp b/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp index acdd15c7b8..f140922fa2 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp @@ -57,6 +57,11 @@ void TGCTask::OnGCResult(TEvBlobStorage::TEvCollectGarbageResult::TPtr ev) { Y_ABORT_UNLESS(itGroup != ListsByGroupId.end()); const auto& keepList = itGroup->second.KeepList; const auto& dontKeepList = itGroup->second.DontKeepList; + + for (auto&& i : dontKeepList) { + Counters->OnReply(i.BlobSize()); + } + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("actor", "OnGCResult")("keep_list", keepList.size())("dont_keep_list", dontKeepList.size()); for (const auto& blobId : keepList) { @@ -83,7 +88,9 @@ THashMap<ui32, std::unique_ptr<NKikimr::TEvBlobStorage::TEvCollectGarbage>> TGCT new TVector<TLogoBlobID>(gl.second.KeepList.begin(), gl.second.KeepList.end()), new TVector<TLogoBlobID>(gl.second.DontKeepList.begin(), gl.second.DontKeepList.end()), TInstant::Max(), true); - + for (auto&& i : gl.second.DontKeepList) { + Counters->OnRequest(i.BlobSize()); + } Y_ABORT_UNLESS(CounterToGroupInFlight.emplace(perGenerationCounter, group).second); perGenerationCounter += requests[group]->PerGenerationCounterStepSize(); } diff --git a/ydb/core/tx/columnshard/blobs_action/bs/gc.h b/ydb/core/tx/columnshard/blobs_action/bs/gc.h index 013f1a1029..c8ee866acb 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/gc.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/gc.h @@ -2,6 +2,7 @@ #include <ydb/core/tx/columnshard/blob_cache.h> #include <ydb/core/tx/columnshard/blobs_action/abstract/gc.h> +#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h> #include <ydb/core/tx/columnshard/blob_manager.h> namespace NKikimr::NOlap::NBlobOperations::NBlobStorage { @@ -23,6 +24,7 @@ private: std::deque<TUnifiedBlobId> KeepsToErase; std::deque<TUnifiedBlobId> DeletesToErase; std::shared_ptr<NColumnShard::TBlobManager> Manager; + std::shared_ptr<TRemoveGCCounters> Counters; protected: virtual void DoOnExecuteTxAfterCleaning(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs) override; virtual bool DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, const std::shared_ptr<IBlobsGCAction>& taskAction) override; @@ -31,6 +33,10 @@ public: return ListsByGroupId.empty(); } + void SetCounters(const std::shared_ptr<TRemoveGCCounters>& counters) { + Counters = counters; + } + TGCTask(const TString& storageId, TGCListsByGroup&& listsByGroupId, const NColumnShard::TGenStep& collectGenStepInFlight, std::deque<TUnifiedBlobId>&& keepsToErase, std::deque<TUnifiedBlobId>&& deletesToErase, const std::shared_ptr<NColumnShard::TBlobManager>& manager); diff --git a/ydb/core/tx/columnshard/blobs_action/bs/read.h b/ydb/core/tx/columnshard/blobs_action/bs/read.h index a30c38eebe..e21c66866a 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/read.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/read.h @@ -13,7 +13,7 @@ private: protected: virtual void DoStartReading(const THashMap<TUnifiedBlobId, THashSet<TBlobRange>>& ranges) override { for (auto&& i : ranges) { - NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = true, .WithDeadline = false}; + NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = GetIsBackgroundProcess(), .WithDeadline = false}; std::vector<TBlobRange> rangesLocal(i.second.begin(), i.second.end()); TActorContext::AsActorContext().Send(BlobCacheActorId, new NBlobCache::TEvBlobCache::TEvReadBlobRangeBatch(std::move(rangesLocal), std::move(readOpts))); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("blob_id", i.first)("count", i.second.size()); diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp index 5293284fb7..7b9b864b8b 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp @@ -20,12 +20,13 @@ std::shared_ptr<NKikimr::NOlap::IBlobsReadingAction> TOperator::DoStartReadingAc return std::make_shared<TReadingAction>(GetStorageId(), BlobCacheActorId); } -std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction() const { +std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const { auto gcTask = Manager->BuildGCTask(GetStorageId(), Manager); if (!gcTask || gcTask->IsEmpty()) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartGCSkipped"); return nullptr; } + gcTask->SetCounters(counters); auto requests = gcTask->BuildRequests(PerGenerationCounter, Manager->GetTabletId(), Manager->GetCurrentGen()); AFL_VERIFY(requests.size()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartGC")("requests_count", requests.size()); diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.h b/ydb/core/tx/columnshard/blobs_action/bs/storage.h index 49e97a61d9..9e39e6173f 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.h @@ -18,7 +18,7 @@ protected: virtual std::shared_ptr<IBlobsDeclareRemovingAction> DoStartDeclareRemovingAction() override; virtual std::shared_ptr<IBlobsWritingAction> DoStartWritingAction() override; virtual std::shared_ptr<IBlobsReadingAction> DoStartReadingAction() override; - virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const override; + virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const override; virtual bool DoLoad(NColumnShard::IBlobManagerDb& dbBlobs) override { return Manager->LoadState(dbBlobs); } diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt index 3f2c60a24c..44ff16687a 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt @@ -19,4 +19,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp ) diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt index 536e94a621..8f18b7f316 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt @@ -20,4 +20,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp ) diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt index 536e94a621..8f18b7f316 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt @@ -20,4 +20,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp ) diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt index 3f2c60a24c..44ff16687a 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt @@ -19,4 +19,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp ) diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp new file mode 100644 index 0000000000..b3fb6ba1ef --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp @@ -0,0 +1,23 @@ +#include "remove_declare.h" +#include "storage.h" + +namespace NKikimr::NOlap::NBlobOperations { + +TRemoveDeclareCounters::TRemoveDeclareCounters(const TConsumerCounters& owner) + : TBase(owner, "RemoveDeclare") +{ + RequestsCount = TBase::GetDeriviative("Requests/Count"); + RequestBytes = TBase::GetDeriviative("Requests/Bytes"); + + RepliesCount = TBase::GetDeriviative("Replies/Count"); + ReplyBytes = TBase::GetDeriviative("Replies/Bytes"); + ReplyDurationBySize = TBase::GetHistogram("Replies/Duration/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1)); + ReplyDurationByCount = TBase::GetHistogram("Replies/Duration/Count", NMonitoring::ExponentialHistogram(15, 2, 1)); + + FailsCount = TBase::GetDeriviative("Fails/Count"); + FailBytes = TBase::GetDeriviative("Fails/Bytes"); + FailDurationBySize = TBase::GetHistogram("Fails/Duration/Bytes", NMonitoring::ExponentialHistogram(15, 2, 2)); + FailDurationByCount = TBase::GetHistogram("Fails/Duration/Count", NMonitoring::ExponentialHistogram(15, 2, 2)); +} + +} diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h new file mode 100644 index 0000000000..4e11a6c68f --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h @@ -0,0 +1,47 @@ +#pragma once +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <ydb/core/tx/columnshard/counters/common/owner.h> + +namespace NKikimr::NOlap::NBlobOperations { + +class TConsumerCounters; + +class TRemoveDeclareCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + NMonitoring::TDynamicCounters::TCounterPtr RequestsCount; + NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; + + NMonitoring::TDynamicCounters::TCounterPtr RepliesCount; + NMonitoring::TDynamicCounters::TCounterPtr ReplyBytes; + NMonitoring::THistogramPtr ReplyDurationByCount; + NMonitoring::THistogramPtr ReplyDurationBySize; + + NMonitoring::TDynamicCounters::TCounterPtr FailsCount; + NMonitoring::TDynamicCounters::TCounterPtr FailBytes; + NMonitoring::THistogramPtr FailDurationByCount; + NMonitoring::THistogramPtr FailDurationBySize; +public: + TRemoveDeclareCounters(const TConsumerCounters& owner); + + void OnRequest(const ui64 bytes) const { + RequestsCount->Add(1); + RequestBytes->Add(bytes); + } + + void OnReply(const ui64 bytes, const TDuration d) const { + RepliesCount->Add(1); + ReplyBytes->Add(bytes); + ReplyDurationByCount->Collect((i64)d.MilliSeconds()); + ReplyDurationBySize->Collect((i64)d.MilliSeconds(), (i64)bytes); + } + + void OnFail(const ui64 bytes, const TDuration d) const { + FailsCount->Add(1); + FailBytes->Add(bytes); + FailDurationByCount->Collect((i64)d.MilliSeconds()); + FailDurationBySize->Collect((i64)d.MilliSeconds(), (i64)bytes); + } +}; + +} diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp new file mode 100644 index 0000000000..599b73689a --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp @@ -0,0 +1,19 @@ +#include "remove_gc.h" +#include "storage.h" + +namespace NKikimr::NOlap::NBlobOperations { + +TRemoveGCCounters::TRemoveGCCounters(const TConsumerCounters& owner) + : TBase(owner, "RemoveGC") +{ + RequestsCount = TBase::GetDeriviative("Requests/Count"); + RequestBytes = TBase::GetDeriviative("Requests/Bytes"); + + RepliesCount = TBase::GetDeriviative("Replies/Count"); + ReplyBytes = TBase::GetDeriviative("Replies/Bytes"); + + FailsCount = TBase::GetDeriviative("Fails/Count"); + FailBytes = TBase::GetDeriviative("Fails/Bytes"); +} + +} diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h new file mode 100644 index 0000000000..d2f7c268c2 --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h @@ -0,0 +1,39 @@ +#pragma once +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <ydb/core/tx/columnshard/counters/common/owner.h> + +namespace NKikimr::NOlap::NBlobOperations { + +class TConsumerCounters; + +class TRemoveGCCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + NMonitoring::TDynamicCounters::TCounterPtr RequestsCount; + NMonitoring::TDynamicCounters::TCounterPtr RequestBytes; + + NMonitoring::TDynamicCounters::TCounterPtr RepliesCount; + NMonitoring::TDynamicCounters::TCounterPtr ReplyBytes; + + NMonitoring::TDynamicCounters::TCounterPtr FailsCount; + NMonitoring::TDynamicCounters::TCounterPtr FailBytes; +public: + TRemoveGCCounters(const TConsumerCounters& owner); + + void OnRequest(const ui64 bytes) const { + RequestsCount->Add(1); + RequestBytes->Add(bytes); + } + + void OnReply(const ui64 bytes) const { + RepliesCount->Add(1); + ReplyBytes->Add(bytes); + } + + void OnFail(const ui64 bytes) const { + FailsCount->Add(1); + FailBytes->Add(bytes); + } +}; + +} diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp b/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp index 4f8b03a9ad..e7c30dc5ee 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp @@ -22,6 +22,8 @@ TConsumerCounters::TConsumerCounters(const TString& consumerId, const TStorageCo DeepSubGroup("Consumer", consumerId); ReadCounters = std::make_shared<TReadCounters>(*this); WriteCounters = std::make_shared<TWriteCounters>(*this); + RemoveDeclareCounters = std::make_shared<TRemoveDeclareCounters>(*this); + RemoveGCCounters = std::make_shared<TRemoveGCCounters>(*this); } } diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.h b/ydb/core/tx/columnshard/blobs_action/counters/storage.h index 8b42b79dfb..e92ca28bda 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.h @@ -1,6 +1,8 @@ #pragma once #include "read.h" #include "write.h" +#include "remove_declare.h" +#include "remove_gc.h" #include <ydb/core/tx/columnshard/counters/common/owner.h> #include <library/cpp/monlib/dynamic_counters/counters.h> #include <util/generic/hash.h> @@ -14,6 +16,8 @@ private: using TBase = NColumnShard::TCommonCountersOwner; YDB_READONLY_DEF(std::shared_ptr<TReadCounters>, ReadCounters); YDB_READONLY_DEF(std::shared_ptr<TWriteCounters>, WriteCounters); + YDB_READONLY_DEF(std::shared_ptr<TRemoveDeclareCounters>, RemoveDeclareCounters); + YDB_READONLY_DEF(std::shared_ptr<TRemoveGCCounters>, RemoveGCCounters); public: TConsumerCounters(const TString& consumerId, const TStorageCounters& parent); }; diff --git a/ydb/core/tx/columnshard/blobs_action/counters/ya.make b/ydb/core/tx/columnshard/blobs_action/counters/ya.make index 7e50b3c7f6..a3e06c138d 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/ya.make +++ b/ydb/core/tx/columnshard/blobs_action/counters/ya.make @@ -4,6 +4,8 @@ SRCS( read.cpp storage.cpp write.cpp + remove_declare.cpp + remove_gc.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp b/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp index 6d311f306c..d1b509cbef 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp @@ -24,6 +24,12 @@ bool TGCTask::DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, cons TActorContext::AsActorContext().Send(self.SelfId(), std::make_unique<NColumnShard::TEvPrivate::TEvGarbageCollectionFinished>(taskAction)); return false; } else { + for (auto&& i : DraftBlobIds) { + Counters->OnReply(i.BlobSize()); + } + for (auto&& i : DeleteBlobIds) { + Counters->OnReply(i.BlobSize()); + } return true; } } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc.h b/ydb/core/tx/columnshard/blobs_action/tier/gc.h index 5917ca0c35..cdae1449f2 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/gc.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/gc.h @@ -1,5 +1,6 @@ #pragma once #include <ydb/core/tx/columnshard/blobs_action/abstract/gc.h> +#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h> #include <ydb/core/tx/columnshard/blob.h> #include <ydb/core/wrappers/abstract.h> #include <ydb/library/accessor/accessor.h> @@ -13,17 +14,25 @@ private: YDB_READONLY_DEF(std::deque<TUnifiedBlobId>, DraftBlobIds); YDB_READONLY_DEF(std::deque<TUnifiedBlobId>, DeleteBlobIds); YDB_READONLY_DEF(NWrappers::NExternalStorage::IExternalStorageOperator::TPtr, ExternalStorageOperator); + const std::shared_ptr<TRemoveGCCounters> Counters; protected: virtual void DoOnExecuteTxAfterCleaning(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs) override; virtual bool DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, const std::shared_ptr<IBlobsGCAction>& taskAction) override; public: TGCTask(const TString& storageId, std::deque<TUnifiedBlobId>&& draftBlobIds, std::deque<TUnifiedBlobId>&& deleteBlobIds, - const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& externalStorageOperator) + const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& externalStorageOperator, const std::shared_ptr<TRemoveGCCounters>& counters) : TBase(storageId) , DraftBlobIds(std::move(draftBlobIds)) , DeleteBlobIds(std::move(deleteBlobIds)) , ExternalStorageOperator(externalStorageOperator) + , Counters(counters) { + for (auto&& i : DraftBlobIds) { + Counters->OnRequest(i.BlobSize()); + } + for (auto&& i : DeleteBlobIds) { + Counters->OnRequest(i.BlobSize()); + } } }; diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index a65dfbb7c2..bf9e02e5f8 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -30,13 +30,13 @@ std::shared_ptr<IBlobsReadingAction> TOperator::DoStartReadingAction() { return std::make_shared<TReadingAction>(GetStorageId(), GetCurrentOperator()); } -std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction() const { +std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const { std::deque<TUnifiedBlobId> draftBlobIds; std::deque<TUnifiedBlobId> deleteBlobIds; if (!GCInfo->ExtractForGC(draftBlobIds, deleteBlobIds, 100000)) { return nullptr; } - auto gcTask = std::make_shared<TGCTask>(GetStorageId(), std::move(draftBlobIds), std::move(deleteBlobIds), GetCurrentOperator()); + auto gcTask = std::make_shared<TGCTask>(GetStorageId(), std::move(draftBlobIds), std::move(deleteBlobIds), GetCurrentOperator(), counters); TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, TabletActorId)); return gcTask; } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.h b/ydb/core/tx/columnshard/blobs_action/tier/storage.h index b79986f999..b153c9b72e 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.h @@ -30,7 +30,7 @@ protected: virtual std::shared_ptr<IBlobsDeclareRemovingAction> DoStartDeclareRemovingAction() override; virtual std::shared_ptr<IBlobsWritingAction> DoStartWritingAction() override; virtual std::shared_ptr<IBlobsReadingAction> DoStartReadingAction() override; - virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const override; + virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const override; virtual bool DoLoad(NColumnShard::IBlobManagerDb& dbBlobs) override { dbBlobs.LoadTierLists(GetStorageId(), GCInfo->MutableBlobsToDelete(), GCInfo->MutableDraftBlobIdsToRemove()); return true; |