aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov <ivanmorozov@yandex-team.com>2023-10-14 21:06:05 +0300
committerivanmorozov <ivanmorozov@yandex-team.com>2023-10-14 21:19:41 +0300
commitb8a327d16e258b2d459ec0c25fe08cc456b066b6 (patch)
tree05be26ac177be03e5069b40aa3afe014c261e2f8
parent3a0ef9c562a551c13d20212dd8c94d630cf1dde0 (diff)
downloadydb-b8a327d16e258b2d459ec0c25fe08cc456b066b6.tar.gz
KIKIMR-19215: additional signals
-rw-r--r--ydb/core/tx/columnshard/blobs_action/abstract/read.h1
-rw-r--r--ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp1
-rw-r--r--ydb/core/tx/columnshard/blobs_action/abstract/remove.h6
-rw-r--r--ydb/core/tx/columnshard/blobs_action/abstract/storage.h15
-rw-r--r--ydb/core/tx/columnshard/blobs_action/bs/gc.cpp9
-rw-r--r--ydb/core/tx/columnshard/blobs_action/bs/gc.h6
-rw-r--r--ydb/core/tx/columnshard/blobs_action/bs/read.h2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/bs/storage.cpp3
-rw-r--r--ydb/core/tx/columnshard/blobs_action/bs/storage.h2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp23
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h47
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp19
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h39
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/storage.cpp2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/storage.h4
-rw-r--r--ydb/core/tx/columnshard/blobs_action/counters/ya.make2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/tier/gc.cpp6
-rw-r--r--ydb/core/tx/columnshard/blobs_action/tier/gc.h11
-rw-r--r--ydb/core/tx/columnshard/blobs_action/tier/storage.cpp4
-rw-r--r--ydb/core/tx/columnshard/blobs_action/tier/storage.h2
24 files changed, 198 insertions, 14 deletions
diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/read.h b/ydb/core/tx/columnshard/blobs_action/abstract/read.h
index cbfa52b32d..70f5129a32 100644
--- a/ydb/core/tx/columnshard/blobs_action/abstract/read.h
+++ b/ydb/core/tx/columnshard/blobs_action/abstract/read.h
@@ -21,6 +21,7 @@ private:
THashMap<TBlobRange, TErrorStatus> Fails;
std::shared_ptr<NBlobOperations::TReadCounters> Counters;
bool Started = false;
+ YDB_ACCESSOR(bool, IsBackgroundProcess, true);
protected:
virtual void DoStartReading(const THashMap<TUnifiedBlobId, THashSet<TBlobRange>>& range) = 0;
void StartReading(THashMap<TUnifiedBlobId, THashSet<TBlobRange>>&& ranges);
diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp
index 47c75900b3..72d058b14d 100644
--- a/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/abstract/remove.cpp
@@ -6,6 +6,7 @@ namespace NKikimr::NOlap {
void IBlobsDeclareRemovingAction::DeclareRemove(const TUnifiedBlobId& blobId) {
if (DeclaredBlobs.emplace(blobId).second) {
ACFL_DEBUG("event", "DeclareRemove")("blob_id", blobId);
+ Counters->OnRequest(blobId.BlobSize());
return DoDeclareRemove(blobId);
}
}
diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/remove.h b/ydb/core/tx/columnshard/blobs_action/abstract/remove.h
index 85b092f5de..6c4daffd12 100644
--- a/ydb/core/tx/columnshard/blobs_action/abstract/remove.h
+++ b/ydb/core/tx/columnshard/blobs_action/abstract/remove.h
@@ -3,6 +3,7 @@
#include <util/generic/hash_set.h>
#include <ydb/core/tx/columnshard/blob.h>
#include <ydb/library/accessor/accessor.h>
+#include <ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h>
namespace NKikimr::NColumnShard {
class TColumnShard;
@@ -14,6 +15,7 @@ namespace NKikimr::NOlap {
class IBlobsDeclareRemovingAction: public ICommonBlobsAction {
private:
using TBase = ICommonBlobsAction;
+ std::shared_ptr<NBlobOperations::TRemoveDeclareCounters> Counters;
YDB_READONLY_DEF(THashSet<TUnifiedBlobId>, DeclaredBlobs);
protected:
virtual void DoDeclareRemove(const TUnifiedBlobId& blobId) = 0;
@@ -26,6 +28,10 @@ public:
}
+ void SetCounters(const std::shared_ptr<NBlobOperations::TRemoveDeclareCounters>& counters) {
+ Counters = counters;
+ }
+
void DeclareRemove(const TUnifiedBlobId& blobId);
void OnExecuteTxAfterRemoving(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs, const bool success) {
return DoOnExecuteTxAfterRemoving(self, dbBlobs, success);
diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h
index 8e9df493bf..b0a565ee55 100644
--- a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h
+++ b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h
@@ -6,6 +6,7 @@
#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h>
#include <ydb/core/tx/columnshard/blobs_action/counters/storage.h>
+#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h>
#include <ydb/library/accessor/accessor.h>
namespace NKikimr::NColumnShard {
@@ -47,9 +48,9 @@ protected:
return "";
}
- virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const = 0;
- std::shared_ptr<IBlobsGCAction> StartGCAction() const {
- return DoStartGCAction();
+ virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<NBlobOperations::TRemoveGCCounters>& counters) const = 0;
+ std::shared_ptr<IBlobsGCAction> StartGCAction(const std::shared_ptr<NBlobOperations::TRemoveGCCounters>& counters) const {
+ return DoStartGCAction(counters);
}
public:
@@ -76,8 +77,10 @@ public:
return DoOnTieringModified(tiers);
}
- std::shared_ptr<IBlobsDeclareRemovingAction> StartDeclareRemovingAction(const TString& /*consumerId*/) {
- return DoStartDeclareRemovingAction();
+ std::shared_ptr<IBlobsDeclareRemovingAction> StartDeclareRemovingAction(const TString& consumerId) {
+ auto result = DoStartDeclareRemovingAction();
+ result->SetCounters(Counters->GetConsumerCounter(consumerId)->GetRemoveDeclareCounters());
+ return result;
}
std::shared_ptr<IBlobsWritingAction> StartWritingAction(const TString& consumerId) {
auto result = DoStartWritingAction();
@@ -96,7 +99,7 @@ public:
if (Stopped) {
return false;
}
- auto task = StartGCAction();
+ auto task = StartGCAction(Counters->GetConsumerCounter("GC")->GetRemoveGCCounters());
if (!task) {
return false;
}
diff --git a/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp b/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp
index acdd15c7b8..f140922fa2 100644
--- a/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/bs/gc.cpp
@@ -57,6 +57,11 @@ void TGCTask::OnGCResult(TEvBlobStorage::TEvCollectGarbageResult::TPtr ev) {
Y_ABORT_UNLESS(itGroup != ListsByGroupId.end());
const auto& keepList = itGroup->second.KeepList;
const auto& dontKeepList = itGroup->second.DontKeepList;
+
+ for (auto&& i : dontKeepList) {
+ Counters->OnReply(i.BlobSize());
+ }
+
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("actor", "OnGCResult")("keep_list", keepList.size())("dont_keep_list", dontKeepList.size());
for (const auto& blobId : keepList) {
@@ -83,7 +88,9 @@ THashMap<ui32, std::unique_ptr<NKikimr::TEvBlobStorage::TEvCollectGarbage>> TGCT
new TVector<TLogoBlobID>(gl.second.KeepList.begin(), gl.second.KeepList.end()),
new TVector<TLogoBlobID>(gl.second.DontKeepList.begin(), gl.second.DontKeepList.end()),
TInstant::Max(), true);
-
+ for (auto&& i : gl.second.DontKeepList) {
+ Counters->OnRequest(i.BlobSize());
+ }
Y_ABORT_UNLESS(CounterToGroupInFlight.emplace(perGenerationCounter, group).second);
perGenerationCounter += requests[group]->PerGenerationCounterStepSize();
}
diff --git a/ydb/core/tx/columnshard/blobs_action/bs/gc.h b/ydb/core/tx/columnshard/blobs_action/bs/gc.h
index 013f1a1029..c8ee866acb 100644
--- a/ydb/core/tx/columnshard/blobs_action/bs/gc.h
+++ b/ydb/core/tx/columnshard/blobs_action/bs/gc.h
@@ -2,6 +2,7 @@
#include <ydb/core/tx/columnshard/blob_cache.h>
#include <ydb/core/tx/columnshard/blobs_action/abstract/gc.h>
+#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h>
#include <ydb/core/tx/columnshard/blob_manager.h>
namespace NKikimr::NOlap::NBlobOperations::NBlobStorage {
@@ -23,6 +24,7 @@ private:
std::deque<TUnifiedBlobId> KeepsToErase;
std::deque<TUnifiedBlobId> DeletesToErase;
std::shared_ptr<NColumnShard::TBlobManager> Manager;
+ std::shared_ptr<TRemoveGCCounters> Counters;
protected:
virtual void DoOnExecuteTxAfterCleaning(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs) override;
virtual bool DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, const std::shared_ptr<IBlobsGCAction>& taskAction) override;
@@ -31,6 +33,10 @@ public:
return ListsByGroupId.empty();
}
+ void SetCounters(const std::shared_ptr<TRemoveGCCounters>& counters) {
+ Counters = counters;
+ }
+
TGCTask(const TString& storageId, TGCListsByGroup&& listsByGroupId, const NColumnShard::TGenStep& collectGenStepInFlight, std::deque<TUnifiedBlobId>&& keepsToErase, std::deque<TUnifiedBlobId>&& deletesToErase,
const std::shared_ptr<NColumnShard::TBlobManager>& manager);
diff --git a/ydb/core/tx/columnshard/blobs_action/bs/read.h b/ydb/core/tx/columnshard/blobs_action/bs/read.h
index a30c38eebe..e21c66866a 100644
--- a/ydb/core/tx/columnshard/blobs_action/bs/read.h
+++ b/ydb/core/tx/columnshard/blobs_action/bs/read.h
@@ -13,7 +13,7 @@ private:
protected:
virtual void DoStartReading(const THashMap<TUnifiedBlobId, THashSet<TBlobRange>>& ranges) override {
for (auto&& i : ranges) {
- NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = true, .WithDeadline = false};
+ NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = GetIsBackgroundProcess(), .WithDeadline = false};
std::vector<TBlobRange> rangesLocal(i.second.begin(), i.second.end());
TActorContext::AsActorContext().Send(BlobCacheActorId, new NBlobCache::TEvBlobCache::TEvReadBlobRangeBatch(std::move(rangesLocal), std::move(readOpts)));
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("blob_id", i.first)("count", i.second.size());
diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp
index 5293284fb7..7b9b864b8b 100644
--- a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp
@@ -20,12 +20,13 @@ std::shared_ptr<NKikimr::NOlap::IBlobsReadingAction> TOperator::DoStartReadingAc
return std::make_shared<TReadingAction>(GetStorageId(), BlobCacheActorId);
}
-std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction() const {
+std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const {
auto gcTask = Manager->BuildGCTask(GetStorageId(), Manager);
if (!gcTask || gcTask->IsEmpty()) {
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartGCSkipped");
return nullptr;
}
+ gcTask->SetCounters(counters);
auto requests = gcTask->BuildRequests(PerGenerationCounter, Manager->GetTabletId(), Manager->GetCurrentGen());
AFL_VERIFY(requests.size());
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartGC")("requests_count", requests.size());
diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.h b/ydb/core/tx/columnshard/blobs_action/bs/storage.h
index 49e97a61d9..9e39e6173f 100644
--- a/ydb/core/tx/columnshard/blobs_action/bs/storage.h
+++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.h
@@ -18,7 +18,7 @@ protected:
virtual std::shared_ptr<IBlobsDeclareRemovingAction> DoStartDeclareRemovingAction() override;
virtual std::shared_ptr<IBlobsWritingAction> DoStartWritingAction() override;
virtual std::shared_ptr<IBlobsReadingAction> DoStartReadingAction() override;
- virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const override;
+ virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const override;
virtual bool DoLoad(NColumnShard::IBlobManagerDb& dbBlobs) override {
return Manager->LoadState(dbBlobs);
}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt
index 3f2c60a24c..44ff16687a 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.darwin-x86_64.txt
@@ -19,4 +19,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
)
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt
index 536e94a621..8f18b7f316 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-aarch64.txt
@@ -20,4 +20,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
)
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt
index 536e94a621..8f18b7f316 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.linux-x86_64.txt
@@ -20,4 +20,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
)
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt
index 3f2c60a24c..44ff16687a 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/columnshard/blobs_action/counters/CMakeLists.windows-x86_64.txt
@@ -19,4 +19,6 @@ target_sources(columnshard-blobs_action-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/read.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/write.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
)
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
new file mode 100644
index 0000000000..b3fb6ba1ef
--- /dev/null
+++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.cpp
@@ -0,0 +1,23 @@
+#include "remove_declare.h"
+#include "storage.h"
+
+namespace NKikimr::NOlap::NBlobOperations {
+
+TRemoveDeclareCounters::TRemoveDeclareCounters(const TConsumerCounters& owner)
+ : TBase(owner, "RemoveDeclare")
+{
+ RequestsCount = TBase::GetDeriviative("Requests/Count");
+ RequestBytes = TBase::GetDeriviative("Requests/Bytes");
+
+ RepliesCount = TBase::GetDeriviative("Replies/Count");
+ ReplyBytes = TBase::GetDeriviative("Replies/Bytes");
+ ReplyDurationBySize = TBase::GetHistogram("Replies/Duration/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1));
+ ReplyDurationByCount = TBase::GetHistogram("Replies/Duration/Count", NMonitoring::ExponentialHistogram(15, 2, 1));
+
+ FailsCount = TBase::GetDeriviative("Fails/Count");
+ FailBytes = TBase::GetDeriviative("Fails/Bytes");
+ FailDurationBySize = TBase::GetHistogram("Fails/Duration/Bytes", NMonitoring::ExponentialHistogram(15, 2, 2));
+ FailDurationByCount = TBase::GetHistogram("Fails/Duration/Count", NMonitoring::ExponentialHistogram(15, 2, 2));
+}
+
+}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h
new file mode 100644
index 0000000000..4e11a6c68f
--- /dev/null
+++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_declare.h
@@ -0,0 +1,47 @@
+#pragma once
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <ydb/core/tx/columnshard/counters/common/owner.h>
+
+namespace NKikimr::NOlap::NBlobOperations {
+
+class TConsumerCounters;
+
+class TRemoveDeclareCounters: public NColumnShard::TCommonCountersOwner {
+private:
+ using TBase = NColumnShard::TCommonCountersOwner;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestsCount;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr RepliesCount;
+ NMonitoring::TDynamicCounters::TCounterPtr ReplyBytes;
+ NMonitoring::THistogramPtr ReplyDurationByCount;
+ NMonitoring::THistogramPtr ReplyDurationBySize;
+
+ NMonitoring::TDynamicCounters::TCounterPtr FailsCount;
+ NMonitoring::TDynamicCounters::TCounterPtr FailBytes;
+ NMonitoring::THistogramPtr FailDurationByCount;
+ NMonitoring::THistogramPtr FailDurationBySize;
+public:
+ TRemoveDeclareCounters(const TConsumerCounters& owner);
+
+ void OnRequest(const ui64 bytes) const {
+ RequestsCount->Add(1);
+ RequestBytes->Add(bytes);
+ }
+
+ void OnReply(const ui64 bytes, const TDuration d) const {
+ RepliesCount->Add(1);
+ ReplyBytes->Add(bytes);
+ ReplyDurationByCount->Collect((i64)d.MilliSeconds());
+ ReplyDurationBySize->Collect((i64)d.MilliSeconds(), (i64)bytes);
+ }
+
+ void OnFail(const ui64 bytes, const TDuration d) const {
+ FailsCount->Add(1);
+ FailBytes->Add(bytes);
+ FailDurationByCount->Collect((i64)d.MilliSeconds());
+ FailDurationBySize->Collect((i64)d.MilliSeconds(), (i64)bytes);
+ }
+};
+
+}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
new file mode 100644
index 0000000000..599b73689a
--- /dev/null
+++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.cpp
@@ -0,0 +1,19 @@
+#include "remove_gc.h"
+#include "storage.h"
+
+namespace NKikimr::NOlap::NBlobOperations {
+
+TRemoveGCCounters::TRemoveGCCounters(const TConsumerCounters& owner)
+ : TBase(owner, "RemoveGC")
+{
+ RequestsCount = TBase::GetDeriviative("Requests/Count");
+ RequestBytes = TBase::GetDeriviative("Requests/Bytes");
+
+ RepliesCount = TBase::GetDeriviative("Replies/Count");
+ ReplyBytes = TBase::GetDeriviative("Replies/Bytes");
+
+ FailsCount = TBase::GetDeriviative("Fails/Count");
+ FailBytes = TBase::GetDeriviative("Fails/Bytes");
+}
+
+}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h
new file mode 100644
index 0000000000..d2f7c268c2
--- /dev/null
+++ b/ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h
@@ -0,0 +1,39 @@
+#pragma once
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <ydb/core/tx/columnshard/counters/common/owner.h>
+
+namespace NKikimr::NOlap::NBlobOperations {
+
+class TConsumerCounters;
+
+class TRemoveGCCounters: public NColumnShard::TCommonCountersOwner {
+private:
+ using TBase = NColumnShard::TCommonCountersOwner;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestsCount;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr RepliesCount;
+ NMonitoring::TDynamicCounters::TCounterPtr ReplyBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr FailsCount;
+ NMonitoring::TDynamicCounters::TCounterPtr FailBytes;
+public:
+ TRemoveGCCounters(const TConsumerCounters& owner);
+
+ void OnRequest(const ui64 bytes) const {
+ RequestsCount->Add(1);
+ RequestBytes->Add(bytes);
+ }
+
+ void OnReply(const ui64 bytes) const {
+ RepliesCount->Add(1);
+ ReplyBytes->Add(bytes);
+ }
+
+ void OnFail(const ui64 bytes) const {
+ FailsCount->Add(1);
+ FailBytes->Add(bytes);
+ }
+};
+
+}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp b/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
index 4f8b03a9ad..e7c30dc5ee 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.cpp
@@ -22,6 +22,8 @@ TConsumerCounters::TConsumerCounters(const TString& consumerId, const TStorageCo
DeepSubGroup("Consumer", consumerId);
ReadCounters = std::make_shared<TReadCounters>(*this);
WriteCounters = std::make_shared<TWriteCounters>(*this);
+ RemoveDeclareCounters = std::make_shared<TRemoveDeclareCounters>(*this);
+ RemoveGCCounters = std::make_shared<TRemoveGCCounters>(*this);
}
}
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.h b/ydb/core/tx/columnshard/blobs_action/counters/storage.h
index 8b42b79dfb..e92ca28bda 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/storage.h
+++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.h
@@ -1,6 +1,8 @@
#pragma once
#include "read.h"
#include "write.h"
+#include "remove_declare.h"
+#include "remove_gc.h"
#include <ydb/core/tx/columnshard/counters/common/owner.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <util/generic/hash.h>
@@ -14,6 +16,8 @@ private:
using TBase = NColumnShard::TCommonCountersOwner;
YDB_READONLY_DEF(std::shared_ptr<TReadCounters>, ReadCounters);
YDB_READONLY_DEF(std::shared_ptr<TWriteCounters>, WriteCounters);
+ YDB_READONLY_DEF(std::shared_ptr<TRemoveDeclareCounters>, RemoveDeclareCounters);
+ YDB_READONLY_DEF(std::shared_ptr<TRemoveGCCounters>, RemoveGCCounters);
public:
TConsumerCounters(const TString& consumerId, const TStorageCounters& parent);
};
diff --git a/ydb/core/tx/columnshard/blobs_action/counters/ya.make b/ydb/core/tx/columnshard/blobs_action/counters/ya.make
index 7e50b3c7f6..a3e06c138d 100644
--- a/ydb/core/tx/columnshard/blobs_action/counters/ya.make
+++ b/ydb/core/tx/columnshard/blobs_action/counters/ya.make
@@ -4,6 +4,8 @@ SRCS(
read.cpp
storage.cpp
write.cpp
+ remove_declare.cpp
+ remove_gc.cpp
)
PEERDIR(
diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp b/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp
index 6d311f306c..d1b509cbef 100644
--- a/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/tier/gc.cpp
@@ -24,6 +24,12 @@ bool TGCTask::DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, cons
TActorContext::AsActorContext().Send(self.SelfId(), std::make_unique<NColumnShard::TEvPrivate::TEvGarbageCollectionFinished>(taskAction));
return false;
} else {
+ for (auto&& i : DraftBlobIds) {
+ Counters->OnReply(i.BlobSize());
+ }
+ for (auto&& i : DeleteBlobIds) {
+ Counters->OnReply(i.BlobSize());
+ }
return true;
}
}
diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc.h b/ydb/core/tx/columnshard/blobs_action/tier/gc.h
index 5917ca0c35..cdae1449f2 100644
--- a/ydb/core/tx/columnshard/blobs_action/tier/gc.h
+++ b/ydb/core/tx/columnshard/blobs_action/tier/gc.h
@@ -1,5 +1,6 @@
#pragma once
#include <ydb/core/tx/columnshard/blobs_action/abstract/gc.h>
+#include <ydb/core/tx/columnshard/blobs_action/counters/remove_gc.h>
#include <ydb/core/tx/columnshard/blob.h>
#include <ydb/core/wrappers/abstract.h>
#include <ydb/library/accessor/accessor.h>
@@ -13,17 +14,25 @@ private:
YDB_READONLY_DEF(std::deque<TUnifiedBlobId>, DraftBlobIds);
YDB_READONLY_DEF(std::deque<TUnifiedBlobId>, DeleteBlobIds);
YDB_READONLY_DEF(NWrappers::NExternalStorage::IExternalStorageOperator::TPtr, ExternalStorageOperator);
+ const std::shared_ptr<TRemoveGCCounters> Counters;
protected:
virtual void DoOnExecuteTxAfterCleaning(NColumnShard::TColumnShard& self, NColumnShard::TBlobManagerDb& dbBlobs) override;
virtual bool DoOnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, const std::shared_ptr<IBlobsGCAction>& taskAction) override;
public:
TGCTask(const TString& storageId, std::deque<TUnifiedBlobId>&& draftBlobIds, std::deque<TUnifiedBlobId>&& deleteBlobIds,
- const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& externalStorageOperator)
+ const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& externalStorageOperator, const std::shared_ptr<TRemoveGCCounters>& counters)
: TBase(storageId)
, DraftBlobIds(std::move(draftBlobIds))
, DeleteBlobIds(std::move(deleteBlobIds))
, ExternalStorageOperator(externalStorageOperator)
+ , Counters(counters)
{
+ for (auto&& i : DraftBlobIds) {
+ Counters->OnRequest(i.BlobSize());
+ }
+ for (auto&& i : DeleteBlobIds) {
+ Counters->OnRequest(i.BlobSize());
+ }
}
};
diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp
index a65dfbb7c2..bf9e02e5f8 100644
--- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp
@@ -30,13 +30,13 @@ std::shared_ptr<IBlobsReadingAction> TOperator::DoStartReadingAction() {
return std::make_shared<TReadingAction>(GetStorageId(), GetCurrentOperator());
}
-std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction() const {
+std::shared_ptr<IBlobsGCAction> TOperator::DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const {
std::deque<TUnifiedBlobId> draftBlobIds;
std::deque<TUnifiedBlobId> deleteBlobIds;
if (!GCInfo->ExtractForGC(draftBlobIds, deleteBlobIds, 100000)) {
return nullptr;
}
- auto gcTask = std::make_shared<TGCTask>(GetStorageId(), std::move(draftBlobIds), std::move(deleteBlobIds), GetCurrentOperator());
+ auto gcTask = std::make_shared<TGCTask>(GetStorageId(), std::move(draftBlobIds), std::move(deleteBlobIds), GetCurrentOperator(), counters);
TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, TabletActorId));
return gcTask;
}
diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.h b/ydb/core/tx/columnshard/blobs_action/tier/storage.h
index b79986f999..b153c9b72e 100644
--- a/ydb/core/tx/columnshard/blobs_action/tier/storage.h
+++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.h
@@ -30,7 +30,7 @@ protected:
virtual std::shared_ptr<IBlobsDeclareRemovingAction> DoStartDeclareRemovingAction() override;
virtual std::shared_ptr<IBlobsWritingAction> DoStartWritingAction() override;
virtual std::shared_ptr<IBlobsReadingAction> DoStartReadingAction() override;
- virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction() const override;
+ virtual std::shared_ptr<IBlobsGCAction> DoStartGCAction(const std::shared_ptr<TRemoveGCCounters>& counters) const override;
virtual bool DoLoad(NColumnShard::IBlobManagerDb& dbBlobs) override {
dbBlobs.LoadTierLists(GetStorageId(), GCInfo->MutableBlobsToDelete(), GCInfo->MutableDraftBlobIdsToRemove());
return true;