aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov <ivanmorozov@yandex-team.com>2023-06-06 11:26:46 +0300
committerivanmorozov <ivanmorozov@yandex-team.com>2023-06-06 11:26:46 +0300
commit8d0a5361300692072804774123e7753775c834d5 (patch)
treebca40d15fec2f97d9ff3802d09b05a7ba095f501
parentb35fea336b768407c64079e8c4ec85488dded825 (diff)
downloadydb-8d0a5361300692072804774123e7753775c834d5.tar.gz
experimental aggregation and new signals
-rw-r--r--ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt6
-rw-r--r--ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt6
-rw-r--r--ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt6
-rw-r--r--ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt6
-rw-r--r--ydb/core/tx/columnshard/counters/blobs_manager.cpp63
-rw-r--r--ydb/core/tx/columnshard/counters/blobs_manager.h104
-rw-r--r--ydb/core/tx/columnshard/counters/columnshard.cpp39
-rw-r--r--ydb/core/tx/columnshard/counters/columnshard.h108
-rw-r--r--ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt22
-rw-r--r--ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt23
-rw-r--r--ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt23
-rw-r--r--ydb/core/tx/columnshard/counters/common/CMakeLists.txt17
-rw-r--r--ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt22
-rw-r--r--ydb/core/tx/columnshard/counters/common/agent.cpp55
-rw-r--r--ydb/core/tx/columnshard/counters/common/agent.h51
-rw-r--r--ydb/core/tx/columnshard/counters/common/client.cpp17
-rw-r--r--ydb/core/tx/columnshard/counters/common/client.h18
-rw-r--r--ydb/core/tx/columnshard/counters/common/owner.cpp43
-rw-r--r--ydb/core/tx/columnshard/counters/common/owner.h27
-rw-r--r--ydb/core/tx/columnshard/counters/common/private.cpp60
-rw-r--r--ydb/core/tx/columnshard/counters/common/private.h12
-rw-r--r--ydb/core/tx/columnshard/counters/common_data.cpp19
-rw-r--r--ydb/core/tx/columnshard/counters/common_data.h44
-rw-r--r--ydb/core/tx/columnshard/counters/engine_logs.cpp32
-rw-r--r--ydb/core/tx/columnshard/counters/engine_logs.h36
-rw-r--r--ydb/core/tx/columnshard/counters/indexation.cpp65
-rw-r--r--ydb/core/tx/columnshard/counters/indexation.h13
-rw-r--r--ydb/core/tx/columnshard/counters/insert_table.cpp15
-rw-r--r--ydb/core/tx/columnshard/counters/insert_table.h71
29 files changed, 974 insertions, 49 deletions
diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt
index 10c00092fd..71ef940f82 100644
--- a/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt
@@ -6,16 +6,22 @@
# original buildsystem will not be accepted.
+add_subdirectory(common)
add_library(tx-columnshard-counters)
target_link_libraries(tx-columnshard-counters PUBLIC
contrib-libs-cxxsupp
yutil
cpp-monlib-dynamic_counters
+ columnshard-counters-common
ydb-core-base
)
target_sources(tx-columnshard-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp
)
diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt
index 0502f0daa8..edfdee32c7 100644
--- a/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(common)
add_library(tx-columnshard-counters)
target_link_libraries(tx-columnshard-counters PUBLIC
@@ -13,10 +14,15 @@ target_link_libraries(tx-columnshard-counters PUBLIC
contrib-libs-cxxsupp
yutil
cpp-monlib-dynamic_counters
+ columnshard-counters-common
ydb-core-base
)
target_sources(tx-columnshard-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp
)
diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt
index 0502f0daa8..edfdee32c7 100644
--- a/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(common)
add_library(tx-columnshard-counters)
target_link_libraries(tx-columnshard-counters PUBLIC
@@ -13,10 +14,15 @@ target_link_libraries(tx-columnshard-counters PUBLIC
contrib-libs-cxxsupp
yutil
cpp-monlib-dynamic_counters
+ columnshard-counters-common
ydb-core-base
)
target_sources(tx-columnshard-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp
)
diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt
index 10c00092fd..71ef940f82 100644
--- a/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt
@@ -6,16 +6,22 @@
# original buildsystem will not be accepted.
+add_subdirectory(common)
add_library(tx-columnshard-counters)
target_link_libraries(tx-columnshard-counters PUBLIC
contrib-libs-cxxsupp
yutil
cpp-monlib-dynamic_counters
+ columnshard-counters-common
ydb-core-base
)
target_sources(tx-columnshard-counters PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp
)
diff --git a/ydb/core/tx/columnshard/counters/blobs_manager.cpp b/ydb/core/tx/columnshard/counters/blobs_manager.cpp
new file mode 100644
index 0000000000..e4c290f7cd
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/blobs_manager.cpp
@@ -0,0 +1,63 @@
+#include "blobs_manager.h"
+#include <ydb/core/base/appdata.h>
+#include <ydb/core/base/counters.h>
+
+namespace NKikimr::NColumnShard {
+
+TBlobsManagerCounters::TBlobsManagerCounters(const TString& module)
+ : TCommonCountersOwner(module)
+{
+ SkipCollection = TBase::GetDeriviative("GC/Skip/Count");
+ StartCollection = TBase::GetDeriviative("GC/Start/Count");
+ CollectDropExplicitBytes = TBase::GetDeriviative("GC/Drop/Explicit/Bytes");
+ CollectDropExplicitCount = TBase::GetDeriviative("GC/Drop/Explicit/Count");
+ CollectDropImplicitBytes = TBase::GetDeriviative("GC/Drop/Implicit/Bytes");
+ CollectDropImplicitCount = TBase::GetDeriviative("GC/Drop/Implicit/Count");
+ CollectKeepBytes = TBase::GetDeriviative("GC/Keep/Bytes");
+ CollectKeepCount = TBase::GetDeriviative("GC/Keep/Count");
+ PutBlobBytes = TBase::GetDeriviative("GC/PutBlob/Bytes");
+ PutBlobCount = TBase::GetDeriviative("GC/PutBlob/Count");
+ CollectGen = TBase::GetValue("GC/Gen");
+ CollectStep = TBase::GetValue("GC/Step");
+
+ DeleteBlobMarkerBytes = TBase::GetDeriviative("GC/MarkerDeleteBlob/Bytes");
+ DeleteBlobMarkerCount = TBase::GetDeriviative("GC/MarkerDeleteBlob/Count");
+ DeleteBlobDelayedMarkerBytes = TBase::GetDeriviative("GC/MarkerDelayedDeleteBlob/Bytes");
+ DeleteBlobDelayedMarkerCount = TBase::GetDeriviative("GC/MarkerDelayedDeleteBlob/Count");
+ AddSmallBlobBytes = TBase::GetDeriviative("GC/AddSmallBlob/Bytes");
+ AddSmallBlobCount = TBase::GetDeriviative("GC/AddSmallBlob/Count");
+ DeleteSmallBlobBytes = TBase::GetDeriviative("GC/DeleteSmallBlob/Bytes");
+ DeleteSmallBlobCount = TBase::GetDeriviative("GC/DeleteSmallBlob/Count");
+
+ BlobsKeepCount = TBase::GetValue("GC/BlobsKeep/Count");
+ BlobsKeepBytes = TBase::GetValue("GC/BlobsKeep/Bytes");
+ BlobsDeleteCount = TBase::GetValue("GC/BlobsDelete/Count");
+ BlobsDeleteBytes = TBase::GetValue("GC/BlobsDelete/Bytes");
+
+ BrokenKeepCount = TBase::GetDeriviative("GC/BrokenKeep/Count");
+ BrokenKeepBytes = TBase::GetDeriviative("GC/BrokenKeep/Bytes");
+
+ KeepMarkerCount = TBase::GetDeriviative("GC/KeepMarker/Count");
+ KeepMarkerBytes = TBase::GetDeriviative("GC/KeepMarker/Bytes");
+}
+
+void TBlobsManagerCounters::OnBlobsKeep(const TSet<TLogoBlobID>& blobs) const {
+ AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnBlobsKeep")("count", blobs.size());
+// BlobsKeepCount->Set(blobs.size());
+// ui64 size = 0;
+// for (auto&& i : blobs) {
+// size += i.BlobSize();
+// }
+// BlobsKeepBytes->Set(size);
+}
+
+void TBlobsManagerCounters::OnBlobsDelete(const TSet<TLogoBlobID>& /*blobs*/) const {
+ // BlobsDeleteCount->Set(blobs.size());
+ // ui64 size = 0;
+ // for (auto&& i : blobs) {
+ // size += i.BlobSize();
+ // }
+ // BlobsDeleteBytes->Set(size);
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/blobs_manager.h b/ydb/core/tx/columnshard/counters/blobs_manager.h
new file mode 100644
index 0000000000..ef11e50826
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/blobs_manager.h
@@ -0,0 +1,104 @@
+#pragma once
+#include <ydb/core/base/logoblob.h>
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include "common/owner.h"
+
+namespace NKikimr::NColumnShard {
+
+class TBlobsManagerCounters: public TCommonCountersOwner {
+private:
+ using TBase = TCommonCountersOwner;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectDropExplicitBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectDropExplicitCount;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectDropImplicitBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectDropImplicitCount;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectKeepBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectKeepCount;
+ NMonitoring::TDynamicCounters::TCounterPtr PutBlobBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr PutBlobCount;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectGen;
+ NMonitoring::TDynamicCounters::TCounterPtr CollectStep;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobMarkerBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobMarkerCount;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobDelayedMarkerBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobDelayedMarkerCount;
+ NMonitoring::TDynamicCounters::TCounterPtr AddSmallBlobBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr AddSmallBlobCount;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteSmallBlobBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr DeleteSmallBlobCount;
+ NMonitoring::TDynamicCounters::TCounterPtr BrokenKeepCount;
+ NMonitoring::TDynamicCounters::TCounterPtr BrokenKeepBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr BlobsKeepCount;
+ NMonitoring::TDynamicCounters::TCounterPtr BlobsKeepBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr BlobsDeleteCount;
+ NMonitoring::TDynamicCounters::TCounterPtr BlobsDeleteBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr KeepMarkerCount;
+ NMonitoring::TDynamicCounters::TCounterPtr KeepMarkerBytes;
+
+public:
+ NMonitoring::TDynamicCounters::TCounterPtr SkipCollection;
+ NMonitoring::TDynamicCounters::TCounterPtr StartCollection;
+
+ TBlobsManagerCounters(const TString& module);
+
+ void OnKeepMarker(const ui64 size) const {
+ KeepMarkerCount->Add(1);
+ KeepMarkerBytes->Add(size);
+ }
+
+ void OnBlobsKeep(const TSet<TLogoBlobID>& blobs) const;
+
+ void OnBlobsDelete(const TSet<TLogoBlobID>& /*blobs*/) const;
+
+ void OnAddSmallBlob(const ui32 bSize) const {
+ AddSmallBlobBytes->Add(bSize);
+ AddSmallBlobCount->Add(1);
+ }
+
+ void OnDeleteBlobDelayedMarker(const ui32 bSize) const {
+ DeleteBlobDelayedMarkerBytes->Add(bSize);
+ DeleteBlobDelayedMarkerCount->Add(1);
+ }
+
+ void OnDeleteBlobMarker(const ui32 bSize) const {
+ DeleteBlobMarkerBytes->Add(bSize);
+ DeleteBlobMarkerCount->Add(1);
+ }
+
+ void OnNewCollectStep(const ui32 gen, const ui32 step) const {
+ CollectGen->Set(gen);
+ CollectStep->Set(step);
+ }
+
+ void OnDeleteSmallBlob(const ui32 bSize) const {
+ DeleteSmallBlobBytes->Add(bSize);
+ DeleteSmallBlobCount->Add(1);
+ }
+
+ void OnPutResult(const ui32 bSize) const {
+ PutBlobBytes->Add(bSize);
+ PutBlobCount->Add(1);
+ }
+
+ void OnCollectKeep(const ui32 bSize) const {
+ CollectKeepBytes->Add(bSize);
+ CollectKeepCount->Add(1);
+ }
+
+ void OnBrokenKeep(const ui32 bSize) const {
+ BrokenKeepBytes->Add(bSize);
+ BrokenKeepCount->Add(1);
+ }
+
+ void OnCollectDropExplicit(const ui32 bSize) const {
+ CollectDropExplicitBytes->Add(bSize);
+ CollectDropExplicitCount->Add(1);
+ }
+
+ void OnCollectDropImplicit(const ui32 bSize) const {
+ CollectDropImplicitBytes->Add(bSize);
+ CollectDropImplicitCount->Add(1);
+ }
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/columnshard.cpp b/ydb/core/tx/columnshard/counters/columnshard.cpp
new file mode 100644
index 0000000000..a399748856
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/columnshard.cpp
@@ -0,0 +1,39 @@
+#include "columnshard.h"
+#include <ydb/core/base/appdata.h>
+#include <ydb/core/base/counters.h>
+
+namespace NKikimr::NColumnShard {
+
+TCSCounters::TCSCounters()
+ : TBase("CS")
+{
+ StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count");
+ TooEarlyBackgroundCount = TBase::GetDeriviative("TooEarlyBackground/Count");
+ SetupCompactionCount = TBase::GetDeriviative("SetupCompaction/Count");
+ SetupIndexationCount = TBase::GetDeriviative("SetupIndexation/Count");
+ SetupTtlCount = TBase::GetDeriviative("SetupTtl/Count");
+ SetupCleanupCount = TBase::GetDeriviative("SetupCleanup/Count");
+
+ SkipIndexationInputDueToGranuleOverloadBytes = TBase::GetDeriviative("SkipIndexationInput/GranuleOverload/Bytes");
+ SkipIndexationInputDueToGranuleOverloadCount = TBase::GetDeriviative("SkipIndexationInput/GranuleOverload/Count");
+ SkipIndexationInputDueToSplitCompactionBytes = TBase::GetDeriviative("SkipIndexationInput/SplitCompaction/Bytes");
+ SkipIndexationInputDueToSplitCompactionCount = TBase::GetDeriviative("SkipIndexationInput/SplitCompaction/Count");
+
+ FutureIndexationInputBytes = TBase::GetDeriviative("FutureIndexationInput/Bytes");
+ IndexationInputBytes = TBase::GetDeriviative("IndexationInput/Bytes");
+
+ OverloadInsertTableBytes = TBase::GetDeriviative("OverloadInsertTable/Bytes");
+ OverloadInsertTableCount = TBase::GetDeriviative("OverloadInsertTable/Count");
+ OverloadGranuleBytes = TBase::GetDeriviative("OverloadGranule/Bytes");
+ OverloadGranuleCount = TBase::GetDeriviative("OverloadGranule/Count");
+ OverloadShardBytes = TBase::GetDeriviative("OverloadShard/Bytes");
+ OverloadShardCount = TBase::GetDeriviative("OverloadShard/Count");
+
+ InternalCompactionGranuleBytes = TBase::GetValueAutoAggregationsClient("InternalCompaction/Bytes");
+ InternalCompactionGranulePortionsCount = TBase::GetValueAutoAggregationsClient("InternalCompaction/PortionsCount");
+
+ SplitCompactionGranuleBytes = TBase::GetValueAutoAggregationsClient("SplitCompaction/Bytes");
+ SplitCompactionGranulePortionsCount = TBase::GetValueAutoAggregationsClient("SplitCompaction/PortionsCount");
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/columnshard.h b/ydb/core/tx/columnshard/counters/columnshard.h
new file mode 100644
index 0000000000..c9b625eb7c
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/columnshard.h
@@ -0,0 +1,108 @@
+#pragma once
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include "common/owner.h"
+
+namespace NKikimr::NColumnShard {
+
+class TCSCounters: public TCommonCountersOwner {
+private:
+ using TBase = TCommonCountersOwner;
+
+ NMonitoring::TDynamicCounters::TCounterPtr StartBackgroundCount;
+ NMonitoring::TDynamicCounters::TCounterPtr TooEarlyBackgroundCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SetupCompactionCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SetupIndexationCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SetupTtlCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SetupCleanupCount;
+
+ NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToGranuleOverloadBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToGranuleOverloadCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToSplitCompactionBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToSplitCompactionCount;
+ NMonitoring::TDynamicCounters::TCounterPtr FutureIndexationInputBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr IndexationInputBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableCount;
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadGranuleBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadGranuleCount;
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadShardBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr OverloadShardCount;
+
+ std::shared_ptr<TValueAggregationClient> InternalCompactionGranuleBytes;
+ std::shared_ptr<TValueAggregationClient> InternalCompactionGranulePortionsCount;
+
+ std::shared_ptr<TValueAggregationClient> SplitCompactionGranuleBytes;
+ std::shared_ptr<TValueAggregationClient> SplitCompactionGranulePortionsCount;
+public:
+ void OnInternalCompactionInfo(const ui64 bytes, const ui32 portionsCount) const {
+ InternalCompactionGranuleBytes->Set(bytes);
+ InternalCompactionGranulePortionsCount->Set(portionsCount);
+ }
+
+ void OnSplitCompactionInfo(const ui64 bytes, const ui32 portionsCount) const {
+ SplitCompactionGranuleBytes->Set(bytes);
+ SplitCompactionGranulePortionsCount->Set(portionsCount);
+ }
+
+ void OnOverloadInsertTable(const ui64 size) const {
+ OverloadInsertTableBytes->Add(size);
+ OverloadInsertTableCount->Add(1);
+ }
+
+ void OnOverloadGranule(const ui64 size) const {
+ OverloadGranuleBytes->Add(size);
+ OverloadGranuleCount->Add(1);
+ }
+
+ void OnOverloadShard(const ui64 size) const {
+ OverloadShardBytes->Add(size);
+ OverloadShardCount->Add(1);
+ }
+
+ void SkipIndexationInputDutToSplitCompaction(const ui64 size) const {
+ SkipIndexationInputDueToSplitCompactionBytes->Add(size);
+ SkipIndexationInputDueToSplitCompactionCount->Add(1);
+ }
+
+ void SkipIndexationInputDutToGranuleOverload(const ui64 size) const {
+ SkipIndexationInputDueToGranuleOverloadBytes->Add(size);
+ SkipIndexationInputDueToGranuleOverloadCount->Add(1);
+ }
+
+ void FutureIndexationInput(const ui64 size) const {
+ FutureIndexationInputBytes->Add(size);
+ }
+
+ void IndexationInput(const ui64 size) const {
+ IndexationInputBytes->Add(size);
+ }
+
+ void OnStartBackground() const {
+ StartBackgroundCount->Add(1);
+ }
+
+ void OnTooEarly() const {
+ TooEarlyBackgroundCount->Add(1);
+ }
+
+ void OnSetupCompaction() const {
+ SetupCompactionCount->Add(1);
+ }
+
+ void OnSetupIndexation() const {
+ SetupIndexationCount->Add(1);
+ }
+
+ void OnSetupTtl() const {
+ SetupTtlCount->Add(1);
+ }
+
+ void OnSetupCleanup() const {
+ SetupCleanupCount->Add(1);
+ }
+
+ TCSCounters();
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..90f970a4a6
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(columnshard-counters-common)
+target_link_libraries(columnshard-counters-common PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-monlib-dynamic_counters
+ ydb-core-base
+)
+target_sources(columnshard-counters-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp
+)
diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..ba3f014509
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(columnshard-counters-common)
+target_link_libraries(columnshard-counters-common PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-monlib-dynamic_counters
+ ydb-core-base
+)
+target_sources(columnshard-counters-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp
+)
diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..ba3f014509
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(columnshard-counters-common)
+target_link_libraries(columnshard-counters-common PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-monlib-dynamic_counters
+ ydb-core-base
+)
+target_sources(columnshard-counters-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp
+)
diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..90f970a4a6
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(columnshard-counters-common)
+target_link_libraries(columnshard-counters-common PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-monlib-dynamic_counters
+ ydb-core-base
+)
+target_sources(columnshard-counters-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp
+)
diff --git a/ydb/core/tx/columnshard/counters/common/agent.cpp b/ydb/core/tx/columnshard/counters/common/agent.cpp
new file mode 100644
index 0000000000..558934e8f9
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/agent.cpp
@@ -0,0 +1,55 @@
+#include "agent.h"
+#include "owner.h"
+
+namespace NKikimr::NColumnShard {
+
+TValueAggregationAgent::TValueAggregationAgent(const TString& signalName, const TCommonCountersOwner& signalsOwner)
+ : ValueSignalSum(signalsOwner.GetValue("SUM/" + signalName))
+ , ValueSignalMin(signalsOwner.GetValue("MIN/" + signalName))
+ , ValueSignalMax(signalsOwner.GetValue("MAX/" + signalName))
+{
+
+}
+
+bool TValueAggregationAgent::CalcAggregations(i64& minValue, i64& maxValue) const {
+ const ui32 count = Values.size();
+ if (!count) {
+ return false;
+ }
+ minValue = Values.front();
+ maxValue = Values.front();
+ for (ui32 i = 0; i < count; ++i) {
+ if (minValue > Values[i]) {
+ minValue = Values[i];
+ }
+ if (maxValue < Values[i]) {
+ maxValue = Values[i];
+ }
+ }
+ return true;
+}
+
+std::optional<NKikimr::NColumnShard::TSignalAggregations> TValueAggregationAgent::GetAggregations() const {
+ i64 min;
+ i64 max;
+ if (!CalcAggregations(min, max)) {
+ return {};
+ }
+ return TSignalAggregations(SumValue, min, max);
+}
+
+void TValueAggregationAgent::ResendStatus() const {
+ TGuard<TMutex> g(Mutex);
+ std::optional<TSignalAggregations> aggr = GetAggregations();
+ if (!!aggr) {
+ ValueSignalMin->Set(aggr->Min);
+ ValueSignalMax->Set(aggr->Max);
+ ValueSignalSum->Set(aggr->Sum);
+ }
+}
+
+std::shared_ptr<NKikimr::NColumnShard::TValueAggregationClient> TValueAggregationAgent::GetClient(std::shared_ptr<TValueAggregationAgent> selfPtr) {
+ return std::make_shared<TValueAggregationClient>(selfPtr);
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/agent.h b/ydb/core/tx/columnshard/counters/common/agent.h
new file mode 100644
index 0000000000..306cc4a421
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/agent.h
@@ -0,0 +1,51 @@
+#pragma once
+#include "client.h"
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <util/system/mutex.h>
+#include <deque>
+
+namespace NKikimr::NColumnShard {
+class TCommonCountersOwner;
+
+class TSignalAggregations {
+public:
+ const i64 Sum = 0;
+ const i64 Min = 0;
+ const i64 Max = 0;
+ TSignalAggregations(const i64 sum, const i64 min, const i64 max)
+ : Sum(sum)
+ , Min(min)
+ , Max(max) {
+
+ }
+};
+
+class TValueAggregationAgent: TNonCopyable {
+private:
+ friend class TRegularSignalBuilderActor;
+ friend class TValueAggregationClient;
+ ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalSum;
+ ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalMin;
+ ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalMax;
+ std::deque<i64> Values;
+ i64 SumValue;
+ TMutex Mutex;
+
+ bool CalcAggregations(i64& minValue, i64& maxValue) const;
+ std::optional<TSignalAggregations> GetAggregations() const;
+
+ i64* RegisterValue(const i64 zeroValue = 0) {
+ TGuard<TMutex> g(Mutex);
+ Values.emplace_back(zeroValue);
+ return &Values.back();
+ }
+
+public:
+ TValueAggregationAgent(const TString& signalName, const TCommonCountersOwner& signalsOwner);
+
+ void ResendStatus() const;
+
+ std::shared_ptr<TValueAggregationClient> GetClient(std::shared_ptr<TValueAggregationAgent> selfPtr);
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/client.cpp b/ydb/core/tx/columnshard/counters/common/client.cpp
new file mode 100644
index 0000000000..95c400b35e
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/client.cpp
@@ -0,0 +1,17 @@
+#include "client.h"
+#include "agent.h"
+
+namespace NKikimr::NColumnShard {
+
+TValueAggregationClient::TValueAggregationClient(std::shared_ptr<TValueAggregationAgent> owner)
+ : Owner(owner)
+ , ValuePtr(Owner->RegisterValue(0))
+{
+
+}
+
+void TValueAggregationClient::Set(const i64 value) const {
+ *ValuePtr = value;
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/client.h b/ydb/core/tx/columnshard/counters/common/client.h
new file mode 100644
index 0000000000..0c8f5a0107
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/client.h
@@ -0,0 +1,18 @@
+#pragma once
+#include <memory>
+#include <util/system/types.h>
+
+namespace NKikimr::NColumnShard {
+class TValueAggregationAgent;
+
+class TValueAggregationClient {
+private:
+ std::shared_ptr<TValueAggregationAgent> Owner;
+ i64* ValuePtr = nullptr;
+public:
+ TValueAggregationClient(std::shared_ptr<TValueAggregationAgent> owner);
+
+ void Set(const i64 value) const;
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/owner.cpp b/ydb/core/tx/columnshard/counters/common/owner.cpp
new file mode 100644
index 0000000000..d674fcf231
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/owner.cpp
@@ -0,0 +1,43 @@
+#include "owner.h"
+#include "private.h"
+#include <ydb/core/base/appdata.h>
+#include <ydb/core/base/counters.h>
+
+namespace NKikimr::NColumnShard {
+
+NMonitoring::TDynamicCounters::TCounterPtr TCommonCountersOwner::GetDeriviative(const TString& name) const {
+ return SubGroup->GetCounter(NormalizeSignalName(ModuleId + "/Deriviative/" + name), true);
+}
+
+NMonitoring::TDynamicCounters::TCounterPtr TCommonCountersOwner::GetValue(const TString& name) const {
+ return SubGroup->GetCounter(NormalizeSignalName(ModuleId + "/Value/" + name), false);
+}
+
+TString TCommonCountersOwner::NormalizeSignalName(const TString& name) const {
+ return TFsPath(name).Fix().GetPath();
+}
+
+TCommonCountersOwner::TCommonCountersOwner(const TString& module)
+ : ModuleId(module)
+{
+ if (NActors::TlsActivationContext) {
+ SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard");
+ } else {
+ SubGroup = new NMonitoring::TDynamicCounters();
+ }
+}
+
+NMonitoring::THistogramPtr TCommonCountersOwner::GetHistogram(const TString& name, NMonitoring::IHistogramCollectorPtr&& hCollector) const {
+ return SubGroup->GetHistogram(NormalizeSignalName(ModuleId + "/Histogram/" + name), std::move(hCollector));
+}
+
+std::shared_ptr<TValueAggregationAgent> TCommonCountersOwner::GetValueAutoAggregations(const TString& name) const {
+ return NPrivate::TAggregationsController::GetAggregation(name, *this);
+}
+
+std::shared_ptr<TValueAggregationClient> TCommonCountersOwner::GetValueAutoAggregationsClient(const TString& name) const {
+ std::shared_ptr<TValueAggregationAgent> agent = NPrivate::TAggregationsController::GetAggregation(name, *this);
+ return std::make_shared<TValueAggregationClient>(agent);
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/owner.h b/ydb/core/tx/columnshard/counters/common/owner.h
new file mode 100644
index 0000000000..172e94860b
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/owner.h
@@ -0,0 +1,27 @@
+#pragma once
+#include "agent.h"
+#include "client.h"
+
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <util/system/mutex.h>
+#include <deque>
+
+namespace NKikimr::NColumnShard {
+
+class TCommonCountersOwner {
+private:
+ friend class TValueAggregationAgent;
+ ::NMonitoring::TDynamicCounterPtr SubGroup;
+ const TString ModuleId;
+ TString NormalizeSignalName(const TString& name) const;
+protected:
+ NMonitoring::TDynamicCounters::TCounterPtr GetDeriviative(const TString& name) const;
+ NMonitoring::THistogramPtr GetHistogram(const TString& name, NMonitoring::IHistogramCollectorPtr&& hCollector) const;
+ NMonitoring::TDynamicCounters::TCounterPtr GetValue(const TString& name) const;
+ std::shared_ptr<TValueAggregationAgent> GetValueAutoAggregations(const TString& name) const;
+ std::shared_ptr<TValueAggregationClient> GetValueAutoAggregationsClient(const TString& name) const;
+public:
+ TCommonCountersOwner(const TString& module);
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/private.cpp b/ydb/core/tx/columnshard/counters/common/private.cpp
new file mode 100644
index 0000000000..2c6d0c92c9
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/private.cpp
@@ -0,0 +1,60 @@
+#include "private.h"
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+
+namespace NKikimr::NColumnShard::NPrivate {
+namespace {
+class TRegularSignalBuilderActor: public NActors::TActorBootstrapped<TRegularSignalBuilderActor> {
+private:
+ std::shared_ptr<TValueAggregationAgent> Agent;
+
+ void Handle(NActors::TEvents::TEvWakeup::TPtr& /*ev*/) {
+ Agent->ResendStatus();
+ Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup);
+ }
+public:
+ TRegularSignalBuilderActor(std::shared_ptr<TValueAggregationAgent> agent)
+ : Agent(agent)
+ {
+
+ }
+
+ void Bootstrap() {
+ Agent->ResendStatus();
+ Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup);
+ Become(&TRegularSignalBuilderActor::StateMain);
+ }
+
+ STATEFN(StateMain) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NActors::TEvents::TEvWakeup, Handle);
+ default:
+ break;
+ }
+ }
+};
+}
+
+
+class TAggregationsControllerImpl {
+private:
+ TMutex Mutex;
+ THashMap<TString, std::shared_ptr<TValueAggregationAgent>> Agents;
+public:
+ std::shared_ptr<TValueAggregationAgent> GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner) {
+ TGuard<TMutex> g(Mutex);
+ auto it = Agents.find(signalName);
+ if (it == Agents.end()) {
+ it = Agents.emplace(signalName, std::make_shared<TValueAggregationAgent>(signalName, signalsOwner)).first;
+ NActors::TActivationContext::Register(new TRegularSignalBuilderActor(it->second));
+ }
+ return it->second;
+ }
+};
+
+std::shared_ptr<TValueAggregationAgent> TAggregationsController::GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner) {
+ return Singleton<TAggregationsControllerImpl>()->GetAggregation(signalName, signalsOwner);
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common/private.h b/ydb/core/tx/columnshard/counters/common/private.h
new file mode 100644
index 0000000000..e07082593e
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common/private.h
@@ -0,0 +1,12 @@
+#pragma once
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include "owner.h"
+#include "agent.h"
+
+namespace NKikimr::NColumnShard::NPrivate {
+class TAggregationsController {
+public:
+ static std::shared_ptr<TValueAggregationAgent> GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner);
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common_data.cpp b/ydb/core/tx/columnshard/counters/common_data.cpp
new file mode 100644
index 0000000000..9f72a45171
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common_data.cpp
@@ -0,0 +1,19 @@
+#include "common_data.h"
+
+namespace NKikimr::NColumnShard {
+
+TDataOwnerSignals::TDataOwnerSignals(const TString& module, const TString dataName)
+ : TBase(module)
+ , DataName(dataName)
+{
+ AddCount = GetDeriviative(DataName + "/Add/Count");
+ AddBytes = GetDeriviative(DataName + "/Add/Bytes");
+ EraseCount = GetDeriviative(DataName + "/Erase/Count");
+ EraseBytes = GetDeriviative(DataName + "/Erase/Bytes");
+ SkipAddCount = GetDeriviative(DataName + "/SkipAdd/Count");
+ SkipAddBytes = GetDeriviative(DataName + "/SkipAdd/Bytes");
+ SkipEraseCount = GetDeriviative(DataName + "/SkipErase/Count");
+ SkipEraseBytes = GetDeriviative(DataName + "/SkipErase/Bytes");
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/common_data.h b/ydb/core/tx/columnshard/counters/common_data.h
new file mode 100644
index 0000000000..b66a4aab2f
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/common_data.h
@@ -0,0 +1,44 @@
+#pragma once
+#include "common/owner.h"
+
+namespace NKikimr::NColumnShard {
+
+class TDataOwnerSignals: public TCommonCountersOwner {
+private:
+ using TBase = TCommonCountersOwner;
+ const TString DataName;
+ NMonitoring::TDynamicCounters::TCounterPtr AddCount;
+ NMonitoring::TDynamicCounters::TCounterPtr AddBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipAddCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipAddBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr EraseCount;
+ NMonitoring::TDynamicCounters::TCounterPtr EraseBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipEraseCount;
+ NMonitoring::TDynamicCounters::TCounterPtr SkipEraseBytes;
+public:
+ TDataOwnerSignals(const TString& module, const TString dataName);
+
+ void Add(const ui64 size) const {
+ AddCount->Add(1);
+ AddBytes->Add(size);
+ }
+
+ void Erase(const ui64 size) const {
+ EraseCount->Add(1);
+ EraseBytes->Add(size);
+ }
+
+ void SkipAdd(const ui64 size) const {
+ SkipAddCount->Add(1);
+ SkipAddBytes->Add(size);
+ }
+
+ void SkipErase(const ui64 size) const {
+ SkipEraseCount->Add(1);
+ SkipEraseBytes->Add(size);
+ }
+
+};
+
+}
diff --git a/ydb/core/tx/columnshard/counters/engine_logs.cpp b/ydb/core/tx/columnshard/counters/engine_logs.cpp
index b855fcff8c..cc208742fa 100644
--- a/ydb/core/tx/columnshard/counters/engine_logs.cpp
+++ b/ydb/core/tx/columnshard/counters/engine_logs.cpp
@@ -4,18 +4,26 @@
namespace NKikimr::NColumnShard {
-TEngineLogsCounters::TEngineLogsCounters() {
- const TString module = "EngineLogs";
- if (NActors::TlsActivationContext) {
- SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard");
- } else {
- SubGroup = new NMonitoring::TDynamicCounters();
- }
- OverloadGranules = SubGroup->GetCounter(module + "/Granules/Overload", false);
- CompactOverloadGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Overload/Count", true);
- NoCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/No/Count", true);
- SplitCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Split/Count", true);
- InternalCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Internal/Count", true);
+TEngineLogsCounters::TEngineLogsCounters()
+ : TBase("EngineLogs")
+{
+ OverloadGranules = TBase::GetValue("Granules/Overload");
+ CompactOverloadGranulesSelection = TBase::GetDeriviative("Granules/Selection/Overload/Count");
+ NoCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/No/Count");
+ SplitCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/Split/Count");
+ InternalCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/Internal/Count");
+
+ PortionToDropCount = TBase::GetDeriviative("Ttl/PortionToDrop/Count");
+ PortionToDropBytes = TBase::GetDeriviative("Ttl/PortionToDrop/Bytes");
+
+ PortionToEvictCount = TBase::GetDeriviative("Ttl/PortionToEvict/Count");
+ PortionToEvictBytes = TBase::GetDeriviative("Ttl/PortionToEvict/Bytes");
+
+ PortionNoTtlColumnCount = TBase::GetDeriviative("Ttl/PortionNoTtlColumn/Count");
+ PortionNoTtlColumnBytes = TBase::GetDeriviative("Ttl/PortionNoTtlColumn/Bytes");
+
+ PortionNoBorderCount = TBase::GetDeriviative("Ttl/PortionNoBorder/Count");
+ PortionNoBorderBytes = TBase::GetDeriviative("Ttl/PortionNoBorder/Bytes");
}
}
diff --git a/ydb/core/tx/columnshard/counters/engine_logs.h b/ydb/core/tx/columnshard/counters/engine_logs.h
index 7774f3c3aa..6beb40308a 100644
--- a/ydb/core/tx/columnshard/counters/engine_logs.h
+++ b/ydb/core/tx/columnshard/counters/engine_logs.h
@@ -1,11 +1,23 @@
#pragma once
#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include "common/owner.h"
namespace NKikimr::NColumnShard {
-class TEngineLogsCounters {
+class TEngineLogsCounters: public TCommonCountersOwner {
private:
- ::NMonitoring::TDynamicCounterPtr SubGroup;
+ using TBase = TCommonCountersOwner;
+ NMonitoring::TDynamicCounters::TCounterPtr PortionToDropCount;
+ NMonitoring::TDynamicCounters::TCounterPtr PortionToDropBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr PortionToEvictCount;
+ NMonitoring::TDynamicCounters::TCounterPtr PortionToEvictBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr PortionNoTtlColumnCount;
+ NMonitoring::TDynamicCounters::TCounterPtr PortionNoTtlColumnBytes;
+
+ NMonitoring::TDynamicCounters::TCounterPtr PortionNoBorderCount;
+ NMonitoring::TDynamicCounters::TCounterPtr PortionNoBorderBytes;
public:
NMonitoring::TDynamicCounters::TCounterPtr OverloadGranules;
NMonitoring::TDynamicCounters::TCounterPtr CompactOverloadGranulesSelection;
@@ -13,6 +25,26 @@ public:
NMonitoring::TDynamicCounters::TCounterPtr SplitCompactGranulesSelection;
NMonitoring::TDynamicCounters::TCounterPtr InternalCompactGranulesSelection;
+ void OnPortionToEvict(const ui64 size) const {
+ PortionToEvictCount->Add(1);
+ PortionToEvictBytes->Add(size);
+ }
+
+ void OnPortionToDrop(const ui64 size) const {
+ PortionToDropCount->Add(1);
+ PortionToDropBytes->Add(size);
+ }
+
+ void OnPortionNoTtlColumn(const ui64 size) const {
+ PortionNoTtlColumnCount->Add(1);
+ PortionNoTtlColumnBytes->Add(size);
+ }
+
+ void OnPortionNoBorder(const ui64 size) const {
+ PortionNoBorderCount->Add(1);
+ PortionNoBorderBytes->Add(size);
+ }
+
TEngineLogsCounters();
};
diff --git a/ydb/core/tx/columnshard/counters/indexation.cpp b/ydb/core/tx/columnshard/counters/indexation.cpp
index 0403832c75..705c0a26ba 100644
--- a/ydb/core/tx/columnshard/counters/indexation.cpp
+++ b/ydb/core/tx/columnshard/counters/indexation.cpp
@@ -4,43 +4,42 @@
namespace NKikimr::NColumnShard {
-TIndexationCounters::TIndexationCounters(const TString& module) {
- if (NActors::TlsActivationContext) {
- SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard");
- } else {
- SubGroup = new NMonitoring::TDynamicCounters();
- }
- ReadBytes = SubGroup->GetCounter(module + "/Counters/Read/Bytes", true);
- AnalizeInsertedPortions = SubGroup->GetCounter(module + "/Counters/AnalizeInsertion/Portions", true);
- AnalizeInsertedBytes = SubGroup->GetCounter(module + "/Counters/AnalizeInsertion/Bytes", true);
- RepackedInsertedPortions = SubGroup->GetCounter(module + "/Counters/RepackedInsertion/Portions", true);
- RepackedInsertedPortionBytes = SubGroup->GetCounter(module + "/Counters/RepackedInsertion/Bytes", true);
+TIndexationCounters::TIndexationCounters(const TString& module)
+ : TBase(module)
+{
+ ReadBytes = TBase::GetDeriviative("Read/Bytes");
+ AnalizeInsertedPortions = TBase::GetDeriviative("AnalizeInsertion/Portions");
+ AnalizeInsertedBytes = TBase::GetDeriviative("AnalizeInsertion/Bytes");
+ RepackedInsertedPortions = TBase::GetDeriviative("RepackedInsertion/Portions");
+ RepackedInsertedPortionBytes = TBase::GetDeriviative("RepackedInsertion/Bytes");
- AnalizeCompactedPortions = SubGroup->GetCounter(module + "/Counters/AnalizeCompaction/Portions", true);
- AnalizeCompactedBytes = SubGroup->GetCounter(module + "/Counters/AnalizeCompaction/Bytes", true);
- SkipPortionsMoveThroughIntersection = SubGroup->GetCounter(module + "/Counters/SkipMoveThroughIntersection/Portions", true);
- SkipPortionBytesMoveThroughIntersection = SubGroup->GetCounter(module + "/Counters/SkipMoveThroughIntersection/Bytes", true);
- RepackedCompactedPortions = SubGroup->GetCounter(module + "/Counters/RepackedCompaction/Portions", true);
- MovedPortions = SubGroup->GetCounter(module + "/Counters/Moved/Portions", true);
- MovedPortionBytes = SubGroup->GetCounter(module + "/Counters/Moved/Bytes", true);
+ AnalizeCompactedPortions = TBase::GetDeriviative("AnalizeCompaction/Portions");
+ AnalizeCompactedBytes = TBase::GetDeriviative("AnalizeCompaction/Bytes");
+ SkipPortionsMoveThroughIntersection = TBase::GetDeriviative("SkipMoveThroughIntersection/Portions");
+ SkipPortionBytesMoveThroughIntersection = TBase::GetDeriviative("SkipMoveThroughIntersection/Bytes");
+ RepackedCompactedPortions = TBase::GetDeriviative("RepackedCompaction/Portions");
+ MovedPortions = TBase::GetDeriviative("Moved/Portions");
+ MovedPortionBytes = TBase::GetDeriviative("Moved/Bytes");
- TrashDataSerializationBytes = SubGroup->GetCounter(module + "/Counters/TrashDataSerialization/Bytes", true);
- TrashDataSerialization = SubGroup->GetCounter(module + "/Counters/TrashDataSerialization/Count", true);
- TrashDataSerializationHistogramBytes = SubGroup->GetHistogram(module + "/Histogram/TrashDataSerialization/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1024));
- CorrectDataSerializationBytes = SubGroup->GetCounter(module + "/Counters/CorrectDataSerialization/Bytes", true);
- CorrectDataSerialization = SubGroup->GetCounter(module + "/Counters/CorrectDataSerialization/Count", true);
+ TrashDataSerializationBytes = TBase::GetDeriviative("TrashDataSerialization/Bytes");
+ TrashDataSerialization = TBase::GetDeriviative("TrashDataSerialization/Count");
+ TrashDataSerializationHistogramBytes = TBase::GetHistogram("TrashDataSerialization/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1024));
+ CorrectDataSerializationBytes = TBase::GetDeriviative("CorrectDataSerialization/Bytes");
+ CorrectDataSerialization = TBase::GetDeriviative("CorrectDataSerialization/Count");
- CompactionDuration = SubGroup->GetHistogram(module + "/Histogram/CompactionDuration", NMonitoring::ExponentialHistogram(18, 2, 20));
- CompactionInputSize = SubGroup->GetHistogram(module + "/Histogram/CompactionInputSize", NMonitoring::ExponentialHistogram(18, 2, 1024));
- CompactionExceptions = SubGroup->GetCounter(module + "/Counters/Exceptions/Count", true);
- CompactionFails = SubGroup->GetCounter(module + "/Counters/CompactionFails/Count", true);
+ CompactionDuration = TBase::GetHistogram("CompactionDuration", NMonitoring::ExponentialHistogram(18, 2, 20));
+ HistogramCompactionInputBytes = TBase::GetHistogram("CompactionInput/Bytes", NMonitoring::ExponentialHistogram(18, 2, 1024));
+ CompactionInputBytes = TBase::GetDeriviative("CompactionInput/Bytes");
+ CompactionInputSize = TBase::GetHistogram("CompactionInput/Bytes", NMonitoring::ExponentialHistogram(18, 2, 1024));
+ CompactionExceptions = TBase::GetDeriviative("Exceptions/Count");
+ CompactionFails = TBase::GetDeriviative("CompactionFails/Count");
- SplittedPortionLargestColumnSize = SubGroup->GetHistogram(module + "/Histogram/SplittedPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
- SplittedPortionColumnSize = SubGroup->GetHistogram(module + "/Histogram/SplittedPortionColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
- SimpleSplitPortionLargestColumnSize = SubGroup->GetHistogram(module + "/Histogram/SimpleSplitPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
- TooSmallBlob = SubGroup->GetCounter(module + "/Counters/TooSmallBlob/Count", true);
- TooSmallBlobFinish = SubGroup->GetCounter(module + "/Counters/TooSmallBlobFinish/Count", true);
- TooSmallBlobStart = SubGroup->GetCounter(module + "/Counters/TooSmallBlobStart/Count", true);
+ SplittedPortionLargestColumnSize = TBase::GetHistogram("SplittedPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
+ SplittedPortionColumnSize = TBase::GetHistogram("SplittedPortionColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
+ SimpleSplitPortionLargestColumnSize = TBase::GetHistogram("SimpleSplitPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024));
+ TooSmallBlob = TBase::GetDeriviative("TooSmallBlob/Count");
+ TooSmallBlobFinish = TBase::GetDeriviative("TooSmallBlobFinish/Count");
+ TooSmallBlobStart = TBase::GetDeriviative("TooSmallBlobStart/Count");
}
}
diff --git a/ydb/core/tx/columnshard/counters/indexation.h b/ydb/core/tx/columnshard/counters/indexation.h
index fb3bb1ba5f..6a1c1e8842 100644
--- a/ydb/core/tx/columnshard/counters/indexation.h
+++ b/ydb/core/tx/columnshard/counters/indexation.h
@@ -1,12 +1,16 @@
#pragma once
#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include "common/owner.h"
namespace NKikimr::NColumnShard {
-class TIndexationCounters {
+class TIndexationCounters: public TCommonCountersOwner {
private:
- ::NMonitoring::TDynamicCounterPtr SubGroup;
+ using TBase = TCommonCountersOwner;
+ NMonitoring::THistogramPtr HistogramCompactionInputBytes;
public:
+ NMonitoring::TDynamicCounters::TCounterPtr CompactionInputBytes;
+
NMonitoring::TDynamicCounters::TCounterPtr ReadBytes;
NMonitoring::TDynamicCounters::TCounterPtr AnalizeCompactedPortions;
NMonitoring::TDynamicCounters::TCounterPtr AnalizeInsertedPortions;
@@ -39,6 +43,11 @@ public:
NMonitoring::TDynamicCounters::TCounterPtr CompactionFails;
TIndexationCounters(const TString& module);
+
+// void CompactionInputSize(const ui64 size) const {
+// HistogramCompactionInputBytes->Collect(size);
+// CompactionInputBytes->Add(size);
+// }
};
}
diff --git a/ydb/core/tx/columnshard/counters/insert_table.cpp b/ydb/core/tx/columnshard/counters/insert_table.cpp
new file mode 100644
index 0000000000..f7c5446dab
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/insert_table.cpp
@@ -0,0 +1,15 @@
+#include "insert_table.h"
+
+namespace NKikimr::NColumnShard {
+
+TInsertTableCounters::TInsertTableCounters()
+ : TBase("InsertTable")
+ , PathIdInserted("InsertTable", "Inserted")
+ , PathIdCommitted("InsertTable", "Committed")
+ , Inserted("InsertTable", "Inserted")
+ , Committed("InsertTable", "Committed")
+ , Aborted("InsertTable", "Aborted")
+{
+}
+
+}
diff --git a/ydb/core/tx/columnshard/counters/insert_table.h b/ydb/core/tx/columnshard/counters/insert_table.h
new file mode 100644
index 0000000000..30dee6ba64
--- /dev/null
+++ b/ydb/core/tx/columnshard/counters/insert_table.h
@@ -0,0 +1,71 @@
+#pragma once
+#include "common_data.h"
+#include "common/client.h"
+#include "common/agent.h"
+
+namespace NKikimr::NColumnShard {
+
+class TPathIdClientCounters {
+private:
+ using TBase = TCommonCountersOwner;
+ const std::shared_ptr<TValueAggregationClient> PathIdCommittedBytes;
+ const std::shared_ptr<TValueAggregationClient> PathIdCommittedChunks;
+public:
+ TPathIdClientCounters(std::shared_ptr<TValueAggregationClient> pathIdCommittedBytes, std::shared_ptr<TValueAggregationClient> pathIdCommittedChunks)
+ : PathIdCommittedBytes(pathIdCommittedBytes)
+ , PathIdCommittedChunks(pathIdCommittedChunks) {
+ }
+
+ void OnPathIdDataInfo(const ui64 bytes, const ui32 chunks) const {
+ PathIdCommittedBytes->Set(bytes);
+ PathIdCommittedChunks->Set(chunks);
+ }
+};
+
+class TPathIdInfoCounters: public TCommonCountersOwner {
+private:
+ using TBase = TCommonCountersOwner;
+ const std::shared_ptr<TValueAggregationAgent> PathIdCommittedBytes;
+ const std::shared_ptr<TValueAggregationAgent> PathIdCommittedChunks;
+public:
+ TPathIdInfoCounters(const TString& countersName, const TString& dataName)
+ : TBase(countersName)
+ , PathIdCommittedBytes(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Bytes"))
+ , PathIdCommittedChunks(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Chunks"))
+ {
+ }
+
+ TPathIdClientCounters GetClient() const {
+ return TPathIdClientCounters(PathIdCommittedBytes->GetClient(PathIdCommittedBytes), PathIdCommittedChunks->GetClient(PathIdCommittedChunks));
+ }
+};
+
+class TPathIdOwnedCounters {
+public:
+ const TPathIdClientCounters Inserted;
+ const TPathIdClientCounters Committed;
+ TPathIdOwnedCounters(TPathIdClientCounters&& inserted, TPathIdClientCounters&& committed)
+ : Inserted(std::move(inserted))
+ , Committed(std::move(committed)) {
+
+ }
+};
+
+class TInsertTableCounters: public TCommonCountersOwner {
+private:
+ using TBase = TCommonCountersOwner;
+ const TPathIdInfoCounters PathIdInserted;
+ const TPathIdInfoCounters PathIdCommitted;
+public:
+ const TDataOwnerSignals Inserted;
+ const TDataOwnerSignals Committed;
+ const TDataOwnerSignals Aborted;
+
+ TInsertTableCounters();
+
+ TPathIdOwnedCounters GetPathIdCounters() const {
+ return TPathIdOwnedCounters(PathIdInserted.GetClient(), PathIdCommitted.GetClient());
+ }
+};
+
+}