diff options
author | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-06-06 11:26:46 +0300 |
---|---|---|
committer | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-06-06 11:26:46 +0300 |
commit | 8d0a5361300692072804774123e7753775c834d5 (patch) | |
tree | bca40d15fec2f97d9ff3802d09b05a7ba095f501 | |
parent | b35fea336b768407c64079e8c4ec85488dded825 (diff) | |
download | ydb-8d0a5361300692072804774123e7753775c834d5.tar.gz |
experimental aggregation and new signals
29 files changed, 974 insertions, 49 deletions
diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt index 10c00092fd..71ef940f82 100644 --- a/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/columnshard/counters/CMakeLists.darwin-x86_64.txt @@ -6,16 +6,22 @@ # original buildsystem will not be accepted. +add_subdirectory(common) add_library(tx-columnshard-counters) target_link_libraries(tx-columnshard-counters PUBLIC contrib-libs-cxxsupp yutil cpp-monlib-dynamic_counters + columnshard-counters-common ydb-core-base ) target_sources(tx-columnshard-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp ) diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt index 0502f0daa8..edfdee32c7 100644 --- a/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/columnshard/counters/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(common) add_library(tx-columnshard-counters) target_link_libraries(tx-columnshard-counters PUBLIC @@ -13,10 +14,15 @@ target_link_libraries(tx-columnshard-counters PUBLIC contrib-libs-cxxsupp yutil cpp-monlib-dynamic_counters + columnshard-counters-common ydb-core-base ) target_sources(tx-columnshard-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp ) diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt index 0502f0daa8..edfdee32c7 100644 --- a/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/columnshard/counters/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(common) add_library(tx-columnshard-counters) target_link_libraries(tx-columnshard-counters PUBLIC @@ -13,10 +14,15 @@ target_link_libraries(tx-columnshard-counters PUBLIC contrib-libs-cxxsupp yutil cpp-monlib-dynamic_counters + columnshard-counters-common ydb-core-base ) target_sources(tx-columnshard-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp ) diff --git a/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt index 10c00092fd..71ef940f82 100644 --- a/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/columnshard/counters/CMakeLists.windows-x86_64.txt @@ -6,16 +6,22 @@ # original buildsystem will not be accepted. +add_subdirectory(common) add_library(tx-columnshard-counters) target_link_libraries(tx-columnshard-counters PUBLIC contrib-libs-cxxsupp yutil cpp-monlib-dynamic_counters + columnshard-counters-common ydb-core-base ) target_sources(tx-columnshard-counters PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/indexation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/scan.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/engine_logs.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/blobs_manager.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/columnshard.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/insert_table.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common_data.cpp ) diff --git a/ydb/core/tx/columnshard/counters/blobs_manager.cpp b/ydb/core/tx/columnshard/counters/blobs_manager.cpp new file mode 100644 index 0000000000..e4c290f7cd --- /dev/null +++ b/ydb/core/tx/columnshard/counters/blobs_manager.cpp @@ -0,0 +1,63 @@ +#include "blobs_manager.h" +#include <ydb/core/base/appdata.h> +#include <ydb/core/base/counters.h> + +namespace NKikimr::NColumnShard { + +TBlobsManagerCounters::TBlobsManagerCounters(const TString& module) + : TCommonCountersOwner(module) +{ + SkipCollection = TBase::GetDeriviative("GC/Skip/Count"); + StartCollection = TBase::GetDeriviative("GC/Start/Count"); + CollectDropExplicitBytes = TBase::GetDeriviative("GC/Drop/Explicit/Bytes"); + CollectDropExplicitCount = TBase::GetDeriviative("GC/Drop/Explicit/Count"); + CollectDropImplicitBytes = TBase::GetDeriviative("GC/Drop/Implicit/Bytes"); + CollectDropImplicitCount = TBase::GetDeriviative("GC/Drop/Implicit/Count"); + CollectKeepBytes = TBase::GetDeriviative("GC/Keep/Bytes"); + CollectKeepCount = TBase::GetDeriviative("GC/Keep/Count"); + PutBlobBytes = TBase::GetDeriviative("GC/PutBlob/Bytes"); + PutBlobCount = TBase::GetDeriviative("GC/PutBlob/Count"); + CollectGen = TBase::GetValue("GC/Gen"); + CollectStep = TBase::GetValue("GC/Step"); + + DeleteBlobMarkerBytes = TBase::GetDeriviative("GC/MarkerDeleteBlob/Bytes"); + DeleteBlobMarkerCount = TBase::GetDeriviative("GC/MarkerDeleteBlob/Count"); + DeleteBlobDelayedMarkerBytes = TBase::GetDeriviative("GC/MarkerDelayedDeleteBlob/Bytes"); + DeleteBlobDelayedMarkerCount = TBase::GetDeriviative("GC/MarkerDelayedDeleteBlob/Count"); + AddSmallBlobBytes = TBase::GetDeriviative("GC/AddSmallBlob/Bytes"); + AddSmallBlobCount = TBase::GetDeriviative("GC/AddSmallBlob/Count"); + DeleteSmallBlobBytes = TBase::GetDeriviative("GC/DeleteSmallBlob/Bytes"); + DeleteSmallBlobCount = TBase::GetDeriviative("GC/DeleteSmallBlob/Count"); + + BlobsKeepCount = TBase::GetValue("GC/BlobsKeep/Count"); + BlobsKeepBytes = TBase::GetValue("GC/BlobsKeep/Bytes"); + BlobsDeleteCount = TBase::GetValue("GC/BlobsDelete/Count"); + BlobsDeleteBytes = TBase::GetValue("GC/BlobsDelete/Bytes"); + + BrokenKeepCount = TBase::GetDeriviative("GC/BrokenKeep/Count"); + BrokenKeepBytes = TBase::GetDeriviative("GC/BrokenKeep/Bytes"); + + KeepMarkerCount = TBase::GetDeriviative("GC/KeepMarker/Count"); + KeepMarkerBytes = TBase::GetDeriviative("GC/KeepMarker/Bytes"); +} + +void TBlobsManagerCounters::OnBlobsKeep(const TSet<TLogoBlobID>& blobs) const { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnBlobsKeep")("count", blobs.size()); +// BlobsKeepCount->Set(blobs.size()); +// ui64 size = 0; +// for (auto&& i : blobs) { +// size += i.BlobSize(); +// } +// BlobsKeepBytes->Set(size); +} + +void TBlobsManagerCounters::OnBlobsDelete(const TSet<TLogoBlobID>& /*blobs*/) const { + // BlobsDeleteCount->Set(blobs.size()); + // ui64 size = 0; + // for (auto&& i : blobs) { + // size += i.BlobSize(); + // } + // BlobsDeleteBytes->Set(size); +} + +} diff --git a/ydb/core/tx/columnshard/counters/blobs_manager.h b/ydb/core/tx/columnshard/counters/blobs_manager.h new file mode 100644 index 0000000000..ef11e50826 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/blobs_manager.h @@ -0,0 +1,104 @@ +#pragma once +#include <ydb/core/base/logoblob.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include "common/owner.h" + +namespace NKikimr::NColumnShard { + +class TBlobsManagerCounters: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + NMonitoring::TDynamicCounters::TCounterPtr CollectDropExplicitBytes; + NMonitoring::TDynamicCounters::TCounterPtr CollectDropExplicitCount; + NMonitoring::TDynamicCounters::TCounterPtr CollectDropImplicitBytes; + NMonitoring::TDynamicCounters::TCounterPtr CollectDropImplicitCount; + NMonitoring::TDynamicCounters::TCounterPtr CollectKeepBytes; + NMonitoring::TDynamicCounters::TCounterPtr CollectKeepCount; + NMonitoring::TDynamicCounters::TCounterPtr PutBlobBytes; + NMonitoring::TDynamicCounters::TCounterPtr PutBlobCount; + NMonitoring::TDynamicCounters::TCounterPtr CollectGen; + NMonitoring::TDynamicCounters::TCounterPtr CollectStep; + NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobMarkerBytes; + NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobMarkerCount; + NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobDelayedMarkerBytes; + NMonitoring::TDynamicCounters::TCounterPtr DeleteBlobDelayedMarkerCount; + NMonitoring::TDynamicCounters::TCounterPtr AddSmallBlobBytes; + NMonitoring::TDynamicCounters::TCounterPtr AddSmallBlobCount; + NMonitoring::TDynamicCounters::TCounterPtr DeleteSmallBlobBytes; + NMonitoring::TDynamicCounters::TCounterPtr DeleteSmallBlobCount; + NMonitoring::TDynamicCounters::TCounterPtr BrokenKeepCount; + NMonitoring::TDynamicCounters::TCounterPtr BrokenKeepBytes; + NMonitoring::TDynamicCounters::TCounterPtr BlobsKeepCount; + NMonitoring::TDynamicCounters::TCounterPtr BlobsKeepBytes; + NMonitoring::TDynamicCounters::TCounterPtr BlobsDeleteCount; + NMonitoring::TDynamicCounters::TCounterPtr BlobsDeleteBytes; + NMonitoring::TDynamicCounters::TCounterPtr KeepMarkerCount; + NMonitoring::TDynamicCounters::TCounterPtr KeepMarkerBytes; + +public: + NMonitoring::TDynamicCounters::TCounterPtr SkipCollection; + NMonitoring::TDynamicCounters::TCounterPtr StartCollection; + + TBlobsManagerCounters(const TString& module); + + void OnKeepMarker(const ui64 size) const { + KeepMarkerCount->Add(1); + KeepMarkerBytes->Add(size); + } + + void OnBlobsKeep(const TSet<TLogoBlobID>& blobs) const; + + void OnBlobsDelete(const TSet<TLogoBlobID>& /*blobs*/) const; + + void OnAddSmallBlob(const ui32 bSize) const { + AddSmallBlobBytes->Add(bSize); + AddSmallBlobCount->Add(1); + } + + void OnDeleteBlobDelayedMarker(const ui32 bSize) const { + DeleteBlobDelayedMarkerBytes->Add(bSize); + DeleteBlobDelayedMarkerCount->Add(1); + } + + void OnDeleteBlobMarker(const ui32 bSize) const { + DeleteBlobMarkerBytes->Add(bSize); + DeleteBlobMarkerCount->Add(1); + } + + void OnNewCollectStep(const ui32 gen, const ui32 step) const { + CollectGen->Set(gen); + CollectStep->Set(step); + } + + void OnDeleteSmallBlob(const ui32 bSize) const { + DeleteSmallBlobBytes->Add(bSize); + DeleteSmallBlobCount->Add(1); + } + + void OnPutResult(const ui32 bSize) const { + PutBlobBytes->Add(bSize); + PutBlobCount->Add(1); + } + + void OnCollectKeep(const ui32 bSize) const { + CollectKeepBytes->Add(bSize); + CollectKeepCount->Add(1); + } + + void OnBrokenKeep(const ui32 bSize) const { + BrokenKeepBytes->Add(bSize); + BrokenKeepCount->Add(1); + } + + void OnCollectDropExplicit(const ui32 bSize) const { + CollectDropExplicitBytes->Add(bSize); + CollectDropExplicitCount->Add(1); + } + + void OnCollectDropImplicit(const ui32 bSize) const { + CollectDropImplicitBytes->Add(bSize); + CollectDropImplicitCount->Add(1); + } +}; + +} diff --git a/ydb/core/tx/columnshard/counters/columnshard.cpp b/ydb/core/tx/columnshard/counters/columnshard.cpp new file mode 100644 index 0000000000..a399748856 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/columnshard.cpp @@ -0,0 +1,39 @@ +#include "columnshard.h" +#include <ydb/core/base/appdata.h> +#include <ydb/core/base/counters.h> + +namespace NKikimr::NColumnShard { + +TCSCounters::TCSCounters() + : TBase("CS") +{ + StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count"); + TooEarlyBackgroundCount = TBase::GetDeriviative("TooEarlyBackground/Count"); + SetupCompactionCount = TBase::GetDeriviative("SetupCompaction/Count"); + SetupIndexationCount = TBase::GetDeriviative("SetupIndexation/Count"); + SetupTtlCount = TBase::GetDeriviative("SetupTtl/Count"); + SetupCleanupCount = TBase::GetDeriviative("SetupCleanup/Count"); + + SkipIndexationInputDueToGranuleOverloadBytes = TBase::GetDeriviative("SkipIndexationInput/GranuleOverload/Bytes"); + SkipIndexationInputDueToGranuleOverloadCount = TBase::GetDeriviative("SkipIndexationInput/GranuleOverload/Count"); + SkipIndexationInputDueToSplitCompactionBytes = TBase::GetDeriviative("SkipIndexationInput/SplitCompaction/Bytes"); + SkipIndexationInputDueToSplitCompactionCount = TBase::GetDeriviative("SkipIndexationInput/SplitCompaction/Count"); + + FutureIndexationInputBytes = TBase::GetDeriviative("FutureIndexationInput/Bytes"); + IndexationInputBytes = TBase::GetDeriviative("IndexationInput/Bytes"); + + OverloadInsertTableBytes = TBase::GetDeriviative("OverloadInsertTable/Bytes"); + OverloadInsertTableCount = TBase::GetDeriviative("OverloadInsertTable/Count"); + OverloadGranuleBytes = TBase::GetDeriviative("OverloadGranule/Bytes"); + OverloadGranuleCount = TBase::GetDeriviative("OverloadGranule/Count"); + OverloadShardBytes = TBase::GetDeriviative("OverloadShard/Bytes"); + OverloadShardCount = TBase::GetDeriviative("OverloadShard/Count"); + + InternalCompactionGranuleBytes = TBase::GetValueAutoAggregationsClient("InternalCompaction/Bytes"); + InternalCompactionGranulePortionsCount = TBase::GetValueAutoAggregationsClient("InternalCompaction/PortionsCount"); + + SplitCompactionGranuleBytes = TBase::GetValueAutoAggregationsClient("SplitCompaction/Bytes"); + SplitCompactionGranulePortionsCount = TBase::GetValueAutoAggregationsClient("SplitCompaction/PortionsCount"); +} + +} diff --git a/ydb/core/tx/columnshard/counters/columnshard.h b/ydb/core/tx/columnshard/counters/columnshard.h new file mode 100644 index 0000000000..c9b625eb7c --- /dev/null +++ b/ydb/core/tx/columnshard/counters/columnshard.h @@ -0,0 +1,108 @@ +#pragma once +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include "common/owner.h" + +namespace NKikimr::NColumnShard { + +class TCSCounters: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + + NMonitoring::TDynamicCounters::TCounterPtr StartBackgroundCount; + NMonitoring::TDynamicCounters::TCounterPtr TooEarlyBackgroundCount; + NMonitoring::TDynamicCounters::TCounterPtr SetupCompactionCount; + NMonitoring::TDynamicCounters::TCounterPtr SetupIndexationCount; + NMonitoring::TDynamicCounters::TCounterPtr SetupTtlCount; + NMonitoring::TDynamicCounters::TCounterPtr SetupCleanupCount; + + NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToGranuleOverloadBytes; + NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToGranuleOverloadCount; + NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToSplitCompactionBytes; + NMonitoring::TDynamicCounters::TCounterPtr SkipIndexationInputDueToSplitCompactionCount; + NMonitoring::TDynamicCounters::TCounterPtr FutureIndexationInputBytes; + NMonitoring::TDynamicCounters::TCounterPtr IndexationInputBytes; + + NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableBytes; + NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableCount; + NMonitoring::TDynamicCounters::TCounterPtr OverloadGranuleBytes; + NMonitoring::TDynamicCounters::TCounterPtr OverloadGranuleCount; + NMonitoring::TDynamicCounters::TCounterPtr OverloadShardBytes; + NMonitoring::TDynamicCounters::TCounterPtr OverloadShardCount; + + std::shared_ptr<TValueAggregationClient> InternalCompactionGranuleBytes; + std::shared_ptr<TValueAggregationClient> InternalCompactionGranulePortionsCount; + + std::shared_ptr<TValueAggregationClient> SplitCompactionGranuleBytes; + std::shared_ptr<TValueAggregationClient> SplitCompactionGranulePortionsCount; +public: + void OnInternalCompactionInfo(const ui64 bytes, const ui32 portionsCount) const { + InternalCompactionGranuleBytes->Set(bytes); + InternalCompactionGranulePortionsCount->Set(portionsCount); + } + + void OnSplitCompactionInfo(const ui64 bytes, const ui32 portionsCount) const { + SplitCompactionGranuleBytes->Set(bytes); + SplitCompactionGranulePortionsCount->Set(portionsCount); + } + + void OnOverloadInsertTable(const ui64 size) const { + OverloadInsertTableBytes->Add(size); + OverloadInsertTableCount->Add(1); + } + + void OnOverloadGranule(const ui64 size) const { + OverloadGranuleBytes->Add(size); + OverloadGranuleCount->Add(1); + } + + void OnOverloadShard(const ui64 size) const { + OverloadShardBytes->Add(size); + OverloadShardCount->Add(1); + } + + void SkipIndexationInputDutToSplitCompaction(const ui64 size) const { + SkipIndexationInputDueToSplitCompactionBytes->Add(size); + SkipIndexationInputDueToSplitCompactionCount->Add(1); + } + + void SkipIndexationInputDutToGranuleOverload(const ui64 size) const { + SkipIndexationInputDueToGranuleOverloadBytes->Add(size); + SkipIndexationInputDueToGranuleOverloadCount->Add(1); + } + + void FutureIndexationInput(const ui64 size) const { + FutureIndexationInputBytes->Add(size); + } + + void IndexationInput(const ui64 size) const { + IndexationInputBytes->Add(size); + } + + void OnStartBackground() const { + StartBackgroundCount->Add(1); + } + + void OnTooEarly() const { + TooEarlyBackgroundCount->Add(1); + } + + void OnSetupCompaction() const { + SetupCompactionCount->Add(1); + } + + void OnSetupIndexation() const { + SetupIndexationCount->Add(1); + } + + void OnSetupTtl() const { + SetupTtlCount->Add(1); + } + + void OnSetupCleanup() const { + SetupCleanupCount->Add(1); + } + + TCSCounters(); +}; + +} diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..90f970a4a6 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(columnshard-counters-common) +target_link_libraries(columnshard-counters-common PUBLIC + contrib-libs-cxxsupp + yutil + cpp-monlib-dynamic_counters + ydb-core-base +) +target_sources(columnshard-counters-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp +) diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..ba3f014509 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-aarch64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(columnshard-counters-common) +target_link_libraries(columnshard-counters-common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-monlib-dynamic_counters + ydb-core-base +) +target_sources(columnshard-counters-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp +) diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..ba3f014509 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.linux-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(columnshard-counters-common) +target_link_libraries(columnshard-counters-common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-monlib-dynamic_counters + ydb-core-base +) +target_sources(columnshard-counters-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp +) diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..90f970a4a6 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(columnshard-counters-common) +target_link_libraries(columnshard-counters-common PUBLIC + contrib-libs-cxxsupp + yutil + cpp-monlib-dynamic_counters + ydb-core-base +) +target_sources(columnshard-counters-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/agent.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/owner.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/counters/common/private.cpp +) diff --git a/ydb/core/tx/columnshard/counters/common/agent.cpp b/ydb/core/tx/columnshard/counters/common/agent.cpp new file mode 100644 index 0000000000..558934e8f9 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/agent.cpp @@ -0,0 +1,55 @@ +#include "agent.h" +#include "owner.h" + +namespace NKikimr::NColumnShard { + +TValueAggregationAgent::TValueAggregationAgent(const TString& signalName, const TCommonCountersOwner& signalsOwner) + : ValueSignalSum(signalsOwner.GetValue("SUM/" + signalName)) + , ValueSignalMin(signalsOwner.GetValue("MIN/" + signalName)) + , ValueSignalMax(signalsOwner.GetValue("MAX/" + signalName)) +{ + +} + +bool TValueAggregationAgent::CalcAggregations(i64& minValue, i64& maxValue) const { + const ui32 count = Values.size(); + if (!count) { + return false; + } + minValue = Values.front(); + maxValue = Values.front(); + for (ui32 i = 0; i < count; ++i) { + if (minValue > Values[i]) { + minValue = Values[i]; + } + if (maxValue < Values[i]) { + maxValue = Values[i]; + } + } + return true; +} + +std::optional<NKikimr::NColumnShard::TSignalAggregations> TValueAggregationAgent::GetAggregations() const { + i64 min; + i64 max; + if (!CalcAggregations(min, max)) { + return {}; + } + return TSignalAggregations(SumValue, min, max); +} + +void TValueAggregationAgent::ResendStatus() const { + TGuard<TMutex> g(Mutex); + std::optional<TSignalAggregations> aggr = GetAggregations(); + if (!!aggr) { + ValueSignalMin->Set(aggr->Min); + ValueSignalMax->Set(aggr->Max); + ValueSignalSum->Set(aggr->Sum); + } +} + +std::shared_ptr<NKikimr::NColumnShard::TValueAggregationClient> TValueAggregationAgent::GetClient(std::shared_ptr<TValueAggregationAgent> selfPtr) { + return std::make_shared<TValueAggregationClient>(selfPtr); +} + +} diff --git a/ydb/core/tx/columnshard/counters/common/agent.h b/ydb/core/tx/columnshard/counters/common/agent.h new file mode 100644 index 0000000000..306cc4a421 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/agent.h @@ -0,0 +1,51 @@ +#pragma once +#include "client.h" +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <util/system/mutex.h> +#include <deque> + +namespace NKikimr::NColumnShard { +class TCommonCountersOwner; + +class TSignalAggregations { +public: + const i64 Sum = 0; + const i64 Min = 0; + const i64 Max = 0; + TSignalAggregations(const i64 sum, const i64 min, const i64 max) + : Sum(sum) + , Min(min) + , Max(max) { + + } +}; + +class TValueAggregationAgent: TNonCopyable { +private: + friend class TRegularSignalBuilderActor; + friend class TValueAggregationClient; + ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalSum; + ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalMin; + ::NMonitoring::TDynamicCounters::TCounterPtr ValueSignalMax; + std::deque<i64> Values; + i64 SumValue; + TMutex Mutex; + + bool CalcAggregations(i64& minValue, i64& maxValue) const; + std::optional<TSignalAggregations> GetAggregations() const; + + i64* RegisterValue(const i64 zeroValue = 0) { + TGuard<TMutex> g(Mutex); + Values.emplace_back(zeroValue); + return &Values.back(); + } + +public: + TValueAggregationAgent(const TString& signalName, const TCommonCountersOwner& signalsOwner); + + void ResendStatus() const; + + std::shared_ptr<TValueAggregationClient> GetClient(std::shared_ptr<TValueAggregationAgent> selfPtr); +}; + +} diff --git a/ydb/core/tx/columnshard/counters/common/client.cpp b/ydb/core/tx/columnshard/counters/common/client.cpp new file mode 100644 index 0000000000..95c400b35e --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/client.cpp @@ -0,0 +1,17 @@ +#include "client.h" +#include "agent.h" + +namespace NKikimr::NColumnShard { + +TValueAggregationClient::TValueAggregationClient(std::shared_ptr<TValueAggregationAgent> owner) + : Owner(owner) + , ValuePtr(Owner->RegisterValue(0)) +{ + +} + +void TValueAggregationClient::Set(const i64 value) const { + *ValuePtr = value; +} + +} diff --git a/ydb/core/tx/columnshard/counters/common/client.h b/ydb/core/tx/columnshard/counters/common/client.h new file mode 100644 index 0000000000..0c8f5a0107 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/client.h @@ -0,0 +1,18 @@ +#pragma once +#include <memory> +#include <util/system/types.h> + +namespace NKikimr::NColumnShard { +class TValueAggregationAgent; + +class TValueAggregationClient { +private: + std::shared_ptr<TValueAggregationAgent> Owner; + i64* ValuePtr = nullptr; +public: + TValueAggregationClient(std::shared_ptr<TValueAggregationAgent> owner); + + void Set(const i64 value) const; +}; + +} diff --git a/ydb/core/tx/columnshard/counters/common/owner.cpp b/ydb/core/tx/columnshard/counters/common/owner.cpp new file mode 100644 index 0000000000..d674fcf231 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/owner.cpp @@ -0,0 +1,43 @@ +#include "owner.h" +#include "private.h" +#include <ydb/core/base/appdata.h> +#include <ydb/core/base/counters.h> + +namespace NKikimr::NColumnShard { + +NMonitoring::TDynamicCounters::TCounterPtr TCommonCountersOwner::GetDeriviative(const TString& name) const { + return SubGroup->GetCounter(NormalizeSignalName(ModuleId + "/Deriviative/" + name), true); +} + +NMonitoring::TDynamicCounters::TCounterPtr TCommonCountersOwner::GetValue(const TString& name) const { + return SubGroup->GetCounter(NormalizeSignalName(ModuleId + "/Value/" + name), false); +} + +TString TCommonCountersOwner::NormalizeSignalName(const TString& name) const { + return TFsPath(name).Fix().GetPath(); +} + +TCommonCountersOwner::TCommonCountersOwner(const TString& module) + : ModuleId(module) +{ + if (NActors::TlsActivationContext) { + SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard"); + } else { + SubGroup = new NMonitoring::TDynamicCounters(); + } +} + +NMonitoring::THistogramPtr TCommonCountersOwner::GetHistogram(const TString& name, NMonitoring::IHistogramCollectorPtr&& hCollector) const { + return SubGroup->GetHistogram(NormalizeSignalName(ModuleId + "/Histogram/" + name), std::move(hCollector)); +} + +std::shared_ptr<TValueAggregationAgent> TCommonCountersOwner::GetValueAutoAggregations(const TString& name) const { + return NPrivate::TAggregationsController::GetAggregation(name, *this); +} + +std::shared_ptr<TValueAggregationClient> TCommonCountersOwner::GetValueAutoAggregationsClient(const TString& name) const { + std::shared_ptr<TValueAggregationAgent> agent = NPrivate::TAggregationsController::GetAggregation(name, *this); + return std::make_shared<TValueAggregationClient>(agent); +} + +} diff --git a/ydb/core/tx/columnshard/counters/common/owner.h b/ydb/core/tx/columnshard/counters/common/owner.h new file mode 100644 index 0000000000..172e94860b --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/owner.h @@ -0,0 +1,27 @@ +#pragma once +#include "agent.h" +#include "client.h" + +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <util/system/mutex.h> +#include <deque> + +namespace NKikimr::NColumnShard { + +class TCommonCountersOwner { +private: + friend class TValueAggregationAgent; + ::NMonitoring::TDynamicCounterPtr SubGroup; + const TString ModuleId; + TString NormalizeSignalName(const TString& name) const; +protected: + NMonitoring::TDynamicCounters::TCounterPtr GetDeriviative(const TString& name) const; + NMonitoring::THistogramPtr GetHistogram(const TString& name, NMonitoring::IHistogramCollectorPtr&& hCollector) const; + NMonitoring::TDynamicCounters::TCounterPtr GetValue(const TString& name) const; + std::shared_ptr<TValueAggregationAgent> GetValueAutoAggregations(const TString& name) const; + std::shared_ptr<TValueAggregationClient> GetValueAutoAggregationsClient(const TString& name) const; +public: + TCommonCountersOwner(const TString& module); +}; + +} diff --git a/ydb/core/tx/columnshard/counters/common/private.cpp b/ydb/core/tx/columnshard/counters/common/private.cpp new file mode 100644 index 0000000000..2c6d0c92c9 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/private.cpp @@ -0,0 +1,60 @@ +#include "private.h" +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> + +namespace NKikimr::NColumnShard::NPrivate { +namespace { +class TRegularSignalBuilderActor: public NActors::TActorBootstrapped<TRegularSignalBuilderActor> { +private: + std::shared_ptr<TValueAggregationAgent> Agent; + + void Handle(NActors::TEvents::TEvWakeup::TPtr& /*ev*/) { + Agent->ResendStatus(); + Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup); + } +public: + TRegularSignalBuilderActor(std::shared_ptr<TValueAggregationAgent> agent) + : Agent(agent) + { + + } + + void Bootstrap() { + Agent->ResendStatus(); + Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup); + Become(&TRegularSignalBuilderActor::StateMain); + } + + STATEFN(StateMain) { + switch (ev->GetTypeRewrite()) { + hFunc(NActors::TEvents::TEvWakeup, Handle); + default: + break; + } + } +}; +} + + +class TAggregationsControllerImpl { +private: + TMutex Mutex; + THashMap<TString, std::shared_ptr<TValueAggregationAgent>> Agents; +public: + std::shared_ptr<TValueAggregationAgent> GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner) { + TGuard<TMutex> g(Mutex); + auto it = Agents.find(signalName); + if (it == Agents.end()) { + it = Agents.emplace(signalName, std::make_shared<TValueAggregationAgent>(signalName, signalsOwner)).first; + NActors::TActivationContext::Register(new TRegularSignalBuilderActor(it->second)); + } + return it->second; + } +}; + +std::shared_ptr<TValueAggregationAgent> TAggregationsController::GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner) { + return Singleton<TAggregationsControllerImpl>()->GetAggregation(signalName, signalsOwner); +} + +} diff --git a/ydb/core/tx/columnshard/counters/common/private.h b/ydb/core/tx/columnshard/counters/common/private.h new file mode 100644 index 0000000000..e07082593e --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common/private.h @@ -0,0 +1,12 @@ +#pragma once +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include "owner.h" +#include "agent.h" + +namespace NKikimr::NColumnShard::NPrivate { +class TAggregationsController { +public: + static std::shared_ptr<TValueAggregationAgent> GetAggregation(const TString& signalName, const TCommonCountersOwner& signalsOwner); +}; + +} diff --git a/ydb/core/tx/columnshard/counters/common_data.cpp b/ydb/core/tx/columnshard/counters/common_data.cpp new file mode 100644 index 0000000000..9f72a45171 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common_data.cpp @@ -0,0 +1,19 @@ +#include "common_data.h" + +namespace NKikimr::NColumnShard { + +TDataOwnerSignals::TDataOwnerSignals(const TString& module, const TString dataName) + : TBase(module) + , DataName(dataName) +{ + AddCount = GetDeriviative(DataName + "/Add/Count"); + AddBytes = GetDeriviative(DataName + "/Add/Bytes"); + EraseCount = GetDeriviative(DataName + "/Erase/Count"); + EraseBytes = GetDeriviative(DataName + "/Erase/Bytes"); + SkipAddCount = GetDeriviative(DataName + "/SkipAdd/Count"); + SkipAddBytes = GetDeriviative(DataName + "/SkipAdd/Bytes"); + SkipEraseCount = GetDeriviative(DataName + "/SkipErase/Count"); + SkipEraseBytes = GetDeriviative(DataName + "/SkipErase/Bytes"); +} + +} diff --git a/ydb/core/tx/columnshard/counters/common_data.h b/ydb/core/tx/columnshard/counters/common_data.h new file mode 100644 index 0000000000..b66a4aab2f --- /dev/null +++ b/ydb/core/tx/columnshard/counters/common_data.h @@ -0,0 +1,44 @@ +#pragma once +#include "common/owner.h" + +namespace NKikimr::NColumnShard { + +class TDataOwnerSignals: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + const TString DataName; + NMonitoring::TDynamicCounters::TCounterPtr AddCount; + NMonitoring::TDynamicCounters::TCounterPtr AddBytes; + NMonitoring::TDynamicCounters::TCounterPtr SkipAddCount; + NMonitoring::TDynamicCounters::TCounterPtr SkipAddBytes; + + NMonitoring::TDynamicCounters::TCounterPtr EraseCount; + NMonitoring::TDynamicCounters::TCounterPtr EraseBytes; + NMonitoring::TDynamicCounters::TCounterPtr SkipEraseCount; + NMonitoring::TDynamicCounters::TCounterPtr SkipEraseBytes; +public: + TDataOwnerSignals(const TString& module, const TString dataName); + + void Add(const ui64 size) const { + AddCount->Add(1); + AddBytes->Add(size); + } + + void Erase(const ui64 size) const { + EraseCount->Add(1); + EraseBytes->Add(size); + } + + void SkipAdd(const ui64 size) const { + SkipAddCount->Add(1); + SkipAddBytes->Add(size); + } + + void SkipErase(const ui64 size) const { + SkipEraseCount->Add(1); + SkipEraseBytes->Add(size); + } + +}; + +} diff --git a/ydb/core/tx/columnshard/counters/engine_logs.cpp b/ydb/core/tx/columnshard/counters/engine_logs.cpp index b855fcff8c..cc208742fa 100644 --- a/ydb/core/tx/columnshard/counters/engine_logs.cpp +++ b/ydb/core/tx/columnshard/counters/engine_logs.cpp @@ -4,18 +4,26 @@ namespace NKikimr::NColumnShard { -TEngineLogsCounters::TEngineLogsCounters() { - const TString module = "EngineLogs"; - if (NActors::TlsActivationContext) { - SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard"); - } else { - SubGroup = new NMonitoring::TDynamicCounters(); - } - OverloadGranules = SubGroup->GetCounter(module + "/Granules/Overload", false); - CompactOverloadGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Overload/Count", true); - NoCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/No/Count", true); - SplitCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Split/Count", true); - InternalCompactGranulesSelection = SubGroup->GetCounter(module + "/Granules/Selection/Internal/Count", true); +TEngineLogsCounters::TEngineLogsCounters() + : TBase("EngineLogs") +{ + OverloadGranules = TBase::GetValue("Granules/Overload"); + CompactOverloadGranulesSelection = TBase::GetDeriviative("Granules/Selection/Overload/Count"); + NoCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/No/Count"); + SplitCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/Split/Count"); + InternalCompactGranulesSelection = TBase::GetDeriviative("Granules/Selection/Internal/Count"); + + PortionToDropCount = TBase::GetDeriviative("Ttl/PortionToDrop/Count"); + PortionToDropBytes = TBase::GetDeriviative("Ttl/PortionToDrop/Bytes"); + + PortionToEvictCount = TBase::GetDeriviative("Ttl/PortionToEvict/Count"); + PortionToEvictBytes = TBase::GetDeriviative("Ttl/PortionToEvict/Bytes"); + + PortionNoTtlColumnCount = TBase::GetDeriviative("Ttl/PortionNoTtlColumn/Count"); + PortionNoTtlColumnBytes = TBase::GetDeriviative("Ttl/PortionNoTtlColumn/Bytes"); + + PortionNoBorderCount = TBase::GetDeriviative("Ttl/PortionNoBorder/Count"); + PortionNoBorderBytes = TBase::GetDeriviative("Ttl/PortionNoBorder/Bytes"); } } diff --git a/ydb/core/tx/columnshard/counters/engine_logs.h b/ydb/core/tx/columnshard/counters/engine_logs.h index 7774f3c3aa..6beb40308a 100644 --- a/ydb/core/tx/columnshard/counters/engine_logs.h +++ b/ydb/core/tx/columnshard/counters/engine_logs.h @@ -1,11 +1,23 @@ #pragma once #include <library/cpp/monlib/dynamic_counters/counters.h> +#include "common/owner.h" namespace NKikimr::NColumnShard { -class TEngineLogsCounters { +class TEngineLogsCounters: public TCommonCountersOwner { private: - ::NMonitoring::TDynamicCounterPtr SubGroup; + using TBase = TCommonCountersOwner; + NMonitoring::TDynamicCounters::TCounterPtr PortionToDropCount; + NMonitoring::TDynamicCounters::TCounterPtr PortionToDropBytes; + + NMonitoring::TDynamicCounters::TCounterPtr PortionToEvictCount; + NMonitoring::TDynamicCounters::TCounterPtr PortionToEvictBytes; + + NMonitoring::TDynamicCounters::TCounterPtr PortionNoTtlColumnCount; + NMonitoring::TDynamicCounters::TCounterPtr PortionNoTtlColumnBytes; + + NMonitoring::TDynamicCounters::TCounterPtr PortionNoBorderCount; + NMonitoring::TDynamicCounters::TCounterPtr PortionNoBorderBytes; public: NMonitoring::TDynamicCounters::TCounterPtr OverloadGranules; NMonitoring::TDynamicCounters::TCounterPtr CompactOverloadGranulesSelection; @@ -13,6 +25,26 @@ public: NMonitoring::TDynamicCounters::TCounterPtr SplitCompactGranulesSelection; NMonitoring::TDynamicCounters::TCounterPtr InternalCompactGranulesSelection; + void OnPortionToEvict(const ui64 size) const { + PortionToEvictCount->Add(1); + PortionToEvictBytes->Add(size); + } + + void OnPortionToDrop(const ui64 size) const { + PortionToDropCount->Add(1); + PortionToDropBytes->Add(size); + } + + void OnPortionNoTtlColumn(const ui64 size) const { + PortionNoTtlColumnCount->Add(1); + PortionNoTtlColumnBytes->Add(size); + } + + void OnPortionNoBorder(const ui64 size) const { + PortionNoBorderCount->Add(1); + PortionNoBorderBytes->Add(size); + } + TEngineLogsCounters(); }; diff --git a/ydb/core/tx/columnshard/counters/indexation.cpp b/ydb/core/tx/columnshard/counters/indexation.cpp index 0403832c75..705c0a26ba 100644 --- a/ydb/core/tx/columnshard/counters/indexation.cpp +++ b/ydb/core/tx/columnshard/counters/indexation.cpp @@ -4,43 +4,42 @@ namespace NKikimr::NColumnShard { -TIndexationCounters::TIndexationCounters(const TString& module) { - if (NActors::TlsActivationContext) { - SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard"); - } else { - SubGroup = new NMonitoring::TDynamicCounters(); - } - ReadBytes = SubGroup->GetCounter(module + "/Counters/Read/Bytes", true); - AnalizeInsertedPortions = SubGroup->GetCounter(module + "/Counters/AnalizeInsertion/Portions", true); - AnalizeInsertedBytes = SubGroup->GetCounter(module + "/Counters/AnalizeInsertion/Bytes", true); - RepackedInsertedPortions = SubGroup->GetCounter(module + "/Counters/RepackedInsertion/Portions", true); - RepackedInsertedPortionBytes = SubGroup->GetCounter(module + "/Counters/RepackedInsertion/Bytes", true); +TIndexationCounters::TIndexationCounters(const TString& module) + : TBase(module) +{ + ReadBytes = TBase::GetDeriviative("Read/Bytes"); + AnalizeInsertedPortions = TBase::GetDeriviative("AnalizeInsertion/Portions"); + AnalizeInsertedBytes = TBase::GetDeriviative("AnalizeInsertion/Bytes"); + RepackedInsertedPortions = TBase::GetDeriviative("RepackedInsertion/Portions"); + RepackedInsertedPortionBytes = TBase::GetDeriviative("RepackedInsertion/Bytes"); - AnalizeCompactedPortions = SubGroup->GetCounter(module + "/Counters/AnalizeCompaction/Portions", true); - AnalizeCompactedBytes = SubGroup->GetCounter(module + "/Counters/AnalizeCompaction/Bytes", true); - SkipPortionsMoveThroughIntersection = SubGroup->GetCounter(module + "/Counters/SkipMoveThroughIntersection/Portions", true); - SkipPortionBytesMoveThroughIntersection = SubGroup->GetCounter(module + "/Counters/SkipMoveThroughIntersection/Bytes", true); - RepackedCompactedPortions = SubGroup->GetCounter(module + "/Counters/RepackedCompaction/Portions", true); - MovedPortions = SubGroup->GetCounter(module + "/Counters/Moved/Portions", true); - MovedPortionBytes = SubGroup->GetCounter(module + "/Counters/Moved/Bytes", true); + AnalizeCompactedPortions = TBase::GetDeriviative("AnalizeCompaction/Portions"); + AnalizeCompactedBytes = TBase::GetDeriviative("AnalizeCompaction/Bytes"); + SkipPortionsMoveThroughIntersection = TBase::GetDeriviative("SkipMoveThroughIntersection/Portions"); + SkipPortionBytesMoveThroughIntersection = TBase::GetDeriviative("SkipMoveThroughIntersection/Bytes"); + RepackedCompactedPortions = TBase::GetDeriviative("RepackedCompaction/Portions"); + MovedPortions = TBase::GetDeriviative("Moved/Portions"); + MovedPortionBytes = TBase::GetDeriviative("Moved/Bytes"); - TrashDataSerializationBytes = SubGroup->GetCounter(module + "/Counters/TrashDataSerialization/Bytes", true); - TrashDataSerialization = SubGroup->GetCounter(module + "/Counters/TrashDataSerialization/Count", true); - TrashDataSerializationHistogramBytes = SubGroup->GetHistogram(module + "/Histogram/TrashDataSerialization/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1024)); - CorrectDataSerializationBytes = SubGroup->GetCounter(module + "/Counters/CorrectDataSerialization/Bytes", true); - CorrectDataSerialization = SubGroup->GetCounter(module + "/Counters/CorrectDataSerialization/Count", true); + TrashDataSerializationBytes = TBase::GetDeriviative("TrashDataSerialization/Bytes"); + TrashDataSerialization = TBase::GetDeriviative("TrashDataSerialization/Count"); + TrashDataSerializationHistogramBytes = TBase::GetHistogram("TrashDataSerialization/Bytes", NMonitoring::ExponentialHistogram(15, 2, 1024)); + CorrectDataSerializationBytes = TBase::GetDeriviative("CorrectDataSerialization/Bytes"); + CorrectDataSerialization = TBase::GetDeriviative("CorrectDataSerialization/Count"); - CompactionDuration = SubGroup->GetHistogram(module + "/Histogram/CompactionDuration", NMonitoring::ExponentialHistogram(18, 2, 20)); - CompactionInputSize = SubGroup->GetHistogram(module + "/Histogram/CompactionInputSize", NMonitoring::ExponentialHistogram(18, 2, 1024)); - CompactionExceptions = SubGroup->GetCounter(module + "/Counters/Exceptions/Count", true); - CompactionFails = SubGroup->GetCounter(module + "/Counters/CompactionFails/Count", true); + CompactionDuration = TBase::GetHistogram("CompactionDuration", NMonitoring::ExponentialHistogram(18, 2, 20)); + HistogramCompactionInputBytes = TBase::GetHistogram("CompactionInput/Bytes", NMonitoring::ExponentialHistogram(18, 2, 1024)); + CompactionInputBytes = TBase::GetDeriviative("CompactionInput/Bytes"); + CompactionInputSize = TBase::GetHistogram("CompactionInput/Bytes", NMonitoring::ExponentialHistogram(18, 2, 1024)); + CompactionExceptions = TBase::GetDeriviative("Exceptions/Count"); + CompactionFails = TBase::GetDeriviative("CompactionFails/Count"); - SplittedPortionLargestColumnSize = SubGroup->GetHistogram(module + "/Histogram/SplittedPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); - SplittedPortionColumnSize = SubGroup->GetHistogram(module + "/Histogram/SplittedPortionColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); - SimpleSplitPortionLargestColumnSize = SubGroup->GetHistogram(module + "/Histogram/SimpleSplitPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); - TooSmallBlob = SubGroup->GetCounter(module + "/Counters/TooSmallBlob/Count", true); - TooSmallBlobFinish = SubGroup->GetCounter(module + "/Counters/TooSmallBlobFinish/Count", true); - TooSmallBlobStart = SubGroup->GetCounter(module + "/Counters/TooSmallBlobStart/Count", true); + SplittedPortionLargestColumnSize = TBase::GetHistogram("SplittedPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); + SplittedPortionColumnSize = TBase::GetHistogram("SplittedPortionColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); + SimpleSplitPortionLargestColumnSize = TBase::GetHistogram("SimpleSplitPortionLargestColumnSize", NMonitoring::ExponentialHistogram(15, 2, 1024)); + TooSmallBlob = TBase::GetDeriviative("TooSmallBlob/Count"); + TooSmallBlobFinish = TBase::GetDeriviative("TooSmallBlobFinish/Count"); + TooSmallBlobStart = TBase::GetDeriviative("TooSmallBlobStart/Count"); } } diff --git a/ydb/core/tx/columnshard/counters/indexation.h b/ydb/core/tx/columnshard/counters/indexation.h index fb3bb1ba5f..6a1c1e8842 100644 --- a/ydb/core/tx/columnshard/counters/indexation.h +++ b/ydb/core/tx/columnshard/counters/indexation.h @@ -1,12 +1,16 @@ #pragma once #include <library/cpp/monlib/dynamic_counters/counters.h> +#include "common/owner.h" namespace NKikimr::NColumnShard { -class TIndexationCounters { +class TIndexationCounters: public TCommonCountersOwner { private: - ::NMonitoring::TDynamicCounterPtr SubGroup; + using TBase = TCommonCountersOwner; + NMonitoring::THistogramPtr HistogramCompactionInputBytes; public: + NMonitoring::TDynamicCounters::TCounterPtr CompactionInputBytes; + NMonitoring::TDynamicCounters::TCounterPtr ReadBytes; NMonitoring::TDynamicCounters::TCounterPtr AnalizeCompactedPortions; NMonitoring::TDynamicCounters::TCounterPtr AnalizeInsertedPortions; @@ -39,6 +43,11 @@ public: NMonitoring::TDynamicCounters::TCounterPtr CompactionFails; TIndexationCounters(const TString& module); + +// void CompactionInputSize(const ui64 size) const { +// HistogramCompactionInputBytes->Collect(size); +// CompactionInputBytes->Add(size); +// } }; } diff --git a/ydb/core/tx/columnshard/counters/insert_table.cpp b/ydb/core/tx/columnshard/counters/insert_table.cpp new file mode 100644 index 0000000000..f7c5446dab --- /dev/null +++ b/ydb/core/tx/columnshard/counters/insert_table.cpp @@ -0,0 +1,15 @@ +#include "insert_table.h" + +namespace NKikimr::NColumnShard { + +TInsertTableCounters::TInsertTableCounters() + : TBase("InsertTable") + , PathIdInserted("InsertTable", "Inserted") + , PathIdCommitted("InsertTable", "Committed") + , Inserted("InsertTable", "Inserted") + , Committed("InsertTable", "Committed") + , Aborted("InsertTable", "Aborted") +{ +} + +} diff --git a/ydb/core/tx/columnshard/counters/insert_table.h b/ydb/core/tx/columnshard/counters/insert_table.h new file mode 100644 index 0000000000..30dee6ba64 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/insert_table.h @@ -0,0 +1,71 @@ +#pragma once +#include "common_data.h" +#include "common/client.h" +#include "common/agent.h" + +namespace NKikimr::NColumnShard { + +class TPathIdClientCounters { +private: + using TBase = TCommonCountersOwner; + const std::shared_ptr<TValueAggregationClient> PathIdCommittedBytes; + const std::shared_ptr<TValueAggregationClient> PathIdCommittedChunks; +public: + TPathIdClientCounters(std::shared_ptr<TValueAggregationClient> pathIdCommittedBytes, std::shared_ptr<TValueAggregationClient> pathIdCommittedChunks) + : PathIdCommittedBytes(pathIdCommittedBytes) + , PathIdCommittedChunks(pathIdCommittedChunks) { + } + + void OnPathIdDataInfo(const ui64 bytes, const ui32 chunks) const { + PathIdCommittedBytes->Set(bytes); + PathIdCommittedChunks->Set(chunks); + } +}; + +class TPathIdInfoCounters: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + const std::shared_ptr<TValueAggregationAgent> PathIdCommittedBytes; + const std::shared_ptr<TValueAggregationAgent> PathIdCommittedChunks; +public: + TPathIdInfoCounters(const TString& countersName, const TString& dataName) + : TBase(countersName) + , PathIdCommittedBytes(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Bytes")) + , PathIdCommittedChunks(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Chunks")) + { + } + + TPathIdClientCounters GetClient() const { + return TPathIdClientCounters(PathIdCommittedBytes->GetClient(PathIdCommittedBytes), PathIdCommittedChunks->GetClient(PathIdCommittedChunks)); + } +}; + +class TPathIdOwnedCounters { +public: + const TPathIdClientCounters Inserted; + const TPathIdClientCounters Committed; + TPathIdOwnedCounters(TPathIdClientCounters&& inserted, TPathIdClientCounters&& committed) + : Inserted(std::move(inserted)) + , Committed(std::move(committed)) { + + } +}; + +class TInsertTableCounters: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + const TPathIdInfoCounters PathIdInserted; + const TPathIdInfoCounters PathIdCommitted; +public: + const TDataOwnerSignals Inserted; + const TDataOwnerSignals Committed; + const TDataOwnerSignals Aborted; + + TInsertTableCounters(); + + TPathIdOwnedCounters GetPathIdCounters() const { + return TPathIdOwnedCounters(PathIdInserted.GetClient(), PathIdCommitted.GetClient()); + } +}; + +} |