From 2ab1e1916633d2f5be41e19ee8d2fb3a8dd8523b Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Wed, 21 May 2025 12:21:00 +0300 Subject: correct writing signals (#18572) Co-authored-by: ivanmorozov333 --- ydb/core/tx/columnshard/columnshard__write.cpp | 5 +++-- ydb/core/tx/columnshard/columnshard_impl.h | 11 +---------- ydb/core/tx/columnshard/counters/columnshard.cpp | 20 +++++++++++++++++-- ydb/core/tx/columnshard/counters/columnshard.h | 25 ++++++++++++++++++++---- ydb/core/tx/columnshard/counters/ya.make | 2 +- ydb/core/tx/columnshard/tablet/write_queue.cpp | 1 + 6 files changed, 45 insertions(+), 19 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 7e407387d86..b3144f6dd4c 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -24,6 +24,7 @@ using namespace NTabletFlatExecutor; void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const NEvWrite::TWriteMeta& writeMeta, const ui64 writeSize, const ui64 cookie, std::unique_ptr&& event, const TActorContext& ctx) { Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL); + Counters.GetCSCounters().OnWriteOverload(overloadReason, writeSize); switch (overloadReason) { case EOverloadStatus::Disk: Counters.OnWriteOverloadDisk(); @@ -80,8 +81,8 @@ TColumnShard::EOverloadStatus TColumnShard::CheckOverloadedImmediate(const TInte return EOverloadStatus::Disk; } ui64 txLimit = Settings.OverloadTxInFlight; - const ui64 writesLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetWritingInFlightRequestsCountLimit() : 1000; - const ui64 writesSizeLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetWritingInFlightRequestBytesLimit() : (((ui64)128) << 20); + const ui64 writesLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetWritingInFlightRequestsCountLimit() : 1000000; + const ui64 writesSizeLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetWritingInFlightRequestBytesLimit() : (((ui64)1) << 30); if (txLimit && Executor()->GetStats().TxInFly > txLimit) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "shard_overload")("reason", "tx_in_fly")("sum", Executor()->GetStats().TxInFly)( "limit", txLimit); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index e7624a3cd80..485a4f09c05 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -351,16 +351,7 @@ public: return shift | Counter.Inc(); } - enum class EOverloadStatus { - ShardTxInFly /* "shard_tx" */, - ShardWritesInFly /* "shard_writes" */, - ShardWritesSizeInFly /* "shard_writes_size" */, - InsertTable /* "insert_table" */, - OverloadMetadata /* "overload_metadata" */, - Disk /* "disk" */, - None /* "none" */, - OverloadCompaction /* "overload_compaction" */ - }; + using EOverloadStatus = EOverloadStatus; // For syslocks void IncCounter(NDataShard::ECumulativeCounters counter, ui64 num = 1) const { diff --git a/ydb/core/tx/columnshard/counters/columnshard.cpp b/ydb/core/tx/columnshard/counters/columnshard.cpp index 78a3c978590..edbee47654b 100644 --- a/ydb/core/tx/columnshard/counters/columnshard.cpp +++ b/ydb/core/tx/columnshard/counters/columnshard.cpp @@ -1,6 +1,4 @@ #include "columnshard.h" -#include -#include #include @@ -11,6 +9,13 @@ TCSCounters::TCSCounters() , WritingCounters(std::make_shared(*this)) , Initialization(*this) , TxProgress(*this) { + for (auto&& i : GetEnumAllValues()) { + AFL_VERIFY((ui32)i == WaitingOverloads.size()); + auto overloadCounters = CreateSubGroup("overload_type", ::ToString(i)); + WaitingOverloads.emplace_back(overloadCounters.GetDeriviative("Overload/Waiting/Count")); + WriteOverloadCount.emplace_back(overloadCounters.GetDeriviative("Overload/Write/Count")); + WriteOverloadBytes.emplace_back(overloadCounters.GetDeriviative("Overload/Write/Bytes")); + } StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count"); TooEarlyBackgroundCount = TBase::GetDeriviative("TooEarlyBackground/Count"); SetupCompactionCount = TBase::GetDeriviative("SetupCompaction/Count"); @@ -75,4 +80,15 @@ void TCSCounters::OnFailedWriteResponse(const EWriteFailReason reason) const { it->second->Add(1); } +void TCSCounters::OnWaitingOverload(const EOverloadStatus status) const { + AFL_VERIFY((ui64)status < WaitingOverloads.size()); + WaitingOverloads[(ui64)status]->Inc(); +} + +void TCSCounters::OnWriteOverload(const EOverloadStatus status, const ui32 size) const { + AFL_VERIFY((ui64)status < WriteOverloadCount.size()); + WriteOverloadCount[(ui64)status]->Inc(); + WriteOverloadBytes[(ui64)status]->Add(size); +} + } diff --git a/ydb/core/tx/columnshard/counters/columnshard.h b/ydb/core/tx/columnshard/counters/columnshard.h index 974bc1433bf..d51085a4028 100644 --- a/ydb/core/tx/columnshard/counters/columnshard.h +++ b/ydb/core/tx/columnshard/counters/columnshard.h @@ -2,18 +2,28 @@ #include "initialization.h" #include "tx_progress.h" -#include - -#include #include +#include + #include #include namespace NKikimr::NColumnShard { +enum class EOverloadStatus { + ShardTxInFly /* "shard_tx" */ = 0, + ShardWritesInFly /* "shard_writes" */, + ShardWritesSizeInFly /* "shard_writes_size" */, + InsertTable /* "insert_table" */, + OverloadMetadata /* "overload_metadata" */, + Disk /* "disk" */, + None /* "none" */, + OverloadCompaction /* "overload_compaction" */ +}; + enum class EWriteFailReason { - Disabled /* "disabled" */, + Disabled /* "disabled" */ = 0, PutBlob /* "put_blob" */, LongTxDuplication /* "long_tx_duplication" */, NoTable /* "no_table" */, @@ -117,12 +127,19 @@ private: NMonitoring::TDynamicCounters::TCounterPtr WriteRequests; THashMap FailedWriteRequests; NMonitoring::TDynamicCounters::TCounterPtr SuccessWriteRequests; + std::vector WaitingOverloads; + std::vector WriteOverloadCount; + std::vector WriteOverloadBytes; public: const std::shared_ptr WritingCounters; const TCSInitialization Initialization; TTxProgressCounters TxProgress; + void OnWaitingOverload(const EOverloadStatus status) const; + + void OnWriteOverload(const EOverloadStatus status, const ui32 size) const; + void OnStartWriteRequest() const { WriteRequests->Add(1); } diff --git a/ydb/core/tx/columnshard/counters/ya.make b/ydb/core/tx/columnshard/counters/ya.make index f27b507d885..fd81a293fbf 100644 --- a/ydb/core/tx/columnshard/counters/ya.make +++ b/ydb/core/tx/columnshard/counters/ya.make @@ -26,7 +26,7 @@ PEERDIR( ydb/library/actors/core ) -GENERATE_ENUM_SERIALIZATION(columnshard.h) +GENERATE_ENUM_SERIALIZATION_WITH_HEADER(columnshard.h) GENERATE_ENUM_SERIALIZATION(scan.h) END() diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 75726189c77..93d33460609 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -36,6 +36,7 @@ bool TWriteTasksQueue::Drain(const bool onWakeup, const TActorContext& ctx) { for (auto&& i : WriteTasks) { auto overloadStatus = Owner->CheckOverloadedWait(i.first); if (overloadStatus != TColumnShard::EOverloadStatus::None) { + Owner->Counters.GetCSCounters().OnWaitingOverload(overloadStatus); countTasks += i.second.size(); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "wait_overload")("status", overloadStatus)("path_id", i.first)( "size", i.second.size()); -- cgit v1.3