Add sensors about AS,

5 files changed, 48 insertions, 13 deletions

diff --git a/library/cpp/actors/core/executor_pool_basic.cpp b/library/cpp/actors/core/executor_pool_basic.cpp
index 5d0ecfa079..00e557fcb4 100644
--- a/library/cpp/actors/core/executor_pool_basic.cpp
+++ b/library/cpp/actors/core/executor_pool_basic.cpp
@@ -334,9 +334,13 @@ namespace NActors {
         poolStats.WrongWakenedThreadCount = RelaxedLoad(&WrongWakenedThreadCount);
         poolStats.CurrentThreadCount = RelaxedLoad(&ThreadCount);
         if (Harmonizer) {
-            TPoolStateFlags flags = Harmonizer->GetPoolFlags(PoolId);
-            poolStats.IsNeedy = flags.IsNeedy;
-            poolStats.IsStarved = flags.IsStarved;
+            TPoolHarmonizedStats stats = Harmonizer->GetPoolStats(PoolId);
+            poolStats.IsNeedy = stats.IsNeedy;
+            poolStats.IsStarved = stats.IsStarved;
+            poolStats.IsHoggish = stats.IsHoggish;
+            poolStats.IncreasingThreadsByNeedyState = stats.IncreasingThreadsByNeedyState;
+            poolStats.DecreasingThreadsByStarvedState = stats.DecreasingThreadsByStarvedState;
+            poolStats.DecreasingThreadsByHoggishState = stats.DecreasingThreadsByHoggishState;
         }
 
         statsCopy.resize(PoolThreads + 1);
diff --git a/library/cpp/actors/core/harmonizer.cpp b/library/cpp/actors/core/harmonizer.cpp
index 2722958486..f318d8909c 100644
--- a/library/cpp/actors/core/harmonizer.cpp
+++ b/library/cpp/actors/core/harmonizer.cpp
@@ -117,7 +117,10 @@ struct TPoolInfo {
     ui32 MaxAvgPingUs = 0;
     ui64 LastUpdateTs = 0;
 
-    TAtomic LastFlags = 0; // 0 - isNeedy; 1 - isStarved
+    TAtomic LastFlags = 0; // 0 - isNeedy; 1 - isStarved; 2 - isHoggish
+    TAtomic IncreasingThreadsByNeedyState = 0;
+    TAtomic DecreasingThreadsByStarvedState = 0;
+    TAtomic DecreasingThreadsByHoggishState = 0;
 
     bool IsBeingStopped(i16 threadIdx);
     double GetBooked(i16 threadIdx);
@@ -212,7 +215,7 @@ public:
     void DeclareEmergency(ui64 ts) override;
     void AddPool(IExecutorPool* pool, TSelfPingInfo *pingInfo) override;
     void Enable(bool enable) override;
-    TPoolStateFlags GetPoolFlags(i16 poolId) const override;
+    TPoolHarmonizedStats GetPoolStats(i16 poolId) const override;
 };
 
 THarmonizer::THarmonizer(ui64 ts) {
@@ -263,7 +266,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) {
             poolConsumed += Rescale(pool.GetConsumed(threadIdx));
             lastSecondPoolConsumed += Rescale(pool.GetlastSecondPoolConsumed(threadIdx));
         }
-        bool isStarved = IsStarved(consumed, booked) || IsStarved(lastSecondPoolConsumed, lastSecondPoolBooked);
+        bool isStarved = IsStarved(poolConsumed, poolBooked) || IsStarved(lastSecondPoolConsumed, lastSecondPoolBooked);
         if (isStarved) {
             isStarvedPresent = true;
         }
@@ -286,7 +289,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) {
         }
         booked += poolBooked;
         consumed += poolConsumed;
-        AtomicSet(pool.LastFlags, (i64)isNeedy | ((i64)isStarved << 1));
+        AtomicSet(pool.LastFlags, (i64)isNeedy | ((i64)isStarved << 1) | ((i64)isHoggish << 2));
         LWPROBE(HarmonizeCheckPool, poolIdx, pool.Pool->GetName(), poolBooked, poolConsumed, lastSecondPoolBooked, lastSecondPoolConsumed, pool.GetThreadCount(), pool.MaxThreadCount, isStarved, isNeedy, isHoggish);
     }
     double budget = total - Max(booked, lastSecondBooked);
@@ -310,6 +313,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) {
               i64 threadCount = pool.GetThreadCount();
               if (threadCount > pool.DefaultThreadCount) {
                   pool.SetThreadCount(threadCount - 1);
+                  AtomicIncrement(pool.DecreasingThreadsByStarvedState);
                   overbooked--;
                   LWPROBE(HarmonizeOperation, poolIdx, pool.Pool->GetName(), "decrease", threadCount - 1, pool.DefaultThreadCount, pool.MaxThreadCount);
                   if (overbooked < 1) {
@@ -324,6 +328,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) {
             if (budget >= 1.0) {
                 i64 threadCount = pool.GetThreadCount();
                 if (threadCount + 1 <= pool.MaxThreadCount) {
+                    AtomicIncrement(pool.IncreasingThreadsByNeedyState);
                     pool.SetThreadCount(threadCount + 1);
                     budget -= 1.0;
                     LWPROBE(HarmonizeOperation, needyPoolIdx, pool.Pool->GetName(), "increase", threadCount + 1, pool.DefaultThreadCount, pool.MaxThreadCount);
@@ -335,6 +340,7 @@ void THarmonizer::HarmonizeImpl(ui64 ts) {
         TPoolInfo &pool = Pools[hoggishPoolIdx];
         i64 threadCount = pool.GetThreadCount();
         if (threadCount > pool.MinThreadCount) {
+            AtomicIncrement(pool.DecreasingThreadsByHoggishState);
             LWPROBE(HarmonizeOperation, hoggishPoolIdx, pool.Pool->GetName(), "decrease", threadCount - 1, pool.DefaultThreadCount, pool.MaxThreadCount);
             pool.SetThreadCount(threadCount - 1);
         }
@@ -409,11 +415,16 @@ IHarmonizer* MakeHarmonizer(ui64 ts) {
     return new THarmonizer(ts);
 }
 
-TPoolStateFlags THarmonizer::GetPoolFlags(i16 poolId) const {
-    ui64 flags = RelaxedLoad(&Pools[poolId].LastFlags);
-    return TPoolStateFlags {
+TPoolHarmonizedStats THarmonizer::GetPoolStats(i16 poolId) const {
+    const TPoolInfo &pool = Pools[poolId];
+    ui64 flags = RelaxedLoad(&pool.LastFlags);
+    return TPoolHarmonizedStats {
+        .IncreasingThreadsByNeedyState = static_cast<ui64>(RelaxedLoad(&pool.IncreasingThreadsByNeedyState)),
+        .DecreasingThreadsByStarvedState = static_cast<ui64>(RelaxedLoad(&pool.DecreasingThreadsByStarvedState)),
+        .DecreasingThreadsByHoggishState = static_cast<ui64>(RelaxedLoad(&pool.DecreasingThreadsByHoggishState)),
         .IsNeedy = static_cast<bool>(flags & 1),
-        .IsStarved = static_cast<bool>(flags & 2)
+        .IsStarved = static_cast<bool>(flags & 2),
+        .IsHoggish = static_cast<bool>(flags & 4),
     };
 }
 
diff --git a/library/cpp/actors/core/harmonizer.h b/library/cpp/actors/core/harmonizer.h
index 5ab44369aa..61f13e43ac 100644
--- a/library/cpp/actors/core/harmonizer.h
+++ b/library/cpp/actors/core/harmonizer.h
@@ -6,9 +6,13 @@
 namespace NActors {
     class IExecutorPool;
 
-    struct TPoolStateFlags {
+    struct TPoolHarmonizedStats {
+        ui64 IncreasingThreadsByNeedyState = 0;
+        ui64 DecreasingThreadsByStarvedState = 0;
+        ui64 DecreasingThreadsByHoggishState = 0;
         bool IsNeedy = false;
         bool IsStarved = false;
+        bool IsHoggish = false;
     };
 
     // Pool cpu harmonizer
@@ -19,7 +23,7 @@ namespace NActors {
         virtual void DeclareEmergency(ui64 ts) = 0;
         virtual void AddPool(IExecutorPool* pool, TSelfPingInfo *pingInfo = nullptr) = 0;
         virtual void Enable(bool enable) = 0;
-        virtual TPoolStateFlags GetPoolFlags(i16 poolId) const = 0;
+        virtual TPoolHarmonizedStats GetPoolStats(i16 poolId) const = 0;
     };
 
     IHarmonizer* MakeHarmonizer(ui64 ts);
diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h
index 037f4bc9d8..117d2ad41d 100644
--- a/library/cpp/actors/core/mon_stats.h
+++ b/library/cpp/actors/core/mon_stats.h
@@ -60,10 +60,14 @@ namespace NActors {
 
     struct TExecutorPoolStats {
         ui64 MaxUtilizationTime = 0;
+        ui64 IncreasingThreadsByNeedyState = 0;
+        ui64 DecreasingThreadsByStarvedState = 0;
+        ui64 DecreasingThreadsByHoggishState = 0;
         i16 WrongWakenedThreadCount = 0;
         i16 CurrentThreadCount = 0;
         bool IsNeedy = false;
         bool IsStarved = false;
+        bool IsHoggish = false;
     };
 
     struct TExecutorThreadStats {
diff --git a/library/cpp/actors/helpers/pool_stats_collector.h b/library/cpp/actors/helpers/pool_stats_collector.h
index 2241abc70c..b1217b1d63 100644
--- a/library/cpp/actors/helpers/pool_stats_collector.h
+++ b/library/cpp/actors/helpers/pool_stats_collector.h
@@ -128,6 +128,10 @@ private:
         NMonitoring::TDynamicCounters::TCounterPtr CurrentThreadCount;
         NMonitoring::TDynamicCounters::TCounterPtr IsNeedy;
         NMonitoring::TDynamicCounters::TCounterPtr IsStarved;
+        NMonitoring::TDynamicCounters::TCounterPtr IsHoggish;
+        NMonitoring::TDynamicCounters::TCounterPtr IncreasingThreadsByNeedyState;
+        NMonitoring::TDynamicCounters::TCounterPtr DecreasingThreadsByStarvedState;
+        NMonitoring::TDynamicCounters::TCounterPtr DecreasingThreadsByHoggishState;
 
 
         THistogramCounters LegacyActivationTimeHistogram;
@@ -176,6 +180,10 @@ private:
             CurrentThreadCount = PoolGroup->GetCounter("CurrentThreadCount", false);
             IsNeedy = PoolGroup->GetCounter("IsNeedy", false);
             IsStarved = PoolGroup->GetCounter("IsStarved", false);
+            IsHoggish = PoolGroup->GetCounter("IsHoggish", false);
+            IncreasingThreadsByNeedyState = PoolGroup->GetCounter("IncreasingThreadsByNeedyState", true);
+            DecreasingThreadsByStarvedState = PoolGroup->GetCounter("DecreasingThreadsByStarvedState", true);
+            DecreasingThreadsByHoggishState = PoolGroup->GetCounter("DecreasingThreadsByHoggishState", true);
 
             LegacyActivationTimeHistogram.Init(PoolGroup.Get(), "ActivationTime", "usec", 5*1000*1000);
             ActivationTimeHistogram = PoolGroup->GetHistogram(
@@ -216,6 +224,10 @@ private:
             *CurrentThreadCount = poolStats.CurrentThreadCount;
             *IsNeedy = poolStats.IsNeedy;
             *IsStarved = poolStats.IsStarved;
+            *IsHoggish = poolStats.IsHoggish;
+            *IncreasingThreadsByNeedyState = poolStats.IncreasingThreadsByNeedyState;
+            *DecreasingThreadsByStarvedState = poolStats.DecreasingThreadsByStarvedState;
+            *DecreasingThreadsByHoggishState = poolStats.DecreasingThreadsByHoggishState;
 
             LegacyActivationTimeHistogram.Set(stats.ActivationTimeHistogram);
             ActivationTimeHistogram->Reset();