diff options
| author | babenko <[email protected]> | 2026-06-24 13:14:54 +0300 |
|---|---|---|
| committer | babenko <[email protected]> | 2026-06-24 14:20:07 +0300 |
| commit | a224436a8d395602cd14b8e5aa3cecb466ba126f (patch) | |
| tree | ff0414f872bd9fc09da53ee4e2ea5610227ecd55 /library/cpp | |
| parent | f0f748358580ca75f9aedce7d9a6572b0c8f4c58 (diff) | |
YT-28458: Add rseq-backed hot sensors
Rename `percpu.{cpp,h}` to `per_cpu.{cpp,h}` and update all includers.
Add an rseq-backed implementation of the hot per-CPU counter, time counter and
gauge (`rseq_sensor_impl.{h,cpp}`): each update commits to the calling CPU's shard
lock-free via an rseq critical section (`library/cpp/yt/rseq`
`AddPerCpu`/`StorePerCpu`) — no atomic and no lock on the fast path. The shard
array is sized to `NRseq::GetCpuCount()` and folded into the sensor allocation
(`NewWithExtraSpace`); reads aggregate with `LoadPerCpu`. The gauge keeps
last-writer-wins semantics. Linux-only.
The existing `TPerCpu{Counter,TimeCounter,Gauge}` stay (`per_cpu_sensor_impl.{h,cpp}`)
as the atomic sharded fallback. The two are interchangeable and chosen per sensor
at construction in `TSolomonRegistry`, so the hot Increment/Update path carries no
dispatch:
* The rseq fast path is **off by default**; opt in via
`singletons/solomon_registry/enable_rseq`. Even when on, a hot sensor uses it
only in a process where the kernel rseq area sits at a fixed thread-pointer
offset (tcmalloc/glibc-owned), per the rseq library's runtime safety probe
(`NRseq::IsPerCpuFastPathSafe`). Everything else — notably a `dlopen`'d YQL UDF
whose `__rseq_abi` lands in dynamically allocated TLS — uses the atomic sharded
sensors.
* `TSolomonRegistry` is a reconfigurable singleton (`solomon_registry`) with an
`enable_rseq` knob (default false), settable in static config and updatable via
dynamic config.
Off Linux the rseq sensors do not exist and the registry uses the atomic sharded
sensors for hot requests. The per-CPU summary is unchanged (TTscp + spinlock).
Unit tests cover the atomic sensors, the rseq sensors (Linux-only), and the simple
sensors. The controller-agent memory-watchdog integration tests are made resilient
to the (core-count-dependent) per-sensor footprint.
Benchmark (hot per-CPU path, 64-core host), atomic sharded vs rseq:
| Benchmark | atomic | rseq | speedup |
| --- | --- | --- | --- |
| BM_PerCpuCounter, threads:1 | 30.7 ns | 3.6 ns | ~8.5x |
| BM_PerCpuCounter, threads:16 | 30.9 ns | 4.4 ns | ~7x |
| BM_PerCpuGauge, threads:1 | 32.8 ns | 12.4 ns | ~2.6x |
| BM_PerCpuGauge, threads:16 | 32.7 ns | 12.5 ns | ~2.5x |
commit_hash:8c633d31f03b2cc862ed2217ae08342bf42adc52
Diffstat (limited to 'library/cpp')
| -rw-r--r-- | library/cpp/yt/rseq/rseq.cpp | 8 | ||||
| -rw-r--r-- | library/cpp/yt/rseq/rseq.h | 20 | ||||
| -rw-r--r-- | library/cpp/yt/rseq/unittests/per_cpu_ut.cpp | 7 |
3 files changed, 26 insertions, 9 deletions
diff --git a/library/cpp/yt/rseq/rseq.cpp b/library/cpp/yt/rseq/rseq.cpp index f8321cbb0bf..12b5a291d8d 100644 --- a/library/cpp/yt/rseq/rseq.cpp +++ b/library/cpp/yt/rseq/rseq.cpp @@ -133,7 +133,7 @@ YT_STATIC_INITIALIZER({ // the static TLS block, incl. tcmalloc) rather than a dlopen'd module's dynamically allocated // TLS, where the offset is valid only on the thread that computed it. Compares addresses // without dereferencing the suspect offset, so it is safe even when the offset is bogus. See -// IsPerCpuFastPathSafe. +// IsPerCpuFastPathSupported. YT_PREVENT_TLS_CACHING bool ValidateFastPathOnFreshThread() { if (OwnsRegistration) { @@ -177,7 +177,7 @@ YT_PREVENT_TLS_CACHING bool EnsureCurrentThreadRegistered() return ReadField<int>(CpuIdFieldOffset) >= 0; } -bool IsPerCpuFastPathSafe() +bool IsPerCpuFastPathSupported() { // Decided once, lazily, on a freshly spawned thread (the check is meaningful only off the // thread that computed the offset) and cached -- cost is one thread spawn at first use. @@ -205,14 +205,12 @@ bool IsPerCpuFastPathSafe() #else // YT_RSEQ_AVAILABLE -#include "per_cpu.h" - namespace NYT::NRseq { //////////////////////////////////////////////////////////////////////////////// // No rseq fast path on this platform; hot sensors use the atomic fallback. -bool IsPerCpuFastPathSafe() +bool IsPerCpuFastPathSupported() { return false; } diff --git a/library/cpp/yt/rseq/rseq.h b/library/cpp/yt/rseq/rseq.h index cfea5101c29..b821e34047a 100644 --- a/library/cpp/yt/rseq/rseq.h +++ b/library/cpp/yt/rseq/rseq.h @@ -6,6 +6,24 @@ #define YT_RSEQ_AVAILABLE #endif +namespace NYT::NRseq { + +//////////////////////////////////////////////////////////////////////////////// + +//! Returns whether the per-CPU rseq fast path is safe to use in this process. +/*! + * The fast path reads the rseq area at a thread-pointer offset cached at startup, which is + * sound only when __rseq_abi sits at a fixed offset from the thread pointer (a glibc-owned + * area or the static TLS block, incl. tcmalloc) -- not when it lands in a dlopen'd module's + * dynamically allocated TLS. Returns false there (and where there is no fast path) so callers + * fall back to atomics. Decided once on a spawned thread and cached: one spawn at first use. + */ +bool IsPerCpuFastPathSupported(); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NRseq + #ifdef YT_RSEQ_AVAILABLE #include <cstddef> @@ -16,7 +34,7 @@ namespace NYT::NRseq { //! Byte offset from the thread pointer to the rseq area's cpu_id field (glibc's area when //! glibc registers rseq, otherwise our own). A fixed offset across threads only when the area -//! is glibc-owned or in the static TLS block; #IsPerCpuFastPathSafe probes this and gates the +//! is glibc-owned or in the static TLS block; #IsPerCpuFastPathSupported probes this and gates the //! fast path. NB: 0 until our startup initializer runs, but nothing reads rseq before then. extern std::ptrdiff_t CpuIdFieldOffset; diff --git a/library/cpp/yt/rseq/unittests/per_cpu_ut.cpp b/library/cpp/yt/rseq/unittests/per_cpu_ut.cpp index f742c8589de..27588aee75e 100644 --- a/library/cpp/yt/rseq/unittests/per_cpu_ut.cpp +++ b/library/cpp/yt/rseq/unittests/per_cpu_ut.cpp @@ -1,6 +1,7 @@ #include <library/cpp/testing/gtest/gtest.h> #include <library/cpp/yt/rseq/per_cpu.h> +#include <library/cpp/yt/rseq/rseq.h> #include <library/cpp/yt/memory/public.h> @@ -58,12 +59,12 @@ TEST(TPerCpuRseqTest, CpuCountIsSane) EXPECT_LE(GetCpuCount(), 1 << 20); } -TEST(TPerCpuRseqTest, FastPathSafetyIsStable) +TEST(TPerCpuRseqTest, FastPathSupportIsStable) { // The probe spawns a thread on first use and caches its verdict, so repeated calls must // agree. We avoid asserting a specific value: it depends on kernel rseq support. - bool safe = IsPerCpuFastPathSafe(); - EXPECT_EQ(safe, IsPerCpuFastPathSafe()); + bool supported = IsPerCpuFastPathSupported(); + EXPECT_EQ(supported, IsPerCpuFastPathSupported()); } TEST(TPerCpuRseqTest, ParsePossibleCpuCount) |
