aboutsummaryrefslogtreecommitdiffstats
path: root/yt
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2023-11-12 21:25:31 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2023-11-12 21:39:54 +0300
commitd28c55ab25cc8cedab8a5f4736c0d66e88b3da95 (patch)
tree73d373709b74fa2baaa4fe02a40a77c0a5baf6b7 /yt
parent35b17f4f3b6e0ed855e7e47d3f1eb57470388a2c (diff)
downloadydb-d28c55ab25cc8cedab8a5f4736c0d66e88b3da95.tar.gz
Intermediate changes
Diffstat (limited to 'yt')
-rw-r--r--yt/yql/plugin/bridge/interface.h13
-rw-r--r--yt/yql/plugin/bridge/plugin.cpp5
-rw-r--r--yt/yql/plugin/dynamic/dylib.exports1
-rw-r--r--yt/yql/plugin/dynamic/impl.cpp16
-rw-r--r--yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt2
-rw-r--r--yt/yql/plugin/native/CMakeLists.linux-aarch64.txt2
-rw-r--r--yt/yql/plugin/native/CMakeLists.linux-x86_64.txt2
-rw-r--r--yt/yql/plugin/native/CMakeLists.windows-x86_64.txt2
-rw-r--r--yt/yql/plugin/native/plugin.cpp9
-rw-r--r--yt/yql/plugin/native/ya.make1
-rw-r--r--yt/yql/plugin/plugin.h4
-rw-r--r--yt/yt/core/CMakeLists.darwin-x86_64.txt2
-rw-r--r--yt/yt/core/CMakeLists.linux-aarch64.txt2
-rw-r--r--yt/yt/core/CMakeLists.linux-x86_64.txt2
-rw-r--r--yt/yt/core/CMakeLists.windows-x86_64.txt2
-rw-r--r--yt/yt/core/rpc/CMakeLists.txt9
-rw-r--r--yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt64
-rw-r--r--yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt65
-rw-r--r--yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt65
-rw-r--r--yt/yt/core/rpc/grpc/CMakeLists.txt17
-rw-r--r--yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt61
-rw-r--r--yt/yt/core/service_discovery/CMakeLists.txt9
-rw-r--r--yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt23
-rw-r--r--yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt24
-rw-r--r--yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt24
-rw-r--r--yt/yt/core/service_discovery/yp/CMakeLists.txt17
-rw-r--r--yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt20
-rw-r--r--yt/yt/core/ya.make1
-rw-r--r--yt/yt/library/CMakeLists.darwin-x86_64.txt4
-rw-r--r--yt/yt/library/CMakeLists.linux-aarch64.txt5
-rw-r--r--yt/yt/library/CMakeLists.linux-x86_64.txt5
-rw-r--r--yt/yt/library/CMakeLists.windows-x86_64.txt4
-rw-r--r--yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt28
-rw-r--r--yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt28
-rw-r--r--yt/yt/library/backtrace_introspector/CMakeLists.txt13
-rw-r--r--yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt24
-rw-r--r--yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt24
-rw-r--r--yt/yt/library/backtrace_introspector/http/CMakeLists.txt13
-rw-r--r--yt/yt/library/backtrace_introspector/http/handler.cpp89
-rw-r--r--yt/yt/library/backtrace_introspector/http/handler.h20
-rw-r--r--yt/yt/library/backtrace_introspector/http/ya.make16
-rw-r--r--yt/yt/library/backtrace_introspector/introspect.cpp216
-rw-r--r--yt/yt/library/backtrace_introspector/introspect.h57
-rw-r--r--yt/yt/library/backtrace_introspector/introspect_dummy.cpp14
-rw-r--r--yt/yt/library/backtrace_introspector/introspect_linux.cpp211
-rw-r--r--yt/yt/library/backtrace_introspector/private.h16
-rw-r--r--yt/yt/library/backtrace_introspector/public.h12
-rw-r--r--yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp198
-rw-r--r--yt/yt/library/backtrace_introspector/unittests/ya.make15
-rw-r--r--yt/yt/library/backtrace_introspector/ya.make31
-rw-r--r--yt/yt/library/containers/CMakeLists.darwin-x86_64.txt30
-rw-r--r--yt/yt/library/containers/CMakeLists.linux-aarch64.txt32
-rw-r--r--yt/yt/library/containers/CMakeLists.linux-x86_64.txt32
-rw-r--r--yt/yt/library/containers/CMakeLists.txt17
-rw-r--r--yt/yt/library/containers/CMakeLists.windows-x86_64.txt27
-rw-r--r--yt/yt/library/containers/cgroup.cpp752
-rw-r--r--yt/yt/library/containers/cgroup.h290
-rw-r--r--yt/yt/library/containers/config.cpp64
-rw-r--r--yt/yt/library/containers/config.h64
-rw-r--r--yt/yt/library/containers/cri/config.cpp54
-rw-r--r--yt/yt/library/containers/cri/config.h70
-rw-r--r--yt/yt/library/containers/cri/cri_api.cpp33
-rw-r--r--yt/yt/library/containers/cri/cri_api.h99
-rw-r--r--yt/yt/library/containers/cri/cri_executor.cpp666
-rw-r--r--yt/yt/library/containers/cri/cri_executor.h207
-rw-r--r--yt/yt/library/containers/cri/private.h13
-rw-r--r--yt/yt/library/containers/cri/public.h17
-rw-r--r--yt/yt/library/containers/cri/ya.make22
-rw-r--r--yt/yt/library/containers/disk_manager/config.cpp61
-rw-r--r--yt/yt/library/containers/disk_manager/config.h79
-rw-r--r--yt/yt/library/containers/disk_manager/disk_info_provider.cpp64
-rw-r--r--yt/yt/library/containers/disk_manager/disk_info_provider.h38
-rw-r--r--yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp49
-rw-r--r--yt/yt/library/containers/disk_manager/disk_manager_proxy.h38
-rw-r--r--yt/yt/library/containers/disk_manager/public.h48
-rw-r--r--yt/yt/library/containers/disk_manager/ya.make19
-rw-r--r--yt/yt/library/containers/instance.cpp812
-rw-r--r--yt/yt/library/containers/instance.h168
-rw-r--r--yt/yt/library/containers/instance_limits_tracker.cpp179
-rw-r--r--yt/yt/library/containers/instance_limits_tracker.h59
-rw-r--r--yt/yt/library/containers/porto_executor.cpp1079
-rw-r--r--yt/yt/library/containers/porto_executor.h142
-rw-r--r--yt/yt/library/containers/porto_health_checker.cpp69
-rw-r--r--yt/yt/library/containers/porto_health_checker.h52
-rw-r--r--yt/yt/library/containers/porto_resource_tracker.cpp711
-rw-r--r--yt/yt/library/containers/porto_resource_tracker.h158
-rw-r--r--yt/yt/library/containers/private.h13
-rw-r--r--yt/yt/library/containers/process.cpp154
-rw-r--r--yt/yt/library/containers/process.h46
-rw-r--r--yt/yt/library/containers/public.h163
-rw-r--r--yt/yt/library/containers/unittests/containers_ut.cpp133
-rw-r--r--yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp251
-rw-r--r--yt/yt/library/containers/unittests/process_ut.cpp302
-rw-r--r--yt/yt/library/containers/unittests/ya.make35
-rw-r--r--yt/yt/library/containers/ya.make37
-rw-r--r--yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt26
-rw-r--r--yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt30
-rw-r--r--yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt30
-rw-r--r--yt/yt/library/monitoring/CMakeLists.txt17
-rw-r--r--yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt23
-rw-r--r--yt/yt/library/monitoring/http_integration.cpp203
-rw-r--r--yt/yt/library/monitoring/http_integration.h28
-rw-r--r--yt/yt/library/monitoring/monitoring_manager.cpp177
-rw-r--r--yt/yt/library/monitoring/monitoring_manager.h54
-rw-r--r--yt/yt/library/monitoring/private.h15
-rw-r--r--yt/yt/library/monitoring/public.h13
-rw-r--r--yt/yt/library/monitoring/ya.make27
-rw-r--r--yt/yt/library/process/CMakeLists.darwin-x86_64.txt26
-rw-r--r--yt/yt/library/process/CMakeLists.linux-aarch64.txt27
-rw-r--r--yt/yt/library/process/CMakeLists.linux-x86_64.txt27
-rw-r--r--yt/yt/library/process/CMakeLists.txt17
-rw-r--r--yt/yt/library/process/CMakeLists.windows-x86_64.txt23
-rw-r--r--yt/yt/library/process/io_dispatcher.cpp37
-rw-r--r--yt/yt/library/process/io_dispatcher.h34
-rw-r--r--yt/yt/library/process/pipe.cpp256
-rw-r--r--yt/yt/library/process/pipe.h114
-rw-r--r--yt/yt/library/process/private.h14
-rw-r--r--yt/yt/library/process/process.cpp697
-rw-r--r--yt/yt/library/process/process.h125
-rw-r--r--yt/yt/library/process/pty.cpp64
-rw-r--r--yt/yt/library/process/pty.h33
-rw-r--r--yt/yt/library/process/public.h14
-rw-r--r--yt/yt/library/process/subprocess.cpp153
-rw-r--r--yt/yt/library/process/subprocess.h48
-rw-r--r--yt/yt/library/process/unittests/pipes_ut.cpp319
-rw-r--r--yt/yt/library/process/unittests/process_ut.cpp235
-rw-r--r--yt/yt/library/process/unittests/subprocess_ut.cpp101
-rw-r--r--yt/yt/library/process/unittests/ya.make24
-rw-r--r--yt/yt/library/process/ya.make22
-rw-r--r--yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt3
-rw-r--r--yt/yt/library/profiling/CMakeLists.linux-aarch64.txt3
-rw-r--r--yt/yt/library/profiling/CMakeLists.linux-x86_64.txt3
-rw-r--r--yt/yt/library/profiling/CMakeLists.windows-x86_64.txt3
-rw-r--r--yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt22
-rw-r--r--yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt23
-rw-r--r--yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt23
-rw-r--r--yt/yt/library/profiling/perf/CMakeLists.txt17
-rw-r--r--yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt19
-rw-r--r--yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt68
-rw-r--r--yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt69
-rw-r--r--yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt69
-rw-r--r--yt/yt/library/profiling/solomon/CMakeLists.txt17
-rw-r--r--yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt65
-rw-r--r--yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt22
-rw-r--r--yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt23
-rw-r--r--yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt23
-rw-r--r--yt/yt/library/profiling/tcmalloc/CMakeLists.txt17
-rw-r--r--yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt19
-rw-r--r--yt/yt/library/program/CMakeLists.darwin-x86_64.txt38
-rw-r--r--yt/yt/library/program/CMakeLists.linux-aarch64.txt39
-rw-r--r--yt/yt/library/program/CMakeLists.linux-x86_64.txt39
-rw-r--r--yt/yt/library/program/CMakeLists.txt17
-rw-r--r--yt/yt/library/program/CMakeLists.windows-x86_64.txt35
-rw-r--r--yt/yt/library/program/build_attributes.cpp107
-rw-r--r--yt/yt/library/program/build_attributes.h44
-rw-r--r--yt/yt/library/program/config.cpp210
-rw-r--r--yt/yt/library/program/config.h224
-rw-r--r--yt/yt/library/program/helpers.cpp335
-rw-r--r--yt/yt/library/program/helpers.h18
-rw-r--r--yt/yt/library/program/private.h15
-rw-r--r--yt/yt/library/program/program.cpp385
-rw-r--r--yt/yt/library/program/program.h148
-rw-r--r--yt/yt/library/program/program_config_mixin.cpp1
-rw-r--r--yt/yt/library/program/program_config_mixin.h166
-rw-r--r--yt/yt/library/program/program_pdeathsig_mixin.cpp36
-rw-r--r--yt/yt/library/program/program_pdeathsig_mixin.h22
-rw-r--r--yt/yt/library/program/program_setsid_mixin.cpp30
-rw-r--r--yt/yt/library/program/program_setsid_mixin.h22
-rw-r--r--yt/yt/library/program/public.h21
-rw-r--r--yt/yt/library/program/ya.make30
-rw-r--r--yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt1
-rw-r--r--yt/yt/library/tracing/CMakeLists.linux-aarch64.txt1
-rw-r--r--yt/yt/library/tracing/CMakeLists.linux-x86_64.txt1
-rw-r--r--yt/yt/library/tracing/CMakeLists.windows-x86_64.txt1
-rw-r--r--yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt67
-rw-r--r--yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt69
-rw-r--r--yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt69
-rw-r--r--yt/yt/library/tracing/jaeger/CMakeLists.txt17
-rw-r--r--yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt61
-rw-r--r--yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt41
-rw-r--r--yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt43
-rw-r--r--yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt43
-rw-r--r--yt/yt/library/ytprof/CMakeLists.txt10
-rw-r--r--yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt40
-rw-r--r--yt/yt/library/ytprof/bundle/ya.make28
-rw-r--r--yt/yt/library/ytprof/example/main.cpp70
-rw-r--r--yt/yt/library/ytprof/example/ya.make19
-rw-r--r--yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt25
-rw-r--r--yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt25
-rw-r--r--yt/yt/library/ytprof/http/CMakeLists.txt13
-rw-r--r--yt/yt/library/ytprof/http/handler.cpp311
-rw-r--r--yt/yt/library/ytprof/http/handler.h24
-rw-r--r--yt/yt/library/ytprof/http/ya.make16
-rw-r--r--yt/yt/library/ytprof/integration/test_http.py113
-rw-r--r--yt/yt/library/ytprof/integration/ya.make20
-rw-r--r--yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt47
-rw-r--r--yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt48
-rw-r--r--yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt48
-rw-r--r--yt/yt/library/ytprof/proto/CMakeLists.txt17
-rw-r--r--yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt47
-rw-r--r--yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp343
-rw-r--r--yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp226
-rw-r--r--yt/yt/library/ytprof/unittests/queue_ut.cpp66
-rw-r--r--yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp172
-rw-r--r--yt/yt/library/ytprof/unittests/symbolizer_ut.cpp83
-rw-r--r--yt/yt/library/ytprof/unittests/testso/testso.cpp9
-rw-r--r--yt/yt/library/ytprof/unittests/testso/ya.make7
-rw-r--r--yt/yt/library/ytprof/unittests/testso1/testso.cpp9
-rw-r--r--yt/yt/library/ytprof/unittests/testso1/ya.make7
-rw-r--r--yt/yt/library/ytprof/unittests/ya.make43
210 files changed, 16794 insertions, 4 deletions
diff --git a/yt/yql/plugin/bridge/interface.h b/yt/yql/plugin/bridge/interface.h
index 08e765b3d0..ba08b657aa 100644
--- a/yt/yql/plugin/bridge/interface.h
+++ b/yt/yql/plugin/bridge/interface.h
@@ -9,8 +9,18 @@
////////////////////////////////////////////////////////////////////////////////
+// NB(mpereskokova): don't forget to update min_required_abi_version at yt/yql/agent/config.cpp and abi_version in yt/yql/plugin/dynamic/impl.cpp during breaking changes
+using TFuncBridgeGetABIVersion = ssize_t();
+
+////////////////////////////////////////////////////////////////////////////////
+
struct TBridgeYqlPluginOptions
{
+ ssize_t RequiredABIVersion;
+
+ const char* SingletonsConfig;
+ ssize_t SingletonsConfigLength;
+
const char* MRJobBinary;
const char* UdfDirectory;
@@ -95,6 +105,7 @@ using TFuncBridgeGetProgress = TBridgeQueryResult*(TBridgeYqlPlugin* plugin, con
XX(BridgeFreeYqlPlugin) \
XX(BridgeFreeQueryResult) \
XX(BridgeRun) \
- XX(BridgeGetProgress)
+ XX(BridgeGetProgress) \
+ XX(BridgeGetABIVersion)
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yql/plugin/bridge/plugin.cpp b/yt/yql/plugin/bridge/plugin.cpp
index 18e2e9e639..5c461616e9 100644
--- a/yt/yql/plugin/bridge/plugin.cpp
+++ b/yt/yql/plugin/bridge/plugin.cpp
@@ -72,7 +72,12 @@ public:
? options.DefaultCluster->data()
: nullptr;
+ TString singletonsConfig = options.SingletonsConfig ? options.SingletonsConfig.ToString() : "{}";
+
TBridgeYqlPluginOptions bridgeOptions {
+ .RequiredABIVersion = options.RequiredABIVersion,
+ .SingletonsConfig = singletonsConfig.data(),
+ .SingletonsConfigLength = static_cast<int>(singletonsConfig.size()),
.MRJobBinary = options.MRJobBinary.data(),
.UdfDirectory = options.UdfDirectory.data(),
.ClusterCount = ssize(bridgeClusters),
diff --git a/yt/yql/plugin/dynamic/dylib.exports b/yt/yql/plugin/dynamic/dylib.exports
index fc77529eaf..b979ce7e18 100644
--- a/yt/yql/plugin/dynamic/dylib.exports
+++ b/yt/yql/plugin/dynamic/dylib.exports
@@ -4,6 +4,7 @@ BridgeFreeYqlPlugin
BridgeFreeQueryResult
BridgeRun
BridgeGetProgress
+BridgeGetABIVersion
# YQL <-> YQL UDFs interface.
UdfAllocateWithSize
diff --git a/yt/yql/plugin/dynamic/impl.cpp b/yt/yql/plugin/dynamic/impl.cpp
index 5966a8a8ec..6beb5bf8c2 100644
--- a/yt/yql/plugin/dynamic/impl.cpp
+++ b/yt/yql/plugin/dynamic/impl.cpp
@@ -10,8 +10,17 @@ extern "C" {
////////////////////////////////////////////////////////////////////////////////
+ssize_t BridgeGetABIVersion()
+{
+ return 0;
+}
+
TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOptions)
{
+ YT_VERIFY(bridgeOptions->RequiredABIVersion == BridgeGetABIVersion());
+
+ static const TYsonString EmptyMap = TYsonString(TString("{}"));
+
THashMap<TString, TString> clusters;
for (auto clusterIndex = 0; clusterIndex < bridgeOptions->ClusterCount; ++clusterIndex) {
const auto& Cluster = bridgeOptions->Clusters[clusterIndex];
@@ -20,9 +29,14 @@ TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOpt
auto operationAttributes = bridgeOptions->OperationAttributes
? TYsonString(TString(bridgeOptions->OperationAttributes, bridgeOptions->OperationAttributesLength))
- : TYsonString();
+ : EmptyMap;
+
+ auto singletonsConfig = bridgeOptions->SingletonsConfig
+ ? TYsonString(TString(bridgeOptions->SingletonsConfig, bridgeOptions->SingletonsConfigLength))
+ : EmptyMap;
TYqlPluginOptions options{
+ .SingletonsConfig = singletonsConfig,
.MRJobBinary = TString(bridgeOptions->MRJobBinary),
.UdfDirectory = TString(bridgeOptions->UdfDirectory),
.Clusters = std::move(clusters),
diff --git a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt
index 7c002160ee..04a6b539a5 100644
--- a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt
+++ b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt
@@ -20,6 +20,7 @@ target_link_libraries(yql-plugin-native PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
@@ -65,6 +66,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
diff --git a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt
index b85008e3c5..b9ac27203c 100644
--- a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt
+++ b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt
@@ -21,6 +21,7 @@ target_link_libraries(yql-plugin-native PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
@@ -67,6 +68,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
diff --git a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt
index b85008e3c5..b9ac27203c 100644
--- a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt
+++ b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt
@@ -21,6 +21,7 @@ target_link_libraries(yql-plugin-native PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
@@ -67,6 +68,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
diff --git a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt
index 7c002160ee..04a6b539a5 100644
--- a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt
+++ b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt
@@ -20,6 +20,7 @@ target_link_libraries(yql-plugin-native PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
@@ -65,6 +66,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC
cpp-yson-node
cpp-mapreduce-client
cpp-mapreduce-common
+ yt-library-program
library-yql-ast
yql-sql-pg
yql-parser-pg_wrapper
diff --git a/yt/yql/plugin/native/plugin.cpp b/yt/yql/plugin/native/plugin.cpp
index 9bb6c2261e..38ccbbdf3b 100644
--- a/yt/yql/plugin/native/plugin.cpp
+++ b/yt/yql/plugin/native/plugin.cpp
@@ -24,7 +24,11 @@
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/backtrace/backtrace.h>
-#include <yt/cpp/mapreduce/interface/config.h>
+#include <yt/yt/core/ytree/convert.h>
+
+#include <yt/yt/library/program/config.h>
+#include <yt/yt/library/program/helpers.h>
+
#include <yt/cpp/mapreduce/interface/logging/logger.h>
#include <library/cpp/yt/threading/rw_spin_lock.h>
@@ -115,6 +119,9 @@ public:
TYqlPlugin(TYqlPluginOptions& options)
{
try {
+ auto singletonsConfig = NYTree::ConvertTo<TSingletonsConfigPtr>(options.SingletonsConfig);
+ ConfigureSingletons(singletonsConfig);
+
NYql::NLog::InitLogger(std::move(options.LogBackend));
auto& logger = NYql::NLog::YqlLogger();
diff --git a/yt/yql/plugin/native/ya.make b/yt/yql/plugin/native/ya.make
index 15f3851411..fe1a657c69 100644
--- a/yt/yql/plugin/native/ya.make
+++ b/yt/yql/plugin/native/ya.make
@@ -13,6 +13,7 @@ PEERDIR(
library/cpp/yson/node
yt/cpp/mapreduce/client
yt/cpp/mapreduce/common
+ yt/yt/library/program
ydb/library/yql/ast
ydb/library/yql/sql/pg
ydb/library/yql/parser/pg_wrapper
diff --git a/yt/yql/plugin/plugin.h b/yt/yql/plugin/plugin.h
index 2d2c45d54c..68b01da922 100644
--- a/yt/yql/plugin/plugin.h
+++ b/yt/yql/plugin/plugin.h
@@ -26,6 +26,10 @@ using TQueryId = TGuid;
class TYqlPluginOptions
{
public:
+ int RequiredABIVersion;
+
+ TYsonString SingletonsConfig;
+
TString MRJobBinary = "./mrjob";
TString UdfDirectory;
diff --git a/yt/yt/core/CMakeLists.darwin-x86_64.txt b/yt/yt/core/CMakeLists.darwin-x86_64.txt
index 0b4da77950..5b4d4b633f 100644
--- a/yt/yt/core/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/core/CMakeLists.darwin-x86_64.txt
@@ -9,6 +9,8 @@
add_subdirectory(http)
add_subdirectory(https)
add_subdirectory(misc)
+add_subdirectory(rpc)
+add_subdirectory(service_discovery)
add_library(yt-yt-core)
target_compile_options(yt-yt-core PRIVATE
diff --git a/yt/yt/core/CMakeLists.linux-aarch64.txt b/yt/yt/core/CMakeLists.linux-aarch64.txt
index 51eeb4ff56..640e862300 100644
--- a/yt/yt/core/CMakeLists.linux-aarch64.txt
+++ b/yt/yt/core/CMakeLists.linux-aarch64.txt
@@ -9,6 +9,8 @@
add_subdirectory(http)
add_subdirectory(https)
add_subdirectory(misc)
+add_subdirectory(rpc)
+add_subdirectory(service_discovery)
add_library(yt-yt-core)
target_compile_options(yt-yt-core PRIVATE
diff --git a/yt/yt/core/CMakeLists.linux-x86_64.txt b/yt/yt/core/CMakeLists.linux-x86_64.txt
index 164e626f9b..ab2ddf3548 100644
--- a/yt/yt/core/CMakeLists.linux-x86_64.txt
+++ b/yt/yt/core/CMakeLists.linux-x86_64.txt
@@ -9,6 +9,8 @@
add_subdirectory(http)
add_subdirectory(https)
add_subdirectory(misc)
+add_subdirectory(rpc)
+add_subdirectory(service_discovery)
add_library(yt-yt-core)
target_compile_options(yt-yt-core PRIVATE
diff --git a/yt/yt/core/CMakeLists.windows-x86_64.txt b/yt/yt/core/CMakeLists.windows-x86_64.txt
index 76ae848b35..741f6b7a8b 100644
--- a/yt/yt/core/CMakeLists.windows-x86_64.txt
+++ b/yt/yt/core/CMakeLists.windows-x86_64.txt
@@ -9,6 +9,8 @@
add_subdirectory(http)
add_subdirectory(https)
add_subdirectory(misc)
+add_subdirectory(rpc)
+add_subdirectory(service_discovery)
add_library(yt-yt-core)
target_compile_options(yt-yt-core PRIVATE
diff --git a/yt/yt/core/rpc/CMakeLists.txt b/yt/yt/core/rpc/CMakeLists.txt
new file mode 100644
index 0000000000..68ea682099
--- /dev/null
+++ b/yt/yt/core/rpc/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(grpc)
diff --git a/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..eb0e7def69
--- /dev/null
+++ b/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,64 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(core-rpc-grpc)
+target_compile_options(core-rpc-grpc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_include_directories(core-rpc-grpc PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_include_directories(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/contrib/libs/grpc
+)
+target_link_libraries(core-rpc-grpc PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto
+)
+target_sources(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp
+)
+target_proto_addincls(core-rpc-grpc
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(core-rpc-grpc
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt b/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..c3945011ba
--- /dev/null
+++ b/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,65 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(core-rpc-grpc)
+target_compile_options(core-rpc-grpc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_include_directories(core-rpc-grpc PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_include_directories(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/contrib/libs/grpc
+)
+target_link_libraries(core-rpc-grpc PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto
+)
+target_sources(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp
+)
+target_proto_addincls(core-rpc-grpc
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(core-rpc-grpc
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..c3945011ba
--- /dev/null
+++ b/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,65 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(core-rpc-grpc)
+target_compile_options(core-rpc-grpc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_include_directories(core-rpc-grpc PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_include_directories(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/contrib/libs/grpc
+)
+target_link_libraries(core-rpc-grpc PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto
+)
+target_sources(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp
+)
+target_proto_addincls(core-rpc-grpc
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(core-rpc-grpc
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/core/rpc/grpc/CMakeLists.txt b/yt/yt/core/rpc/grpc/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/core/rpc/grpc/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..629594878e
--- /dev/null
+++ b/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,61 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(core-rpc-grpc)
+target_include_directories(core-rpc-grpc PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_include_directories(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/contrib/libs/grpc
+)
+target_link_libraries(core-rpc-grpc PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto
+)
+target_sources(core-rpc-grpc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp
+)
+target_proto_addincls(core-rpc-grpc
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(core-rpc-grpc
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/core/service_discovery/CMakeLists.txt b/yt/yt/core/service_discovery/CMakeLists.txt
new file mode 100644
index 0000000000..7f1d154d54
--- /dev/null
+++ b/yt/yt/core/service_discovery/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(yp)
diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..78ca098926
--- /dev/null
+++ b/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-service_discovery-yp)
+target_compile_options(core-service_discovery-yp PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(core-service_discovery-yp PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-rpc-grpc
+)
+target_sources(core-service_discovery-yp PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp
+)
diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..eb8048e099
--- /dev/null
+++ b/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-service_discovery-yp)
+target_compile_options(core-service_discovery-yp PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(core-service_discovery-yp PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-rpc-grpc
+)
+target_sources(core-service_discovery-yp PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp
+)
diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..eb8048e099
--- /dev/null
+++ b/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-service_discovery-yp)
+target_compile_options(core-service_discovery-yp PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(core-service_discovery-yp PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-rpc-grpc
+)
+target_sources(core-service_discovery-yp PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp
+)
diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.txt b/yt/yt/core/service_discovery/yp/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/core/service_discovery/yp/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..2aa18009a5
--- /dev/null
+++ b/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-service_discovery-yp)
+target_link_libraries(core-service_discovery-yp PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-rpc-grpc
+)
+target_sources(core-service_discovery-yp PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp
+)
diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make
index e985dd82ae..043d3a1dd9 100644
--- a/yt/yt/core/ya.make
+++ b/yt/yt/core/ya.make
@@ -324,6 +324,7 @@ PEERDIR(
library/cpp/ytalloc/api
yt/yt/build
+
yt/yt/core/misc/isa_crc64
yt/yt_proto/yt/core
diff --git a/yt/yt/library/CMakeLists.darwin-x86_64.txt b/yt/yt/library/CMakeLists.darwin-x86_64.txt
index d26f3bae28..2debbb7626 100644
--- a/yt/yt/library/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/library/CMakeLists.darwin-x86_64.txt
@@ -7,10 +7,14 @@
add_subdirectory(auth)
+add_subdirectory(containers)
add_subdirectory(decimal)
add_subdirectory(erasure)
+add_subdirectory(monitoring)
add_subdirectory(numeric)
+add_subdirectory(process)
add_subdirectory(profiling)
+add_subdirectory(program)
add_subdirectory(quantile_digest)
add_subdirectory(re2)
add_subdirectory(syncmap)
diff --git a/yt/yt/library/CMakeLists.linux-aarch64.txt b/yt/yt/library/CMakeLists.linux-aarch64.txt
index d26f3bae28..524ffcf525 100644
--- a/yt/yt/library/CMakeLists.linux-aarch64.txt
+++ b/yt/yt/library/CMakeLists.linux-aarch64.txt
@@ -7,10 +7,15 @@
add_subdirectory(auth)
+add_subdirectory(backtrace_introspector)
+add_subdirectory(containers)
add_subdirectory(decimal)
add_subdirectory(erasure)
+add_subdirectory(monitoring)
add_subdirectory(numeric)
+add_subdirectory(process)
add_subdirectory(profiling)
+add_subdirectory(program)
add_subdirectory(quantile_digest)
add_subdirectory(re2)
add_subdirectory(syncmap)
diff --git a/yt/yt/library/CMakeLists.linux-x86_64.txt b/yt/yt/library/CMakeLists.linux-x86_64.txt
index d26f3bae28..524ffcf525 100644
--- a/yt/yt/library/CMakeLists.linux-x86_64.txt
+++ b/yt/yt/library/CMakeLists.linux-x86_64.txt
@@ -7,10 +7,15 @@
add_subdirectory(auth)
+add_subdirectory(backtrace_introspector)
+add_subdirectory(containers)
add_subdirectory(decimal)
add_subdirectory(erasure)
+add_subdirectory(monitoring)
add_subdirectory(numeric)
+add_subdirectory(process)
add_subdirectory(profiling)
+add_subdirectory(program)
add_subdirectory(quantile_digest)
add_subdirectory(re2)
add_subdirectory(syncmap)
diff --git a/yt/yt/library/CMakeLists.windows-x86_64.txt b/yt/yt/library/CMakeLists.windows-x86_64.txt
index 20f7fe76fa..4502da4e61 100644
--- a/yt/yt/library/CMakeLists.windows-x86_64.txt
+++ b/yt/yt/library/CMakeLists.windows-x86_64.txt
@@ -6,7 +6,11 @@
# original buildsystem will not be accepted.
+add_subdirectory(containers)
+add_subdirectory(monitoring)
+add_subdirectory(process)
add_subdirectory(profiling)
+add_subdirectory(program)
add_subdirectory(syncmap)
add_subdirectory(tracing)
add_subdirectory(tvm)
diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt b/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..215573de83
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,28 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(http)
+
+add_library(yt-library-backtrace_introspector)
+target_compile_options(yt-library-backtrace_introspector PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-backtrace_introspector PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ backtrace-cursors-interop
+ backtrace-cursors-libunwind
+ backtrace-cursors-frame_pointer
+ cpp-yt-misc
+)
+target_sources(yt-library-backtrace_introspector PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect_linux.cpp
+)
diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt b/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..215573de83
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,28 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(http)
+
+add_library(yt-library-backtrace_introspector)
+target_compile_options(yt-library-backtrace_introspector PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-backtrace_introspector PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ backtrace-cursors-interop
+ backtrace-cursors-libunwind
+ backtrace-cursors-frame_pointer
+ cpp-yt-misc
+)
+target_sources(yt-library-backtrace_introspector PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect_linux.cpp
+)
diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.txt b/yt/yt/library/backtrace_introspector/CMakeLists.txt
new file mode 100644
index 0000000000..4d48dcdee6
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/CMakeLists.txt
@@ -0,0 +1,13 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..5b8a9100d2
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-backtrace_introspector-http)
+target_compile_options(library-backtrace_introspector-http PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-backtrace_introspector-http PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-core-http
+ yt-library-backtrace_introspector
+)
+target_sources(library-backtrace_introspector-http PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/http/handler.cpp
+)
diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..5b8a9100d2
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-backtrace_introspector-http)
+target_compile_options(library-backtrace_introspector-http PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-backtrace_introspector-http PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-core-http
+ yt-library-backtrace_introspector
+)
+target_sources(library-backtrace_introspector-http PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/http/handler.cpp
+)
diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.txt
new file mode 100644
index 0000000000..4d48dcdee6
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.txt
@@ -0,0 +1,13 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/backtrace_introspector/http/handler.cpp b/yt/yt/library/backtrace_introspector/http/handler.cpp
new file mode 100644
index 0000000000..367e3105c0
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/handler.cpp
@@ -0,0 +1,89 @@
+#include "handler.h"
+
+#include <yt/yt/core/http/server.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+
+#include <yt/yt/library/backtrace_introspector/introspect.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+using namespace NHttp;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class THandlerBase
+ : public IHttpHandler
+{
+public:
+ void HandleRequest(const IRequestPtr& /*req*/, const IResponseWriterPtr& rsp) override
+ {
+ try {
+ auto dumpFuture = BIND(&THandlerBase::Dump, MakeStrong(this))
+ .AsyncVia(Queue_->GetInvoker())
+ .Run();
+
+ auto dump = WaitFor(dumpFuture)
+ .ValueOrThrow();
+
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(dump)))
+ .ThrowOnError();
+
+ WaitFor(rsp->Close())
+ .ThrowOnError();
+ } catch (const std::exception& ex) {
+ if (!rsp->AreHeadersFlushed()) {
+ rsp->SetStatus(EStatusCode::InternalServerError);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what())))
+ .ThrowOnError();
+ }
+ throw;
+ }
+ }
+
+private:
+ static inline const TActionQueuePtr Queue_ = New<TActionQueue>("BacktraceIntro");
+
+protected:
+ virtual TString Dump() = 0;
+};
+
+class TThreadsHandler
+ : public THandlerBase
+{
+private:
+ TString Dump() override
+ {
+ return FormatIntrospectionInfos(IntrospectThreads());
+ }
+};
+
+class TFibersHandler
+ : public THandlerBase
+{
+private:
+ TString Dump() override
+ {
+ return FormatIntrospectionInfos(IntrospectFibers());
+ }
+};
+
+void Register(
+ const IRequestPathMatcherPtr& handlers,
+ const TString& prefix)
+{
+ handlers->Add(prefix + "/threads", New<TThreadsHandler>());
+ handlers->Add(prefix + "/fibers", New<TFibersHandler>());
+}
+
+void Register(
+ const IServerPtr& server,
+ const TString& prefix)
+{
+ Register(server->GetPathMatcher(), prefix);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/http/handler.h b/yt/yt/library/backtrace_introspector/http/handler.h
new file mode 100644
index 0000000000..be795b7e5d
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/handler.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <yt/yt/core/http/public.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Registers introspector handlers.
+void Register(
+ const NHttp::IRequestPathMatcherPtr& handlers,
+ const TString& prefix = {});
+
+void Register(
+ const NHttp::IServerPtr& server,
+ const TString& prefix = {});
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/http/ya.make b/yt/yt/library/backtrace_introspector/http/ya.make
new file mode 100644
index 0000000000..504d20a2e3
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/http/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ handler.cpp
+)
+
+PEERDIR(
+ yt/yt/core
+ yt/yt/core/http
+
+ yt/yt/library/backtrace_introspector
+)
+
+END()
diff --git a/yt/yt/library/backtrace_introspector/introspect.cpp b/yt/yt/library/backtrace_introspector/introspect.cpp
new file mode 100644
index 0000000000..592c232f0f
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/introspect.cpp
@@ -0,0 +1,216 @@
+#include "introspect.h"
+
+#include "private.h"
+
+#include <yt/yt/core/misc/finally.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/concurrency/fiber.h>
+#include <yt/yt/core/concurrency/scheduler_api.h>
+
+#include <yt/yt/core/tracing/trace_context.h>
+
+#include <library/cpp/yt/memory/safe_memory_reader.h>
+
+#include <library/cpp/yt/backtrace/backtrace.h>
+
+#include <library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h>
+
+#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h>
+
+#include <library/cpp/yt/backtrace/cursors/interop/interop.h>
+
+#include <util/system/yield.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+using namespace NConcurrency;
+using namespace NThreading;
+using namespace NTracing;
+using namespace NBacktrace;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = BacktraceIntrospectorLogger;
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::vector<TFiberIntrospectionInfo> IntrospectFibers()
+{
+ YT_LOG_INFO("Fiber introspection started");
+
+ auto fibers = TFiber::List();
+
+ YT_LOG_INFO("Collecting waiting fibers backtraces");
+
+ std::vector<TFiberIntrospectionInfo> infos;
+ THashSet<TFiberId> waitingFiberIds;
+ THashSet<TFiberId> fiberIds;
+ for (const auto& fiber : fibers) {
+ auto fiberId = fiber->GetFiberId();
+ if (fiberId == InvalidFiberId) {
+ continue;
+ }
+
+ InsertOrCrash(fiberIds, fiberId);
+
+ EFiberState state;
+ if (!fiber->TryIntrospectWaiting(state, [&] {
+ YT_LOG_DEBUG("Waiting fiber is successfully locked for introspection (FiberId: %x)",
+ fiberId);
+
+ const auto& propagatingStorage = fiber->GetPropagatingStorage();
+ const auto* traceContext = TryGetTraceContextFromPropagatingStorage(propagatingStorage);
+
+ TFiberIntrospectionInfo info{
+ .State = EFiberState::Waiting,
+ .FiberId = fiberId,
+ .WaitingSince = fiber->GetWaitingSince(),
+ .TraceId = traceContext ? traceContext->GetTraceId() : TTraceId(),
+ .TraceLoggingTag = traceContext ? traceContext->GetLoggingTag() : TString(),
+ };
+
+ auto optionalContext = TrySynthesizeLibunwindContextFromMachineContext(*fiber->GetMachineContext());
+ if (!optionalContext) {
+ YT_LOG_WARNING("Failed to synthesize libunwind context (FiberId: %x)",
+ fiberId);
+ return;
+ }
+
+ TLibunwindCursor cursor(*optionalContext);
+ while (!cursor.IsFinished()) {
+ info.Backtrace.push_back(cursor.GetCurrentIP());
+ cursor.MoveNext();
+ }
+
+ infos.push_back(std::move(info));
+ InsertOrCrash(waitingFiberIds, fiberId);
+
+ YT_LOG_DEBUG("Fiber introspection completed (FiberId: %x)",
+ info.FiberId);
+ })) {
+ YT_LOG_DEBUG("Failed to lock fiber for introspection (FiberId: %x, State: %v)",
+ fiberId,
+ state);
+ }
+ }
+
+ YT_LOG_INFO("Collecting running fibers backtraces");
+
+ THashSet<TFiberId> runningFiberIds;
+ for (auto& info : IntrospectThreads()) {
+ if (info.FiberId == InvalidFiberId) {
+ continue;
+ }
+
+ if (waitingFiberIds.contains(info.FiberId)) {
+ continue;
+ }
+
+ if (!runningFiberIds.insert(info.FiberId).second) {
+ continue;
+ }
+
+ infos.push_back(TFiberIntrospectionInfo{
+ .State = EFiberState::Running,
+ .FiberId = info.FiberId,
+ .ThreadId = info.ThreadId,
+ .ThreadName = std::move(info.ThreadName),
+ .TraceId = info.TraceId,
+ .TraceLoggingTag = std::move(info.TraceLoggingTag),
+ .Backtrace = std::move(info.Backtrace),
+ });
+ }
+
+ for (const auto& fiber : fibers) {
+ auto fiberId = fiber->GetFiberId();
+ if (fiberId == InvalidFiberId) {
+ continue;
+ }
+ if (runningFiberIds.contains(fiberId)) {
+ continue;
+ }
+ if (waitingFiberIds.contains(fiberId)) {
+ continue;
+ }
+
+ infos.push_back(TFiberIntrospectionInfo{
+ .State = fiber->GetState(),
+ .FiberId = fiberId,
+ });
+ }
+
+ YT_LOG_INFO("Fiber introspection completed");
+
+ return infos;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+void FormatBacktrace(TStringBuilder* builder, const std::vector<const void*>& backtrace)
+{
+ if (!backtrace.empty()) {
+ builder->AppendString("Backtrace:\n");
+ SymbolizeBacktrace(
+ MakeRange(backtrace),
+ [&] (TStringBuf str) {
+ builder->AppendFormat(" %v", str);
+ });
+ }
+}
+
+} // namespace
+
+TString FormatIntrospectionInfos(const std::vector<TThreadIntrospectionInfo>& infos)
+{
+ TStringBuilder builder;
+ for (const auto& info : infos) {
+ builder.AppendFormat("Thread id: %v\n", info.ThreadId);
+ builder.AppendFormat("Thread name: %v\n", info.ThreadName);
+ if (info.FiberId != InvalidFiberId) {
+ builder.AppendFormat("Fiber id: %x\n", info.FiberId);
+ }
+ if (info.TraceId) {
+ builder.AppendFormat("Trace id: %v\n", info.TraceId);
+ }
+ if (info.TraceLoggingTag) {
+ builder.AppendFormat("Trace logging tag: %v\n", info.TraceLoggingTag);
+ }
+ FormatBacktrace(&builder, info.Backtrace);
+ builder.AppendString("\n");
+ }
+ return builder.Flush();
+}
+
+TString FormatIntrospectionInfos(const std::vector<TFiberIntrospectionInfo>& infos)
+{
+ TStringBuilder builder;
+ for (const auto& info : infos) {
+ builder.AppendFormat("Fiber id: %x\n", info.FiberId);
+ builder.AppendFormat("State: %v\n", info.State);
+ if (info.WaitingSince) {
+ builder.AppendFormat("Waiting since: %v\n", info.WaitingSince);
+ }
+ if (info.ThreadId != InvalidThreadId) {
+ builder.AppendFormat("Thread id: %v\n", info.ThreadId);
+ }
+ if (!info.ThreadName.empty()) {
+ builder.AppendFormat("Thread name: %v\n", info.ThreadName);
+ }
+ if (info.TraceId) {
+ builder.AppendFormat("Trace id: %v\n", info.TraceId);
+ }
+ if (info.TraceLoggingTag) {
+ builder.AppendFormat("Trace logging tag: %v\n", info.TraceLoggingTag);
+ }
+ FormatBacktrace(&builder, info.Backtrace);
+ builder.AppendString("\n");
+ }
+ return builder.Flush();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/introspect.h b/yt/yt/library/backtrace_introspector/introspect.h
new file mode 100644
index 0000000000..2be09d2ec8
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/introspect.h
@@ -0,0 +1,57 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/concurrency/public.h>
+
+#include <yt/yt/core/threading/public.h>
+
+#include <yt/yt/core/tracing/public.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+////////////////////////////////////////////////////////////////////////////////
+// Thread introspection API
+
+struct TThreadIntrospectionInfo
+{
+ NThreading::TThreadId ThreadId;
+ NConcurrency::TFiberId FiberId;
+ TString ThreadName;
+ NTracing::TTraceId TraceId;
+ //! Empty if no trace context is known.
+ TString TraceLoggingTag;
+ std::vector<const void*> Backtrace;
+};
+
+std::vector<TThreadIntrospectionInfo> IntrospectThreads();
+
+////////////////////////////////////////////////////////////////////////////////
+// Fiber introspection API
+
+struct TFiberIntrospectionInfo
+{
+ NConcurrency::EFiberState State;
+ NConcurrency::TFiberId FiberId;
+ //! Zero if fiber is not waiting.
+ TInstant WaitingSince;
+ //! |InvalidThreadId| is fiber is not running.
+ NThreading::TThreadId ThreadId;
+ //! Empty if fiber is not running.
+ TString ThreadName;
+ NTracing::TTraceId TraceId;
+ //! Empty if no trace context is known.
+ TString TraceLoggingTag;
+ std::vector<const void*> Backtrace;
+};
+
+std::vector<TFiberIntrospectionInfo> IntrospectFibers();
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString FormatIntrospectionInfos(const std::vector<TThreadIntrospectionInfo>& infos);
+TString FormatIntrospectionInfos(const std::vector<TFiberIntrospectionInfo>& infos);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/introspect_dummy.cpp b/yt/yt/library/backtrace_introspector/introspect_dummy.cpp
new file mode 100644
index 0000000000..e29293c7f5
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/introspect_dummy.cpp
@@ -0,0 +1,14 @@
+#include "introspect.h"
+
+namespace NYT::NBacktraceIntrospector {
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::vector<TThreadIntrospectionInfo> IntrospectThreads()
+{
+ return {};
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/introspect_linux.cpp b/yt/yt/library/backtrace_introspector/introspect_linux.cpp
new file mode 100644
index 0000000000..3fc1a077f6
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/introspect_linux.cpp
@@ -0,0 +1,211 @@
+#include "introspect.h"
+
+#include "private.h"
+
+#include <yt/yt/core/misc/finally.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/concurrency/fiber.h>
+#include <yt/yt/core/concurrency/scheduler_api.h>
+
+#include <yt/yt/core/tracing/trace_context.h>
+
+#include <library/cpp/yt/memory/safe_memory_reader.h>
+
+#include <library/cpp/yt/backtrace/backtrace.h>
+
+#include <library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h>
+
+#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h>
+
+#include <library/cpp/yt/backtrace/cursors/interop/interop.h>
+
+#include <library/cpp/yt/misc/thread_name.h>
+
+#include <util/system/yield.h>
+
+#include <sys/syscall.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+using namespace NConcurrency;
+using namespace NTracing;
+using namespace NBacktrace;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = BacktraceIntrospectorLogger;
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+struct TStaticString
+{
+ TStaticString() = default;
+
+ explicit TStaticString(TStringBuf str)
+ {
+ Length = std::min(std::ssize(str), std::ssize(Buffer));
+ std::copy(str.data(), str.data() + Length, Buffer.data());
+ }
+
+ operator TString() const
+ {
+ return TString(Buffer.data(), static_cast<size_t>(Length));
+ }
+
+ std::array<char, 256> Buffer;
+ int Length = 0;
+};
+
+struct TStaticBacktrace
+{
+ operator std::vector<const void*>() const
+ {
+ return std::vector<const void*>(Frames.data(), Frames.data() + FrameCount);
+ }
+
+ std::array<const void*, 100> Frames;
+ int FrameCount = 0;
+};
+
+struct TSignalHandlerContext
+{
+ TSignalHandlerContext();
+ ~TSignalHandlerContext();
+
+ std::atomic<bool> Finished = false;
+
+ TFiberId FiberId = {};
+ TTraceId TraceId = {};
+ TStaticString TraceLoggingTag;
+ TStaticBacktrace Backtrace;
+ TThreadName ThreadName = {};
+
+ TSafeMemoryReader* MemoryReader = Singleton<TSafeMemoryReader>();
+
+ void SetFinished()
+ {
+ Finished.store(true);
+ }
+
+ void WaitUntilFinished()
+ {
+ while (!Finished.load()) {
+ ThreadYield();
+ }
+ }
+};
+
+static TSignalHandlerContext* SignalHandlerContext;
+
+TSignalHandlerContext::TSignalHandlerContext()
+{
+ YT_VERIFY(!SignalHandlerContext);
+ SignalHandlerContext = this;
+}
+
+TSignalHandlerContext::~TSignalHandlerContext()
+{
+ YT_VERIFY(SignalHandlerContext == this);
+ SignalHandlerContext = nullptr;
+}
+
+void SignalHandler(int sig, siginfo_t* /*info*/, void* threadContext)
+{
+ YT_VERIFY(sig == SIGUSR1);
+
+ SignalHandlerContext->FiberId = GetCurrentFiberId();
+ SignalHandlerContext->ThreadName = GetCurrentThreadName();
+ if (const auto* traceContext = TryGetCurrentTraceContext()) {
+ SignalHandlerContext->TraceId = traceContext->GetTraceId();
+ SignalHandlerContext->TraceLoggingTag = TStaticString(traceContext->GetLoggingTag());
+ }
+
+ auto cursorContext = FramePointerCursorContextFromUcontext(*static_cast<const ucontext_t*>(threadContext));
+ TFramePointerCursor cursor(SignalHandlerContext->MemoryReader, cursorContext);
+ while (!cursor.IsFinished() && SignalHandlerContext->Backtrace.FrameCount < std::ssize(SignalHandlerContext->Backtrace.Frames)) {
+ SignalHandlerContext->Backtrace.Frames[SignalHandlerContext->Backtrace.FrameCount++] = cursor.GetCurrentIP();
+ cursor.MoveNext();
+ }
+
+ SignalHandlerContext->SetFinished();
+}
+
+} // namespace
+
+std::vector<TThreadIntrospectionInfo> IntrospectThreads()
+{
+ static std::atomic<bool> IntrospectionLock;
+
+ if (IntrospectionLock.exchange(true)) {
+ THROW_ERROR_EXCEPTION("Thread introspection is already in progress");
+ }
+
+ auto introspectionLockGuard = Finally([] {
+ YT_VERIFY(IntrospectionLock.exchange(false));
+ });
+
+ YT_LOG_INFO("Thread introspection started");
+
+ {
+ struct sigaction action;
+ action.sa_flags = SA_SIGINFO | SA_RESTART;
+ ::sigemptyset(&action.sa_mask);
+ action.sa_sigaction = SignalHandler;
+
+ if (::sigaction(SIGUSR1, &action, nullptr) != 0) {
+ THROW_ERROR_EXCEPTION("Failed to install signal handler")
+ << TError::FromSystem();
+ }
+ }
+
+ std::vector<TThreadIntrospectionInfo> infos;
+ for (auto threadId : GetCurrentProcessThreadIds()) {
+ TSignalHandlerContext signalHandlerContext;
+ if (::syscall(SYS_tkill, threadId, SIGUSR1) != 0) {
+ YT_LOG_DEBUG(TError::FromSystem(), "Failed to signal to thread (ThreadId: %v)",
+ threadId);
+ continue;
+ }
+
+ YT_LOG_DEBUG("Sent signal to thread (ThreadId: %v)",
+ threadId);
+
+ signalHandlerContext.WaitUntilFinished();
+
+ YT_LOG_DEBUG("Signal handler finished (ThreadId: %v, FiberId: %x)",
+ threadId,
+ signalHandlerContext.FiberId);
+
+ infos.push_back(TThreadIntrospectionInfo{
+ .ThreadId = threadId,
+ .FiberId = signalHandlerContext.FiberId,
+ .ThreadName = TString(signalHandlerContext.ThreadName.Buffer.data(), static_cast<size_t>(signalHandlerContext.ThreadName.Length)),
+ .TraceId = signalHandlerContext.TraceId,
+ .TraceLoggingTag = signalHandlerContext.TraceLoggingTag,
+ .Backtrace = signalHandlerContext.Backtrace,
+ });
+ }
+
+ {
+ struct sigaction action;
+ action.sa_flags = SA_RESTART;
+ ::sigemptyset(&action.sa_mask);
+ action.sa_handler = SIG_IGN;
+
+ if (::sigaction(SIGUSR1, &action, nullptr) != 0) {
+ THROW_ERROR_EXCEPTION("Failed to de-install signal handler")
+ << TError::FromSystem();
+ }
+ }
+
+ YT_LOG_INFO("Thread introspection completed");
+
+ return infos;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/private.h b/yt/yt/library/backtrace_introspector/private.h
new file mode 100644
index 0000000000..59f25e6023
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/private.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT::NBacktraceIntrospector {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger BacktraceIntrospectorLogger("BacktraceIntrospector");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
+
diff --git a/yt/yt/library/backtrace_introspector/public.h b/yt/yt/library/backtrace_introspector/public.h
new file mode 100644
index 0000000000..54a8bd06ed
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/public.h
@@ -0,0 +1,12 @@
+#pragma once
+
+namespace NYT::NBacktraceIntrospector {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TThreadIntrospectionInfo;
+struct TFiberIntrospectionInfo;
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp b/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp
new file mode 100644
index 0000000000..a939417958
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp
@@ -0,0 +1,198 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/library/backtrace_introspector/introspect.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/delayed_executor.h>
+
+#include <yt/yt/core/actions/bind.h>
+#include <yt/yt/core/actions/future.h>
+
+#include <yt/yt/core/tracing/trace_context.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/collection_helpers.h>
+
+namespace NYT::NBacktraceIntrospector {
+namespace {
+
+using namespace NConcurrency;
+using namespace NTracing;
+
+////////////////////////////////////////////////////////////////////////////////
+
+NLogging::TLogger Logger("Test");
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TBacktraceIntrospectorTest, Fibers)
+{
+ constexpr int HeavyQueueCount = 5;
+ std::vector<TActionQueuePtr> heavyQueues;
+ const TString HeavyThreadNamePrefix("Heavy:");
+ for (int index = 0; index < HeavyQueueCount; ++index) {
+ heavyQueues.push_back(New<TActionQueue>(HeavyThreadNamePrefix + ToString(index)));
+ }
+
+ constexpr int LightQueueCount = 3;
+ std::vector<TActionQueuePtr> lightQueues;
+ const TString LightThreadNamePrefix("Light:");
+ for (int index = 0; index < LightQueueCount; ++index) {
+ lightQueues.push_back(New<TActionQueue>(LightThreadNamePrefix + ToString(index)));
+ }
+
+ constexpr int HeavyCallbackCount = 3;
+ std::vector<TTraceContextPtr> heavyTraceContexts;
+ std::set<TTraceId> expectedHeavyTraceIds;
+ for (int index = 0; index < HeavyCallbackCount; ++index) {
+ auto traceContext = TTraceContext::NewRoot("Heavy");
+ traceContext->SetLoggingTag(Format("HeavyLoggingTag:%v", index));
+ heavyTraceContexts.push_back(traceContext);
+ InsertOrCrash(expectedHeavyTraceIds, traceContext->GetTraceId());
+ }
+
+ std::vector<TFuture<void>> heavyFutures;
+ for (int index = 0; index < HeavyCallbackCount; ++index) {
+ heavyFutures.push_back(
+ BIND([&, index] {
+ TTraceContextGuard traceContextGuard(heavyTraceContexts[index]);
+ YT_LOG_INFO("Heavy callback started (Index: %v)", index);
+ Sleep(TDuration::Seconds(3));
+ YT_LOG_INFO("Heavy callback finished (Index: %v)", index);
+ })
+ .AsyncVia(heavyQueues[index % HeavyQueueCount]->GetInvoker())
+ .Run());
+ }
+
+ constexpr int LightCallbackCount = 10;
+ std::vector<TTraceContextPtr> lightTraceContexts;
+ std::set<TTraceId> expectedLightTraceIds;
+ for (int index = 0; index < LightCallbackCount; ++index) {
+ auto traceContext = TTraceContext::NewRoot("Light");
+ traceContext->SetLoggingTag(Format("LightLoggingTag:%v", index));
+ lightTraceContexts.push_back(traceContext);
+ InsertOrCrash(expectedLightTraceIds, traceContext->GetTraceId());
+ }
+
+ std::vector<TFuture<void>> lightFutures;
+ for (int index = 0; index < LightCallbackCount; ++index) {
+ lightFutures.push_back(
+ BIND([&, index] {
+ TTraceContextGuard traceContextGuard(lightTraceContexts[index]);
+ YT_LOG_INFO("Light callback started (Index: %v)", index);
+ TDelayedExecutor::WaitForDuration(TDuration::Seconds(1));
+ YT_LOG_INFO("Light callback finished (Index: %v)", index);
+ })
+ .AsyncVia(lightQueues[index % LightQueueCount]->GetInvoker())
+ .Run());
+ }
+
+ Sleep(TDuration::MilliSeconds(100));
+
+ auto infos = IntrospectFibers();
+ Cerr << FormatIntrospectionInfos(infos);
+
+ std::set<TTraceId> actualHeavyTraceIds;
+ std::set<TTraceId> actualLightTraceIds;
+ for (const auto& info : infos) {
+ if (!info.TraceId) {
+ continue;
+ }
+ switch (info.State) {
+ case EFiberState::Running:
+ EXPECT_TRUE(actualHeavyTraceIds.insert(info.TraceId).second);
+ if (expectedHeavyTraceIds.contains(info.TraceId)) {
+ EXPECT_TRUE(info.ThreadName.StartsWith(HeavyThreadNamePrefix));
+ }
+ break;
+
+ case EFiberState::Waiting:
+ EXPECT_TRUE(actualLightTraceIds.insert(info.TraceId).second);
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ EXPECT_EQ(expectedLightTraceIds, actualLightTraceIds);
+ EXPECT_EQ(expectedHeavyTraceIds, actualHeavyTraceIds);
+
+ for (const auto& future : heavyFutures) {
+ future.Get().ThrowOnError();
+ }
+
+ for (const auto& future : lightFutures) {
+ future.Get().ThrowOnError();
+ }
+
+ for (const auto& queue : heavyQueues) {
+ queue->Shutdown(/*graceful*/ true);
+ }
+ for (const auto& queue : lightQueues) {
+ queue->Shutdown(/*graceful*/ true);
+ }
+}
+
+TEST(TBacktraceIntrospectorTest, Threads)
+{
+ constexpr int QueueCount = 5;
+ std::vector<TActionQueuePtr> queues;
+ const TString ThreadNamePrefix("Queue:");
+ for (int index = 0; index < QueueCount; ++index) {
+ queues.push_back(New<TActionQueue>(ThreadNamePrefix + ToString(index)));
+ }
+
+ constexpr int CallbackCount = 3;
+ std::vector<TTraceContextPtr> traceContexts;
+ std::set<TTraceId> expectedTraceIds;
+ for (int index = 0; index < CallbackCount; ++index) {
+ auto traceContext = TTraceContext::NewRoot("Heavy");
+ traceContexts.push_back(traceContext);
+ InsertOrCrash(expectedTraceIds, traceContext->GetTraceId());
+ }
+
+ std::vector<TFuture<void>> futures;
+ for (int index = 0; index < CallbackCount; ++index) {
+ futures.push_back(
+ BIND([&, index] {
+ TTraceContextGuard traceContextGuard(traceContexts[index]);
+ YT_LOG_INFO("Callback started (Index: %v)", index);
+ Sleep(TDuration::Seconds(3));
+ YT_LOG_INFO("Callback finished (Index: %v)", index);
+ })
+ .AsyncVia(queues[index % QueueCount]->GetInvoker())
+ .Run());
+ }
+
+ Sleep(TDuration::MilliSeconds(100));
+
+ auto infos = IntrospectThreads();
+ Cerr << FormatIntrospectionInfos(infos);
+
+ std::set<TTraceId> actualTraceIds;
+ for (const auto& info : infos) {
+ if (!info.TraceId) {
+ continue;
+ }
+ EXPECT_TRUE(actualTraceIds.insert(info.TraceId).second);
+ if (expectedTraceIds.contains(info.TraceId)) {
+ EXPECT_TRUE(info.ThreadName.StartsWith(ThreadNamePrefix));
+ }
+ }
+
+ EXPECT_EQ(expectedTraceIds, actualTraceIds);
+
+ for (const auto& future : futures) {
+ future.Get().ThrowOnError();
+ }
+ for (const auto& queue : queues) {
+ queue->Shutdown(/*graceful*/ true);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NBacktraceIntrospector
diff --git a/yt/yt/library/backtrace_introspector/unittests/ya.make b/yt/yt/library/backtrace_introspector/unittests/ya.make
new file mode 100644
index 0000000000..953dc020a8
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/unittests/ya.make
@@ -0,0 +1,15 @@
+GTEST()
+
+SRCS(
+ introspect_ut.cpp
+)
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc)
+
+PEERDIR(
+ yt/yt/library/backtrace_introspector
+
+ yt/yt/core/test_framework
+)
+
+END()
diff --git a/yt/yt/library/backtrace_introspector/ya.make b/yt/yt/library/backtrace_introspector/ya.make
new file mode 100644
index 0000000000..884b8fb562
--- /dev/null
+++ b/yt/yt/library/backtrace_introspector/ya.make
@@ -0,0 +1,31 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ introspect.cpp
+)
+IF (OS_LINUX)
+ SRCS(introspect_linux.cpp)
+ELSE()
+ SRCS(introspect_dummy.cpp)
+ENDIF()
+
+PEERDIR(
+ yt/yt/core
+
+ library/cpp/yt/backtrace/cursors/interop
+ library/cpp/yt/backtrace/cursors/libunwind
+ library/cpp/yt/backtrace/cursors/frame_pointer
+ library/cpp/yt/misc
+)
+
+END()
+
+RECURSE(
+ http
+)
+
+RECURSE_FOR_TESTS(
+ unittests
+)
diff --git a/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt b/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..faab79bbf6
--- /dev/null
+++ b/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-containers)
+target_compile_options(yt-library-containers PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-containers PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-porto-proto
+ yt-library-process
+ yt-yt-core
+)
+target_sources(yt-library-containers PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp
+)
diff --git a/yt/yt/library/containers/CMakeLists.linux-aarch64.txt b/yt/yt/library/containers/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..d3ab3811e0
--- /dev/null
+++ b/yt/yt/library/containers/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-containers)
+target_compile_options(yt-library-containers PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-containers PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-porto-proto
+ yt-library-process
+ yt-yt-core
+ library-cpp-porto
+)
+target_sources(yt-library-containers PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp
+)
diff --git a/yt/yt/library/containers/CMakeLists.linux-x86_64.txt b/yt/yt/library/containers/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..d3ab3811e0
--- /dev/null
+++ b/yt/yt/library/containers/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-containers)
+target_compile_options(yt-library-containers PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-containers PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-porto-proto
+ yt-library-process
+ yt-yt-core
+ library-cpp-porto
+)
+target_sources(yt-library-containers PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp
+)
diff --git a/yt/yt/library/containers/CMakeLists.txt b/yt/yt/library/containers/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/containers/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/containers/CMakeLists.windows-x86_64.txt b/yt/yt/library/containers/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..998e1690fa
--- /dev/null
+++ b/yt/yt/library/containers/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,27 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-containers)
+target_link_libraries(yt-library-containers PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-porto-proto
+ yt-library-process
+ yt-yt-core
+)
+target_sources(yt-library-containers PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp
+)
diff --git a/yt/yt/library/containers/cgroup.cpp b/yt/yt/library/containers/cgroup.cpp
new file mode 100644
index 0000000000..b43ab1e14b
--- /dev/null
+++ b/yt/yt/library/containers/cgroup.cpp
@@ -0,0 +1,752 @@
+#include "cgroup.h"
+#include "private.h"
+
+#include <yt/yt/core/misc/fs.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/ytree/fluent.h>
+
+#include <util/string/split.h>
+#include <util/system/filemap.h>
+
+#include <util/system/yield.h>
+
+#ifdef _linux_
+ #include <unistd.h>
+ #include <sys/stat.h>
+ #include <errno.h>
+#endif
+
+namespace NYT::NContainers {
+
+using namespace NYTree;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = ContainersLogger;
+static const TString CGroupRootPath("/sys/fs/cgroup");
+#ifdef _linux_
+static const int ReadByAll = S_IRUSR | S_IRGRP | S_IROTH;
+static const int ReadExecuteByAll = ReadByAll | S_IXUSR | S_IXGRP | S_IXOTH;
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+TString GetParentFor(const TString& type)
+{
+#ifdef _linux_
+ auto rawData = TUnbufferedFileInput("/proc/self/cgroup")
+ .ReadAll();
+ auto result = ParseProcessCGroups(rawData);
+ return result[type];
+#else
+ Y_UNUSED(type);
+ return "_parent_";
+#endif
+}
+
+#ifdef _linux_
+
+std::vector<TString> ReadAllValues(const TString& fileName)
+{
+ auto raw = TUnbufferedFileInput(fileName)
+ .ReadAll();
+
+ YT_LOG_DEBUG("File %v contains %Qv",
+ fileName,
+ raw);
+
+ TVector<TString> values;
+ StringSplitter(raw.data())
+ .SplitBySet(" \n")
+ .SkipEmpty()
+ .Collect(&values);
+ return values;
+}
+
+TDuration FromJiffies(ui64 jiffies)
+{
+ static const auto TicksPerSecond = sysconf(_SC_CLK_TCK);
+ return TDuration::MicroSeconds(1000 * 1000 * jiffies / TicksPerSecond);
+}
+
+#endif
+
+} // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TKillProcessGroupTool::operator()(const TString& processGroupPath) const
+{
+ SafeSetUid(0);
+ TNonOwningCGroup group(processGroupPath);
+ group.Kill();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TNonOwningCGroup::TNonOwningCGroup(const TString& fullPath)
+ : FullPath_(fullPath)
+{ }
+
+TNonOwningCGroup::TNonOwningCGroup(const TString& type, const TString& name)
+ : FullPath_(NFS::CombinePaths({
+ CGroupRootPath,
+ type,
+ GetParentFor(type),
+ name
+ }))
+{ }
+
+TNonOwningCGroup::TNonOwningCGroup(TNonOwningCGroup&& other)
+ : FullPath_(std::move(other.FullPath_))
+{ }
+
+void TNonOwningCGroup::AddTask(int pid) const
+{
+ YT_LOG_INFO(
+ "Adding task to cgroup (Task: %v, Cgroup: %v)",
+ pid,
+ FullPath_);
+ Append("tasks", ToString(pid));
+}
+
+void TNonOwningCGroup::AddCurrentTask() const
+{
+ YT_VERIFY(!IsNull());
+#ifdef _linux_
+ auto pid = getpid();
+ AddTask(pid);
+#endif
+}
+
+TString TNonOwningCGroup::Get(const TString& name) const
+{
+ YT_VERIFY(!IsNull());
+ TString result;
+#ifdef _linux_
+ const auto path = GetPath(name);
+ result = TFileInput(path).ReadLine();
+#else
+ Y_UNUSED(name);
+#endif
+ return result;
+}
+
+void TNonOwningCGroup::Set(const TString& name, const TString& value) const
+{
+ YT_VERIFY(!IsNull());
+#ifdef _linux_
+ auto path = GetPath(name);
+ TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::WrOnly));
+ output << value;
+#else
+ Y_UNUSED(name);
+ Y_UNUSED(value);
+#endif
+}
+
+void TNonOwningCGroup::Append(const TString& name, const TString& value) const
+{
+ YT_VERIFY(!IsNull());
+#ifdef _linux_
+ auto path = GetPath(name);
+ TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::ForAppend));
+ output << value;
+#else
+ Y_UNUSED(name);
+ Y_UNUSED(value);
+#endif
+}
+
+bool TNonOwningCGroup::IsRoot() const
+{
+ return FullPath_ == CGroupRootPath;
+}
+
+bool TNonOwningCGroup::IsNull() const
+{
+ return FullPath_.empty();
+}
+
+bool TNonOwningCGroup::Exists() const
+{
+ return NFS::Exists(FullPath_);
+}
+
+std::vector<int> TNonOwningCGroup::GetProcesses() const
+{
+ std::vector<int> results;
+ if (!IsNull()) {
+#ifdef _linux_
+ auto values = ReadAllValues(GetPath("cgroup.procs"));
+ for (const auto& value : values) {
+ int pid = FromString<int>(value);
+ results.push_back(pid);
+ }
+#endif
+ }
+ return results;
+}
+
+std::vector<int> TNonOwningCGroup::GetTasks() const
+{
+ std::vector<int> results;
+ if (!IsNull()) {
+#ifdef _linux_
+ auto values = ReadAllValues(GetPath("tasks"));
+ for (const auto& value : values) {
+ int pid = FromString<int>(value);
+ results.push_back(pid);
+ }
+#endif
+ }
+ return results;
+}
+
+const TString& TNonOwningCGroup::GetFullPath() const
+{
+ return FullPath_;
+}
+
+std::vector<TNonOwningCGroup> TNonOwningCGroup::GetChildren() const
+{
+ // We retry enumerating directories, since it may fail with weird diagnostics if
+ // number of subcgroups changes.
+ while (true) {
+ try {
+ std::vector<TNonOwningCGroup> result;
+
+ if (IsNull()) {
+ return result;
+ }
+
+ auto directories = NFS::EnumerateDirectories(FullPath_);
+ for (const auto& directory : directories) {
+ result.emplace_back(NFS::CombinePaths(FullPath_, directory));
+ }
+ return result;
+ } catch (const std::exception& ex) {
+ YT_LOG_WARNING(ex, "Failed to list subcgroups (Path: %v)", FullPath_);
+ }
+ }
+}
+
+void TNonOwningCGroup::EnsureExistence() const
+{
+ YT_LOG_INFO("Creating cgroup (Cgroup: %v)", FullPath_);
+
+ YT_VERIFY(!IsNull());
+
+#ifdef _linux_
+ NFS::MakeDirRecursive(FullPath_, 0755);
+#endif
+}
+
+void TNonOwningCGroup::Lock() const
+{
+ Traverse(
+ BIND([] (const TNonOwningCGroup& group) { group.DoLock(); }),
+ BIND([] (const TNonOwningCGroup& /*group*/) {}));
+}
+
+void TNonOwningCGroup::Unlock() const
+{
+ Traverse(
+ BIND([] (const TNonOwningCGroup& /*group*/) {}),
+ BIND([] (const TNonOwningCGroup& group) { group.DoUnlock(); }));
+}
+
+void TNonOwningCGroup::Kill() const
+{
+ YT_VERIFY(!IsRoot());
+
+ Traverse(
+ BIND([] (const TNonOwningCGroup& group) { group.DoKill(); }),
+ BIND([] (const TNonOwningCGroup& /*group*/) {}));
+}
+
+void TNonOwningCGroup::RemoveAllSubcgroups() const
+{
+ Traverse(
+ BIND([] (const TNonOwningCGroup& group) {
+ group.TryUnlock();
+ }),
+ BIND([this_ = this] (const TNonOwningCGroup& group) {
+ if (this_ != &group) {
+ group.DoRemove();
+ }
+ }));
+}
+
+void TNonOwningCGroup::RemoveRecursive() const
+{
+ RemoveAllSubcgroups();
+ DoRemove();
+}
+
+void TNonOwningCGroup::DoLock() const
+{
+ YT_LOG_INFO("Locking cgroup (Cgroup: %v)", FullPath_);
+
+#ifdef _linux_
+ if (!IsNull()) {
+ int code = chmod(FullPath_.data(), ReadExecuteByAll);
+ YT_VERIFY(code == 0);
+
+ code = chmod(GetPath("tasks").data(), ReadByAll);
+ YT_VERIFY(code == 0);
+ }
+#endif
+}
+
+bool TNonOwningCGroup::TryUnlock() const
+{
+ YT_LOG_INFO("Unlocking cgroup (Cgroup: %v)", FullPath_);
+
+ if (!Exists()) {
+ return true;
+ }
+
+ bool result = true;
+
+#ifdef _linux_
+ if (!IsNull()) {
+ int code = chmod(GetPath("tasks").data(), ReadByAll | S_IWUSR);
+ if (code != 0) {
+ result = false;
+ }
+
+ code = chmod(FullPath_.data(), ReadExecuteByAll | S_IWUSR);
+ if (code != 0) {
+ result = false;
+ }
+ }
+#endif
+
+ return result;
+}
+
+void TNonOwningCGroup::DoUnlock() const
+{
+ YT_VERIFY(TryUnlock());
+}
+
+void TNonOwningCGroup::DoKill() const
+{
+ YT_LOG_DEBUG("Started killing processes in cgroup (Cgroup: %v)", FullPath_);
+
+#ifdef _linux_
+ while (true) {
+ auto pids = GetTasks();
+ if (pids.empty())
+ break;
+
+ YT_LOG_DEBUG("Killing processes (Pids: %v)", pids);
+
+ for (int pid : pids) {
+ auto result = kill(pid, SIGKILL);
+ if (result == -1) {
+ YT_VERIFY(errno == ESRCH);
+ }
+ }
+
+ ThreadYield();
+ }
+#endif
+
+ YT_LOG_DEBUG("Finished killing processes in cgroup (Cgroup: %v)", FullPath_);
+}
+
+void TNonOwningCGroup::DoRemove() const
+{
+ if (NFS::Exists(FullPath_)) {
+ NFS::Remove(FullPath_);
+ }
+}
+
+void TNonOwningCGroup::Traverse(
+ const TCallback<void(const TNonOwningCGroup&)>& preorderAction,
+ const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const
+{
+ preorderAction(*this);
+
+ for (const auto& child : GetChildren()) {
+ child.Traverse(preorderAction, postorderAction);
+ }
+
+ postorderAction(*this);
+}
+
+TString TNonOwningCGroup::GetPath(const TString& filename) const
+{
+ return NFS::CombinePaths(FullPath_, filename);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TCGroup::TCGroup(const TString& type, const TString& name)
+ : TNonOwningCGroup(type, name)
+{ }
+
+TCGroup::TCGroup(TCGroup&& other)
+ : TNonOwningCGroup(std::move(other))
+ , Created_(other.Created_)
+{
+ other.Created_ = false;
+}
+
+TCGroup::TCGroup(TNonOwningCGroup&& other)
+ : TNonOwningCGroup(std::move(other))
+ , Created_(false)
+{ }
+
+TCGroup::~TCGroup()
+{
+ if (Created_) {
+ Destroy();
+ }
+}
+
+void TCGroup::Create()
+{
+ EnsureExistence();
+ Created_ = true;
+}
+
+void TCGroup::Destroy()
+{
+ YT_LOG_INFO("Destroying cgroup (Cgroup: %v)", FullPath_);
+ YT_VERIFY(Created_);
+
+#ifdef _linux_
+ try {
+ NFS::Remove(FullPath_);
+ } catch (const std::exception& ex) {
+ YT_LOG_FATAL(ex, "Failed to destroy cgroup (Cgroup: %v)", FullPath_);
+ }
+#endif
+ Created_ = false;
+}
+
+bool TCGroup::IsCreated() const
+{
+ return Created_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+const TString TCpuAccounting::Name = "cpuacct";
+
+TCpuAccounting::TStatistics& operator-=(TCpuAccounting::TStatistics& lhs, const TCpuAccounting::TStatistics& rhs)
+{
+ #define XX(name) lhs.name = lhs.name.ValueOrThrow() - rhs.name.ValueOrThrow();
+ XX(UserUsageTime)
+ XX(SystemUsageTime)
+ XX(WaitTime)
+ XX(ThrottledTime)
+ XX(ContextSwitchesDelta)
+ XX(PeakThreadCount)
+ #undef XX
+ return lhs;
+}
+
+TCpuAccounting::TCpuAccounting(const TString& name)
+ : TCGroup(Name, name)
+{ }
+
+TCpuAccounting::TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup)
+ : TCGroup(std::move(nonOwningCGroup))
+{ }
+
+TCpuAccounting::TStatistics TCpuAccounting::GetStatisticsRecursive() const
+{
+ TCpuAccounting::TStatistics result;
+#ifdef _linux_
+ try {
+ auto path = NFS::CombinePaths(GetFullPath(), "cpuacct.stat");
+ auto values = ReadAllValues(path);
+ YT_VERIFY(values.size() == 4);
+
+ TString type[2];
+ ui64 jiffies[2];
+
+ for (int i = 0; i < 2; ++i) {
+ type[i] = values[2 * i];
+ jiffies[i] = FromString<ui64>(values[2 * i + 1]);
+ }
+
+ for (int i = 0; i < 2; ++i) {
+ if (type[i] == "user") {
+ result.UserUsageTime = FromJiffies(jiffies[i]);
+ } else if (type[i] == "system") {
+ result.SystemUsageTime = FromJiffies(jiffies[i]);
+ }
+ }
+ } catch (const std::exception& ex) {
+ YT_LOG_FATAL(
+ ex,
+ "Failed to retrieve CPU statistics from cgroup (Cgroup: %v)",
+ GetFullPath());
+ }
+#endif
+ return result;
+}
+
+TCpuAccounting::TStatistics TCpuAccounting::GetStatistics() const
+{
+ auto statistics = GetStatisticsRecursive();
+ for (auto& cgroup : GetChildren()) {
+ auto cpuCGroup = TCpuAccounting(std::move(cgroup));
+ statistics -= cpuCGroup.GetStatisticsRecursive();
+ }
+ return statistics;
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+const TString TCpu::Name = "cpu";
+
+static const int DefaultCpuShare = 1024;
+
+TCpu::TCpu(const TString& name)
+ : TCGroup(Name, name)
+{ }
+
+void TCpu::SetShare(double share)
+{
+ int cpuShare = static_cast<int>(share * DefaultCpuShare);
+ Set("cpu.shares", ToString(cpuShare));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+const TString TBlockIO::Name = "blkio";
+
+TBlockIO::TBlockIO(const TString& name)
+ : TCGroup(Name, name)
+{ }
+
+// For more information about format of data
+// read https://www.kernel.org/doc/Documentation/cgroups/blkio-controller.txt
+
+TBlockIO::TStatistics TBlockIO::GetStatistics() const
+{
+ TBlockIO::TStatistics result;
+#ifdef _linux_
+ auto bytesStats = GetDetailedStatistics("blkio.io_service_bytes");
+ for (const auto& item : bytesStats) {
+ if (item.Type == "Read") {
+ result.IOReadByte = result.IOReadByte.ValueOrThrow() + item.Value;
+ } else if (item.Type == "Write") {
+ result.IOWriteByte = result.IOReadByte.ValueOrThrow() + item.Value;
+ }
+ }
+
+ auto ioStats = GetDetailedStatistics("blkio.io_serviced");
+ for (const auto& item : ioStats) {
+ if (item.Type == "Read") {
+ result.IOReadOps = result.IOReadOps.ValueOrThrow() + item.Value;
+ result.IOOps = result.IOOps.ValueOrThrow() + item.Value;
+ } else if (item.Type == "Write") {
+ result.IOWriteOps = result.IOWriteOps.ValueOrThrow() + item.Value;
+ result.IOOps = result.IOOps.ValueOrThrow() + item.Value;
+ }
+ }
+#endif
+ return result;
+}
+
+std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiceBytes() const
+{
+ return GetDetailedStatistics("blkio.io_service_bytes");
+}
+
+std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiced() const
+{
+ return GetDetailedStatistics("blkio.io_serviced");
+}
+
+std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetDetailedStatistics(const char* filename) const
+{
+ std::vector<TBlockIO::TStatisticsItem> result;
+#ifdef _linux_
+ try {
+ auto path = NFS::CombinePaths(GetFullPath(), filename);
+ auto values = ReadAllValues(path);
+
+ int lineNumber = 0;
+ while (3 * lineNumber + 2 < std::ssize(values)) {
+ TStatisticsItem item;
+ item.DeviceId = values[3 * lineNumber];
+ item.Type = values[3 * lineNumber + 1];
+ item.Value = FromString<ui64>(values[3 * lineNumber + 2]);
+
+ {
+ auto guard = Guard(SpinLock_);
+ DeviceIds_.insert(item.DeviceId);
+ }
+
+ if (item.Type == "Read" || item.Type == "Write") {
+ result.push_back(item);
+
+ YT_LOG_DEBUG("IO operations serviced (OperationCount: %v, OperationType: %v, DeviceId: %v)",
+ item.Value,
+ item.Type,
+ item.DeviceId);
+ }
+ ++lineNumber;
+ }
+ } catch (const std::exception& ex) {
+ YT_LOG_FATAL(
+ ex,
+ "Failed to retrieve block IO statistics from cgroup (Cgroup: %v)",
+ GetFullPath());
+ }
+#else
+ Y_UNUSED(filename);
+#endif
+ return result;
+}
+
+void TBlockIO::ThrottleOperations(i64 operations) const
+{
+ auto guard = Guard(SpinLock_);
+ for (const auto& deviceId : DeviceIds_) {
+ auto value = Format("%v %v", deviceId, operations);
+ Append("blkio.throttle.read_iops_device", value);
+ Append("blkio.throttle.write_iops_device", value);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+const TString TMemory::Name = "memory";
+
+TMemory::TMemory(const TString& name)
+ : TCGroup(Name, name)
+{ }
+
+TMemory::TStatistics TMemory::GetStatistics() const
+{
+ TMemory::TStatistics result;
+#ifdef _linux_
+ try {
+ auto values = ReadAllValues(GetPath("memory.stat"));
+ int lineNumber = 0;
+ while (2 * lineNumber + 1 < std::ssize(values)) {
+ const auto& type = values[2 * lineNumber];
+ const auto& unparsedValue = values[2 * lineNumber + 1];
+ if (type == "rss") {
+ result.Rss = FromString<ui64>(unparsedValue);
+ }
+ if (type == "mapped_file") {
+ result.MappedFile = FromString<ui64>(unparsedValue);
+ }
+ if (type == "pgmajfault") {
+ result.MajorPageFaults = FromString<ui64>(unparsedValue);
+ }
+ ++lineNumber;
+ }
+ } catch (const std::exception& ex) {
+ YT_LOG_FATAL(
+ ex,
+ "Failed to retrieve memory statistics from cgroup (Cgroup: %v)",
+ GetFullPath());
+ }
+#endif
+ return result;
+}
+
+i64 TMemory::GetMaxMemoryUsage() const
+{
+ return FromString<i64>(Get("memory.max_usage_in_bytes"));
+}
+
+void TMemory::SetLimitInBytes(i64 bytes) const
+{
+ Set("memory.limit_in_bytes", ToString(bytes));
+}
+
+void TMemory::ForceEmpty() const
+{
+ Set("memory.force_empty", "0");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+const TString TFreezer::Name = "freezer";
+
+TFreezer::TFreezer(const TString& name)
+ : TCGroup(Name, name)
+{ }
+
+TString TFreezer::GetState() const
+{
+ return Get("freezer.state");
+}
+
+void TFreezer::Freeze() const
+{
+ Set("freezer.state", "FROZEN");
+}
+
+void TFreezer::Unfreeze() const
+{
+ Set("freezer.state", "THAWED");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::map<TString, TString> ParseProcessCGroups(const TString& str)
+{
+ std::map<TString, TString> result;
+
+ TVector<TString> values;
+ StringSplitter(str.data()).SplitBySet(":\n").SkipEmpty().Collect(&values);
+ for (size_t i = 0; i + 2 < values.size(); i += 3) {
+ // Check format.
+ FromString<int>(values[i]);
+
+ const auto& subsystemsSet = values[i + 1];
+ const auto& name = values[i + 2];
+
+ TVector<TString> subsystems;
+ StringSplitter(subsystemsSet.data()).Split(',').SkipEmpty().Collect(&subsystems);
+ for (const auto& subsystem : subsystems) {
+ if (!subsystem.StartsWith("name=")) {
+ int start = 0;
+ if (name.StartsWith("/")) {
+ start = 1;
+ }
+ result[subsystem] = name.substr(start);
+ }
+ }
+ }
+
+ return result;
+}
+
+std::map<TString, TString> GetProcessCGroups(pid_t pid)
+{
+ auto cgroupsPath = Format("/proc/%v/cgroup", pid);
+ auto rawCgroups = TFileInput{cgroupsPath}.ReadAll();
+ return ParseProcessCGroups(rawCgroups);
+}
+
+bool IsValidCGroupType(const TString& type)
+{
+ return
+ type == TCpuAccounting::Name ||
+ type == TCpu::Name ||
+ type == TBlockIO::Name ||
+ type == TMemory::Name ||
+ type == TFreezer::Name;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/cgroup.h b/yt/yt/library/containers/cgroup.h
new file mode 100644
index 0000000000..a61fbbddc3
--- /dev/null
+++ b/yt/yt/library/containers/cgroup.h
@@ -0,0 +1,290 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/actions/public.h>
+
+#include <yt/yt/core/ytree/yson_struct.h>
+#include <yt/yt/core/yson/public.h>
+
+#include <yt/yt/core/misc/property.h>
+
+#include <library/cpp/yt/threading/spin_lock.h>
+
+#include <vector>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void RemoveAllSubcgroups(const TString& path);
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TKillProcessGroupTool
+{
+ void operator()(const TString& processGroupPath) const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNonOwningCGroup
+ : private TNonCopyable
+{
+public:
+ DEFINE_BYREF_RO_PROPERTY(TString, FullPath);
+
+public:
+ TNonOwningCGroup() = default;
+ explicit TNonOwningCGroup(const TString& fullPath);
+ TNonOwningCGroup(const TString& type, const TString& name);
+ TNonOwningCGroup(TNonOwningCGroup&& other);
+
+ void AddTask(int pid) const;
+ void AddCurrentTask() const;
+
+ bool IsRoot() const;
+ bool IsNull() const;
+ bool Exists() const;
+
+ std::vector<int> GetProcesses() const;
+ std::vector<int> GetTasks() const;
+ const TString& GetFullPath() const;
+
+ std::vector<TNonOwningCGroup> GetChildren() const;
+
+ void EnsureExistence() const;
+
+ void Lock() const;
+ void Unlock() const;
+
+ void Kill() const;
+
+ void RemoveAllSubcgroups() const;
+ void RemoveRecursive() const;
+
+protected:
+ TString Get(const TString& name) const;
+ void Set(const TString& name, const TString& value) const;
+ void Append(const TString& name, const TString& value) const;
+
+ void DoLock() const;
+ void DoUnlock() const;
+
+ bool TryUnlock() const;
+
+ void DoKill() const;
+
+ void DoRemove() const;
+
+ void Traverse(
+ const TCallback<void(const TNonOwningCGroup&)>& preorderAction,
+ const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const;
+
+ TString GetPath(const TString& filename) const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCGroup
+ : public TNonOwningCGroup
+{
+protected:
+ TCGroup(const TString& type, const TString& name);
+ TCGroup(TNonOwningCGroup&& other);
+ TCGroup(TCGroup&& other);
+
+public:
+ ~TCGroup();
+
+ void Create();
+ void Destroy();
+
+ bool IsCreated() const;
+
+private:
+ bool Created_ = false;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCpuAccounting
+ : public TCGroup
+{
+public:
+ static const TString Name;
+
+ struct TStatistics
+ {
+ TErrorOr<TDuration> TotalUsageTime;
+ TErrorOr<TDuration> UserUsageTime;
+ TErrorOr<TDuration> SystemUsageTime;
+ TErrorOr<TDuration> WaitTime;
+ TErrorOr<TDuration> ThrottledTime;
+
+ TErrorOr<ui64> ThreadCount;
+ TErrorOr<ui64> ContextSwitches;
+ TErrorOr<ui64> ContextSwitchesDelta;
+ TErrorOr<ui64> PeakThreadCount;
+
+ TErrorOr<TDuration> LimitTime;
+ TErrorOr<TDuration> GuaranteeTime;
+ };
+
+ explicit TCpuAccounting(const TString& name);
+
+ TStatistics GetStatisticsRecursive() const;
+ TStatistics GetStatistics() const;
+
+private:
+ explicit TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup);
+};
+
+void Serialize(const TCpuAccounting::TStatistics& statistics, NYson::IYsonConsumer* consumer);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCpu
+ : public TCGroup
+{
+public:
+ static const TString Name;
+
+ explicit TCpu(const TString& name);
+
+ void SetShare(double share);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TBlockIO
+ : public TCGroup
+{
+public:
+ static const TString Name;
+
+ struct TStatistics
+ {
+ TErrorOr<ui64> IOReadByte;
+ TErrorOr<ui64> IOWriteByte;
+ TErrorOr<ui64> IOBytesLimit;
+
+ TErrorOr<ui64> IOReadOps;
+ TErrorOr<ui64> IOWriteOps;
+ TErrorOr<ui64> IOOps;
+ TErrorOr<ui64> IOOpsLimit;
+
+ TErrorOr<TDuration> IOTotalTime;
+ TErrorOr<TDuration> IOWaitTime;
+ };
+
+ struct TStatisticsItem
+ {
+ TString DeviceId;
+ TString Type;
+ ui64 Value = 0;
+ };
+
+ explicit TBlockIO(const TString& name);
+
+ TStatistics GetStatistics() const;
+ void ThrottleOperations(i64 iops) const;
+
+private:
+ //! Guards device ids.
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
+ //! Set of all seen device ids.
+ mutable THashSet<TString> DeviceIds_;
+
+ std::vector<TBlockIO::TStatisticsItem> GetDetailedStatistics(const char* filename) const;
+
+ std::vector<TStatisticsItem> GetIOServiceBytes() const;
+ std::vector<TStatisticsItem> GetIOServiced() const;
+};
+
+void Serialize(const TBlockIO::TStatistics& statistics, NYson::IYsonConsumer* consumer);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TMemory
+ : public TCGroup
+{
+public:
+ static const TString Name;
+
+ struct TStatistics
+ {
+ TErrorOr<ui64> Rss;
+ TErrorOr<ui64> MappedFile;
+ TErrorOr<ui64> MinorPageFaults;
+ TErrorOr<ui64> MajorPageFaults;
+
+ TErrorOr<ui64> FileCacheUsage;
+ TErrorOr<ui64> AnonUsage;
+ TErrorOr<ui64> AnonLimit;
+ TErrorOr<ui64> MemoryUsage;
+ TErrorOr<ui64> MemoryGuarantee;
+ TErrorOr<ui64> MemoryLimit;
+ TErrorOr<ui64> MaxMemoryUsage;
+
+ TErrorOr<ui64> OomKills;
+ TErrorOr<ui64> OomKillsTotal;
+ };
+
+ explicit TMemory(const TString& name);
+
+ TStatistics GetStatistics() const;
+ i64 GetMaxMemoryUsage() const;
+
+ void SetLimitInBytes(i64 bytes) const;
+
+ void ForceEmpty() const;
+};
+
+void Serialize(const TMemory::TStatistics& statistics, NYson::IYsonConsumer* consumer);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNetwork
+{
+public:
+ struct TStatistics
+ {
+ TErrorOr<ui64> TxBytes;
+ TErrorOr<ui64> TxPackets;
+ TErrorOr<ui64> TxDrops;
+ TErrorOr<ui64> TxLimit;
+
+ TErrorOr<ui64> RxBytes;
+ TErrorOr<ui64> RxPackets;
+ TErrorOr<ui64> RxDrops;
+ TErrorOr<ui64> RxLimit;
+ };
+};
+
+void Serialize(const TNetwork::TStatistics& statistics, NYson::IYsonConsumer* consumer);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TFreezer
+ : public TCGroup
+{
+public:
+ static const TString Name;
+
+ explicit TFreezer(const TString& name);
+
+ TString GetState() const;
+ void Freeze() const;
+ void Unfreeze() const;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::map<TString, TString> ParseProcessCGroups(const TString& str);
+std::map<TString, TString> GetProcessCGroups(pid_t pid);
+bool IsValidCGroupType(const TString& type);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/config.cpp b/yt/yt/library/containers/config.cpp
new file mode 100644
index 0000000000..39e46f2372
--- /dev/null
+++ b/yt/yt/library/containers/config.cpp
@@ -0,0 +1,64 @@
+#include "config.h"
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TPodSpecConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("cpu_to_vcpu_factor", &TThis::CpuToVCpuFactor)
+ .Default();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool TCGroupConfig::IsCGroupSupported(const TString& cgroupType) const
+{
+ auto it = std::find_if(
+ SupportedCGroups.begin(),
+ SupportedCGroups.end(),
+ [&] (const TString& type) {
+ return type == cgroupType;
+ });
+ return it != SupportedCGroups.end();
+}
+
+void TCGroupConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("supported_cgroups", &TThis::SupportedCGroups)
+ .Default();
+
+ registrar.Postprocessor([] (TThis* config) {
+ for (const auto& type : config->SupportedCGroups) {
+ if (!IsValidCGroupType(type)) {
+ THROW_ERROR_EXCEPTION("Invalid cgroup type %Qv", type);
+ }
+ }
+ });
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TPortoExecutorDynamicConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("retries_timeout", &TThis::RetriesTimeout)
+ .Default(TDuration::Seconds(10));
+ registrar.Parameter("poll_period", &TThis::PollPeriod)
+ .Default(TDuration::MilliSeconds(100));
+ registrar.Parameter("api_timeout", &TThis::ApiTimeout)
+ .Default(TDuration::Minutes(5));
+ registrar.Parameter("api_disk_timeout", &TThis::ApiDiskTimeout)
+ .Default(TDuration::Minutes(30));
+ registrar.Parameter("enable_network_isolation", &TThis::EnableNetworkIsolation)
+ .Default(true);
+ registrar.Parameter("enable_test_porto_failures", &TThis::EnableTestPortoFailures)
+ .Default(false);
+ registrar.Parameter("stub_error_code", &TThis::StubErrorCode)
+ .Default(EPortoErrorCode::SocketError);
+ registrar.Parameter("enable_test_porto_not_responding", &TThis::EnableTestPortoNotResponding)
+ .Default(false);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/config.h b/yt/yt/library/containers/config.h
new file mode 100644
index 0000000000..3639274cff
--- /dev/null
+++ b/yt/yt/library/containers/config.h
@@ -0,0 +1,64 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/ytree/yson_struct.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPodSpecConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ std::optional<double> CpuToVCpuFactor;
+
+ REGISTER_YSON_STRUCT(TPodSpecConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TPodSpecConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCGroupConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ std::vector<TString> SupportedCGroups;
+
+ bool IsCGroupSupported(const TString& cgroupType) const;
+
+ REGISTER_YSON_STRUCT(TCGroupConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoExecutorDynamicConfig
+ : public NYTree::TYsonStruct
+{
+public:
+ TDuration RetriesTimeout;
+ TDuration PollPeriod;
+ TDuration ApiTimeout;
+ TDuration ApiDiskTimeout;
+ bool EnableNetworkIsolation;
+ bool EnableTestPortoFailures;
+ bool EnableTestPortoNotResponding;
+
+ EPortoErrorCode StubErrorCode;
+
+ REGISTER_YSON_STRUCT(TPortoExecutorDynamicConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TPortoExecutorDynamicConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/cri/config.cpp b/yt/yt/library/containers/cri/config.cpp
new file mode 100644
index 0000000000..5572f4d980
--- /dev/null
+++ b/yt/yt/library/containers/cri/config.cpp
@@ -0,0 +1,54 @@
+#include "config.h"
+#include "cri_api.h"
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TCriExecutorConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("runtime_endpoint", &TThis::RuntimeEndpoint)
+ .Default(TString(DefaultCriEndpoint));
+
+ registrar.Parameter("image_endpoint", &TThis::ImageEndpoint)
+ .Default(TString(DefaultCriEndpoint));
+
+ registrar.Parameter("namespace", &TThis::Namespace)
+ .NonEmpty();
+
+ registrar.Parameter("runtime_handler", &TThis::RuntimeHandler)
+ .Optional();
+
+ registrar.Parameter("base_cgroup", &TThis::BaseCgroup)
+ .NonEmpty();
+
+ registrar.Parameter("cpu_period", &TThis::CpuPeriod)
+ .Default(TDuration::MilliSeconds(100));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TCriAuthConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("username", &TThis::Username)
+ .Optional();
+
+ registrar.Parameter("password", &TThis::Password)
+ .Optional();
+
+ registrar.Parameter("auth", &TThis::Auth)
+ .Optional();
+
+ registrar.Parameter("server_address", &TThis::ServerAddress)
+ .Optional();
+
+ registrar.Parameter("identity_token", &TThis::IdentityToken)
+ .Optional();
+
+ registrar.Parameter("registry_token", &TThis::RegistryToken)
+ .Optional();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/config.h b/yt/yt/library/containers/cri/config.h
new file mode 100644
index 0000000000..4ea33fd390
--- /dev/null
+++ b/yt/yt/library/containers/cri/config.h
@@ -0,0 +1,70 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/rpc/config.h>
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCriExecutorConfig
+ : public NRpc::TRetryingChannelConfig
+{
+public:
+ //! gRPC endpoint for CRI container runtime service.
+ TString RuntimeEndpoint;
+
+ //! gRPC endpoint for CRI image manager service.
+ TString ImageEndpoint;
+
+ //! CRI namespace where this executor operates.
+ TString Namespace;
+
+ //! Name of CRI runtime configuration to use.
+ TString RuntimeHandler;
+
+ //! Common parent cgroup for all pods.
+ TString BaseCgroup;
+
+ //! Cpu quota period for cpu limits.
+ TDuration CpuPeriod;
+
+ REGISTER_YSON_STRUCT(TCriExecutorConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TCriExecutorConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+// TODO(khlebnikov): split docker registry stuff into common "docker" library.
+
+//! TCriAuthConfig depicts docker registry authentification
+class TCriAuthConfig
+ : public NYTree::TYsonStruct
+{
+public:
+ TString Username;
+
+ TString Password;
+
+ TString Auth;
+
+ TString ServerAddress;
+
+ TString IdentityToken;
+
+ TString RegistryToken;
+
+ REGISTER_YSON_STRUCT(TCriAuthConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TCriAuthConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/cri_api.cpp b/yt/yt/library/containers/cri/cri_api.cpp
new file mode 100644
index 0000000000..93457017ba
--- /dev/null
+++ b/yt/yt/library/containers/cri/cri_api.cpp
@@ -0,0 +1,33 @@
+#include "cri_api.h"
+
+namespace NYT::NContainers::NCri {
+
+using namespace NRpc;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TCriRuntimeApi::TCriRuntimeApi(IChannelPtr channel)
+ : TProxyBase(std::move(channel), GetDescriptor())
+{ }
+
+const TServiceDescriptor& TCriRuntimeApi::GetDescriptor()
+{
+ static const auto Descriptor = TServiceDescriptor(NProto::RuntimeService::service_full_name());
+ return Descriptor;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TCriImageApi::TCriImageApi(IChannelPtr channel)
+ : TProxyBase(std::move(channel), GetDescriptor())
+{ }
+
+const TServiceDescriptor& TCriImageApi::GetDescriptor()
+{
+ static const auto Descriptor = TServiceDescriptor(NProto::ImageService::service_full_name());
+ return Descriptor;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/cri_api.h b/yt/yt/library/containers/cri/cri_api.h
new file mode 100644
index 0000000000..74fe9a64a0
--- /dev/null
+++ b/yt/yt/library/containers/cri/cri_api.h
@@ -0,0 +1,99 @@
+#pragma once
+
+#include <yt/yt/core/rpc/client.h>
+
+#include <k8s.io/cri-api/pkg/apis/runtime/v1/api.grpc.pb.h>
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NProto = ::runtime::v1;
+
+//! Reasonable default for CRI gRPC socket address.
+constexpr TStringBuf DefaultCriEndpoint = "unix:///run/containerd/containerd.sock";
+
+//! RuntimeReady means the runtime is up and ready to accept basic containers.
+constexpr TStringBuf RuntimeReady = "RuntimeReady";
+
+//! NetworkReady means the runtime network is up and ready to accept containers which require network.
+constexpr TStringBuf NetworkReady = "NetworkReady";
+
+//! CRI uses cgroupfs notation for systemd slices, but each name must ends with ".slice".
+constexpr TStringBuf SystemdSliceSuffix = ".slice";
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! CRI labels for pods and containers managed by YT
+constexpr TStringBuf YTPodNamespaceLabel = "tech.ytsaurus.pod.namespace";
+constexpr TStringBuf YTPodNameLabel = "tech.ytsaurus.pod.name";
+constexpr TStringBuf YTContainerNameLabel = "tech.ytsaurus.container.name";
+constexpr TStringBuf YTJobIdLabel = "tech.ytsaurus.job.id";
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define DEFINE_CRI_API_METHOD(method, ...) \
+ DEFINE_RPC_PROXY_METHOD_GENERIC(method, NProto::method##Request, NProto::method##Response, __VA_ARGS__)
+
+//! See https://github.com/kubernetes/cri-api
+class TCriRuntimeApi
+ : public NRpc::TProxyBase
+{
+public:
+ explicit TCriRuntimeApi(NRpc::IChannelPtr channel);
+
+ static const NRpc::TServiceDescriptor& GetDescriptor();
+
+ DEFINE_CRI_API_METHOD(Version);
+ DEFINE_CRI_API_METHOD(RunPodSandbox);
+ DEFINE_CRI_API_METHOD(StopPodSandbox);
+ DEFINE_CRI_API_METHOD(RemovePodSandbox);
+ DEFINE_CRI_API_METHOD(PodSandboxStatus);
+ DEFINE_CRI_API_METHOD(ListPodSandbox);
+ DEFINE_CRI_API_METHOD(CreateContainer);
+ DEFINE_CRI_API_METHOD(StartContainer);
+ DEFINE_CRI_API_METHOD(StopContainer);
+ DEFINE_CRI_API_METHOD(RemoveContainer);
+ DEFINE_CRI_API_METHOD(ListContainers);
+ DEFINE_CRI_API_METHOD(ContainerStatus);
+ DEFINE_CRI_API_METHOD(UpdateContainerResources);
+ DEFINE_CRI_API_METHOD(ReopenContainerLog);
+ DEFINE_CRI_API_METHOD(ExecSync);
+ DEFINE_CRI_API_METHOD(Exec);
+ DEFINE_CRI_API_METHOD(Attach);
+ DEFINE_CRI_API_METHOD(PortForward);
+ DEFINE_CRI_API_METHOD(ContainerStats);
+ DEFINE_CRI_API_METHOD(ListContainerStats);
+ DEFINE_CRI_API_METHOD(PodSandboxStats);
+ DEFINE_CRI_API_METHOD(ListPodSandboxStats);
+ DEFINE_CRI_API_METHOD(UpdateRuntimeConfig);
+ DEFINE_CRI_API_METHOD(Status);
+ DEFINE_CRI_API_METHOD(CheckpointContainer);
+ DEFINE_CRI_API_METHOD(ListMetricDescriptors);
+ DEFINE_CRI_API_METHOD(ListPodSandboxMetrics);
+
+ // FIXME(khlebnikov): figure out streaming results
+ // DEFINE_RPC_PROXY_METHOD_GENERIC(GetContainerEvents, NProto::GetEventsRequest, NProto::ContainerEventResponse,
+ // .SetStreamingEnabled(true));
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCriImageApi
+ : public NRpc::TProxyBase
+{
+public:
+ explicit TCriImageApi(NRpc::IChannelPtr channel);
+
+ static const NRpc::TServiceDescriptor& GetDescriptor();
+
+ DEFINE_CRI_API_METHOD(ListImages);
+ DEFINE_CRI_API_METHOD(ImageStatus);
+ DEFINE_CRI_API_METHOD(PullImage);
+ DEFINE_CRI_API_METHOD(RemoveImage);
+ DEFINE_CRI_API_METHOD(ImageFsInfo);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/cri_executor.cpp b/yt/yt/library/containers/cri/cri_executor.cpp
new file mode 100644
index 0000000000..428fd93165
--- /dev/null
+++ b/yt/yt/library/containers/cri/cri_executor.cpp
@@ -0,0 +1,666 @@
+#include "cri_executor.h"
+#include "private.h"
+
+#include <yt/yt/core/actions/bind.h>
+
+#include <yt/yt/core/rpc/grpc/channel.h>
+
+#include <yt/yt/core/rpc/retrying_channel.h>
+
+#include <yt/yt/core/misc/error.h>
+#include <yt/yt/core/misc/proc.h>
+#include <yt/yt/core/misc/protobuf_helpers.h>
+
+#include <yt/yt/core/concurrency/periodic_executor.h>
+
+namespace NYT::NContainers::NCri {
+
+using namespace NRpc;
+using namespace NRpc::NGrpc;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf /*spec*/)
+{
+ builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name);
+}
+
+void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf /*spec*/)
+{
+ builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name);
+}
+
+void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf /*spec*/)
+{
+ builder->AppendString(descriptor.Image);
+}
+
+static TError DecodeExitCode(int exitCode, const TString& reason)
+{
+ if (exitCode == 0) {
+ return TError();
+ }
+
+ // TODO(khkebnikov) map reason == "OOMKilled"
+
+ // Common bash notation for signals: 128 + signal
+ if (exitCode > 128) {
+ int signalNumber = exitCode - 128;
+ return TError(
+ EProcessErrorCode::Signal,
+ "Process terminated by signal %v",
+ signalNumber)
+ << TErrorAttribute("signal", signalNumber)
+ << TErrorAttribute("reason", reason);
+ }
+
+ // TODO(khkebnikov) check these
+ // 125 - container failed to run
+ // 126 - non executable
+ // 127 - command not found
+ // 128 - invalid exit code
+ // 255 - exit code out of range
+
+ return TError(
+ EProcessErrorCode::NonZeroExitCode,
+ "Process exited with code %v",
+ exitCode)
+ << TErrorAttribute("exit_code", exitCode)
+ << TErrorAttribute("reason", reason);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCriProcess
+ : public TProcessBase
+{
+public:
+ TCriProcess(
+ const TString& path,
+ ICriExecutorPtr executor,
+ TCriContainerSpecPtr containerSpec,
+ const TCriPodDescriptor& podDescriptor,
+ TCriPodSpecPtr podSpec,
+ TDuration pollPeriod = TDuration::MilliSeconds(100))
+ : TProcessBase(path)
+ , Executor_(std::move(executor))
+ , ContainerSpec_(std::move(containerSpec))
+ , PodDescriptor_(podDescriptor)
+ , PodSpec_(std::move(podSpec))
+ , PollPeriod_(pollPeriod)
+ {
+ // Just for symmetry with sibling classes.
+ AddArgument(Path_);
+ }
+
+ void Kill(int /*signal*/) override
+ {
+ WaitFor(Executor_->StopContainer(ContainerDescriptor_))
+ .ThrowOnError();
+ }
+
+ NNet::IConnectionWriterPtr GetStdInWriter() override
+ {
+ THROW_ERROR_EXCEPTION("Not implemented for CRI process");
+ }
+
+ NNet::IConnectionReaderPtr GetStdOutReader() override
+ {
+ THROW_ERROR_EXCEPTION("Not implemented for CRI process");
+ }
+
+ NNet::IConnectionReaderPtr GetStdErrReader() override
+ {
+ THROW_ERROR_EXCEPTION("Not implemented for CRI process");
+ }
+
+private:
+ const ICriExecutorPtr Executor_;
+ const TCriContainerSpecPtr ContainerSpec_;
+ const TCriPodDescriptor PodDescriptor_;
+ const TCriPodSpecPtr PodSpec_;
+ const TDuration PollPeriod_;
+
+ TCriDescriptor ContainerDescriptor_;
+
+ TPeriodicExecutorPtr AsyncWaitExecutor_;
+
+ void DoSpawn() override
+ {
+ if (ContainerSpec_->Command.empty()) {
+ ContainerSpec_->Command = {Path_};
+ }
+ ContainerSpec_->Arguments = std::vector<TString>(Args_.begin() + 1, Args_.end());
+ ContainerSpec_->WorkingDirectory = WorkingDirectory_;
+
+ ContainerSpec_->BindMounts.emplace_back(
+ NCri::TCriBindMount {
+ .ContainerPath = WorkingDirectory_,
+ .HostPath = WorkingDirectory_,
+ .ReadOnly = false,
+ }
+ );
+
+ for (const auto& keyVal : Env_) {
+ TStringBuf key, val;
+ if (TStringBuf(keyVal).TrySplit('=', key, val)) {
+ ContainerSpec_->Environment[key] = val;
+ }
+ }
+
+ ContainerDescriptor_ = WaitFor(Executor_->CreateContainer(ContainerSpec_, PodDescriptor_, PodSpec_))
+ .ValueOrThrow();
+
+ YT_LOG_DEBUG("Spawning process (Command: %v, Container: %v)", ContainerSpec_->Command[0], ContainerDescriptor_);
+ WaitFor(Executor_->StartContainer(ContainerDescriptor_))
+ .ThrowOnError();
+
+ // TODO(khkebnikov) replace polling with CRI event
+ AsyncWaitExecutor_ = New<TPeriodicExecutor>(
+ GetSyncInvoker(),
+ BIND(&TCriProcess::PollContainerStatus, MakeStrong(this)),
+ PollPeriod_);
+
+ AsyncWaitExecutor_->Start();
+ }
+
+ void PollContainerStatus()
+ {
+ Executor_->GetContainerStatus(ContainerDescriptor_)
+ .SubscribeUnique(BIND(&TCriProcess::OnContainerStatus, MakeStrong(this)));
+ }
+
+ void OnContainerStatus(TErrorOr<TCriRuntimeApi::TRspContainerStatusPtr>&& responseOrError)
+ {
+ auto response = responseOrError.ValueOrThrow();
+ if (!response->has_status()) {
+ return;
+ }
+ auto status = response->status();
+ if (status.state() == NProto::CONTAINER_EXITED) {
+ auto error = DecodeExitCode(status.exit_code(), status.reason());
+ YT_LOG_DEBUG(error, "Process finished (Container: %v)", ContainerDescriptor_);
+ YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop());
+ FinishedPromise_.TrySet(error);
+ }
+ }
+};
+
+DEFINE_REFCOUNTED_TYPE(TCriProcess)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCriExecutor
+ : public ICriExecutor
+{
+public:
+ TCriExecutor(
+ TCriExecutorConfigPtr config,
+ IChannelFactoryPtr channelFactory)
+ : Config_(std::move(config))
+ , RuntimeApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->RuntimeEndpoint)))
+ , ImageApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->ImageEndpoint)))
+ { }
+
+ TString GetPodCgroup(TString podName) const override
+ {
+ TStringBuilder cgroup;
+ cgroup.AppendString(Config_->BaseCgroup);
+ cgroup.AppendString("/");
+ cgroup.AppendString(podName);
+ if (Config_->BaseCgroup.EndsWith(SystemdSliceSuffix)) {
+ cgroup.AppendString(SystemdSliceSuffix);
+ }
+ return cgroup.Flush();
+ }
+
+ TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) override
+ {
+ auto req = RuntimeApi_.Status();
+ req->set_verbose(verbose);
+ return req->Invoke();
+ }
+
+ TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox(
+ std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) override
+ {
+ auto req = RuntimeApi_.ListPodSandbox();
+
+ {
+ auto* filter = req->mutable_filter();
+
+ if (auto namespace_ = Config_->Namespace) {
+ auto& labels = *filter->mutable_label_selector();
+ labels[YTPodNamespaceLabel] = namespace_;
+ }
+
+ if (initFilter) {
+ initFilter(*filter);
+ }
+ }
+
+ return req->Invoke();
+ }
+
+ TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers(
+ std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override
+ {
+ auto req = RuntimeApi_.ListContainers();
+
+ {
+ auto* filter = req->mutable_filter();
+
+ if (auto namespace_ = Config_->Namespace) {
+ auto& labels = *filter->mutable_label_selector();
+ labels[YTPodNamespaceLabel] = namespace_;
+ }
+
+ if (initFilter) {
+ initFilter(*filter);
+ }
+ }
+
+ return req->Invoke();
+ }
+
+ TFuture<void> ForEachPodSandbox(
+ const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback,
+ std::function<void(NProto::PodSandboxFilter&)> initFilter) override
+ {
+ return ListPodSandbox(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListPodSandboxPtr& rsp) {
+ for (const auto& pod : rsp->items()) {
+ TCriPodDescriptor descriptor{.Name=pod.metadata().name(), .Id=pod.id()};
+ callback(descriptor, pod);
+ }
+ }));
+ }
+
+ TFuture<void> ForEachContainer(
+ const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback,
+ std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override
+ {
+ return ListContainers(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListContainersPtr& rsp) {
+ for (const auto& ct : rsp->containers()) {
+ TCriDescriptor descriptor{.Name=ct.metadata().name(), .Id=ct.id()};
+ callback(descriptor, ct);
+ }
+ }));
+ }
+
+ TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus(
+ const TCriPodDescriptor& podDescriptor, bool verbose = false) override
+ {
+ auto req = RuntimeApi_.PodSandboxStatus();
+ req->set_pod_sandbox_id(podDescriptor.Id);
+ req->set_verbose(verbose);
+ return req->Invoke();
+ }
+
+ TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus(
+ const TCriDescriptor& descriptor, bool verbose = false) override
+ {
+ auto req = RuntimeApi_.ContainerStatus();
+ req->set_container_id(descriptor.Id);
+ req->set_verbose(verbose);
+ return req->Invoke();
+ }
+
+ TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) override
+ {
+ auto req = RuntimeApi_.RunPodSandbox();
+
+ FillPodSandboxConfig(req->mutable_config(), *podSpec);
+
+ if (Config_->RuntimeHandler) {
+ req->set_runtime_handler(Config_->RuntimeHandler);
+ }
+
+ return req->Invoke().Apply(BIND([name = podSpec->Name] (const TCriRuntimeApi::TRspRunPodSandboxPtr& rsp) -> TCriPodDescriptor {
+ return TCriPodDescriptor{.Name = name, .Id = rsp->pod_sandbox_id()};
+ }));
+ }
+
+ TFuture<void> StopPodSandbox(const TCriPodDescriptor& podDescriptor) override
+ {
+ auto req = RuntimeApi_.StopPodSandbox();
+ req->set_pod_sandbox_id(podDescriptor.Id);
+ return req->Invoke().AsVoid();
+ }
+
+ TFuture<void> RemovePodSandbox(const TCriPodDescriptor& podDescriptor) override
+ {
+ auto req = RuntimeApi_.RemovePodSandbox();
+ req->set_pod_sandbox_id(podDescriptor.Id);
+ return req->Invoke().AsVoid();
+ }
+
+ TFuture<void> UpdatePodResources(
+ const TCriPodDescriptor& /*pod*/,
+ const TCriContainerResources& /*resources*/) override
+ {
+ return MakeFuture(TError("Not implemented"));
+ }
+
+ TFuture<TCriDescriptor> CreateContainer(
+ TCriContainerSpecPtr ctSpec,
+ const TCriPodDescriptor& podDescriptor,
+ TCriPodSpecPtr podSpec) override
+ {
+ auto req = RuntimeApi_.CreateContainer();
+ req->set_pod_sandbox_id(podDescriptor.Id);
+
+ auto* config = req->mutable_config();
+
+ {
+ auto* metadata = config->mutable_metadata();
+ metadata->set_name(ctSpec->Name);
+ }
+
+ {
+ auto& labels = *config->mutable_labels();
+
+ for (const auto& [key, val] : ctSpec->Labels) {
+ labels[key] = val;
+ }
+
+ labels[YTPodNamespaceLabel] = Config_->Namespace;
+ labels[YTPodNameLabel] = podSpec->Name;
+ labels[YTContainerNameLabel] = ctSpec->Name;
+ }
+
+ FillImageSpec(config->mutable_image(), ctSpec->Image);
+
+ for (const auto& mountSpec : ctSpec->BindMounts) {
+ auto* mount = config->add_mounts();
+ mount->set_container_path(mountSpec.ContainerPath);
+ mount->set_host_path(mountSpec.HostPath);
+ mount->set_readonly(mountSpec.ReadOnly);
+ mount->set_propagation(NProto::PROPAGATION_PRIVATE);
+ }
+
+ {
+ ToProto(config->mutable_command(), ctSpec->Command);
+ ToProto(config->mutable_args(), ctSpec->Arguments);
+
+ config->set_working_dir(ctSpec->WorkingDirectory);
+
+ for (const auto& [key, val] : ctSpec->Environment) {
+ auto* env = config->add_envs();
+ env->set_key(key);
+ env->set_value(val);
+ }
+ }
+
+ {
+ auto* linux = config->mutable_linux();
+ FillLinuxContainerResources(linux->mutable_resources(), ctSpec->Resources);
+
+ auto* security = linux->mutable_security_context();
+
+ auto* namespaces = security->mutable_namespace_options();
+ namespaces->set_network(NProto::NODE);
+
+ security->set_readonly_rootfs(ctSpec->ReadOnlyRootFS);
+
+ if (ctSpec->Credentials.Uid) {
+ security->mutable_run_as_user()->set_value(*ctSpec->Credentials.Uid);
+ }
+ if (ctSpec->Credentials.Gid) {
+ security->mutable_run_as_group()->set_value(*ctSpec->Credentials.Gid);
+ }
+ ToProto(security->mutable_supplemental_groups(), ctSpec->Credentials.Groups);
+ }
+
+ FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec);
+
+ return req->Invoke().Apply(BIND([name = ctSpec->Name] (const TCriRuntimeApi::TRspCreateContainerPtr& rsp) -> TCriDescriptor {
+ return TCriDescriptor{.Name = "", .Id = rsp->container_id()};
+ }));
+ }
+
+ TFuture<void> StartContainer(const TCriDescriptor& descriptor) override
+ {
+ auto req = RuntimeApi_.StartContainer();
+ req->set_container_id(descriptor.Id);
+ return req->Invoke().AsVoid();
+ }
+
+ TFuture<void> StopContainer(const TCriDescriptor& descriptor, TDuration timeout) override
+ {
+ auto req = RuntimeApi_.StopContainer();
+ req->set_container_id(descriptor.Id);
+ req->set_timeout(timeout.Seconds());
+ return req->Invoke().AsVoid();
+ }
+
+ TFuture<void> RemoveContainer(const TCriDescriptor& descriptor) override
+ {
+ auto req = RuntimeApi_.RemoveContainer();
+ req->set_container_id(descriptor.Id);
+ return req->Invoke().AsVoid();
+ }
+
+ TFuture<void> UpdateContainerResources(const TCriDescriptor& descriptor, const TCriContainerResources& resources) override
+ {
+ auto req = RuntimeApi_.UpdateContainerResources();
+ req->set_container_id(descriptor.Id);
+ FillLinuxContainerResources(req->mutable_linux(), resources);
+ return req->Invoke().AsVoid();
+ }
+
+ void CleanNamespace() override
+ {
+ YT_VERIFY(Config_->Namespace);
+ auto pods = WaitFor(ListPodSandbox())
+ .ValueOrThrow();
+
+ {
+ std::vector<TFuture<void>> futures;
+ futures.reserve(pods->items_size());
+ for (const auto& pod : pods->items()) {
+ TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id() };
+ futures.push_back(StopPodSandbox(podDescriptor));
+ }
+ WaitFor(AllSucceeded(std::move(futures)))
+ .ThrowOnError();
+ }
+
+ {
+ std::vector<TFuture<void>> futures;
+ futures.reserve(pods->items_size());
+ for (const auto& pod : pods->items()) {
+ TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id()};
+ futures.push_back(RemovePodSandbox(podDescriptor));
+ }
+ WaitFor(AllSucceeded(std::move(futures)))
+ .ThrowOnError();
+ }
+ }
+
+ void CleanPodSandbox(const TCriPodDescriptor& podDescriptor) override
+ {
+ auto containers = WaitFor(ListContainers([=] (NProto::ContainerFilter& filter) {
+ filter.set_pod_sandbox_id(podDescriptor.Id);
+ }))
+ .ValueOrThrow();
+
+ {
+ std::vector<TFuture<void>> futures;
+ futures.reserve(containers->containers_size());
+ for (const auto& ct : containers->containers()) {
+ TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()};
+ futures.push_back(StopContainer(ctDescriptor, TDuration::Zero()));
+ }
+ WaitFor(AllSucceeded(std::move(futures)))
+ .ThrowOnError();
+ }
+
+ {
+ std::vector<TFuture<void>> futures;
+ futures.reserve(containers->containers_size());
+ for (const auto& ct : containers->containers()) {
+ TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()};
+ futures.push_back(RemoveContainer(ctDescriptor));
+ }
+ WaitFor(AllSucceeded(std::move(futures)))
+ .ThrowOnError();
+ }
+ }
+
+ TFuture<TCriImageApi::TRspListImagesPtr> ListImages(
+ std::function<void(NProto::ImageFilter&)> initFilter = nullptr) override
+ {
+ auto req = ImageApi_.ListImages();
+ if (initFilter) {
+ initFilter(*req->mutable_filter());
+ }
+ return req->Invoke();
+ }
+
+ TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus(
+ const TCriImageDescriptor& image,
+ bool verbose = false) override
+ {
+ auto req = ImageApi_.ImageStatus();
+ FillImageSpec(req->mutable_image(), image);
+ req->set_verbose(verbose);
+ return req->Invoke();
+ }
+
+ TFuture<TCriImageDescriptor> PullImage(
+ const TCriImageDescriptor& image,
+ bool always,
+ TCriAuthConfigPtr authConfig,
+ TCriPodSpecPtr podSpec) override
+ {
+ if (!always) {
+ return GetImageStatus(image)
+ .Apply(BIND([=, this, this_ = MakeStrong(this)] (const TCriImageApi::TRspImageStatusPtr& imageStatus) {
+ if (imageStatus->has_image()) {
+ return MakeFuture(TCriImageDescriptor{.Image = imageStatus->image().id()});
+ }
+ return PullImage(image, /*always*/ true, authConfig, podSpec);
+ }));
+ }
+
+ auto req = ImageApi_.PullImage();
+ FillImageSpec(req->mutable_image(), image);
+ if (authConfig) {
+ FillAuthConfig(req->mutable_auth(), *authConfig);
+ }
+ if (podSpec) {
+ FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec);
+ }
+ return req->Invoke().Apply(BIND([] (const TCriImageApi::TRspPullImagePtr& rsp) -> TCriImageDescriptor {
+ return TCriImageDescriptor{.Image = rsp->image_ref()};
+ }));
+ }
+
+ TFuture<void> RemoveImage(const TCriImageDescriptor& image) override
+ {
+ auto req = ImageApi_.RemoveImage();
+ FillImageSpec(req->mutable_image(), image);
+ return req->Invoke().AsVoid();
+ }
+
+ TProcessBasePtr CreateProcess(
+ const TString& path,
+ TCriContainerSpecPtr containerSpec,
+ const TCriPodDescriptor& podDescriptor,
+ TCriPodSpecPtr podSpec) override
+ {
+ return New<TCriProcess>(path, this, std::move(containerSpec), podDescriptor, std::move(podSpec));
+ }
+
+private:
+ const TCriExecutorConfigPtr Config_;
+ TCriRuntimeApi RuntimeApi_;
+ TCriImageApi ImageApi_;
+
+ void FillLinuxContainerResources(NProto::LinuxContainerResources* resources, const TCriContainerResources& spec)
+ {
+ auto* unified = resources->mutable_unified();
+
+ if (spec.CpuLimit) {
+ i64 period = Config_->CpuPeriod.MicroSeconds();
+ i64 quota = period * *spec.CpuLimit;
+
+ resources->set_cpu_period(period);
+ resources->set_cpu_quota(quota);
+ }
+
+ if (spec.MemoryLimit) {
+ resources->set_memory_limit_in_bytes(*spec.MemoryLimit);
+ }
+
+ if (spec.MemoryRequest) {
+ (*unified)["memory.low"] = ToString(*spec.MemoryRequest);
+ }
+ }
+
+ void FillPodSandboxConfig(NProto::PodSandboxConfig* config, const TCriPodSpec& spec)
+ {
+ {
+ auto* metadata = config->mutable_metadata();
+ metadata->set_namespace_(Config_->Namespace);
+ metadata->set_name(spec.Name);
+ metadata->set_uid(spec.Name);
+ }
+
+ {
+ auto& labels = *config->mutable_labels();
+ labels[YTPodNamespaceLabel] = Config_->Namespace;
+ labels[YTPodNameLabel] = spec.Name;
+ }
+
+ {
+ auto* linux = config->mutable_linux();
+ linux->set_cgroup_parent(GetPodCgroup(spec.Name));
+
+ auto* security = linux->mutable_security_context();
+ auto* namespaces = security->mutable_namespace_options();
+ namespaces->set_network(NProto::NODE);
+ }
+ }
+
+ void FillImageSpec(NProto::ImageSpec* spec, const TCriImageDescriptor& image)
+ {
+ spec->set_image(image.Image);
+ }
+
+ void FillAuthConfig(NProto::AuthConfig* auth, const TCriAuthConfig& authConfig)
+ {
+ if (!authConfig.Username.empty()) {
+ auth->set_username(authConfig.Username);
+ }
+ if (!authConfig.Password.empty()) {
+ auth->set_password(authConfig.Password);
+ }
+ if (!authConfig.Auth.empty()) {
+ auth->set_auth(authConfig.Auth);
+ }
+ if (!authConfig.ServerAddress.empty()) {
+ auth->set_server_address(authConfig.ServerAddress);
+ }
+ if (!authConfig.IdentityToken.empty()) {
+ auth->set_identity_token(authConfig.IdentityToken);
+ }
+ if (!authConfig.RegistryToken.empty()) {
+ auth->set_registry_token(authConfig.RegistryToken);
+ }
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config)
+{
+ return New<TCriExecutor>(
+ std::move(config),
+ GetGrpcChannelFactory());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/cri_executor.h b/yt/yt/library/containers/cri/cri_executor.h
new file mode 100644
index 0000000000..de9741721f
--- /dev/null
+++ b/yt/yt/library/containers/cri/cri_executor.h
@@ -0,0 +1,207 @@
+#pragma once
+
+#include "public.h"
+#include "config.h"
+#include "cri_api.h"
+
+#include <yt/yt/library/process/process.h>
+
+#include <yt/yt/core/ytree/yson_struct.h>
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TCriDescriptor
+{
+ TString Name;
+ TString Id;
+};
+
+struct TCriPodDescriptor
+{
+ TString Name;
+ TString Id;
+};
+
+struct TCriImageDescriptor
+{
+ TString Image;
+};
+
+void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf spec);
+void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf spec);
+void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf spec);
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TCriContainerResources
+{
+ std::optional<double> CpuLimit;
+ std::optional<double> CpuRequest;
+ std::optional<i64> MemoryLimit;
+ std::optional<i64> MemoryRequest;
+};
+
+struct TCriPodSpec
+ : public TRefCounted
+{
+ TString Name;
+ TCriContainerResources Resources;
+};
+
+DEFINE_REFCOUNTED_TYPE(TCriPodSpec)
+
+struct TCriBindMount
+{
+ TString ContainerPath;
+ TString HostPath;
+ bool ReadOnly;
+};
+
+struct TCriCredentials
+{
+ std::optional<i64> Uid;
+ std::optional<i64> Gid;
+ std::vector<i64> Groups;
+};
+
+struct TCriContainerSpec
+ : public TRefCounted
+{
+ TString Name;
+
+ THashMap<TString, TString> Labels;
+
+ TCriImageDescriptor Image;
+
+ bool ReadOnlyRootFS;
+
+ std::vector<TCriBindMount> BindMounts;
+
+ TCriCredentials Credentials;
+
+ TCriContainerResources Resources;
+
+ //! Command to execute (i.e., entrypoint for docker).
+ std::vector<TString> Command;
+
+ //! Arguments for the Command (i.e., command for docker).
+ std::vector<TString> Arguments;
+
+ //! Current working directory of the command.
+ TString WorkingDirectory;
+
+ //! Environment variable to set in the container.
+ THashMap<TString, TString> Environment;
+};
+
+DEFINE_REFCOUNTED_TYPE(TCriContainerSpec)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Wrapper around CRI gRPC API
+//!
+//! @see yt/yt/contrib/cri-api/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto
+//! @see https://github.com/kubernetes/cri-api
+struct ICriExecutor
+ : public TRefCounted
+{
+ //! Returns status of the CRI runtime.
+ //! @param verbose fill field "info" with runtime-specific debug.
+ virtual TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) = 0;
+
+ // PodSandbox
+
+ virtual TString GetPodCgroup(TString podName) const = 0;
+
+ virtual TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox(
+ std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0;
+
+ virtual TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers(
+ std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0;
+
+ virtual TFuture<void> ForEachPodSandbox(
+ const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback,
+ std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0;
+
+ virtual TFuture<void> ForEachContainer(
+ const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback,
+ std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0;
+
+ //! Returns status of the pod.
+ //! @param verbose fill field "info" with runtime-specific debug.
+ virtual TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus(
+ const TCriPodDescriptor& pod, bool verbose = false) = 0;
+
+ //! Returns status of the container.
+ //! @param verbose fill "info" with runtime-specific debug information.
+ virtual TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus(
+ const TCriDescriptor& ct, bool verbose = false) = 0;
+
+ virtual TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) = 0;
+ virtual TFuture<void> StopPodSandbox(const TCriPodDescriptor& pod) = 0;
+ virtual TFuture<void> RemovePodSandbox(const TCriPodDescriptor& pod) = 0;
+ virtual TFuture<void> UpdatePodResources(
+ const TCriPodDescriptor& pod,
+ const TCriContainerResources& resources) = 0;
+
+ //! Remove all pods and containers in namespace managed by executor.
+ virtual void CleanNamespace() = 0;
+
+ //! Remove all containers in one pod.
+ virtual void CleanPodSandbox(const TCriPodDescriptor& pod) = 0;
+
+ virtual TFuture<TCriDescriptor> CreateContainer(
+ TCriContainerSpecPtr containerSpec,
+ const TCriPodDescriptor& pod,
+ TCriPodSpecPtr podSpec) = 0;
+
+ virtual TFuture<void> StartContainer(const TCriDescriptor& ct) = 0;
+
+ //! Stops container if it's running.
+ //! @param timeout defines timeout for graceful stop, timeout=0 - kill instantly.
+ virtual TFuture<void> StopContainer(
+ const TCriDescriptor& ct,
+ TDuration timeout = TDuration::Zero()) = 0;
+
+ virtual TFuture<void> RemoveContainer(const TCriDescriptor& ct) = 0;
+
+ virtual TFuture<void> UpdateContainerResources(
+ const TCriDescriptor& ct,
+ const TCriContainerResources& resources) = 0;
+
+ virtual TFuture<TCriImageApi::TRspListImagesPtr> ListImages(
+ std::function<void(NProto::ImageFilter&)> initFilter = nullptr) = 0;
+
+ //! Returns status of the image.
+ //! @param verbose fill field "info" with runtime-specific debug.
+ virtual TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus(
+ const TCriImageDescriptor& image,
+ bool verbose = false) = 0;
+
+ virtual TFuture<TCriImageDescriptor> PullImage(
+ const TCriImageDescriptor& image,
+ bool always = false,
+ TCriAuthConfigPtr authConfig = nullptr,
+ TCriPodSpecPtr podSpec = nullptr) = 0;
+
+ virtual TFuture<void> RemoveImage(const TCriImageDescriptor& image) = 0;
+
+ // FIXME(khlebnikov): temporary compat
+ virtual TProcessBasePtr CreateProcess(
+ const TString& path,
+ TCriContainerSpecPtr containerSpec,
+ const TCriPodDescriptor& pod,
+ TCriPodSpecPtr podSpec) = 0;
+};
+
+DEFINE_REFCOUNTED_TYPE(ICriExecutor)
+
+////////////////////////////////////////////////////////////////////////////////
+
+ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/private.h b/yt/yt/library/containers/cri/private.h
new file mode 100644
index 0000000000..36fdf194f5
--- /dev/null
+++ b/yt/yt/library/containers/cri/private.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger Logger("Cri");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/public.h b/yt/yt/library/containers/cri/public.h
new file mode 100644
index 0000000000..a12ee86d57
--- /dev/null
+++ b/yt/yt/library/containers/cri/public.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <yt/yt/core/misc/intrusive_ptr.h>
+
+namespace NYT::NContainers::NCri {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_STRUCT(TCriPodSpec)
+DECLARE_REFCOUNTED_STRUCT(TCriContainerSpec)
+DECLARE_REFCOUNTED_CLASS(TCriExecutorConfig)
+DECLARE_REFCOUNTED_CLASS(TCriAuthConfig)
+DECLARE_REFCOUNTED_STRUCT(ICriExecutor)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers::NCri
diff --git a/yt/yt/library/containers/cri/ya.make b/yt/yt/library/containers/cri/ya.make
new file mode 100644
index 0000000000..dc9dd15a0b
--- /dev/null
+++ b/yt/yt/library/containers/cri/ya.make
@@ -0,0 +1,22 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+PEERDIR(
+ yt/yt/core
+ yt/yt/core/rpc/grpc
+ yt/yt/contrib/cri-api
+)
+
+SRCS(
+ cri_api.cpp
+ cri_executor.cpp
+ config.cpp
+)
+
+ADDINCL(
+ ONE_LEVEL
+ yt/yt/contrib/cri-api
+)
+
+END()
diff --git a/yt/yt/library/containers/disk_manager/config.cpp b/yt/yt/library/containers/disk_manager/config.cpp
new file mode 100644
index 0000000000..84484db630
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/config.cpp
@@ -0,0 +1,61 @@
+#include "config.h"
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TMockedDiskConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("disk_id", &TThis::DiskId)
+ .Default();
+ registrar.Parameter("device_path", &TThis::DevicePath)
+ .Default();
+ registrar.Parameter("device_name", &TThis::DeviceName)
+ .Default();
+ registrar.Parameter("disk_model", &TThis::DiskModel)
+ .Default();
+ registrar.Parameter("partition_fs_labels", &TThis::PartitionFsLabels)
+ .Default();
+ registrar.Parameter("state", &TThis::State)
+ .Default(EDiskState::Ok);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TDiskInfoProviderConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("disk_ids", &TThis::DiskIds)
+ .Default();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TDiskManagerProxyConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("disk_manager_address", &TThis::DiskManagerAddress)
+ .Default("unix:/run/yandex-diskmanager/yandex-diskmanager.sock");
+ registrar.Parameter("disk_manager_service_name", &TThis::DiskManagerServiceName)
+ .Default("diskman.DiskManager");
+
+ registrar.Parameter("is_mock", &TThis::IsMock)
+ .Default(false);
+ registrar.Parameter("mock_disks", &TThis::MockDisks)
+ .Default();
+ registrar.Parameter("mock_yt_paths", &TThis::MockYtPaths)
+ .Default();
+
+ registrar.Parameter("request_timeout", &TThis::RequestTimeout)
+ .Default(TDuration::Seconds(10));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TDiskManagerProxyDynamicConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("request_timeout", &TThis::RequestTimeout)
+ .Default();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/config.h b/yt/yt/library/containers/disk_manager/config.h
new file mode 100644
index 0000000000..4f01d378b9
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/config.h
@@ -0,0 +1,79 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/ytree/yson_struct.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TMockedDiskConfig
+ : public NYTree::TYsonStruct
+{
+ TString DiskId;
+ TString DevicePath;
+ TString DeviceName;
+ TString DiskModel;
+ std::vector<TString> PartitionFsLabels;
+ EDiskState State;
+
+ REGISTER_YSON_STRUCT(TMockedDiskConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TMockedDiskConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDiskManagerProxyConfig
+ : public NYTree::TYsonStruct
+{
+ TString DiskManagerAddress;
+ TString DiskManagerServiceName;
+
+ bool IsMock;
+ std::vector<TMockedDiskConfigPtr> MockDisks;
+ std::vector<TString> MockYtPaths;
+
+ TDuration RequestTimeout;
+
+ REGISTER_YSON_STRUCT(TDiskManagerProxyConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDiskInfoProviderConfig
+ : public NYTree::TYsonStruct
+{
+ std::vector<TString> DiskIds;
+
+ REGISTER_YSON_STRUCT(TDiskInfoProviderConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiskInfoProviderConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDiskManagerProxyDynamicConfig
+ : public NYTree::TYsonStruct
+{
+ std::optional<TDuration> RequestTimeout;
+
+ REGISTER_YSON_STRUCT(TDiskManagerProxyDynamicConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyDynamicConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.cpp b/yt/yt/library/containers/disk_manager/disk_info_provider.cpp
new file mode 100644
index 0000000000..0ee3a5b6cb
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/disk_info_provider.cpp
@@ -0,0 +1,64 @@
+#include "disk_info_provider.h"
+
+#include <yt/yt/library/containers/disk_manager/disk_manager_proxy.h>
+
+#include <yt/yt/core/actions/future.h>
+#include <yt/yt/core/actions/invoker_util.h>
+
+#include <yt/yt/core/concurrency/public.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TDiskInfoProvider::TDiskInfoProvider(
+ IDiskManagerProxyPtr diskManagerProxy,
+ TDiskInfoProviderConfigPtr config)
+ : DiskManagerProxy_(std::move(diskManagerProxy))
+ , Config_(std::move(config))
+{ }
+
+const std::vector<TString>& TDiskInfoProvider::GetConfigDiskIds() const
+{
+ return Config_->DiskIds;
+}
+
+TFuture<std::vector<TDiskInfo>> TDiskInfoProvider::GetYTDiskInfos()
+{
+ auto diskInfosFuture = DiskManagerProxy_->GetDisks();
+ auto ytDiskPathsFuture = DiskManagerProxy_->GetYtDiskMountPaths();
+
+ // Merge two futures and filter disks placed in /yt.
+ return diskInfosFuture.Apply(BIND([=] (const std::vector<TDiskInfo>& diskInfos) {
+ return ytDiskPathsFuture.Apply(BIND([=] (const THashSet<TString>& diskPaths) {
+ std::vector<TDiskInfo> disks;
+
+ for (const auto& diskInfo : diskInfos) {
+ for (const auto& partitionFsLabel : diskInfo.PartitionFsLabels) {
+ if (diskPaths.contains(partitionFsLabel)) {
+ disks.push_back(diskInfo);
+ break;
+ }
+ }
+ }
+
+ return disks;
+ }));
+ }));
+}
+
+TFuture<void> TDiskInfoProvider::RecoverDisk(const TString& diskId)
+{
+ return DiskManagerProxy_->RecoverDiskById(diskId, ERecoverPolicy::RecoverAuto);
+}
+
+TFuture<void> TDiskInfoProvider::FailDisk(
+ const TString& diskId,
+ const TString& reason)
+{
+ return DiskManagerProxy_->FailDiskById(diskId, reason);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.h b/yt/yt/library/containers/disk_manager/disk_info_provider.h
new file mode 100644
index 0000000000..b8d686438d
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/disk_info_provider.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/actions/future.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDiskInfoProvider
+ : public TRefCounted
+{
+public:
+ TDiskInfoProvider(
+ IDiskManagerProxyPtr diskManagerProxy,
+ TDiskInfoProviderConfigPtr config);
+
+ const std::vector<TString>& GetConfigDiskIds() const;
+
+ TFuture<std::vector<TDiskInfo>> GetYTDiskInfos();
+
+ TFuture<void> RecoverDisk(const TString& diskId);
+
+ TFuture<void> FailDisk(
+ const TString& diskId,
+ const TString& reason);
+
+private:
+ const IDiskManagerProxyPtr DiskManagerProxy_;
+ const TDiskInfoProviderConfigPtr Config_;
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiskInfoProvider)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp b/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp
new file mode 100644
index 0000000000..961723c51f
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp
@@ -0,0 +1,49 @@
+#include "disk_manager_proxy.h"
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDiskManagerProxyMock
+ : public IDiskManagerProxy
+{
+ virtual TFuture<THashSet<TString>> GetYtDiskMountPaths()
+ {
+ THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration");
+ }
+
+ virtual TFuture<std::vector<TDiskInfo>> GetDisks()
+ {
+ THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration");
+ }
+
+ virtual TFuture<void> RecoverDiskById(const TString& /*diskId*/, ERecoverPolicy /*recoverPolicy*/)
+ {
+ THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration");
+ }
+
+ virtual TFuture<void> FailDiskById(const TString& /*diskId*/, const TString& /*reason*/)
+ {
+ THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration");
+ }
+
+ virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& /*newConfig*/)
+ {
+ // Do nothing
+ }
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyMock)
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_WEAK IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr /*config*/)
+{
+ // This implementation is used when disk_manager_proxy_impl.cpp is not linked.
+
+ return New<TDiskManagerProxyMock>();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.h b/yt/yt/library/containers/disk_manager/disk_manager_proxy.h
new file mode 100644
index 0000000000..d2da5c1873
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/disk_manager_proxy.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/library/containers/disk_manager/config.h>
+
+#include <yt/yt/core/misc/atomic_object.h>
+
+#include <yt/yt/core/rpc/client.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IDiskManagerProxy
+ : public virtual TRefCounted
+{
+ virtual TFuture<THashSet<TString>> GetYtDiskMountPaths() = 0;
+
+ virtual TFuture<std::vector<TDiskInfo>> GetDisks() = 0;
+
+ virtual TFuture<void> RecoverDiskById(const TString& diskId, ERecoverPolicy recoverPolicy) = 0;
+
+ virtual TFuture<void> FailDiskById(const TString& diskId, const TString& reason) = 0;
+
+ virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& newConfig) = 0;
+
+};
+
+DEFINE_REFCOUNTED_TYPE(IDiskManagerProxy)
+
+////////////////////////////////////////////////////////////////////////////////
+
+IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr config);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/public.h b/yt/yt/library/containers/disk_manager/public.h
new file mode 100644
index 0000000000..8a812638f3
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/public.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DEFINE_ENUM(EDiskState,
+ ((Unknown) (0))
+ ((Ok) (1))
+ ((Failed) (2))
+ ((RecoverWait) (3))
+);
+
+// 1. Remount all disk volumes to it's default state
+// 2. Recreate disk layout, all data on disk will be lost
+// 3. Replace phisical disk
+DEFINE_ENUM(ERecoverPolicy,
+ ((RecoverAuto) (0))
+ ((RecoverMount) (1))
+ ((RecoverLayout) (2))
+ ((RecoverDisk) (3))
+);
+
+struct TDiskInfo
+{
+ TString DiskId;
+ TString DevicePath;
+ TString DeviceName;
+ TString DiskModel;
+ THashSet<TString> PartitionFsLabels;
+ EDiskState State;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_STRUCT(TMockedDiskConfig)
+DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyConfig)
+DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyDynamicConfig)
+DECLARE_REFCOUNTED_STRUCT(TDiskInfoProviderConfig)
+
+DECLARE_REFCOUNTED_STRUCT(IDiskManagerProxy)
+DECLARE_REFCOUNTED_CLASS(TDiskInfoProvider)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/disk_manager/ya.make b/yt/yt/library/containers/disk_manager/ya.make
new file mode 100644
index 0000000000..dcb260cf38
--- /dev/null
+++ b/yt/yt/library/containers/disk_manager/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+PEERDIR(
+ yt/yt/core
+)
+
+SRCS(
+ config.cpp
+ disk_info_provider.cpp
+ disk_manager_proxy.cpp
+)
+
+IF (NOT OPENSOURCE)
+ INCLUDE(ya_non_opensource.inc)
+ENDIF()
+
+END()
diff --git a/yt/yt/library/containers/instance.cpp b/yt/yt/library/containers/instance.cpp
new file mode 100644
index 0000000000..0a56987e1b
--- /dev/null
+++ b/yt/yt/library/containers/instance.cpp
@@ -0,0 +1,812 @@
+#ifdef __linux__
+
+#include "instance.h"
+
+#include "porto_executor.h"
+#include "private.h"
+
+#include <yt/yt/library/containers/cgroup.h>
+#include <yt/yt/library/containers/config.h>
+
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/collection_helpers.h>
+#include <yt/yt/core/misc/error.h>
+#include <yt/yt/core/misc/fs.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <library/cpp/porto/libporto.hpp>
+
+#include <util/stream/file.h>
+
+#include <util/string/cast.h>
+#include <util/string/split.h>
+
+#include <util/system/env.h>
+
+#include <initializer_list>
+#include <string>
+
+namespace NYT::NContainers {
+
+using namespace NConcurrency;
+using namespace NNet;
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+// Porto passes command string to wordexp, where quota (') symbol
+// is delimiter. So we must replace it with concatenation ('"'"').
+TString EscapeForWordexp(const char* in)
+{
+ TString buffer;
+ while (*in) {
+ if (*in == '\'') {
+ buffer.append(R"('"'"')");
+ } else {
+ buffer.append(*in);
+ }
+ in++;
+ }
+ return buffer;
+}
+
+i64 Extract(
+ const TString& input,
+ const TString& pattern,
+ const TString& terminator = "\n")
+{
+ auto start = input.find(pattern) + pattern.length();
+ auto end = input.find(terminator, start);
+ return std::stol(input.substr(start, (end == input.npos) ? end : end - start));
+}
+
+i64 ExtractSum(
+ const TString& input,
+ const TString& pattern,
+ const TString& delimiter,
+ const TString& terminator = "\n")
+{
+ i64 sum = 0;
+ TString::size_type pos = 0;
+ while (pos < input.length()) {
+ pos = input.find(pattern, pos);
+ if (pos == input.npos) {
+ break;
+ }
+ pos += pattern.length();
+
+ pos = input.find(delimiter, pos);
+ if (pos == input.npos) {
+ break;
+ }
+
+ pos++;
+ auto end = input.find(terminator, pos);
+ sum += std::stol(input.substr(pos, (end == input.npos) ? end : end - pos));
+ }
+ return sum;
+}
+
+using TPortoStatRule = std::pair<TString, std::function<i64(const TString& input)>>;
+
+static const std::function<i64(const TString&)> LongExtractor = [] (const TString& in) {
+ return std::stol(in);
+};
+
+static const std::function<i64(const TString&)> CoreNsPerSecondExtractor = [] (const TString& in) {
+ int pos = in.find("c", 0);
+ return (std::stod(in.substr(0, pos))) * 1'000'000'000;
+};
+
+static const std::function<i64(const TString&)> GetIOStatExtractor(const TString& rwMode = "")
+{
+ return [rwMode] (const TString& in) {
+ return ExtractSum(in, "hw", rwMode + ":", ";");
+ };
+}
+
+static const std::function<i64(const TString&)> GetStatByKeyExtractor(const TString& statKey)
+{
+ return [statKey] (const TString& in) {
+ return Extract(in, statKey);
+ };
+}
+
+const THashMap<EStatField, TPortoStatRule> PortoStatRules = {
+ {EStatField::CpuUsage, {"cpu_usage", LongExtractor}},
+ {EStatField::CpuSystemUsage, {"cpu_usage_system", LongExtractor}},
+ {EStatField::CpuWait, {"cpu_wait", LongExtractor}},
+ {EStatField::CpuThrottled, {"cpu_throttled", LongExtractor}},
+ {EStatField::ThreadCount, {"thread_count", LongExtractor}},
+ {EStatField::CpuLimit, {"cpu_limit_bound", CoreNsPerSecondExtractor}},
+ {EStatField::CpuGuarantee, {"cpu_guarantee_bound", CoreNsPerSecondExtractor}},
+ {EStatField::Rss, {"memory.stat", GetStatByKeyExtractor("total_rss")}},
+ {EStatField::MappedFile, {"memory.stat", GetStatByKeyExtractor("total_mapped_file")}},
+ {EStatField::MinorPageFaults, {"minor_faults", LongExtractor}},
+ {EStatField::MajorPageFaults, {"major_faults", LongExtractor}},
+ {EStatField::FileCacheUsage, {"cache_usage", LongExtractor}},
+ {EStatField::AnonMemoryUsage, {"anon_usage", LongExtractor}},
+ {EStatField::AnonMemoryLimit, {"anon_limit_total", LongExtractor}},
+ {EStatField::MemoryUsage, {"memory_usage", LongExtractor}},
+ {EStatField::MemoryGuarantee, {"memory_guarantee", LongExtractor}},
+ {EStatField::MemoryLimit, {"memory_limit_total", LongExtractor}},
+ {EStatField::MaxMemoryUsage, {"memory.max_usage_in_bytes", LongExtractor}},
+ {EStatField::OomKills, {"oom_kills", LongExtractor}},
+ {EStatField::OomKillsTotal, {"oom_kills_total", LongExtractor}},
+
+ {EStatField::IOReadByte, {"io_read", GetIOStatExtractor()}},
+ {EStatField::IOWriteByte, {"io_write", GetIOStatExtractor()}},
+ {EStatField::IOBytesLimit, {"io_limit", GetIOStatExtractor()}},
+ {EStatField::IOReadOps, {"io_read_ops", GetIOStatExtractor()}},
+ {EStatField::IOWriteOps, {"io_write_ops", GetIOStatExtractor()}},
+ {EStatField::IOOps, {"io_ops", GetIOStatExtractor()}},
+ {EStatField::IOOpsLimit, {"io_ops_limit", GetIOStatExtractor()}},
+ {EStatField::IOTotalTime, {"io_time", GetIOStatExtractor()}},
+ {EStatField::IOWaitTime, {"io_wait", GetIOStatExtractor()}},
+
+ {EStatField::NetTxBytes, {"net_tx_bytes[veth]", LongExtractor}},
+ {EStatField::NetTxPackets, {"net_tx_packets[veth]", LongExtractor}},
+ {EStatField::NetTxDrops, {"net_tx_drops[veth]", LongExtractor}},
+ {EStatField::NetTxLimit, {"net_limit[veth]", LongExtractor}},
+ {EStatField::NetRxBytes, {"net_rx_bytes[veth]", LongExtractor}},
+ {EStatField::NetRxPackets, {"net_rx_packets[veth]", LongExtractor}},
+ {EStatField::NetRxDrops, {"net_rx_drops[veth]", LongExtractor}},
+ {EStatField::NetRxLimit, {"net_rx_limit[veth]", LongExtractor}},
+};
+
+std::optional<TString> GetParentName(const TString& name)
+{
+ if (name.empty()) {
+ return std::nullopt;
+ }
+
+ auto slashPosition = name.rfind('/');
+ if (slashPosition == TString::npos) {
+ return "";
+ }
+
+ return name.substr(0, slashPosition);
+}
+
+std::optional<TString> GetRootName(const TString& name)
+{
+ if (name.empty()) {
+ return std::nullopt;
+ }
+
+ if (name == "/") {
+ return name;
+ }
+
+ auto slashPosition = name.find('/');
+ if (slashPosition == TString::npos) {
+ return name;
+ }
+
+ return name.substr(0, slashPosition);
+}
+
+} // namespace NDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoInstanceLauncher
+ : public IInstanceLauncher
+{
+public:
+ TPortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor)
+ : Executor_(std::move(executor))
+ , Logger(ContainersLogger.WithTag("Container: %v", name))
+ {
+ Spec_.Name = name;
+ Spec_.CGroupControllers = {
+ "freezer",
+ "cpu",
+ "cpuacct",
+ "net_cls",
+ "blkio",
+ "devices",
+ "pids"
+ };
+ }
+
+ const TString& GetName() const override
+ {
+ return Spec_.Name;
+ }
+
+ bool HasRoot() const override
+ {
+ return static_cast<bool>(Spec_.RootFS);
+ }
+
+ void SetStdIn(const TString& inputPath) override
+ {
+ Spec_.StdinPath = inputPath;
+ }
+
+ void SetStdOut(const TString& outPath) override
+ {
+ Spec_.StdoutPath = outPath;
+ }
+
+ void SetStdErr(const TString& errorPath) override
+ {
+ Spec_.StderrPath = errorPath;
+ }
+
+ void SetCwd(const TString& pwd) override
+ {
+ Spec_.CurrentWorkingDirectory = pwd;
+ }
+
+ void SetCoreDumpHandler(const std::optional<TString>& handler) override
+ {
+ if (handler) {
+ Spec_.CoreCommand = *handler;
+ Spec_.EnableCoreDumps = true;
+ } else {
+ Spec_.EnableCoreDumps = false;
+ }
+ }
+
+ void SetRoot(const TRootFS& rootFS) override
+ {
+ Spec_.RootFS = rootFS;
+ }
+
+ void SetThreadLimit(i64 threadLimit) override
+ {
+ Spec_.ThreadLimit = threadLimit;
+ }
+
+ void SetDevices(const std::vector<TDevice>& devices) override
+ {
+ Spec_.Devices = devices;
+ }
+
+ void SetEnablePorto(EEnablePorto enablePorto) override
+ {
+ Spec_.EnablePorto = enablePorto;
+ }
+
+ void SetIsolate(bool isolate) override
+ {
+ Spec_.Isolate = isolate;
+ }
+
+ void EnableMemoryTracking() override
+ {
+ Spec_.CGroupControllers.push_back("memory");
+ }
+
+ void SetGroup(int groupId) override
+ {
+ Spec_.GroupId = groupId;
+ }
+
+ void SetUser(const TString& user) override
+ {
+ Spec_.User = user;
+ }
+
+ void SetIPAddresses(const std::vector<NNet::TIP6Address>& addresses, bool enableNat64) override
+ {
+ Spec_.IPAddresses = addresses;
+ Spec_.EnableNat64 = enableNat64;
+ Spec_.DisableNetwork = false;
+ }
+
+ void DisableNetwork() override
+ {
+ Spec_.DisableNetwork = true;
+ Spec_.IPAddresses.clear();
+ Spec_.EnableNat64 = false;
+ }
+
+ void SetHostName(const TString& hostName) override
+ {
+ Spec_.HostName = hostName;
+ }
+
+ TFuture<IInstancePtr> Launch(
+ const TString& path,
+ const std::vector<TString>& args,
+ const THashMap<TString, TString>& env) override
+ {
+ TStringBuilder commandBuilder;
+ auto append = [&] (const auto& value) {
+ commandBuilder.AppendString("'");
+ commandBuilder.AppendString(NDetail::EscapeForWordexp(value.c_str()));
+ commandBuilder.AppendString("' ");
+ };
+
+ append(path);
+ for (const auto& arg : args) {
+ append(arg);
+ }
+
+ Spec_.Command = commandBuilder.Flush();
+ YT_LOG_DEBUG("Executing Porto container (Name: %v, Command: %v)",
+ Spec_.Name,
+ Spec_.Command);
+
+ Spec_.Env = env;
+
+ auto onContainerCreated = [this, this_ = MakeStrong(this)] (const TError& error) -> IInstancePtr {
+ if (!error.IsOK()) {
+ THROW_ERROR_EXCEPTION(EErrorCode::FailedToStartContainer, "Unable to start container")
+ << error;
+ }
+
+ return GetPortoInstance(Executor_, Spec_.Name);
+ };
+
+ return Executor_->CreateContainer(Spec_, /* start */ true)
+ .Apply(BIND(onContainerCreated));
+ }
+
+private:
+ IPortoExecutorPtr Executor_;
+ TRunnableContainerSpec Spec_;
+ const NLogging::TLogger Logger;
+};
+
+IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor)
+{
+ return New<TPortoInstanceLauncher>(name, executor);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoInstance
+ : public IInstance
+{
+public:
+ static IInstancePtr GetSelf(IPortoExecutorPtr executor)
+ {
+ return New<TPortoInstance>(GetSelfContainerName(executor), executor);
+ }
+
+ static IInstancePtr GetInstance(IPortoExecutorPtr executor, const TString& name)
+ {
+ return New<TPortoInstance>(name, executor);
+ }
+
+ void Kill(int signal) override
+ {
+ auto error = WaitFor(Executor_->KillContainer(Name_, signal));
+ // Killing already finished process is not an error.
+ if (error.FindMatching(EPortoErrorCode::InvalidState)) {
+ return;
+ }
+ if (!error.IsOK()) {
+ THROW_ERROR_EXCEPTION("Failed to send signal to Porto instance")
+ << TErrorAttribute("signal", signal)
+ << TErrorAttribute("container", Name_)
+ << error;
+ }
+ }
+
+ void Destroy() override
+ {
+ WaitFor(Executor_->DestroyContainer(Name_))
+ .ThrowOnError();
+ Destroyed_ = true;
+ }
+
+ void Stop() override
+ {
+ WaitFor(Executor_->StopContainer(Name_))
+ .ThrowOnError();
+ }
+
+ TErrorOr<ui64> CalculateCpuUserUsage(
+ TErrorOr<ui64>& cpuUsage,
+ TErrorOr<ui64>& cpuSystemUsage) const
+ {
+ if (cpuUsage.IsOK() && cpuSystemUsage.IsOK()) {
+ return cpuUsage.Value() > cpuSystemUsage.Value() ? cpuUsage.Value() - cpuSystemUsage.Value() : 0;
+ } else if (cpuUsage.IsOK()) {
+ return TError("Missing property %Qlv in Porto response", EStatField::CpuSystemUsage)
+ << TErrorAttribute("container", Name_);
+ } else {
+ return TError("Missing property %Qlv in Porto response", EStatField::CpuUsage)
+ << TErrorAttribute("container", Name_);
+ }
+ }
+
+ TResourceUsage GetResourceUsage(
+ const std::vector<EStatField>& fields) const override
+ {
+ std::vector<TString> properties;
+ properties.push_back("absolute_name");
+
+ bool userTimeRequested = false;
+ bool contextSwitchesRequested = false;
+ for (auto field : fields) {
+ if (auto it = NDetail::PortoStatRules.find(field)) {
+ const auto& rule = it->second;
+ properties.push_back(rule.first);
+ } else if (field == EStatField::ContextSwitchesDelta || field == EStatField::ContextSwitches) {
+ contextSwitchesRequested = true;
+ } else if (field == EStatField::CpuUserUsage) {
+ userTimeRequested = true;
+ } else {
+ THROW_ERROR_EXCEPTION("Unknown resource field %Qlv requested", field)
+ << TErrorAttribute("container", Name_);
+ }
+ }
+
+ auto propertyMap = WaitFor(Executor_->GetContainerProperties(Name_, properties))
+ .ValueOrThrow();
+
+ TResourceUsage result;
+
+ for (auto field : fields) {
+ auto ruleIt = NDetail::PortoStatRules.find(field);
+ if (ruleIt == NDetail::PortoStatRules.end()) {
+ continue;
+ }
+
+ const auto& [property, callback] = ruleIt->second;
+ auto& record = result[field];
+ if (auto responseIt = propertyMap.find(property); responseIt != propertyMap.end()) {
+ const auto& valueOrError = responseIt->second;
+ if (valueOrError.IsOK()) {
+ const auto& value = valueOrError.Value();
+
+ try {
+ record = callback(value);
+ } catch (const std::exception& ex) {
+ record = TError("Error parsing Porto property %Qlv", field)
+ << TErrorAttribute("container", Name_)
+ << TErrorAttribute("property_value", value)
+ << ex;
+ }
+ } else {
+ record = TError("Error getting Porto property %Qlv", field)
+ << TErrorAttribute("container", Name_)
+ << valueOrError;
+ }
+ } else {
+ record = TError("Missing property %Qlv in Porto response", field)
+ << TErrorAttribute("container", Name_);
+ }
+ }
+
+ // We should maintain context switch information even if this field
+ // is not requested since metrics of individual containers can go up and down.
+ auto subcontainers = WaitFor(Executor_->ListSubcontainers(Name_, /*includeRoot*/ true))
+ .ValueOrThrow();
+
+ auto metricMap = WaitFor(Executor_->GetContainerMetrics(subcontainers, "ctxsw"))
+ .ValueOrThrow();
+
+ // TODO(don-dron): remove diff calculation from GetResourceUsage, because GetResourceUsage must return only snapshot stat.
+ {
+ auto guard = Guard(ContextSwitchMapLock_);
+
+ for (const auto& [container, newValue] : metricMap) {
+ auto& prevValue = ContextSwitchMap_[container];
+ TotalContextSwitches_ += std::max<i64>(0LL, newValue - prevValue);
+ prevValue = newValue;
+ }
+
+ if (contextSwitchesRequested) {
+ result[EStatField::ContextSwitchesDelta] = TotalContextSwitches_;
+ }
+ }
+
+ if (contextSwitchesRequested) {
+ ui64 totalContextSwitches = 0;
+
+ for (const auto& [container, newValue] : metricMap) {
+ totalContextSwitches += std::max<ui64>(0UL, newValue);
+ }
+
+ result[EStatField::ContextSwitches] = totalContextSwitches;
+ }
+
+ if (userTimeRequested) {
+ result[EStatField::CpuUserUsage] = CalculateCpuUserUsage(
+ result[EStatField::CpuUsage],
+ result[EStatField::CpuSystemUsage]);
+ }
+
+ return result;
+ }
+
+ TResourceLimits GetResourceLimits() const override
+ {
+ std::vector<TString> properties;
+ static TString memoryLimitProperty = "memory_limit_total";
+ static TString cpuLimitProperty = "cpu_limit_bound";
+ static TString cpuGuaranteeProperty = "cpu_guarantee_bound";
+ properties.push_back(memoryLimitProperty);
+ properties.push_back(cpuLimitProperty);
+ properties.push_back(cpuGuaranteeProperty);
+
+ auto responseOrError = WaitFor(Executor_->GetContainerProperties(Name_, properties));
+ THROW_ERROR_EXCEPTION_IF_FAILED(responseOrError, "Failed to get Porto container resource limits");
+
+ const auto& response = responseOrError.Value();
+
+ const auto& memoryLimitRsp = response.at(memoryLimitProperty);
+ THROW_ERROR_EXCEPTION_IF_FAILED(memoryLimitRsp, "Failed to get memory limit from Porto");
+
+ i64 memoryLimit;
+ if (!TryFromString<i64>(memoryLimitRsp.Value(), memoryLimit)) {
+ THROW_ERROR_EXCEPTION("Failed to parse memory limit value from Porto")
+ << TErrorAttribute(memoryLimitProperty, memoryLimitRsp.Value());
+ }
+
+ const auto& cpuLimitRsp = response.at(cpuLimitProperty);
+ THROW_ERROR_EXCEPTION_IF_FAILED(cpuLimitRsp, "Failed to get CPU limit from Porto");
+
+ double cpuLimit;
+ YT_VERIFY(cpuLimitRsp.Value().EndsWith('c'));
+ auto cpuLimitValue = TStringBuf(cpuLimitRsp.Value().begin(), cpuLimitRsp.Value().size() - 1);
+ if (!TryFromString<double>(cpuLimitValue, cpuLimit)) {
+ THROW_ERROR_EXCEPTION("Failed to parse CPU limit value from Porto")
+ << TErrorAttribute(cpuLimitProperty, cpuLimitRsp.Value());
+ }
+
+ const auto& cpuGuaranteeRsp = response.at(cpuGuaranteeProperty);
+ THROW_ERROR_EXCEPTION_IF_FAILED(cpuGuaranteeRsp, "Failed to get CPU guarantee from Porto");
+
+ double cpuGuarantee;
+ if (!cpuGuaranteeRsp.Value()) {
+ // XXX(ignat): hack for missing response from Porto.
+ cpuGuarantee = 0.0;
+ } else {
+ YT_VERIFY(cpuGuaranteeRsp.Value().EndsWith('c'));
+ auto cpuGuaranteeValue = TStringBuf(cpuGuaranteeRsp.Value().begin(), cpuGuaranteeRsp.Value().size() - 1);
+ if (!TryFromString<double>(cpuGuaranteeValue, cpuGuarantee)) {
+ THROW_ERROR_EXCEPTION("Failed to parse CPU guarantee value from Porto")
+ << TErrorAttribute(cpuGuaranteeProperty, cpuGuaranteeRsp.Value());
+ }
+ }
+
+ return TResourceLimits{
+ .CpuLimit = cpuLimit,
+ .CpuGuarantee = cpuGuarantee,
+ .Memory = memoryLimit,
+ };
+ }
+
+ void SetCpuGuarantee(double cores) override
+ {
+ SetProperty("cpu_guarantee", ToString(cores) + "c");
+ }
+
+ void SetCpuLimit(double cores) override
+ {
+ SetProperty("cpu_limit", ToString(cores) + "c");
+ }
+
+ void SetCpuWeight(double weight) override
+ {
+ SetProperty("cpu_weight", weight);
+ }
+
+ void SetMemoryGuarantee(i64 memoryGuarantee) override
+ {
+ SetProperty("memory_guarantee", memoryGuarantee);
+ }
+
+ void SetIOWeight(double weight) override
+ {
+ SetProperty("io_weight", weight);
+ }
+
+ void SetIOThrottle(i64 operations) override
+ {
+ SetProperty("io_ops_limit", operations);
+ }
+
+ TString GetStderr() const override
+ {
+ return *WaitFor(Executor_->GetContainerProperty(Name_, "stderr"))
+ .ValueOrThrow();
+ }
+
+ TString GetName() const override
+ {
+ return Name_;
+ }
+
+ std::optional<TString> GetParentName() const override
+ {
+ return NDetail::GetParentName(Name_);
+ }
+
+ std::optional<TString> GetRootName() const override
+ {
+ return NDetail::GetRootName(Name_);
+ }
+
+ pid_t GetPid() const override
+ {
+ auto pid = *WaitFor(Executor_->GetContainerProperty(Name_, "root_pid"))
+ .ValueOrThrow();
+ return std::stoi(pid);
+ }
+
+ i64 GetMajorPageFaultCount() const override
+ {
+ auto faults = WaitFor(Executor_->GetContainerProperty(Name_, "major_faults"))
+ .ValueOrThrow();
+ return faults
+ ? std::stoll(*faults)
+ : 0;
+ }
+
+ double GetCpuGuarantee() const override
+ {
+ auto result = WaitFor(Executor_->GetContainerProperty(Name_, "cpu_guarantee"))
+ .ValueOrThrow();
+ return result
+ ? std::stod(*result)
+ : 0;
+ }
+
+ std::vector<pid_t> GetPids() const override
+ {
+ auto getPidCgroup = [&] (const TString& cgroups) {
+ for (TStringBuf cgroup : StringSplitter(cgroups).SplitByString("; ")) {
+ if (cgroup.StartsWith("pids:")) {
+ auto startPosition = cgroup.find('/');
+ YT_VERIFY(startPosition != TString::npos);
+ return cgroup.substr(startPosition);
+ }
+ }
+ THROW_ERROR_EXCEPTION("Pids cgroup not found for container %Qv", GetName())
+ << TErrorAttribute("cgroups", cgroups);
+ };
+
+ auto cgroups = *WaitFor(Executor_->GetContainerProperty(Name_, "cgroups"))
+ .ValueOrThrow();
+ // Porto returns full cgroup name, with mount prefix, such as "/sys/fs/cgroup/pids".
+ auto instanceCgroup = getPidCgroup(cgroups);
+
+ std::vector<pid_t> pids;
+ for (auto pid : ListPids()) {
+ std::map<TString, TString> cgroups;
+ try {
+ cgroups = GetProcessCGroups(pid);
+ } catch (const std::exception& ex) {
+ YT_LOG_DEBUG(ex, "Failed to get CGroups for process (Pid: %v)", pid);
+ continue;
+ }
+
+ // Pid cgroups are returned in short form.
+ auto processPidCgroup = cgroups["pids"];
+ if (!processPidCgroup.empty() && instanceCgroup.EndsWith(processPidCgroup)) {
+ pids.push_back(pid);
+ }
+ }
+
+ return pids;
+ }
+
+ TFuture<void> Wait() override
+ {
+ return Executor_->PollContainer(Name_)
+ .Apply(BIND([] (int status) {
+ StatusToError(status)
+ .ThrowOnError();
+ }));
+ }
+
+private:
+ const TString Name_;
+ const IPortoExecutorPtr Executor_;
+ const NLogging::TLogger Logger;
+
+ bool Destroyed_ = false;
+
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, ContextSwitchMapLock_);
+ mutable i64 TotalContextSwitches_ = 0;
+ mutable THashMap<TString, i64> ContextSwitchMap_;
+
+ TPortoInstance(TString name, IPortoExecutorPtr executor)
+ : Name_(std::move(name))
+ , Executor_(std::move(executor))
+ , Logger(ContainersLogger.WithTag("Container: %v", Name_))
+ { }
+
+ void SetProperty(const TString& key, const TString& value)
+ {
+ WaitFor(Executor_->SetContainerProperty(Name_, key, value))
+ .ThrowOnError();
+ }
+
+ void SetProperty(const TString& key, i64 value)
+ {
+ SetProperty(key, ToString(value));
+ }
+
+ void SetProperty(const TString& key, double value)
+ {
+ SetProperty(key, ToString(value));
+ }
+
+ DECLARE_NEW_FRIEND()
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString GetSelfContainerName(const IPortoExecutorPtr& executor)
+{
+ try {
+ auto properties = WaitFor(executor->GetContainerProperties(
+ "self",
+ std::vector<TString>{"absolute_name", "absolute_namespace"}))
+ .ValueOrThrow();
+
+ auto absoluteName = properties.at("absolute_name")
+ .ValueOrThrow();
+ auto absoluteNamespace = properties.at("absolute_namespace")
+ .ValueOrThrow();
+
+ if (absoluteName == "/") {
+ return absoluteName;
+ }
+
+ if (absoluteName.length() < absoluteNamespace.length()) {
+ YT_VERIFY(absoluteName + "/" == absoluteNamespace);
+ return "";
+ } else {
+ YT_VERIFY(absoluteName.StartsWith(absoluteNamespace));
+ return absoluteName.substr(absoluteNamespace.length());
+ }
+ } catch (const std::exception& ex) {
+ THROW_ERROR_EXCEPTION("Failed to get name for container \"self\"")
+ << ex;
+ }
+}
+
+IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor)
+{
+ return TPortoInstance::GetSelf(executor);
+}
+
+IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name)
+{
+ return TPortoInstance::GetInstance(executor, name);
+}
+
+IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor)
+{
+ auto self = GetSelfPortoInstance(executor);
+ return TPortoInstance::GetInstance(executor, *self->GetRootName());
+}
+
+double GetSelfPortoInstanceVCpuFactor()
+{
+ auto config = New<TPortoExecutorDynamicConfig>();
+ auto executorPtr = CreatePortoExecutor(config, "");
+ auto currentContainer = GetSelfPortoInstance(executorPtr);
+ double cpuLimit = currentContainer->GetResourceLimits().CpuLimit;
+ if (cpuLimit <= 0) {
+ THROW_ERROR_EXCEPTION("Cpu limit must be greater than 0");
+ }
+
+ // DEPLOY_VCPU_LIMIT stores value in millicores
+ if (TString vcpuLimitStr = GetEnv("DEPLOY_VCPU_LIMIT"); !vcpuLimitStr.Empty()) {
+ double vcpuLimit = FromString<double>(vcpuLimitStr) / 1000.0;
+ return vcpuLimit / cpuLimit;
+ }
+ THROW_ERROR_EXCEPTION("Failed to get vcpu limit from env variable");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
+
+#endif
diff --git a/yt/yt/library/containers/instance.h b/yt/yt/library/containers/instance.h
new file mode 100644
index 0000000000..ff6e0b3ce1
--- /dev/null
+++ b/yt/yt/library/containers/instance.h
@@ -0,0 +1,168 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/actions/future.h>
+
+#include <yt/yt/core/net/address.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+using TResourceUsage = THashMap<EStatField, TErrorOr<ui64>>;
+
+const std::vector<EStatField> InstanceStatFields{
+ EStatField::CpuUsage,
+ EStatField::CpuUserUsage,
+ EStatField::CpuSystemUsage,
+ EStatField::CpuWait,
+ EStatField::CpuThrottled,
+ EStatField::ContextSwitches,
+ EStatField::ContextSwitchesDelta,
+ EStatField::ThreadCount,
+ EStatField::CpuLimit,
+ EStatField::CpuGuarantee,
+
+ EStatField::Rss,
+ EStatField::MappedFile,
+ EStatField::MajorPageFaults,
+ EStatField::MinorPageFaults,
+ EStatField::FileCacheUsage,
+ EStatField::AnonMemoryUsage,
+ EStatField::AnonMemoryLimit,
+ EStatField::MemoryUsage,
+ EStatField::MemoryGuarantee,
+ EStatField::MemoryLimit,
+ EStatField::MaxMemoryUsage,
+ EStatField::OomKills,
+ EStatField::OomKillsTotal,
+
+ EStatField::IOReadByte,
+ EStatField::IOWriteByte,
+ EStatField::IOBytesLimit,
+ EStatField::IOReadOps,
+ EStatField::IOWriteOps,
+ EStatField::IOOps,
+ EStatField::IOOpsLimit,
+ EStatField::IOTotalTime,
+ EStatField::IOWaitTime,
+
+ EStatField::NetTxBytes,
+ EStatField::NetTxPackets,
+ EStatField::NetTxDrops,
+ EStatField::NetTxLimit,
+ EStatField::NetRxBytes,
+ EStatField::NetRxPackets,
+ EStatField::NetRxDrops,
+ EStatField::NetRxLimit,
+};
+
+struct TResourceLimits
+{
+ double CpuLimit;
+ double CpuGuarantee;
+ i64 Memory;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IInstanceLauncher
+ : public TRefCounted
+{
+ virtual bool HasRoot() const = 0;
+ virtual const TString& GetName() const = 0;
+
+ virtual void SetStdIn(const TString& inputPath) = 0;
+ virtual void SetStdOut(const TString& outPath) = 0;
+ virtual void SetStdErr(const TString& errorPath) = 0;
+ virtual void SetCwd(const TString& pwd) = 0;
+
+ // Null core dump handler implies disabled core dumps.
+ virtual void SetCoreDumpHandler(const std::optional<TString>& handler) = 0;
+ virtual void SetRoot(const TRootFS& rootFS) = 0;
+
+ virtual void SetThreadLimit(i64 threadLimit) = 0;
+ virtual void SetDevices(const std::vector<TDevice>& devices) = 0;
+
+ virtual void SetEnablePorto(EEnablePorto enablePorto) = 0;
+ virtual void SetIsolate(bool isolate) = 0;
+ virtual void EnableMemoryTracking() = 0;
+ virtual void SetGroup(int groupId) = 0;
+ virtual void SetUser(const TString& user) = 0;
+ virtual void SetIPAddresses(
+ const std::vector<NNet::TIP6Address>& addresses,
+ bool enableNat64 = false) = 0;
+ virtual void DisableNetwork() = 0;
+ virtual void SetHostName(const TString& hostName) = 0;
+
+ virtual TFuture<IInstancePtr> Launch(
+ const TString& path,
+ const std::vector<TString>& args,
+ const THashMap<TString, TString>& env) = 0;
+};
+
+DEFINE_REFCOUNTED_TYPE(IInstanceLauncher)
+
+#ifdef _linux_
+IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor);
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IInstance
+ : public TRefCounted
+{
+ virtual void Kill(int signal) = 0;
+ virtual void Stop() = 0;
+ virtual void Destroy() = 0;
+
+ virtual TResourceUsage GetResourceUsage(
+ const std::vector<EStatField>& fields = InstanceStatFields) const = 0;
+ virtual TResourceLimits GetResourceLimits() const = 0;
+ virtual void SetCpuGuarantee(double cores) = 0;
+ virtual void SetCpuLimit(double cores) = 0;
+ virtual void SetCpuWeight(double weight) = 0;
+ virtual void SetIOWeight(double weight) = 0;
+ virtual void SetIOThrottle(i64 operations) = 0;
+ virtual void SetMemoryGuarantee(i64 memoryGuarantee) = 0;
+
+ virtual TString GetStderr() const = 0;
+
+ virtual TString GetName() const = 0;
+ virtual std::optional<TString> GetParentName() const = 0;
+ virtual std::optional<TString> GetRootName() const = 0;
+
+ //! Returns externally visible pid of the root process inside container.
+ //! Throws if container is not running.
+ virtual pid_t GetPid() const = 0;
+ //! Returns the list of externally visible pids of processes running inside container.
+ virtual std::vector<pid_t> GetPids() const = 0;
+
+ virtual i64 GetMajorPageFaultCount() const = 0;
+ virtual double GetCpuGuarantee() const = 0;
+
+ //! Future is set when container reaches terminal state (stopped or dead).
+ //! Resulting error is OK iff container exited with code 0.
+ virtual TFuture<void> Wait() = 0;
+};
+
+DEFINE_REFCOUNTED_TYPE(IInstance)
+
+////////////////////////////////////////////////////////////////////////////////
+
+#ifdef _linux_
+TString GetSelfContainerName(const IPortoExecutorPtr& executor);
+
+IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor);
+IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor);
+IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name);
+
+//! Works only in Yandex.Deploy pod environment where env DEPLOY_VCPU_LIMIT is set.
+//! Throws if this env is absent.
+double GetSelfPortoInstanceVCpuFactor();
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/instance_limits_tracker.cpp b/yt/yt/library/containers/instance_limits_tracker.cpp
new file mode 100644
index 0000000000..55ef7d2d67
--- /dev/null
+++ b/yt/yt/library/containers/instance_limits_tracker.cpp
@@ -0,0 +1,179 @@
+#include "public.h"
+#include "instance_limits_tracker.h"
+#include "instance.h"
+#include "porto_resource_tracker.h"
+#include "private.h"
+
+#include <yt/yt/core/concurrency/periodic_executor.h>
+
+#include <yt/yt/core/ytree/fluent.h>
+#include <yt/yt/core/ytree/ypath_service.h>
+
+namespace NYT::NContainers {
+
+using namespace NYTree;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = ContainersLogger;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TInstanceLimitsTracker::TInstanceLimitsTracker(
+ IInstancePtr instance,
+ IInstancePtr root,
+ IInvokerPtr invoker,
+ TDuration updatePeriod)
+ : Invoker_(std::move(invoker))
+ , Executor_(New<NConcurrency::TPeriodicExecutor>(
+ Invoker_,
+ BIND(&TInstanceLimitsTracker::DoUpdateLimits, MakeWeak(this)),
+ updatePeriod))
+{
+#ifdef _linux_
+ SelfTracker_ = New<TPortoResourceTracker>(std::move(instance), updatePeriod / 2);
+ RootTracker_ = New<TPortoResourceTracker>(std::move(root), updatePeriod / 2);
+#else
+ Y_UNUSED(instance);
+ Y_UNUSED(root);
+#endif
+}
+
+void TInstanceLimitsTracker::Start()
+{
+ if (!Running_) {
+ Executor_->Start();
+ Running_ = true;
+ YT_LOG_INFO("Instance limits tracker started");
+ }
+}
+
+void TInstanceLimitsTracker::Stop()
+{
+ if (Running_) {
+ YT_UNUSED_FUTURE(Executor_->Stop());
+ Running_ = false;
+ YT_LOG_INFO("Instance limits tracker stopped");
+ }
+}
+
+void TInstanceLimitsTracker::DoUpdateLimits()
+{
+ VERIFY_INVOKER_AFFINITY(Invoker_);
+
+#ifdef _linux_
+ YT_LOG_DEBUG("Checking for instance limits update");
+
+ auto setIfOk = [] (auto* destination, const auto& valueOrError, const TString& fieldName, bool alert = true) {
+ if (valueOrError.IsOK()) {
+ *destination = valueOrError.Value();
+ } else {
+ YT_LOG_ALERT_IF(alert, valueOrError, "Failed to get container property (Field: %v)",
+ fieldName);
+
+ YT_LOG_DEBUG(valueOrError, "Failed to get container property (Field: %v)",
+ fieldName);
+ }
+ };
+
+ try {
+ auto memoryStatistics = SelfTracker_->GetMemoryStatistics();
+ auto netStatistics = RootTracker_->GetNetworkStatistics();
+ auto cpuStatistics = SelfTracker_->GetCpuStatistics();
+
+ setIfOk(&MemoryUsage_, memoryStatistics.Rss, "MemoryRss");
+
+ TDuration cpuGuarantee;
+ TDuration cpuLimit;
+
+ if (cpuStatistics.GuaranteeTime.IsOK()) {
+ setIfOk(&cpuGuarantee, cpuStatistics.GuaranteeTime, "CpuGuarantee");
+ } else {
+ // XXX(don-dron, ignat): do nothing, see NContainers::TPortoInstance::GetResourceLimits, hack for missing response from Porto.
+ }
+
+ setIfOk(&cpuLimit, cpuStatistics.LimitTime, "CpuLimit");
+
+ if (CpuGuarantee_ != cpuGuarantee) {
+ YT_LOG_INFO("Instance CPU guarantee updated (OldCpuGuarantee: %v, NewCpuGuarantee: %v)",
+ CpuGuarantee_,
+ cpuGuarantee);
+ CpuGuarantee_ = cpuGuarantee;
+ // NB: We do not fire LimitsUpdated since this value used only for diagnostics.
+ }
+
+ TInstanceLimits limits;
+ limits.Cpu = cpuLimit.SecondsFloat();
+
+ if (memoryStatistics.AnonLimit.IsOK() && memoryStatistics.MemoryLimit.IsOK()) {
+ i64 anonLimit = memoryStatistics.AnonLimit.Value();
+ i64 memoryLimit = memoryStatistics.MemoryLimit.Value();
+
+ if (anonLimit > 0 && memoryLimit > 0) {
+ limits.Memory = std::min(anonLimit, memoryLimit);
+ } else if (anonLimit > 0) {
+ limits.Memory = anonLimit;
+ } else {
+ limits.Memory = memoryLimit;
+ }
+ } else {
+ setIfOk(&limits.Memory, memoryStatistics.MemoryLimit, "MemoryLimit");
+ }
+
+ static constexpr bool DontFireAlertOnError = {};
+ setIfOk(&limits.NetTx, netStatistics.TxLimit, "NetTxLimit", DontFireAlertOnError);
+ setIfOk(&limits.NetRx, netStatistics.RxLimit, "NetRxLimit", DontFireAlertOnError);
+
+ if (InstanceLimits_ != limits) {
+ YT_LOG_INFO("Instance limits updated (OldLimits: %v, NewLimits: %v)",
+ InstanceLimits_,
+ limits);
+ InstanceLimits_ = limits;
+ LimitsUpdated_.Fire(limits);
+ }
+ } catch (const std::exception& ex) {
+ YT_LOG_WARNING(ex, "Failed to get instance limits");
+ }
+#endif
+}
+
+IYPathServicePtr TInstanceLimitsTracker::GetOrchidService()
+{
+ return IYPathService::FromProducer(BIND(&TInstanceLimitsTracker::DoBuildOrchid, MakeStrong(this)))
+ ->Via(Invoker_);
+}
+
+void TInstanceLimitsTracker::DoBuildOrchid(NYson::IYsonConsumer* consumer) const
+{
+ NYTree::BuildYsonFluently(consumer)
+ .BeginMap()
+ .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) {
+ fluent.Item("cpu_limit").Value(InstanceLimits_->Cpu);
+ })
+ .DoIf(static_cast<bool>(CpuGuarantee_), [&] (auto fluent) {
+ fluent.Item("cpu_guarantee").Value(*CpuGuarantee_);
+ })
+ .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) {
+ fluent.Item("memory_limit").Value(InstanceLimits_->Memory);
+ })
+ .DoIf(static_cast<bool>(MemoryUsage_), [&] (auto fluent) {
+ fluent.Item("memory_usage").Value(*MemoryUsage_);
+ })
+ .EndMap();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf /*format*/)
+{
+ builder->AppendFormat(
+ "{Cpu: %v, Memory: %v, NetTx: %v, NetRx: %v}",
+ limits.Cpu,
+ limits.Memory,
+ limits.NetTx,
+ limits.NetRx);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/instance_limits_tracker.h b/yt/yt/library/containers/instance_limits_tracker.h
new file mode 100644
index 0000000000..e652fff446
--- /dev/null
+++ b/yt/yt/library/containers/instance_limits_tracker.h
@@ -0,0 +1,59 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/actions/signal.h>
+
+#include <yt/yt/core/concurrency/public.h>
+
+#include <yt/yt/core/yson/public.h>
+
+#include <yt/yt/core/ytree/public.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TInstanceLimitsTracker
+ : public TRefCounted
+{
+public:
+ //! Raises when container limits change.
+ DEFINE_SIGNAL(void(const TInstanceLimits&), LimitsUpdated);
+
+public:
+ TInstanceLimitsTracker(
+ IInstancePtr instance,
+ IInstancePtr root,
+ IInvokerPtr invoker,
+ TDuration updatePeriod);
+
+ void Start();
+ void Stop();
+
+ NYTree::IYPathServicePtr GetOrchidService();
+
+private:
+ void DoUpdateLimits();
+ void DoBuildOrchid(NYson::IYsonConsumer* consumer) const;
+
+ TPortoResourceTrackerPtr SelfTracker_;
+ TPortoResourceTrackerPtr RootTracker_;
+ const IInvokerPtr Invoker_;
+ const NConcurrency::TPeriodicExecutorPtr Executor_;
+
+ std::optional<TDuration> CpuGuarantee_;
+ std::optional<TInstanceLimits> InstanceLimits_;
+ std::optional<i64> MemoryUsage_;
+ bool Running_ = false;
+};
+
+DEFINE_REFCOUNTED_TYPE(TInstanceLimitsTracker)
+
+////////////////////////////////////////////////////////////////////////////////
+
+void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf format);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_executor.cpp b/yt/yt/library/containers/porto_executor.cpp
new file mode 100644
index 0000000000..a6a44fd20f
--- /dev/null
+++ b/yt/yt/library/containers/porto_executor.cpp
@@ -0,0 +1,1079 @@
+#include "porto_executor.h"
+#include "config.h"
+
+#include "private.h"
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/periodic_executor.h>
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/fs.h>
+
+#include <yt/yt/core/profiling/timing.h>
+
+#include <yt/yt/core/ytree/convert.h>
+
+#include <library/cpp/porto/proto/rpc.pb.h>
+
+#include <library/cpp/yt/memory/atomic_intrusive_ptr.h>
+
+#include <string>
+
+namespace NYT::NContainers {
+
+using namespace NConcurrency;
+using Porto::EError;
+
+////////////////////////////////////////////////////////////////////////////////
+
+#ifdef _linux_
+
+static const NLogging::TLogger& Logger = ContainersLogger;
+static constexpr auto RetryInterval = TDuration::MilliSeconds(100);
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString PortoErrorCodeFormatter(int code)
+{
+ return TEnumTraits<EPortoErrorCode>::ToString(static_cast<EPortoErrorCode>(code));
+}
+
+YT_DEFINE_ERROR_CODE_RANGE(12000, 13999, "NYT::NContainers::EPortoErrorCode", PortoErrorCodeFormatter);
+
+////////////////////////////////////////////////////////////////////////////////
+
+EPortoErrorCode ConvertPortoErrorCode(EError portoError)
+{
+ return static_cast<EPortoErrorCode>(PortoErrorCodeBase + portoError);
+}
+
+bool IsRetriableErrorCode(EPortoErrorCode error, bool idempotent)
+{
+ return
+ error == EPortoErrorCode::Unknown ||
+ // TODO(babenko): it's not obvious that we can always retry SocketError
+ // but this is how it has used to work for a while.
+ error == EPortoErrorCode::SocketError ||
+ error == EPortoErrorCode::SocketTimeout && idempotent;
+}
+
+THashMap<TString, TErrorOr<TString>> ParsePortoGetResponse(
+ const Porto::TGetResponse_TContainerGetListResponse& response)
+{
+ THashMap<TString, TErrorOr<TString>> result;
+ for (const auto& property : response.keyval()) {
+ if (property.error() == EError::Success) {
+ result[property.variable()] = property.value();
+ } else {
+ result[property.variable()] = TError(ConvertPortoErrorCode(property.error()), property.errormsg())
+ << TErrorAttribute("porto_error", ConvertPortoErrorCode(property.error()));
+ }
+ }
+ return result;
+}
+
+THashMap<TString, TErrorOr<TString>> ParseSinglePortoGetResponse(
+ const TString& name,
+ const Porto::TGetResponse& getResponse)
+{
+ for (const auto& container : getResponse.list()) {
+ if (container.name() == name) {
+ return ParsePortoGetResponse(container);
+ }
+ }
+ THROW_ERROR_EXCEPTION("Unable to get properties from Porto")
+ << TErrorAttribute("container", name);
+}
+
+THashMap<TString, THashMap<TString, TErrorOr<TString>>> ParseMultiplePortoGetResponse(
+ const Porto::TGetResponse& getResponse)
+{
+ THashMap<TString, THashMap<TString, TErrorOr<TString>>> result;
+ for (const auto& container : getResponse.list()) {
+ result[container.name()] = ParsePortoGetResponse(container);
+ }
+ return result;
+}
+
+TString FormatEnablePorto(EEnablePorto value)
+{
+ switch (value) {
+ case EEnablePorto::None: return "none";
+ case EEnablePorto::Isolate: return "isolate";
+ case EEnablePorto::Full: return "full";
+ default: YT_ABORT();
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoExecutor
+ : public IPortoExecutor
+{
+public:
+ TPortoExecutor(
+ TPortoExecutorDynamicConfigPtr config,
+ const TString& threadNameSuffix,
+ const NProfiling::TProfiler& profiler)
+ : Config_(std::move(config))
+ , Queue_(New<TActionQueue>(Format("Porto:%v", threadNameSuffix)))
+ , Profiler_(profiler)
+ , PollExecutor_(New<TPeriodicExecutor>(
+ Queue_->GetInvoker(),
+ BIND(&TPortoExecutor::DoPoll, MakeWeak(this)),
+ Config_->PollPeriod))
+ {
+ DynamicConfig_.Store(New<TPortoExecutorDynamicConfig>());
+
+ Api_->SetTimeout(Config_->ApiTimeout.Seconds());
+ Api_->SetDiskTimeout(Config_->ApiDiskTimeout.Seconds());
+
+ PollExecutor_->Start();
+ }
+
+ void SubscribeFailed(const TCallback<void (const TError&)>& callback) override
+ {
+ Failed_.Subscribe(callback);
+ }
+
+ void UnsubscribeFailed(const TCallback<void (const TError&)>& callback) override
+ {
+ Failed_.Unsubscribe(callback);
+ }
+
+ void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) override
+ {
+ DynamicConfig_.Store(newConfig);
+ }
+
+private:
+ template <class T, class... TArgs1, class... TArgs2>
+ auto ExecutePortoApiAction(
+ T(TPortoExecutor::*Method)(TArgs1...),
+ const TString& command,
+ TArgs2&&... args)
+ {
+ YT_LOG_DEBUG("Enqueue Porto API action (Command: %v)", command);
+ return BIND(Method, MakeStrong(this), std::forward<TArgs2>(args)...)
+ .AsyncVia(Queue_->GetInvoker())
+ .Run();
+ };
+
+public:
+ TFuture<void> CreateContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoCreateContainer,
+ "CreateContainer",
+ container);
+ }
+
+ TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoCreateContainerFromSpec,
+ "CreateContainerFromSpec",
+ containerSpec,
+ start);
+ }
+
+ TFuture<std::optional<TString>> GetContainerProperty(
+ const TString& container,
+ const TString& property) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoGetContainerProperty,
+ "GetContainerProperty",
+ container,
+ property);
+ }
+
+ TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties(
+ const TString& container,
+ const std::vector<TString>& properties) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoGetContainerProperties,
+ "GetContainerProperty",
+ container,
+ properties);
+ }
+
+ TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties(
+ const std::vector<TString>& containers,
+ const std::vector<TString>& properties) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoGetContainerMultipleProperties,
+ "GetContainerProperty",
+ containers,
+ properties);
+ }
+
+ TFuture<THashMap<TString, i64>> GetContainerMetrics(
+ const std::vector<TString>& containers,
+ const TString& metric) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoGetContainerMetrics,
+ "GetContainerMetrics",
+ containers,
+ metric);
+ }
+
+ TFuture<void> SetContainerProperty(
+ const TString& container,
+ const TString& property,
+ const TString& value) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoSetContainerProperty,
+ "SetContainerProperty",
+ container,
+ property,
+ value);
+ }
+
+ TFuture<void> DestroyContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoDestroyContainer,
+ "DestroyContainer",
+ container);
+ }
+
+ TFuture<void> StopContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoStopContainer,
+ "StopContainer",
+ container);
+ }
+
+ TFuture<void> StartContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoStartContainer,
+ "StartContainer",
+ container);
+ }
+
+ TFuture<TString> ConvertPath(const TString& path, const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoConvertPath,
+ "ConvertPath",
+ path,
+ container);
+ }
+
+ TFuture<void> KillContainer(const TString& container, int signal) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoKillContainer,
+ "KillContainer",
+ container,
+ signal);
+ }
+
+ TFuture<std::vector<TString>> ListSubcontainers(
+ const TString& rootContainer,
+ bool includeRoot) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoListSubcontainers,
+ "ListSubcontainers",
+ rootContainer,
+ includeRoot);
+ }
+
+ TFuture<int> PollContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoPollContainer,
+ "PollContainer",
+ container);
+ }
+
+ TFuture<int> WaitContainer(const TString& container) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoWaitContainer,
+ "WaitContainer",
+ container);
+ }
+
+ // This method allocates Porto "resources", so it should be uncancellable.
+ TFuture<TString> CreateVolume(
+ const TString& path,
+ const THashMap<TString, TString>& properties) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoCreateVolume,
+ "CreateVolume",
+ path,
+ properties)
+ .ToUncancelable();
+ }
+
+ // This method allocates Porto "resources", so it should be uncancellable.
+ TFuture<void> LinkVolume(
+ const TString& path,
+ const TString& name) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoLinkVolume,
+ "LinkVolume",
+ path,
+ name)
+ .ToUncancelable();
+ }
+
+ // This method deallocates Porto "resources", so it should be uncancellable.
+ TFuture<void> UnlinkVolume(
+ const TString& path,
+ const TString& name) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoUnlinkVolume,
+ "UnlinkVolume",
+ path,
+ name)
+ .ToUncancelable();
+ }
+
+ TFuture<std::vector<TString>> ListVolumePaths() override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoListVolumePaths,
+ "ListVolumePaths");
+ }
+
+ // This method allocates Porto "resources", so it should be uncancellable.
+ TFuture<void> ImportLayer(const TString& archivePath, const TString& layerId, const TString& place) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoImportLayer,
+ "ImportLayer",
+ archivePath,
+ layerId,
+ place)
+ .ToUncancelable();
+ }
+
+ // This method deallocates Porto "resources", so it should be uncancellable.
+ TFuture<void> RemoveLayer(const TString& layerId, const TString& place, bool async) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoRemoveLayer,
+ "RemoveLayer",
+ layerId,
+ place,
+ async)
+ .ToUncancelable();
+ }
+
+ TFuture<std::vector<TString>> ListLayers(const TString& place) override
+ {
+ return ExecutePortoApiAction(
+ &TPortoExecutor::DoListLayers,
+ "ListLayers",
+ place);
+ }
+
+ IInvokerPtr GetInvoker() const override
+ {
+ return Queue_->GetInvoker();
+ }
+
+private:
+ const TPortoExecutorDynamicConfigPtr Config_;
+ const TActionQueuePtr Queue_;
+ const NProfiling::TProfiler Profiler_;
+ const std::unique_ptr<Porto::TPortoApi> Api_ = std::make_unique<Porto::TPortoApi>();
+ const TPeriodicExecutorPtr PollExecutor_;
+ TAtomicIntrusivePtr<TPortoExecutorDynamicConfig> DynamicConfig_;
+
+ std::vector<TString> Containers_;
+ THashMap<TString, TPromise<int>> ContainerMap_;
+ TSingleShotCallbackList<void(const TError&)> Failed_;
+
+ struct TCommandEntry
+ {
+ explicit TCommandEntry(const NProfiling::TProfiler& registry)
+ : TimeGauge(registry.Timer("/command_time"))
+ , RetryCounter(registry.Counter("/command_retries"))
+ , SuccessCounter(registry.Counter("/command_successes"))
+ , FailureCounter(registry.Counter("/command_failures"))
+ { }
+
+ NProfiling::TEventTimer TimeGauge;
+ NProfiling::TCounter RetryCounter;
+ NProfiling::TCounter SuccessCounter;
+ NProfiling::TCounter FailureCounter;
+ };
+
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, CommandLock_);
+ THashMap<TString, TCommandEntry> CommandToEntry_;
+
+ static const std::vector<TString> ContainerRequestVars_;
+
+ bool IsTestPortoFailureEnabled() const
+ {
+ auto config = DynamicConfig_.Acquire();
+ return config->EnableTestPortoFailures;
+ }
+
+ bool IsTestPortoTimeout() const
+ {
+ auto config = DynamicConfig_.Acquire();
+ return config->EnableTestPortoNotResponding;
+ }
+
+ EPortoErrorCode GetFailedStubError() const
+ {
+ auto config = DynamicConfig_.Acquire();
+ return config->StubErrorCode;
+ }
+
+ static TError CreatePortoError(EPortoErrorCode errorCode, const TString& message)
+ {
+ return TError(errorCode, "Porto API error")
+ << TErrorAttribute("original_porto_error_code", static_cast<int>(errorCode) - PortoErrorCodeBase)
+ << TErrorAttribute("porto_error_message", message);
+ }
+
+ THashMap<TString, TErrorOr<TString>> DoGetContainerProperties(
+ const TString& container,
+ const std::vector<TString>& properties)
+ {
+ auto response = DoRequestContainerProperties({container}, properties);
+ return ParseSinglePortoGetResponse(container, response);
+ }
+
+ THashMap<TString, THashMap<TString, TErrorOr<TString>>> DoGetContainerMultipleProperties(
+ const std::vector<TString>& containers,
+ const std::vector<TString>& properties)
+ {
+ auto response = DoRequestContainerProperties(containers, properties);
+ return ParseMultiplePortoGetResponse(response);
+ }
+
+ std::optional<TString> DoGetContainerProperty(
+ const TString& container,
+ const TString& property)
+ {
+ auto response = DoRequestContainerProperties({container}, {property});
+ auto parsedResponse = ParseSinglePortoGetResponse(container, response);
+ auto it = parsedResponse.find(property);
+ if (it == parsedResponse.end()) {
+ return std::nullopt;
+ } else {
+ return it->second.ValueOrThrow();
+ }
+ }
+
+ void DoCreateContainer(const TString& container)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->Create(container); },
+ "Create",
+ /*idempotent*/ false);
+ }
+
+ void DoCreateContainerFromSpec(const TRunnableContainerSpec& spec, bool start)
+ {
+ Porto::TContainerSpec portoSpec;
+
+ // Required properties.
+ portoSpec.set_name(spec.Name);
+ portoSpec.set_command(spec.Command);
+
+ portoSpec.set_enable_porto(FormatEnablePorto(spec.EnablePorto));
+ portoSpec.set_isolate(spec.Isolate);
+
+ if (spec.StdinPath) {
+ portoSpec.set_stdin_path(*spec.StdinPath);
+ }
+ if (spec.StdoutPath) {
+ portoSpec.set_stdout_path(*spec.StdoutPath);
+ }
+ if (spec.StderrPath) {
+ portoSpec.set_stderr_path(*spec.StderrPath);
+ }
+
+ if (spec.CurrentWorkingDirectory) {
+ portoSpec.set_cwd(*spec.CurrentWorkingDirectory);
+ }
+
+ if (spec.CoreCommand) {
+ portoSpec.set_core_command(*spec.CoreCommand);
+ }
+ if (spec.User) {
+ portoSpec.set_user(*spec.User);
+ }
+
+ // Useful for jobs, where we operate with numeric group ids.
+ if (spec.GroupId) {
+ portoSpec.set_group(ToString(*spec.GroupId));
+ }
+
+ if (spec.ThreadLimit) {
+ portoSpec.set_thread_limit(*spec.ThreadLimit);
+ }
+
+ if (spec.HostName) {
+ // To get a reasonable and unique host name inside container.
+ portoSpec.set_hostname(*spec.HostName);
+ if (!spec.IPAddresses.empty()) {
+ const auto& address = spec.IPAddresses[0];
+ auto etcHosts = Format("%v %v\n", address, *spec.HostName);
+ // To be able to resolve hostname into IP inside container.
+ portoSpec.set_etc_hosts(etcHosts);
+ }
+ }
+
+ if (spec.DisableNetwork) {
+ auto* netConfig = portoSpec.mutable_net()->add_cfg();
+ netConfig->set_opt("none");
+ } else if (!spec.IPAddresses.empty() && Config_->EnableNetworkIsolation) {
+ // This label is intended for HBF-agent: YT-12512.
+ auto* label = portoSpec.mutable_labels()->add_map();
+ label->set_key("HBF.ignore_address");
+ label->set_val("1");
+
+ auto* netConfig = portoSpec.mutable_net()->add_cfg();
+ netConfig->set_opt("L3");
+ netConfig->add_arg("veth0");
+
+ for (const auto& address : spec.IPAddresses) {
+ auto* ipConfig = portoSpec.mutable_ip()->add_cfg();
+ ipConfig->set_dev("veth0");
+ ipConfig->set_ip(ToString(address));
+ }
+
+ if (spec.EnableNat64) {
+ // Behave like nanny does.
+ portoSpec.set_resolv_conf("nameserver fd64::1;nameserver 2a02:6b8:0:3400::5005;options attempts:1 timeout:1");
+ }
+ }
+
+ for (const auto& [key, value] : spec.Labels) {
+ auto* map = portoSpec.mutable_labels()->add_map();
+ map->set_key(key);
+ map->set_val(value);
+ }
+
+ for (const auto& [name, value] : spec.Env) {
+ auto* var = portoSpec.mutable_env()->add_var();
+ var->set_name(name);
+ var->set_value(value);
+ }
+
+ for (const auto& controller : spec.CGroupControllers) {
+ portoSpec.mutable_controllers()->add_controller(controller);
+ }
+
+ for (const auto& device : spec.Devices) {
+ auto* portoDevice = portoSpec.mutable_devices()->add_device();
+ portoDevice->set_device(device.DeviceName);
+ portoDevice->set_access(device.Enabled ? "rw" : "-");
+ }
+
+ auto addBind = [&] (const TBind& bind) {
+ auto* portoBind = portoSpec.mutable_bind()->add_bind();
+ portoBind->set_target(bind.TargetPath);
+ portoBind->set_source(bind.SourcePath);
+ portoBind->add_flag(bind.ReadOnly ? "ro" : "rw");
+ };
+
+ if (spec.RootFS) {
+ portoSpec.set_root_readonly(spec.RootFS->IsRootReadOnly);
+ portoSpec.set_root(spec.RootFS->RootPath);
+
+ for (const auto& bind : spec.RootFS->Binds) {
+ addBind(bind);
+ }
+ }
+
+ {
+ auto* ulimit = portoSpec.mutable_ulimit()->add_ulimit();
+ ulimit->set_type("core");
+ if (spec.EnableCoreDumps) {
+ ulimit->set_unlimited(true);
+ } else {
+ ulimit->set_hard(0);
+ ulimit->set_soft(0);
+ }
+ }
+
+ // Set some universal defaults.
+ portoSpec.set_oom_is_fatal(false);
+
+ ExecuteApiCall(
+ [&] { return Api_->CreateFromSpec(portoSpec, {}, start); },
+ "CreateFromSpec",
+ /*idempotent*/ false);
+ }
+
+ void DoSetContainerProperty(const TString& container, const TString& property, const TString& value)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->SetProperty(container, property, value); },
+ "SetProperty",
+ /*idempotent*/ true);
+ }
+
+ void DoDestroyContainer(const TString& container)
+ {
+ try {
+ ExecuteApiCall(
+ [&] { return Api_->Destroy(container); },
+ "Destroy",
+ /*idempotent*/ true);
+ } catch (const TErrorException& ex) {
+ if (!ex.Error().FindMatching(EPortoErrorCode::ContainerDoesNotExist)) {
+ throw;
+ }
+ }
+ }
+
+ void DoStopContainer(const TString& container)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->Stop(container); },
+ "Stop",
+ /*idempotent*/ true);
+ }
+
+ void DoStartContainer(const TString& container)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->Start(container); },
+ "Start",
+ /*idempotent*/ false);
+ }
+
+ TString DoConvertPath(const TString& path, const TString& container)
+ {
+ TString result;
+ ExecuteApiCall(
+ [&] { return Api_->ConvertPath(path, container, "self", result); },
+ "ConvertPath",
+ /*idempotent*/ true);
+ return result;
+ }
+
+ void DoKillContainer(const TString& container, int signal)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->Kill(container, signal); },
+ "Kill",
+ /*idempotent*/ false);
+ }
+
+ std::vector<TString> DoListSubcontainers(const TString& rootContainer, bool includeRoot)
+ {
+ Porto::TListContainersRequest req;
+ auto filter = req.add_filters();
+ filter->set_name(rootContainer + "/*");
+ if (includeRoot) {
+ auto rootFilter = req.add_filters();
+ rootFilter->set_name(rootContainer);
+ }
+ auto fieldOptions = req.mutable_field_options();
+ fieldOptions->add_properties("absolute_name");
+ TVector<Porto::TContainer> containers;
+ ExecuteApiCall(
+ [&] { return Api_->ListContainersBy(req, containers); },
+ "ListContainersBy",
+ /*idempotent*/ true);
+
+ std::vector<TString> containerNames;
+ containerNames.reserve(containers.size());
+ for (const auto& container : containers) {
+ const auto& absoluteName = container.status().absolute_name();
+ if (!absoluteName.empty()) {
+ containerNames.push_back(absoluteName);
+ }
+ }
+ return containerNames;
+ }
+
+ TFuture<int> DoWaitContainer(const TString& container)
+ {
+ auto result = NewPromise<int>();
+ auto waitCallback = [=, this, this_ = MakeStrong(this)] (const Porto::TWaitResponse& rsp) {
+ return OnContainerTerminated(rsp, result);
+ };
+
+ ExecuteApiCall(
+ [&] { return Api_->AsyncWait({container}, {}, waitCallback); },
+ "AsyncWait",
+ /*idempotent*/ false);
+
+ return result.ToFuture().ToImmediatelyCancelable();
+ }
+
+ void OnContainerTerminated(const Porto::TWaitResponse& portoWaitResponse, TPromise<int> result)
+ {
+ const auto& container = portoWaitResponse.name();
+ const auto& state = portoWaitResponse.state();
+ if (state != "dead" && state != "stopped") {
+ result.TrySet(TError("Container finished with unexpected state")
+ << TErrorAttribute("container_name", container)
+ << TErrorAttribute("container_state", state));
+ return;
+ }
+
+ // TODO(max42): switch to Subscribe.
+ YT_UNUSED_FUTURE(GetContainerProperty(container, "exit_status").Apply(BIND(
+ [=] (const TErrorOr<std::optional<TString>>& errorOrExitCode) {
+ if (!errorOrExitCode.IsOK()) {
+ result.TrySet(TError("Container finished, but exit status is unknown")
+ << errorOrExitCode);
+ return;
+ }
+
+ const auto& optionalExitCode = errorOrExitCode.Value();
+ if (!optionalExitCode) {
+ result.TrySet(TError("Container finished, but exit status is unknown")
+ << TErrorAttribute("container_name", container)
+ << TErrorAttribute("container_state", state));
+ return;
+ }
+
+ try {
+ int exitStatus = FromString<int>(*optionalExitCode);
+ result.TrySet(exitStatus);
+ } catch (const std::exception& ex) {
+ auto error = TError("Failed to parse Porto exit status")
+ << TErrorAttribute("container_name", container)
+ << TErrorAttribute("exit_status", optionalExitCode.value());
+ error.MutableInnerErrors()->push_back(TError(ex));
+ result.TrySet(error);
+ }
+ })));
+ }
+
+ TFuture<int> DoPollContainer(const TString& container)
+ {
+ auto [it, inserted] = ContainerMap_.insert({container, NewPromise<int>()});
+ if (!inserted) {
+ YT_LOG_WARNING("Container already added for polling (Container: %v)",
+ container);
+ } else {
+ Containers_.push_back(container);
+ }
+ return it->second.ToFuture();
+ }
+
+ Porto::TGetResponse DoRequestContainerProperties(
+ const std::vector<TString>& containers,
+ const std::vector<TString>& vars)
+ {
+ TVector<TString> containers_(containers.begin(), containers.end());
+ TVector<TString> vars_(vars.begin(), vars.end());
+
+ const Porto::TGetResponse* getResponse;
+
+ ExecuteApiCall(
+ [&] {
+ getResponse = Api_->Get(containers_, vars_);
+ return getResponse ? EError::Success : EError::Unknown;
+ },
+ "Get",
+ /*idempotent*/ true);
+
+ YT_VERIFY(getResponse);
+ return *getResponse;
+ }
+
+ THashMap<TString, i64> DoGetContainerMetrics(
+ const std::vector<TString>& containers,
+ const TString& metric)
+ {
+ TVector<TString> containers_(containers.begin(), containers.end());
+
+ TMap<TString, uint64_t> result;
+
+ ExecuteApiCall(
+ [&] { return Api_->GetProcMetric(containers_, metric, result); },
+ "GetProcMetric",
+ /*idempotent*/ true);
+
+ return {result.begin(), result.end()};
+ }
+
+ void DoPoll()
+ {
+ try {
+ if (Containers_.empty()) {
+ return;
+ }
+
+ auto getResponse = DoRequestContainerProperties(Containers_, ContainerRequestVars_);
+
+ if (getResponse.list().empty()) {
+ return;
+ }
+
+ auto getProperty = [] (
+ const Porto::TGetResponse::TContainerGetListResponse& container,
+ const TString& name) -> Porto::TGetResponse::TContainerGetValueResponse
+ {
+ for (const auto& property : container.keyval()) {
+ if (property.variable() == name) {
+ return property;
+ }
+ }
+
+ return {};
+ };
+
+ for (const auto& container : getResponse.list()) {
+ auto state = getProperty(container, "state");
+ if (state.error() == EError::ContainerDoesNotExist) {
+ HandleResult(container.name(), state);
+ } else if (state.value() == "dead" || state.value() == "stopped") {
+ HandleResult(container.name(), getProperty(container, "exit_status"));
+ }
+ //TODO(dcherednik): other states
+ }
+ } catch (const std::exception& ex) {
+ YT_LOG_ERROR(ex, "Fatal exception occurred while polling Porto");
+ Failed_.Fire(TError(ex));
+ }
+ }
+
+ TString DoCreateVolume(
+ const TString& path,
+ const THashMap<TString, TString>& properties)
+ {
+ auto volume = path;
+ TMap<TString, TString> propertyMap(properties.begin(), properties.end());
+ ExecuteApiCall(
+ [&] { return Api_->CreateVolume(volume, propertyMap); },
+ "CreateVolume",
+ /*idempotent*/ false);
+ return volume;
+ }
+
+ void DoLinkVolume(const TString& path, const TString& container)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->LinkVolume(path, container); },
+ "LinkVolume",
+ /*idempotent*/ false);
+ }
+
+ void DoUnlinkVolume(const TString& path, const TString& container)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->UnlinkVolume(path, container); },
+ "UnlinkVolume",
+ /*idempotent*/ false);
+ }
+
+ std::vector<TString> DoListVolumePaths()
+ {
+ TVector<TString> volumes;
+ ExecuteApiCall(
+ [&] { return Api_->ListVolumes(volumes); },
+ "ListVolume",
+ /*idempotent*/ true);
+ return {volumes.begin(), volumes.end()};
+ }
+
+ void DoImportLayer(const TString& archivePath, const TString& layerId, const TString& place)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->ImportLayer(layerId, archivePath, false, place); },
+ "ImportLayer",
+ /*idempotent*/ false);
+ }
+
+ void DoRemoveLayer(const TString& layerId, const TString& place, bool async)
+ {
+ ExecuteApiCall(
+ [&] { return Api_->RemoveLayer(layerId, place, async); },
+ "RemoveLayer",
+ /*idempotent*/ false);
+ }
+
+ std::vector<TString> DoListLayers(const TString& place)
+ {
+ TVector<TString> layers;
+ ExecuteApiCall(
+ [&] { return Api_->ListLayers(layers, place); },
+ "ListLayers",
+ /*idempotent*/ true);
+ return {layers.begin(), layers.end()};
+ }
+
+ TCommandEntry* GetCommandEntry(const TString& command)
+ {
+ auto guard = Guard(CommandLock_);
+ if (auto it = CommandToEntry_.find(command)) {
+ return &it->second;
+ }
+ return &CommandToEntry_.emplace(command, TCommandEntry(Profiler_.WithTag("command", command))).first->second;
+ }
+
+ void ExecuteApiCall(
+ std::function<EError()> callback,
+ const TString& command,
+ bool idempotent)
+ {
+ YT_LOG_DEBUG("Porto API call started (Command: %v)", command);
+
+ if (IsTestPortoTimeout()) {
+ YT_LOG_DEBUG("Testing Porto timeout (Command: %v)", command);
+
+ auto config = DynamicConfig_.Acquire();
+ TDelayedExecutor::WaitForDuration(config->ApiTimeout);
+
+ THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto timeout");
+ }
+
+ if (IsTestPortoFailureEnabled()) {
+ YT_LOG_DEBUG("Testing Porto failure (Command: %v)", command);
+ THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto stub error");
+ }
+
+ auto* entry = GetCommandEntry(command);
+ auto startTime = NProfiling::GetInstant();
+ while (true) {
+ EError error;
+
+ {
+ NProfiling::TWallTimer timer;
+ error = callback();
+ entry->TimeGauge.Record(timer.GetElapsedTime());
+ }
+
+ if (error == EError::Success) {
+ entry->SuccessCounter.Increment();
+ break;
+ }
+
+ entry->FailureCounter.Increment();
+ HandleApiError(command, startTime, idempotent);
+
+ YT_LOG_DEBUG("Sleeping and retrying Porto API call (Command: %v)", command);
+ entry->RetryCounter.Increment();
+
+ TDelayedExecutor::WaitForDuration(RetryInterval);
+ }
+
+ YT_LOG_DEBUG("Porto API call completed (Command: %v)", command);
+ }
+
+ void HandleApiError(
+ const TString& command,
+ TInstant startTime,
+ bool idempotent)
+ {
+ TString errorMessage;
+ auto error = ConvertPortoErrorCode(Api_->GetLastError(errorMessage));
+
+ // These errors are typical during job cleanup: we might try to kill a container that is already stopped.
+ bool debug = (error == EPortoErrorCode::ContainerDoesNotExist || error == EPortoErrorCode::InvalidState);
+ YT_LOG_EVENT(
+ Logger,
+ debug ? NLogging::ELogLevel::Debug : NLogging::ELogLevel::Error,
+ "Porto API call error (Error: %v, Command: %v, Message: %v)",
+ error,
+ command,
+ errorMessage);
+
+ if (!IsRetriableErrorCode(error, idempotent) || NProfiling::GetInstant() - startTime > Config_->RetriesTimeout) {
+ THROW_ERROR CreatePortoError(error, errorMessage);
+ }
+ }
+
+ void HandleResult(const TString& container, const Porto::TGetResponse::TContainerGetValueResponse& rsp)
+ {
+ auto portoErrorCode = ConvertPortoErrorCode(rsp.error());
+ auto it = ContainerMap_.find(container);
+ if (it == ContainerMap_.end()) {
+ YT_LOG_ERROR("Got an unexpected container "
+ "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)",
+ container,
+ portoErrorCode,
+ rsp.errormsg(),
+ rsp.value());
+ return;
+ } else {
+ if (portoErrorCode != EPortoErrorCode::Success) {
+ YT_LOG_ERROR("Container finished with Porto API error "
+ "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)",
+ container,
+ portoErrorCode,
+ rsp.errormsg(),
+ rsp.value());
+ it->second.Set(CreatePortoError(portoErrorCode, rsp.errormsg()));
+ } else {
+ try {
+ int exitStatus = std::stoi(rsp.value());
+ YT_LOG_DEBUG("Container finished with exit code (Container: %v, ExitCode: %v)",
+ container,
+ exitStatus);
+
+ it->second.Set(exitStatus);
+ } catch (const std::exception& ex) {
+ it->second.Set(TError("Failed to parse Porto exit status") << ex);
+ }
+ }
+ }
+ RemoveFromPoller(container);
+ }
+
+ void RemoveFromPoller(const TString& container)
+ {
+ ContainerMap_.erase(container);
+
+ Containers_.clear();
+ for (const auto& [name, pid] : ContainerMap_) {
+ Containers_.push_back(name);
+ }
+ }
+};
+
+const std::vector<TString> TPortoExecutor::ContainerRequestVars_ = {
+ "state",
+ "exit_status"
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+IPortoExecutorPtr CreatePortoExecutor(
+ TPortoExecutorDynamicConfigPtr config,
+ const TString& threadNameSuffix,
+ const NProfiling::TProfiler& profiler)
+{
+ return New<TPortoExecutor>(
+ std::move(config),
+ threadNameSuffix,
+ profiler);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+#else
+
+IPortoExecutorPtr CreatePortoExecutor(
+ TPortoExecutorDynamicConfigPtr /* config */,
+ const TString& /* threadNameSuffix */,
+ const NProfiling::TProfiler& /* profiler */)
+{
+ THROW_ERROR_EXCEPTION("Porto executor is not available on this platform");
+}
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_executor.h b/yt/yt/library/containers/porto_executor.h
new file mode 100644
index 0000000000..d629ab6275
--- /dev/null
+++ b/yt/yt/library/containers/porto_executor.h
@@ -0,0 +1,142 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/library/profiling/sensor.h>
+
+#include <yt/yt/core/actions/future.h>
+#include <yt/yt/core/actions/signal.h>
+
+#include <yt/yt/core/net/address.h>
+
+#include <library/cpp/porto/libporto.hpp>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TVolumeId
+{
+ TString Path;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TRunnableContainerSpec
+{
+ TString Name;
+ TString Command;
+
+ EEnablePorto EnablePorto = EEnablePorto::None;
+ bool Isolate = true;
+
+ std::optional<TString> StdinPath;
+ std::optional<TString> StdoutPath;
+ std::optional<TString> StderrPath;
+ std::optional<TString> CurrentWorkingDirectory;
+ std::optional<TString> CoreCommand;
+ std::optional<TString> User;
+ std::optional<int> GroupId;
+
+ bool EnableCoreDumps = true;
+
+ std::optional<i64> ThreadLimit;
+
+ std::optional<TString> HostName;
+ std::vector<NYT::NNet::TIP6Address> IPAddresses;
+ bool EnableNat64 = false;
+ bool DisableNetwork = false;
+
+ THashMap<TString, TString> Labels;
+ THashMap<TString, TString> Env;
+ std::vector<TString> CGroupControllers;
+ std::vector<TDevice> Devices;
+ std::optional<TRootFS> RootFS;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IPortoExecutor
+ : public TRefCounted
+{
+ virtual void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) = 0;
+
+ virtual TFuture<void> CreateContainer(const TString& container) = 0;
+
+ virtual TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) = 0;
+
+ virtual TFuture<void> SetContainerProperty(
+ const TString& container,
+ const TString& property,
+ const TString& value) = 0;
+
+ virtual TFuture<std::optional<TString>> GetContainerProperty(
+ const TString& container,
+ const TString& property) = 0;
+
+ virtual TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties(
+ const TString& container,
+ const std::vector<TString>& properties) = 0;
+ virtual TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties(
+ const std::vector<TString>& containers,
+ const std::vector<TString>& properties) = 0;
+
+ virtual TFuture<THashMap<TString, i64>> GetContainerMetrics(
+ const std::vector<TString>& containers,
+ const TString& metric) = 0;
+ virtual TFuture<void> DestroyContainer(const TString& container) = 0;
+ virtual TFuture<void> StopContainer(const TString& container) = 0;
+ virtual TFuture<void> StartContainer(const TString& container) = 0;
+ virtual TFuture<void> KillContainer(const TString& container, int signal) = 0;
+
+ virtual TFuture<TString> ConvertPath(const TString& path, const TString& container) = 0;
+
+ // Returns absolute names of immediate children only.
+ virtual TFuture<std::vector<TString>> ListSubcontainers(
+ const TString& rootContainer,
+ bool includeRoot) = 0;
+ // Starts polling a given container, returns future with exit code of finished process.
+ virtual TFuture<int> PollContainer(const TString& container) = 0;
+
+ // Returns future with exit code of finished process.
+ // NB: temporarily broken, see https://st.yandex-team.ru/PORTO-846 for details.
+ virtual TFuture<int> WaitContainer(const TString& container) = 0;
+
+ virtual TFuture<TString> CreateVolume(
+ const TString& path,
+ const THashMap<TString, TString>& properties) = 0;
+ virtual TFuture<void> LinkVolume(
+ const TString& path,
+ const TString& name) = 0;
+ virtual TFuture<void> UnlinkVolume(
+ const TString& path,
+ const TString& name) = 0;
+ virtual TFuture<std::vector<TString>> ListVolumePaths() = 0;
+
+ virtual TFuture<void> ImportLayer(
+ const TString& archivePath,
+ const TString& layerId,
+ const TString& place) = 0;
+ virtual TFuture<void> RemoveLayer(
+ const TString& layerId,
+ const TString& place,
+ bool async) = 0;
+ virtual TFuture<std::vector<TString>> ListLayers(const TString& place) = 0;
+
+ virtual IInvokerPtr GetInvoker() const = 0;
+
+ DECLARE_INTERFACE_SIGNAL(void(const TError&), Failed);
+};
+
+DEFINE_REFCOUNTED_TYPE(IPortoExecutor)
+
+////////////////////////////////////////////////////////////////////////////////
+
+IPortoExecutorPtr CreatePortoExecutor(
+ TPortoExecutorDynamicConfigPtr config,
+ const TString& threadNameSuffix,
+ const NProfiling::TProfiler& profiler = {});
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_health_checker.cpp b/yt/yt/library/containers/porto_health_checker.cpp
new file mode 100644
index 0000000000..5a5d358441
--- /dev/null
+++ b/yt/yt/library/containers/porto_health_checker.cpp
@@ -0,0 +1,69 @@
+
+#include "porto_health_checker.h"
+
+#include "porto_executor.h"
+#include "private.h"
+#include "config.h"
+
+#include <yt/yt/core/actions/future.h>
+
+#include <yt/yt/core/misc/fs.h>
+
+#include <util/random/random.h>
+
+namespace NYT::NContainers {
+
+using namespace NConcurrency;
+using namespace NLogging;
+using namespace NProfiling;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPortoHealthChecker::TPortoHealthChecker(
+ TPortoExecutorDynamicConfigPtr config,
+ IInvokerPtr invoker,
+ TLogger logger)
+ : Config_(std::move(config))
+ , Logger(std::move(logger))
+ , CheckInvoker_(std::move(invoker))
+ , Executor_(CreatePortoExecutor(
+ Config_,
+ "porto_check"))
+{ }
+
+void TPortoHealthChecker::Start()
+{
+ YT_LOG_DEBUG("Porto health checker started");
+
+ PeriodicExecutor_ = New<TPeriodicExecutor>(
+ CheckInvoker_,
+ BIND(&TPortoHealthChecker::OnCheck, MakeWeak(this)),
+ Config_->RetriesTimeout);
+ PeriodicExecutor_->Start();
+}
+
+void TPortoHealthChecker::OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig)
+{
+ YT_LOG_DEBUG(
+ "Porto health checker dynamic config changed (EnableTestPortoFailures: %v, StubErrorCode: %v)",
+ Config_->EnableTestPortoFailures,
+ Config_->StubErrorCode);
+
+ Executor_->OnDynamicConfigChanged(newConfig);
+}
+
+void TPortoHealthChecker::OnCheck()
+{
+ YT_LOG_DEBUG("Run Porto health check");
+
+ auto result = WaitFor(Executor_->ListVolumePaths().AsVoid());
+ if (result.IsOK()) {
+ Success_.Fire();
+ } else {
+ Failed_.Fire(result);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_health_checker.h b/yt/yt/library/containers/porto_health_checker.h
new file mode 100644
index 0000000000..f0fb8f0908
--- /dev/null
+++ b/yt/yt/library/containers/porto_health_checker.h
@@ -0,0 +1,52 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/library/profiling/sensor.h>
+
+#include <yt/yt/core/actions/signal.h>
+
+#include <yt/yt/core/concurrency/periodic_executor.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/error.h>
+
+#include <atomic>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoHealthChecker
+ : public TRefCounted
+{
+public:
+ TPortoHealthChecker(
+ TPortoExecutorDynamicConfigPtr config,
+ IInvokerPtr invoker,
+ NLogging::TLogger logger);
+
+ void Start();
+
+ void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig);
+
+ DEFINE_SIGNAL(void(), Success);
+
+ DEFINE_SIGNAL(void(const TError&), Failed);
+
+private:
+ const TPortoExecutorDynamicConfigPtr Config_;
+ const NLogging::TLogger Logger;
+ const IInvokerPtr CheckInvoker_;
+ const IPortoExecutorPtr Executor_;
+ NConcurrency::TPeriodicExecutorPtr PeriodicExecutor_;
+
+ void OnCheck();
+};
+
+DEFINE_REFCOUNTED_TYPE(TPortoHealthChecker)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_resource_tracker.cpp b/yt/yt/library/containers/porto_resource_tracker.cpp
new file mode 100644
index 0000000000..c1fe48d6af
--- /dev/null
+++ b/yt/yt/library/containers/porto_resource_tracker.cpp
@@ -0,0 +1,711 @@
+#include "porto_resource_tracker.h"
+#include "private.h"
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/error.h>
+
+#include <yt/yt/core/net/address.h>
+
+#include <yt/yt/core/ytree/public.h>
+
+#include <yt/yt/library/process/process.h>
+
+#include <yt/yt/library/containers/cgroup.h>
+#include <yt/yt/library/containers/config.h>
+#include <yt/yt/library/containers/instance.h>
+#include <yt/yt/library/containers/porto_executor.h>
+#include <yt/yt/library/containers/public.h>
+
+namespace NYT::NContainers {
+
+using namespace NProfiling;
+
+static const auto& Logger = ContainersLogger;
+
+#ifdef _linux_
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TPortoProfilers
+ : public TRefCounted
+{
+ TPortoResourceProfilerPtr DaemonProfiler;
+ TPortoResourceProfilerPtr ContainerProfiler;
+
+ TPortoProfilers(
+ TPortoResourceProfilerPtr daemonProfiler,
+ TPortoResourceProfilerPtr containerProfiler)
+ : DaemonProfiler(std::move(daemonProfiler))
+ , ContainerProfiler(std::move(containerProfiler))
+ { }
+};
+
+DEFINE_REFCOUNTED_TYPE(TPortoProfilers)
+
+////////////////////////////////////////////////////////////////////////////////
+
+static TErrorOr<ui64> GetFieldOrError(
+ const TResourceUsage& usage,
+ EStatField field)
+{
+ auto it = usage.find(field);
+ if (it == usage.end()) {
+ return TError("Resource usage is missing %Qlv field", field);
+ }
+ const auto& errorOrValue = it->second;
+ if (errorOrValue.FindMatching(EPortoErrorCode::NotSupported)) {
+ return TError("Property %Qlv not supported in Porto response", field);
+ }
+ return errorOrValue;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPortoResourceTracker::TPortoResourceTracker(
+ IInstancePtr instance,
+ TDuration updatePeriod,
+ bool isDeltaTracker,
+ bool isForceUpdate)
+ : Instance_(std::move(instance))
+ , UpdatePeriod_(updatePeriod)
+ , IsDeltaTracker_(isDeltaTracker)
+ , IsForceUpdate_(isForceUpdate)
+{
+ ResourceUsage_ = {
+ {EStatField::IOReadByte, 0},
+ {EStatField::IOWriteByte, 0},
+ {EStatField::IOBytesLimit, 0},
+ {EStatField::IOReadOps, 0},
+ {EStatField::IOWriteOps, 0},
+ {EStatField::IOOps, 0},
+ {EStatField::IOOpsLimit, 0},
+ {EStatField::IOTotalTime, 0},
+ {EStatField::IOWaitTime, 0}
+ };
+ ResourceUsageDelta_ = ResourceUsage_;
+}
+
+static TErrorOr<TDuration> ExtractDuration(TErrorOr<ui64> timeNs)
+{
+ if (timeNs.IsOK()) {
+ return TErrorOr<TDuration>(TDuration::MicroSeconds(timeNs.Value() / 1000));
+ } else {
+ return TError(timeNs);
+ }
+}
+
+TCpuStatistics TPortoResourceTracker::ExtractCpuStatistics(const TResourceUsage& resourceUsage) const
+{
+ // NB: Job proxy uses last sample of CPU statistics but we are interested in
+ // peak thread count value.
+ auto currentThreadCountPeak = GetFieldOrError(resourceUsage, EStatField::ThreadCount);
+
+ PeakThreadCount_ = currentThreadCountPeak.IsOK() && PeakThreadCount_.IsOK()
+ ? std::max<ui64>(
+ PeakThreadCount_.Value(),
+ currentThreadCountPeak.Value())
+ : currentThreadCountPeak.IsOK() ? currentThreadCountPeak : PeakThreadCount_;
+
+ auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUsage);
+ auto systemTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuSystemUsage);
+ auto userTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUserUsage);
+ auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuWait);
+ auto throttledNs = GetFieldOrError(resourceUsage, EStatField::CpuThrottled);
+ auto limitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuLimit);
+ auto guaranteeTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuGuarantee);
+
+ return TCpuStatistics{
+ .TotalUsageTime = ExtractDuration(totalTimeNs),
+ .UserUsageTime = ExtractDuration(userTimeNs),
+ .SystemUsageTime = ExtractDuration(systemTimeNs),
+ .WaitTime = ExtractDuration(waitTimeNs),
+ .ThrottledTime = ExtractDuration(throttledNs),
+ .ThreadCount = GetFieldOrError(resourceUsage, EStatField::ThreadCount),
+ .ContextSwitches = GetFieldOrError(resourceUsage, EStatField::ContextSwitches),
+ .ContextSwitchesDelta = GetFieldOrError(resourceUsage, EStatField::ContextSwitchesDelta),
+ .PeakThreadCount = PeakThreadCount_,
+ .LimitTime = ExtractDuration(limitTimeNs),
+ .GuaranteeTime = ExtractDuration(guaranteeTimeNs),
+ };
+}
+
+TMemoryStatistics TPortoResourceTracker::ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const
+{
+ return TMemoryStatistics{
+ .Rss = GetFieldOrError(resourceUsage, EStatField::Rss),
+ .MappedFile = GetFieldOrError(resourceUsage, EStatField::MappedFile),
+ .MinorPageFaults = GetFieldOrError(resourceUsage, EStatField::MinorPageFaults),
+ .MajorPageFaults = GetFieldOrError(resourceUsage, EStatField::MajorPageFaults),
+ .FileCacheUsage = GetFieldOrError(resourceUsage, EStatField::FileCacheUsage),
+ .AnonUsage = GetFieldOrError(resourceUsage, EStatField::AnonMemoryUsage),
+ .AnonLimit = GetFieldOrError(resourceUsage, EStatField::AnonMemoryLimit),
+ .MemoryUsage = GetFieldOrError(resourceUsage, EStatField::MemoryUsage),
+ .MemoryGuarantee = GetFieldOrError(resourceUsage, EStatField::MemoryGuarantee),
+ .MemoryLimit = GetFieldOrError(resourceUsage, EStatField::MemoryLimit),
+ .MaxMemoryUsage = GetFieldOrError(resourceUsage, EStatField::MaxMemoryUsage),
+ .OomKills = GetFieldOrError(resourceUsage, EStatField::OomKills),
+ .OomKillsTotal = GetFieldOrError(resourceUsage, EStatField::OomKillsTotal)
+ };
+}
+
+TBlockIOStatistics TPortoResourceTracker::ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const
+{
+ auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::IOTotalTime);
+ auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::IOWaitTime);
+
+ return TBlockIOStatistics{
+ .IOReadByte = GetFieldOrError(resourceUsage, EStatField::IOReadByte),
+ .IOWriteByte = GetFieldOrError(resourceUsage, EStatField::IOWriteByte),
+ .IOBytesLimit = GetFieldOrError(resourceUsage, EStatField::IOBytesLimit),
+ .IOReadOps = GetFieldOrError(resourceUsage, EStatField::IOReadOps),
+ .IOWriteOps = GetFieldOrError(resourceUsage, EStatField::IOWriteOps),
+ .IOOps = GetFieldOrError(resourceUsage, EStatField::IOOps),
+ .IOOpsLimit = GetFieldOrError(resourceUsage, EStatField::IOOpsLimit),
+ .IOTotalTime = ExtractDuration(totalTimeNs),
+ .IOWaitTime = ExtractDuration(waitTimeNs)
+ };
+}
+
+TNetworkStatistics TPortoResourceTracker::ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const
+{
+ return TNetworkStatistics{
+ .TxBytes = GetFieldOrError(resourceUsage, EStatField::NetTxBytes),
+ .TxPackets = GetFieldOrError(resourceUsage, EStatField::NetTxPackets),
+ .TxDrops = GetFieldOrError(resourceUsage, EStatField::NetTxDrops),
+ .TxLimit = GetFieldOrError(resourceUsage, EStatField::NetTxLimit),
+
+ .RxBytes = GetFieldOrError(resourceUsage, EStatField::NetRxBytes),
+ .RxPackets = GetFieldOrError(resourceUsage, EStatField::NetRxPackets),
+ .RxDrops = GetFieldOrError(resourceUsage, EStatField::NetRxDrops),
+ .RxLimit = GetFieldOrError(resourceUsage, EStatField::NetRxLimit),
+ };
+}
+
+TTotalStatistics TPortoResourceTracker::ExtractTotalStatistics(const TResourceUsage& resourceUsage) const
+{
+ return TTotalStatistics{
+ .CpuStatistics = ExtractCpuStatistics(resourceUsage),
+ .MemoryStatistics = ExtractMemoryStatistics(resourceUsage),
+ .BlockIOStatistics = ExtractBlockIOStatistics(resourceUsage),
+ .NetworkStatistics = ExtractNetworkStatistics(resourceUsage),
+ };
+}
+
+TCpuStatistics TPortoResourceTracker::GetCpuStatistics() const
+{
+ return GetStatistics(
+ CachedCpuStatistics_,
+ "CPU",
+ [&] (TResourceUsage& resourceUsage) {
+ return ExtractCpuStatistics(resourceUsage);
+ });
+}
+
+TMemoryStatistics TPortoResourceTracker::GetMemoryStatistics() const
+{
+ return GetStatistics(
+ CachedMemoryStatistics_,
+ "memory",
+ [&] (TResourceUsage& resourceUsage) {
+ return ExtractMemoryStatistics(resourceUsage);
+ });
+}
+
+TBlockIOStatistics TPortoResourceTracker::GetBlockIOStatistics() const
+{
+ return GetStatistics(
+ CachedBlockIOStatistics_,
+ "block IO",
+ [&] (TResourceUsage& resourceUsage) {
+ return ExtractBlockIOStatistics(resourceUsage);
+ });
+}
+
+TNetworkStatistics TPortoResourceTracker::GetNetworkStatistics() const
+{
+ return GetStatistics(
+ CachedNetworkStatistics_,
+ "network",
+ [&] (TResourceUsage& resourceUsage) {
+ return ExtractNetworkStatistics(resourceUsage);
+ });
+}
+
+TTotalStatistics TPortoResourceTracker::GetTotalStatistics() const
+{
+ return GetStatistics(
+ CachedTotalStatistics_,
+ "total",
+ [&] (TResourceUsage& resourceUsage) {
+ return ExtractTotalStatistics(resourceUsage);
+ });
+}
+
+template <class T, class F>
+T TPortoResourceTracker::GetStatistics(
+ std::optional<T>& cachedStatistics,
+ const TString& statisticsKind,
+ F extractor) const
+{
+ UpdateResourceUsageStatisticsIfExpired();
+
+ auto guard = Guard(SpinLock_);
+ try {
+ auto newStatistics = extractor(IsDeltaTracker_ ? ResourceUsageDelta_ : ResourceUsage_);
+ cachedStatistics = newStatistics;
+ return newStatistics;
+ } catch (const std::exception& ex) {
+ if (!cachedStatistics) {
+ THROW_ERROR_EXCEPTION("Unable to get %v statistics", statisticsKind)
+ << ex;
+ }
+ YT_LOG_WARNING(ex, "Unable to get %v statistics; using the last one", statisticsKind);
+ return *cachedStatistics;
+ }
+}
+
+bool TPortoResourceTracker::AreResourceUsageStatisticsExpired() const
+{
+ return TInstant::Now() - LastUpdateTime_.load() > UpdatePeriod_;
+}
+
+TInstant TPortoResourceTracker::GetLastUpdateTime() const
+{
+ return LastUpdateTime_.load();
+}
+
+void TPortoResourceTracker::UpdateResourceUsageStatisticsIfExpired() const
+{
+ if (IsForceUpdate_ || AreResourceUsageStatisticsExpired()) {
+ DoUpdateResourceUsage();
+ }
+}
+
+TErrorOr<ui64> TPortoResourceTracker::CalculateCounterDelta(
+ const TErrorOr<ui64>& oldValue,
+ const TErrorOr<ui64>& newValue) const
+{
+ if (oldValue.IsOK() && newValue.IsOK()) {
+ return newValue.Value() - oldValue.Value();
+ } else if (newValue.IsOK()) {
+ // It is better to return an error than an incorrect value.
+ return oldValue;
+ } else {
+ return newValue;
+ }
+}
+
+static bool IsCumulativeStatistics(EStatField statistic)
+{
+ return
+ statistic == EStatField::CpuUsage ||
+ statistic == EStatField::CpuUserUsage ||
+ statistic == EStatField::CpuSystemUsage ||
+ statistic == EStatField::CpuWait ||
+ statistic == EStatField::CpuThrottled ||
+
+ statistic == EStatField::ContextSwitches ||
+
+ statistic == EStatField::MinorPageFaults ||
+ statistic == EStatField::MajorPageFaults ||
+
+ statistic == EStatField::IOReadByte ||
+ statistic == EStatField::IOWriteByte ||
+ statistic == EStatField::IOReadOps ||
+ statistic == EStatField::IOWriteOps ||
+ statistic == EStatField::IOOps ||
+ statistic == EStatField::IOTotalTime ||
+ statistic == EStatField::IOWaitTime ||
+
+ statistic == EStatField::NetTxBytes ||
+ statistic == EStatField::NetTxPackets ||
+ statistic == EStatField::NetTxDrops ||
+ statistic == EStatField::NetRxBytes ||
+ statistic == EStatField::NetRxPackets ||
+ statistic == EStatField::NetRxDrops;
+}
+
+void TPortoResourceTracker::ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const
+{
+ auto guard = Guard(SpinLock_);
+
+ TResourceUsage resourceUsage;
+ TResourceUsage resourceUsageDelta;
+
+ for (const auto& stat : InstanceStatFields) {
+ TErrorOr<ui64> oldValue;
+ TErrorOr<ui64> newValue;
+
+ if (auto newValueIt = newResourceUsage.find(stat); newValueIt.IsEnd()) {
+ newValue = TError("Missing property %Qlv in Porto response", stat)
+ << TErrorAttribute("container", Instance_->GetName());
+ } else {
+ newValue = newValueIt->second;
+ }
+
+ if (auto oldValueIt = ResourceUsage_.find(stat); oldValueIt.IsEnd()) {
+ oldValue = newValue;
+ } else {
+ oldValue = oldValueIt->second;
+ }
+
+ if (newValue.IsOK()) {
+ resourceUsage[stat] = newValue;
+ } else {
+ resourceUsage[stat] = oldValue;
+ }
+
+ if (IsCumulativeStatistics(stat)) {
+ resourceUsageDelta[stat] = CalculateCounterDelta(oldValue, newValue);
+ } else {
+ if (newValue.IsOK()) {
+ resourceUsageDelta[stat] = newValue;
+ } else {
+ resourceUsageDelta[stat] = oldValue;
+ }
+ }
+ }
+
+ ResourceUsage_ = resourceUsage;
+ ResourceUsageDelta_ = resourceUsageDelta;
+ LastUpdateTime_.store(TInstant::Now());
+}
+
+void TPortoResourceTracker::DoUpdateResourceUsage() const
+{
+ try {
+ ReCalculateResourceUsage(Instance_->GetResourceUsage());
+ } catch (const std::exception& ex) {
+ YT_LOG_ERROR(
+ ex,
+ "Couldn't get metrics from Porto");
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPortoResourceProfiler::TPortoResourceProfiler(
+ TPortoResourceTrackerPtr tracker,
+ TPodSpecConfigPtr podSpec,
+ const TProfiler& profiler)
+ : ResourceTracker_(std::move(tracker))
+ , PodSpec_(std::move(podSpec))
+{
+ profiler.AddProducer("", MakeStrong(this));
+}
+
+static void WriteGaugeIfOk(
+ ISensorWriter* writer,
+ const TString& path,
+ TErrorOr<ui64> valueOrError)
+{
+ if (valueOrError.IsOK()) {
+ i64 value = static_cast<i64>(valueOrError.Value());
+
+ if (value >= 0) {
+ writer->AddGauge(path, value);
+ }
+ }
+}
+
+static void WriteCumulativeGaugeIfOk(
+ ISensorWriter* writer,
+ const TString& path,
+ TErrorOr<ui64> valueOrError,
+ i64 timeDeltaUsec)
+{
+ if (valueOrError.IsOK()) {
+ i64 value = static_cast<i64>(valueOrError.Value());
+
+ if (value >= 0) {
+ writer->AddGauge(path,
+ 1.0 * value * ResourceUsageUpdatePeriod.MicroSeconds() / timeDeltaUsec);
+ }
+ }
+}
+
+void TPortoResourceProfiler::WriteCpuMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec)
+{
+ {
+ if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) {
+ i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds();
+ double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs / timeDeltaUsec);
+ writer->AddGauge("/cpu/user", userUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) {
+ i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds();
+ double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs / timeDeltaUsec);
+ writer->AddGauge("/cpu/system", systemUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.WaitTime.IsOK()) {
+ i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds();
+ double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec);
+ writer->AddGauge("/cpu/wait", waitPercent);
+ }
+
+ if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) {
+ i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds();
+ double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs / timeDeltaUsec);
+ writer->AddGauge("/cpu/throttled", throttledPercent);
+ }
+
+ if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) {
+ i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds();
+ double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs / timeDeltaUsec);
+ writer->AddGauge("/cpu/total", totalUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) {
+ i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds();
+ double guaranteePercent = std::max<double>(0.0, (100. * guaranteeTimeUs) / (1'000'000L));
+ writer->AddGauge("/cpu/guarantee", guaranteePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.LimitTime.IsOK()) {
+ i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds();
+ double limitPercent = std::max<double>(0.0, (100. * limitTimeUs) / (1'000'000L));
+ writer->AddGauge("/cpu/limit", limitPercent);
+ }
+ }
+
+ if (PodSpec_->CpuToVCpuFactor) {
+ auto factor = *PodSpec_->CpuToVCpuFactor;
+
+ writer->AddGauge("/cpu_to_vcpu_factor", factor);
+
+ if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) {
+ i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds();
+ double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs * factor / timeDeltaUsec);
+ writer->AddGauge("/vcpu/user", userUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) {
+ i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds();
+ double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs * factor / timeDeltaUsec);
+ writer->AddGauge("/vcpu/system", systemUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.WaitTime.IsOK()) {
+ i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds();
+ double waitPercent = std::max<double>(0.0, 100. * waitTimeUs * factor / timeDeltaUsec);
+ writer->AddGauge("/vcpu/wait", waitPercent);
+ }
+
+ if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) {
+ i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds();
+ double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs * factor / timeDeltaUsec);
+ writer->AddGauge("/vcpu/throttled", throttledPercent);
+ }
+
+ if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) {
+ i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds();
+ double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs * factor / timeDeltaUsec);
+ writer->AddGauge("/vcpu/total", totalUsagePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) {
+ i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds();
+ double guaranteePercent = std::max<double>(0.0, 100. * guaranteeTimeUs * factor / 1'000'000L);
+ writer->AddGauge("/vcpu/guarantee", guaranteePercent);
+ }
+
+ if (totalStatistics.CpuStatistics.LimitTime.IsOK()) {
+ i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds();
+ double limitPercent = std::max<double>(0.0, 100. * limitTimeUs * factor / 1'000'000L);
+ writer->AddGauge("/vcpu/limit", limitPercent);
+ }
+ }
+
+ WriteGaugeIfOk(writer, "/cpu/thread_count", totalStatistics.CpuStatistics.ThreadCount);
+ WriteGaugeIfOk(writer, "/cpu/context_switches", totalStatistics.CpuStatistics.ContextSwitches);
+}
+
+void TPortoResourceProfiler::WriteMemoryMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec)
+{
+ WriteCumulativeGaugeIfOk(writer,
+ "/memory/minor_page_faults",
+ totalStatistics.MemoryStatistics.MinorPageFaults,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(writer,
+ "/memory/major_page_faults",
+ totalStatistics.MemoryStatistics.MajorPageFaults,
+ timeDeltaUsec);
+
+ WriteGaugeIfOk(writer, "/memory/oom_kills", totalStatistics.MemoryStatistics.OomKills);
+ WriteGaugeIfOk(writer, "/memory/oom_kills_total", totalStatistics.MemoryStatistics.OomKillsTotal);
+
+ WriteGaugeIfOk(writer, "/memory/file_cache_usage", totalStatistics.MemoryStatistics.FileCacheUsage);
+ WriteGaugeIfOk(writer, "/memory/anon_usage", totalStatistics.MemoryStatistics.AnonUsage);
+ WriteGaugeIfOk(writer, "/memory/anon_limit", totalStatistics.MemoryStatistics.AnonLimit);
+ WriteGaugeIfOk(writer, "/memory/memory_usage", totalStatistics.MemoryStatistics.MemoryUsage);
+ WriteGaugeIfOk(writer, "/memory/memory_guarantee", totalStatistics.MemoryStatistics.MemoryGuarantee);
+ WriteGaugeIfOk(writer, "/memory/memory_limit", totalStatistics.MemoryStatistics.MemoryLimit);
+}
+
+void TPortoResourceProfiler::WriteBlockingIOMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec)
+{
+ WriteCumulativeGaugeIfOk(writer,
+ "/io/read_bytes",
+ totalStatistics.BlockIOStatistics.IOReadByte,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(writer,
+ "/io/write_bytes",
+ totalStatistics.BlockIOStatistics.IOWriteByte,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(writer,
+ "/io/read_ops",
+ totalStatistics.BlockIOStatistics.IOReadOps,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(writer,
+ "/io/write_ops",
+ totalStatistics.BlockIOStatistics.IOWriteOps,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(writer,
+ "/io/ops",
+ totalStatistics.BlockIOStatistics.IOOps,
+ timeDeltaUsec);
+
+ WriteGaugeIfOk(writer,
+ "/io/bytes_limit",
+ totalStatistics.BlockIOStatistics.IOBytesLimit);
+ WriteGaugeIfOk(writer,
+ "/io/ops_limit",
+ totalStatistics.BlockIOStatistics.IOOpsLimit);
+
+ if (totalStatistics.BlockIOStatistics.IOTotalTime.IsOK()) {
+ i64 totalTimeUs = totalStatistics.BlockIOStatistics.IOTotalTime.Value().MicroSeconds();
+ double totalPercent = std::max<double>(0.0, 100. * totalTimeUs / timeDeltaUsec);
+ writer->AddGauge("/io/total", totalPercent);
+ }
+
+ if (totalStatistics.BlockIOStatistics.IOWaitTime.IsOK()) {
+ i64 waitTimeUs = totalStatistics.BlockIOStatistics.IOWaitTime.Value().MicroSeconds();
+ double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec);
+ writer->AddGauge("/io/wait", waitPercent);
+ }
+}
+
+void TPortoResourceProfiler::WriteNetworkMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec)
+{
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/rx_bytes",
+ totalStatistics.NetworkStatistics.RxBytes,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/rx_drops",
+ totalStatistics.NetworkStatistics.RxDrops,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/rx_packets",
+ totalStatistics.NetworkStatistics.RxPackets,
+ timeDeltaUsec);
+ WriteGaugeIfOk(
+ writer,
+ "/network/rx_limit",
+ totalStatistics.NetworkStatistics.RxLimit);
+
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/tx_bytes",
+ totalStatistics.NetworkStatistics.TxBytes,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/tx_drops",
+ totalStatistics.NetworkStatistics.TxDrops,
+ timeDeltaUsec);
+ WriteCumulativeGaugeIfOk(
+ writer,
+ "/network/tx_packets",
+ totalStatistics.NetworkStatistics.TxPackets,
+ timeDeltaUsec);
+ WriteGaugeIfOk(
+ writer,
+ "/network/tx_limit",
+ totalStatistics.NetworkStatistics.TxLimit);
+}
+
+void TPortoResourceProfiler::CollectSensors(ISensorWriter* writer)
+{
+ i64 lastUpdate = ResourceTracker_->GetLastUpdateTime().MicroSeconds();
+
+ auto totalStatistics = ResourceTracker_->GetTotalStatistics();
+ i64 timeDeltaUsec = TInstant::Now().MicroSeconds() - lastUpdate;
+
+ WriteCpuMetrics(writer, totalStatistics, timeDeltaUsec);
+ WriteMemoryMetrics(writer, totalStatistics, timeDeltaUsec);
+ WriteBlockingIOMetrics(writer, totalStatistics, timeDeltaUsec);
+ WriteNetworkMetrics(writer, totalStatistics, timeDeltaUsec);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPortoResourceProfilerPtr CreatePortoProfilerWithTags(
+ const IInstancePtr& instance,
+ const TString containerCategory,
+ const TPodSpecConfigPtr& podSpec)
+{
+ auto portoResourceTracker = New<TPortoResourceTracker>(
+ instance,
+ ResourceUsageUpdatePeriod,
+ true,
+ true);
+
+ return New<TPortoResourceProfiler>(
+ portoResourceTracker,
+ podSpec,
+ TProfiler("/porto")
+ .WithTag("container_category", containerCategory));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+#endif
+
+#ifdef __linux__
+void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec)
+{
+ BIND([=] {
+ auto executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "porto-tracker");
+
+ executor->SubscribeFailed(BIND([=] (const TError& error) {
+ YT_LOG_ERROR(error, "Fatal error during Porto polling");
+ }));
+
+ LeakyRefCountedSingleton<TPortoProfilers>(
+ CreatePortoProfilerWithTags(GetSelfPortoInstance(executor), "daemon", podSpec),
+ CreatePortoProfilerWithTags(GetRootPortoInstance(executor), "pod", podSpec));
+ }).AsyncVia(GetCurrentInvoker())
+ .Run()
+ .Subscribe(BIND([] (const TError& error) {
+ YT_LOG_ERROR_IF(!error.IsOK(), error, "Failed to enable Porto profiler");
+ }));
+}
+#else
+void EnablePortoResourceTracker(const TPodSpecConfigPtr& /*podSpec*/)
+{
+ YT_LOG_WARNING("Porto resource tracker not supported");
+}
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/porto_resource_tracker.h b/yt/yt/library/containers/porto_resource_tracker.h
new file mode 100644
index 0000000000..8a0f781949
--- /dev/null
+++ b/yt/yt/library/containers/porto_resource_tracker.h
@@ -0,0 +1,158 @@
+#pragma once
+
+#include <yt/yt/library/containers/instance.h>
+#include <yt/yt/library/containers/public.h>
+
+#include <yt/yt/library/containers/cgroup.h>
+
+#include <yt/yt/core/misc/singleton.h>
+#include <yt/yt/core/net/address.h>
+#include <yt/yt/core/ytree/public.h>
+
+#include <yt/yt/library/process/process.h>
+#include <yt/yt/library/profiling/producer.h>
+
+namespace NYT::NContainers {
+
+using namespace NProfiling;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static constexpr auto ResourceUsageUpdatePeriod = TDuration::MilliSeconds(1000);
+
+////////////////////////////////////////////////////////////////////////////////
+
+using TCpuStatistics = TCpuAccounting::TStatistics;
+using TBlockIOStatistics = TBlockIO::TStatistics;
+using TMemoryStatistics = TMemory::TStatistics;
+using TNetworkStatistics = TNetwork::TStatistics;
+
+struct TTotalStatistics
+{
+public:
+ TCpuStatistics CpuStatistics;
+ TMemoryStatistics MemoryStatistics;
+ TBlockIOStatistics BlockIOStatistics;
+ TNetworkStatistics NetworkStatistics;
+};
+
+#ifdef _linux_
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoResourceTracker
+ : public TRefCounted
+{
+public:
+ TPortoResourceTracker(
+ IInstancePtr instance,
+ TDuration updatePeriod,
+ bool isDeltaTracker = false,
+ bool isForceUpdate = false);
+
+ TCpuStatistics GetCpuStatistics() const;
+
+ TBlockIOStatistics GetBlockIOStatistics() const;
+
+ TMemoryStatistics GetMemoryStatistics() const;
+
+ TNetworkStatistics GetNetworkStatistics() const;
+
+ TTotalStatistics GetTotalStatistics() const;
+
+ bool AreResourceUsageStatisticsExpired() const;
+
+ TInstant GetLastUpdateTime() const;
+
+private:
+ const IInstancePtr Instance_;
+ const TDuration UpdatePeriod_;
+ const bool IsDeltaTracker_;
+ const bool IsForceUpdate_;
+
+ mutable std::atomic<TInstant> LastUpdateTime_ = {};
+
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
+ mutable TResourceUsage ResourceUsage_;
+ mutable TResourceUsage ResourceUsageDelta_;
+
+ mutable std::optional<TCpuStatistics> CachedCpuStatistics_;
+ mutable std::optional<TMemoryStatistics> CachedMemoryStatistics_;
+ mutable std::optional<TBlockIOStatistics> CachedBlockIOStatistics_;
+ mutable std::optional<TNetworkStatistics> CachedNetworkStatistics_;
+ mutable std::optional<TTotalStatistics> CachedTotalStatistics_;
+ mutable TErrorOr<ui64> PeakThreadCount_ = 0;
+
+ template <class T, class F>
+ T GetStatistics(
+ std::optional<T>& cachedStatistics,
+ const TString& statisticsKind,
+ F extractor) const;
+
+ TCpuStatistics ExtractCpuStatistics(const TResourceUsage& resourceUsage) const;
+ TMemoryStatistics ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const;
+ TBlockIOStatistics ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const;
+ TNetworkStatistics ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const;
+ TTotalStatistics ExtractTotalStatistics(const TResourceUsage& resourceUsage) const;
+
+ TErrorOr<ui64> CalculateCounterDelta(
+ const TErrorOr<ui64>& oldValue,
+ const TErrorOr<ui64>& newValue) const;
+
+ void ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const;
+
+ void UpdateResourceUsageStatisticsIfExpired() const;
+
+ void DoUpdateResourceUsage() const;
+};
+
+DEFINE_REFCOUNTED_TYPE(TPortoResourceTracker)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoResourceProfiler
+ : public ISensorProducer
+{
+public:
+ TPortoResourceProfiler(
+ TPortoResourceTrackerPtr tracker,
+ TPodSpecConfigPtr podSpec,
+ const TProfiler& profiler = TProfiler{"/porto"});
+
+ void CollectSensors(ISensorWriter* writer) override;
+
+private:
+ const TPortoResourceTrackerPtr ResourceTracker_;
+ const TPodSpecConfigPtr PodSpec_;
+
+ void WriteCpuMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec);
+
+ void WriteMemoryMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec);
+
+ void WriteBlockingIOMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec);
+
+ void WriteNetworkMetrics(
+ ISensorWriter* writer,
+ TTotalStatistics& totalStatistics,
+ i64 timeDeltaUsec);
+};
+
+DECLARE_REFCOUNTED_TYPE(TPortoResourceProfiler)
+DEFINE_REFCOUNTED_TYPE(TPortoResourceProfiler)
+
+////////////////////////////////////////////////////////////////////////////////
+
+#endif
+
+void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec);
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/private.h b/yt/yt/library/containers/private.h
new file mode 100644
index 0000000000..62682cb364
--- /dev/null
+++ b/yt/yt/library/containers/private.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger ContainersLogger("Containers");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/process.cpp b/yt/yt/library/containers/process.cpp
new file mode 100644
index 0000000000..ad1c8d35dc
--- /dev/null
+++ b/yt/yt/library/containers/process.cpp
@@ -0,0 +1,154 @@
+#ifdef __linux__
+
+#include "process.h"
+
+#include <yt/yt/library/containers/instance.h>
+
+#include <yt/yt/core/misc/proc.h>
+#include <yt/yt/core/misc/fs.h>
+
+namespace NYT::NContainers {
+
+using namespace NPipes;
+using namespace NNet;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline const NLogging::TLogger Logger("Process");
+
+static constexpr pid_t InvalidProcessId = -1;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPortoProcess::TPortoProcess(
+ const TString& path,
+ IInstanceLauncherPtr containerLauncher,
+ bool copyEnv)
+ : TProcessBase(path)
+ , ContainerLauncher_(std::move(containerLauncher))
+{
+ AddArgument(NFS::GetFileName(path));
+ if (copyEnv) {
+ for (char** envIt = environ; *envIt; ++envIt) {
+ Env_.push_back(Capture(*envIt));
+ }
+ }
+}
+
+void TPortoProcess::Kill(int signal)
+{
+ if (auto instance = GetInstance()) {
+ instance->Kill(signal);
+ }
+}
+
+void TPortoProcess::DoSpawn()
+{
+ YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_);
+ YT_VERIFY(!GetInstance());
+ YT_VERIFY(!Started_);
+ YT_VERIFY(!Args_.empty());
+
+ if (!WorkingDirectory_.empty()) {
+ ContainerLauncher_->SetCwd(WorkingDirectory_);
+ }
+
+ Started_ = true;
+
+ try {
+ // TPortoProcess doesn't support running processes inside rootFS.
+ YT_VERIFY(!ContainerLauncher_->HasRoot());
+ std::vector<TString> args(Args_.begin() + 1, Args_.end());
+ auto instance = WaitFor(ContainerLauncher_->Launch(ResolvedPath_, args, DecomposeEnv()))
+ .ValueOrThrow();
+ ContainerInstance_.Store(instance);
+ FinishedPromise_.SetFrom(instance->Wait());
+
+ try {
+ ProcessId_ = instance->GetPid();
+ } catch (const std::exception& ex) {
+ // This could happen if Porto container has already died or pid namespace of
+ // parent container is not a parent of pid namespace of child container.
+ // It's not a problem, since for Porto process pid is used for logging purposes only.
+ YT_LOG_DEBUG(ex, "Failed to get pid of root process (Container: %v)",
+ instance->GetName());
+ }
+
+ YT_LOG_DEBUG("Process inside Porto spawned successfully (Path: %v, ExternalPid: %v, Container: %v)",
+ ResolvedPath_,
+ ProcessId_,
+ instance->GetName());
+
+ FinishedPromise_.ToFuture().Subscribe(BIND([=, this, this_ = MakeStrong(this)] (const TError& exitStatus) {
+ Finished_ = true;
+ if (exitStatus.IsOK()) {
+ YT_LOG_DEBUG("Process inside Porto exited gracefully (ExternalPid: %v, Container: %v)",
+ ProcessId_,
+ instance->GetName());
+ } else {
+ YT_LOG_DEBUG(exitStatus, "Process inside Porto exited with an error (ExternalPid: %v, Container: %v)",
+ ProcessId_,
+ instance->GetName());
+ }
+ }));
+ } catch (const std::exception& ex) {
+ Finished_ = true;
+ THROW_ERROR_EXCEPTION("Failed to start child process inside Porto")
+ << TErrorAttribute("path", ResolvedPath_)
+ << TErrorAttribute("container", ContainerLauncher_->GetName())
+ << ex;
+ }
+}
+
+IInstancePtr TPortoProcess::GetInstance()
+{
+ return ContainerInstance_.Acquire();
+}
+
+THashMap<TString, TString> TPortoProcess::DecomposeEnv() const
+{
+ THashMap<TString, TString> result;
+ for (const auto& env : Env_) {
+ TStringBuf name, value;
+ TStringBuf(env).TrySplit('=', name, value);
+ result[name] = value;
+ }
+ return result;
+}
+
+static TString CreateStdIONamedPipePath()
+{
+ const TString name = ToString(TGuid::Create());
+ return NFS::GetRealPath(NFS::CombinePaths("/tmp", name));
+}
+
+IConnectionWriterPtr TPortoProcess::GetStdInWriter()
+{
+ auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath());
+ ContainerLauncher_->SetStdIn(pipe->GetPath());
+ NamedPipes_.push_back(pipe);
+ return pipe->CreateAsyncWriter();
+}
+
+IConnectionReaderPtr TPortoProcess::GetStdOutReader()
+{
+ auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath());
+ ContainerLauncher_->SetStdOut(pipe->GetPath());
+ NamedPipes_.push_back(pipe);
+ return pipe->CreateAsyncReader();
+}
+
+IConnectionReaderPtr TPortoProcess::GetStdErrReader()
+{
+ auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath());
+ ContainerLauncher_->SetStdErr(pipe->GetPath());
+ NamedPipes_.push_back(pipe);
+ return pipe->CreateAsyncReader();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
+
+#endif
diff --git a/yt/yt/library/containers/process.h b/yt/yt/library/containers/process.h
new file mode 100644
index 0000000000..75255165d8
--- /dev/null
+++ b/yt/yt/library/containers/process.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/library/process/process.h>
+
+#include <library/cpp/yt/memory/atomic_intrusive_ptr.h>
+
+#include <library/cpp/porto/libporto.hpp>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// NB(psushin): this class is deprecated and only used to run job proxy.
+// ToDo(psushin): kill me.
+class TPortoProcess
+ : public TProcessBase
+{
+public:
+ TPortoProcess(
+ const TString& path,
+ NContainers::IInstanceLauncherPtr containerLauncher,
+ bool copyEnv = true);
+ void Kill(int signal) override;
+ NNet::IConnectionWriterPtr GetStdInWriter() override;
+ NNet::IConnectionReaderPtr GetStdOutReader() override;
+ NNet::IConnectionReaderPtr GetStdErrReader() override;
+
+ NContainers::IInstancePtr GetInstance();
+
+private:
+ const NContainers::IInstanceLauncherPtr ContainerLauncher_;
+
+ TAtomicIntrusivePtr<NContainers::IInstance> ContainerInstance_;
+ std::vector<NPipes::TNamedPipePtr> NamedPipes_;
+
+ void DoSpawn() override;
+ THashMap<TString, TString> DecomposeEnv() const;
+};
+
+DEFINE_REFCOUNTED_TYPE(TPortoProcess)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/public.h b/yt/yt/library/containers/public.h
new file mode 100644
index 0000000000..d8e3cf3491
--- /dev/null
+++ b/yt/yt/library/containers/public.h
@@ -0,0 +1,163 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+#include <library/cpp/porto/proto/rpc.pb.h>
+#include <library/cpp/yt/misc/enum.h>
+
+namespace NYT::NContainers {
+
+////////////////////////////////////////////////////////////////////////////////
+
+const int PortoErrorCodeBase = 12000;
+
+DEFINE_ENUM(EPortoErrorCode,
+ ((Success) ((PortoErrorCodeBase + Porto::EError::Success)))
+ ((Unknown) ((PortoErrorCodeBase + Porto::EError::Unknown)))
+ ((InvalidMethod) ((PortoErrorCodeBase + Porto::EError::InvalidMethod)))
+ ((ContainerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::ContainerAlreadyExists)))
+ ((ContainerDoesNotExist) ((PortoErrorCodeBase + Porto::EError::ContainerDoesNotExist)))
+ ((InvalidProperty) ((PortoErrorCodeBase + Porto::EError::InvalidProperty)))
+ ((InvalidData) ((PortoErrorCodeBase + Porto::EError::InvalidData)))
+ ((InvalidValue) ((PortoErrorCodeBase + Porto::EError::InvalidValue)))
+ ((InvalidState) ((PortoErrorCodeBase + Porto::EError::InvalidState)))
+ ((NotSupported) ((PortoErrorCodeBase + Porto::EError::NotSupported)))
+ ((ResourceNotAvailable) ((PortoErrorCodeBase + Porto::EError::ResourceNotAvailable)))
+ ((Permission) ((PortoErrorCodeBase + Porto::EError::Permission)))
+ ((VolumeAlreadyExists) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyExists)))
+ ((VolumeNotFound) ((PortoErrorCodeBase + Porto::EError::VolumeNotFound)))
+ ((NoSpace) ((PortoErrorCodeBase + Porto::EError::NoSpace)))
+ ((Busy) ((PortoErrorCodeBase + Porto::EError::Busy)))
+ ((VolumeAlreadyLinked) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyLinked)))
+ ((VolumeNotLinked) ((PortoErrorCodeBase + Porto::EError::VolumeNotLinked)))
+ ((LayerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::LayerAlreadyExists)))
+ ((LayerNotFound) ((PortoErrorCodeBase + Porto::EError::LayerNotFound)))
+ ((NoValue) ((PortoErrorCodeBase + Porto::EError::NoValue)))
+ ((VolumeNotReady) ((PortoErrorCodeBase + Porto::EError::VolumeNotReady)))
+ ((InvalidCommand) ((PortoErrorCodeBase + Porto::EError::InvalidCommand)))
+ ((LostError) ((PortoErrorCodeBase + Porto::EError::LostError)))
+ ((DeviceNotFound) ((PortoErrorCodeBase + Porto::EError::DeviceNotFound)))
+ ((InvalidPath) ((PortoErrorCodeBase + Porto::EError::InvalidPath)))
+ ((InvalidNetworkAddress) ((PortoErrorCodeBase + Porto::EError::InvalidNetworkAddress)))
+ ((PortoFrozen) ((PortoErrorCodeBase + Porto::EError::PortoFrozen)))
+ ((LabelNotFound) ((PortoErrorCodeBase + Porto::EError::LabelNotFound)))
+ ((InvalidLabel) ((PortoErrorCodeBase + Porto::EError::InvalidLabel)))
+ ((NotFound) ((PortoErrorCodeBase + Porto::EError::NotFound)))
+ ((SocketError) ((PortoErrorCodeBase + Porto::EError::SocketError)))
+ ((SocketUnavailable) ((PortoErrorCodeBase + Porto::EError::SocketUnavailable)))
+ ((SocketTimeout) ((PortoErrorCodeBase + Porto::EError::SocketTimeout)))
+ ((Taint) ((PortoErrorCodeBase + Porto::EError::Taint)))
+ ((Queued) ((PortoErrorCodeBase + Porto::EError::Queued)))
+);
+
+////////////////////////////////////////////////////////////////////////////////
+
+YT_DEFINE_ERROR_ENUM(
+ ((FailedToStartContainer) (14000))
+);
+
+DEFINE_ENUM(EStatField,
+ // CPU
+ (CpuUsage)
+ (CpuUserUsage)
+ (CpuSystemUsage)
+ (CpuWait)
+ (CpuThrottled)
+ (ContextSwitches)
+ (ContextSwitchesDelta)
+ (ThreadCount)
+ (CpuLimit)
+ (CpuGuarantee)
+
+ // Memory
+ (Rss)
+ (MappedFile)
+ (MajorPageFaults)
+ (MinorPageFaults)
+ (FileCacheUsage)
+ (AnonMemoryUsage)
+ (AnonMemoryLimit)
+ (MemoryUsage)
+ (MemoryGuarantee)
+ (MemoryLimit)
+ (MaxMemoryUsage)
+ (OomKills)
+ (OomKillsTotal)
+
+ // IO
+ (IOReadByte)
+ (IOWriteByte)
+ (IOBytesLimit)
+ (IOReadOps)
+ (IOWriteOps)
+ (IOOps)
+ (IOOpsLimit)
+ (IOTotalTime)
+ (IOWaitTime)
+
+ // Network
+ (NetTxBytes)
+ (NetTxPackets)
+ (NetTxDrops)
+ (NetTxLimit)
+ (NetRxBytes)
+ (NetRxPackets)
+ (NetRxDrops)
+ (NetRxLimit)
+);
+
+DEFINE_ENUM(EEnablePorto,
+ (None)
+ (Isolate)
+ (Full)
+);
+
+struct TBind
+{
+ TString SourcePath;
+ TString TargetPath;
+ bool ReadOnly;
+};
+
+struct TRootFS
+{
+ TString RootPath;
+ bool IsRootReadOnly;
+ std::vector<TBind> Binds;
+};
+
+struct TDevice
+{
+ TString DeviceName;
+ bool Enabled;
+};
+
+struct TInstanceLimits
+{
+ double Cpu = 0;
+ i64 Memory = 0;
+ std::optional<i64> NetTx;
+ std::optional<i64> NetRx;
+
+ bool operator==(const TInstanceLimits&) const = default;
+};
+
+DECLARE_REFCOUNTED_STRUCT(IContainerManager)
+DECLARE_REFCOUNTED_STRUCT(IInstanceLauncher)
+DECLARE_REFCOUNTED_STRUCT(IInstance)
+DECLARE_REFCOUNTED_STRUCT(IPortoExecutor)
+
+DECLARE_REFCOUNTED_CLASS(TPortoHealthChecker)
+DECLARE_REFCOUNTED_CLASS(TInstanceLimitsTracker)
+DECLARE_REFCOUNTED_CLASS(TPortoProcess)
+DECLARE_REFCOUNTED_CLASS(TPortoResourceTracker)
+DECLARE_REFCOUNTED_CLASS(TPortoExecutorDynamicConfig)
+DECLARE_REFCOUNTED_CLASS(TPodSpecConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool IsValidCGroupType(const TString& type);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/unittests/containers_ut.cpp b/yt/yt/library/containers/unittests/containers_ut.cpp
new file mode 100644
index 0000000000..4f1c10a435
--- /dev/null
+++ b/yt/yt/library/containers/unittests/containers_ut.cpp
@@ -0,0 +1,133 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#ifdef _linux_
+
+#include <yt/yt/library/containers/config.h>
+#include <yt/yt/library/containers/porto_executor.h>
+#include <yt/yt/library/containers/instance.h>
+
+#include <util/system/platform.h>
+#include <util/system/env.h>
+
+namespace NYT::NContainers {
+namespace {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TContainersTest
+ : public ::testing::Test
+{
+ void SetUp() override
+ {
+ if (GetEnv("SKIP_PORTO_TESTS") != "") {
+ GTEST_SKIP();
+ }
+ }
+};
+
+static TString GetUniqueName()
+{
+ return "yt_ut_" + ToString(TGuid::Create());
+}
+
+IPortoExecutorPtr CreatePortoExecutor()
+{
+ return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default");
+}
+
+TEST_F(TContainersTest, ListSubcontainers)
+{
+ auto executor = CreatePortoExecutor();
+ auto name = GetUniqueName();
+
+ WaitFor(executor->CreateContainer(name))
+ .ThrowOnError();
+
+ auto absoluteName = *WaitFor(executor->GetContainerProperty(name, "absolute_name"))
+ .ValueOrThrow();
+
+ auto nestedName = absoluteName + "/nested";
+ WaitFor(executor->CreateContainer(nestedName))
+ .ThrowOnError();
+
+ auto withRoot = WaitFor(executor->ListSubcontainers(name, true))
+ .ValueOrThrow();
+ EXPECT_EQ(std::vector<TString>({absoluteName, nestedName}), withRoot);
+
+ auto withoutRoot = WaitFor(executor->ListSubcontainers(name, false))
+ .ValueOrThrow();
+ EXPECT_EQ(std::vector<TString>({nestedName}), withoutRoot);
+
+ WaitFor(executor->DestroyContainer(absoluteName))
+ .ThrowOnError();
+}
+
+// See https://st.yandex-team.ru/PORTO-846.
+TEST_F(TContainersTest, DISABLED_WaitContainer)
+{
+ auto executor = CreatePortoExecutor();
+ auto name = GetUniqueName();
+
+ WaitFor(executor->CreateContainer(name))
+ .ThrowOnError();
+
+ WaitFor(executor->SetContainerProperty(name, "command", "sleep 10"))
+ .ThrowOnError();
+
+ WaitFor(executor->StartContainer(name))
+ .ThrowOnError();
+
+ auto exitCode = WaitFor(executor->WaitContainer(name))
+ .ValueOrThrow();
+
+ EXPECT_EQ(0, exitCode);
+
+ WaitFor(executor->DestroyContainer(name))
+ .ThrowOnError();
+}
+
+TEST_F(TContainersTest, CreateFromSpec)
+{
+ auto executor = CreatePortoExecutor();
+ auto name = GetUniqueName();
+
+ auto spec = TRunnableContainerSpec {
+ .Name = name,
+ .Command = "sleep 2",
+ };
+
+ WaitFor(executor->CreateContainer(spec, /*start*/ true))
+ .ThrowOnError();
+
+ auto exitCode = WaitFor(executor->PollContainer(name))
+ .ValueOrThrow();
+
+ EXPECT_EQ(0, exitCode);
+
+ WaitFor(executor->DestroyContainer(name))
+ .ThrowOnError();
+}
+
+TEST_F(TContainersTest, ListPids)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+
+ auto instance = WaitFor(launcher->Launch("sleep", {"5"}, {}))
+ .ValueOrThrow();
+
+ auto pids = instance->GetPids();
+ EXPECT_LT(0u, pids.size());
+
+ instance->Destroy();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NContainers
+
+#endif
diff --git a/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp b/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp
new file mode 100644
index 0000000000..04d169ba4e
--- /dev/null
+++ b/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp
@@ -0,0 +1,251 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/core/ytree/convert.h>
+
+#include <util/system/fs.h>
+#include <util/system/tempfile.h>
+
+#include <yt/yt/library/profiling/producer.h>
+#include <yt/yt/library/containers/config.h>
+#include <yt/yt/library/containers/porto_executor.h>
+#include <yt/yt/library/containers/porto_resource_tracker.h>
+#include <yt/yt/library/containers/instance.h>
+
+#include <util/system/platform.h>
+#include <util/system/env.h>
+
+namespace NYT::NContainers {
+namespace {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static constexpr auto TestUpdatePeriod = TDuration::MilliSeconds(10);
+
+class TPortoTrackerTest
+ : public ::testing::Test
+{
+public:
+ IPortoExecutorPtr Executor;
+
+ void SetUp() override
+ {
+ if (GetEnv("SKIP_PORTO_TESTS") != "") {
+ GTEST_SKIP();
+ }
+
+ Executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default");
+ }
+};
+
+TString GetUniqueName()
+{
+ return "yt_porto_ut_" + ToString(TGuid::Create());
+}
+
+TPortoResourceTrackerPtr CreateSumPortoTracker(IPortoExecutorPtr Executor, const TString& name)
+{
+ return New<TPortoResourceTracker>(
+ GetPortoInstance(Executor, name),
+ TestUpdatePeriod,
+ false);
+}
+
+TPortoResourceProfilerPtr CreateDeltaPortoProfiler(IPortoExecutorPtr executor, const TString& name)
+{
+ auto instance = GetPortoInstance(executor, name);
+ auto portoResourceTracker = New<TPortoResourceTracker>(
+ instance,
+ ResourceUsageUpdatePeriod,
+ true,
+ true
+ );
+
+ // Init metrics for delta tracker.
+ portoResourceTracker->GetTotalStatistics();
+
+ return LeakyRefCountedSingleton<TPortoResourceProfiler>(
+ portoResourceTracker,
+ New<TPodSpecConfig>(),
+ TProfiler("/porto")
+ .WithTag("porto_name", instance->GetName())
+ .WithTag("container_category", "yt_daemon"));
+}
+
+void AssertGauges(const std::vector<std::tuple<TString, TTagList, double>>& gauges) {
+ THashSet<TString> sensors{
+ "/cpu/user",
+ "/cpu/total",
+ "/cpu/system",
+ "/cpu/wait",
+ "/cpu/throttled",
+ "/cpu/guarantee",
+ "/cpu/limit",
+ "/cpu/thread_count",
+ "/cpu/context_switches",
+
+ "/memory/minor_page_faults",
+ "/memory/major_page_faults",
+ "/memory/file_cache_usage",
+ "/memory/anon_usage",
+ "/memory/anon_limit",
+ "/memory/memory_usage",
+ "/memory/memory_guarantee",
+ "/memory/memory_limit",
+
+ "/io/read_bytes",
+ "/io/write_bytes",
+ "/io/bytes_limit",
+
+ "/io/read_ops",
+ "/io/write_ops",
+ "/io/ops",
+ "/io/ops_limit",
+ "/io/total",
+
+ "/network/rx_bytes",
+ "/network/rx_drops",
+ "/network/rx_packets",
+ "/network/rx_limit",
+ "/network/tx_bytes",
+ "/network/tx_drops",
+ "/network/tx_packets",
+ "/network/tx_limit"
+ };
+
+ THashSet<TString> mayBeEmpty{
+ "/cpu/wait",
+ "/cpu/throttled",
+ "/cpu/guarantee",
+ "/cpu/context_switches",
+ "/memory/major_page_faults",
+ "/memory/memory_guarantee",
+ "/io/ops_limit",
+ "/io/read_ops",
+ "/io/write_ops",
+ "/io/wait",
+ "/io/bytes_limit",
+ "/network/rx_bytes",
+ "/network/rx_drops",
+ "/network/rx_packets",
+ "/network/rx_limit",
+ "/network/tx_bytes",
+ "/network/tx_drops",
+ "/network/tx_packets",
+ "/network/tx_limit"
+ };
+
+ for (const auto& [name, tags, value] : gauges) {
+ EXPECT_TRUE(value >= 0 && sensors.find(name) || mayBeEmpty.find(name));
+ }
+}
+
+TEST_F(TPortoTrackerTest, ValidateSummaryPortoTracker)
+{
+ auto name = GetUniqueName();
+
+ WaitFor(Executor->CreateContainer(
+ TRunnableContainerSpec {
+ .Name = name,
+ .Command = "sleep .1",
+ }, true))
+ .ThrowOnError();
+
+ auto tracker = CreateSumPortoTracker(Executor, name);
+
+ auto firstStatistics = tracker->GetTotalStatistics();
+
+ WaitFor(Executor->StopContainer(name))
+ .ThrowOnError();
+ WaitFor(Executor->SetContainerProperty(
+ name,
+ "command",
+ "find /"))
+ .ThrowOnError();
+ WaitFor(Executor->StartContainer(name))
+ .ThrowOnError();
+ Sleep(TDuration::MilliSeconds(500));
+
+ auto secondStatistics = tracker->GetTotalStatistics();
+
+ WaitFor(Executor->DestroyContainer(name))
+ .ThrowOnError();
+}
+
+TEST_F(TPortoTrackerTest, ValidateDeltaPortoTracker)
+{
+ auto name = GetUniqueName();
+
+ auto spec = TRunnableContainerSpec {
+ .Name = name,
+ .Command = "sleep .1",
+ };
+
+ WaitFor(Executor->CreateContainer(spec, true))
+ .ThrowOnError();
+
+ auto profiler = CreateDeltaPortoProfiler(Executor, name);
+
+ WaitFor(Executor->StopContainer(name))
+ .ThrowOnError();
+ WaitFor(Executor->SetContainerProperty(
+ name,
+ "command",
+ "find /"))
+ .ThrowOnError();
+ WaitFor(Executor->StartContainer(name))
+ .ThrowOnError();
+
+ Sleep(TDuration::MilliSeconds(500));
+
+ auto buffer = New<TSensorBuffer>();
+ profiler->CollectSensors(buffer.Get());
+ AssertGauges(buffer->GetGauges());
+
+ WaitFor(Executor->DestroyContainer(name))
+ .ThrowOnError();
+}
+
+TEST_F(TPortoTrackerTest, ValidateDeltaRootPortoTracker)
+{
+ auto name = GetUniqueName();
+
+ auto spec = TRunnableContainerSpec {
+ .Name = name,
+ .Command = "sleep .1",
+ };
+
+ WaitFor(Executor->CreateContainer(spec, true))
+ .ThrowOnError();
+
+ auto profiler = CreateDeltaPortoProfiler(
+ Executor,
+ GetPortoInstance(
+ Executor,
+ *GetPortoInstance(Executor, name)->GetRootName())->GetName());
+
+ WaitFor(Executor->StopContainer(name))
+ .ThrowOnError();
+ WaitFor(Executor->SetContainerProperty(
+ name,
+ "command",
+ "find /"))
+ .ThrowOnError();
+ WaitFor(Executor->StartContainer(name))
+ .ThrowOnError();
+
+ Sleep(TDuration::MilliSeconds(500));
+
+ auto buffer = New<TSensorBuffer>();
+ profiler->CollectSensors(buffer.Get());
+ AssertGauges(buffer->GetGauges());
+
+ WaitFor(Executor->DestroyContainer(name))
+ .ThrowOnError();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NContainers
diff --git a/yt/yt/library/containers/unittests/process_ut.cpp b/yt/yt/library/containers/unittests/process_ut.cpp
new file mode 100644
index 0000000000..b9c0d844f4
--- /dev/null
+++ b/yt/yt/library/containers/unittests/process_ut.cpp
@@ -0,0 +1,302 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#ifdef _linux_
+
+#include <yt/yt/core/actions/bind.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/delayed_executor.h>
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/misc/guid.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/net/connection.h>
+
+#include <yt/yt/library/containers/process.h>
+
+#include <yt/yt/library/containers/config.h>
+#include <yt/yt/library/containers/porto_executor.h>
+#include <yt/yt/library/containers/instance.h>
+
+#include <util/system/platform.h>
+#include <util/system/env.h>
+
+namespace NYT::NContainers {
+namespace {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPortoProcessTest
+ : public ::testing::Test
+{
+ void SetUp() override
+ {
+ if (GetEnv("SKIP_PORTO_TESTS") != "") {
+ GTEST_SKIP();
+ }
+ }
+};
+
+static TString GetUniqueName()
+{
+ return "yt_ut_" + ToString(TGuid::Create());
+}
+
+IPortoExecutorPtr CreatePortoExecutor()
+{
+ return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default");
+}
+
+TEST_F(TPortoProcessTest, Basic)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/ls", launcher, true);
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, RunFromPathEnv)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("ls", launcher, true);
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, MultiBasic)
+{
+ auto portoExecutor = CreatePortoExecutor();
+ auto l1 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor);
+ auto l2 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor);
+ auto p1 = New<TPortoProcess>("/bin/ls", l1, true);
+ auto p2 = New<TPortoProcess>("/bin/ls", l2, true);
+ TFuture<void> f1;
+ TFuture<void> f2;
+ ASSERT_NO_THROW(f1 = p1->Spawn());
+ ASSERT_NO_THROW(f2 = p2->Spawn());
+ auto error = WaitFor((AllSucceeded(std::vector<TFuture<void>>{f1, f2})));
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p1->IsFinished());
+ EXPECT_TRUE(p2->IsFinished());
+ p1->GetInstance()->Destroy();
+ p2->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, InvalidPath)
+{
+ auto portoExecutor = CreatePortoExecutor();
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ portoExecutor);
+ auto p = New<TPortoProcess>("/some/bad/path/binary", launcher, true);
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_FALSE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_FALSE(p->IsFinished());
+ EXPECT_FALSE(error.IsOK());
+ WaitFor(portoExecutor->DestroyContainer(launcher->GetName()))
+ .ThrowOnError();
+}
+
+TEST_F(TPortoProcessTest, StdOut)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/date", launcher, true);
+
+ auto outStream = p->GetStdOutReader();
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+
+ auto buffer = TSharedMutableRef::Allocate(4_KB, {.InitializeStorage = false});
+ auto future = outStream->Read(buffer);
+ TErrorOr<size_t> result = WaitFor(future);
+ size_t sz = result.ValueOrThrow();
+ EXPECT_TRUE(sz > 0);
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, GetCommandLine)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ EXPECT_EQ("/bin/bash", p->GetCommandLine());
+ p->AddArgument("-c");
+ EXPECT_EQ("/bin/bash -c", p->GetCommandLine());
+ p->AddArgument("exit 0");
+ EXPECT_EQ("/bin/bash -c \"exit 0\"", p->GetCommandLine());
+}
+
+TEST_F(TPortoProcessTest, ProcessReturnCode0)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("exit 0");
+
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, ProcessReturnCode123)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("exit 123");
+
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_EQ(EProcessErrorCode::NonZeroExitCode, error.GetCode());
+ EXPECT_EQ(123, error.Attributes().Get<int>("exit_code"));
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, Params1)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("if test 3 -gt 1; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_FALSE(error.IsOK());
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, Params2)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("if test 1 -gt 3; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, InheritEnvironment)
+{
+ const char* name = "SPAWN_TEST_ENV_VAR";
+ const char* value = "42";
+ setenv(name, value, 1);
+
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("if test $SPAWN_TEST_ENV_VAR = 42; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_FALSE(error.IsOK());
+ EXPECT_TRUE(p->IsFinished());
+
+ unsetenv(name);
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, Kill)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/sleep", launcher, true);
+ p->AddArgument("5");
+
+ auto finished = p->Spawn();
+
+ NConcurrency::TDelayedExecutor::Submit(
+ BIND([&] () {
+ p->Kill(SIGKILL);
+ }),
+ TDuration::MilliSeconds(100));
+
+ auto error = WaitFor(finished);
+ EXPECT_FALSE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, KillFinished)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/bash", launcher, true);
+ p->AddArgument("-c");
+ p->AddArgument("true");
+
+ auto finished = p->Spawn();
+
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK());
+
+ p->Kill(SIGKILL);
+ p->GetInstance()->Destroy();
+}
+
+TEST_F(TPortoProcessTest, PollDuration)
+{
+ auto launcher = CreatePortoInstanceLauncher(
+ GetUniqueName(),
+ CreatePortoExecutor());
+ auto p = New<TPortoProcess>("/bin/sleep", launcher, true);
+ p->AddArgument("1");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+ p->GetInstance()->Destroy();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NContainers
+
+#endif
diff --git a/yt/yt/library/containers/unittests/ya.make b/yt/yt/library/containers/unittests/ya.make
new file mode 100644
index 0000000000..42984e2dc7
--- /dev/null
+++ b/yt/yt/library/containers/unittests/ya.make
@@ -0,0 +1,35 @@
+GTEST(unittester-containers)
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+ALLOCATOR(TCMALLOC)
+
+IF (AUTOCHECK)
+ ENV(SKIP_PORTO_TESTS=1)
+ENDIF()
+
+IF (DISTBUILD) # TODO(prime@): this is always on
+ ENV(SKIP_PORTO_TESTS=1)
+ENDIF()
+
+SRCS(
+ containers_ut.cpp
+ process_ut.cpp
+)
+
+IF(OS_LINUX)
+ SRCS(
+ porto_resource_tracker_ut.cpp
+ )
+ENDIF()
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc)
+
+PEERDIR(
+ yt/yt/build
+ yt/yt/library/containers
+)
+
+SIZE(MEDIUM)
+
+END()
diff --git a/yt/yt/library/containers/ya.make b/yt/yt/library/containers/ya.make
new file mode 100644
index 0000000000..499b8d9da8
--- /dev/null
+++ b/yt/yt/library/containers/ya.make
@@ -0,0 +1,37 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ cgroup.cpp
+ config.cpp
+ instance.cpp
+ instance_limits_tracker.cpp
+ process.cpp
+ porto_executor.cpp
+ porto_resource_tracker.cpp
+ porto_health_checker.cpp
+)
+
+PEERDIR(
+ library/cpp/porto/proto
+ yt/yt/library/process
+ yt/yt/core
+)
+
+IF(OS_LINUX)
+ PEERDIR(
+ library/cpp/porto
+ )
+ENDIF()
+
+END()
+
+RECURSE(
+ disk_manager
+ cri
+)
+
+RECURSE_FOR_TESTS(
+ unittests
+)
diff --git a/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..cbddba12a3
--- /dev/null
+++ b/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,26 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-monitoring)
+target_compile_options(yt-library-monitoring PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-monitoring PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-yt-build
+ yt-library-profiling
+ library-profiling-solomon
+ library-cpp-cgiparam
+)
+target_sources(yt-library-monitoring PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp
+)
diff --git a/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt b/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..192d29cfb1
--- /dev/null
+++ b/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-monitoring)
+target_compile_options(yt-library-monitoring PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-monitoring PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-yt-build
+ yt-library-profiling
+ library-profiling-solomon
+ library-cpp-cgiparam
+ yt-library-ytprof
+ library-ytprof-http
+ library-backtrace_introspector-http
+)
+target_sources(yt-library-monitoring PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp
+)
diff --git a/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..192d29cfb1
--- /dev/null
+++ b/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-monitoring)
+target_compile_options(yt-library-monitoring PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-monitoring PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-yt-build
+ yt-library-profiling
+ library-profiling-solomon
+ library-cpp-cgiparam
+ yt-library-ytprof
+ library-ytprof-http
+ library-backtrace_introspector-http
+)
+target_sources(yt-library-monitoring PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp
+)
diff --git a/yt/yt/library/monitoring/CMakeLists.txt b/yt/yt/library/monitoring/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/monitoring/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..adebf50e70
--- /dev/null
+++ b/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-monitoring)
+target_link_libraries(yt-library-monitoring PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ yt-yt-build
+ yt-library-profiling
+ library-profiling-solomon
+ library-cpp-cgiparam
+)
+target_sources(yt-library-monitoring PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp
+)
diff --git a/yt/yt/library/monitoring/http_integration.cpp b/yt/yt/library/monitoring/http_integration.cpp
new file mode 100644
index 0000000000..a526d2ede6
--- /dev/null
+++ b/yt/yt/library/monitoring/http_integration.cpp
@@ -0,0 +1,203 @@
+#include "http_integration.h"
+
+#include "monitoring_manager.h"
+
+#include <yt/yt/build/build.h>
+
+#include <yt/yt/core/json/config.h>
+#include <yt/yt/core/json/json_writer.h>
+
+#include <yt/yt/core/ytree/fluent.h>
+
+#include <yt/yt/core/yson/parser.h>
+#include <yt/yt/core/yson/consumer.h>
+
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/ytree/helpers.h>
+#include <yt/yt/core/ytree/virtual.h>
+#include <yt/yt/core/ytree/ypath_detail.h>
+#include <yt/yt/core/ytree/ypath_proxy.h>
+
+#include <yt/yt/core/http/http.h>
+#include <yt/yt/core/http/helpers.h>
+#include <yt/yt/core/http/server.h>
+
+#include <yt/yt/core/misc/ref_counted_tracker_statistics_producer.h>
+
+#include <yt/yt/library/profiling/solomon/exporter.h>
+
+#ifdef _linux_
+#include <yt/yt/library/ytprof/http/handler.h>
+#include <yt/yt/library/ytprof/build_info.h>
+
+#include <yt/yt/library/backtrace_introspector/http/handler.h>
+#endif
+
+#include <library/cpp/cgiparam/cgiparam.h>
+
+#include <util/string/vector.h>
+
+namespace NYT::NMonitoring {
+
+using namespace NYTree;
+using namespace NYson;
+using namespace NHttp;
+using namespace NConcurrency;
+using namespace NJson;
+
+////////////////////////////////////////////////////////////////////////////////
+
+DEFINE_ENUM(EVerb,
+ (Get)
+ (List)
+);
+
+////////////////////////////////////////////////////////////////////////////////
+
+void Initialize(
+ const NHttp::IServerPtr& monitoringServer,
+ const NProfiling::TSolomonExporterConfigPtr& config,
+ TMonitoringManagerPtr* monitoringManager,
+ NYTree::IMapNodePtr* orchidRoot)
+{
+ *monitoringManager = New<TMonitoringManager>();
+ (*monitoringManager)->Register("/ref_counted", CreateRefCountedTrackerStatisticsProducer());
+ (*monitoringManager)->Register("/solomon", BIND([] (NYson::IYsonConsumer* consumer) {
+ auto tags = NProfiling::TSolomonRegistry::Get()->GetDynamicTags();
+
+ BuildYsonFluently(consumer)
+ .BeginMap()
+ .Item("dynamic_tags").Value(THashMap<TString, TString>(tags.begin(), tags.end()))
+ .EndMap();
+ }));
+ (*monitoringManager)->Start();
+
+ *orchidRoot = NYTree::GetEphemeralNodeFactory(true)->CreateMap();
+ SetNodeByYPath(
+ *orchidRoot,
+ "/monitoring",
+ CreateVirtualNode((*monitoringManager)->GetService()));
+
+#ifdef _linux_
+ auto buildInfo = NYTProf::TBuildInfo::GetDefault();
+ buildInfo.BinaryVersion = GetVersion();
+
+ SetNodeByYPath(
+ *orchidRoot,
+ "/build_info",
+ NYTree::BuildYsonNodeFluently()
+ .BeginMap()
+ .Item("arc_revision").Value(buildInfo.ArcRevision)
+ .Item("binary_version").Value(buildInfo.BinaryVersion)
+ .Item("build_type").Value(buildInfo.BuildType)
+ .EndMap());
+#endif
+
+ if (monitoringServer) {
+ auto exporter = New<NProfiling::TSolomonExporter>(config);
+ exporter->Register("/solomon", monitoringServer);
+ exporter->Start();
+
+ SetNodeByYPath(
+ *orchidRoot,
+ "/sensors",
+ CreateVirtualNode(exporter->GetSensorService()));
+
+#ifdef _linux_
+ NYTProf::Register(monitoringServer, "/ytprof", buildInfo);
+ NBacktraceIntrospector::Register(monitoringServer, "/backtrace");
+#endif
+ monitoringServer->AddHandler(
+ "/orchid/",
+ GetOrchidYPathHttpHandler(*orchidRoot));
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TYPathHttpHandler
+ : public IHttpHandler
+{
+public:
+ explicit TYPathHttpHandler(IYPathServicePtr service)
+ : Service_(std::move(service))
+ { }
+
+ void HandleRequest(
+ const IRequestPtr& req,
+ const IResponseWriterPtr& rsp) override
+ {
+ const TStringBuf orchidPrefix = "/orchid";
+
+ TString path{req->GetUrl().Path};
+ if (!path.StartsWith(orchidPrefix)) {
+ THROW_ERROR_EXCEPTION("HTTP request must start with %Qv prefix",
+ orchidPrefix)
+ << TErrorAttribute("path", path);
+ }
+
+ path = path.substr(orchidPrefix.size(), TString::npos);
+ TCgiParameters params(req->GetUrl().RawQuery);
+
+ auto verb = EVerb::Get;
+
+ auto options = CreateEphemeralAttributes();
+ for (const auto& param : params) {
+ if (param.first == "verb") {
+ verb = ParseEnum<EVerb>(param.second);
+ } else {
+ // Just a check, IAttributeDictionary takes raw YSON anyway.
+ try {
+ ValidateYson(TYsonString(param.second), DefaultYsonParserNestingLevelLimit);
+ } catch (const std::exception& ex) {
+ THROW_ERROR_EXCEPTION("Error parsing value of query parameter %Qv",
+ param.first)
+ << ex;
+ }
+
+ options->SetYson(param.first, TYsonString(param.second));
+ }
+ }
+
+ TYsonString result;
+ switch (verb) {
+ case EVerb::Get: {
+ auto ypathReq = TYPathProxy::Get(path);
+ ToProto(ypathReq->mutable_options(), *options);
+ auto ypathRsp = WaitFor(ExecuteVerb(Service_, ypathReq))
+ .ValueOrThrow();
+ result = TYsonString(ypathRsp->value());
+ break;
+ }
+ case EVerb::List: {
+ auto ypathReq = TYPathProxy::List(path);
+ auto ypathRsp = WaitFor(ExecuteVerb(Service_, ypathReq))
+ .ValueOrThrow();
+ result = TYsonString(ypathRsp->value());
+ break;
+ }
+ default:
+ YT_ABORT();
+ }
+
+ rsp->SetStatus(EStatusCode::OK);
+ NHttp::ReplyJson(rsp, [&] (NYson::IYsonConsumer* writer) {
+ Serialize(result, writer);
+ });
+ WaitFor(rsp->Close())
+ .ThrowOnError();
+ }
+
+private:
+ const IYPathServicePtr Service_;
+};
+
+IHttpHandlerPtr GetOrchidYPathHttpHandler(const IYPathServicePtr& service)
+{
+ return WrapYTException(New<TYPathHttpHandler>(service));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NMonitoring
diff --git a/yt/yt/library/monitoring/http_integration.h b/yt/yt/library/monitoring/http_integration.h
new file mode 100644
index 0000000000..48c12ca8a8
--- /dev/null
+++ b/yt/yt/library/monitoring/http_integration.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/ytree/ypath_service.h>
+
+#include <yt/yt/core/http/public.h>
+
+#include <yt/yt/library/profiling/solomon/public.h>
+
+namespace NYT::NMonitoring {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void Initialize(
+ const NHttp::IServerPtr& monitoringServer,
+ const NProfiling::TSolomonExporterConfigPtr& solomonExporterConfig,
+ TMonitoringManagerPtr* monitoringManager,
+ NYTree::IMapNodePtr* orchidRoot);
+
+NHttp::IHttpHandlerPtr CreateTracingHttpHandler();
+
+NHttp::IHttpHandlerPtr GetOrchidYPathHttpHandler(
+ const NYTree::IYPathServicePtr& service);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NMonitoring
diff --git a/yt/yt/library/monitoring/monitoring_manager.cpp b/yt/yt/library/monitoring/monitoring_manager.cpp
new file mode 100644
index 0000000000..ef642034a4
--- /dev/null
+++ b/yt/yt/library/monitoring/monitoring_manager.cpp
@@ -0,0 +1,177 @@
+#include "monitoring_manager.h"
+#include "private.h"
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/periodic_executor.h>
+
+#include <yt/yt/core/ytree/convert.h>
+#include <yt/yt/core/ytree/ephemeral_node_factory.h>
+#include <yt/yt/core/ytree/node.h>
+#include <yt/yt/core/ytree/tree_visitor.h>
+#include <yt/yt/core/ytree/ypath_detail.h>
+#include <yt/yt/core/ytree/ypath_client.h>
+
+#include <yt/yt/library/profiling/sensor.h>
+
+namespace NYT::NMonitoring {
+
+using namespace NYTree;
+using namespace NYPath;
+using namespace NYson;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = MonitoringLogger;
+
+static const auto UpdatePeriod = TDuration::Seconds(3);
+static const auto EmptyRoot = GetEphemeralNodeFactory()->CreateMap();
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TMonitoringManager::TImpl
+ : public TRefCounted
+{
+public:
+ void Register(const TYPath& path, TYsonProducer producer)
+ {
+ auto guard = Guard(SpinLock_);
+ YT_VERIFY(PathToProducer_.emplace(path, producer).second);
+ }
+
+ void Unregister(const TYPath& path)
+ {
+ auto guard = Guard(SpinLock_);
+ YT_VERIFY(PathToProducer_.erase(path) == 1);
+ }
+
+ IYPathServicePtr GetService()
+ {
+ return New<TYPathService>(this);
+ }
+
+ void Start()
+ {
+ auto guard = Guard(SpinLock_);
+
+ YT_VERIFY(!Started_);
+
+ PeriodicExecutor_ = New<TPeriodicExecutor>(
+ ActionQueue_->GetInvoker(),
+ BIND(&TImpl::Update, MakeWeak(this)),
+ UpdatePeriod);
+ PeriodicExecutor_->Start();
+
+ Started_ = true;
+ }
+
+ void Stop()
+ {
+ auto guard = Guard(SpinLock_);
+
+ if (!Started_)
+ return;
+
+ Started_ = false;
+ YT_UNUSED_FUTURE(PeriodicExecutor_->Stop());
+ Root_.Reset();
+ }
+
+private:
+ class TYPathService
+ : public TYPathServiceBase
+ {
+ public:
+ explicit TYPathService(TIntrusivePtr<TImpl> owner)
+ : Owner_(std::move(owner))
+ { }
+
+ TResolveResult Resolve(const TYPath& path, const IYPathServiceContextPtr& /*context*/) override
+ {
+ return TResolveResultThere{Owner_->GetRoot(), path};
+ }
+
+ private:
+ const TIntrusivePtr<TImpl> Owner_;
+
+ };
+
+ bool Started_ = false;
+ TActionQueuePtr ActionQueue_ = New<TActionQueue>("Monitoring");
+ TPeriodicExecutorPtr PeriodicExecutor_;
+
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
+ THashMap<TString, NYson::TYsonProducer> PathToProducer_;
+ IMapNodePtr Root_;
+
+ void Update()
+ {
+ YT_LOG_DEBUG("Started updating monitoring state");
+
+ YT_PROFILE_TIMING("/monitoring/update_time") {
+ auto newRoot = GetEphemeralNodeFactory()->CreateMap();
+
+ THashMap<TString, NYson::TYsonProducer> pathToProducer;;
+ {
+ auto guard = Guard(SpinLock_);
+ pathToProducer = PathToProducer_;
+ }
+
+ for (const auto& [path, producer] : pathToProducer) {
+ auto value = ConvertToYsonString(producer);
+ SyncYPathSet(newRoot, path, value);
+ }
+
+ if (Started_) {
+ auto guard = Guard(SpinLock_);
+ std::swap(Root_, newRoot);
+ }
+ }
+ YT_LOG_DEBUG("Finished updating monitoring state");
+ }
+
+ IMapNodePtr GetRoot()
+ {
+ auto guard = Guard(SpinLock_);
+ return Root_ ? Root_ : EmptyRoot;
+ }
+};
+
+DEFINE_REFCOUNTED_TYPE(TMonitoringManager)
+
+////////////////////////////////////////////////////////////////////////////////
+
+TMonitoringManager::TMonitoringManager()
+ : Impl_(New<TImpl>())
+{ }
+
+TMonitoringManager::~TMonitoringManager() = default;
+
+void TMonitoringManager::Register(const TYPath& path, TYsonProducer producer)
+{
+ Impl_->Register(path, producer);
+}
+
+void TMonitoringManager::Unregister(const TYPath& path)
+{
+ Impl_->Unregister(path);
+}
+
+IYPathServicePtr TMonitoringManager::GetService()
+{
+ return Impl_->GetService();
+}
+
+void TMonitoringManager::Start()
+{
+ Impl_->Start();
+}
+
+void TMonitoringManager::Stop()
+{
+ Impl_->Stop();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NMonitoring
diff --git a/yt/yt/library/monitoring/monitoring_manager.h b/yt/yt/library/monitoring/monitoring_manager.h
new file mode 100644
index 0000000000..fc5c3de6c7
--- /dev/null
+++ b/yt/yt/library/monitoring/monitoring_manager.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/yson/consumer.h>
+
+#include <yt/yt/core/ypath/public.h>
+
+#include <yt/yt/core/ytree/public.h>
+
+namespace NYT::NMonitoring {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Exposes a tree assembled from results returned by a set of
+//! registered NYson::TYsonProducer-s.
+/*!
+ * \note
+ * The results are cached and periodically updated.
+ */
+class TMonitoringManager
+ : public TRefCounted
+{
+public:
+ TMonitoringManager();
+ ~TMonitoringManager();
+
+ //! Registers a new #producer for a given #path.
+ void Register(const NYPath::TYPath& path, NYson::TYsonProducer producer);
+
+ //! Unregisters an existing producer for the specified #path.
+ void Unregister(const NYPath::TYPath& path);
+
+ //! Returns the service representing the whole tree.
+ /*!
+ * \note The service is thread-safe.
+ */
+ NYTree::IYPathServicePtr GetService();
+
+ //! Starts periodic updates.
+ void Start();
+
+ //! Stops periodic updates.
+ void Stop();
+
+private:
+ class TImpl;
+ TIntrusivePtr<TImpl> Impl_;
+
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NMonitoring
diff --git a/yt/yt/library/monitoring/private.h b/yt/yt/library/monitoring/private.h
new file mode 100644
index 0000000000..e2bfb31c78
--- /dev/null
+++ b/yt/yt/library/monitoring/private.h
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT::NMonitoring {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger MonitoringLogger("Monitoring");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NJournalClient
diff --git a/yt/yt/library/monitoring/public.h b/yt/yt/library/monitoring/public.h
new file mode 100644
index 0000000000..3514bdd858
--- /dev/null
+++ b/yt/yt/library/monitoring/public.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+namespace NYT::NMonitoring {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_CLASS(TMonitoringManager)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NMonitoring
diff --git a/yt/yt/library/monitoring/ya.make b/yt/yt/library/monitoring/ya.make
new file mode 100644
index 0000000000..c2fccd99ac
--- /dev/null
+++ b/yt/yt/library/monitoring/ya.make
@@ -0,0 +1,27 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ http_integration.cpp
+ monitoring_manager.cpp
+)
+
+PEERDIR(
+ yt/yt/core
+ yt/yt/build
+ yt/yt/library/profiling
+ yt/yt/library/profiling/solomon
+ library/cpp/cgiparam
+)
+
+IF (OS_LINUX)
+ PEERDIR(
+ yt/yt/library/ytprof
+ yt/yt/library/ytprof/http
+
+ yt/yt/library/backtrace_introspector/http
+ )
+ENDIF()
+
+END()
diff --git a/yt/yt/library/process/CMakeLists.darwin-x86_64.txt b/yt/yt/library/process/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..b66c679390
--- /dev/null
+++ b/yt/yt/library/process/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,26 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-process)
+target_compile_options(yt-library-process PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-process PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-re2
+)
+target_sources(yt-library-process PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp
+)
diff --git a/yt/yt/library/process/CMakeLists.linux-aarch64.txt b/yt/yt/library/process/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..e065bba3b1
--- /dev/null
+++ b/yt/yt/library/process/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,27 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-process)
+target_compile_options(yt-library-process PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-process PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-re2
+)
+target_sources(yt-library-process PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp
+)
diff --git a/yt/yt/library/process/CMakeLists.linux-x86_64.txt b/yt/yt/library/process/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..e065bba3b1
--- /dev/null
+++ b/yt/yt/library/process/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,27 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-process)
+target_compile_options(yt-library-process PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-process PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-re2
+)
+target_sources(yt-library-process PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp
+)
diff --git a/yt/yt/library/process/CMakeLists.txt b/yt/yt/library/process/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/process/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/process/CMakeLists.windows-x86_64.txt b/yt/yt/library/process/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..3637ee7dae
--- /dev/null
+++ b/yt/yt/library/process/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-process)
+target_link_libraries(yt-library-process PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ contrib-libs-re2
+)
+target_sources(yt-library-process PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp
+)
diff --git a/yt/yt/library/process/io_dispatcher.cpp b/yt/yt/library/process/io_dispatcher.cpp
new file mode 100644
index 0000000000..7da757658d
--- /dev/null
+++ b/yt/yt/library/process/io_dispatcher.cpp
@@ -0,0 +1,37 @@
+#include "io_dispatcher.h"
+
+#include <yt/yt/core/concurrency/thread_pool_poller.h>
+#include <yt/yt/core/concurrency/poller.h>
+
+#include <yt/yt/core/misc/singleton.h>
+
+namespace NYT::NPipes {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TIODispatcher::TIODispatcher()
+ : Poller_(BIND([] { return CreateThreadPoolPoller(1, "Pipes"); }))
+{ }
+
+TIODispatcher::~TIODispatcher() = default;
+
+TIODispatcher* TIODispatcher::Get()
+{
+ return Singleton<TIODispatcher>();
+}
+
+IInvokerPtr TIODispatcher::GetInvoker()
+{
+ return Poller_.Value()->GetInvoker();
+}
+
+IPollerPtr TIODispatcher::GetPoller()
+{
+ return Poller_.Value();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/io_dispatcher.h b/yt/yt/library/process/io_dispatcher.h
new file mode 100644
index 0000000000..2db1b34386
--- /dev/null
+++ b/yt/yt/library/process/io_dispatcher.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/concurrency/public.h>
+
+#include <yt/yt/core/misc/lazy_ptr.h>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TIODispatcher
+{
+public:
+ ~TIODispatcher();
+
+ static TIODispatcher* Get();
+
+ IInvokerPtr GetInvoker();
+
+ NConcurrency::IPollerPtr GetPoller();
+
+private:
+ TIODispatcher();
+
+ Y_DECLARE_SINGLETON_FRIEND()
+
+ TLazyIntrusivePtr<NConcurrency::IThreadPoolPoller> Poller_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/pipe.cpp b/yt/yt/library/process/pipe.cpp
new file mode 100644
index 0000000000..f51d043f22
--- /dev/null
+++ b/yt/yt/library/process/pipe.cpp
@@ -0,0 +1,256 @@
+#include "pipe.h"
+#include "private.h"
+#include "io_dispatcher.h"
+
+#include <yt/yt/core/net/connection.h>
+
+#include <yt/yt/core/misc/proc.h>
+#include <yt/yt/core/misc/fs.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+namespace NYT::NPipes {
+
+using namespace NNet;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static const auto& Logger = PipesLogger;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TNamedPipe::TNamedPipe(const TString& path, bool owning)
+ : Path_(path)
+ , Owning_(owning)
+{ }
+
+TNamedPipe::~TNamedPipe()
+{
+ if (!Owning_) {
+ return;
+ }
+
+ if (unlink(Path_.c_str()) == -1) {
+ YT_LOG_INFO(TError::FromSystem(), "Failed to unlink pipe %v", Path_);
+ }
+}
+
+TNamedPipePtr TNamedPipe::Create(const TString& path, int permissions)
+{
+ auto pipe = New<TNamedPipe>(path, /* owning */ true);
+ pipe->Open(permissions);
+ YT_LOG_DEBUG("Named pipe created (Path: %v, Permissions: %v)", path, permissions);
+ return pipe;
+}
+
+TNamedPipePtr TNamedPipe::FromPath(const TString& path)
+{
+ return New<TNamedPipe>(path, /* owning */ false);
+}
+
+void TNamedPipe::Open(int permissions)
+{
+ if (mkfifo(Path_.c_str(), permissions) == -1) {
+ THROW_ERROR_EXCEPTION("Failed to create named pipe %v", Path_)
+ << TError::FromSystem();
+ }
+}
+
+IConnectionReaderPtr TNamedPipe::CreateAsyncReader()
+{
+ YT_VERIFY(!Path_.empty());
+ return CreateInputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this));
+}
+
+IConnectionWriterPtr TNamedPipe::CreateAsyncWriter()
+{
+ YT_VERIFY(!Path_.empty());
+ return CreateOutputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this));
+}
+
+TString TNamedPipe::GetPath() const
+{
+ return Path_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TNamedPipeConfigPtr TNamedPipeConfig::Create(TString path, int fd, bool write)
+{
+ auto result = New<TNamedPipeConfig>();
+ result->Path = std::move(path);
+ result->FD = fd;
+ result->Write = write;
+
+ return result;
+}
+
+void TNamedPipeConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("path", &TThis::Path)
+ .Default();
+
+ registrar.Parameter("fd", &TThis::FD)
+ .Default(0);
+
+ registrar.Parameter("write", &TThis::Write)
+ .Default(false);
+}
+
+DEFINE_REFCOUNTED_TYPE(TNamedPipeConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPipe::TPipe()
+{ }
+
+TPipe::TPipe(TPipe&& pipe)
+{
+ Init(std::move(pipe));
+}
+
+TPipe::TPipe(int fd[2])
+ : ReadFD_(fd[0])
+ , WriteFD_(fd[1])
+{ }
+
+void TPipe::Init(TPipe&& other)
+{
+ ReadFD_ = other.ReadFD_;
+ WriteFD_ = other.WriteFD_;
+ other.ReadFD_ = InvalidFD;
+ other.WriteFD_ = InvalidFD;
+}
+
+TPipe::~TPipe()
+{
+ if (ReadFD_ != InvalidFD) {
+ YT_VERIFY(TryClose(ReadFD_, false));
+ }
+
+ if (WriteFD_ != InvalidFD) {
+ YT_VERIFY(TryClose(WriteFD_, false));
+ }
+}
+
+void TPipe::operator=(TPipe&& other)
+{
+ if (this == &other) {
+ return;
+ }
+
+ Init(std::move(other));
+}
+
+IConnectionWriterPtr TPipe::CreateAsyncWriter()
+{
+ YT_VERIFY(WriteFD_ != InvalidFD);
+ SafeMakeNonblocking(WriteFD_);
+ return CreateConnectionFromFD(ReleaseWriteFD(), {}, {}, TIODispatcher::Get()->GetPoller());
+}
+
+IConnectionReaderPtr TPipe::CreateAsyncReader()
+{
+ YT_VERIFY(ReadFD_ != InvalidFD);
+ SafeMakeNonblocking(ReadFD_);
+ return CreateConnectionFromFD(ReleaseReadFD(), {}, {}, TIODispatcher::Get()->GetPoller());
+}
+
+int TPipe::ReleaseReadFD()
+{
+ YT_VERIFY(ReadFD_ != InvalidFD);
+ auto fd = ReadFD_;
+ ReadFD_ = InvalidFD;
+ return fd;
+}
+
+int TPipe::ReleaseWriteFD()
+{
+ YT_VERIFY(WriteFD_ != InvalidFD);
+ auto fd = WriteFD_;
+ WriteFD_ = InvalidFD;
+ return fd;
+}
+
+int TPipe::GetReadFD() const
+{
+ YT_VERIFY(ReadFD_ != InvalidFD);
+ return ReadFD_;
+}
+
+int TPipe::GetWriteFD() const
+{
+ YT_VERIFY(WriteFD_ != InvalidFD);
+ return WriteFD_;
+}
+
+void TPipe::CloseReadFD()
+{
+ if (ReadFD_ == InvalidFD) {
+ return;
+ }
+ auto fd = ReadFD_;
+ ReadFD_ = InvalidFD;
+ SafeClose(fd, false);
+}
+
+void TPipe::CloseWriteFD()
+{
+ if (WriteFD_ == InvalidFD) {
+ return;
+ }
+ auto fd = WriteFD_;
+ WriteFD_ = InvalidFD;
+ SafeClose(fd, false);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString ToString(const TPipe& pipe)
+{
+ return Format("{ReadFD: %v, WriteFD: %v}",
+ pipe.GetReadFD(),
+ pipe.GetWriteFD());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPipeFactory::TPipeFactory(int minFD)
+ : MinFD_(minFD)
+{ }
+
+TPipeFactory::~TPipeFactory()
+{
+ for (int fd : ReservedFDs_) {
+ YT_VERIFY(TryClose(fd, false));
+ }
+}
+
+TPipe TPipeFactory::Create()
+{
+ while (true) {
+ int fd[2];
+ SafePipe(fd);
+ if (fd[0] >= MinFD_ && fd[1] >= MinFD_) {
+ TPipe pipe(fd);
+ return pipe;
+ } else {
+ ReservedFDs_.push_back(fd[0]);
+ ReservedFDs_.push_back(fd[1]);
+ }
+ }
+}
+
+void TPipeFactory::Clear()
+{
+ for (int& fd : ReservedFDs_) {
+ YT_VERIFY(TryClose(fd, false));
+ fd = TPipe::InvalidFD;
+ }
+ ReservedFDs_.clear();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/pipe.h b/yt/yt/library/process/pipe.h
new file mode 100644
index 0000000000..10da81cc8a
--- /dev/null
+++ b/yt/yt/library/process/pipe.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/net/public.h>
+
+#include <yt/yt/core/ytree/yson_struct.h>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNamedPipe
+ : public TRefCounted
+{
+public:
+ ~TNamedPipe();
+ static TNamedPipePtr Create(const TString& path, int permissions = 0660);
+ static TNamedPipePtr FromPath(const TString& path);
+
+ NNet::IConnectionReaderPtr CreateAsyncReader();
+ NNet::IConnectionWriterPtr CreateAsyncWriter();
+
+ TString GetPath() const;
+
+private:
+ const TString Path_;
+
+ //! Whether pipe was created by this class
+ //! and should be removed in destructor.
+ const bool Owning_;
+
+ explicit TNamedPipe(const TString& path, bool owning);
+ void Open(int permissions);
+ DECLARE_NEW_FRIEND()
+};
+
+DEFINE_REFCOUNTED_TYPE(TNamedPipe)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNamedPipeConfig
+ : public NYTree::TYsonStruct
+{
+public:
+ TString Path;
+ int FD = 0;
+ bool Write = false;
+
+ static TNamedPipeConfigPtr Create(TString path, int fd, bool write);
+
+ REGISTER_YSON_STRUCT(TNamedPipeConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPipe
+ : public TNonCopyable
+{
+public:
+ static const int InvalidFD = -1;
+
+ TPipe();
+ TPipe(TPipe&& pipe);
+ ~TPipe();
+
+ void operator=(TPipe&& other);
+
+ void CloseReadFD();
+ void CloseWriteFD();
+
+ NNet::IConnectionReaderPtr CreateAsyncReader();
+ NNet::IConnectionWriterPtr CreateAsyncWriter();
+
+ int ReleaseReadFD();
+ int ReleaseWriteFD();
+
+ int GetReadFD() const;
+ int GetWriteFD() const;
+
+private:
+ int ReadFD_ = InvalidFD;
+ int WriteFD_ = InvalidFD;
+
+ TPipe(int fd[2]);
+ void Init(TPipe&& other);
+
+ friend class TPipeFactory;
+};
+
+TString ToString(const TPipe& pipe);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPipeFactory
+{
+public:
+ explicit TPipeFactory(int minFD = 0);
+ ~TPipeFactory();
+
+ TPipe Create();
+
+ void Clear();
+
+private:
+ const int MinFD_;
+ std::vector<int> ReservedFDs_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/private.h b/yt/yt/library/process/private.h
new file mode 100644
index 0000000000..95b2ffb0f5
--- /dev/null
+++ b/yt/yt/library/process/private.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger PipesLogger("Pipes");
+inline const NLogging::TLogger PtyLogger("Pty");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/process.cpp b/yt/yt/library/process/process.cpp
new file mode 100644
index 0000000000..809a50ed9a
--- /dev/null
+++ b/yt/yt/library/process/process.cpp
@@ -0,0 +1,697 @@
+#include "process.h"
+#include "pipe.h"
+
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/misc/error.h>
+#include <yt/yt/core/misc/fs.h>
+#include <yt/yt/core/misc/finally.h>
+
+#include <yt/yt/core/concurrency/periodic_executor.h>
+#include <yt/yt/core/concurrency/delayed_executor.h>
+
+#include <library/cpp/yt/system/handle_eintr.h>
+
+#include <util/folder/dirut.h>
+
+#include <util/generic/guid.h>
+
+#include <util/string/ascii.h>
+
+#include <util/string/util.h>
+
+#include <util/system/env.h>
+#include <util/system/execpath.h>
+#include <util/system/maxlen.h>
+#include <util/system/shellcommand.h>
+
+#ifdef _unix_
+ #include <unistd.h>
+ #include <errno.h>
+ #include <sys/wait.h>
+ #include <sys/resource.h>
+#endif
+
+#ifdef _darwin_
+ #include <crt_externs.h>
+ #define environ (*_NSGetEnviron())
+#endif
+
+namespace NYT {
+
+using namespace NPipes;
+using namespace NNet;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline const NLogging::TLogger Logger("Process");
+
+static constexpr pid_t InvalidProcessId = -1;
+
+static constexpr int ExecveRetryCount = 5;
+static constexpr auto ExecveRetryTimeout = TDuration::Seconds(1);
+
+static constexpr int ResolveRetryCount = 5;
+static constexpr auto ResolveRetryTimeout = TDuration::Seconds(1);
+
+////////////////////////////////////////////////////////////////////////////////
+
+TErrorOr<TString> ResolveBinaryPath(const TString& binary)
+{
+ auto Logger = NYT::Logger
+ .WithTag("Binary: %v", binary);
+
+ YT_LOG_DEBUG("Resolving binary path");
+
+ std::vector<TError> accumulatedErrors;
+
+ auto test = [&] (const char* path) {
+ YT_LOG_DEBUG("Probing path (Path: %v)", path);
+ if (access(path, R_OK | X_OK) == 0) {
+ return true;
+ } else {
+ auto error = TError("Cannot run %Qlv", path) << TError::FromSystem();
+ accumulatedErrors.push_back(std::move(error));
+ return false;
+ }
+ };
+
+ auto failure = [&] {
+ auto error = TError(
+ EProcessErrorCode::CannotResolveBinary,
+ "Cannot resolve binary %Qlv",
+ binary);
+ error.MutableInnerErrors()->swap(accumulatedErrors);
+ YT_LOG_DEBUG(error, "Error resolving binary path");
+ return error;
+ };
+
+ auto success = [&] (const TString& path) {
+ YT_LOG_DEBUG("Binary resolved (Path: %v)", path);
+ return path;
+ };
+
+ if (test(binary.c_str())) {
+ return success(binary);
+ }
+
+ // If this is an absolute path, stop here.
+ if (binary.empty() || binary[0] == '/') {
+ return failure();
+ }
+
+ // XXX(sandello): Sometimes we drop PATH from environment when spawning isolated processes.
+ // In this case, try to locate somewhere nearby.
+ {
+ auto execPathDirName = GetDirName(GetExecPath());
+ YT_LOG_DEBUG("Looking in our exec path directory (ExecPathDir: %v)", execPathDirName);
+ auto probe = TString::Join(execPathDirName, "/", binary);
+ if (test(probe.c_str())) {
+ return success(probe);
+ }
+ }
+
+ std::array<char, MAX_PATH> buffer;
+
+ auto envPathStr = GetEnv("PATH");
+ TStringBuf envPath(envPathStr);
+ TStringBuf envPathItem;
+
+ YT_LOG_DEBUG("Looking for binary in PATH (Path: %v)", envPathStr);
+
+ while (envPath.NextTok(':', envPathItem)) {
+ if (buffer.size() < 2 + envPathItem.size() + binary.size()) {
+ continue;
+ }
+
+ size_t index = 0;
+ std::copy(envPathItem.begin(), envPathItem.end(), buffer.begin() + index);
+ index += envPathItem.size();
+ buffer[index] = '/';
+ index += 1;
+ std::copy(binary.begin(), binary.end(), buffer.begin() + index);
+ index += binary.size();
+ buffer[index] = 0;
+
+ if (test(buffer.data())) {
+ return success(TString(buffer.data(), index));
+ }
+ }
+
+ return failure();
+}
+
+bool TryKillProcessByPid(int pid, int signal)
+{
+#ifdef _unix_
+ YT_VERIFY(pid != -1);
+ int result = ::kill(pid, signal);
+ // Ignore ESRCH because process may have died just before TryKillProcessByPid.
+ if (result < 0 && errno != ESRCH) {
+ return false;
+ }
+ return true;
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace {
+
+#ifdef _unix_
+
+bool TryWaitid(idtype_t idtype, id_t id, siginfo_t *infop, int options)
+{
+ if (infop != nullptr) {
+ // See comment below.
+ infop->si_pid = 0;
+ }
+
+ siginfo_t info;
+ ::memset(&info, 0, sizeof(info));
+ auto res = HandleEintr(::waitid, idtype, id, infop != nullptr ? infop : &info, options);
+
+ if (res == 0) {
+ // According to man wait.
+ // If WNOHANG was specified in options and there were
+ // no children in a waitable state, then waitid() returns 0 immediately.
+ // To distinguish this case from that where a child
+ // was in a waitable state, zero out the si_pid field
+ // before the call and check for a nonzero value in this field after
+ // the call returns.
+ if (infop && infop->si_pid == 0) {
+ return false;
+ }
+ return true;
+ }
+
+ return false;
+}
+
+void Wait4OrDie(pid_t id, int* status, int options, rusage* rusage)
+{
+ auto res = HandleEintr(::wait4, id, status, options, rusage);
+ if (res == -1) {
+ YT_LOG_FATAL(TError::FromSystem(), "Wait4 failed");
+ }
+}
+
+void Cleanup(int pid)
+{
+ YT_VERIFY(pid > 0);
+
+ YT_VERIFY(TryKillProcessByPid(pid, 9));
+ YT_VERIFY(TryWaitid(P_PID, pid, nullptr, WEXITED));
+}
+
+bool TrySetSignalMask(const sigset_t* sigmask, sigset_t* oldSigmask)
+{
+ int error = pthread_sigmask(SIG_SETMASK, sigmask, oldSigmask);
+ if (error != 0) {
+ return false;
+ }
+ return true;
+}
+
+bool TryResetSignals()
+{
+ for (int sig = 1; sig < NSIG; ++sig) {
+ // Ignore invalid signal errors.
+ ::signal(sig, SIG_DFL);
+ }
+ return true;
+}
+
+#endif
+
+} // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+TProcessBase::TProcessBase(const TString& path)
+ : Path_(path)
+ , ProcessId_(InvalidProcessId)
+{ }
+
+void TProcessBase::AddArgument(TStringBuf arg)
+{
+ YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_);
+
+ Args_.push_back(Capture(arg));
+}
+
+void TProcessBase::AddEnvVar(TStringBuf var)
+{
+ YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_);
+
+ Env_.push_back(Capture(var));
+}
+
+void TProcessBase::AddArguments(std::initializer_list<TStringBuf> args)
+{
+ for (auto arg : args) {
+ AddArgument(arg);
+ }
+}
+
+void TProcessBase::AddArguments(const std::vector<TString>& args)
+{
+ for (const auto& arg : args) {
+ AddArgument(arg);
+ }
+}
+
+void TProcessBase::SetWorkingDirectory(const TString& path)
+{
+ WorkingDirectory_ = path;
+}
+
+void TProcessBase::CreateProcessGroup()
+{
+ CreateProcessGroup_ = true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TSimpleProcess::TSimpleProcess(const TString& path, bool copyEnv, TDuration pollPeriod)
+ // TString is guaranteed to be zero-terminated.
+ // https://wiki.yandex-team.ru/Development/Poisk/arcadia/util/TStringAndTStringBuf#sobstvennosimvoly
+ : TProcessBase(path)
+ , PollPeriod_(pollPeriod)
+ , PipeFactory_(3)
+{
+ AddArgument(path);
+
+ if (copyEnv) {
+ for (char** envIt = environ; *envIt; ++envIt) {
+ Env_.push_back(Capture(*envIt));
+ }
+ }
+}
+
+void TSimpleProcess::AddDup2FileAction(int oldFD, int newFD)
+{
+ TSpawnAction action{
+ std::bind(TryDup2, oldFD, newFD),
+ Format("Error duplicating %v file descriptor to %v in child process", oldFD, newFD)
+ };
+
+ MaxSpawnActionFD_ = std::max(MaxSpawnActionFD_, newFD);
+ SpawnActions_.push_back(action);
+}
+
+IConnectionReaderPtr TSimpleProcess::GetStdOutReader()
+{
+ auto& pipe = StdPipes_[STDOUT_FILENO];
+ pipe = PipeFactory_.Create();
+ AddDup2FileAction(pipe.GetWriteFD(), STDOUT_FILENO);
+ return pipe.CreateAsyncReader();
+}
+
+IConnectionReaderPtr TSimpleProcess::GetStdErrReader()
+{
+ auto& pipe = StdPipes_[STDERR_FILENO];
+ pipe = PipeFactory_.Create();
+ AddDup2FileAction(pipe.GetWriteFD(), STDERR_FILENO);
+ return pipe.CreateAsyncReader();
+}
+
+IConnectionWriterPtr TSimpleProcess::GetStdInWriter()
+{
+ auto& pipe = StdPipes_[STDIN_FILENO];
+ pipe = PipeFactory_.Create();
+ AddDup2FileAction(pipe.GetReadFD(), STDIN_FILENO);
+ return pipe.CreateAsyncWriter();
+}
+
+TFuture<void> TProcessBase::Spawn()
+{
+ try {
+ // Resolve binary path.
+ std::vector<TError> innerErrors;
+ for (int retryIndex = ResolveRetryCount; retryIndex >= 0; --retryIndex) {
+ auto errorOrPath = ResolveBinaryPath(Path_);
+ if (errorOrPath.IsOK()) {
+ ResolvedPath_ = errorOrPath.Value();
+ break;
+ }
+
+ innerErrors.push_back(errorOrPath);
+
+ if (retryIndex == 0) {
+ THROW_ERROR_EXCEPTION("Failed to resolve binary path %v", Path_)
+ << innerErrors;
+ }
+
+ TDelayedExecutor::WaitForDuration(ResolveRetryTimeout);
+ }
+
+ DoSpawn();
+ } catch (const std::exception& ex) {
+ FinishedPromise_.TrySet(ex);
+ }
+ return FinishedPromise_;
+}
+
+void TSimpleProcess::DoSpawn()
+{
+#ifdef _unix_
+ auto finally = Finally([&] () {
+ StdPipes_[STDIN_FILENO].CloseReadFD();
+ StdPipes_[STDOUT_FILENO].CloseWriteFD();
+ StdPipes_[STDERR_FILENO].CloseWriteFD();
+ PipeFactory_.Clear();
+ });
+
+ YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_);
+
+ // Make sure no spawn action closes Pipe_.WriteFD
+ TPipeFactory pipeFactory(MaxSpawnActionFD_ + 1);
+ Pipe_ = pipeFactory.Create();
+ pipeFactory.Clear();
+
+ YT_LOG_DEBUG("Spawning new process (Path: %v, ErrorPipe: %v, Arguments: %v, Environment: %v)",
+ ResolvedPath_,
+ Pipe_,
+ Args_,
+ Env_);
+
+ Env_.push_back(nullptr);
+ Args_.push_back(nullptr);
+
+ // Block all signals around vfork; see http://ewontfix.com/7/
+
+ // As the child may run in the same address space as the parent until
+ // the actual execve() system call, any (custom) signal handlers that
+ // the parent has might alter parent's memory if invoked in the child,
+ // with undefined results. So we block all signals in the parent before
+ // vfork(), which will cause them to be blocked in the child as well (we
+ // rely on the fact that Linux, just like all sane implementations, only
+ // clones the calling thread). Then, in the child, we reset all signals
+ // to their default dispositions (while still blocked), and unblock them
+ // (so the exec()ed process inherits the parent's signal mask)
+
+ sigset_t allBlocked;
+ sigfillset(&allBlocked);
+ sigset_t oldSignals;
+
+ if (!TrySetSignalMask(&allBlocked, &oldSignals)) {
+ THROW_ERROR_EXCEPTION("Failed to block all signals")
+ << TError::FromSystem();
+ }
+
+ SpawnActions_.push_back(TSpawnAction{
+ TryResetSignals,
+ "Error resetting signals to default disposition in child process: signal failed"
+ });
+
+ SpawnActions_.push_back(TSpawnAction{
+ std::bind(TrySetSignalMask, &oldSignals, nullptr),
+ "Error unblocking signals in child process: pthread_sigmask failed"
+ });
+
+ if (!WorkingDirectory_.empty()) {
+ SpawnActions_.push_back(TSpawnAction{
+ [&] () {
+ NFs::SetCurrentWorkingDirectory(WorkingDirectory_);
+ return true;
+ },
+ "Error changing working directory"
+ });
+ }
+
+ if (CreateProcessGroup_) {
+ SpawnActions_.push_back(TSpawnAction{
+ [&] () {
+ setpgrp();
+ return true;
+ },
+ "Error creating process group"
+ });
+ }
+
+ SpawnActions_.push_back(TSpawnAction{
+ [this] {
+ for (int retryIndex = 0; retryIndex < ExecveRetryCount; ++retryIndex) {
+ // Execve may fail, if called binary is being updated, e.g. during yandex-yt package update.
+ // So we'd better retry several times.
+ // For example see YT-6352.
+ TryExecve(ResolvedPath_.c_str(), Args_.data(), Env_.data());
+ if (retryIndex < ExecveRetryCount - 1) {
+ Sleep(ExecveRetryTimeout);
+ }
+ }
+ // If we are still here, return failure.
+ return false;
+ },
+ "Error starting child process: execve failed"
+ });
+
+ SpawnChild();
+
+ // This should not fail ever.
+ YT_VERIFY(TrySetSignalMask(&oldSignals, nullptr));
+
+ Pipe_.CloseWriteFD();
+
+ ValidateSpawnResult();
+
+ AsyncWaitExecutor_ = New<TPeriodicExecutor>(
+ GetSyncInvoker(),
+ BIND(&TSimpleProcess::AsyncPeriodicTryWait, MakeStrong(this)),
+ PollPeriod_);
+
+ AsyncWaitExecutor_->Start();
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+void TSimpleProcess::SpawnChild()
+{
+ // NB: fork() will cause data corruption when run concurrently with
+ // Disk IO on O_DIRECT file descriptor. Seems like vfork don't suffer from the same issue.
+
+#ifdef _unix_
+ int pid = vfork();
+
+ if (pid < 0) {
+ THROW_ERROR_EXCEPTION("Error starting child process: vfork failed")
+ << TErrorAttribute("path", ResolvedPath_)
+ << TError::FromSystem();
+ }
+
+ if (pid == 0) {
+ try {
+ Child();
+ } catch (...) {
+ YT_ABORT();
+ }
+ }
+
+ ProcessId_ = pid;
+ Started_ = true;
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+void TSimpleProcess::ValidateSpawnResult()
+{
+#ifdef _unix_
+ int data[2];
+ ssize_t res;
+ res = HandleEintr(::read, Pipe_.GetReadFD(), &data, sizeof(data));
+ Pipe_.CloseReadFD();
+
+ if (res == 0) {
+ // Child successfully spawned or was killed by a signal.
+ // But there is no way to distinguish between these two cases:
+ // * child killed by signal before exec
+ // * child killed by signal after exec
+ // So we treat kill-before-exec the same way as kill-after-exec.
+ YT_LOG_DEBUG("Child process spawned successfully (Pid: %v)", ProcessId_);
+ return;
+ }
+
+ YT_VERIFY(res == sizeof(data));
+ Finished_ = true;
+
+ Cleanup(ProcessId_);
+ ProcessId_ = InvalidProcessId;
+
+ int actionIndex = data[0];
+ int errorCode = data[1];
+
+ YT_VERIFY(0 <= actionIndex && actionIndex < std::ssize(SpawnActions_));
+ const auto& action = SpawnActions_[actionIndex];
+ THROW_ERROR_EXCEPTION("%v", action.ErrorMessage)
+ << TError::FromSystem(errorCode);
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+#ifdef _unix_
+void TSimpleProcess::AsyncPeriodicTryWait()
+{
+ siginfo_t processInfo;
+ memset(&processInfo, 0, sizeof(siginfo_t));
+
+ // Note WNOWAIT flag.
+ // This call just waits for a process to be finished but does not clear zombie flag.
+
+ if (!TryWaitid(P_PID, ProcessId_, &processInfo, WEXITED | WNOWAIT | WNOHANG) ||
+ processInfo.si_pid != ProcessId_)
+ {
+ return;
+ }
+
+ YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop());
+ AsyncWaitExecutor_ = nullptr;
+
+ // This call just should return immediately
+ // because we have already waited for this process with WNOHANG
+ rusage rusage;
+ Wait4OrDie(ProcessId_, nullptr, WNOHANG, &rusage);
+
+ Finished_ = true;
+ auto error = ProcessInfoToError(processInfo);
+ YT_LOG_DEBUG("Process finished (Pid: %v, MajFaults: %d, Error: %v)", ProcessId_, rusage.ru_majflt, error);
+
+ FinishedPromise_.Set(error);
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+void TSimpleProcess::Kill(int signal)
+{
+#ifdef _unix_
+ if (!Started_) {
+ THROW_ERROR_EXCEPTION("Process is not started yet");
+ }
+
+ if (Finished_) {
+ return;
+ }
+
+ YT_LOG_DEBUG("Killing child process (Pid: %v)", ProcessId_);
+
+ bool result = false;
+ if (!CreateProcessGroup_) {
+ result = TryKillProcessByPid(ProcessId_, signal);
+ } else {
+ result = TryKillProcessByPid(-1 * ProcessId_, signal);
+ }
+
+ if (!result) {
+ THROW_ERROR_EXCEPTION("Failed to kill child process %v", ProcessId_)
+ << TError::FromSystem();
+ }
+ return;
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+TString TProcessBase::GetPath() const
+{
+ return Path_;
+}
+
+int TProcessBase::GetProcessId() const
+{
+ return ProcessId_;
+}
+
+bool TProcessBase::IsStarted() const
+{
+ return Started_;
+}
+
+bool TProcessBase::IsFinished() const
+{
+ return Finished_;
+}
+
+TString TProcessBase::GetCommandLine() const
+{
+ TStringBuilder builder;
+ builder.AppendString(Path_);
+
+ bool first = true;
+ for (const auto& arg_ : Args_) {
+ TStringBuf arg(arg_);
+ if (first) {
+ first = false;
+ } else {
+ if (arg) {
+ builder.AppendChar(' ');
+ bool needQuote = false;
+ for (size_t i = 0; i < arg.length(); ++i) {
+ if (!IsAsciiAlnum(arg[i]) &&
+ arg[i] != '-' && arg[i] != '_' && arg[i] != '=' && arg[i] != '/')
+ {
+ needQuote = true;
+ break;
+ }
+ }
+ if (needQuote) {
+ builder.AppendChar('"');
+ TStringBuf left, right;
+ while (arg.TrySplit('"', left, right)) {
+ builder.AppendString(left);
+ builder.AppendString("\\\"");
+ arg = right;
+ }
+ builder.AppendString(arg);
+ builder.AppendChar('"');
+ } else {
+ builder.AppendString(arg);
+ }
+ }
+ }
+ }
+
+ return builder.Flush();
+}
+
+const char* TProcessBase::Capture(TStringBuf arg)
+{
+ StringHolders_.push_back(TString(arg));
+ return StringHolders_.back().c_str();
+}
+
+void TSimpleProcess::Child()
+{
+#ifdef _unix_
+ for (int actionIndex = 0; actionIndex < std::ssize(SpawnActions_); ++actionIndex) {
+ auto& action = SpawnActions_[actionIndex];
+ if (!action.Callback()) {
+ // Report error through the pipe.
+ int data[] = {
+ actionIndex,
+ errno
+ };
+
+ // According to pipe(7) write of small buffer is atomic.
+ ssize_t size = HandleEintr(::write, Pipe_.GetWriteFD(), &data, sizeof(data));
+ YT_VERIFY(size == sizeof(data));
+ _exit(1);
+ }
+ }
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+ YT_ABORT();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/process/process.h b/yt/yt/library/process/process.h
new file mode 100644
index 0000000000..b38ae3f4b3
--- /dev/null
+++ b/yt/yt/library/process/process.h
@@ -0,0 +1,125 @@
+#pragma once
+
+#include "pipe.h"
+
+#include <yt/yt/core/misc/error.h>
+
+#include <yt/yt/core/actions/future.h>
+
+#include <yt/yt/core/concurrency/public.h>
+
+#include <atomic>
+#include <vector>
+#include <array>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TErrorOr<TString> ResolveBinaryPath(const TString& binary);
+bool TryKillProcessByPid(int pid, int signal);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TProcessBase
+ : public TRefCounted
+{
+public:
+ explicit TProcessBase(const TString& path);
+
+ void AddArgument(TStringBuf arg);
+ void AddEnvVar(TStringBuf var);
+
+ void AddArguments(std::initializer_list<TStringBuf> args);
+ void AddArguments(const std::vector<TString>& args);
+
+ void SetWorkingDirectory(const TString& path);
+ void CreateProcessGroup();
+
+ virtual NNet::IConnectionWriterPtr GetStdInWriter() = 0;
+ virtual NNet::IConnectionReaderPtr GetStdOutReader() = 0;
+ virtual NNet::IConnectionReaderPtr GetStdErrReader() = 0;
+
+ TFuture<void> Spawn();
+ virtual void Kill(int signal) = 0;
+
+ TString GetPath() const;
+ int GetProcessId() const;
+ bool IsStarted() const;
+ bool IsFinished() const;
+
+ TString GetCommandLine() const;
+
+protected:
+ const TString Path_;
+
+ int ProcessId_;
+ std::atomic<bool> Started_ = {false};
+ std::atomic<bool> Finished_ = {false};
+ int MaxSpawnActionFD_ = - 1;
+ NPipes::TPipe Pipe_;
+ // Container for owning string data. Use std::deque because it never moves contained objects.
+ std::deque<std::string> StringHolders_;
+ std::vector<const char*> Args_;
+ std::vector<const char*> Env_;
+ TString ResolvedPath_;
+ TString WorkingDirectory_;
+ bool CreateProcessGroup_ = false;
+ TPromise<void> FinishedPromise_ = NewPromise<void>();
+
+ virtual void DoSpawn() = 0;
+ const char* Capture(TStringBuf arg);
+
+private:
+ void SpawnChild();
+ void ValidateSpawnResult();
+ void Child();
+ void AsyncPeriodicTryWait();
+};
+
+DEFINE_REFCOUNTED_TYPE(TProcessBase)
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Read this
+// http://ewontfix.com/7/
+// before making any changes.
+class TSimpleProcess
+ : public TProcessBase
+{
+public:
+ explicit TSimpleProcess(
+ const TString& path,
+ bool copyEnv = true,
+ TDuration pollPeriod = TDuration::MilliSeconds(100));
+ void Kill(int signal) override;
+ NNet::IConnectionWriterPtr GetStdInWriter() override;
+ NNet::IConnectionReaderPtr GetStdOutReader() override;
+ NNet::IConnectionReaderPtr GetStdErrReader() override;
+
+private:
+ const TDuration PollPeriod_;
+
+ NPipes::TPipeFactory PipeFactory_;
+ std::array<NPipes::TPipe, 3> StdPipes_;
+
+ NConcurrency::TPeriodicExecutorPtr AsyncWaitExecutor_;
+ struct TSpawnAction
+ {
+ std::function<bool()> Callback;
+ TString ErrorMessage;
+ };
+
+ std::vector<TSpawnAction> SpawnActions_;
+
+ void AddDup2FileAction(int oldFD, int newFD);
+ void DoSpawn() override;
+ void SpawnChild();
+ void ValidateSpawnResult();
+ void AsyncPeriodicTryWait();
+ void Child();
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/process/pty.cpp b/yt/yt/library/process/pty.cpp
new file mode 100644
index 0000000000..fc972d38ea
--- /dev/null
+++ b/yt/yt/library/process/pty.cpp
@@ -0,0 +1,64 @@
+#include "pty.h"
+
+#include "io_dispatcher.h"
+
+#include <yt/yt/core/misc/common.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/net/connection.h>
+
+namespace NYT::NPipes {
+
+using namespace NNet;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TPty::TPty(int height, int width)
+{
+ SafeOpenPty(&MasterFD_, &SlaveFD_, height, width);
+}
+
+TPty::~TPty()
+{
+ if (MasterFD_ != InvalidFD) {
+ YT_VERIFY(TryClose(MasterFD_, false));
+ }
+
+ if (SlaveFD_ != InvalidFD) {
+ YT_VERIFY(TryClose(SlaveFD_, false));
+ }
+}
+
+IConnectionWriterPtr TPty::CreateMasterAsyncWriter()
+{
+ YT_VERIFY(MasterFD_ != InvalidFD);
+ int fd = SafeDup(MasterFD_);
+ SafeSetCloexec(fd);
+ SafeMakeNonblocking(fd);
+ return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller());
+}
+
+IConnectionReaderPtr TPty::CreateMasterAsyncReader()
+{
+ YT_VERIFY(MasterFD_ != InvalidFD);
+ int fd = SafeDup(MasterFD_);
+ SafeSetCloexec(fd);
+ SafeMakeNonblocking(fd);
+ return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller());
+}
+
+int TPty::GetMasterFD() const
+{
+ YT_VERIFY(MasterFD_ != InvalidFD);
+ return MasterFD_;
+}
+
+int TPty::GetSlaveFD() const
+{
+ YT_VERIFY(SlaveFD_ != InvalidFD);
+ return SlaveFD_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/pty.h b/yt/yt/library/process/pty.h
new file mode 100644
index 0000000000..b585782d12
--- /dev/null
+++ b/yt/yt/library/process/pty.h
@@ -0,0 +1,33 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/net/public.h>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPty
+ : public TNonCopyable
+{
+public:
+ static const int InvalidFD = -1;
+
+ TPty(int height, int width);
+ ~TPty();
+
+ NNet::IConnectionReaderPtr CreateMasterAsyncReader();
+ NNet::IConnectionWriterPtr CreateMasterAsyncWriter();
+
+ int GetMasterFD() const;
+ int GetSlaveFD() const;
+
+private:
+ int MasterFD_ = InvalidFD;
+ int SlaveFD_ = InvalidFD;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/public.h b/yt/yt/library/process/public.h
new file mode 100644
index 0000000000..0fa1d3d0a9
--- /dev/null
+++ b/yt/yt/library/process/public.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_CLASS(TNamedPipe)
+DECLARE_REFCOUNTED_CLASS(TNamedPipeConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/subprocess.cpp b/yt/yt/library/process/subprocess.cpp
new file mode 100644
index 0000000000..02555b0c9b
--- /dev/null
+++ b/yt/yt/library/process/subprocess.cpp
@@ -0,0 +1,153 @@
+#include "subprocess.h"
+
+#include <yt/yt/core/misc/blob.h>
+#include <yt/yt/core/misc/proc.h>
+#include <yt/yt/core/misc/finally.h>
+
+#include <yt/yt/core/logging/log.h>
+
+#include <yt/yt/core/net/connection.h>
+
+#include <util/system/execpath.h>
+
+#include <array>
+
+namespace NYT {
+
+using namespace NConcurrency;
+using namespace NPipes;
+
+////////////////////////////////////////////////////////////////////////////////
+
+const static size_t PipeBlockSize = 64 * 1024;
+static NLogging::TLogger Logger("Subprocess");
+
+////////////////////////////////////////////////////////////////////////////////
+
+TSubprocess::TSubprocess(const TString& path, bool copyEnv)
+ : Process_(New<TSimpleProcess>(path, copyEnv))
+{ }
+
+TSubprocess TSubprocess::CreateCurrentProcessSpawner()
+{
+ return TSubprocess(GetExecPath());
+}
+
+void TSubprocess::AddArgument(TStringBuf arg)
+{
+ Process_->AddArgument(arg);
+}
+
+void TSubprocess::AddArguments(std::initializer_list<TStringBuf> args)
+{
+ Process_->AddArguments(args);
+}
+
+TSubprocessResult TSubprocess::Execute(const TSharedRef& input)
+{
+#ifdef _unix_
+ auto inputStream = Process_->GetStdInWriter();
+ auto outputStream = Process_->GetStdOutReader();
+ auto errorStream = Process_->GetStdErrReader();
+ auto finished = Process_->Spawn();
+
+ auto readIntoBlob = [] (IAsyncInputStreamPtr stream) {
+ TBlob output;
+ auto buffer = TSharedMutableRef::Allocate(PipeBlockSize, {.InitializeStorage = false});
+ while (true) {
+ auto size = WaitFor(stream->Read(buffer))
+ .ValueOrThrow();
+
+ if (size == 0)
+ break;
+
+ // ToDo(psushin): eliminate copying.
+ output.Append(buffer.Begin(), size);
+ }
+ return TSharedRef::FromBlob(std::move(output));
+ };
+
+ auto writeStdin = BIND([=] {
+ if (input.Size() > 0) {
+ WaitFor(inputStream->Write(input))
+ .ThrowOnError();
+ }
+
+ WaitFor(inputStream->Close())
+ .ThrowOnError();
+
+ //! Return dummy ref, so later we cat put Future into vector
+ //! along with stdout and stderr.
+ return TSharedRef::MakeEmpty();
+ });
+
+ std::vector<TFuture<TSharedRef>> futures = {
+ BIND(readIntoBlob, outputStream).AsyncVia(GetCurrentInvoker()).Run(),
+ BIND(readIntoBlob, errorStream).AsyncVia(GetCurrentInvoker()).Run(),
+ writeStdin.AsyncVia(GetCurrentInvoker()).Run(),
+ };
+
+ try {
+ auto outputsOrError = WaitFor(AllSucceeded(futures));
+ THROW_ERROR_EXCEPTION_IF_FAILED(
+ outputsOrError,
+ "IO error occurred during subprocess call");
+
+ const auto& outputs = outputsOrError.Value();
+ YT_VERIFY(outputs.size() == 3);
+
+ // This can block indefinitely.
+ auto exitCode = WaitFor(finished);
+ return TSubprocessResult{outputs[0], outputs[1], exitCode};
+ } catch (...) {
+ try {
+ Process_->Kill(SIGKILL);
+ } catch (...) { }
+ Y_UNUSED(WaitFor(finished));
+ throw;
+ }
+#else
+ THROW_ERROR_EXCEPTION("Unsupported platform");
+#endif
+}
+
+void TSubprocess::Kill(int signal)
+{
+ Process_->Kill(signal);
+}
+
+TString TSubprocess::GetCommandLine() const
+{
+ return Process_->GetCommandLine();
+}
+
+TProcessBasePtr TSubprocess::GetProcess() const
+{
+ return Process_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void RunSubprocess(const std::vector<TString>& cmd)
+{
+ if (cmd.empty()) {
+ THROW_ERROR_EXCEPTION("Command can't be empty");
+ }
+
+ auto process = TSubprocess(cmd[0]);
+ for (int index = 1; index < std::ssize(cmd); ++index) {
+ process.AddArgument(cmd[index]);
+ }
+
+ auto result = process.Execute();
+ if (!result.Status.IsOK()) {
+ THROW_ERROR_EXCEPTION("Failed to run %v", cmd[0])
+ << result.Status
+ << TErrorAttribute("command_line", process.GetCommandLine())
+ << TErrorAttribute("error", TString(result.Error.Begin(), result.Error.End()));
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/process/subprocess.h b/yt/yt/library/process/subprocess.h
new file mode 100644
index 0000000000..223db533f6
--- /dev/null
+++ b/yt/yt/library/process/subprocess.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include "public.h"
+#include "process.h"
+
+#include <library/cpp/yt/memory/ref.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TSubprocessResult
+{
+ TSharedRef Output;
+ TSharedRef Error;
+ TError Status;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSubprocess
+{
+public:
+ explicit TSubprocess(const TString& path, bool copyEnv = true);
+
+ static TSubprocess CreateCurrentProcessSpawner();
+
+ void AddArgument(TStringBuf arg);
+ void AddArguments(std::initializer_list<TStringBuf> args);
+
+ TSubprocessResult Execute(const TSharedRef& input = TSharedRef::MakeEmpty());
+ void Kill(int signal);
+
+ TString GetCommandLine() const;
+
+ TProcessBasePtr GetProcess() const;
+
+private:
+ const TProcessBasePtr Process_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+void RunSubprocess(const std::vector<TString>& cmd);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/process/unittests/pipes_ut.cpp b/yt/yt/library/process/unittests/pipes_ut.cpp
new file mode 100644
index 0000000000..0c7e2a0cf2
--- /dev/null
+++ b/yt/yt/library/process/unittests/pipes_ut.cpp
@@ -0,0 +1,319 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/misc/blob.h>
+#include <yt/yt/core/misc/proc.h>
+
+#include <yt/yt/core/net/connection.h>
+
+#include <yt/yt/library/process/pipe.h>
+
+#include <random>
+
+namespace NYT::NPipes {
+
+////////////////////////////////////////////////////////////////////////////////
+
+using namespace NConcurrency;
+using namespace NNet;
+
+#ifndef _win_
+
+TEST(TPipeIOHolder, CanInstantiate)
+{
+ auto pipe = TPipeFactory().Create();
+
+ auto readerHolder = pipe.CreateAsyncReader();
+ auto writerHolder = pipe.CreateAsyncWriter();
+
+ readerHolder->Abort().Get();
+ writerHolder->Abort().Get();
+}
+
+TEST(TPipeTest, PrematureEOF)
+{
+ auto pipe = TNamedPipe::Create("./namedpipe");
+ auto reader = pipe->CreateAsyncReader();
+
+ auto buffer = TSharedMutableRef::Allocate(1024 * 1024);
+ EXPECT_THROW(reader->Read(buffer).WithTimeout(TDuration::Seconds(1)).Get().ValueOrThrow(), TErrorException);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TBlob ReadAll(IConnectionReaderPtr reader, bool useWaitFor)
+{
+ auto buffer = TSharedMutableRef::Allocate(1_MB, {.InitializeStorage = false});
+ auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>());
+
+ while (true) {
+ TErrorOr<size_t> result;
+ auto future = reader->Read(buffer);
+ if (useWaitFor) {
+ result = WaitFor(future);
+ } else {
+ result = future.Get();
+ }
+
+ if (result.ValueOrThrow() == 0) {
+ break;
+ }
+
+ whole.Append(buffer.Begin(), result.Value());
+ }
+ return whole;
+}
+
+TEST(TAsyncWriterTest, AsyncCloseFail)
+{
+ auto pipe = TPipeFactory().Create();
+
+ auto reader = pipe.CreateAsyncReader();
+ auto writer = pipe.CreateAsyncWriter();
+
+ auto queue = New<NConcurrency::TActionQueue>();
+ auto readFromPipe =
+ BIND(&ReadAll, reader, false)
+ .AsyncVia(queue->GetInvoker())
+ .Run();
+
+ int length = 200*1024;
+ auto buffer = TSharedMutableRef::Allocate(length);
+ ::memset(buffer.Begin(), 'a', buffer.Size());
+
+ auto writeResult = writer->Write(buffer).Get();
+
+ EXPECT_TRUE(writeResult.IsOK())
+ << ToString(writeResult);
+
+ auto error = writer->Close();
+
+ auto readResult = readFromPipe.Get();
+ ASSERT_TRUE(readResult.IsOK())
+ << ToString(readResult);
+
+ auto closeStatus = error.Get();
+}
+
+TEST(TAsyncWriterTest, WriteFailed)
+{
+ auto pipe = TPipeFactory().Create();
+ auto reader = pipe.CreateAsyncReader();
+ auto writer = pipe.CreateAsyncWriter();
+
+ int length = 200*1024;
+ auto buffer = TSharedMutableRef::Allocate(length);
+ ::memset(buffer.Begin(), 'a', buffer.Size());
+
+ auto asyncWriteResult = writer->Write(buffer);
+ reader->Abort();
+
+ EXPECT_FALSE(asyncWriteResult.Get().IsOK())
+ << ToString(asyncWriteResult.Get());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPipeReadWriteTest
+ : public ::testing::Test
+{
+protected:
+ void SetUp() override
+ {
+ auto pipe = TPipeFactory().Create();
+
+ Reader = pipe.CreateAsyncReader();
+ Writer = pipe.CreateAsyncWriter();
+ }
+
+ void TearDown() override
+ { }
+
+ IConnectionReaderPtr Reader;
+ IConnectionWriterPtr Writer;
+};
+
+class TNamedPipeReadWriteTest
+ : public ::testing::Test
+{
+protected:
+ void SetUp() override
+ {
+ auto pipe = TNamedPipe::Create("./namedpipe");
+ Reader = pipe->CreateAsyncReader();
+ Writer = pipe->CreateAsyncWriter();
+ }
+
+ void TearDown() override
+ { }
+
+ IConnectionReaderPtr Reader;
+ IConnectionWriterPtr Writer;
+};
+
+TEST_F(TPipeReadWriteTest, ReadSomethingSpin)
+{
+ TString message("Hello pipe!\n");
+ auto buffer = TSharedRef::FromString(message);
+ Writer->Write(buffer).Get().ThrowOnError();
+ Writer->Close().Get().ThrowOnError();
+
+ auto data = TSharedMutableRef::Allocate(1);
+ auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>());
+
+ while (true) {
+ auto result = Reader->Read(data).Get();
+ if (result.ValueOrThrow() == 0) {
+ break;
+ }
+ whole.Append(data.Begin(), result.Value());
+ }
+
+ EXPECT_EQ(message, TString(whole.Begin(), whole.End()));
+}
+
+TEST_F(TNamedPipeReadWriteTest, ReadSomethingSpin)
+{
+ TString message("Hello pipe!\n");
+ auto buffer = TSharedRef::FromString(message);
+
+ Writer->Write(buffer).Get().ThrowOnError();
+ Writer->Close().Get().ThrowOnError();
+
+ auto data = TSharedMutableRef::Allocate(1);
+ auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>());
+
+ while (true) {
+ auto result = Reader->Read(data).Get();
+ if (result.ValueOrThrow() == 0) {
+ break;
+ }
+ whole.Append(data.Begin(), result.Value());
+ }
+ EXPECT_EQ(message, TString(whole.Begin(), whole.End()));
+}
+
+
+TEST_F(TPipeReadWriteTest, ReadSomethingWait)
+{
+ TString message("Hello pipe!\n");
+ auto buffer = TSharedRef::FromString(message);
+ EXPECT_TRUE(Writer->Write(buffer).Get().IsOK());
+ WaitFor(Writer->Close())
+ .ThrowOnError();
+ auto whole = ReadAll(Reader, false);
+ EXPECT_EQ(message, TString(whole.Begin(), whole.End()));
+}
+
+TEST_F(TNamedPipeReadWriteTest, ReadSomethingWait)
+{
+ TString message("Hello pipe!\n");
+ auto buffer = TSharedRef::FromString(message);
+ EXPECT_TRUE(Writer->Write(buffer).Get().IsOK());
+ WaitFor(Writer->Close())
+ .ThrowOnError();
+ auto whole = ReadAll(Reader, false);
+ EXPECT_EQ(message, TString(whole.Begin(), whole.End()));
+}
+
+TEST_F(TPipeReadWriteTest, ReadWrite)
+{
+ TString text("Hello cruel world!\n");
+ auto buffer = TSharedRef::FromString(text);
+ Writer->Write(buffer).Get();
+ auto errorsOnClose = Writer->Close();
+
+ auto textFromPipe = ReadAll(Reader, false);
+
+ auto error = errorsOnClose.Get();
+ EXPECT_TRUE(error.IsOK()) << error.GetMessage();
+ EXPECT_EQ(text, TString(textFromPipe.Begin(), textFromPipe.End()));
+}
+
+TEST_F(TNamedPipeReadWriteTest, ReadWrite)
+{
+ TString text("Hello cruel world!\n");
+ auto buffer = TSharedRef::FromString(text);
+ Writer->Write(buffer).Get();
+ auto errorsOnClose = Writer->Close();
+
+ auto textFromPipe = ReadAll(Reader, false);
+
+ auto error = errorsOnClose.Get();
+ EXPECT_TRUE(error.IsOK()) << error.GetMessage();
+ EXPECT_EQ(text, TString(textFromPipe.Begin(), textFromPipe.End()));
+}
+
+void WriteAll(IConnectionWriterPtr writer, const char* data, size_t size, size_t blockSize)
+{
+ while (size > 0) {
+ const size_t currentBlockSize = std::min(blockSize, size);
+ auto buffer = TSharedRef(data, currentBlockSize, nullptr);
+ auto error = WaitFor(writer->Write(buffer));
+ THROW_ERROR_EXCEPTION_IF_FAILED(error);
+ size -= currentBlockSize;
+ data += currentBlockSize;
+ }
+
+ {
+ auto error = WaitFor(writer->Close());
+ THROW_ERROR_EXCEPTION_IF_FAILED(error);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TPipeBigReadWriteTest
+ : public TPipeReadWriteTest
+ , public ::testing::WithParamInterface<std::pair<size_t, size_t>>
+{ };
+
+TEST_P(TPipeBigReadWriteTest, RealReadWrite)
+{
+ size_t dataSize, blockSize;
+ std::tie(dataSize, blockSize) = GetParam();
+
+ auto queue = New<NConcurrency::TActionQueue>();
+
+ std::vector<char> data(dataSize, 'a');
+
+ BIND([&] () {
+ auto dice = std::bind(
+ std::uniform_int_distribution<int>(0, 127),
+ std::default_random_engine());
+ for (size_t i = 0; i < data.size(); ++i) {
+ data[i] = dice();
+ }
+ })
+ .AsyncVia(queue->GetInvoker()).Run();
+
+ auto writeError = BIND(&WriteAll, Writer, data.data(), data.size(), blockSize)
+ .AsyncVia(queue->GetInvoker())
+ .Run();
+ auto readFromPipe = BIND(&ReadAll, Reader, true)
+ .AsyncVia(queue->GetInvoker())
+ .Run();
+
+ auto textFromPipe = readFromPipe.Get().ValueOrThrow();
+ EXPECT_EQ(data.size(), textFromPipe.Size());
+ auto result = std::mismatch(textFromPipe.Begin(), textFromPipe.End(), data.begin());
+ EXPECT_TRUE(std::equal(textFromPipe.Begin(), textFromPipe.End(), data.begin())) <<
+ (result.first - textFromPipe.Begin()) << " " << (int)(*result.first);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ ValueParametrized,
+ TPipeBigReadWriteTest,
+ ::testing::Values(
+ std::make_pair(2000 * 4096, 4096),
+ std::make_pair(100 * 4096, 10000),
+ std::make_pair(100 * 4096, 100),
+ std::make_pair(100, 4096)));
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NPipes
diff --git a/yt/yt/library/process/unittests/process_ut.cpp b/yt/yt/library/process/unittests/process_ut.cpp
new file mode 100644
index 0000000000..55f7fc65c8
--- /dev/null
+++ b/yt/yt/library/process/unittests/process_ut.cpp
@@ -0,0 +1,235 @@
+#include <yt/yt/library/process/process.h>
+
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/core/actions/bind.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/delayed_executor.h>
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/net/connection.h>
+
+#include <library/cpp/yt/system/handle_eintr.h>
+
+namespace NYT {
+namespace {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+#if defined(_unix_) and not defined(_asan_enabled_)
+
+TEST(TProcessTest, Basic)
+{
+ auto p = New<TSimpleProcess>("/bin/ls");
+ TFuture<void> finished;
+
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+}
+
+// NB: We cannot rely on 'ls' and 'sleep' in arcadia tests.
+TEST(TProcessTest, RunFromPathEnv)
+{
+ auto p = New<TSimpleProcess>("/bin/ls", false);
+ TFuture<void> finished;
+
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, PollDuration)
+{
+ auto p = New<TSimpleProcess>("/bin/sleep", true, TDuration::MilliSeconds(1));
+ p->AddArgument("0.1");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, InvalidPath)
+{
+ auto p = New<TSimpleProcess>("/some/bad/path/binary");
+
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_FALSE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_FALSE(p->IsFinished());
+ EXPECT_FALSE(error.IsOK());
+}
+
+TEST(TProcessTest, StdOut)
+{
+ auto p = New<TSimpleProcess>("/bin/date");
+
+ auto outStream = p->GetStdOutReader();
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+
+ auto buffer = TSharedMutableRef::Allocate(4_KB, {.InitializeStorage = false});
+ auto future = outStream->Read(buffer);
+ auto result = WaitFor(future);
+ size_t sz = result.ValueOrThrow();
+ EXPECT_TRUE(sz > 0);
+}
+
+TEST(TSimpleProcess, GetCommandLine1)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ EXPECT_EQ("/bin/bash", p->GetCommandLine());
+ p->AddArgument("-c");
+ EXPECT_EQ("/bin/bash -c", p->GetCommandLine());
+ p->AddArgument("exit 0");
+ EXPECT_EQ("/bin/bash -c \"exit 0\"", p->GetCommandLine());
+}
+
+TEST(TProcessBase, GetCommandLine2)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ EXPECT_EQ("/bin/bash", p->GetCommandLine());
+ p->AddArgument("-c");
+ EXPECT_EQ("/bin/bash -c", p->GetCommandLine());
+ p->AddArgument("\"quoted\"");
+ EXPECT_EQ("/bin/bash -c \"\\\"quoted\\\"\"", p->GetCommandLine());
+}
+
+TEST(TProcessTest, ProcessReturnCode0)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("exit 0");
+
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, ProcessReturnCode123)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("exit 123");
+
+ TFuture<void> finished;
+ ASSERT_NO_THROW(finished = p->Spawn());
+ ASSERT_TRUE(p->IsStarted());
+ auto error = WaitFor(finished);
+ EXPECT_EQ(EProcessErrorCode::NonZeroExitCode, error.GetCode());
+ EXPECT_EQ(123, error.Attributes().Get<int>("exit_code"));
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, Params1)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("if test 3 -gt 1; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_FALSE(error.IsOK());
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, Params2)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("if test 1 -gt 3; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_TRUE(error.IsOK()) << ToString(error);
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, InheritEnvironment)
+{
+ const char* name = "SPAWN_TEST_ENV_VAR";
+ const char* value = "42";
+ setenv(name, value, 1);
+
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("if test $SPAWN_TEST_ENV_VAR = 42; then exit 7; fi");
+
+ auto error = WaitFor(p->Spawn());
+ EXPECT_FALSE(error.IsOK());
+ EXPECT_TRUE(p->IsFinished());
+
+ unsetenv(name);
+}
+
+TEST(TProcessTest, Kill)
+{
+ auto p = New<TSimpleProcess>("/bin/sleep");
+ p->AddArgument("5");
+
+ auto finished = p->Spawn();
+
+ NConcurrency::TDelayedExecutor::Submit(
+ BIND([&] () {
+ p->Kill(SIGKILL);
+ }),
+ TDuration::MilliSeconds(100));
+
+ auto error = WaitFor(finished);
+ EXPECT_FALSE(error.IsOK());
+ EXPECT_TRUE(p->IsFinished());
+}
+
+TEST(TProcessTest, KillFinished)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("true");
+
+ auto finished = p->Spawn();
+
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK());
+
+ p->Kill(SIGKILL);
+}
+
+TEST(TProcessTest, KillZombie)
+{
+ auto p = New<TSimpleProcess>("/bin/bash");
+ p->AddArgument("-c");
+ p->AddArgument("/bin/sleep 1; /bin/true");
+
+ auto finished = p->Spawn();
+
+ siginfo_t infop;
+ auto res = HandleEintr(::waitid, P_PID, p->GetProcessId(), &infop, WEXITED | WNOWAIT);
+ EXPECT_EQ(0, res)
+ << "errno = " << errno;
+ EXPECT_EQ(p->GetProcessId(), infop.si_pid);
+
+ p->Kill(SIGKILL);
+ auto error = WaitFor(finished);
+ EXPECT_TRUE(error.IsOK())
+ << ToString(error);
+}
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/yt/yt/library/process/unittests/subprocess_ut.cpp b/yt/yt/library/process/unittests/subprocess_ut.cpp
new file mode 100644
index 0000000000..932d250784
--- /dev/null
+++ b/yt/yt/library/process/unittests/subprocess_ut.cpp
@@ -0,0 +1,101 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/core/actions/future.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+
+#include <yt/yt/library/process/subprocess.h>
+
+namespace NYT {
+namespace {
+
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+#if defined(_unix_) and not defined(_asan_enabled_)
+
+TEST(TSubprocessTest, Basic)
+{
+ TSubprocess subprocess("/bin/bash");
+
+ subprocess.AddArgument("-c");
+ subprocess.AddArgument("true");
+
+ auto result = subprocess.Execute();
+ EXPECT_TRUE(result.Status.IsOK());
+}
+
+
+TEST(TSubprocessTest, PipeOutput)
+{
+ TSubprocess subprocess("/bin/echo");
+
+ subprocess.AddArgument("hello");
+
+ auto result = subprocess.Execute();
+ EXPECT_TRUE(result.Status.IsOK());
+ TString output(result.Output.Begin(), result.Output.End());
+ EXPECT_TRUE(output == "hello\n") << output;
+}
+
+TEST(TSubprocessTest, PipeStdin)
+{
+ auto queue = New<TActionQueue>();
+
+ BIND([] () {
+ TSubprocess subprocess("/bin/cat");
+ subprocess.AddArgument("-");
+
+ auto input = TString("TEST test TEST");
+ auto inputRef = TSharedRef::FromString(input);
+ auto result = subprocess.Execute(inputRef);
+ EXPECT_TRUE(result.Status.IsOK());
+
+ TString output(result.Output.Begin(), result.Output.End());
+ EXPECT_EQ(input, output);
+ }).AsyncVia(queue->GetInvoker()).Run().Get().ThrowOnError();
+}
+
+TEST(TSubprocessTest, PipeBigOutput)
+{
+ auto queue = New<TActionQueue>();
+
+ auto result = BIND([] () {
+ TSubprocess subprocess("/bin/bash");
+
+ subprocess.AddArgument("-c");
+ subprocess.AddArgument("for i in `/usr/bin/seq 100000`; do echo hello; done; echo world");
+
+ auto result = subprocess.Execute();
+ return result.Status.IsOK();
+ }).AsyncVia(queue->GetInvoker()).Run().Get().Value();
+
+ EXPECT_TRUE(result);
+}
+
+
+TEST(TSubprocessTest, PipeBigError)
+{
+ auto queue = New<TActionQueue>();
+
+ auto result = BIND([] () {
+ TSubprocess subprocess("/bin/bash");
+
+ subprocess.AddArgument("-c");
+ subprocess.AddArgument("for i in `/usr/bin/seq 100000`; do echo hello 1>&2; done; echo world");
+
+ auto result = subprocess.Execute();
+ return result;
+ }).AsyncVia(queue->GetInvoker()).Run().Get().Value();
+
+ EXPECT_TRUE(result.Status.IsOK());
+ EXPECT_EQ(6*100000, std::ssize(result.Error));
+}
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/yt/yt/library/process/unittests/ya.make b/yt/yt/library/process/unittests/ya.make
new file mode 100644
index 0000000000..6e476c5702
--- /dev/null
+++ b/yt/yt/library/process/unittests/ya.make
@@ -0,0 +1,24 @@
+GTEST(unittester-library-process)
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+ALLOCATOR(YT)
+
+SRCS(
+ pipes_ut.cpp
+ process_ut.cpp
+ subprocess_ut.cpp
+)
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc)
+
+PEERDIR(
+ yt/yt/build
+ yt/yt/core
+ yt/yt/core/test_framework
+ yt/yt/library/process
+)
+
+SIZE(MEDIUM)
+
+END()
diff --git a/yt/yt/library/process/ya.make b/yt/yt/library/process/ya.make
new file mode 100644
index 0000000000..79763c7267
--- /dev/null
+++ b/yt/yt/library/process/ya.make
@@ -0,0 +1,22 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ io_dispatcher.cpp
+ pipe.cpp
+ process.cpp
+ pty.cpp
+ subprocess.cpp
+)
+
+PEERDIR(
+ yt/yt/core
+ contrib/libs/re2
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ unittests
+)
diff --git a/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt
index bafd86d310..0f1318b348 100644
--- a/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt
@@ -6,7 +6,10 @@
# original buildsystem will not be accepted.
+add_subdirectory(perf)
add_subdirectory(resource_tracker)
+add_subdirectory(solomon)
+add_subdirectory(tcmalloc)
add_library(yt-library-profiling)
target_compile_options(yt-library-profiling PRIVATE
diff --git a/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt
index e4524762fc..b1ff65e70f 100644
--- a/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt
+++ b/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt
@@ -6,7 +6,10 @@
# original buildsystem will not be accepted.
+add_subdirectory(perf)
add_subdirectory(resource_tracker)
+add_subdirectory(solomon)
+add_subdirectory(tcmalloc)
add_library(yt-library-profiling)
target_compile_options(yt-library-profiling PRIVATE
diff --git a/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt
index e4524762fc..b1ff65e70f 100644
--- a/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt
+++ b/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt
@@ -6,7 +6,10 @@
# original buildsystem will not be accepted.
+add_subdirectory(perf)
add_subdirectory(resource_tracker)
+add_subdirectory(solomon)
+add_subdirectory(tcmalloc)
add_library(yt-library-profiling)
target_compile_options(yt-library-profiling PRIVATE
diff --git a/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt
index 6a6f1a6dde..34e12beaad 100644
--- a/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt
+++ b/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt
@@ -6,7 +6,10 @@
# original buildsystem will not be accepted.
+add_subdirectory(perf)
add_subdirectory(resource_tracker)
+add_subdirectory(solomon)
+add_subdirectory(tcmalloc)
add_library(yt-library-profiling)
target_link_libraries(yt-library-profiling PUBLIC
diff --git a/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..fd65c86abb
--- /dev/null
+++ b/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-perf)
+target_compile_options(library-profiling-perf PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-perf PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+)
+target_sources(library-profiling-perf PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters_other.cpp
+)
diff --git a/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..8615b211d9
--- /dev/null
+++ b/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-perf)
+target_compile_options(library-profiling-perf PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-perf PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+)
+target_sources(library-profiling-perf PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters.cpp
+)
diff --git a/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..8615b211d9
--- /dev/null
+++ b/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-perf)
+target_compile_options(library-profiling-perf PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-perf PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+)
+target_sources(library-profiling-perf PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters.cpp
+)
diff --git a/yt/yt/library/profiling/perf/CMakeLists.txt b/yt/yt/library/profiling/perf/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/profiling/perf/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..f6897f44de
--- /dev/null
+++ b/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-perf)
+target_link_libraries(library-profiling-perf PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+)
+target_sources(library-profiling-perf PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters_other.cpp
+)
diff --git a/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..b4c6b378dd
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,68 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-profiling-solomon)
+target_compile_options(library-profiling-solomon PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-solomon PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+ yt-core-http
+ library-cpp-cgiparam
+ cpp-monlib-metrics
+ monlib-encode-prometheus
+ monlib-encode-spack
+ monlib-encode-json
+ cpp-yt-threading
+ contrib-libs-protobuf
+)
+target_proto_messages(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto
+)
+target_sources(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp
+)
+target_proto_addincls(library-profiling-solomon
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-profiling-solomon
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..57e8fb3c6e
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-profiling-solomon)
+target_compile_options(library-profiling-solomon PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-solomon PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+ yt-core-http
+ library-cpp-cgiparam
+ cpp-monlib-metrics
+ monlib-encode-prometheus
+ monlib-encode-spack
+ monlib-encode-json
+ cpp-yt-threading
+ contrib-libs-protobuf
+)
+target_proto_messages(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto
+)
+target_sources(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp
+)
+target_proto_addincls(library-profiling-solomon
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-profiling-solomon
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..57e8fb3c6e
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-profiling-solomon)
+target_compile_options(library-profiling-solomon PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-solomon PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+ yt-core-http
+ library-cpp-cgiparam
+ cpp-monlib-metrics
+ monlib-encode-prometheus
+ monlib-encode-spack
+ monlib-encode-json
+ cpp-yt-threading
+ contrib-libs-protobuf
+)
+target_proto_messages(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto
+)
+target_sources(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp
+)
+target_proto_addincls(library-profiling-solomon
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-profiling-solomon
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/yt/yt/library/profiling/solomon/CMakeLists.txt b/yt/yt/library/profiling/solomon/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..958f7e64fc
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,65 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-profiling-solomon)
+target_link_libraries(library-profiling-solomon PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ yt-yt-core
+ yt-core-http
+ library-cpp-cgiparam
+ cpp-monlib-metrics
+ monlib-encode-prometheus
+ monlib-encode-spack
+ monlib-encode-json
+ cpp-yt-threading
+ contrib-libs-protobuf
+)
+target_proto_messages(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto
+)
+target_sources(library-profiling-solomon PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp
+)
+target_proto_addincls(library-profiling-solomon
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-profiling-solomon
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..341ff6724c
--- /dev/null
+++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-tcmalloc)
+target_compile_options(library-profiling-tcmalloc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-tcmalloc PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ libs-tcmalloc-malloc_extension
+)
+target_sources(library-profiling-tcmalloc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp
+)
diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..3ebf7b2fd2
--- /dev/null
+++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-tcmalloc)
+target_compile_options(library-profiling-tcmalloc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-tcmalloc PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ libs-tcmalloc-malloc_extension
+)
+target_sources(library-profiling-tcmalloc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp
+)
diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..3ebf7b2fd2
--- /dev/null
+++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-tcmalloc)
+target_compile_options(library-profiling-tcmalloc PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-profiling-tcmalloc PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ libs-tcmalloc-malloc_extension
+)
+target_sources(library-profiling-tcmalloc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp
+)
diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..3fcdf372c8
--- /dev/null
+++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-profiling-tcmalloc)
+target_link_libraries(library-profiling-tcmalloc PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-profiling
+ libs-tcmalloc-malloc_extension
+)
+target_sources(library-profiling-tcmalloc PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp
+)
diff --git a/yt/yt/library/program/CMakeLists.darwin-x86_64.txt b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..fc3e796e79
--- /dev/null
+++ b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,38 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-program)
+target_compile_options(yt-library-program PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-program PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-service_discovery-yp
+ yt-library-monitoring
+ yt-library-containers
+ library-profiling-solomon
+ library-profiling-tcmalloc
+ library-profiling-perf
+ yt-library-ytprof
+ library-tracing-jaeger
+ cpp-yt-mlock
+ cpp-yt-stockpile
+ cpp-yt-string
+)
+target_sources(yt-library-program PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp
+)
diff --git a/yt/yt/library/program/CMakeLists.linux-aarch64.txt b/yt/yt/library/program/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..be31ba10cf
--- /dev/null
+++ b/yt/yt/library/program/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-program)
+target_compile_options(yt-library-program PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-program PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-service_discovery-yp
+ yt-library-monitoring
+ yt-library-containers
+ library-profiling-solomon
+ library-profiling-tcmalloc
+ library-profiling-perf
+ yt-library-ytprof
+ library-tracing-jaeger
+ cpp-yt-mlock
+ cpp-yt-stockpile
+ cpp-yt-string
+)
+target_sources(yt-library-program PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp
+)
diff --git a/yt/yt/library/program/CMakeLists.linux-x86_64.txt b/yt/yt/library/program/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..be31ba10cf
--- /dev/null
+++ b/yt/yt/library/program/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-program)
+target_compile_options(yt-library-program PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(yt-library-program PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-service_discovery-yp
+ yt-library-monitoring
+ yt-library-containers
+ library-profiling-solomon
+ library-profiling-tcmalloc
+ library-profiling-perf
+ yt-library-ytprof
+ library-tracing-jaeger
+ cpp-yt-mlock
+ cpp-yt-stockpile
+ cpp-yt-string
+)
+target_sources(yt-library-program PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp
+)
diff --git a/yt/yt/library/program/CMakeLists.txt b/yt/yt/library/program/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/program/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/program/CMakeLists.windows-x86_64.txt b/yt/yt/library/program/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..1f2aea4bd0
--- /dev/null
+++ b/yt/yt/library/program/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,35 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-library-program)
+target_link_libraries(yt-library-program PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-yt-core
+ core-service_discovery-yp
+ yt-library-monitoring
+ yt-library-containers
+ library-profiling-solomon
+ library-profiling-tcmalloc
+ library-profiling-perf
+ yt-library-ytprof
+ library-tracing-jaeger
+ cpp-yt-mlock
+ cpp-yt-stockpile
+ cpp-yt-string
+)
+target_sources(yt-library-program PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp
+)
diff --git a/yt/yt/library/program/build_attributes.cpp b/yt/yt/library/program/build_attributes.cpp
new file mode 100644
index 0000000000..38caf57997
--- /dev/null
+++ b/yt/yt/library/program/build_attributes.cpp
@@ -0,0 +1,107 @@
+#include "build_attributes.h"
+
+#include <yt/yt/build/build.h>
+
+#include <yt/yt/core/ytree/fluent.h>
+#include <yt/yt/core/ytree/ypath_client.h>
+
+#include <yt/yt/core/misc/error_code.h>
+
+namespace NYT {
+
+using namespace NYTree;
+using namespace NYson;
+
+static const NLogging::TLogger Logger("Build");
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TBuildInfo::Register(TRegistrar registrar)
+{
+ registrar.Parameter("name", &TThis::Name)
+ .Default();
+
+ registrar.Parameter("version", &TThis::Version)
+ .Default(GetVersion());
+
+ registrar.Parameter("build_host", &TThis::BuildHost)
+ .Default(GetBuildHost());
+
+ registrar.Parameter("build_time", &TThis::BuildTime)
+ .Default(ParseBuildTime());
+
+ registrar.Parameter("start_time", &TThis::StartTime)
+ .Default(TInstant::Now());
+}
+
+std::optional<TInstant> TBuildInfo::ParseBuildTime()
+{
+ TString rawBuildTime(GetBuildTime());
+
+ // Build time may be empty if code is building
+ // without -DBUILD_DATE (for example, in opensource build).
+ if (rawBuildTime.empty()) {
+ return std::nullopt;
+ }
+
+ try {
+ return TInstant::ParseIso8601(rawBuildTime);
+ } catch (const std::exception& ex) {
+ YT_LOG_ERROR(ex, "Error parsing build time");
+ return std::nullopt;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TBuildInfoPtr BuildBuildAttributes(const char* serviceName)
+{
+ auto info = New<TBuildInfo>();
+ if (serviceName) {
+ info->Name = serviceName;
+ }
+ return info;
+}
+
+void SetBuildAttributes(IYPathServicePtr orchidRoot, const char* serviceName)
+{
+ SyncYPathSet(
+ orchidRoot,
+ "/service",
+ BuildYsonStringFluently()
+ .BeginAttributes()
+ .Item("opaque").Value(true)
+ .EndAttributes()
+ .Value(BuildBuildAttributes(serviceName)));
+ SyncYPathSet(
+ orchidRoot,
+ "/error_codes",
+ BuildYsonStringFluently()
+ .BeginAttributes()
+ .Item("opaque").Value(true)
+ .EndAttributes()
+ .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodes(), [] (TFluentMap fluent, const auto& pair) {
+ fluent
+ .Item(ToString(pair.first)).BeginMap()
+ .Item("cpp_literal").Value(ToString(pair.second))
+ .EndMap();
+ }));
+ SyncYPathSet(
+ orchidRoot,
+ "/error_code_ranges",
+ BuildYsonStringFluently()
+ .BeginAttributes()
+ .Item("opaque").Value(true)
+ .EndAttributes()
+ .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodeRanges(), [] (TFluentMap fluent, const TErrorCodeRegistry::TErrorCodeRangeInfo& range) {
+ fluent
+ .Item(ToString(range)).BeginMap()
+ .Item("cpp_enum").Value(range.Namespace)
+ .EndMap();
+ }));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
diff --git a/yt/yt/library/program/build_attributes.h b/yt/yt/library/program/build_attributes.h
new file mode 100644
index 0000000000..e02f86b351
--- /dev/null
+++ b/yt/yt/library/program/build_attributes.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/ytree/public.h>
+#include <yt/yt/core/ytree/yson_struct.h>
+
+#include <yt/yt/core/yson/public.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TBuildInfo
+ : public NYTree::TYsonStruct
+{
+public:
+ std::optional<TString> Name;
+ TString Version;
+ TString BuildHost;
+ std::optional<TInstant> BuildTime;
+ TInstant StartTime;
+
+ REGISTER_YSON_STRUCT(TBuildInfo);
+
+ static void Register(TRegistrar registrar);
+
+private:
+ static std::optional<TInstant> ParseBuildTime();
+};
+
+DEFINE_REFCOUNTED_TYPE(TBuildInfo)
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Build build (pun intended) attributes as a TBuildInfo a-la /orchid/service. If service name is not provided,
+//! it is omitted from the result.
+TBuildInfoPtr BuildBuildAttributes(const char* serviceName = nullptr);
+
+void SetBuildAttributes(NYTree::IYPathServicePtr orchidRoot, const char* serviceName);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp
new file mode 100644
index 0000000000..0705fb48fc
--- /dev/null
+++ b/yt/yt/library/program/config.cpp
@@ -0,0 +1,210 @@
+#include "config.h"
+
+namespace NYT {
+
+using namespace NYTree;
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TRpcConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("tracing", &TThis::Tracing)
+ .Default();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void THeapSizeLimit::Register(TRegistrar registrar)
+{
+ registrar.Parameter("container_memory_ratio", &TThis::ContainerMemoryRatio)
+ .Optional();
+ registrar.Parameter("is_hard", &TThis::IsHard)
+ .Default(false);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TTCMallocConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("background_release_rate", &TThis::BackgroundReleaseRate)
+ .Default(32_MB);
+ registrar.Parameter("max_per_cpu_cache_size", &TThis::MaxPerCpuCacheSize)
+ .Default(3_MB);
+
+ registrar.Parameter("aggressive_release_threshold", &TThis::AggressiveReleaseThreshold)
+ .Default(20_GB);
+ registrar.Parameter("aggressive_release_threshold_ratio", &TThis::AggressiveReleaseThresholdRatio)
+ .Optional();
+
+ registrar.Parameter("aggressive_release_size", &TThis::AggressiveReleaseSize)
+ .Default(128_MB);
+ registrar.Parameter("aggressive_release_period", &TThis::AggressiveReleasePeriod)
+ .Default(TDuration::MilliSeconds(100));
+ registrar.Parameter("guarded_sampling_rate", &TThis::GuardedSamplingRate)
+ .Default(128_MB);
+
+ registrar.Parameter("heap_size_limit", &TThis::HeapSizeLimit)
+ .DefaultNew();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TStockpileConfig::Register(TRegistrar registrar)
+{
+ registrar.BaseClassParameter("buffer_size", &TThis::BufferSize)
+ .Default(DefaultBufferSize);
+ registrar.BaseClassParameter("thread_count", &TThis::ThreadCount)
+ .Default(DefaultThreadCount);
+ registrar.BaseClassParameter("period", &TThis::Period)
+ .Default(DefaultPeriod);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void THeapProfilerConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("snapshot_update_period", &TThis::SnapshotUpdatePeriod)
+ .Default(TDuration::Seconds(5));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TSingletonsConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("spin_wait_slow_path_logging_threshold", &TThis::SpinWaitSlowPathLoggingThreshold)
+ .Default(TDuration::MicroSeconds(100));
+ registrar.Parameter("yt_alloc", &TThis::YTAlloc)
+ .DefaultNew();
+ registrar.Parameter("fiber_stack_pool_sizes", &TThis::FiberStackPoolSizes)
+ .Default({});
+ registrar.Parameter("address_resolver", &TThis::AddressResolver)
+ .DefaultNew();
+ registrar.Parameter("tcp_dispatcher", &TThis::TcpDispatcher)
+ .DefaultNew();
+ registrar.Parameter("rpc_dispatcher", &TThis::RpcDispatcher)
+ .DefaultNew();
+ registrar.Parameter("grpc_dispatcher", &TThis::GrpcDispatcher)
+ .DefaultNew();
+ registrar.Parameter("yp_service_discovery", &TThis::YPServiceDiscovery)
+ .DefaultNew();
+ registrar.Parameter("solomon_exporter", &TThis::SolomonExporter)
+ .DefaultNew();
+ registrar.Parameter("logging", &TThis::Logging)
+ .DefaultCtor([] () { return NLogging::TLogManagerConfig::CreateDefault(); });
+ registrar.Parameter("jaeger", &TThis::Jaeger)
+ .DefaultNew();
+ registrar.Parameter("rpc", &TThis::Rpc)
+ .DefaultNew();
+ registrar.Parameter("tcmalloc", &TThis::TCMalloc)
+ .DefaultNew();
+ registrar.Parameter("stockpile", &TThis::Stockpile)
+ .DefaultNew();
+ registrar.Parameter("enable_ref_counted_tracker_profiling", &TThis::EnableRefCountedTrackerProfiling)
+ .Default(true);
+ registrar.Parameter("enable_resource_tracker", &TThis::EnableResourceTracker)
+ .Default(true);
+ registrar.Parameter("enable_porto_resource_tracker", &TThis::EnablePortoResourceTracker)
+ .Default(false);
+ registrar.Parameter("resource_tracker_vcpu_factor", &TThis::ResourceTrackerVCpuFactor)
+ .Optional();
+ registrar.Parameter("pod_spec", &TThis::PodSpec)
+ .DefaultNew();
+ registrar.Parameter("heap_profiler", &TThis::HeapProfiler)
+ .DefaultNew();
+
+ registrar.Postprocessor([] (TThis* config) {
+ if (config->ResourceTrackerVCpuFactor && !config->EnableResourceTracker) {
+ THROW_ERROR_EXCEPTION("Option \"resource_tracker_vcpu_factor\" can be specified only if resource tracker is enabled");
+ }
+ });
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TSingletonsDynamicConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("spin_lock_slow_path_logging_threshold", &TThis::SpinWaitSlowPathLoggingThreshold)
+ .Optional();
+ registrar.Parameter("yt_alloc", &TThis::YTAlloc)
+ .Optional();
+ registrar.Parameter("tcp_dispatcher", &TThis::TcpDispatcher)
+ .DefaultNew();
+ registrar.Parameter("rpc_dispatcher", &TThis::RpcDispatcher)
+ .DefaultNew();
+ registrar.Parameter("logging", &TThis::Logging)
+ .DefaultNew();
+ registrar.Parameter("jaeger", &TThis::Jaeger)
+ .DefaultNew();
+ registrar.Parameter("rpc", &TThis::Rpc)
+ .DefaultNew();
+ registrar.Parameter("tcmalloc", &TThis::TCMalloc)
+ .Optional();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TDiagnosticDumpConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("yt_alloc_dump_period", &TThis::YTAllocDumpPeriod)
+ .Default();
+ registrar.Parameter("ref_counted_tracker_dump_period", &TThis::RefCountedTrackerDumpPeriod)
+ .Default();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void WarnForUnrecognizedOptionsImpl(
+ const NLogging::TLogger& logger,
+ const IMapNodePtr& unrecognized)
+{
+ const auto& Logger = logger;
+ if (unrecognized && unrecognized->GetChildCount() > 0) {
+ YT_LOG_WARNING("Bootstrap config contains unrecognized options (Unrecognized: %v)",
+ ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text));
+ }
+}
+
+void WarnForUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonStructPtr& config)
+{
+ WarnForUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized());
+}
+
+void WarnForUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonSerializablePtr& config)
+{
+ WarnForUnrecognizedOptionsImpl(logger, config->GetUnrecognizedRecursively());
+}
+
+void AbortOnUnrecognizedOptionsImpl(
+ const NLogging::TLogger& logger,
+ const IMapNodePtr& unrecognized)
+{
+ const auto& Logger = logger;
+ if (unrecognized && unrecognized->GetChildCount() > 0) {
+ YT_LOG_ERROR("Bootstrap config contains unrecognized options, terminating (Unrecognized: %v)",
+ ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text));
+ YT_ABORT();
+ }
+}
+
+void AbortOnUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonStructPtr& config)
+{
+ AbortOnUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized());
+}
+
+void AbortOnUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonSerializablePtr& config)
+{
+ AbortOnUnrecognizedOptionsImpl(logger, config->GetUnrecognizedRecursively());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h
new file mode 100644
index 0000000000..7d92939f1c
--- /dev/null
+++ b/yt/yt/library/program/config.h
@@ -0,0 +1,224 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/ytree/yson_serializable.h>
+#include <yt/yt/core/ytree/yson_struct.h>
+
+#include <yt/yt/core/ytalloc/config.h>
+
+#include <yt/yt/core/net/config.h>
+
+#include <yt/yt/core/rpc/config.h>
+#include <yt/yt/core/rpc/grpc/config.h>
+
+#include <yt/yt/core/bus/tcp/config.h>
+
+#include <yt/yt/core/logging/config.h>
+
+#include <yt/yt/core/tracing/config.h>
+
+#include <yt/yt/core/service_discovery/yp/config.h>
+
+#include <yt/yt/library/profiling/solomon/exporter.h>
+
+#include <yt/yt/library/containers/config.h>
+
+#include <yt/yt/library/tracing/jaeger/tracer.h>
+
+#include <library/cpp/yt/stockpile/stockpile.h>
+
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TRpcConfig
+ : public NYTree::TYsonStruct
+{
+public:
+ NTracing::TTracingConfigPtr Tracing;
+
+ REGISTER_YSON_STRUCT(TRpcConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TRpcConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class THeapSizeLimit
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ //! Limit program memory in terms of container memory.
+ // If program heap size exceeds the limit tcmalloc is instructed to release memory to the kernel.
+ std::optional<double> ContainerMemoryRatio;
+
+ //! If true tcmalloc crashes when system allocates more memory than #ContainerMemoryRatio.
+ bool IsHard;
+
+ REGISTER_YSON_STRUCT(THeapSizeLimit);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(THeapSizeLimit)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TTCMallocConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ i64 BackgroundReleaseRate;
+ int MaxPerCpuCacheSize;
+
+ //! Threshold in bytes
+ i64 AggressiveReleaseThreshold;
+
+ //! Threshold in fractions of total memory of the container
+ std::optional<double> AggressiveReleaseThresholdRatio;
+
+ i64 AggressiveReleaseSize;
+ TDuration AggressiveReleasePeriod;
+
+ //! Approximately 1/#GuardedSamplingRate of all allocations of
+ //! size <= 256 KiB will be under GWP-ASAN.
+ std::optional<i64> GuardedSamplingRate;
+
+ THeapSizeLimitPtr HeapSizeLimit;
+
+ REGISTER_YSON_STRUCT(TTCMallocConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TTCMallocConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TStockpileConfig
+ : public TStockpileOptions
+ , public NYTree::TYsonStruct
+{
+public:
+ REGISTER_YSON_STRUCT(TStockpileConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TStockpileConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class THeapProfilerConfig
+ : public NYTree::TYsonStruct
+{
+public:
+ // Period of update snapshot in heap profiler.
+ std::optional<TDuration> SnapshotUpdatePeriod;
+
+ REGISTER_YSON_STRUCT(THeapProfilerConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(THeapProfilerConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSingletonsConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ TDuration SpinWaitSlowPathLoggingThreshold;
+ NYTAlloc::TYTAllocConfigPtr YTAlloc;
+ THashMap<TString, int> FiberStackPoolSizes;
+ NNet::TAddressResolverConfigPtr AddressResolver;
+ NBus::TTcpDispatcherConfigPtr TcpDispatcher;
+ NRpc::TDispatcherConfigPtr RpcDispatcher;
+ NRpc::NGrpc::TDispatcherConfigPtr GrpcDispatcher;
+ NServiceDiscovery::NYP::TServiceDiscoveryConfigPtr YPServiceDiscovery;
+ NProfiling::TSolomonExporterConfigPtr SolomonExporter;
+ NLogging::TLogManagerConfigPtr Logging;
+ NTracing::TJaegerTracerConfigPtr Jaeger;
+ TRpcConfigPtr Rpc;
+ TTCMallocConfigPtr TCMalloc;
+ TStockpileConfigPtr Stockpile;
+ bool EnableRefCountedTrackerProfiling;
+ bool EnableResourceTracker;
+ bool EnablePortoResourceTracker;
+ std::optional<double> ResourceTrackerVCpuFactor;
+ NContainers::TPodSpecConfigPtr PodSpec;
+ THeapProfilerConfigPtr HeapProfiler;
+
+ REGISTER_YSON_STRUCT(TSingletonsConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TSingletonsConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TSingletonsDynamicConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ std::optional<TDuration> SpinWaitSlowPathLoggingThreshold;
+ NYTAlloc::TYTAllocConfigPtr YTAlloc;
+ NBus::TTcpDispatcherDynamicConfigPtr TcpDispatcher;
+ NRpc::TDispatcherDynamicConfigPtr RpcDispatcher;
+ NLogging::TLogManagerDynamicConfigPtr Logging;
+ NTracing::TJaegerTracerDynamicConfigPtr Jaeger;
+ TRpcConfigPtr Rpc;
+ TTCMallocConfigPtr TCMalloc;
+
+ REGISTER_YSON_STRUCT(TSingletonsDynamicConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TSingletonsDynamicConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDiagnosticDumpConfig
+ : public virtual NYTree::TYsonStruct
+{
+public:
+ std::optional<TDuration> YTAllocDumpPeriod;
+ std::optional<TDuration> RefCountedTrackerDumpPeriod;
+
+ REGISTER_YSON_STRUCT(TDiagnosticDumpConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TDiagnosticDumpConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+// NB: These functions should not be called from bootstrap
+// config validator since logger is not set up yet.
+void WarnForUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonStructPtr& config);
+
+void WarnForUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonSerializablePtr& config);
+
+void AbortOnUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonStructPtr& config);
+
+void AbortOnUnrecognizedOptions(
+ const NLogging::TLogger& logger,
+ const NYTree::TYsonSerializablePtr& config);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp
new file mode 100644
index 0000000000..5c7ff29db1
--- /dev/null
+++ b/yt/yt/library/program/helpers.cpp
@@ -0,0 +1,335 @@
+#include "helpers.h"
+#include "config.h"
+#include "private.h"
+
+#include <yt/yt/core/ytalloc/bindings.h>
+
+#include <yt/yt/core/misc/lazy_ptr.h>
+#include <yt/yt/core/misc/ref_counted_tracker.h>
+#include <yt/yt/core/misc/ref_counted_tracker_profiler.h>
+
+#include <yt/yt/core/bus/tcp/dispatcher.h>
+
+#include <yt/yt/library/tracing/jaeger/tracer.h>
+
+#include <yt/yt/library/profiling/perf/counters.h>
+
+#include <yt/yt/library/profiling/resource_tracker/resource_tracker.h>
+
+#include <yt/yt/library/containers/config.h>
+#include <yt/yt/library/containers/porto_resource_tracker.h>
+
+#include <yt/yt/core/logging/log_manager.h>
+
+#include <yt/yt/core/concurrency/execution_stack.h>
+#include <yt/yt/core/concurrency/periodic_executor.h>
+#include <yt/yt/core/concurrency/private.h>
+
+#include <tcmalloc/malloc_extension.h>
+
+#include <yt/yt/core/net/address.h>
+#include <yt/yt/core/net/local_address.h>
+
+#include <yt/yt/core/rpc/dispatcher.h>
+#include <yt/yt/core/rpc/grpc/dispatcher.h>
+
+#include <yt/yt/core/service_discovery/yp/service_discovery.h>
+
+#include <yt/yt/core/threading/spin_wait_slow_path_logger.h>
+
+#include <library/cpp/yt/threading/spin_wait_hook.h>
+
+#include <library/cpp/yt/memory/atomic_intrusive_ptr.h>
+
+#include <util/string/split.h>
+#include <util/system/thread.h>
+
+#include <mutex>
+#include <thread>
+
+namespace NYT {
+
+using namespace NConcurrency;
+using namespace NThreading;
+
+////////////////////////////////////////////////////////////////////////////////
+
+static std::once_flag InitAggressiveReleaseThread;
+static auto& Logger = ProgramLogger;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TCMallocLimitsAdjuster
+{
+public:
+ void Adjust(const TTCMallocConfigPtr& config)
+ {
+ i64 totalMemory = GetContainerMemoryLimit();
+ AdjustPageHeapLimit(totalMemory, config);
+ AdjustAggressiveReleaseThreshold(totalMemory, config);
+ }
+
+ i64 GetAggressiveReleaseThreshold()
+ {
+ return AggressiveReleaseThreshold_;
+ }
+
+private:
+ using TAllocatorMemoryLimit = tcmalloc::MallocExtension::MemoryLimit;
+
+ TAllocatorMemoryLimit AppliedLimit_;
+ i64 AggressiveReleaseThreshold_ = 0;
+
+
+ void AdjustPageHeapLimit(i64 totalMemory, const TTCMallocConfigPtr& config)
+ {
+ auto proposed = ProposeHeapMemoryLimit(totalMemory, config);
+
+ if (proposed.limit == AppliedLimit_.limit && proposed.hard == AppliedLimit_.hard) {
+ // Already applied
+ return;
+ }
+
+ YT_LOG_INFO("Changing tcmalloc memory limit (Limit: %v, IsHard: %v)",
+ proposed.limit,
+ proposed.hard);
+
+ tcmalloc::MallocExtension::SetMemoryLimit(proposed);
+ AppliedLimit_ = proposed;
+ }
+
+ void AdjustAggressiveReleaseThreshold(i64 totalMemory, const TTCMallocConfigPtr& config)
+ {
+ if (totalMemory && config->AggressiveReleaseThresholdRatio) {
+ AggressiveReleaseThreshold_ = *config->AggressiveReleaseThresholdRatio * totalMemory;
+ } else {
+ AggressiveReleaseThreshold_ = config->AggressiveReleaseThreshold;
+ }
+ }
+
+ i64 GetContainerMemoryLimit() const
+ {
+ auto resourceTracker = NProfiling::GetResourceTracker();
+ if (!resourceTracker) {
+ return 0;
+ }
+
+ return resourceTracker->GetTotalMemoryLimit();
+ }
+
+ TAllocatorMemoryLimit ProposeHeapMemoryLimit(i64 totalMemory, const TTCMallocConfigPtr& config) const
+ {
+ const auto& heapLimitConfig = config->HeapSizeLimit;
+
+ if (totalMemory == 0 || !heapLimitConfig->ContainerMemoryRatio) {
+ return {};
+ }
+
+ TAllocatorMemoryLimit proposed;
+ proposed.limit = *heapLimitConfig->ContainerMemoryRatio * totalMemory;
+ proposed.hard = heapLimitConfig->IsHard;
+
+ return proposed;
+ }
+};
+
+void ConfigureTCMalloc(const TTCMallocConfigPtr& config)
+{
+ tcmalloc::MallocExtension::SetBackgroundReleaseRate(
+ tcmalloc::MallocExtension::BytesPerSecond{static_cast<size_t>(config->BackgroundReleaseRate)});
+
+ tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(config->MaxPerCpuCacheSize);
+
+ if (config->GuardedSamplingRate) {
+ tcmalloc::MallocExtension::SetGuardedSamplingRate(*config->GuardedSamplingRate);
+ tcmalloc::MallocExtension::ActivateGuardedSampling();
+ }
+
+ struct TConfigSingleton
+ {
+ TAtomicIntrusivePtr<TTCMallocConfig> Config;
+ };
+
+ LeakySingleton<TConfigSingleton>()->Config.Store(config);
+
+ if (tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) {
+ std::call_once(InitAggressiveReleaseThread, [] {
+ std::thread([] {
+ ::TThread::SetCurrentThreadName("TCAllocYT");
+
+ TCMallocLimitsAdjuster limitsAdjuster;
+
+ while (true) {
+ auto config = LeakySingleton<TConfigSingleton>()->Config.Acquire();
+ limitsAdjuster.Adjust(config);
+
+ auto freeBytes = tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.page_heap_free");
+ YT_VERIFY(freeBytes);
+
+ if (static_cast<i64>(*freeBytes) > limitsAdjuster.GetAggressiveReleaseThreshold()) {
+
+ YT_LOG_DEBUG("Aggressively releasing memory (FreeBytes: %v, Threshold: %v)",
+ static_cast<i64>(*freeBytes),
+ limitsAdjuster.GetAggressiveReleaseThreshold());
+
+ tcmalloc::MallocExtension::ReleaseMemoryToSystem(config->AggressiveReleaseSize);
+ }
+
+ Sleep(config->AggressiveReleasePeriod);
+ }
+ }).detach();
+ });
+ }
+}
+
+template <class TConfig>
+void ConfigureSingletonsImpl(const TConfig& config)
+{
+ SetSpinWaitSlowPathLoggingThreshold(config->SpinWaitSlowPathLoggingThreshold);
+
+ if (!NYTAlloc::ConfigureFromEnv()) {
+ NYTAlloc::Configure(config->YTAlloc);
+ }
+
+ for (const auto& [kind, size] : config->FiberStackPoolSizes) {
+ NConcurrency::SetFiberStackPoolSize(ParseEnum<NConcurrency::EExecutionStackKind>(kind), size);
+ }
+
+ NLogging::TLogManager::Get()->EnableReopenOnSighup();
+ if (!NLogging::TLogManager::Get()->IsConfiguredFromEnv()) {
+ NLogging::TLogManager::Get()->Configure(config->Logging);
+ }
+
+ NNet::TAddressResolver::Get()->Configure(config->AddressResolver);
+ // By default, server components must have a reasonable FQDN.
+ // Failure to do so may result in issues like YT-4561.
+ NNet::TAddressResolver::Get()->EnsureLocalHostName();
+
+ NBus::TTcpDispatcher::Get()->Configure(config->TcpDispatcher);
+
+ NRpc::TDispatcher::Get()->Configure(config->RpcDispatcher);
+
+ NRpc::NGrpc::TDispatcher::Get()->Configure(config->GrpcDispatcher);
+
+ NRpc::TDispatcher::Get()->SetServiceDiscovery(
+ NServiceDiscovery::NYP::CreateServiceDiscovery(config->YPServiceDiscovery));
+
+ NTracing::SetGlobalTracer(New<NTracing::TJaegerTracer>(config->Jaeger));
+
+ NProfiling::EnablePerfCounters();
+
+ if (auto tracingConfig = config->Rpc->Tracing) {
+ NTracing::SetTracingConfig(tracingConfig);
+ }
+
+ ConfigureTCMalloc(config->TCMalloc);
+
+ ConfigureStockpile(*config->Stockpile);
+
+ if (config->EnableRefCountedTrackerProfiling) {
+ EnableRefCountedTrackerProfiling();
+ }
+
+ if (config->EnableResourceTracker) {
+ NProfiling::EnableResourceTracker();
+ if (config->ResourceTrackerVCpuFactor.has_value()) {
+ NProfiling::SetVCpuFactor(config->ResourceTrackerVCpuFactor.value());
+ }
+ }
+
+ if (config->EnablePortoResourceTracker) {
+ NContainers::EnablePortoResourceTracker(config->PodSpec);
+ }
+}
+
+void ConfigureSingletons(const TSingletonsConfigPtr& config)
+{
+ ConfigureSingletonsImpl(config);
+}
+
+template <class TStaticConfig, class TDynamicConfig>
+void ReconfigureSingletonsImpl(const TStaticConfig& config, const TDynamicConfig& dynamicConfig)
+{
+ SetSpinWaitSlowPathLoggingThreshold(dynamicConfig->SpinWaitSlowPathLoggingThreshold.value_or(config->SpinWaitSlowPathLoggingThreshold));
+
+ if (!NYTAlloc::IsConfiguredFromEnv()) {
+ NYTAlloc::Configure(dynamicConfig->YTAlloc ? dynamicConfig->YTAlloc : config->YTAlloc);
+ }
+
+ if (!NLogging::TLogManager::Get()->IsConfiguredFromEnv()) {
+ NLogging::TLogManager::Get()->Configure(
+ config->Logging->ApplyDynamic(dynamicConfig->Logging),
+ /*sync*/ false);
+ }
+
+ auto tracer = NTracing::GetGlobalTracer();
+ if (auto jaeger = DynamicPointerCast<NTracing::TJaegerTracer>(tracer); jaeger) {
+ jaeger->Configure(config->Jaeger->ApplyDynamic(dynamicConfig->Jaeger));
+ }
+
+ NBus::TTcpDispatcher::Get()->Configure(config->TcpDispatcher->ApplyDynamic(dynamicConfig->TcpDispatcher));
+
+ NRpc::TDispatcher::Get()->Configure(config->RpcDispatcher->ApplyDynamic(dynamicConfig->RpcDispatcher));
+
+ if (dynamicConfig->Rpc->Tracing) {
+ NTracing::SetTracingConfig(dynamicConfig->Rpc->Tracing);
+ } else if (config->Rpc->Tracing) {
+ NTracing::SetTracingConfig(config->Rpc->Tracing);
+ }
+
+ if (dynamicConfig->TCMalloc) {
+ ConfigureTCMalloc(dynamicConfig->TCMalloc);
+ } else if (config->TCMalloc) {
+ ConfigureTCMalloc(config->TCMalloc);
+ }
+}
+
+void ReconfigureSingletons(const TSingletonsConfigPtr& config, const TSingletonsDynamicConfigPtr& dynamicConfig)
+{
+ ReconfigureSingletonsImpl(config, dynamicConfig);
+}
+
+template <class TConfig>
+void StartDiagnosticDumpImpl(const TConfig& config)
+{
+ static NLogging::TLogger Logger("DiagDump");
+
+ auto logDumpString = [&] (TStringBuf banner, const TString& str) {
+ for (const auto& line : StringSplitter(str).Split('\n')) {
+ YT_LOG_DEBUG("%v %v", banner, line.Token());
+ }
+ };
+
+ if (config->YTAllocDumpPeriod) {
+ static const TLazyIntrusivePtr<TPeriodicExecutor> Executor(BIND([&] {
+ return New<TPeriodicExecutor>(
+ NRpc::TDispatcher::Get()->GetHeavyInvoker(),
+ BIND([&] {
+ logDumpString("YTAlloc", NYTAlloc::FormatAllocationCounters());
+ }));
+ }));
+ Executor->SetPeriod(config->YTAllocDumpPeriod);
+ Executor->Start();
+ }
+
+ if (config->RefCountedTrackerDumpPeriod) {
+ static const TLazyIntrusivePtr<TPeriodicExecutor> Executor(BIND([&] {
+ return New<TPeriodicExecutor>(
+ NRpc::TDispatcher::Get()->GetHeavyInvoker(),
+ BIND([&] {
+ logDumpString("RCT", TRefCountedTracker::Get()->GetDebugInfo());
+ }));
+ }));
+ Executor->SetPeriod(config->RefCountedTrackerDumpPeriod);
+ Executor->Start();
+ }
+}
+
+void StartDiagnosticDump(const TDiagnosticDumpConfigPtr& config)
+{
+ StartDiagnosticDumpImpl(config);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/helpers.h b/yt/yt/library/program/helpers.h
new file mode 100644
index 0000000000..be09ec889c
--- /dev/null
+++ b/yt/yt/library/program/helpers.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include "public.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void ConfigureSingletons(const TSingletonsConfigPtr& config);
+void ReconfigureSingletons(
+ const TSingletonsConfigPtr& config,
+ const TSingletonsDynamicConfigPtr& dynamicConfig);
+
+void StartDiagnosticDump(const TDiagnosticDumpConfigPtr& config);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/private.h b/yt/yt/library/program/private.h
new file mode 100644
index 0000000000..e328f30667
--- /dev/null
+++ b/yt/yt/library/program/private.h
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "public.h"
+
+#include <yt/yt/core/logging/log.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline const NLogging::TLogger ProgramLogger("Program");
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program.cpp b/yt/yt/library/program/program.cpp
new file mode 100644
index 0000000000..318d998b3a
--- /dev/null
+++ b/yt/yt/library/program/program.cpp
@@ -0,0 +1,385 @@
+#include "program.h"
+
+#include "build_attributes.h"
+
+#include <yt/yt/build/build.h>
+
+#include <yt/yt/core/misc/crash_handler.h>
+#include <yt/yt/core/misc/signal_registry.h>
+#include <yt/yt/core/misc/fs.h>
+#include <yt/yt/core/misc/shutdown.h>
+
+#include <yt/yt/core/ytalloc/bindings.h>
+
+#include <yt/yt/core/yson/writer.h>
+#include <yt/yt/core/yson/null_consumer.h>
+
+#include <yt/yt/core/logging/log_manager.h>
+
+#include <yt/yt/library/ytprof/heap_profiler.h>
+
+#include <yt/yt/library/profiling/tcmalloc/profiler.h>
+
+#include <library/cpp/ytalloc/api/ytalloc.h>
+
+#include <library/cpp/yt/mlock/mlock.h>
+#include <library/cpp/yt/stockpile/stockpile.h>
+
+#include <tcmalloc/malloc_extension.h>
+
+#include <absl/debugging/stacktrace.h>
+
+#include <util/system/thread.h>
+#include <util/system/sigset.h>
+
+#include <util/string/subst.h>
+
+#include <thread>
+
+#include <stdlib.h>
+
+#ifdef _unix_
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#endif
+
+#ifdef _linux_
+#include <grp.h>
+#include <sys/prctl.h>
+#endif
+
+#if defined(_linux_) && defined(CLANG_COVERAGE)
+extern "C" int __llvm_profile_write_file(void);
+extern "C" void __llvm_profile_set_filename(const char* name);
+#endif
+
+namespace NYT {
+
+using namespace NYson;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TProgram::TOptsParseResult
+ : public NLastGetopt::TOptsParseResult
+{
+public:
+ TOptsParseResult(TProgram* owner, int argc, const char** argv)
+ : Owner_(owner)
+ {
+ Init(&Owner_->Opts_, argc, argv);
+ }
+
+ void HandleError() const override
+ {
+ Owner_->OnError(CurrentExceptionMessage());
+ Cerr << Endl << "Try running '" << Owner_->Argv0_ << " --help' for more information." << Endl;
+ Owner_->Exit(EProgramExitCode::OptionsError);
+ }
+
+private:
+ TProgram* const Owner_;
+};
+
+TProgram::TProgram()
+{
+ Opts_.AddHelpOption();
+ Opts_.AddLongOption("yt-version", "print YT version and exit")
+ .NoArgument()
+ .StoreValue(&PrintYTVersion_, true);
+ Opts_.AddLongOption("version", "print version and exit")
+ .NoArgument()
+ .StoreValue(&PrintVersion_, true);
+ Opts_.AddLongOption("yson", "print build information in YSON")
+ .NoArgument()
+ .StoreValue(&UseYson_, true);
+ Opts_.AddLongOption("build", "print build information and exit")
+ .NoArgument()
+ .StoreValue(&PrintBuild_, true);
+ Opts_.SetFreeArgsNum(0);
+
+ ConfigureCoverageOutput();
+}
+
+void TProgram::SetCrashOnError()
+{
+ CrashOnError_ = true;
+}
+
+TProgram::~TProgram() = default;
+
+void TProgram::HandleVersionAndBuild()
+{
+ if (PrintVersion_) {
+ PrintVersionAndExit();
+ }
+ if (PrintYTVersion_) {
+ PrintYTVersionAndExit();
+ }
+ if (PrintBuild_) {
+ PrintBuildAndExit();
+ }
+}
+
+int TProgram::Run(int argc, const char** argv)
+{
+ ::srand(time(nullptr));
+
+ auto run = [&] {
+ Argv0_ = TString(argv[0]);
+ TOptsParseResult result(this, argc, argv);
+
+ HandleVersionAndBuild();
+
+ DoRun(result);
+ };
+
+ if (!CrashOnError_) {
+ try {
+ run();
+ Exit(EProgramExitCode::OK);
+ } catch (...) {
+ OnError(CurrentExceptionMessage());
+ Exit(EProgramExitCode::ProgramError);
+ }
+ } else {
+ run();
+ Exit(EProgramExitCode::OK);
+ }
+
+ // Cannot reach this due to #Exit calls above.
+ YT_ABORT();
+}
+
+void TProgram::Abort(EProgramExitCode code) noexcept
+{
+ Abort(static_cast<int>(code));
+}
+
+void TProgram::Abort(int code) noexcept
+{
+ NLogging::TLogManager::Get()->Shutdown();
+
+ ::_exit(code);
+}
+
+void TProgram::Exit(EProgramExitCode code) noexcept
+{
+ Exit(static_cast<int>(code));
+}
+
+void TProgram::Exit(int code) noexcept
+{
+#if defined(_linux_) && defined(CLANG_COVERAGE)
+ __llvm_profile_write_file();
+#endif
+
+ // This explicit call may become obsolete some day;
+ // cf. the comment section for NYT::Shutdown.
+ Shutdown({
+ .AbortOnHang = ShouldAbortOnHungShutdown(),
+ .HungExitCode = code
+ });
+
+ ::exit(code);
+}
+
+bool TProgram::ShouldAbortOnHungShutdown() noexcept
+{
+ return true;
+}
+
+void TProgram::OnError(const TString& message) noexcept
+{
+ try {
+ Cerr << message << Endl;
+ } catch (...) {
+ // Just ignore it; STDERR might be closed already,
+ // and write() would result in EPIPE.
+ }
+}
+
+void TProgram::PrintYTVersionAndExit()
+{
+ if (UseYson_) {
+ THROW_ERROR_EXCEPTION("--yson is not supported when printing version");
+ }
+ Cout << GetVersion() << Endl;
+ Exit(0);
+}
+
+void TProgram::PrintBuildAndExit()
+{
+ if (UseYson_) {
+ TYsonWriter writer(&Cout, EYsonFormat::Pretty);
+ Serialize(BuildBuildAttributes(), &writer);
+ Cout << Endl;
+ } else {
+ Cout << "Build Time: " << GetBuildTime() << Endl;
+ Cout << "Build Host: " << GetBuildHost() << Endl;
+ }
+ Exit(0);
+}
+
+void TProgram::PrintVersionAndExit()
+{
+ PrintYTVersionAndExit();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TProgramException::TProgramException(TString what)
+ : What_(std::move(what))
+{ }
+
+const char* TProgramException::what() const noexcept
+{
+ return What_.c_str();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString CheckPathExistsArgMapper(const TString& arg)
+{
+ if (!NFS::Exists(arg)) {
+ throw TProgramException(Format("File %v does not exist", arg));
+ }
+ return arg;
+}
+
+TGuid CheckGuidArgMapper(const TString& arg)
+{
+ TGuid result;
+ if (!TGuid::FromString(arg, &result)) {
+ throw TProgramException(Format("Error parsing guid %Qv", arg));
+ }
+ return result;
+}
+
+NYson::TYsonString CheckYsonArgMapper(const TString& arg)
+{
+ ParseYsonStringBuffer(arg, EYsonType::Node, GetNullYsonConsumer());
+ return NYson::TYsonString(arg);
+}
+
+void ConfigureUids()
+{
+#ifdef _unix_
+ uid_t ruid, euid;
+#ifdef _linux_
+ uid_t suid;
+ YT_VERIFY(getresuid(&ruid, &euid, &suid) == 0);
+#else
+ ruid = getuid();
+ euid = geteuid();
+#endif
+ if (euid == 0) {
+ // if real uid is already root do not set root as supplementary ids.
+ if (ruid != 0) {
+ YT_VERIFY(setgroups(0, nullptr) == 0);
+ }
+ // if effective uid == 0 (e. g. set-uid-root), alter saved = effective, effective = real.
+#ifdef _linux_
+ YT_VERIFY(setresuid(ruid, ruid, euid) == 0);
+ // Make server suid_dumpable = 1.
+ YT_VERIFY(prctl(PR_SET_DUMPABLE, 1) == 0);
+#else
+ YT_VERIFY(setuid(euid) == 0);
+ YT_VERIFY(seteuid(ruid) == 0);
+ YT_VERIFY(setruid(ruid) == 0);
+#endif
+ }
+ umask(0000);
+#endif
+}
+
+void ConfigureCoverageOutput()
+{
+#if defined(_linux_) && defined(CLANG_COVERAGE)
+ // YT tests use pid namespaces. We can't use process id as unique identifier for output file.
+ if (auto profileFile = getenv("LLVM_PROFILE_FILE")) {
+ TString fixedProfile{profileFile};
+ SubstGlobal(fixedProfile, "%e", "ytserver-all");
+ SubstGlobal(fixedProfile, "%p", ToString(TInstant::Now().NanoSeconds()));
+ __llvm_profile_set_filename(fixedProfile.c_str());
+ }
+#endif
+}
+
+void ConfigureIgnoreSigpipe()
+{
+#ifdef _unix_
+ signal(SIGPIPE, SIG_IGN);
+#endif
+}
+
+void ConfigureCrashHandler()
+{
+ TSignalRegistry::Get()->PushCallback(AllCrashSignals, CrashSignalHandler);
+ TSignalRegistry::Get()->PushDefaultSignalHandler(AllCrashSignals);
+}
+
+namespace {
+
+void ExitZero(int /*unused*/)
+{
+#if defined(_linux_) && defined(CLANG_COVERAGE)
+ __llvm_profile_write_file();
+#endif
+ // TODO(babenko): replace with pure "exit" some day.
+ // Currently this causes some RPC requests to master to be replied with "Promise abandoned" error,
+ // which is not retriable.
+ _exit(0);
+}
+
+} // namespace
+
+void ConfigureExitZeroOnSigterm()
+{
+#ifdef _unix_
+ signal(SIGTERM, ExitZero);
+#endif
+}
+
+void ConfigureAllocator(const TAllocatorOptions& options)
+{
+ NYT::MlockFileMappings();
+
+#ifdef _linux_
+ NYTAlloc::EnableYTLogging();
+ NYTAlloc::EnableYTProfiling();
+ NYTAlloc::InitializeLibunwindInterop();
+ NYTAlloc::SetEnableEagerMemoryRelease(options.YTAllocEagerMemoryRelease);
+
+ if (tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) {
+ std::thread backgroundThread([] {
+ TThread::SetCurrentThreadName("TCAllocBack");
+ tcmalloc::MallocExtension::ProcessBackgroundActions();
+ YT_ABORT();
+ });
+ backgroundThread.detach();
+ }
+
+ NProfiling::EnableTCMallocProfiler();
+
+ NYTProf::EnableMemoryProfilingTags(options.SnapshotUpdatePeriod);
+
+ absl::SetStackUnwinder(NYTProf::AbslStackUnwinder);
+ // TODO(prime@): tune parameters.
+ tcmalloc::MallocExtension::SetProfileSamplingRate(2_MB);
+ if (options.TCMallocGuardedSamplingRate) {
+ tcmalloc::MallocExtension::SetGuardedSamplingRate(*options.TCMallocGuardedSamplingRate);
+ tcmalloc::MallocExtension::ActivateGuardedSampling();
+ }
+ tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(3_MB);
+ tcmalloc::MallocExtension::SetMaxTotalThreadCacheBytes(24_MB);
+ tcmalloc::MallocExtension::SetBackgroundReleaseRate(tcmalloc::MallocExtension::BytesPerSecond{32_MB});
+ tcmalloc::MallocExtension::EnableForkSupport();
+#else
+ Y_UNUSED(options);
+#endif
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program.h b/yt/yt/library/program/program.h
new file mode 100644
index 0000000000..f1f236cfab
--- /dev/null
+++ b/yt/yt/library/program/program.h
@@ -0,0 +1,148 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+#include <library/cpp/yt/stockpile/stockpile.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+
+#include <yt/yt/core/yson/string.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DEFINE_ENUM(EProgramExitCode,
+ ((OK)(0))
+ ((OptionsError)(1))
+ ((ProgramError)(2))
+);
+
+class TProgram
+{
+public:
+ TProgram();
+ ~TProgram();
+
+ TProgram(const TProgram&) = delete;
+ TProgram(TProgram&&) = delete;
+
+ // This call actually never returns;
+ // |int| return type is just for the symmetry with |main|.
+ [[noreturn]]
+ int Run(int argc, const char** argv);
+
+ //! Handles --version/--yt-version/--build [--yson] if they are present.
+ void HandleVersionAndBuild();
+
+ //! Nongracefully aborts the program.
+ /*!
+ * Tries to flush logging messages.
+ * Aborts via |_exit| call.
+ */
+ [[noreturn]]
+ static void Abort(EProgramExitCode code) noexcept;
+ [[noreturn]]
+ static void Abort(int code) noexcept;
+
+protected:
+ NLastGetopt::TOpts Opts_;
+ TString Argv0_;
+ bool PrintYTVersion_ = false;
+ bool PrintVersion_ = false;
+ bool PrintBuild_ = false;
+ bool UseYson_ = false;
+
+ virtual void DoRun(const NLastGetopt::TOptsParseResult& parseResult) = 0;
+
+ virtual void OnError(const TString& message) noexcept;
+
+ virtual bool ShouldAbortOnHungShutdown() noexcept;
+
+ void SetCrashOnError();
+
+ //! Handler for --yt-version command argument.
+ [[noreturn]]
+ void PrintYTVersionAndExit();
+
+ //! Handler for --build command argument.
+ [[noreturn]]
+ void PrintBuildAndExit();
+
+ //! Handler for --version command argument.
+ //! By default, --version and --yt-version work the same way,
+ //! but some YT components (e.g. CHYT) can override it to provide its own version.
+ [[noreturn]]
+ virtual void PrintVersionAndExit();
+
+ [[noreturn]]
+ void Exit(EProgramExitCode code) noexcept;
+
+ [[noreturn]]
+ void Exit(int code) noexcept;
+
+private:
+ bool CrashOnError_ = false;
+
+ // Custom handler for option parsing errors.
+ class TOptsParseResult;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! The simplest exception possible.
+//! Here we refrain from using TErrorException, as it relies on proper configuration of singleton subsystems,
+//! which might not be the case during startup.
+class TProgramException
+ : public std::exception
+{
+public:
+ explicit TProgramException(TString what);
+
+ const char* what() const noexcept override;
+
+private:
+ const TString What_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Helper for TOpt::StoreMappedResult to validate file paths for existence.
+TString CheckPathExistsArgMapper(const TString& arg);
+
+//! Helper for TOpt::StoreMappedResult to parse GUIDs.
+TGuid CheckGuidArgMapper(const TString& arg);
+
+//! Helper for TOpt::StoreMappedResult to parse YSON strings.
+NYson::TYsonString CheckYsonArgMapper(const TString& arg);
+
+//! Drop privileges and save them if running with suid-bit.
+void ConfigureUids();
+
+void ConfigureCoverageOutput();
+
+void ConfigureIgnoreSigpipe();
+
+//! Intercepts standard crash signals (see signal_registry.h for full list) with a nice handler.
+void ConfigureCrashHandler();
+
+//! Intercepts SIGTERM and terminates the process immediately with zero exit code.
+void ConfigureExitZeroOnSigterm();
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TAllocatorOptions
+{
+ bool YTAllocEagerMemoryRelease = false;
+
+ bool TCMallocOptimizeSize = false;
+ std::optional<i64> TCMallocGuardedSamplingRate = 128_MB;
+
+ std::optional<TDuration> SnapshotUpdatePeriod;
+};
+
+void ConfigureAllocator(const TAllocatorOptions& options = {});
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program_config_mixin.cpp b/yt/yt/library/program/program_config_mixin.cpp
new file mode 100644
index 0000000000..9ced4de64f
--- /dev/null
+++ b/yt/yt/library/program/program_config_mixin.cpp
@@ -0,0 +1 @@
+#include "program_config_mixin.h"
diff --git a/yt/yt/library/program/program_config_mixin.h b/yt/yt/library/program/program_config_mixin.h
new file mode 100644
index 0000000000..80f681d06e
--- /dev/null
+++ b/yt/yt/library/program/program_config_mixin.h
@@ -0,0 +1,166 @@
+#pragma once
+
+#include "program.h"
+
+#include <library/cpp/yt/string/enum.h>
+
+#include <yt/yt/core/ytree/convert.h>
+#include <yt/yt/core/ytree/yson_serializable.h>
+
+#include <util/stream/file.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class TConfig, class TDynamicConfig = void>
+class TProgramConfigMixin
+{
+protected:
+ explicit TProgramConfigMixin(
+ NLastGetopt::TOpts& opts,
+ bool required = true,
+ const TString& argumentName = "config")
+ : ArgumentName_(argumentName)
+ {
+ auto opt = opts
+ .AddLongOption(TString(argumentName), Format("path to %v file", argumentName))
+ .StoreMappedResult(&ConfigPath_, &CheckPathExistsArgMapper)
+ .RequiredArgument("FILE");
+ if (required) {
+ opt.Required();
+ } else {
+ opt.Optional();
+ }
+ opts
+ .AddLongOption(
+ Format("%v-template", argumentName),
+ Format("print %v template and exit", argumentName))
+ .SetFlag(&ConfigTemplate_);
+ opts
+ .AddLongOption(
+ Format("%v-actual", argumentName),
+ Format("print actual %v and exit", argumentName))
+ .SetFlag(&ConfigActual_);
+ opts
+ .AddLongOption(
+ Format("%v-unrecognized-strategy", argumentName),
+ Format("configure strategy for unrecognized attributes in %v", argumentName))
+ .Handler1T<TStringBuf>([this](TStringBuf value) {
+ UnrecognizedStrategy_ = ParseEnum<NYTree::EUnrecognizedStrategy>(value);
+ });
+
+ if constexpr (std::is_same_v<TDynamicConfig, void>) {
+ return;
+ }
+
+ opts
+ .AddLongOption(
+ Format("dynamic-%v-template", argumentName),
+ Format("print dynamic %v template and exit", argumentName))
+ .SetFlag(&DynamicConfigTemplate_);
+ }
+
+ TIntrusivePtr<TConfig> GetConfig(bool returnNullIfNotSupplied = false)
+ {
+ if (returnNullIfNotSupplied && !ConfigPath_) {
+ return nullptr;
+ }
+
+ if (!Config_) {
+ LoadConfig();
+ }
+ return Config_;
+ }
+
+ NYTree::INodePtr GetConfigNode(bool returnNullIfNotSupplied = false)
+ {
+ if (returnNullIfNotSupplied && !ConfigPath_) {
+ return nullptr;
+ }
+
+ if (!ConfigNode_) {
+ LoadConfigNode();
+ }
+ return ConfigNode_;
+ }
+
+ bool HandleConfigOptions()
+ {
+ auto print = [] (const auto& config) {
+ using namespace NYson;
+ TYsonWriter writer(&Cout, EYsonFormat::Pretty);
+ config->Save(&writer);
+ Cout << Flush;
+ };
+ if (ConfigTemplate_) {
+ print(New<TConfig>());
+ return true;
+ }
+ if (ConfigActual_) {
+ print(GetConfig());
+ return true;
+ }
+
+ if constexpr (!std::is_same_v<TDynamicConfig, void>) {
+ if (DynamicConfigTemplate_) {
+ print(New<TDynamicConfig>());
+ return true;
+ }
+ }
+ return false;
+ }
+
+private:
+ void LoadConfigNode()
+ {
+ using namespace NYTree;
+
+ if (!ConfigPath_){
+ THROW_ERROR_EXCEPTION("Missing --%v option", ArgumentName_);
+ }
+
+ try {
+ TIFStream stream(ConfigPath_);
+ ConfigNode_ = ConvertToNode(&stream);
+ } catch (const std::exception& ex) {
+ THROW_ERROR_EXCEPTION("Error parsing %v file %v",
+ ArgumentName_,
+ ConfigPath_)
+ << ex;
+ }
+ }
+
+ void LoadConfig()
+ {
+ if (!ConfigNode_) {
+ LoadConfigNode();
+ }
+
+ try {
+ Config_ = New<TConfig>();
+ Config_->SetUnrecognizedStrategy(UnrecognizedStrategy_);
+ Config_->Load(ConfigNode_);
+ } catch (const std::exception& ex) {
+ THROW_ERROR_EXCEPTION("Error loading %v file %v",
+ ArgumentName_,
+ ConfigPath_)
+ << ex;
+ }
+ }
+
+ const TString ArgumentName_;
+
+ TString ConfigPath_;
+ bool ConfigTemplate_;
+ bool ConfigActual_;
+ bool DynamicConfigTemplate_ = false;
+ NYTree::EUnrecognizedStrategy UnrecognizedStrategy_ = NYTree::EUnrecognizedStrategy::KeepRecursive;
+
+ TIntrusivePtr<TConfig> Config_;
+ NYTree::INodePtr ConfigNode_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program_pdeathsig_mixin.cpp b/yt/yt/library/program/program_pdeathsig_mixin.cpp
new file mode 100644
index 0000000000..34f1f3b9a8
--- /dev/null
+++ b/yt/yt/library/program/program_pdeathsig_mixin.cpp
@@ -0,0 +1,36 @@
+#include "program_pdeathsig_mixin.h"
+
+#ifdef _linux_
+#include <sys/prctl.h>
+#endif
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TProgramPdeathsigMixin::TProgramPdeathsigMixin(NLastGetopt::TOpts& opts)
+{
+ opts.AddLongOption("pdeathsig", "parent death signal")
+ .StoreResult(&ParentDeathSignal_)
+ .RequiredArgument("PDEATHSIG");
+}
+
+bool TProgramPdeathsigMixin::HandlePdeathsigOptions()
+{
+ if (ParentDeathSignal_ > 0) {
+#ifdef _linux_
+ // Parent death signal is set by testing framework to avoid dangling processes when test runner crashes.
+ // Unfortunately, setting pdeathsig in preexec_fn in subprocess call in test runner is not working
+ // when the program has suid bit (pdeath_sig is reset after exec call in this case)
+ // More details can be found in
+ // http://linux.die.net/man/2/prctl
+ // http://www.isec.pl/vulnerabilities/isec-0024-death-signal.txt
+ YT_VERIFY(prctl(PR_SET_PDEATHSIG, ParentDeathSignal_) == 0);
+#endif
+ }
+ return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program_pdeathsig_mixin.h b/yt/yt/library/program/program_pdeathsig_mixin.h
new file mode 100644
index 0000000000..3e4bcfd4a6
--- /dev/null
+++ b/yt/yt/library/program/program_pdeathsig_mixin.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "program.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TProgramPdeathsigMixin
+{
+protected:
+ explicit TProgramPdeathsigMixin(NLastGetopt::TOpts& opts);
+
+ bool HandlePdeathsigOptions();
+
+private:
+ int ParentDeathSignal_ = -1;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program_setsid_mixin.cpp b/yt/yt/library/program/program_setsid_mixin.cpp
new file mode 100644
index 0000000000..a745fcd3a2
--- /dev/null
+++ b/yt/yt/library/program/program_setsid_mixin.cpp
@@ -0,0 +1,30 @@
+#include "program_setsid_mixin.h"
+
+#ifdef _linux_
+#include <unistd.h>
+#endif
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TProgramSetsidMixin::TProgramSetsidMixin(NLastGetopt::TOpts& opts)
+{
+ opts.AddLongOption("setsid", "create a new session")
+ .StoreTrue(&Setsid_)
+ .Optional();
+}
+
+bool TProgramSetsidMixin::HandleSetsidOptions()
+{
+ if (Setsid_) {
+#ifdef _linux_
+ setsid();
+#endif
+ }
+ return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/program_setsid_mixin.h b/yt/yt/library/program/program_setsid_mixin.h
new file mode 100644
index 0000000000..00b3dff50e
--- /dev/null
+++ b/yt/yt/library/program/program_setsid_mixin.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "program.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TProgramSetsidMixin
+{
+protected:
+ explicit TProgramSetsidMixin(NLastGetopt::TOpts& opts);
+
+ bool HandleSetsidOptions();
+
+private:
+ bool Setsid_ = false;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/public.h b/yt/yt/library/program/public.h
new file mode 100644
index 0000000000..b10575778e
--- /dev/null
+++ b/yt/yt/library/program/public.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include <yt/yt/core/misc/public.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_CLASS(TBuildInfo)
+DECLARE_REFCOUNTED_CLASS(TRpcConfig)
+DECLARE_REFCOUNTED_CLASS(TTCMallocConfig)
+DECLARE_REFCOUNTED_CLASS(TStockpileConfig)
+DECLARE_REFCOUNTED_CLASS(TSingletonsConfig)
+DECLARE_REFCOUNTED_CLASS(TSingletonsDynamicConfig)
+DECLARE_REFCOUNTED_CLASS(TDiagnosticDumpConfig)
+DECLARE_REFCOUNTED_CLASS(THeapSizeLimit)
+DECLARE_REFCOUNTED_CLASS(THeapProfilerConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/yt/yt/library/program/ya.make b/yt/yt/library/program/ya.make
new file mode 100644
index 0000000000..5742ce9287
--- /dev/null
+++ b/yt/yt/library/program/ya.make
@@ -0,0 +1,30 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ build_attributes.cpp
+ config.cpp
+ helpers.cpp
+ program.cpp
+ program_config_mixin.cpp
+ program_pdeathsig_mixin.cpp
+ program_setsid_mixin.cpp
+)
+
+PEERDIR(
+ yt/yt/core
+ yt/yt/core/service_discovery/yp
+ yt/yt/library/monitoring
+ yt/yt/library/containers
+ yt/yt/library/profiling/solomon
+ yt/yt/library/profiling/tcmalloc
+ yt/yt/library/profiling/perf
+ yt/yt/library/ytprof
+ yt/yt/library/tracing/jaeger
+ library/cpp/yt/mlock
+ library/cpp/yt/stockpile
+ library/cpp/yt/string
+)
+
+END()
diff --git a/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt
index 54f67f1ec5..68eb83d171 100644
--- a/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt
+++ b/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(jaeger)
add_library(yt-library-tracing)
target_compile_options(yt-library-tracing PRIVATE
diff --git a/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt b/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt
index 923b49e3e7..5681c6a06e 100644
--- a/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt
+++ b/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(jaeger)
add_library(yt-library-tracing)
target_compile_options(yt-library-tracing PRIVATE
diff --git a/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt b/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt
index 923b49e3e7..5681c6a06e 100644
--- a/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt
+++ b/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(jaeger)
add_library(yt-library-tracing)
target_compile_options(yt-library-tracing PRIVATE
diff --git a/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt b/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt
index e33cbed391..0adf7dc67a 100644
--- a/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt
+++ b/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(jaeger)
add_library(yt-library-tracing)
target_link_libraries(yt-library-tracing PUBLIC
diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..ebc07421e3
--- /dev/null
+++ b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,67 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto
+)
+target_sources(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp
+)
+target_proto_addincls(library-tracing-jaeger
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-tracing-jaeger
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
+
+add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger.global PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_sources(library-tracing-jaeger.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp
+)
diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..0946a55b6c
--- /dev/null
+++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto
+)
+target_sources(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp
+)
+target_proto_addincls(library-tracing-jaeger
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-tracing-jaeger
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
+
+add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger.global PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_sources(library-tracing-jaeger.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp
+)
diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..0946a55b6c
--- /dev/null
+++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto
+)
+target_sources(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp
+)
+target_proto_addincls(library-tracing-jaeger
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-tracing-jaeger
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
+
+add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger)
+target_compile_options(library-tracing-jaeger.global PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-tracing-jaeger.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_sources(library-tracing-jaeger.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp
+)
diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.txt b/yt/yt/library/tracing/jaeger/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/tracing/jaeger/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..e6ee22e85b
--- /dev/null
+++ b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,61 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-tracing-jaeger)
+target_link_libraries(library-tracing-jaeger PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_proto_messages(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto
+)
+target_sources(library-tracing-jaeger PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp
+)
+target_proto_addincls(library-tracing-jaeger
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-tracing-jaeger
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
+
+add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger)
+target_link_libraries(library-tracing-jaeger.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yt-library-tracing
+ core-rpc-grpc
+ contrib-libs-protobuf
+)
+target_sources(library-tracing-jaeger.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp
+)
diff --git a/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..ac3d970f12
--- /dev/null
+++ b/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,41 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(api)
+add_subdirectory(proto)
+
+add_library(yt-library-ytprof)
+target_compile_options(yt-library-ytprof PRIVATE
+ -Wdeprecated-this-capture
+ -DYTPROF_BUILD_TYPE="RELEASE"
+)
+target_link_libraries(yt-library-ytprof PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-memory
+ cpp-yt-threading
+ backtrace-cursors-interop
+ backtrace-cursors-frame_pointer
+ backtrace-cursors-libunwind
+ library-ytprof-api
+ library-ytprof-proto
+ contrib-libs-libunwind
+ libs-tcmalloc-malloc_extension
+ library-cpp-svnversion
+ yt-yt-core
+)
+target_sources(yt-library-ytprof PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize_other.cpp
+)
diff --git a/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..c015a8d3da
--- /dev/null
+++ b/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,43 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(api)
+add_subdirectory(http)
+add_subdirectory(proto)
+
+add_library(yt-library-ytprof)
+target_compile_options(yt-library-ytprof PRIVATE
+ -Wdeprecated-this-capture
+ -DYTPROF_BUILD_TYPE="RELEASE"
+)
+target_link_libraries(yt-library-ytprof PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-memory
+ cpp-yt-threading
+ backtrace-cursors-interop
+ backtrace-cursors-frame_pointer
+ backtrace-cursors-libunwind
+ library-ytprof-api
+ library-ytprof-proto
+ contrib-libs-libunwind
+ libs-tcmalloc-malloc_extension
+ library-cpp-svnversion
+ yt-yt-core
+)
+target_sources(yt-library-ytprof PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize.cpp
+)
diff --git a/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..c015a8d3da
--- /dev/null
+++ b/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,43 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(api)
+add_subdirectory(http)
+add_subdirectory(proto)
+
+add_library(yt-library-ytprof)
+target_compile_options(yt-library-ytprof PRIVATE
+ -Wdeprecated-this-capture
+ -DYTPROF_BUILD_TYPE="RELEASE"
+)
+target_link_libraries(yt-library-ytprof PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-memory
+ cpp-yt-threading
+ backtrace-cursors-interop
+ backtrace-cursors-frame_pointer
+ backtrace-cursors-libunwind
+ library-ytprof-api
+ library-ytprof-proto
+ contrib-libs-libunwind
+ libs-tcmalloc-malloc_extension
+ library-cpp-svnversion
+ yt-yt-core
+)
+target_sources(yt-library-ytprof PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize.cpp
+)
diff --git a/yt/yt/library/ytprof/CMakeLists.txt b/yt/yt/library/ytprof/CMakeLists.txt
index dbfb934bae..f8b31df0c1 100644
--- a/yt/yt/library/ytprof/CMakeLists.txt
+++ b/yt/yt/library/ytprof/CMakeLists.txt
@@ -6,4 +6,12 @@
# original buildsystem will not be accepted.
-add_subdirectory(api)
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..ec5652c9e9
--- /dev/null
+++ b/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(api)
+add_subdirectory(proto)
+
+add_library(yt-library-ytprof)
+target_compile_options(yt-library-ytprof PRIVATE
+ -DYTPROF_BUILD_TYPE="RELEASE"
+)
+target_link_libraries(yt-library-ytprof PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-yt-memory
+ cpp-yt-threading
+ backtrace-cursors-interop
+ backtrace-cursors-frame_pointer
+ backtrace-cursors-libunwind
+ library-ytprof-api
+ library-ytprof-proto
+ contrib-libs-libunwind
+ libs-tcmalloc-malloc_extension
+ library-cpp-svnversion
+ yt-yt-core
+)
+target_sources(yt-library-ytprof PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize_other.cpp
+)
diff --git a/yt/yt/library/ytprof/bundle/ya.make b/yt/yt/library/ytprof/bundle/ya.make
new file mode 100644
index 0000000000..7f88583494
--- /dev/null
+++ b/yt/yt/library/ytprof/bundle/ya.make
@@ -0,0 +1,28 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+# Built with ya make -DNO_DEBUGINFO=yes -r --musl contrib/libs/llvm12/tools/llvm-symbolizer
+FROM_SANDBOX(
+ FILE 2531143113
+ OUT_NOAUTO llvm-symbolizer
+)
+
+RESOURCE(
+ yt/yt/library/ytprof/bundle/llvm-symbolizer
+ /ytprof/llvm-symbolizer
+)
+
+# Built with env CGO_ENABLED=0 ya tool go install github.com/google/pprof@latest
+FROM_SANDBOX(
+ FILE 2531135322
+ OUT_NOAUTO pprof
+)
+
+RESOURCE(
+ yt/yt/library/ytprof/bundle/pprof
+ /ytprof/pprof
+)
+
+END()
+
diff --git a/yt/yt/library/ytprof/example/main.cpp b/yt/yt/library/ytprof/example/main.cpp
new file mode 100644
index 0000000000..bc9dec690a
--- /dev/null
+++ b/yt/yt/library/ytprof/example/main.cpp
@@ -0,0 +1,70 @@
+#include <yt/yt/core/concurrency/poller.h>
+#include <yt/yt/core/concurrency/thread_pool_poller.h>
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/thread_pool.h>
+#include <yt/yt/core/http/server.h>
+
+#include <yt/yt/library/ytprof/http/handler.h>
+#include <yt/yt/library/ytprof/heap_profiler.h>
+
+#include <absl/debugging/stacktrace.h>
+
+using namespace NYT;
+using namespace NYT::NHttp;
+using namespace NYT::NConcurrency;
+using namespace NYT::NYTProf;
+
+int main(int argc, char* argv[])
+{
+ absl::SetStackUnwinder(AbslStackUnwinder);
+ tcmalloc::MallocExtension::SetProfileSamplingRate(2_MB);
+
+ try {
+ if (argc != 2 && argc != 3) {
+ throw yexception() << "usage: " << argv[0] << " PORT";
+ }
+
+ auto port = FromString<int>(argv[1]);
+ auto poller = CreateThreadPoolPoller(1, "Example");
+ auto server = CreateServer(port, poller);
+
+ Register(server, "");
+ server->Start();
+
+ THashMap<TString, std::vector<int>> data;
+ for (int i = 0; i < 1024 * 16; i++) {
+ data[ToString(i)].resize(1024);
+ }
+
+ auto burnCpu = [] {
+ ui64 value = 0;
+ while (true) {
+ THash<TString> hasher;
+ for (int i = 0; i < 10000000; i++) {
+ value += hasher(ToString(i));
+ }
+
+ std::vector<TString> data;
+ for (int i = 0; i < 10000; i++) {
+ data.push_back(TString(1024, 'x'));
+ }
+
+ if (value == 1) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+ };
+
+ auto pool = CreateThreadPool(64, "Pool");
+ for (int i = 0; i < 64; i++) {
+ pool->GetInvoker()->Invoke(BIND(burnCpu));
+ }
+
+ burnCpu();
+ } catch (const std::exception& ex) {
+ Cerr << ex.what() << Endl;
+ _exit(1);
+ }
+
+ return 0;
+}
diff --git a/yt/yt/library/ytprof/example/ya.make b/yt/yt/library/ytprof/example/ya.make
new file mode 100644
index 0000000000..2f79922400
--- /dev/null
+++ b/yt/yt/library/ytprof/example/ya.make
@@ -0,0 +1,19 @@
+PROGRAM(ytprof-example)
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+IF (OS_LINUX)
+ ALLOCATOR(TCMALLOC_256K)
+ENDIF()
+
+SRCS(main.cpp)
+
+IF (OS_LINUX)
+ LDFLAGS("-Wl,--build-id=sha1")
+ENDIF()
+
+PEERDIR(
+ yt/yt/library/ytprof/http
+)
+
+END()
diff --git a/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..334ca90c02
--- /dev/null
+++ b/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,25 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-ytprof-http)
+target_compile_options(library-ytprof-http PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-ytprof-http PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cgiparam
+ yt-core-http
+ yt-library-ytprof
+ yt-library-process
+)
+target_sources(library-ytprof-http PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/http/handler.cpp
+)
diff --git a/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..334ca90c02
--- /dev/null
+++ b/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,25 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(library-ytprof-http)
+target_compile_options(library-ytprof-http PRIVATE
+ -Wdeprecated-this-capture
+)
+target_link_libraries(library-ytprof-http PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cgiparam
+ yt-core-http
+ yt-library-ytprof
+ yt-library-process
+)
+target_sources(library-ytprof-http PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/http/handler.cpp
+)
diff --git a/yt/yt/library/ytprof/http/CMakeLists.txt b/yt/yt/library/ytprof/http/CMakeLists.txt
new file mode 100644
index 0000000000..4d48dcdee6
--- /dev/null
+++ b/yt/yt/library/ytprof/http/CMakeLists.txt
@@ -0,0 +1,13 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/ytprof/http/handler.cpp b/yt/yt/library/ytprof/http/handler.cpp
new file mode 100644
index 0000000000..382ffc1fec
--- /dev/null
+++ b/yt/yt/library/ytprof/http/handler.cpp
@@ -0,0 +1,311 @@
+#include "handler.h"
+
+#include <yt/yt/core/concurrency/async_stream.h>
+
+#include <yt/yt/core/http/http.h>
+#include <yt/yt/core/http/server.h>
+
+#include <yt/yt/library/ytprof/cpu_profiler.h>
+#include <yt/yt/library/ytprof/spinlock_profiler.h>
+#include <yt/yt/library/ytprof/heap_profiler.h>
+#include <yt/yt/library/ytprof/profile.h>
+#include <yt/yt/library/ytprof/symbolize.h>
+#include <yt/yt/library/ytprof/external_pprof.h>
+
+#include <yt/yt/library/process/subprocess.h>
+
+#include <yt/yt/core/misc/finally.h>
+
+#include <library/cpp/cgiparam/cgiparam.h>
+
+#include <util/system/mutex.h>
+
+namespace NYT::NYTProf {
+
+using namespace NHttp;
+using namespace NConcurrency;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TBaseHandler
+ : public IHttpHandler
+{
+public:
+ explicit TBaseHandler(const TBuildInfo& buildInfo)
+ : BuildInfo_(buildInfo)
+ { }
+
+ virtual NProto::Profile BuildProfile(const TCgiParameters& params) = 0;
+
+ void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override
+ {
+ try {
+ TTryGuard guard(Lock_);
+ if (!guard) {
+ rsp->SetStatus(EStatusCode::TooManyRequests);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString("Profile fetch already running")))
+ .ThrowOnError();
+ return;
+ }
+
+ TCgiParameters params(req->GetUrl().RawQuery);
+ auto profile = BuildProfile(params);
+ Symbolize(&profile, true);
+ AddBuildInfo(&profile, BuildInfo_);
+
+ if (auto it = params.Find("symbolize"); it == params.end() || it->second != "0") {
+ SymbolizeByExternalPProf(&profile, TSymbolizationOptions{
+ .RunTool = RunSubprocess,
+ });
+ }
+
+ TStringStream profileBlob;
+ WriteProfile(&profileBlob, profile);
+
+ rsp->SetStatus(EStatusCode::OK);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(profileBlob.Str())))
+ .ThrowOnError();
+ } catch (const std::exception& ex) {
+ if (rsp->AreHeadersFlushed()) {
+ throw;
+ }
+
+ rsp->SetStatus(EStatusCode::InternalServerError);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what())))
+ .ThrowOnError();
+
+ throw;
+ }
+ }
+
+protected:
+ const TBuildInfo BuildInfo_;
+
+private:
+ YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_);
+};
+
+class TCpuProfilerHandler
+ : public TBaseHandler
+{
+public:
+ using TBaseHandler::TBaseHandler;
+
+ NProto::Profile BuildProfile(const TCgiParameters& params) override
+ {
+ auto duration = TDuration::Seconds(15);
+ if (auto it = params.Find("d"); it != params.end()) {
+ duration = TDuration::Parse(it->second);
+ }
+
+ TCpuProfilerOptions options;
+ if (auto it = params.Find("freq"); it != params.end()) {
+ options.SamplingFrequency = FromString<int>(it->second);
+ }
+
+ if (auto it = params.Find("record_action_run_time"); it != params.end()) {
+ options.RecordActionRunTime = true;
+ }
+
+ if (auto it = params.Find("action_min_exec_time"); it != params.end()) {
+ options.SampleFilters.push_back(GetActionMinExecTimeFilter(TDuration::Parse(it->second)));
+ }
+
+ TCpuProfiler profiler{options};
+ profiler.Start();
+ TDelayedExecutor::WaitForDuration(duration);
+ profiler.Stop();
+
+ return profiler.ReadProfile();
+ }
+};
+
+class TSpinlockProfilerHandler
+ : public TBaseHandler
+{
+public:
+ TSpinlockProfilerHandler(const TBuildInfo& buildInfo, bool yt)
+ : TBaseHandler(buildInfo)
+ , YT_(yt)
+ { }
+
+ NProto::Profile BuildProfile(const TCgiParameters& params) override
+ {
+ auto duration = TDuration::Seconds(15);
+ if (auto it = params.Find("d"); it != params.end()) {
+ duration = TDuration::Parse(it->second);
+ }
+
+ TSpinlockProfilerOptions options;
+ if (auto it = params.Find("frac"); it != params.end()) {
+ options.ProfileFraction = FromString<int>(it->second);
+ }
+
+ if (YT_) {
+ TBlockingProfiler profiler{options};
+ profiler.Start();
+ TDelayedExecutor::WaitForDuration(duration);
+ profiler.Stop();
+
+ return profiler.ReadProfile();
+ } else {
+ TSpinlockProfiler profiler{options};
+ profiler.Start();
+ TDelayedExecutor::WaitForDuration(duration);
+ profiler.Stop();
+
+ return profiler.ReadProfile();
+ }
+ }
+
+private:
+ const bool YT_;
+};
+
+class TTCMallocSnapshotProfilerHandler
+ : public TBaseHandler
+{
+public:
+ TTCMallocSnapshotProfilerHandler(const TBuildInfo& buildInfo, tcmalloc::ProfileType profileType)
+ : TBaseHandler(buildInfo)
+ , ProfileType_(profileType)
+ { }
+
+ NProto::Profile BuildProfile(const TCgiParameters& /*params*/) override
+ {
+ return ReadHeapProfile(ProfileType_);
+ }
+
+private:
+ tcmalloc::ProfileType ProfileType_;
+};
+
+class TTCMallocAllocationProfilerHandler
+ : public TBaseHandler
+{
+public:
+ using TBaseHandler::TBaseHandler;
+
+ NProto::Profile BuildProfile(const TCgiParameters& params) override
+ {
+ auto duration = TDuration::Seconds(15);
+ if (auto it = params.Find("d"); it != params.end()) {
+ duration = TDuration::Parse(it->second);
+ }
+
+ auto token = tcmalloc::MallocExtension::StartAllocationProfiling();
+ TDelayedExecutor::WaitForDuration(duration);
+ return ConvertAllocationProfile(std::move(token).Stop());
+ }
+};
+
+class TTCMallocStatHandler
+ : public IHttpHandler
+{
+public:
+ void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override
+ {
+ auto stat = tcmalloc::MallocExtension::GetStats();
+ rsp->SetStatus(EStatusCode::OK);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(TString{stat})))
+ .ThrowOnError();
+ }
+};
+
+class TBinaryHandler
+ : public IHttpHandler
+{
+public:
+ void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override
+ {
+ try {
+ auto buildId = GetBuildId();
+ TCgiParameters params(req->GetUrl().RawQuery);
+
+ if (auto it = params.Find("check_build_id"); it != params.end()) {
+ if (it->second != buildId) {
+ THROW_ERROR_EXCEPTION("Wrong build id: %v != %v", it->second, buildId);
+ }
+ }
+
+ rsp->SetStatus(EStatusCode::OK);
+
+ TFileInput file{"/proc/self/exe"};
+ auto adapter = CreateBufferedSyncAdapter(rsp);
+ file.ReadAll(*adapter);
+ adapter->Finish();
+
+ WaitFor(rsp->Close())
+ .ThrowOnError();
+ } catch (const std::exception& ex) {
+ if (rsp->AreHeadersFlushed()) {
+ throw;
+ }
+
+ rsp->SetStatus(EStatusCode::InternalServerError);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what())))
+ .ThrowOnError();
+
+ throw;
+ }
+ }
+};
+
+class TVersionHandler
+ : public IHttpHandler
+{
+public:
+ void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override
+ {
+ rsp->SetStatus(EStatusCode::OK);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion())))
+ .ThrowOnError();
+ }
+};
+
+class TBuildIdHandler
+ : public IHttpHandler
+{
+public:
+ void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override
+ {
+ rsp->SetStatus(EStatusCode::OK);
+ WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion())))
+ .ThrowOnError();
+ }
+};
+
+void Register(
+ const NHttp::IServerPtr& server,
+ const TString& prefix,
+ const TBuildInfo& buildInfo)
+{
+ Register(server->GetPathMatcher(), prefix, buildInfo);
+}
+
+void Register(
+ const IRequestPathMatcherPtr& handlers,
+ const TString& prefix,
+ const TBuildInfo& buildInfo)
+{
+ handlers->Add(prefix + "/profile", New<TCpuProfilerHandler>(buildInfo));
+
+ handlers->Add(prefix + "/lock", New<TSpinlockProfilerHandler>(buildInfo, false));
+ handlers->Add(prefix + "/block", New<TSpinlockProfilerHandler>(buildInfo, true));
+
+ handlers->Add(prefix + "/heap", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kHeap));
+ handlers->Add(prefix + "/peak", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kPeakHeap));
+ handlers->Add(prefix + "/fragmentation", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kFragmentation));
+ handlers->Add(prefix + "/allocations", New<TTCMallocAllocationProfilerHandler>(buildInfo));
+
+ handlers->Add(prefix + "/tcmalloc", New<TTCMallocStatHandler>());
+
+ handlers->Add(prefix + "/binary", New<TBinaryHandler>());
+
+ handlers->Add(prefix + "/version", New<TVersionHandler>());
+ handlers->Add(prefix + "/buildid", New<TBuildIdHandler>());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/http/handler.h b/yt/yt/library/ytprof/http/handler.h
new file mode 100644
index 0000000000..fa96412d95
--- /dev/null
+++ b/yt/yt/library/ytprof/http/handler.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <yt/yt/core/http/public.h>
+
+#include <yt/yt/library/ytprof/build_info.h>
+
+namespace NYT::NYTProf {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Register profiling handlers.
+void Register(
+ const NHttp::IServerPtr& server,
+ const TString& prefix,
+ const TBuildInfo& buildInfo = TBuildInfo::GetDefault());
+
+void Register(
+ const NHttp::IRequestPathMatcherPtr& handlers,
+ const TString& prefix,
+ const TBuildInfo& buildInfo = TBuildInfo::GetDefault());
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/http/ya.make b/yt/yt/library/ytprof/http/ya.make
new file mode 100644
index 0000000000..1a1f3ff20c
--- /dev/null
+++ b/yt/yt/library/ytprof/http/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ handler.cpp
+)
+
+PEERDIR(
+ library/cpp/cgiparam
+ yt/yt/core/http
+ yt/yt/library/ytprof
+ yt/yt/library/process
+)
+
+END()
diff --git a/yt/yt/library/ytprof/integration/test_http.py b/yt/yt/library/ytprof/integration/test_http.py
new file mode 100644
index 0000000000..2135d8daf9
--- /dev/null
+++ b/yt/yt/library/ytprof/integration/test_http.py
@@ -0,0 +1,113 @@
+import pytest
+import requests
+import time
+
+import asyncio
+import httpx
+
+import yatest.common
+import yatest.common.network
+
+
+TIMEOUT = 5000
+
+
+@pytest.fixture(scope="session")
+def running_example():
+ with yatest.common.network.PortManager() as pm:
+ port = pm.get_port()
+
+ cmd = [
+ yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"),
+ str(port)
+ ]
+
+ p = yatest.common.execute(cmd, wait=False, env={"YT_LOG_LEVEL": "DEBUG"})
+ time.sleep(1)
+ assert p.running
+
+ try:
+ yield {"port": port}
+ finally:
+ p.kill()
+
+
+def fetch_data(running_example, name):
+ rsp = requests.get(f"http://localhost:{running_example['port']}/{name}")
+ if rsp.status_code == 200:
+ return rsp.content
+
+ if rsp.status_code == 500:
+ raise Exception(rsp.text)
+
+ rsp.raise_for_status()
+
+
+def test_smoke_tcmalloc(running_example):
+ fetch_data(running_example, "heap")
+ fetch_data(running_example, "allocations?d=1")
+ fetch_data(running_example, "peak")
+ fetch_data(running_example, "fragmentation")
+
+
+async def get_async(url):
+ async with httpx.AsyncClient() as client:
+ return await client.get(url, timeout=TIMEOUT)
+
+
+async def launch(running_example, name):
+ url = f"http://localhost:{running_example['port']}/{name}"
+
+ urls = [url, url, url]
+
+ resps = await asyncio.gather(*map(get_async, urls))
+ data = [resp.status_code for resp in resps]
+
+ assert data == [200, 429, 429]
+
+
+def test_async(running_example):
+ fetch_data(running_example, "heap")
+ asyncio.run(launch(running_example, "heap"))
+ fetch_data(running_example, "heap")
+
+
+def test_status_handlers(running_example):
+ assert fetch_data(running_example, "buildid")
+ assert fetch_data(running_example, "version")
+
+
+def test_cpu_profile(running_example):
+ if yatest.common.context.build_type != "profile":
+ pytest.skip()
+
+ fetch_data(running_example, "profile?d=1")
+ fetch_data(running_example, "profile?d=1&freq=1000")
+
+
+def test_spinlock_profile(running_example):
+ if yatest.common.context.build_type != "profile":
+ pytest.skip()
+
+ fetch_data(running_example, "lock?d=1")
+ fetch_data(running_example, "lock?d=1&frac=1")
+
+
+def test_block_profile(running_example):
+ if yatest.common.context.build_type != "profile":
+ pytest.skip()
+
+ fetch_data(running_example, "block?d=1")
+ fetch_data(running_example, "block?d=1&frac=1")
+
+
+def test_binary_handler(running_example):
+ binary = fetch_data(running_example, "binary")
+
+ with open(yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"), "rb") as f:
+ real_binary = f.read()
+
+ assert binary == real_binary
+
+ with pytest.raises(Exception):
+ fetch_data(running_example, "binary?check_build_id=1234")
diff --git a/yt/yt/library/ytprof/integration/ya.make b/yt/yt/library/ytprof/integration/ya.make
new file mode 100644
index 0000000000..889c25c0e1
--- /dev/null
+++ b/yt/yt/library/ytprof/integration/ya.make
@@ -0,0 +1,20 @@
+PY3TEST()
+
+SIZE(MEDIUM)
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc)
+
+PEERDIR(
+ contrib/python/requests
+ contrib/python/httpx
+)
+
+TEST_SRCS(
+ test_http.py
+)
+
+DEPENDS(
+ yt/yt/library/ytprof/example
+)
+
+END()
diff --git a/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..2a3265d317
--- /dev/null
+++ b/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,47 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-ytprof-proto)
+target_include_directories(library-ytprof-proto PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_link_libraries(library-ytprof-proto PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(library-ytprof-proto PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto
+)
+target_proto_addincls(library-ytprof-proto
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-ytprof-proto
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..d16987cc1d
--- /dev/null
+++ b/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,48 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-ytprof-proto)
+target_include_directories(library-ytprof-proto PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_link_libraries(library-ytprof-proto PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(library-ytprof-proto PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto
+)
+target_proto_addincls(library-ytprof-proto
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-ytprof-proto
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..d16987cc1d
--- /dev/null
+++ b/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,48 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-ytprof-proto)
+target_include_directories(library-ytprof-proto PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_link_libraries(library-ytprof-proto PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(library-ytprof-proto PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto
+)
+target_proto_addincls(library-ytprof-proto
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-ytprof-proto
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/library/ytprof/proto/CMakeLists.txt b/yt/yt/library/ytprof/proto/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yt/library/ytprof/proto/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..2a3265d317
--- /dev/null
+++ b/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,47 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(library-ytprof-proto)
+target_include_directories(library-ytprof-proto PUBLIC
+ ${CMAKE_BINARY_DIR}/yt
+)
+target_link_libraries(library-ytprof-proto PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(library-ytprof-proto PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto
+)
+target_proto_addincls(library-ytprof-proto
+ ./yt
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/yt
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(library-ytprof-proto
+ --cpp_out=${CMAKE_BINARY_DIR}/yt
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt
+)
diff --git a/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp
new file mode 100644
index 0000000000..3edf6a6f89
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp
@@ -0,0 +1,343 @@
+#include <dlfcn.h>
+
+#include <gtest/gtest.h>
+
+#include <library/cpp/testing/common/env.h>
+
+#include <library/cpp/yt/memory/new.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/scheduler_api.h>
+
+#include <yt/yt/core/actions/bind.h>
+
+#include <yt/yt/core/tracing/trace_context.h>
+
+#include <yt/yt/library/ytprof/cpu_profiler.h>
+#include <yt/yt/library/ytprof/symbolize.h>
+#include <yt/yt/library/ytprof/profile.h>
+#include <yt/yt/library/ytprof/external_pprof.h>
+
+#include <util/string/cast.h>
+#include <util/stream/file.h>
+#include <util/datetime/base.h>
+#include <util/system/shellcommand.h>
+
+#include <yt/yt/core/concurrency/thread_pool.h>
+
+namespace NYT::NYTProf {
+namespace {
+
+using namespace NConcurrency;
+using namespace NTracing;
+
+using TSampleFilter = TCpuProfilerOptions::TSampleFilter;
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <size_t Index>
+Y_NO_INLINE void BurnCpu()
+{
+ THash<TString> hasher;
+ ui64 value = 0;
+ for (int i = 0; i < 10000000; i++) {
+ value += hasher(ToString(i));
+ }
+ EXPECT_NE(Index, value);
+}
+
+static std::atomic<int> Counter{0};
+
+struct TNoTailCall
+{
+ ~TNoTailCall()
+ {
+ Counter++;
+ }
+};
+
+static Y_NO_INLINE void StaticFunction()
+{
+ TNoTailCall noTail;
+ BurnCpu<0>();
+}
+
+void RunUnderProfiler(
+ const TString& name,
+ std::function<void()> work,
+ bool checkSamples = true,
+ const std::vector<TSampleFilter>& filters = {},
+ bool expectEmpty = false)
+{
+ TCpuProfilerOptions options;
+ options.SampleFilters = filters;
+ options.SamplingFrequency = 100000;
+ options.RecordActionRunTime = true;
+
+#ifdef YTPROF_DEBUG_BUILD
+ options.SamplingFrequency = 100;
+#endif
+
+ TCpuProfiler profiler(options);
+
+ profiler.Start();
+
+ work();
+
+ profiler.Stop();
+
+ auto profile = profiler.ReadProfile();
+ if (checkSamples) {
+ ASSERT_EQ(expectEmpty, profile.sampleSize() == 0);
+ }
+
+ Symbolize(&profile, true);
+ AddBuildInfo(&profile, TBuildInfo::GetDefault());
+ SymbolizeByExternalPProf(&profile, TSymbolizationOptions{
+ .TmpDir = GetOutputPath(),
+ .KeepTmpDir = true,
+ .RunTool = [] (const std::vector<TString>& args) {
+ TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}};
+ command.Run();
+
+ EXPECT_TRUE(command.GetExitCode() == 0)
+ << command.GetError();
+ },
+ });
+
+ TFileOutput output(GetOutputPath() / name);
+ WriteProfile(&output, profile);
+ output.Finish();
+}
+
+class TCpuProfilerTest
+ : public ::testing::Test
+{
+ void SetUp() override
+ {
+ if (!IsProfileBuild()) {
+ GTEST_SKIP() << "rebuild with --build=profile";
+ }
+ }
+};
+
+TEST_F(TCpuProfilerTest, SingleThreadRun)
+{
+ RunUnderProfiler("single_thread.pb.gz", [] {
+ BurnCpu<0>();
+ });
+}
+
+TEST_F(TCpuProfilerTest, MultipleThreads)
+{
+ RunUnderProfiler("multiple_threads.pb.gz", [] {
+ std::thread t1([] {
+ BurnCpu<1>();
+ });
+
+ std::thread t2([] {
+ BurnCpu<2>();
+ });
+
+ t1.join();
+ t2.join();
+ });
+}
+
+TEST_F(TCpuProfilerTest, StaticFunction)
+{
+ RunUnderProfiler("static_function.pb.gz", [] {
+ StaticFunction();
+ });
+}
+
+Y_NO_INLINE void RecursiveFunction(int n)
+{
+ TNoTailCall noTail;
+ if (n == 0) {
+ BurnCpu<0>();
+ } else {
+ RecursiveFunction(n-1);
+ }
+}
+
+TEST_F(TCpuProfilerTest, DeepRecursion)
+{
+ RunUnderProfiler("recursive_function.pb.gz", [] {
+ RecursiveFunction(1024);
+ });
+}
+
+TEST_F(TCpuProfilerTest, DlOpen)
+{
+ RunUnderProfiler("dlopen.pb.gz", [] {
+ auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso/libtestso.so");
+
+ auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY);
+ ASSERT_TRUE(dl);
+
+ auto sym = dlsym(dl, "CallNext");
+ ASSERT_TRUE(sym);
+
+ auto callNext = reinterpret_cast<void(*)(void(*)())>(sym);
+ callNext(BurnCpu<0>);
+ });
+}
+
+TEST_F(TCpuProfilerTest, DlClose)
+{
+ RunUnderProfiler("dlclose.pb.gz", [] {
+ auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso1/libtestso1.so");
+
+ auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY);
+ ASSERT_TRUE(dl);
+
+ auto sym = dlsym(dl, "CallOtherNext");
+ ASSERT_TRUE(sym);
+
+ auto callNext = reinterpret_cast<void(*)(void(*)())>(sym);
+ callNext(BurnCpu<0>);
+
+ ASSERT_EQ(dlclose(dl), 0);
+ });
+}
+
+void ReadUrandom()
+{
+ TIFStream input("/dev/urandom");
+
+ std::array<char, 1 << 20> buffer;
+
+ for (int i = 0; i < 100; i++) {
+ input.Read(buffer.data(), buffer.size());
+ }
+}
+
+TEST_F(TCpuProfilerTest, Syscalls)
+{
+ RunUnderProfiler("syscalls.pb.gz", [] {
+ ReadUrandom();
+ });
+}
+
+TEST_F(TCpuProfilerTest, VDSO)
+{
+ RunUnderProfiler("vdso.pb.gz", [] {
+ auto now = TInstant::Now();
+ while (TInstant::Now() < now + TDuration::MilliSeconds(100))
+ { }
+ }, false);
+}
+
+TEST_F(TCpuProfilerTest, ProfilerTags)
+{
+ auto userTag = New<TProfilerTag>("user", "prime");
+ auto intTag = New<TProfilerTag>("block_size", 1024);
+
+ RunUnderProfiler("tags.pb.gz", [&] {
+ {
+ TCpuProfilerTagGuard guard(userTag);
+ BurnCpu<0>();
+ }
+ {
+ TCpuProfilerTagGuard guard(intTag);
+ BurnCpu<1>();
+ }
+ {
+ TCpuProfilerTagGuard guard(userTag);
+ TCpuProfilerTagGuard secondGuard(intTag);
+ BurnCpu<2>();
+ }
+ });
+}
+
+TEST_F(TCpuProfilerTest, MultipleProfilers)
+{
+ TCpuProfiler profiler, secondProfiler;
+
+ profiler.Start();
+ EXPECT_THROW(secondProfiler.Start(), std::exception);
+}
+
+TEST_F(TCpuProfilerTest, TraceContext)
+{
+ RunUnderProfiler("trace_context.pb.gz", [] {
+ auto actionQueue = New<TActionQueue>("CpuProfileTest");
+
+ BIND([] {
+ auto rootTraceContext = TTraceContext::NewRoot("");
+ rootTraceContext->AddProfilingTag("user", "prime");
+ TCurrentTraceContextGuard guard(rootTraceContext);
+
+ auto asyncSubrequest = BIND([&] {
+ TChildTraceContextGuard guard("");
+ AnnotateTraceContext([] (const auto traceContext) {
+ traceContext->AddProfilingTag("table", "//foo");
+ });
+
+ BurnCpu<0>();
+ })
+ .AsyncVia(GetCurrentInvoker())
+ .Run();
+
+ BurnCpu<1>();
+ WaitFor(asyncSubrequest)
+ .ThrowOnError();
+ })
+ .AsyncVia(actionQueue->GetInvoker())
+ .Run()
+ .Get();
+
+ actionQueue->Shutdown();
+ });
+}
+
+TEST_F(TCpuProfilerTest, SlowActions)
+{
+ static const TString WorkerThreadName = "Heavy";
+ static const auto TraceThreshold = TDuration::MilliSeconds(50);
+
+ const std::vector<TSampleFilter> filters = {
+ GetActionMinExecTimeFilter(TraceThreshold),
+ };
+
+ auto busyWait = [] (TDuration duration) {
+ auto now = TInstant::Now();
+ while (TInstant::Now() < now + duration)
+ { }
+ };
+
+ auto traceContext = NTracing::TTraceContext::NewRoot("Test");
+
+ const bool ExpectEmptyTraces = true;
+
+ auto threadPool = CreateThreadPool(2, WorkerThreadName);
+
+ // No slow actions.
+ RunUnderProfiler("slow_actions_empty.pb.gz", [&] {
+ NTracing::TTraceContextGuard guard(traceContext);
+ auto future = BIND(busyWait, TraceThreshold / 2)
+ .AsyncVia(threadPool->GetInvoker())
+ .Run();
+ future.Get();
+ },
+ true,
+ filters,
+ ExpectEmptyTraces);
+
+ // Slow actions = non empty traces.
+ RunUnderProfiler("slow_actions.pb.gz", [&] {
+ NTracing::TTraceContextGuard guard(traceContext);
+ auto future = BIND(busyWait, TraceThreshold * 3)
+ .AsyncVia(threadPool->GetInvoker())
+ .Run();
+ future.Get();
+ },
+ true,
+ filters);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp
new file mode 100644
index 0000000000..42dcf6e802
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp
@@ -0,0 +1,226 @@
+#include <gtest/gtest.h>
+
+#include <yt/yt/library/ytprof/heap_profiler.h>
+#include <yt/yt/library/ytprof/symbolize.h>
+#include <yt/yt/library/ytprof/profile.h>
+
+#include <yt/yt/core/actions/current_invoker.h>
+#include <yt/yt/core/actions/invoker_detail.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+
+#include <yt/yt/core/misc/lazy_ptr.h>
+
+#include <yt/yt/core/tracing/allocation_tags.h>
+#include <yt/yt/core/tracing/trace_context.h>
+
+#include <library/cpp/testing/common/env.h>
+
+#include <util/string/cast.h>
+#include <util/stream/file.h>
+#include <util/generic/hash_set.h>
+#include <util/datetime/base.h>
+#include <util/generic/size_literals.h>
+
+#include <tcmalloc/common.h>
+
+#include <absl/debugging/stacktrace.h>
+
+namespace NYT::NYTProf {
+namespace {
+
+using namespace NTracing;
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr auto MemoryAllocationTag = "memory_allocation_tag";
+const std::vector<TString> MemoryAllocationTags = {"0", "1", "2", "3", "4", "5", "6", "7"};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <size_t Index>
+Y_NO_INLINE auto BlowHeap()
+{
+ std::vector<TString> data;
+ for (int i = 0; i < 10240; i++) {
+ data.push_back(TString(1024, 'x'));
+ }
+ return data;
+}
+
+TEST(HeapProfiler, ReadProfile)
+{
+ absl::SetStackUnwinder(AbslStackUnwinder);
+ tcmalloc::MallocExtension::SetProfileSamplingRate(256_KB);
+
+ auto token = tcmalloc::MallocExtension::StartAllocationProfiling();
+
+ EnableMemoryProfilingTags();
+ auto traceContext = TTraceContext::NewRoot("Root");
+ TTraceContextGuard guard(traceContext);
+
+ traceContext->SetAllocationTags({{"user", "first"}, {"sometag", "my"}});
+
+ auto h0 = BlowHeap<0>();
+
+ auto tag = TMemoryTag(1);
+ traceContext->SetAllocationTags({{"user", "second"}, {"sometag", "notmy"}, {MemoryAllocationTag, ToString(tag)}});
+ auto currentTag = traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag);
+ ASSERT_EQ(currentTag, tag);
+
+ auto h1 = BlowHeap<1>();
+
+ traceContext->ClearAllocationTagsPtr();
+
+ auto h2 = BlowHeap<2>();
+ h2.clear();
+
+ auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, ToString(tag));
+ ASSERT_GE(usage, 5_MB);
+
+ auto dumpProfile = [] (auto name, auto type) {
+ auto profile = ReadHeapProfile(type);
+
+ TFileOutput output(GetOutputPath() / name);
+ WriteProfile(&output, profile);
+ output.Finish();
+ };
+
+ dumpProfile("heap.pb.gz", tcmalloc::ProfileType::kHeap);
+ dumpProfile("peak.pb.gz", tcmalloc::ProfileType::kPeakHeap);
+ dumpProfile("fragmentation.pb.gz", tcmalloc::ProfileType::kFragmentation);
+ dumpProfile("allocations.pb.gz", tcmalloc::ProfileType::kAllocations);
+
+ auto profile = std::move(token).Stop();
+
+ TFileOutput output(GetOutputPath() / "allocations.pb.gz");
+ WriteProfile(&output, ConvertAllocationProfile(profile));
+ output.Finish();
+}
+
+TEST(HeapProfiler, AllocationTagsWithMemoryTag)
+{
+ EnableMemoryProfilingTags();
+ auto traceContext = TTraceContext::NewRoot("Root");
+ TTraceContextGuard guard(traceContext);
+
+ ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), std::nullopt);
+ traceContext->SetAllocationTags({{"user", "first user"}, {MemoryAllocationTag, MemoryAllocationTags[0]}});
+ ASSERT_EQ(traceContext->FindAllocationTag<TString>("user"), "first user");
+ ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), MemoryAllocationTags[0]);
+
+ std::vector<std::vector<TString>> heap;
+ heap.push_back(BlowHeap<0>());
+
+ traceContext->SetAllocationTags({{"user", "second user"}, {MemoryAllocationTag, MemoryAllocationTags[1]}});
+ ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag), 1);
+
+ heap.push_back(BlowHeap<1>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[0]);
+
+ auto usage1 = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[1]);
+
+ ASSERT_NEAR(usage1, 12_MB, 8_MB);
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[2]);
+ ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), MemoryAllocationTags[2]);
+
+ {
+ volatile auto h = BlowHeap<2>();
+ }
+
+ traceContext->ClearAllocationTagsPtr();
+ ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), std::nullopt);
+
+ heap.push_back(BlowHeap<0>());
+
+ {
+ auto snapshot = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag);
+ ASSERT_EQ(snapshot[MemoryAllocationTags[1]], usage1);
+ ASSERT_LE(snapshot[MemoryAllocationTags[2]], 1_MB);
+ }
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[6]);
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[3]);
+ heap.push_back(BlowHeap<3>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[4]);
+ heap.push_back(BlowHeap<4>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[7]);
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[5]);
+ heap.push_back(BlowHeap<5>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[4]);
+ heap.push_back(BlowHeap<4>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[7]);
+
+ traceContext->SetAllocationTagsPtr(nullptr);
+
+ auto snapshot = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag);
+
+ constexpr auto maxDifference = 10_MB;
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[1]], snapshot[MemoryAllocationTags[3]], maxDifference);
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[3]], snapshot[MemoryAllocationTags[5]], maxDifference);
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[1]], snapshot[MemoryAllocationTags[5]], maxDifference);
+
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], 20_MB, 15_MB);
+
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[1]] + snapshot[MemoryAllocationTags[3]], 2 * maxDifference);
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[1]] + snapshot[MemoryAllocationTags[5]], 2 * maxDifference);
+ ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[3]] + snapshot[MemoryAllocationTags[5]], 2 * maxDifference);
+
+ ASSERT_LE(snapshot[MemoryAllocationTags[6]], 1_MB);
+ ASSERT_LE(snapshot[MemoryAllocationTags[7]], 1_MB);
+}
+
+template <size_t Index>
+Y_NO_INLINE auto BlowHeap(int64_t megabytes)
+{
+ std::vector<TString> data;
+ megabytes <<= 10;
+ for (int64_t i = 0; i < megabytes; i++) {
+ data.push_back(TString( 1024, 'x'));
+ }
+ return data;
+}
+
+TEST(HeapProfiler, HugeAllocationsTagsWithMemoryTag)
+{
+ EnableMemoryProfilingTags();
+ auto traceContext = TTraceContext::NewRoot("Root");
+ TCurrentTraceContextGuard guard(traceContext);
+
+ std::vector<std::vector<TString>> heap;
+
+ heap.push_back(BlowHeap<0>());
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[1]);
+ ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag), 1);
+
+ heap.push_back(BlowHeap<1>(100));
+
+ {
+ traceContext->SetAllocationTagsPtr(nullptr);
+ auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[1]);
+ ASSERT_GE(usage, 100_MB);
+ ASSERT_LE(usage, 150_MB);
+ }
+
+ traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[2]);
+ heap.push_back(BlowHeap<1>(1000));
+
+ traceContext->SetAllocationTagsPtr(nullptr);
+ auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[2]);
+ ASSERT_GE(usage, 1000_MB);
+ ASSERT_LE(usage, 1300_MB);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/unittests/queue_ut.cpp b/yt/yt/library/ytprof/unittests/queue_ut.cpp
new file mode 100644
index 0000000000..f44ae8da32
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/queue_ut.cpp
@@ -0,0 +1,66 @@
+#include <gtest/gtest.h>
+
+#include <yt/yt/library/ytprof/queue.h>
+
+namespace NYT::NYTProf {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(StaticQueue, PushPop)
+{
+ TStaticQueue queue(10);
+
+ int a, b, c;
+ void* aptr = reinterpret_cast<void*>(&a);
+ void* bptr = reinterpret_cast<void*>(&b);
+ void* cptr = reinterpret_cast<void*>(&c);
+
+ std::vector<void*> backtrace;
+ auto getBacktrace = [&] () -> std::pair<void*, bool> {
+ if (backtrace.empty()) {
+ return {nullptr, false};
+ }
+
+ auto ip = backtrace.front();
+ backtrace.erase(backtrace.begin());
+ return {ip, true};
+ };
+
+ ASSERT_TRUE(queue.TryPush(getBacktrace));
+
+ backtrace = {aptr, bptr, cptr};
+ ASSERT_TRUE(queue.TryPush(getBacktrace));
+
+ backtrace.push_back(aptr);
+ ASSERT_TRUE(queue.TryPush(getBacktrace));
+
+ auto readBacktrace = [&] (void *ip) {
+ backtrace.push_back(ip);
+ };
+
+ ASSERT_TRUE(queue.TryPop(readBacktrace));
+ ASSERT_EQ(backtrace, std::vector<void*>{});
+
+ backtrace.clear();
+ ASSERT_TRUE(queue.TryPop(readBacktrace));
+ ASSERT_EQ(backtrace, (std::vector<void*>{aptr, bptr, cptr}));
+
+ backtrace.clear();
+ ASSERT_TRUE(queue.TryPop(readBacktrace));
+ ASSERT_EQ(backtrace, (std::vector<void*>{aptr}));
+
+ ASSERT_FALSE(queue.TryPop(readBacktrace));
+}
+
+TEST(StaticQueue, Overflow)
+{
+ TStaticQueue queue(10);
+
+ ASSERT_FALSE(queue.TryPush([] () -> std::pair<void*, bool> {
+ return {nullptr, true};
+ }));
+}
+
+} // namespace
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp
new file mode 100644
index 0000000000..1cf315155c
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp
@@ -0,0 +1,172 @@
+#include <gtest/gtest.h>
+
+#include <library/cpp/testing/common/env.h>
+
+#include <yt/yt/library/ytprof/spinlock_profiler.h>
+#include <yt/yt/library/ytprof/symbolize.h>
+#include <yt/yt/library/ytprof/profile.h>
+#include <yt/yt/library/ytprof/external_pprof.h>
+
+#include <tcmalloc/malloc_extension.h>
+
+#include <library/cpp/yt/threading/spin_lock.h>
+
+#include <util/string/cast.h>
+#include <util/stream/file.h>
+#include <util/datetime/base.h>
+#include <util/system/compiler.h>
+#include <util/system/shellcommand.h>
+#include <util/thread/lfstack.h>
+#include <util/generic/size_literals.h>
+
+namespace NYT::NYTProf {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class TProfiler>
+void RunUnderProfiler(const TString& name, std::function<void()> work, bool checkSamples = true)
+{
+ TSpinlockProfilerOptions options;
+ options.ProfileFraction = 10;
+
+ TProfiler profiler(options);
+
+ profiler.Start();
+
+ work();
+
+ profiler.Stop();
+
+ auto profile = profiler.ReadProfile();
+ if (checkSamples) {
+ ASSERT_NE(0, profile.sample_size());
+ }
+
+ Symbolize(&profile, true);
+ AddBuildInfo(&profile, TBuildInfo::GetDefault());
+ SymbolizeByExternalPProf(&profile, TSymbolizationOptions{
+ .TmpDir = GetOutputPath(),
+ .KeepTmpDir = true,
+ .RunTool = [] (const std::vector<TString>& args) {
+ TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}};
+ command.Run();
+
+ EXPECT_TRUE(command.GetExitCode() == 0)
+ << command.GetError();
+ },
+ });
+
+ TFileOutput output(GetOutputPath() / name);
+ WriteProfile(&output, profile);
+ output.Finish();
+}
+
+class TSpinlockProfilerTest
+ : public ::testing::Test
+{
+ void SetUp() override
+ {
+ if (!IsProfileBuild()) {
+ GTEST_SKIP() << "rebuild with --build=profile";
+ }
+ }
+};
+
+TEST_F(TSpinlockProfilerTest, PageHeapLock)
+{
+ RunUnderProfiler<TSpinlockProfiler>("pageheap_lock.pb.gz", [] {
+ std::atomic<bool> Stop = false;
+
+ std::thread release([&] {
+ while (!Stop) {
+ tcmalloc::MallocExtension::ReleaseMemoryToSystem(1_GB);
+ }
+ });
+
+ std::vector<std::thread> allocators;
+ for (int i = 0; i < 16; i++) {
+ allocators.emplace_back([&] {
+ while (!Stop) {
+ auto ptr = malloc(4_MB);
+ DoNotOptimizeAway(ptr);
+ free(ptr);
+ }
+ });
+ }
+
+ Sleep(TDuration::Seconds(5));
+ Stop = true;
+
+ release.join();
+ for (auto& t : allocators) {
+ t.join();
+ }
+ });
+}
+
+
+TEST_F(TSpinlockProfilerTest, TransferCacheLock)
+{
+ RunUnderProfiler<TSpinlockProfiler>("transfer_cache_lock.pb.gz", [] {
+ std::atomic<bool> Stop = false;
+
+ TLockFreeStack<int> stack;
+ std::thread producer([&] {
+ while (!Stop) {
+ stack.Enqueue(1);
+ }
+ });
+
+ std::thread consumer([&] {
+ while (!Stop) {
+ int value;
+ stack.Dequeue(&value);
+ }
+ });
+
+ Sleep(TDuration::Seconds(5));
+
+ Stop = true;
+ producer.join();
+ consumer.join();
+ });
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST_F(TSpinlockProfilerTest, YTLocks)
+{
+ RunUnderProfiler<TBlockingProfiler>("ytlock.pb.gz", [] {
+ std::atomic<bool> Stop = false;
+
+ NThreading::TSpinLock lock;
+ std::thread slow([&] {
+ while (!Stop) {
+ lock.Acquire();
+ Sleep(TDuration::MilliSeconds(10));
+ lock.Release();
+ Sleep(TDuration::MilliSeconds(10));
+ }
+ });
+
+ std::thread fast([&] {
+ while (!Stop) {
+ lock.Acquire();
+ lock.Release();
+ }
+ });
+
+ Sleep(TDuration::Seconds(5));
+
+ Stop = true;
+ slow.join();
+ fast.join();
+ });
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+
+} // namespace
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp b/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp
new file mode 100644
index 0000000000..646ecaafad
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp
@@ -0,0 +1,83 @@
+#include <gtest/gtest.h>
+
+#include <yt/yt/library/ytprof/symbolize.h>
+#include <yt/yt/library/ytprof/build_info.h>
+
+namespace NYT::NYTProf {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_NO_INLINE void* GetIP()
+{
+ return __builtin_return_address(0);
+}
+
+TEST(Symbolize, EmptyProfile)
+{
+ NProto::Profile profile;
+ profile.add_string_table();
+
+ Symbolize(&profile);
+ AddBuildInfo(&profile, TBuildInfo::GetDefault());
+}
+
+TEST(Symbolize, SingleLocation)
+{
+ NProto::Profile profile;
+ profile.add_string_table();
+
+ auto thisIP = GetIP();
+
+ {
+ auto location = profile.add_location();
+ location->set_address(reinterpret_cast<ui64>(thisIP));
+
+ auto line = location->add_line();
+ line->set_function_id(reinterpret_cast<ui64>(thisIP));
+
+ auto function = profile.add_function();
+ function->set_id(reinterpret_cast<ui64>(thisIP));
+ }
+
+ Symbolize(&profile);
+
+ ASSERT_EQ(1, profile.function_size());
+ auto function = profile.function(0);
+
+ auto name = profile.string_table(function.name());
+ ASSERT_TRUE(name.find("SingleLocation") != TString::npos)
+ << "function name is " << name;
+}
+
+TEST(Symbolize, GetBuildId)
+{
+ if (!IsProfileBuild()) {
+ GTEST_SKIP();
+ }
+
+ return;
+
+ auto buildId = GetBuildId();
+ ASSERT_TRUE(buildId);
+ ASSERT_NE(*buildId, TString{""});
+}
+
+TEST(BuildInfo, Test)
+{
+ if (!IsProfileBuild()) {
+ GTEST_SKIP();
+ }
+
+ auto info = TBuildInfo::GetDefault();
+ if (IsProfileBuild()) {
+ ASSERT_EQ(info.BuildType, "profile");
+ }
+
+ ASSERT_NE(info.ArcRevision, "");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NYTProf
diff --git a/yt/yt/library/ytprof/unittests/testso/testso.cpp b/yt/yt/library/ytprof/unittests/testso/testso.cpp
new file mode 100644
index 0000000000..4209d5f6cc
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/testso/testso.cpp
@@ -0,0 +1,9 @@
+#include <atomic>
+
+static std::atomic<int> CallCount;
+
+extern "C" void CallNext(void (*next)())
+{
+ next();
+ CallCount++;
+}
diff --git a/yt/yt/library/ytprof/unittests/testso/ya.make b/yt/yt/library/ytprof/unittests/testso/ya.make
new file mode 100644
index 0000000000..8851d8eed6
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/testso/ya.make
@@ -0,0 +1,7 @@
+DLL(testso)
+
+SRCS(
+ testso.cpp
+)
+
+END()
diff --git a/yt/yt/library/ytprof/unittests/testso1/testso.cpp b/yt/yt/library/ytprof/unittests/testso1/testso.cpp
new file mode 100644
index 0000000000..56e348c020
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/testso1/testso.cpp
@@ -0,0 +1,9 @@
+#include <atomic>
+
+static std::atomic<int> CallCount;
+
+extern "C" void CallOtherNext(void (*next)())
+{
+ next();
+ CallCount++;
+}
diff --git a/yt/yt/library/ytprof/unittests/testso1/ya.make b/yt/yt/library/ytprof/unittests/testso1/ya.make
new file mode 100644
index 0000000000..d0d2f90143
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/testso1/ya.make
@@ -0,0 +1,7 @@
+DLL(testso1)
+
+SRCS(
+ testso.cpp
+)
+
+END()
diff --git a/yt/yt/library/ytprof/unittests/ya.make b/yt/yt/library/ytprof/unittests/ya.make
new file mode 100644
index 0000000000..8c8bce98f5
--- /dev/null
+++ b/yt/yt/library/ytprof/unittests/ya.make
@@ -0,0 +1,43 @@
+GTEST()
+
+INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
+
+SRCS(
+ symbolizer_ut.cpp
+ cpu_profiler_ut.cpp
+ heap_profiler_ut.cpp
+ spinlock_profiler_ut.cpp
+ queue_ut.cpp
+)
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc)
+
+PEERDIR(
+ yt/yt/library/ytprof
+ yt/yt/library/profiling
+ yt/yt/core
+)
+
+IF (OS_LINUX)
+ LDFLAGS("-Wl,--build-id=sha1")
+ENDIF()
+
+DEPENDS(
+ yt/yt/library/ytprof/unittests/testso
+ yt/yt/library/ytprof/unittests/testso1
+)
+
+IF (BUILD_TYPE != "release" AND BUILD_TYPE != "relwithdebinfo")
+ CFLAGS(-DYTPROF_DEBUG_BUILD)
+ENDIF()
+
+SIZE(MEDIUM)
+
+ALLOCATOR(TCMALLOC_256K)
+
+END()
+
+RECURSE(
+ testso
+ testso1
+)