diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-12 21:25:31 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-12 21:39:54 +0300 |
commit | d28c55ab25cc8cedab8a5f4736c0d66e88b3da95 (patch) | |
tree | 73d373709b74fa2baaa4fe02a40a77c0a5baf6b7 /yt | |
parent | 35b17f4f3b6e0ed855e7e47d3f1eb57470388a2c (diff) | |
download | ydb-d28c55ab25cc8cedab8a5f4736c0d66e88b3da95.tar.gz |
Intermediate changes
Diffstat (limited to 'yt')
210 files changed, 16794 insertions, 4 deletions
diff --git a/yt/yql/plugin/bridge/interface.h b/yt/yql/plugin/bridge/interface.h index 08e765b3d0..ba08b657aa 100644 --- a/yt/yql/plugin/bridge/interface.h +++ b/yt/yql/plugin/bridge/interface.h @@ -9,8 +9,18 @@ //////////////////////////////////////////////////////////////////////////////// +// NB(mpereskokova): don't forget to update min_required_abi_version at yt/yql/agent/config.cpp and abi_version in yt/yql/plugin/dynamic/impl.cpp during breaking changes +using TFuncBridgeGetABIVersion = ssize_t(); + +//////////////////////////////////////////////////////////////////////////////// + struct TBridgeYqlPluginOptions { + ssize_t RequiredABIVersion; + + const char* SingletonsConfig; + ssize_t SingletonsConfigLength; + const char* MRJobBinary; const char* UdfDirectory; @@ -95,6 +105,7 @@ using TFuncBridgeGetProgress = TBridgeQueryResult*(TBridgeYqlPlugin* plugin, con XX(BridgeFreeYqlPlugin) \ XX(BridgeFreeQueryResult) \ XX(BridgeRun) \ - XX(BridgeGetProgress) + XX(BridgeGetProgress) \ + XX(BridgeGetABIVersion) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yql/plugin/bridge/plugin.cpp b/yt/yql/plugin/bridge/plugin.cpp index 18e2e9e639..5c461616e9 100644 --- a/yt/yql/plugin/bridge/plugin.cpp +++ b/yt/yql/plugin/bridge/plugin.cpp @@ -72,7 +72,12 @@ public: ? options.DefaultCluster->data() : nullptr; + TString singletonsConfig = options.SingletonsConfig ? options.SingletonsConfig.ToString() : "{}"; + TBridgeYqlPluginOptions bridgeOptions { + .RequiredABIVersion = options.RequiredABIVersion, + .SingletonsConfig = singletonsConfig.data(), + .SingletonsConfigLength = static_cast<int>(singletonsConfig.size()), .MRJobBinary = options.MRJobBinary.data(), .UdfDirectory = options.UdfDirectory.data(), .ClusterCount = ssize(bridgeClusters), diff --git a/yt/yql/plugin/dynamic/dylib.exports b/yt/yql/plugin/dynamic/dylib.exports index fc77529eaf..b979ce7e18 100644 --- a/yt/yql/plugin/dynamic/dylib.exports +++ b/yt/yql/plugin/dynamic/dylib.exports @@ -4,6 +4,7 @@ BridgeFreeYqlPlugin BridgeFreeQueryResult BridgeRun BridgeGetProgress +BridgeGetABIVersion # YQL <-> YQL UDFs interface. UdfAllocateWithSize diff --git a/yt/yql/plugin/dynamic/impl.cpp b/yt/yql/plugin/dynamic/impl.cpp index 5966a8a8ec..6beb5bf8c2 100644 --- a/yt/yql/plugin/dynamic/impl.cpp +++ b/yt/yql/plugin/dynamic/impl.cpp @@ -10,8 +10,17 @@ extern "C" { //////////////////////////////////////////////////////////////////////////////// +ssize_t BridgeGetABIVersion() +{ + return 0; +} + TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOptions) { + YT_VERIFY(bridgeOptions->RequiredABIVersion == BridgeGetABIVersion()); + + static const TYsonString EmptyMap = TYsonString(TString("{}")); + THashMap<TString, TString> clusters; for (auto clusterIndex = 0; clusterIndex < bridgeOptions->ClusterCount; ++clusterIndex) { const auto& Cluster = bridgeOptions->Clusters[clusterIndex]; @@ -20,9 +29,14 @@ TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOpt auto operationAttributes = bridgeOptions->OperationAttributes ? TYsonString(TString(bridgeOptions->OperationAttributes, bridgeOptions->OperationAttributesLength)) - : TYsonString(); + : EmptyMap; + + auto singletonsConfig = bridgeOptions->SingletonsConfig + ? TYsonString(TString(bridgeOptions->SingletonsConfig, bridgeOptions->SingletonsConfigLength)) + : EmptyMap; TYqlPluginOptions options{ + .SingletonsConfig = singletonsConfig, .MRJobBinary = TString(bridgeOptions->MRJobBinary), .UdfDirectory = TString(bridgeOptions->UdfDirectory), .Clusters = std::move(clusters), diff --git a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt index 7c002160ee..04a6b539a5 100644 --- a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt +++ b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt @@ -20,6 +20,7 @@ target_link_libraries(yql-plugin-native PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper @@ -65,6 +66,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper diff --git a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt index b85008e3c5..b9ac27203c 100644 --- a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt +++ b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt @@ -21,6 +21,7 @@ target_link_libraries(yql-plugin-native PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper @@ -67,6 +68,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper diff --git a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt index b85008e3c5..b9ac27203c 100644 --- a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt +++ b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt @@ -21,6 +21,7 @@ target_link_libraries(yql-plugin-native PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper @@ -67,6 +68,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper diff --git a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt index 7c002160ee..04a6b539a5 100644 --- a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt +++ b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt @@ -20,6 +20,7 @@ target_link_libraries(yql-plugin-native PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper @@ -65,6 +66,7 @@ target_link_libraries(yql-plugin-native.global PUBLIC cpp-yson-node cpp-mapreduce-client cpp-mapreduce-common + yt-library-program library-yql-ast yql-sql-pg yql-parser-pg_wrapper diff --git a/yt/yql/plugin/native/plugin.cpp b/yt/yql/plugin/native/plugin.cpp index 9bb6c2261e..38ccbbdf3b 100644 --- a/yt/yql/plugin/native/plugin.cpp +++ b/yt/yql/plugin/native/plugin.cpp @@ -24,7 +24,11 @@ #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/backtrace/backtrace.h> -#include <yt/cpp/mapreduce/interface/config.h> +#include <yt/yt/core/ytree/convert.h> + +#include <yt/yt/library/program/config.h> +#include <yt/yt/library/program/helpers.h> + #include <yt/cpp/mapreduce/interface/logging/logger.h> #include <library/cpp/yt/threading/rw_spin_lock.h> @@ -115,6 +119,9 @@ public: TYqlPlugin(TYqlPluginOptions& options) { try { + auto singletonsConfig = NYTree::ConvertTo<TSingletonsConfigPtr>(options.SingletonsConfig); + ConfigureSingletons(singletonsConfig); + NYql::NLog::InitLogger(std::move(options.LogBackend)); auto& logger = NYql::NLog::YqlLogger(); diff --git a/yt/yql/plugin/native/ya.make b/yt/yql/plugin/native/ya.make index 15f3851411..fe1a657c69 100644 --- a/yt/yql/plugin/native/ya.make +++ b/yt/yql/plugin/native/ya.make @@ -13,6 +13,7 @@ PEERDIR( library/cpp/yson/node yt/cpp/mapreduce/client yt/cpp/mapreduce/common + yt/yt/library/program ydb/library/yql/ast ydb/library/yql/sql/pg ydb/library/yql/parser/pg_wrapper diff --git a/yt/yql/plugin/plugin.h b/yt/yql/plugin/plugin.h index 2d2c45d54c..68b01da922 100644 --- a/yt/yql/plugin/plugin.h +++ b/yt/yql/plugin/plugin.h @@ -26,6 +26,10 @@ using TQueryId = TGuid; class TYqlPluginOptions { public: + int RequiredABIVersion; + + TYsonString SingletonsConfig; + TString MRJobBinary = "./mrjob"; TString UdfDirectory; diff --git a/yt/yt/core/CMakeLists.darwin-x86_64.txt b/yt/yt/core/CMakeLists.darwin-x86_64.txt index 0b4da77950..5b4d4b633f 100644 --- a/yt/yt/core/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/core/CMakeLists.darwin-x86_64.txt @@ -9,6 +9,8 @@ add_subdirectory(http) add_subdirectory(https) add_subdirectory(misc) +add_subdirectory(rpc) +add_subdirectory(service_discovery) add_library(yt-yt-core) target_compile_options(yt-yt-core PRIVATE diff --git a/yt/yt/core/CMakeLists.linux-aarch64.txt b/yt/yt/core/CMakeLists.linux-aarch64.txt index 51eeb4ff56..640e862300 100644 --- a/yt/yt/core/CMakeLists.linux-aarch64.txt +++ b/yt/yt/core/CMakeLists.linux-aarch64.txt @@ -9,6 +9,8 @@ add_subdirectory(http) add_subdirectory(https) add_subdirectory(misc) +add_subdirectory(rpc) +add_subdirectory(service_discovery) add_library(yt-yt-core) target_compile_options(yt-yt-core PRIVATE diff --git a/yt/yt/core/CMakeLists.linux-x86_64.txt b/yt/yt/core/CMakeLists.linux-x86_64.txt index 164e626f9b..ab2ddf3548 100644 --- a/yt/yt/core/CMakeLists.linux-x86_64.txt +++ b/yt/yt/core/CMakeLists.linux-x86_64.txt @@ -9,6 +9,8 @@ add_subdirectory(http) add_subdirectory(https) add_subdirectory(misc) +add_subdirectory(rpc) +add_subdirectory(service_discovery) add_library(yt-yt-core) target_compile_options(yt-yt-core PRIVATE diff --git a/yt/yt/core/CMakeLists.windows-x86_64.txt b/yt/yt/core/CMakeLists.windows-x86_64.txt index 76ae848b35..741f6b7a8b 100644 --- a/yt/yt/core/CMakeLists.windows-x86_64.txt +++ b/yt/yt/core/CMakeLists.windows-x86_64.txt @@ -9,6 +9,8 @@ add_subdirectory(http) add_subdirectory(https) add_subdirectory(misc) +add_subdirectory(rpc) +add_subdirectory(service_discovery) add_library(yt-yt-core) target_compile_options(yt-yt-core PRIVATE diff --git a/yt/yt/core/rpc/CMakeLists.txt b/yt/yt/core/rpc/CMakeLists.txt new file mode 100644 index 0000000000..68ea682099 --- /dev/null +++ b/yt/yt/core/rpc/CMakeLists.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(grpc) diff --git a/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..eb0e7def69 --- /dev/null +++ b/yt/yt/core/rpc/grpc/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,64 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(core-rpc-grpc) +target_compile_options(core-rpc-grpc PRIVATE + -Wdeprecated-this-capture +) +target_include_directories(core-rpc-grpc PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_include_directories(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc +) +target_link_libraries(core-rpc-grpc PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-grpc + contrib-libs-protobuf +) +target_proto_messages(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto +) +target_sources(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp +) +target_proto_addincls(core-rpc-grpc + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(core-rpc-grpc + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt b/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..c3945011ba --- /dev/null +++ b/yt/yt/core/rpc/grpc/CMakeLists.linux-aarch64.txt @@ -0,0 +1,65 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(core-rpc-grpc) +target_compile_options(core-rpc-grpc PRIVATE + -Wdeprecated-this-capture +) +target_include_directories(core-rpc-grpc PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_include_directories(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc +) +target_link_libraries(core-rpc-grpc PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-grpc + contrib-libs-protobuf +) +target_proto_messages(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto +) +target_sources(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp +) +target_proto_addincls(core-rpc-grpc + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(core-rpc-grpc + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..c3945011ba --- /dev/null +++ b/yt/yt/core/rpc/grpc/CMakeLists.linux-x86_64.txt @@ -0,0 +1,65 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(core-rpc-grpc) +target_compile_options(core-rpc-grpc PRIVATE + -Wdeprecated-this-capture +) +target_include_directories(core-rpc-grpc PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_include_directories(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc +) +target_link_libraries(core-rpc-grpc PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-grpc + contrib-libs-protobuf +) +target_proto_messages(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto +) +target_sources(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp +) +target_proto_addincls(core-rpc-grpc + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(core-rpc-grpc + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/core/rpc/grpc/CMakeLists.txt b/yt/yt/core/rpc/grpc/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/core/rpc/grpc/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt b/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..629594878e --- /dev/null +++ b/yt/yt/core/rpc/grpc/CMakeLists.windows-x86_64.txt @@ -0,0 +1,61 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(core-rpc-grpc) +target_include_directories(core-rpc-grpc PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_include_directories(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc +) +target_link_libraries(core-rpc-grpc PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-grpc + contrib-libs-protobuf +) +target_proto_messages(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/proto/grpc.proto +) +target_sources(core-rpc-grpc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/public.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/server.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/rpc/grpc/channel.cpp +) +target_proto_addincls(core-rpc-grpc + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(core-rpc-grpc + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/core/service_discovery/CMakeLists.txt b/yt/yt/core/service_discovery/CMakeLists.txt new file mode 100644 index 0000000000..7f1d154d54 --- /dev/null +++ b/yt/yt/core/service_discovery/CMakeLists.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(yp) diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..78ca098926 --- /dev/null +++ b/yt/yt/core/service_discovery/yp/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-service_discovery-yp) +target_compile_options(core-service_discovery-yp PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(core-service_discovery-yp PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + core-rpc-grpc +) +target_sources(core-service_discovery-yp PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp +) diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..eb8048e099 --- /dev/null +++ b/yt/yt/core/service_discovery/yp/CMakeLists.linux-aarch64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-service_discovery-yp) +target_compile_options(core-service_discovery-yp PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(core-service_discovery-yp PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + core-rpc-grpc +) +target_sources(core-service_discovery-yp PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp +) diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..eb8048e099 --- /dev/null +++ b/yt/yt/core/service_discovery/yp/CMakeLists.linux-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-service_discovery-yp) +target_compile_options(core-service_discovery-yp PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(core-service_discovery-yp PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + core-rpc-grpc +) +target_sources(core-service_discovery-yp PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp +) diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.txt b/yt/yt/core/service_discovery/yp/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/core/service_discovery/yp/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt b/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..2aa18009a5 --- /dev/null +++ b/yt/yt/core/service_discovery/yp/CMakeLists.windows-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-service_discovery-yp) +target_link_libraries(core-service_discovery-yp PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + core-rpc-grpc +) +target_sources(core-service_discovery-yp PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/core/service_discovery/yp/service_discovery_dummy.cpp +) diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index e985dd82ae..043d3a1dd9 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -324,6 +324,7 @@ PEERDIR( library/cpp/ytalloc/api yt/yt/build + yt/yt/core/misc/isa_crc64 yt/yt_proto/yt/core diff --git a/yt/yt/library/CMakeLists.darwin-x86_64.txt b/yt/yt/library/CMakeLists.darwin-x86_64.txt index d26f3bae28..2debbb7626 100644 --- a/yt/yt/library/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/CMakeLists.darwin-x86_64.txt @@ -7,10 +7,14 @@ add_subdirectory(auth) +add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) +add_subdirectory(monitoring) add_subdirectory(numeric) +add_subdirectory(process) add_subdirectory(profiling) +add_subdirectory(program) add_subdirectory(quantile_digest) add_subdirectory(re2) add_subdirectory(syncmap) diff --git a/yt/yt/library/CMakeLists.linux-aarch64.txt b/yt/yt/library/CMakeLists.linux-aarch64.txt index d26f3bae28..524ffcf525 100644 --- a/yt/yt/library/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/CMakeLists.linux-aarch64.txt @@ -7,10 +7,15 @@ add_subdirectory(auth) +add_subdirectory(backtrace_introspector) +add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) +add_subdirectory(monitoring) add_subdirectory(numeric) +add_subdirectory(process) add_subdirectory(profiling) +add_subdirectory(program) add_subdirectory(quantile_digest) add_subdirectory(re2) add_subdirectory(syncmap) diff --git a/yt/yt/library/CMakeLists.linux-x86_64.txt b/yt/yt/library/CMakeLists.linux-x86_64.txt index d26f3bae28..524ffcf525 100644 --- a/yt/yt/library/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/CMakeLists.linux-x86_64.txt @@ -7,10 +7,15 @@ add_subdirectory(auth) +add_subdirectory(backtrace_introspector) +add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) +add_subdirectory(monitoring) add_subdirectory(numeric) +add_subdirectory(process) add_subdirectory(profiling) +add_subdirectory(program) add_subdirectory(quantile_digest) add_subdirectory(re2) add_subdirectory(syncmap) diff --git a/yt/yt/library/CMakeLists.windows-x86_64.txt b/yt/yt/library/CMakeLists.windows-x86_64.txt index 20f7fe76fa..4502da4e61 100644 --- a/yt/yt/library/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/CMakeLists.windows-x86_64.txt @@ -6,7 +6,11 @@ # original buildsystem will not be accepted. +add_subdirectory(containers) +add_subdirectory(monitoring) +add_subdirectory(process) add_subdirectory(profiling) +add_subdirectory(program) add_subdirectory(syncmap) add_subdirectory(tracing) add_subdirectory(tvm) diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt b/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..215573de83 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/CMakeLists.linux-aarch64.txt @@ -0,0 +1,28 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(http) + +add_library(yt-library-backtrace_introspector) +target_compile_options(yt-library-backtrace_introspector PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-backtrace_introspector PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + backtrace-cursors-interop + backtrace-cursors-libunwind + backtrace-cursors-frame_pointer + cpp-yt-misc +) +target_sources(yt-library-backtrace_introspector PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect_linux.cpp +) diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt b/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..215573de83 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/CMakeLists.linux-x86_64.txt @@ -0,0 +1,28 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(http) + +add_library(yt-library-backtrace_introspector) +target_compile_options(yt-library-backtrace_introspector PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-backtrace_introspector PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + backtrace-cursors-interop + backtrace-cursors-libunwind + backtrace-cursors-frame_pointer + cpp-yt-misc +) +target_sources(yt-library-backtrace_introspector PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/introspect_linux.cpp +) diff --git a/yt/yt/library/backtrace_introspector/CMakeLists.txt b/yt/yt/library/backtrace_introspector/CMakeLists.txt new file mode 100644 index 0000000000..4d48dcdee6 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..5b8a9100d2 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-backtrace_introspector-http) +target_compile_options(library-backtrace_introspector-http PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-backtrace_introspector-http PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-core-http + yt-library-backtrace_introspector +) +target_sources(library-backtrace_introspector-http PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/http/handler.cpp +) diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..5b8a9100d2 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-backtrace_introspector-http) +target_compile_options(library-backtrace_introspector-http PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-backtrace_introspector-http PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-core-http + yt-library-backtrace_introspector +) +target_sources(library-backtrace_introspector-http PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/backtrace_introspector/http/handler.cpp +) diff --git a/yt/yt/library/backtrace_introspector/http/CMakeLists.txt b/yt/yt/library/backtrace_introspector/http/CMakeLists.txt new file mode 100644 index 0000000000..4d48dcdee6 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/backtrace_introspector/http/handler.cpp b/yt/yt/library/backtrace_introspector/http/handler.cpp new file mode 100644 index 0000000000..367e3105c0 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/handler.cpp @@ -0,0 +1,89 @@ +#include "handler.h" + +#include <yt/yt/core/http/server.h> + +#include <yt/yt/core/concurrency/action_queue.h> + +#include <yt/yt/library/backtrace_introspector/introspect.h> + +namespace NYT::NBacktraceIntrospector { + +using namespace NHttp; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +class THandlerBase + : public IHttpHandler +{ +public: + void HandleRequest(const IRequestPtr& /*req*/, const IResponseWriterPtr& rsp) override + { + try { + auto dumpFuture = BIND(&THandlerBase::Dump, MakeStrong(this)) + .AsyncVia(Queue_->GetInvoker()) + .Run(); + + auto dump = WaitFor(dumpFuture) + .ValueOrThrow(); + + WaitFor(rsp->WriteBody(TSharedRef::FromString(dump))) + .ThrowOnError(); + + WaitFor(rsp->Close()) + .ThrowOnError(); + } catch (const std::exception& ex) { + if (!rsp->AreHeadersFlushed()) { + rsp->SetStatus(EStatusCode::InternalServerError); + WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what()))) + .ThrowOnError(); + } + throw; + } + } + +private: + static inline const TActionQueuePtr Queue_ = New<TActionQueue>("BacktraceIntro"); + +protected: + virtual TString Dump() = 0; +}; + +class TThreadsHandler + : public THandlerBase +{ +private: + TString Dump() override + { + return FormatIntrospectionInfos(IntrospectThreads()); + } +}; + +class TFibersHandler + : public THandlerBase +{ +private: + TString Dump() override + { + return FormatIntrospectionInfos(IntrospectFibers()); + } +}; + +void Register( + const IRequestPathMatcherPtr& handlers, + const TString& prefix) +{ + handlers->Add(prefix + "/threads", New<TThreadsHandler>()); + handlers->Add(prefix + "/fibers", New<TFibersHandler>()); +} + +void Register( + const IServerPtr& server, + const TString& prefix) +{ + Register(server->GetPathMatcher(), prefix); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/http/handler.h b/yt/yt/library/backtrace_introspector/http/handler.h new file mode 100644 index 0000000000..be795b7e5d --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/handler.h @@ -0,0 +1,20 @@ +#pragma once + +#include <yt/yt/core/http/public.h> + +namespace NYT::NBacktraceIntrospector { + +//////////////////////////////////////////////////////////////////////////////// + +//! Registers introspector handlers. +void Register( + const NHttp::IRequestPathMatcherPtr& handlers, + const TString& prefix = {}); + +void Register( + const NHttp::IServerPtr& server, + const TString& prefix = {}); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/http/ya.make b/yt/yt/library/backtrace_introspector/http/ya.make new file mode 100644 index 0000000000..504d20a2e3 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/http/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + handler.cpp +) + +PEERDIR( + yt/yt/core + yt/yt/core/http + + yt/yt/library/backtrace_introspector +) + +END() diff --git a/yt/yt/library/backtrace_introspector/introspect.cpp b/yt/yt/library/backtrace_introspector/introspect.cpp new file mode 100644 index 0000000000..592c232f0f --- /dev/null +++ b/yt/yt/library/backtrace_introspector/introspect.cpp @@ -0,0 +1,216 @@ +#include "introspect.h" + +#include "private.h" + +#include <yt/yt/core/misc/finally.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/concurrency/fiber.h> +#include <yt/yt/core/concurrency/scheduler_api.h> + +#include <yt/yt/core/tracing/trace_context.h> + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/interop/interop.h> + +#include <util/system/yield.h> + +namespace NYT::NBacktraceIntrospector { + +using namespace NConcurrency; +using namespace NThreading; +using namespace NTracing; +using namespace NBacktrace; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = BacktraceIntrospectorLogger; + +//////////////////////////////////////////////////////////////////////////////// + +std::vector<TFiberIntrospectionInfo> IntrospectFibers() +{ + YT_LOG_INFO("Fiber introspection started"); + + auto fibers = TFiber::List(); + + YT_LOG_INFO("Collecting waiting fibers backtraces"); + + std::vector<TFiberIntrospectionInfo> infos; + THashSet<TFiberId> waitingFiberIds; + THashSet<TFiberId> fiberIds; + for (const auto& fiber : fibers) { + auto fiberId = fiber->GetFiberId(); + if (fiberId == InvalidFiberId) { + continue; + } + + InsertOrCrash(fiberIds, fiberId); + + EFiberState state; + if (!fiber->TryIntrospectWaiting(state, [&] { + YT_LOG_DEBUG("Waiting fiber is successfully locked for introspection (FiberId: %x)", + fiberId); + + const auto& propagatingStorage = fiber->GetPropagatingStorage(); + const auto* traceContext = TryGetTraceContextFromPropagatingStorage(propagatingStorage); + + TFiberIntrospectionInfo info{ + .State = EFiberState::Waiting, + .FiberId = fiberId, + .WaitingSince = fiber->GetWaitingSince(), + .TraceId = traceContext ? traceContext->GetTraceId() : TTraceId(), + .TraceLoggingTag = traceContext ? traceContext->GetLoggingTag() : TString(), + }; + + auto optionalContext = TrySynthesizeLibunwindContextFromMachineContext(*fiber->GetMachineContext()); + if (!optionalContext) { + YT_LOG_WARNING("Failed to synthesize libunwind context (FiberId: %x)", + fiberId); + return; + } + + TLibunwindCursor cursor(*optionalContext); + while (!cursor.IsFinished()) { + info.Backtrace.push_back(cursor.GetCurrentIP()); + cursor.MoveNext(); + } + + infos.push_back(std::move(info)); + InsertOrCrash(waitingFiberIds, fiberId); + + YT_LOG_DEBUG("Fiber introspection completed (FiberId: %x)", + info.FiberId); + })) { + YT_LOG_DEBUG("Failed to lock fiber for introspection (FiberId: %x, State: %v)", + fiberId, + state); + } + } + + YT_LOG_INFO("Collecting running fibers backtraces"); + + THashSet<TFiberId> runningFiberIds; + for (auto& info : IntrospectThreads()) { + if (info.FiberId == InvalidFiberId) { + continue; + } + + if (waitingFiberIds.contains(info.FiberId)) { + continue; + } + + if (!runningFiberIds.insert(info.FiberId).second) { + continue; + } + + infos.push_back(TFiberIntrospectionInfo{ + .State = EFiberState::Running, + .FiberId = info.FiberId, + .ThreadId = info.ThreadId, + .ThreadName = std::move(info.ThreadName), + .TraceId = info.TraceId, + .TraceLoggingTag = std::move(info.TraceLoggingTag), + .Backtrace = std::move(info.Backtrace), + }); + } + + for (const auto& fiber : fibers) { + auto fiberId = fiber->GetFiberId(); + if (fiberId == InvalidFiberId) { + continue; + } + if (runningFiberIds.contains(fiberId)) { + continue; + } + if (waitingFiberIds.contains(fiberId)) { + continue; + } + + infos.push_back(TFiberIntrospectionInfo{ + .State = fiber->GetState(), + .FiberId = fiberId, + }); + } + + YT_LOG_INFO("Fiber introspection completed"); + + return infos; +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +void FormatBacktrace(TStringBuilder* builder, const std::vector<const void*>& backtrace) +{ + if (!backtrace.empty()) { + builder->AppendString("Backtrace:\n"); + SymbolizeBacktrace( + MakeRange(backtrace), + [&] (TStringBuf str) { + builder->AppendFormat(" %v", str); + }); + } +} + +} // namespace + +TString FormatIntrospectionInfos(const std::vector<TThreadIntrospectionInfo>& infos) +{ + TStringBuilder builder; + for (const auto& info : infos) { + builder.AppendFormat("Thread id: %v\n", info.ThreadId); + builder.AppendFormat("Thread name: %v\n", info.ThreadName); + if (info.FiberId != InvalidFiberId) { + builder.AppendFormat("Fiber id: %x\n", info.FiberId); + } + if (info.TraceId) { + builder.AppendFormat("Trace id: %v\n", info.TraceId); + } + if (info.TraceLoggingTag) { + builder.AppendFormat("Trace logging tag: %v\n", info.TraceLoggingTag); + } + FormatBacktrace(&builder, info.Backtrace); + builder.AppendString("\n"); + } + return builder.Flush(); +} + +TString FormatIntrospectionInfos(const std::vector<TFiberIntrospectionInfo>& infos) +{ + TStringBuilder builder; + for (const auto& info : infos) { + builder.AppendFormat("Fiber id: %x\n", info.FiberId); + builder.AppendFormat("State: %v\n", info.State); + if (info.WaitingSince) { + builder.AppendFormat("Waiting since: %v\n", info.WaitingSince); + } + if (info.ThreadId != InvalidThreadId) { + builder.AppendFormat("Thread id: %v\n", info.ThreadId); + } + if (!info.ThreadName.empty()) { + builder.AppendFormat("Thread name: %v\n", info.ThreadName); + } + if (info.TraceId) { + builder.AppendFormat("Trace id: %v\n", info.TraceId); + } + if (info.TraceLoggingTag) { + builder.AppendFormat("Trace logging tag: %v\n", info.TraceLoggingTag); + } + FormatBacktrace(&builder, info.Backtrace); + builder.AppendString("\n"); + } + return builder.Flush(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/introspect.h b/yt/yt/library/backtrace_introspector/introspect.h new file mode 100644 index 0000000000..2be09d2ec8 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/introspect.h @@ -0,0 +1,57 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/concurrency/public.h> + +#include <yt/yt/core/threading/public.h> + +#include <yt/yt/core/tracing/public.h> + +namespace NYT::NBacktraceIntrospector { + +//////////////////////////////////////////////////////////////////////////////// +// Thread introspection API + +struct TThreadIntrospectionInfo +{ + NThreading::TThreadId ThreadId; + NConcurrency::TFiberId FiberId; + TString ThreadName; + NTracing::TTraceId TraceId; + //! Empty if no trace context is known. + TString TraceLoggingTag; + std::vector<const void*> Backtrace; +}; + +std::vector<TThreadIntrospectionInfo> IntrospectThreads(); + +//////////////////////////////////////////////////////////////////////////////// +// Fiber introspection API + +struct TFiberIntrospectionInfo +{ + NConcurrency::EFiberState State; + NConcurrency::TFiberId FiberId; + //! Zero if fiber is not waiting. + TInstant WaitingSince; + //! |InvalidThreadId| is fiber is not running. + NThreading::TThreadId ThreadId; + //! Empty if fiber is not running. + TString ThreadName; + NTracing::TTraceId TraceId; + //! Empty if no trace context is known. + TString TraceLoggingTag; + std::vector<const void*> Backtrace; +}; + +std::vector<TFiberIntrospectionInfo> IntrospectFibers(); + +//////////////////////////////////////////////////////////////////////////////// + +TString FormatIntrospectionInfos(const std::vector<TThreadIntrospectionInfo>& infos); +TString FormatIntrospectionInfos(const std::vector<TFiberIntrospectionInfo>& infos); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/introspect_dummy.cpp b/yt/yt/library/backtrace_introspector/introspect_dummy.cpp new file mode 100644 index 0000000000..e29293c7f5 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/introspect_dummy.cpp @@ -0,0 +1,14 @@ +#include "introspect.h" + +namespace NYT::NBacktraceIntrospector { + +//////////////////////////////////////////////////////////////////////////////// + +std::vector<TThreadIntrospectionInfo> IntrospectThreads() +{ + return {}; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/introspect_linux.cpp b/yt/yt/library/backtrace_introspector/introspect_linux.cpp new file mode 100644 index 0000000000..3fc1a077f6 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/introspect_linux.cpp @@ -0,0 +1,211 @@ +#include "introspect.h" + +#include "private.h" + +#include <yt/yt/core/misc/finally.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/concurrency/fiber.h> +#include <yt/yt/core/concurrency/scheduler_api.h> + +#include <yt/yt/core/tracing/trace_context.h> + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/interop/interop.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <util/system/yield.h> + +#include <sys/syscall.h> + +namespace NYT::NBacktraceIntrospector { + +using namespace NConcurrency; +using namespace NTracing; +using namespace NBacktrace; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = BacktraceIntrospectorLogger; + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +struct TStaticString +{ + TStaticString() = default; + + explicit TStaticString(TStringBuf str) + { + Length = std::min(std::ssize(str), std::ssize(Buffer)); + std::copy(str.data(), str.data() + Length, Buffer.data()); + } + + operator TString() const + { + return TString(Buffer.data(), static_cast<size_t>(Length)); + } + + std::array<char, 256> Buffer; + int Length = 0; +}; + +struct TStaticBacktrace +{ + operator std::vector<const void*>() const + { + return std::vector<const void*>(Frames.data(), Frames.data() + FrameCount); + } + + std::array<const void*, 100> Frames; + int FrameCount = 0; +}; + +struct TSignalHandlerContext +{ + TSignalHandlerContext(); + ~TSignalHandlerContext(); + + std::atomic<bool> Finished = false; + + TFiberId FiberId = {}; + TTraceId TraceId = {}; + TStaticString TraceLoggingTag; + TStaticBacktrace Backtrace; + TThreadName ThreadName = {}; + + TSafeMemoryReader* MemoryReader = Singleton<TSafeMemoryReader>(); + + void SetFinished() + { + Finished.store(true); + } + + void WaitUntilFinished() + { + while (!Finished.load()) { + ThreadYield(); + } + } +}; + +static TSignalHandlerContext* SignalHandlerContext; + +TSignalHandlerContext::TSignalHandlerContext() +{ + YT_VERIFY(!SignalHandlerContext); + SignalHandlerContext = this; +} + +TSignalHandlerContext::~TSignalHandlerContext() +{ + YT_VERIFY(SignalHandlerContext == this); + SignalHandlerContext = nullptr; +} + +void SignalHandler(int sig, siginfo_t* /*info*/, void* threadContext) +{ + YT_VERIFY(sig == SIGUSR1); + + SignalHandlerContext->FiberId = GetCurrentFiberId(); + SignalHandlerContext->ThreadName = GetCurrentThreadName(); + if (const auto* traceContext = TryGetCurrentTraceContext()) { + SignalHandlerContext->TraceId = traceContext->GetTraceId(); + SignalHandlerContext->TraceLoggingTag = TStaticString(traceContext->GetLoggingTag()); + } + + auto cursorContext = FramePointerCursorContextFromUcontext(*static_cast<const ucontext_t*>(threadContext)); + TFramePointerCursor cursor(SignalHandlerContext->MemoryReader, cursorContext); + while (!cursor.IsFinished() && SignalHandlerContext->Backtrace.FrameCount < std::ssize(SignalHandlerContext->Backtrace.Frames)) { + SignalHandlerContext->Backtrace.Frames[SignalHandlerContext->Backtrace.FrameCount++] = cursor.GetCurrentIP(); + cursor.MoveNext(); + } + + SignalHandlerContext->SetFinished(); +} + +} // namespace + +std::vector<TThreadIntrospectionInfo> IntrospectThreads() +{ + static std::atomic<bool> IntrospectionLock; + + if (IntrospectionLock.exchange(true)) { + THROW_ERROR_EXCEPTION("Thread introspection is already in progress"); + } + + auto introspectionLockGuard = Finally([] { + YT_VERIFY(IntrospectionLock.exchange(false)); + }); + + YT_LOG_INFO("Thread introspection started"); + + { + struct sigaction action; + action.sa_flags = SA_SIGINFO | SA_RESTART; + ::sigemptyset(&action.sa_mask); + action.sa_sigaction = SignalHandler; + + if (::sigaction(SIGUSR1, &action, nullptr) != 0) { + THROW_ERROR_EXCEPTION("Failed to install signal handler") + << TError::FromSystem(); + } + } + + std::vector<TThreadIntrospectionInfo> infos; + for (auto threadId : GetCurrentProcessThreadIds()) { + TSignalHandlerContext signalHandlerContext; + if (::syscall(SYS_tkill, threadId, SIGUSR1) != 0) { + YT_LOG_DEBUG(TError::FromSystem(), "Failed to signal to thread (ThreadId: %v)", + threadId); + continue; + } + + YT_LOG_DEBUG("Sent signal to thread (ThreadId: %v)", + threadId); + + signalHandlerContext.WaitUntilFinished(); + + YT_LOG_DEBUG("Signal handler finished (ThreadId: %v, FiberId: %x)", + threadId, + signalHandlerContext.FiberId); + + infos.push_back(TThreadIntrospectionInfo{ + .ThreadId = threadId, + .FiberId = signalHandlerContext.FiberId, + .ThreadName = TString(signalHandlerContext.ThreadName.Buffer.data(), static_cast<size_t>(signalHandlerContext.ThreadName.Length)), + .TraceId = signalHandlerContext.TraceId, + .TraceLoggingTag = signalHandlerContext.TraceLoggingTag, + .Backtrace = signalHandlerContext.Backtrace, + }); + } + + { + struct sigaction action; + action.sa_flags = SA_RESTART; + ::sigemptyset(&action.sa_mask); + action.sa_handler = SIG_IGN; + + if (::sigaction(SIGUSR1, &action, nullptr) != 0) { + THROW_ERROR_EXCEPTION("Failed to de-install signal handler") + << TError::FromSystem(); + } + } + + YT_LOG_INFO("Thread introspection completed"); + + return infos; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/private.h b/yt/yt/library/backtrace_introspector/private.h new file mode 100644 index 0000000000..59f25e6023 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/private.h @@ -0,0 +1,16 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NBacktraceIntrospector { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger BacktraceIntrospectorLogger("BacktraceIntrospector"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector + diff --git a/yt/yt/library/backtrace_introspector/public.h b/yt/yt/library/backtrace_introspector/public.h new file mode 100644 index 0000000000..54a8bd06ed --- /dev/null +++ b/yt/yt/library/backtrace_introspector/public.h @@ -0,0 +1,12 @@ +#pragma once + +namespace NYT::NBacktraceIntrospector { + +//////////////////////////////////////////////////////////////////////////////// + +struct TThreadIntrospectionInfo; +struct TFiberIntrospectionInfo; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp b/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp new file mode 100644 index 0000000000..a939417958 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/unittests/introspect_ut.cpp @@ -0,0 +1,198 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/library/backtrace_introspector/introspect.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/delayed_executor.h> + +#include <yt/yt/core/actions/bind.h> +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/tracing/trace_context.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/collection_helpers.h> + +namespace NYT::NBacktraceIntrospector { +namespace { + +using namespace NConcurrency; +using namespace NTracing; + +//////////////////////////////////////////////////////////////////////////////// + +NLogging::TLogger Logger("Test"); + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TBacktraceIntrospectorTest, Fibers) +{ + constexpr int HeavyQueueCount = 5; + std::vector<TActionQueuePtr> heavyQueues; + const TString HeavyThreadNamePrefix("Heavy:"); + for (int index = 0; index < HeavyQueueCount; ++index) { + heavyQueues.push_back(New<TActionQueue>(HeavyThreadNamePrefix + ToString(index))); + } + + constexpr int LightQueueCount = 3; + std::vector<TActionQueuePtr> lightQueues; + const TString LightThreadNamePrefix("Light:"); + for (int index = 0; index < LightQueueCount; ++index) { + lightQueues.push_back(New<TActionQueue>(LightThreadNamePrefix + ToString(index))); + } + + constexpr int HeavyCallbackCount = 3; + std::vector<TTraceContextPtr> heavyTraceContexts; + std::set<TTraceId> expectedHeavyTraceIds; + for (int index = 0; index < HeavyCallbackCount; ++index) { + auto traceContext = TTraceContext::NewRoot("Heavy"); + traceContext->SetLoggingTag(Format("HeavyLoggingTag:%v", index)); + heavyTraceContexts.push_back(traceContext); + InsertOrCrash(expectedHeavyTraceIds, traceContext->GetTraceId()); + } + + std::vector<TFuture<void>> heavyFutures; + for (int index = 0; index < HeavyCallbackCount; ++index) { + heavyFutures.push_back( + BIND([&, index] { + TTraceContextGuard traceContextGuard(heavyTraceContexts[index]); + YT_LOG_INFO("Heavy callback started (Index: %v)", index); + Sleep(TDuration::Seconds(3)); + YT_LOG_INFO("Heavy callback finished (Index: %v)", index); + }) + .AsyncVia(heavyQueues[index % HeavyQueueCount]->GetInvoker()) + .Run()); + } + + constexpr int LightCallbackCount = 10; + std::vector<TTraceContextPtr> lightTraceContexts; + std::set<TTraceId> expectedLightTraceIds; + for (int index = 0; index < LightCallbackCount; ++index) { + auto traceContext = TTraceContext::NewRoot("Light"); + traceContext->SetLoggingTag(Format("LightLoggingTag:%v", index)); + lightTraceContexts.push_back(traceContext); + InsertOrCrash(expectedLightTraceIds, traceContext->GetTraceId()); + } + + std::vector<TFuture<void>> lightFutures; + for (int index = 0; index < LightCallbackCount; ++index) { + lightFutures.push_back( + BIND([&, index] { + TTraceContextGuard traceContextGuard(lightTraceContexts[index]); + YT_LOG_INFO("Light callback started (Index: %v)", index); + TDelayedExecutor::WaitForDuration(TDuration::Seconds(1)); + YT_LOG_INFO("Light callback finished (Index: %v)", index); + }) + .AsyncVia(lightQueues[index % LightQueueCount]->GetInvoker()) + .Run()); + } + + Sleep(TDuration::MilliSeconds(100)); + + auto infos = IntrospectFibers(); + Cerr << FormatIntrospectionInfos(infos); + + std::set<TTraceId> actualHeavyTraceIds; + std::set<TTraceId> actualLightTraceIds; + for (const auto& info : infos) { + if (!info.TraceId) { + continue; + } + switch (info.State) { + case EFiberState::Running: + EXPECT_TRUE(actualHeavyTraceIds.insert(info.TraceId).second); + if (expectedHeavyTraceIds.contains(info.TraceId)) { + EXPECT_TRUE(info.ThreadName.StartsWith(HeavyThreadNamePrefix)); + } + break; + + case EFiberState::Waiting: + EXPECT_TRUE(actualLightTraceIds.insert(info.TraceId).second); + break; + + default: + break; + } + } + + EXPECT_EQ(expectedLightTraceIds, actualLightTraceIds); + EXPECT_EQ(expectedHeavyTraceIds, actualHeavyTraceIds); + + for (const auto& future : heavyFutures) { + future.Get().ThrowOnError(); + } + + for (const auto& future : lightFutures) { + future.Get().ThrowOnError(); + } + + for (const auto& queue : heavyQueues) { + queue->Shutdown(/*graceful*/ true); + } + for (const auto& queue : lightQueues) { + queue->Shutdown(/*graceful*/ true); + } +} + +TEST(TBacktraceIntrospectorTest, Threads) +{ + constexpr int QueueCount = 5; + std::vector<TActionQueuePtr> queues; + const TString ThreadNamePrefix("Queue:"); + for (int index = 0; index < QueueCount; ++index) { + queues.push_back(New<TActionQueue>(ThreadNamePrefix + ToString(index))); + } + + constexpr int CallbackCount = 3; + std::vector<TTraceContextPtr> traceContexts; + std::set<TTraceId> expectedTraceIds; + for (int index = 0; index < CallbackCount; ++index) { + auto traceContext = TTraceContext::NewRoot("Heavy"); + traceContexts.push_back(traceContext); + InsertOrCrash(expectedTraceIds, traceContext->GetTraceId()); + } + + std::vector<TFuture<void>> futures; + for (int index = 0; index < CallbackCount; ++index) { + futures.push_back( + BIND([&, index] { + TTraceContextGuard traceContextGuard(traceContexts[index]); + YT_LOG_INFO("Callback started (Index: %v)", index); + Sleep(TDuration::Seconds(3)); + YT_LOG_INFO("Callback finished (Index: %v)", index); + }) + .AsyncVia(queues[index % QueueCount]->GetInvoker()) + .Run()); + } + + Sleep(TDuration::MilliSeconds(100)); + + auto infos = IntrospectThreads(); + Cerr << FormatIntrospectionInfos(infos); + + std::set<TTraceId> actualTraceIds; + for (const auto& info : infos) { + if (!info.TraceId) { + continue; + } + EXPECT_TRUE(actualTraceIds.insert(info.TraceId).second); + if (expectedTraceIds.contains(info.TraceId)) { + EXPECT_TRUE(info.ThreadName.StartsWith(ThreadNamePrefix)); + } + } + + EXPECT_EQ(expectedTraceIds, actualTraceIds); + + for (const auto& future : futures) { + future.Get().ThrowOnError(); + } + for (const auto& queue : queues) { + queue->Shutdown(/*graceful*/ true); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NBacktraceIntrospector diff --git a/yt/yt/library/backtrace_introspector/unittests/ya.make b/yt/yt/library/backtrace_introspector/unittests/ya.make new file mode 100644 index 0000000000..953dc020a8 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/unittests/ya.make @@ -0,0 +1,15 @@ +GTEST() + +SRCS( + introspect_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + yt/yt/library/backtrace_introspector + + yt/yt/core/test_framework +) + +END() diff --git a/yt/yt/library/backtrace_introspector/ya.make b/yt/yt/library/backtrace_introspector/ya.make new file mode 100644 index 0000000000..884b8fb562 --- /dev/null +++ b/yt/yt/library/backtrace_introspector/ya.make @@ -0,0 +1,31 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + introspect.cpp +) +IF (OS_LINUX) + SRCS(introspect_linux.cpp) +ELSE() + SRCS(introspect_dummy.cpp) +ENDIF() + +PEERDIR( + yt/yt/core + + library/cpp/yt/backtrace/cursors/interop + library/cpp/yt/backtrace/cursors/libunwind + library/cpp/yt/backtrace/cursors/frame_pointer + library/cpp/yt/misc +) + +END() + +RECURSE( + http +) + +RECURSE_FOR_TESTS( + unittests +) diff --git a/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt b/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..faab79bbf6 --- /dev/null +++ b/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-containers) +target_compile_options(yt-library-containers PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-containers PUBLIC + contrib-libs-cxxsupp + yutil + cpp-porto-proto + yt-library-process + yt-yt-core +) +target_sources(yt-library-containers PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp +) diff --git a/yt/yt/library/containers/CMakeLists.linux-aarch64.txt b/yt/yt/library/containers/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..d3ab3811e0 --- /dev/null +++ b/yt/yt/library/containers/CMakeLists.linux-aarch64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-containers) +target_compile_options(yt-library-containers PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-containers PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-porto-proto + yt-library-process + yt-yt-core + library-cpp-porto +) +target_sources(yt-library-containers PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp +) diff --git a/yt/yt/library/containers/CMakeLists.linux-x86_64.txt b/yt/yt/library/containers/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..d3ab3811e0 --- /dev/null +++ b/yt/yt/library/containers/CMakeLists.linux-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-containers) +target_compile_options(yt-library-containers PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-containers PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-porto-proto + yt-library-process + yt-yt-core + library-cpp-porto +) +target_sources(yt-library-containers PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp +) diff --git a/yt/yt/library/containers/CMakeLists.txt b/yt/yt/library/containers/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/containers/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/containers/CMakeLists.windows-x86_64.txt b/yt/yt/library/containers/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..998e1690fa --- /dev/null +++ b/yt/yt/library/containers/CMakeLists.windows-x86_64.txt @@ -0,0 +1,27 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-containers) +target_link_libraries(yt-library-containers PUBLIC + contrib-libs-cxxsupp + yutil + cpp-porto-proto + yt-library-process + yt-yt-core +) +target_sources(yt-library-containers PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp +) diff --git a/yt/yt/library/containers/cgroup.cpp b/yt/yt/library/containers/cgroup.cpp new file mode 100644 index 0000000000..b43ab1e14b --- /dev/null +++ b/yt/yt/library/containers/cgroup.cpp @@ -0,0 +1,752 @@ +#include "cgroup.h" +#include "private.h" + +#include <yt/yt/core/misc/fs.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/ytree/fluent.h> + +#include <util/string/split.h> +#include <util/system/filemap.h> + +#include <util/system/yield.h> + +#ifdef _linux_ + #include <unistd.h> + #include <sys/stat.h> + #include <errno.h> +#endif + +namespace NYT::NContainers { + +using namespace NYTree; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = ContainersLogger; +static const TString CGroupRootPath("/sys/fs/cgroup"); +#ifdef _linux_ +static const int ReadByAll = S_IRUSR | S_IRGRP | S_IROTH; +static const int ReadExecuteByAll = ReadByAll | S_IXUSR | S_IXGRP | S_IXOTH; +#endif + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +TString GetParentFor(const TString& type) +{ +#ifdef _linux_ + auto rawData = TUnbufferedFileInput("/proc/self/cgroup") + .ReadAll(); + auto result = ParseProcessCGroups(rawData); + return result[type]; +#else + Y_UNUSED(type); + return "_parent_"; +#endif +} + +#ifdef _linux_ + +std::vector<TString> ReadAllValues(const TString& fileName) +{ + auto raw = TUnbufferedFileInput(fileName) + .ReadAll(); + + YT_LOG_DEBUG("File %v contains %Qv", + fileName, + raw); + + TVector<TString> values; + StringSplitter(raw.data()) + .SplitBySet(" \n") + .SkipEmpty() + .Collect(&values); + return values; +} + +TDuration FromJiffies(ui64 jiffies) +{ + static const auto TicksPerSecond = sysconf(_SC_CLK_TCK); + return TDuration::MicroSeconds(1000 * 1000 * jiffies / TicksPerSecond); +} + +#endif + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +void TKillProcessGroupTool::operator()(const TString& processGroupPath) const +{ + SafeSetUid(0); + TNonOwningCGroup group(processGroupPath); + group.Kill(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TNonOwningCGroup::TNonOwningCGroup(const TString& fullPath) + : FullPath_(fullPath) +{ } + +TNonOwningCGroup::TNonOwningCGroup(const TString& type, const TString& name) + : FullPath_(NFS::CombinePaths({ + CGroupRootPath, + type, + GetParentFor(type), + name + })) +{ } + +TNonOwningCGroup::TNonOwningCGroup(TNonOwningCGroup&& other) + : FullPath_(std::move(other.FullPath_)) +{ } + +void TNonOwningCGroup::AddTask(int pid) const +{ + YT_LOG_INFO( + "Adding task to cgroup (Task: %v, Cgroup: %v)", + pid, + FullPath_); + Append("tasks", ToString(pid)); +} + +void TNonOwningCGroup::AddCurrentTask() const +{ + YT_VERIFY(!IsNull()); +#ifdef _linux_ + auto pid = getpid(); + AddTask(pid); +#endif +} + +TString TNonOwningCGroup::Get(const TString& name) const +{ + YT_VERIFY(!IsNull()); + TString result; +#ifdef _linux_ + const auto path = GetPath(name); + result = TFileInput(path).ReadLine(); +#else + Y_UNUSED(name); +#endif + return result; +} + +void TNonOwningCGroup::Set(const TString& name, const TString& value) const +{ + YT_VERIFY(!IsNull()); +#ifdef _linux_ + auto path = GetPath(name); + TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::WrOnly)); + output << value; +#else + Y_UNUSED(name); + Y_UNUSED(value); +#endif +} + +void TNonOwningCGroup::Append(const TString& name, const TString& value) const +{ + YT_VERIFY(!IsNull()); +#ifdef _linux_ + auto path = GetPath(name); + TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::ForAppend)); + output << value; +#else + Y_UNUSED(name); + Y_UNUSED(value); +#endif +} + +bool TNonOwningCGroup::IsRoot() const +{ + return FullPath_ == CGroupRootPath; +} + +bool TNonOwningCGroup::IsNull() const +{ + return FullPath_.empty(); +} + +bool TNonOwningCGroup::Exists() const +{ + return NFS::Exists(FullPath_); +} + +std::vector<int> TNonOwningCGroup::GetProcesses() const +{ + std::vector<int> results; + if (!IsNull()) { +#ifdef _linux_ + auto values = ReadAllValues(GetPath("cgroup.procs")); + for (const auto& value : values) { + int pid = FromString<int>(value); + results.push_back(pid); + } +#endif + } + return results; +} + +std::vector<int> TNonOwningCGroup::GetTasks() const +{ + std::vector<int> results; + if (!IsNull()) { +#ifdef _linux_ + auto values = ReadAllValues(GetPath("tasks")); + for (const auto& value : values) { + int pid = FromString<int>(value); + results.push_back(pid); + } +#endif + } + return results; +} + +const TString& TNonOwningCGroup::GetFullPath() const +{ + return FullPath_; +} + +std::vector<TNonOwningCGroup> TNonOwningCGroup::GetChildren() const +{ + // We retry enumerating directories, since it may fail with weird diagnostics if + // number of subcgroups changes. + while (true) { + try { + std::vector<TNonOwningCGroup> result; + + if (IsNull()) { + return result; + } + + auto directories = NFS::EnumerateDirectories(FullPath_); + for (const auto& directory : directories) { + result.emplace_back(NFS::CombinePaths(FullPath_, directory)); + } + return result; + } catch (const std::exception& ex) { + YT_LOG_WARNING(ex, "Failed to list subcgroups (Path: %v)", FullPath_); + } + } +} + +void TNonOwningCGroup::EnsureExistence() const +{ + YT_LOG_INFO("Creating cgroup (Cgroup: %v)", FullPath_); + + YT_VERIFY(!IsNull()); + +#ifdef _linux_ + NFS::MakeDirRecursive(FullPath_, 0755); +#endif +} + +void TNonOwningCGroup::Lock() const +{ + Traverse( + BIND([] (const TNonOwningCGroup& group) { group.DoLock(); }), + BIND([] (const TNonOwningCGroup& /*group*/) {})); +} + +void TNonOwningCGroup::Unlock() const +{ + Traverse( + BIND([] (const TNonOwningCGroup& /*group*/) {}), + BIND([] (const TNonOwningCGroup& group) { group.DoUnlock(); })); +} + +void TNonOwningCGroup::Kill() const +{ + YT_VERIFY(!IsRoot()); + + Traverse( + BIND([] (const TNonOwningCGroup& group) { group.DoKill(); }), + BIND([] (const TNonOwningCGroup& /*group*/) {})); +} + +void TNonOwningCGroup::RemoveAllSubcgroups() const +{ + Traverse( + BIND([] (const TNonOwningCGroup& group) { + group.TryUnlock(); + }), + BIND([this_ = this] (const TNonOwningCGroup& group) { + if (this_ != &group) { + group.DoRemove(); + } + })); +} + +void TNonOwningCGroup::RemoveRecursive() const +{ + RemoveAllSubcgroups(); + DoRemove(); +} + +void TNonOwningCGroup::DoLock() const +{ + YT_LOG_INFO("Locking cgroup (Cgroup: %v)", FullPath_); + +#ifdef _linux_ + if (!IsNull()) { + int code = chmod(FullPath_.data(), ReadExecuteByAll); + YT_VERIFY(code == 0); + + code = chmod(GetPath("tasks").data(), ReadByAll); + YT_VERIFY(code == 0); + } +#endif +} + +bool TNonOwningCGroup::TryUnlock() const +{ + YT_LOG_INFO("Unlocking cgroup (Cgroup: %v)", FullPath_); + + if (!Exists()) { + return true; + } + + bool result = true; + +#ifdef _linux_ + if (!IsNull()) { + int code = chmod(GetPath("tasks").data(), ReadByAll | S_IWUSR); + if (code != 0) { + result = false; + } + + code = chmod(FullPath_.data(), ReadExecuteByAll | S_IWUSR); + if (code != 0) { + result = false; + } + } +#endif + + return result; +} + +void TNonOwningCGroup::DoUnlock() const +{ + YT_VERIFY(TryUnlock()); +} + +void TNonOwningCGroup::DoKill() const +{ + YT_LOG_DEBUG("Started killing processes in cgroup (Cgroup: %v)", FullPath_); + +#ifdef _linux_ + while (true) { + auto pids = GetTasks(); + if (pids.empty()) + break; + + YT_LOG_DEBUG("Killing processes (Pids: %v)", pids); + + for (int pid : pids) { + auto result = kill(pid, SIGKILL); + if (result == -1) { + YT_VERIFY(errno == ESRCH); + } + } + + ThreadYield(); + } +#endif + + YT_LOG_DEBUG("Finished killing processes in cgroup (Cgroup: %v)", FullPath_); +} + +void TNonOwningCGroup::DoRemove() const +{ + if (NFS::Exists(FullPath_)) { + NFS::Remove(FullPath_); + } +} + +void TNonOwningCGroup::Traverse( + const TCallback<void(const TNonOwningCGroup&)>& preorderAction, + const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const +{ + preorderAction(*this); + + for (const auto& child : GetChildren()) { + child.Traverse(preorderAction, postorderAction); + } + + postorderAction(*this); +} + +TString TNonOwningCGroup::GetPath(const TString& filename) const +{ + return NFS::CombinePaths(FullPath_, filename); +} + +//////////////////////////////////////////////////////////////////////////////// + +TCGroup::TCGroup(const TString& type, const TString& name) + : TNonOwningCGroup(type, name) +{ } + +TCGroup::TCGroup(TCGroup&& other) + : TNonOwningCGroup(std::move(other)) + , Created_(other.Created_) +{ + other.Created_ = false; +} + +TCGroup::TCGroup(TNonOwningCGroup&& other) + : TNonOwningCGroup(std::move(other)) + , Created_(false) +{ } + +TCGroup::~TCGroup() +{ + if (Created_) { + Destroy(); + } +} + +void TCGroup::Create() +{ + EnsureExistence(); + Created_ = true; +} + +void TCGroup::Destroy() +{ + YT_LOG_INFO("Destroying cgroup (Cgroup: %v)", FullPath_); + YT_VERIFY(Created_); + +#ifdef _linux_ + try { + NFS::Remove(FullPath_); + } catch (const std::exception& ex) { + YT_LOG_FATAL(ex, "Failed to destroy cgroup (Cgroup: %v)", FullPath_); + } +#endif + Created_ = false; +} + +bool TCGroup::IsCreated() const +{ + return Created_; +} + +//////////////////////////////////////////////////////////////////////////////// + +const TString TCpuAccounting::Name = "cpuacct"; + +TCpuAccounting::TStatistics& operator-=(TCpuAccounting::TStatistics& lhs, const TCpuAccounting::TStatistics& rhs) +{ + #define XX(name) lhs.name = lhs.name.ValueOrThrow() - rhs.name.ValueOrThrow(); + XX(UserUsageTime) + XX(SystemUsageTime) + XX(WaitTime) + XX(ThrottledTime) + XX(ContextSwitchesDelta) + XX(PeakThreadCount) + #undef XX + return lhs; +} + +TCpuAccounting::TCpuAccounting(const TString& name) + : TCGroup(Name, name) +{ } + +TCpuAccounting::TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup) + : TCGroup(std::move(nonOwningCGroup)) +{ } + +TCpuAccounting::TStatistics TCpuAccounting::GetStatisticsRecursive() const +{ + TCpuAccounting::TStatistics result; +#ifdef _linux_ + try { + auto path = NFS::CombinePaths(GetFullPath(), "cpuacct.stat"); + auto values = ReadAllValues(path); + YT_VERIFY(values.size() == 4); + + TString type[2]; + ui64 jiffies[2]; + + for (int i = 0; i < 2; ++i) { + type[i] = values[2 * i]; + jiffies[i] = FromString<ui64>(values[2 * i + 1]); + } + + for (int i = 0; i < 2; ++i) { + if (type[i] == "user") { + result.UserUsageTime = FromJiffies(jiffies[i]); + } else if (type[i] == "system") { + result.SystemUsageTime = FromJiffies(jiffies[i]); + } + } + } catch (const std::exception& ex) { + YT_LOG_FATAL( + ex, + "Failed to retrieve CPU statistics from cgroup (Cgroup: %v)", + GetFullPath()); + } +#endif + return result; +} + +TCpuAccounting::TStatistics TCpuAccounting::GetStatistics() const +{ + auto statistics = GetStatisticsRecursive(); + for (auto& cgroup : GetChildren()) { + auto cpuCGroup = TCpuAccounting(std::move(cgroup)); + statistics -= cpuCGroup.GetStatisticsRecursive(); + } + return statistics; +} + + +//////////////////////////////////////////////////////////////////////////////// + +const TString TCpu::Name = "cpu"; + +static const int DefaultCpuShare = 1024; + +TCpu::TCpu(const TString& name) + : TCGroup(Name, name) +{ } + +void TCpu::SetShare(double share) +{ + int cpuShare = static_cast<int>(share * DefaultCpuShare); + Set("cpu.shares", ToString(cpuShare)); +} + +//////////////////////////////////////////////////////////////////////////////// + +const TString TBlockIO::Name = "blkio"; + +TBlockIO::TBlockIO(const TString& name) + : TCGroup(Name, name) +{ } + +// For more information about format of data +// read https://www.kernel.org/doc/Documentation/cgroups/blkio-controller.txt + +TBlockIO::TStatistics TBlockIO::GetStatistics() const +{ + TBlockIO::TStatistics result; +#ifdef _linux_ + auto bytesStats = GetDetailedStatistics("blkio.io_service_bytes"); + for (const auto& item : bytesStats) { + if (item.Type == "Read") { + result.IOReadByte = result.IOReadByte.ValueOrThrow() + item.Value; + } else if (item.Type == "Write") { + result.IOWriteByte = result.IOReadByte.ValueOrThrow() + item.Value; + } + } + + auto ioStats = GetDetailedStatistics("blkio.io_serviced"); + for (const auto& item : ioStats) { + if (item.Type == "Read") { + result.IOReadOps = result.IOReadOps.ValueOrThrow() + item.Value; + result.IOOps = result.IOOps.ValueOrThrow() + item.Value; + } else if (item.Type == "Write") { + result.IOWriteOps = result.IOWriteOps.ValueOrThrow() + item.Value; + result.IOOps = result.IOOps.ValueOrThrow() + item.Value; + } + } +#endif + return result; +} + +std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiceBytes() const +{ + return GetDetailedStatistics("blkio.io_service_bytes"); +} + +std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiced() const +{ + return GetDetailedStatistics("blkio.io_serviced"); +} + +std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetDetailedStatistics(const char* filename) const +{ + std::vector<TBlockIO::TStatisticsItem> result; +#ifdef _linux_ + try { + auto path = NFS::CombinePaths(GetFullPath(), filename); + auto values = ReadAllValues(path); + + int lineNumber = 0; + while (3 * lineNumber + 2 < std::ssize(values)) { + TStatisticsItem item; + item.DeviceId = values[3 * lineNumber]; + item.Type = values[3 * lineNumber + 1]; + item.Value = FromString<ui64>(values[3 * lineNumber + 2]); + + { + auto guard = Guard(SpinLock_); + DeviceIds_.insert(item.DeviceId); + } + + if (item.Type == "Read" || item.Type == "Write") { + result.push_back(item); + + YT_LOG_DEBUG("IO operations serviced (OperationCount: %v, OperationType: %v, DeviceId: %v)", + item.Value, + item.Type, + item.DeviceId); + } + ++lineNumber; + } + } catch (const std::exception& ex) { + YT_LOG_FATAL( + ex, + "Failed to retrieve block IO statistics from cgroup (Cgroup: %v)", + GetFullPath()); + } +#else + Y_UNUSED(filename); +#endif + return result; +} + +void TBlockIO::ThrottleOperations(i64 operations) const +{ + auto guard = Guard(SpinLock_); + for (const auto& deviceId : DeviceIds_) { + auto value = Format("%v %v", deviceId, operations); + Append("blkio.throttle.read_iops_device", value); + Append("blkio.throttle.write_iops_device", value); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +const TString TMemory::Name = "memory"; + +TMemory::TMemory(const TString& name) + : TCGroup(Name, name) +{ } + +TMemory::TStatistics TMemory::GetStatistics() const +{ + TMemory::TStatistics result; +#ifdef _linux_ + try { + auto values = ReadAllValues(GetPath("memory.stat")); + int lineNumber = 0; + while (2 * lineNumber + 1 < std::ssize(values)) { + const auto& type = values[2 * lineNumber]; + const auto& unparsedValue = values[2 * lineNumber + 1]; + if (type == "rss") { + result.Rss = FromString<ui64>(unparsedValue); + } + if (type == "mapped_file") { + result.MappedFile = FromString<ui64>(unparsedValue); + } + if (type == "pgmajfault") { + result.MajorPageFaults = FromString<ui64>(unparsedValue); + } + ++lineNumber; + } + } catch (const std::exception& ex) { + YT_LOG_FATAL( + ex, + "Failed to retrieve memory statistics from cgroup (Cgroup: %v)", + GetFullPath()); + } +#endif + return result; +} + +i64 TMemory::GetMaxMemoryUsage() const +{ + return FromString<i64>(Get("memory.max_usage_in_bytes")); +} + +void TMemory::SetLimitInBytes(i64 bytes) const +{ + Set("memory.limit_in_bytes", ToString(bytes)); +} + +void TMemory::ForceEmpty() const +{ + Set("memory.force_empty", "0"); +} + +//////////////////////////////////////////////////////////////////////////////// + +const TString TFreezer::Name = "freezer"; + +TFreezer::TFreezer(const TString& name) + : TCGroup(Name, name) +{ } + +TString TFreezer::GetState() const +{ + return Get("freezer.state"); +} + +void TFreezer::Freeze() const +{ + Set("freezer.state", "FROZEN"); +} + +void TFreezer::Unfreeze() const +{ + Set("freezer.state", "THAWED"); +} + +//////////////////////////////////////////////////////////////////////////////// + +std::map<TString, TString> ParseProcessCGroups(const TString& str) +{ + std::map<TString, TString> result; + + TVector<TString> values; + StringSplitter(str.data()).SplitBySet(":\n").SkipEmpty().Collect(&values); + for (size_t i = 0; i + 2 < values.size(); i += 3) { + // Check format. + FromString<int>(values[i]); + + const auto& subsystemsSet = values[i + 1]; + const auto& name = values[i + 2]; + + TVector<TString> subsystems; + StringSplitter(subsystemsSet.data()).Split(',').SkipEmpty().Collect(&subsystems); + for (const auto& subsystem : subsystems) { + if (!subsystem.StartsWith("name=")) { + int start = 0; + if (name.StartsWith("/")) { + start = 1; + } + result[subsystem] = name.substr(start); + } + } + } + + return result; +} + +std::map<TString, TString> GetProcessCGroups(pid_t pid) +{ + auto cgroupsPath = Format("/proc/%v/cgroup", pid); + auto rawCgroups = TFileInput{cgroupsPath}.ReadAll(); + return ParseProcessCGroups(rawCgroups); +} + +bool IsValidCGroupType(const TString& type) +{ + return + type == TCpuAccounting::Name || + type == TCpu::Name || + type == TBlockIO::Name || + type == TMemory::Name || + type == TFreezer::Name; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/cgroup.h b/yt/yt/library/containers/cgroup.h new file mode 100644 index 0000000000..a61fbbddc3 --- /dev/null +++ b/yt/yt/library/containers/cgroup.h @@ -0,0 +1,290 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/actions/public.h> + +#include <yt/yt/core/ytree/yson_struct.h> +#include <yt/yt/core/yson/public.h> + +#include <yt/yt/core/misc/property.h> + +#include <library/cpp/yt/threading/spin_lock.h> + +#include <vector> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +void RemoveAllSubcgroups(const TString& path); + +//////////////////////////////////////////////////////////////////////////////// + +struct TKillProcessGroupTool +{ + void operator()(const TString& processGroupPath) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TNonOwningCGroup + : private TNonCopyable +{ +public: + DEFINE_BYREF_RO_PROPERTY(TString, FullPath); + +public: + TNonOwningCGroup() = default; + explicit TNonOwningCGroup(const TString& fullPath); + TNonOwningCGroup(const TString& type, const TString& name); + TNonOwningCGroup(TNonOwningCGroup&& other); + + void AddTask(int pid) const; + void AddCurrentTask() const; + + bool IsRoot() const; + bool IsNull() const; + bool Exists() const; + + std::vector<int> GetProcesses() const; + std::vector<int> GetTasks() const; + const TString& GetFullPath() const; + + std::vector<TNonOwningCGroup> GetChildren() const; + + void EnsureExistence() const; + + void Lock() const; + void Unlock() const; + + void Kill() const; + + void RemoveAllSubcgroups() const; + void RemoveRecursive() const; + +protected: + TString Get(const TString& name) const; + void Set(const TString& name, const TString& value) const; + void Append(const TString& name, const TString& value) const; + + void DoLock() const; + void DoUnlock() const; + + bool TryUnlock() const; + + void DoKill() const; + + void DoRemove() const; + + void Traverse( + const TCallback<void(const TNonOwningCGroup&)>& preorderAction, + const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const; + + TString GetPath(const TString& filename) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TCGroup + : public TNonOwningCGroup +{ +protected: + TCGroup(const TString& type, const TString& name); + TCGroup(TNonOwningCGroup&& other); + TCGroup(TCGroup&& other); + +public: + ~TCGroup(); + + void Create(); + void Destroy(); + + bool IsCreated() const; + +private: + bool Created_ = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TCpuAccounting + : public TCGroup +{ +public: + static const TString Name; + + struct TStatistics + { + TErrorOr<TDuration> TotalUsageTime; + TErrorOr<TDuration> UserUsageTime; + TErrorOr<TDuration> SystemUsageTime; + TErrorOr<TDuration> WaitTime; + TErrorOr<TDuration> ThrottledTime; + + TErrorOr<ui64> ThreadCount; + TErrorOr<ui64> ContextSwitches; + TErrorOr<ui64> ContextSwitchesDelta; + TErrorOr<ui64> PeakThreadCount; + + TErrorOr<TDuration> LimitTime; + TErrorOr<TDuration> GuaranteeTime; + }; + + explicit TCpuAccounting(const TString& name); + + TStatistics GetStatisticsRecursive() const; + TStatistics GetStatistics() const; + +private: + explicit TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup); +}; + +void Serialize(const TCpuAccounting::TStatistics& statistics, NYson::IYsonConsumer* consumer); + +//////////////////////////////////////////////////////////////////////////////// + +class TCpu + : public TCGroup +{ +public: + static const TString Name; + + explicit TCpu(const TString& name); + + void SetShare(double share); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TBlockIO + : public TCGroup +{ +public: + static const TString Name; + + struct TStatistics + { + TErrorOr<ui64> IOReadByte; + TErrorOr<ui64> IOWriteByte; + TErrorOr<ui64> IOBytesLimit; + + TErrorOr<ui64> IOReadOps; + TErrorOr<ui64> IOWriteOps; + TErrorOr<ui64> IOOps; + TErrorOr<ui64> IOOpsLimit; + + TErrorOr<TDuration> IOTotalTime; + TErrorOr<TDuration> IOWaitTime; + }; + + struct TStatisticsItem + { + TString DeviceId; + TString Type; + ui64 Value = 0; + }; + + explicit TBlockIO(const TString& name); + + TStatistics GetStatistics() const; + void ThrottleOperations(i64 iops) const; + +private: + //! Guards device ids. + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); + //! Set of all seen device ids. + mutable THashSet<TString> DeviceIds_; + + std::vector<TBlockIO::TStatisticsItem> GetDetailedStatistics(const char* filename) const; + + std::vector<TStatisticsItem> GetIOServiceBytes() const; + std::vector<TStatisticsItem> GetIOServiced() const; +}; + +void Serialize(const TBlockIO::TStatistics& statistics, NYson::IYsonConsumer* consumer); + +//////////////////////////////////////////////////////////////////////////////// + +class TMemory + : public TCGroup +{ +public: + static const TString Name; + + struct TStatistics + { + TErrorOr<ui64> Rss; + TErrorOr<ui64> MappedFile; + TErrorOr<ui64> MinorPageFaults; + TErrorOr<ui64> MajorPageFaults; + + TErrorOr<ui64> FileCacheUsage; + TErrorOr<ui64> AnonUsage; + TErrorOr<ui64> AnonLimit; + TErrorOr<ui64> MemoryUsage; + TErrorOr<ui64> MemoryGuarantee; + TErrorOr<ui64> MemoryLimit; + TErrorOr<ui64> MaxMemoryUsage; + + TErrorOr<ui64> OomKills; + TErrorOr<ui64> OomKillsTotal; + }; + + explicit TMemory(const TString& name); + + TStatistics GetStatistics() const; + i64 GetMaxMemoryUsage() const; + + void SetLimitInBytes(i64 bytes) const; + + void ForceEmpty() const; +}; + +void Serialize(const TMemory::TStatistics& statistics, NYson::IYsonConsumer* consumer); + +//////////////////////////////////////////////////////////////////////////////// + +class TNetwork +{ +public: + struct TStatistics + { + TErrorOr<ui64> TxBytes; + TErrorOr<ui64> TxPackets; + TErrorOr<ui64> TxDrops; + TErrorOr<ui64> TxLimit; + + TErrorOr<ui64> RxBytes; + TErrorOr<ui64> RxPackets; + TErrorOr<ui64> RxDrops; + TErrorOr<ui64> RxLimit; + }; +}; + +void Serialize(const TNetwork::TStatistics& statistics, NYson::IYsonConsumer* consumer); + +//////////////////////////////////////////////////////////////////////////////// + +class TFreezer + : public TCGroup +{ +public: + static const TString Name; + + explicit TFreezer(const TString& name); + + TString GetState() const; + void Freeze() const; + void Unfreeze() const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +std::map<TString, TString> ParseProcessCGroups(const TString& str); +std::map<TString, TString> GetProcessCGroups(pid_t pid); +bool IsValidCGroupType(const TString& type); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/config.cpp b/yt/yt/library/containers/config.cpp new file mode 100644 index 0000000000..39e46f2372 --- /dev/null +++ b/yt/yt/library/containers/config.cpp @@ -0,0 +1,64 @@ +#include "config.h" + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +void TPodSpecConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("cpu_to_vcpu_factor", &TThis::CpuToVCpuFactor) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TCGroupConfig::IsCGroupSupported(const TString& cgroupType) const +{ + auto it = std::find_if( + SupportedCGroups.begin(), + SupportedCGroups.end(), + [&] (const TString& type) { + return type == cgroupType; + }); + return it != SupportedCGroups.end(); +} + +void TCGroupConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("supported_cgroups", &TThis::SupportedCGroups) + .Default(); + + registrar.Postprocessor([] (TThis* config) { + for (const auto& type : config->SupportedCGroups) { + if (!IsValidCGroupType(type)) { + THROW_ERROR_EXCEPTION("Invalid cgroup type %Qv", type); + } + } + }); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TPortoExecutorDynamicConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("retries_timeout", &TThis::RetriesTimeout) + .Default(TDuration::Seconds(10)); + registrar.Parameter("poll_period", &TThis::PollPeriod) + .Default(TDuration::MilliSeconds(100)); + registrar.Parameter("api_timeout", &TThis::ApiTimeout) + .Default(TDuration::Minutes(5)); + registrar.Parameter("api_disk_timeout", &TThis::ApiDiskTimeout) + .Default(TDuration::Minutes(30)); + registrar.Parameter("enable_network_isolation", &TThis::EnableNetworkIsolation) + .Default(true); + registrar.Parameter("enable_test_porto_failures", &TThis::EnableTestPortoFailures) + .Default(false); + registrar.Parameter("stub_error_code", &TThis::StubErrorCode) + .Default(EPortoErrorCode::SocketError); + registrar.Parameter("enable_test_porto_not_responding", &TThis::EnableTestPortoNotResponding) + .Default(false); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/config.h b/yt/yt/library/containers/config.h new file mode 100644 index 0000000000..3639274cff --- /dev/null +++ b/yt/yt/library/containers/config.h @@ -0,0 +1,64 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +class TPodSpecConfig + : public virtual NYTree::TYsonStruct +{ +public: + std::optional<double> CpuToVCpuFactor; + + REGISTER_YSON_STRUCT(TPodSpecConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TPodSpecConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class TCGroupConfig + : public virtual NYTree::TYsonStruct +{ +public: + std::vector<TString> SupportedCGroups; + + bool IsCGroupSupported(const TString& cgroupType) const; + + REGISTER_YSON_STRUCT(TCGroupConfig); + + static void Register(TRegistrar registrar); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoExecutorDynamicConfig + : public NYTree::TYsonStruct +{ +public: + TDuration RetriesTimeout; + TDuration PollPeriod; + TDuration ApiTimeout; + TDuration ApiDiskTimeout; + bool EnableNetworkIsolation; + bool EnableTestPortoFailures; + bool EnableTestPortoNotResponding; + + EPortoErrorCode StubErrorCode; + + REGISTER_YSON_STRUCT(TPortoExecutorDynamicConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TPortoExecutorDynamicConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/cri/config.cpp b/yt/yt/library/containers/cri/config.cpp new file mode 100644 index 0000000000..5572f4d980 --- /dev/null +++ b/yt/yt/library/containers/cri/config.cpp @@ -0,0 +1,54 @@ +#include "config.h" +#include "cri_api.h" + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +void TCriExecutorConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("runtime_endpoint", &TThis::RuntimeEndpoint) + .Default(TString(DefaultCriEndpoint)); + + registrar.Parameter("image_endpoint", &TThis::ImageEndpoint) + .Default(TString(DefaultCriEndpoint)); + + registrar.Parameter("namespace", &TThis::Namespace) + .NonEmpty(); + + registrar.Parameter("runtime_handler", &TThis::RuntimeHandler) + .Optional(); + + registrar.Parameter("base_cgroup", &TThis::BaseCgroup) + .NonEmpty(); + + registrar.Parameter("cpu_period", &TThis::CpuPeriod) + .Default(TDuration::MilliSeconds(100)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TCriAuthConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("username", &TThis::Username) + .Optional(); + + registrar.Parameter("password", &TThis::Password) + .Optional(); + + registrar.Parameter("auth", &TThis::Auth) + .Optional(); + + registrar.Parameter("server_address", &TThis::ServerAddress) + .Optional(); + + registrar.Parameter("identity_token", &TThis::IdentityToken) + .Optional(); + + registrar.Parameter("registry_token", &TThis::RegistryToken) + .Optional(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/config.h b/yt/yt/library/containers/cri/config.h new file mode 100644 index 0000000000..4ea33fd390 --- /dev/null +++ b/yt/yt/library/containers/cri/config.h @@ -0,0 +1,70 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/rpc/config.h> + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +class TCriExecutorConfig + : public NRpc::TRetryingChannelConfig +{ +public: + //! gRPC endpoint for CRI container runtime service. + TString RuntimeEndpoint; + + //! gRPC endpoint for CRI image manager service. + TString ImageEndpoint; + + //! CRI namespace where this executor operates. + TString Namespace; + + //! Name of CRI runtime configuration to use. + TString RuntimeHandler; + + //! Common parent cgroup for all pods. + TString BaseCgroup; + + //! Cpu quota period for cpu limits. + TDuration CpuPeriod; + + REGISTER_YSON_STRUCT(TCriExecutorConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TCriExecutorConfig) + +//////////////////////////////////////////////////////////////////////////////// + +// TODO(khlebnikov): split docker registry stuff into common "docker" library. + +//! TCriAuthConfig depicts docker registry authentification +class TCriAuthConfig + : public NYTree::TYsonStruct +{ +public: + TString Username; + + TString Password; + + TString Auth; + + TString ServerAddress; + + TString IdentityToken; + + TString RegistryToken; + + REGISTER_YSON_STRUCT(TCriAuthConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TCriAuthConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_api.cpp b/yt/yt/library/containers/cri/cri_api.cpp new file mode 100644 index 0000000000..93457017ba --- /dev/null +++ b/yt/yt/library/containers/cri/cri_api.cpp @@ -0,0 +1,33 @@ +#include "cri_api.h" + +namespace NYT::NContainers::NCri { + +using namespace NRpc; + +//////////////////////////////////////////////////////////////////////////////// + +TCriRuntimeApi::TCriRuntimeApi(IChannelPtr channel) + : TProxyBase(std::move(channel), GetDescriptor()) +{ } + +const TServiceDescriptor& TCriRuntimeApi::GetDescriptor() +{ + static const auto Descriptor = TServiceDescriptor(NProto::RuntimeService::service_full_name()); + return Descriptor; +} + +//////////////////////////////////////////////////////////////////////////////// + +TCriImageApi::TCriImageApi(IChannelPtr channel) + : TProxyBase(std::move(channel), GetDescriptor()) +{ } + +const TServiceDescriptor& TCriImageApi::GetDescriptor() +{ + static const auto Descriptor = TServiceDescriptor(NProto::ImageService::service_full_name()); + return Descriptor; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_api.h b/yt/yt/library/containers/cri/cri_api.h new file mode 100644 index 0000000000..74fe9a64a0 --- /dev/null +++ b/yt/yt/library/containers/cri/cri_api.h @@ -0,0 +1,99 @@ +#pragma once + +#include <yt/yt/core/rpc/client.h> + +#include <k8s.io/cri-api/pkg/apis/runtime/v1/api.grpc.pb.h> + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NProto = ::runtime::v1; + +//! Reasonable default for CRI gRPC socket address. +constexpr TStringBuf DefaultCriEndpoint = "unix:///run/containerd/containerd.sock"; + +//! RuntimeReady means the runtime is up and ready to accept basic containers. +constexpr TStringBuf RuntimeReady = "RuntimeReady"; + +//! NetworkReady means the runtime network is up and ready to accept containers which require network. +constexpr TStringBuf NetworkReady = "NetworkReady"; + +//! CRI uses cgroupfs notation for systemd slices, but each name must ends with ".slice". +constexpr TStringBuf SystemdSliceSuffix = ".slice"; + +//////////////////////////////////////////////////////////////////////////////// + +//! CRI labels for pods and containers managed by YT +constexpr TStringBuf YTPodNamespaceLabel = "tech.ytsaurus.pod.namespace"; +constexpr TStringBuf YTPodNameLabel = "tech.ytsaurus.pod.name"; +constexpr TStringBuf YTContainerNameLabel = "tech.ytsaurus.container.name"; +constexpr TStringBuf YTJobIdLabel = "tech.ytsaurus.job.id"; + +//////////////////////////////////////////////////////////////////////////////// + +#define DEFINE_CRI_API_METHOD(method, ...) \ + DEFINE_RPC_PROXY_METHOD_GENERIC(method, NProto::method##Request, NProto::method##Response, __VA_ARGS__) + +//! See https://github.com/kubernetes/cri-api +class TCriRuntimeApi + : public NRpc::TProxyBase +{ +public: + explicit TCriRuntimeApi(NRpc::IChannelPtr channel); + + static const NRpc::TServiceDescriptor& GetDescriptor(); + + DEFINE_CRI_API_METHOD(Version); + DEFINE_CRI_API_METHOD(RunPodSandbox); + DEFINE_CRI_API_METHOD(StopPodSandbox); + DEFINE_CRI_API_METHOD(RemovePodSandbox); + DEFINE_CRI_API_METHOD(PodSandboxStatus); + DEFINE_CRI_API_METHOD(ListPodSandbox); + DEFINE_CRI_API_METHOD(CreateContainer); + DEFINE_CRI_API_METHOD(StartContainer); + DEFINE_CRI_API_METHOD(StopContainer); + DEFINE_CRI_API_METHOD(RemoveContainer); + DEFINE_CRI_API_METHOD(ListContainers); + DEFINE_CRI_API_METHOD(ContainerStatus); + DEFINE_CRI_API_METHOD(UpdateContainerResources); + DEFINE_CRI_API_METHOD(ReopenContainerLog); + DEFINE_CRI_API_METHOD(ExecSync); + DEFINE_CRI_API_METHOD(Exec); + DEFINE_CRI_API_METHOD(Attach); + DEFINE_CRI_API_METHOD(PortForward); + DEFINE_CRI_API_METHOD(ContainerStats); + DEFINE_CRI_API_METHOD(ListContainerStats); + DEFINE_CRI_API_METHOD(PodSandboxStats); + DEFINE_CRI_API_METHOD(ListPodSandboxStats); + DEFINE_CRI_API_METHOD(UpdateRuntimeConfig); + DEFINE_CRI_API_METHOD(Status); + DEFINE_CRI_API_METHOD(CheckpointContainer); + DEFINE_CRI_API_METHOD(ListMetricDescriptors); + DEFINE_CRI_API_METHOD(ListPodSandboxMetrics); + + // FIXME(khlebnikov): figure out streaming results + // DEFINE_RPC_PROXY_METHOD_GENERIC(GetContainerEvents, NProto::GetEventsRequest, NProto::ContainerEventResponse, + // .SetStreamingEnabled(true)); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TCriImageApi + : public NRpc::TProxyBase +{ +public: + explicit TCriImageApi(NRpc::IChannelPtr channel); + + static const NRpc::TServiceDescriptor& GetDescriptor(); + + DEFINE_CRI_API_METHOD(ListImages); + DEFINE_CRI_API_METHOD(ImageStatus); + DEFINE_CRI_API_METHOD(PullImage); + DEFINE_CRI_API_METHOD(RemoveImage); + DEFINE_CRI_API_METHOD(ImageFsInfo); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_executor.cpp b/yt/yt/library/containers/cri/cri_executor.cpp new file mode 100644 index 0000000000..428fd93165 --- /dev/null +++ b/yt/yt/library/containers/cri/cri_executor.cpp @@ -0,0 +1,666 @@ +#include "cri_executor.h" +#include "private.h" + +#include <yt/yt/core/actions/bind.h> + +#include <yt/yt/core/rpc/grpc/channel.h> + +#include <yt/yt/core/rpc/retrying_channel.h> + +#include <yt/yt/core/misc/error.h> +#include <yt/yt/core/misc/proc.h> +#include <yt/yt/core/misc/protobuf_helpers.h> + +#include <yt/yt/core/concurrency/periodic_executor.h> + +namespace NYT::NContainers::NCri { + +using namespace NRpc; +using namespace NRpc::NGrpc; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf /*spec*/) +{ + builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name); +} + +void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf /*spec*/) +{ + builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name); +} + +void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf /*spec*/) +{ + builder->AppendString(descriptor.Image); +} + +static TError DecodeExitCode(int exitCode, const TString& reason) +{ + if (exitCode == 0) { + return TError(); + } + + // TODO(khkebnikov) map reason == "OOMKilled" + + // Common bash notation for signals: 128 + signal + if (exitCode > 128) { + int signalNumber = exitCode - 128; + return TError( + EProcessErrorCode::Signal, + "Process terminated by signal %v", + signalNumber) + << TErrorAttribute("signal", signalNumber) + << TErrorAttribute("reason", reason); + } + + // TODO(khkebnikov) check these + // 125 - container failed to run + // 126 - non executable + // 127 - command not found + // 128 - invalid exit code + // 255 - exit code out of range + + return TError( + EProcessErrorCode::NonZeroExitCode, + "Process exited with code %v", + exitCode) + << TErrorAttribute("exit_code", exitCode) + << TErrorAttribute("reason", reason); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TCriProcess + : public TProcessBase +{ +public: + TCriProcess( + const TString& path, + ICriExecutorPtr executor, + TCriContainerSpecPtr containerSpec, + const TCriPodDescriptor& podDescriptor, + TCriPodSpecPtr podSpec, + TDuration pollPeriod = TDuration::MilliSeconds(100)) + : TProcessBase(path) + , Executor_(std::move(executor)) + , ContainerSpec_(std::move(containerSpec)) + , PodDescriptor_(podDescriptor) + , PodSpec_(std::move(podSpec)) + , PollPeriod_(pollPeriod) + { + // Just for symmetry with sibling classes. + AddArgument(Path_); + } + + void Kill(int /*signal*/) override + { + WaitFor(Executor_->StopContainer(ContainerDescriptor_)) + .ThrowOnError(); + } + + NNet::IConnectionWriterPtr GetStdInWriter() override + { + THROW_ERROR_EXCEPTION("Not implemented for CRI process"); + } + + NNet::IConnectionReaderPtr GetStdOutReader() override + { + THROW_ERROR_EXCEPTION("Not implemented for CRI process"); + } + + NNet::IConnectionReaderPtr GetStdErrReader() override + { + THROW_ERROR_EXCEPTION("Not implemented for CRI process"); + } + +private: + const ICriExecutorPtr Executor_; + const TCriContainerSpecPtr ContainerSpec_; + const TCriPodDescriptor PodDescriptor_; + const TCriPodSpecPtr PodSpec_; + const TDuration PollPeriod_; + + TCriDescriptor ContainerDescriptor_; + + TPeriodicExecutorPtr AsyncWaitExecutor_; + + void DoSpawn() override + { + if (ContainerSpec_->Command.empty()) { + ContainerSpec_->Command = {Path_}; + } + ContainerSpec_->Arguments = std::vector<TString>(Args_.begin() + 1, Args_.end()); + ContainerSpec_->WorkingDirectory = WorkingDirectory_; + + ContainerSpec_->BindMounts.emplace_back( + NCri::TCriBindMount { + .ContainerPath = WorkingDirectory_, + .HostPath = WorkingDirectory_, + .ReadOnly = false, + } + ); + + for (const auto& keyVal : Env_) { + TStringBuf key, val; + if (TStringBuf(keyVal).TrySplit('=', key, val)) { + ContainerSpec_->Environment[key] = val; + } + } + + ContainerDescriptor_ = WaitFor(Executor_->CreateContainer(ContainerSpec_, PodDescriptor_, PodSpec_)) + .ValueOrThrow(); + + YT_LOG_DEBUG("Spawning process (Command: %v, Container: %v)", ContainerSpec_->Command[0], ContainerDescriptor_); + WaitFor(Executor_->StartContainer(ContainerDescriptor_)) + .ThrowOnError(); + + // TODO(khkebnikov) replace polling with CRI event + AsyncWaitExecutor_ = New<TPeriodicExecutor>( + GetSyncInvoker(), + BIND(&TCriProcess::PollContainerStatus, MakeStrong(this)), + PollPeriod_); + + AsyncWaitExecutor_->Start(); + } + + void PollContainerStatus() + { + Executor_->GetContainerStatus(ContainerDescriptor_) + .SubscribeUnique(BIND(&TCriProcess::OnContainerStatus, MakeStrong(this))); + } + + void OnContainerStatus(TErrorOr<TCriRuntimeApi::TRspContainerStatusPtr>&& responseOrError) + { + auto response = responseOrError.ValueOrThrow(); + if (!response->has_status()) { + return; + } + auto status = response->status(); + if (status.state() == NProto::CONTAINER_EXITED) { + auto error = DecodeExitCode(status.exit_code(), status.reason()); + YT_LOG_DEBUG(error, "Process finished (Container: %v)", ContainerDescriptor_); + YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop()); + FinishedPromise_.TrySet(error); + } + } +}; + +DEFINE_REFCOUNTED_TYPE(TCriProcess) + +//////////////////////////////////////////////////////////////////////////////// + +class TCriExecutor + : public ICriExecutor +{ +public: + TCriExecutor( + TCriExecutorConfigPtr config, + IChannelFactoryPtr channelFactory) + : Config_(std::move(config)) + , RuntimeApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->RuntimeEndpoint))) + , ImageApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->ImageEndpoint))) + { } + + TString GetPodCgroup(TString podName) const override + { + TStringBuilder cgroup; + cgroup.AppendString(Config_->BaseCgroup); + cgroup.AppendString("/"); + cgroup.AppendString(podName); + if (Config_->BaseCgroup.EndsWith(SystemdSliceSuffix)) { + cgroup.AppendString(SystemdSliceSuffix); + } + return cgroup.Flush(); + } + + TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) override + { + auto req = RuntimeApi_.Status(); + req->set_verbose(verbose); + return req->Invoke(); + } + + TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox( + std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) override + { + auto req = RuntimeApi_.ListPodSandbox(); + + { + auto* filter = req->mutable_filter(); + + if (auto namespace_ = Config_->Namespace) { + auto& labels = *filter->mutable_label_selector(); + labels[YTPodNamespaceLabel] = namespace_; + } + + if (initFilter) { + initFilter(*filter); + } + } + + return req->Invoke(); + } + + TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers( + std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override + { + auto req = RuntimeApi_.ListContainers(); + + { + auto* filter = req->mutable_filter(); + + if (auto namespace_ = Config_->Namespace) { + auto& labels = *filter->mutable_label_selector(); + labels[YTPodNamespaceLabel] = namespace_; + } + + if (initFilter) { + initFilter(*filter); + } + } + + return req->Invoke(); + } + + TFuture<void> ForEachPodSandbox( + const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback, + std::function<void(NProto::PodSandboxFilter&)> initFilter) override + { + return ListPodSandbox(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListPodSandboxPtr& rsp) { + for (const auto& pod : rsp->items()) { + TCriPodDescriptor descriptor{.Name=pod.metadata().name(), .Id=pod.id()}; + callback(descriptor, pod); + } + })); + } + + TFuture<void> ForEachContainer( + const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback, + std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override + { + return ListContainers(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListContainersPtr& rsp) { + for (const auto& ct : rsp->containers()) { + TCriDescriptor descriptor{.Name=ct.metadata().name(), .Id=ct.id()}; + callback(descriptor, ct); + } + })); + } + + TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus( + const TCriPodDescriptor& podDescriptor, bool verbose = false) override + { + auto req = RuntimeApi_.PodSandboxStatus(); + req->set_pod_sandbox_id(podDescriptor.Id); + req->set_verbose(verbose); + return req->Invoke(); + } + + TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus( + const TCriDescriptor& descriptor, bool verbose = false) override + { + auto req = RuntimeApi_.ContainerStatus(); + req->set_container_id(descriptor.Id); + req->set_verbose(verbose); + return req->Invoke(); + } + + TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) override + { + auto req = RuntimeApi_.RunPodSandbox(); + + FillPodSandboxConfig(req->mutable_config(), *podSpec); + + if (Config_->RuntimeHandler) { + req->set_runtime_handler(Config_->RuntimeHandler); + } + + return req->Invoke().Apply(BIND([name = podSpec->Name] (const TCriRuntimeApi::TRspRunPodSandboxPtr& rsp) -> TCriPodDescriptor { + return TCriPodDescriptor{.Name = name, .Id = rsp->pod_sandbox_id()}; + })); + } + + TFuture<void> StopPodSandbox(const TCriPodDescriptor& podDescriptor) override + { + auto req = RuntimeApi_.StopPodSandbox(); + req->set_pod_sandbox_id(podDescriptor.Id); + return req->Invoke().AsVoid(); + } + + TFuture<void> RemovePodSandbox(const TCriPodDescriptor& podDescriptor) override + { + auto req = RuntimeApi_.RemovePodSandbox(); + req->set_pod_sandbox_id(podDescriptor.Id); + return req->Invoke().AsVoid(); + } + + TFuture<void> UpdatePodResources( + const TCriPodDescriptor& /*pod*/, + const TCriContainerResources& /*resources*/) override + { + return MakeFuture(TError("Not implemented")); + } + + TFuture<TCriDescriptor> CreateContainer( + TCriContainerSpecPtr ctSpec, + const TCriPodDescriptor& podDescriptor, + TCriPodSpecPtr podSpec) override + { + auto req = RuntimeApi_.CreateContainer(); + req->set_pod_sandbox_id(podDescriptor.Id); + + auto* config = req->mutable_config(); + + { + auto* metadata = config->mutable_metadata(); + metadata->set_name(ctSpec->Name); + } + + { + auto& labels = *config->mutable_labels(); + + for (const auto& [key, val] : ctSpec->Labels) { + labels[key] = val; + } + + labels[YTPodNamespaceLabel] = Config_->Namespace; + labels[YTPodNameLabel] = podSpec->Name; + labels[YTContainerNameLabel] = ctSpec->Name; + } + + FillImageSpec(config->mutable_image(), ctSpec->Image); + + for (const auto& mountSpec : ctSpec->BindMounts) { + auto* mount = config->add_mounts(); + mount->set_container_path(mountSpec.ContainerPath); + mount->set_host_path(mountSpec.HostPath); + mount->set_readonly(mountSpec.ReadOnly); + mount->set_propagation(NProto::PROPAGATION_PRIVATE); + } + + { + ToProto(config->mutable_command(), ctSpec->Command); + ToProto(config->mutable_args(), ctSpec->Arguments); + + config->set_working_dir(ctSpec->WorkingDirectory); + + for (const auto& [key, val] : ctSpec->Environment) { + auto* env = config->add_envs(); + env->set_key(key); + env->set_value(val); + } + } + + { + auto* linux = config->mutable_linux(); + FillLinuxContainerResources(linux->mutable_resources(), ctSpec->Resources); + + auto* security = linux->mutable_security_context(); + + auto* namespaces = security->mutable_namespace_options(); + namespaces->set_network(NProto::NODE); + + security->set_readonly_rootfs(ctSpec->ReadOnlyRootFS); + + if (ctSpec->Credentials.Uid) { + security->mutable_run_as_user()->set_value(*ctSpec->Credentials.Uid); + } + if (ctSpec->Credentials.Gid) { + security->mutable_run_as_group()->set_value(*ctSpec->Credentials.Gid); + } + ToProto(security->mutable_supplemental_groups(), ctSpec->Credentials.Groups); + } + + FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec); + + return req->Invoke().Apply(BIND([name = ctSpec->Name] (const TCriRuntimeApi::TRspCreateContainerPtr& rsp) -> TCriDescriptor { + return TCriDescriptor{.Name = "", .Id = rsp->container_id()}; + })); + } + + TFuture<void> StartContainer(const TCriDescriptor& descriptor) override + { + auto req = RuntimeApi_.StartContainer(); + req->set_container_id(descriptor.Id); + return req->Invoke().AsVoid(); + } + + TFuture<void> StopContainer(const TCriDescriptor& descriptor, TDuration timeout) override + { + auto req = RuntimeApi_.StopContainer(); + req->set_container_id(descriptor.Id); + req->set_timeout(timeout.Seconds()); + return req->Invoke().AsVoid(); + } + + TFuture<void> RemoveContainer(const TCriDescriptor& descriptor) override + { + auto req = RuntimeApi_.RemoveContainer(); + req->set_container_id(descriptor.Id); + return req->Invoke().AsVoid(); + } + + TFuture<void> UpdateContainerResources(const TCriDescriptor& descriptor, const TCriContainerResources& resources) override + { + auto req = RuntimeApi_.UpdateContainerResources(); + req->set_container_id(descriptor.Id); + FillLinuxContainerResources(req->mutable_linux(), resources); + return req->Invoke().AsVoid(); + } + + void CleanNamespace() override + { + YT_VERIFY(Config_->Namespace); + auto pods = WaitFor(ListPodSandbox()) + .ValueOrThrow(); + + { + std::vector<TFuture<void>> futures; + futures.reserve(pods->items_size()); + for (const auto& pod : pods->items()) { + TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id() }; + futures.push_back(StopPodSandbox(podDescriptor)); + } + WaitFor(AllSucceeded(std::move(futures))) + .ThrowOnError(); + } + + { + std::vector<TFuture<void>> futures; + futures.reserve(pods->items_size()); + for (const auto& pod : pods->items()) { + TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id()}; + futures.push_back(RemovePodSandbox(podDescriptor)); + } + WaitFor(AllSucceeded(std::move(futures))) + .ThrowOnError(); + } + } + + void CleanPodSandbox(const TCriPodDescriptor& podDescriptor) override + { + auto containers = WaitFor(ListContainers([=] (NProto::ContainerFilter& filter) { + filter.set_pod_sandbox_id(podDescriptor.Id); + })) + .ValueOrThrow(); + + { + std::vector<TFuture<void>> futures; + futures.reserve(containers->containers_size()); + for (const auto& ct : containers->containers()) { + TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()}; + futures.push_back(StopContainer(ctDescriptor, TDuration::Zero())); + } + WaitFor(AllSucceeded(std::move(futures))) + .ThrowOnError(); + } + + { + std::vector<TFuture<void>> futures; + futures.reserve(containers->containers_size()); + for (const auto& ct : containers->containers()) { + TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()}; + futures.push_back(RemoveContainer(ctDescriptor)); + } + WaitFor(AllSucceeded(std::move(futures))) + .ThrowOnError(); + } + } + + TFuture<TCriImageApi::TRspListImagesPtr> ListImages( + std::function<void(NProto::ImageFilter&)> initFilter = nullptr) override + { + auto req = ImageApi_.ListImages(); + if (initFilter) { + initFilter(*req->mutable_filter()); + } + return req->Invoke(); + } + + TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus( + const TCriImageDescriptor& image, + bool verbose = false) override + { + auto req = ImageApi_.ImageStatus(); + FillImageSpec(req->mutable_image(), image); + req->set_verbose(verbose); + return req->Invoke(); + } + + TFuture<TCriImageDescriptor> PullImage( + const TCriImageDescriptor& image, + bool always, + TCriAuthConfigPtr authConfig, + TCriPodSpecPtr podSpec) override + { + if (!always) { + return GetImageStatus(image) + .Apply(BIND([=, this, this_ = MakeStrong(this)] (const TCriImageApi::TRspImageStatusPtr& imageStatus) { + if (imageStatus->has_image()) { + return MakeFuture(TCriImageDescriptor{.Image = imageStatus->image().id()}); + } + return PullImage(image, /*always*/ true, authConfig, podSpec); + })); + } + + auto req = ImageApi_.PullImage(); + FillImageSpec(req->mutable_image(), image); + if (authConfig) { + FillAuthConfig(req->mutable_auth(), *authConfig); + } + if (podSpec) { + FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec); + } + return req->Invoke().Apply(BIND([] (const TCriImageApi::TRspPullImagePtr& rsp) -> TCriImageDescriptor { + return TCriImageDescriptor{.Image = rsp->image_ref()}; + })); + } + + TFuture<void> RemoveImage(const TCriImageDescriptor& image) override + { + auto req = ImageApi_.RemoveImage(); + FillImageSpec(req->mutable_image(), image); + return req->Invoke().AsVoid(); + } + + TProcessBasePtr CreateProcess( + const TString& path, + TCriContainerSpecPtr containerSpec, + const TCriPodDescriptor& podDescriptor, + TCriPodSpecPtr podSpec) override + { + return New<TCriProcess>(path, this, std::move(containerSpec), podDescriptor, std::move(podSpec)); + } + +private: + const TCriExecutorConfigPtr Config_; + TCriRuntimeApi RuntimeApi_; + TCriImageApi ImageApi_; + + void FillLinuxContainerResources(NProto::LinuxContainerResources* resources, const TCriContainerResources& spec) + { + auto* unified = resources->mutable_unified(); + + if (spec.CpuLimit) { + i64 period = Config_->CpuPeriod.MicroSeconds(); + i64 quota = period * *spec.CpuLimit; + + resources->set_cpu_period(period); + resources->set_cpu_quota(quota); + } + + if (spec.MemoryLimit) { + resources->set_memory_limit_in_bytes(*spec.MemoryLimit); + } + + if (spec.MemoryRequest) { + (*unified)["memory.low"] = ToString(*spec.MemoryRequest); + } + } + + void FillPodSandboxConfig(NProto::PodSandboxConfig* config, const TCriPodSpec& spec) + { + { + auto* metadata = config->mutable_metadata(); + metadata->set_namespace_(Config_->Namespace); + metadata->set_name(spec.Name); + metadata->set_uid(spec.Name); + } + + { + auto& labels = *config->mutable_labels(); + labels[YTPodNamespaceLabel] = Config_->Namespace; + labels[YTPodNameLabel] = spec.Name; + } + + { + auto* linux = config->mutable_linux(); + linux->set_cgroup_parent(GetPodCgroup(spec.Name)); + + auto* security = linux->mutable_security_context(); + auto* namespaces = security->mutable_namespace_options(); + namespaces->set_network(NProto::NODE); + } + } + + void FillImageSpec(NProto::ImageSpec* spec, const TCriImageDescriptor& image) + { + spec->set_image(image.Image); + } + + void FillAuthConfig(NProto::AuthConfig* auth, const TCriAuthConfig& authConfig) + { + if (!authConfig.Username.empty()) { + auth->set_username(authConfig.Username); + } + if (!authConfig.Password.empty()) { + auth->set_password(authConfig.Password); + } + if (!authConfig.Auth.empty()) { + auth->set_auth(authConfig.Auth); + } + if (!authConfig.ServerAddress.empty()) { + auth->set_server_address(authConfig.ServerAddress); + } + if (!authConfig.IdentityToken.empty()) { + auth->set_identity_token(authConfig.IdentityToken); + } + if (!authConfig.RegistryToken.empty()) { + auth->set_registry_token(authConfig.RegistryToken); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config) +{ + return New<TCriExecutor>( + std::move(config), + GetGrpcChannelFactory()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_executor.h b/yt/yt/library/containers/cri/cri_executor.h new file mode 100644 index 0000000000..de9741721f --- /dev/null +++ b/yt/yt/library/containers/cri/cri_executor.h @@ -0,0 +1,207 @@ +#pragma once + +#include "public.h" +#include "config.h" +#include "cri_api.h" + +#include <yt/yt/library/process/process.h> + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +struct TCriDescriptor +{ + TString Name; + TString Id; +}; + +struct TCriPodDescriptor +{ + TString Name; + TString Id; +}; + +struct TCriImageDescriptor +{ + TString Image; +}; + +void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf spec); +void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf spec); +void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf spec); + +//////////////////////////////////////////////////////////////////////////////// + +struct TCriContainerResources +{ + std::optional<double> CpuLimit; + std::optional<double> CpuRequest; + std::optional<i64> MemoryLimit; + std::optional<i64> MemoryRequest; +}; + +struct TCriPodSpec + : public TRefCounted +{ + TString Name; + TCriContainerResources Resources; +}; + +DEFINE_REFCOUNTED_TYPE(TCriPodSpec) + +struct TCriBindMount +{ + TString ContainerPath; + TString HostPath; + bool ReadOnly; +}; + +struct TCriCredentials +{ + std::optional<i64> Uid; + std::optional<i64> Gid; + std::vector<i64> Groups; +}; + +struct TCriContainerSpec + : public TRefCounted +{ + TString Name; + + THashMap<TString, TString> Labels; + + TCriImageDescriptor Image; + + bool ReadOnlyRootFS; + + std::vector<TCriBindMount> BindMounts; + + TCriCredentials Credentials; + + TCriContainerResources Resources; + + //! Command to execute (i.e., entrypoint for docker). + std::vector<TString> Command; + + //! Arguments for the Command (i.e., command for docker). + std::vector<TString> Arguments; + + //! Current working directory of the command. + TString WorkingDirectory; + + //! Environment variable to set in the container. + THashMap<TString, TString> Environment; +}; + +DEFINE_REFCOUNTED_TYPE(TCriContainerSpec) + +//////////////////////////////////////////////////////////////////////////////// + +//! Wrapper around CRI gRPC API +//! +//! @see yt/yt/contrib/cri-api/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto +//! @see https://github.com/kubernetes/cri-api +struct ICriExecutor + : public TRefCounted +{ + //! Returns status of the CRI runtime. + //! @param verbose fill field "info" with runtime-specific debug. + virtual TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) = 0; + + // PodSandbox + + virtual TString GetPodCgroup(TString podName) const = 0; + + virtual TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox( + std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0; + + virtual TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers( + std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0; + + virtual TFuture<void> ForEachPodSandbox( + const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback, + std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0; + + virtual TFuture<void> ForEachContainer( + const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback, + std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0; + + //! Returns status of the pod. + //! @param verbose fill field "info" with runtime-specific debug. + virtual TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus( + const TCriPodDescriptor& pod, bool verbose = false) = 0; + + //! Returns status of the container. + //! @param verbose fill "info" with runtime-specific debug information. + virtual TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus( + const TCriDescriptor& ct, bool verbose = false) = 0; + + virtual TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) = 0; + virtual TFuture<void> StopPodSandbox(const TCriPodDescriptor& pod) = 0; + virtual TFuture<void> RemovePodSandbox(const TCriPodDescriptor& pod) = 0; + virtual TFuture<void> UpdatePodResources( + const TCriPodDescriptor& pod, + const TCriContainerResources& resources) = 0; + + //! Remove all pods and containers in namespace managed by executor. + virtual void CleanNamespace() = 0; + + //! Remove all containers in one pod. + virtual void CleanPodSandbox(const TCriPodDescriptor& pod) = 0; + + virtual TFuture<TCriDescriptor> CreateContainer( + TCriContainerSpecPtr containerSpec, + const TCriPodDescriptor& pod, + TCriPodSpecPtr podSpec) = 0; + + virtual TFuture<void> StartContainer(const TCriDescriptor& ct) = 0; + + //! Stops container if it's running. + //! @param timeout defines timeout for graceful stop, timeout=0 - kill instantly. + virtual TFuture<void> StopContainer( + const TCriDescriptor& ct, + TDuration timeout = TDuration::Zero()) = 0; + + virtual TFuture<void> RemoveContainer(const TCriDescriptor& ct) = 0; + + virtual TFuture<void> UpdateContainerResources( + const TCriDescriptor& ct, + const TCriContainerResources& resources) = 0; + + virtual TFuture<TCriImageApi::TRspListImagesPtr> ListImages( + std::function<void(NProto::ImageFilter&)> initFilter = nullptr) = 0; + + //! Returns status of the image. + //! @param verbose fill field "info" with runtime-specific debug. + virtual TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus( + const TCriImageDescriptor& image, + bool verbose = false) = 0; + + virtual TFuture<TCriImageDescriptor> PullImage( + const TCriImageDescriptor& image, + bool always = false, + TCriAuthConfigPtr authConfig = nullptr, + TCriPodSpecPtr podSpec = nullptr) = 0; + + virtual TFuture<void> RemoveImage(const TCriImageDescriptor& image) = 0; + + // FIXME(khlebnikov): temporary compat + virtual TProcessBasePtr CreateProcess( + const TString& path, + TCriContainerSpecPtr containerSpec, + const TCriPodDescriptor& pod, + TCriPodSpecPtr podSpec) = 0; +}; + +DEFINE_REFCOUNTED_TYPE(ICriExecutor) + +//////////////////////////////////////////////////////////////////////////////// + +ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/private.h b/yt/yt/library/containers/cri/private.h new file mode 100644 index 0000000000..36fdf194f5 --- /dev/null +++ b/yt/yt/library/containers/cri/private.h @@ -0,0 +1,13 @@ +#pragma once + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger Logger("Cri"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/public.h b/yt/yt/library/containers/cri/public.h new file mode 100644 index 0000000000..a12ee86d57 --- /dev/null +++ b/yt/yt/library/containers/cri/public.h @@ -0,0 +1,17 @@ +#pragma once + +#include <yt/yt/core/misc/intrusive_ptr.h> + +namespace NYT::NContainers::NCri { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_STRUCT(TCriPodSpec) +DECLARE_REFCOUNTED_STRUCT(TCriContainerSpec) +DECLARE_REFCOUNTED_CLASS(TCriExecutorConfig) +DECLARE_REFCOUNTED_CLASS(TCriAuthConfig) +DECLARE_REFCOUNTED_STRUCT(ICriExecutor) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/ya.make b/yt/yt/library/containers/cri/ya.make new file mode 100644 index 0000000000..dc9dd15a0b --- /dev/null +++ b/yt/yt/library/containers/cri/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +PEERDIR( + yt/yt/core + yt/yt/core/rpc/grpc + yt/yt/contrib/cri-api +) + +SRCS( + cri_api.cpp + cri_executor.cpp + config.cpp +) + +ADDINCL( + ONE_LEVEL + yt/yt/contrib/cri-api +) + +END() diff --git a/yt/yt/library/containers/disk_manager/config.cpp b/yt/yt/library/containers/disk_manager/config.cpp new file mode 100644 index 0000000000..84484db630 --- /dev/null +++ b/yt/yt/library/containers/disk_manager/config.cpp @@ -0,0 +1,61 @@ +#include "config.h" + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +void TMockedDiskConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("disk_id", &TThis::DiskId) + .Default(); + registrar.Parameter("device_path", &TThis::DevicePath) + .Default(); + registrar.Parameter("device_name", &TThis::DeviceName) + .Default(); + registrar.Parameter("disk_model", &TThis::DiskModel) + .Default(); + registrar.Parameter("partition_fs_labels", &TThis::PartitionFsLabels) + .Default(); + registrar.Parameter("state", &TThis::State) + .Default(EDiskState::Ok); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDiskInfoProviderConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("disk_ids", &TThis::DiskIds) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDiskManagerProxyConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("disk_manager_address", &TThis::DiskManagerAddress) + .Default("unix:/run/yandex-diskmanager/yandex-diskmanager.sock"); + registrar.Parameter("disk_manager_service_name", &TThis::DiskManagerServiceName) + .Default("diskman.DiskManager"); + + registrar.Parameter("is_mock", &TThis::IsMock) + .Default(false); + registrar.Parameter("mock_disks", &TThis::MockDisks) + .Default(); + registrar.Parameter("mock_yt_paths", &TThis::MockYtPaths) + .Default(); + + registrar.Parameter("request_timeout", &TThis::RequestTimeout) + .Default(TDuration::Seconds(10)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDiskManagerProxyDynamicConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("request_timeout", &TThis::RequestTimeout) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/config.h b/yt/yt/library/containers/disk_manager/config.h new file mode 100644 index 0000000000..4f01d378b9 --- /dev/null +++ b/yt/yt/library/containers/disk_manager/config.h @@ -0,0 +1,79 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +struct TMockedDiskConfig + : public NYTree::TYsonStruct +{ + TString DiskId; + TString DevicePath; + TString DeviceName; + TString DiskModel; + std::vector<TString> PartitionFsLabels; + EDiskState State; + + REGISTER_YSON_STRUCT(TMockedDiskConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TMockedDiskConfig) + +//////////////////////////////////////////////////////////////////////////////// + +struct TDiskManagerProxyConfig + : public NYTree::TYsonStruct +{ + TString DiskManagerAddress; + TString DiskManagerServiceName; + + bool IsMock; + std::vector<TMockedDiskConfigPtr> MockDisks; + std::vector<TString> MockYtPaths; + + TDuration RequestTimeout; + + REGISTER_YSON_STRUCT(TDiskManagerProxyConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyConfig) + +//////////////////////////////////////////////////////////////////////////////// + +struct TDiskInfoProviderConfig + : public NYTree::TYsonStruct +{ + std::vector<TString> DiskIds; + + REGISTER_YSON_STRUCT(TDiskInfoProviderConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TDiskInfoProviderConfig) + +//////////////////////////////////////////////////////////////////////////////// + +struct TDiskManagerProxyDynamicConfig + : public NYTree::TYsonStruct +{ + std::optional<TDuration> RequestTimeout; + + REGISTER_YSON_STRUCT(TDiskManagerProxyDynamicConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyDynamicConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.cpp b/yt/yt/library/containers/disk_manager/disk_info_provider.cpp new file mode 100644 index 0000000000..0ee3a5b6cb --- /dev/null +++ b/yt/yt/library/containers/disk_manager/disk_info_provider.cpp @@ -0,0 +1,64 @@ +#include "disk_info_provider.h" + +#include <yt/yt/library/containers/disk_manager/disk_manager_proxy.h> + +#include <yt/yt/core/actions/future.h> +#include <yt/yt/core/actions/invoker_util.h> + +#include <yt/yt/core/concurrency/public.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +TDiskInfoProvider::TDiskInfoProvider( + IDiskManagerProxyPtr diskManagerProxy, + TDiskInfoProviderConfigPtr config) + : DiskManagerProxy_(std::move(diskManagerProxy)) + , Config_(std::move(config)) +{ } + +const std::vector<TString>& TDiskInfoProvider::GetConfigDiskIds() const +{ + return Config_->DiskIds; +} + +TFuture<std::vector<TDiskInfo>> TDiskInfoProvider::GetYTDiskInfos() +{ + auto diskInfosFuture = DiskManagerProxy_->GetDisks(); + auto ytDiskPathsFuture = DiskManagerProxy_->GetYtDiskMountPaths(); + + // Merge two futures and filter disks placed in /yt. + return diskInfosFuture.Apply(BIND([=] (const std::vector<TDiskInfo>& diskInfos) { + return ytDiskPathsFuture.Apply(BIND([=] (const THashSet<TString>& diskPaths) { + std::vector<TDiskInfo> disks; + + for (const auto& diskInfo : diskInfos) { + for (const auto& partitionFsLabel : diskInfo.PartitionFsLabels) { + if (diskPaths.contains(partitionFsLabel)) { + disks.push_back(diskInfo); + break; + } + } + } + + return disks; + })); + })); +} + +TFuture<void> TDiskInfoProvider::RecoverDisk(const TString& diskId) +{ + return DiskManagerProxy_->RecoverDiskById(diskId, ERecoverPolicy::RecoverAuto); +} + +TFuture<void> TDiskInfoProvider::FailDisk( + const TString& diskId, + const TString& reason) +{ + return DiskManagerProxy_->FailDiskById(diskId, reason); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.h b/yt/yt/library/containers/disk_manager/disk_info_provider.h new file mode 100644 index 0000000000..b8d686438d --- /dev/null +++ b/yt/yt/library/containers/disk_manager/disk_info_provider.h @@ -0,0 +1,38 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/actions/future.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +class TDiskInfoProvider + : public TRefCounted +{ +public: + TDiskInfoProvider( + IDiskManagerProxyPtr diskManagerProxy, + TDiskInfoProviderConfigPtr config); + + const std::vector<TString>& GetConfigDiskIds() const; + + TFuture<std::vector<TDiskInfo>> GetYTDiskInfos(); + + TFuture<void> RecoverDisk(const TString& diskId); + + TFuture<void> FailDisk( + const TString& diskId, + const TString& reason); + +private: + const IDiskManagerProxyPtr DiskManagerProxy_; + const TDiskInfoProviderConfigPtr Config_; +}; + +DEFINE_REFCOUNTED_TYPE(TDiskInfoProvider) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp b/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp new file mode 100644 index 0000000000..961723c51f --- /dev/null +++ b/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp @@ -0,0 +1,49 @@ +#include "disk_manager_proxy.h" + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +struct TDiskManagerProxyMock + : public IDiskManagerProxy +{ + virtual TFuture<THashSet<TString>> GetYtDiskMountPaths() + { + THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); + } + + virtual TFuture<std::vector<TDiskInfo>> GetDisks() + { + THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); + } + + virtual TFuture<void> RecoverDiskById(const TString& /*diskId*/, ERecoverPolicy /*recoverPolicy*/) + { + THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); + } + + virtual TFuture<void> FailDiskById(const TString& /*diskId*/, const TString& /*reason*/) + { + THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); + } + + virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& /*newConfig*/) + { + // Do nothing + } +}; + +DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyMock) + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr /*config*/) +{ + // This implementation is used when disk_manager_proxy_impl.cpp is not linked. + + return New<TDiskManagerProxyMock>(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.h b/yt/yt/library/containers/disk_manager/disk_manager_proxy.h new file mode 100644 index 0000000000..d2da5c1873 --- /dev/null +++ b/yt/yt/library/containers/disk_manager/disk_manager_proxy.h @@ -0,0 +1,38 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/containers/disk_manager/config.h> + +#include <yt/yt/core/misc/atomic_object.h> + +#include <yt/yt/core/rpc/client.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +struct IDiskManagerProxy + : public virtual TRefCounted +{ + virtual TFuture<THashSet<TString>> GetYtDiskMountPaths() = 0; + + virtual TFuture<std::vector<TDiskInfo>> GetDisks() = 0; + + virtual TFuture<void> RecoverDiskById(const TString& diskId, ERecoverPolicy recoverPolicy) = 0; + + virtual TFuture<void> FailDiskById(const TString& diskId, const TString& reason) = 0; + + virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& newConfig) = 0; + +}; + +DEFINE_REFCOUNTED_TYPE(IDiskManagerProxy) + +//////////////////////////////////////////////////////////////////////////////// + +IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr config); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/public.h b/yt/yt/library/containers/disk_manager/public.h new file mode 100644 index 0000000000..8a812638f3 --- /dev/null +++ b/yt/yt/library/containers/disk_manager/public.h @@ -0,0 +1,48 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(EDiskState, + ((Unknown) (0)) + ((Ok) (1)) + ((Failed) (2)) + ((RecoverWait) (3)) +); + +// 1. Remount all disk volumes to it's default state +// 2. Recreate disk layout, all data on disk will be lost +// 3. Replace phisical disk +DEFINE_ENUM(ERecoverPolicy, + ((RecoverAuto) (0)) + ((RecoverMount) (1)) + ((RecoverLayout) (2)) + ((RecoverDisk) (3)) +); + +struct TDiskInfo +{ + TString DiskId; + TString DevicePath; + TString DeviceName; + TString DiskModel; + THashSet<TString> PartitionFsLabels; + EDiskState State; +}; + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_STRUCT(TMockedDiskConfig) +DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyConfig) +DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyDynamicConfig) +DECLARE_REFCOUNTED_STRUCT(TDiskInfoProviderConfig) + +DECLARE_REFCOUNTED_STRUCT(IDiskManagerProxy) +DECLARE_REFCOUNTED_CLASS(TDiskInfoProvider) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/ya.make b/yt/yt/library/containers/disk_manager/ya.make new file mode 100644 index 0000000000..dcb260cf38 --- /dev/null +++ b/yt/yt/library/containers/disk_manager/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +PEERDIR( + yt/yt/core +) + +SRCS( + config.cpp + disk_info_provider.cpp + disk_manager_proxy.cpp +) + +IF (NOT OPENSOURCE) + INCLUDE(ya_non_opensource.inc) +ENDIF() + +END() diff --git a/yt/yt/library/containers/instance.cpp b/yt/yt/library/containers/instance.cpp new file mode 100644 index 0000000000..0a56987e1b --- /dev/null +++ b/yt/yt/library/containers/instance.cpp @@ -0,0 +1,812 @@ +#ifdef __linux__ + +#include "instance.h" + +#include "porto_executor.h" +#include "private.h" + +#include <yt/yt/library/containers/cgroup.h> +#include <yt/yt/library/containers/config.h> + +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/collection_helpers.h> +#include <yt/yt/core/misc/error.h> +#include <yt/yt/core/misc/fs.h> +#include <yt/yt/core/misc/proc.h> + +#include <library/cpp/porto/libporto.hpp> + +#include <util/stream/file.h> + +#include <util/string/cast.h> +#include <util/string/split.h> + +#include <util/system/env.h> + +#include <initializer_list> +#include <string> + +namespace NYT::NContainers { + +using namespace NConcurrency; +using namespace NNet; + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +// Porto passes command string to wordexp, where quota (') symbol +// is delimiter. So we must replace it with concatenation ('"'"'). +TString EscapeForWordexp(const char* in) +{ + TString buffer; + while (*in) { + if (*in == '\'') { + buffer.append(R"('"'"')"); + } else { + buffer.append(*in); + } + in++; + } + return buffer; +} + +i64 Extract( + const TString& input, + const TString& pattern, + const TString& terminator = "\n") +{ + auto start = input.find(pattern) + pattern.length(); + auto end = input.find(terminator, start); + return std::stol(input.substr(start, (end == input.npos) ? end : end - start)); +} + +i64 ExtractSum( + const TString& input, + const TString& pattern, + const TString& delimiter, + const TString& terminator = "\n") +{ + i64 sum = 0; + TString::size_type pos = 0; + while (pos < input.length()) { + pos = input.find(pattern, pos); + if (pos == input.npos) { + break; + } + pos += pattern.length(); + + pos = input.find(delimiter, pos); + if (pos == input.npos) { + break; + } + + pos++; + auto end = input.find(terminator, pos); + sum += std::stol(input.substr(pos, (end == input.npos) ? end : end - pos)); + } + return sum; +} + +using TPortoStatRule = std::pair<TString, std::function<i64(const TString& input)>>; + +static const std::function<i64(const TString&)> LongExtractor = [] (const TString& in) { + return std::stol(in); +}; + +static const std::function<i64(const TString&)> CoreNsPerSecondExtractor = [] (const TString& in) { + int pos = in.find("c", 0); + return (std::stod(in.substr(0, pos))) * 1'000'000'000; +}; + +static const std::function<i64(const TString&)> GetIOStatExtractor(const TString& rwMode = "") +{ + return [rwMode] (const TString& in) { + return ExtractSum(in, "hw", rwMode + ":", ";"); + }; +} + +static const std::function<i64(const TString&)> GetStatByKeyExtractor(const TString& statKey) +{ + return [statKey] (const TString& in) { + return Extract(in, statKey); + }; +} + +const THashMap<EStatField, TPortoStatRule> PortoStatRules = { + {EStatField::CpuUsage, {"cpu_usage", LongExtractor}}, + {EStatField::CpuSystemUsage, {"cpu_usage_system", LongExtractor}}, + {EStatField::CpuWait, {"cpu_wait", LongExtractor}}, + {EStatField::CpuThrottled, {"cpu_throttled", LongExtractor}}, + {EStatField::ThreadCount, {"thread_count", LongExtractor}}, + {EStatField::CpuLimit, {"cpu_limit_bound", CoreNsPerSecondExtractor}}, + {EStatField::CpuGuarantee, {"cpu_guarantee_bound", CoreNsPerSecondExtractor}}, + {EStatField::Rss, {"memory.stat", GetStatByKeyExtractor("total_rss")}}, + {EStatField::MappedFile, {"memory.stat", GetStatByKeyExtractor("total_mapped_file")}}, + {EStatField::MinorPageFaults, {"minor_faults", LongExtractor}}, + {EStatField::MajorPageFaults, {"major_faults", LongExtractor}}, + {EStatField::FileCacheUsage, {"cache_usage", LongExtractor}}, + {EStatField::AnonMemoryUsage, {"anon_usage", LongExtractor}}, + {EStatField::AnonMemoryLimit, {"anon_limit_total", LongExtractor}}, + {EStatField::MemoryUsage, {"memory_usage", LongExtractor}}, + {EStatField::MemoryGuarantee, {"memory_guarantee", LongExtractor}}, + {EStatField::MemoryLimit, {"memory_limit_total", LongExtractor}}, + {EStatField::MaxMemoryUsage, {"memory.max_usage_in_bytes", LongExtractor}}, + {EStatField::OomKills, {"oom_kills", LongExtractor}}, + {EStatField::OomKillsTotal, {"oom_kills_total", LongExtractor}}, + + {EStatField::IOReadByte, {"io_read", GetIOStatExtractor()}}, + {EStatField::IOWriteByte, {"io_write", GetIOStatExtractor()}}, + {EStatField::IOBytesLimit, {"io_limit", GetIOStatExtractor()}}, + {EStatField::IOReadOps, {"io_read_ops", GetIOStatExtractor()}}, + {EStatField::IOWriteOps, {"io_write_ops", GetIOStatExtractor()}}, + {EStatField::IOOps, {"io_ops", GetIOStatExtractor()}}, + {EStatField::IOOpsLimit, {"io_ops_limit", GetIOStatExtractor()}}, + {EStatField::IOTotalTime, {"io_time", GetIOStatExtractor()}}, + {EStatField::IOWaitTime, {"io_wait", GetIOStatExtractor()}}, + + {EStatField::NetTxBytes, {"net_tx_bytes[veth]", LongExtractor}}, + {EStatField::NetTxPackets, {"net_tx_packets[veth]", LongExtractor}}, + {EStatField::NetTxDrops, {"net_tx_drops[veth]", LongExtractor}}, + {EStatField::NetTxLimit, {"net_limit[veth]", LongExtractor}}, + {EStatField::NetRxBytes, {"net_rx_bytes[veth]", LongExtractor}}, + {EStatField::NetRxPackets, {"net_rx_packets[veth]", LongExtractor}}, + {EStatField::NetRxDrops, {"net_rx_drops[veth]", LongExtractor}}, + {EStatField::NetRxLimit, {"net_rx_limit[veth]", LongExtractor}}, +}; + +std::optional<TString> GetParentName(const TString& name) +{ + if (name.empty()) { + return std::nullopt; + } + + auto slashPosition = name.rfind('/'); + if (slashPosition == TString::npos) { + return ""; + } + + return name.substr(0, slashPosition); +} + +std::optional<TString> GetRootName(const TString& name) +{ + if (name.empty()) { + return std::nullopt; + } + + if (name == "/") { + return name; + } + + auto slashPosition = name.find('/'); + if (slashPosition == TString::npos) { + return name; + } + + return name.substr(0, slashPosition); +} + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoInstanceLauncher + : public IInstanceLauncher +{ +public: + TPortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor) + : Executor_(std::move(executor)) + , Logger(ContainersLogger.WithTag("Container: %v", name)) + { + Spec_.Name = name; + Spec_.CGroupControllers = { + "freezer", + "cpu", + "cpuacct", + "net_cls", + "blkio", + "devices", + "pids" + }; + } + + const TString& GetName() const override + { + return Spec_.Name; + } + + bool HasRoot() const override + { + return static_cast<bool>(Spec_.RootFS); + } + + void SetStdIn(const TString& inputPath) override + { + Spec_.StdinPath = inputPath; + } + + void SetStdOut(const TString& outPath) override + { + Spec_.StdoutPath = outPath; + } + + void SetStdErr(const TString& errorPath) override + { + Spec_.StderrPath = errorPath; + } + + void SetCwd(const TString& pwd) override + { + Spec_.CurrentWorkingDirectory = pwd; + } + + void SetCoreDumpHandler(const std::optional<TString>& handler) override + { + if (handler) { + Spec_.CoreCommand = *handler; + Spec_.EnableCoreDumps = true; + } else { + Spec_.EnableCoreDumps = false; + } + } + + void SetRoot(const TRootFS& rootFS) override + { + Spec_.RootFS = rootFS; + } + + void SetThreadLimit(i64 threadLimit) override + { + Spec_.ThreadLimit = threadLimit; + } + + void SetDevices(const std::vector<TDevice>& devices) override + { + Spec_.Devices = devices; + } + + void SetEnablePorto(EEnablePorto enablePorto) override + { + Spec_.EnablePorto = enablePorto; + } + + void SetIsolate(bool isolate) override + { + Spec_.Isolate = isolate; + } + + void EnableMemoryTracking() override + { + Spec_.CGroupControllers.push_back("memory"); + } + + void SetGroup(int groupId) override + { + Spec_.GroupId = groupId; + } + + void SetUser(const TString& user) override + { + Spec_.User = user; + } + + void SetIPAddresses(const std::vector<NNet::TIP6Address>& addresses, bool enableNat64) override + { + Spec_.IPAddresses = addresses; + Spec_.EnableNat64 = enableNat64; + Spec_.DisableNetwork = false; + } + + void DisableNetwork() override + { + Spec_.DisableNetwork = true; + Spec_.IPAddresses.clear(); + Spec_.EnableNat64 = false; + } + + void SetHostName(const TString& hostName) override + { + Spec_.HostName = hostName; + } + + TFuture<IInstancePtr> Launch( + const TString& path, + const std::vector<TString>& args, + const THashMap<TString, TString>& env) override + { + TStringBuilder commandBuilder; + auto append = [&] (const auto& value) { + commandBuilder.AppendString("'"); + commandBuilder.AppendString(NDetail::EscapeForWordexp(value.c_str())); + commandBuilder.AppendString("' "); + }; + + append(path); + for (const auto& arg : args) { + append(arg); + } + + Spec_.Command = commandBuilder.Flush(); + YT_LOG_DEBUG("Executing Porto container (Name: %v, Command: %v)", + Spec_.Name, + Spec_.Command); + + Spec_.Env = env; + + auto onContainerCreated = [this, this_ = MakeStrong(this)] (const TError& error) -> IInstancePtr { + if (!error.IsOK()) { + THROW_ERROR_EXCEPTION(EErrorCode::FailedToStartContainer, "Unable to start container") + << error; + } + + return GetPortoInstance(Executor_, Spec_.Name); + }; + + return Executor_->CreateContainer(Spec_, /* start */ true) + .Apply(BIND(onContainerCreated)); + } + +private: + IPortoExecutorPtr Executor_; + TRunnableContainerSpec Spec_; + const NLogging::TLogger Logger; +}; + +IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor) +{ + return New<TPortoInstanceLauncher>(name, executor); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoInstance + : public IInstance +{ +public: + static IInstancePtr GetSelf(IPortoExecutorPtr executor) + { + return New<TPortoInstance>(GetSelfContainerName(executor), executor); + } + + static IInstancePtr GetInstance(IPortoExecutorPtr executor, const TString& name) + { + return New<TPortoInstance>(name, executor); + } + + void Kill(int signal) override + { + auto error = WaitFor(Executor_->KillContainer(Name_, signal)); + // Killing already finished process is not an error. + if (error.FindMatching(EPortoErrorCode::InvalidState)) { + return; + } + if (!error.IsOK()) { + THROW_ERROR_EXCEPTION("Failed to send signal to Porto instance") + << TErrorAttribute("signal", signal) + << TErrorAttribute("container", Name_) + << error; + } + } + + void Destroy() override + { + WaitFor(Executor_->DestroyContainer(Name_)) + .ThrowOnError(); + Destroyed_ = true; + } + + void Stop() override + { + WaitFor(Executor_->StopContainer(Name_)) + .ThrowOnError(); + } + + TErrorOr<ui64> CalculateCpuUserUsage( + TErrorOr<ui64>& cpuUsage, + TErrorOr<ui64>& cpuSystemUsage) const + { + if (cpuUsage.IsOK() && cpuSystemUsage.IsOK()) { + return cpuUsage.Value() > cpuSystemUsage.Value() ? cpuUsage.Value() - cpuSystemUsage.Value() : 0; + } else if (cpuUsage.IsOK()) { + return TError("Missing property %Qlv in Porto response", EStatField::CpuSystemUsage) + << TErrorAttribute("container", Name_); + } else { + return TError("Missing property %Qlv in Porto response", EStatField::CpuUsage) + << TErrorAttribute("container", Name_); + } + } + + TResourceUsage GetResourceUsage( + const std::vector<EStatField>& fields) const override + { + std::vector<TString> properties; + properties.push_back("absolute_name"); + + bool userTimeRequested = false; + bool contextSwitchesRequested = false; + for (auto field : fields) { + if (auto it = NDetail::PortoStatRules.find(field)) { + const auto& rule = it->second; + properties.push_back(rule.first); + } else if (field == EStatField::ContextSwitchesDelta || field == EStatField::ContextSwitches) { + contextSwitchesRequested = true; + } else if (field == EStatField::CpuUserUsage) { + userTimeRequested = true; + } else { + THROW_ERROR_EXCEPTION("Unknown resource field %Qlv requested", field) + << TErrorAttribute("container", Name_); + } + } + + auto propertyMap = WaitFor(Executor_->GetContainerProperties(Name_, properties)) + .ValueOrThrow(); + + TResourceUsage result; + + for (auto field : fields) { + auto ruleIt = NDetail::PortoStatRules.find(field); + if (ruleIt == NDetail::PortoStatRules.end()) { + continue; + } + + const auto& [property, callback] = ruleIt->second; + auto& record = result[field]; + if (auto responseIt = propertyMap.find(property); responseIt != propertyMap.end()) { + const auto& valueOrError = responseIt->second; + if (valueOrError.IsOK()) { + const auto& value = valueOrError.Value(); + + try { + record = callback(value); + } catch (const std::exception& ex) { + record = TError("Error parsing Porto property %Qlv", field) + << TErrorAttribute("container", Name_) + << TErrorAttribute("property_value", value) + << ex; + } + } else { + record = TError("Error getting Porto property %Qlv", field) + << TErrorAttribute("container", Name_) + << valueOrError; + } + } else { + record = TError("Missing property %Qlv in Porto response", field) + << TErrorAttribute("container", Name_); + } + } + + // We should maintain context switch information even if this field + // is not requested since metrics of individual containers can go up and down. + auto subcontainers = WaitFor(Executor_->ListSubcontainers(Name_, /*includeRoot*/ true)) + .ValueOrThrow(); + + auto metricMap = WaitFor(Executor_->GetContainerMetrics(subcontainers, "ctxsw")) + .ValueOrThrow(); + + // TODO(don-dron): remove diff calculation from GetResourceUsage, because GetResourceUsage must return only snapshot stat. + { + auto guard = Guard(ContextSwitchMapLock_); + + for (const auto& [container, newValue] : metricMap) { + auto& prevValue = ContextSwitchMap_[container]; + TotalContextSwitches_ += std::max<i64>(0LL, newValue - prevValue); + prevValue = newValue; + } + + if (contextSwitchesRequested) { + result[EStatField::ContextSwitchesDelta] = TotalContextSwitches_; + } + } + + if (contextSwitchesRequested) { + ui64 totalContextSwitches = 0; + + for (const auto& [container, newValue] : metricMap) { + totalContextSwitches += std::max<ui64>(0UL, newValue); + } + + result[EStatField::ContextSwitches] = totalContextSwitches; + } + + if (userTimeRequested) { + result[EStatField::CpuUserUsage] = CalculateCpuUserUsage( + result[EStatField::CpuUsage], + result[EStatField::CpuSystemUsage]); + } + + return result; + } + + TResourceLimits GetResourceLimits() const override + { + std::vector<TString> properties; + static TString memoryLimitProperty = "memory_limit_total"; + static TString cpuLimitProperty = "cpu_limit_bound"; + static TString cpuGuaranteeProperty = "cpu_guarantee_bound"; + properties.push_back(memoryLimitProperty); + properties.push_back(cpuLimitProperty); + properties.push_back(cpuGuaranteeProperty); + + auto responseOrError = WaitFor(Executor_->GetContainerProperties(Name_, properties)); + THROW_ERROR_EXCEPTION_IF_FAILED(responseOrError, "Failed to get Porto container resource limits"); + + const auto& response = responseOrError.Value(); + + const auto& memoryLimitRsp = response.at(memoryLimitProperty); + THROW_ERROR_EXCEPTION_IF_FAILED(memoryLimitRsp, "Failed to get memory limit from Porto"); + + i64 memoryLimit; + if (!TryFromString<i64>(memoryLimitRsp.Value(), memoryLimit)) { + THROW_ERROR_EXCEPTION("Failed to parse memory limit value from Porto") + << TErrorAttribute(memoryLimitProperty, memoryLimitRsp.Value()); + } + + const auto& cpuLimitRsp = response.at(cpuLimitProperty); + THROW_ERROR_EXCEPTION_IF_FAILED(cpuLimitRsp, "Failed to get CPU limit from Porto"); + + double cpuLimit; + YT_VERIFY(cpuLimitRsp.Value().EndsWith('c')); + auto cpuLimitValue = TStringBuf(cpuLimitRsp.Value().begin(), cpuLimitRsp.Value().size() - 1); + if (!TryFromString<double>(cpuLimitValue, cpuLimit)) { + THROW_ERROR_EXCEPTION("Failed to parse CPU limit value from Porto") + << TErrorAttribute(cpuLimitProperty, cpuLimitRsp.Value()); + } + + const auto& cpuGuaranteeRsp = response.at(cpuGuaranteeProperty); + THROW_ERROR_EXCEPTION_IF_FAILED(cpuGuaranteeRsp, "Failed to get CPU guarantee from Porto"); + + double cpuGuarantee; + if (!cpuGuaranteeRsp.Value()) { + // XXX(ignat): hack for missing response from Porto. + cpuGuarantee = 0.0; + } else { + YT_VERIFY(cpuGuaranteeRsp.Value().EndsWith('c')); + auto cpuGuaranteeValue = TStringBuf(cpuGuaranteeRsp.Value().begin(), cpuGuaranteeRsp.Value().size() - 1); + if (!TryFromString<double>(cpuGuaranteeValue, cpuGuarantee)) { + THROW_ERROR_EXCEPTION("Failed to parse CPU guarantee value from Porto") + << TErrorAttribute(cpuGuaranteeProperty, cpuGuaranteeRsp.Value()); + } + } + + return TResourceLimits{ + .CpuLimit = cpuLimit, + .CpuGuarantee = cpuGuarantee, + .Memory = memoryLimit, + }; + } + + void SetCpuGuarantee(double cores) override + { + SetProperty("cpu_guarantee", ToString(cores) + "c"); + } + + void SetCpuLimit(double cores) override + { + SetProperty("cpu_limit", ToString(cores) + "c"); + } + + void SetCpuWeight(double weight) override + { + SetProperty("cpu_weight", weight); + } + + void SetMemoryGuarantee(i64 memoryGuarantee) override + { + SetProperty("memory_guarantee", memoryGuarantee); + } + + void SetIOWeight(double weight) override + { + SetProperty("io_weight", weight); + } + + void SetIOThrottle(i64 operations) override + { + SetProperty("io_ops_limit", operations); + } + + TString GetStderr() const override + { + return *WaitFor(Executor_->GetContainerProperty(Name_, "stderr")) + .ValueOrThrow(); + } + + TString GetName() const override + { + return Name_; + } + + std::optional<TString> GetParentName() const override + { + return NDetail::GetParentName(Name_); + } + + std::optional<TString> GetRootName() const override + { + return NDetail::GetRootName(Name_); + } + + pid_t GetPid() const override + { + auto pid = *WaitFor(Executor_->GetContainerProperty(Name_, "root_pid")) + .ValueOrThrow(); + return std::stoi(pid); + } + + i64 GetMajorPageFaultCount() const override + { + auto faults = WaitFor(Executor_->GetContainerProperty(Name_, "major_faults")) + .ValueOrThrow(); + return faults + ? std::stoll(*faults) + : 0; + } + + double GetCpuGuarantee() const override + { + auto result = WaitFor(Executor_->GetContainerProperty(Name_, "cpu_guarantee")) + .ValueOrThrow(); + return result + ? std::stod(*result) + : 0; + } + + std::vector<pid_t> GetPids() const override + { + auto getPidCgroup = [&] (const TString& cgroups) { + for (TStringBuf cgroup : StringSplitter(cgroups).SplitByString("; ")) { + if (cgroup.StartsWith("pids:")) { + auto startPosition = cgroup.find('/'); + YT_VERIFY(startPosition != TString::npos); + return cgroup.substr(startPosition); + } + } + THROW_ERROR_EXCEPTION("Pids cgroup not found for container %Qv", GetName()) + << TErrorAttribute("cgroups", cgroups); + }; + + auto cgroups = *WaitFor(Executor_->GetContainerProperty(Name_, "cgroups")) + .ValueOrThrow(); + // Porto returns full cgroup name, with mount prefix, such as "/sys/fs/cgroup/pids". + auto instanceCgroup = getPidCgroup(cgroups); + + std::vector<pid_t> pids; + for (auto pid : ListPids()) { + std::map<TString, TString> cgroups; + try { + cgroups = GetProcessCGroups(pid); + } catch (const std::exception& ex) { + YT_LOG_DEBUG(ex, "Failed to get CGroups for process (Pid: %v)", pid); + continue; + } + + // Pid cgroups are returned in short form. + auto processPidCgroup = cgroups["pids"]; + if (!processPidCgroup.empty() && instanceCgroup.EndsWith(processPidCgroup)) { + pids.push_back(pid); + } + } + + return pids; + } + + TFuture<void> Wait() override + { + return Executor_->PollContainer(Name_) + .Apply(BIND([] (int status) { + StatusToError(status) + .ThrowOnError(); + })); + } + +private: + const TString Name_; + const IPortoExecutorPtr Executor_; + const NLogging::TLogger Logger; + + bool Destroyed_ = false; + + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, ContextSwitchMapLock_); + mutable i64 TotalContextSwitches_ = 0; + mutable THashMap<TString, i64> ContextSwitchMap_; + + TPortoInstance(TString name, IPortoExecutorPtr executor) + : Name_(std::move(name)) + , Executor_(std::move(executor)) + , Logger(ContainersLogger.WithTag("Container: %v", Name_)) + { } + + void SetProperty(const TString& key, const TString& value) + { + WaitFor(Executor_->SetContainerProperty(Name_, key, value)) + .ThrowOnError(); + } + + void SetProperty(const TString& key, i64 value) + { + SetProperty(key, ToString(value)); + } + + void SetProperty(const TString& key, double value) + { + SetProperty(key, ToString(value)); + } + + DECLARE_NEW_FRIEND() +}; + +//////////////////////////////////////////////////////////////////////////////// + +TString GetSelfContainerName(const IPortoExecutorPtr& executor) +{ + try { + auto properties = WaitFor(executor->GetContainerProperties( + "self", + std::vector<TString>{"absolute_name", "absolute_namespace"})) + .ValueOrThrow(); + + auto absoluteName = properties.at("absolute_name") + .ValueOrThrow(); + auto absoluteNamespace = properties.at("absolute_namespace") + .ValueOrThrow(); + + if (absoluteName == "/") { + return absoluteName; + } + + if (absoluteName.length() < absoluteNamespace.length()) { + YT_VERIFY(absoluteName + "/" == absoluteNamespace); + return ""; + } else { + YT_VERIFY(absoluteName.StartsWith(absoluteNamespace)); + return absoluteName.substr(absoluteNamespace.length()); + } + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Failed to get name for container \"self\"") + << ex; + } +} + +IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor) +{ + return TPortoInstance::GetSelf(executor); +} + +IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name) +{ + return TPortoInstance::GetInstance(executor, name); +} + +IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor) +{ + auto self = GetSelfPortoInstance(executor); + return TPortoInstance::GetInstance(executor, *self->GetRootName()); +} + +double GetSelfPortoInstanceVCpuFactor() +{ + auto config = New<TPortoExecutorDynamicConfig>(); + auto executorPtr = CreatePortoExecutor(config, ""); + auto currentContainer = GetSelfPortoInstance(executorPtr); + double cpuLimit = currentContainer->GetResourceLimits().CpuLimit; + if (cpuLimit <= 0) { + THROW_ERROR_EXCEPTION("Cpu limit must be greater than 0"); + } + + // DEPLOY_VCPU_LIMIT stores value in millicores + if (TString vcpuLimitStr = GetEnv("DEPLOY_VCPU_LIMIT"); !vcpuLimitStr.Empty()) { + double vcpuLimit = FromString<double>(vcpuLimitStr) / 1000.0; + return vcpuLimit / cpuLimit; + } + THROW_ERROR_EXCEPTION("Failed to get vcpu limit from env variable"); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers + +#endif diff --git a/yt/yt/library/containers/instance.h b/yt/yt/library/containers/instance.h new file mode 100644 index 0000000000..ff6e0b3ce1 --- /dev/null +++ b/yt/yt/library/containers/instance.h @@ -0,0 +1,168 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/net/address.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +using TResourceUsage = THashMap<EStatField, TErrorOr<ui64>>; + +const std::vector<EStatField> InstanceStatFields{ + EStatField::CpuUsage, + EStatField::CpuUserUsage, + EStatField::CpuSystemUsage, + EStatField::CpuWait, + EStatField::CpuThrottled, + EStatField::ContextSwitches, + EStatField::ContextSwitchesDelta, + EStatField::ThreadCount, + EStatField::CpuLimit, + EStatField::CpuGuarantee, + + EStatField::Rss, + EStatField::MappedFile, + EStatField::MajorPageFaults, + EStatField::MinorPageFaults, + EStatField::FileCacheUsage, + EStatField::AnonMemoryUsage, + EStatField::AnonMemoryLimit, + EStatField::MemoryUsage, + EStatField::MemoryGuarantee, + EStatField::MemoryLimit, + EStatField::MaxMemoryUsage, + EStatField::OomKills, + EStatField::OomKillsTotal, + + EStatField::IOReadByte, + EStatField::IOWriteByte, + EStatField::IOBytesLimit, + EStatField::IOReadOps, + EStatField::IOWriteOps, + EStatField::IOOps, + EStatField::IOOpsLimit, + EStatField::IOTotalTime, + EStatField::IOWaitTime, + + EStatField::NetTxBytes, + EStatField::NetTxPackets, + EStatField::NetTxDrops, + EStatField::NetTxLimit, + EStatField::NetRxBytes, + EStatField::NetRxPackets, + EStatField::NetRxDrops, + EStatField::NetRxLimit, +}; + +struct TResourceLimits +{ + double CpuLimit; + double CpuGuarantee; + i64 Memory; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct IInstanceLauncher + : public TRefCounted +{ + virtual bool HasRoot() const = 0; + virtual const TString& GetName() const = 0; + + virtual void SetStdIn(const TString& inputPath) = 0; + virtual void SetStdOut(const TString& outPath) = 0; + virtual void SetStdErr(const TString& errorPath) = 0; + virtual void SetCwd(const TString& pwd) = 0; + + // Null core dump handler implies disabled core dumps. + virtual void SetCoreDumpHandler(const std::optional<TString>& handler) = 0; + virtual void SetRoot(const TRootFS& rootFS) = 0; + + virtual void SetThreadLimit(i64 threadLimit) = 0; + virtual void SetDevices(const std::vector<TDevice>& devices) = 0; + + virtual void SetEnablePorto(EEnablePorto enablePorto) = 0; + virtual void SetIsolate(bool isolate) = 0; + virtual void EnableMemoryTracking() = 0; + virtual void SetGroup(int groupId) = 0; + virtual void SetUser(const TString& user) = 0; + virtual void SetIPAddresses( + const std::vector<NNet::TIP6Address>& addresses, + bool enableNat64 = false) = 0; + virtual void DisableNetwork() = 0; + virtual void SetHostName(const TString& hostName) = 0; + + virtual TFuture<IInstancePtr> Launch( + const TString& path, + const std::vector<TString>& args, + const THashMap<TString, TString>& env) = 0; +}; + +DEFINE_REFCOUNTED_TYPE(IInstanceLauncher) + +#ifdef _linux_ +IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor); +#endif + +//////////////////////////////////////////////////////////////////////////////// + +struct IInstance + : public TRefCounted +{ + virtual void Kill(int signal) = 0; + virtual void Stop() = 0; + virtual void Destroy() = 0; + + virtual TResourceUsage GetResourceUsage( + const std::vector<EStatField>& fields = InstanceStatFields) const = 0; + virtual TResourceLimits GetResourceLimits() const = 0; + virtual void SetCpuGuarantee(double cores) = 0; + virtual void SetCpuLimit(double cores) = 0; + virtual void SetCpuWeight(double weight) = 0; + virtual void SetIOWeight(double weight) = 0; + virtual void SetIOThrottle(i64 operations) = 0; + virtual void SetMemoryGuarantee(i64 memoryGuarantee) = 0; + + virtual TString GetStderr() const = 0; + + virtual TString GetName() const = 0; + virtual std::optional<TString> GetParentName() const = 0; + virtual std::optional<TString> GetRootName() const = 0; + + //! Returns externally visible pid of the root process inside container. + //! Throws if container is not running. + virtual pid_t GetPid() const = 0; + //! Returns the list of externally visible pids of processes running inside container. + virtual std::vector<pid_t> GetPids() const = 0; + + virtual i64 GetMajorPageFaultCount() const = 0; + virtual double GetCpuGuarantee() const = 0; + + //! Future is set when container reaches terminal state (stopped or dead). + //! Resulting error is OK iff container exited with code 0. + virtual TFuture<void> Wait() = 0; +}; + +DEFINE_REFCOUNTED_TYPE(IInstance) + +//////////////////////////////////////////////////////////////////////////////// + +#ifdef _linux_ +TString GetSelfContainerName(const IPortoExecutorPtr& executor); + +IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor); +IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor); +IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name); + +//! Works only in Yandex.Deploy pod environment where env DEPLOY_VCPU_LIMIT is set. +//! Throws if this env is absent. +double GetSelfPortoInstanceVCpuFactor(); +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/instance_limits_tracker.cpp b/yt/yt/library/containers/instance_limits_tracker.cpp new file mode 100644 index 0000000000..55ef7d2d67 --- /dev/null +++ b/yt/yt/library/containers/instance_limits_tracker.cpp @@ -0,0 +1,179 @@ +#include "public.h" +#include "instance_limits_tracker.h" +#include "instance.h" +#include "porto_resource_tracker.h" +#include "private.h" + +#include <yt/yt/core/concurrency/periodic_executor.h> + +#include <yt/yt/core/ytree/fluent.h> +#include <yt/yt/core/ytree/ypath_service.h> + +namespace NYT::NContainers { + +using namespace NYTree; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = ContainersLogger; + +//////////////////////////////////////////////////////////////////////////////// + +TInstanceLimitsTracker::TInstanceLimitsTracker( + IInstancePtr instance, + IInstancePtr root, + IInvokerPtr invoker, + TDuration updatePeriod) + : Invoker_(std::move(invoker)) + , Executor_(New<NConcurrency::TPeriodicExecutor>( + Invoker_, + BIND(&TInstanceLimitsTracker::DoUpdateLimits, MakeWeak(this)), + updatePeriod)) +{ +#ifdef _linux_ + SelfTracker_ = New<TPortoResourceTracker>(std::move(instance), updatePeriod / 2); + RootTracker_ = New<TPortoResourceTracker>(std::move(root), updatePeriod / 2); +#else + Y_UNUSED(instance); + Y_UNUSED(root); +#endif +} + +void TInstanceLimitsTracker::Start() +{ + if (!Running_) { + Executor_->Start(); + Running_ = true; + YT_LOG_INFO("Instance limits tracker started"); + } +} + +void TInstanceLimitsTracker::Stop() +{ + if (Running_) { + YT_UNUSED_FUTURE(Executor_->Stop()); + Running_ = false; + YT_LOG_INFO("Instance limits tracker stopped"); + } +} + +void TInstanceLimitsTracker::DoUpdateLimits() +{ + VERIFY_INVOKER_AFFINITY(Invoker_); + +#ifdef _linux_ + YT_LOG_DEBUG("Checking for instance limits update"); + + auto setIfOk = [] (auto* destination, const auto& valueOrError, const TString& fieldName, bool alert = true) { + if (valueOrError.IsOK()) { + *destination = valueOrError.Value(); + } else { + YT_LOG_ALERT_IF(alert, valueOrError, "Failed to get container property (Field: %v)", + fieldName); + + YT_LOG_DEBUG(valueOrError, "Failed to get container property (Field: %v)", + fieldName); + } + }; + + try { + auto memoryStatistics = SelfTracker_->GetMemoryStatistics(); + auto netStatistics = RootTracker_->GetNetworkStatistics(); + auto cpuStatistics = SelfTracker_->GetCpuStatistics(); + + setIfOk(&MemoryUsage_, memoryStatistics.Rss, "MemoryRss"); + + TDuration cpuGuarantee; + TDuration cpuLimit; + + if (cpuStatistics.GuaranteeTime.IsOK()) { + setIfOk(&cpuGuarantee, cpuStatistics.GuaranteeTime, "CpuGuarantee"); + } else { + // XXX(don-dron, ignat): do nothing, see NContainers::TPortoInstance::GetResourceLimits, hack for missing response from Porto. + } + + setIfOk(&cpuLimit, cpuStatistics.LimitTime, "CpuLimit"); + + if (CpuGuarantee_ != cpuGuarantee) { + YT_LOG_INFO("Instance CPU guarantee updated (OldCpuGuarantee: %v, NewCpuGuarantee: %v)", + CpuGuarantee_, + cpuGuarantee); + CpuGuarantee_ = cpuGuarantee; + // NB: We do not fire LimitsUpdated since this value used only for diagnostics. + } + + TInstanceLimits limits; + limits.Cpu = cpuLimit.SecondsFloat(); + + if (memoryStatistics.AnonLimit.IsOK() && memoryStatistics.MemoryLimit.IsOK()) { + i64 anonLimit = memoryStatistics.AnonLimit.Value(); + i64 memoryLimit = memoryStatistics.MemoryLimit.Value(); + + if (anonLimit > 0 && memoryLimit > 0) { + limits.Memory = std::min(anonLimit, memoryLimit); + } else if (anonLimit > 0) { + limits.Memory = anonLimit; + } else { + limits.Memory = memoryLimit; + } + } else { + setIfOk(&limits.Memory, memoryStatistics.MemoryLimit, "MemoryLimit"); + } + + static constexpr bool DontFireAlertOnError = {}; + setIfOk(&limits.NetTx, netStatistics.TxLimit, "NetTxLimit", DontFireAlertOnError); + setIfOk(&limits.NetRx, netStatistics.RxLimit, "NetRxLimit", DontFireAlertOnError); + + if (InstanceLimits_ != limits) { + YT_LOG_INFO("Instance limits updated (OldLimits: %v, NewLimits: %v)", + InstanceLimits_, + limits); + InstanceLimits_ = limits; + LimitsUpdated_.Fire(limits); + } + } catch (const std::exception& ex) { + YT_LOG_WARNING(ex, "Failed to get instance limits"); + } +#endif +} + +IYPathServicePtr TInstanceLimitsTracker::GetOrchidService() +{ + return IYPathService::FromProducer(BIND(&TInstanceLimitsTracker::DoBuildOrchid, MakeStrong(this))) + ->Via(Invoker_); +} + +void TInstanceLimitsTracker::DoBuildOrchid(NYson::IYsonConsumer* consumer) const +{ + NYTree::BuildYsonFluently(consumer) + .BeginMap() + .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) { + fluent.Item("cpu_limit").Value(InstanceLimits_->Cpu); + }) + .DoIf(static_cast<bool>(CpuGuarantee_), [&] (auto fluent) { + fluent.Item("cpu_guarantee").Value(*CpuGuarantee_); + }) + .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) { + fluent.Item("memory_limit").Value(InstanceLimits_->Memory); + }) + .DoIf(static_cast<bool>(MemoryUsage_), [&] (auto fluent) { + fluent.Item("memory_usage").Value(*MemoryUsage_); + }) + .EndMap(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf /*format*/) +{ + builder->AppendFormat( + "{Cpu: %v, Memory: %v, NetTx: %v, NetRx: %v}", + limits.Cpu, + limits.Memory, + limits.NetTx, + limits.NetRx); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/instance_limits_tracker.h b/yt/yt/library/containers/instance_limits_tracker.h new file mode 100644 index 0000000000..e652fff446 --- /dev/null +++ b/yt/yt/library/containers/instance_limits_tracker.h @@ -0,0 +1,59 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/actions/signal.h> + +#include <yt/yt/core/concurrency/public.h> + +#include <yt/yt/core/yson/public.h> + +#include <yt/yt/core/ytree/public.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +class TInstanceLimitsTracker + : public TRefCounted +{ +public: + //! Raises when container limits change. + DEFINE_SIGNAL(void(const TInstanceLimits&), LimitsUpdated); + +public: + TInstanceLimitsTracker( + IInstancePtr instance, + IInstancePtr root, + IInvokerPtr invoker, + TDuration updatePeriod); + + void Start(); + void Stop(); + + NYTree::IYPathServicePtr GetOrchidService(); + +private: + void DoUpdateLimits(); + void DoBuildOrchid(NYson::IYsonConsumer* consumer) const; + + TPortoResourceTrackerPtr SelfTracker_; + TPortoResourceTrackerPtr RootTracker_; + const IInvokerPtr Invoker_; + const NConcurrency::TPeriodicExecutorPtr Executor_; + + std::optional<TDuration> CpuGuarantee_; + std::optional<TInstanceLimits> InstanceLimits_; + std::optional<i64> MemoryUsage_; + bool Running_ = false; +}; + +DEFINE_REFCOUNTED_TYPE(TInstanceLimitsTracker) + +//////////////////////////////////////////////////////////////////////////////// + +void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf format); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_executor.cpp b/yt/yt/library/containers/porto_executor.cpp new file mode 100644 index 0000000000..a6a44fd20f --- /dev/null +++ b/yt/yt/library/containers/porto_executor.cpp @@ -0,0 +1,1079 @@ +#include "porto_executor.h" +#include "config.h" + +#include "private.h" + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/periodic_executor.h> +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/fs.h> + +#include <yt/yt/core/profiling/timing.h> + +#include <yt/yt/core/ytree/convert.h> + +#include <library/cpp/porto/proto/rpc.pb.h> + +#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> + +#include <string> + +namespace NYT::NContainers { + +using namespace NConcurrency; +using Porto::EError; + +//////////////////////////////////////////////////////////////////////////////// + +#ifdef _linux_ + +static const NLogging::TLogger& Logger = ContainersLogger; +static constexpr auto RetryInterval = TDuration::MilliSeconds(100); + +//////////////////////////////////////////////////////////////////////////////// + +TString PortoErrorCodeFormatter(int code) +{ + return TEnumTraits<EPortoErrorCode>::ToString(static_cast<EPortoErrorCode>(code)); +} + +YT_DEFINE_ERROR_CODE_RANGE(12000, 13999, "NYT::NContainers::EPortoErrorCode", PortoErrorCodeFormatter); + +//////////////////////////////////////////////////////////////////////////////// + +EPortoErrorCode ConvertPortoErrorCode(EError portoError) +{ + return static_cast<EPortoErrorCode>(PortoErrorCodeBase + portoError); +} + +bool IsRetriableErrorCode(EPortoErrorCode error, bool idempotent) +{ + return + error == EPortoErrorCode::Unknown || + // TODO(babenko): it's not obvious that we can always retry SocketError + // but this is how it has used to work for a while. + error == EPortoErrorCode::SocketError || + error == EPortoErrorCode::SocketTimeout && idempotent; +} + +THashMap<TString, TErrorOr<TString>> ParsePortoGetResponse( + const Porto::TGetResponse_TContainerGetListResponse& response) +{ + THashMap<TString, TErrorOr<TString>> result; + for (const auto& property : response.keyval()) { + if (property.error() == EError::Success) { + result[property.variable()] = property.value(); + } else { + result[property.variable()] = TError(ConvertPortoErrorCode(property.error()), property.errormsg()) + << TErrorAttribute("porto_error", ConvertPortoErrorCode(property.error())); + } + } + return result; +} + +THashMap<TString, TErrorOr<TString>> ParseSinglePortoGetResponse( + const TString& name, + const Porto::TGetResponse& getResponse) +{ + for (const auto& container : getResponse.list()) { + if (container.name() == name) { + return ParsePortoGetResponse(container); + } + } + THROW_ERROR_EXCEPTION("Unable to get properties from Porto") + << TErrorAttribute("container", name); +} + +THashMap<TString, THashMap<TString, TErrorOr<TString>>> ParseMultiplePortoGetResponse( + const Porto::TGetResponse& getResponse) +{ + THashMap<TString, THashMap<TString, TErrorOr<TString>>> result; + for (const auto& container : getResponse.list()) { + result[container.name()] = ParsePortoGetResponse(container); + } + return result; +} + +TString FormatEnablePorto(EEnablePorto value) +{ + switch (value) { + case EEnablePorto::None: return "none"; + case EEnablePorto::Isolate: return "isolate"; + case EEnablePorto::Full: return "full"; + default: YT_ABORT(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoExecutor + : public IPortoExecutor +{ +public: + TPortoExecutor( + TPortoExecutorDynamicConfigPtr config, + const TString& threadNameSuffix, + const NProfiling::TProfiler& profiler) + : Config_(std::move(config)) + , Queue_(New<TActionQueue>(Format("Porto:%v", threadNameSuffix))) + , Profiler_(profiler) + , PollExecutor_(New<TPeriodicExecutor>( + Queue_->GetInvoker(), + BIND(&TPortoExecutor::DoPoll, MakeWeak(this)), + Config_->PollPeriod)) + { + DynamicConfig_.Store(New<TPortoExecutorDynamicConfig>()); + + Api_->SetTimeout(Config_->ApiTimeout.Seconds()); + Api_->SetDiskTimeout(Config_->ApiDiskTimeout.Seconds()); + + PollExecutor_->Start(); + } + + void SubscribeFailed(const TCallback<void (const TError&)>& callback) override + { + Failed_.Subscribe(callback); + } + + void UnsubscribeFailed(const TCallback<void (const TError&)>& callback) override + { + Failed_.Unsubscribe(callback); + } + + void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) override + { + DynamicConfig_.Store(newConfig); + } + +private: + template <class T, class... TArgs1, class... TArgs2> + auto ExecutePortoApiAction( + T(TPortoExecutor::*Method)(TArgs1...), + const TString& command, + TArgs2&&... args) + { + YT_LOG_DEBUG("Enqueue Porto API action (Command: %v)", command); + return BIND(Method, MakeStrong(this), std::forward<TArgs2>(args)...) + .AsyncVia(Queue_->GetInvoker()) + .Run(); + }; + +public: + TFuture<void> CreateContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoCreateContainer, + "CreateContainer", + container); + } + + TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoCreateContainerFromSpec, + "CreateContainerFromSpec", + containerSpec, + start); + } + + TFuture<std::optional<TString>> GetContainerProperty( + const TString& container, + const TString& property) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoGetContainerProperty, + "GetContainerProperty", + container, + property); + } + + TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties( + const TString& container, + const std::vector<TString>& properties) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoGetContainerProperties, + "GetContainerProperty", + container, + properties); + } + + TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties( + const std::vector<TString>& containers, + const std::vector<TString>& properties) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoGetContainerMultipleProperties, + "GetContainerProperty", + containers, + properties); + } + + TFuture<THashMap<TString, i64>> GetContainerMetrics( + const std::vector<TString>& containers, + const TString& metric) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoGetContainerMetrics, + "GetContainerMetrics", + containers, + metric); + } + + TFuture<void> SetContainerProperty( + const TString& container, + const TString& property, + const TString& value) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoSetContainerProperty, + "SetContainerProperty", + container, + property, + value); + } + + TFuture<void> DestroyContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoDestroyContainer, + "DestroyContainer", + container); + } + + TFuture<void> StopContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoStopContainer, + "StopContainer", + container); + } + + TFuture<void> StartContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoStartContainer, + "StartContainer", + container); + } + + TFuture<TString> ConvertPath(const TString& path, const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoConvertPath, + "ConvertPath", + path, + container); + } + + TFuture<void> KillContainer(const TString& container, int signal) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoKillContainer, + "KillContainer", + container, + signal); + } + + TFuture<std::vector<TString>> ListSubcontainers( + const TString& rootContainer, + bool includeRoot) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoListSubcontainers, + "ListSubcontainers", + rootContainer, + includeRoot); + } + + TFuture<int> PollContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoPollContainer, + "PollContainer", + container); + } + + TFuture<int> WaitContainer(const TString& container) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoWaitContainer, + "WaitContainer", + container); + } + + // This method allocates Porto "resources", so it should be uncancellable. + TFuture<TString> CreateVolume( + const TString& path, + const THashMap<TString, TString>& properties) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoCreateVolume, + "CreateVolume", + path, + properties) + .ToUncancelable(); + } + + // This method allocates Porto "resources", so it should be uncancellable. + TFuture<void> LinkVolume( + const TString& path, + const TString& name) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoLinkVolume, + "LinkVolume", + path, + name) + .ToUncancelable(); + } + + // This method deallocates Porto "resources", so it should be uncancellable. + TFuture<void> UnlinkVolume( + const TString& path, + const TString& name) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoUnlinkVolume, + "UnlinkVolume", + path, + name) + .ToUncancelable(); + } + + TFuture<std::vector<TString>> ListVolumePaths() override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoListVolumePaths, + "ListVolumePaths"); + } + + // This method allocates Porto "resources", so it should be uncancellable. + TFuture<void> ImportLayer(const TString& archivePath, const TString& layerId, const TString& place) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoImportLayer, + "ImportLayer", + archivePath, + layerId, + place) + .ToUncancelable(); + } + + // This method deallocates Porto "resources", so it should be uncancellable. + TFuture<void> RemoveLayer(const TString& layerId, const TString& place, bool async) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoRemoveLayer, + "RemoveLayer", + layerId, + place, + async) + .ToUncancelable(); + } + + TFuture<std::vector<TString>> ListLayers(const TString& place) override + { + return ExecutePortoApiAction( + &TPortoExecutor::DoListLayers, + "ListLayers", + place); + } + + IInvokerPtr GetInvoker() const override + { + return Queue_->GetInvoker(); + } + +private: + const TPortoExecutorDynamicConfigPtr Config_; + const TActionQueuePtr Queue_; + const NProfiling::TProfiler Profiler_; + const std::unique_ptr<Porto::TPortoApi> Api_ = std::make_unique<Porto::TPortoApi>(); + const TPeriodicExecutorPtr PollExecutor_; + TAtomicIntrusivePtr<TPortoExecutorDynamicConfig> DynamicConfig_; + + std::vector<TString> Containers_; + THashMap<TString, TPromise<int>> ContainerMap_; + TSingleShotCallbackList<void(const TError&)> Failed_; + + struct TCommandEntry + { + explicit TCommandEntry(const NProfiling::TProfiler& registry) + : TimeGauge(registry.Timer("/command_time")) + , RetryCounter(registry.Counter("/command_retries")) + , SuccessCounter(registry.Counter("/command_successes")) + , FailureCounter(registry.Counter("/command_failures")) + { } + + NProfiling::TEventTimer TimeGauge; + NProfiling::TCounter RetryCounter; + NProfiling::TCounter SuccessCounter; + NProfiling::TCounter FailureCounter; + }; + + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, CommandLock_); + THashMap<TString, TCommandEntry> CommandToEntry_; + + static const std::vector<TString> ContainerRequestVars_; + + bool IsTestPortoFailureEnabled() const + { + auto config = DynamicConfig_.Acquire(); + return config->EnableTestPortoFailures; + } + + bool IsTestPortoTimeout() const + { + auto config = DynamicConfig_.Acquire(); + return config->EnableTestPortoNotResponding; + } + + EPortoErrorCode GetFailedStubError() const + { + auto config = DynamicConfig_.Acquire(); + return config->StubErrorCode; + } + + static TError CreatePortoError(EPortoErrorCode errorCode, const TString& message) + { + return TError(errorCode, "Porto API error") + << TErrorAttribute("original_porto_error_code", static_cast<int>(errorCode) - PortoErrorCodeBase) + << TErrorAttribute("porto_error_message", message); + } + + THashMap<TString, TErrorOr<TString>> DoGetContainerProperties( + const TString& container, + const std::vector<TString>& properties) + { + auto response = DoRequestContainerProperties({container}, properties); + return ParseSinglePortoGetResponse(container, response); + } + + THashMap<TString, THashMap<TString, TErrorOr<TString>>> DoGetContainerMultipleProperties( + const std::vector<TString>& containers, + const std::vector<TString>& properties) + { + auto response = DoRequestContainerProperties(containers, properties); + return ParseMultiplePortoGetResponse(response); + } + + std::optional<TString> DoGetContainerProperty( + const TString& container, + const TString& property) + { + auto response = DoRequestContainerProperties({container}, {property}); + auto parsedResponse = ParseSinglePortoGetResponse(container, response); + auto it = parsedResponse.find(property); + if (it == parsedResponse.end()) { + return std::nullopt; + } else { + return it->second.ValueOrThrow(); + } + } + + void DoCreateContainer(const TString& container) + { + ExecuteApiCall( + [&] { return Api_->Create(container); }, + "Create", + /*idempotent*/ false); + } + + void DoCreateContainerFromSpec(const TRunnableContainerSpec& spec, bool start) + { + Porto::TContainerSpec portoSpec; + + // Required properties. + portoSpec.set_name(spec.Name); + portoSpec.set_command(spec.Command); + + portoSpec.set_enable_porto(FormatEnablePorto(spec.EnablePorto)); + portoSpec.set_isolate(spec.Isolate); + + if (spec.StdinPath) { + portoSpec.set_stdin_path(*spec.StdinPath); + } + if (spec.StdoutPath) { + portoSpec.set_stdout_path(*spec.StdoutPath); + } + if (spec.StderrPath) { + portoSpec.set_stderr_path(*spec.StderrPath); + } + + if (spec.CurrentWorkingDirectory) { + portoSpec.set_cwd(*spec.CurrentWorkingDirectory); + } + + if (spec.CoreCommand) { + portoSpec.set_core_command(*spec.CoreCommand); + } + if (spec.User) { + portoSpec.set_user(*spec.User); + } + + // Useful for jobs, where we operate with numeric group ids. + if (spec.GroupId) { + portoSpec.set_group(ToString(*spec.GroupId)); + } + + if (spec.ThreadLimit) { + portoSpec.set_thread_limit(*spec.ThreadLimit); + } + + if (spec.HostName) { + // To get a reasonable and unique host name inside container. + portoSpec.set_hostname(*spec.HostName); + if (!spec.IPAddresses.empty()) { + const auto& address = spec.IPAddresses[0]; + auto etcHosts = Format("%v %v\n", address, *spec.HostName); + // To be able to resolve hostname into IP inside container. + portoSpec.set_etc_hosts(etcHosts); + } + } + + if (spec.DisableNetwork) { + auto* netConfig = portoSpec.mutable_net()->add_cfg(); + netConfig->set_opt("none"); + } else if (!spec.IPAddresses.empty() && Config_->EnableNetworkIsolation) { + // This label is intended for HBF-agent: YT-12512. + auto* label = portoSpec.mutable_labels()->add_map(); + label->set_key("HBF.ignore_address"); + label->set_val("1"); + + auto* netConfig = portoSpec.mutable_net()->add_cfg(); + netConfig->set_opt("L3"); + netConfig->add_arg("veth0"); + + for (const auto& address : spec.IPAddresses) { + auto* ipConfig = portoSpec.mutable_ip()->add_cfg(); + ipConfig->set_dev("veth0"); + ipConfig->set_ip(ToString(address)); + } + + if (spec.EnableNat64) { + // Behave like nanny does. + portoSpec.set_resolv_conf("nameserver fd64::1;nameserver 2a02:6b8:0:3400::5005;options attempts:1 timeout:1"); + } + } + + for (const auto& [key, value] : spec.Labels) { + auto* map = portoSpec.mutable_labels()->add_map(); + map->set_key(key); + map->set_val(value); + } + + for (const auto& [name, value] : spec.Env) { + auto* var = portoSpec.mutable_env()->add_var(); + var->set_name(name); + var->set_value(value); + } + + for (const auto& controller : spec.CGroupControllers) { + portoSpec.mutable_controllers()->add_controller(controller); + } + + for (const auto& device : spec.Devices) { + auto* portoDevice = portoSpec.mutable_devices()->add_device(); + portoDevice->set_device(device.DeviceName); + portoDevice->set_access(device.Enabled ? "rw" : "-"); + } + + auto addBind = [&] (const TBind& bind) { + auto* portoBind = portoSpec.mutable_bind()->add_bind(); + portoBind->set_target(bind.TargetPath); + portoBind->set_source(bind.SourcePath); + portoBind->add_flag(bind.ReadOnly ? "ro" : "rw"); + }; + + if (spec.RootFS) { + portoSpec.set_root_readonly(spec.RootFS->IsRootReadOnly); + portoSpec.set_root(spec.RootFS->RootPath); + + for (const auto& bind : spec.RootFS->Binds) { + addBind(bind); + } + } + + { + auto* ulimit = portoSpec.mutable_ulimit()->add_ulimit(); + ulimit->set_type("core"); + if (spec.EnableCoreDumps) { + ulimit->set_unlimited(true); + } else { + ulimit->set_hard(0); + ulimit->set_soft(0); + } + } + + // Set some universal defaults. + portoSpec.set_oom_is_fatal(false); + + ExecuteApiCall( + [&] { return Api_->CreateFromSpec(portoSpec, {}, start); }, + "CreateFromSpec", + /*idempotent*/ false); + } + + void DoSetContainerProperty(const TString& container, const TString& property, const TString& value) + { + ExecuteApiCall( + [&] { return Api_->SetProperty(container, property, value); }, + "SetProperty", + /*idempotent*/ true); + } + + void DoDestroyContainer(const TString& container) + { + try { + ExecuteApiCall( + [&] { return Api_->Destroy(container); }, + "Destroy", + /*idempotent*/ true); + } catch (const TErrorException& ex) { + if (!ex.Error().FindMatching(EPortoErrorCode::ContainerDoesNotExist)) { + throw; + } + } + } + + void DoStopContainer(const TString& container) + { + ExecuteApiCall( + [&] { return Api_->Stop(container); }, + "Stop", + /*idempotent*/ true); + } + + void DoStartContainer(const TString& container) + { + ExecuteApiCall( + [&] { return Api_->Start(container); }, + "Start", + /*idempotent*/ false); + } + + TString DoConvertPath(const TString& path, const TString& container) + { + TString result; + ExecuteApiCall( + [&] { return Api_->ConvertPath(path, container, "self", result); }, + "ConvertPath", + /*idempotent*/ true); + return result; + } + + void DoKillContainer(const TString& container, int signal) + { + ExecuteApiCall( + [&] { return Api_->Kill(container, signal); }, + "Kill", + /*idempotent*/ false); + } + + std::vector<TString> DoListSubcontainers(const TString& rootContainer, bool includeRoot) + { + Porto::TListContainersRequest req; + auto filter = req.add_filters(); + filter->set_name(rootContainer + "/*"); + if (includeRoot) { + auto rootFilter = req.add_filters(); + rootFilter->set_name(rootContainer); + } + auto fieldOptions = req.mutable_field_options(); + fieldOptions->add_properties("absolute_name"); + TVector<Porto::TContainer> containers; + ExecuteApiCall( + [&] { return Api_->ListContainersBy(req, containers); }, + "ListContainersBy", + /*idempotent*/ true); + + std::vector<TString> containerNames; + containerNames.reserve(containers.size()); + for (const auto& container : containers) { + const auto& absoluteName = container.status().absolute_name(); + if (!absoluteName.empty()) { + containerNames.push_back(absoluteName); + } + } + return containerNames; + } + + TFuture<int> DoWaitContainer(const TString& container) + { + auto result = NewPromise<int>(); + auto waitCallback = [=, this, this_ = MakeStrong(this)] (const Porto::TWaitResponse& rsp) { + return OnContainerTerminated(rsp, result); + }; + + ExecuteApiCall( + [&] { return Api_->AsyncWait({container}, {}, waitCallback); }, + "AsyncWait", + /*idempotent*/ false); + + return result.ToFuture().ToImmediatelyCancelable(); + } + + void OnContainerTerminated(const Porto::TWaitResponse& portoWaitResponse, TPromise<int> result) + { + const auto& container = portoWaitResponse.name(); + const auto& state = portoWaitResponse.state(); + if (state != "dead" && state != "stopped") { + result.TrySet(TError("Container finished with unexpected state") + << TErrorAttribute("container_name", container) + << TErrorAttribute("container_state", state)); + return; + } + + // TODO(max42): switch to Subscribe. + YT_UNUSED_FUTURE(GetContainerProperty(container, "exit_status").Apply(BIND( + [=] (const TErrorOr<std::optional<TString>>& errorOrExitCode) { + if (!errorOrExitCode.IsOK()) { + result.TrySet(TError("Container finished, but exit status is unknown") + << errorOrExitCode); + return; + } + + const auto& optionalExitCode = errorOrExitCode.Value(); + if (!optionalExitCode) { + result.TrySet(TError("Container finished, but exit status is unknown") + << TErrorAttribute("container_name", container) + << TErrorAttribute("container_state", state)); + return; + } + + try { + int exitStatus = FromString<int>(*optionalExitCode); + result.TrySet(exitStatus); + } catch (const std::exception& ex) { + auto error = TError("Failed to parse Porto exit status") + << TErrorAttribute("container_name", container) + << TErrorAttribute("exit_status", optionalExitCode.value()); + error.MutableInnerErrors()->push_back(TError(ex)); + result.TrySet(error); + } + }))); + } + + TFuture<int> DoPollContainer(const TString& container) + { + auto [it, inserted] = ContainerMap_.insert({container, NewPromise<int>()}); + if (!inserted) { + YT_LOG_WARNING("Container already added for polling (Container: %v)", + container); + } else { + Containers_.push_back(container); + } + return it->second.ToFuture(); + } + + Porto::TGetResponse DoRequestContainerProperties( + const std::vector<TString>& containers, + const std::vector<TString>& vars) + { + TVector<TString> containers_(containers.begin(), containers.end()); + TVector<TString> vars_(vars.begin(), vars.end()); + + const Porto::TGetResponse* getResponse; + + ExecuteApiCall( + [&] { + getResponse = Api_->Get(containers_, vars_); + return getResponse ? EError::Success : EError::Unknown; + }, + "Get", + /*idempotent*/ true); + + YT_VERIFY(getResponse); + return *getResponse; + } + + THashMap<TString, i64> DoGetContainerMetrics( + const std::vector<TString>& containers, + const TString& metric) + { + TVector<TString> containers_(containers.begin(), containers.end()); + + TMap<TString, uint64_t> result; + + ExecuteApiCall( + [&] { return Api_->GetProcMetric(containers_, metric, result); }, + "GetProcMetric", + /*idempotent*/ true); + + return {result.begin(), result.end()}; + } + + void DoPoll() + { + try { + if (Containers_.empty()) { + return; + } + + auto getResponse = DoRequestContainerProperties(Containers_, ContainerRequestVars_); + + if (getResponse.list().empty()) { + return; + } + + auto getProperty = [] ( + const Porto::TGetResponse::TContainerGetListResponse& container, + const TString& name) -> Porto::TGetResponse::TContainerGetValueResponse + { + for (const auto& property : container.keyval()) { + if (property.variable() == name) { + return property; + } + } + + return {}; + }; + + for (const auto& container : getResponse.list()) { + auto state = getProperty(container, "state"); + if (state.error() == EError::ContainerDoesNotExist) { + HandleResult(container.name(), state); + } else if (state.value() == "dead" || state.value() == "stopped") { + HandleResult(container.name(), getProperty(container, "exit_status")); + } + //TODO(dcherednik): other states + } + } catch (const std::exception& ex) { + YT_LOG_ERROR(ex, "Fatal exception occurred while polling Porto"); + Failed_.Fire(TError(ex)); + } + } + + TString DoCreateVolume( + const TString& path, + const THashMap<TString, TString>& properties) + { + auto volume = path; + TMap<TString, TString> propertyMap(properties.begin(), properties.end()); + ExecuteApiCall( + [&] { return Api_->CreateVolume(volume, propertyMap); }, + "CreateVolume", + /*idempotent*/ false); + return volume; + } + + void DoLinkVolume(const TString& path, const TString& container) + { + ExecuteApiCall( + [&] { return Api_->LinkVolume(path, container); }, + "LinkVolume", + /*idempotent*/ false); + } + + void DoUnlinkVolume(const TString& path, const TString& container) + { + ExecuteApiCall( + [&] { return Api_->UnlinkVolume(path, container); }, + "UnlinkVolume", + /*idempotent*/ false); + } + + std::vector<TString> DoListVolumePaths() + { + TVector<TString> volumes; + ExecuteApiCall( + [&] { return Api_->ListVolumes(volumes); }, + "ListVolume", + /*idempotent*/ true); + return {volumes.begin(), volumes.end()}; + } + + void DoImportLayer(const TString& archivePath, const TString& layerId, const TString& place) + { + ExecuteApiCall( + [&] { return Api_->ImportLayer(layerId, archivePath, false, place); }, + "ImportLayer", + /*idempotent*/ false); + } + + void DoRemoveLayer(const TString& layerId, const TString& place, bool async) + { + ExecuteApiCall( + [&] { return Api_->RemoveLayer(layerId, place, async); }, + "RemoveLayer", + /*idempotent*/ false); + } + + std::vector<TString> DoListLayers(const TString& place) + { + TVector<TString> layers; + ExecuteApiCall( + [&] { return Api_->ListLayers(layers, place); }, + "ListLayers", + /*idempotent*/ true); + return {layers.begin(), layers.end()}; + } + + TCommandEntry* GetCommandEntry(const TString& command) + { + auto guard = Guard(CommandLock_); + if (auto it = CommandToEntry_.find(command)) { + return &it->second; + } + return &CommandToEntry_.emplace(command, TCommandEntry(Profiler_.WithTag("command", command))).first->second; + } + + void ExecuteApiCall( + std::function<EError()> callback, + const TString& command, + bool idempotent) + { + YT_LOG_DEBUG("Porto API call started (Command: %v)", command); + + if (IsTestPortoTimeout()) { + YT_LOG_DEBUG("Testing Porto timeout (Command: %v)", command); + + auto config = DynamicConfig_.Acquire(); + TDelayedExecutor::WaitForDuration(config->ApiTimeout); + + THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto timeout"); + } + + if (IsTestPortoFailureEnabled()) { + YT_LOG_DEBUG("Testing Porto failure (Command: %v)", command); + THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto stub error"); + } + + auto* entry = GetCommandEntry(command); + auto startTime = NProfiling::GetInstant(); + while (true) { + EError error; + + { + NProfiling::TWallTimer timer; + error = callback(); + entry->TimeGauge.Record(timer.GetElapsedTime()); + } + + if (error == EError::Success) { + entry->SuccessCounter.Increment(); + break; + } + + entry->FailureCounter.Increment(); + HandleApiError(command, startTime, idempotent); + + YT_LOG_DEBUG("Sleeping and retrying Porto API call (Command: %v)", command); + entry->RetryCounter.Increment(); + + TDelayedExecutor::WaitForDuration(RetryInterval); + } + + YT_LOG_DEBUG("Porto API call completed (Command: %v)", command); + } + + void HandleApiError( + const TString& command, + TInstant startTime, + bool idempotent) + { + TString errorMessage; + auto error = ConvertPortoErrorCode(Api_->GetLastError(errorMessage)); + + // These errors are typical during job cleanup: we might try to kill a container that is already stopped. + bool debug = (error == EPortoErrorCode::ContainerDoesNotExist || error == EPortoErrorCode::InvalidState); + YT_LOG_EVENT( + Logger, + debug ? NLogging::ELogLevel::Debug : NLogging::ELogLevel::Error, + "Porto API call error (Error: %v, Command: %v, Message: %v)", + error, + command, + errorMessage); + + if (!IsRetriableErrorCode(error, idempotent) || NProfiling::GetInstant() - startTime > Config_->RetriesTimeout) { + THROW_ERROR CreatePortoError(error, errorMessage); + } + } + + void HandleResult(const TString& container, const Porto::TGetResponse::TContainerGetValueResponse& rsp) + { + auto portoErrorCode = ConvertPortoErrorCode(rsp.error()); + auto it = ContainerMap_.find(container); + if (it == ContainerMap_.end()) { + YT_LOG_ERROR("Got an unexpected container " + "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)", + container, + portoErrorCode, + rsp.errormsg(), + rsp.value()); + return; + } else { + if (portoErrorCode != EPortoErrorCode::Success) { + YT_LOG_ERROR("Container finished with Porto API error " + "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)", + container, + portoErrorCode, + rsp.errormsg(), + rsp.value()); + it->second.Set(CreatePortoError(portoErrorCode, rsp.errormsg())); + } else { + try { + int exitStatus = std::stoi(rsp.value()); + YT_LOG_DEBUG("Container finished with exit code (Container: %v, ExitCode: %v)", + container, + exitStatus); + + it->second.Set(exitStatus); + } catch (const std::exception& ex) { + it->second.Set(TError("Failed to parse Porto exit status") << ex); + } + } + } + RemoveFromPoller(container); + } + + void RemoveFromPoller(const TString& container) + { + ContainerMap_.erase(container); + + Containers_.clear(); + for (const auto& [name, pid] : ContainerMap_) { + Containers_.push_back(name); + } + } +}; + +const std::vector<TString> TPortoExecutor::ContainerRequestVars_ = { + "state", + "exit_status" +}; + +//////////////////////////////////////////////////////////////////////////////// + +IPortoExecutorPtr CreatePortoExecutor( + TPortoExecutorDynamicConfigPtr config, + const TString& threadNameSuffix, + const NProfiling::TProfiler& profiler) +{ + return New<TPortoExecutor>( + std::move(config), + threadNameSuffix, + profiler); +} + +//////////////////////////////////////////////////////////////////////////////// + +#else + +IPortoExecutorPtr CreatePortoExecutor( + TPortoExecutorDynamicConfigPtr /* config */, + const TString& /* threadNameSuffix */, + const NProfiling::TProfiler& /* profiler */) +{ + THROW_ERROR_EXCEPTION("Porto executor is not available on this platform"); +} + +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_executor.h b/yt/yt/library/containers/porto_executor.h new file mode 100644 index 0000000000..d629ab6275 --- /dev/null +++ b/yt/yt/library/containers/porto_executor.h @@ -0,0 +1,142 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/profiling/sensor.h> + +#include <yt/yt/core/actions/future.h> +#include <yt/yt/core/actions/signal.h> + +#include <yt/yt/core/net/address.h> + +#include <library/cpp/porto/libporto.hpp> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +struct TVolumeId +{ + TString Path; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TRunnableContainerSpec +{ + TString Name; + TString Command; + + EEnablePorto EnablePorto = EEnablePorto::None; + bool Isolate = true; + + std::optional<TString> StdinPath; + std::optional<TString> StdoutPath; + std::optional<TString> StderrPath; + std::optional<TString> CurrentWorkingDirectory; + std::optional<TString> CoreCommand; + std::optional<TString> User; + std::optional<int> GroupId; + + bool EnableCoreDumps = true; + + std::optional<i64> ThreadLimit; + + std::optional<TString> HostName; + std::vector<NYT::NNet::TIP6Address> IPAddresses; + bool EnableNat64 = false; + bool DisableNetwork = false; + + THashMap<TString, TString> Labels; + THashMap<TString, TString> Env; + std::vector<TString> CGroupControllers; + std::vector<TDevice> Devices; + std::optional<TRootFS> RootFS; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct IPortoExecutor + : public TRefCounted +{ + virtual void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) = 0; + + virtual TFuture<void> CreateContainer(const TString& container) = 0; + + virtual TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) = 0; + + virtual TFuture<void> SetContainerProperty( + const TString& container, + const TString& property, + const TString& value) = 0; + + virtual TFuture<std::optional<TString>> GetContainerProperty( + const TString& container, + const TString& property) = 0; + + virtual TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties( + const TString& container, + const std::vector<TString>& properties) = 0; + virtual TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties( + const std::vector<TString>& containers, + const std::vector<TString>& properties) = 0; + + virtual TFuture<THashMap<TString, i64>> GetContainerMetrics( + const std::vector<TString>& containers, + const TString& metric) = 0; + virtual TFuture<void> DestroyContainer(const TString& container) = 0; + virtual TFuture<void> StopContainer(const TString& container) = 0; + virtual TFuture<void> StartContainer(const TString& container) = 0; + virtual TFuture<void> KillContainer(const TString& container, int signal) = 0; + + virtual TFuture<TString> ConvertPath(const TString& path, const TString& container) = 0; + + // Returns absolute names of immediate children only. + virtual TFuture<std::vector<TString>> ListSubcontainers( + const TString& rootContainer, + bool includeRoot) = 0; + // Starts polling a given container, returns future with exit code of finished process. + virtual TFuture<int> PollContainer(const TString& container) = 0; + + // Returns future with exit code of finished process. + // NB: temporarily broken, see https://st.yandex-team.ru/PORTO-846 for details. + virtual TFuture<int> WaitContainer(const TString& container) = 0; + + virtual TFuture<TString> CreateVolume( + const TString& path, + const THashMap<TString, TString>& properties) = 0; + virtual TFuture<void> LinkVolume( + const TString& path, + const TString& name) = 0; + virtual TFuture<void> UnlinkVolume( + const TString& path, + const TString& name) = 0; + virtual TFuture<std::vector<TString>> ListVolumePaths() = 0; + + virtual TFuture<void> ImportLayer( + const TString& archivePath, + const TString& layerId, + const TString& place) = 0; + virtual TFuture<void> RemoveLayer( + const TString& layerId, + const TString& place, + bool async) = 0; + virtual TFuture<std::vector<TString>> ListLayers(const TString& place) = 0; + + virtual IInvokerPtr GetInvoker() const = 0; + + DECLARE_INTERFACE_SIGNAL(void(const TError&), Failed); +}; + +DEFINE_REFCOUNTED_TYPE(IPortoExecutor) + +//////////////////////////////////////////////////////////////////////////////// + +IPortoExecutorPtr CreatePortoExecutor( + TPortoExecutorDynamicConfigPtr config, + const TString& threadNameSuffix, + const NProfiling::TProfiler& profiler = {}); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_health_checker.cpp b/yt/yt/library/containers/porto_health_checker.cpp new file mode 100644 index 0000000000..5a5d358441 --- /dev/null +++ b/yt/yt/library/containers/porto_health_checker.cpp @@ -0,0 +1,69 @@ + +#include "porto_health_checker.h" + +#include "porto_executor.h" +#include "private.h" +#include "config.h" + +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/misc/fs.h> + +#include <util/random/random.h> + +namespace NYT::NContainers { + +using namespace NConcurrency; +using namespace NLogging; +using namespace NProfiling; + +//////////////////////////////////////////////////////////////////////////////// + +TPortoHealthChecker::TPortoHealthChecker( + TPortoExecutorDynamicConfigPtr config, + IInvokerPtr invoker, + TLogger logger) + : Config_(std::move(config)) + , Logger(std::move(logger)) + , CheckInvoker_(std::move(invoker)) + , Executor_(CreatePortoExecutor( + Config_, + "porto_check")) +{ } + +void TPortoHealthChecker::Start() +{ + YT_LOG_DEBUG("Porto health checker started"); + + PeriodicExecutor_ = New<TPeriodicExecutor>( + CheckInvoker_, + BIND(&TPortoHealthChecker::OnCheck, MakeWeak(this)), + Config_->RetriesTimeout); + PeriodicExecutor_->Start(); +} + +void TPortoHealthChecker::OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) +{ + YT_LOG_DEBUG( + "Porto health checker dynamic config changed (EnableTestPortoFailures: %v, StubErrorCode: %v)", + Config_->EnableTestPortoFailures, + Config_->StubErrorCode); + + Executor_->OnDynamicConfigChanged(newConfig); +} + +void TPortoHealthChecker::OnCheck() +{ + YT_LOG_DEBUG("Run Porto health check"); + + auto result = WaitFor(Executor_->ListVolumePaths().AsVoid()); + if (result.IsOK()) { + Success_.Fire(); + } else { + Failed_.Fire(result); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_health_checker.h b/yt/yt/library/containers/porto_health_checker.h new file mode 100644 index 0000000000..f0fb8f0908 --- /dev/null +++ b/yt/yt/library/containers/porto_health_checker.h @@ -0,0 +1,52 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/profiling/sensor.h> + +#include <yt/yt/core/actions/signal.h> + +#include <yt/yt/core/concurrency/periodic_executor.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/error.h> + +#include <atomic> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoHealthChecker + : public TRefCounted +{ +public: + TPortoHealthChecker( + TPortoExecutorDynamicConfigPtr config, + IInvokerPtr invoker, + NLogging::TLogger logger); + + void Start(); + + void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig); + + DEFINE_SIGNAL(void(), Success); + + DEFINE_SIGNAL(void(const TError&), Failed); + +private: + const TPortoExecutorDynamicConfigPtr Config_; + const NLogging::TLogger Logger; + const IInvokerPtr CheckInvoker_; + const IPortoExecutorPtr Executor_; + NConcurrency::TPeriodicExecutorPtr PeriodicExecutor_; + + void OnCheck(); +}; + +DEFINE_REFCOUNTED_TYPE(TPortoHealthChecker) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_resource_tracker.cpp b/yt/yt/library/containers/porto_resource_tracker.cpp new file mode 100644 index 0000000000..c1fe48d6af --- /dev/null +++ b/yt/yt/library/containers/porto_resource_tracker.cpp @@ -0,0 +1,711 @@ +#include "porto_resource_tracker.h" +#include "private.h" + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/error.h> + +#include <yt/yt/core/net/address.h> + +#include <yt/yt/core/ytree/public.h> + +#include <yt/yt/library/process/process.h> + +#include <yt/yt/library/containers/cgroup.h> +#include <yt/yt/library/containers/config.h> +#include <yt/yt/library/containers/instance.h> +#include <yt/yt/library/containers/porto_executor.h> +#include <yt/yt/library/containers/public.h> + +namespace NYT::NContainers { + +using namespace NProfiling; + +static const auto& Logger = ContainersLogger; + +#ifdef _linux_ + +//////////////////////////////////////////////////////////////////////////////// + +struct TPortoProfilers + : public TRefCounted +{ + TPortoResourceProfilerPtr DaemonProfiler; + TPortoResourceProfilerPtr ContainerProfiler; + + TPortoProfilers( + TPortoResourceProfilerPtr daemonProfiler, + TPortoResourceProfilerPtr containerProfiler) + : DaemonProfiler(std::move(daemonProfiler)) + , ContainerProfiler(std::move(containerProfiler)) + { } +}; + +DEFINE_REFCOUNTED_TYPE(TPortoProfilers) + +//////////////////////////////////////////////////////////////////////////////// + +static TErrorOr<ui64> GetFieldOrError( + const TResourceUsage& usage, + EStatField field) +{ + auto it = usage.find(field); + if (it == usage.end()) { + return TError("Resource usage is missing %Qlv field", field); + } + const auto& errorOrValue = it->second; + if (errorOrValue.FindMatching(EPortoErrorCode::NotSupported)) { + return TError("Property %Qlv not supported in Porto response", field); + } + return errorOrValue; +} + +//////////////////////////////////////////////////////////////////////////////// + +TPortoResourceTracker::TPortoResourceTracker( + IInstancePtr instance, + TDuration updatePeriod, + bool isDeltaTracker, + bool isForceUpdate) + : Instance_(std::move(instance)) + , UpdatePeriod_(updatePeriod) + , IsDeltaTracker_(isDeltaTracker) + , IsForceUpdate_(isForceUpdate) +{ + ResourceUsage_ = { + {EStatField::IOReadByte, 0}, + {EStatField::IOWriteByte, 0}, + {EStatField::IOBytesLimit, 0}, + {EStatField::IOReadOps, 0}, + {EStatField::IOWriteOps, 0}, + {EStatField::IOOps, 0}, + {EStatField::IOOpsLimit, 0}, + {EStatField::IOTotalTime, 0}, + {EStatField::IOWaitTime, 0} + }; + ResourceUsageDelta_ = ResourceUsage_; +} + +static TErrorOr<TDuration> ExtractDuration(TErrorOr<ui64> timeNs) +{ + if (timeNs.IsOK()) { + return TErrorOr<TDuration>(TDuration::MicroSeconds(timeNs.Value() / 1000)); + } else { + return TError(timeNs); + } +} + +TCpuStatistics TPortoResourceTracker::ExtractCpuStatistics(const TResourceUsage& resourceUsage) const +{ + // NB: Job proxy uses last sample of CPU statistics but we are interested in + // peak thread count value. + auto currentThreadCountPeak = GetFieldOrError(resourceUsage, EStatField::ThreadCount); + + PeakThreadCount_ = currentThreadCountPeak.IsOK() && PeakThreadCount_.IsOK() + ? std::max<ui64>( + PeakThreadCount_.Value(), + currentThreadCountPeak.Value()) + : currentThreadCountPeak.IsOK() ? currentThreadCountPeak : PeakThreadCount_; + + auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUsage); + auto systemTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuSystemUsage); + auto userTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUserUsage); + auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuWait); + auto throttledNs = GetFieldOrError(resourceUsage, EStatField::CpuThrottled); + auto limitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuLimit); + auto guaranteeTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuGuarantee); + + return TCpuStatistics{ + .TotalUsageTime = ExtractDuration(totalTimeNs), + .UserUsageTime = ExtractDuration(userTimeNs), + .SystemUsageTime = ExtractDuration(systemTimeNs), + .WaitTime = ExtractDuration(waitTimeNs), + .ThrottledTime = ExtractDuration(throttledNs), + .ThreadCount = GetFieldOrError(resourceUsage, EStatField::ThreadCount), + .ContextSwitches = GetFieldOrError(resourceUsage, EStatField::ContextSwitches), + .ContextSwitchesDelta = GetFieldOrError(resourceUsage, EStatField::ContextSwitchesDelta), + .PeakThreadCount = PeakThreadCount_, + .LimitTime = ExtractDuration(limitTimeNs), + .GuaranteeTime = ExtractDuration(guaranteeTimeNs), + }; +} + +TMemoryStatistics TPortoResourceTracker::ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const +{ + return TMemoryStatistics{ + .Rss = GetFieldOrError(resourceUsage, EStatField::Rss), + .MappedFile = GetFieldOrError(resourceUsage, EStatField::MappedFile), + .MinorPageFaults = GetFieldOrError(resourceUsage, EStatField::MinorPageFaults), + .MajorPageFaults = GetFieldOrError(resourceUsage, EStatField::MajorPageFaults), + .FileCacheUsage = GetFieldOrError(resourceUsage, EStatField::FileCacheUsage), + .AnonUsage = GetFieldOrError(resourceUsage, EStatField::AnonMemoryUsage), + .AnonLimit = GetFieldOrError(resourceUsage, EStatField::AnonMemoryLimit), + .MemoryUsage = GetFieldOrError(resourceUsage, EStatField::MemoryUsage), + .MemoryGuarantee = GetFieldOrError(resourceUsage, EStatField::MemoryGuarantee), + .MemoryLimit = GetFieldOrError(resourceUsage, EStatField::MemoryLimit), + .MaxMemoryUsage = GetFieldOrError(resourceUsage, EStatField::MaxMemoryUsage), + .OomKills = GetFieldOrError(resourceUsage, EStatField::OomKills), + .OomKillsTotal = GetFieldOrError(resourceUsage, EStatField::OomKillsTotal) + }; +} + +TBlockIOStatistics TPortoResourceTracker::ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const +{ + auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::IOTotalTime); + auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::IOWaitTime); + + return TBlockIOStatistics{ + .IOReadByte = GetFieldOrError(resourceUsage, EStatField::IOReadByte), + .IOWriteByte = GetFieldOrError(resourceUsage, EStatField::IOWriteByte), + .IOBytesLimit = GetFieldOrError(resourceUsage, EStatField::IOBytesLimit), + .IOReadOps = GetFieldOrError(resourceUsage, EStatField::IOReadOps), + .IOWriteOps = GetFieldOrError(resourceUsage, EStatField::IOWriteOps), + .IOOps = GetFieldOrError(resourceUsage, EStatField::IOOps), + .IOOpsLimit = GetFieldOrError(resourceUsage, EStatField::IOOpsLimit), + .IOTotalTime = ExtractDuration(totalTimeNs), + .IOWaitTime = ExtractDuration(waitTimeNs) + }; +} + +TNetworkStatistics TPortoResourceTracker::ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const +{ + return TNetworkStatistics{ + .TxBytes = GetFieldOrError(resourceUsage, EStatField::NetTxBytes), + .TxPackets = GetFieldOrError(resourceUsage, EStatField::NetTxPackets), + .TxDrops = GetFieldOrError(resourceUsage, EStatField::NetTxDrops), + .TxLimit = GetFieldOrError(resourceUsage, EStatField::NetTxLimit), + + .RxBytes = GetFieldOrError(resourceUsage, EStatField::NetRxBytes), + .RxPackets = GetFieldOrError(resourceUsage, EStatField::NetRxPackets), + .RxDrops = GetFieldOrError(resourceUsage, EStatField::NetRxDrops), + .RxLimit = GetFieldOrError(resourceUsage, EStatField::NetRxLimit), + }; +} + +TTotalStatistics TPortoResourceTracker::ExtractTotalStatistics(const TResourceUsage& resourceUsage) const +{ + return TTotalStatistics{ + .CpuStatistics = ExtractCpuStatistics(resourceUsage), + .MemoryStatistics = ExtractMemoryStatistics(resourceUsage), + .BlockIOStatistics = ExtractBlockIOStatistics(resourceUsage), + .NetworkStatistics = ExtractNetworkStatistics(resourceUsage), + }; +} + +TCpuStatistics TPortoResourceTracker::GetCpuStatistics() const +{ + return GetStatistics( + CachedCpuStatistics_, + "CPU", + [&] (TResourceUsage& resourceUsage) { + return ExtractCpuStatistics(resourceUsage); + }); +} + +TMemoryStatistics TPortoResourceTracker::GetMemoryStatistics() const +{ + return GetStatistics( + CachedMemoryStatistics_, + "memory", + [&] (TResourceUsage& resourceUsage) { + return ExtractMemoryStatistics(resourceUsage); + }); +} + +TBlockIOStatistics TPortoResourceTracker::GetBlockIOStatistics() const +{ + return GetStatistics( + CachedBlockIOStatistics_, + "block IO", + [&] (TResourceUsage& resourceUsage) { + return ExtractBlockIOStatistics(resourceUsage); + }); +} + +TNetworkStatistics TPortoResourceTracker::GetNetworkStatistics() const +{ + return GetStatistics( + CachedNetworkStatistics_, + "network", + [&] (TResourceUsage& resourceUsage) { + return ExtractNetworkStatistics(resourceUsage); + }); +} + +TTotalStatistics TPortoResourceTracker::GetTotalStatistics() const +{ + return GetStatistics( + CachedTotalStatistics_, + "total", + [&] (TResourceUsage& resourceUsage) { + return ExtractTotalStatistics(resourceUsage); + }); +} + +template <class T, class F> +T TPortoResourceTracker::GetStatistics( + std::optional<T>& cachedStatistics, + const TString& statisticsKind, + F extractor) const +{ + UpdateResourceUsageStatisticsIfExpired(); + + auto guard = Guard(SpinLock_); + try { + auto newStatistics = extractor(IsDeltaTracker_ ? ResourceUsageDelta_ : ResourceUsage_); + cachedStatistics = newStatistics; + return newStatistics; + } catch (const std::exception& ex) { + if (!cachedStatistics) { + THROW_ERROR_EXCEPTION("Unable to get %v statistics", statisticsKind) + << ex; + } + YT_LOG_WARNING(ex, "Unable to get %v statistics; using the last one", statisticsKind); + return *cachedStatistics; + } +} + +bool TPortoResourceTracker::AreResourceUsageStatisticsExpired() const +{ + return TInstant::Now() - LastUpdateTime_.load() > UpdatePeriod_; +} + +TInstant TPortoResourceTracker::GetLastUpdateTime() const +{ + return LastUpdateTime_.load(); +} + +void TPortoResourceTracker::UpdateResourceUsageStatisticsIfExpired() const +{ + if (IsForceUpdate_ || AreResourceUsageStatisticsExpired()) { + DoUpdateResourceUsage(); + } +} + +TErrorOr<ui64> TPortoResourceTracker::CalculateCounterDelta( + const TErrorOr<ui64>& oldValue, + const TErrorOr<ui64>& newValue) const +{ + if (oldValue.IsOK() && newValue.IsOK()) { + return newValue.Value() - oldValue.Value(); + } else if (newValue.IsOK()) { + // It is better to return an error than an incorrect value. + return oldValue; + } else { + return newValue; + } +} + +static bool IsCumulativeStatistics(EStatField statistic) +{ + return + statistic == EStatField::CpuUsage || + statistic == EStatField::CpuUserUsage || + statistic == EStatField::CpuSystemUsage || + statistic == EStatField::CpuWait || + statistic == EStatField::CpuThrottled || + + statistic == EStatField::ContextSwitches || + + statistic == EStatField::MinorPageFaults || + statistic == EStatField::MajorPageFaults || + + statistic == EStatField::IOReadByte || + statistic == EStatField::IOWriteByte || + statistic == EStatField::IOReadOps || + statistic == EStatField::IOWriteOps || + statistic == EStatField::IOOps || + statistic == EStatField::IOTotalTime || + statistic == EStatField::IOWaitTime || + + statistic == EStatField::NetTxBytes || + statistic == EStatField::NetTxPackets || + statistic == EStatField::NetTxDrops || + statistic == EStatField::NetRxBytes || + statistic == EStatField::NetRxPackets || + statistic == EStatField::NetRxDrops; +} + +void TPortoResourceTracker::ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const +{ + auto guard = Guard(SpinLock_); + + TResourceUsage resourceUsage; + TResourceUsage resourceUsageDelta; + + for (const auto& stat : InstanceStatFields) { + TErrorOr<ui64> oldValue; + TErrorOr<ui64> newValue; + + if (auto newValueIt = newResourceUsage.find(stat); newValueIt.IsEnd()) { + newValue = TError("Missing property %Qlv in Porto response", stat) + << TErrorAttribute("container", Instance_->GetName()); + } else { + newValue = newValueIt->second; + } + + if (auto oldValueIt = ResourceUsage_.find(stat); oldValueIt.IsEnd()) { + oldValue = newValue; + } else { + oldValue = oldValueIt->second; + } + + if (newValue.IsOK()) { + resourceUsage[stat] = newValue; + } else { + resourceUsage[stat] = oldValue; + } + + if (IsCumulativeStatistics(stat)) { + resourceUsageDelta[stat] = CalculateCounterDelta(oldValue, newValue); + } else { + if (newValue.IsOK()) { + resourceUsageDelta[stat] = newValue; + } else { + resourceUsageDelta[stat] = oldValue; + } + } + } + + ResourceUsage_ = resourceUsage; + ResourceUsageDelta_ = resourceUsageDelta; + LastUpdateTime_.store(TInstant::Now()); +} + +void TPortoResourceTracker::DoUpdateResourceUsage() const +{ + try { + ReCalculateResourceUsage(Instance_->GetResourceUsage()); + } catch (const std::exception& ex) { + YT_LOG_ERROR( + ex, + "Couldn't get metrics from Porto"); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TPortoResourceProfiler::TPortoResourceProfiler( + TPortoResourceTrackerPtr tracker, + TPodSpecConfigPtr podSpec, + const TProfiler& profiler) + : ResourceTracker_(std::move(tracker)) + , PodSpec_(std::move(podSpec)) +{ + profiler.AddProducer("", MakeStrong(this)); +} + +static void WriteGaugeIfOk( + ISensorWriter* writer, + const TString& path, + TErrorOr<ui64> valueOrError) +{ + if (valueOrError.IsOK()) { + i64 value = static_cast<i64>(valueOrError.Value()); + + if (value >= 0) { + writer->AddGauge(path, value); + } + } +} + +static void WriteCumulativeGaugeIfOk( + ISensorWriter* writer, + const TString& path, + TErrorOr<ui64> valueOrError, + i64 timeDeltaUsec) +{ + if (valueOrError.IsOK()) { + i64 value = static_cast<i64>(valueOrError.Value()); + + if (value >= 0) { + writer->AddGauge(path, + 1.0 * value * ResourceUsageUpdatePeriod.MicroSeconds() / timeDeltaUsec); + } + } +} + +void TPortoResourceProfiler::WriteCpuMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec) +{ + { + if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) { + i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds(); + double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs / timeDeltaUsec); + writer->AddGauge("/cpu/user", userUsagePercent); + } + + if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) { + i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds(); + double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs / timeDeltaUsec); + writer->AddGauge("/cpu/system", systemUsagePercent); + } + + if (totalStatistics.CpuStatistics.WaitTime.IsOK()) { + i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds(); + double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec); + writer->AddGauge("/cpu/wait", waitPercent); + } + + if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) { + i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds(); + double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs / timeDeltaUsec); + writer->AddGauge("/cpu/throttled", throttledPercent); + } + + if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) { + i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds(); + double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs / timeDeltaUsec); + writer->AddGauge("/cpu/total", totalUsagePercent); + } + + if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) { + i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds(); + double guaranteePercent = std::max<double>(0.0, (100. * guaranteeTimeUs) / (1'000'000L)); + writer->AddGauge("/cpu/guarantee", guaranteePercent); + } + + if (totalStatistics.CpuStatistics.LimitTime.IsOK()) { + i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds(); + double limitPercent = std::max<double>(0.0, (100. * limitTimeUs) / (1'000'000L)); + writer->AddGauge("/cpu/limit", limitPercent); + } + } + + if (PodSpec_->CpuToVCpuFactor) { + auto factor = *PodSpec_->CpuToVCpuFactor; + + writer->AddGauge("/cpu_to_vcpu_factor", factor); + + if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) { + i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds(); + double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs * factor / timeDeltaUsec); + writer->AddGauge("/vcpu/user", userUsagePercent); + } + + if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) { + i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds(); + double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs * factor / timeDeltaUsec); + writer->AddGauge("/vcpu/system", systemUsagePercent); + } + + if (totalStatistics.CpuStatistics.WaitTime.IsOK()) { + i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds(); + double waitPercent = std::max<double>(0.0, 100. * waitTimeUs * factor / timeDeltaUsec); + writer->AddGauge("/vcpu/wait", waitPercent); + } + + if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) { + i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds(); + double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs * factor / timeDeltaUsec); + writer->AddGauge("/vcpu/throttled", throttledPercent); + } + + if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) { + i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds(); + double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs * factor / timeDeltaUsec); + writer->AddGauge("/vcpu/total", totalUsagePercent); + } + + if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) { + i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds(); + double guaranteePercent = std::max<double>(0.0, 100. * guaranteeTimeUs * factor / 1'000'000L); + writer->AddGauge("/vcpu/guarantee", guaranteePercent); + } + + if (totalStatistics.CpuStatistics.LimitTime.IsOK()) { + i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds(); + double limitPercent = std::max<double>(0.0, 100. * limitTimeUs * factor / 1'000'000L); + writer->AddGauge("/vcpu/limit", limitPercent); + } + } + + WriteGaugeIfOk(writer, "/cpu/thread_count", totalStatistics.CpuStatistics.ThreadCount); + WriteGaugeIfOk(writer, "/cpu/context_switches", totalStatistics.CpuStatistics.ContextSwitches); +} + +void TPortoResourceProfiler::WriteMemoryMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec) +{ + WriteCumulativeGaugeIfOk(writer, + "/memory/minor_page_faults", + totalStatistics.MemoryStatistics.MinorPageFaults, + timeDeltaUsec); + WriteCumulativeGaugeIfOk(writer, + "/memory/major_page_faults", + totalStatistics.MemoryStatistics.MajorPageFaults, + timeDeltaUsec); + + WriteGaugeIfOk(writer, "/memory/oom_kills", totalStatistics.MemoryStatistics.OomKills); + WriteGaugeIfOk(writer, "/memory/oom_kills_total", totalStatistics.MemoryStatistics.OomKillsTotal); + + WriteGaugeIfOk(writer, "/memory/file_cache_usage", totalStatistics.MemoryStatistics.FileCacheUsage); + WriteGaugeIfOk(writer, "/memory/anon_usage", totalStatistics.MemoryStatistics.AnonUsage); + WriteGaugeIfOk(writer, "/memory/anon_limit", totalStatistics.MemoryStatistics.AnonLimit); + WriteGaugeIfOk(writer, "/memory/memory_usage", totalStatistics.MemoryStatistics.MemoryUsage); + WriteGaugeIfOk(writer, "/memory/memory_guarantee", totalStatistics.MemoryStatistics.MemoryGuarantee); + WriteGaugeIfOk(writer, "/memory/memory_limit", totalStatistics.MemoryStatistics.MemoryLimit); +} + +void TPortoResourceProfiler::WriteBlockingIOMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec) +{ + WriteCumulativeGaugeIfOk(writer, + "/io/read_bytes", + totalStatistics.BlockIOStatistics.IOReadByte, + timeDeltaUsec); + WriteCumulativeGaugeIfOk(writer, + "/io/write_bytes", + totalStatistics.BlockIOStatistics.IOWriteByte, + timeDeltaUsec); + WriteCumulativeGaugeIfOk(writer, + "/io/read_ops", + totalStatistics.BlockIOStatistics.IOReadOps, + timeDeltaUsec); + WriteCumulativeGaugeIfOk(writer, + "/io/write_ops", + totalStatistics.BlockIOStatistics.IOWriteOps, + timeDeltaUsec); + WriteCumulativeGaugeIfOk(writer, + "/io/ops", + totalStatistics.BlockIOStatistics.IOOps, + timeDeltaUsec); + + WriteGaugeIfOk(writer, + "/io/bytes_limit", + totalStatistics.BlockIOStatistics.IOBytesLimit); + WriteGaugeIfOk(writer, + "/io/ops_limit", + totalStatistics.BlockIOStatistics.IOOpsLimit); + + if (totalStatistics.BlockIOStatistics.IOTotalTime.IsOK()) { + i64 totalTimeUs = totalStatistics.BlockIOStatistics.IOTotalTime.Value().MicroSeconds(); + double totalPercent = std::max<double>(0.0, 100. * totalTimeUs / timeDeltaUsec); + writer->AddGauge("/io/total", totalPercent); + } + + if (totalStatistics.BlockIOStatistics.IOWaitTime.IsOK()) { + i64 waitTimeUs = totalStatistics.BlockIOStatistics.IOWaitTime.Value().MicroSeconds(); + double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec); + writer->AddGauge("/io/wait", waitPercent); + } +} + +void TPortoResourceProfiler::WriteNetworkMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec) +{ + WriteCumulativeGaugeIfOk( + writer, + "/network/rx_bytes", + totalStatistics.NetworkStatistics.RxBytes, + timeDeltaUsec); + WriteCumulativeGaugeIfOk( + writer, + "/network/rx_drops", + totalStatistics.NetworkStatistics.RxDrops, + timeDeltaUsec); + WriteCumulativeGaugeIfOk( + writer, + "/network/rx_packets", + totalStatistics.NetworkStatistics.RxPackets, + timeDeltaUsec); + WriteGaugeIfOk( + writer, + "/network/rx_limit", + totalStatistics.NetworkStatistics.RxLimit); + + WriteCumulativeGaugeIfOk( + writer, + "/network/tx_bytes", + totalStatistics.NetworkStatistics.TxBytes, + timeDeltaUsec); + WriteCumulativeGaugeIfOk( + writer, + "/network/tx_drops", + totalStatistics.NetworkStatistics.TxDrops, + timeDeltaUsec); + WriteCumulativeGaugeIfOk( + writer, + "/network/tx_packets", + totalStatistics.NetworkStatistics.TxPackets, + timeDeltaUsec); + WriteGaugeIfOk( + writer, + "/network/tx_limit", + totalStatistics.NetworkStatistics.TxLimit); +} + +void TPortoResourceProfiler::CollectSensors(ISensorWriter* writer) +{ + i64 lastUpdate = ResourceTracker_->GetLastUpdateTime().MicroSeconds(); + + auto totalStatistics = ResourceTracker_->GetTotalStatistics(); + i64 timeDeltaUsec = TInstant::Now().MicroSeconds() - lastUpdate; + + WriteCpuMetrics(writer, totalStatistics, timeDeltaUsec); + WriteMemoryMetrics(writer, totalStatistics, timeDeltaUsec); + WriteBlockingIOMetrics(writer, totalStatistics, timeDeltaUsec); + WriteNetworkMetrics(writer, totalStatistics, timeDeltaUsec); +} + +//////////////////////////////////////////////////////////////////////////////// + +TPortoResourceProfilerPtr CreatePortoProfilerWithTags( + const IInstancePtr& instance, + const TString containerCategory, + const TPodSpecConfigPtr& podSpec) +{ + auto portoResourceTracker = New<TPortoResourceTracker>( + instance, + ResourceUsageUpdatePeriod, + true, + true); + + return New<TPortoResourceProfiler>( + portoResourceTracker, + podSpec, + TProfiler("/porto") + .WithTag("container_category", containerCategory)); +} + +//////////////////////////////////////////////////////////////////////////////// + +#endif + +#ifdef __linux__ +void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec) +{ + BIND([=] { + auto executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "porto-tracker"); + + executor->SubscribeFailed(BIND([=] (const TError& error) { + YT_LOG_ERROR(error, "Fatal error during Porto polling"); + })); + + LeakyRefCountedSingleton<TPortoProfilers>( + CreatePortoProfilerWithTags(GetSelfPortoInstance(executor), "daemon", podSpec), + CreatePortoProfilerWithTags(GetRootPortoInstance(executor), "pod", podSpec)); + }).AsyncVia(GetCurrentInvoker()) + .Run() + .Subscribe(BIND([] (const TError& error) { + YT_LOG_ERROR_IF(!error.IsOK(), error, "Failed to enable Porto profiler"); + })); +} +#else +void EnablePortoResourceTracker(const TPodSpecConfigPtr& /*podSpec*/) +{ + YT_LOG_WARNING("Porto resource tracker not supported"); +} +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_resource_tracker.h b/yt/yt/library/containers/porto_resource_tracker.h new file mode 100644 index 0000000000..8a0f781949 --- /dev/null +++ b/yt/yt/library/containers/porto_resource_tracker.h @@ -0,0 +1,158 @@ +#pragma once + +#include <yt/yt/library/containers/instance.h> +#include <yt/yt/library/containers/public.h> + +#include <yt/yt/library/containers/cgroup.h> + +#include <yt/yt/core/misc/singleton.h> +#include <yt/yt/core/net/address.h> +#include <yt/yt/core/ytree/public.h> + +#include <yt/yt/library/process/process.h> +#include <yt/yt/library/profiling/producer.h> + +namespace NYT::NContainers { + +using namespace NProfiling; + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto ResourceUsageUpdatePeriod = TDuration::MilliSeconds(1000); + +//////////////////////////////////////////////////////////////////////////////// + +using TCpuStatistics = TCpuAccounting::TStatistics; +using TBlockIOStatistics = TBlockIO::TStatistics; +using TMemoryStatistics = TMemory::TStatistics; +using TNetworkStatistics = TNetwork::TStatistics; + +struct TTotalStatistics +{ +public: + TCpuStatistics CpuStatistics; + TMemoryStatistics MemoryStatistics; + TBlockIOStatistics BlockIOStatistics; + TNetworkStatistics NetworkStatistics; +}; + +#ifdef _linux_ + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoResourceTracker + : public TRefCounted +{ +public: + TPortoResourceTracker( + IInstancePtr instance, + TDuration updatePeriod, + bool isDeltaTracker = false, + bool isForceUpdate = false); + + TCpuStatistics GetCpuStatistics() const; + + TBlockIOStatistics GetBlockIOStatistics() const; + + TMemoryStatistics GetMemoryStatistics() const; + + TNetworkStatistics GetNetworkStatistics() const; + + TTotalStatistics GetTotalStatistics() const; + + bool AreResourceUsageStatisticsExpired() const; + + TInstant GetLastUpdateTime() const; + +private: + const IInstancePtr Instance_; + const TDuration UpdatePeriod_; + const bool IsDeltaTracker_; + const bool IsForceUpdate_; + + mutable std::atomic<TInstant> LastUpdateTime_ = {}; + + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); + mutable TResourceUsage ResourceUsage_; + mutable TResourceUsage ResourceUsageDelta_; + + mutable std::optional<TCpuStatistics> CachedCpuStatistics_; + mutable std::optional<TMemoryStatistics> CachedMemoryStatistics_; + mutable std::optional<TBlockIOStatistics> CachedBlockIOStatistics_; + mutable std::optional<TNetworkStatistics> CachedNetworkStatistics_; + mutable std::optional<TTotalStatistics> CachedTotalStatistics_; + mutable TErrorOr<ui64> PeakThreadCount_ = 0; + + template <class T, class F> + T GetStatistics( + std::optional<T>& cachedStatistics, + const TString& statisticsKind, + F extractor) const; + + TCpuStatistics ExtractCpuStatistics(const TResourceUsage& resourceUsage) const; + TMemoryStatistics ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const; + TBlockIOStatistics ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const; + TNetworkStatistics ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const; + TTotalStatistics ExtractTotalStatistics(const TResourceUsage& resourceUsage) const; + + TErrorOr<ui64> CalculateCounterDelta( + const TErrorOr<ui64>& oldValue, + const TErrorOr<ui64>& newValue) const; + + void ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const; + + void UpdateResourceUsageStatisticsIfExpired() const; + + void DoUpdateResourceUsage() const; +}; + +DEFINE_REFCOUNTED_TYPE(TPortoResourceTracker) + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoResourceProfiler + : public ISensorProducer +{ +public: + TPortoResourceProfiler( + TPortoResourceTrackerPtr tracker, + TPodSpecConfigPtr podSpec, + const TProfiler& profiler = TProfiler{"/porto"}); + + void CollectSensors(ISensorWriter* writer) override; + +private: + const TPortoResourceTrackerPtr ResourceTracker_; + const TPodSpecConfigPtr PodSpec_; + + void WriteCpuMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec); + + void WriteMemoryMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec); + + void WriteBlockingIOMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec); + + void WriteNetworkMetrics( + ISensorWriter* writer, + TTotalStatistics& totalStatistics, + i64 timeDeltaUsec); +}; + +DECLARE_REFCOUNTED_TYPE(TPortoResourceProfiler) +DEFINE_REFCOUNTED_TYPE(TPortoResourceProfiler) + +//////////////////////////////////////////////////////////////////////////////// + +#endif + +void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec); + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/private.h b/yt/yt/library/containers/private.h new file mode 100644 index 0000000000..62682cb364 --- /dev/null +++ b/yt/yt/library/containers/private.h @@ -0,0 +1,13 @@ +#pragma once + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger ContainersLogger("Containers"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/process.cpp b/yt/yt/library/containers/process.cpp new file mode 100644 index 0000000000..ad1c8d35dc --- /dev/null +++ b/yt/yt/library/containers/process.cpp @@ -0,0 +1,154 @@ +#ifdef __linux__ + +#include "process.h" + +#include <yt/yt/library/containers/instance.h> + +#include <yt/yt/core/misc/proc.h> +#include <yt/yt/core/misc/fs.h> + +namespace NYT::NContainers { + +using namespace NPipes; +using namespace NNet; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +static inline const NLogging::TLogger Logger("Process"); + +static constexpr pid_t InvalidProcessId = -1; + +//////////////////////////////////////////////////////////////////////////////// + +TPortoProcess::TPortoProcess( + const TString& path, + IInstanceLauncherPtr containerLauncher, + bool copyEnv) + : TProcessBase(path) + , ContainerLauncher_(std::move(containerLauncher)) +{ + AddArgument(NFS::GetFileName(path)); + if (copyEnv) { + for (char** envIt = environ; *envIt; ++envIt) { + Env_.push_back(Capture(*envIt)); + } + } +} + +void TPortoProcess::Kill(int signal) +{ + if (auto instance = GetInstance()) { + instance->Kill(signal); + } +} + +void TPortoProcess::DoSpawn() +{ + YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); + YT_VERIFY(!GetInstance()); + YT_VERIFY(!Started_); + YT_VERIFY(!Args_.empty()); + + if (!WorkingDirectory_.empty()) { + ContainerLauncher_->SetCwd(WorkingDirectory_); + } + + Started_ = true; + + try { + // TPortoProcess doesn't support running processes inside rootFS. + YT_VERIFY(!ContainerLauncher_->HasRoot()); + std::vector<TString> args(Args_.begin() + 1, Args_.end()); + auto instance = WaitFor(ContainerLauncher_->Launch(ResolvedPath_, args, DecomposeEnv())) + .ValueOrThrow(); + ContainerInstance_.Store(instance); + FinishedPromise_.SetFrom(instance->Wait()); + + try { + ProcessId_ = instance->GetPid(); + } catch (const std::exception& ex) { + // This could happen if Porto container has already died or pid namespace of + // parent container is not a parent of pid namespace of child container. + // It's not a problem, since for Porto process pid is used for logging purposes only. + YT_LOG_DEBUG(ex, "Failed to get pid of root process (Container: %v)", + instance->GetName()); + } + + YT_LOG_DEBUG("Process inside Porto spawned successfully (Path: %v, ExternalPid: %v, Container: %v)", + ResolvedPath_, + ProcessId_, + instance->GetName()); + + FinishedPromise_.ToFuture().Subscribe(BIND([=, this, this_ = MakeStrong(this)] (const TError& exitStatus) { + Finished_ = true; + if (exitStatus.IsOK()) { + YT_LOG_DEBUG("Process inside Porto exited gracefully (ExternalPid: %v, Container: %v)", + ProcessId_, + instance->GetName()); + } else { + YT_LOG_DEBUG(exitStatus, "Process inside Porto exited with an error (ExternalPid: %v, Container: %v)", + ProcessId_, + instance->GetName()); + } + })); + } catch (const std::exception& ex) { + Finished_ = true; + THROW_ERROR_EXCEPTION("Failed to start child process inside Porto") + << TErrorAttribute("path", ResolvedPath_) + << TErrorAttribute("container", ContainerLauncher_->GetName()) + << ex; + } +} + +IInstancePtr TPortoProcess::GetInstance() +{ + return ContainerInstance_.Acquire(); +} + +THashMap<TString, TString> TPortoProcess::DecomposeEnv() const +{ + THashMap<TString, TString> result; + for (const auto& env : Env_) { + TStringBuf name, value; + TStringBuf(env).TrySplit('=', name, value); + result[name] = value; + } + return result; +} + +static TString CreateStdIONamedPipePath() +{ + const TString name = ToString(TGuid::Create()); + return NFS::GetRealPath(NFS::CombinePaths("/tmp", name)); +} + +IConnectionWriterPtr TPortoProcess::GetStdInWriter() +{ + auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); + ContainerLauncher_->SetStdIn(pipe->GetPath()); + NamedPipes_.push_back(pipe); + return pipe->CreateAsyncWriter(); +} + +IConnectionReaderPtr TPortoProcess::GetStdOutReader() +{ + auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); + ContainerLauncher_->SetStdOut(pipe->GetPath()); + NamedPipes_.push_back(pipe); + return pipe->CreateAsyncReader(); +} + +IConnectionReaderPtr TPortoProcess::GetStdErrReader() +{ + auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); + ContainerLauncher_->SetStdErr(pipe->GetPath()); + NamedPipes_.push_back(pipe); + return pipe->CreateAsyncReader(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers + +#endif diff --git a/yt/yt/library/containers/process.h b/yt/yt/library/containers/process.h new file mode 100644 index 0000000000..75255165d8 --- /dev/null +++ b/yt/yt/library/containers/process.h @@ -0,0 +1,46 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/process/process.h> + +#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> + +#include <library/cpp/porto/libporto.hpp> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +// NB(psushin): this class is deprecated and only used to run job proxy. +// ToDo(psushin): kill me. +class TPortoProcess + : public TProcessBase +{ +public: + TPortoProcess( + const TString& path, + NContainers::IInstanceLauncherPtr containerLauncher, + bool copyEnv = true); + void Kill(int signal) override; + NNet::IConnectionWriterPtr GetStdInWriter() override; + NNet::IConnectionReaderPtr GetStdOutReader() override; + NNet::IConnectionReaderPtr GetStdErrReader() override; + + NContainers::IInstancePtr GetInstance(); + +private: + const NContainers::IInstanceLauncherPtr ContainerLauncher_; + + TAtomicIntrusivePtr<NContainers::IInstance> ContainerInstance_; + std::vector<NPipes::TNamedPipePtr> NamedPipes_; + + void DoSpawn() override; + THashMap<TString, TString> DecomposeEnv() const; +}; + +DEFINE_REFCOUNTED_TYPE(TPortoProcess) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/public.h b/yt/yt/library/containers/public.h new file mode 100644 index 0000000000..d8e3cf3491 --- /dev/null +++ b/yt/yt/library/containers/public.h @@ -0,0 +1,163 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +#include <library/cpp/porto/proto/rpc.pb.h> +#include <library/cpp/yt/misc/enum.h> + +namespace NYT::NContainers { + +//////////////////////////////////////////////////////////////////////////////// + +const int PortoErrorCodeBase = 12000; + +DEFINE_ENUM(EPortoErrorCode, + ((Success) ((PortoErrorCodeBase + Porto::EError::Success))) + ((Unknown) ((PortoErrorCodeBase + Porto::EError::Unknown))) + ((InvalidMethod) ((PortoErrorCodeBase + Porto::EError::InvalidMethod))) + ((ContainerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::ContainerAlreadyExists))) + ((ContainerDoesNotExist) ((PortoErrorCodeBase + Porto::EError::ContainerDoesNotExist))) + ((InvalidProperty) ((PortoErrorCodeBase + Porto::EError::InvalidProperty))) + ((InvalidData) ((PortoErrorCodeBase + Porto::EError::InvalidData))) + ((InvalidValue) ((PortoErrorCodeBase + Porto::EError::InvalidValue))) + ((InvalidState) ((PortoErrorCodeBase + Porto::EError::InvalidState))) + ((NotSupported) ((PortoErrorCodeBase + Porto::EError::NotSupported))) + ((ResourceNotAvailable) ((PortoErrorCodeBase + Porto::EError::ResourceNotAvailable))) + ((Permission) ((PortoErrorCodeBase + Porto::EError::Permission))) + ((VolumeAlreadyExists) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyExists))) + ((VolumeNotFound) ((PortoErrorCodeBase + Porto::EError::VolumeNotFound))) + ((NoSpace) ((PortoErrorCodeBase + Porto::EError::NoSpace))) + ((Busy) ((PortoErrorCodeBase + Porto::EError::Busy))) + ((VolumeAlreadyLinked) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyLinked))) + ((VolumeNotLinked) ((PortoErrorCodeBase + Porto::EError::VolumeNotLinked))) + ((LayerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::LayerAlreadyExists))) + ((LayerNotFound) ((PortoErrorCodeBase + Porto::EError::LayerNotFound))) + ((NoValue) ((PortoErrorCodeBase + Porto::EError::NoValue))) + ((VolumeNotReady) ((PortoErrorCodeBase + Porto::EError::VolumeNotReady))) + ((InvalidCommand) ((PortoErrorCodeBase + Porto::EError::InvalidCommand))) + ((LostError) ((PortoErrorCodeBase + Porto::EError::LostError))) + ((DeviceNotFound) ((PortoErrorCodeBase + Porto::EError::DeviceNotFound))) + ((InvalidPath) ((PortoErrorCodeBase + Porto::EError::InvalidPath))) + ((InvalidNetworkAddress) ((PortoErrorCodeBase + Porto::EError::InvalidNetworkAddress))) + ((PortoFrozen) ((PortoErrorCodeBase + Porto::EError::PortoFrozen))) + ((LabelNotFound) ((PortoErrorCodeBase + Porto::EError::LabelNotFound))) + ((InvalidLabel) ((PortoErrorCodeBase + Porto::EError::InvalidLabel))) + ((NotFound) ((PortoErrorCodeBase + Porto::EError::NotFound))) + ((SocketError) ((PortoErrorCodeBase + Porto::EError::SocketError))) + ((SocketUnavailable) ((PortoErrorCodeBase + Porto::EError::SocketUnavailable))) + ((SocketTimeout) ((PortoErrorCodeBase + Porto::EError::SocketTimeout))) + ((Taint) ((PortoErrorCodeBase + Porto::EError::Taint))) + ((Queued) ((PortoErrorCodeBase + Porto::EError::Queued))) +); + +//////////////////////////////////////////////////////////////////////////////// + +YT_DEFINE_ERROR_ENUM( + ((FailedToStartContainer) (14000)) +); + +DEFINE_ENUM(EStatField, + // CPU + (CpuUsage) + (CpuUserUsage) + (CpuSystemUsage) + (CpuWait) + (CpuThrottled) + (ContextSwitches) + (ContextSwitchesDelta) + (ThreadCount) + (CpuLimit) + (CpuGuarantee) + + // Memory + (Rss) + (MappedFile) + (MajorPageFaults) + (MinorPageFaults) + (FileCacheUsage) + (AnonMemoryUsage) + (AnonMemoryLimit) + (MemoryUsage) + (MemoryGuarantee) + (MemoryLimit) + (MaxMemoryUsage) + (OomKills) + (OomKillsTotal) + + // IO + (IOReadByte) + (IOWriteByte) + (IOBytesLimit) + (IOReadOps) + (IOWriteOps) + (IOOps) + (IOOpsLimit) + (IOTotalTime) + (IOWaitTime) + + // Network + (NetTxBytes) + (NetTxPackets) + (NetTxDrops) + (NetTxLimit) + (NetRxBytes) + (NetRxPackets) + (NetRxDrops) + (NetRxLimit) +); + +DEFINE_ENUM(EEnablePorto, + (None) + (Isolate) + (Full) +); + +struct TBind +{ + TString SourcePath; + TString TargetPath; + bool ReadOnly; +}; + +struct TRootFS +{ + TString RootPath; + bool IsRootReadOnly; + std::vector<TBind> Binds; +}; + +struct TDevice +{ + TString DeviceName; + bool Enabled; +}; + +struct TInstanceLimits +{ + double Cpu = 0; + i64 Memory = 0; + std::optional<i64> NetTx; + std::optional<i64> NetRx; + + bool operator==(const TInstanceLimits&) const = default; +}; + +DECLARE_REFCOUNTED_STRUCT(IContainerManager) +DECLARE_REFCOUNTED_STRUCT(IInstanceLauncher) +DECLARE_REFCOUNTED_STRUCT(IInstance) +DECLARE_REFCOUNTED_STRUCT(IPortoExecutor) + +DECLARE_REFCOUNTED_CLASS(TPortoHealthChecker) +DECLARE_REFCOUNTED_CLASS(TInstanceLimitsTracker) +DECLARE_REFCOUNTED_CLASS(TPortoProcess) +DECLARE_REFCOUNTED_CLASS(TPortoResourceTracker) +DECLARE_REFCOUNTED_CLASS(TPortoExecutorDynamicConfig) +DECLARE_REFCOUNTED_CLASS(TPodSpecConfig) + +//////////////////////////////////////////////////////////////////////////////// + +bool IsValidCGroupType(const TString& type); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/unittests/containers_ut.cpp b/yt/yt/library/containers/unittests/containers_ut.cpp new file mode 100644 index 0000000000..4f1c10a435 --- /dev/null +++ b/yt/yt/library/containers/unittests/containers_ut.cpp @@ -0,0 +1,133 @@ +#include <yt/yt/core/test_framework/framework.h> + +#ifdef _linux_ + +#include <yt/yt/library/containers/config.h> +#include <yt/yt/library/containers/porto_executor.h> +#include <yt/yt/library/containers/instance.h> + +#include <util/system/platform.h> +#include <util/system/env.h> + +namespace NYT::NContainers { +namespace { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +class TContainersTest + : public ::testing::Test +{ + void SetUp() override + { + if (GetEnv("SKIP_PORTO_TESTS") != "") { + GTEST_SKIP(); + } + } +}; + +static TString GetUniqueName() +{ + return "yt_ut_" + ToString(TGuid::Create()); +} + +IPortoExecutorPtr CreatePortoExecutor() +{ + return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); +} + +TEST_F(TContainersTest, ListSubcontainers) +{ + auto executor = CreatePortoExecutor(); + auto name = GetUniqueName(); + + WaitFor(executor->CreateContainer(name)) + .ThrowOnError(); + + auto absoluteName = *WaitFor(executor->GetContainerProperty(name, "absolute_name")) + .ValueOrThrow(); + + auto nestedName = absoluteName + "/nested"; + WaitFor(executor->CreateContainer(nestedName)) + .ThrowOnError(); + + auto withRoot = WaitFor(executor->ListSubcontainers(name, true)) + .ValueOrThrow(); + EXPECT_EQ(std::vector<TString>({absoluteName, nestedName}), withRoot); + + auto withoutRoot = WaitFor(executor->ListSubcontainers(name, false)) + .ValueOrThrow(); + EXPECT_EQ(std::vector<TString>({nestedName}), withoutRoot); + + WaitFor(executor->DestroyContainer(absoluteName)) + .ThrowOnError(); +} + +// See https://st.yandex-team.ru/PORTO-846. +TEST_F(TContainersTest, DISABLED_WaitContainer) +{ + auto executor = CreatePortoExecutor(); + auto name = GetUniqueName(); + + WaitFor(executor->CreateContainer(name)) + .ThrowOnError(); + + WaitFor(executor->SetContainerProperty(name, "command", "sleep 10")) + .ThrowOnError(); + + WaitFor(executor->StartContainer(name)) + .ThrowOnError(); + + auto exitCode = WaitFor(executor->WaitContainer(name)) + .ValueOrThrow(); + + EXPECT_EQ(0, exitCode); + + WaitFor(executor->DestroyContainer(name)) + .ThrowOnError(); +} + +TEST_F(TContainersTest, CreateFromSpec) +{ + auto executor = CreatePortoExecutor(); + auto name = GetUniqueName(); + + auto spec = TRunnableContainerSpec { + .Name = name, + .Command = "sleep 2", + }; + + WaitFor(executor->CreateContainer(spec, /*start*/ true)) + .ThrowOnError(); + + auto exitCode = WaitFor(executor->PollContainer(name)) + .ValueOrThrow(); + + EXPECT_EQ(0, exitCode); + + WaitFor(executor->DestroyContainer(name)) + .ThrowOnError(); +} + +TEST_F(TContainersTest, ListPids) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + + auto instance = WaitFor(launcher->Launch("sleep", {"5"}, {})) + .ValueOrThrow(); + + auto pids = instance->GetPids(); + EXPECT_LT(0u, pids.size()); + + instance->Destroy(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NContainers + +#endif diff --git a/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp b/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp new file mode 100644 index 0000000000..04d169ba4e --- /dev/null +++ b/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp @@ -0,0 +1,251 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/ytree/convert.h> + +#include <util/system/fs.h> +#include <util/system/tempfile.h> + +#include <yt/yt/library/profiling/producer.h> +#include <yt/yt/library/containers/config.h> +#include <yt/yt/library/containers/porto_executor.h> +#include <yt/yt/library/containers/porto_resource_tracker.h> +#include <yt/yt/library/containers/instance.h> + +#include <util/system/platform.h> +#include <util/system/env.h> + +namespace NYT::NContainers { +namespace { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto TestUpdatePeriod = TDuration::MilliSeconds(10); + +class TPortoTrackerTest + : public ::testing::Test +{ +public: + IPortoExecutorPtr Executor; + + void SetUp() override + { + if (GetEnv("SKIP_PORTO_TESTS") != "") { + GTEST_SKIP(); + } + + Executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); + } +}; + +TString GetUniqueName() +{ + return "yt_porto_ut_" + ToString(TGuid::Create()); +} + +TPortoResourceTrackerPtr CreateSumPortoTracker(IPortoExecutorPtr Executor, const TString& name) +{ + return New<TPortoResourceTracker>( + GetPortoInstance(Executor, name), + TestUpdatePeriod, + false); +} + +TPortoResourceProfilerPtr CreateDeltaPortoProfiler(IPortoExecutorPtr executor, const TString& name) +{ + auto instance = GetPortoInstance(executor, name); + auto portoResourceTracker = New<TPortoResourceTracker>( + instance, + ResourceUsageUpdatePeriod, + true, + true + ); + + // Init metrics for delta tracker. + portoResourceTracker->GetTotalStatistics(); + + return LeakyRefCountedSingleton<TPortoResourceProfiler>( + portoResourceTracker, + New<TPodSpecConfig>(), + TProfiler("/porto") + .WithTag("porto_name", instance->GetName()) + .WithTag("container_category", "yt_daemon")); +} + +void AssertGauges(const std::vector<std::tuple<TString, TTagList, double>>& gauges) { + THashSet<TString> sensors{ + "/cpu/user", + "/cpu/total", + "/cpu/system", + "/cpu/wait", + "/cpu/throttled", + "/cpu/guarantee", + "/cpu/limit", + "/cpu/thread_count", + "/cpu/context_switches", + + "/memory/minor_page_faults", + "/memory/major_page_faults", + "/memory/file_cache_usage", + "/memory/anon_usage", + "/memory/anon_limit", + "/memory/memory_usage", + "/memory/memory_guarantee", + "/memory/memory_limit", + + "/io/read_bytes", + "/io/write_bytes", + "/io/bytes_limit", + + "/io/read_ops", + "/io/write_ops", + "/io/ops", + "/io/ops_limit", + "/io/total", + + "/network/rx_bytes", + "/network/rx_drops", + "/network/rx_packets", + "/network/rx_limit", + "/network/tx_bytes", + "/network/tx_drops", + "/network/tx_packets", + "/network/tx_limit" + }; + + THashSet<TString> mayBeEmpty{ + "/cpu/wait", + "/cpu/throttled", + "/cpu/guarantee", + "/cpu/context_switches", + "/memory/major_page_faults", + "/memory/memory_guarantee", + "/io/ops_limit", + "/io/read_ops", + "/io/write_ops", + "/io/wait", + "/io/bytes_limit", + "/network/rx_bytes", + "/network/rx_drops", + "/network/rx_packets", + "/network/rx_limit", + "/network/tx_bytes", + "/network/tx_drops", + "/network/tx_packets", + "/network/tx_limit" + }; + + for (const auto& [name, tags, value] : gauges) { + EXPECT_TRUE(value >= 0 && sensors.find(name) || mayBeEmpty.find(name)); + } +} + +TEST_F(TPortoTrackerTest, ValidateSummaryPortoTracker) +{ + auto name = GetUniqueName(); + + WaitFor(Executor->CreateContainer( + TRunnableContainerSpec { + .Name = name, + .Command = "sleep .1", + }, true)) + .ThrowOnError(); + + auto tracker = CreateSumPortoTracker(Executor, name); + + auto firstStatistics = tracker->GetTotalStatistics(); + + WaitFor(Executor->StopContainer(name)) + .ThrowOnError(); + WaitFor(Executor->SetContainerProperty( + name, + "command", + "find /")) + .ThrowOnError(); + WaitFor(Executor->StartContainer(name)) + .ThrowOnError(); + Sleep(TDuration::MilliSeconds(500)); + + auto secondStatistics = tracker->GetTotalStatistics(); + + WaitFor(Executor->DestroyContainer(name)) + .ThrowOnError(); +} + +TEST_F(TPortoTrackerTest, ValidateDeltaPortoTracker) +{ + auto name = GetUniqueName(); + + auto spec = TRunnableContainerSpec { + .Name = name, + .Command = "sleep .1", + }; + + WaitFor(Executor->CreateContainer(spec, true)) + .ThrowOnError(); + + auto profiler = CreateDeltaPortoProfiler(Executor, name); + + WaitFor(Executor->StopContainer(name)) + .ThrowOnError(); + WaitFor(Executor->SetContainerProperty( + name, + "command", + "find /")) + .ThrowOnError(); + WaitFor(Executor->StartContainer(name)) + .ThrowOnError(); + + Sleep(TDuration::MilliSeconds(500)); + + auto buffer = New<TSensorBuffer>(); + profiler->CollectSensors(buffer.Get()); + AssertGauges(buffer->GetGauges()); + + WaitFor(Executor->DestroyContainer(name)) + .ThrowOnError(); +} + +TEST_F(TPortoTrackerTest, ValidateDeltaRootPortoTracker) +{ + auto name = GetUniqueName(); + + auto spec = TRunnableContainerSpec { + .Name = name, + .Command = "sleep .1", + }; + + WaitFor(Executor->CreateContainer(spec, true)) + .ThrowOnError(); + + auto profiler = CreateDeltaPortoProfiler( + Executor, + GetPortoInstance( + Executor, + *GetPortoInstance(Executor, name)->GetRootName())->GetName()); + + WaitFor(Executor->StopContainer(name)) + .ThrowOnError(); + WaitFor(Executor->SetContainerProperty( + name, + "command", + "find /")) + .ThrowOnError(); + WaitFor(Executor->StartContainer(name)) + .ThrowOnError(); + + Sleep(TDuration::MilliSeconds(500)); + + auto buffer = New<TSensorBuffer>(); + profiler->CollectSensors(buffer.Get()); + AssertGauges(buffer->GetGauges()); + + WaitFor(Executor->DestroyContainer(name)) + .ThrowOnError(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/unittests/process_ut.cpp b/yt/yt/library/containers/unittests/process_ut.cpp new file mode 100644 index 0000000000..b9c0d844f4 --- /dev/null +++ b/yt/yt/library/containers/unittests/process_ut.cpp @@ -0,0 +1,302 @@ +#include <yt/yt/core/test_framework/framework.h> + +#ifdef _linux_ + +#include <yt/yt/core/actions/bind.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/delayed_executor.h> +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/misc/guid.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/net/connection.h> + +#include <yt/yt/library/containers/process.h> + +#include <yt/yt/library/containers/config.h> +#include <yt/yt/library/containers/porto_executor.h> +#include <yt/yt/library/containers/instance.h> + +#include <util/system/platform.h> +#include <util/system/env.h> + +namespace NYT::NContainers { +namespace { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +class TPortoProcessTest + : public ::testing::Test +{ + void SetUp() override + { + if (GetEnv("SKIP_PORTO_TESTS") != "") { + GTEST_SKIP(); + } + } +}; + +static TString GetUniqueName() +{ + return "yt_ut_" + ToString(TGuid::Create()); +} + +IPortoExecutorPtr CreatePortoExecutor() +{ + return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); +} + +TEST_F(TPortoProcessTest, Basic) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/ls", launcher, true); + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, RunFromPathEnv) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("ls", launcher, true); + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, MultiBasic) +{ + auto portoExecutor = CreatePortoExecutor(); + auto l1 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor); + auto l2 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor); + auto p1 = New<TPortoProcess>("/bin/ls", l1, true); + auto p2 = New<TPortoProcess>("/bin/ls", l2, true); + TFuture<void> f1; + TFuture<void> f2; + ASSERT_NO_THROW(f1 = p1->Spawn()); + ASSERT_NO_THROW(f2 = p2->Spawn()); + auto error = WaitFor((AllSucceeded(std::vector<TFuture<void>>{f1, f2}))); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p1->IsFinished()); + EXPECT_TRUE(p2->IsFinished()); + p1->GetInstance()->Destroy(); + p2->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, InvalidPath) +{ + auto portoExecutor = CreatePortoExecutor(); + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + portoExecutor); + auto p = New<TPortoProcess>("/some/bad/path/binary", launcher, true); + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_FALSE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_FALSE(p->IsFinished()); + EXPECT_FALSE(error.IsOK()); + WaitFor(portoExecutor->DestroyContainer(launcher->GetName())) + .ThrowOnError(); +} + +TEST_F(TPortoProcessTest, StdOut) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/date", launcher, true); + + auto outStream = p->GetStdOutReader(); + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + + auto buffer = TSharedMutableRef::Allocate(4_KB, {.InitializeStorage = false}); + auto future = outStream->Read(buffer); + TErrorOr<size_t> result = WaitFor(future); + size_t sz = result.ValueOrThrow(); + EXPECT_TRUE(sz > 0); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, GetCommandLine) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + EXPECT_EQ("/bin/bash", p->GetCommandLine()); + p->AddArgument("-c"); + EXPECT_EQ("/bin/bash -c", p->GetCommandLine()); + p->AddArgument("exit 0"); + EXPECT_EQ("/bin/bash -c \"exit 0\"", p->GetCommandLine()); +} + +TEST_F(TPortoProcessTest, ProcessReturnCode0) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("exit 0"); + + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, ProcessReturnCode123) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("exit 123"); + + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_EQ(EProcessErrorCode::NonZeroExitCode, error.GetCode()); + EXPECT_EQ(123, error.Attributes().Get<int>("exit_code")); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, Params1) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("if test 3 -gt 1; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_FALSE(error.IsOK()); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, Params2) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("if test 1 -gt 3; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, InheritEnvironment) +{ + const char* name = "SPAWN_TEST_ENV_VAR"; + const char* value = "42"; + setenv(name, value, 1); + + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("if test $SPAWN_TEST_ENV_VAR = 42; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_FALSE(error.IsOK()); + EXPECT_TRUE(p->IsFinished()); + + unsetenv(name); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, Kill) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/sleep", launcher, true); + p->AddArgument("5"); + + auto finished = p->Spawn(); + + NConcurrency::TDelayedExecutor::Submit( + BIND([&] () { + p->Kill(SIGKILL); + }), + TDuration::MilliSeconds(100)); + + auto error = WaitFor(finished); + EXPECT_FALSE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, KillFinished) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/bash", launcher, true); + p->AddArgument("-c"); + p->AddArgument("true"); + + auto finished = p->Spawn(); + + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()); + + p->Kill(SIGKILL); + p->GetInstance()->Destroy(); +} + +TEST_F(TPortoProcessTest, PollDuration) +{ + auto launcher = CreatePortoInstanceLauncher( + GetUniqueName(), + CreatePortoExecutor()); + auto p = New<TPortoProcess>("/bin/sleep", launcher, true); + p->AddArgument("1"); + + auto error = WaitFor(p->Spawn()); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + p->GetInstance()->Destroy(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NContainers + +#endif diff --git a/yt/yt/library/containers/unittests/ya.make b/yt/yt/library/containers/unittests/ya.make new file mode 100644 index 0000000000..42984e2dc7 --- /dev/null +++ b/yt/yt/library/containers/unittests/ya.make @@ -0,0 +1,35 @@ +GTEST(unittester-containers) + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +ALLOCATOR(TCMALLOC) + +IF (AUTOCHECK) + ENV(SKIP_PORTO_TESTS=1) +ENDIF() + +IF (DISTBUILD) # TODO(prime@): this is always on + ENV(SKIP_PORTO_TESTS=1) +ENDIF() + +SRCS( + containers_ut.cpp + process_ut.cpp +) + +IF(OS_LINUX) + SRCS( + porto_resource_tracker_ut.cpp + ) +ENDIF() + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + yt/yt/build + yt/yt/library/containers +) + +SIZE(MEDIUM) + +END() diff --git a/yt/yt/library/containers/ya.make b/yt/yt/library/containers/ya.make new file mode 100644 index 0000000000..499b8d9da8 --- /dev/null +++ b/yt/yt/library/containers/ya.make @@ -0,0 +1,37 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + cgroup.cpp + config.cpp + instance.cpp + instance_limits_tracker.cpp + process.cpp + porto_executor.cpp + porto_resource_tracker.cpp + porto_health_checker.cpp +) + +PEERDIR( + library/cpp/porto/proto + yt/yt/library/process + yt/yt/core +) + +IF(OS_LINUX) + PEERDIR( + library/cpp/porto + ) +ENDIF() + +END() + +RECURSE( + disk_manager + cri +) + +RECURSE_FOR_TESTS( + unittests +) diff --git a/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..cbddba12a3 --- /dev/null +++ b/yt/yt/library/monitoring/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-monitoring) +target_compile_options(yt-library-monitoring PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-monitoring PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-yt-build + yt-library-profiling + library-profiling-solomon + library-cpp-cgiparam +) +target_sources(yt-library-monitoring PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp +) diff --git a/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt b/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..192d29cfb1 --- /dev/null +++ b/yt/yt/library/monitoring/CMakeLists.linux-aarch64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-monitoring) +target_compile_options(yt-library-monitoring PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-monitoring PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-yt-build + yt-library-profiling + library-profiling-solomon + library-cpp-cgiparam + yt-library-ytprof + library-ytprof-http + library-backtrace_introspector-http +) +target_sources(yt-library-monitoring PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp +) diff --git a/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..192d29cfb1 --- /dev/null +++ b/yt/yt/library/monitoring/CMakeLists.linux-x86_64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-monitoring) +target_compile_options(yt-library-monitoring PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-monitoring PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-yt-build + yt-library-profiling + library-profiling-solomon + library-cpp-cgiparam + yt-library-ytprof + library-ytprof-http + library-backtrace_introspector-http +) +target_sources(yt-library-monitoring PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp +) diff --git a/yt/yt/library/monitoring/CMakeLists.txt b/yt/yt/library/monitoring/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/monitoring/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt b/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..adebf50e70 --- /dev/null +++ b/yt/yt/library/monitoring/CMakeLists.windows-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-monitoring) +target_link_libraries(yt-library-monitoring PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + yt-yt-build + yt-library-profiling + library-profiling-solomon + library-cpp-cgiparam +) +target_sources(yt-library-monitoring PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/http_integration.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/monitoring/monitoring_manager.cpp +) diff --git a/yt/yt/library/monitoring/http_integration.cpp b/yt/yt/library/monitoring/http_integration.cpp new file mode 100644 index 0000000000..a526d2ede6 --- /dev/null +++ b/yt/yt/library/monitoring/http_integration.cpp @@ -0,0 +1,203 @@ +#include "http_integration.h" + +#include "monitoring_manager.h" + +#include <yt/yt/build/build.h> + +#include <yt/yt/core/json/config.h> +#include <yt/yt/core/json/json_writer.h> + +#include <yt/yt/core/ytree/fluent.h> + +#include <yt/yt/core/yson/parser.h> +#include <yt/yt/core/yson/consumer.h> + +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/ytree/helpers.h> +#include <yt/yt/core/ytree/virtual.h> +#include <yt/yt/core/ytree/ypath_detail.h> +#include <yt/yt/core/ytree/ypath_proxy.h> + +#include <yt/yt/core/http/http.h> +#include <yt/yt/core/http/helpers.h> +#include <yt/yt/core/http/server.h> + +#include <yt/yt/core/misc/ref_counted_tracker_statistics_producer.h> + +#include <yt/yt/library/profiling/solomon/exporter.h> + +#ifdef _linux_ +#include <yt/yt/library/ytprof/http/handler.h> +#include <yt/yt/library/ytprof/build_info.h> + +#include <yt/yt/library/backtrace_introspector/http/handler.h> +#endif + +#include <library/cpp/cgiparam/cgiparam.h> + +#include <util/string/vector.h> + +namespace NYT::NMonitoring { + +using namespace NYTree; +using namespace NYson; +using namespace NHttp; +using namespace NConcurrency; +using namespace NJson; + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(EVerb, + (Get) + (List) +); + +//////////////////////////////////////////////////////////////////////////////// + +void Initialize( + const NHttp::IServerPtr& monitoringServer, + const NProfiling::TSolomonExporterConfigPtr& config, + TMonitoringManagerPtr* monitoringManager, + NYTree::IMapNodePtr* orchidRoot) +{ + *monitoringManager = New<TMonitoringManager>(); + (*monitoringManager)->Register("/ref_counted", CreateRefCountedTrackerStatisticsProducer()); + (*monitoringManager)->Register("/solomon", BIND([] (NYson::IYsonConsumer* consumer) { + auto tags = NProfiling::TSolomonRegistry::Get()->GetDynamicTags(); + + BuildYsonFluently(consumer) + .BeginMap() + .Item("dynamic_tags").Value(THashMap<TString, TString>(tags.begin(), tags.end())) + .EndMap(); + })); + (*monitoringManager)->Start(); + + *orchidRoot = NYTree::GetEphemeralNodeFactory(true)->CreateMap(); + SetNodeByYPath( + *orchidRoot, + "/monitoring", + CreateVirtualNode((*monitoringManager)->GetService())); + +#ifdef _linux_ + auto buildInfo = NYTProf::TBuildInfo::GetDefault(); + buildInfo.BinaryVersion = GetVersion(); + + SetNodeByYPath( + *orchidRoot, + "/build_info", + NYTree::BuildYsonNodeFluently() + .BeginMap() + .Item("arc_revision").Value(buildInfo.ArcRevision) + .Item("binary_version").Value(buildInfo.BinaryVersion) + .Item("build_type").Value(buildInfo.BuildType) + .EndMap()); +#endif + + if (monitoringServer) { + auto exporter = New<NProfiling::TSolomonExporter>(config); + exporter->Register("/solomon", monitoringServer); + exporter->Start(); + + SetNodeByYPath( + *orchidRoot, + "/sensors", + CreateVirtualNode(exporter->GetSensorService())); + +#ifdef _linux_ + NYTProf::Register(monitoringServer, "/ytprof", buildInfo); + NBacktraceIntrospector::Register(monitoringServer, "/backtrace"); +#endif + monitoringServer->AddHandler( + "/orchid/", + GetOrchidYPathHttpHandler(*orchidRoot)); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TYPathHttpHandler + : public IHttpHandler +{ +public: + explicit TYPathHttpHandler(IYPathServicePtr service) + : Service_(std::move(service)) + { } + + void HandleRequest( + const IRequestPtr& req, + const IResponseWriterPtr& rsp) override + { + const TStringBuf orchidPrefix = "/orchid"; + + TString path{req->GetUrl().Path}; + if (!path.StartsWith(orchidPrefix)) { + THROW_ERROR_EXCEPTION("HTTP request must start with %Qv prefix", + orchidPrefix) + << TErrorAttribute("path", path); + } + + path = path.substr(orchidPrefix.size(), TString::npos); + TCgiParameters params(req->GetUrl().RawQuery); + + auto verb = EVerb::Get; + + auto options = CreateEphemeralAttributes(); + for (const auto& param : params) { + if (param.first == "verb") { + verb = ParseEnum<EVerb>(param.second); + } else { + // Just a check, IAttributeDictionary takes raw YSON anyway. + try { + ValidateYson(TYsonString(param.second), DefaultYsonParserNestingLevelLimit); + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Error parsing value of query parameter %Qv", + param.first) + << ex; + } + + options->SetYson(param.first, TYsonString(param.second)); + } + } + + TYsonString result; + switch (verb) { + case EVerb::Get: { + auto ypathReq = TYPathProxy::Get(path); + ToProto(ypathReq->mutable_options(), *options); + auto ypathRsp = WaitFor(ExecuteVerb(Service_, ypathReq)) + .ValueOrThrow(); + result = TYsonString(ypathRsp->value()); + break; + } + case EVerb::List: { + auto ypathReq = TYPathProxy::List(path); + auto ypathRsp = WaitFor(ExecuteVerb(Service_, ypathReq)) + .ValueOrThrow(); + result = TYsonString(ypathRsp->value()); + break; + } + default: + YT_ABORT(); + } + + rsp->SetStatus(EStatusCode::OK); + NHttp::ReplyJson(rsp, [&] (NYson::IYsonConsumer* writer) { + Serialize(result, writer); + }); + WaitFor(rsp->Close()) + .ThrowOnError(); + } + +private: + const IYPathServicePtr Service_; +}; + +IHttpHandlerPtr GetOrchidYPathHttpHandler(const IYPathServicePtr& service) +{ + return WrapYTException(New<TYPathHttpHandler>(service)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NMonitoring diff --git a/yt/yt/library/monitoring/http_integration.h b/yt/yt/library/monitoring/http_integration.h new file mode 100644 index 0000000000..48c12ca8a8 --- /dev/null +++ b/yt/yt/library/monitoring/http_integration.h @@ -0,0 +1,28 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/ypath_service.h> + +#include <yt/yt/core/http/public.h> + +#include <yt/yt/library/profiling/solomon/public.h> + +namespace NYT::NMonitoring { + +//////////////////////////////////////////////////////////////////////////////// + +void Initialize( + const NHttp::IServerPtr& monitoringServer, + const NProfiling::TSolomonExporterConfigPtr& solomonExporterConfig, + TMonitoringManagerPtr* monitoringManager, + NYTree::IMapNodePtr* orchidRoot); + +NHttp::IHttpHandlerPtr CreateTracingHttpHandler(); + +NHttp::IHttpHandlerPtr GetOrchidYPathHttpHandler( + const NYTree::IYPathServicePtr& service); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NMonitoring diff --git a/yt/yt/library/monitoring/monitoring_manager.cpp b/yt/yt/library/monitoring/monitoring_manager.cpp new file mode 100644 index 0000000000..ef642034a4 --- /dev/null +++ b/yt/yt/library/monitoring/monitoring_manager.cpp @@ -0,0 +1,177 @@ +#include "monitoring_manager.h" +#include "private.h" + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/periodic_executor.h> + +#include <yt/yt/core/ytree/convert.h> +#include <yt/yt/core/ytree/ephemeral_node_factory.h> +#include <yt/yt/core/ytree/node.h> +#include <yt/yt/core/ytree/tree_visitor.h> +#include <yt/yt/core/ytree/ypath_detail.h> +#include <yt/yt/core/ytree/ypath_client.h> + +#include <yt/yt/library/profiling/sensor.h> + +namespace NYT::NMonitoring { + +using namespace NYTree; +using namespace NYPath; +using namespace NYson; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = MonitoringLogger; + +static const auto UpdatePeriod = TDuration::Seconds(3); +static const auto EmptyRoot = GetEphemeralNodeFactory()->CreateMap(); + +//////////////////////////////////////////////////////////////////////////////// + +class TMonitoringManager::TImpl + : public TRefCounted +{ +public: + void Register(const TYPath& path, TYsonProducer producer) + { + auto guard = Guard(SpinLock_); + YT_VERIFY(PathToProducer_.emplace(path, producer).second); + } + + void Unregister(const TYPath& path) + { + auto guard = Guard(SpinLock_); + YT_VERIFY(PathToProducer_.erase(path) == 1); + } + + IYPathServicePtr GetService() + { + return New<TYPathService>(this); + } + + void Start() + { + auto guard = Guard(SpinLock_); + + YT_VERIFY(!Started_); + + PeriodicExecutor_ = New<TPeriodicExecutor>( + ActionQueue_->GetInvoker(), + BIND(&TImpl::Update, MakeWeak(this)), + UpdatePeriod); + PeriodicExecutor_->Start(); + + Started_ = true; + } + + void Stop() + { + auto guard = Guard(SpinLock_); + + if (!Started_) + return; + + Started_ = false; + YT_UNUSED_FUTURE(PeriodicExecutor_->Stop()); + Root_.Reset(); + } + +private: + class TYPathService + : public TYPathServiceBase + { + public: + explicit TYPathService(TIntrusivePtr<TImpl> owner) + : Owner_(std::move(owner)) + { } + + TResolveResult Resolve(const TYPath& path, const IYPathServiceContextPtr& /*context*/) override + { + return TResolveResultThere{Owner_->GetRoot(), path}; + } + + private: + const TIntrusivePtr<TImpl> Owner_; + + }; + + bool Started_ = false; + TActionQueuePtr ActionQueue_ = New<TActionQueue>("Monitoring"); + TPeriodicExecutorPtr PeriodicExecutor_; + + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); + THashMap<TString, NYson::TYsonProducer> PathToProducer_; + IMapNodePtr Root_; + + void Update() + { + YT_LOG_DEBUG("Started updating monitoring state"); + + YT_PROFILE_TIMING("/monitoring/update_time") { + auto newRoot = GetEphemeralNodeFactory()->CreateMap(); + + THashMap<TString, NYson::TYsonProducer> pathToProducer;; + { + auto guard = Guard(SpinLock_); + pathToProducer = PathToProducer_; + } + + for (const auto& [path, producer] : pathToProducer) { + auto value = ConvertToYsonString(producer); + SyncYPathSet(newRoot, path, value); + } + + if (Started_) { + auto guard = Guard(SpinLock_); + std::swap(Root_, newRoot); + } + } + YT_LOG_DEBUG("Finished updating monitoring state"); + } + + IMapNodePtr GetRoot() + { + auto guard = Guard(SpinLock_); + return Root_ ? Root_ : EmptyRoot; + } +}; + +DEFINE_REFCOUNTED_TYPE(TMonitoringManager) + +//////////////////////////////////////////////////////////////////////////////// + +TMonitoringManager::TMonitoringManager() + : Impl_(New<TImpl>()) +{ } + +TMonitoringManager::~TMonitoringManager() = default; + +void TMonitoringManager::Register(const TYPath& path, TYsonProducer producer) +{ + Impl_->Register(path, producer); +} + +void TMonitoringManager::Unregister(const TYPath& path) +{ + Impl_->Unregister(path); +} + +IYPathServicePtr TMonitoringManager::GetService() +{ + return Impl_->GetService(); +} + +void TMonitoringManager::Start() +{ + Impl_->Start(); +} + +void TMonitoringManager::Stop() +{ + Impl_->Stop(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NMonitoring diff --git a/yt/yt/library/monitoring/monitoring_manager.h b/yt/yt/library/monitoring/monitoring_manager.h new file mode 100644 index 0000000000..fc5c3de6c7 --- /dev/null +++ b/yt/yt/library/monitoring/monitoring_manager.h @@ -0,0 +1,54 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/yson/consumer.h> + +#include <yt/yt/core/ypath/public.h> + +#include <yt/yt/core/ytree/public.h> + +namespace NYT::NMonitoring { + +//////////////////////////////////////////////////////////////////////////////// + +//! Exposes a tree assembled from results returned by a set of +//! registered NYson::TYsonProducer-s. +/*! + * \note + * The results are cached and periodically updated. + */ +class TMonitoringManager + : public TRefCounted +{ +public: + TMonitoringManager(); + ~TMonitoringManager(); + + //! Registers a new #producer for a given #path. + void Register(const NYPath::TYPath& path, NYson::TYsonProducer producer); + + //! Unregisters an existing producer for the specified #path. + void Unregister(const NYPath::TYPath& path); + + //! Returns the service representing the whole tree. + /*! + * \note The service is thread-safe. + */ + NYTree::IYPathServicePtr GetService(); + + //! Starts periodic updates. + void Start(); + + //! Stops periodic updates. + void Stop(); + +private: + class TImpl; + TIntrusivePtr<TImpl> Impl_; + +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NMonitoring diff --git a/yt/yt/library/monitoring/private.h b/yt/yt/library/monitoring/private.h new file mode 100644 index 0000000000..e2bfb31c78 --- /dev/null +++ b/yt/yt/library/monitoring/private.h @@ -0,0 +1,15 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NMonitoring { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger MonitoringLogger("Monitoring"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NJournalClient diff --git a/yt/yt/library/monitoring/public.h b/yt/yt/library/monitoring/public.h new file mode 100644 index 0000000000..3514bdd858 --- /dev/null +++ b/yt/yt/library/monitoring/public.h @@ -0,0 +1,13 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +namespace NYT::NMonitoring { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_CLASS(TMonitoringManager) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NMonitoring diff --git a/yt/yt/library/monitoring/ya.make b/yt/yt/library/monitoring/ya.make new file mode 100644 index 0000000000..c2fccd99ac --- /dev/null +++ b/yt/yt/library/monitoring/ya.make @@ -0,0 +1,27 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + http_integration.cpp + monitoring_manager.cpp +) + +PEERDIR( + yt/yt/core + yt/yt/build + yt/yt/library/profiling + yt/yt/library/profiling/solomon + library/cpp/cgiparam +) + +IF (OS_LINUX) + PEERDIR( + yt/yt/library/ytprof + yt/yt/library/ytprof/http + + yt/yt/library/backtrace_introspector/http + ) +ENDIF() + +END() diff --git a/yt/yt/library/process/CMakeLists.darwin-x86_64.txt b/yt/yt/library/process/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b66c679390 --- /dev/null +++ b/yt/yt/library/process/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-process) +target_compile_options(yt-library-process PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-process PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-re2 +) +target_sources(yt-library-process PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp +) diff --git a/yt/yt/library/process/CMakeLists.linux-aarch64.txt b/yt/yt/library/process/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..e065bba3b1 --- /dev/null +++ b/yt/yt/library/process/CMakeLists.linux-aarch64.txt @@ -0,0 +1,27 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-process) +target_compile_options(yt-library-process PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-process PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-re2 +) +target_sources(yt-library-process PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp +) diff --git a/yt/yt/library/process/CMakeLists.linux-x86_64.txt b/yt/yt/library/process/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..e065bba3b1 --- /dev/null +++ b/yt/yt/library/process/CMakeLists.linux-x86_64.txt @@ -0,0 +1,27 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-process) +target_compile_options(yt-library-process PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-process PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-re2 +) +target_sources(yt-library-process PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp +) diff --git a/yt/yt/library/process/CMakeLists.txt b/yt/yt/library/process/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/process/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/process/CMakeLists.windows-x86_64.txt b/yt/yt/library/process/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..3637ee7dae --- /dev/null +++ b/yt/yt/library/process/CMakeLists.windows-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-process) +target_link_libraries(yt-library-process PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + contrib-libs-re2 +) +target_sources(yt-library-process PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp +) diff --git a/yt/yt/library/process/io_dispatcher.cpp b/yt/yt/library/process/io_dispatcher.cpp new file mode 100644 index 0000000000..7da757658d --- /dev/null +++ b/yt/yt/library/process/io_dispatcher.cpp @@ -0,0 +1,37 @@ +#include "io_dispatcher.h" + +#include <yt/yt/core/concurrency/thread_pool_poller.h> +#include <yt/yt/core/concurrency/poller.h> + +#include <yt/yt/core/misc/singleton.h> + +namespace NYT::NPipes { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +TIODispatcher::TIODispatcher() + : Poller_(BIND([] { return CreateThreadPoolPoller(1, "Pipes"); })) +{ } + +TIODispatcher::~TIODispatcher() = default; + +TIODispatcher* TIODispatcher::Get() +{ + return Singleton<TIODispatcher>(); +} + +IInvokerPtr TIODispatcher::GetInvoker() +{ + return Poller_.Value()->GetInvoker(); +} + +IPollerPtr TIODispatcher::GetPoller() +{ + return Poller_.Value(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/io_dispatcher.h b/yt/yt/library/process/io_dispatcher.h new file mode 100644 index 0000000000..2db1b34386 --- /dev/null +++ b/yt/yt/library/process/io_dispatcher.h @@ -0,0 +1,34 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/concurrency/public.h> + +#include <yt/yt/core/misc/lazy_ptr.h> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +class TIODispatcher +{ +public: + ~TIODispatcher(); + + static TIODispatcher* Get(); + + IInvokerPtr GetInvoker(); + + NConcurrency::IPollerPtr GetPoller(); + +private: + TIODispatcher(); + + Y_DECLARE_SINGLETON_FRIEND() + + TLazyIntrusivePtr<NConcurrency::IThreadPoolPoller> Poller_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pipe.cpp b/yt/yt/library/process/pipe.cpp new file mode 100644 index 0000000000..f51d043f22 --- /dev/null +++ b/yt/yt/library/process/pipe.cpp @@ -0,0 +1,256 @@ +#include "pipe.h" +#include "private.h" +#include "io_dispatcher.h" + +#include <yt/yt/core/net/connection.h> + +#include <yt/yt/core/misc/proc.h> +#include <yt/yt/core/misc/fs.h> + +#include <sys/types.h> +#include <sys/stat.h> + +namespace NYT::NPipes { + +using namespace NNet; + +//////////////////////////////////////////////////////////////////////////////// + +static const auto& Logger = PipesLogger; + +//////////////////////////////////////////////////////////////////////////////// + +TNamedPipe::TNamedPipe(const TString& path, bool owning) + : Path_(path) + , Owning_(owning) +{ } + +TNamedPipe::~TNamedPipe() +{ + if (!Owning_) { + return; + } + + if (unlink(Path_.c_str()) == -1) { + YT_LOG_INFO(TError::FromSystem(), "Failed to unlink pipe %v", Path_); + } +} + +TNamedPipePtr TNamedPipe::Create(const TString& path, int permissions) +{ + auto pipe = New<TNamedPipe>(path, /* owning */ true); + pipe->Open(permissions); + YT_LOG_DEBUG("Named pipe created (Path: %v, Permissions: %v)", path, permissions); + return pipe; +} + +TNamedPipePtr TNamedPipe::FromPath(const TString& path) +{ + return New<TNamedPipe>(path, /* owning */ false); +} + +void TNamedPipe::Open(int permissions) +{ + if (mkfifo(Path_.c_str(), permissions) == -1) { + THROW_ERROR_EXCEPTION("Failed to create named pipe %v", Path_) + << TError::FromSystem(); + } +} + +IConnectionReaderPtr TNamedPipe::CreateAsyncReader() +{ + YT_VERIFY(!Path_.empty()); + return CreateInputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this)); +} + +IConnectionWriterPtr TNamedPipe::CreateAsyncWriter() +{ + YT_VERIFY(!Path_.empty()); + return CreateOutputConnectionFromPath(Path_, TIODispatcher::Get()->GetPoller(), MakeStrong(this)); +} + +TString TNamedPipe::GetPath() const +{ + return Path_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TNamedPipeConfigPtr TNamedPipeConfig::Create(TString path, int fd, bool write) +{ + auto result = New<TNamedPipeConfig>(); + result->Path = std::move(path); + result->FD = fd; + result->Write = write; + + return result; +} + +void TNamedPipeConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("path", &TThis::Path) + .Default(); + + registrar.Parameter("fd", &TThis::FD) + .Default(0); + + registrar.Parameter("write", &TThis::Write) + .Default(false); +} + +DEFINE_REFCOUNTED_TYPE(TNamedPipeConfig) + +//////////////////////////////////////////////////////////////////////////////// + +TPipe::TPipe() +{ } + +TPipe::TPipe(TPipe&& pipe) +{ + Init(std::move(pipe)); +} + +TPipe::TPipe(int fd[2]) + : ReadFD_(fd[0]) + , WriteFD_(fd[1]) +{ } + +void TPipe::Init(TPipe&& other) +{ + ReadFD_ = other.ReadFD_; + WriteFD_ = other.WriteFD_; + other.ReadFD_ = InvalidFD; + other.WriteFD_ = InvalidFD; +} + +TPipe::~TPipe() +{ + if (ReadFD_ != InvalidFD) { + YT_VERIFY(TryClose(ReadFD_, false)); + } + + if (WriteFD_ != InvalidFD) { + YT_VERIFY(TryClose(WriteFD_, false)); + } +} + +void TPipe::operator=(TPipe&& other) +{ + if (this == &other) { + return; + } + + Init(std::move(other)); +} + +IConnectionWriterPtr TPipe::CreateAsyncWriter() +{ + YT_VERIFY(WriteFD_ != InvalidFD); + SafeMakeNonblocking(WriteFD_); + return CreateConnectionFromFD(ReleaseWriteFD(), {}, {}, TIODispatcher::Get()->GetPoller()); +} + +IConnectionReaderPtr TPipe::CreateAsyncReader() +{ + YT_VERIFY(ReadFD_ != InvalidFD); + SafeMakeNonblocking(ReadFD_); + return CreateConnectionFromFD(ReleaseReadFD(), {}, {}, TIODispatcher::Get()->GetPoller()); +} + +int TPipe::ReleaseReadFD() +{ + YT_VERIFY(ReadFD_ != InvalidFD); + auto fd = ReadFD_; + ReadFD_ = InvalidFD; + return fd; +} + +int TPipe::ReleaseWriteFD() +{ + YT_VERIFY(WriteFD_ != InvalidFD); + auto fd = WriteFD_; + WriteFD_ = InvalidFD; + return fd; +} + +int TPipe::GetReadFD() const +{ + YT_VERIFY(ReadFD_ != InvalidFD); + return ReadFD_; +} + +int TPipe::GetWriteFD() const +{ + YT_VERIFY(WriteFD_ != InvalidFD); + return WriteFD_; +} + +void TPipe::CloseReadFD() +{ + if (ReadFD_ == InvalidFD) { + return; + } + auto fd = ReadFD_; + ReadFD_ = InvalidFD; + SafeClose(fd, false); +} + +void TPipe::CloseWriteFD() +{ + if (WriteFD_ == InvalidFD) { + return; + } + auto fd = WriteFD_; + WriteFD_ = InvalidFD; + SafeClose(fd, false); +} + +//////////////////////////////////////////////////////////////////////////////// + +TString ToString(const TPipe& pipe) +{ + return Format("{ReadFD: %v, WriteFD: %v}", + pipe.GetReadFD(), + pipe.GetWriteFD()); +} + +//////////////////////////////////////////////////////////////////////////////// + +TPipeFactory::TPipeFactory(int minFD) + : MinFD_(minFD) +{ } + +TPipeFactory::~TPipeFactory() +{ + for (int fd : ReservedFDs_) { + YT_VERIFY(TryClose(fd, false)); + } +} + +TPipe TPipeFactory::Create() +{ + while (true) { + int fd[2]; + SafePipe(fd); + if (fd[0] >= MinFD_ && fd[1] >= MinFD_) { + TPipe pipe(fd); + return pipe; + } else { + ReservedFDs_.push_back(fd[0]); + ReservedFDs_.push_back(fd[1]); + } + } +} + +void TPipeFactory::Clear() +{ + for (int& fd : ReservedFDs_) { + YT_VERIFY(TryClose(fd, false)); + fd = TPipe::InvalidFD; + } + ReservedFDs_.clear(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pipe.h b/yt/yt/library/process/pipe.h new file mode 100644 index 0000000000..10da81cc8a --- /dev/null +++ b/yt/yt/library/process/pipe.h @@ -0,0 +1,114 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/net/public.h> + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +class TNamedPipe + : public TRefCounted +{ +public: + ~TNamedPipe(); + static TNamedPipePtr Create(const TString& path, int permissions = 0660); + static TNamedPipePtr FromPath(const TString& path); + + NNet::IConnectionReaderPtr CreateAsyncReader(); + NNet::IConnectionWriterPtr CreateAsyncWriter(); + + TString GetPath() const; + +private: + const TString Path_; + + //! Whether pipe was created by this class + //! and should be removed in destructor. + const bool Owning_; + + explicit TNamedPipe(const TString& path, bool owning); + void Open(int permissions); + DECLARE_NEW_FRIEND() +}; + +DEFINE_REFCOUNTED_TYPE(TNamedPipe) + +//////////////////////////////////////////////////////////////////////////////// + +class TNamedPipeConfig + : public NYTree::TYsonStruct +{ +public: + TString Path; + int FD = 0; + bool Write = false; + + static TNamedPipeConfigPtr Create(TString path, int fd, bool write); + + REGISTER_YSON_STRUCT(TNamedPipeConfig); + + static void Register(TRegistrar registrar); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TPipe + : public TNonCopyable +{ +public: + static const int InvalidFD = -1; + + TPipe(); + TPipe(TPipe&& pipe); + ~TPipe(); + + void operator=(TPipe&& other); + + void CloseReadFD(); + void CloseWriteFD(); + + NNet::IConnectionReaderPtr CreateAsyncReader(); + NNet::IConnectionWriterPtr CreateAsyncWriter(); + + int ReleaseReadFD(); + int ReleaseWriteFD(); + + int GetReadFD() const; + int GetWriteFD() const; + +private: + int ReadFD_ = InvalidFD; + int WriteFD_ = InvalidFD; + + TPipe(int fd[2]); + void Init(TPipe&& other); + + friend class TPipeFactory; +}; + +TString ToString(const TPipe& pipe); + +//////////////////////////////////////////////////////////////////////////////// + +class TPipeFactory +{ +public: + explicit TPipeFactory(int minFD = 0); + ~TPipeFactory(); + + TPipe Create(); + + void Clear(); + +private: + const int MinFD_; + std::vector<int> ReservedFDs_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/private.h b/yt/yt/library/process/private.h new file mode 100644 index 0000000000..95b2ffb0f5 --- /dev/null +++ b/yt/yt/library/process/private.h @@ -0,0 +1,14 @@ +#pragma once + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger PipesLogger("Pipes"); +inline const NLogging::TLogger PtyLogger("Pty"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/process.cpp b/yt/yt/library/process/process.cpp new file mode 100644 index 0000000000..809a50ed9a --- /dev/null +++ b/yt/yt/library/process/process.cpp @@ -0,0 +1,697 @@ +#include "process.h" +#include "pipe.h" + +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/error.h> +#include <yt/yt/core/misc/fs.h> +#include <yt/yt/core/misc/finally.h> + +#include <yt/yt/core/concurrency/periodic_executor.h> +#include <yt/yt/core/concurrency/delayed_executor.h> + +#include <library/cpp/yt/system/handle_eintr.h> + +#include <util/folder/dirut.h> + +#include <util/generic/guid.h> + +#include <util/string/ascii.h> + +#include <util/string/util.h> + +#include <util/system/env.h> +#include <util/system/execpath.h> +#include <util/system/maxlen.h> +#include <util/system/shellcommand.h> + +#ifdef _unix_ + #include <unistd.h> + #include <errno.h> + #include <sys/wait.h> + #include <sys/resource.h> +#endif + +#ifdef _darwin_ + #include <crt_externs.h> + #define environ (*_NSGetEnviron()) +#endif + +namespace NYT { + +using namespace NPipes; +using namespace NNet; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +static inline const NLogging::TLogger Logger("Process"); + +static constexpr pid_t InvalidProcessId = -1; + +static constexpr int ExecveRetryCount = 5; +static constexpr auto ExecveRetryTimeout = TDuration::Seconds(1); + +static constexpr int ResolveRetryCount = 5; +static constexpr auto ResolveRetryTimeout = TDuration::Seconds(1); + +//////////////////////////////////////////////////////////////////////////////// + +TErrorOr<TString> ResolveBinaryPath(const TString& binary) +{ + auto Logger = NYT::Logger + .WithTag("Binary: %v", binary); + + YT_LOG_DEBUG("Resolving binary path"); + + std::vector<TError> accumulatedErrors; + + auto test = [&] (const char* path) { + YT_LOG_DEBUG("Probing path (Path: %v)", path); + if (access(path, R_OK | X_OK) == 0) { + return true; + } else { + auto error = TError("Cannot run %Qlv", path) << TError::FromSystem(); + accumulatedErrors.push_back(std::move(error)); + return false; + } + }; + + auto failure = [&] { + auto error = TError( + EProcessErrorCode::CannotResolveBinary, + "Cannot resolve binary %Qlv", + binary); + error.MutableInnerErrors()->swap(accumulatedErrors); + YT_LOG_DEBUG(error, "Error resolving binary path"); + return error; + }; + + auto success = [&] (const TString& path) { + YT_LOG_DEBUG("Binary resolved (Path: %v)", path); + return path; + }; + + if (test(binary.c_str())) { + return success(binary); + } + + // If this is an absolute path, stop here. + if (binary.empty() || binary[0] == '/') { + return failure(); + } + + // XXX(sandello): Sometimes we drop PATH from environment when spawning isolated processes. + // In this case, try to locate somewhere nearby. + { + auto execPathDirName = GetDirName(GetExecPath()); + YT_LOG_DEBUG("Looking in our exec path directory (ExecPathDir: %v)", execPathDirName); + auto probe = TString::Join(execPathDirName, "/", binary); + if (test(probe.c_str())) { + return success(probe); + } + } + + std::array<char, MAX_PATH> buffer; + + auto envPathStr = GetEnv("PATH"); + TStringBuf envPath(envPathStr); + TStringBuf envPathItem; + + YT_LOG_DEBUG("Looking for binary in PATH (Path: %v)", envPathStr); + + while (envPath.NextTok(':', envPathItem)) { + if (buffer.size() < 2 + envPathItem.size() + binary.size()) { + continue; + } + + size_t index = 0; + std::copy(envPathItem.begin(), envPathItem.end(), buffer.begin() + index); + index += envPathItem.size(); + buffer[index] = '/'; + index += 1; + std::copy(binary.begin(), binary.end(), buffer.begin() + index); + index += binary.size(); + buffer[index] = 0; + + if (test(buffer.data())) { + return success(TString(buffer.data(), index)); + } + } + + return failure(); +} + +bool TryKillProcessByPid(int pid, int signal) +{ +#ifdef _unix_ + YT_VERIFY(pid != -1); + int result = ::kill(pid, signal); + // Ignore ESRCH because process may have died just before TryKillProcessByPid. + if (result < 0 && errno != ESRCH) { + return false; + } + return true; +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +#ifdef _unix_ + +bool TryWaitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) +{ + if (infop != nullptr) { + // See comment below. + infop->si_pid = 0; + } + + siginfo_t info; + ::memset(&info, 0, sizeof(info)); + auto res = HandleEintr(::waitid, idtype, id, infop != nullptr ? infop : &info, options); + + if (res == 0) { + // According to man wait. + // If WNOHANG was specified in options and there were + // no children in a waitable state, then waitid() returns 0 immediately. + // To distinguish this case from that where a child + // was in a waitable state, zero out the si_pid field + // before the call and check for a nonzero value in this field after + // the call returns. + if (infop && infop->si_pid == 0) { + return false; + } + return true; + } + + return false; +} + +void Wait4OrDie(pid_t id, int* status, int options, rusage* rusage) +{ + auto res = HandleEintr(::wait4, id, status, options, rusage); + if (res == -1) { + YT_LOG_FATAL(TError::FromSystem(), "Wait4 failed"); + } +} + +void Cleanup(int pid) +{ + YT_VERIFY(pid > 0); + + YT_VERIFY(TryKillProcessByPid(pid, 9)); + YT_VERIFY(TryWaitid(P_PID, pid, nullptr, WEXITED)); +} + +bool TrySetSignalMask(const sigset_t* sigmask, sigset_t* oldSigmask) +{ + int error = pthread_sigmask(SIG_SETMASK, sigmask, oldSigmask); + if (error != 0) { + return false; + } + return true; +} + +bool TryResetSignals() +{ + for (int sig = 1; sig < NSIG; ++sig) { + // Ignore invalid signal errors. + ::signal(sig, SIG_DFL); + } + return true; +} + +#endif + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TProcessBase::TProcessBase(const TString& path) + : Path_(path) + , ProcessId_(InvalidProcessId) +{ } + +void TProcessBase::AddArgument(TStringBuf arg) +{ + YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); + + Args_.push_back(Capture(arg)); +} + +void TProcessBase::AddEnvVar(TStringBuf var) +{ + YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); + + Env_.push_back(Capture(var)); +} + +void TProcessBase::AddArguments(std::initializer_list<TStringBuf> args) +{ + for (auto arg : args) { + AddArgument(arg); + } +} + +void TProcessBase::AddArguments(const std::vector<TString>& args) +{ + for (const auto& arg : args) { + AddArgument(arg); + } +} + +void TProcessBase::SetWorkingDirectory(const TString& path) +{ + WorkingDirectory_ = path; +} + +void TProcessBase::CreateProcessGroup() +{ + CreateProcessGroup_ = true; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSimpleProcess::TSimpleProcess(const TString& path, bool copyEnv, TDuration pollPeriod) + // TString is guaranteed to be zero-terminated. + // https://wiki.yandex-team.ru/Development/Poisk/arcadia/util/TStringAndTStringBuf#sobstvennosimvoly + : TProcessBase(path) + , PollPeriod_(pollPeriod) + , PipeFactory_(3) +{ + AddArgument(path); + + if (copyEnv) { + for (char** envIt = environ; *envIt; ++envIt) { + Env_.push_back(Capture(*envIt)); + } + } +} + +void TSimpleProcess::AddDup2FileAction(int oldFD, int newFD) +{ + TSpawnAction action{ + std::bind(TryDup2, oldFD, newFD), + Format("Error duplicating %v file descriptor to %v in child process", oldFD, newFD) + }; + + MaxSpawnActionFD_ = std::max(MaxSpawnActionFD_, newFD); + SpawnActions_.push_back(action); +} + +IConnectionReaderPtr TSimpleProcess::GetStdOutReader() +{ + auto& pipe = StdPipes_[STDOUT_FILENO]; + pipe = PipeFactory_.Create(); + AddDup2FileAction(pipe.GetWriteFD(), STDOUT_FILENO); + return pipe.CreateAsyncReader(); +} + +IConnectionReaderPtr TSimpleProcess::GetStdErrReader() +{ + auto& pipe = StdPipes_[STDERR_FILENO]; + pipe = PipeFactory_.Create(); + AddDup2FileAction(pipe.GetWriteFD(), STDERR_FILENO); + return pipe.CreateAsyncReader(); +} + +IConnectionWriterPtr TSimpleProcess::GetStdInWriter() +{ + auto& pipe = StdPipes_[STDIN_FILENO]; + pipe = PipeFactory_.Create(); + AddDup2FileAction(pipe.GetReadFD(), STDIN_FILENO); + return pipe.CreateAsyncWriter(); +} + +TFuture<void> TProcessBase::Spawn() +{ + try { + // Resolve binary path. + std::vector<TError> innerErrors; + for (int retryIndex = ResolveRetryCount; retryIndex >= 0; --retryIndex) { + auto errorOrPath = ResolveBinaryPath(Path_); + if (errorOrPath.IsOK()) { + ResolvedPath_ = errorOrPath.Value(); + break; + } + + innerErrors.push_back(errorOrPath); + + if (retryIndex == 0) { + THROW_ERROR_EXCEPTION("Failed to resolve binary path %v", Path_) + << innerErrors; + } + + TDelayedExecutor::WaitForDuration(ResolveRetryTimeout); + } + + DoSpawn(); + } catch (const std::exception& ex) { + FinishedPromise_.TrySet(ex); + } + return FinishedPromise_; +} + +void TSimpleProcess::DoSpawn() +{ +#ifdef _unix_ + auto finally = Finally([&] () { + StdPipes_[STDIN_FILENO].CloseReadFD(); + StdPipes_[STDOUT_FILENO].CloseWriteFD(); + StdPipes_[STDERR_FILENO].CloseWriteFD(); + PipeFactory_.Clear(); + }); + + YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); + + // Make sure no spawn action closes Pipe_.WriteFD + TPipeFactory pipeFactory(MaxSpawnActionFD_ + 1); + Pipe_ = pipeFactory.Create(); + pipeFactory.Clear(); + + YT_LOG_DEBUG("Spawning new process (Path: %v, ErrorPipe: %v, Arguments: %v, Environment: %v)", + ResolvedPath_, + Pipe_, + Args_, + Env_); + + Env_.push_back(nullptr); + Args_.push_back(nullptr); + + // Block all signals around vfork; see http://ewontfix.com/7/ + + // As the child may run in the same address space as the parent until + // the actual execve() system call, any (custom) signal handlers that + // the parent has might alter parent's memory if invoked in the child, + // with undefined results. So we block all signals in the parent before + // vfork(), which will cause them to be blocked in the child as well (we + // rely on the fact that Linux, just like all sane implementations, only + // clones the calling thread). Then, in the child, we reset all signals + // to their default dispositions (while still blocked), and unblock them + // (so the exec()ed process inherits the parent's signal mask) + + sigset_t allBlocked; + sigfillset(&allBlocked); + sigset_t oldSignals; + + if (!TrySetSignalMask(&allBlocked, &oldSignals)) { + THROW_ERROR_EXCEPTION("Failed to block all signals") + << TError::FromSystem(); + } + + SpawnActions_.push_back(TSpawnAction{ + TryResetSignals, + "Error resetting signals to default disposition in child process: signal failed" + }); + + SpawnActions_.push_back(TSpawnAction{ + std::bind(TrySetSignalMask, &oldSignals, nullptr), + "Error unblocking signals in child process: pthread_sigmask failed" + }); + + if (!WorkingDirectory_.empty()) { + SpawnActions_.push_back(TSpawnAction{ + [&] () { + NFs::SetCurrentWorkingDirectory(WorkingDirectory_); + return true; + }, + "Error changing working directory" + }); + } + + if (CreateProcessGroup_) { + SpawnActions_.push_back(TSpawnAction{ + [&] () { + setpgrp(); + return true; + }, + "Error creating process group" + }); + } + + SpawnActions_.push_back(TSpawnAction{ + [this] { + for (int retryIndex = 0; retryIndex < ExecveRetryCount; ++retryIndex) { + // Execve may fail, if called binary is being updated, e.g. during yandex-yt package update. + // So we'd better retry several times. + // For example see YT-6352. + TryExecve(ResolvedPath_.c_str(), Args_.data(), Env_.data()); + if (retryIndex < ExecveRetryCount - 1) { + Sleep(ExecveRetryTimeout); + } + } + // If we are still here, return failure. + return false; + }, + "Error starting child process: execve failed" + }); + + SpawnChild(); + + // This should not fail ever. + YT_VERIFY(TrySetSignalMask(&oldSignals, nullptr)); + + Pipe_.CloseWriteFD(); + + ValidateSpawnResult(); + + AsyncWaitExecutor_ = New<TPeriodicExecutor>( + GetSyncInvoker(), + BIND(&TSimpleProcess::AsyncPeriodicTryWait, MakeStrong(this)), + PollPeriod_); + + AsyncWaitExecutor_->Start(); +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +void TSimpleProcess::SpawnChild() +{ + // NB: fork() will cause data corruption when run concurrently with + // Disk IO on O_DIRECT file descriptor. Seems like vfork don't suffer from the same issue. + +#ifdef _unix_ + int pid = vfork(); + + if (pid < 0) { + THROW_ERROR_EXCEPTION("Error starting child process: vfork failed") + << TErrorAttribute("path", ResolvedPath_) + << TError::FromSystem(); + } + + if (pid == 0) { + try { + Child(); + } catch (...) { + YT_ABORT(); + } + } + + ProcessId_ = pid; + Started_ = true; +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +void TSimpleProcess::ValidateSpawnResult() +{ +#ifdef _unix_ + int data[2]; + ssize_t res; + res = HandleEintr(::read, Pipe_.GetReadFD(), &data, sizeof(data)); + Pipe_.CloseReadFD(); + + if (res == 0) { + // Child successfully spawned or was killed by a signal. + // But there is no way to distinguish between these two cases: + // * child killed by signal before exec + // * child killed by signal after exec + // So we treat kill-before-exec the same way as kill-after-exec. + YT_LOG_DEBUG("Child process spawned successfully (Pid: %v)", ProcessId_); + return; + } + + YT_VERIFY(res == sizeof(data)); + Finished_ = true; + + Cleanup(ProcessId_); + ProcessId_ = InvalidProcessId; + + int actionIndex = data[0]; + int errorCode = data[1]; + + YT_VERIFY(0 <= actionIndex && actionIndex < std::ssize(SpawnActions_)); + const auto& action = SpawnActions_[actionIndex]; + THROW_ERROR_EXCEPTION("%v", action.ErrorMessage) + << TError::FromSystem(errorCode); +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +#ifdef _unix_ +void TSimpleProcess::AsyncPeriodicTryWait() +{ + siginfo_t processInfo; + memset(&processInfo, 0, sizeof(siginfo_t)); + + // Note WNOWAIT flag. + // This call just waits for a process to be finished but does not clear zombie flag. + + if (!TryWaitid(P_PID, ProcessId_, &processInfo, WEXITED | WNOWAIT | WNOHANG) || + processInfo.si_pid != ProcessId_) + { + return; + } + + YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop()); + AsyncWaitExecutor_ = nullptr; + + // This call just should return immediately + // because we have already waited for this process with WNOHANG + rusage rusage; + Wait4OrDie(ProcessId_, nullptr, WNOHANG, &rusage); + + Finished_ = true; + auto error = ProcessInfoToError(processInfo); + YT_LOG_DEBUG("Process finished (Pid: %v, MajFaults: %d, Error: %v)", ProcessId_, rusage.ru_majflt, error); + + FinishedPromise_.Set(error); +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +void TSimpleProcess::Kill(int signal) +{ +#ifdef _unix_ + if (!Started_) { + THROW_ERROR_EXCEPTION("Process is not started yet"); + } + + if (Finished_) { + return; + } + + YT_LOG_DEBUG("Killing child process (Pid: %v)", ProcessId_); + + bool result = false; + if (!CreateProcessGroup_) { + result = TryKillProcessByPid(ProcessId_, signal); + } else { + result = TryKillProcessByPid(-1 * ProcessId_, signal); + } + + if (!result) { + THROW_ERROR_EXCEPTION("Failed to kill child process %v", ProcessId_) + << TError::FromSystem(); + } + return; +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +TString TProcessBase::GetPath() const +{ + return Path_; +} + +int TProcessBase::GetProcessId() const +{ + return ProcessId_; +} + +bool TProcessBase::IsStarted() const +{ + return Started_; +} + +bool TProcessBase::IsFinished() const +{ + return Finished_; +} + +TString TProcessBase::GetCommandLine() const +{ + TStringBuilder builder; + builder.AppendString(Path_); + + bool first = true; + for (const auto& arg_ : Args_) { + TStringBuf arg(arg_); + if (first) { + first = false; + } else { + if (arg) { + builder.AppendChar(' '); + bool needQuote = false; + for (size_t i = 0; i < arg.length(); ++i) { + if (!IsAsciiAlnum(arg[i]) && + arg[i] != '-' && arg[i] != '_' && arg[i] != '=' && arg[i] != '/') + { + needQuote = true; + break; + } + } + if (needQuote) { + builder.AppendChar('"'); + TStringBuf left, right; + while (arg.TrySplit('"', left, right)) { + builder.AppendString(left); + builder.AppendString("\\\""); + arg = right; + } + builder.AppendString(arg); + builder.AppendChar('"'); + } else { + builder.AppendString(arg); + } + } + } + } + + return builder.Flush(); +} + +const char* TProcessBase::Capture(TStringBuf arg) +{ + StringHolders_.push_back(TString(arg)); + return StringHolders_.back().c_str(); +} + +void TSimpleProcess::Child() +{ +#ifdef _unix_ + for (int actionIndex = 0; actionIndex < std::ssize(SpawnActions_); ++actionIndex) { + auto& action = SpawnActions_[actionIndex]; + if (!action.Callback()) { + // Report error through the pipe. + int data[] = { + actionIndex, + errno + }; + + // According to pipe(7) write of small buffer is atomic. + ssize_t size = HandleEintr(::write, Pipe_.GetWriteFD(), &data, sizeof(data)); + YT_VERIFY(size == sizeof(data)); + _exit(1); + } + } +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif + YT_ABORT(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/process/process.h b/yt/yt/library/process/process.h new file mode 100644 index 0000000000..b38ae3f4b3 --- /dev/null +++ b/yt/yt/library/process/process.h @@ -0,0 +1,125 @@ +#pragma once + +#include "pipe.h" + +#include <yt/yt/core/misc/error.h> + +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/concurrency/public.h> + +#include <atomic> +#include <vector> +#include <array> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TErrorOr<TString> ResolveBinaryPath(const TString& binary); +bool TryKillProcessByPid(int pid, int signal); + +//////////////////////////////////////////////////////////////////////////////// + +class TProcessBase + : public TRefCounted +{ +public: + explicit TProcessBase(const TString& path); + + void AddArgument(TStringBuf arg); + void AddEnvVar(TStringBuf var); + + void AddArguments(std::initializer_list<TStringBuf> args); + void AddArguments(const std::vector<TString>& args); + + void SetWorkingDirectory(const TString& path); + void CreateProcessGroup(); + + virtual NNet::IConnectionWriterPtr GetStdInWriter() = 0; + virtual NNet::IConnectionReaderPtr GetStdOutReader() = 0; + virtual NNet::IConnectionReaderPtr GetStdErrReader() = 0; + + TFuture<void> Spawn(); + virtual void Kill(int signal) = 0; + + TString GetPath() const; + int GetProcessId() const; + bool IsStarted() const; + bool IsFinished() const; + + TString GetCommandLine() const; + +protected: + const TString Path_; + + int ProcessId_; + std::atomic<bool> Started_ = {false}; + std::atomic<bool> Finished_ = {false}; + int MaxSpawnActionFD_ = - 1; + NPipes::TPipe Pipe_; + // Container for owning string data. Use std::deque because it never moves contained objects. + std::deque<std::string> StringHolders_; + std::vector<const char*> Args_; + std::vector<const char*> Env_; + TString ResolvedPath_; + TString WorkingDirectory_; + bool CreateProcessGroup_ = false; + TPromise<void> FinishedPromise_ = NewPromise<void>(); + + virtual void DoSpawn() = 0; + const char* Capture(TStringBuf arg); + +private: + void SpawnChild(); + void ValidateSpawnResult(); + void Child(); + void AsyncPeriodicTryWait(); +}; + +DEFINE_REFCOUNTED_TYPE(TProcessBase) + +//////////////////////////////////////////////////////////////////////////////// + +// Read this +// http://ewontfix.com/7/ +// before making any changes. +class TSimpleProcess + : public TProcessBase +{ +public: + explicit TSimpleProcess( + const TString& path, + bool copyEnv = true, + TDuration pollPeriod = TDuration::MilliSeconds(100)); + void Kill(int signal) override; + NNet::IConnectionWriterPtr GetStdInWriter() override; + NNet::IConnectionReaderPtr GetStdOutReader() override; + NNet::IConnectionReaderPtr GetStdErrReader() override; + +private: + const TDuration PollPeriod_; + + NPipes::TPipeFactory PipeFactory_; + std::array<NPipes::TPipe, 3> StdPipes_; + + NConcurrency::TPeriodicExecutorPtr AsyncWaitExecutor_; + struct TSpawnAction + { + std::function<bool()> Callback; + TString ErrorMessage; + }; + + std::vector<TSpawnAction> SpawnActions_; + + void AddDup2FileAction(int oldFD, int newFD); + void DoSpawn() override; + void SpawnChild(); + void ValidateSpawnResult(); + void AsyncPeriodicTryWait(); + void Child(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/process/pty.cpp b/yt/yt/library/process/pty.cpp new file mode 100644 index 0000000000..fc972d38ea --- /dev/null +++ b/yt/yt/library/process/pty.cpp @@ -0,0 +1,64 @@ +#include "pty.h" + +#include "io_dispatcher.h" + +#include <yt/yt/core/misc/common.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/net/connection.h> + +namespace NYT::NPipes { + +using namespace NNet; + +//////////////////////////////////////////////////////////////////////////////// + +TPty::TPty(int height, int width) +{ + SafeOpenPty(&MasterFD_, &SlaveFD_, height, width); +} + +TPty::~TPty() +{ + if (MasterFD_ != InvalidFD) { + YT_VERIFY(TryClose(MasterFD_, false)); + } + + if (SlaveFD_ != InvalidFD) { + YT_VERIFY(TryClose(SlaveFD_, false)); + } +} + +IConnectionWriterPtr TPty::CreateMasterAsyncWriter() +{ + YT_VERIFY(MasterFD_ != InvalidFD); + int fd = SafeDup(MasterFD_); + SafeSetCloexec(fd); + SafeMakeNonblocking(fd); + return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller()); +} + +IConnectionReaderPtr TPty::CreateMasterAsyncReader() +{ + YT_VERIFY(MasterFD_ != InvalidFD); + int fd = SafeDup(MasterFD_); + SafeSetCloexec(fd); + SafeMakeNonblocking(fd); + return CreateConnectionFromFD(fd, {}, {}, TIODispatcher::Get()->GetPoller()); +} + +int TPty::GetMasterFD() const +{ + YT_VERIFY(MasterFD_ != InvalidFD); + return MasterFD_; +} + +int TPty::GetSlaveFD() const +{ + YT_VERIFY(SlaveFD_ != InvalidFD); + return SlaveFD_; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/pty.h b/yt/yt/library/process/pty.h new file mode 100644 index 0000000000..b585782d12 --- /dev/null +++ b/yt/yt/library/process/pty.h @@ -0,0 +1,33 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/net/public.h> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +class TPty + : public TNonCopyable +{ +public: + static const int InvalidFD = -1; + + TPty(int height, int width); + ~TPty(); + + NNet::IConnectionReaderPtr CreateMasterAsyncReader(); + NNet::IConnectionWriterPtr CreateMasterAsyncWriter(); + + int GetMasterFD() const; + int GetSlaveFD() const; + +private: + int MasterFD_ = InvalidFD; + int SlaveFD_ = InvalidFD; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/public.h b/yt/yt/library/process/public.h new file mode 100644 index 0000000000..0fa1d3d0a9 --- /dev/null +++ b/yt/yt/library/process/public.h @@ -0,0 +1,14 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_CLASS(TNamedPipe) +DECLARE_REFCOUNTED_CLASS(TNamedPipeConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/subprocess.cpp b/yt/yt/library/process/subprocess.cpp new file mode 100644 index 0000000000..02555b0c9b --- /dev/null +++ b/yt/yt/library/process/subprocess.cpp @@ -0,0 +1,153 @@ +#include "subprocess.h" + +#include <yt/yt/core/misc/blob.h> +#include <yt/yt/core/misc/proc.h> +#include <yt/yt/core/misc/finally.h> + +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/net/connection.h> + +#include <util/system/execpath.h> + +#include <array> + +namespace NYT { + +using namespace NConcurrency; +using namespace NPipes; + +//////////////////////////////////////////////////////////////////////////////// + +const static size_t PipeBlockSize = 64 * 1024; +static NLogging::TLogger Logger("Subprocess"); + +//////////////////////////////////////////////////////////////////////////////// + +TSubprocess::TSubprocess(const TString& path, bool copyEnv) + : Process_(New<TSimpleProcess>(path, copyEnv)) +{ } + +TSubprocess TSubprocess::CreateCurrentProcessSpawner() +{ + return TSubprocess(GetExecPath()); +} + +void TSubprocess::AddArgument(TStringBuf arg) +{ + Process_->AddArgument(arg); +} + +void TSubprocess::AddArguments(std::initializer_list<TStringBuf> args) +{ + Process_->AddArguments(args); +} + +TSubprocessResult TSubprocess::Execute(const TSharedRef& input) +{ +#ifdef _unix_ + auto inputStream = Process_->GetStdInWriter(); + auto outputStream = Process_->GetStdOutReader(); + auto errorStream = Process_->GetStdErrReader(); + auto finished = Process_->Spawn(); + + auto readIntoBlob = [] (IAsyncInputStreamPtr stream) { + TBlob output; + auto buffer = TSharedMutableRef::Allocate(PipeBlockSize, {.InitializeStorage = false}); + while (true) { + auto size = WaitFor(stream->Read(buffer)) + .ValueOrThrow(); + + if (size == 0) + break; + + // ToDo(psushin): eliminate copying. + output.Append(buffer.Begin(), size); + } + return TSharedRef::FromBlob(std::move(output)); + }; + + auto writeStdin = BIND([=] { + if (input.Size() > 0) { + WaitFor(inputStream->Write(input)) + .ThrowOnError(); + } + + WaitFor(inputStream->Close()) + .ThrowOnError(); + + //! Return dummy ref, so later we cat put Future into vector + //! along with stdout and stderr. + return TSharedRef::MakeEmpty(); + }); + + std::vector<TFuture<TSharedRef>> futures = { + BIND(readIntoBlob, outputStream).AsyncVia(GetCurrentInvoker()).Run(), + BIND(readIntoBlob, errorStream).AsyncVia(GetCurrentInvoker()).Run(), + writeStdin.AsyncVia(GetCurrentInvoker()).Run(), + }; + + try { + auto outputsOrError = WaitFor(AllSucceeded(futures)); + THROW_ERROR_EXCEPTION_IF_FAILED( + outputsOrError, + "IO error occurred during subprocess call"); + + const auto& outputs = outputsOrError.Value(); + YT_VERIFY(outputs.size() == 3); + + // This can block indefinitely. + auto exitCode = WaitFor(finished); + return TSubprocessResult{outputs[0], outputs[1], exitCode}; + } catch (...) { + try { + Process_->Kill(SIGKILL); + } catch (...) { } + Y_UNUSED(WaitFor(finished)); + throw; + } +#else + THROW_ERROR_EXCEPTION("Unsupported platform"); +#endif +} + +void TSubprocess::Kill(int signal) +{ + Process_->Kill(signal); +} + +TString TSubprocess::GetCommandLine() const +{ + return Process_->GetCommandLine(); +} + +TProcessBasePtr TSubprocess::GetProcess() const +{ + return Process_; +} + +//////////////////////////////////////////////////////////////////////////////// + +void RunSubprocess(const std::vector<TString>& cmd) +{ + if (cmd.empty()) { + THROW_ERROR_EXCEPTION("Command can't be empty"); + } + + auto process = TSubprocess(cmd[0]); + for (int index = 1; index < std::ssize(cmd); ++index) { + process.AddArgument(cmd[index]); + } + + auto result = process.Execute(); + if (!result.Status.IsOK()) { + THROW_ERROR_EXCEPTION("Failed to run %v", cmd[0]) + << result.Status + << TErrorAttribute("command_line", process.GetCommandLine()) + << TErrorAttribute("error", TString(result.Error.Begin(), result.Error.End())); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/process/subprocess.h b/yt/yt/library/process/subprocess.h new file mode 100644 index 0000000000..223db533f6 --- /dev/null +++ b/yt/yt/library/process/subprocess.h @@ -0,0 +1,48 @@ +#pragma once + +#include "public.h" +#include "process.h" + +#include <library/cpp/yt/memory/ref.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +struct TSubprocessResult +{ + TSharedRef Output; + TSharedRef Error; + TError Status; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TSubprocess +{ +public: + explicit TSubprocess(const TString& path, bool copyEnv = true); + + static TSubprocess CreateCurrentProcessSpawner(); + + void AddArgument(TStringBuf arg); + void AddArguments(std::initializer_list<TStringBuf> args); + + TSubprocessResult Execute(const TSharedRef& input = TSharedRef::MakeEmpty()); + void Kill(int signal); + + TString GetCommandLine() const; + + TProcessBasePtr GetProcess() const; + +private: + const TProcessBasePtr Process_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +void RunSubprocess(const std::vector<TString>& cmd); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/process/unittests/pipes_ut.cpp b/yt/yt/library/process/unittests/pipes_ut.cpp new file mode 100644 index 0000000000..0c7e2a0cf2 --- /dev/null +++ b/yt/yt/library/process/unittests/pipes_ut.cpp @@ -0,0 +1,319 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/misc/blob.h> +#include <yt/yt/core/misc/proc.h> + +#include <yt/yt/core/net/connection.h> + +#include <yt/yt/library/process/pipe.h> + +#include <random> + +namespace NYT::NPipes { + +//////////////////////////////////////////////////////////////////////////////// + +using namespace NConcurrency; +using namespace NNet; + +#ifndef _win_ + +TEST(TPipeIOHolder, CanInstantiate) +{ + auto pipe = TPipeFactory().Create(); + + auto readerHolder = pipe.CreateAsyncReader(); + auto writerHolder = pipe.CreateAsyncWriter(); + + readerHolder->Abort().Get(); + writerHolder->Abort().Get(); +} + +TEST(TPipeTest, PrematureEOF) +{ + auto pipe = TNamedPipe::Create("./namedpipe"); + auto reader = pipe->CreateAsyncReader(); + + auto buffer = TSharedMutableRef::Allocate(1024 * 1024); + EXPECT_THROW(reader->Read(buffer).WithTimeout(TDuration::Seconds(1)).Get().ValueOrThrow(), TErrorException); +} + +//////////////////////////////////////////////////////////////////////////////// + +TBlob ReadAll(IConnectionReaderPtr reader, bool useWaitFor) +{ + auto buffer = TSharedMutableRef::Allocate(1_MB, {.InitializeStorage = false}); + auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>()); + + while (true) { + TErrorOr<size_t> result; + auto future = reader->Read(buffer); + if (useWaitFor) { + result = WaitFor(future); + } else { + result = future.Get(); + } + + if (result.ValueOrThrow() == 0) { + break; + } + + whole.Append(buffer.Begin(), result.Value()); + } + return whole; +} + +TEST(TAsyncWriterTest, AsyncCloseFail) +{ + auto pipe = TPipeFactory().Create(); + + auto reader = pipe.CreateAsyncReader(); + auto writer = pipe.CreateAsyncWriter(); + + auto queue = New<NConcurrency::TActionQueue>(); + auto readFromPipe = + BIND(&ReadAll, reader, false) + .AsyncVia(queue->GetInvoker()) + .Run(); + + int length = 200*1024; + auto buffer = TSharedMutableRef::Allocate(length); + ::memset(buffer.Begin(), 'a', buffer.Size()); + + auto writeResult = writer->Write(buffer).Get(); + + EXPECT_TRUE(writeResult.IsOK()) + << ToString(writeResult); + + auto error = writer->Close(); + + auto readResult = readFromPipe.Get(); + ASSERT_TRUE(readResult.IsOK()) + << ToString(readResult); + + auto closeStatus = error.Get(); +} + +TEST(TAsyncWriterTest, WriteFailed) +{ + auto pipe = TPipeFactory().Create(); + auto reader = pipe.CreateAsyncReader(); + auto writer = pipe.CreateAsyncWriter(); + + int length = 200*1024; + auto buffer = TSharedMutableRef::Allocate(length); + ::memset(buffer.Begin(), 'a', buffer.Size()); + + auto asyncWriteResult = writer->Write(buffer); + reader->Abort(); + + EXPECT_FALSE(asyncWriteResult.Get().IsOK()) + << ToString(asyncWriteResult.Get()); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPipeReadWriteTest + : public ::testing::Test +{ +protected: + void SetUp() override + { + auto pipe = TPipeFactory().Create(); + + Reader = pipe.CreateAsyncReader(); + Writer = pipe.CreateAsyncWriter(); + } + + void TearDown() override + { } + + IConnectionReaderPtr Reader; + IConnectionWriterPtr Writer; +}; + +class TNamedPipeReadWriteTest + : public ::testing::Test +{ +protected: + void SetUp() override + { + auto pipe = TNamedPipe::Create("./namedpipe"); + Reader = pipe->CreateAsyncReader(); + Writer = pipe->CreateAsyncWriter(); + } + + void TearDown() override + { } + + IConnectionReaderPtr Reader; + IConnectionWriterPtr Writer; +}; + +TEST_F(TPipeReadWriteTest, ReadSomethingSpin) +{ + TString message("Hello pipe!\n"); + auto buffer = TSharedRef::FromString(message); + Writer->Write(buffer).Get().ThrowOnError(); + Writer->Close().Get().ThrowOnError(); + + auto data = TSharedMutableRef::Allocate(1); + auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>()); + + while (true) { + auto result = Reader->Read(data).Get(); + if (result.ValueOrThrow() == 0) { + break; + } + whole.Append(data.Begin(), result.Value()); + } + + EXPECT_EQ(message, TString(whole.Begin(), whole.End())); +} + +TEST_F(TNamedPipeReadWriteTest, ReadSomethingSpin) +{ + TString message("Hello pipe!\n"); + auto buffer = TSharedRef::FromString(message); + + Writer->Write(buffer).Get().ThrowOnError(); + Writer->Close().Get().ThrowOnError(); + + auto data = TSharedMutableRef::Allocate(1); + auto whole = TBlob(GetRefCountedTypeCookie<TDefaultBlobTag>()); + + while (true) { + auto result = Reader->Read(data).Get(); + if (result.ValueOrThrow() == 0) { + break; + } + whole.Append(data.Begin(), result.Value()); + } + EXPECT_EQ(message, TString(whole.Begin(), whole.End())); +} + + +TEST_F(TPipeReadWriteTest, ReadSomethingWait) +{ + TString message("Hello pipe!\n"); + auto buffer = TSharedRef::FromString(message); + EXPECT_TRUE(Writer->Write(buffer).Get().IsOK()); + WaitFor(Writer->Close()) + .ThrowOnError(); + auto whole = ReadAll(Reader, false); + EXPECT_EQ(message, TString(whole.Begin(), whole.End())); +} + +TEST_F(TNamedPipeReadWriteTest, ReadSomethingWait) +{ + TString message("Hello pipe!\n"); + auto buffer = TSharedRef::FromString(message); + EXPECT_TRUE(Writer->Write(buffer).Get().IsOK()); + WaitFor(Writer->Close()) + .ThrowOnError(); + auto whole = ReadAll(Reader, false); + EXPECT_EQ(message, TString(whole.Begin(), whole.End())); +} + +TEST_F(TPipeReadWriteTest, ReadWrite) +{ + TString text("Hello cruel world!\n"); + auto buffer = TSharedRef::FromString(text); + Writer->Write(buffer).Get(); + auto errorsOnClose = Writer->Close(); + + auto textFromPipe = ReadAll(Reader, false); + + auto error = errorsOnClose.Get(); + EXPECT_TRUE(error.IsOK()) << error.GetMessage(); + EXPECT_EQ(text, TString(textFromPipe.Begin(), textFromPipe.End())); +} + +TEST_F(TNamedPipeReadWriteTest, ReadWrite) +{ + TString text("Hello cruel world!\n"); + auto buffer = TSharedRef::FromString(text); + Writer->Write(buffer).Get(); + auto errorsOnClose = Writer->Close(); + + auto textFromPipe = ReadAll(Reader, false); + + auto error = errorsOnClose.Get(); + EXPECT_TRUE(error.IsOK()) << error.GetMessage(); + EXPECT_EQ(text, TString(textFromPipe.Begin(), textFromPipe.End())); +} + +void WriteAll(IConnectionWriterPtr writer, const char* data, size_t size, size_t blockSize) +{ + while (size > 0) { + const size_t currentBlockSize = std::min(blockSize, size); + auto buffer = TSharedRef(data, currentBlockSize, nullptr); + auto error = WaitFor(writer->Write(buffer)); + THROW_ERROR_EXCEPTION_IF_FAILED(error); + size -= currentBlockSize; + data += currentBlockSize; + } + + { + auto error = WaitFor(writer->Close()); + THROW_ERROR_EXCEPTION_IF_FAILED(error); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TPipeBigReadWriteTest + : public TPipeReadWriteTest + , public ::testing::WithParamInterface<std::pair<size_t, size_t>> +{ }; + +TEST_P(TPipeBigReadWriteTest, RealReadWrite) +{ + size_t dataSize, blockSize; + std::tie(dataSize, blockSize) = GetParam(); + + auto queue = New<NConcurrency::TActionQueue>(); + + std::vector<char> data(dataSize, 'a'); + + BIND([&] () { + auto dice = std::bind( + std::uniform_int_distribution<int>(0, 127), + std::default_random_engine()); + for (size_t i = 0; i < data.size(); ++i) { + data[i] = dice(); + } + }) + .AsyncVia(queue->GetInvoker()).Run(); + + auto writeError = BIND(&WriteAll, Writer, data.data(), data.size(), blockSize) + .AsyncVia(queue->GetInvoker()) + .Run(); + auto readFromPipe = BIND(&ReadAll, Reader, true) + .AsyncVia(queue->GetInvoker()) + .Run(); + + auto textFromPipe = readFromPipe.Get().ValueOrThrow(); + EXPECT_EQ(data.size(), textFromPipe.Size()); + auto result = std::mismatch(textFromPipe.Begin(), textFromPipe.End(), data.begin()); + EXPECT_TRUE(std::equal(textFromPipe.Begin(), textFromPipe.End(), data.begin())) << + (result.first - textFromPipe.Begin()) << " " << (int)(*result.first); +} + +INSTANTIATE_TEST_SUITE_P( + ValueParametrized, + TPipeBigReadWriteTest, + ::testing::Values( + std::make_pair(2000 * 4096, 4096), + std::make_pair(100 * 4096, 10000), + std::make_pair(100 * 4096, 100), + std::make_pair(100, 4096))); + +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NPipes diff --git a/yt/yt/library/process/unittests/process_ut.cpp b/yt/yt/library/process/unittests/process_ut.cpp new file mode 100644 index 0000000000..55f7fc65c8 --- /dev/null +++ b/yt/yt/library/process/unittests/process_ut.cpp @@ -0,0 +1,235 @@ +#include <yt/yt/library/process/process.h> + +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/actions/bind.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/delayed_executor.h> +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/net/connection.h> + +#include <library/cpp/yt/system/handle_eintr.h> + +namespace NYT { +namespace { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +#if defined(_unix_) and not defined(_asan_enabled_) + +TEST(TProcessTest, Basic) +{ + auto p = New<TSimpleProcess>("/bin/ls"); + TFuture<void> finished; + + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); +} + +// NB: We cannot rely on 'ls' and 'sleep' in arcadia tests. +TEST(TProcessTest, RunFromPathEnv) +{ + auto p = New<TSimpleProcess>("/bin/ls", false); + TFuture<void> finished; + + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, PollDuration) +{ + auto p = New<TSimpleProcess>("/bin/sleep", true, TDuration::MilliSeconds(1)); + p->AddArgument("0.1"); + + auto error = WaitFor(p->Spawn()); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, InvalidPath) +{ + auto p = New<TSimpleProcess>("/some/bad/path/binary"); + + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_FALSE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_FALSE(p->IsFinished()); + EXPECT_FALSE(error.IsOK()); +} + +TEST(TProcessTest, StdOut) +{ + auto p = New<TSimpleProcess>("/bin/date"); + + auto outStream = p->GetStdOutReader(); + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); + + auto buffer = TSharedMutableRef::Allocate(4_KB, {.InitializeStorage = false}); + auto future = outStream->Read(buffer); + auto result = WaitFor(future); + size_t sz = result.ValueOrThrow(); + EXPECT_TRUE(sz > 0); +} + +TEST(TSimpleProcess, GetCommandLine1) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + EXPECT_EQ("/bin/bash", p->GetCommandLine()); + p->AddArgument("-c"); + EXPECT_EQ("/bin/bash -c", p->GetCommandLine()); + p->AddArgument("exit 0"); + EXPECT_EQ("/bin/bash -c \"exit 0\"", p->GetCommandLine()); +} + +TEST(TProcessBase, GetCommandLine2) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + EXPECT_EQ("/bin/bash", p->GetCommandLine()); + p->AddArgument("-c"); + EXPECT_EQ("/bin/bash -c", p->GetCommandLine()); + p->AddArgument("\"quoted\""); + EXPECT_EQ("/bin/bash -c \"\\\"quoted\\\"\"", p->GetCommandLine()); +} + +TEST(TProcessTest, ProcessReturnCode0) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("exit 0"); + + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, ProcessReturnCode123) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("exit 123"); + + TFuture<void> finished; + ASSERT_NO_THROW(finished = p->Spawn()); + ASSERT_TRUE(p->IsStarted()); + auto error = WaitFor(finished); + EXPECT_EQ(EProcessErrorCode::NonZeroExitCode, error.GetCode()); + EXPECT_EQ(123, error.Attributes().Get<int>("exit_code")); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, Params1) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("if test 3 -gt 1; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_FALSE(error.IsOK()); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, Params2) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("if test 1 -gt 3; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_TRUE(error.IsOK()) << ToString(error); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, InheritEnvironment) +{ + const char* name = "SPAWN_TEST_ENV_VAR"; + const char* value = "42"; + setenv(name, value, 1); + + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("if test $SPAWN_TEST_ENV_VAR = 42; then exit 7; fi"); + + auto error = WaitFor(p->Spawn()); + EXPECT_FALSE(error.IsOK()); + EXPECT_TRUE(p->IsFinished()); + + unsetenv(name); +} + +TEST(TProcessTest, Kill) +{ + auto p = New<TSimpleProcess>("/bin/sleep"); + p->AddArgument("5"); + + auto finished = p->Spawn(); + + NConcurrency::TDelayedExecutor::Submit( + BIND([&] () { + p->Kill(SIGKILL); + }), + TDuration::MilliSeconds(100)); + + auto error = WaitFor(finished); + EXPECT_FALSE(error.IsOK()); + EXPECT_TRUE(p->IsFinished()); +} + +TEST(TProcessTest, KillFinished) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("true"); + + auto finished = p->Spawn(); + + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()); + + p->Kill(SIGKILL); +} + +TEST(TProcessTest, KillZombie) +{ + auto p = New<TSimpleProcess>("/bin/bash"); + p->AddArgument("-c"); + p->AddArgument("/bin/sleep 1; /bin/true"); + + auto finished = p->Spawn(); + + siginfo_t infop; + auto res = HandleEintr(::waitid, P_PID, p->GetProcessId(), &infop, WEXITED | WNOWAIT); + EXPECT_EQ(0, res) + << "errno = " << errno; + EXPECT_EQ(p->GetProcessId(), infop.si_pid); + + p->Kill(SIGKILL); + auto error = WaitFor(finished); + EXPECT_TRUE(error.IsOK()) + << ToString(error); +} + +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/yt/yt/library/process/unittests/subprocess_ut.cpp b/yt/yt/library/process/unittests/subprocess_ut.cpp new file mode 100644 index 0000000000..932d250784 --- /dev/null +++ b/yt/yt/library/process/unittests/subprocess_ut.cpp @@ -0,0 +1,101 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/concurrency/action_queue.h> + +#include <yt/yt/library/process/subprocess.h> + +namespace NYT { +namespace { + +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +#if defined(_unix_) and not defined(_asan_enabled_) + +TEST(TSubprocessTest, Basic) +{ + TSubprocess subprocess("/bin/bash"); + + subprocess.AddArgument("-c"); + subprocess.AddArgument("true"); + + auto result = subprocess.Execute(); + EXPECT_TRUE(result.Status.IsOK()); +} + + +TEST(TSubprocessTest, PipeOutput) +{ + TSubprocess subprocess("/bin/echo"); + + subprocess.AddArgument("hello"); + + auto result = subprocess.Execute(); + EXPECT_TRUE(result.Status.IsOK()); + TString output(result.Output.Begin(), result.Output.End()); + EXPECT_TRUE(output == "hello\n") << output; +} + +TEST(TSubprocessTest, PipeStdin) +{ + auto queue = New<TActionQueue>(); + + BIND([] () { + TSubprocess subprocess("/bin/cat"); + subprocess.AddArgument("-"); + + auto input = TString("TEST test TEST"); + auto inputRef = TSharedRef::FromString(input); + auto result = subprocess.Execute(inputRef); + EXPECT_TRUE(result.Status.IsOK()); + + TString output(result.Output.Begin(), result.Output.End()); + EXPECT_EQ(input, output); + }).AsyncVia(queue->GetInvoker()).Run().Get().ThrowOnError(); +} + +TEST(TSubprocessTest, PipeBigOutput) +{ + auto queue = New<TActionQueue>(); + + auto result = BIND([] () { + TSubprocess subprocess("/bin/bash"); + + subprocess.AddArgument("-c"); + subprocess.AddArgument("for i in `/usr/bin/seq 100000`; do echo hello; done; echo world"); + + auto result = subprocess.Execute(); + return result.Status.IsOK(); + }).AsyncVia(queue->GetInvoker()).Run().Get().Value(); + + EXPECT_TRUE(result); +} + + +TEST(TSubprocessTest, PipeBigError) +{ + auto queue = New<TActionQueue>(); + + auto result = BIND([] () { + TSubprocess subprocess("/bin/bash"); + + subprocess.AddArgument("-c"); + subprocess.AddArgument("for i in `/usr/bin/seq 100000`; do echo hello 1>&2; done; echo world"); + + auto result = subprocess.Execute(); + return result; + }).AsyncVia(queue->GetInvoker()).Run().Get().Value(); + + EXPECT_TRUE(result.Status.IsOK()); + EXPECT_EQ(6*100000, std::ssize(result.Error)); +} + +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/yt/yt/library/process/unittests/ya.make b/yt/yt/library/process/unittests/ya.make new file mode 100644 index 0000000000..6e476c5702 --- /dev/null +++ b/yt/yt/library/process/unittests/ya.make @@ -0,0 +1,24 @@ +GTEST(unittester-library-process) + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +ALLOCATOR(YT) + +SRCS( + pipes_ut.cpp + process_ut.cpp + subprocess_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + yt/yt/build + yt/yt/core + yt/yt/core/test_framework + yt/yt/library/process +) + +SIZE(MEDIUM) + +END() diff --git a/yt/yt/library/process/ya.make b/yt/yt/library/process/ya.make new file mode 100644 index 0000000000..79763c7267 --- /dev/null +++ b/yt/yt/library/process/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + io_dispatcher.cpp + pipe.cpp + process.cpp + pty.cpp + subprocess.cpp +) + +PEERDIR( + yt/yt/core + contrib/libs/re2 +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt index bafd86d310..0f1318b348 100644 --- a/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt @@ -6,7 +6,10 @@ # original buildsystem will not be accepted. +add_subdirectory(perf) add_subdirectory(resource_tracker) +add_subdirectory(solomon) +add_subdirectory(tcmalloc) add_library(yt-library-profiling) target_compile_options(yt-library-profiling PRIVATE diff --git a/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt index e4524762fc..b1ff65e70f 100644 --- a/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/profiling/CMakeLists.linux-aarch64.txt @@ -6,7 +6,10 @@ # original buildsystem will not be accepted. +add_subdirectory(perf) add_subdirectory(resource_tracker) +add_subdirectory(solomon) +add_subdirectory(tcmalloc) add_library(yt-library-profiling) target_compile_options(yt-library-profiling PRIVATE diff --git a/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt index e4524762fc..b1ff65e70f 100644 --- a/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/profiling/CMakeLists.linux-x86_64.txt @@ -6,7 +6,10 @@ # original buildsystem will not be accepted. +add_subdirectory(perf) add_subdirectory(resource_tracker) +add_subdirectory(solomon) +add_subdirectory(tcmalloc) add_library(yt-library-profiling) target_compile_options(yt-library-profiling PRIVATE diff --git a/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt index 6a6f1a6dde..34e12beaad 100644 --- a/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/profiling/CMakeLists.windows-x86_64.txt @@ -6,7 +6,10 @@ # original buildsystem will not be accepted. +add_subdirectory(perf) add_subdirectory(resource_tracker) +add_subdirectory(solomon) +add_subdirectory(tcmalloc) add_library(yt-library-profiling) target_link_libraries(yt-library-profiling PUBLIC diff --git a/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..fd65c86abb --- /dev/null +++ b/yt/yt/library/profiling/perf/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-perf) +target_compile_options(library-profiling-perf PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-perf PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core +) +target_sources(library-profiling-perf PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters_other.cpp +) diff --git a/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..8615b211d9 --- /dev/null +++ b/yt/yt/library/profiling/perf/CMakeLists.linux-aarch64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-perf) +target_compile_options(library-profiling-perf PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-perf PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core +) +target_sources(library-profiling-perf PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters.cpp +) diff --git a/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..8615b211d9 --- /dev/null +++ b/yt/yt/library/profiling/perf/CMakeLists.linux-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-perf) +target_compile_options(library-profiling-perf PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-perf PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core +) +target_sources(library-profiling-perf PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters.cpp +) diff --git a/yt/yt/library/profiling/perf/CMakeLists.txt b/yt/yt/library/profiling/perf/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/profiling/perf/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..f6897f44de --- /dev/null +++ b/yt/yt/library/profiling/perf/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-perf) +target_link_libraries(library-profiling-perf PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core +) +target_sources(library-profiling-perf PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/perf/counters_other.cpp +) diff --git a/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..b4c6b378dd --- /dev/null +++ b/yt/yt/library/profiling/solomon/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,68 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-profiling-solomon) +target_compile_options(library-profiling-solomon PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-solomon PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core + yt-core-http + library-cpp-cgiparam + cpp-monlib-metrics + monlib-encode-prometheus + monlib-encode-spack + monlib-encode-json + cpp-yt-threading + contrib-libs-protobuf +) +target_proto_messages(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto +) +target_sources(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp +) +target_proto_addincls(library-profiling-solomon + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-profiling-solomon + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..57e8fb3c6e --- /dev/null +++ b/yt/yt/library/profiling/solomon/CMakeLists.linux-aarch64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-profiling-solomon) +target_compile_options(library-profiling-solomon PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-solomon PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core + yt-core-http + library-cpp-cgiparam + cpp-monlib-metrics + monlib-encode-prometheus + monlib-encode-spack + monlib-encode-json + cpp-yt-threading + contrib-libs-protobuf +) +target_proto_messages(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto +) +target_sources(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp +) +target_proto_addincls(library-profiling-solomon + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-profiling-solomon + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..57e8fb3c6e --- /dev/null +++ b/yt/yt/library/profiling/solomon/CMakeLists.linux-x86_64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-profiling-solomon) +target_compile_options(library-profiling-solomon PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-solomon PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core + yt-core-http + library-cpp-cgiparam + cpp-monlib-metrics + monlib-encode-prometheus + monlib-encode-spack + monlib-encode-json + cpp-yt-threading + contrib-libs-protobuf +) +target_proto_messages(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto +) +target_sources(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp +) +target_proto_addincls(library-profiling-solomon + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-profiling-solomon + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/yt/yt/library/profiling/solomon/CMakeLists.txt b/yt/yt/library/profiling/solomon/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/profiling/solomon/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..958f7e64fc --- /dev/null +++ b/yt/yt/library/profiling/solomon/CMakeLists.windows-x86_64.txt @@ -0,0 +1,65 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-profiling-solomon) +target_link_libraries(library-profiling-solomon PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + yt-yt-core + yt-core-http + library-cpp-cgiparam + cpp-monlib-metrics + monlib-encode-prometheus + monlib-encode-spack + monlib-encode-json + cpp-yt-threading + contrib-libs-protobuf +) +target_proto_messages(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_dump.proto +) +target_sources(library-profiling-solomon PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/cube.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/exporter.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/percpu.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/producer.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/registry.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/remote.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_service.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/sensor_set.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/solomon/tag_registry.cpp +) +target_proto_addincls(library-profiling-solomon + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-profiling-solomon + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..341ff6724c --- /dev/null +++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-tcmalloc) +target_compile_options(library-profiling-tcmalloc PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-tcmalloc PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + libs-tcmalloc-malloc_extension +) +target_sources(library-profiling-tcmalloc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp +) diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..3ebf7b2fd2 --- /dev/null +++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-aarch64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-tcmalloc) +target_compile_options(library-profiling-tcmalloc PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-tcmalloc PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + libs-tcmalloc-malloc_extension +) +target_sources(library-profiling-tcmalloc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp +) diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..3ebf7b2fd2 --- /dev/null +++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.linux-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-tcmalloc) +target_compile_options(library-profiling-tcmalloc PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-profiling-tcmalloc PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-profiling + libs-tcmalloc-malloc_extension +) +target_sources(library-profiling-tcmalloc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp +) diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt b/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..3fcdf372c8 --- /dev/null +++ b/yt/yt/library/profiling/tcmalloc/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-profiling-tcmalloc) +target_link_libraries(library-profiling-tcmalloc PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-profiling + libs-tcmalloc-malloc_extension +) +target_sources(library-profiling-tcmalloc PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/profiling/tcmalloc/profiler.cpp +) diff --git a/yt/yt/library/program/CMakeLists.darwin-x86_64.txt b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..fc3e796e79 --- /dev/null +++ b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-program) +target_compile_options(yt-library-program PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-program PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + core-service_discovery-yp + yt-library-monitoring + yt-library-containers + library-profiling-solomon + library-profiling-tcmalloc + library-profiling-perf + yt-library-ytprof + library-tracing-jaeger + cpp-yt-mlock + cpp-yt-stockpile + cpp-yt-string +) +target_sources(yt-library-program PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp +) diff --git a/yt/yt/library/program/CMakeLists.linux-aarch64.txt b/yt/yt/library/program/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..be31ba10cf --- /dev/null +++ b/yt/yt/library/program/CMakeLists.linux-aarch64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-program) +target_compile_options(yt-library-program PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-program PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + core-service_discovery-yp + yt-library-monitoring + yt-library-containers + library-profiling-solomon + library-profiling-tcmalloc + library-profiling-perf + yt-library-ytprof + library-tracing-jaeger + cpp-yt-mlock + cpp-yt-stockpile + cpp-yt-string +) +target_sources(yt-library-program PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp +) diff --git a/yt/yt/library/program/CMakeLists.linux-x86_64.txt b/yt/yt/library/program/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..be31ba10cf --- /dev/null +++ b/yt/yt/library/program/CMakeLists.linux-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-program) +target_compile_options(yt-library-program PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-library-program PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-yt-core + core-service_discovery-yp + yt-library-monitoring + yt-library-containers + library-profiling-solomon + library-profiling-tcmalloc + library-profiling-perf + yt-library-ytprof + library-tracing-jaeger + cpp-yt-mlock + cpp-yt-stockpile + cpp-yt-string +) +target_sources(yt-library-program PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp +) diff --git a/yt/yt/library/program/CMakeLists.txt b/yt/yt/library/program/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/program/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/program/CMakeLists.windows-x86_64.txt b/yt/yt/library/program/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..1f2aea4bd0 --- /dev/null +++ b/yt/yt/library/program/CMakeLists.windows-x86_64.txt @@ -0,0 +1,35 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-library-program) +target_link_libraries(yt-library-program PUBLIC + contrib-libs-cxxsupp + yutil + yt-yt-core + core-service_discovery-yp + yt-library-monitoring + yt-library-containers + library-profiling-solomon + library-profiling-tcmalloc + library-profiling-perf + yt-library-ytprof + library-tracing-jaeger + cpp-yt-mlock + cpp-yt-stockpile + cpp-yt-string +) +target_sources(yt-library-program PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/build_attributes.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/config.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/helpers.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_config_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_pdeathsig_mixin.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/program/program_setsid_mixin.cpp +) diff --git a/yt/yt/library/program/build_attributes.cpp b/yt/yt/library/program/build_attributes.cpp new file mode 100644 index 0000000000..38caf57997 --- /dev/null +++ b/yt/yt/library/program/build_attributes.cpp @@ -0,0 +1,107 @@ +#include "build_attributes.h" + +#include <yt/yt/build/build.h> + +#include <yt/yt/core/ytree/fluent.h> +#include <yt/yt/core/ytree/ypath_client.h> + +#include <yt/yt/core/misc/error_code.h> + +namespace NYT { + +using namespace NYTree; +using namespace NYson; + +static const NLogging::TLogger Logger("Build"); + +//////////////////////////////////////////////////////////////////////////////// + +void TBuildInfo::Register(TRegistrar registrar) +{ + registrar.Parameter("name", &TThis::Name) + .Default(); + + registrar.Parameter("version", &TThis::Version) + .Default(GetVersion()); + + registrar.Parameter("build_host", &TThis::BuildHost) + .Default(GetBuildHost()); + + registrar.Parameter("build_time", &TThis::BuildTime) + .Default(ParseBuildTime()); + + registrar.Parameter("start_time", &TThis::StartTime) + .Default(TInstant::Now()); +} + +std::optional<TInstant> TBuildInfo::ParseBuildTime() +{ + TString rawBuildTime(GetBuildTime()); + + // Build time may be empty if code is building + // without -DBUILD_DATE (for example, in opensource build). + if (rawBuildTime.empty()) { + return std::nullopt; + } + + try { + return TInstant::ParseIso8601(rawBuildTime); + } catch (const std::exception& ex) { + YT_LOG_ERROR(ex, "Error parsing build time"); + return std::nullopt; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TBuildInfoPtr BuildBuildAttributes(const char* serviceName) +{ + auto info = New<TBuildInfo>(); + if (serviceName) { + info->Name = serviceName; + } + return info; +} + +void SetBuildAttributes(IYPathServicePtr orchidRoot, const char* serviceName) +{ + SyncYPathSet( + orchidRoot, + "/service", + BuildYsonStringFluently() + .BeginAttributes() + .Item("opaque").Value(true) + .EndAttributes() + .Value(BuildBuildAttributes(serviceName))); + SyncYPathSet( + orchidRoot, + "/error_codes", + BuildYsonStringFluently() + .BeginAttributes() + .Item("opaque").Value(true) + .EndAttributes() + .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodes(), [] (TFluentMap fluent, const auto& pair) { + fluent + .Item(ToString(pair.first)).BeginMap() + .Item("cpp_literal").Value(ToString(pair.second)) + .EndMap(); + })); + SyncYPathSet( + orchidRoot, + "/error_code_ranges", + BuildYsonStringFluently() + .BeginAttributes() + .Item("opaque").Value(true) + .EndAttributes() + .DoMapFor(TErrorCodeRegistry::Get()->GetAllErrorCodeRanges(), [] (TFluentMap fluent, const TErrorCodeRegistry::TErrorCodeRangeInfo& range) { + fluent + .Item(ToString(range)).BeginMap() + .Item("cpp_enum").Value(range.Namespace) + .EndMap(); + })); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + diff --git a/yt/yt/library/program/build_attributes.h b/yt/yt/library/program/build_attributes.h new file mode 100644 index 0000000000..e02f86b351 --- /dev/null +++ b/yt/yt/library/program/build_attributes.h @@ -0,0 +1,44 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/public.h> +#include <yt/yt/core/ytree/yson_struct.h> + +#include <yt/yt/core/yson/public.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TBuildInfo + : public NYTree::TYsonStruct +{ +public: + std::optional<TString> Name; + TString Version; + TString BuildHost; + std::optional<TInstant> BuildTime; + TInstant StartTime; + + REGISTER_YSON_STRUCT(TBuildInfo); + + static void Register(TRegistrar registrar); + +private: + static std::optional<TInstant> ParseBuildTime(); +}; + +DEFINE_REFCOUNTED_TYPE(TBuildInfo) + +//////////////////////////////////////////////////////////////////////////////// + +//! Build build (pun intended) attributes as a TBuildInfo a-la /orchid/service. If service name is not provided, +//! it is omitted from the result. +TBuildInfoPtr BuildBuildAttributes(const char* serviceName = nullptr); + +void SetBuildAttributes(NYTree::IYPathServicePtr orchidRoot, const char* serviceName); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp new file mode 100644 index 0000000000..0705fb48fc --- /dev/null +++ b/yt/yt/library/program/config.cpp @@ -0,0 +1,210 @@ +#include "config.h" + +namespace NYT { + +using namespace NYTree; + +//////////////////////////////////////////////////////////////////////////////// + +void TRpcConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("tracing", &TThis::Tracing) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void THeapSizeLimit::Register(TRegistrar registrar) +{ + registrar.Parameter("container_memory_ratio", &TThis::ContainerMemoryRatio) + .Optional(); + registrar.Parameter("is_hard", &TThis::IsHard) + .Default(false); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TTCMallocConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("background_release_rate", &TThis::BackgroundReleaseRate) + .Default(32_MB); + registrar.Parameter("max_per_cpu_cache_size", &TThis::MaxPerCpuCacheSize) + .Default(3_MB); + + registrar.Parameter("aggressive_release_threshold", &TThis::AggressiveReleaseThreshold) + .Default(20_GB); + registrar.Parameter("aggressive_release_threshold_ratio", &TThis::AggressiveReleaseThresholdRatio) + .Optional(); + + registrar.Parameter("aggressive_release_size", &TThis::AggressiveReleaseSize) + .Default(128_MB); + registrar.Parameter("aggressive_release_period", &TThis::AggressiveReleasePeriod) + .Default(TDuration::MilliSeconds(100)); + registrar.Parameter("guarded_sampling_rate", &TThis::GuardedSamplingRate) + .Default(128_MB); + + registrar.Parameter("heap_size_limit", &TThis::HeapSizeLimit) + .DefaultNew(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TStockpileConfig::Register(TRegistrar registrar) +{ + registrar.BaseClassParameter("buffer_size", &TThis::BufferSize) + .Default(DefaultBufferSize); + registrar.BaseClassParameter("thread_count", &TThis::ThreadCount) + .Default(DefaultThreadCount); + registrar.BaseClassParameter("period", &TThis::Period) + .Default(DefaultPeriod); +} + +//////////////////////////////////////////////////////////////////////////////// + +void THeapProfilerConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("snapshot_update_period", &TThis::SnapshotUpdatePeriod) + .Default(TDuration::Seconds(5)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TSingletonsConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("spin_wait_slow_path_logging_threshold", &TThis::SpinWaitSlowPathLoggingThreshold) + .Default(TDuration::MicroSeconds(100)); + registrar.Parameter("yt_alloc", &TThis::YTAlloc) + .DefaultNew(); + registrar.Parameter("fiber_stack_pool_sizes", &TThis::FiberStackPoolSizes) + .Default({}); + registrar.Parameter("address_resolver", &TThis::AddressResolver) + .DefaultNew(); + registrar.Parameter("tcp_dispatcher", &TThis::TcpDispatcher) + .DefaultNew(); + registrar.Parameter("rpc_dispatcher", &TThis::RpcDispatcher) + .DefaultNew(); + registrar.Parameter("grpc_dispatcher", &TThis::GrpcDispatcher) + .DefaultNew(); + registrar.Parameter("yp_service_discovery", &TThis::YPServiceDiscovery) + .DefaultNew(); + registrar.Parameter("solomon_exporter", &TThis::SolomonExporter) + .DefaultNew(); + registrar.Parameter("logging", &TThis::Logging) + .DefaultCtor([] () { return NLogging::TLogManagerConfig::CreateDefault(); }); + registrar.Parameter("jaeger", &TThis::Jaeger) + .DefaultNew(); + registrar.Parameter("rpc", &TThis::Rpc) + .DefaultNew(); + registrar.Parameter("tcmalloc", &TThis::TCMalloc) + .DefaultNew(); + registrar.Parameter("stockpile", &TThis::Stockpile) + .DefaultNew(); + registrar.Parameter("enable_ref_counted_tracker_profiling", &TThis::EnableRefCountedTrackerProfiling) + .Default(true); + registrar.Parameter("enable_resource_tracker", &TThis::EnableResourceTracker) + .Default(true); + registrar.Parameter("enable_porto_resource_tracker", &TThis::EnablePortoResourceTracker) + .Default(false); + registrar.Parameter("resource_tracker_vcpu_factor", &TThis::ResourceTrackerVCpuFactor) + .Optional(); + registrar.Parameter("pod_spec", &TThis::PodSpec) + .DefaultNew(); + registrar.Parameter("heap_profiler", &TThis::HeapProfiler) + .DefaultNew(); + + registrar.Postprocessor([] (TThis* config) { + if (config->ResourceTrackerVCpuFactor && !config->EnableResourceTracker) { + THROW_ERROR_EXCEPTION("Option \"resource_tracker_vcpu_factor\" can be specified only if resource tracker is enabled"); + } + }); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TSingletonsDynamicConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("spin_lock_slow_path_logging_threshold", &TThis::SpinWaitSlowPathLoggingThreshold) + .Optional(); + registrar.Parameter("yt_alloc", &TThis::YTAlloc) + .Optional(); + registrar.Parameter("tcp_dispatcher", &TThis::TcpDispatcher) + .DefaultNew(); + registrar.Parameter("rpc_dispatcher", &TThis::RpcDispatcher) + .DefaultNew(); + registrar.Parameter("logging", &TThis::Logging) + .DefaultNew(); + registrar.Parameter("jaeger", &TThis::Jaeger) + .DefaultNew(); + registrar.Parameter("rpc", &TThis::Rpc) + .DefaultNew(); + registrar.Parameter("tcmalloc", &TThis::TCMalloc) + .Optional(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDiagnosticDumpConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("yt_alloc_dump_period", &TThis::YTAllocDumpPeriod) + .Default(); + registrar.Parameter("ref_counted_tracker_dump_period", &TThis::RefCountedTrackerDumpPeriod) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void WarnForUnrecognizedOptionsImpl( + const NLogging::TLogger& logger, + const IMapNodePtr& unrecognized) +{ + const auto& Logger = logger; + if (unrecognized && unrecognized->GetChildCount() > 0) { + YT_LOG_WARNING("Bootstrap config contains unrecognized options (Unrecognized: %v)", + ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text)); + } +} + +void WarnForUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonStructPtr& config) +{ + WarnForUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized()); +} + +void WarnForUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonSerializablePtr& config) +{ + WarnForUnrecognizedOptionsImpl(logger, config->GetUnrecognizedRecursively()); +} + +void AbortOnUnrecognizedOptionsImpl( + const NLogging::TLogger& logger, + const IMapNodePtr& unrecognized) +{ + const auto& Logger = logger; + if (unrecognized && unrecognized->GetChildCount() > 0) { + YT_LOG_ERROR("Bootstrap config contains unrecognized options, terminating (Unrecognized: %v)", + ConvertToYsonString(unrecognized, NYson::EYsonFormat::Text)); + YT_ABORT(); + } +} + +void AbortOnUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonStructPtr& config) +{ + AbortOnUnrecognizedOptionsImpl(logger, config->GetRecursiveUnrecognized()); +} + +void AbortOnUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonSerializablePtr& config) +{ + AbortOnUnrecognizedOptionsImpl(logger, config->GetUnrecognizedRecursively()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h new file mode 100644 index 0000000000..7d92939f1c --- /dev/null +++ b/yt/yt/library/program/config.h @@ -0,0 +1,224 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/yson_serializable.h> +#include <yt/yt/core/ytree/yson_struct.h> + +#include <yt/yt/core/ytalloc/config.h> + +#include <yt/yt/core/net/config.h> + +#include <yt/yt/core/rpc/config.h> +#include <yt/yt/core/rpc/grpc/config.h> + +#include <yt/yt/core/bus/tcp/config.h> + +#include <yt/yt/core/logging/config.h> + +#include <yt/yt/core/tracing/config.h> + +#include <yt/yt/core/service_discovery/yp/config.h> + +#include <yt/yt/library/profiling/solomon/exporter.h> + +#include <yt/yt/library/containers/config.h> + +#include <yt/yt/library/tracing/jaeger/tracer.h> + +#include <library/cpp/yt/stockpile/stockpile.h> + + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TRpcConfig + : public NYTree::TYsonStruct +{ +public: + NTracing::TTracingConfigPtr Tracing; + + REGISTER_YSON_STRUCT(TRpcConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TRpcConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class THeapSizeLimit + : public virtual NYTree::TYsonStruct +{ +public: + //! Limit program memory in terms of container memory. + // If program heap size exceeds the limit tcmalloc is instructed to release memory to the kernel. + std::optional<double> ContainerMemoryRatio; + + //! If true tcmalloc crashes when system allocates more memory than #ContainerMemoryRatio. + bool IsHard; + + REGISTER_YSON_STRUCT(THeapSizeLimit); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(THeapSizeLimit) + +//////////////////////////////////////////////////////////////////////////////// + +class TTCMallocConfig + : public virtual NYTree::TYsonStruct +{ +public: + i64 BackgroundReleaseRate; + int MaxPerCpuCacheSize; + + //! Threshold in bytes + i64 AggressiveReleaseThreshold; + + //! Threshold in fractions of total memory of the container + std::optional<double> AggressiveReleaseThresholdRatio; + + i64 AggressiveReleaseSize; + TDuration AggressiveReleasePeriod; + + //! Approximately 1/#GuardedSamplingRate of all allocations of + //! size <= 256 KiB will be under GWP-ASAN. + std::optional<i64> GuardedSamplingRate; + + THeapSizeLimitPtr HeapSizeLimit; + + REGISTER_YSON_STRUCT(TTCMallocConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TTCMallocConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class TStockpileConfig + : public TStockpileOptions + , public NYTree::TYsonStruct +{ +public: + REGISTER_YSON_STRUCT(TStockpileConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TStockpileConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class THeapProfilerConfig + : public NYTree::TYsonStruct +{ +public: + // Period of update snapshot in heap profiler. + std::optional<TDuration> SnapshotUpdatePeriod; + + REGISTER_YSON_STRUCT(THeapProfilerConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(THeapProfilerConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class TSingletonsConfig + : public virtual NYTree::TYsonStruct +{ +public: + TDuration SpinWaitSlowPathLoggingThreshold; + NYTAlloc::TYTAllocConfigPtr YTAlloc; + THashMap<TString, int> FiberStackPoolSizes; + NNet::TAddressResolverConfigPtr AddressResolver; + NBus::TTcpDispatcherConfigPtr TcpDispatcher; + NRpc::TDispatcherConfigPtr RpcDispatcher; + NRpc::NGrpc::TDispatcherConfigPtr GrpcDispatcher; + NServiceDiscovery::NYP::TServiceDiscoveryConfigPtr YPServiceDiscovery; + NProfiling::TSolomonExporterConfigPtr SolomonExporter; + NLogging::TLogManagerConfigPtr Logging; + NTracing::TJaegerTracerConfigPtr Jaeger; + TRpcConfigPtr Rpc; + TTCMallocConfigPtr TCMalloc; + TStockpileConfigPtr Stockpile; + bool EnableRefCountedTrackerProfiling; + bool EnableResourceTracker; + bool EnablePortoResourceTracker; + std::optional<double> ResourceTrackerVCpuFactor; + NContainers::TPodSpecConfigPtr PodSpec; + THeapProfilerConfigPtr HeapProfiler; + + REGISTER_YSON_STRUCT(TSingletonsConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TSingletonsConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class TSingletonsDynamicConfig + : public virtual NYTree::TYsonStruct +{ +public: + std::optional<TDuration> SpinWaitSlowPathLoggingThreshold; + NYTAlloc::TYTAllocConfigPtr YTAlloc; + NBus::TTcpDispatcherDynamicConfigPtr TcpDispatcher; + NRpc::TDispatcherDynamicConfigPtr RpcDispatcher; + NLogging::TLogManagerDynamicConfigPtr Logging; + NTracing::TJaegerTracerDynamicConfigPtr Jaeger; + TRpcConfigPtr Rpc; + TTCMallocConfigPtr TCMalloc; + + REGISTER_YSON_STRUCT(TSingletonsDynamicConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TSingletonsDynamicConfig) + +//////////////////////////////////////////////////////////////////////////////// + +class TDiagnosticDumpConfig + : public virtual NYTree::TYsonStruct +{ +public: + std::optional<TDuration> YTAllocDumpPeriod; + std::optional<TDuration> RefCountedTrackerDumpPeriod; + + REGISTER_YSON_STRUCT(TDiagnosticDumpConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TDiagnosticDumpConfig) + +//////////////////////////////////////////////////////////////////////////////// + +// NB: These functions should not be called from bootstrap +// config validator since logger is not set up yet. +void WarnForUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonStructPtr& config); + +void WarnForUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonSerializablePtr& config); + +void AbortOnUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonStructPtr& config); + +void AbortOnUnrecognizedOptions( + const NLogging::TLogger& logger, + const NYTree::TYsonSerializablePtr& config); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp new file mode 100644 index 0000000000..5c7ff29db1 --- /dev/null +++ b/yt/yt/library/program/helpers.cpp @@ -0,0 +1,335 @@ +#include "helpers.h" +#include "config.h" +#include "private.h" + +#include <yt/yt/core/ytalloc/bindings.h> + +#include <yt/yt/core/misc/lazy_ptr.h> +#include <yt/yt/core/misc/ref_counted_tracker.h> +#include <yt/yt/core/misc/ref_counted_tracker_profiler.h> + +#include <yt/yt/core/bus/tcp/dispatcher.h> + +#include <yt/yt/library/tracing/jaeger/tracer.h> + +#include <yt/yt/library/profiling/perf/counters.h> + +#include <yt/yt/library/profiling/resource_tracker/resource_tracker.h> + +#include <yt/yt/library/containers/config.h> +#include <yt/yt/library/containers/porto_resource_tracker.h> + +#include <yt/yt/core/logging/log_manager.h> + +#include <yt/yt/core/concurrency/execution_stack.h> +#include <yt/yt/core/concurrency/periodic_executor.h> +#include <yt/yt/core/concurrency/private.h> + +#include <tcmalloc/malloc_extension.h> + +#include <yt/yt/core/net/address.h> +#include <yt/yt/core/net/local_address.h> + +#include <yt/yt/core/rpc/dispatcher.h> +#include <yt/yt/core/rpc/grpc/dispatcher.h> + +#include <yt/yt/core/service_discovery/yp/service_discovery.h> + +#include <yt/yt/core/threading/spin_wait_slow_path_logger.h> + +#include <library/cpp/yt/threading/spin_wait_hook.h> + +#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> + +#include <util/string/split.h> +#include <util/system/thread.h> + +#include <mutex> +#include <thread> + +namespace NYT { + +using namespace NConcurrency; +using namespace NThreading; + +//////////////////////////////////////////////////////////////////////////////// + +static std::once_flag InitAggressiveReleaseThread; +static auto& Logger = ProgramLogger; + +//////////////////////////////////////////////////////////////////////////////// + +class TCMallocLimitsAdjuster +{ +public: + void Adjust(const TTCMallocConfigPtr& config) + { + i64 totalMemory = GetContainerMemoryLimit(); + AdjustPageHeapLimit(totalMemory, config); + AdjustAggressiveReleaseThreshold(totalMemory, config); + } + + i64 GetAggressiveReleaseThreshold() + { + return AggressiveReleaseThreshold_; + } + +private: + using TAllocatorMemoryLimit = tcmalloc::MallocExtension::MemoryLimit; + + TAllocatorMemoryLimit AppliedLimit_; + i64 AggressiveReleaseThreshold_ = 0; + + + void AdjustPageHeapLimit(i64 totalMemory, const TTCMallocConfigPtr& config) + { + auto proposed = ProposeHeapMemoryLimit(totalMemory, config); + + if (proposed.limit == AppliedLimit_.limit && proposed.hard == AppliedLimit_.hard) { + // Already applied + return; + } + + YT_LOG_INFO("Changing tcmalloc memory limit (Limit: %v, IsHard: %v)", + proposed.limit, + proposed.hard); + + tcmalloc::MallocExtension::SetMemoryLimit(proposed); + AppliedLimit_ = proposed; + } + + void AdjustAggressiveReleaseThreshold(i64 totalMemory, const TTCMallocConfigPtr& config) + { + if (totalMemory && config->AggressiveReleaseThresholdRatio) { + AggressiveReleaseThreshold_ = *config->AggressiveReleaseThresholdRatio * totalMemory; + } else { + AggressiveReleaseThreshold_ = config->AggressiveReleaseThreshold; + } + } + + i64 GetContainerMemoryLimit() const + { + auto resourceTracker = NProfiling::GetResourceTracker(); + if (!resourceTracker) { + return 0; + } + + return resourceTracker->GetTotalMemoryLimit(); + } + + TAllocatorMemoryLimit ProposeHeapMemoryLimit(i64 totalMemory, const TTCMallocConfigPtr& config) const + { + const auto& heapLimitConfig = config->HeapSizeLimit; + + if (totalMemory == 0 || !heapLimitConfig->ContainerMemoryRatio) { + return {}; + } + + TAllocatorMemoryLimit proposed; + proposed.limit = *heapLimitConfig->ContainerMemoryRatio * totalMemory; + proposed.hard = heapLimitConfig->IsHard; + + return proposed; + } +}; + +void ConfigureTCMalloc(const TTCMallocConfigPtr& config) +{ + tcmalloc::MallocExtension::SetBackgroundReleaseRate( + tcmalloc::MallocExtension::BytesPerSecond{static_cast<size_t>(config->BackgroundReleaseRate)}); + + tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(config->MaxPerCpuCacheSize); + + if (config->GuardedSamplingRate) { + tcmalloc::MallocExtension::SetGuardedSamplingRate(*config->GuardedSamplingRate); + tcmalloc::MallocExtension::ActivateGuardedSampling(); + } + + struct TConfigSingleton + { + TAtomicIntrusivePtr<TTCMallocConfig> Config; + }; + + LeakySingleton<TConfigSingleton>()->Config.Store(config); + + if (tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) { + std::call_once(InitAggressiveReleaseThread, [] { + std::thread([] { + ::TThread::SetCurrentThreadName("TCAllocYT"); + + TCMallocLimitsAdjuster limitsAdjuster; + + while (true) { + auto config = LeakySingleton<TConfigSingleton>()->Config.Acquire(); + limitsAdjuster.Adjust(config); + + auto freeBytes = tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.page_heap_free"); + YT_VERIFY(freeBytes); + + if (static_cast<i64>(*freeBytes) > limitsAdjuster.GetAggressiveReleaseThreshold()) { + + YT_LOG_DEBUG("Aggressively releasing memory (FreeBytes: %v, Threshold: %v)", + static_cast<i64>(*freeBytes), + limitsAdjuster.GetAggressiveReleaseThreshold()); + + tcmalloc::MallocExtension::ReleaseMemoryToSystem(config->AggressiveReleaseSize); + } + + Sleep(config->AggressiveReleasePeriod); + } + }).detach(); + }); + } +} + +template <class TConfig> +void ConfigureSingletonsImpl(const TConfig& config) +{ + SetSpinWaitSlowPathLoggingThreshold(config->SpinWaitSlowPathLoggingThreshold); + + if (!NYTAlloc::ConfigureFromEnv()) { + NYTAlloc::Configure(config->YTAlloc); + } + + for (const auto& [kind, size] : config->FiberStackPoolSizes) { + NConcurrency::SetFiberStackPoolSize(ParseEnum<NConcurrency::EExecutionStackKind>(kind), size); + } + + NLogging::TLogManager::Get()->EnableReopenOnSighup(); + if (!NLogging::TLogManager::Get()->IsConfiguredFromEnv()) { + NLogging::TLogManager::Get()->Configure(config->Logging); + } + + NNet::TAddressResolver::Get()->Configure(config->AddressResolver); + // By default, server components must have a reasonable FQDN. + // Failure to do so may result in issues like YT-4561. + NNet::TAddressResolver::Get()->EnsureLocalHostName(); + + NBus::TTcpDispatcher::Get()->Configure(config->TcpDispatcher); + + NRpc::TDispatcher::Get()->Configure(config->RpcDispatcher); + + NRpc::NGrpc::TDispatcher::Get()->Configure(config->GrpcDispatcher); + + NRpc::TDispatcher::Get()->SetServiceDiscovery( + NServiceDiscovery::NYP::CreateServiceDiscovery(config->YPServiceDiscovery)); + + NTracing::SetGlobalTracer(New<NTracing::TJaegerTracer>(config->Jaeger)); + + NProfiling::EnablePerfCounters(); + + if (auto tracingConfig = config->Rpc->Tracing) { + NTracing::SetTracingConfig(tracingConfig); + } + + ConfigureTCMalloc(config->TCMalloc); + + ConfigureStockpile(*config->Stockpile); + + if (config->EnableRefCountedTrackerProfiling) { + EnableRefCountedTrackerProfiling(); + } + + if (config->EnableResourceTracker) { + NProfiling::EnableResourceTracker(); + if (config->ResourceTrackerVCpuFactor.has_value()) { + NProfiling::SetVCpuFactor(config->ResourceTrackerVCpuFactor.value()); + } + } + + if (config->EnablePortoResourceTracker) { + NContainers::EnablePortoResourceTracker(config->PodSpec); + } +} + +void ConfigureSingletons(const TSingletonsConfigPtr& config) +{ + ConfigureSingletonsImpl(config); +} + +template <class TStaticConfig, class TDynamicConfig> +void ReconfigureSingletonsImpl(const TStaticConfig& config, const TDynamicConfig& dynamicConfig) +{ + SetSpinWaitSlowPathLoggingThreshold(dynamicConfig->SpinWaitSlowPathLoggingThreshold.value_or(config->SpinWaitSlowPathLoggingThreshold)); + + if (!NYTAlloc::IsConfiguredFromEnv()) { + NYTAlloc::Configure(dynamicConfig->YTAlloc ? dynamicConfig->YTAlloc : config->YTAlloc); + } + + if (!NLogging::TLogManager::Get()->IsConfiguredFromEnv()) { + NLogging::TLogManager::Get()->Configure( + config->Logging->ApplyDynamic(dynamicConfig->Logging), + /*sync*/ false); + } + + auto tracer = NTracing::GetGlobalTracer(); + if (auto jaeger = DynamicPointerCast<NTracing::TJaegerTracer>(tracer); jaeger) { + jaeger->Configure(config->Jaeger->ApplyDynamic(dynamicConfig->Jaeger)); + } + + NBus::TTcpDispatcher::Get()->Configure(config->TcpDispatcher->ApplyDynamic(dynamicConfig->TcpDispatcher)); + + NRpc::TDispatcher::Get()->Configure(config->RpcDispatcher->ApplyDynamic(dynamicConfig->RpcDispatcher)); + + if (dynamicConfig->Rpc->Tracing) { + NTracing::SetTracingConfig(dynamicConfig->Rpc->Tracing); + } else if (config->Rpc->Tracing) { + NTracing::SetTracingConfig(config->Rpc->Tracing); + } + + if (dynamicConfig->TCMalloc) { + ConfigureTCMalloc(dynamicConfig->TCMalloc); + } else if (config->TCMalloc) { + ConfigureTCMalloc(config->TCMalloc); + } +} + +void ReconfigureSingletons(const TSingletonsConfigPtr& config, const TSingletonsDynamicConfigPtr& dynamicConfig) +{ + ReconfigureSingletonsImpl(config, dynamicConfig); +} + +template <class TConfig> +void StartDiagnosticDumpImpl(const TConfig& config) +{ + static NLogging::TLogger Logger("DiagDump"); + + auto logDumpString = [&] (TStringBuf banner, const TString& str) { + for (const auto& line : StringSplitter(str).Split('\n')) { + YT_LOG_DEBUG("%v %v", banner, line.Token()); + } + }; + + if (config->YTAllocDumpPeriod) { + static const TLazyIntrusivePtr<TPeriodicExecutor> Executor(BIND([&] { + return New<TPeriodicExecutor>( + NRpc::TDispatcher::Get()->GetHeavyInvoker(), + BIND([&] { + logDumpString("YTAlloc", NYTAlloc::FormatAllocationCounters()); + })); + })); + Executor->SetPeriod(config->YTAllocDumpPeriod); + Executor->Start(); + } + + if (config->RefCountedTrackerDumpPeriod) { + static const TLazyIntrusivePtr<TPeriodicExecutor> Executor(BIND([&] { + return New<TPeriodicExecutor>( + NRpc::TDispatcher::Get()->GetHeavyInvoker(), + BIND([&] { + logDumpString("RCT", TRefCountedTracker::Get()->GetDebugInfo()); + })); + })); + Executor->SetPeriod(config->RefCountedTrackerDumpPeriod); + Executor->Start(); + } +} + +void StartDiagnosticDump(const TDiagnosticDumpConfigPtr& config) +{ + StartDiagnosticDumpImpl(config); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/helpers.h b/yt/yt/library/program/helpers.h new file mode 100644 index 0000000000..be09ec889c --- /dev/null +++ b/yt/yt/library/program/helpers.h @@ -0,0 +1,18 @@ +#pragma once + +#include "public.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +void ConfigureSingletons(const TSingletonsConfigPtr& config); +void ReconfigureSingletons( + const TSingletonsConfigPtr& config, + const TSingletonsDynamicConfigPtr& dynamicConfig); + +void StartDiagnosticDump(const TDiagnosticDumpConfigPtr& config); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/private.h b/yt/yt/library/program/private.h new file mode 100644 index 0000000000..e328f30667 --- /dev/null +++ b/yt/yt/library/program/private.h @@ -0,0 +1,15 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/logging/log.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +inline const NLogging::TLogger ProgramLogger("Program"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program.cpp b/yt/yt/library/program/program.cpp new file mode 100644 index 0000000000..318d998b3a --- /dev/null +++ b/yt/yt/library/program/program.cpp @@ -0,0 +1,385 @@ +#include "program.h" + +#include "build_attributes.h" + +#include <yt/yt/build/build.h> + +#include <yt/yt/core/misc/crash_handler.h> +#include <yt/yt/core/misc/signal_registry.h> +#include <yt/yt/core/misc/fs.h> +#include <yt/yt/core/misc/shutdown.h> + +#include <yt/yt/core/ytalloc/bindings.h> + +#include <yt/yt/core/yson/writer.h> +#include <yt/yt/core/yson/null_consumer.h> + +#include <yt/yt/core/logging/log_manager.h> + +#include <yt/yt/library/ytprof/heap_profiler.h> + +#include <yt/yt/library/profiling/tcmalloc/profiler.h> + +#include <library/cpp/ytalloc/api/ytalloc.h> + +#include <library/cpp/yt/mlock/mlock.h> +#include <library/cpp/yt/stockpile/stockpile.h> + +#include <tcmalloc/malloc_extension.h> + +#include <absl/debugging/stacktrace.h> + +#include <util/system/thread.h> +#include <util/system/sigset.h> + +#include <util/string/subst.h> + +#include <thread> + +#include <stdlib.h> + +#ifdef _unix_ +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#endif + +#ifdef _linux_ +#include <grp.h> +#include <sys/prctl.h> +#endif + +#if defined(_linux_) && defined(CLANG_COVERAGE) +extern "C" int __llvm_profile_write_file(void); +extern "C" void __llvm_profile_set_filename(const char* name); +#endif + +namespace NYT { + +using namespace NYson; + +//////////////////////////////////////////////////////////////////////////////// + +class TProgram::TOptsParseResult + : public NLastGetopt::TOptsParseResult +{ +public: + TOptsParseResult(TProgram* owner, int argc, const char** argv) + : Owner_(owner) + { + Init(&Owner_->Opts_, argc, argv); + } + + void HandleError() const override + { + Owner_->OnError(CurrentExceptionMessage()); + Cerr << Endl << "Try running '" << Owner_->Argv0_ << " --help' for more information." << Endl; + Owner_->Exit(EProgramExitCode::OptionsError); + } + +private: + TProgram* const Owner_; +}; + +TProgram::TProgram() +{ + Opts_.AddHelpOption(); + Opts_.AddLongOption("yt-version", "print YT version and exit") + .NoArgument() + .StoreValue(&PrintYTVersion_, true); + Opts_.AddLongOption("version", "print version and exit") + .NoArgument() + .StoreValue(&PrintVersion_, true); + Opts_.AddLongOption("yson", "print build information in YSON") + .NoArgument() + .StoreValue(&UseYson_, true); + Opts_.AddLongOption("build", "print build information and exit") + .NoArgument() + .StoreValue(&PrintBuild_, true); + Opts_.SetFreeArgsNum(0); + + ConfigureCoverageOutput(); +} + +void TProgram::SetCrashOnError() +{ + CrashOnError_ = true; +} + +TProgram::~TProgram() = default; + +void TProgram::HandleVersionAndBuild() +{ + if (PrintVersion_) { + PrintVersionAndExit(); + } + if (PrintYTVersion_) { + PrintYTVersionAndExit(); + } + if (PrintBuild_) { + PrintBuildAndExit(); + } +} + +int TProgram::Run(int argc, const char** argv) +{ + ::srand(time(nullptr)); + + auto run = [&] { + Argv0_ = TString(argv[0]); + TOptsParseResult result(this, argc, argv); + + HandleVersionAndBuild(); + + DoRun(result); + }; + + if (!CrashOnError_) { + try { + run(); + Exit(EProgramExitCode::OK); + } catch (...) { + OnError(CurrentExceptionMessage()); + Exit(EProgramExitCode::ProgramError); + } + } else { + run(); + Exit(EProgramExitCode::OK); + } + + // Cannot reach this due to #Exit calls above. + YT_ABORT(); +} + +void TProgram::Abort(EProgramExitCode code) noexcept +{ + Abort(static_cast<int>(code)); +} + +void TProgram::Abort(int code) noexcept +{ + NLogging::TLogManager::Get()->Shutdown(); + + ::_exit(code); +} + +void TProgram::Exit(EProgramExitCode code) noexcept +{ + Exit(static_cast<int>(code)); +} + +void TProgram::Exit(int code) noexcept +{ +#if defined(_linux_) && defined(CLANG_COVERAGE) + __llvm_profile_write_file(); +#endif + + // This explicit call may become obsolete some day; + // cf. the comment section for NYT::Shutdown. + Shutdown({ + .AbortOnHang = ShouldAbortOnHungShutdown(), + .HungExitCode = code + }); + + ::exit(code); +} + +bool TProgram::ShouldAbortOnHungShutdown() noexcept +{ + return true; +} + +void TProgram::OnError(const TString& message) noexcept +{ + try { + Cerr << message << Endl; + } catch (...) { + // Just ignore it; STDERR might be closed already, + // and write() would result in EPIPE. + } +} + +void TProgram::PrintYTVersionAndExit() +{ + if (UseYson_) { + THROW_ERROR_EXCEPTION("--yson is not supported when printing version"); + } + Cout << GetVersion() << Endl; + Exit(0); +} + +void TProgram::PrintBuildAndExit() +{ + if (UseYson_) { + TYsonWriter writer(&Cout, EYsonFormat::Pretty); + Serialize(BuildBuildAttributes(), &writer); + Cout << Endl; + } else { + Cout << "Build Time: " << GetBuildTime() << Endl; + Cout << "Build Host: " << GetBuildHost() << Endl; + } + Exit(0); +} + +void TProgram::PrintVersionAndExit() +{ + PrintYTVersionAndExit(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TProgramException::TProgramException(TString what) + : What_(std::move(what)) +{ } + +const char* TProgramException::what() const noexcept +{ + return What_.c_str(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TString CheckPathExistsArgMapper(const TString& arg) +{ + if (!NFS::Exists(arg)) { + throw TProgramException(Format("File %v does not exist", arg)); + } + return arg; +} + +TGuid CheckGuidArgMapper(const TString& arg) +{ + TGuid result; + if (!TGuid::FromString(arg, &result)) { + throw TProgramException(Format("Error parsing guid %Qv", arg)); + } + return result; +} + +NYson::TYsonString CheckYsonArgMapper(const TString& arg) +{ + ParseYsonStringBuffer(arg, EYsonType::Node, GetNullYsonConsumer()); + return NYson::TYsonString(arg); +} + +void ConfigureUids() +{ +#ifdef _unix_ + uid_t ruid, euid; +#ifdef _linux_ + uid_t suid; + YT_VERIFY(getresuid(&ruid, &euid, &suid) == 0); +#else + ruid = getuid(); + euid = geteuid(); +#endif + if (euid == 0) { + // if real uid is already root do not set root as supplementary ids. + if (ruid != 0) { + YT_VERIFY(setgroups(0, nullptr) == 0); + } + // if effective uid == 0 (e. g. set-uid-root), alter saved = effective, effective = real. +#ifdef _linux_ + YT_VERIFY(setresuid(ruid, ruid, euid) == 0); + // Make server suid_dumpable = 1. + YT_VERIFY(prctl(PR_SET_DUMPABLE, 1) == 0); +#else + YT_VERIFY(setuid(euid) == 0); + YT_VERIFY(seteuid(ruid) == 0); + YT_VERIFY(setruid(ruid) == 0); +#endif + } + umask(0000); +#endif +} + +void ConfigureCoverageOutput() +{ +#if defined(_linux_) && defined(CLANG_COVERAGE) + // YT tests use pid namespaces. We can't use process id as unique identifier for output file. + if (auto profileFile = getenv("LLVM_PROFILE_FILE")) { + TString fixedProfile{profileFile}; + SubstGlobal(fixedProfile, "%e", "ytserver-all"); + SubstGlobal(fixedProfile, "%p", ToString(TInstant::Now().NanoSeconds())); + __llvm_profile_set_filename(fixedProfile.c_str()); + } +#endif +} + +void ConfigureIgnoreSigpipe() +{ +#ifdef _unix_ + signal(SIGPIPE, SIG_IGN); +#endif +} + +void ConfigureCrashHandler() +{ + TSignalRegistry::Get()->PushCallback(AllCrashSignals, CrashSignalHandler); + TSignalRegistry::Get()->PushDefaultSignalHandler(AllCrashSignals); +} + +namespace { + +void ExitZero(int /*unused*/) +{ +#if defined(_linux_) && defined(CLANG_COVERAGE) + __llvm_profile_write_file(); +#endif + // TODO(babenko): replace with pure "exit" some day. + // Currently this causes some RPC requests to master to be replied with "Promise abandoned" error, + // which is not retriable. + _exit(0); +} + +} // namespace + +void ConfigureExitZeroOnSigterm() +{ +#ifdef _unix_ + signal(SIGTERM, ExitZero); +#endif +} + +void ConfigureAllocator(const TAllocatorOptions& options) +{ + NYT::MlockFileMappings(); + +#ifdef _linux_ + NYTAlloc::EnableYTLogging(); + NYTAlloc::EnableYTProfiling(); + NYTAlloc::InitializeLibunwindInterop(); + NYTAlloc::SetEnableEagerMemoryRelease(options.YTAllocEagerMemoryRelease); + + if (tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) { + std::thread backgroundThread([] { + TThread::SetCurrentThreadName("TCAllocBack"); + tcmalloc::MallocExtension::ProcessBackgroundActions(); + YT_ABORT(); + }); + backgroundThread.detach(); + } + + NProfiling::EnableTCMallocProfiler(); + + NYTProf::EnableMemoryProfilingTags(options.SnapshotUpdatePeriod); + + absl::SetStackUnwinder(NYTProf::AbslStackUnwinder); + // TODO(prime@): tune parameters. + tcmalloc::MallocExtension::SetProfileSamplingRate(2_MB); + if (options.TCMallocGuardedSamplingRate) { + tcmalloc::MallocExtension::SetGuardedSamplingRate(*options.TCMallocGuardedSamplingRate); + tcmalloc::MallocExtension::ActivateGuardedSampling(); + } + tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(3_MB); + tcmalloc::MallocExtension::SetMaxTotalThreadCacheBytes(24_MB); + tcmalloc::MallocExtension::SetBackgroundReleaseRate(tcmalloc::MallocExtension::BytesPerSecond{32_MB}); + tcmalloc::MallocExtension::EnableForkSupport(); +#else + Y_UNUSED(options); +#endif +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program.h b/yt/yt/library/program/program.h new file mode 100644 index 0000000000..f1f236cfab --- /dev/null +++ b/yt/yt/library/program/program.h @@ -0,0 +1,148 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +#include <library/cpp/yt/stockpile/stockpile.h> + +#include <library/cpp/getopt/last_getopt.h> + +#include <yt/yt/core/yson/string.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(EProgramExitCode, + ((OK)(0)) + ((OptionsError)(1)) + ((ProgramError)(2)) +); + +class TProgram +{ +public: + TProgram(); + ~TProgram(); + + TProgram(const TProgram&) = delete; + TProgram(TProgram&&) = delete; + + // This call actually never returns; + // |int| return type is just for the symmetry with |main|. + [[noreturn]] + int Run(int argc, const char** argv); + + //! Handles --version/--yt-version/--build [--yson] if they are present. + void HandleVersionAndBuild(); + + //! Nongracefully aborts the program. + /*! + * Tries to flush logging messages. + * Aborts via |_exit| call. + */ + [[noreturn]] + static void Abort(EProgramExitCode code) noexcept; + [[noreturn]] + static void Abort(int code) noexcept; + +protected: + NLastGetopt::TOpts Opts_; + TString Argv0_; + bool PrintYTVersion_ = false; + bool PrintVersion_ = false; + bool PrintBuild_ = false; + bool UseYson_ = false; + + virtual void DoRun(const NLastGetopt::TOptsParseResult& parseResult) = 0; + + virtual void OnError(const TString& message) noexcept; + + virtual bool ShouldAbortOnHungShutdown() noexcept; + + void SetCrashOnError(); + + //! Handler for --yt-version command argument. + [[noreturn]] + void PrintYTVersionAndExit(); + + //! Handler for --build command argument. + [[noreturn]] + void PrintBuildAndExit(); + + //! Handler for --version command argument. + //! By default, --version and --yt-version work the same way, + //! but some YT components (e.g. CHYT) can override it to provide its own version. + [[noreturn]] + virtual void PrintVersionAndExit(); + + [[noreturn]] + void Exit(EProgramExitCode code) noexcept; + + [[noreturn]] + void Exit(int code) noexcept; + +private: + bool CrashOnError_ = false; + + // Custom handler for option parsing errors. + class TOptsParseResult; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! The simplest exception possible. +//! Here we refrain from using TErrorException, as it relies on proper configuration of singleton subsystems, +//! which might not be the case during startup. +class TProgramException + : public std::exception +{ +public: + explicit TProgramException(TString what); + + const char* what() const noexcept override; + +private: + const TString What_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +//! Helper for TOpt::StoreMappedResult to validate file paths for existence. +TString CheckPathExistsArgMapper(const TString& arg); + +//! Helper for TOpt::StoreMappedResult to parse GUIDs. +TGuid CheckGuidArgMapper(const TString& arg); + +//! Helper for TOpt::StoreMappedResult to parse YSON strings. +NYson::TYsonString CheckYsonArgMapper(const TString& arg); + +//! Drop privileges and save them if running with suid-bit. +void ConfigureUids(); + +void ConfigureCoverageOutput(); + +void ConfigureIgnoreSigpipe(); + +//! Intercepts standard crash signals (see signal_registry.h for full list) with a nice handler. +void ConfigureCrashHandler(); + +//! Intercepts SIGTERM and terminates the process immediately with zero exit code. +void ConfigureExitZeroOnSigterm(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TAllocatorOptions +{ + bool YTAllocEagerMemoryRelease = false; + + bool TCMallocOptimizeSize = false; + std::optional<i64> TCMallocGuardedSamplingRate = 128_MB; + + std::optional<TDuration> SnapshotUpdatePeriod; +}; + +void ConfigureAllocator(const TAllocatorOptions& options = {}); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program_config_mixin.cpp b/yt/yt/library/program/program_config_mixin.cpp new file mode 100644 index 0000000000..9ced4de64f --- /dev/null +++ b/yt/yt/library/program/program_config_mixin.cpp @@ -0,0 +1 @@ +#include "program_config_mixin.h" diff --git a/yt/yt/library/program/program_config_mixin.h b/yt/yt/library/program/program_config_mixin.h new file mode 100644 index 0000000000..80f681d06e --- /dev/null +++ b/yt/yt/library/program/program_config_mixin.h @@ -0,0 +1,166 @@ +#pragma once + +#include "program.h" + +#include <library/cpp/yt/string/enum.h> + +#include <yt/yt/core/ytree/convert.h> +#include <yt/yt/core/ytree/yson_serializable.h> + +#include <util/stream/file.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TConfig, class TDynamicConfig = void> +class TProgramConfigMixin +{ +protected: + explicit TProgramConfigMixin( + NLastGetopt::TOpts& opts, + bool required = true, + const TString& argumentName = "config") + : ArgumentName_(argumentName) + { + auto opt = opts + .AddLongOption(TString(argumentName), Format("path to %v file", argumentName)) + .StoreMappedResult(&ConfigPath_, &CheckPathExistsArgMapper) + .RequiredArgument("FILE"); + if (required) { + opt.Required(); + } else { + opt.Optional(); + } + opts + .AddLongOption( + Format("%v-template", argumentName), + Format("print %v template and exit", argumentName)) + .SetFlag(&ConfigTemplate_); + opts + .AddLongOption( + Format("%v-actual", argumentName), + Format("print actual %v and exit", argumentName)) + .SetFlag(&ConfigActual_); + opts + .AddLongOption( + Format("%v-unrecognized-strategy", argumentName), + Format("configure strategy for unrecognized attributes in %v", argumentName)) + .Handler1T<TStringBuf>([this](TStringBuf value) { + UnrecognizedStrategy_ = ParseEnum<NYTree::EUnrecognizedStrategy>(value); + }); + + if constexpr (std::is_same_v<TDynamicConfig, void>) { + return; + } + + opts + .AddLongOption( + Format("dynamic-%v-template", argumentName), + Format("print dynamic %v template and exit", argumentName)) + .SetFlag(&DynamicConfigTemplate_); + } + + TIntrusivePtr<TConfig> GetConfig(bool returnNullIfNotSupplied = false) + { + if (returnNullIfNotSupplied && !ConfigPath_) { + return nullptr; + } + + if (!Config_) { + LoadConfig(); + } + return Config_; + } + + NYTree::INodePtr GetConfigNode(bool returnNullIfNotSupplied = false) + { + if (returnNullIfNotSupplied && !ConfigPath_) { + return nullptr; + } + + if (!ConfigNode_) { + LoadConfigNode(); + } + return ConfigNode_; + } + + bool HandleConfigOptions() + { + auto print = [] (const auto& config) { + using namespace NYson; + TYsonWriter writer(&Cout, EYsonFormat::Pretty); + config->Save(&writer); + Cout << Flush; + }; + if (ConfigTemplate_) { + print(New<TConfig>()); + return true; + } + if (ConfigActual_) { + print(GetConfig()); + return true; + } + + if constexpr (!std::is_same_v<TDynamicConfig, void>) { + if (DynamicConfigTemplate_) { + print(New<TDynamicConfig>()); + return true; + } + } + return false; + } + +private: + void LoadConfigNode() + { + using namespace NYTree; + + if (!ConfigPath_){ + THROW_ERROR_EXCEPTION("Missing --%v option", ArgumentName_); + } + + try { + TIFStream stream(ConfigPath_); + ConfigNode_ = ConvertToNode(&stream); + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Error parsing %v file %v", + ArgumentName_, + ConfigPath_) + << ex; + } + } + + void LoadConfig() + { + if (!ConfigNode_) { + LoadConfigNode(); + } + + try { + Config_ = New<TConfig>(); + Config_->SetUnrecognizedStrategy(UnrecognizedStrategy_); + Config_->Load(ConfigNode_); + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Error loading %v file %v", + ArgumentName_, + ConfigPath_) + << ex; + } + } + + const TString ArgumentName_; + + TString ConfigPath_; + bool ConfigTemplate_; + bool ConfigActual_; + bool DynamicConfigTemplate_ = false; + NYTree::EUnrecognizedStrategy UnrecognizedStrategy_ = NYTree::EUnrecognizedStrategy::KeepRecursive; + + TIntrusivePtr<TConfig> Config_; + NYTree::INodePtr ConfigNode_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program_pdeathsig_mixin.cpp b/yt/yt/library/program/program_pdeathsig_mixin.cpp new file mode 100644 index 0000000000..34f1f3b9a8 --- /dev/null +++ b/yt/yt/library/program/program_pdeathsig_mixin.cpp @@ -0,0 +1,36 @@ +#include "program_pdeathsig_mixin.h" + +#ifdef _linux_ +#include <sys/prctl.h> +#endif + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TProgramPdeathsigMixin::TProgramPdeathsigMixin(NLastGetopt::TOpts& opts) +{ + opts.AddLongOption("pdeathsig", "parent death signal") + .StoreResult(&ParentDeathSignal_) + .RequiredArgument("PDEATHSIG"); +} + +bool TProgramPdeathsigMixin::HandlePdeathsigOptions() +{ + if (ParentDeathSignal_ > 0) { +#ifdef _linux_ + // Parent death signal is set by testing framework to avoid dangling processes when test runner crashes. + // Unfortunately, setting pdeathsig in preexec_fn in subprocess call in test runner is not working + // when the program has suid bit (pdeath_sig is reset after exec call in this case) + // More details can be found in + // http://linux.die.net/man/2/prctl + // http://www.isec.pl/vulnerabilities/isec-0024-death-signal.txt + YT_VERIFY(prctl(PR_SET_PDEATHSIG, ParentDeathSignal_) == 0); +#endif + } + return false; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program_pdeathsig_mixin.h b/yt/yt/library/program/program_pdeathsig_mixin.h new file mode 100644 index 0000000000..3e4bcfd4a6 --- /dev/null +++ b/yt/yt/library/program/program_pdeathsig_mixin.h @@ -0,0 +1,22 @@ +#pragma once + +#include "program.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TProgramPdeathsigMixin +{ +protected: + explicit TProgramPdeathsigMixin(NLastGetopt::TOpts& opts); + + bool HandlePdeathsigOptions(); + +private: + int ParentDeathSignal_ = -1; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program_setsid_mixin.cpp b/yt/yt/library/program/program_setsid_mixin.cpp new file mode 100644 index 0000000000..a745fcd3a2 --- /dev/null +++ b/yt/yt/library/program/program_setsid_mixin.cpp @@ -0,0 +1,30 @@ +#include "program_setsid_mixin.h" + +#ifdef _linux_ +#include <unistd.h> +#endif + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TProgramSetsidMixin::TProgramSetsidMixin(NLastGetopt::TOpts& opts) +{ + opts.AddLongOption("setsid", "create a new session") + .StoreTrue(&Setsid_) + .Optional(); +} + +bool TProgramSetsidMixin::HandleSetsidOptions() +{ + if (Setsid_) { +#ifdef _linux_ + setsid(); +#endif + } + return false; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/program_setsid_mixin.h b/yt/yt/library/program/program_setsid_mixin.h new file mode 100644 index 0000000000..00b3dff50e --- /dev/null +++ b/yt/yt/library/program/program_setsid_mixin.h @@ -0,0 +1,22 @@ +#pragma once + +#include "program.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TProgramSetsidMixin +{ +protected: + explicit TProgramSetsidMixin(NLastGetopt::TOpts& opts); + + bool HandleSetsidOptions(); + +private: + bool Setsid_ = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/public.h b/yt/yt/library/program/public.h new file mode 100644 index 0000000000..b10575778e --- /dev/null +++ b/yt/yt/library/program/public.h @@ -0,0 +1,21 @@ +#pragma once + +#include <yt/yt/core/misc/public.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_CLASS(TBuildInfo) +DECLARE_REFCOUNTED_CLASS(TRpcConfig) +DECLARE_REFCOUNTED_CLASS(TTCMallocConfig) +DECLARE_REFCOUNTED_CLASS(TStockpileConfig) +DECLARE_REFCOUNTED_CLASS(TSingletonsConfig) +DECLARE_REFCOUNTED_CLASS(TSingletonsDynamicConfig) +DECLARE_REFCOUNTED_CLASS(TDiagnosticDumpConfig) +DECLARE_REFCOUNTED_CLASS(THeapSizeLimit) +DECLARE_REFCOUNTED_CLASS(THeapProfilerConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/library/program/ya.make b/yt/yt/library/program/ya.make new file mode 100644 index 0000000000..5742ce9287 --- /dev/null +++ b/yt/yt/library/program/ya.make @@ -0,0 +1,30 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + build_attributes.cpp + config.cpp + helpers.cpp + program.cpp + program_config_mixin.cpp + program_pdeathsig_mixin.cpp + program_setsid_mixin.cpp +) + +PEERDIR( + yt/yt/core + yt/yt/core/service_discovery/yp + yt/yt/library/monitoring + yt/yt/library/containers + yt/yt/library/profiling/solomon + yt/yt/library/profiling/tcmalloc + yt/yt/library/profiling/perf + yt/yt/library/ytprof + yt/yt/library/tracing/jaeger + library/cpp/yt/mlock + library/cpp/yt/stockpile + library/cpp/yt/string +) + +END() diff --git a/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt index 54f67f1ec5..68eb83d171 100644 --- a/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/tracing/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(jaeger) add_library(yt-library-tracing) target_compile_options(yt-library-tracing PRIVATE diff --git a/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt b/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt index 923b49e3e7..5681c6a06e 100644 --- a/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/tracing/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(jaeger) add_library(yt-library-tracing) target_compile_options(yt-library-tracing PRIVATE diff --git a/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt b/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt index 923b49e3e7..5681c6a06e 100644 --- a/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/tracing/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(jaeger) add_library(yt-library-tracing) target_compile_options(yt-library-tracing PRIVATE diff --git a/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt b/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt index e33cbed391..0adf7dc67a 100644 --- a/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/tracing/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(jaeger) add_library(yt-library-tracing) target_link_libraries(yt-library-tracing PUBLIC diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..ebc07421e3 --- /dev/null +++ b/yt/yt/library/tracing/jaeger/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,67 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-tracing-jaeger) +target_compile_options(library-tracing-jaeger PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_proto_messages(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto +) +target_sources(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp +) +target_proto_addincls(library-tracing-jaeger + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-tracing-jaeger + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) + +add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger) +target_compile_options(library-tracing-jaeger.global PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger.global PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_sources(library-tracing-jaeger.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp +) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..0946a55b6c --- /dev/null +++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-aarch64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-tracing-jaeger) +target_compile_options(library-tracing-jaeger PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_proto_messages(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto +) +target_sources(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp +) +target_proto_addincls(library-tracing-jaeger + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-tracing-jaeger + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) + +add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger) +target_compile_options(library-tracing-jaeger.global PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_sources(library-tracing-jaeger.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp +) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..0946a55b6c --- /dev/null +++ b/yt/yt/library/tracing/jaeger/CMakeLists.linux-x86_64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-tracing-jaeger) +target_compile_options(library-tracing-jaeger PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_proto_messages(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto +) +target_sources(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp +) +target_proto_addincls(library-tracing-jaeger + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-tracing-jaeger + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) + +add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger) +target_compile_options(library-tracing-jaeger.global PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-tracing-jaeger.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_sources(library-tracing-jaeger.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp +) diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.txt b/yt/yt/library/tracing/jaeger/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/tracing/jaeger/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..e6ee22e85b --- /dev/null +++ b/yt/yt/library/tracing/jaeger/CMakeLists.windows-x86_64.txt @@ -0,0 +1,61 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-tracing-jaeger) +target_link_libraries(library-tracing-jaeger PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_proto_messages(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/model.proto +) +target_sources(library-tracing-jaeger PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/sampler.cpp +) +target_proto_addincls(library-tracing-jaeger + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-tracing-jaeger + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) + +add_global_library_for(library-tracing-jaeger.global library-tracing-jaeger) +target_link_libraries(library-tracing-jaeger.global PUBLIC + contrib-libs-cxxsupp + yutil + yt-library-tracing + core-rpc-grpc + contrib-libs-protobuf +) +target_sources(library-tracing-jaeger.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/tracing/jaeger/tracer.cpp +) diff --git a/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..ac3d970f12 --- /dev/null +++ b/yt/yt/library/ytprof/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,41 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(api) +add_subdirectory(proto) + +add_library(yt-library-ytprof) +target_compile_options(yt-library-ytprof PRIVATE + -Wdeprecated-this-capture + -DYTPROF_BUILD_TYPE="RELEASE" +) +target_link_libraries(yt-library-ytprof PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-threading + backtrace-cursors-interop + backtrace-cursors-frame_pointer + backtrace-cursors-libunwind + library-ytprof-api + library-ytprof-proto + contrib-libs-libunwind + libs-tcmalloc-malloc_extension + library-cpp-svnversion + yt-yt-core +) +target_sources(yt-library-ytprof PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize_other.cpp +) diff --git a/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..c015a8d3da --- /dev/null +++ b/yt/yt/library/ytprof/CMakeLists.linux-aarch64.txt @@ -0,0 +1,43 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(api) +add_subdirectory(http) +add_subdirectory(proto) + +add_library(yt-library-ytprof) +target_compile_options(yt-library-ytprof PRIVATE + -Wdeprecated-this-capture + -DYTPROF_BUILD_TYPE="RELEASE" +) +target_link_libraries(yt-library-ytprof PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-threading + backtrace-cursors-interop + backtrace-cursors-frame_pointer + backtrace-cursors-libunwind + library-ytprof-api + library-ytprof-proto + contrib-libs-libunwind + libs-tcmalloc-malloc_extension + library-cpp-svnversion + yt-yt-core +) +target_sources(yt-library-ytprof PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize.cpp +) diff --git a/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..c015a8d3da --- /dev/null +++ b/yt/yt/library/ytprof/CMakeLists.linux-x86_64.txt @@ -0,0 +1,43 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(api) +add_subdirectory(http) +add_subdirectory(proto) + +add_library(yt-library-ytprof) +target_compile_options(yt-library-ytprof PRIVATE + -Wdeprecated-this-capture + -DYTPROF_BUILD_TYPE="RELEASE" +) +target_link_libraries(yt-library-ytprof PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-threading + backtrace-cursors-interop + backtrace-cursors-frame_pointer + backtrace-cursors-libunwind + library-ytprof-api + library-ytprof-proto + contrib-libs-libunwind + libs-tcmalloc-malloc_extension + library-cpp-svnversion + yt-yt-core +) +target_sources(yt-library-ytprof PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize.cpp +) diff --git a/yt/yt/library/ytprof/CMakeLists.txt b/yt/yt/library/ytprof/CMakeLists.txt index dbfb934bae..f8b31df0c1 100644 --- a/yt/yt/library/ytprof/CMakeLists.txt +++ b/yt/yt/library/ytprof/CMakeLists.txt @@ -6,4 +6,12 @@ # original buildsystem will not be accepted. -add_subdirectory(api) +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt b/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..ec5652c9e9 --- /dev/null +++ b/yt/yt/library/ytprof/CMakeLists.windows-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(api) +add_subdirectory(proto) + +add_library(yt-library-ytprof) +target_compile_options(yt-library-ytprof PRIVATE + -DYTPROF_BUILD_TYPE="RELEASE" +) +target_link_libraries(yt-library-ytprof PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-memory + cpp-yt-threading + backtrace-cursors-interop + backtrace-cursors-frame_pointer + backtrace-cursors-libunwind + library-ytprof-api + library-ytprof-proto + contrib-libs-libunwind + libs-tcmalloc-malloc_extension + library-cpp-svnversion + yt-yt-core +) +target_sources(yt-library-ytprof PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/signal_safe_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/cpu_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/heap_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/spinlock_profiler.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/profile.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/build_info.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/external_pprof.cpp + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/symbolize_other.cpp +) diff --git a/yt/yt/library/ytprof/bundle/ya.make b/yt/yt/library/ytprof/bundle/ya.make new file mode 100644 index 0000000000..7f88583494 --- /dev/null +++ b/yt/yt/library/ytprof/bundle/ya.make @@ -0,0 +1,28 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +# Built with ya make -DNO_DEBUGINFO=yes -r --musl contrib/libs/llvm12/tools/llvm-symbolizer +FROM_SANDBOX( + FILE 2531143113 + OUT_NOAUTO llvm-symbolizer +) + +RESOURCE( + yt/yt/library/ytprof/bundle/llvm-symbolizer + /ytprof/llvm-symbolizer +) + +# Built with env CGO_ENABLED=0 ya tool go install github.com/google/pprof@latest +FROM_SANDBOX( + FILE 2531135322 + OUT_NOAUTO pprof +) + +RESOURCE( + yt/yt/library/ytprof/bundle/pprof + /ytprof/pprof +) + +END() + diff --git a/yt/yt/library/ytprof/example/main.cpp b/yt/yt/library/ytprof/example/main.cpp new file mode 100644 index 0000000000..bc9dec690a --- /dev/null +++ b/yt/yt/library/ytprof/example/main.cpp @@ -0,0 +1,70 @@ +#include <yt/yt/core/concurrency/poller.h> +#include <yt/yt/core/concurrency/thread_pool_poller.h> +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/thread_pool.h> +#include <yt/yt/core/http/server.h> + +#include <yt/yt/library/ytprof/http/handler.h> +#include <yt/yt/library/ytprof/heap_profiler.h> + +#include <absl/debugging/stacktrace.h> + +using namespace NYT; +using namespace NYT::NHttp; +using namespace NYT::NConcurrency; +using namespace NYT::NYTProf; + +int main(int argc, char* argv[]) +{ + absl::SetStackUnwinder(AbslStackUnwinder); + tcmalloc::MallocExtension::SetProfileSamplingRate(2_MB); + + try { + if (argc != 2 && argc != 3) { + throw yexception() << "usage: " << argv[0] << " PORT"; + } + + auto port = FromString<int>(argv[1]); + auto poller = CreateThreadPoolPoller(1, "Example"); + auto server = CreateServer(port, poller); + + Register(server, ""); + server->Start(); + + THashMap<TString, std::vector<int>> data; + for (int i = 0; i < 1024 * 16; i++) { + data[ToString(i)].resize(1024); + } + + auto burnCpu = [] { + ui64 value = 0; + while (true) { + THash<TString> hasher; + for (int i = 0; i < 10000000; i++) { + value += hasher(ToString(i)); + } + + std::vector<TString> data; + for (int i = 0; i < 10000; i++) { + data.push_back(TString(1024, 'x')); + } + + if (value == 1) { + Sleep(TDuration::Seconds(1)); + } + } + }; + + auto pool = CreateThreadPool(64, "Pool"); + for (int i = 0; i < 64; i++) { + pool->GetInvoker()->Invoke(BIND(burnCpu)); + } + + burnCpu(); + } catch (const std::exception& ex) { + Cerr << ex.what() << Endl; + _exit(1); + } + + return 0; +} diff --git a/yt/yt/library/ytprof/example/ya.make b/yt/yt/library/ytprof/example/ya.make new file mode 100644 index 0000000000..2f79922400 --- /dev/null +++ b/yt/yt/library/ytprof/example/ya.make @@ -0,0 +1,19 @@ +PROGRAM(ytprof-example) + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +IF (OS_LINUX) + ALLOCATOR(TCMALLOC_256K) +ENDIF() + +SRCS(main.cpp) + +IF (OS_LINUX) + LDFLAGS("-Wl,--build-id=sha1") +ENDIF() + +PEERDIR( + yt/yt/library/ytprof/http +) + +END() diff --git a/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..334ca90c02 --- /dev/null +++ b/yt/yt/library/ytprof/http/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-ytprof-http) +target_compile_options(library-ytprof-http PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-ytprof-http PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cgiparam + yt-core-http + yt-library-ytprof + yt-library-process +) +target_sources(library-ytprof-http PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/http/handler.cpp +) diff --git a/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..334ca90c02 --- /dev/null +++ b/yt/yt/library/ytprof/http/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(library-ytprof-http) +target_compile_options(library-ytprof-http PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(library-ytprof-http PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cgiparam + yt-core-http + yt-library-ytprof + yt-library-process +) +target_sources(library-ytprof-http PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/http/handler.cpp +) diff --git a/yt/yt/library/ytprof/http/CMakeLists.txt b/yt/yt/library/ytprof/http/CMakeLists.txt new file mode 100644 index 0000000000..4d48dcdee6 --- /dev/null +++ b/yt/yt/library/ytprof/http/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/ytprof/http/handler.cpp b/yt/yt/library/ytprof/http/handler.cpp new file mode 100644 index 0000000000..382ffc1fec --- /dev/null +++ b/yt/yt/library/ytprof/http/handler.cpp @@ -0,0 +1,311 @@ +#include "handler.h" + +#include <yt/yt/core/concurrency/async_stream.h> + +#include <yt/yt/core/http/http.h> +#include <yt/yt/core/http/server.h> + +#include <yt/yt/library/ytprof/cpu_profiler.h> +#include <yt/yt/library/ytprof/spinlock_profiler.h> +#include <yt/yt/library/ytprof/heap_profiler.h> +#include <yt/yt/library/ytprof/profile.h> +#include <yt/yt/library/ytprof/symbolize.h> +#include <yt/yt/library/ytprof/external_pprof.h> + +#include <yt/yt/library/process/subprocess.h> + +#include <yt/yt/core/misc/finally.h> + +#include <library/cpp/cgiparam/cgiparam.h> + +#include <util/system/mutex.h> + +namespace NYT::NYTProf { + +using namespace NHttp; +using namespace NConcurrency; + +//////////////////////////////////////////////////////////////////////////////// + +class TBaseHandler + : public IHttpHandler +{ +public: + explicit TBaseHandler(const TBuildInfo& buildInfo) + : BuildInfo_(buildInfo) + { } + + virtual NProto::Profile BuildProfile(const TCgiParameters& params) = 0; + + void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override + { + try { + TTryGuard guard(Lock_); + if (!guard) { + rsp->SetStatus(EStatusCode::TooManyRequests); + WaitFor(rsp->WriteBody(TSharedRef::FromString("Profile fetch already running"))) + .ThrowOnError(); + return; + } + + TCgiParameters params(req->GetUrl().RawQuery); + auto profile = BuildProfile(params); + Symbolize(&profile, true); + AddBuildInfo(&profile, BuildInfo_); + + if (auto it = params.Find("symbolize"); it == params.end() || it->second != "0") { + SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ + .RunTool = RunSubprocess, + }); + } + + TStringStream profileBlob; + WriteProfile(&profileBlob, profile); + + rsp->SetStatus(EStatusCode::OK); + WaitFor(rsp->WriteBody(TSharedRef::FromString(profileBlob.Str()))) + .ThrowOnError(); + } catch (const std::exception& ex) { + if (rsp->AreHeadersFlushed()) { + throw; + } + + rsp->SetStatus(EStatusCode::InternalServerError); + WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what()))) + .ThrowOnError(); + + throw; + } + } + +protected: + const TBuildInfo BuildInfo_; + +private: + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_); +}; + +class TCpuProfilerHandler + : public TBaseHandler +{ +public: + using TBaseHandler::TBaseHandler; + + NProto::Profile BuildProfile(const TCgiParameters& params) override + { + auto duration = TDuration::Seconds(15); + if (auto it = params.Find("d"); it != params.end()) { + duration = TDuration::Parse(it->second); + } + + TCpuProfilerOptions options; + if (auto it = params.Find("freq"); it != params.end()) { + options.SamplingFrequency = FromString<int>(it->second); + } + + if (auto it = params.Find("record_action_run_time"); it != params.end()) { + options.RecordActionRunTime = true; + } + + if (auto it = params.Find("action_min_exec_time"); it != params.end()) { + options.SampleFilters.push_back(GetActionMinExecTimeFilter(TDuration::Parse(it->second))); + } + + TCpuProfiler profiler{options}; + profiler.Start(); + TDelayedExecutor::WaitForDuration(duration); + profiler.Stop(); + + return profiler.ReadProfile(); + } +}; + +class TSpinlockProfilerHandler + : public TBaseHandler +{ +public: + TSpinlockProfilerHandler(const TBuildInfo& buildInfo, bool yt) + : TBaseHandler(buildInfo) + , YT_(yt) + { } + + NProto::Profile BuildProfile(const TCgiParameters& params) override + { + auto duration = TDuration::Seconds(15); + if (auto it = params.Find("d"); it != params.end()) { + duration = TDuration::Parse(it->second); + } + + TSpinlockProfilerOptions options; + if (auto it = params.Find("frac"); it != params.end()) { + options.ProfileFraction = FromString<int>(it->second); + } + + if (YT_) { + TBlockingProfiler profiler{options}; + profiler.Start(); + TDelayedExecutor::WaitForDuration(duration); + profiler.Stop(); + + return profiler.ReadProfile(); + } else { + TSpinlockProfiler profiler{options}; + profiler.Start(); + TDelayedExecutor::WaitForDuration(duration); + profiler.Stop(); + + return profiler.ReadProfile(); + } + } + +private: + const bool YT_; +}; + +class TTCMallocSnapshotProfilerHandler + : public TBaseHandler +{ +public: + TTCMallocSnapshotProfilerHandler(const TBuildInfo& buildInfo, tcmalloc::ProfileType profileType) + : TBaseHandler(buildInfo) + , ProfileType_(profileType) + { } + + NProto::Profile BuildProfile(const TCgiParameters& /*params*/) override + { + return ReadHeapProfile(ProfileType_); + } + +private: + tcmalloc::ProfileType ProfileType_; +}; + +class TTCMallocAllocationProfilerHandler + : public TBaseHandler +{ +public: + using TBaseHandler::TBaseHandler; + + NProto::Profile BuildProfile(const TCgiParameters& params) override + { + auto duration = TDuration::Seconds(15); + if (auto it = params.Find("d"); it != params.end()) { + duration = TDuration::Parse(it->second); + } + + auto token = tcmalloc::MallocExtension::StartAllocationProfiling(); + TDelayedExecutor::WaitForDuration(duration); + return ConvertAllocationProfile(std::move(token).Stop()); + } +}; + +class TTCMallocStatHandler + : public IHttpHandler +{ +public: + void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override + { + auto stat = tcmalloc::MallocExtension::GetStats(); + rsp->SetStatus(EStatusCode::OK); + WaitFor(rsp->WriteBody(TSharedRef::FromString(TString{stat}))) + .ThrowOnError(); + } +}; + +class TBinaryHandler + : public IHttpHandler +{ +public: + void HandleRequest(const IRequestPtr& req, const IResponseWriterPtr& rsp) override + { + try { + auto buildId = GetBuildId(); + TCgiParameters params(req->GetUrl().RawQuery); + + if (auto it = params.Find("check_build_id"); it != params.end()) { + if (it->second != buildId) { + THROW_ERROR_EXCEPTION("Wrong build id: %v != %v", it->second, buildId); + } + } + + rsp->SetStatus(EStatusCode::OK); + + TFileInput file{"/proc/self/exe"}; + auto adapter = CreateBufferedSyncAdapter(rsp); + file.ReadAll(*adapter); + adapter->Finish(); + + WaitFor(rsp->Close()) + .ThrowOnError(); + } catch (const std::exception& ex) { + if (rsp->AreHeadersFlushed()) { + throw; + } + + rsp->SetStatus(EStatusCode::InternalServerError); + WaitFor(rsp->WriteBody(TSharedRef::FromString(ex.what()))) + .ThrowOnError(); + + throw; + } + } +}; + +class TVersionHandler + : public IHttpHandler +{ +public: + void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override + { + rsp->SetStatus(EStatusCode::OK); + WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion()))) + .ThrowOnError(); + } +}; + +class TBuildIdHandler + : public IHttpHandler +{ +public: + void HandleRequest(const IRequestPtr& /* req */, const IResponseWriterPtr& rsp) override + { + rsp->SetStatus(EStatusCode::OK); + WaitFor(rsp->WriteBody(TSharedRef::FromString(GetVersion()))) + .ThrowOnError(); + } +}; + +void Register( + const NHttp::IServerPtr& server, + const TString& prefix, + const TBuildInfo& buildInfo) +{ + Register(server->GetPathMatcher(), prefix, buildInfo); +} + +void Register( + const IRequestPathMatcherPtr& handlers, + const TString& prefix, + const TBuildInfo& buildInfo) +{ + handlers->Add(prefix + "/profile", New<TCpuProfilerHandler>(buildInfo)); + + handlers->Add(prefix + "/lock", New<TSpinlockProfilerHandler>(buildInfo, false)); + handlers->Add(prefix + "/block", New<TSpinlockProfilerHandler>(buildInfo, true)); + + handlers->Add(prefix + "/heap", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kHeap)); + handlers->Add(prefix + "/peak", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kPeakHeap)); + handlers->Add(prefix + "/fragmentation", New<TTCMallocSnapshotProfilerHandler>(buildInfo, tcmalloc::ProfileType::kFragmentation)); + handlers->Add(prefix + "/allocations", New<TTCMallocAllocationProfilerHandler>(buildInfo)); + + handlers->Add(prefix + "/tcmalloc", New<TTCMallocStatHandler>()); + + handlers->Add(prefix + "/binary", New<TBinaryHandler>()); + + handlers->Add(prefix + "/version", New<TVersionHandler>()); + handlers->Add(prefix + "/buildid", New<TBuildIdHandler>()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/http/handler.h b/yt/yt/library/ytprof/http/handler.h new file mode 100644 index 0000000000..fa96412d95 --- /dev/null +++ b/yt/yt/library/ytprof/http/handler.h @@ -0,0 +1,24 @@ +#pragma once + +#include <yt/yt/core/http/public.h> + +#include <yt/yt/library/ytprof/build_info.h> + +namespace NYT::NYTProf { + +//////////////////////////////////////////////////////////////////////////////// + +//! Register profiling handlers. +void Register( + const NHttp::IServerPtr& server, + const TString& prefix, + const TBuildInfo& buildInfo = TBuildInfo::GetDefault()); + +void Register( + const NHttp::IRequestPathMatcherPtr& handlers, + const TString& prefix, + const TBuildInfo& buildInfo = TBuildInfo::GetDefault()); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/http/ya.make b/yt/yt/library/ytprof/http/ya.make new file mode 100644 index 0000000000..1a1f3ff20c --- /dev/null +++ b/yt/yt/library/ytprof/http/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + handler.cpp +) + +PEERDIR( + library/cpp/cgiparam + yt/yt/core/http + yt/yt/library/ytprof + yt/yt/library/process +) + +END() diff --git a/yt/yt/library/ytprof/integration/test_http.py b/yt/yt/library/ytprof/integration/test_http.py new file mode 100644 index 0000000000..2135d8daf9 --- /dev/null +++ b/yt/yt/library/ytprof/integration/test_http.py @@ -0,0 +1,113 @@ +import pytest +import requests +import time + +import asyncio +import httpx + +import yatest.common +import yatest.common.network + + +TIMEOUT = 5000 + + +@pytest.fixture(scope="session") +def running_example(): + with yatest.common.network.PortManager() as pm: + port = pm.get_port() + + cmd = [ + yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"), + str(port) + ] + + p = yatest.common.execute(cmd, wait=False, env={"YT_LOG_LEVEL": "DEBUG"}) + time.sleep(1) + assert p.running + + try: + yield {"port": port} + finally: + p.kill() + + +def fetch_data(running_example, name): + rsp = requests.get(f"http://localhost:{running_example['port']}/{name}") + if rsp.status_code == 200: + return rsp.content + + if rsp.status_code == 500: + raise Exception(rsp.text) + + rsp.raise_for_status() + + +def test_smoke_tcmalloc(running_example): + fetch_data(running_example, "heap") + fetch_data(running_example, "allocations?d=1") + fetch_data(running_example, "peak") + fetch_data(running_example, "fragmentation") + + +async def get_async(url): + async with httpx.AsyncClient() as client: + return await client.get(url, timeout=TIMEOUT) + + +async def launch(running_example, name): + url = f"http://localhost:{running_example['port']}/{name}" + + urls = [url, url, url] + + resps = await asyncio.gather(*map(get_async, urls)) + data = [resp.status_code for resp in resps] + + assert data == [200, 429, 429] + + +def test_async(running_example): + fetch_data(running_example, "heap") + asyncio.run(launch(running_example, "heap")) + fetch_data(running_example, "heap") + + +def test_status_handlers(running_example): + assert fetch_data(running_example, "buildid") + assert fetch_data(running_example, "version") + + +def test_cpu_profile(running_example): + if yatest.common.context.build_type != "profile": + pytest.skip() + + fetch_data(running_example, "profile?d=1") + fetch_data(running_example, "profile?d=1&freq=1000") + + +def test_spinlock_profile(running_example): + if yatest.common.context.build_type != "profile": + pytest.skip() + + fetch_data(running_example, "lock?d=1") + fetch_data(running_example, "lock?d=1&frac=1") + + +def test_block_profile(running_example): + if yatest.common.context.build_type != "profile": + pytest.skip() + + fetch_data(running_example, "block?d=1") + fetch_data(running_example, "block?d=1&frac=1") + + +def test_binary_handler(running_example): + binary = fetch_data(running_example, "binary") + + with open(yatest.common.binary_path("yt/yt/library/ytprof/example/ytprof-example"), "rb") as f: + real_binary = f.read() + + assert binary == real_binary + + with pytest.raises(Exception): + fetch_data(running_example, "binary?check_build_id=1234") diff --git a/yt/yt/library/ytprof/integration/ya.make b/yt/yt/library/ytprof/integration/ya.make new file mode 100644 index 0000000000..889c25c0e1 --- /dev/null +++ b/yt/yt/library/ytprof/integration/ya.make @@ -0,0 +1,20 @@ +PY3TEST() + +SIZE(MEDIUM) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + contrib/python/requests + contrib/python/httpx +) + +TEST_SRCS( + test_http.py +) + +DEPENDS( + yt/yt/library/ytprof/example +) + +END() diff --git a/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..2a3265d317 --- /dev/null +++ b/yt/yt/library/ytprof/proto/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,47 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-ytprof-proto) +target_include_directories(library-ytprof-proto PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_link_libraries(library-ytprof-proto PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(library-ytprof-proto PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto +) +target_proto_addincls(library-ytprof-proto + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-ytprof-proto + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt b/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..d16987cc1d --- /dev/null +++ b/yt/yt/library/ytprof/proto/CMakeLists.linux-aarch64.txt @@ -0,0 +1,48 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-ytprof-proto) +target_include_directories(library-ytprof-proto PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_link_libraries(library-ytprof-proto PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(library-ytprof-proto PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto +) +target_proto_addincls(library-ytprof-proto + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-ytprof-proto + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..d16987cc1d --- /dev/null +++ b/yt/yt/library/ytprof/proto/CMakeLists.linux-x86_64.txt @@ -0,0 +1,48 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-ytprof-proto) +target_include_directories(library-ytprof-proto PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_link_libraries(library-ytprof-proto PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(library-ytprof-proto PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto +) +target_proto_addincls(library-ytprof-proto + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-ytprof-proto + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/library/ytprof/proto/CMakeLists.txt b/yt/yt/library/ytprof/proto/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yt/library/ytprof/proto/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt b/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..2a3265d317 --- /dev/null +++ b/yt/yt/library/ytprof/proto/CMakeLists.windows-x86_64.txt @@ -0,0 +1,47 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(library-ytprof-proto) +target_include_directories(library-ytprof-proto PUBLIC + ${CMAKE_BINARY_DIR}/yt +) +target_link_libraries(library-ytprof-proto PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(library-ytprof-proto PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yt/library/ytprof/proto/profile.proto +) +target_proto_addincls(library-ytprof-proto + ./yt + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/yt + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(library-ytprof-proto + --cpp_out=${CMAKE_BINARY_DIR}/yt + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/yt +) diff --git a/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp new file mode 100644 index 0000000000..3edf6a6f89 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/cpu_profiler_ut.cpp @@ -0,0 +1,343 @@ +#include <dlfcn.h> + +#include <gtest/gtest.h> + +#include <library/cpp/testing/common/env.h> + +#include <library/cpp/yt/memory/new.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/scheduler_api.h> + +#include <yt/yt/core/actions/bind.h> + +#include <yt/yt/core/tracing/trace_context.h> + +#include <yt/yt/library/ytprof/cpu_profiler.h> +#include <yt/yt/library/ytprof/symbolize.h> +#include <yt/yt/library/ytprof/profile.h> +#include <yt/yt/library/ytprof/external_pprof.h> + +#include <util/string/cast.h> +#include <util/stream/file.h> +#include <util/datetime/base.h> +#include <util/system/shellcommand.h> + +#include <yt/yt/core/concurrency/thread_pool.h> + +namespace NYT::NYTProf { +namespace { + +using namespace NConcurrency; +using namespace NTracing; + +using TSampleFilter = TCpuProfilerOptions::TSampleFilter; + +//////////////////////////////////////////////////////////////////////////////// + +template <size_t Index> +Y_NO_INLINE void BurnCpu() +{ + THash<TString> hasher; + ui64 value = 0; + for (int i = 0; i < 10000000; i++) { + value += hasher(ToString(i)); + } + EXPECT_NE(Index, value); +} + +static std::atomic<int> Counter{0}; + +struct TNoTailCall +{ + ~TNoTailCall() + { + Counter++; + } +}; + +static Y_NO_INLINE void StaticFunction() +{ + TNoTailCall noTail; + BurnCpu<0>(); +} + +void RunUnderProfiler( + const TString& name, + std::function<void()> work, + bool checkSamples = true, + const std::vector<TSampleFilter>& filters = {}, + bool expectEmpty = false) +{ + TCpuProfilerOptions options; + options.SampleFilters = filters; + options.SamplingFrequency = 100000; + options.RecordActionRunTime = true; + +#ifdef YTPROF_DEBUG_BUILD + options.SamplingFrequency = 100; +#endif + + TCpuProfiler profiler(options); + + profiler.Start(); + + work(); + + profiler.Stop(); + + auto profile = profiler.ReadProfile(); + if (checkSamples) { + ASSERT_EQ(expectEmpty, profile.sampleSize() == 0); + } + + Symbolize(&profile, true); + AddBuildInfo(&profile, TBuildInfo::GetDefault()); + SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ + .TmpDir = GetOutputPath(), + .KeepTmpDir = true, + .RunTool = [] (const std::vector<TString>& args) { + TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}}; + command.Run(); + + EXPECT_TRUE(command.GetExitCode() == 0) + << command.GetError(); + }, + }); + + TFileOutput output(GetOutputPath() / name); + WriteProfile(&output, profile); + output.Finish(); +} + +class TCpuProfilerTest + : public ::testing::Test +{ + void SetUp() override + { + if (!IsProfileBuild()) { + GTEST_SKIP() << "rebuild with --build=profile"; + } + } +}; + +TEST_F(TCpuProfilerTest, SingleThreadRun) +{ + RunUnderProfiler("single_thread.pb.gz", [] { + BurnCpu<0>(); + }); +} + +TEST_F(TCpuProfilerTest, MultipleThreads) +{ + RunUnderProfiler("multiple_threads.pb.gz", [] { + std::thread t1([] { + BurnCpu<1>(); + }); + + std::thread t2([] { + BurnCpu<2>(); + }); + + t1.join(); + t2.join(); + }); +} + +TEST_F(TCpuProfilerTest, StaticFunction) +{ + RunUnderProfiler("static_function.pb.gz", [] { + StaticFunction(); + }); +} + +Y_NO_INLINE void RecursiveFunction(int n) +{ + TNoTailCall noTail; + if (n == 0) { + BurnCpu<0>(); + } else { + RecursiveFunction(n-1); + } +} + +TEST_F(TCpuProfilerTest, DeepRecursion) +{ + RunUnderProfiler("recursive_function.pb.gz", [] { + RecursiveFunction(1024); + }); +} + +TEST_F(TCpuProfilerTest, DlOpen) +{ + RunUnderProfiler("dlopen.pb.gz", [] { + auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso/libtestso.so"); + + auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY); + ASSERT_TRUE(dl); + + auto sym = dlsym(dl, "CallNext"); + ASSERT_TRUE(sym); + + auto callNext = reinterpret_cast<void(*)(void(*)())>(sym); + callNext(BurnCpu<0>); + }); +} + +TEST_F(TCpuProfilerTest, DlClose) +{ + RunUnderProfiler("dlclose.pb.gz", [] { + auto libraryPath = BinaryPath("yt/yt/library/ytprof/unittests/testso1/libtestso1.so"); + + auto dl = dlopen(libraryPath.c_str(), RTLD_LAZY); + ASSERT_TRUE(dl); + + auto sym = dlsym(dl, "CallOtherNext"); + ASSERT_TRUE(sym); + + auto callNext = reinterpret_cast<void(*)(void(*)())>(sym); + callNext(BurnCpu<0>); + + ASSERT_EQ(dlclose(dl), 0); + }); +} + +void ReadUrandom() +{ + TIFStream input("/dev/urandom"); + + std::array<char, 1 << 20> buffer; + + for (int i = 0; i < 100; i++) { + input.Read(buffer.data(), buffer.size()); + } +} + +TEST_F(TCpuProfilerTest, Syscalls) +{ + RunUnderProfiler("syscalls.pb.gz", [] { + ReadUrandom(); + }); +} + +TEST_F(TCpuProfilerTest, VDSO) +{ + RunUnderProfiler("vdso.pb.gz", [] { + auto now = TInstant::Now(); + while (TInstant::Now() < now + TDuration::MilliSeconds(100)) + { } + }, false); +} + +TEST_F(TCpuProfilerTest, ProfilerTags) +{ + auto userTag = New<TProfilerTag>("user", "prime"); + auto intTag = New<TProfilerTag>("block_size", 1024); + + RunUnderProfiler("tags.pb.gz", [&] { + { + TCpuProfilerTagGuard guard(userTag); + BurnCpu<0>(); + } + { + TCpuProfilerTagGuard guard(intTag); + BurnCpu<1>(); + } + { + TCpuProfilerTagGuard guard(userTag); + TCpuProfilerTagGuard secondGuard(intTag); + BurnCpu<2>(); + } + }); +} + +TEST_F(TCpuProfilerTest, MultipleProfilers) +{ + TCpuProfiler profiler, secondProfiler; + + profiler.Start(); + EXPECT_THROW(secondProfiler.Start(), std::exception); +} + +TEST_F(TCpuProfilerTest, TraceContext) +{ + RunUnderProfiler("trace_context.pb.gz", [] { + auto actionQueue = New<TActionQueue>("CpuProfileTest"); + + BIND([] { + auto rootTraceContext = TTraceContext::NewRoot(""); + rootTraceContext->AddProfilingTag("user", "prime"); + TCurrentTraceContextGuard guard(rootTraceContext); + + auto asyncSubrequest = BIND([&] { + TChildTraceContextGuard guard(""); + AnnotateTraceContext([] (const auto traceContext) { + traceContext->AddProfilingTag("table", "//foo"); + }); + + BurnCpu<0>(); + }) + .AsyncVia(GetCurrentInvoker()) + .Run(); + + BurnCpu<1>(); + WaitFor(asyncSubrequest) + .ThrowOnError(); + }) + .AsyncVia(actionQueue->GetInvoker()) + .Run() + .Get(); + + actionQueue->Shutdown(); + }); +} + +TEST_F(TCpuProfilerTest, SlowActions) +{ + static const TString WorkerThreadName = "Heavy"; + static const auto TraceThreshold = TDuration::MilliSeconds(50); + + const std::vector<TSampleFilter> filters = { + GetActionMinExecTimeFilter(TraceThreshold), + }; + + auto busyWait = [] (TDuration duration) { + auto now = TInstant::Now(); + while (TInstant::Now() < now + duration) + { } + }; + + auto traceContext = NTracing::TTraceContext::NewRoot("Test"); + + const bool ExpectEmptyTraces = true; + + auto threadPool = CreateThreadPool(2, WorkerThreadName); + + // No slow actions. + RunUnderProfiler("slow_actions_empty.pb.gz", [&] { + NTracing::TTraceContextGuard guard(traceContext); + auto future = BIND(busyWait, TraceThreshold / 2) + .AsyncVia(threadPool->GetInvoker()) + .Run(); + future.Get(); + }, + true, + filters, + ExpectEmptyTraces); + + // Slow actions = non empty traces. + RunUnderProfiler("slow_actions.pb.gz", [&] { + NTracing::TTraceContextGuard guard(traceContext); + auto future = BIND(busyWait, TraceThreshold * 3) + .AsyncVia(threadPool->GetInvoker()) + .Run(); + future.Get(); + }, + true, + filters); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp new file mode 100644 index 0000000000..42dcf6e802 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp @@ -0,0 +1,226 @@ +#include <gtest/gtest.h> + +#include <yt/yt/library/ytprof/heap_profiler.h> +#include <yt/yt/library/ytprof/symbolize.h> +#include <yt/yt/library/ytprof/profile.h> + +#include <yt/yt/core/actions/current_invoker.h> +#include <yt/yt/core/actions/invoker_detail.h> + +#include <yt/yt/core/concurrency/action_queue.h> + +#include <yt/yt/core/misc/lazy_ptr.h> + +#include <yt/yt/core/tracing/allocation_tags.h> +#include <yt/yt/core/tracing/trace_context.h> + +#include <library/cpp/testing/common/env.h> + +#include <util/string/cast.h> +#include <util/stream/file.h> +#include <util/generic/hash_set.h> +#include <util/datetime/base.h> +#include <util/generic/size_literals.h> + +#include <tcmalloc/common.h> + +#include <absl/debugging/stacktrace.h> + +namespace NYT::NYTProf { +namespace { + +using namespace NTracing; + +//////////////////////////////////////////////////////////////////////////////// + +constexpr auto MemoryAllocationTag = "memory_allocation_tag"; +const std::vector<TString> MemoryAllocationTags = {"0", "1", "2", "3", "4", "5", "6", "7"}; + +//////////////////////////////////////////////////////////////////////////////// + +template <size_t Index> +Y_NO_INLINE auto BlowHeap() +{ + std::vector<TString> data; + for (int i = 0; i < 10240; i++) { + data.push_back(TString(1024, 'x')); + } + return data; +} + +TEST(HeapProfiler, ReadProfile) +{ + absl::SetStackUnwinder(AbslStackUnwinder); + tcmalloc::MallocExtension::SetProfileSamplingRate(256_KB); + + auto token = tcmalloc::MallocExtension::StartAllocationProfiling(); + + EnableMemoryProfilingTags(); + auto traceContext = TTraceContext::NewRoot("Root"); + TTraceContextGuard guard(traceContext); + + traceContext->SetAllocationTags({{"user", "first"}, {"sometag", "my"}}); + + auto h0 = BlowHeap<0>(); + + auto tag = TMemoryTag(1); + traceContext->SetAllocationTags({{"user", "second"}, {"sometag", "notmy"}, {MemoryAllocationTag, ToString(tag)}}); + auto currentTag = traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag); + ASSERT_EQ(currentTag, tag); + + auto h1 = BlowHeap<1>(); + + traceContext->ClearAllocationTagsPtr(); + + auto h2 = BlowHeap<2>(); + h2.clear(); + + auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, ToString(tag)); + ASSERT_GE(usage, 5_MB); + + auto dumpProfile = [] (auto name, auto type) { + auto profile = ReadHeapProfile(type); + + TFileOutput output(GetOutputPath() / name); + WriteProfile(&output, profile); + output.Finish(); + }; + + dumpProfile("heap.pb.gz", tcmalloc::ProfileType::kHeap); + dumpProfile("peak.pb.gz", tcmalloc::ProfileType::kPeakHeap); + dumpProfile("fragmentation.pb.gz", tcmalloc::ProfileType::kFragmentation); + dumpProfile("allocations.pb.gz", tcmalloc::ProfileType::kAllocations); + + auto profile = std::move(token).Stop(); + + TFileOutput output(GetOutputPath() / "allocations.pb.gz"); + WriteProfile(&output, ConvertAllocationProfile(profile)); + output.Finish(); +} + +TEST(HeapProfiler, AllocationTagsWithMemoryTag) +{ + EnableMemoryProfilingTags(); + auto traceContext = TTraceContext::NewRoot("Root"); + TTraceContextGuard guard(traceContext); + + ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), std::nullopt); + traceContext->SetAllocationTags({{"user", "first user"}, {MemoryAllocationTag, MemoryAllocationTags[0]}}); + ASSERT_EQ(traceContext->FindAllocationTag<TString>("user"), "first user"); + ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), MemoryAllocationTags[0]); + + std::vector<std::vector<TString>> heap; + heap.push_back(BlowHeap<0>()); + + traceContext->SetAllocationTags({{"user", "second user"}, {MemoryAllocationTag, MemoryAllocationTags[1]}}); + ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag), 1); + + heap.push_back(BlowHeap<1>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[0]); + + auto usage1 = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[1]); + + ASSERT_NEAR(usage1, 12_MB, 8_MB); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[2]); + ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), MemoryAllocationTags[2]); + + { + volatile auto h = BlowHeap<2>(); + } + + traceContext->ClearAllocationTagsPtr(); + ASSERT_EQ(traceContext->FindAllocationTag<TString>(MemoryAllocationTag), std::nullopt); + + heap.push_back(BlowHeap<0>()); + + { + auto snapshot = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag); + ASSERT_EQ(snapshot[MemoryAllocationTags[1]], usage1); + ASSERT_LE(snapshot[MemoryAllocationTags[2]], 1_MB); + } + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[6]); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[3]); + heap.push_back(BlowHeap<3>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[4]); + heap.push_back(BlowHeap<4>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[7]); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[5]); + heap.push_back(BlowHeap<5>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[4]); + heap.push_back(BlowHeap<4>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[7]); + + traceContext->SetAllocationTagsPtr(nullptr); + + auto snapshot = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag); + + constexpr auto maxDifference = 10_MB; + ASSERT_NEAR(snapshot[MemoryAllocationTags[1]], snapshot[MemoryAllocationTags[3]], maxDifference); + ASSERT_NEAR(snapshot[MemoryAllocationTags[3]], snapshot[MemoryAllocationTags[5]], maxDifference); + ASSERT_NEAR(snapshot[MemoryAllocationTags[1]], snapshot[MemoryAllocationTags[5]], maxDifference); + + ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], 20_MB, 15_MB); + + ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[1]] + snapshot[MemoryAllocationTags[3]], 2 * maxDifference); + ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[1]] + snapshot[MemoryAllocationTags[5]], 2 * maxDifference); + ASSERT_NEAR(snapshot[MemoryAllocationTags[4]], snapshot[MemoryAllocationTags[3]] + snapshot[MemoryAllocationTags[5]], 2 * maxDifference); + + ASSERT_LE(snapshot[MemoryAllocationTags[6]], 1_MB); + ASSERT_LE(snapshot[MemoryAllocationTags[7]], 1_MB); +} + +template <size_t Index> +Y_NO_INLINE auto BlowHeap(int64_t megabytes) +{ + std::vector<TString> data; + megabytes <<= 10; + for (int64_t i = 0; i < megabytes; i++) { + data.push_back(TString( 1024, 'x')); + } + return data; +} + +TEST(HeapProfiler, HugeAllocationsTagsWithMemoryTag) +{ + EnableMemoryProfilingTags(); + auto traceContext = TTraceContext::NewRoot("Root"); + TCurrentTraceContextGuard guard(traceContext); + + std::vector<std::vector<TString>> heap; + + heap.push_back(BlowHeap<0>()); + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[1]); + ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag), 1); + + heap.push_back(BlowHeap<1>(100)); + + { + traceContext->SetAllocationTagsPtr(nullptr); + auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[1]); + ASSERT_GE(usage, 100_MB); + ASSERT_LE(usage, 150_MB); + } + + traceContext->SetAllocationTag(MemoryAllocationTag, MemoryAllocationTags[2]); + heap.push_back(BlowHeap<1>(1000)); + + traceContext->SetAllocationTagsPtr(nullptr); + auto usage = CollectMemoryUsageSnapshot()->GetUsage(MemoryAllocationTag, MemoryAllocationTags[2]); + ASSERT_GE(usage, 1000_MB); + ASSERT_LE(usage, 1300_MB); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/queue_ut.cpp b/yt/yt/library/ytprof/unittests/queue_ut.cpp new file mode 100644 index 0000000000..f44ae8da32 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/queue_ut.cpp @@ -0,0 +1,66 @@ +#include <gtest/gtest.h> + +#include <yt/yt/library/ytprof/queue.h> + +namespace NYT::NYTProf { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(StaticQueue, PushPop) +{ + TStaticQueue queue(10); + + int a, b, c; + void* aptr = reinterpret_cast<void*>(&a); + void* bptr = reinterpret_cast<void*>(&b); + void* cptr = reinterpret_cast<void*>(&c); + + std::vector<void*> backtrace; + auto getBacktrace = [&] () -> std::pair<void*, bool> { + if (backtrace.empty()) { + return {nullptr, false}; + } + + auto ip = backtrace.front(); + backtrace.erase(backtrace.begin()); + return {ip, true}; + }; + + ASSERT_TRUE(queue.TryPush(getBacktrace)); + + backtrace = {aptr, bptr, cptr}; + ASSERT_TRUE(queue.TryPush(getBacktrace)); + + backtrace.push_back(aptr); + ASSERT_TRUE(queue.TryPush(getBacktrace)); + + auto readBacktrace = [&] (void *ip) { + backtrace.push_back(ip); + }; + + ASSERT_TRUE(queue.TryPop(readBacktrace)); + ASSERT_EQ(backtrace, std::vector<void*>{}); + + backtrace.clear(); + ASSERT_TRUE(queue.TryPop(readBacktrace)); + ASSERT_EQ(backtrace, (std::vector<void*>{aptr, bptr, cptr})); + + backtrace.clear(); + ASSERT_TRUE(queue.TryPop(readBacktrace)); + ASSERT_EQ(backtrace, (std::vector<void*>{aptr})); + + ASSERT_FALSE(queue.TryPop(readBacktrace)); +} + +TEST(StaticQueue, Overflow) +{ + TStaticQueue queue(10); + + ASSERT_FALSE(queue.TryPush([] () -> std::pair<void*, bool> { + return {nullptr, true}; + })); +} + +} // namespace +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp new file mode 100644 index 0000000000..1cf315155c --- /dev/null +++ b/yt/yt/library/ytprof/unittests/spinlock_profiler_ut.cpp @@ -0,0 +1,172 @@ +#include <gtest/gtest.h> + +#include <library/cpp/testing/common/env.h> + +#include <yt/yt/library/ytprof/spinlock_profiler.h> +#include <yt/yt/library/ytprof/symbolize.h> +#include <yt/yt/library/ytprof/profile.h> +#include <yt/yt/library/ytprof/external_pprof.h> + +#include <tcmalloc/malloc_extension.h> + +#include <library/cpp/yt/threading/spin_lock.h> + +#include <util/string/cast.h> +#include <util/stream/file.h> +#include <util/datetime/base.h> +#include <util/system/compiler.h> +#include <util/system/shellcommand.h> +#include <util/thread/lfstack.h> +#include <util/generic/size_literals.h> + +namespace NYT::NYTProf { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TProfiler> +void RunUnderProfiler(const TString& name, std::function<void()> work, bool checkSamples = true) +{ + TSpinlockProfilerOptions options; + options.ProfileFraction = 10; + + TProfiler profiler(options); + + profiler.Start(); + + work(); + + profiler.Stop(); + + auto profile = profiler.ReadProfile(); + if (checkSamples) { + ASSERT_NE(0, profile.sample_size()); + } + + Symbolize(&profile, true); + AddBuildInfo(&profile, TBuildInfo::GetDefault()); + SymbolizeByExternalPProf(&profile, TSymbolizationOptions{ + .TmpDir = GetOutputPath(), + .KeepTmpDir = true, + .RunTool = [] (const std::vector<TString>& args) { + TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}}; + command.Run(); + + EXPECT_TRUE(command.GetExitCode() == 0) + << command.GetError(); + }, + }); + + TFileOutput output(GetOutputPath() / name); + WriteProfile(&output, profile); + output.Finish(); +} + +class TSpinlockProfilerTest + : public ::testing::Test +{ + void SetUp() override + { + if (!IsProfileBuild()) { + GTEST_SKIP() << "rebuild with --build=profile"; + } + } +}; + +TEST_F(TSpinlockProfilerTest, PageHeapLock) +{ + RunUnderProfiler<TSpinlockProfiler>("pageheap_lock.pb.gz", [] { + std::atomic<bool> Stop = false; + + std::thread release([&] { + while (!Stop) { + tcmalloc::MallocExtension::ReleaseMemoryToSystem(1_GB); + } + }); + + std::vector<std::thread> allocators; + for (int i = 0; i < 16; i++) { + allocators.emplace_back([&] { + while (!Stop) { + auto ptr = malloc(4_MB); + DoNotOptimizeAway(ptr); + free(ptr); + } + }); + } + + Sleep(TDuration::Seconds(5)); + Stop = true; + + release.join(); + for (auto& t : allocators) { + t.join(); + } + }); +} + + +TEST_F(TSpinlockProfilerTest, TransferCacheLock) +{ + RunUnderProfiler<TSpinlockProfiler>("transfer_cache_lock.pb.gz", [] { + std::atomic<bool> Stop = false; + + TLockFreeStack<int> stack; + std::thread producer([&] { + while (!Stop) { + stack.Enqueue(1); + } + }); + + std::thread consumer([&] { + while (!Stop) { + int value; + stack.Dequeue(&value); + } + }); + + Sleep(TDuration::Seconds(5)); + + Stop = true; + producer.join(); + consumer.join(); + }); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST_F(TSpinlockProfilerTest, YTLocks) +{ + RunUnderProfiler<TBlockingProfiler>("ytlock.pb.gz", [] { + std::atomic<bool> Stop = false; + + NThreading::TSpinLock lock; + std::thread slow([&] { + while (!Stop) { + lock.Acquire(); + Sleep(TDuration::MilliSeconds(10)); + lock.Release(); + Sleep(TDuration::MilliSeconds(10)); + } + }); + + std::thread fast([&] { + while (!Stop) { + lock.Acquire(); + lock.Release(); + } + }); + + Sleep(TDuration::Seconds(5)); + + Stop = true; + slow.join(); + fast.join(); + }); +} + +//////////////////////////////////////////////////////////////////////////////// + + +} // namespace +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp b/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp new file mode 100644 index 0000000000..646ecaafad --- /dev/null +++ b/yt/yt/library/ytprof/unittests/symbolizer_ut.cpp @@ -0,0 +1,83 @@ +#include <gtest/gtest.h> + +#include <yt/yt/library/ytprof/symbolize.h> +#include <yt/yt/library/ytprof/build_info.h> + +namespace NYT::NYTProf { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +Y_NO_INLINE void* GetIP() +{ + return __builtin_return_address(0); +} + +TEST(Symbolize, EmptyProfile) +{ + NProto::Profile profile; + profile.add_string_table(); + + Symbolize(&profile); + AddBuildInfo(&profile, TBuildInfo::GetDefault()); +} + +TEST(Symbolize, SingleLocation) +{ + NProto::Profile profile; + profile.add_string_table(); + + auto thisIP = GetIP(); + + { + auto location = profile.add_location(); + location->set_address(reinterpret_cast<ui64>(thisIP)); + + auto line = location->add_line(); + line->set_function_id(reinterpret_cast<ui64>(thisIP)); + + auto function = profile.add_function(); + function->set_id(reinterpret_cast<ui64>(thisIP)); + } + + Symbolize(&profile); + + ASSERT_EQ(1, profile.function_size()); + auto function = profile.function(0); + + auto name = profile.string_table(function.name()); + ASSERT_TRUE(name.find("SingleLocation") != TString::npos) + << "function name is " << name; +} + +TEST(Symbolize, GetBuildId) +{ + if (!IsProfileBuild()) { + GTEST_SKIP(); + } + + return; + + auto buildId = GetBuildId(); + ASSERT_TRUE(buildId); + ASSERT_NE(*buildId, TString{""}); +} + +TEST(BuildInfo, Test) +{ + if (!IsProfileBuild()) { + GTEST_SKIP(); + } + + auto info = TBuildInfo::GetDefault(); + if (IsProfileBuild()) { + ASSERT_EQ(info.BuildType, "profile"); + } + + ASSERT_NE(info.ArcRevision, ""); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYTProf diff --git a/yt/yt/library/ytprof/unittests/testso/testso.cpp b/yt/yt/library/ytprof/unittests/testso/testso.cpp new file mode 100644 index 0000000000..4209d5f6cc --- /dev/null +++ b/yt/yt/library/ytprof/unittests/testso/testso.cpp @@ -0,0 +1,9 @@ +#include <atomic> + +static std::atomic<int> CallCount; + +extern "C" void CallNext(void (*next)()) +{ + next(); + CallCount++; +} diff --git a/yt/yt/library/ytprof/unittests/testso/ya.make b/yt/yt/library/ytprof/unittests/testso/ya.make new file mode 100644 index 0000000000..8851d8eed6 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/testso/ya.make @@ -0,0 +1,7 @@ +DLL(testso) + +SRCS( + testso.cpp +) + +END() diff --git a/yt/yt/library/ytprof/unittests/testso1/testso.cpp b/yt/yt/library/ytprof/unittests/testso1/testso.cpp new file mode 100644 index 0000000000..56e348c020 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/testso1/testso.cpp @@ -0,0 +1,9 @@ +#include <atomic> + +static std::atomic<int> CallCount; + +extern "C" void CallOtherNext(void (*next)()) +{ + next(); + CallCount++; +} diff --git a/yt/yt/library/ytprof/unittests/testso1/ya.make b/yt/yt/library/ytprof/unittests/testso1/ya.make new file mode 100644 index 0000000000..d0d2f90143 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/testso1/ya.make @@ -0,0 +1,7 @@ +DLL(testso1) + +SRCS( + testso.cpp +) + +END() diff --git a/yt/yt/library/ytprof/unittests/ya.make b/yt/yt/library/ytprof/unittests/ya.make new file mode 100644 index 0000000000..8c8bce98f5 --- /dev/null +++ b/yt/yt/library/ytprof/unittests/ya.make @@ -0,0 +1,43 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + symbolizer_ut.cpp + cpu_profiler_ut.cpp + heap_profiler_ut.cpp + spinlock_profiler_ut.cpp + queue_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) + +PEERDIR( + yt/yt/library/ytprof + yt/yt/library/profiling + yt/yt/core +) + +IF (OS_LINUX) + LDFLAGS("-Wl,--build-id=sha1") +ENDIF() + +DEPENDS( + yt/yt/library/ytprof/unittests/testso + yt/yt/library/ytprof/unittests/testso1 +) + +IF (BUILD_TYPE != "release" AND BUILD_TYPE != "relwithdebinfo") + CFLAGS(-DYTPROF_DEBUG_BUILD) +ENDIF() + +SIZE(MEDIUM) + +ALLOCATOR(TCMALLOC_256K) + +END() + +RECURSE( + testso + testso1 +) |