aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2024-12-05 09:35:47 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2024-12-05 09:48:57 +0300
commit1fef94960daef6658da8883f15004a0c2e9be983 (patch)
tree1ea968d1fd85f97f43a5e80148c2563385e59c35
parentd2daa783976633567d093f5c5cb8051c4640f680 (diff)
downloadydb-1fef94960daef6658da8883f15004a0c2e9be983.tar.gz
Intermediate changes
commit_hash:1202dc6865ea9a18b824d684fc2c165ad191e1bd
-rw-r--r--yt/yt/library/oom/oom.h17
-rw-r--r--yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp312
-rw-r--r--yt/yt/library/oom/ya.make2
-rw-r--r--yt/yt/library/tcmalloc/config.cpp14
-rw-r--r--yt/yt/library/tcmalloc/config.h16
-rw-r--r--yt/yt/library/tcmalloc/tcmalloc_manager.cpp338
-rw-r--r--yt/yt/library/tcmalloc/ya.make2
7 files changed, 336 insertions, 365 deletions
diff --git a/yt/yt/library/oom/oom.h b/yt/yt/library/oom/oom.h
index 1fa01194dc..7a5892918a 100644
--- a/yt/yt/library/oom/oom.h
+++ b/yt/yt/library/oom/oom.h
@@ -2,9 +2,7 @@
#include <optional>
-#include <util/system/types.h>
#include <util/generic/string.h>
-#include <util/datetime/base.h>
namespace NYT {
@@ -20,19 +18,4 @@ void EnableEarlyOomWatchdog(TOomWatchdogOptions options);
////////////////////////////////////////////////////////////////////////////////
-struct TTCMallocLimitHandlerOptions
-{
- TString HeapDumpDirectory;
-
- // Files structure would have the following form:
- // HeapDumpDirectory/<ActualName>_FilenameSuffix_Timestamp.ext.
- std::optional<TString> FilenameSuffix;
- TDuration Timeout = TDuration::Minutes(5);
-};
-
-void EnableTCMallocLimitHandler(TTCMallocLimitHandlerOptions options);
-void DisableTCMallocLimitHandler();
-
-////////////////////////////////////////////////////////////////////////////////
-
} // namespace NYT
diff --git a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp b/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp
deleted file mode 100644
index cad154d211..0000000000
--- a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp
+++ /dev/null
@@ -1,312 +0,0 @@
-#include "oom.h"
-
-#include <yt/yt/library/ytprof/external_pprof.h>
-#include <yt/yt/library/ytprof/heap_profiler.h>
-#include <yt/yt/library/ytprof/profile.h>
-#include <yt/yt/library/ytprof/spinlock_profiler.h>
-#include <yt/yt/library/ytprof/symbolize.h>
-
-#include <yt/yt/core/misc/crash_handler.h>
-#include <yt/yt/core/misc/error.h>
-
-#include <yt/yt/core/ytree/yson_struct.h>
-
-#include <yt/yt/core/yson/writer.h>
-
-#include <library/cpp/yt/string/format.h>
-
-#include <library/cpp/yt/memory/atomic_intrusive_ptr.h>
-
-#include <library/cpp/yt/system/exit.h>
-
-#include <util/datetime/base.h>
-#include <util/stream/file.h>
-#include <util/stream/output.h>
-#include <util/string/split.h>
-#include <util/system/env.h>
-#include <util/system/file.h>
-#include <util/system/fs.h>
-#include <util/system/shellcommand.h>
-
-#include <mutex>
-#include <thread>
-
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
-namespace {
-
-TString MakeIncompletePath(const TString& path)
-{
- return NYT::Format("%v_incomplete", path);
-}
-
-} // namespace
-
-////////////////////////////////////////////////////////////////////////////////
-
-void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::ProfileType profileType)
-{
- auto profile = NYTProf::ReadHeapProfile(profileType);
- SymbolizeByExternalPProf(&profile, NYTProf::TSymbolizationOptions{
- .RunTool = [] (const std::vector<TString>& args) {
- TShellCommand command{args[0], TList<TString>{args.begin()+1, args.end()}};
- command.Run();
- },
- });
-
- auto incompletePath = NYT::MakeIncompletePath(memoryProfilePath);
-
- TFileOutput output(incompletePath);
- NYTProf::WriteProfile(&output, profile);
- output.Finish();
- NFs::Rename(incompletePath, memoryProfilePath);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-namespace {
-
-void MemoryProfileTimeoutHandler(int /*signal*/)
-{
- AbortProcessDramatically(
- EProcessExitCode::GenericError,
- "Process hung while dumping heap profile");
-}
-
-} // namespace
-
-void SetupMemoryProfileTimeout(int timeout)
-{
- ::signal(SIGALRM, &MemoryProfileTimeoutHandler);
- ::alarm(timeout);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-DECLARE_REFCOUNTED_STRUCT(TOomProfilePaths)
-
-struct TOomProfilePaths
- : public NYTree::TYsonStruct
-{
- TString HeapProfilePath;
- TString PeakProfilePath;
-
- REGISTER_YSON_STRUCT(TOomProfilePaths);
-
- static void Register(TRegistrar registrar)
- {
- registrar.Parameter("heap_profile_path", &TThis::HeapProfilePath)
- .Default();
- registrar.Parameter("peak_profile_path", &TThis::PeakProfilePath)
- .Default();
- }
-};
-
-DEFINE_REFCOUNTED_TYPE(TOomProfilePaths)
-
-////////////////////////////////////////////////////////////////////////////////
-
-void DumpProfilePaths(const TOomProfilePathsPtr& links, const TString& fileName)
-{
- TFileOutput output(fileName);
- NYson::TYsonWriter writer(&output, NYson::EYsonFormat::Pretty);
- Serialize(links, &writer);
- writer.Flush();
- output.Finish();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TTCMallocLimitHandler
- : public TRefCounted
-{
-public:
- explicit TTCMallocLimitHandler(TTCMallocLimitHandlerOptions options)
- : Options_(options)
- {
- Thread_ = std::thread([this] {
- Handle();
- });
- }
-
- ~TTCMallocLimitHandler()
- {
- {
- std::unique_lock<std::mutex> lock(Mutex_);
- Fired_ = true;
- CV_.notify_all();
- }
-
- Thread_.join();
- }
-
- void Fire()
- {
- std::unique_lock<std::mutex> lock(Mutex_);
- Fired_ = true;
- NeedToHandle_ = true;
- CV_.notify_all();
- }
-
-private:
- const TTCMallocLimitHandlerOptions Options_;
-
- bool Fired_ = false;
- bool NeedToHandle_ = false;
- std::mutex Mutex_;
- std::condition_variable CV_;
- std::thread Thread_;
-
- void Handle()
- {
- std::unique_lock<std::mutex> lock(Mutex_);
- CV_.wait(lock, [&] {
- return Fired_;
- });
-
- if (!NeedToHandle_) {
- return;
- }
-
- auto timestamp = TInstant::Now().FormatLocalTime("%Y%m%dT%H%M%S");
- auto profilePaths = New<TOomProfilePaths>();
- auto profilePathsFile = GetProfilePaths(timestamp);
- profilePaths->HeapProfilePath = GetHeapDumpPath(timestamp);
- profilePaths->PeakProfilePath = GetPeakDumpPath(timestamp);
-
- Cerr << "TTCMallocLimitHandler: Fork process to write heap profile: "
- << profilePaths->HeapProfilePath
- << " peak profile path: " << profilePaths->PeakProfilePath
- << " profiles path file: " << profilePathsFile
- << Endl;
-
- SetupMemoryProfileTimeout(Options_.Timeout.Seconds());
- auto childPid = fork();
-
- if (childPid == 0) {
- NFs::MakeDirectoryRecursive(Options_.HeapDumpDirectory);
- SetupMemoryProfileTimeout(Options_.Timeout.Seconds());
- CollectAndDumpMemoryProfile(profilePaths->HeapProfilePath, tcmalloc::ProfileType::kHeap);
- CollectAndDumpMemoryProfile(profilePaths->PeakProfilePath, tcmalloc::ProfileType::kPeakHeap);
- DumpProfilePaths(profilePaths, profilePathsFile);
-
- Cerr << "TTCMallocLimitHandler: Heap profiles are written" << Endl;
- AbortProcessSilently(EProcessExitCode::OK);
- }
-
- if (childPid < 0) {
- Cerr << "TTCMallocLimitHandler: Fork failed: " << LastSystemErrorText() << Endl;
- AbortProcessSilently(EProcessExitCode::GenericError);
- }
-
- ExecWaitForChild(childPid);
- AbortProcessSilently(EProcessExitCode::OK);
- }
-
- auto MakeSuffixFormatter(const TString& timestamp) const
- {
- return NYT::MakeFormatterWrapper([this, &timestamp] (TStringBuilderBase* builder) {
- if (Options_.FilenameSuffix) {
- builder->AppendFormat("%v_", *Options_.FilenameSuffix);
- }
- FormatValue(builder, timestamp, "v");
- });
- }
-
- TString GetHeapDumpPath(const TString& timestamp) const
- {
- return Format(
- "%v/heap_%v.pb.gz",
- Options_.HeapDumpDirectory,
- MakeSuffixFormatter(timestamp));
- }
-
- TString GetPeakDumpPath(const TString& timestamp) const
- {
- return Format(
- "%v/peak_%v.pb.gz",
- Options_.HeapDumpDirectory,
- MakeSuffixFormatter(timestamp));
- }
-
- TString GetProfilePaths(const TString& timestamp) const
- {
- return Format(
- "%v/oom_profile_paths_%v.yson",
- Options_.HeapDumpDirectory,
- MakeSuffixFormatter(timestamp));
- }
-
- void ExecWaitForChild(int pid)
- {
- Cerr << "TTCMallocLimitHandler: Start waiting for the child" << Endl;
-
- auto command = Format("while [ -e /proc/%v ]; do sleep 1; done;", pid);
- execl("/bin/bash", "/bin/bash", "-c", command.c_str(), (void*)nullptr);
-
- Cerr << "TTCMallocLimitHandler: Failed to switch main process to dummy child waiter: "
- << LastSystemErrorText() << Endl;
- }
-};
-
-DEFINE_REFCOUNTED_TYPE(TTCMallocLimitHandler);
-
-////////////////////////////////////////////////////////////////////////////////
-
-template <class TMallocExtension>
-concept CSupportsLimitHandler = requires (TMallocExtension extension)
-{
- { extension.GetSoftMemoryLimitHandler() };
-};
-
-template <typename TMallocExtension, typename THandler>
-void SetSoftMemoryLimitHandler(THandler)
-{
- WriteToStderr("TCMalloc does not support memory limit handler\n");
-}
-
-template <CSupportsLimitHandler TMallocExtension, typename THandler>
-void SetSoftMemoryLimitHandler(THandler handler)
-{
- TMallocExtension::SetSoftMemoryLimitHandler(handler);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-namespace {
-
-YT_DEFINE_GLOBAL(TAtomicIntrusivePtr<TTCMallocLimitHandler>, LimitHandler);
-
-void HandleTCMallocLimit()
-{
- if (auto handler = LimitHandler().Acquire()) {
- handler->Fire();
- }
-}
-
-} // namespace
-
-void EnableTCMallocLimitHandler(TTCMallocLimitHandlerOptions options)
-{
- {
- if (LimitHandler().Acquire()) {
- return;
- }
-
- TAtomicIntrusivePtr<TTCMallocLimitHandler>::TRawPtr expected = nullptr;
- LimitHandler().CompareAndSwap(expected, New<TTCMallocLimitHandler>(options));
- }
-
- SetSoftMemoryLimitHandler<tcmalloc::MallocExtension>(&HandleTCMallocLimit);
-}
-
-void DisableTCMallocLimitHandler()
-{
- LimitHandler().Reset();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
diff --git a/yt/yt/library/oom/ya.make b/yt/yt/library/oom/ya.make
index 5ad7018578..f4845495d8 100644
--- a/yt/yt/library/oom/ya.make
+++ b/yt/yt/library/oom/ya.make
@@ -4,13 +4,11 @@ INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc)
SRCS(
oom.cpp
- tcmalloc_memory_limit_handler.cpp
)
PEERDIR(
yt/yt/core
yt/yt/library/ytprof
- library/cpp/yt/logging
)
END()
diff --git a/yt/yt/library/tcmalloc/config.cpp b/yt/yt/library/tcmalloc/config.cpp
index 7629617ba2..58e3927e5a 100644
--- a/yt/yt/library/tcmalloc/config.cpp
+++ b/yt/yt/library/tcmalloc/config.cpp
@@ -16,17 +16,25 @@ void THeapSizeLimitConfig::Register(TRegistrar registrar)
.Default(false);
registrar.Parameter("dump_memory_profile_on_violation", &TThis::DumpMemoryProfileOnViolation)
.Default(false);
- registrar.Parameter("dump_memory_profile_timeout", &TThis::DumpMemoryProfileTimeout)
+ registrar.Parameter("memory_profile_dump_timeout", &TThis::MemoryProfileDumpTimeout)
.Default(TDuration::Minutes(10));
- registrar.Parameter("dump_memory_profile_path", &TThis::DumpMemoryProfilePath)
+ registrar.Parameter("memory_profile_dump_path", &TThis::MemoryProfileDumpPath)
.Default();
+ registrar.Parameter("emory_profile_dump_filename_suffix", &TThis::MemoryProfileDumpFilenameSuffix)
+ .Default();
+
+ registrar.Postprocessor([] (THeapSizeLimitConfig* config) {
+ if (config->DumpMemoryProfileOnViolation && !config->MemoryProfileDumpPath) {
+ THROW_ERROR_EXCEPTION("\"memory_profile_dump_path\" must be set when \"dump_memory_profile_on_violation\" is true");
+ }
+ });
}
TTCMallocConfigPtr TTCMallocConfig::ApplyDynamic(const TTCMallocConfigPtr& dynamicConfig) const
{
// TODO(babenko): fix this mess
auto mergedConfig = CloneYsonStruct(dynamicConfig);
- mergedConfig->HeapSizeLimit->DumpMemoryProfilePath = HeapSizeLimit->DumpMemoryProfilePath;
+ mergedConfig->HeapSizeLimit->MemoryProfileDumpPath = HeapSizeLimit->MemoryProfileDumpPath;
return mergedConfig;
}
diff --git a/yt/yt/library/tcmalloc/config.h b/yt/yt/library/tcmalloc/config.h
index c2060c33bd..a4bf5a313e 100644
--- a/yt/yt/library/tcmalloc/config.h
+++ b/yt/yt/library/tcmalloc/config.h
@@ -12,21 +12,27 @@ struct THeapSizeLimitConfig
: public NYTree::TYsonStruct
{
//! Limit program memory in terms of container memory.
- // If program heap size exceeds the limit tcmalloc is instructed to release memory to the kernel.
+ //! If program heap size exceeds the limit tcmalloc is instructed to release memory to the kernel.
std::optional<double> ContainerMemoryRatio;
//! Similar to #ContainerMemoryRatio, but is set in terms of absolute difference from
//! the container memory limit.
//! For example, if ContainerMemoryLimit=200Gb and ContainerMemoryMargin=1Gb
- // then tcmalloc limit will be 199Gb.
- std::optional<double> ContainerMemoryMargin;
+ //! then tcmalloc limit will be 199Gb.
+ std::optional<i64> ContainerMemoryMargin;
//! If true tcmalloc crashes when system allocates more memory than #ContainerMemoryRatio/#ContainerMemoryMargin.
bool Hard;
bool DumpMemoryProfileOnViolation;
- TDuration DumpMemoryProfileTimeout;
- TString DumpMemoryProfilePath;
+
+ TDuration MemoryProfileDumpTimeout;
+
+ //! Filenames are as follows:
+ //! $(MemoryProfileDumpPath)/$(Name)_$(MemoryProfileDumpFilenameSuffix)_$(Timestamp).$(Ext) or
+ //! $(MemoryProfileDumpPath)/$(Name)_$(Timestamp).$(Ext) (if MemoryProfileDumpFilenameSuffix is missing)
+ std::optional<TString> MemoryProfileDumpPath;
+ std::optional<TString> MemoryProfileDumpFilenameSuffix;
REGISTER_YSON_STRUCT(THeapSizeLimitConfig);
diff --git a/yt/yt/library/tcmalloc/tcmalloc_manager.cpp b/yt/yt/library/tcmalloc/tcmalloc_manager.cpp
index e50e8b59ef..55dd203f1a 100644
--- a/yt/yt/library/tcmalloc/tcmalloc_manager.cpp
+++ b/yt/yt/library/tcmalloc/tcmalloc_manager.cpp
@@ -4,11 +4,20 @@
#include <yt/yt/library/profiling/resource_tracker/resource_tracker.h>
-#include <yt/yt/library/oom/oom.h>
+#include <yt/yt/library/ytprof/external_pprof.h>
+#include <yt/yt/library/ytprof/heap_profiler.h>
+#include <yt/yt/library/ytprof/profile.h>
#include <library/cpp/yt/memory/leaky_singleton.h>
+#include <library/cpp/yt/memory/atomic_intrusive_ptr.h>
+
+#include <library/cpp/yt/system/exit.h>
+
+#include <yt/yt/core/misc/fs.h>
+#include <yt/yt/core/misc/crash_handler.h>
#include <util/system/thread.h>
+#include <util/system/shellcommand.h>
#include <tcmalloc/malloc_extension.h>
@@ -19,19 +28,305 @@ namespace NYT::NTCMalloc {
////////////////////////////////////////////////////////////////////////////////
-static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TCMalloc");
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TCMalloc");
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_STRUCT(TOomProfileManifest)
+
+struct TOomProfileManifest
+ : public NYTree::TYsonStruct
+{
+ TString CurrentProfilePath;
+ TString PeakProfilePath;
+
+ REGISTER_YSON_STRUCT(TOomProfileManifest);
+
+ static void Register(TRegistrar registrar)
+ {
+ // TODO(babenko): fix names, see ytserver_mail_profile_link.py
+ registrar.Parameter("heap_profile_path", &TThis::CurrentProfilePath)
+ .Default();
+ registrar.Parameter("peak_profile_path", &TThis::PeakProfilePath)
+ .Default();
+ }
+};
+
+DEFINE_REFCOUNTED_TYPE(TOomProfileManifest)
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString MakeIncompletePath(const TString& path)
+{
+ return NYT::Format("%v_incomplete", path);
+}
+
+void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::ProfileType profileType)
+{
+ auto profile = NYTProf::ReadHeapProfile(profileType);
+ SymbolizeByExternalPProf(&profile, NYTProf::TSymbolizationOptions{
+ .RunTool = [] (const std::vector<TString>& args) {
+ TShellCommand command{args[0], TList<TString>{args.begin() + 1, args.end()}};
+ command.Run();
+ },
+ });
+
+ auto incompletePath = MakeIncompletePath(memoryProfilePath);
+
+ TFileOutput output(incompletePath);
+ NYTProf::WriteProfile(&output, profile);
+ output.Finish();
+ NFS::Rename(incompletePath, memoryProfilePath);
+}
+
+void MemoryProfileTimeoutHandler(int /*signal*/)
+{
+ AbortProcessDramatically(
+ EProcessExitCode::GenericError,
+ "Process hung while dumping heap profile");
+}
+
+void SetupMemoryProfileTimeout(int timeout)
+{
+ ::signal(SIGALRM, &MemoryProfileTimeoutHandler);
+ ::alarm(timeout);
+}
+
+void DumpManifest(const TOomProfileManifestPtr& manifest, const TString& fileName)
+{
+ TFileOutput output(fileName);
+ NYson::TYsonWriter writer(&output, NYson::EYsonFormat::Pretty);
+ Serialize(manifest, &writer);
+ writer.Flush();
+ output.Finish();
+}
////////////////////////////////////////////////////////////////////////////////
-class TCMallocLimitsAdjuster
+class TMemoryLimitHandler
+ : public TRefCounted
{
public:
- void Adjust(const TTCMallocConfigPtr& config)
+ explicit TMemoryLimitHandler(THeapSizeLimitConfigPtr config)
+ : Config_(std::move(config))
+ {
+ Thread_ = std::thread([this] {
+ Handle();
+ });
+ }
+
+ ~TMemoryLimitHandler()
+ {
+ {
+ std::unique_lock<std::mutex> lock(Mutex_);
+ Fired_ = true;
+ CV_.notify_all();
+ }
+
+ Thread_.join();
+ }
+
+ void Fire()
+ {
+ std::unique_lock<std::mutex> lock(Mutex_);
+ Fired_ = true;
+ NeedToHandle_ = true;
+ CV_.notify_all();
+ }
+
+private:
+ const THeapSizeLimitConfigPtr Config_;
+
+ bool Fired_ = false;
+ bool NeedToHandle_ = false;
+ std::mutex Mutex_;
+ std::condition_variable CV_;
+ std::thread Thread_;
+
+ void Handle()
+ {
+ std::unique_lock<std::mutex> lock(Mutex_);
+ CV_.wait(lock, [&] {
+ return Fired_;
+ });
+
+ if (!NeedToHandle_) {
+ return;
+ }
+
+ auto timestamp = TInstant::Now().FormatLocalTime("%Y%m%dT%H%M%S");
+ auto manifest = New<TOomProfileManifest>();
+ auto manifestPath = GetManifestPath(timestamp);
+ manifest->CurrentProfilePath = GetCurrentDumpPath(timestamp);
+ manifest->PeakProfilePath = GetPeakDumpPath(timestamp);
+
+ Cerr << "*** Forking process to write heap profiles" << Endl
+ << " current: " << manifest->CurrentProfilePath << Endl
+ << " peak: " << manifest->PeakProfilePath << Endl
+ << " manifest: " << manifestPath << Endl;
+
+ SetupMemoryProfileTimeout(Config_->MemoryProfileDumpTimeout.Seconds());
+
+ auto childPid = fork();
+ if (childPid == 0) {
+ NFS::MakeDirRecursive(*Config_->MemoryProfileDumpPath);
+ SetupMemoryProfileTimeout(Config_->MemoryProfileDumpTimeout.Seconds());
+ CollectAndDumpMemoryProfile(manifest->CurrentProfilePath, tcmalloc::ProfileType::kHeap);
+ CollectAndDumpMemoryProfile(manifest->PeakProfilePath, tcmalloc::ProfileType::kPeakHeap);
+ DumpManifest(manifest, manifestPath);
+
+ Cerr << "*** Heap profiles are written" << Endl;
+ AbortProcessSilently(EProcessExitCode::OK);
+ }
+
+ if (childPid < 0) {
+ Cerr << "*** Fork failed: " << LastSystemErrorText() << Endl;
+ AbortProcessSilently(EProcessExitCode::GenericError);
+ }
+
+ ExecWaitForChild(childPid);
+ AbortProcessSilently(EProcessExitCode::OK);
+ }
+
+ auto MakeSuffixFormatter(const TString& timestamp) const
+ {
+ return NYT::MakeFormatterWrapper([this, &timestamp] (TStringBuilderBase* builder) {
+ if (Config_->MemoryProfileDumpFilenameSuffix) {
+ builder->AppendFormat("%v_", *Config_->MemoryProfileDumpFilenameSuffix);
+ }
+ FormatValue(builder, timestamp, "v");
+ });
+ }
+
+ TString GetCurrentDumpPath(const TString& timestamp) const
+ {
+ return Format(
+ "%v/current_%v.pb.gz",
+ Config_->MemoryProfileDumpPath,
+ MakeSuffixFormatter(timestamp));
+ }
+
+ TString GetPeakDumpPath(const TString& timestamp) const
+ {
+ return Format(
+ "%v/peak_%v.pb.gz",
+ Config_->MemoryProfileDumpPath,
+ MakeSuffixFormatter(timestamp));
+ }
+
+ TString GetManifestPath(const TString& timestamp) const
+ {
+ return Format(
+ "%v/oom_profile_paths_%v.yson",
+ Config_->MemoryProfileDumpPath,
+ MakeSuffixFormatter(timestamp));
+ }
+
+ void ExecWaitForChild(int pid)
+ {
+ Cerr << "*** Start waiting for the child" << Endl;
+
+ auto command = Format("while [ -e /proc/%v ]; do sleep 1; done;", pid);
+ execl("/bin/bash", "/bin/bash", "-c", command.c_str(), (void*)nullptr);
+
+ Cerr << "*** Failed to switch main process to dummy child waiter: "
+ << LastSystemErrorText() << Endl;
+ }
+};
+
+DEFINE_REFCOUNTED_TYPE(TMemoryLimitHandler);
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class TMallocExtension>
+concept CSupportsLimitHandler = requires (TMallocExtension extension)
+{
+ { extension.GetSoftMemoryLimitHandler() };
+};
+
+template <typename TMallocExtension, typename THandler>
+void SetSoftMemoryLimitHandler(THandler)
+{
+ WriteToStderr("TCMalloc does not support memory limit handler\n");
+}
+
+template <CSupportsLimitHandler TMallocExtension, typename THandler>
+void SetSoftMemoryLimitHandler(THandler handler)
+{
+ TMallocExtension::SetSoftMemoryLimitHandler(handler);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TMemoryLimitHandlerInstaller
+{
+public:
+ static TMemoryLimitHandlerInstaller* Get()
+ {
+ return LeakySingleton<TMemoryLimitHandlerInstaller>();
+ }
+
+ void Reconfigure(const THeapSizeLimitConfigPtr& config)
+ {
+ if (config->DumpMemoryProfileOnViolation) {
+ Enable(config);
+ } else {
+ Disable();
+ }
+ }
+
+private:
+ DECLARE_LEAKY_SINGLETON_FRIEND();
+
+ TAtomicIntrusivePtr<TMemoryLimitHandler> LimitHandler_;
+
+ TMemoryLimitHandlerInstaller()
+ {
+ SetSoftMemoryLimitHandler<tcmalloc::MallocExtension>(&HandleTCMallocLimit);
+ }
+
+ static void HandleTCMallocLimit()
+ {
+ if (auto handler = Get()->LimitHandler_.Acquire()) {
+ handler->Fire();
+ }
+ }
+
+ void Enable(const THeapSizeLimitConfigPtr& config)
+ {
+ if (LimitHandler_.Acquire()) {
+ return;
+ }
+
+ TAtomicIntrusivePtr<TMemoryLimitHandler>::TRawPtr expected = nullptr;
+ LimitHandler_.CompareAndSwap(expected, New<TMemoryLimitHandler>(config));
+ }
+
+ void Disable()
+ {
+ LimitHandler_.Reset();
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TLimitsAdjuster
+{
+public:
+ static TLimitsAdjuster* Get()
+ {
+ return LeakySingleton<TLimitsAdjuster>();
+ }
+
+ void Reconfigure(const TTCMallocConfigPtr& config)
{
i64 totalMemory = GetAnonymousMemoryLimit();
AdjustPageHeapLimit(totalMemory, config);
AdjustAggressiveReleaseThreshold(totalMemory, config);
- SetupMemoryLimitHandler(config);
}
i64 GetAggressiveReleaseThreshold()
@@ -40,6 +335,10 @@ public:
}
private:
+ DECLARE_LEAKY_SINGLETON_FRIEND();
+
+ TLimitsAdjuster() = default;
+
using TAllocatorMemoryLimit = tcmalloc::MallocExtension::MemoryLimit;
TAllocatorMemoryLimit AppliedLimit_;
@@ -51,7 +350,7 @@ private:
auto proposed = ProposeHeapMemoryLimit(totalMemory, config);
if (proposed.limit == AppliedLimit_.limit && proposed.hard == AppliedLimit_.hard) {
- // Already applied
+ // Already applied.
return;
}
@@ -72,20 +371,6 @@ private:
}
}
- void SetupMemoryLimitHandler(const TTCMallocConfigPtr& config)
- {
- TTCMallocLimitHandlerOptions handlerOptions {
- .HeapDumpDirectory = config->HeapSizeLimit->DumpMemoryProfilePath,
- .Timeout = config->HeapSizeLimit->DumpMemoryProfileTimeout,
- };
-
- if (config->HeapSizeLimit->DumpMemoryProfileOnViolation) {
- EnableTCMallocLimitHandler(handlerOptions);
- } else {
- DisableTCMallocLimitHandler();
- }
- }
-
i64 GetAnonymousMemoryLimit() const
{
return NProfiling::TResourceTracker::GetAnonymousMemoryLimit();
@@ -114,6 +399,10 @@ private:
////////////////////////////////////////////////////////////////////////////////
+} // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
class TTCMallocManagerImpl
{
public:
@@ -141,20 +430,19 @@ public:
std::thread([&] {
::TThread::SetCurrentThreadName("TCMallocBack");
- TCMallocLimitsAdjuster limitsAdjuster;
-
while (true) {
auto config = Config_.Acquire();
- limitsAdjuster.Adjust(config);
+ TLimitsAdjuster::Get()->Reconfigure(config);
+ TMemoryLimitHandlerInstaller::Get()->Reconfigure(config->HeapSizeLimit);
auto freeBytes = tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.page_heap_free");
YT_VERIFY(freeBytes);
- if (static_cast<i64>(*freeBytes) > limitsAdjuster.GetAggressiveReleaseThreshold()) {
+ if (static_cast<i64>(*freeBytes) > TLimitsAdjuster::Get()->GetAggressiveReleaseThreshold()) {
YT_LOG_DEBUG("Aggressively releasing memory (FreeBytes: %v, Threshold: %v)",
static_cast<i64>(*freeBytes),
- limitsAdjuster.GetAggressiveReleaseThreshold());
+ TLimitsAdjuster::Get()->GetAggressiveReleaseThreshold());
tcmalloc::MallocExtension::ReleaseMemoryToSystem(config->AggressiveReleaseSize);
}
diff --git a/yt/yt/library/tcmalloc/ya.make b/yt/yt/library/tcmalloc/ya.make
index e92d0fb137..4042a08971 100644
--- a/yt/yt/library/tcmalloc/ya.make
+++ b/yt/yt/library/tcmalloc/ya.make
@@ -9,7 +9,7 @@ SRCS(
PEERDIR(
yt/yt/core
- yt/yt/library/oom
+ yt/yt/library/ytprof
yt/yt/library/profiling/resource_tracker
contrib/libs/tcmalloc/malloc_extension
)