Intermediate changes

12 files changed, 269 insertions, 0 deletions

diff --git a/library/cpp/yt/mlock/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/mlock/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..ca50021faf
--- /dev/null
+++ b/library/cpp/yt/mlock/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-yt-mlock)
+target_compile_options(cpp-yt-mlock PRIVATE
+  -Wdeprecated-this-capture
+)
+target_link_libraries(cpp-yt-mlock PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(cpp-yt-mlock PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/yt/mlock/mlock_other.cpp
+)
diff --git a/library/cpp/yt/mlock/CMakeLists.linux-aarch64.txt b/library/cpp/yt/mlock/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..68f500a75a
--- /dev/null
+++ b/library/cpp/yt/mlock/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-yt-mlock)
+target_compile_options(cpp-yt-mlock PRIVATE
+  -Wdeprecated-this-capture
+)
+target_link_libraries(cpp-yt-mlock PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(cpp-yt-mlock PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/yt/mlock/mlock_linux.cpp
+)
diff --git a/library/cpp/yt/mlock/CMakeLists.linux-x86_64.txt b/library/cpp/yt/mlock/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..68f500a75a
--- /dev/null
+++ b/library/cpp/yt/mlock/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-yt-mlock)
+target_compile_options(cpp-yt-mlock PRIVATE
+  -Wdeprecated-this-capture
+)
+target_link_libraries(cpp-yt-mlock PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(cpp-yt-mlock PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/yt/mlock/mlock_linux.cpp
+)
diff --git a/library/cpp/yt/mlock/CMakeLists.txt b/library/cpp/yt/mlock/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/library/cpp/yt/mlock/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/library/cpp/yt/mlock/CMakeLists.windows-x86_64.txt b/library/cpp/yt/mlock/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..1537ee764b
--- /dev/null
+++ b/library/cpp/yt/mlock/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-yt-mlock)
+target_link_libraries(cpp-yt-mlock PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(cpp-yt-mlock PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/yt/mlock/mlock_other.cpp
+)
diff --git a/library/cpp/yt/mlock/README.md b/library/cpp/yt/mlock/README.md
new file mode 100644
index 0000000000..b61b6072c4
--- /dev/null
+++ b/library/cpp/yt/mlock/README.md
@@ -0,0 +1,11 @@
+# mlock
+
+MlockFileMappings подгружает и лочит в память все страницы исполняемого файла.
+
+В отличии от вызова mlockall, функция не лочит другие страницы процесса.
+mlockall явно выделяет физическую память под все vma. Типичный процесс сначала
+стартует и инициализирует аллокатор, а потом уже вызывает функцию для mlock страниц.
+Аллокатор при старте выделяет большие диапазоны через mmap, но реально их не использует.
+Поэтому mlockall приводит в повышенному потреблению памяти.
+
+Также, в отличии от mlockall, функция может подгрузить страницы в память сразу. 
diff --git a/library/cpp/yt/mlock/mlock.h b/library/cpp/yt/mlock/mlock.h
new file mode 100644
index 0000000000..035fc47e37
--- /dev/null
+++ b/library/cpp/yt/mlock/mlock.h
@@ -0,0 +1,11 @@
+#pragma once
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool MlockFileMappings(bool populate = true);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/mlock/mlock_linux.cpp b/library/cpp/yt/mlock/mlock_linux.cpp
new file mode 100644
index 0000000000..8791869f95
--- /dev/null
+++ b/library/cpp/yt/mlock/mlock_linux.cpp
@@ -0,0 +1,89 @@
+#include "mlock.h"
+
+#include <stdio.h>
+#include <sys/mman.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+void PopulateFile(void* ptr, size_t size)
+{
+    constexpr size_t PageSize = 4096;
+
+    auto* begin = static_cast<volatile char*>(ptr);
+    for (auto* current = begin; current < begin + size; current += PageSize) {
+        *current;
+    }
+}
+
+bool MlockFileMappings(bool populate)
+{
+    auto* file = ::fopen("/proc/self/maps", "r");
+    if (!file) {
+        return false;
+    }
+
+    // Each line of /proc/<pid>/smaps has the following format:
+    // address           perms offset  dev   inode   path
+    // E.g.
+    // 08048000-08056000 r-xp 00000000 03:0c 64593   /usr/sbin/gpm
+
+    bool failed = false;
+    while (true) {
+        char line[1024];
+        if (!fgets(line, sizeof(line), file)) {
+            break;
+        }
+
+        char addressStr[64];
+        char permsStr[64];
+        char offsetStr[64];
+        char devStr[64];
+        int inode;
+        if (sscanf(line, "%s %s %s %s %d",
+            addressStr,
+            permsStr,
+            offsetStr,
+            devStr,
+            &inode) != 5)
+        {
+            continue;
+        }
+
+        if (inode == 0) {
+            continue;
+        }
+
+        if (permsStr[0] != 'r') {
+            continue;
+        }
+
+        uintptr_t startAddress;
+        uintptr_t endAddress;
+        if (sscanf(addressStr, "%" PRIx64 "-%" PRIx64,
+                &startAddress,
+                &endAddress) != 2)
+        {
+            continue;
+        }
+
+        if (::mlock(reinterpret_cast<const void*>(startAddress), endAddress - startAddress) != 0) {
+            failed = true;
+            continue;
+        }
+
+        if (populate) {
+            PopulateFile(reinterpret_cast<void*>(startAddress), endAddress - startAddress);
+        }
+    }
+
+    ::fclose(file);
+    return !failed;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/mlock/mlock_other.cpp b/library/cpp/yt/mlock/mlock_other.cpp
new file mode 100644
index 0000000000..269c5c3cb9
--- /dev/null
+++ b/library/cpp/yt/mlock/mlock_other.cpp
@@ -0,0 +1,14 @@
+#include "mlock.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool MlockFileMappings(bool /* populate */)
+{
+    return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/mlock/unittests/mlock_ut.cpp b/library/cpp/yt/mlock/unittests/mlock_ut.cpp
new file mode 100644
index 0000000000..98386622e8
--- /dev/null
+++ b/library/cpp/yt/mlock/unittests/mlock_ut.cpp
@@ -0,0 +1,19 @@
+#include <gtest/gtest.h>
+
+#include <library/cpp/yt/mlock/mlock.h>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TMlockTest, Call)
+{
+    ASSERT_TRUE(MlockFileMappings(false));
+    ASSERT_TRUE(MlockFileMappings(true));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
\ No newline at end of file
diff --git a/library/cpp/yt/mlock/unittests/ya.make b/library/cpp/yt/mlock/unittests/ya.make
new file mode 100644
index 0000000000..f1f956d468
--- /dev/null
+++ b/library/cpp/yt/mlock/unittests/ya.make
@@ -0,0 +1,13 @@
+GTEST()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+SRCS(
+    mlock_ut.cpp
+)
+
+PEERDIR(
+    library/cpp/yt/mlock
+)
+
+END()
diff --git a/library/cpp/yt/mlock/ya.make b/library/cpp/yt/mlock/ya.make
new file mode 100644
index 0000000000..2603d128ed
--- /dev/null
+++ b/library/cpp/yt/mlock/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
+
+IF (OS_LINUX AND NOT SANITIZER_TYPE)
+    SRCS(mlock_linux.cpp)
+ELSE()
+    SRCS(mlock_other.cpp)
+ENDIF()
+
+END()
+
+IF (OS_LINUX AND NOT SANITIZER_TYPE)
+    RECURSE(unittests)
+ENDIF()
+