aboutsummaryrefslogtreecommitdiffstats
path: root/util/system
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /util/system
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'util/system')
-rw-r--r--util/system/align.cpp1
-rw-r--r--util/system/align.h49
-rw-r--r--util/system/align_ut.cpp35
-rw-r--r--util/system/atexit.cpp136
-rw-r--r--util/system/atexit.h22
-rw-r--r--util/system/atexit_ut.cpp85
-rw-r--r--util/system/atomic.cpp3
-rw-r--r--util/system/atomic.h51
-rw-r--r--util/system/atomic_gcc.h90
-rw-r--r--util/system/atomic_ops.h189
-rw-r--r--util/system/atomic_ut.cpp227
-rw-r--r--util/system/atomic_win.h114
-rw-r--r--util/system/backtrace.cpp307
-rw-r--r--util/system/backtrace.h44
-rw-r--r--util/system/backtrace_ut.cpp85
-rw-r--r--util/system/benchmark/cpu_id/main.cpp51
-rw-r--r--util/system/benchmark/cpu_id/metrics/main.py5
-rw-r--r--util/system/benchmark/cpu_id/metrics/ya.make21
-rw-r--r--util/system/benchmark/cpu_id/ya.make13
-rw-r--r--util/system/benchmark/create_destroy_thread/main.cpp26
-rw-r--r--util/system/benchmark/create_destroy_thread/metrics/main.py7
-rw-r--r--util/system/benchmark/create_destroy_thread/metrics/ya.make21
-rw-r--r--util/system/benchmark/create_destroy_thread/ya.make9
-rw-r--r--util/system/benchmark/rdtsc/main.cpp61
-rw-r--r--util/system/benchmark/rdtsc/ya.make10
-rw-r--r--util/system/benchmark/ya.make18
-rw-r--r--util/system/byteorder.cpp1
-rw-r--r--util/system/byteorder.h141
-rw-r--r--util/system/byteorder_ut.cpp26
-rw-r--r--util/system/compat.cpp45
-rw-r--r--util/system/compat.h84
-rw-r--r--util/system/compat_ut.cpp12
-rw-r--r--util/system/compiler.cpp9
-rw-r--r--util/system/compiler.h650
-rw-r--r--util/system/compiler_ut.cpp72
-rw-r--r--util/system/condvar.cpp147
-rw-r--r--util/system/condvar.h71
-rw-r--r--util/system/condvar_ut.cpp200
-rw-r--r--util/system/context.cpp336
-rw-r--r--util/system/context.h181
-rw-r--r--util/system/context_aarch64.S52
-rw-r--r--util/system/context_aarch64.h8
-rw-r--r--util/system/context_i686.asm43
-rw-r--r--util/system/context_i686.h9
-rw-r--r--util/system/context_ut.cpp71
-rw-r--r--util/system/context_x86.asm15
-rw-r--r--util/system/context_x86.h12
-rw-r--r--util/system/context_x86_64.asm40
-rw-r--r--util/system/context_x86_64.h7
-rw-r--r--util/system/cpu_id.cpp263
-rw-r--r--util/system/cpu_id.h157
-rw-r--r--util/system/cpu_id_ut.cpp450
-rw-r--r--util/system/daemon.cpp168
-rw-r--r--util/system/daemon.h27
-rw-r--r--util/system/daemon_ut.cpp94
-rw-r--r--util/system/datetime.cpp103
-rw-r--r--util/system/datetime.h98
-rw-r--r--util/system/datetime_ut.cpp1
-rw-r--r--util/system/defaults.c2
-rw-r--r--util/system/defaults.h155
-rw-r--r--util/system/demangle_impl.h21
-rw-r--r--util/system/direct_io.cpp266
-rw-r--r--util/system/direct_io.h75
-rw-r--r--util/system/direct_io_ut.cpp115
-rw-r--r--util/system/dynlib.cpp138
-rw-r--r--util/system/dynlib.h119
-rw-r--r--util/system/env.cpp67
-rw-r--r--util/system/env.h32
-rw-r--r--util/system/env_ut.cpp31
-rw-r--r--util/system/err.cpp79
-rw-r--r--util/system/error.cpp96
-rw-r--r--util/system/error.h95
-rw-r--r--util/system/error_ut.cpp41
-rw-r--r--util/system/event.cpp137
-rw-r--r--util/system/event.h122
-rw-r--r--util/system/event_ut.cpp132
-rw-r--r--util/system/execpath.cpp199
-rw-r--r--util/system/execpath.h17
-rw-r--r--util/system/execpath_ut.cpp22
-rw-r--r--util/system/fasttime.cpp242
-rw-r--r--util/system/fasttime.h6
-rw-r--r--util/system/fhandle.cpp1
-rw-r--r--util/system/fhandle.h27
-rw-r--r--util/system/file.cpp1302
-rw-r--r--util/system/file.h225
-rw-r--r--util/system/file_lock.cpp46
-rw-r--r--util/system/file_lock.h34
-rw-r--r--util/system/file_ut.cpp416
-rw-r--r--util/system/filemap.cpp585
-rw-r--r--util/system/filemap.h381
-rw-r--r--util/system/filemap_ut.cpp359
-rw-r--r--util/system/flock.cpp71
-rw-r--r--util/system/flock.h35
-rw-r--r--util/system/flock_ut.cpp57
-rw-r--r--util/system/fs.cpp179
-rw-r--r--util/system/fs.h156
-rw-r--r--util/system/fs_ut.cpp325
-rw-r--r--util/system/fs_win.cpp233
-rw-r--r--util/system/fs_win.h29
-rw-r--r--util/system/fstat.cpp215
-rw-r--r--util/system/fstat.h47
-rw-r--r--util/system/fstat_ut.cpp157
-rw-r--r--util/system/getpid.cpp23
-rw-r--r--util/system/getpid.h12
-rw-r--r--util/system/getpid_ut.cpp19
-rw-r--r--util/system/guard.cpp1
-rw-r--r--util/system/guard.h176
-rw-r--r--util/system/guard_ut.cpp180
-rw-r--r--util/system/hi_lo.cpp1
-rw-r--r--util/system/hi_lo.h149
-rw-r--r--util/system/hi_lo_ut.cpp69
-rw-r--r--util/system/hostname.cpp96
-rw-r--r--util/system/hostname.h10
-rw-r--r--util/system/hostname_ut.cpp25
-rw-r--r--util/system/hp_timer.cpp118
-rw-r--r--util/system/hp_timer.h36
-rw-r--r--util/system/info.cpp239
-rw-r--r--util/system/info.h12
-rw-r--r--util/system/info_ut.cpp22
-rw-r--r--util/system/interrupt_signals.cpp60
-rw-r--r--util/system/interrupt_signals.h22
-rw-r--r--util/system/interrupt_signals_ut.cpp46
-rw-r--r--util/system/madvise.cpp126
-rw-r--r--util/system/madvise.h30
-rw-r--r--util/system/maxlen.cpp1
-rw-r--r--util/system/maxlen.h32
-rw-r--r--util/system/mem_info.cpp219
-rw-r--r--util/system/mem_info.h18
-rw-r--r--util/system/mem_info_ut.cpp21
-rw-r--r--util/system/mincore.cpp35
-rw-r--r--util/system/mincore.h38
-rw-r--r--util/system/mincore_ut.cpp47
-rw-r--r--util/system/mktemp.cpp73
-rw-r--r--util/system/mktemp_system.cpp175
-rw-r--r--util/system/mlock.cpp86
-rw-r--r--util/system/mlock.h43
-rw-r--r--util/system/mutex.cpp146
-rw-r--r--util/system/mutex.h64
-rw-r--r--util/system/mutex_ut.cpp129
-rw-r--r--util/system/nice.cpp15
-rw-r--r--util/system/nice.h3
-rw-r--r--util/system/nice_ut.cpp42
-rw-r--r--util/system/pipe.cpp161
-rw-r--r--util/system/pipe.h90
-rw-r--r--util/system/pipe_ut.cpp15
-rw-r--r--util/system/platform.cpp18
-rw-r--r--util/system/platform.h246
-rw-r--r--util/system/platform_ut.cpp27
-rw-r--r--util/system/progname.cpp26
-rw-r--r--util/system/progname.h13
-rw-r--r--util/system/progname_ut.cpp18
-rw-r--r--util/system/protect.cpp95
-rw-r--r--util/system/protect.h25
-rw-r--r--util/system/rusage.cpp121
-rw-r--r--util/system/rusage.h26
-rw-r--r--util/system/rusage_ut.cpp11
-rw-r--r--util/system/rwlock.cpp250
-rw-r--r--util/system/rwlock.h78
-rw-r--r--util/system/rwlock_ut.cpp124
-rw-r--r--util/system/sanitizers.cpp142
-rw-r--r--util/system/sanitizers.h141
-rw-r--r--util/system/sanitizers_ut.cpp15
-rw-r--r--util/system/sem.cpp278
-rw-r--r--util/system/sem.h41
-rw-r--r--util/system/shellcommand.cpp1200
-rw-r--r--util/system/shellcommand.h485
-rw-r--r--util/system/shellcommand_ut.cpp493
-rw-r--r--util/system/shmat.cpp231
-rw-r--r--util/system/shmat.h32
-rw-r--r--util/system/shmat_ut.cpp17
-rw-r--r--util/system/sigset.cpp1
-rw-r--r--util/system/sigset.h78
-rw-r--r--util/system/spin_wait.cpp40
-rw-r--r--util/system/spin_wait.h10
-rw-r--r--util/system/spinlock.cpp1
-rw-r--r--util/system/spinlock.h121
-rw-r--r--util/system/spinlock_ut.cpp37
-rw-r--r--util/system/src_location.cpp17
-rw-r--r--util/system/src_location.h25
-rw-r--r--util/system/src_location_ut.cpp18
-rw-r--r--util/system/src_root.h68
-rw-r--r--util/system/src_root_ut.cpp27
-rw-r--r--util/system/sys_alloc.cpp1
-rw-r--r--util/system/sys_alloc.h43
-rw-r--r--util/system/sysstat.cpp47
-rw-r--r--util/system/sysstat.h52
-rw-r--r--util/system/tempfile.cpp25
-rw-r--r--util/system/tempfile.h50
-rw-r--r--util/system/tempfile_ut.cpp151
-rw-r--r--util/system/thread.cpp557
-rw-r--r--util/system/thread.h174
-rw-r--r--util/system/thread.i52
-rw-r--r--util/system/thread_ut.cpp229
-rw-r--r--util/system/tls.cpp260
-rw-r--r--util/system/tls.h307
-rw-r--r--util/system/tls_ut.cpp58
-rw-r--r--util/system/type_name.cpp60
-rw-r--r--util/system/type_name.h30
-rw-r--r--util/system/type_name_ut.cpp184
-rw-r--r--util/system/types.cpp18
-rw-r--r--util/system/types.h119
-rw-r--r--util/system/types.pxd13
-rw-r--r--util/system/types_ut.cpp23
-rw-r--r--util/system/types_ut.pyx40
-rw-r--r--util/system/unaligned_mem.cpp1
-rw-r--r--util/system/unaligned_mem.h67
-rw-r--r--util/system/unaligned_mem_ut.cpp96
-rw-r--r--util/system/user.cpp58
-rw-r--r--util/system/user.h5
-rw-r--r--util/system/user_ut.cpp9
-rw-r--r--util/system/ut/stdin_osfhandle/main.cpp15
-rw-r--r--util/system/ut/stdin_osfhandle/ya.make14
-rw-r--r--util/system/ut/ya.make88
-rw-r--r--util/system/utime.cpp20
-rw-r--r--util/system/utime.h6
-rw-r--r--util/system/valgrind.cpp1
-rw-r--r--util/system/valgrind.h48
-rw-r--r--util/system/winint.cpp1
-rw-r--r--util/system/winint.h43
-rw-r--r--util/system/ya.make6
-rw-r--r--util/system/yassert.cpp94
-rw-r--r--util/system/yassert.h126
-rw-r--r--util/system/yassert_ut.cpp35
-rw-r--r--util/system/yield.cpp25
-rw-r--r--util/system/yield.h4
225 files changed, 23138 insertions, 0 deletions
diff --git a/util/system/align.cpp b/util/system/align.cpp
new file mode 100644
index 0000000000..711a4f12ff
--- /dev/null
+++ b/util/system/align.cpp
@@ -0,0 +1 @@
+#include "align.h"
diff --git a/util/system/align.h b/util/system/align.h
new file mode 100644
index 0000000000..ea0bbc5b46
--- /dev/null
+++ b/util/system/align.h
@@ -0,0 +1,49 @@
+#pragma once
+
+#include "yassert.h"
+#include "defaults.h"
+#include <util/generic/bitops.h>
+
+template <class T>
+static inline T AlignDown(T len, T align) noexcept {
+ Y_ASSERT(IsPowerOf2(align)); // align should be power of 2
+ return len & ~(align - 1);
+}
+
+template <class T>
+static inline T AlignUp(T len, T align) noexcept {
+ const T alignedResult = AlignDown(len + (align - 1), align);
+ Y_ASSERT(alignedResult >= len); // check for overflow
+ return alignedResult;
+}
+
+template <class T>
+static inline T AlignUpSpace(T len, T align) noexcept {
+ Y_ASSERT(IsPowerOf2(align)); // align should be power of 2
+ return ((T)0 - len) & (align - 1); // AlignUp(len, align) - len;
+}
+
+template <class T>
+static inline T* AlignUp(T* ptr, size_t align) noexcept {
+ return (T*)AlignUp((uintptr_t)ptr, align);
+}
+
+template <class T>
+static inline T* AlignDown(T* ptr, size_t align) noexcept {
+ return (T*)AlignDown((uintptr_t)ptr, align);
+}
+
+template <class T>
+static inline T AlignUp(T t) noexcept {
+ return AlignUp(t, (size_t)PLATFORM_DATA_ALIGN);
+}
+
+template <class T>
+static inline T AlignDown(T t) noexcept {
+ return AlignDown(t, (size_t)PLATFORM_DATA_ALIGN);
+}
+
+template <class T>
+static inline T Align(T t) noexcept {
+ return AlignUp(t);
+}
diff --git a/util/system/align_ut.cpp b/util/system/align_ut.cpp
new file mode 100644
index 0000000000..3ba3a3442b
--- /dev/null
+++ b/util/system/align_ut.cpp
@@ -0,0 +1,35 @@
+#include "align.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+class TAlignTest: public TTestBase {
+ UNIT_TEST_SUITE(TAlignTest);
+ UNIT_TEST(TestDown)
+ UNIT_TEST(TestUp)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestDown() {
+ UNIT_ASSERT(AlignDown(0, 4) == 0);
+ UNIT_ASSERT(AlignDown(1, 4) == 0);
+ UNIT_ASSERT(AlignDown(2, 4) == 0);
+ UNIT_ASSERT(AlignDown(3, 4) == 0);
+ UNIT_ASSERT(AlignDown(4, 4) == 4);
+ UNIT_ASSERT(AlignDown(5, 4) == 4);
+ UNIT_ASSERT(AlignDown(0, 8) == 0);
+ UNIT_ASSERT(AlignDown(1, 8) == 0);
+ }
+
+ inline void TestUp() {
+ UNIT_ASSERT(AlignUp(0, 4) == 0);
+ UNIT_ASSERT(AlignUp(1, 4) == 4);
+ UNIT_ASSERT(AlignUp(2, 4) == 4);
+ UNIT_ASSERT(AlignUp(3, 4) == 4);
+ UNIT_ASSERT(AlignUp(4, 4) == 4);
+ UNIT_ASSERT(AlignUp(5, 4) == 8);
+ UNIT_ASSERT(AlignUp(0, 8) == 0);
+ UNIT_ASSERT(AlignUp(1, 8) == 8);
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TAlignTest);
diff --git a/util/system/atexit.cpp b/util/system/atexit.cpp
new file mode 100644
index 0000000000..74fb10b6b1
--- /dev/null
+++ b/util/system/atexit.cpp
@@ -0,0 +1,136 @@
+#include "atexit.h"
+#include "atomic.h"
+#include "yassert.h"
+#include "spinlock.h"
+#include "thread.h"
+
+#include <util/generic/ylimits.h>
+#include <util/generic/utility.h>
+#include <util/generic/deque.h>
+#include <util/generic/queue.h>
+
+#include <tuple>
+
+#include <cstdlib>
+
+namespace {
+ class TAtExit {
+ struct TFunc {
+ TAtExitFunc Func;
+ void* Ctx;
+ size_t Priority;
+ size_t Number;
+ };
+
+ struct TCmp {
+ inline bool operator()(const TFunc* l, const TFunc* r) const noexcept {
+ return std::tie(l->Priority, l->Number) < std::tie(r->Priority, r->Number);
+ }
+ };
+
+ public:
+ inline TAtExit() noexcept
+ : FinishStarted_(0)
+ {
+ }
+
+ inline void Finish() noexcept {
+ AtomicSet(FinishStarted_, 1);
+
+ auto guard = Guard(Lock_);
+
+ while (Items_) {
+ auto c = Items_.top();
+
+ Y_ASSERT(c);
+
+ Items_.pop();
+
+ {
+ auto unguard = Unguard(guard);
+
+ try {
+ c->Func(c->Ctx);
+ } catch (...) {
+ // ¯\_(ツ)_/¯
+ }
+ }
+ }
+ }
+
+ inline void Register(TAtExitFunc func, void* ctx, size_t priority) {
+ with_lock (Lock_) {
+ Store_.push_back({func, ctx, priority, Store_.size()});
+ Items_.push(&Store_.back());
+ }
+ }
+
+ inline bool FinishStarted() const {
+ return AtomicGet(FinishStarted_);
+ }
+
+ private:
+ TAdaptiveLock Lock_;
+ TAtomic FinishStarted_;
+ TDeque<TFunc> Store_;
+ TPriorityQueue<TFunc*, TVector<TFunc*>, TCmp> Items_;
+ };
+
+ static TAtomic atExitLock = 0;
+ static TAtExit* volatile atExitPtr = nullptr;
+ alignas(TAtExit) static char atExitMem[sizeof(TAtExit)];
+
+ static void OnExit() {
+ if (TAtExit* const atExit = AtomicGet(atExitPtr)) {
+ atExit->Finish();
+ atExit->~TAtExit();
+ AtomicSet(atExitPtr, nullptr);
+ }
+ }
+
+ static inline TAtExit* Instance() {
+ if (TAtExit* const atExit = AtomicGet(atExitPtr)) {
+ return atExit;
+ }
+ with_lock (atExitLock) {
+ if (TAtExit* const atExit = AtomicGet(atExitPtr)) {
+ return atExit;
+ }
+ atexit(OnExit);
+ TAtExit* const atExit = new (atExitMem) TAtExit;
+ AtomicSet(atExitPtr, atExit);
+ return atExit;
+ }
+ }
+}
+
+void ManualRunAtExitFinalizers() {
+ OnExit();
+}
+
+bool ExitStarted() {
+ if (TAtExit* const atExit = AtomicGet(atExitPtr)) {
+ return atExit->FinishStarted();
+ }
+ return false;
+}
+
+void AtExit(TAtExitFunc func, void* ctx, size_t priority) {
+ Instance()->Register(func, ctx, priority);
+}
+
+void AtExit(TAtExitFunc func, void* ctx) {
+ AtExit(func, ctx, Max<size_t>());
+}
+
+static void TraditionalCloser(void* ctx) {
+ reinterpret_cast<TTraditionalAtExitFunc>(ctx)();
+}
+
+void AtExit(TTraditionalAtExitFunc func) {
+ AtExit(TraditionalCloser, reinterpret_cast<void*>(func));
+}
+
+void AtExit(TTraditionalAtExitFunc func, size_t priority) {
+ AtExit(TraditionalCloser, reinterpret_cast<void*>(func), priority);
+}
diff --git a/util/system/atexit.h b/util/system/atexit.h
new file mode 100644
index 0000000000..eb3188615c
--- /dev/null
+++ b/util/system/atexit.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "defaults.h"
+
+using TAtExitFunc = void (*)(void*);
+using TTraditionalAtExitFunc = void (*)();
+
+void AtExit(TAtExitFunc func, void* ctx);
+void AtExit(TAtExitFunc func, void* ctx, size_t priority);
+
+void AtExit(TTraditionalAtExitFunc func);
+void AtExit(TTraditionalAtExitFunc func, size_t priority);
+
+bool ExitStarted();
+
+/**
+ * Generally it's a bad idea to call this method except for some rare cases,
+ * like graceful python DLL module unload.
+ * This function is not threadsafe.
+ * Calls in the moment when application is not terminating - bad idea.
+ */
+void ManualRunAtExitFinalizers();
diff --git a/util/system/atexit_ut.cpp b/util/system/atexit_ut.cpp
new file mode 100644
index 0000000000..953f432811
--- /dev/null
+++ b/util/system/atexit_ut.cpp
@@ -0,0 +1,85 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "atexit.h"
+
+#include <errno.h>
+
+#ifdef _win_
+// not implemented
+#else
+ #include <sys/types.h>
+ #include <sys/wait.h>
+#endif //_win_
+
+#include <stdio.h>
+
+#ifdef _win_
+// not implemented
+#else
+struct TAtExitParams {
+ TAtExitParams(int fd_, const char* str_)
+ : fd(fd_)
+ , str(str_)
+ {
+ }
+
+ int fd;
+ const char* str;
+};
+
+void MyAtExitFunc(void* ptr) {
+ THolder<TAtExitParams> params{static_cast<TAtExitParams*>(ptr)};
+ if (write(params->fd, params->str, strlen(params->str)) < 0) {
+ abort();
+ }
+}
+#endif
+
+class TAtExitTest: public TTestBase {
+ UNIT_TEST_SUITE(TAtExitTest);
+ UNIT_TEST(TestAtExit)
+ UNIT_TEST_SUITE_END();
+
+ void TestAtExit() {
+#ifdef _win_
+// not implemented
+#else
+ int ret;
+ int pipefd[2];
+
+ ret = pipe(pipefd);
+ UNIT_ASSERT(ret == 0);
+
+ pid_t pid = fork();
+
+ if (pid < 0) {
+ UNIT_ASSERT(0);
+ }
+
+ if (pid > 0) {
+ char data[1024];
+ int last = 0;
+
+ close(pipefd[1]);
+
+ while (read(pipefd[0], data + last++, 1) > 0 && last < 1024) {
+ }
+ data[--last] = 0;
+
+ UNIT_ASSERT(strcmp(data, "High prio\nMiddle prio\nLow-middle prio\nLow prio\nVery low prio\n") == 0);
+ } else {
+ close(pipefd[0]);
+
+ AtExit(MyAtExitFunc, new TAtExitParams(pipefd[1], "Low prio\n"), 3);
+ AtExit(MyAtExitFunc, new TAtExitParams(pipefd[1], "Middle prio\n"), 5);
+ AtExit(MyAtExitFunc, new TAtExitParams(pipefd[1], "High prio\n"), 7);
+ AtExit(MyAtExitFunc, new TAtExitParams(pipefd[1], "Very low prio\n"), 1);
+ AtExit(MyAtExitFunc, new TAtExitParams(pipefd[1], "Low-middle prio\n"), 4);
+
+ exit(0);
+ }
+#endif //_win_
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TAtExitTest);
diff --git a/util/system/atomic.cpp b/util/system/atomic.cpp
new file mode 100644
index 0000000000..fe85ee00b7
--- /dev/null
+++ b/util/system/atomic.cpp
@@ -0,0 +1,3 @@
+#include "atomic.h"
+
+static_assert(sizeof(TAtomic) == sizeof(void*), "expect sizeof(TAtomic) == sizeof(void*)");
diff --git a/util/system/atomic.h b/util/system/atomic.h
new file mode 100644
index 0000000000..80265babfd
--- /dev/null
+++ b/util/system/atomic.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#include "defaults.h"
+
+using TAtomicBase = intptr_t;
+using TAtomic = volatile TAtomicBase;
+
+#if defined(__GNUC__)
+ #include "atomic_gcc.h"
+#elif defined(_MSC_VER)
+ #include "atomic_win.h"
+#else
+ #error unsupported platform
+#endif
+
+#if !defined(ATOMIC_COMPILER_BARRIER)
+ #define ATOMIC_COMPILER_BARRIER()
+#endif
+
+static inline TAtomicBase AtomicSub(TAtomic& a, TAtomicBase v) {
+ return AtomicAdd(a, -v);
+}
+
+static inline TAtomicBase AtomicGetAndSub(TAtomic& a, TAtomicBase v) {
+ return AtomicGetAndAdd(a, -v);
+}
+
+#if defined(USE_GENERIC_SETGET)
+static inline TAtomicBase AtomicGet(const TAtomic& a) {
+ return a;
+}
+
+static inline void AtomicSet(TAtomic& a, TAtomicBase v) {
+ a = v;
+}
+#endif
+
+static inline bool AtomicTryLock(TAtomic* a) {
+ return AtomicCas(a, 1, 0);
+}
+
+static inline bool AtomicTryAndTryLock(TAtomic* a) {
+ return (AtomicGet(*a) == 0) && AtomicTryLock(a);
+}
+
+static inline void AtomicUnlock(TAtomic* a) {
+ ATOMIC_COMPILER_BARRIER();
+ AtomicSet(*a, 0);
+}
+
+#include "atomic_ops.h"
diff --git a/util/system/atomic_gcc.h b/util/system/atomic_gcc.h
new file mode 100644
index 0000000000..ed8dc2bdc5
--- /dev/null
+++ b/util/system/atomic_gcc.h
@@ -0,0 +1,90 @@
+#pragma once
+
+#define ATOMIC_COMPILER_BARRIER() __asm__ __volatile__("" \
+ : \
+ : \
+ : "memory")
+
+static inline TAtomicBase AtomicGet(const TAtomic& a) {
+ TAtomicBase tmp;
+#if defined(_arm64_)
+ __asm__ __volatile__(
+ "ldar %x[value], %[ptr] \n\t"
+ : [value] "=r"(tmp)
+ : [ptr] "Q"(a)
+ : "memory");
+#else
+ __atomic_load(&a, &tmp, __ATOMIC_ACQUIRE);
+#endif
+ return tmp;
+}
+
+static inline void AtomicSet(TAtomic& a, TAtomicBase v) {
+#if defined(_arm64_)
+ __asm__ __volatile__(
+ "stlr %x[value], %[ptr] \n\t"
+ : [ptr] "=Q"(a)
+ : [value] "r"(v)
+ : "memory");
+#else
+ __atomic_store(&a, &v, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline intptr_t AtomicIncrement(TAtomic& p) {
+ return __atomic_add_fetch(&p, 1, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicGetAndIncrement(TAtomic& p) {
+ return __atomic_fetch_add(&p, 1, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicDecrement(TAtomic& p) {
+ return __atomic_sub_fetch(&p, 1, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicGetAndDecrement(TAtomic& p) {
+ return __atomic_fetch_sub(&p, 1, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicAdd(TAtomic& p, intptr_t v) {
+ return __atomic_add_fetch(&p, v, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicGetAndAdd(TAtomic& p, intptr_t v) {
+ return __atomic_fetch_add(&p, v, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicSwap(TAtomic* p, intptr_t v) {
+ (void)p; // disable strange 'parameter set but not used' warning on gcc
+ intptr_t ret;
+ __atomic_exchange(p, &v, &ret, __ATOMIC_SEQ_CST);
+ return ret;
+}
+
+static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ (void)a; // disable strange 'parameter set but not used' warning on gcc
+ return __atomic_compare_exchange(a, &compare, &exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ (void)a; // disable strange 'parameter set but not used' warning on gcc
+ __atomic_compare_exchange(a, &compare, &exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+ return compare;
+}
+
+static inline intptr_t AtomicOr(TAtomic& a, intptr_t b) {
+ return __atomic_or_fetch(&a, b, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicXor(TAtomic& a, intptr_t b) {
+ return __atomic_xor_fetch(&a, b, __ATOMIC_SEQ_CST);
+}
+
+static inline intptr_t AtomicAnd(TAtomic& a, intptr_t b) {
+ return __atomic_and_fetch(&a, b, __ATOMIC_SEQ_CST);
+}
+
+static inline void AtomicBarrier() {
+ __sync_synchronize();
+}
diff --git a/util/system/atomic_ops.h b/util/system/atomic_ops.h
new file mode 100644
index 0000000000..76766b4a0a
--- /dev/null
+++ b/util/system/atomic_ops.h
@@ -0,0 +1,189 @@
+#pragma once
+
+#include <util/generic/typetraits.h>
+
+template <typename T>
+inline TAtomic* AsAtomicPtr(T volatile* target) {
+ return reinterpret_cast<TAtomic*>(target);
+}
+
+template <typename T>
+inline const TAtomic* AsAtomicPtr(T const volatile* target) {
+ return reinterpret_cast<const TAtomic*>(target);
+}
+
+// integral types
+
+template <typename T>
+struct TAtomicTraits {
+ enum {
+ Castable = std::is_integral<T>::value && sizeof(T) == sizeof(TAtomicBase) && !std::is_const<T>::value,
+ };
+};
+
+template <typename T, typename TT>
+using TEnableIfCastable = std::enable_if_t<TAtomicTraits<T>::Castable, TT>;
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGet(T const volatile& target) {
+ return static_cast<T>(AtomicGet(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, void> AtomicSet(T volatile& target, TAtomicBase value) {
+ AtomicSet(*AsAtomicPtr(&target), value);
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicIncrement(T volatile& target) {
+ return static_cast<T>(AtomicIncrement(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGetAndIncrement(T volatile& target) {
+ return static_cast<T>(AtomicGetAndIncrement(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicDecrement(T volatile& target) {
+ return static_cast<T>(AtomicDecrement(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGetAndDecrement(T volatile& target) {
+ return static_cast<T>(AtomicGetAndDecrement(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicAdd(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicAdd(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGetAndAdd(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicGetAndAdd(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicSub(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicSub(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGetAndSub(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicGetAndSub(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicSwap(T volatile* target, TAtomicBase exchange) {
+ return static_cast<T>(AtomicSwap(AsAtomicPtr(target), exchange));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, bool> AtomicCas(T volatile* target, TAtomicBase exchange, TAtomicBase compare) {
+ return AtomicCas(AsAtomicPtr(target), exchange, compare);
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicGetAndCas(T volatile* target, TAtomicBase exchange, TAtomicBase compare) {
+ return static_cast<T>(AtomicGetAndCas(AsAtomicPtr(target), exchange, compare));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, bool> AtomicTryLock(T volatile* target) {
+ return AtomicTryLock(AsAtomicPtr(target));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, bool> AtomicTryAndTryLock(T volatile* target) {
+ return AtomicTryAndTryLock(AsAtomicPtr(target));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, void> AtomicUnlock(T volatile* target) {
+ AtomicUnlock(AsAtomicPtr(target));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicOr(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicOr(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicAnd(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicAnd(*AsAtomicPtr(&target), value));
+}
+
+template <typename T>
+inline TEnableIfCastable<T, T> AtomicXor(T volatile& target, TAtomicBase value) {
+ return static_cast<T>(AtomicXor(*AsAtomicPtr(&target), value));
+}
+
+// pointer types
+
+template <typename T>
+inline T* AtomicGet(T* const volatile& target) {
+ return reinterpret_cast<T*>(AtomicGet(*AsAtomicPtr(&target)));
+}
+
+template <typename T>
+inline void AtomicSet(T* volatile& target, T* value) {
+ AtomicSet(*AsAtomicPtr(&target), reinterpret_cast<TAtomicBase>(value));
+}
+
+using TNullPtr = decltype(nullptr);
+
+template <typename T>
+inline void AtomicSet(T* volatile& target, TNullPtr) {
+ AtomicSet(*AsAtomicPtr(&target), 0);
+}
+
+template <typename T>
+inline T* AtomicSwap(T* volatile* target, T* exchange) {
+ return reinterpret_cast<T*>(AtomicSwap(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange)));
+}
+
+template <typename T>
+inline T* AtomicSwap(T* volatile* target, TNullPtr) {
+ return reinterpret_cast<T*>(AtomicSwap(AsAtomicPtr(target), 0));
+}
+
+template <typename T>
+inline bool AtomicCas(T* volatile* target, T* exchange, T* compare) {
+ return AtomicCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), reinterpret_cast<TAtomicBase>(compare));
+}
+
+template <typename T>
+inline T* AtomicGetAndCas(T* volatile* target, T* exchange, T* compare) {
+ return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), reinterpret_cast<TAtomicBase>(compare)));
+}
+
+template <typename T>
+inline bool AtomicCas(T* volatile* target, T* exchange, TNullPtr) {
+ return AtomicCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), 0);
+}
+
+template <typename T>
+inline T* AtomicGetAndCas(T* volatile* target, T* exchange, TNullPtr) {
+ return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), reinterpret_cast<TAtomicBase>(exchange), 0));
+}
+
+template <typename T>
+inline bool AtomicCas(T* volatile* target, TNullPtr, T* compare) {
+ return AtomicCas(AsAtomicPtr(target), 0, reinterpret_cast<TAtomicBase>(compare));
+}
+
+template <typename T>
+inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, T* compare) {
+ return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, reinterpret_cast<TAtomicBase>(compare)));
+}
+
+template <typename T>
+inline bool AtomicCas(T* volatile* target, TNullPtr, TNullPtr) {
+ return AtomicCas(AsAtomicPtr(target), 0, 0);
+}
+
+template <typename T>
+inline T* AtomicGetAndCas(T* volatile* target, TNullPtr, TNullPtr) {
+ return reinterpret_cast<T*>(AtomicGetAndCas(AsAtomicPtr(target), 0, 0));
+}
diff --git a/util/system/atomic_ut.cpp b/util/system/atomic_ut.cpp
new file mode 100644
index 0000000000..07211ffba7
--- /dev/null
+++ b/util/system/atomic_ut.cpp
@@ -0,0 +1,227 @@
+#include "atomic.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/ylimits.h>
+
+template <typename TAtomic>
+class TAtomicTest
+ : public TTestBase {
+ UNIT_TEST_SUITE(TAtomicTest);
+ UNIT_TEST(TestAtomicInc1)
+ UNIT_TEST(TestAtomicInc2)
+ UNIT_TEST(TestAtomicGetAndInc)
+ UNIT_TEST(TestAtomicDec)
+ UNIT_TEST(TestAtomicGetAndDec)
+ UNIT_TEST(TestAtomicAdd)
+ UNIT_TEST(TestAtomicGetAndAdd)
+ UNIT_TEST(TestAtomicSub)
+ UNIT_TEST(TestAtomicGetAndSub)
+ UNIT_TEST(TestAtomicSwap)
+ UNIT_TEST(TestAtomicOr)
+ UNIT_TEST(TestAtomicAnd)
+ UNIT_TEST(TestAtomicXor)
+ UNIT_TEST(TestCAS)
+ UNIT_TEST(TestGetAndCAS)
+ UNIT_TEST(TestLockUnlock)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestLockUnlock() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT(AtomicTryLock(&v));
+ UNIT_ASSERT(!AtomicTryLock(&v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ AtomicUnlock(&v);
+ UNIT_ASSERT_VALUES_EQUAL(v, 0);
+ }
+
+ inline void TestCAS() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT(AtomicCas(&v, 1, 0));
+ UNIT_ASSERT(!AtomicCas(&v, 1, 0));
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT(AtomicCas(&v, 0, 1));
+ UNIT_ASSERT_VALUES_EQUAL(v, 0);
+ UNIT_ASSERT(AtomicCas(&v, Max<intptr_t>(), 0));
+ UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>());
+ }
+
+ inline void TestGetAndCAS() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 1, 0), 0);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 2, 0), 1);
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, 0, 1), 1);
+ UNIT_ASSERT_VALUES_EQUAL(v, 0);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndCas(&v, Max<intptr_t>(), 0), 0);
+ UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>());
+ }
+
+ inline void TestAtomicInc1() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT(AtomicAdd(v, 1));
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT(AtomicAdd(v, 10));
+ UNIT_ASSERT_VALUES_EQUAL(v, 11);
+ }
+
+ inline void TestAtomicInc2() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT(AtomicIncrement(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT(AtomicIncrement(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 2);
+ }
+
+ inline void TestAtomicGetAndInc() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 0);
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT_EQUAL(AtomicGetAndIncrement(v), 1);
+ UNIT_ASSERT_VALUES_EQUAL(v, 2);
+ }
+
+ inline void TestAtomicDec() {
+ TAtomic v = 2;
+
+ UNIT_ASSERT(AtomicDecrement(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT(!AtomicDecrement(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 0);
+ }
+
+ inline void TestAtomicGetAndDec() {
+ TAtomic v = 2;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 2);
+ UNIT_ASSERT_VALUES_EQUAL(v, 1);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndDecrement(v), 1);
+ UNIT_ASSERT_VALUES_EQUAL(v, 0);
+ }
+
+ inline void TestAtomicAdd() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicAdd(v, 1), 1);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicAdd(v, 2), 3);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicAdd(v, -4), -1);
+ UNIT_ASSERT_VALUES_EQUAL(v, -1);
+ }
+
+ inline void TestAtomicGetAndAdd() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 1), 0);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, 2), 1);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndAdd(v, -4), 3);
+ UNIT_ASSERT_VALUES_EQUAL(v, -1);
+ }
+
+ inline void TestAtomicSub() {
+ TAtomic v = 4;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSub(v, 1), 3);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSub(v, 2), 1);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSub(v, 3), -2);
+ UNIT_ASSERT_VALUES_EQUAL(v, -2);
+ }
+
+ inline void TestAtomicGetAndSub() {
+ TAtomic v = 4;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 1), 4);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 2), 3);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGetAndSub(v, 3), 1);
+ UNIT_ASSERT_VALUES_EQUAL(v, -2);
+ }
+
+ inline void TestAtomicSwap() {
+ TAtomic v = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSwap(&v, 3), 0);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSwap(&v, 5), 3);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSwap(&v, -7), 5);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSwap(&v, Max<intptr_t>()), -7);
+ UNIT_ASSERT_VALUES_EQUAL(v, Max<intptr_t>());
+ }
+
+ inline void TestAtomicOr() {
+ TAtomic v = 0xf0;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicOr(v, 0x0f), 0xff);
+ UNIT_ASSERT_VALUES_EQUAL(v, 0xff);
+ }
+
+ inline void TestAtomicAnd() {
+ TAtomic v = 0xff;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicAnd(v, 0xf0), 0xf0);
+ UNIT_ASSERT_VALUES_EQUAL(v, 0xf0);
+ }
+
+ inline void TestAtomicXor() {
+ TAtomic v = 0x00;
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicXor(v, 0xff), 0xff);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicXor(v, 0xff), 0x00);
+ }
+
+ inline void TestAtomicPtr() {
+ int* p;
+ AtomicSet(p, nullptr);
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGet(p), 0);
+
+ int i;
+ AtomicSet(p, &i);
+
+ UNIT_ASSERT_VALUES_EQUAL(AtomicGet(p), &i);
+ UNIT_ASSERT_VALUES_EQUAL(AtomicSwap(&p, nullptr), &i);
+ UNIT_ASSERT(AtomicCas(&p, &i, nullptr));
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TAtomicTest<TAtomic>);
+
+#ifndef _MSC_VER
+// chooses type *other than* T1
+template <typename T1, typename T2, typename T3>
+struct TChooser {
+ using TdType = T2;
+};
+
+template <typename T1, typename T2>
+struct TChooser<T1, T1, T2> {
+ using TdType = T2;
+};
+
+template <typename T1>
+struct TChooser<T1, T1, T1> {};
+
+ #if defined(__IOS__) && defined(_32_)
+using TAltAtomic = int;
+ #else
+using TAltAtomic = volatile TChooser<TAtomicBase, long, long long>::TdType;
+ #endif
+
+class TTTest: public TAtomicTest<TAltAtomic> {
+public:
+ TString Name() const noexcept override {
+ return "TAtomicTest<TAltAtomic>";
+ }
+
+ static TString StaticName() noexcept {
+ return "TAtomicTest<TAltAtomic>";
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TTTest);
+
+#endif
diff --git a/util/system/atomic_win.h b/util/system/atomic_win.h
new file mode 100644
index 0000000000..65c290e6cc
--- /dev/null
+++ b/util/system/atomic_win.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include <intrin.h>
+
+#define USE_GENERIC_SETGET
+
+#if defined(_i386_)
+
+ #pragma intrinsic(_InterlockedIncrement)
+ #pragma intrinsic(_InterlockedDecrement)
+ #pragma intrinsic(_InterlockedExchangeAdd)
+ #pragma intrinsic(_InterlockedExchange)
+ #pragma intrinsic(_InterlockedCompareExchange)
+
+static inline intptr_t AtomicIncrement(TAtomic& a) {
+ return _InterlockedIncrement((volatile long*)&a);
+}
+
+static inline intptr_t AtomicGetAndIncrement(TAtomic& a) {
+ return _InterlockedIncrement((volatile long*)&a) - 1;
+}
+
+static inline intptr_t AtomicDecrement(TAtomic& a) {
+ return _InterlockedDecrement((volatile long*)&a);
+}
+
+static inline intptr_t AtomicGetAndDecrement(TAtomic& a) {
+ return _InterlockedDecrement((volatile long*)&a) + 1;
+}
+
+static inline intptr_t AtomicAdd(TAtomic& a, intptr_t b) {
+ return _InterlockedExchangeAdd((volatile long*)&a, b) + b;
+}
+
+static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) {
+ return _InterlockedExchangeAdd((volatile long*)&a, b);
+}
+
+static inline intptr_t AtomicSwap(TAtomic* a, intptr_t b) {
+ return _InterlockedExchange((volatile long*)a, b);
+}
+
+static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ return _InterlockedCompareExchange((volatile long*)a, exchange, compare) == compare;
+}
+
+static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ return _InterlockedCompareExchange((volatile long*)a, exchange, compare);
+}
+
+#else // _x86_64_
+
+ #pragma intrinsic(_InterlockedIncrement64)
+ #pragma intrinsic(_InterlockedDecrement64)
+ #pragma intrinsic(_InterlockedExchangeAdd64)
+ #pragma intrinsic(_InterlockedExchange64)
+ #pragma intrinsic(_InterlockedCompareExchange64)
+
+static inline intptr_t AtomicIncrement(TAtomic& a) {
+ return _InterlockedIncrement64((volatile __int64*)&a);
+}
+
+static inline intptr_t AtomicGetAndIncrement(TAtomic& a) {
+ return _InterlockedIncrement64((volatile __int64*)&a) - 1;
+}
+
+static inline intptr_t AtomicDecrement(TAtomic& a) {
+ return _InterlockedDecrement64((volatile __int64*)&a);
+}
+
+static inline intptr_t AtomicGetAndDecrement(TAtomic& a) {
+ return _InterlockedDecrement64((volatile __int64*)&a) + 1;
+}
+
+static inline intptr_t AtomicAdd(TAtomic& a, intptr_t b) {
+ return _InterlockedExchangeAdd64((volatile __int64*)&a, b) + b;
+}
+
+static inline intptr_t AtomicGetAndAdd(TAtomic& a, intptr_t b) {
+ return _InterlockedExchangeAdd64((volatile __int64*)&a, b);
+}
+
+static inline intptr_t AtomicSwap(TAtomic* a, intptr_t b) {
+ return _InterlockedExchange64((volatile __int64*)a, b);
+}
+
+static inline bool AtomicCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ return _InterlockedCompareExchange64((volatile __int64*)a, exchange, compare) == compare;
+}
+
+static inline intptr_t AtomicGetAndCas(TAtomic* a, intptr_t exchange, intptr_t compare) {
+ return _InterlockedCompareExchange64((volatile __int64*)a, exchange, compare);
+}
+
+static inline intptr_t AtomicOr(TAtomic& a, intptr_t b) {
+ return _InterlockedOr64(&a, b) | b;
+}
+
+static inline intptr_t AtomicAnd(TAtomic& a, intptr_t b) {
+ return _InterlockedAnd64(&a, b) & b;
+}
+
+static inline intptr_t AtomicXor(TAtomic& a, intptr_t b) {
+ return _InterlockedXor64(&a, b) ^ b;
+}
+
+#endif // _x86_
+
+//TODO
+static inline void AtomicBarrier() {
+ TAtomic val = 0;
+
+ AtomicSwap(&val, 0);
+}
diff --git a/util/system/backtrace.cpp b/util/system/backtrace.cpp
new file mode 100644
index 0000000000..b77fe58fb1
--- /dev/null
+++ b/util/system/backtrace.cpp
@@ -0,0 +1,307 @@
+#include "dynlib.h"
+#include "demangle_impl.h"
+#include "platform.h"
+#include "backtrace.h"
+
+#include <util/stream/output.h>
+#include <util/stream/format.h>
+#include <util/generic/array_ref.h>
+#include <util/generic/string.h>
+
+#ifdef _win_
+ #include "mutex.h"
+
+ #ifndef OPTIONAL
+ #define OPTIONAL
+ #endif
+ #include <dbghelp.h>
+#endif
+
+#if defined(_bionic_)
+//TODO
+#else
+ #if !defined(HAVE_BACKTRACE) && defined(_cygwin_)
+ #define CaptureStackBackTrace RtlCaptureStackBackTrace
+extern "C" __stdcall unsigned short CaptureStackBackTrace(unsigned long FramesToSkip, unsigned long FramesToCapture, void** BackTrace, unsigned long* BackTraceHash);
+
+ #define USE_WIN_BACKTRACE
+ #define HAVE_BACKTRACE
+ #endif
+
+ #if !defined(HAVE_BACKTRACE) && defined(__IOS__)
+ #define USE_GLIBC_BACKTRACE
+ #define HAVE_BACKTRACE
+ #endif
+
+ #if !defined(HAVE_BACKTRACE) && defined(__GNUC__)
+ #define USE_GCC_BACKTRACE
+ #define HAVE_BACKTRACE
+ #endif
+
+ #if !defined(HAVE_BACKTRACE) && defined(_win_)
+ #define USE_WIN_BACKTRACE
+ #define HAVE_BACKTRACE
+ #endif
+
+ #if !defined(HAVE_BACKTRACE) && defined(_glibc_)
+ #define USE_GLIBC_BACKTRACE
+ #define HAVE_BACKTRACE
+ #endif
+#endif
+
+#if defined(USE_GLIBC_BACKTRACE)
+ #include <execinfo.h>
+
+size_t BackTrace(void** p, size_t len) {
+ return (size_t)backtrace(p, len);
+}
+#endif
+
+#if defined(USE_GCC_BACKTRACE)
+ #include <cxxabi.h>
+ #include <unwind.h>
+
+namespace {
+ namespace NGCCBacktrace {
+ struct TBackTraceContext {
+ void** sym;
+ size_t cnt;
+ size_t size;
+ };
+
+ static _Unwind_Reason_Code Helper(struct _Unwind_Context* c, void* h) {
+ TBackTraceContext* bt = (TBackTraceContext*)h;
+
+ if (bt->cnt != 0) {
+ bt->sym[bt->cnt - 1] = (void*)_Unwind_GetIP(c);
+ }
+
+ if (bt->cnt == bt->size) {
+ return _URC_END_OF_STACK;
+ }
+
+ ++bt->cnt;
+
+ return _URC_NO_REASON;
+ }
+
+ static inline size_t BackTrace(void** p, size_t len) {
+ if (len >= 1) {
+ TBackTraceContext bt = {p, 0, len};
+
+ _Unwind_Backtrace(Helper, &bt);
+
+ return bt.cnt - 1;
+ }
+
+ return 0;
+ }
+ }
+}
+
+size_t BackTrace(void** p, size_t len) {
+ return NGCCBacktrace::BackTrace(p, len);
+}
+#endif
+
+#if defined(USE_WIN_BACKTRACE)
+size_t BackTrace(void** p, size_t len) {
+ return CaptureStackBackTrace(0, len, p, nullptr);
+}
+#endif
+
+#if !defined(HAVE_BACKTRACE)
+size_t BackTrace(void**, size_t) {
+ return 0;
+}
+#endif
+
+#if defined(_unix_) && !defined(_cygwin_)
+ #include <util/generic/strfcpy.h>
+
+ #include <dlfcn.h>
+
+ #if defined(_darwin_)
+ #include <execinfo.h>
+ #endif
+
+static inline const char* CopyTo(const char* from, char* buf, size_t len) {
+ strfcpy(buf, from, len);
+
+ return buf;
+}
+
+TResolvedSymbol ResolveSymbol(void* sym, char* buf, size_t len) {
+ TResolvedSymbol ret = {
+ "??",
+ sym,
+ };
+
+ Dl_info dli;
+
+ Zero(dli);
+
+ if (dladdr(sym, &dli) && dli.dli_sname) {
+ ret.Name = CopyTo(NPrivate::TCppDemangler().Demangle(dli.dli_sname), buf, len);
+ ret.NearestSymbol = dli.dli_saddr;
+ }
+
+ return ret;
+}
+#elif defined(_win_)
+ #include <util/generic/singleton.h>
+
+namespace {
+ struct TWinSymbolResolverImpl {
+ typedef BOOL(WINAPI* TSymInitializeFunc)(HANDLE, PCSTR, BOOL);
+ typedef BOOL(WINAPI* TSymCleanupFunc)(HANDLE);
+ typedef BOOL(WINAPI* TSymFromAddrFunc)(HANDLE, DWORD64, PDWORD64, PSYMBOL_INFO);
+
+ TWinSymbolResolverImpl()
+ : InitOk(FALSE)
+ {
+ Library = LoadLibraryA("Dbghelp.dll");
+ if (!Library) {
+ return;
+ }
+
+ SymInitializeFunc = (TSymInitializeFunc)GetProcAddress(Library, "SymInitialize");
+ SymCleanupFunc = (TSymCleanupFunc)GetProcAddress(Library, "SymCleanup");
+ SymFromAddrFunc = (TSymFromAddrFunc)GetProcAddress(Library, "SymFromAddr");
+ if (SymInitializeFunc && SymCleanupFunc && SymFromAddrFunc) {
+ InitOk = SymInitializeFunc(GetCurrentProcess(), nullptr, TRUE);
+ }
+ }
+
+ ~TWinSymbolResolverImpl() {
+ if (InitOk) {
+ SymCleanupFunc(GetCurrentProcess());
+ }
+
+ if (Library) {
+ FreeLibrary(Library);
+ }
+ }
+
+ TResolvedSymbol Resolve(void* sym, char* buf, size_t len) {
+ TGuard<TMutex> guard(Mutex);
+
+ TResolvedSymbol ret = {
+ "??",
+ sym};
+
+ if (!InitOk || (len <= 1 + sizeof(SYMBOL_INFO))) {
+ return ret;
+ }
+
+ SYMBOL_INFO* symbol = (SYMBOL_INFO*)buf;
+ Zero(*symbol);
+
+ symbol->MaxNameLen = len - sizeof(SYMBOL_INFO) - 1;
+ symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+
+ DWORD64 displacement = 0;
+ BOOL res = SymFromAddrFunc(GetCurrentProcess(), (DWORD64)sym, &displacement, symbol);
+ if (res) {
+ ret.NearestSymbol = (void*)symbol->Address;
+ ret.Name = symbol->Name;
+ }
+
+ return ret;
+ }
+
+ TMutex Mutex;
+ HMODULE Library;
+ TSymInitializeFunc SymInitializeFunc;
+ TSymCleanupFunc SymCleanupFunc;
+ TSymFromAddrFunc SymFromAddrFunc;
+ BOOL InitOk;
+ };
+}
+
+TResolvedSymbol ResolveSymbol(void* sym, char* buf, size_t len) {
+ return Singleton<TWinSymbolResolverImpl>()->Resolve(sym, buf, len);
+}
+#else
+TResolvedSymbol ResolveSymbol(void* sym, char*, size_t) {
+ TResolvedSymbol ret = {
+ "??",
+ sym,
+ };
+
+ return ret;
+}
+#endif
+
+void FormatBackTrace(IOutputStream* out, void* const* backtrace, size_t backtraceSize) {
+ char tmpBuf[1024];
+
+ for (size_t i = 0; i < backtraceSize; ++i) {
+ TResolvedSymbol rs = ResolveSymbol(backtrace[i], tmpBuf, sizeof(tmpBuf));
+
+ *out << rs.Name << "+" << ((ptrdiff_t)backtrace[i] - (ptrdiff_t)rs.NearestSymbol) << " (" << Hex((ptrdiff_t)backtrace[i], HF_ADDX) << ')' << '\n';
+ }
+}
+
+TFormatBackTraceFn FormatBackTraceFn = FormatBackTrace;
+
+TFormatBackTraceFn SetFormatBackTraceFn(TFormatBackTraceFn f) {
+ TFormatBackTraceFn prevFn = FormatBackTraceFn;
+ FormatBackTraceFn = f;
+ return prevFn;
+}
+
+void FormatBackTrace(IOutputStream* out) {
+ void* array[300];
+ const size_t s = BackTrace(array, Y_ARRAY_SIZE(array));
+ FormatBackTraceFn(out, array, s);
+}
+
+TFormatBackTraceFn GetFormatBackTraceFn() {
+ return FormatBackTraceFn;
+}
+
+void PrintBackTrace() {
+ FormatBackTrace(&Cerr);
+}
+
+TBackTrace::TBackTrace()
+ : Size(0)
+{
+}
+
+void TBackTrace::Capture() {
+ Size = BackTrace(Data, CAPACITY);
+}
+
+void TBackTrace::PrintTo(IOutputStream& out) const {
+ FormatBackTraceFn(&out, Data, Size);
+}
+
+TString TBackTrace::PrintToString() const {
+ TStringStream ss;
+ PrintTo(ss);
+ return ss.Str();
+}
+
+size_t TBackTrace::size() const {
+ return Size;
+}
+
+const void* const* TBackTrace::data() const {
+ return Data;
+}
+
+TBackTrace::operator TBackTraceView() const {
+ return TBackTraceView(Data, Size);
+}
+
+TBackTrace TBackTrace::FromCurrentException() {
+#ifdef _YNDX_LIBUNWIND_EXCEPTION_BACKTRACE_SIZE
+ TBackTrace result;
+ result.Size = __cxxabiv1::__cxa_collect_current_exception_backtrace(result.Data, CAPACITY);
+ return result;
+#else
+ return TBackTrace();
+#endif
+}
diff --git a/util/system/backtrace.h b/util/system/backtrace.h
new file mode 100644
index 0000000000..2fce7585c3
--- /dev/null
+++ b/util/system/backtrace.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+#include <util/system/defaults.h>
+
+class IOutputStream;
+
+size_t BackTrace(void** p, size_t len);
+
+struct TResolvedSymbol {
+ const char* Name;
+ void* NearestSymbol;
+};
+
+TResolvedSymbol ResolveSymbol(void* sym, char* buf, size_t len);
+
+void FormatBackTrace(IOutputStream* out, void* const* backtrace, size_t backtraceSize);
+void FormatBackTrace(IOutputStream* out);
+void PrintBackTrace();
+
+using TFormatBackTraceFn = void (*)(IOutputStream*, void* const* backtrace, size_t backtraceSize);
+
+TFormatBackTraceFn SetFormatBackTraceFn(TFormatBackTraceFn f);
+TFormatBackTraceFn GetFormatBackTraceFn();
+
+using TBackTraceView = TArrayRef<void* const>;
+
+class TBackTrace {
+private:
+ static constexpr size_t CAPACITY = 300;
+ void* Data[CAPACITY];
+ size_t Size;
+
+public:
+ TBackTrace();
+ void Capture();
+ void PrintTo(IOutputStream&) const;
+ TString PrintToString() const;
+ size_t size() const;
+ const void* const* data() const;
+ operator TBackTraceView() const;
+
+ static TBackTrace FromCurrentException();
+};
diff --git a/util/system/backtrace_ut.cpp b/util/system/backtrace_ut.cpp
new file mode 100644
index 0000000000..9b5ead71bc
--- /dev/null
+++ b/util/system/backtrace_ut.cpp
@@ -0,0 +1,85 @@
+#include "backtrace.h"
+
+#include <util/generic/array_ref.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/output.h>
+
+using PFunc = int (*)(void**, size_t);
+
+int Dbg1(void** buf, size_t len) {
+ volatile int ret = (int)BackTrace(buf, len);
+ return ret;
+}
+
+int Dbg2(void** buf, size_t len) {
+ volatile int ret = (int)BackTrace(buf, len);
+ return ret;
+}
+
+void FormatBackTraceReplacement(IOutputStream* out, void* const*, size_t) {
+ *out << "WorksLikeACharm" << Endl;
+}
+
+void SomeMethod() {
+ TStringStream out;
+
+ FormatBackTrace(&out);
+
+#if defined(_musl_)
+ // musl dladdr broken for us now
+ return;
+#endif
+
+ UNIT_ASSERT(out.Empty() || out.Str().find("SomeMethod") != TString::npos);
+}
+
+class TBackTraceTest: public TTestBase {
+ UNIT_TEST_SUITE(TBackTraceTest);
+ UNIT_TEST(TestBackTrace)
+ UNIT_TEST(TestBackTraceView)
+ UNIT_TEST(TestPrintBackTrace)
+ UNIT_TEST(TestSetFormatBackTraceFn)
+ UNIT_TEST_SUITE_END();
+
+ void TestPrintBackTrace() {
+ SomeMethod();
+ }
+
+ void TestSetFormatBackTraceFn() {
+ TFormatBackTraceFn prevFn = SetFormatBackTraceFn(FormatBackTraceReplacement);
+ TStringStream out;
+ FormatBackTrace(&out);
+ SetFormatBackTraceFn(prevFn);
+ UNIT_ASSERT(out.Str().Contains("WorksLikeACharm"));
+ TestPrintBackTrace();
+ }
+
+ void TestBackTrace() {
+ //PrintBackTrace();
+ void* buf1[100];
+ size_t ret1;
+
+ void* buf2[100];
+ size_t ret2;
+
+ volatile PFunc func = &Dbg1;
+ ret1 = (*func)(buf1, 100);
+ func = &Dbg2;
+ ret2 = (*func)(buf2, 100);
+
+ UNIT_ASSERT_EQUAL(ret1, ret2);
+ }
+
+ void TestBackTraceView() {
+ try {
+ throw TWithBackTrace<yexception>();
+ } catch (const yexception& e) {
+ const TBackTrace bt = *e.BackTrace();
+ const TBackTraceView btView = bt;
+ UNIT_ASSERT_VALUES_EQUAL(btView.size(), bt.size());
+ }
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TBackTraceTest);
diff --git a/util/system/benchmark/cpu_id/main.cpp b/util/system/benchmark/cpu_id/main.cpp
new file mode 100644
index 0000000000..8efe539983
--- /dev/null
+++ b/util/system/benchmark/cpu_id/main.cpp
@@ -0,0 +1,51 @@
+#include <library/cpp/testing/benchmark/bench.h>
+
+#include <util/system/cpu_id.h>
+
+#include <util/generic/xrange.h>
+
+#define DEFINE_BENCHMARK_PAIR(name) \
+ Y_CPU_BENCHMARK(Have##name, iface) { \
+ for (const auto i : xrange(iface.Iterations())) { \
+ Y_UNUSED(i); \
+ Y_DO_NOT_OPTIMIZE_AWAY(NX86::Have##name()); \
+ } \
+ } \
+ \
+ Y_CPU_BENCHMARK(CachedHave##name, iface) { \
+ for (const auto i : xrange(iface.Iterations())) { \
+ Y_UNUSED(i); \
+ Y_DO_NOT_OPTIMIZE_AWAY(NX86::CachedHave##name()); \
+ } \
+ }
+
+DEFINE_BENCHMARK_PAIR(SSE)
+DEFINE_BENCHMARK_PAIR(SSE2)
+DEFINE_BENCHMARK_PAIR(SSE3)
+DEFINE_BENCHMARK_PAIR(SSSE3)
+DEFINE_BENCHMARK_PAIR(SSE41)
+DEFINE_BENCHMARK_PAIR(SSE42)
+DEFINE_BENCHMARK_PAIR(POPCNT)
+DEFINE_BENCHMARK_PAIR(BMI1)
+DEFINE_BENCHMARK_PAIR(AES)
+DEFINE_BENCHMARK_PAIR(AVX)
+DEFINE_BENCHMARK_PAIR(AVX2)
+DEFINE_BENCHMARK_PAIR(AVX512F)
+DEFINE_BENCHMARK_PAIR(AVX512DQ)
+DEFINE_BENCHMARK_PAIR(AVX512IFMA)
+DEFINE_BENCHMARK_PAIR(AVX512PF)
+DEFINE_BENCHMARK_PAIR(AVX512ER)
+DEFINE_BENCHMARK_PAIR(AVX512CD)
+DEFINE_BENCHMARK_PAIR(AVX512BW)
+DEFINE_BENCHMARK_PAIR(AVX512VL)
+DEFINE_BENCHMARK_PAIR(AVX512VBMI)
+DEFINE_BENCHMARK_PAIR(PREFETCHWT1)
+DEFINE_BENCHMARK_PAIR(SHA)
+DEFINE_BENCHMARK_PAIR(ADX)
+DEFINE_BENCHMARK_PAIR(RDRAND)
+DEFINE_BENCHMARK_PAIR(RDSEED)
+DEFINE_BENCHMARK_PAIR(PCOMMIT)
+DEFINE_BENCHMARK_PAIR(CLFLUSHOPT)
+DEFINE_BENCHMARK_PAIR(CLWB)
+
+#undef DEFINE_BENCHMARK_PAIR
diff --git a/util/system/benchmark/cpu_id/metrics/main.py b/util/system/benchmark/cpu_id/metrics/main.py
new file mode 100644
index 0000000000..d9a86e825c
--- /dev/null
+++ b/util/system/benchmark/cpu_id/metrics/main.py
@@ -0,0 +1,5 @@
+import yatest.common as yc
+
+
+def test_export_metrics(metrics):
+ metrics.set_benchmark(yc.execute_benchmark('util/system/benchmark/cpu_id/cpu_id', threads=8))
diff --git a/util/system/benchmark/cpu_id/metrics/ya.make b/util/system/benchmark/cpu_id/metrics/ya.make
new file mode 100644
index 0000000000..8c55def99b
--- /dev/null
+++ b/util/system/benchmark/cpu_id/metrics/ya.make
@@ -0,0 +1,21 @@
+OWNER(
+ yazevnul
+ g:util
+)
+SUBSCRIBER(g:util-subscribers)
+
+PY2TEST()
+
+SIZE(LARGE)
+
+TAG(
+ ya:force_sandbox
+ sb:intel_e5_2660v1
+ ya:fat
+)
+
+TEST_SRCS(main.py)
+
+DEPENDS(util/system/benchmark/cpu_id)
+
+END()
diff --git a/util/system/benchmark/cpu_id/ya.make b/util/system/benchmark/cpu_id/ya.make
new file mode 100644
index 0000000000..976977014f
--- /dev/null
+++ b/util/system/benchmark/cpu_id/ya.make
@@ -0,0 +1,13 @@
+OWNER(
+ yazevnul
+ g:util
+)
+SUBSCRIBER(g:util-subscribers)
+
+Y_BENCHMARK()
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/util/system/benchmark/create_destroy_thread/main.cpp b/util/system/benchmark/create_destroy_thread/main.cpp
new file mode 100644
index 0000000000..0ca2a9d96f
--- /dev/null
+++ b/util/system/benchmark/create_destroy_thread/main.cpp
@@ -0,0 +1,26 @@
+#include <library/cpp/testing/benchmark/bench.h>
+
+#include <util/system/thread.h>
+
+static void* DoNothing(void*) noexcept {
+ return nullptr;
+}
+
+Y_CPU_BENCHMARK(CreateDestroyThread, iface) {
+ for (size_t i = 0, iEnd = iface.Iterations(); i < iEnd; ++i) {
+ NBench::Clobber();
+ TThread t(&DoNothing, nullptr);
+ Y_DO_NOT_OPTIMIZE_AWAY(t);
+ NBench::Clobber();
+ }
+}
+
+Y_CPU_BENCHMARK(CreateRunDestroyThread, iface) {
+ for (size_t i = 0, iEnd = iface.Iterations(); i < iEnd; ++i) {
+ NBench::Clobber();
+ TThread t(&DoNothing, nullptr);
+ t.Start();
+ NBench::Escape(t.Join());
+ NBench::Clobber();
+ }
+}
diff --git a/util/system/benchmark/create_destroy_thread/metrics/main.py b/util/system/benchmark/create_destroy_thread/metrics/main.py
new file mode 100644
index 0000000000..45564cda7f
--- /dev/null
+++ b/util/system/benchmark/create_destroy_thread/metrics/main.py
@@ -0,0 +1,7 @@
+import yatest.common as yc
+
+
+def test_export_metrics(metrics):
+ metrics.set_benchmark(
+ yc.execute_benchmark('util/system/benchmark/create_destroy_thread/create_destroy_thread', threads=8)
+ )
diff --git a/util/system/benchmark/create_destroy_thread/metrics/ya.make b/util/system/benchmark/create_destroy_thread/metrics/ya.make
new file mode 100644
index 0000000000..d526487e1a
--- /dev/null
+++ b/util/system/benchmark/create_destroy_thread/metrics/ya.make
@@ -0,0 +1,21 @@
+OWNER(
+ yazevnul
+ g:util
+)
+SUBSCRIBER(g:util-subscribers)
+
+PY2TEST()
+
+SIZE(LARGE)
+
+TAG(
+ ya:force_sandbox
+ sb:intel_e5_2660v1
+ ya:fat
+)
+
+TEST_SRCS(main.py)
+
+DEPENDS(util/system/benchmark/create_destroy_thread)
+
+END()
diff --git a/util/system/benchmark/create_destroy_thread/ya.make b/util/system/benchmark/create_destroy_thread/ya.make
new file mode 100644
index 0000000000..03eb0ec8e0
--- /dev/null
+++ b/util/system/benchmark/create_destroy_thread/ya.make
@@ -0,0 +1,9 @@
+OWNER(yazevnul)
+
+Y_BENCHMARK()
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/util/system/benchmark/rdtsc/main.cpp b/util/system/benchmark/rdtsc/main.cpp
new file mode 100644
index 0000000000..8189d10f06
--- /dev/null
+++ b/util/system/benchmark/rdtsc/main.cpp
@@ -0,0 +1,61 @@
+#include <library/cpp/testing/benchmark/bench.h>
+
+#include <util/system/datetime.h>
+#include <util/generic/xrange.h>
+
+Y_FORCE_INLINE ui64 GetCycleCountLinux() {
+ unsigned hi, lo;
+ __asm__ __volatile__("lfence\n"
+ "rdtsc"
+ : "=a"(lo), "=d"(hi));
+ return ((unsigned long long)lo) | (((unsigned long long)hi) << 32);
+}
+
+Y_FORCE_INLINE ui64 GetCycleCountAgri1() {
+ unsigned hi, lo;
+
+ __asm__ __volatile__("rdtscp\n"
+ : "=a"(lo), "=d"(hi)::"%rbx", "%rcx");
+
+ return ((unsigned long long)lo) | (((unsigned long long)hi) << 32);
+}
+
+Y_FORCE_INLINE ui64 GetCycleCountAgri2() {
+ unsigned hi, lo;
+ __asm__ __volatile__("rdtscp\n"
+ : "=a"(lo), "=d"(hi)::"%rbx", "%rcx");
+ /* call cpuid to prevent out of order execution */
+ __asm__ __volatile__("mov $0, %%eax\n"
+ "cpuid\n" ::
+ : "%eax");
+
+ return ((unsigned long long)lo) | (((unsigned long long)hi) << 32);
+}
+
+Y_CPU_BENCHMARK(RdtscUtil, iface) {
+ for (const auto i : xrange(iface.Iterations())) {
+ Y_UNUSED(i);
+ Y_DO_NOT_OPTIMIZE_AWAY(GetCycleCount());
+ }
+}
+
+Y_CPU_BENCHMARK(RdtscLinux, iface) {
+ for (const auto i : xrange(iface.Iterations())) {
+ Y_UNUSED(i);
+ Y_DO_NOT_OPTIMIZE_AWAY(GetCycleCountLinux());
+ }
+}
+
+Y_CPU_BENCHMARK(RdtscAgri1, iface) {
+ for (const auto i : xrange(iface.Iterations())) {
+ Y_UNUSED(i);
+ Y_DO_NOT_OPTIMIZE_AWAY(GetCycleCountAgri1());
+ }
+}
+
+Y_CPU_BENCHMARK(RdtscAgri2, iface) {
+ for (const auto i : xrange(iface.Iterations())) {
+ Y_UNUSED(i);
+ Y_DO_NOT_OPTIMIZE_AWAY(GetCycleCountAgri2());
+ }
+}
diff --git a/util/system/benchmark/rdtsc/ya.make b/util/system/benchmark/rdtsc/ya.make
new file mode 100644
index 0000000000..7059abc3a4
--- /dev/null
+++ b/util/system/benchmark/rdtsc/ya.make
@@ -0,0 +1,10 @@
+Y_BENCHMARK()
+
+OWNER(g:util)
+SUBSCRIBER(g:util-subscribers)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/util/system/benchmark/ya.make b/util/system/benchmark/ya.make
new file mode 100644
index 0000000000..12fa9af9d6
--- /dev/null
+++ b/util/system/benchmark/ya.make
@@ -0,0 +1,18 @@
+OWNER(
+ yazevnul
+ g:util
+)
+SUBSCRIBER(g:util-subscribers)
+
+RECURSE(
+ cpu_id
+ cpu_id/metrics
+ create_destroy_thread
+ create_destroy_thread/metrics
+)
+
+IF (NOT OS_WINDOWS)
+ RECURSE(
+ rdtsc
+ )
+ENDIF()
diff --git a/util/system/byteorder.cpp b/util/system/byteorder.cpp
new file mode 100644
index 0000000000..5db3d30297
--- /dev/null
+++ b/util/system/byteorder.cpp
@@ -0,0 +1 @@
+#include "byteorder.h"
diff --git a/util/system/byteorder.h b/util/system/byteorder.h
new file mode 100644
index 0000000000..94b9fea515
--- /dev/null
+++ b/util/system/byteorder.h
@@ -0,0 +1,141 @@
+#pragma once
+
+#include "defaults.h"
+
+//#define USE_GENERIC_ENDIAN_CVT
+
+#if defined(_linux_) && !defined(USE_GENERIC_ENDIAN_CVT)
+ #include <byteswap.h>
+#elif defined(_darwin_)
+ #if defined(_arm_) || defined(__IOS__)
+ #include <architecture/byte_order.h>
+ #else
+ #include <machine/byte_order.h>
+ #endif
+#else
+ #include <util/generic/utility.h>
+#endif
+
+#if defined(_linux_) && !defined(USE_GENERIC_ENDIAN_CVT)
+ #define SwapBytes16 bswap_16
+ #define SwapBytes32 bswap_32
+ #define SwapBytes64 bswap_64
+#elif defined(_darwin_)
+ #ifdef _arm_
+ #define SwapBytes16 _OSSwapInt16
+ #define SwapBytes32 _OSSwapInt32
+ #define SwapBytes64 _OSSwapInt64
+ #else
+ #define SwapBytes16 OSSwapInt16
+ #define SwapBytes32 OSSwapInt32
+ #define SwapBytes64 OSSwapInt64
+ #endif
+#endif
+
+#ifndef SwapBytes16
+inline ui16 SwapBytes16(ui16 val) noexcept {
+ #define byte_n(__val, __n) ((((unsigned char*)(&__val))[__n]))
+ DoSwap(byte_n(val, 0), byte_n(val, 1));
+ return val;
+ #undef byte_n
+}
+#endif
+
+#ifndef SwapBytes32
+inline ui32 SwapBytes32(ui32 val) noexcept {
+ #define byte_n(__val, __n) ((((unsigned char*)(&__val))[__n]))
+ DoSwap(byte_n(val, 0), byte_n(val, 3));
+ DoSwap(byte_n(val, 1), byte_n(val, 2));
+ return val;
+ #undef byte_n
+}
+#endif
+
+#ifndef SwapBytes64
+inline ui64 SwapBytes64(ui64 val) noexcept {
+ union {
+ ui64 val;
+ ui32 p[2];
+ } tmp, ret;
+
+ tmp.val = val;
+ ret.p[0] = SwapBytes32(tmp.p[1]);
+ ret.p[1] = SwapBytes32(tmp.p[0]);
+
+ return ret.val;
+}
+#endif
+
+//for convenience
+static inline ui8 SwapBytes8(ui8 v) noexcept {
+ return v;
+}
+
+namespace NSwapBytes {
+ template <unsigned N>
+ struct TSwapBytesHelper {
+ };
+
+#define DEF_SB(X) \
+ template <> \
+ struct TSwapBytesHelper<X> { \
+ template <class T> \
+ static inline T Swap(T t) noexcept { \
+ return (T)SwapBytes##X((ui##X)t); \
+ } \
+ };
+
+ DEF_SB(8)
+ DEF_SB(16)
+ DEF_SB(32)
+ DEF_SB(64)
+
+#undef DEF_SB
+}
+
+template <class T>
+inline T SwapBytes(T val) noexcept {
+ return NSwapBytes::TSwapBytesHelper<sizeof(T) * 8>::Swap(val);
+}
+
+template <class T>
+inline T LittleToBig(T val) noexcept {
+ return SwapBytes(val);
+}
+
+template <class T>
+inline T BigToLittle(T val) noexcept {
+ return LittleToBig(val);
+}
+
+template <class T>
+inline T HostToInet(T val) noexcept {
+#if defined(_big_endian_)
+ return val;
+#elif defined(_little_endian_)
+ return LittleToBig(val);
+#else
+ #error todo
+#endif
+}
+
+template <class T>
+inline T InetToHost(T val) noexcept {
+ return HostToInet(val);
+}
+
+template <class T>
+inline T HostToLittle(T val) noexcept {
+#if defined(_big_endian_)
+ return BigToLittle(val);
+#elif defined(_little_endian_)
+ return val;
+#else
+ #error todo
+#endif
+}
+
+template <class T>
+inline T LittleToHost(T val) noexcept {
+ return HostToLittle(val);
+}
diff --git a/util/system/byteorder_ut.cpp b/util/system/byteorder_ut.cpp
new file mode 100644
index 0000000000..39b8603d3f
--- /dev/null
+++ b/util/system/byteorder_ut.cpp
@@ -0,0 +1,26 @@
+#include "byteorder.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+class TByteOrderTest: public TTestBase {
+ UNIT_TEST_SUITE(TByteOrderTest);
+ UNIT_TEST(TestSwap16)
+ UNIT_TEST(TestSwap32)
+ UNIT_TEST(TestSwap64)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestSwap16() {
+ UNIT_ASSERT_EQUAL((ui16)0x1234, SwapBytes((ui16)0x3412));
+ }
+
+ inline void TestSwap32() {
+ UNIT_ASSERT_EQUAL(0x12345678, SwapBytes(0x78563412));
+ }
+
+ inline void TestSwap64() {
+ UNIT_ASSERT_EQUAL(0x1234567890abcdefULL, SwapBytes((ui64)ULL(0xefcdab9078563412)));
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TByteOrderTest);
diff --git a/util/system/compat.cpp b/util/system/compat.cpp
new file mode 100644
index 0000000000..18fbfa296a
--- /dev/null
+++ b/util/system/compat.cpp
@@ -0,0 +1,45 @@
+#include "compat.h"
+#include "defaults.h"
+#include "progname.h"
+
+#include <cctype>
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <cstdarg>
+#include <cstdlib>
+
+#include <util/generic/string.h>
+
+#ifdef _win_
+ #include "winint.h"
+ #include <io.h>
+#endif
+
+#ifndef HAVE_NATIVE_GETPROGNAME
+const char* getprogname() {
+ return GetProgramName().data();
+}
+#endif
+
+#ifdef _win_
+
+void sleep(i64 len) {
+ Sleep((unsigned long)len * 1000);
+}
+
+void usleep(i64 len) {
+ Sleep((unsigned long)len / 1000);
+}
+
+ #include <fcntl.h>
+int ftruncate(int fd, i64 length) {
+ return _chsize_s(fd, length);
+}
+int truncate(const char* name, i64 length) {
+ int fd = ::_open(name, _O_WRONLY);
+ int ret = ftruncate(fd, length);
+ ::close(fd);
+ return ret;
+}
+#endif
diff --git a/util/system/compat.h b/util/system/compat.h
new file mode 100644
index 0000000000..c53dbcca17
--- /dev/null
+++ b/util/system/compat.h
@@ -0,0 +1,84 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <cstdarg>
+
+#include <csignal>
+
+#if defined(_unix_)
+ #include <unistd.h>
+#endif
+
+#if defined(_win_)
+ #include <process.h>
+#endif
+
+extern "C" {
+#if defined(_win_)
+ using pid_t = int;
+
+ inline unsigned int alarm(unsigned int /*seconds*/) {
+ return 0; // no alarm is currently set :)
+ }
+
+ #define SIGQUIT SIGBREAK // instead of 3
+ #define SIGKILL SIGTERM // instead of 9
+ #define SIGPIPE 13 //will not receive under win?
+ #define SIGALRM 14 //will not receive under win?
+#endif
+
+#if defined(__FreeBSD__) || defined(_darwin_)
+ #define HAVE_NATIVE_GETPROGNAME
+#endif
+
+#ifndef HAVE_NATIVE_GETPROGNAME
+ const char* getprogname();
+#endif
+
+#if defined(_MSC_VER)
+ void err(int e, const char* m, ...);
+ void errx(int e, const char* m, ...);
+ void warn(const char* m, ...);
+ void warnx(const char* m, ...);
+ void vwarnx(const char* format, va_list ap);
+ void vwarn(const char* format, va_list ap);
+ void verrx(int status, const char* format, va_list ap);
+#else
+ #include <err.h>
+#endif
+}
+
+#ifdef _MSC_VER
+ #define popen _popen
+ #define pclose _pclose
+#endif
+
+#ifdef _win_
+ #define NAME_MAX FILENAME_MAX
+#endif
+#ifdef _sun_
+ #define NAME_MAX PATH_MAX
+#endif
+
+#ifdef _win_
+
+ #ifdef sleep // may be defined by perl
+ #undef sleep
+ #endif
+
+void sleep(i64 len);
+void usleep(i64 len);
+
+#endif
+
+#ifdef _win_
+int ftruncate(int fd, i64 length);
+int truncate(const char* name, i64 length);
+#endif
+
+#if defined(GNUC)
+ #ifndef va_copy
+ #define va_copy __va_copy
+ #endif
+#endif
diff --git a/util/system/compat_ut.cpp b/util/system/compat_ut.cpp
new file mode 100644
index 0000000000..dbd9289c17
--- /dev/null
+++ b/util/system/compat_ut.cpp
@@ -0,0 +1,12 @@
+#include "compat.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/folder/dirut.h>
+#include <util/stream/output.h>
+
+Y_UNIT_TEST_SUITE(TCompatTest) {
+ Y_UNIT_TEST(TestGetprogname) {
+ getprogname(); // just check it links
+ }
+}
diff --git a/util/system/compiler.cpp b/util/system/compiler.cpp
new file mode 100644
index 0000000000..d4b3cca0af
--- /dev/null
+++ b/util/system/compiler.cpp
@@ -0,0 +1,9 @@
+#include "compiler.h"
+#include <cstdlib>
+
+[[noreturn]] Y_HIDDEN void _YandexAbort() {
+ std::abort();
+}
+
+void UseCharPointerImpl(volatile const char*) {
+}
diff --git a/util/system/compiler.h b/util/system/compiler.h
new file mode 100644
index 0000000000..b373edcc46
--- /dev/null
+++ b/util/system/compiler.h
@@ -0,0 +1,650 @@
+#pragma once
+
+#if defined(_MSC_VER)
+ #include <intrin.h>
+#endif
+
+// useful cross-platfrom definitions for compilers
+
+/**
+ * @def Y_FUNC_SIGNATURE
+ *
+ * Use this macro to get pretty function name (see example).
+ *
+ * @code
+ * void Hi() {
+ * Cout << Y_FUNC_SIGNATURE << Endl;
+ * }
+
+ * template <typename T>
+ * void Do() {
+ * Cout << Y_FUNC_SIGNATURE << Endl;
+ * }
+
+ * int main() {
+ * Hi(); // void Hi()
+ * Do<int>(); // void Do() [T = int]
+ * Do<TString>(); // void Do() [T = TString]
+ * }
+ * @endcode
+ */
+#if defined(__GNUC__)
+ #define Y_FUNC_SIGNATURE __PRETTY_FUNCTION__
+#elif defined(_MSC_VER)
+ #define Y_FUNC_SIGNATURE __FUNCSIG__
+#else
+ #define Y_FUNC_SIGNATURE ""
+#endif
+
+#ifdef __GNUC__
+ #define Y_PRINTF_FORMAT(n, m) __attribute__((__format__(__printf__, n, m)))
+#endif
+
+#ifndef Y_PRINTF_FORMAT
+ #define Y_PRINTF_FORMAT(n, m)
+#endif
+
+#if defined(__clang__)
+ #define Y_NO_SANITIZE(...) __attribute__((no_sanitize(__VA_ARGS__)))
+#endif
+
+#if !defined(Y_NO_SANITIZE)
+ #define Y_NO_SANITIZE(...)
+#endif
+
+/**
+ * @def Y_DECLARE_UNUSED
+ *
+ * Macro is needed to silence compiler warning about unused entities (e.g. function or argument).
+ *
+ * @code
+ * Y_DECLARE_UNUSED int FunctionUsedSolelyForDebugPurposes();
+ * assert(FunctionUsedSolelyForDebugPurposes() == 42);
+ *
+ * void Foo(const int argumentUsedOnlyForDebugPurposes Y_DECLARE_UNUSED) {
+ * assert(argumentUsedOnlyForDebugPurposes == 42);
+ * // however you may as well omit `Y_DECLARE_UNUSED` and use `UNUSED` macro instead
+ * Y_UNUSED(argumentUsedOnlyForDebugPurposes);
+ * }
+ * @endcode
+ */
+#ifdef __GNUC__
+ #define Y_DECLARE_UNUSED __attribute__((unused))
+#endif
+
+#ifndef Y_DECLARE_UNUSED
+ #define Y_DECLARE_UNUSED
+#endif
+
+#if defined(__GNUC__)
+ #define Y_LIKELY(Cond) __builtin_expect(!!(Cond), 1)
+ #define Y_UNLIKELY(Cond) __builtin_expect(!!(Cond), 0)
+ #define Y_PREFETCH_READ(Pointer, Priority) __builtin_prefetch((const void*)(Pointer), 0, Priority)
+ #define Y_PREFETCH_WRITE(Pointer, Priority) __builtin_prefetch((const void*)(Pointer), 1, Priority)
+#endif
+
+/**
+ * @def Y_FORCE_INLINE
+ *
+ * Macro to use in place of 'inline' in function declaration/definition to force
+ * it to be inlined.
+ */
+#if !defined(Y_FORCE_INLINE)
+ #if defined(CLANG_COVERAGE)
+ #/* excessive __always_inline__ might significantly slow down compilation of an instrumented unit */
+ #define Y_FORCE_INLINE inline
+ #elif defined(_MSC_VER)
+ #define Y_FORCE_INLINE __forceinline
+ #elif defined(__GNUC__)
+ #/* Clang also defines __GNUC__ (as 4) */
+ #define Y_FORCE_INLINE inline __attribute__((__always_inline__))
+ #else
+ #define Y_FORCE_INLINE inline
+ #endif
+#endif
+
+/**
+ * @def Y_NO_INLINE
+ *
+ * Macro to use in place of 'inline' in function declaration/definition to
+ * prevent it from being inlined.
+ */
+#if !defined(Y_NO_INLINE)
+ #if defined(_MSC_VER)
+ #define Y_NO_INLINE __declspec(noinline)
+ #elif defined(__GNUC__) || defined(__INTEL_COMPILER)
+ #/* Clang also defines __GNUC__ (as 4) */
+ #define Y_NO_INLINE __attribute__((__noinline__))
+ #else
+ #define Y_NO_INLINE
+ #endif
+#endif
+
+//to cheat compiler about strict aliasing or similar problems
+#if defined(__GNUC__)
+ #define Y_FAKE_READ(X) \
+ do { \
+ __asm__ __volatile__("" \
+ : \
+ : "m"(X)); \
+ } while (0)
+
+ #define Y_FAKE_WRITE(X) \
+ do { \
+ __asm__ __volatile__("" \
+ : "=m"(X)); \
+ } while (0)
+#endif
+
+#if !defined(Y_FAKE_READ)
+ #define Y_FAKE_READ(X)
+#endif
+
+#if !defined(Y_FAKE_WRITE)
+ #define Y_FAKE_WRITE(X)
+#endif
+
+#ifndef Y_PREFETCH_READ
+ #define Y_PREFETCH_READ(Pointer, Priority) (void)(const void*)(Pointer), (void)Priority
+#endif
+
+#ifndef Y_PREFETCH_WRITE
+ #define Y_PREFETCH_WRITE(Pointer, Priority) (void)(const void*)(Pointer), (void)Priority
+#endif
+
+#ifndef Y_LIKELY
+ #define Y_LIKELY(Cond) (Cond)
+ #define Y_UNLIKELY(Cond) (Cond)
+#endif
+
+#ifdef __GNUC__
+ #define Y_PACKED __attribute__((packed))
+#else
+ #define Y_PACKED
+#endif
+
+#if defined(__GNUC__)
+ #define Y_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#endif
+
+#ifndef Y_WARN_UNUSED_RESULT
+ #define Y_WARN_UNUSED_RESULT
+#endif
+
+#if defined(__GNUC__)
+ #define Y_HIDDEN __attribute__((visibility("hidden")))
+#endif
+
+#if !defined(Y_HIDDEN)
+ #define Y_HIDDEN
+#endif
+
+#if defined(__GNUC__)
+ #define Y_PUBLIC __attribute__((visibility("default")))
+#endif
+
+#if !defined(Y_PUBLIC)
+ #define Y_PUBLIC
+#endif
+
+#if !defined(Y_UNUSED) && !defined(__cplusplus)
+ #define Y_UNUSED(var) (void)(var)
+#endif
+#if !defined(Y_UNUSED) && defined(__cplusplus)
+template <class... Types>
+constexpr Y_FORCE_INLINE int Y_UNUSED(Types&&...) {
+ return 0;
+};
+#endif
+
+/**
+ * @def Y_ASSUME
+ *
+ * Macro that tells the compiler that it can generate optimized code
+ * as if the given expression will always evaluate true.
+ * The behavior is undefined if it ever evaluates false.
+ *
+ * @code
+ * // factored into a function so that it's testable
+ * inline int Avg(int x, int y) {
+ * if (x >= 0 && y >= 0) {
+ * return (static_cast<unsigned>(x) + static_cast<unsigned>(y)) >> 1;
+ * } else {
+ * // a slower implementation
+ * }
+ * }
+ *
+ * // we know that xs and ys are non-negative from domain knowledge,
+ * // but we can't change the types of xs and ys because of API constrains
+ * int Foo(const TVector<int>& xs, const TVector<int>& ys) {
+ * TVector<int> avgs;
+ * avgs.resize(xs.size());
+ * for (size_t i = 0; i < xs.size(); ++i) {
+ * auto x = xs[i];
+ * auto y = ys[i];
+ * Y_ASSUME(x >= 0);
+ * Y_ASSUME(y >= 0);
+ * xs[i] = Avg(x, y);
+ * }
+ * }
+ * @endcode
+ */
+#if defined(__GNUC__)
+ #define Y_ASSUME(condition) ((condition) ? (void)0 : __builtin_unreachable())
+#elif defined(_MSC_VER)
+ #define Y_ASSUME(condition) __assume(condition)
+#else
+ #define Y_ASSUME(condition) Y_UNUSED(condition)
+#endif
+
+#ifdef __cplusplus
+[[noreturn]]
+#endif
+Y_HIDDEN void
+_YandexAbort();
+
+/**
+ * @def Y_UNREACHABLE
+ *
+ * Macro that marks the rest of the code branch unreachable.
+ * The behavior is undefined if it's ever reached.
+ *
+ * @code
+ * switch (i % 3) {
+ * case 0:
+ * return foo;
+ * case 1:
+ * return bar;
+ * case 2:
+ * return baz;
+ * default:
+ * Y_UNREACHABLE();
+ * }
+ * @endcode
+ */
+#if defined(__GNUC__)
+ #define Y_UNREACHABLE() __builtin_unreachable()
+#elif defined(_MSC_VER)
+ #define Y_UNREACHABLE() __assume(false)
+#else
+ #define Y_UNREACHABLE() _YandexAbort()
+#endif
+
+#if defined(undefined_sanitizer_enabled)
+ #define _ubsan_enabled_
+#endif
+
+#ifdef __clang__
+
+ #if __has_feature(thread_sanitizer)
+ #define _tsan_enabled_
+ #endif
+ #if __has_feature(memory_sanitizer)
+ #define _msan_enabled_
+ #endif
+ #if __has_feature(address_sanitizer)
+ #define _asan_enabled_
+ #endif
+
+#else
+
+ #if defined(thread_sanitizer_enabled) || defined(__SANITIZE_THREAD__)
+ #define _tsan_enabled_
+ #endif
+ #if defined(memory_sanitizer_enabled)
+ #define _msan_enabled_
+ #endif
+ #if defined(address_sanitizer_enabled) || defined(__SANITIZE_ADDRESS__)
+ #define _asan_enabled_
+ #endif
+
+#endif
+
+#if defined(_asan_enabled_) || defined(_msan_enabled_) || defined(_tsan_enabled_) || defined(_ubsan_enabled_)
+ #define _san_enabled_
+#endif
+
+#if defined(_MSC_VER)
+ #define __PRETTY_FUNCTION__ __FUNCSIG__
+#endif
+
+#if defined(__GNUC__)
+ #define Y_WEAK __attribute__((weak))
+#else
+ #define Y_WEAK
+#endif
+
+#if defined(__CUDACC_VER_MAJOR__)
+ #define Y_CUDA_AT_LEAST(x, y) (__CUDACC_VER_MAJOR__ > x || (__CUDACC_VER_MAJOR__ == x && __CUDACC_VER_MINOR__ >= y))
+#else
+ #define Y_CUDA_AT_LEAST(x, y) 0
+#endif
+
+// NVidia CUDA C++ Compiler did not know about noexcept keyword until version 9.0
+#if !Y_CUDA_AT_LEAST(9, 0)
+ #if defined(__CUDACC__) && !defined(noexcept)
+ #define noexcept throw()
+ #endif
+#endif
+
+#if defined(__GNUC__)
+ #define Y_COLD __attribute__((cold))
+ #define Y_LEAF __attribute__((leaf))
+ #define Y_WRAPPER __attribute__((artificial))
+#else
+ #define Y_COLD
+ #define Y_LEAF
+ #define Y_WRAPPER
+#endif
+
+/**
+ * @def Y_PRAGMA
+ *
+ * Macro for use in other macros to define compiler pragma
+ * See below for other usage examples
+ *
+ * @code
+ * #if defined(__clang__) || defined(__GNUC__)
+ * #define Y_PRAGMA_NO_WSHADOW \
+ * Y_PRAGMA("GCC diagnostic ignored \"-Wshadow\"")
+ * #elif defined(_MSC_VER)
+ * #define Y_PRAGMA_NO_WSHADOW \
+ * Y_PRAGMA("warning(disable:4456 4457")
+ * #else
+ * #define Y_PRAGMA_NO_WSHADOW
+ * #endif
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA(x) _Pragma(x)
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA(x) __pragma(x)
+#else
+ #define Y_PRAGMA(x)
+#endif
+
+/**
+ * @def Y_PRAGMA_DIAGNOSTIC_PUSH
+ *
+ * Cross-compiler pragma to save diagnostic settings
+ *
+ * @see
+ * GCC: https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Pragmas.html
+ * MSVC: https://msdn.microsoft.com/en-us/library/2c8f766e.aspx
+ * Clang: https://clang.llvm.org/docs/UsersManual.html#controlling-diagnostics-via-pragmas
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_PUSH
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_DIAGNOSTIC_PUSH \
+ Y_PRAGMA("GCC diagnostic push")
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA_DIAGNOSTIC_PUSH \
+ Y_PRAGMA(warning(push))
+#else
+ #define Y_PRAGMA_DIAGNOSTIC_PUSH
+#endif
+
+/**
+ * @def Y_PRAGMA_DIAGNOSTIC_POP
+ *
+ * Cross-compiler pragma to restore diagnostic settings
+ *
+ * @see
+ * GCC: https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Pragmas.html
+ * MSVC: https://msdn.microsoft.com/en-us/library/2c8f766e.aspx
+ * Clang: https://clang.llvm.org/docs/UsersManual.html#controlling-diagnostics-via-pragmas
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_POP
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_DIAGNOSTIC_POP \
+ Y_PRAGMA("GCC diagnostic pop")
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA_DIAGNOSTIC_POP \
+ Y_PRAGMA(warning(pop))
+#else
+ #define Y_PRAGMA_DIAGNOSTIC_POP
+#endif
+
+/**
+ * @def Y_PRAGMA_NO_WSHADOW
+ *
+ * Cross-compiler pragma to disable warnings about shadowing variables
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_PUSH
+ * Y_PRAGMA_NO_WSHADOW
+ *
+ * // some code which use variable shadowing, e.g.:
+ *
+ * for (int i = 0; i < 100; ++i) {
+ * Use(i);
+ *
+ * for (int i = 42; i < 100500; ++i) { // this i is shadowing previous i
+ * AnotherUse(i);
+ * }
+ * }
+ *
+ * Y_PRAGMA_DIAGNOSTIC_POP
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_NO_WSHADOW \
+ Y_PRAGMA("GCC diagnostic ignored \"-Wshadow\"")
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA_NO_WSHADOW \
+ Y_PRAGMA(warning(disable : 4456 4457))
+#else
+ #define Y_PRAGMA_NO_WSHADOW
+#endif
+
+/**
+ * @ def Y_PRAGMA_NO_UNUSED_FUNCTION
+ *
+ * Cross-compiler pragma to disable warnings about unused functions
+ *
+ * @see
+ * GCC: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
+ * Clang: https://clang.llvm.org/docs/DiagnosticsReference.html#wunused-function
+ * MSVC: there is no such warning
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_PUSH
+ * Y_PRAGMA_NO_UNUSED_FUNCTION
+ *
+ * // some code which introduces a function which later will not be used, e.g.:
+ *
+ * void Foo() {
+ * }
+ *
+ * int main() {
+ * return 0; // Foo() never called
+ * }
+ *
+ * Y_PRAGMA_DIAGNOSTIC_POP
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_NO_UNUSED_FUNCTION \
+ Y_PRAGMA("GCC diagnostic ignored \"-Wunused-function\"")
+#else
+ #define Y_PRAGMA_NO_UNUSED_FUNCTION
+#endif
+
+/**
+ * @ def Y_PRAGMA_NO_UNUSED_PARAMETER
+ *
+ * Cross-compiler pragma to disable warnings about unused function parameters
+ *
+ * @see
+ * GCC: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
+ * Clang: https://clang.llvm.org/docs/DiagnosticsReference.html#wunused-parameter
+ * MSVC: https://msdn.microsoft.com/en-us/library/26kb9fy0.aspx
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_PUSH
+ * Y_PRAGMA_NO_UNUSED_PARAMETER
+ *
+ * // some code which introduces a function with unused parameter, e.g.:
+ *
+ * void foo(int a) {
+ * // a is not referenced
+ * }
+ *
+ * int main() {
+ * foo(1);
+ * return 0;
+ * }
+ *
+ * Y_PRAGMA_DIAGNOSTIC_POP
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_NO_UNUSED_PARAMETER \
+ Y_PRAGMA("GCC diagnostic ignored \"-Wunused-parameter\"")
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA_NO_UNUSED_PARAMETER \
+ Y_PRAGMA(warning(disable : 4100))
+#else
+ #define Y_PRAGMA_NO_UNUSED_PARAMETER
+#endif
+
+/**
+ * @def Y_PRAGMA_NO_DEPRECATED
+ *
+ * Cross compiler pragma to disable warnings and errors about deprecated
+ *
+ * @see
+ * GCC: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
+ * Clang: https://clang.llvm.org/docs/DiagnosticsReference.html#wdeprecated
+ * MSVC: https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-3-c4996?view=vs-2017
+ *
+ * @code
+ * Y_PRAGMA_DIAGNOSTIC_PUSH
+ * Y_PRAGMA_NO_DEPRECATED
+ *
+ * [deprecated] void foo() {
+ * // ...
+ * }
+ *
+ * int main() {
+ * foo();
+ * return 0;
+ * }
+ *
+ * Y_PRAGMA_DIAGNOSTIC_POP
+ * @endcode
+ */
+#if defined(__clang__) || defined(__GNUC__)
+ #define Y_PRAGMA_NO_DEPRECATED \
+ Y_PRAGMA("GCC diagnostic ignored \"-Wdeprecated\"")
+#elif defined(_MSC_VER)
+ #define Y_PRAGMA_NO_DEPRECATED \
+ Y_PRAGMA(warning(disable : 4996))
+#else
+ #define Y_PRAGMA_NO_DEPRECATED
+#endif
+
+// Memory sanitizer sometimes doesn't correctly set parameter shadow of constant functions.
+#if (defined(__clang__) || defined(__GNUC__)) && !defined(_msan_enabled_)
+ /**
+ * @def Y_CONST_FUNCTION
+ methods and functions, marked with this method are promised to:
+ 1. do not have side effects
+ 2. this method do not read global memory
+ NOTE: this attribute can't be set for methods that depend on data, pointed by this
+ this allow compilers to do hard optimization of that functions
+ NOTE: in common case this attribute can't be set if method have pointer-arguments
+ NOTE: as result there no any reason to discard result of such method
+*/
+ #define Y_CONST_FUNCTION [[gnu::const]]
+#endif
+
+#if !defined(Y_CONST_FUNCTION)
+ #define Y_CONST_FUNCTION
+#endif
+
+#if defined(__clang__) || defined(__GNUC__)
+ /**
+ * @def Y_PURE_FUNCTION
+ methods and functions, marked with this method are promised to:
+ 1. do not have side effects
+ 2. result will be the same if no global memory changed
+ this allow compilers to do hard optimization of that functions
+ NOTE: as result there no any reason to discard result of such method
+*/
+ #define Y_PURE_FUNCTION [[gnu::pure]]
+#endif
+
+#if !defined(Y_PURE_FUNCTION)
+ #define Y_PURE_FUNCTION
+#endif
+
+/**
+ * @ def Y_HAVE_INT128
+ *
+ * Defined when the compiler supports __int128 extension
+ *
+ * @code
+ *
+ * #if defined(Y_HAVE_INT128)
+ * __int128 myVeryBigInt = 12345678901234567890;
+ * #endif
+ *
+ * @endcode
+ */
+#if defined(__SIZEOF_INT128__)
+ #define Y_HAVE_INT128 1
+#endif
+
+/**
+ * XRAY macro must be passed to compiler if XRay is enabled.
+ *
+ * Define everything XRay-specific as a macro so that it doesn't cause errors
+ * for compilers that doesn't support XRay.
+ */
+#if defined(XRAY) && defined(__cplusplus)
+ #include <xray/xray_interface.h>
+ #define Y_XRAY_ALWAYS_INSTRUMENT [[clang::xray_always_instrument]]
+ #define Y_XRAY_NEVER_INSTRUMENT [[clang::xray_never_instrument]]
+ #define Y_XRAY_CUSTOM_EVENT(__string, __length) \
+ do { \
+ __xray_customevent(__string, __length); \
+ } while (0)
+#else
+ #define Y_XRAY_ALWAYS_INSTRUMENT
+ #define Y_XRAY_NEVER_INSTRUMENT
+ #define Y_XRAY_CUSTOM_EVENT(__string, __length) \
+ do { \
+ } while (0)
+#endif
+
+#ifdef __cplusplus
+
+void UseCharPointerImpl(volatile const char*);
+
+template <typename T>
+Y_FORCE_INLINE void DoNotOptimizeAway(T&& datum) {
+ #if defined(_MSC_VER)
+ UseCharPointerImpl(&reinterpret_cast<volatile const char&>(datum));
+ _ReadWriteBarrier();
+ #elif defined(__GNUC__) && defined(_x86_)
+ asm volatile(""
+ :
+ : "X"(datum));
+ #else
+ Y_FAKE_READ(datum);
+ #endif
+}
+
+ /**
+ * Use this macro to prevent unused variables elimination.
+ */
+ #define Y_DO_NOT_OPTIMIZE_AWAY(X) ::DoNotOptimizeAway(X)
+
+#endif
diff --git a/util/system/compiler_ut.cpp b/util/system/compiler_ut.cpp
new file mode 100644
index 0000000000..f93b1c0850
--- /dev/null
+++ b/util/system/compiler_ut.cpp
@@ -0,0 +1,72 @@
+#include "compiler.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TCompilerTest) {
+ Y_UNIT_TEST(TestPragmaNoWshadow) {
+ Y_PRAGMA_DIAGNOSTIC_PUSH
+ Y_PRAGMA_NO_WSHADOW
+
+ // define two variables with similar names, latest must shadow first
+ // and there will be no warning for this
+
+ for (int i = 0; i < 1; ++i) {
+ for (int i = 100500; i < 100501; ++i) {
+ UNIT_ASSERT_EQUAL(i, 100500);
+ }
+ }
+
+ Y_PRAGMA_DIAGNOSTIC_POP
+ }
+
+ Y_PRAGMA_DIAGNOSTIC_PUSH
+ Y_PRAGMA_NO_UNUSED_PARAMETER
+
+ // define function with unused named parameter
+ // and there will be no warning for this
+ int Foo(int a) {
+ return 0;
+ }
+
+ Y_PRAGMA_DIAGNOSTIC_POP
+
+ Y_UNIT_TEST(TestPragmaNoUnusedParameter) {
+ UNIT_ASSERT_EQUAL(Foo(1), 0);
+ }
+
+ Y_UNIT_TEST(TestHaveInt128) {
+#ifdef Y_HAVE_INT128
+ // will be compiled without errors
+ unsigned __int128 a = 1;
+ __int128 b = 1;
+ UNIT_ASSERT_EQUAL(a, 1);
+ UNIT_ASSERT_EQUAL(b, 1);
+ UNIT_ASSERT_EQUAL(sizeof(a), sizeof(b));
+
+ // and we can set a type alias for __int128 and unsigned __int128 without compiler errors
+ using TMyInt128 = __int128;
+ using TMyUnsignedInt128 = unsigned __int128;
+
+ TMyInt128 i128value;
+ TMyUnsignedInt128 ui128value;
+ Y_UNUSED(i128value);
+ Y_UNUSED(ui128value);
+
+#endif
+ }
+
+ // define deprecated function
+ [[deprecated]] void Bar() {
+ return;
+ }
+
+ Y_UNIT_TEST(TestNoDeprecated) {
+ Y_PRAGMA_DIAGNOSTIC_PUSH
+ Y_PRAGMA_NO_DEPRECATED
+
+ // will be compiled without errors
+ Bar();
+
+ Y_PRAGMA_DIAGNOSTIC_POP
+ }
+}
diff --git a/util/system/condvar.cpp b/util/system/condvar.cpp
new file mode 100644
index 0000000000..62f3d22356
--- /dev/null
+++ b/util/system/condvar.cpp
@@ -0,0 +1,147 @@
+#include "event.h"
+#include "mutex.h"
+#include "yassert.h"
+#include "condvar.h"
+#include "datetime.h"
+#include "spinlock.h"
+
+#include <util/generic/ylimits.h>
+#include <util/generic/intrlist.h>
+#include <util/generic/yexception.h>
+
+#include <cstdlib>
+
+#if defined(_unix_)
+ #include <sys/time.h>
+ #include <pthread.h>
+ #include <cerrno>
+#endif
+
+namespace {
+ class TCondVarImpl {
+ using TLock = TAdaptiveLock;
+
+ struct TWaitEvent: public TIntrusiveListItem<TWaitEvent>, public TSystemEvent {
+ };
+
+ using TWaitEvents = TIntrusiveList<TWaitEvent>;
+
+ public:
+ inline ~TCondVarImpl() {
+ Y_ASSERT(Events_.Empty());
+ }
+
+ inline void Signal() noexcept {
+ with_lock (Lock_) {
+ if (!Events_.Empty()) {
+ Events_.PopFront()->Signal();
+ }
+ }
+ }
+
+ inline void BroadCast() noexcept {
+ with_lock (Lock_) {
+ //TODO
+ while (!Events_.Empty()) {
+ Events_.PopFront()->Signal();
+ }
+ }
+ }
+
+ inline bool WaitD(TMutex& m, TInstant deadLine) noexcept {
+ TWaitEvent event;
+
+ with_lock (Lock_) {
+ Events_.PushBack(&event);
+ }
+
+ m.Release();
+
+ const bool signalled = event.WaitD(deadLine);
+
+ m.Acquire();
+
+ with_lock (Lock_) {
+ event.Unlink();
+ }
+
+ return signalled;
+ }
+
+ private:
+ TWaitEvents Events_;
+ TLock Lock_;
+ };
+}
+
+#if defined(_win_)
+class TCondVar::TImpl: public TCondVarImpl {
+};
+#else
+class TCondVar::TImpl {
+public:
+ inline TImpl() {
+ if (pthread_cond_init(&Cond_, nullptr)) {
+ ythrow yexception() << "can not create condvar(" << LastSystemErrorText() << ")";
+ }
+ }
+
+ inline ~TImpl() {
+ int ret = pthread_cond_destroy(&Cond_);
+ Y_VERIFY(ret == 0, "pthread_cond_destroy failed: %s", LastSystemErrorText(ret));
+ }
+
+ inline void Signal() noexcept {
+ int ret = pthread_cond_signal(&Cond_);
+ Y_VERIFY(ret == 0, "pthread_cond_signal failed: %s", LastSystemErrorText(ret));
+ }
+
+ inline bool WaitD(TMutex& lock, TInstant deadLine) noexcept {
+ if (deadLine == TInstant::Max()) {
+ int ret = pthread_cond_wait(&Cond_, (pthread_mutex_t*)lock.Handle());
+ Y_VERIFY(ret == 0, "pthread_cond_wait failed: %s", LastSystemErrorText(ret));
+ return true;
+ } else {
+ struct timespec spec;
+
+ Zero(spec);
+
+ spec.tv_sec = deadLine.Seconds();
+ spec.tv_nsec = deadLine.NanoSecondsOfSecond();
+
+ int ret = pthread_cond_timedwait(&Cond_, (pthread_mutex_t*)lock.Handle(), &spec);
+
+ Y_VERIFY(ret == 0 || ret == ETIMEDOUT, "pthread_cond_timedwait failed: %s", LastSystemErrorText(ret));
+
+ return ret == 0;
+ }
+ }
+
+ inline void BroadCast() noexcept {
+ int ret = pthread_cond_broadcast(&Cond_);
+ Y_VERIFY(ret == 0, "pthread_cond_broadcast failed: %s", LastSystemErrorText(ret));
+ }
+
+private:
+ pthread_cond_t Cond_;
+};
+#endif
+
+TCondVar::TCondVar()
+ : Impl_(new TImpl)
+{
+}
+
+TCondVar::~TCondVar() = default;
+
+void TCondVar::BroadCast() noexcept {
+ Impl_->BroadCast();
+}
+
+void TCondVar::Signal() noexcept {
+ Impl_->Signal();
+}
+
+bool TCondVar::WaitD(TMutex& mutex, TInstant deadLine) noexcept {
+ return Impl_->WaitD(mutex, deadLine);
+}
diff --git a/util/system/condvar.h b/util/system/condvar.h
new file mode 100644
index 0000000000..569162717c
--- /dev/null
+++ b/util/system/condvar.h
@@ -0,0 +1,71 @@
+#pragma once
+
+#include "mutex.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+#include <util/datetime/base.h>
+
+#include <utility>
+
+class TCondVar {
+public:
+ TCondVar();
+ ~TCondVar();
+
+ void BroadCast() noexcept;
+ void Signal() noexcept;
+
+ /*
+ * returns false if failed by timeout
+ */
+ bool WaitD(TMutex& m, TInstant deadline) noexcept;
+
+ template <typename P>
+ inline bool WaitD(TMutex& m, TInstant deadline, P pred) noexcept {
+ while (!pred()) {
+ if (!WaitD(m, deadline)) {
+ return pred();
+ }
+ }
+ return true;
+ }
+
+ /*
+ * returns false if failed by timeout
+ */
+ inline bool WaitT(TMutex& m, TDuration timeout) noexcept {
+ return WaitD(m, timeout.ToDeadLine());
+ }
+
+ template <typename P>
+ inline bool WaitT(TMutex& m, TDuration timeout, P pred) noexcept {
+ return WaitD(m, timeout.ToDeadLine(), std::move(pred));
+ }
+
+ /*
+ * infinite wait
+ */
+ inline void WaitI(TMutex& m) noexcept {
+ WaitD(m, TInstant::Max());
+ }
+
+ template <typename P>
+ inline void WaitI(TMutex& m, P pred) noexcept {
+ WaitD(m, TInstant::Max(), std::move(pred));
+ }
+
+ //deprecated
+ inline void Wait(TMutex& m) noexcept {
+ WaitI(m);
+ }
+
+ template <typename P>
+ inline void Wait(TMutex& m, P pred) noexcept {
+ WaitI(m, std::move(pred));
+ }
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
diff --git a/util/system/condvar_ut.cpp b/util/system/condvar_ut.cpp
new file mode 100644
index 0000000000..5130a18d32
--- /dev/null
+++ b/util/system/condvar_ut.cpp
@@ -0,0 +1,200 @@
+#include "mutex.h"
+#include "guard.h"
+#include "condvar.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/system/atomic.h>
+#include <util/system/atomic_ops.h>
+#include <util/thread/pool.h>
+
+class TCondVarTest: public TTestBase {
+ UNIT_TEST_SUITE(TCondVarTest);
+ UNIT_TEST(TestBasics)
+ UNIT_TEST(TestSyncronize)
+ UNIT_TEST_SUITE_END();
+
+ struct TSharedData {
+ TSharedData()
+ : stopWaiting(false)
+ , in(0)
+ , out(0)
+ , waited(0)
+ , failed(false)
+ {
+ }
+
+ TMutex mutex;
+ TCondVar condVar1;
+ TCondVar condVar2;
+
+ TAtomic stopWaiting;
+
+ TAtomic in;
+ TAtomic out;
+
+ TAtomic waited;
+
+ bool failed;
+ };
+
+ class TThreadTask: public IObjectInQueue {
+ public:
+ using PFunc = void (TThreadTask::*)(void);
+
+ TThreadTask(PFunc func, size_t id, size_t totalIds, TSharedData& data)
+ : Func_(func)
+ , Id_(id)
+ , TotalIds_(totalIds)
+ , Data_(data)
+ {
+ }
+
+ void Process(void*) override {
+ THolder<TThreadTask> This(this);
+
+ (this->*Func_)();
+ }
+
+#define FAIL_ASSERT(cond) \
+ if (!(cond)) { \
+ Data_.failed = true; \
+ }
+ void RunBasics() {
+ Y_ASSERT(TotalIds_ == 3);
+
+ if (Id_ < 2) {
+ TGuard<TMutex> guard(Data_.mutex);
+ while (!AtomicGet(Data_.stopWaiting)) {
+ bool res = Data_.condVar1.WaitT(Data_.mutex, TDuration::Seconds(1));
+ FAIL_ASSERT(res == true);
+ }
+ } else {
+ usleep(100000);
+ AtomicSet(Data_.stopWaiting, true);
+
+ TGuard<TMutex> guard(Data_.mutex);
+ Data_.condVar1.Signal();
+ Data_.condVar1.Signal();
+ }
+ }
+
+ void RunBasicsWithPredicate() {
+ Y_ASSERT(TotalIds_ == 3);
+
+ if (Id_ < 2) {
+ TGuard<TMutex> guard(Data_.mutex);
+ const auto res = Data_.condVar1.WaitT(Data_.mutex, TDuration::Seconds(1), [&] {
+ return AtomicGet(Data_.stopWaiting);
+ });
+ FAIL_ASSERT(res == true);
+ } else {
+ usleep(100000);
+ AtomicSet(Data_.stopWaiting, true);
+
+ TGuard<TMutex> guard(Data_.mutex);
+ Data_.condVar1.Signal();
+ Data_.condVar1.Signal();
+ }
+ }
+
+ void RunSyncronize() {
+ for (size_t i = 0; i < 10; ++i) {
+ TGuard<TMutex> guard(Data_.mutex);
+ AtomicIncrement(Data_.in);
+ if (AtomicGet(Data_.in) == TotalIds_) {
+ AtomicSet(Data_.out, 0);
+ Data_.condVar1.BroadCast();
+ } else {
+ AtomicIncrement(Data_.waited);
+ while (AtomicGet(Data_.in) < TotalIds_) {
+ bool res = Data_.condVar1.WaitT(Data_.mutex, TDuration::Seconds(1));
+ FAIL_ASSERT(res == true);
+ }
+ }
+
+ AtomicIncrement(Data_.out);
+ if (AtomicGet(Data_.out) == TotalIds_) {
+ AtomicSet(Data_.in, 0);
+ Data_.condVar2.BroadCast();
+ } else {
+ while (AtomicGet(Data_.out) < TotalIds_) {
+ bool res = Data_.condVar2.WaitT(Data_.mutex, TDuration::Seconds(1));
+ FAIL_ASSERT(res == true);
+ }
+ }
+ }
+
+ FAIL_ASSERT(AtomicGet(Data_.waited) == (TotalIds_ - 1) * 10);
+ }
+
+ void RunSyncronizeWithPredicate() {
+ for (size_t i = 0; i < 10; ++i) {
+ TGuard<TMutex> guard(Data_.mutex);
+ AtomicIncrement(Data_.in);
+ if (AtomicGet(Data_.in) == TotalIds_) {
+ AtomicSet(Data_.out, 0);
+ Data_.condVar1.BroadCast();
+ } else {
+ AtomicIncrement(Data_.waited);
+ const auto res = Data_.condVar1.WaitT(Data_.mutex, TDuration::Seconds(1), [&] {
+ return AtomicGet(Data_.in) >= TotalIds_;
+ });
+ FAIL_ASSERT(res == true);
+ }
+
+ AtomicIncrement(Data_.out);
+ if (AtomicGet(Data_.out) == TotalIds_) {
+ AtomicSet(Data_.in, 0);
+ Data_.condVar2.BroadCast();
+ } else {
+ const auto res = Data_.condVar2.WaitT(Data_.mutex, TDuration::Seconds(1), [&] {
+ return AtomicGet(Data_.out) >= TotalIds_;
+ });
+ FAIL_ASSERT(res == true);
+ }
+ }
+
+ FAIL_ASSERT(Data_.waited == (TotalIds_ - 1) * 10);
+ }
+#undef FAIL_ASSERT
+
+ private:
+ PFunc Func_;
+ size_t Id_;
+ TAtomicBase TotalIds_;
+ TSharedData& Data_;
+ };
+
+private:
+#define RUN_CYCLE(what, count) \
+ Q_.Start(count); \
+ for (size_t i = 0; i < count; ++i) { \
+ UNIT_ASSERT(Q_.Add(new TThreadTask(&TThreadTask::what, i, count, Data_))); \
+ } \
+ Q_.Stop(); \
+ bool b = Data_.failed; \
+ Data_.failed = false; \
+ UNIT_ASSERT(!b);
+
+ inline void TestBasics() {
+ RUN_CYCLE(RunBasics, 3);
+ }
+
+ inline void TestBasicsWithPredicate() {
+ RUN_CYCLE(RunBasicsWithPredicate, 3);
+ }
+
+ inline void TestSyncronize() {
+ RUN_CYCLE(RunSyncronize, 6);
+ }
+
+ inline void TestSyncronizeWithPredicate() {
+ RUN_CYCLE(RunSyncronizeWithPredicate, 6);
+ }
+#undef RUN_CYCLE
+ TSharedData Data_;
+ TThreadPool Q_;
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TCondVarTest);
diff --git a/util/system/context.cpp b/util/system/context.cpp
new file mode 100644
index 0000000000..ad99309088
--- /dev/null
+++ b/util/system/context.cpp
@@ -0,0 +1,336 @@
+#include "compiler.h"
+#include "defaults.h"
+#include "event.h"
+#include "thread.h"
+
+#include <cstdlib> //for abort()
+
+#if defined(_win_)
+ #include "winint.h"
+#endif
+
+#if defined(_unix_)
+ #include <cxxabi.h>
+
+ #if !defined(Y_CXA_EH_GLOBALS_COMPLETE)
+namespace __cxxabiv1 {
+ struct __cxa_eh_globals {
+ void* caughtExceptions;
+ unsigned int uncaughtExceptions;
+ };
+
+ extern "C" __cxa_eh_globals* __cxa_get_globals();
+}
+ #endif
+#endif
+
+#include <util/stream/output.h>
+#include <util/generic/yexception.h>
+
+#define FROM_CONTEXT_IMPL
+#include "context.h"
+
+void ITrampoLine::DoRun() {
+}
+
+void ITrampoLine::DoRunNaked() {
+ try {
+ DoRun();
+ } catch (...) {
+ Cerr << "Uncaught exception in coroutine: " << CurrentExceptionMessage() << "\n";
+ }
+
+ abort();
+}
+
+static inline void Run(void* arg) {
+ ((ITrampoLine*)arg)->DoRunNaked();
+}
+
+#if defined(USE_JUMP_CONT)
+extern "C" void __mylongjmp(__myjmp_buf env, int val) __attribute__((__noreturn__));
+extern "C" int __mysetjmp(__myjmp_buf env) __attribute__((__returns_twice__));
+
+namespace {
+ class TStackType {
+ public:
+ inline TStackType(TArrayRef<char> range) noexcept
+ #if defined(STACK_GROW_DOWN)
+ : Data_(range.data() + range.size())
+ #else
+ : Data_(range.data() + STACK_ALIGN)
+ #endif
+ {
+ ReAlign();
+ }
+
+ inline ~TStackType() = default;
+
+ inline void ReAlign() noexcept {
+ Data_ = AlignStackPtr(Data_);
+ }
+
+ template <class T>
+ inline void Push(T t) noexcept {
+ #if defined(STACK_GROW_DOWN)
+ Data_ -= sizeof(T);
+ *((T*)Data_) = t;
+ #else
+ *((T*)Data_) = t;
+ Data_ += sizeof(T);
+ #endif
+ }
+
+ inline char* StackPtr() noexcept {
+ return Data_;
+ }
+
+ private:
+ static inline char* AlignStackPtr(char* ptr) noexcept {
+ #if defined(STACK_GROW_DOWN)
+ return AlignDown(ptr, STACK_ALIGN);
+ #else
+ return AlignUp(ptr, STACK_ALIGN);
+ #endif
+ }
+
+ private:
+ char* Data_;
+ };
+
+ static inline void*& JmpBufReg(__myjmp_buf& buf, size_t n) noexcept {
+ return (((void**)(void*)(buf))[n]);
+ }
+
+ static inline void*& JmpBufStackReg(__myjmp_buf& buf) noexcept {
+ return JmpBufReg(buf, STACK_CNT);
+ }
+
+ static inline void*& JmpBufProgrReg(__myjmp_buf& buf) noexcept {
+ return JmpBufReg(buf, PROGR_CNT);
+ }
+
+ static inline void*& JmpBufFrameReg(__myjmp_buf& buf) noexcept {
+ return JmpBufReg(buf, FRAME_CNT);
+ }
+
+ #if defined(_x86_64_)
+ // not sure if Y_NO_SANITIZE is needed
+ Y_NO_SANITIZE("address")
+ Y_NO_SANITIZE("memory") extern "C" void ContextTrampoLine(void*, void*, void*, void*, void*, void*, // register arguments, no defined value
+ /* first argument passed through the stack */ void* t1,
+ /* second argument passed through the stack */ void* t2) {
+ Y_ASSERT(t1 == t2);
+ Run(t1);
+ }
+ #else
+ Y_NO_SANITIZE("address")
+ Y_NO_SANITIZE("memory") static void ContextTrampoLine() {
+ void** argPtr = (void**)((char*)AlignUp(&argPtr + EXTRA_PUSH_ARGS, STACK_ALIGN) + STACK_ALIGN);
+ Y_ASSERT(*(argPtr - 1) == *(argPtr - 2));
+
+ Run(*(argPtr - 1));
+ }
+ #endif
+}
+
+TContMachineContext::TSan::TSan() noexcept
+ : TL(nullptr)
+{
+}
+
+TContMachineContext::TSan::TSan(const TContClosure& c) noexcept
+ : NSan::TFiberContext(c.Stack.data(), c.Stack.size(), c.ContName)
+ , TL(c.TrampoLine)
+{
+}
+
+void TContMachineContext::TSan::DoRunNaked() {
+ AfterSwitch();
+ TL->DoRunNaked();
+ BeforeFinish();
+}
+
+TContMachineContext::TContMachineContext(const TContClosure& c)
+ #if defined(_asan_enabled_) || defined(_tsan_enabled_)
+ : San_(c)
+ #endif
+{
+ TStackType stack(c.Stack);
+
+ /*
+ * arg, and align data
+ */
+
+ #if defined(_asan_enabled_)
+ auto trampoline = &San_;
+ #else
+ auto trampoline = c.TrampoLine;
+ #endif
+
+ #if defined(_x86_64_)
+ stack.ReAlign();
+ // push twice to preserve alignment by 16
+ stack.Push(trampoline); // second stack argument
+ stack.Push(trampoline); // first stack argument
+
+ stack.Push(nullptr); // fake return address
+ #else
+ stack.Push(trampoline);
+ stack.Push(trampoline);
+ stack.ReAlign();
+ /*
+ * fake return address
+ */
+ for (size_t i = 0; i < EXTRA_PUSH_ARGS; ++i) {
+ stack.Push(nullptr);
+ }
+ #endif
+
+ __mysetjmp(Buf_);
+
+ JmpBufProgrReg(Buf_) = reinterpret_cast<void*>(ContextTrampoLine);
+ JmpBufStackReg(Buf_) = stack.StackPtr();
+ JmpBufFrameReg(Buf_) = nullptr;
+}
+
+void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
+ if (Y_LIKELY(__mysetjmp(Buf_) == 0)) {
+ #if defined(_asan_enabled_) || defined(_tsan_enabled_)
+ next->San_.BeforeSwitch(&San_);
+ #endif
+ __mylongjmp(next->Buf_, 1);
+ } else {
+ #if defined(_asan_enabled_)
+ San_.AfterSwitch();
+ #endif
+ }
+}
+#elif defined(_win_) && defined(_32_)
+void __stdcall ContextTrampoLine(void* arg) {
+ Run(arg);
+}
+#else
+void ContextTrampoLine(void* arg) {
+ Run(arg);
+}
+#endif
+
+#if defined(USE_FIBER_CONT)
+TContMachineContext::TContMachineContext()
+ : Fiber_(ConvertThreadToFiber(this))
+ , MainFiber_(true)
+{
+ Y_ENSURE(Fiber_, TStringBuf("fiber error"));
+}
+
+TContMachineContext::TContMachineContext(const TContClosure& c)
+ : Fiber_(CreateFiber(c.Stack.size(), (LPFIBER_START_ROUTINE)ContextTrampoLine, (LPVOID)c.TrampoLine))
+ , MainFiber_(false)
+{
+ Y_ENSURE(Fiber_, TStringBuf("fiber error"));
+}
+
+TContMachineContext::~TContMachineContext() {
+ if (MainFiber_) {
+ ConvertFiberToThread();
+ } else {
+ DeleteFiber(Fiber_);
+ }
+}
+
+void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
+ SwitchToFiber(next->Fiber_);
+}
+#endif
+
+#if defined(USE_GENERIC_CONT)
+ #include <pthread.h>
+
+struct TContMachineContext::TImpl {
+ inline TImpl()
+ : TL(nullptr)
+ , Finish(false)
+ {
+ }
+
+ inline TImpl(const TContClosure& c)
+ : TL(c.TrampoLine)
+ , Finish(false)
+ {
+ Thread.Reset(new TThread(TThread::TParams(Run, this).SetStackSize(c.Stack.size()).SetStackPointer((void*)c.Stack.data())));
+ Thread->Start();
+ }
+
+ inline ~TImpl() {
+ if (Thread) {
+ Finish = true;
+ Signal();
+ Thread->Join();
+ }
+ }
+
+ inline void SwitchTo(TImpl* next) noexcept {
+ next->Signal();
+ Wait();
+ }
+
+ static void* Run(void* self) {
+ ((TImpl*)self)->DoRun();
+
+ return nullptr;
+ }
+
+ inline void DoRun() {
+ Wait();
+ TL->DoRun();
+ }
+
+ inline void Signal() noexcept {
+ Event.Signal();
+ }
+
+ inline void Wait() noexcept {
+ Event.Wait();
+
+ if (Finish) {
+ // TODO - need proper TThread::Exit(), have some troubles in win32 now
+ pthread_exit(0);
+ }
+ }
+
+ TAutoEvent Event;
+ THolder<TThread> Thread;
+ ITrampoLine* TL;
+ bool Finish;
+};
+
+TContMachineContext::TContMachineContext()
+ : Impl_(new TImpl())
+{
+}
+
+TContMachineContext::TContMachineContext(const TContClosure& c)
+ : Impl_(new TImpl(c))
+{
+}
+
+TContMachineContext::~TContMachineContext() {
+}
+
+void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
+ Impl_->SwitchTo(next->Impl_.Get());
+}
+#endif
+
+void TExceptionSafeContext::SwitchTo(TExceptionSafeContext* to) noexcept {
+#if defined(_unix_)
+ static_assert(sizeof(__cxxabiv1::__cxa_eh_globals) == sizeof(Buf_), "size mismatch of __cxa_eh_globals structure");
+
+ auto* eh = __cxxabiv1::__cxa_get_globals();
+ ::memcpy(Buf_, eh, sizeof(Buf_));
+ ::memcpy(eh, to->Buf_, sizeof(Buf_));
+#endif
+
+ TContMachineContext::SwitchTo(to);
+}
diff --git a/util/system/context.h b/util/system/context.h
new file mode 100644
index 0000000000..d2a349bfc5
--- /dev/null
+++ b/util/system/context.h
@@ -0,0 +1,181 @@
+#pragma once
+
+#include "align.h"
+#include "defaults.h"
+#include "compiler.h"
+#include "sanitizers.h"
+
+#include <util/generic/array_ref.h>
+#include <util/generic/utility.h>
+#include <util/generic/yexception.h>
+
+#define STACK_ALIGN (8 * PLATFORM_DATA_ALIGN)
+
+#if defined(_x86_64_) || defined(_i386_) || defined(_arm_) || defined(_ppc64_)
+ #define STACK_GROW_DOWN 1
+#else
+ #error todo
+#endif
+
+/*
+ * switch method
+ */
+#if defined(_bionic_) || defined(__IOS__)
+ #define USE_GENERIC_CONT
+#elif defined(_cygwin_)
+ #define USE_UCONTEXT_CONT
+#elif defined(_win_)
+ #define USE_FIBER_CONT
+#elif (defined(_i386_) || defined(_x86_64_) || defined(_arm64_)) && !defined(_k1om_)
+ #define USE_JUMP_CONT
+#else
+ #define USE_UCONTEXT_CONT
+#endif
+
+#if defined(USE_JUMP_CONT)
+ #if defined(_arm64_)
+ #include "context_aarch64.h"
+ #else
+ #include "context_x86.h"
+ #endif
+#endif
+
+#if defined(USE_UCONTEXT_CONT)
+ #include <ucontext.h>
+#endif
+
+struct ITrampoLine {
+ virtual ~ITrampoLine() = default;
+
+ virtual void DoRun();
+ virtual void DoRunNaked();
+};
+
+struct TContClosure {
+ ITrampoLine* TrampoLine;
+ TArrayRef<char> Stack;
+ const char* ContName = nullptr;
+};
+
+#if defined(USE_UCONTEXT_CONT)
+class TContMachineContext {
+ typedef void (*ucontext_func_t)(void);
+
+public:
+ inline TContMachineContext() {
+ getcontext(&Ctx_);
+ }
+
+ inline TContMachineContext(const TContClosure& c) {
+ getcontext(&Ctx_);
+
+ Ctx_.uc_link = 0;
+ Ctx_.uc_stack.ss_sp = (void*)c.Stack.data();
+ Ctx_.uc_stack.ss_size = c.Stack.size();
+ Ctx_.uc_stack.ss_flags = 0;
+
+ extern void ContextTrampoLine(void* arg);
+ makecontext(&Ctx_, (ucontext_func_t)ContextTrampoLine, 1, c.TrampoLine);
+ }
+
+ inline ~TContMachineContext() {
+ }
+
+ inline void SwitchTo(TContMachineContext* next) noexcept {
+ swapcontext(&Ctx_, &next->Ctx_);
+ }
+
+private:
+ ucontext_t Ctx_;
+};
+#endif
+
+#if defined(USE_GENERIC_CONT)
+class TContMachineContext {
+ struct TImpl;
+
+public:
+ TContMachineContext();
+ TContMachineContext(const TContClosure& c);
+
+ ~TContMachineContext();
+
+ void SwitchTo(TContMachineContext* next) noexcept;
+
+private:
+ THolder<TImpl> Impl_;
+};
+#endif
+
+#if defined(USE_FIBER_CONT)
+class TContMachineContext {
+public:
+ TContMachineContext();
+ TContMachineContext(const TContClosure& c);
+ ~TContMachineContext();
+
+ void SwitchTo(TContMachineContext* next) noexcept;
+
+private:
+ void* Fiber_;
+ bool MainFiber_;
+};
+#endif
+
+#if defined(USE_JUMP_CONT)
+class TContMachineContext {
+public:
+ inline TContMachineContext() {
+ Zero(Buf_);
+ }
+
+ TContMachineContext(const TContClosure& c);
+
+ inline ~TContMachineContext() = default;
+
+ void SwitchTo(TContMachineContext* next) noexcept;
+
+private:
+ __myjmp_buf Buf_;
+
+ struct TSan: public ITrampoLine, public ::NSan::TFiberContext {
+ TSan() noexcept;
+ TSan(const TContClosure& c) noexcept;
+
+ void DoRunNaked() override;
+
+ ITrampoLine* TL;
+ };
+
+ #if defined(_asan_enabled_) || defined(_tsan_enabled_)
+ TSan San_;
+ #endif
+};
+#endif
+
+static inline size_t MachineContextSize() noexcept {
+ return sizeof(TContMachineContext);
+}
+
+/*
+ * be polite
+ */
+#if !defined(FROM_CONTEXT_IMPL)
+ #undef USE_JUMP_CONT
+ #undef USE_FIBER_CONT
+ #undef USE_GENERIC_CONT
+ #undef USE_UCONTEXT_CONT
+ #undef PROGR_CNT
+ #undef STACK_CNT
+ #undef EXTRA_PUSH_ARGS
+#endif
+
+struct TExceptionSafeContext: public TContMachineContext {
+ using TContMachineContext::TContMachineContext;
+
+ void SwitchTo(TExceptionSafeContext* to) noexcept;
+
+#if defined(_unix_)
+ void* Buf_[2] = {nullptr, nullptr};
+#endif
+};
diff --git a/util/system/context_aarch64.S b/util/system/context_aarch64.S
new file mode 100644
index 0000000000..0b2ef4e4a6
--- /dev/null
+++ b/util/system/context_aarch64.S
@@ -0,0 +1,52 @@
+.p2align 2
+#if !(defined __darwin__) && !(defined __arm64__)
+.global __mysetjmp
+.type __mysetjmp,@function
+__mysetjmp:
+#else
+.global ___mysetjmp
+___mysetjmp:
+#endif
+ // IHI0055B_aapcs64.pdf 5.1.1, 5.1.2 callee saved registers
+ stp x19, x20, [x0,#0]
+ stp x21, x22, [x0,#16]
+ stp x23, x24, [x0,#32]
+ stp x25, x26, [x0,#48]
+ stp x27, x28, [x0,#64]
+ stp x29, x30, [x0,#80]
+ mov x2, sp
+ str x2, [x0,#104]
+ stp d8, d9, [x0,#112]
+ stp d10, d11, [x0,#128]
+ stp d12, d13, [x0,#144]
+ stp d14, d15, [x0,#160]
+ mov x0, #0
+ ret
+
+.p2align 2
+#if !(defined __darwin__) && !(defined __arm64__)
+.global __mylongjmp
+.type __mylongjump,@function
+__mylongjmp:
+#else
+.global ___mylongjmp
+___mylongjmp:
+#endif
+ // IHI0055B_aapcs64.pdf 5.1.1, 5.1.2 callee saved registers
+ ldp x19, x20, [x0,#0]
+ ldp x21, x22, [x0,#16]
+ ldp x23, x24, [x0,#32]
+ ldp x25, x26, [x0,#48]
+ ldp x27, x28, [x0,#64]
+ ldp x29, x30, [x0,#80]
+ ldr x2, [x0,#104]
+ mov sp, x2
+ ldp d8 , d9, [x0,#112]
+ ldp d10, d11, [x0,#128]
+ ldp d12, d13, [x0,#144]
+ ldp d14, d15, [x0,#160]
+
+ mov x0, x1
+ cbnz x1, 1f
+ mov x0, #1
+1: br x30
diff --git a/util/system/context_aarch64.h b/util/system/context_aarch64.h
new file mode 100644
index 0000000000..589521d8ae
--- /dev/null
+++ b/util/system/context_aarch64.h
@@ -0,0 +1,8 @@
+#pragma once
+
+typedef unsigned long __myjmp_buf[22];
+
+#define FRAME_CNT 10
+#define PROGR_CNT 11
+#define STACK_CNT 13
+#define EXTRA_PUSH_ARGS 2
diff --git a/util/system/context_i686.asm b/util/system/context_i686.asm
new file mode 100644
index 0000000000..11f8cecc8e
--- /dev/null
+++ b/util/system/context_i686.asm
@@ -0,0 +1,43 @@
+ [bits 32]
+
+ %define MJB_BX 0
+ %define MJB_SI 1
+ %define MJB_DI 2
+ %define MJB_BP 3
+ %define MJB_SP 4
+ %define MJB_PC 5
+ %define MJB_RSP MJB_SP
+ %define MJB_SIZE 24
+
+ %define LINKAGE 4
+ %define PCOFF 0
+ %define PTR_SIZE 4
+
+ %define PARMS LINKAGE
+ %define JMPBUF PARMS
+ %define JBUF PARMS
+ %define VAL JBUF + PTR_SIZE
+
+EXPORT __mylongjmp
+ mov ecx, [esp + JBUF]
+ mov eax, [esp + VAL]
+ mov edx, [ecx + MJB_PC*4]
+ mov ebx, [ecx + MJB_BX*4]
+ mov esi, [ecx + MJB_SI*4]
+ mov edi, [ecx + MJB_DI*4]
+ mov ebp, [ecx + MJB_BP*4]
+ mov esp, [ecx + MJB_SP*4]
+ jmp edx
+
+EXPORT __mysetjmp
+ mov eax, [esp + JMPBUF]
+ mov [eax + MJB_BX*4], ebx
+ mov [eax + MJB_SI*4], esi
+ mov [eax + MJB_DI*4], edi
+ lea ecx, [esp + JMPBUF]
+ mov [eax + MJB_SP*4], ecx
+ mov ecx, [esp + PCOFF]
+ mov [eax + MJB_PC*4], ecx
+ mov [eax + MJB_BP*4], ebp
+ xor eax, eax
+ ret
diff --git a/util/system/context_i686.h b/util/system/context_i686.h
new file mode 100644
index 0000000000..1abfd5dada
--- /dev/null
+++ b/util/system/context_i686.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#define MJB_BP 3
+#define MJB_SP 4
+#define MJB_PC 5
+#define MJB_RBP MJB_BP
+#define MJB_RSP MJB_SP
+
+typedef int __myjmp_buf[6];
diff --git a/util/system/context_ut.cpp b/util/system/context_ut.cpp
new file mode 100644
index 0000000000..45cb0249f7
--- /dev/null
+++ b/util/system/context_ut.cpp
@@ -0,0 +1,71 @@
+#include "context.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/deque.h>
+#include <util/generic/yexception.h>
+
+Y_UNIT_TEST_SUITE(TestContext) {
+ template <class F>
+ static TContClosure Wrap(F& f) {
+ struct TW: public ITrampoLine {
+ inline TW(F* ff) noexcept
+ : F_(ff)
+ {
+ }
+
+ void DoRun() override {
+ (*F_)();
+ }
+
+ F* F_;
+ char Buf[1000000];
+ };
+
+ static TDeque<TW> w;
+
+ auto& tw = w.emplace_back(&f);
+
+ return {&tw, TArrayRef(tw.Buf, sizeof(tw.Buf))};
+ }
+
+ Y_UNIT_TEST(TestExceptionSafety) {
+ TExceptionSafeContext main;
+ TExceptionSafeContext* volatile nextPtr = nullptr;
+
+ bool hasUncaught = true;
+
+ auto func = [&]() {
+ hasUncaught = UncaughtException();
+ nextPtr->SwitchTo(&main);
+ };
+
+ auto closure = Wrap(func);
+
+ TExceptionSafeContext next(closure);
+
+ nextPtr = &next;
+
+ struct THelper {
+ inline ~THelper() {
+ M->SwitchTo(N);
+ }
+
+ TExceptionSafeContext* M;
+ TExceptionSafeContext* N;
+ };
+
+ bool throwed = false;
+
+ try {
+ THelper helper{&main, &next};
+
+ throw 1;
+ } catch (...) {
+ throwed = true;
+ }
+
+ UNIT_ASSERT(throwed);
+ UNIT_ASSERT(!hasUncaught);
+ }
+}
diff --git a/util/system/context_x86.asm b/util/system/context_x86.asm
new file mode 100644
index 0000000000..e825d5d087
--- /dev/null
+++ b/util/system/context_x86.asm
@@ -0,0 +1,15 @@
+%macro EXPORT 1
+ %ifdef DARWIN
+ global _%1
+ _%1:
+ %else
+ global %1
+ %1:
+ %endif
+%endmacro
+
+%ifdef _x86_64_
+ %include "context_x86_64.asm"
+%else
+ %include "context_i686.asm"
+%endif
diff --git a/util/system/context_x86.h b/util/system/context_x86.h
new file mode 100644
index 0000000000..6ea066ff88
--- /dev/null
+++ b/util/system/context_x86.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#if defined(_x86_64_)
+ #include "context_x86_64.h"
+#elif defined(_i386_)
+ #include "context_i686.h"
+#endif
+
+#define PROGR_CNT MJB_PC
+#define STACK_CNT MJB_RSP
+#define FRAME_CNT MJB_RBP
+#define EXTRA_PUSH_ARGS 1
diff --git a/util/system/context_x86_64.asm b/util/system/context_x86_64.asm
new file mode 100644
index 0000000000..8bcc01e4fc
--- /dev/null
+++ b/util/system/context_x86_64.asm
@@ -0,0 +1,40 @@
+ [bits 64]
+
+ %define MJB_RBX 0
+ %define MJB_RBP 1
+ %define MJB_R12 2
+ %define MJB_R13 3
+ %define MJB_R14 4
+ %define MJB_R15 5
+ %define MJB_RSP 6
+ %define MJB_PC 7
+ %define MJB_SIZE (8*8)
+
+EXPORT __mylongjmp
+ mov rbx, [rdi + MJB_RBX * 8]
+ mov rbp, [rdi + MJB_RBP * 8]
+ mov r12, [rdi + MJB_R12 * 8]
+ mov r13, [rdi + MJB_R13 * 8]
+ mov r14, [rdi + MJB_R14 * 8]
+ mov r15, [rdi + MJB_R15 * 8]
+ test esi, esi
+ mov eax, 1
+ cmove esi, eax
+ mov eax, esi
+ mov rdx, [rdi + MJB_PC * 8]
+ mov rsp, [rdi + MJB_RSP * 8]
+ jmp rdx
+
+EXPORT __mysetjmp
+ mov [rdi + MJB_RBX * 8], rbx
+ mov [rdi + MJB_RBP * 8], rbp
+ mov [rdi + MJB_R12 * 8], r12
+ mov [rdi + MJB_R13 * 8], r13
+ mov [rdi + MJB_R14 * 8], r14
+ mov [rdi + MJB_R15 * 8], r15
+ lea rdx, [rsp + 8]
+ mov [rdi + MJB_RSP * 8], rdx
+ mov rax, [rsp]
+ mov [rdi + MJB_PC * 8], rax
+ mov eax, 0
+ ret
diff --git a/util/system/context_x86_64.h b/util/system/context_x86_64.h
new file mode 100644
index 0000000000..94e4f37eb3
--- /dev/null
+++ b/util/system/context_x86_64.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#define MJB_RBP 1
+#define MJB_RSP 6
+#define MJB_PC 7
+
+typedef long int __myjmp_buf[8];
diff --git a/util/system/cpu_id.cpp b/util/system/cpu_id.cpp
new file mode 100644
index 0000000000..598c71f4d9
--- /dev/null
+++ b/util/system/cpu_id.cpp
@@ -0,0 +1,263 @@
+#include "cpu_id.h"
+#include "types.h"
+#include "platform.h"
+
+#include <util/generic/singleton.h>
+
+#if defined(_win_)
+ #include <intrin.h>
+ #include <immintrin.h>
+#elif defined(_x86_)
+ #include <cpuid.h>
+#endif
+
+#include <string.h>
+
+#if defined(_x86_) && !defined(_win_)
+static ui64 _xgetbv(ui32 xcr) {
+ ui32 eax;
+ ui32 edx;
+ __asm__ volatile(
+ "xgetbv"
+ : "=a"(eax), "=d"(edx)
+ : "c"(xcr));
+ return (static_cast<ui64>(edx) << 32) | eax;
+}
+#endif
+
+bool NX86::CpuId(ui32 op, ui32 subOp, ui32* res) noexcept {
+#if defined(_x86_)
+ #if defined(_MSC_VER)
+ static_assert(sizeof(int) == sizeof(ui32), "ups, something wrong here");
+ __cpuidex((int*)res, op, subOp);
+ #else
+ __cpuid_count(op, subOp, res[0], res[1], res[2], res[3]);
+ #endif
+ return true;
+#else
+ (void)op;
+ (void)subOp;
+
+ memset(res, 0, 4 * sizeof(ui32));
+
+ return false;
+#endif
+}
+
+bool NX86::CpuId(ui32 op, ui32* res) noexcept {
+#if defined(_x86_)
+ #if defined(_MSC_VER)
+ static_assert(sizeof(int) == sizeof(ui32), "ups, something wrong here");
+ __cpuid((int*)res, op);
+ #else
+ __cpuid(op, res[0], res[1], res[2], res[3]);
+ #endif
+ return true;
+#else
+ (void)op;
+
+ memset(res, 0, 4 * sizeof(ui32));
+
+ return false;
+#endif
+}
+
+namespace {
+ union TX86CpuInfo {
+ ui32 Info[4];
+
+ struct {
+ ui32 EAX;
+ ui32 EBX;
+ ui32 ECX;
+ ui32 EDX;
+ };
+
+ inline TX86CpuInfo(ui32 op) noexcept {
+ NX86::CpuId(op, Info);
+ }
+
+ inline TX86CpuInfo(ui32 op, ui32 subOp) noexcept {
+ NX86::CpuId(op, subOp, Info);
+ }
+ };
+
+ static_assert(sizeof(TX86CpuInfo) == 16, "please, fix me");
+}
+
+// https://en.wikipedia.org/wiki/CPUID
+bool NX86::HaveRDTSCP() noexcept {
+ return (TX86CpuInfo(0x80000001).EDX >> 27) & 1u;
+}
+
+bool NX86::HaveSSE() noexcept {
+ return (TX86CpuInfo(0x1).EDX >> 25) & 1u;
+}
+
+bool NX86::HaveSSE2() noexcept {
+ return (TX86CpuInfo(0x1).EDX >> 26) & 1u;
+}
+
+bool NX86::HaveSSE3() noexcept {
+ return TX86CpuInfo(0x1).ECX & 1u;
+}
+
+bool NX86::HavePCLMUL() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 1) & 1u;
+}
+
+bool NX86::HaveSSSE3() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 9) & 1u;
+}
+
+bool NX86::HaveSSE41() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 19) & 1u;
+}
+
+bool NX86::HaveSSE42() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 20) & 1u;
+}
+
+bool NX86::HaveF16C() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 29) & 1u;
+}
+
+bool NX86::HavePOPCNT() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 23) & 1u;
+}
+
+bool NX86::HaveAES() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 25) & 1u;
+}
+
+bool NX86::HaveXSAVE() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 26) & 1u;
+}
+
+bool NX86::HaveOSXSAVE() noexcept {
+ return (TX86CpuInfo(0x1).ECX >> 27) & 1u;
+}
+
+bool NX86::HaveAVX() noexcept {
+#if defined(_x86_)
+ // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=375968
+ return HaveOSXSAVE() // implies HaveXSAVE()
+ && (_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS
+ && ((TX86CpuInfo(0x1).ECX >> 28) & 1u); // AVX bit
+#else
+ return false;
+#endif
+}
+
+bool NX86::HaveFMA() noexcept {
+ return HaveAVX() && ((TX86CpuInfo(0x1).ECX >> 12) & 1u);
+}
+
+bool NX86::HaveAVX2() noexcept {
+ return HaveAVX() && ((TX86CpuInfo(0x7, 0).EBX >> 5) & 1u);
+}
+
+bool NX86::HaveBMI1() noexcept {
+ return (TX86CpuInfo(0x7, 0).EBX >> 3) & 1u;
+}
+
+bool NX86::HaveBMI2() noexcept {
+ return (TX86CpuInfo(0x7, 0).EBX >> 8) & 1u;
+}
+
+bool NX86::HaveAVX512F() noexcept {
+#if defined(_x86_)
+ // https://software.intel.com/en-us/articles/how-to-detect-knl-instruction-support
+ return HaveOSXSAVE() // implies HaveXSAVE()
+ && (_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS
+ && ((_xgetbv(0) >> 5) & 7u) == 7u // ZMM state is enabled by OS
+ && TX86CpuInfo(0x0).EAX >= 0x7 // leaf 7 is present
+ && ((TX86CpuInfo(0x7).EBX >> 16) & 1u); // AVX512F bit
+#else
+ return false;
+#endif
+}
+
+bool NX86::HaveAVX512DQ() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 17) & 1u);
+}
+
+bool NX86::HaveRDSEED() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 18) & 1u);
+}
+
+bool NX86::HaveADX() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 19) & 1u);
+}
+
+bool NX86::HaveAVX512IFMA() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 21) & 1u);
+}
+
+bool NX86::HavePCOMMIT() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 22) & 1u);
+}
+
+bool NX86::HaveCLFLUSHOPT() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 23) & 1u);
+}
+
+bool NX86::HaveCLWB() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 24) & 1u);
+}
+
+bool NX86::HaveAVX512PF() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 26) & 1u);
+}
+
+bool NX86::HaveAVX512ER() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 27) & 1u);
+}
+
+bool NX86::HaveAVX512CD() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 28) & 1u);
+}
+
+bool NX86::HaveSHA() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).EBX >> 29) & 1u);
+}
+
+bool NX86::HaveAVX512BW() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 30) & 1u);
+}
+
+bool NX86::HaveAVX512VL() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).EBX >> 31) & 1u);
+}
+
+bool NX86::HavePREFETCHWT1() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x7, 0).ECX >> 0) & 1u);
+}
+
+bool NX86::HaveAVX512VBMI() noexcept {
+ return HaveAVX512F() && ((TX86CpuInfo(0x7, 0).ECX >> 1) & 1u);
+}
+
+bool NX86::HaveRDRAND() noexcept {
+ return TX86CpuInfo(0x0).EAX >= 0x7 && ((TX86CpuInfo(0x1).ECX >> 30) & 1u);
+}
+
+const char* CpuBrand(ui32* store) noexcept {
+ memset(store, 0, 12 * sizeof(*store));
+
+#if defined(_x86_)
+ NX86::CpuId(0x80000002, store);
+ NX86::CpuId(0x80000003, store + 4);
+ NX86::CpuId(0x80000004, store + 8);
+#endif
+
+ return (const char*)store;
+}
+
+#define Y_DEF_NAME(X) \
+ bool NX86::CachedHave##X() noexcept { \
+ return SingletonWithPriority<TFlagsCache, 0>()->Have##X##_; \
+ }
+Y_CPU_ID_ENUMERATE_OUTLINED_CACHED_DEFINE(Y_DEF_NAME)
+#undef Y_DEF_NAME
diff --git a/util/system/cpu_id.h b/util/system/cpu_id.h
new file mode 100644
index 0000000000..3c49e728a7
--- /dev/null
+++ b/util/system/cpu_id.h
@@ -0,0 +1,157 @@
+#pragma once
+
+#include "types.h"
+#include "compiler.h"
+#include <util/generic/singleton.h>
+
+#define Y_CPU_ID_ENUMERATE(F) \
+ F(SSE) \
+ F(SSE2) \
+ F(SSE3) \
+ F(SSSE3) \
+ F(SSE41) \
+ F(SSE42) \
+ F(F16C) \
+ F(POPCNT) \
+ F(BMI1) \
+ F(BMI2) \
+ F(PCLMUL) \
+ F(AES) \
+ F(AVX) \
+ F(FMA) \
+ F(AVX2) \
+ F(AVX512F) \
+ F(AVX512DQ) \
+ F(AVX512IFMA) \
+ F(AVX512PF) \
+ F(AVX512ER) \
+ F(AVX512CD) \
+ F(AVX512BW) \
+ F(AVX512VL) \
+ F(AVX512VBMI) \
+ F(PREFETCHWT1) \
+ F(SHA) \
+ F(ADX) \
+ F(RDRAND) \
+ F(RDSEED) \
+ F(PCOMMIT) \
+ F(RDTSCP) \
+ F(CLFLUSHOPT) \
+ F(CLWB) \
+ F(XSAVE) \
+ F(OSXSAVE)
+
+#define Y_CPU_ID_ENUMERATE_OUTLINED_CACHED_DEFINE(F) \
+ F(F16C) \
+ F(BMI1) \
+ F(BMI2) \
+ F(PCLMUL) \
+ F(AES) \
+ F(AVX) \
+ F(FMA) \
+ F(AVX2) \
+ F(AVX512F) \
+ F(AVX512DQ) \
+ F(AVX512IFMA) \
+ F(AVX512PF) \
+ F(AVX512ER) \
+ F(AVX512CD) \
+ F(AVX512BW) \
+ F(AVX512VL) \
+ F(AVX512VBMI) \
+ F(PREFETCHWT1) \
+ F(SHA) \
+ F(ADX) \
+ F(RDRAND) \
+ F(RDSEED) \
+ F(PCOMMIT) \
+ F(RDTSCP) \
+ F(CLFLUSHOPT) \
+ F(CLWB) \
+ F(XSAVE) \
+ F(OSXSAVE)
+
+namespace NX86 {
+ /**
+ * returns false on non-x86 platforms
+ */
+ bool CpuId(ui32 op, ui32 res[4]) noexcept;
+ bool CpuId(ui32 op, ui32 subOp, ui32 res[4]) noexcept;
+
+#define Y_DEF_NAME(X) Y_CONST_FUNCTION bool Have##X() noexcept;
+ Y_CPU_ID_ENUMERATE(Y_DEF_NAME)
+#undef Y_DEF_NAME
+
+#define Y_DEF_NAME(X) Y_CONST_FUNCTION bool CachedHave##X() noexcept;
+ Y_CPU_ID_ENUMERATE_OUTLINED_CACHED_DEFINE(Y_DEF_NAME)
+#undef Y_DEF_NAME
+
+ struct TFlagsCache {
+#define Y_DEF_NAME(X) const bool Have##X##_ = NX86::Have##X();
+ Y_CPU_ID_ENUMERATE(Y_DEF_NAME)
+#undef Y_DEF_NAME
+ };
+
+#define Y_LOOKUP_CPU_ID_IMPL(X) return SingletonWithPriority<TFlagsCache, 0>()->Have##X##_;
+
+ inline bool CachedHaveSSE() noexcept {
+#ifdef _sse_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSE)
+#endif
+ }
+
+ inline bool CachedHaveSSE2() noexcept {
+#ifdef _sse2_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSE2)
+#endif
+ }
+
+ inline bool CachedHaveSSE3() noexcept {
+#ifdef _sse3_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSE3)
+#endif
+ }
+
+ inline bool CachedHaveSSSE3() noexcept {
+#ifdef _ssse3_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSSE3)
+#endif
+ }
+
+ inline bool CachedHaveSSE41() noexcept {
+#ifdef _sse4_1_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSE41)
+#endif
+ }
+
+ inline bool CachedHaveSSE42() noexcept {
+#ifdef _sse4_2_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(SSE42)
+#endif
+ }
+
+ inline bool CachedHavePOPCNT() noexcept {
+#ifdef _popcnt_
+ return true;
+#else
+ Y_LOOKUP_CPU_ID_IMPL(POPCNT)
+#endif
+ }
+
+#undef Y_LOOKUP_CPU_ID_IMPL
+
+}
+
+const char* CpuBrand(ui32 store[12]) noexcept;
diff --git a/util/system/cpu_id_ut.cpp b/util/system/cpu_id_ut.cpp
new file mode 100644
index 0000000000..68f1f8aac7
--- /dev/null
+++ b/util/system/cpu_id_ut.cpp
@@ -0,0 +1,450 @@
+#include "cpu_id.h"
+
+#include "platform.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+// There are no tests yet for instructions that use 512-bit wide registers because they are not
+// supported by some compilers yet.
+// Relevant review in LLVM https://reviews.llvm.org/D16757, we should wait untill it will be in our
+// version of Clang.
+//
+// There are also no tests for PREFETCHWT1, PCOMMIT, CLFLUSHOPT and CLWB as they are not supported
+// by our compilers yet (and there are no available processors yet :).
+
+static void ExecuteSSEInstruction();
+static void ExecuteSSE2Instruction();
+static void ExecuteSSE3Instruction();
+static void ExecuteSSSE3Instruction();
+static void ExecuteSSE41Instruction();
+static void ExecuteSSE42Instruction();
+static void ExecuteF16CInstruction();
+static void ExecuteAVXInstruction();
+static void ExecuteAVX2Instruction();
+static void ExecutePOPCNTInstruction();
+static void ExecuteBMI1Instruction();
+static void ExecuteBMI2Instruction();
+static void ExecutePCLMULInstruction();
+static void ExecuteAESInstruction();
+static void ExecuteAVXInstruction();
+static void ExecuteAVX2Instruction();
+static void ExecuteAVX512FInstruction();
+static void ExecuteAVX512DQInstruction();
+static void ExecuteAVX512IFMAInstruction();
+static void ExecuteAVX512PFInstruction();
+static void ExecuteAVX512ERInstruction();
+static void ExecuteAVX512CDInstruction();
+static void ExecuteAVX512BWInstruction();
+static void ExecuteAVX512VLInstruction();
+static void ExecuteAVX512VBMIInstruction();
+static void ExecutePREFETCHWT1Instruction();
+static void ExecuteSHAInstruction();
+static void ExecuteADXInstruction();
+static void ExecuteRDRANDInstruction();
+static void ExecuteRDSEEDInstruction();
+static void ExecutePCOMMITInstruction();
+static void ExecuteCLFLUSHOPTInstruction();
+static void ExecuteCLWBInstruction();
+
+static void ExecuteFMAInstruction() {
+}
+
+static void ExecuteRDTSCPInstruction() {
+}
+
+static void ExecuteXSAVEInstruction() {
+}
+
+static void ExecuteOSXSAVEInstruction() {
+}
+
+Y_UNIT_TEST_SUITE(TestCpuId) {
+#define DECLARE_TEST_HAVE_INSTRUCTION(name) \
+ Y_UNIT_TEST(Test##Have##name) { \
+ if (NX86::Have##name()) { \
+ Execute##name##Instruction(); \
+ } \
+ }
+
+ Y_CPU_ID_ENUMERATE(DECLARE_TEST_HAVE_INSTRUCTION)
+#undef DECLARE_TEST_HAVE_INSTRUCTION
+
+ Y_UNIT_TEST(TestSSE2) {
+#if defined(_x86_64_)
+ UNIT_ASSERT(NX86::HaveSSE2());
+#endif
+ }
+
+ Y_UNIT_TEST(TestCpuBrand) {
+ ui32 store[12];
+
+ //Cout << CpuBrand(store) << Endl;;
+
+ UNIT_ASSERT(strlen(CpuBrand(store)) > 0);
+ }
+
+ Y_UNIT_TEST(TestCachedAndNoncached) {
+#define Y_DEF_NAME(X) UNIT_ASSERT_VALUES_EQUAL(NX86::Have##X(), NX86::CachedHave##X());
+ Y_CPU_ID_ENUMERATE(Y_DEF_NAME)
+#undef Y_DEF_NAME
+ }
+}
+
+#if defined(_x86_64_)
+ #if defined(__GNUC__)
+void ExecuteSSEInstruction() {
+ __asm__ __volatile__("xorps %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteSSE2Instruction() {
+ __asm__ __volatile__("psrldq $0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteSSE3Instruction() {
+ __asm__ __volatile__("addsubpd %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteSSSE3Instruction() {
+ __asm__ __volatile__("psignb %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteSSE41Instruction() {
+ __asm__ __volatile__("pmuldq %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteSSE42Instruction() {
+ __asm__ __volatile__("crc32 %%eax, %%eax\n"
+ :
+ :
+ : "eax");
+}
+
+void ExecuteF16CInstruction() {
+ __asm__ __volatile__("vcvtph2ps %%xmm0, %%ymm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteAVXInstruction() {
+ __asm__ __volatile__("vzeroupper\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteAVX2Instruction() {
+ __asm__ __volatile__("vpunpcklbw %%ymm0, %%ymm0, %%ymm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecutePOPCNTInstruction() {
+ __asm__ __volatile__("popcnt %%eax, %%eax\n"
+ :
+ :
+ : "eax");
+}
+
+void ExecuteBMI1Instruction() {
+ __asm__ __volatile__("tzcnt %%eax, %%eax\n"
+ :
+ :
+ : "eax");
+}
+
+void ExecuteBMI2Instruction() {
+ __asm__ __volatile__("pdep %%rax, %%rdi, %%rax\n"
+ :
+ :
+ : "rax");
+}
+
+void ExecutePCLMULInstruction() {
+ __asm__ __volatile__("pclmullqlqdq %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteAESInstruction() {
+ __asm__ __volatile__("aesimc %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteAVX512FInstruction() {
+}
+
+void ExecuteAVX512DQInstruction() {
+}
+
+void ExecuteAVX512IFMAInstruction() {
+}
+
+void ExecuteAVX512PFInstruction() {
+}
+
+void ExecuteAVX512ERInstruction() {
+}
+
+void ExecuteAVX512CDInstruction() {
+}
+
+void ExecuteAVX512BWInstruction() {
+}
+
+void ExecuteAVX512VLInstruction() {
+}
+
+void ExecuteAVX512VBMIInstruction() {
+}
+
+void ExecutePREFETCHWT1Instruction() {
+}
+
+void ExecuteSHAInstruction() {
+ __asm__ __volatile__("sha1msg1 %%xmm0, %%xmm0\n"
+ :
+ :
+ : "xmm0");
+}
+
+void ExecuteADXInstruction() {
+ __asm__ __volatile__("adcx %%eax, %%eax\n"
+ :
+ :
+ : "eax");
+}
+
+void ExecuteRDRANDInstruction() {
+ __asm__ __volatile__("rdrand %%eax"
+ :
+ :
+ : "eax");
+}
+
+void ExecuteRDSEEDInstruction() {
+ __asm__ __volatile__("rdseed %%eax"
+ :
+ :
+ : "eax");
+}
+
+void ExecutePCOMMITInstruction() {
+}
+
+void ExecuteCLFLUSHOPTInstruction() {
+}
+
+void ExecuteCLWBInstruction() {
+}
+
+ #elif defined(_MSC_VER)
+void ExecuteSSEInstruction() {
+}
+
+void ExecuteSSE2Instruction() {
+}
+
+void ExecuteSSE3Instruction() {
+}
+
+void ExecuteSSSE3Instruction() {
+}
+
+void ExecuteSSE41Instruction() {
+}
+
+void ExecuteSSE42Instruction() {
+}
+
+void ExecuteF16CInstruction() {
+}
+
+void ExecuteAVXInstruction() {
+}
+
+void ExecuteAVX2Instruction() {
+}
+
+void ExecutePOPCNTInstruction() {
+}
+
+void ExecuteBMI1Instruction() {
+}
+
+void ExecuteBMI2Instruction() {
+}
+
+void ExecutePCLMULInstruction() {
+}
+
+void ExecuteAESInstruction() {
+}
+
+void ExecuteAVX512FInstruction() {
+}
+
+void ExecuteAVX512DQInstruction() {
+}
+
+void ExecuteAVX512IFMAInstruction() {
+}
+
+void ExecuteAVX512PFInstruction() {
+}
+
+void ExecuteAVX512ERInstruction() {
+}
+
+void ExecuteAVX512CDInstruction() {
+}
+
+void ExecuteAVX512BWInstruction() {
+}
+
+void ExecuteAVX512VLInstruction() {
+}
+
+void ExecuteAVX512VBMIInstruction() {
+}
+
+void ExecutePREFETCHWT1Instruction() {
+}
+
+void ExecuteSHAInstruction() {
+}
+
+void ExecuteADXInstruction() {
+}
+
+void ExecuteRDRANDInstruction() {
+}
+
+void ExecuteRDSEEDInstruction() {
+}
+
+void ExecutePCOMMITInstruction() {
+}
+
+void ExecuteCLFLUSHOPTInstruction() {
+}
+
+void ExecuteCLWBInstruction() {
+}
+
+ #else
+ #error "unknown compiler"
+ #endif
+#else
+void ExecuteSSEInstruction() {
+}
+
+void ExecuteSSE2Instruction() {
+}
+
+void ExecuteSSE3Instruction() {
+}
+
+void ExecuteSSSE3Instruction() {
+}
+
+void ExecuteSSE41Instruction() {
+}
+
+void ExecuteSSE42Instruction() {
+}
+
+void ExecuteF16CInstruction() {
+}
+
+void ExecuteAVXInstruction() {
+}
+
+void ExecuteAVX2Instruction() {
+}
+
+void ExecutePOPCNTInstruction() {
+}
+
+void ExecuteBMI1Instruction() {
+}
+
+void ExecuteBMI2Instruction() {
+}
+
+void ExecutePCLMULInstruction() {
+}
+
+void ExecuteAESInstruction() {
+}
+
+void ExecuteAVX512FInstruction() {
+}
+
+void ExecuteAVX512DQInstruction() {
+}
+
+void ExecuteAVX512IFMAInstruction() {
+}
+
+void ExecuteAVX512PFInstruction() {
+}
+
+void ExecuteAVX512ERInstruction() {
+}
+
+void ExecuteAVX512CDInstruction() {
+}
+
+void ExecuteAVX512BWInstruction() {
+}
+
+void ExecuteAVX512VLInstruction() {
+}
+
+void ExecuteAVX512VBMIInstruction() {
+}
+
+void ExecutePREFETCHWT1Instruction() {
+}
+
+void ExecuteSHAInstruction() {
+}
+
+void ExecuteADXInstruction() {
+}
+
+void ExecuteRDRANDInstruction() {
+}
+
+void ExecuteRDSEEDInstruction() {
+}
+
+void ExecutePCOMMITInstruction() {
+}
+
+void ExecuteCLFLUSHOPTInstruction() {
+}
+
+void ExecuteCLWBInstruction() {
+}
+#endif
diff --git a/util/system/daemon.cpp b/util/system/daemon.cpp
new file mode 100644
index 0000000000..130e6c8f45
--- /dev/null
+++ b/util/system/daemon.cpp
@@ -0,0 +1,168 @@
+#include <util/generic/yexception.h>
+
+#include <cerrno>
+#include <cstdlib>
+#include <util/system/info.h>
+
+#if defined(_win_)
+ #include <io.h>
+#else
+ #include <sys/wait.h>
+ #include <unistd.h>
+ #include <fcntl.h>
+#endif
+
+#include "daemon.h"
+
+#ifdef _unix_
+using namespace NDaemonMaker;
+
+static bool Fork(EParent parent) {
+ pid_t pid = fork();
+
+ if (pid > 0) {
+ int status = 0;
+ while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {
+ }
+ if (parent == callExitFromParent) {
+ _exit(0);
+ } else {
+ return true;
+ }
+ } else if (pid < 0) {
+ ythrow TSystemError() << "Cannot fork";
+ }
+
+ if (setsid() < 0) {
+ ythrow TSystemError() << "Cannot setsid";
+ }
+
+ pid = fork();
+
+ if (pid > 0) {
+ _exit(0);
+ } else if (pid < 0) {
+ ythrow TSystemError() << "Cannot second fork";
+ }
+ return false;
+}
+
+#endif
+
+static void CloseFromToExcept(int from, int to, const int* except) {
+ (void)from;
+ (void)to;
+ (void)except;
+
+#ifdef _unix_
+ int mfd = NSystemInfo::MaxOpenFiles();
+ for (int s = from; s < mfd && (to == -1 || s < to); s++) {
+ for (const int* ex = except; *ex >= 0; ++ex) {
+ if (s == *ex) {
+ goto dontclose;
+ }
+ }
+ while (close(s) == -1) {
+ if (errno == EBADF) {
+ break;
+ }
+ if (errno != EINTR) {
+ ythrow TSystemError() << "close(" << s << ") failed";
+ }
+ }
+ dontclose:;
+ }
+#endif /* _unix_ */
+}
+
+bool NDaemonMaker::MakeMeDaemon(ECloseDescriptors cd, EStdIoDescriptors iod, EChDir chd, EParent parent) {
+ (void)cd;
+ (void)iod;
+ (void)chd;
+
+#ifdef _unix_
+ if (Fork(parent)) {
+ return true;
+ }
+
+ if (chd == chdirRoot) {
+ if (chdir("/")) {
+ ythrow TSystemError() << "chdir(\"/\") failed";
+ }
+ }
+
+ int fd[4] = {-1, -1, -1, -1};
+ switch (iod) {
+ case openYandexStd:
+ fd[0] = open("yandex.stdin", O_RDONLY);
+ if (fd[0] < 0) {
+ ythrow TSystemError() << "Cannot open 'yandex.stdin'";
+ }
+ fd[1] = open("yandex.stdout", O_WRONLY | O_APPEND | O_CREAT, 660);
+ if (fd[1] < 0) {
+ ythrow TSystemError() << "Cannot open 'yandex.stdout'";
+ }
+ fd[2] = open("yandex.stderr", O_WRONLY | O_APPEND | O_CREAT, 660);
+ if (fd[2] < 0) {
+ ythrow TSystemError() << "Cannot open 'yandex.stderr'";
+ }
+ break;
+ case openDevNull:
+ fd[0] = open("/dev/null", O_RDWR, 0);
+ break;
+ case openNone:
+ break;
+ default:
+ ythrow yexception() << "Unknown open descriptors mode: " << (int)iod;
+ }
+
+ const int except[4] = {
+ fd[0],
+ fd[1],
+ fd[2],
+ -1};
+ if (closeAll == cd) {
+ CloseFromToExcept(0, -1, except);
+ } else if (closeStdIoOnly == cd) {
+ CloseFromToExcept(0, 3, except);
+ } else {
+ ythrow yexception() << "Unknown close descriptors mode: " << (int)cd;
+ }
+
+ switch (iod) {
+ case openYandexStd:
+ /* Assuming that open(2) acquires fds in order. */
+ dup2(fd[0], STDIN_FILENO);
+ if (fd[0] > 2) {
+ close(fd[0]);
+ }
+ dup2(fd[1], STDOUT_FILENO);
+ if (fd[1] > 2) {
+ close(fd[1]);
+ }
+ dup2(fd[2], STDERR_FILENO);
+ if (fd[2] > 2) {
+ close(fd[2]);
+ }
+ break;
+ case openDevNull:
+ dup2(fd[0], STDIN_FILENO);
+ dup2(fd[0], STDOUT_FILENO);
+ dup2(fd[0], STDERR_FILENO);
+ if (fd[0] > 2) {
+ close(fd[0]);
+ }
+ break;
+ default:
+ break;
+ }
+ return false;
+#else
+ return true;
+#endif
+}
+
+void NDaemonMaker::CloseFrom(int fd) {
+ static const int except[1] = {-1};
+ CloseFromToExcept(fd, -1, except);
+}
diff --git a/util/system/daemon.h b/util/system/daemon.h
new file mode 100644
index 0000000000..b00793b9c9
--- /dev/null
+++ b/util/system/daemon.h
@@ -0,0 +1,27 @@
+#pragma once
+
+namespace NDaemonMaker {
+ enum ECloseDescriptors {
+ closeAll = 0,
+ closeStdIoOnly
+ };
+
+ enum EStdIoDescriptors {
+ openNone = 0,
+ openDevNull,
+ openYandexStd
+ };
+
+ enum EChDir {
+ chdirNone = 0,
+ chdirRoot
+ };
+
+ enum EParent {
+ callExitFromParent = 0,
+ returnFromParent
+ };
+
+ bool MakeMeDaemon(ECloseDescriptors cd = closeAll, EStdIoDescriptors iod = openDevNull, EChDir chd = chdirRoot, EParent parent = callExitFromParent);
+ void CloseFrom(int fd);
+}
diff --git a/util/system/daemon_ut.cpp b/util/system/daemon_ut.cpp
new file mode 100644
index 0000000000..f93b9a9645
--- /dev/null
+++ b/util/system/daemon_ut.cpp
@@ -0,0 +1,94 @@
+#include "daemon.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/network/pair.h>
+#include <util/network/socket.h>
+#include <util/system/pipe.h>
+
+Y_UNIT_TEST_SUITE(TDaemonTest) {
+#ifdef _unix_
+ template <typename Func>
+ static bool ProcessBuffer(Func&& func, void* bufin, size_t size) {
+ char* buf = (char*)bufin;
+ do {
+ const ssize_t bytesDone = func(buf, size);
+ if (bytesDone == 0) {
+ return false;
+ }
+
+ if (bytesDone < 0) {
+ if (errno == EAGAIN || errno == EINTR) {
+ continue;
+ } else {
+ return false;
+ }
+ }
+
+ buf += bytesDone;
+ size -= bytesDone;
+ } while (size != 0);
+
+ return true;
+ }
+
+ const int size = 1024 * 4;
+ const int pagesSize = sizeof(int) * size;
+
+ Y_UNIT_TEST(WaitForMessageSocket) {
+ using namespace NDaemonMaker;
+ SOCKET sockets[2];
+ SocketPair(sockets, false, true);
+ TSocket sender(sockets[0]);
+ TSocket receiver(sockets[1]);
+
+ int status = -1;
+ int* pages = new int[size];
+
+ memset(pages, 0, pagesSize);
+ if (MakeMeDaemon(closeStdIoOnly, openDevNull, chdirNone, returnFromParent)) {
+ sender.Close();
+ UNIT_ASSERT(ProcessBuffer([&receiver](char* ptr, size_t sz) -> size_t { return receiver.Recv(ptr, sz); }, &status, sizeof(status)));
+ UNIT_ASSERT(ProcessBuffer([&receiver](char* ptr, size_t sz) -> size_t { return receiver.Recv(ptr, sz); }, pages, pagesSize));
+ UNIT_ASSERT(memchr(pages, 0, pagesSize) == nullptr);
+ } else {
+ receiver.Close();
+ status = 0;
+ UNIT_ASSERT(ProcessBuffer([&sender](char* ptr, size_t sz) -> size_t { return sender.Send(ptr, sz); }, &status, sizeof(status)));
+ memset(pages, 1, pagesSize);
+ UNIT_ASSERT(ProcessBuffer([&sender](char* ptr, size_t sz) -> size_t { return sender.Send(ptr, sz); }, pages, pagesSize));
+ exit(0);
+ }
+ UNIT_ASSERT(status == 0);
+
+ delete[] pages;
+ }
+
+ Y_UNIT_TEST(WaitForMessagePipe) {
+ using namespace NDaemonMaker;
+ TPipeHandle sender;
+ TPipeHandle receiver;
+ TPipeHandle::Pipe(receiver, sender);
+
+ int status = -1;
+ int* pages = new int[size];
+ memset(pages, 0, pagesSize);
+ if (MakeMeDaemon(closeStdIoOnly, openDevNull, chdirNone, returnFromParent)) {
+ sender.Close();
+ UNIT_ASSERT(ProcessBuffer([&receiver](char* ptr, size_t sz) -> size_t { return receiver.Read(ptr, sz); }, &status, sizeof(status)));
+ UNIT_ASSERT(ProcessBuffer([&receiver](char* ptr, size_t sz) -> size_t { return receiver.Read(ptr, sz); }, pages, pagesSize));
+ UNIT_ASSERT(memchr(pages, 0, pagesSize) == nullptr);
+ } else {
+ receiver.Close();
+ status = 0;
+ UNIT_ASSERT(ProcessBuffer([&sender](char* ptr, size_t sz) -> size_t { return sender.Write(ptr, sz); }, &status, sizeof(status)));
+ memset(pages, 1, pagesSize);
+ UNIT_ASSERT(ProcessBuffer([&sender](char* ptr, size_t sz) -> size_t { return sender.Write(ptr, sz); }, pages, pagesSize));
+ exit(0);
+ }
+ UNIT_ASSERT(status == 0);
+
+ delete[] pages;
+ }
+#endif
+}
diff --git a/util/system/datetime.cpp b/util/system/datetime.cpp
new file mode 100644
index 0000000000..b07b50679a
--- /dev/null
+++ b/util/system/datetime.cpp
@@ -0,0 +1,103 @@
+#include "datetime.h"
+#include "yassert.h"
+#include "platform.h"
+#include "cpu_id.h"
+
+#include <util/datetime/systime.h>
+
+#include <ctime>
+#include <cerrno>
+
+#ifdef _darwin_
+ #include <AvailabilityMacros.h>
+ #if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12
+ #define Y_HAS_CLOCK_GETTIME
+ #endif
+#elif defined(_linux_) || defined(_freebsd_) || defined(_cygwin_)
+ #define Y_HAS_CLOCK_GETTIME
+#endif
+
+static ui64 ToMicroSeconds(const struct timeval& tv) {
+ return (ui64)tv.tv_sec * 1000000 + (ui64)tv.tv_usec;
+}
+
+#if defined(_win_)
+static ui64 ToMicroSeconds(const FILETIME& ft) {
+ return (((ui64)ft.dwHighDateTime << 32) + (ui64)ft.dwLowDateTime) / (ui64)10;
+}
+#elif defined(Y_HAS_CLOCK_GETTIME)
+static ui64 ToMicroSeconds(const struct timespec& ts) {
+ return (ui64)ts.tv_sec * 1000000 + (ui64)ts.tv_nsec / 1000;
+}
+#endif
+
+ui64 MicroSeconds() noexcept {
+ struct timeval tv;
+ gettimeofday(&tv, nullptr);
+
+ return ToMicroSeconds(tv);
+}
+
+ui64 ThreadCPUUserTime() noexcept {
+#if defined(_win_)
+ FILETIME creationTime, exitTime, kernelTime, userTime;
+ GetThreadTimes(GetCurrentThread(), &creationTime, &exitTime, &kernelTime, &userTime);
+ return ToMicroSeconds(userTime);
+#else
+ return 0;
+#endif
+}
+
+ui64 ThreadCPUSystemTime() noexcept {
+#if defined(_win_)
+ FILETIME creationTime, exitTime, kernelTime, userTime;
+ GetThreadTimes(GetCurrentThread(), &creationTime, &exitTime, &kernelTime, &userTime);
+ return ToMicroSeconds(kernelTime);
+#else
+ return 0;
+#endif
+}
+
+ui64 ThreadCPUTime() noexcept {
+#if defined(_win_)
+ FILETIME creationTime, exitTime, kernelTime, userTime;
+ GetThreadTimes(GetCurrentThread(), &creationTime, &exitTime, &kernelTime, &userTime);
+ return ToMicroSeconds(userTime) + ToMicroSeconds(kernelTime);
+#elif defined(Y_HAS_CLOCK_GETTIME)
+ struct timespec ts;
+ clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
+ return ToMicroSeconds(ts);
+#else
+ return 0;
+#endif
+}
+
+ui32 Seconds() noexcept {
+ struct timeval tv;
+ gettimeofday(&tv, nullptr);
+ return tv.tv_sec;
+}
+
+void NanoSleep(ui64 ns) noexcept {
+#if defined(_win_)
+ Sleep(ns / 1000000);
+#else
+ const ui64 NS = 1000 * 1000 * 1000;
+ struct timespec req;
+ req.tv_sec = ns / NS;
+ req.tv_nsec = ns % NS;
+ struct timespec left;
+ while (nanosleep(&req, &left) < 0) {
+ Y_ASSERT(errno == EINTR);
+ req = left;
+ }
+#endif
+}
+
+#if defined(_x86_)
+extern const bool HaveRdtscp = NX86::HaveRDTSCP();
+#endif
+
+#ifdef Y_HAS_CLOCK_GETTIME
+ #undef Y_HAS_CLOCK_GETTIME
+#endif
diff --git a/util/system/datetime.h b/util/system/datetime.h
new file mode 100644
index 0000000000..aa009974e0
--- /dev/null
+++ b/util/system/datetime.h
@@ -0,0 +1,98 @@
+#pragma once
+
+#include "defaults.h"
+#include "platform.h"
+
+#if defined(_win_)
+ #include <intrin.h>
+ #pragma intrinsic(__rdtsc)
+#endif // _win_
+
+#if defined(_darwin_) && !defined(_x86_)
+ #include <mach/mach_time.h>
+#endif
+
+/// util/system/datetime.h contains only system time providers
+/// for handy datetime utilities include util/datetime/base.h
+
+/// Current time in microseconds since epoch
+ui64 MicroSeconds() noexcept;
+/// Current time in milliseconds since epoch
+inline ui64 MilliSeconds() {
+ return MicroSeconds() / ui64(1000);
+}
+/// Current time in milliseconds since epoch (deprecated, use MilliSeconds instead)
+inline ui64 millisec() {
+ return MilliSeconds();
+}
+/// Current time in seconds since epoch
+ui32 Seconds() noexcept;
+///Current thread time in microseconds
+ui64 ThreadCPUUserTime() noexcept;
+ui64 ThreadCPUSystemTime() noexcept;
+ui64 ThreadCPUTime() noexcept;
+
+void NanoSleep(ui64 ns) noexcept;
+
+// GetCycleCount guarantees to return synchronous values on different cores
+// and provide constant rate only on modern Intel and AMD processors
+// NOTE: rdtscp is used to prevent out of order execution
+// rdtsc can be reordered, while rdtscp cannot be reordered
+// with preceding instructions
+// PERFORMANCE: rdtsc - 15 cycles per call , rdtscp - 19 cycles per call
+// WARNING: following instruction can be executed out-of-order
+Y_FORCE_INLINE ui64 GetCycleCount() noexcept {
+#if defined(_MSC_VER)
+ // Generates the rdtscp instruction, which returns the processor time stamp.
+ // The processor time stamp records the number of clock cycles since the last reset.
+ extern const bool HaveRdtscp;
+
+ if (HaveRdtscp) {
+ unsigned int aux;
+ return __rdtscp(&aux);
+ } else {
+ return __rdtsc();
+ }
+#elif defined(_x86_64_)
+ extern const bool HaveRdtscp;
+
+ unsigned hi, lo;
+
+ if (HaveRdtscp) {
+ __asm__ __volatile__("rdtscp"
+ : "=a"(lo), "=d"(hi)::"%rcx");
+ } else {
+ __asm__ __volatile__("rdtsc"
+ : "=a"(lo), "=d"(hi));
+ }
+
+ return ((unsigned long long)lo) | (((unsigned long long)hi) << 32);
+#elif defined(_i386_)
+ extern const bool HaveRdtscp;
+
+ ui64 x;
+ if (HaveRdtscp) {
+ __asm__ volatile("rdtscp\n\t"
+ : "=A"(x)::"%ecx");
+ } else {
+ __asm__ volatile("rdtsc\n\t"
+ : "=A"(x));
+ }
+ return x;
+#elif defined(_darwin_)
+ return mach_absolute_time();
+#elif defined(__clang__) && !defined(_arm_)
+ return __builtin_readcyclecounter();
+#elif defined(_arm32_)
+ return MicroSeconds();
+#elif defined(_arm64_)
+ ui64 x;
+
+ __asm__ __volatile__("isb; mrs %0, cntvct_el0"
+ : "=r"(x));
+
+ return x;
+#else
+ #error "unsupported arch"
+#endif
+}
diff --git a/util/system/datetime_ut.cpp b/util/system/datetime_ut.cpp
new file mode 100644
index 0000000000..a865a888ca
--- /dev/null
+++ b/util/system/datetime_ut.cpp
@@ -0,0 +1 @@
+#include "datetime.h"
diff --git a/util/system/defaults.c b/util/system/defaults.c
new file mode 100644
index 0000000000..75710009ac
--- /dev/null
+++ b/util/system/defaults.c
@@ -0,0 +1,2 @@
+//test for C-compiler compileability
+#include "defaults.h"
diff --git a/util/system/defaults.h b/util/system/defaults.h
new file mode 100644
index 0000000000..dcd7abea38
--- /dev/null
+++ b/util/system/defaults.h
@@ -0,0 +1,155 @@
+#pragma once
+
+#include "platform.h"
+
+#if defined _unix_
+ #define LOCSLASH_C '/'
+ #define LOCSLASH_S "/"
+#else
+ #define LOCSLASH_C '\\'
+ #define LOCSLASH_S "\\"
+#endif // _unix_
+
+#if defined(__INTEL_COMPILER) && defined(__cplusplus)
+ #include <new>
+#endif
+
+// low and high parts of integers
+#if !defined(_win_)
+ #include <sys/param.h>
+#endif
+
+#if defined(BSD) || defined(_android_)
+
+ #if defined(BSD)
+ #include <machine/endian.h>
+ #endif
+
+ #if defined(_android_)
+ #include <endian.h>
+ #endif
+
+ #if (BYTE_ORDER == LITTLE_ENDIAN)
+ #define _little_endian_
+ #elif (BYTE_ORDER == BIG_ENDIAN)
+ #define _big_endian_
+ #else
+ #error unknown endian not supported
+ #endif
+
+#elif (defined(_sun_) && !defined(__i386__)) || defined(_hpux_) || defined(WHATEVER_THAT_HAS_BIG_ENDIAN)
+ #define _big_endian_
+#else
+ #define _little_endian_
+#endif
+
+// alignment
+#if (defined(_sun_) && !defined(__i386__)) || defined(_hpux_) || defined(__alpha__) || defined(__ia64__) || defined(WHATEVER_THAT_NEEDS_ALIGNING_QUADS)
+ #define _must_align8_
+#endif
+
+#if (defined(_sun_) && !defined(__i386__)) || defined(_hpux_) || defined(__alpha__) || defined(__ia64__) || defined(WHATEVER_THAT_NEEDS_ALIGNING_LONGS)
+ #define _must_align4_
+#endif
+
+#if (defined(_sun_) && !defined(__i386__)) || defined(_hpux_) || defined(__alpha__) || defined(__ia64__) || defined(WHATEVER_THAT_NEEDS_ALIGNING_SHORTS)
+ #define _must_align2_
+#endif
+
+#if defined(__GNUC__)
+ #define alias_hack __attribute__((__may_alias__))
+#endif
+
+#ifndef alias_hack
+ #define alias_hack
+#endif
+
+#include "types.h"
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+ #define PRAGMA(x) _Pragma(#x)
+ #define RCSID(idstr) PRAGMA(comment(exestr, idstr))
+#else
+ #define RCSID(idstr) static const char rcsid[] = idstr
+#endif
+
+#include "compiler.h"
+
+#ifdef _win_
+ #include <malloc.h>
+#elif defined(_sun_)
+ #include <alloca.h>
+#endif
+
+#ifdef NDEBUG
+ #define Y_IF_DEBUG(X)
+ #ifdef __cplusplus
+constexpr bool Y_IS_DEBUG_BUILD = false;
+ #endif
+#else
+ #define Y_IF_DEBUG(X) X
+ #ifdef __cplusplus
+constexpr bool Y_IS_DEBUG_BUILD = true;
+ #endif
+#endif
+
+/**
+ * @def Y_ARRAY_SIZE
+ *
+ * This macro is needed to get number of elements in a statically allocated fixed size array. The
+ * expression is a compile-time constant and therefore can be used in compile time computations.
+ *
+ * @code
+ * enum ENumbers {
+ * EN_ONE,
+ * EN_TWO,
+ * EN_SIZE
+ * }
+ *
+ * const char* NAMES[] = {
+ * "one",
+ * "two"
+ * }
+ *
+ * static_assert(Y_ARRAY_SIZE(NAMES) == EN_SIZE, "you should define `NAME` for each enumeration");
+ * @endcode
+ *
+ * This macro also catches type errors. If you see a compiler error like "warning: division by zero
+ * is undefined" when using `Y_ARRAY_SIZE` then you are probably giving it a pointer.
+ *
+ * Since all of our code is expected to work on a 64 bit platform where pointers are 8 bytes we may
+ * falsefully accept pointers to types of sizes that are divisors of 8 (1, 2, 4 and 8).
+ */
+#if defined(__cplusplus)
+ #include <util/generic/array_size.h>
+#else
+ #undef Y_ARRAY_SIZE
+ #define Y_ARRAY_SIZE(arr) \
+ ((sizeof(arr) / sizeof((arr)[0])) / static_cast<size_t>(!(sizeof(arr) % sizeof((arr)[0]))))
+#endif
+
+#undef Y_ARRAY_BEGIN
+#define Y_ARRAY_BEGIN(arr) (arr)
+
+#undef Y_ARRAY_END
+#define Y_ARRAY_END(arr) ((arr) + Y_ARRAY_SIZE(arr))
+
+/**
+ * Concatenates two symbols, even if one of them is itself a macro.
+ */
+#define Y_CAT(X, Y) Y_CAT_I(X, Y)
+#define Y_CAT_I(X, Y) Y_CAT_II(X, Y)
+#define Y_CAT_II(X, Y) X##Y
+
+#define Y_STRINGIZE(X) UTIL_PRIVATE_STRINGIZE_AUX(X)
+#define UTIL_PRIVATE_STRINGIZE_AUX(X) #X
+
+#if defined(__COUNTER__)
+ #define Y_GENERATE_UNIQUE_ID(N) Y_CAT(N, __COUNTER__)
+#endif
+
+#if !defined(Y_GENERATE_UNIQUE_ID)
+ #define Y_GENERATE_UNIQUE_ID(N) Y_CAT(N, __LINE__)
+#endif
+
+#define NPOS ((size_t)-1)
diff --git a/util/system/demangle_impl.h b/util/system/demangle_impl.h
new file mode 100644
index 0000000000..081004f976
--- /dev/null
+++ b/util/system/demangle_impl.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
+namespace NPrivate {
+
+ /*
+ * cxxabi::__cxa_demangle (and thus TCppDemanger) have terrible memory ownership model.
+ *
+ * Consider using CppDemangle instead. It is slow, but robust.
+ */
+ class TCppDemangler {
+ public:
+ const char* Demangle(const char* name);
+
+ private:
+ THolder<char, TFree> TmpBuf_;
+ };
+
+} //namespace NPrivate
diff --git a/util/system/direct_io.cpp b/util/system/direct_io.cpp
new file mode 100644
index 0000000000..f59c54b0cb
--- /dev/null
+++ b/util/system/direct_io.cpp
@@ -0,0 +1,266 @@
+#include "direct_io.h"
+
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+#include <util/system/info.h>
+#include "align.h"
+
+#ifdef _linux_
+ #include <util/string/cast.h>
+ #include <linux/version.h>
+ #include <sys/utsname.h>
+#endif
+
+namespace {
+ struct TAlignmentCalcer {
+ inline TAlignmentCalcer()
+ : Alignment(0)
+ {
+#ifdef _linux_
+ utsname sysInfo;
+
+ Y_VERIFY(!uname(&sysInfo), "Error while call uname: %s", LastSystemErrorText());
+
+ TStringBuf release(sysInfo.release);
+ release = release.substr(0, release.find_first_not_of(".0123456789"));
+
+ int v1 = FromString<int>(release.NextTok('.'));
+ int v2 = FromString<int>(release.NextTok('.'));
+ int v3 = FromString<int>(release.NextTok('.'));
+ int linuxVersionCode = KERNEL_VERSION(v1, v2, v3);
+
+ if (linuxVersionCode < KERNEL_VERSION(2, 4, 10)) {
+ Alignment = 0;
+ } else if (linuxVersionCode < KERNEL_VERSION(2, 6, 0)) {
+ Alignment = NSystemInfo::GetPageSize();
+ } else {
+ // Default alignment used to be 512, but most modern devices rely on 4k physical blocks.
+ // 4k alignment works well for both 512 and 4k blocks and doesn't require 512e support in the kernel.
+ // See IGNIETFERRO-946.
+ Alignment = 4096;
+ }
+#endif
+ }
+
+ size_t Alignment;
+ };
+}
+
+TDirectIOBufferedFile::TDirectIOBufferedFile(const TString& path, EOpenMode oMode, size_t buflen /*= 1 << 17*/)
+ : File(path, oMode)
+ , Alignment(0)
+ , DataLen(0)
+ , ReadPosition(0)
+ , WritePosition(0)
+ , DirectIO(false)
+{
+ if (buflen == 0) {
+ ythrow TFileError() << "unbuffered usage is not supported";
+ }
+
+ if (oMode & Direct) {
+ Alignment = Singleton<TAlignmentCalcer>()->Alignment;
+ SetDirectIO(true);
+ }
+
+ WritePosition = File.GetLength();
+ FlushedBytes = WritePosition;
+ FlushedToDisk = FlushedBytes;
+ BufLen = (!!Alignment) ? AlignUp(buflen, Alignment) : buflen;
+ BufferStorage.Resize(BufLen + Alignment);
+ Buffer = (!!Alignment) ? AlignUp(BufferStorage.Data(), Alignment) : BufferStorage.Data();
+}
+
+#define DIRECT_IO_FLAGS (O_DIRECT | O_SYNC)
+
+void TDirectIOBufferedFile::SetDirectIO(bool value) {
+#ifdef _linux_
+ if (DirectIO == value) {
+ return;
+ }
+
+ if (!!Alignment && value) {
+ (void)fcntl(File.GetHandle(), F_SETFL, fcntl(File.GetHandle(), F_GETFL) | DIRECT_IO_FLAGS);
+ } else {
+ (void)fcntl(File.GetHandle(), F_SETFL, fcntl(File.GetHandle(), F_GETFL) & ~DIRECT_IO_FLAGS);
+ }
+
+ DirectIO = value;
+#else
+ DirectIO = value;
+#endif
+}
+
+TDirectIOBufferedFile::~TDirectIOBufferedFile() {
+ try {
+ Finish();
+ } catch (...) {
+ }
+}
+
+void TDirectIOBufferedFile::FlushData() {
+ WriteToFile(Buffer, DataLen, FlushedBytes);
+ DataLen = 0;
+ File.FlushData();
+}
+
+void TDirectIOBufferedFile::Finish() {
+ FlushData();
+ File.Flush();
+ File.Close();
+}
+
+void TDirectIOBufferedFile::Write(const void* buffer, size_t byteCount) {
+ WriteToBuffer(buffer, byteCount, DataLen);
+ WritePosition += byteCount;
+}
+
+void TDirectIOBufferedFile::WriteToBuffer(const void* buf, size_t len, ui64 position) {
+ while (len > 0) {
+ size_t writeLen = Min<size_t>(BufLen - position, len);
+
+ if (writeLen > 0) {
+ memcpy((char*)Buffer + position, buf, writeLen);
+ buf = (char*)buf + writeLen;
+ len -= writeLen;
+ DataLen = (size_t)Max(position + writeLen, (ui64)DataLen);
+ position += writeLen;
+ }
+
+ if (DataLen == BufLen) {
+ WriteToFile(Buffer, DataLen, FlushedBytes);
+ DataLen = 0;
+ position = 0;
+ }
+ }
+}
+
+void TDirectIOBufferedFile::WriteToFile(const void* buf, size_t len, ui64 position) {
+ if (!!len) {
+ SetDirectIO(IsAligned(buf) && IsAligned(len) && IsAligned(position));
+
+ File.Pwrite(buf, len, position);
+
+ FlushedBytes = Max(FlushedBytes, position + len);
+ FlushedToDisk = Min(FlushedToDisk, position);
+ }
+}
+
+size_t TDirectIOBufferedFile::PreadSafe(void* buffer, size_t byteCount, ui64 offset) {
+ if (FlushedToDisk < offset + byteCount) {
+ File.FlushData();
+ FlushedToDisk = FlushedBytes;
+ }
+
+#ifdef _linux_
+ ssize_t bytesRead = 0;
+ do {
+ bytesRead = pread(File.GetHandle(), buffer, byteCount, offset);
+ } while (bytesRead == -1 && errno == EINTR);
+
+ if (bytesRead < 0) {
+ ythrow yexception() << "error while pread file: " << LastSystemError() << "(" << LastSystemErrorText() << ")";
+ }
+
+ return bytesRead;
+#else
+ return File.Pread(buffer, byteCount, offset);
+#endif
+}
+
+size_t TDirectIOBufferedFile::ReadFromFile(void* buffer, size_t byteCount, ui64 offset) {
+ SetDirectIO(true);
+
+ ui64 bytesRead = 0;
+
+ while (byteCount) {
+ if (!Alignment || IsAligned(buffer) && IsAligned(byteCount) && IsAligned(offset)) {
+ if (const ui64 fromFile = PreadSafe(buffer, byteCount, offset)) {
+ buffer = (char*)buffer + fromFile;
+ byteCount -= fromFile;
+ offset += fromFile;
+ bytesRead += fromFile;
+ } else {
+ return bytesRead;
+ }
+ } else {
+ break;
+ }
+ }
+
+ if (!byteCount) {
+ return bytesRead;
+ }
+
+ ui64 bufSize = AlignUp(Min<size_t>(BufferStorage.Size(), byteCount + (Alignment << 1)), Alignment);
+ TBuffer readBufferStorage(bufSize + Alignment);
+ char* readBuffer = AlignUp((char*)readBufferStorage.Data(), Alignment);
+
+ while (byteCount) {
+ ui64 begin = AlignDown(offset, (ui64)Alignment);
+ ui64 end = AlignUp(offset + byteCount, (ui64)Alignment);
+ ui64 toRead = Min(end - begin, bufSize);
+ ui64 fromFile = PreadSafe(readBuffer, toRead, begin);
+
+ if (!fromFile) {
+ break;
+ }
+
+ ui64 delta = offset - begin;
+ ui64 count = Min<ui64>(fromFile - delta, byteCount);
+
+ memcpy(buffer, readBuffer + delta, count);
+ buffer = (char*)buffer + count;
+ byteCount -= count;
+ offset += count;
+ bytesRead += count;
+ }
+ return bytesRead;
+}
+
+size_t TDirectIOBufferedFile::Read(void* buffer, size_t byteCount) {
+ size_t bytesRead = Pread(buffer, byteCount, ReadPosition);
+ ReadPosition += bytesRead;
+ return bytesRead;
+}
+
+size_t TDirectIOBufferedFile::Pread(void* buffer, size_t byteCount, ui64 offset) {
+ if (!byteCount) {
+ return 0;
+ }
+
+ size_t readFromFile = 0;
+ if (offset < FlushedBytes) {
+ readFromFile = Min<ui64>(byteCount, FlushedBytes - offset);
+ size_t bytesRead = ReadFromFile(buffer, readFromFile, offset);
+ if (bytesRead != readFromFile || readFromFile == byteCount) {
+ return bytesRead;
+ }
+ }
+ ui64 start = offset > FlushedBytes ? offset - FlushedBytes : 0;
+ ui64 count = Min<ui64>(DataLen - start, byteCount - readFromFile);
+ if (count) {
+ memcpy((char*)buffer + readFromFile, (const char*)Buffer + start, count);
+ }
+ return count + readFromFile;
+}
+
+void TDirectIOBufferedFile::Pwrite(const void* buffer, size_t byteCount, ui64 offset) {
+ if (offset > WritePosition) {
+ ythrow yexception() << "cannot frite to position" << offset;
+ }
+
+ size_t writeToBufer = byteCount;
+ size_t writeToFile = 0;
+
+ if (FlushedBytes > offset) {
+ writeToFile = Min<ui64>(byteCount, FlushedBytes - offset);
+ WriteToFile(buffer, writeToFile, offset);
+ writeToBufer -= writeToFile;
+ }
+
+ if (writeToBufer > 0) {
+ ui64 bufferOffset = offset + writeToFile - FlushedBytes;
+ WriteToBuffer((const char*)buffer + writeToFile, writeToBufer, bufferOffset);
+ }
+}
diff --git a/util/system/direct_io.h b/util/system/direct_io.h
new file mode 100644
index 0000000000..6a3325a960
--- /dev/null
+++ b/util/system/direct_io.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include "align.h"
+
+#include "file.h"
+#include <util/generic/buffer.h>
+
+// Supports Linux Direct-IO:
+// - Simple buffering logic.
+// - Default buffer size of 128KB matches VM page writeback granularity, to maximize IO throughput.
+// - Supports writing odd sized files by turning off direct IO for the last chunk.
+class TDirectIOBufferedFile {
+public:
+ TDirectIOBufferedFile(const TString& path, EOpenMode oMode, size_t buflen = 1 << 17);
+ ~TDirectIOBufferedFile();
+
+ void FlushData();
+ void Finish();
+ size_t Read(void* buffer, size_t byteCount);
+ void Write(const void* buffer, size_t byteCount);
+ size_t Pread(void* buffer, size_t byteCount, ui64 offset);
+ void Pwrite(const void* buffer, size_t byteCount, ui64 offset);
+
+ inline bool IsOpen() const {
+ return true;
+ }
+
+ inline ui64 GetWritePosition() const {
+ return WritePosition;
+ }
+
+ inline ui64 GetLength() const {
+ return FlushedBytes + DataLen;
+ }
+
+ inline FHANDLE GetHandle() {
+ return File.GetHandle();
+ }
+
+ inline void FallocateNoResize(ui64 length) {
+ File.FallocateNoResize(length);
+ }
+
+ inline void ShrinkToFit() {
+ File.ShrinkToFit();
+ }
+
+private:
+ inline bool IsAligned(i64 value) {
+ return Alignment ? value == AlignDown<i64>(value, Alignment) : true;
+ }
+
+ inline bool IsAligned(const void* value) {
+ return Alignment ? value == AlignDown(value, Alignment) : true;
+ }
+
+ size_t PreadSafe(void* buffer, size_t byteCount, ui64 offset);
+ size_t ReadFromFile(void* buffer, size_t byteCount, ui64 offset);
+ void WriteToFile(const void* buf, size_t len, ui64 position);
+ void WriteToBuffer(const void* buf, size_t len, ui64 position);
+ void SetDirectIO(bool value);
+
+private:
+ TFile File;
+ size_t Alignment;
+ size_t BufLen;
+ size_t DataLen;
+ void* Buffer;
+ TBuffer BufferStorage;
+ ui64 ReadPosition;
+ ui64 WritePosition;
+ ui64 FlushedBytes;
+ ui64 FlushedToDisk;
+ bool DirectIO;
+};
diff --git a/util/system/direct_io_ut.cpp b/util/system/direct_io_ut.cpp
new file mode 100644
index 0000000000..839c3de7ca
--- /dev/null
+++ b/util/system/direct_io_ut.cpp
@@ -0,0 +1,115 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/yexception.h>
+#include <util/system/fs.h>
+#include <util/system/tempfile.h>
+#include <util/random/random.h>
+
+#include "direct_io.h"
+
+static const char* FileName_("./test.file");
+
+Y_UNIT_TEST_SUITE(TDirectIoTestSuite) {
+ Y_UNIT_TEST(TestDirectFile) {
+ TDirectIOBufferedFile file(FileName_, RdWr | Direct | Seq | CreateAlways, 1 << 15);
+ TVector<ui64> data((1 << 15) + 1);
+ TVector<ui64> readResult(data.size());
+ for (auto& i : data) {
+ i = RandomNumber<ui64>();
+ }
+ for (size_t writePos = 0; writePos < data.size();) {
+ size_t writeCount = Min<size_t>(1 + RandomNumber<size_t>(1 << 10), data.ysize() - writePos);
+ file.Write(&data[writePos], sizeof(ui64) * writeCount);
+ writePos += writeCount;
+ size_t readPos = RandomNumber(writePos);
+ size_t readCount = RandomNumber(writePos - readPos);
+ UNIT_ASSERT_VALUES_EQUAL(
+ file.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)),
+ readCount * sizeof(ui64));
+ for (size_t i = 0; i < readCount; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i + readPos]);
+ }
+ }
+ file.Finish();
+ TDirectIOBufferedFile fileNew(FileName_, RdOnly | Direct | Seq | OpenAlways, 1 << 15);
+ for (int i = 0; i < 1000; ++i) {
+ size_t readPos = RandomNumber(data.size());
+ size_t readCount = RandomNumber(data.size() - readPos);
+ UNIT_ASSERT_VALUES_EQUAL(
+ fileNew.Pread(&readResult[0], readCount * sizeof(ui64), readPos * sizeof(ui64)),
+ readCount * sizeof(ui64));
+ for (size_t j = 0; j < readCount; ++j) {
+ UNIT_ASSERT_VALUES_EQUAL(readResult[j], data[j + readPos]);
+ }
+ }
+ size_t readCount = data.size();
+ UNIT_ASSERT_VALUES_EQUAL(
+ fileNew.Pread(&readResult[0], readCount * sizeof(ui64), 0),
+ readCount * sizeof(ui64));
+ for (size_t i = 0; i < readCount; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(readResult[i], data[i]);
+ }
+ NFs::Remove(FileName_);
+ }
+
+ void TestHugeFile(size_t size) {
+ TTempFile tmpFile("test.file");
+
+ {
+ TDirectIOBufferedFile directIOFile(tmpFile.Name(), WrOnly | CreateAlways | Direct);
+ TVector<ui8> data(size, 'x');
+ directIOFile.Write(&data[0], data.size());
+ }
+
+ {
+ TDirectIOBufferedFile directIOFile(tmpFile.Name(), RdOnly | Direct);
+ TVector<ui8> data(size + 1, 'y');
+
+ const size_t readResult = directIOFile.Read(&data[0], data.size());
+
+ UNIT_ASSERT_VALUES_EQUAL(readResult, size);
+
+ UNIT_ASSERT_VALUES_EQUAL(data[0], 'x');
+ UNIT_ASSERT_VALUES_EQUAL(data[size / 2], 'x');
+ UNIT_ASSERT_VALUES_EQUAL(data[size - 1], 'x');
+ UNIT_ASSERT_VALUES_EQUAL(data[size], 'y');
+ }
+ }
+
+ Y_UNIT_TEST(TestHugeFile1) {
+ if constexpr (sizeof(size_t) > 4) {
+ TestHugeFile(5 * 1024 * 1024 * 1024ULL);
+ }
+ }
+ Y_UNIT_TEST(TestHugeFile2) {
+ if constexpr (sizeof(size_t) > 4) {
+ TestHugeFile(5 * 1024 * 1024 * 1024ULL + 1111);
+ }
+ }
+}
+
+Y_UNIT_TEST_SUITE(TDirectIoErrorHandling) {
+ Y_UNIT_TEST(Constructor) {
+ // A non-existent file should not be opened for reading
+ UNIT_ASSERT_EXCEPTION(TDirectIOBufferedFile(FileName_, RdOnly, 1 << 15), TFileError);
+ }
+
+ Y_UNIT_TEST(WritingReadOnlyFileBufferFlushed) {
+ // Note the absence of Direct
+ TDirectIOBufferedFile file(FileName_, RdOnly | OpenAlways, 1);
+ TString buffer = "Hello";
+ UNIT_ASSERT_EXCEPTION(file.Write(buffer.data(), buffer.size()), TFileError);
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(WritingReadOnlyFileAllInBuffer) {
+ TDirectIOBufferedFile file(FileName_, RdOnly | Direct | Seq | OpenAlways, 1 << 15);
+ TString buffer = "Hello";
+
+ // Doesn't throw because of buffering.
+ file.Write(buffer.data(), buffer.size());
+
+ UNIT_ASSERT_EXCEPTION(file.Finish(), TFileError);
+ NFs::Remove(FileName_);
+ }
+}
diff --git a/util/system/dynlib.cpp b/util/system/dynlib.cpp
new file mode 100644
index 0000000000..9d2541c25f
--- /dev/null
+++ b/util/system/dynlib.cpp
@@ -0,0 +1,138 @@
+#include "dynlib.h"
+
+#include "guard.h"
+#include "mutex.h"
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+
+#ifdef _win32_
+ #include "winint.h"
+
+ #define DLLOPEN(path, flags) LoadLibrary(path)
+ #define DLLCLOSE(hndl) FreeLibrary(hndl)
+ #define DLLSYM(hndl, name) GetProcAddress(hndl, name)
+#else
+ #include <dlfcn.h>
+
+ #ifndef RTLD_GLOBAL
+ #define RTLD_GLOBAL (0)
+ #endif
+
+using HINSTANCE = void*;
+
+ #define DLLOPEN(path, flags) dlopen(path, flags)
+ #define DLLCLOSE(hndl) dlclose(hndl)
+ #define DLLSYM(hndl, name) dlsym(hndl, name)
+#endif
+
+inline TString DLLERR() {
+#ifdef _unix_
+ return dlerror();
+#endif
+
+#ifdef _win32_
+ char* msg = 0;
+ DWORD cnt = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ nullptr, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (char*)&msg, 0, nullptr);
+ if (!msg)
+ return "DLLERR() unknown error";
+ while (cnt && isspace(msg[cnt - 1]))
+ --cnt;
+ TString err(msg, 0, cnt);
+ LocalFree(msg);
+ return err;
+#endif
+}
+
+class TDynamicLibrary::TImpl {
+private:
+ inline TImpl(const char* path, int flags)
+ : Module(DLLOPEN(path, flags))
+ , Unloadable(true)
+ {
+ (void)flags;
+
+ if (!Module) {
+ ythrow yexception() << DLLERR().data();
+ }
+ }
+
+ class TCreateMutex: public TMutex {
+ };
+
+public:
+ static inline TImpl* SafeCreate(const char* path, int flags) {
+ auto guard = Guard(*Singleton<TCreateMutex>());
+
+ return new TImpl(path, flags);
+ }
+
+ inline ~TImpl() {
+ if (Module && Unloadable) {
+ DLLCLOSE(Module);
+ }
+ }
+
+ inline void* SymOptional(const char* name) noexcept {
+ return (void*)DLLSYM(Module, name);
+ }
+
+ inline void* Sym(const char* name) {
+ void* symbol = SymOptional(name);
+
+ if (symbol == nullptr) {
+ ythrow yexception() << DLLERR().data();
+ }
+
+ return symbol;
+ }
+
+ inline void SetUnloadable(bool unloadable) {
+ Unloadable = unloadable;
+ }
+
+private:
+ HINSTANCE Module;
+ bool Unloadable;
+};
+
+TDynamicLibrary::TDynamicLibrary() noexcept {
+}
+
+TDynamicLibrary::TDynamicLibrary(const TString& path, int flags) {
+ Open(path.data(), flags);
+}
+
+TDynamicLibrary::~TDynamicLibrary() = default;
+
+void TDynamicLibrary::Open(const char* path, int flags) {
+ Impl_.Reset(TImpl::SafeCreate(path, flags));
+}
+
+void TDynamicLibrary::Close() noexcept {
+ Impl_.Destroy();
+}
+
+void* TDynamicLibrary::SymOptional(const char* name) noexcept {
+ if (!IsLoaded()) {
+ return nullptr;
+ }
+
+ return Impl_->SymOptional(name);
+}
+
+void* TDynamicLibrary::Sym(const char* name) {
+ if (!IsLoaded()) {
+ ythrow yexception() << "library not loaded";
+ }
+
+ return Impl_->Sym(name);
+}
+
+bool TDynamicLibrary::IsLoaded() const noexcept {
+ return (bool)Impl_.Get();
+}
+
+void TDynamicLibrary::SetUnloadable(bool unloadable) {
+ Impl_->SetUnloadable(unloadable);
+}
diff --git a/util/system/dynlib.h b/util/system/dynlib.h
new file mode 100644
index 0000000000..66eaf4a5c1
--- /dev/null
+++ b/util/system/dynlib.h
@@ -0,0 +1,119 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
+#define Y_GET_FUNC(dll, name) FUNC_##name((dll).Sym(#name))
+#define Y_GET_FUNC_OPTIONAL(dll, name) FUNC_##name((dll).SymOptional(#name))
+
+#ifdef _win32_
+ #define DEFAULT_DLLOPEN_FLAGS 0
+#else
+ #include <dlfcn.h>
+
+ #ifndef RTLD_GLOBAL
+ #define RTLD_GLOBAL (0)
+ #endif
+
+ #define DEFAULT_DLLOPEN_FLAGS (RTLD_NOW | RTLD_GLOBAL)
+#endif
+
+class TDynamicLibrary {
+public:
+ TDynamicLibrary() noexcept;
+ TDynamicLibrary(const TString& path, int flags = DEFAULT_DLLOPEN_FLAGS);
+ ~TDynamicLibrary();
+
+ void Open(const char* path, int flags = DEFAULT_DLLOPEN_FLAGS);
+ void Close() noexcept;
+ void* SymOptional(const char* name) noexcept;
+ void* Sym(const char* name);
+ bool IsLoaded() const noexcept;
+ void SetUnloadable(bool unloadable); // Set to false to avoid unloading on destructor
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+// a wrapper for a symbol
+template <class TLib>
+class TExternalSymbol {
+private:
+ TLib* PLib;
+ TDynamicLibrary* DLib;
+ TString lname;
+ TString vname;
+
+public:
+ TExternalSymbol() noexcept {
+ PLib = nullptr;
+ DLib = nullptr;
+ }
+ TExternalSymbol(const TExternalSymbol& es) {
+ PLib = nullptr;
+ DLib = nullptr;
+ if (es.IsDynamic())
+ Open(es.LibName().data(), es.VtblName().data());
+ else if (es.IsStatic())
+ SetSym(es.Symbol());
+ }
+ TExternalSymbol& operator=(const TExternalSymbol& es) {
+ if (this != &es) {
+ Close();
+ if (es.IsDynamic())
+ Open(es.LibName().data(), es.VtblName().data());
+ else if (es.IsStatic())
+ SetSym(es.Symbol());
+ }
+ return *this;
+ }
+ ~TExternalSymbol() {
+ delete DLib;
+ }
+ // set the symbol from dynamic source
+ void Open(const char* lib_name, const char* vtbl_name) {
+ if (DLib != nullptr || PLib != nullptr)
+ return;
+ try {
+ DLib = new TDynamicLibrary();
+ DLib->Open(lib_name);
+ PLib = (TLib*)DLib->Sym(vtbl_name);
+ } catch (...) {
+ delete DLib;
+ DLib = nullptr;
+ throw;
+ }
+ lname = lib_name;
+ vname = vtbl_name;
+ }
+ // set the symbol from static source
+ void SetSym(TLib* pl) noexcept {
+ if (DLib == nullptr && PLib == nullptr)
+ PLib = pl;
+ }
+ void Close() noexcept {
+ delete DLib;
+ DLib = 0;
+ PLib = 0;
+ lname.remove();
+ vname.remove();
+ }
+ TLib* Symbol() const noexcept {
+ return PLib;
+ }
+ const TString& LibName() const noexcept {
+ return lname;
+ }
+ const TString& VtblName() const noexcept {
+ return vname;
+ }
+ bool IsStatic() const noexcept {
+ return DLib == nullptr && PLib != nullptr;
+ }
+ bool IsDynamic() const noexcept {
+ return DLib && DLib->IsLoaded() && PLib != nullptr;
+ }
+};
diff --git a/util/system/env.cpp b/util/system/env.cpp
new file mode 100644
index 0000000000..ead9b566a5
--- /dev/null
+++ b/util/system/env.cpp
@@ -0,0 +1,67 @@
+#include "env.h"
+
+#include <util/generic/string.h>
+#include <util/generic/yexception.h>
+
+#ifdef _win_
+ #include <util/generic/vector.h>
+ #include "winint.h"
+#else
+ #include <cerrno>
+ #include <cstdlib>
+#endif
+
+/**
+ * On Windows there may be many copies of enviroment variables, there at least two known, one is
+ * manipulated by Win32 API, another by C runtime, so we must be consistent in the choice of
+ * functions used to manipulate them.
+ *
+ * Relevant links:
+ * - http://bugs.python.org/issue16633
+ * - https://a.yandex-team.ru/review/108892/details
+ */
+
+TString GetEnv(const TString& key, const TString& def) {
+#ifdef _win_
+ size_t len = GetEnvironmentVariableA(key.data(), nullptr, 0);
+
+ if (len == 0) {
+ if (GetLastError() == ERROR_ENVVAR_NOT_FOUND) {
+ return def;
+ }
+ return TString{};
+ }
+
+ TVector<char> buffer(len);
+ size_t bufferSize;
+ do {
+ bufferSize = buffer.size();
+ len = GetEnvironmentVariableA(key.data(), buffer.data(), static_cast<DWORD>(bufferSize));
+ if (len > bufferSize) {
+ buffer.resize(len);
+ }
+ } while (len > bufferSize);
+
+ return TString(buffer.data(), len);
+#else
+ const char* env = getenv(key.data());
+ return env ? TString(env) : def;
+#endif
+}
+
+void SetEnv(const TString& key, const TString& value) {
+ bool isOk = false;
+ int errorCode = 0;
+#ifdef _win_
+ isOk = SetEnvironmentVariable(key.data(), value.data());
+ if (!isOk) {
+ errorCode = GetLastError();
+ }
+#else
+ isOk = (0 == setenv(key.data(), value.data(), true /*replace*/));
+ if (!isOk) {
+ errorCode = errno;
+ }
+#endif
+ Y_ENSURE_EX(isOk, TSystemError() << "failed to SetEnv with error-code " << errorCode);
+}
diff --git a/util/system/env.h b/util/system/env.h
new file mode 100644
index 0000000000..e2ccdd1e95
--- /dev/null
+++ b/util/system/env.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+/**
+ * Search the environment list provided by the host environment for associated variable.
+ *
+ * @param key String identifying the name of the environmental variable to look for
+ * @param def String that returns if environmental variable not found by key
+ *
+ * @return String that is associated with the matched environment variable or empty string if
+ * such variable is missing.
+ *
+ * @note Use it only in pair with `SetEnv` as there may be inconsistency in their behaviour
+ * otherwise.
+ * @note Calls to `GetEnv` and `SetEnv` from different threads must be synchronized.
+ * @see SetEnv
+ */
+TString GetEnv(const TString& key, const TString& def = TString());
+
+/**
+ * Add or change environment variable provided by the host environment.
+ *
+ * @key String identifying the name of the environment variable to set or change
+ * @value Value to assign
+
+ * @note Use it only in pair with `GetEnv` as there may be inconsistency in their behaviour
+ * otherwise.
+ * @note Calls to `GetEnv` and `SetEnv` from different threads must be synchronized.
+ * @see GetEnv
+ */
+void SetEnv(const TString& key, const TString& value);
diff --git a/util/system/env_ut.cpp b/util/system/env_ut.cpp
new file mode 100644
index 0000000000..e03cc01658
--- /dev/null
+++ b/util/system/env_ut.cpp
@@ -0,0 +1,31 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/string.h>
+#include "env.h"
+
+Y_UNIT_TEST_SUITE(EnvTest) {
+ Y_UNIT_TEST(GetSetEnvTest) {
+ TString key = "util_GETENV_TestVar";
+ TString value = "Some value for env var";
+ TString def = "Some default value for env var";
+ // first of all, it should be clear
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(key), TString());
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(key, def), def);
+ SetEnv(key, value);
+ // set and see what value we get here
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(key), value);
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(key, def), value);
+ // set empty value
+ SetEnv(key, TString());
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(key), TString());
+
+ // check for long values, see IGNIETFERRO-214
+ TString longKey = "util_GETENV_TestVarLong";
+ TString longValue{1500, 't'};
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(longKey), TString());
+ SetEnv(longKey, longValue);
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(longKey), longValue);
+ SetEnv(longKey, TString());
+ UNIT_ASSERT_VALUES_EQUAL(GetEnv(longKey), TString());
+ }
+}
diff --git a/util/system/err.cpp b/util/system/err.cpp
new file mode 100644
index 0000000000..5573ea1ee9
--- /dev/null
+++ b/util/system/err.cpp
@@ -0,0 +1,79 @@
+#include "defaults.h"
+#include "progname.h"
+#include "compat.h"
+#include "error.h"
+
+#include <util/generic/scope.h>
+
+#include <util/stream/printf.h>
+#include <util/stream/output.h>
+
+void vwarnx(const char* fmt, va_list args) {
+ Cerr << GetProgramName() << ": ";
+
+ if (fmt) {
+ Printf(Cerr, fmt, args);
+ }
+
+ Cerr << '\n';
+}
+
+void vwarn(const char* fmt, va_list args) {
+ int curErrNo = errno;
+ auto curErrText = LastSystemErrorText();
+
+ Y_DEFER {
+ errno = curErrNo;
+ };
+
+ Cerr << GetProgramName() << ": ";
+
+ if (fmt) {
+ Printf(Cerr, fmt, args);
+ Cerr << ": ";
+ }
+
+ Cerr << curErrText << '\n';
+}
+
+void warn(const char* fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ vwarn(fmt, args);
+ va_end(args);
+}
+
+void warnx(const char* fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ vwarnx(fmt, args);
+ va_end(args);
+}
+
+void verr(int status, const char* fmt, va_list args) {
+ vwarn(fmt, args);
+ exit(status);
+}
+
+void err(int status, const char* fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ verr(status, fmt, args);
+ va_end(args);
+}
+
+void verrx(int status, const char* fmt, va_list args) {
+ vwarnx(fmt, args);
+ exit(status);
+}
+
+void errx(int status, const char* fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ verrx(status, fmt, args);
+ va_end(args);
+}
diff --git a/util/system/error.cpp b/util/system/error.cpp
new file mode 100644
index 0000000000..f778ec42cb
--- /dev/null
+++ b/util/system/error.cpp
@@ -0,0 +1,96 @@
+#include "tls.h"
+#include "error.h"
+
+#include <util/string/strip.h>
+#include <util/generic/strfcpy.h>
+
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+
+#if defined(_win_)
+ #include <util/network/socket.h>
+ #include <util/generic/singleton.h>
+ #include "winint.h"
+#elif defined(_unix_)
+ #include <unistd.h>
+#endif
+
+void ClearLastSystemError() {
+#if defined(_win_)
+ SetLastError(0);
+#else
+ errno = 0;
+#endif
+}
+
+int LastSystemError() {
+#if defined(_win_)
+ int ret = GetLastError();
+
+ if (ret)
+ return ret;
+
+ ret = WSAGetLastError();
+
+ if (ret)
+ return ret;
+ // when descriptors number are over maximum, errno set in this variable
+ ret = *(_errno());
+ return ret;
+
+#else
+ return errno;
+#endif
+}
+
+#if defined(_win_)
+namespace {
+ struct TErrString {
+ inline TErrString() noexcept {
+ data[0] = 0;
+ }
+
+ char data[1024];
+ };
+}
+#endif
+
+const char* LastSystemErrorText(int code) {
+#if defined(_win_)
+ TErrString& text(*Singleton<TErrString>());
+ LastSystemErrorText(text.data, sizeof(text.data), code);
+
+ return text.data;
+#else
+ return strerror(code);
+#endif
+}
+
+#ifdef _win_
+static char* Strip(char* s) {
+ size_t len = strlen(s);
+ const char* ptr = s;
+ Strip(ptr, len);
+ if (ptr != s)
+ memmove(s, ptr, len);
+ s[len] = 0;
+ return s;
+}
+#endif // _win_
+
+void LastSystemErrorText(char* str, size_t size, int code) {
+#if defined(_win_)
+ FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, 0, code, 0, str, DWORD(size), 0);
+ Strip(str);
+#elif defined(_sun_)
+ strfcpy(str, strerror(code), size);
+#elif defined(_freebsd_) || defined(_darwin_) || defined(_musl_) || defined(_bionic_)
+ strerror_r(code, str, size);
+#elif defined(_linux_) | defined(_cygwin_)
+ char* msg = strerror_r(code, str, size);
+ strncpy(str, msg, size);
+#else
+ #error port me gently!
+#endif
+}
diff --git a/util/system/error.h b/util/system/error.h
new file mode 100644
index 0000000000..5f2d4cc547
--- /dev/null
+++ b/util/system/error.h
@@ -0,0 +1,95 @@
+#pragma once
+
+#include "defaults.h"
+
+#if defined(_win_)
+ #include <winerror.h>
+ #include <errno.h>
+
+ #undef E_FAIL
+ #undef ERROR_TIMEOUT
+
+ #if defined(_MSC_VER)
+ #undef EADDRINUSE
+ #undef EADDRNOTAVAIL
+ #undef EAFNOSUPPORT
+ #undef EALREADY
+ #undef ECANCELED
+ #undef ECONNABORTED
+ #undef ECONNREFUSED
+ #undef ECONNRESET
+ #undef EDESTADDRREQ
+ #undef EHOSTUNREACH
+ #undef EINPROGRESS
+ #undef EISCONN
+ #undef ELOOP
+ #undef EMSGSIZE
+ #undef ENETDOWN
+ #undef ENETRESET
+ #undef ENETUNREACH
+ #undef ENOBUFS
+ #undef ENOPROTOOPT
+ #undef ENOTCONN
+ #undef ENOTSOCK
+ #undef EOPNOTSUPP
+ #undef EPROTONOSUPPORT
+ #undef EPROTOTYPE
+ #undef ETIMEDOUT
+ #undef EWOULDBLOCK
+ #undef ENAMETOOLONG
+ #undef ENOTEMPTY
+
+ #define EWOULDBLOCK WSAEWOULDBLOCK
+ #define EINPROGRESS WSAEINPROGRESS
+ #define EALREADY WSAEALREADY
+ #define ENOTSOCK WSAENOTSOCK
+ #define EDESTADDRREQ WSAEDESTADDRREQ
+ #define EMSGSIZE WSAEMSGSIZE
+ #define EPROTOTYPE WSAEPROTOTYPE
+ #define ENOPROTOOPT WSAENOPROTOOPT
+ #define EPROTONOSUPPORT WSAEPROTONOSUPPORT
+ #define ESOCKTNOSUPPORT WSAESOCKTNOSUPPORT
+ #define EOPNOTSUPP WSAEOPNOTSUPP
+ #define EPFNOSUPPORT WSAEPFNOSUPPORT
+ #define EAFNOSUPPORT WSAEAFNOSUPPORT
+ #define EADDRINUSE WSAEADDRINUSE
+ #define EADDRNOTAVAIL WSAEADDRNOTAVAIL
+ #define ENETDOWN WSAENETDOWN
+ #define ENETUNREACH WSAENETUNREACH
+ #define ENETRESET WSAENETRESET
+ #define ECONNABORTED WSAECONNABORTED
+ #define ECONNRESET WSAECONNRESET
+ #define ENOBUFS WSAENOBUFS
+ #define EISCONN WSAEISCONN
+ #define ENOTCONN WSAENOTCONN
+ #define ESHUTDOWN WSAESHUTDOWN
+ #define ETOOMANYREFS WSAETOOMANYREFS
+ #define ETIMEDOUT WSAETIMEDOUT
+ #define ECONNREFUSED WSAECONNREFUSED
+ #define ELOOP WSAELOOP
+ #define ENAMETOOLONG WSAENAMETOOLONG
+ #define EHOSTDOWN WSAEHOSTDOWN
+ #define EHOSTUNREACH WSAEHOSTUNREACH
+ #define ENOTEMPTY WSAENOTEMPTY
+ #define EPROCLIM WSAEPROCLIM
+ #define EUSERS WSAEUSERS
+ #define ESTALE WSAESTALE
+ #define EREMOTE WSAEREMOTE
+ #define ECANCELED WSAECANCELLED
+ #endif
+
+ #define EDQUOT WSAEDQUOT
+#endif
+
+void ClearLastSystemError();
+int LastSystemError();
+void LastSystemErrorText(char* str, size_t size, int code);
+const char* LastSystemErrorText(int code);
+
+inline const char* LastSystemErrorText() {
+ return LastSystemErrorText(LastSystemError());
+}
+
+inline void LastSystemErrorText(char* str, size_t size) {
+ LastSystemErrorText(str, size, LastSystemError());
+}
diff --git a/util/system/error_ut.cpp b/util/system/error_ut.cpp
new file mode 100644
index 0000000000..763b0dddb7
--- /dev/null
+++ b/util/system/error_ut.cpp
@@ -0,0 +1,41 @@
+#include "error.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/ylimits.h>
+
+#ifdef _win_
+ #include "winint.h"
+#else
+ #include <fcntl.h>
+#endif
+
+class TSysErrorTest: public TTestBase {
+ UNIT_TEST_SUITE(TSysErrorTest);
+ UNIT_TEST(TestErrorCode)
+ UNIT_TEST(TestErrorMessage)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestErrorCode() {
+ GenFailure();
+
+ UNIT_ASSERT(LastSystemError() != 0);
+ }
+
+ inline void TestErrorMessage() {
+ GenFailure();
+
+ UNIT_ASSERT(*LastSystemErrorText() != 0);
+ }
+
+ inline void GenFailure() {
+#ifdef _win_
+ SetLastError(3);
+#else
+ UNIT_ASSERT(open("/non-existent", O_RDONLY) < 0);
+#endif
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TSysErrorTest);
diff --git a/util/system/event.cpp b/util/system/event.cpp
new file mode 100644
index 0000000000..79b3cdb291
--- /dev/null
+++ b/util/system/event.cpp
@@ -0,0 +1,137 @@
+#include "datetime.h"
+#include "defaults.h"
+
+#include <cstdio>
+
+#include "atomic.h"
+#include "event.h"
+#include "mutex.h"
+#include "condvar.h"
+
+#ifdef _win_
+ #include "winint.h"
+#endif
+
+class TSystemEvent::TEvImpl: public TAtomicRefCount<TSystemEvent::TEvImpl> {
+public:
+#ifdef _win_
+ inline TEvImpl(ResetMode rmode) {
+ cond = CreateEvent(nullptr, rmode == rManual ? true : false, false, nullptr);
+ }
+
+ inline ~TEvImpl() {
+ CloseHandle(cond);
+ }
+
+ inline void Reset() noexcept {
+ ResetEvent(cond);
+ }
+
+ inline void Signal() noexcept {
+ SetEvent(cond);
+ }
+
+ inline bool WaitD(TInstant deadLine) noexcept {
+ if (deadLine == TInstant::Max()) {
+ return WaitForSingleObject(cond, INFINITE) == WAIT_OBJECT_0;
+ }
+
+ const TInstant now = Now();
+
+ if (now < deadLine) {
+ //TODO
+ return WaitForSingleObject(cond, (deadLine - now).MilliSeconds()) == WAIT_OBJECT_0;
+ }
+
+ return (WaitForSingleObject(cond, 0) == WAIT_OBJECT_0);
+ }
+#else
+ inline TEvImpl(ResetMode rmode)
+ : Manual(rmode == rManual ? true : false)
+ {
+ }
+
+ inline void Signal() noexcept {
+ if (Manual && AtomicGet(Signaled)) {
+ return; // shortcut
+ }
+
+ with_lock (Mutex) {
+ AtomicSet(Signaled, 1);
+ }
+
+ if (Manual) {
+ Cond.BroadCast();
+ } else {
+ Cond.Signal();
+ }
+ }
+
+ inline void Reset() noexcept {
+ AtomicSet(Signaled, 0);
+ }
+
+ inline bool WaitD(TInstant deadLine) noexcept {
+ if (Manual && AtomicGet(Signaled)) {
+ return true; // shortcut
+ }
+
+ bool resSignaled = true;
+
+ with_lock (Mutex) {
+ while (!AtomicGet(Signaled)) {
+ if (!Cond.WaitD(Mutex, deadLine)) {
+ resSignaled = AtomicGet(Signaled); // timed out, but Signaled could have been set
+
+ break;
+ }
+ }
+
+ if (!Manual) {
+ AtomicSet(Signaled, 0);
+ }
+ }
+
+ return resSignaled;
+ }
+#endif
+
+private:
+#ifdef _win_
+ HANDLE cond;
+#else
+ TCondVar Cond;
+ TMutex Mutex;
+ TAtomic Signaled = 0;
+ bool Manual;
+#endif
+};
+
+TSystemEvent::TSystemEvent(ResetMode rmode)
+ : EvImpl_(new TEvImpl(rmode))
+{
+}
+
+TSystemEvent::TSystemEvent(const TSystemEvent& other) noexcept
+ : EvImpl_(other.EvImpl_)
+{
+}
+
+TSystemEvent& TSystemEvent::operator=(const TSystemEvent& other) noexcept {
+ EvImpl_ = other.EvImpl_;
+ return *this;
+}
+
+TSystemEvent::~TSystemEvent() = default;
+
+void TSystemEvent::Reset() noexcept {
+ EvImpl_->Reset();
+}
+
+void TSystemEvent::Signal() noexcept {
+ EvImpl_->Signal();
+}
+
+bool TSystemEvent::WaitD(TInstant deadLine) noexcept {
+ return EvImpl_->WaitD(deadLine);
+}
diff --git a/util/system/event.h b/util/system/event.h
new file mode 100644
index 0000000000..cab2fc478a
--- /dev/null
+++ b/util/system/event.h
@@ -0,0 +1,122 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/datetime/base.h>
+
+struct TEventResetType {
+ enum ResetMode {
+ rAuto, // the state will be nonsignaled after Wait() returns
+ rManual, // we need call Reset() to set the state to nonsignaled.
+ };
+};
+
+/**
+ * DEPRECATED!
+ *
+ * Use TAutoEvent, TManualEvent for the direct replacement.
+ * Use TManualEvent to prevent SEGFAULT (http://nga.at.yandex-team.ru/5772).
+ */
+class TSystemEvent: public TEventResetType {
+public:
+ TSystemEvent(ResetMode rmode = rManual);
+ TSystemEvent(const TSystemEvent& other) noexcept;
+ TSystemEvent& operator=(const TSystemEvent& other) noexcept;
+
+ ~TSystemEvent();
+
+ void Reset() noexcept;
+ void Signal() noexcept;
+
+ /*
+ * return true if signaled, false if timed out.
+ */
+ bool WaitD(TInstant deadLine) noexcept;
+
+ /*
+ * return true if signaled, false if timed out.
+ */
+ inline bool WaitT(TDuration timeOut) noexcept {
+ return WaitD(timeOut.ToDeadLine());
+ }
+
+ /*
+ * wait infinite time
+ */
+ inline void WaitI() noexcept {
+ WaitD(TInstant::Max());
+ }
+
+ //return true if signaled, false if timed out.
+ inline bool Wait(ui32 timer) noexcept {
+ return WaitT(TDuration::MilliSeconds(timer));
+ }
+
+ inline bool Wait() noexcept {
+ WaitI();
+
+ return true;
+ }
+
+private:
+ class TEvImpl;
+ TIntrusivePtr<TEvImpl> EvImpl_;
+};
+
+class TAutoEvent: public TSystemEvent {
+public:
+ TAutoEvent()
+ : TSystemEvent(TSystemEvent::rAuto)
+ {
+ }
+
+private:
+ void Reset() noexcept;
+};
+
+/**
+ * Prevents from a "shortcut problem" (see http://nga.at.yandex-team.ru/5772): if Wait will be called after Signaled
+ * flag set to true in Signal method but before CondVar.BroadCast - Wait will shortcut (without actual wait on condvar).
+ * If Wait thread will destruct event - Signal thread will do broadcast on a destructed CondVar.
+ */
+class TManualEvent {
+public:
+ TManualEvent()
+ : Ev(TEventResetType::rManual)
+ {
+ }
+
+ void Reset() noexcept {
+ TSystemEvent{Ev}.Reset();
+ }
+
+ void Signal() noexcept {
+ TSystemEvent{Ev}.Signal();
+ }
+
+ /** return true if signaled, false if timed out. */
+ bool WaitD(TInstant deadLine) noexcept {
+ return TSystemEvent{Ev}.WaitD(deadLine);
+ }
+
+ /** return true if signaled, false if timed out. */
+ inline bool WaitT(TDuration timeOut) noexcept {
+ return TSystemEvent{Ev}.WaitT(timeOut);
+ }
+
+ /** Wait infinite time */
+ inline void WaitI() noexcept {
+ TSystemEvent{Ev}.WaitI();
+ }
+
+ /** return true if signaled, false if timed out. */
+ inline bool Wait(ui32 timer) noexcept {
+ return TSystemEvent{Ev}.Wait(timer);
+ }
+
+ inline bool Wait() noexcept {
+ return TSystemEvent{Ev}.Wait();
+ }
+
+private:
+ TSystemEvent Ev;
+};
diff --git a/util/system/event_ut.cpp b/util/system/event_ut.cpp
new file mode 100644
index 0000000000..2506cb7a91
--- /dev/null
+++ b/util/system/event_ut.cpp
@@ -0,0 +1,132 @@
+#include "event.h"
+#include "atomic.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/thread/pool.h>
+
+namespace {
+ struct TSharedData {
+ TSharedData()
+ : Counter(0)
+ , failed(false)
+ {
+ }
+
+ TAtomic Counter;
+ TManualEvent event;
+ bool failed;
+ };
+
+ struct TThreadTask: public IObjectInQueue {
+ public:
+ TThreadTask(TSharedData& data, size_t id)
+ : Data_(data)
+ , Id_(id)
+ {
+ }
+
+ void Process(void*) override {
+ THolder<TThreadTask> This(this);
+
+ if (Id_ == 0) {
+ usleep(100);
+ bool cond = Data_.Counter == 0;
+ if (!cond) {
+ Data_.failed = true;
+ }
+ Data_.event.Signal();
+ } else {
+ while (!Data_.event.WaitT(TDuration::Seconds(100))) {
+ }
+ AtomicAdd(Data_.Counter, Id_);
+ }
+ }
+
+ private:
+ TSharedData& Data_;
+ size_t Id_;
+ };
+
+ class TSignalTask: public IObjectInQueue {
+ private:
+ TManualEvent& Ev_;
+
+ public:
+ TSignalTask(TManualEvent& ev)
+ : Ev_(ev)
+ {
+ }
+
+ void Process(void*) override {
+ Ev_.Signal();
+ }
+ };
+
+ class TOwnerTask: public IObjectInQueue {
+ public:
+ TManualEvent Barrier;
+ THolder<TManualEvent> Ev;
+
+ public:
+ TOwnerTask()
+ : Ev(new TManualEvent)
+ {
+ }
+
+ void Process(void*) override {
+ Ev->WaitI();
+ Ev.Destroy();
+ }
+ };
+
+}
+
+Y_UNIT_TEST_SUITE(EventTest) {
+ Y_UNIT_TEST(WaitAndSignalTest) {
+ TSharedData data;
+ TThreadPool queue;
+ queue.Start(5);
+ for (size_t i = 0; i < 5; ++i) {
+ UNIT_ASSERT(queue.Add(new TThreadTask(data, i)));
+ }
+ queue.Stop();
+ UNIT_ASSERT(data.Counter == 10);
+ UNIT_ASSERT(!data.failed);
+ }
+
+ Y_UNIT_TEST(ConcurrentSignalAndWaitTest) {
+ // test for problem detected by thread-sanitizer (signal/wait race) SEARCH-2113
+ const size_t limit = 200;
+ TManualEvent event[limit];
+ TThreadPool queue;
+ queue.Start(limit);
+ TVector<THolder<IObjectInQueue>> tasks;
+ for (size_t i = 0; i < limit; ++i) {
+ tasks.emplace_back(MakeHolder<TSignalTask>(event[i]));
+ UNIT_ASSERT(queue.Add(tasks.back().Get()));
+ }
+ for (size_t i = limit; i != 0; --i) {
+ UNIT_ASSERT(event[i - 1].WaitT(TDuration::Seconds(90)));
+ }
+ queue.Stop();
+ }
+
+ /** Test for a problem: http://nga.at.yandex-team.ru/5772 */
+ Y_UNIT_TEST(DestructorBeforeSignalFinishTest) {
+ return;
+ TVector<THolder<IObjectInQueue>> tasks;
+ for (size_t i = 0; i < 1000; ++i) {
+ auto owner = MakeHolder<TOwnerTask>();
+ tasks.emplace_back(MakeHolder<TSignalTask>(*owner->Ev));
+ tasks.emplace_back(std::move(owner));
+ }
+
+ TThreadPool queue;
+ queue.Start(4);
+ for (auto& task : tasks) {
+ UNIT_ASSERT(queue.Add(task.Get()));
+ }
+ queue.Stop();
+ }
+}
diff --git a/util/system/execpath.cpp b/util/system/execpath.cpp
new file mode 100644
index 0000000000..33198af58b
--- /dev/null
+++ b/util/system/execpath.cpp
@@ -0,0 +1,199 @@
+#include "platform.h"
+
+#include <stdlib.h>
+
+#if defined(_solaris_)
+ #include <stdlib.h>
+#elif defined(_darwin_)
+ #include <mach-o/dyld.h>
+#elif defined(_win_)
+ #include "winint.h"
+ #include <io.h>
+#elif defined(_linux_)
+ #include <unistd.h>
+#elif defined(_freebsd_)
+ #include <string.h>
+ #include <sys/types.h> // for u_int not defined in sysctl.h
+ #include <sys/sysctl.h>
+ #include <unistd.h>
+#endif
+
+#include <util/folder/dirut.h>
+#include <util/generic/singleton.h>
+#include <util/generic/function.h>
+#include <util/generic/yexception.h>
+#include <util/memory/tempbuf.h>
+#include <util/stream/file.h>
+#include <util/stream/pipe.h>
+#include <util/string/cast.h>
+
+#include "filemap.h"
+#include "execpath.h"
+#include "fs.h"
+
+#if defined(_freebsd_)
+static inline bool GoodPath(const TString& path) {
+ return path.find('/') != TString::npos;
+}
+
+static inline int FreeBSDSysCtl(int* mib, size_t mibSize, TTempBuf& res) {
+ for (size_t i = 0; i < 2; ++i) {
+ size_t cb = res.Size();
+ if (sysctl(mib, mibSize, res.Data(), &cb, nullptr, 0) == 0) {
+ res.Proceed(cb);
+ return 0;
+ } else if (errno == ENOMEM) {
+ res = TTempBuf(cb);
+ } else {
+ return errno;
+ }
+ }
+ return errno;
+}
+
+static inline TString FreeBSDGetExecPath() {
+ int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
+ TTempBuf buf;
+ int r = FreeBSDSysCtl(mib, Y_ARRAY_SIZE(mib), buf);
+ if (r == 0) {
+ return TString(buf.Data(), buf.Filled() - 1);
+ } else if (r == ENOTSUP) { // older FreeBSD version
+ /*
+ * BSD analogue for /proc/self is /proc/curproc.
+ * See:
+ * https://www.freebsd.org/cgi/man.cgi?query=procfs&sektion=5&format=html
+ */
+ TString path("/proc/curproc/file");
+ return NFs::ReadLink(path);
+ } else {
+ return TString();
+ }
+}
+
+static inline TString FreeBSDGetArgv0() {
+ int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_ARGS, getpid()};
+ TTempBuf buf;
+ int r = FreeBSDSysCtl(mib, Y_ARRAY_SIZE(mib), buf);
+ if (r == 0) {
+ return TString(buf.Data());
+ } else if (r == ENOTSUP) {
+ return TString();
+ } else {
+ ythrow yexception() << "FreeBSDGetArgv0() failed: " << LastSystemErrorText();
+ }
+}
+
+static inline bool FreeBSDGuessExecPath(const TString& guessPath, TString& execPath) {
+ if (NFs::Exists(guessPath)) {
+ // now it should work for real
+ execPath = FreeBSDGetExecPath();
+ if (RealPath(execPath) == RealPath(guessPath)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static inline bool FreeBSDGuessExecBasePath(const TString& guessBasePath, TString& execPath) {
+ return FreeBSDGuessExecPath(TString(guessBasePath) + "/" + getprogname(), execPath);
+}
+
+#endif
+
+static TString GetExecPathImpl() {
+#if defined(_solaris_)
+ return execname();
+#elif defined(_darwin_)
+ TTempBuf execNameBuf;
+ for (size_t i = 0; i < 2; ++i) {
+ std::remove_pointer_t<TFunctionArg<decltype(_NSGetExecutablePath), 1>> bufsize = execNameBuf.Size();
+ int r = _NSGetExecutablePath(execNameBuf.Data(), &bufsize);
+ if (r == 0) {
+ return execNameBuf.Data();
+ } else if (r == -1) {
+ execNameBuf = TTempBuf(bufsize);
+ }
+ }
+ ythrow yexception() << "GetExecPathImpl() failed";
+#elif defined(_win_)
+ TTempBuf execNameBuf;
+ for (;;) {
+ DWORD r = GetModuleFileName(nullptr, execNameBuf.Data(), execNameBuf.Size());
+ if (r == execNameBuf.Size()) {
+ execNameBuf = TTempBuf(execNameBuf.Size() * 2);
+ } else if (r == 0) {
+ ythrow yexception() << "GetExecPathImpl() failed: " << LastSystemErrorText();
+ } else {
+ return execNameBuf.Data();
+ }
+ }
+#elif defined(_linux_) || defined(_cygwin_)
+ TString path("/proc/self/exe");
+ return NFs::ReadLink(path);
+// TODO(yoda): check if the filename ends with " (deleted)"
+#elif defined(_freebsd_)
+ TString execPath = FreeBSDGetExecPath();
+ if (GoodPath(execPath)) {
+ return execPath;
+ }
+ if (FreeBSDGuessExecPath(FreeBSDGetArgv0(), execPath)) {
+ return execPath;
+ }
+ if (FreeBSDGuessExecPath(getenv("_"), execPath)) {
+ return execPath;
+ }
+ if (FreeBSDGuessExecBasePath(getenv("PWD"), execPath)) {
+ return execPath;
+ }
+ if (FreeBSDGuessExecBasePath(NFs::CurrentWorkingDirectory(), execPath)) {
+ return execPath;
+ }
+
+ ythrow yexception() << "can not resolve exec path";
+#else
+ #error dont know how to implement GetExecPath on this platform
+#endif
+}
+
+static bool GetPersistentExecPathImpl(TString& to) {
+#if defined(_solaris_)
+ to = TString("/proc/self/object/a.out");
+ return true;
+#elif defined(_linux_) || defined(_cygwin_)
+ to = TString("/proc/self/exe");
+ return true;
+#elif defined(_freebsd_)
+ to = TString("/proc/curproc/file");
+ return true;
+#else // defined(_win_) || defined(_darwin_) or unknown
+ Y_UNUSED(to);
+ return false;
+#endif
+}
+
+namespace {
+ struct TExecPathsHolder {
+ inline TExecPathsHolder() {
+ ExecPath = GetExecPathImpl();
+
+ if (!GetPersistentExecPathImpl(PersistentExecPath)) {
+ PersistentExecPath = ExecPath;
+ }
+ }
+
+ static inline auto Instance() {
+ return SingletonWithPriority<TExecPathsHolder, 1>();
+ }
+
+ TString ExecPath;
+ TString PersistentExecPath;
+ };
+}
+
+const TString& GetExecPath() {
+ return TExecPathsHolder::Instance()->ExecPath;
+}
+
+const TString& GetPersistentExecPath() {
+ return TExecPathsHolder::Instance()->PersistentExecPath;
+}
diff --git a/util/system/execpath.h b/util/system/execpath.h
new file mode 100644
index 0000000000..4b914b8e85
--- /dev/null
+++ b/util/system/execpath.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+
+// NOTE: This function has rare sporadic failures (throws exceptions) on FreeBSD. See REVIEW:54297
+const TString& GetExecPath();
+
+/**
+ * Get openable path to the binary currently being executed.
+ *
+ * The path does not match the original binary location, but stays openable even
+ * if the binary was moved or removed.
+ *
+ * On UNIX variants, utilizes the /proc FS. On Windows, equivalent to
+ * GetExecPath.
+ */
+const TString& GetPersistentExecPath();
diff --git a/util/system/execpath_ut.cpp b/util/system/execpath_ut.cpp
new file mode 100644
index 0000000000..16b01466f5
--- /dev/null
+++ b/util/system/execpath_ut.cpp
@@ -0,0 +1,22 @@
+#include "execpath.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "platform.h"
+#include <util/folder/dirut.h>
+
+Y_UNIT_TEST_SUITE(TExecPathTest) {
+ Y_UNIT_TEST(TestIt) {
+ TString execPath = GetExecPath();
+ TString persistentExecPath = GetPersistentExecPath();
+
+ try {
+ UNIT_ASSERT(NFs::Exists(execPath));
+ UNIT_ASSERT(NFs::Exists(persistentExecPath));
+ } catch (...) {
+ Cerr << execPath << Endl;
+
+ throw;
+ }
+ }
+}
diff --git a/util/system/fasttime.cpp b/util/system/fasttime.cpp
new file mode 100644
index 0000000000..057a814f0a
--- /dev/null
+++ b/util/system/fasttime.cpp
@@ -0,0 +1,242 @@
+#include "dynlib.h"
+#include "fasttime.h"
+
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+#include <utility>
+
+#include <util/thread/singleton.h>
+
+#if defined(_win_) || defined(_arm32_) || defined(_cygwin_)
+ui64 InterpolatedMicroSeconds() {
+ return MicroSeconds();
+}
+#else
+
+ #include <dlfcn.h>
+ #include <sys/time.h>
+
+ #if defined(_musl_)
+ #include <util/generic/hash.h>
+ #include <util/generic/vector.h>
+ #include <util/generic/string.h>
+
+ #include <contrib/libs/linuxvdso/interface.h>
+ #endif
+
+namespace {
+ using TTime = ui64;
+
+ struct TSymbols {
+ using TFunc = int (*)(struct timeval*, struct timezone*);
+
+ inline TSymbols()
+ : Func(nullptr)
+ {
+ // not DEFAULT, cause library/cpp/gettimeofday
+ Func = reinterpret_cast<TFunc>(dlsym(RTLD_NEXT, "gettimeofday"));
+
+ #if defined(_musl_)
+ if (!Func) {
+ Func = reinterpret_cast<TFunc>(NVdso::Function("__vdso_gettimeofday", "LINUX_2.6"));
+ }
+ #endif
+
+ if (!Func) {
+ Func = reinterpret_cast<TFunc>(Libc()->Sym("gettimeofday"));
+ }
+ }
+
+ inline TTime SystemTime() {
+ timeval tv;
+
+ Zero(tv);
+
+ Func(&tv, nullptr);
+
+ return (((TTime)1000000) * (TTime)tv.tv_sec) + (TTime)tv.tv_usec;
+ }
+
+ static inline THolder<TDynamicLibrary> OpenLibc() {
+ const char* libs[] = {
+ "/lib/libc.so.8",
+ "/lib/libc.so.7",
+ "/lib/libc.so.6",
+ };
+
+ for (auto& lib : libs) {
+ try {
+ return MakeHolder<TDynamicLibrary>(lib);
+ } catch (...) {
+ // ¯\_(ツ)_/¯
+ }
+ }
+
+ ythrow yexception() << "can not load libc";
+ }
+
+ inline TDynamicLibrary* Libc() {
+ if (!Lib) {
+ Lib = OpenLibc();
+ }
+
+ return Lib.Get();
+ }
+
+ THolder<TDynamicLibrary> Lib;
+ TFunc Func;
+ };
+
+ static inline TTime SystemTime() {
+ return Singleton<TSymbols>()->SystemTime();
+ }
+
+ struct TInitialTimes {
+ inline TInitialTimes()
+ : ITime(TimeBase())
+ , IProc(RdtscBase())
+ {
+ }
+
+ static TTime RdtscBase() {
+ return GetCycleCount() / (TTime)1000;
+ }
+
+ static TTime TimeBase() {
+ return SystemTime();
+ }
+
+ inline TTime Rdtsc() {
+ return RdtscBase() - IProc;
+ }
+
+ inline TTime Time() {
+ return TimeBase() - ITime;
+ }
+
+ const TTime ITime;
+ const TTime IProc;
+ };
+
+ template <size_t N, class A, class B>
+ class TLinePredictor {
+ public:
+ using TSample = std::pair<A, B>;
+
+ inline TLinePredictor()
+ : C_(0)
+ , A_(0)
+ , B_(0)
+ {
+ }
+
+ inline void Add(const A& a, const B& b) noexcept {
+ Add(TSample(a, b));
+ }
+
+ inline void Add(const TSample& s) noexcept {
+ S_[(C_++) % N] = s;
+ if (C_ > 1) {
+ ReCalc();
+ }
+ }
+
+ inline B Predict(A a) const noexcept {
+ return A_ + a * B_;
+ }
+
+ inline size_t Size() const noexcept {
+ return C_;
+ }
+
+ inline bool Enough() const noexcept {
+ return Size() >= N;
+ }
+
+ inline A LastX() const noexcept {
+ return S_[(C_ - 1) % N].first;
+ }
+
+ private:
+ inline void ReCalc() noexcept {
+ const size_t n = Min(N, C_);
+
+ double sx = 0;
+ double sy = 0;
+ double sxx = 0;
+ double syy = 0;
+ double sxy = 0;
+
+ for (size_t i = 0; i < n; ++i) {
+ const double x = S_[i].first;
+ const double y = S_[i].second;
+
+ sx += x;
+ sy += y;
+ sxx += x * x;
+ syy += y * y;
+ sxy += x * y;
+ }
+
+ B_ = (n * sxy - sx * sy) / (n * sxx - sx * sx);
+ A_ = (sy - B_ * sx) / n;
+ }
+
+ private:
+ size_t C_;
+ TSample S_[N];
+ double A_;
+ double B_;
+ };
+
+ class TTimePredictor: public TInitialTimes {
+ public:
+ inline TTimePredictor()
+ : Next_(1)
+ {
+ }
+
+ inline TTime Get() {
+ return GetBase() + ITime;
+ }
+
+ private:
+ inline TTime GetBase() {
+ const TTime x = Rdtsc();
+
+ if (TimeToSync(x)) {
+ const TTime y = Time();
+
+ P_.Add(x, y);
+
+ return y;
+ }
+
+ if (P_.Enough()) {
+ return P_.Predict(x);
+ }
+
+ return Time();
+ }
+
+ inline bool TimeToSync(TTime x) {
+ if (x > Next_) {
+ Next_ = Min(x + x / 10, x + 1000000);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ private:
+ TLinePredictor<16, TTime, TTime> P_;
+ TTime Next_;
+ };
+}
+
+ui64 InterpolatedMicroSeconds() {
+ return FastTlsSingleton<TTimePredictor>()->Get();
+}
+
+#endif
diff --git a/util/system/fasttime.h b/util/system/fasttime.h
new file mode 100644
index 0000000000..544cb4bd19
--- /dev/null
+++ b/util/system/fasttime.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#include "datetime.h"
+
+/// Fast but possibly less accurate microseconds since epoch
+ui64 InterpolatedMicroSeconds();
diff --git a/util/system/fhandle.cpp b/util/system/fhandle.cpp
new file mode 100644
index 0000000000..67250ae1e5
--- /dev/null
+++ b/util/system/fhandle.cpp
@@ -0,0 +1 @@
+#include "fhandle.h"
diff --git a/util/system/fhandle.h b/util/system/fhandle.h
new file mode 100644
index 0000000000..f8033e3c14
--- /dev/null
+++ b/util/system/fhandle.h
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "defaults.h"
+
+using WIN_HANDLE = void*;
+#define INVALID_WIN_HANDLE ((WIN_HANDLE)(long)-1)
+
+using UNIX_HANDLE = int;
+#define INVALID_UNIX_HANDLE -1
+
+#if defined(_win_)
+using FHANDLE = WIN_HANDLE;
+ #define INVALID_FHANDLE INVALID_WIN_HANDLE
+#elif defined(_unix_)
+using FHANDLE = UNIX_HANDLE;
+ #define INVALID_FHANDLE INVALID_UNIX_HANDLE
+#else
+ #error
+#endif
+
+#if defined(_cygwin_)
+using OS_HANDLE = WIN_HANDLE;
+ #define INVALID_OS_HANDLE INVALID_WIN_HANDLE
+#else
+using OS_HANDLE = FHANDLE;
+ #define INVALID_OS_HANDLE INVALID_FHANDLE
+#endif
diff --git a/util/system/file.cpp b/util/system/file.cpp
new file mode 100644
index 0000000000..4a261d020c
--- /dev/null
+++ b/util/system/file.cpp
@@ -0,0 +1,1302 @@
+#include "file.h"
+#include "flock.h"
+#include "fstat.h"
+#include "sysstat.h"
+#include "align.h"
+#include "info.h"
+
+#include <array>
+
+#include <util/string/util.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+
+#include <util/stream/hex.h>
+#include <util/stream/format.h>
+
+#include <util/random/random.h>
+
+#include <util/generic/size_literals.h>
+#include <util/generic/string.h>
+#include <util/generic/ylimits.h>
+#include <util/generic/yexception.h>
+
+#include <util/datetime/base.h>
+
+#include <errno.h>
+
+#if defined(_unix_)
+ #include <fcntl.h>
+
+ #if defined(_linux_) && (!defined(_android_) || __ANDROID_API__ >= 21) && !defined(FALLOC_FL_KEEP_SIZE)
+ #include <linux/falloc.h>
+ #endif
+
+ #include <stdlib.h>
+ #include <unistd.h>
+ #include <sys/mman.h>
+#elif defined(_win_)
+ #include "winint.h"
+ #include "fs_win.h"
+ #include <io.h>
+#endif
+
+#if defined(_bionic_)
+ #include <sys/sendfile.h>
+ #define HAVE_POSIX_FADVISE 0
+ #define HAVE_SYNC_FILE_RANGE 0
+#elif defined(_linux_)
+ #include <sys/sendfile.h>
+ #define HAVE_POSIX_FADVISE 1
+ #define HAVE_SYNC_FILE_RANGE 1
+#elif defined(__FreeBSD__) && !defined(WITH_VALGRIND)
+ #include <sys/param.h>
+ #define HAVE_POSIX_FADVISE (__FreeBSD_version >= 900501)
+ #define HAVE_SYNC_FILE_RANGE 0
+#else
+ #define HAVE_POSIX_FADVISE 0
+ #define HAVE_SYNC_FILE_RANGE 0
+#endif
+
+static bool IsStupidFlagCombination(EOpenMode oMode) {
+ // ForAppend will actually not be applied in the following combinations:
+ return (oMode & (CreateAlways | ForAppend)) == (CreateAlways | ForAppend) || (oMode & (TruncExisting | ForAppend)) == (TruncExisting | ForAppend) || (oMode & (CreateNew | ForAppend)) == (CreateNew | ForAppend);
+}
+
+TFileHandle::TFileHandle(const TString& fName, EOpenMode oMode) noexcept {
+ ui32 fcMode = 0;
+ EOpenMode createMode = oMode & MaskCreation;
+ Y_VERIFY(!IsStupidFlagCombination(oMode), "oMode %d makes no sense", static_cast<int>(oMode));
+ if (!(oMode & MaskRW)) {
+ oMode |= RdWr;
+ }
+ if (!(oMode & AMask)) {
+ oMode |= ARW;
+ }
+
+#ifdef _win_
+
+ switch (createMode) {
+ case OpenExisting:
+ fcMode = OPEN_EXISTING;
+ break;
+ case TruncExisting:
+ fcMode = TRUNCATE_EXISTING;
+ break;
+ case OpenAlways:
+ fcMode = OPEN_ALWAYS;
+ break;
+ case CreateNew:
+ fcMode = CREATE_NEW;
+ break;
+ case CreateAlways:
+ fcMode = CREATE_ALWAYS;
+ break;
+ default:
+ abort();
+ break;
+ }
+
+ ui32 faMode = 0;
+ if (oMode & RdOnly) {
+ faMode |= GENERIC_READ;
+ }
+ if (oMode & WrOnly) {
+ // WrOnly or RdWr
+ faMode |= GENERIC_WRITE;
+ }
+ if (oMode & ::ForAppend) {
+ faMode |= GENERIC_WRITE;
+ faMode |= FILE_APPEND_DATA;
+ faMode &= ~FILE_WRITE_DATA;
+ }
+
+ bool inheritHandle = !(oMode & CloseOnExec);
+
+ ui32 shMode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
+
+ ui32 attrMode = FILE_ATTRIBUTE_NORMAL;
+ if ((createMode == OpenExisting || createMode == OpenAlways) && ((oMode & AMask) == (oMode & AR))) {
+ attrMode |= FILE_ATTRIBUTE_READONLY;
+ }
+ if (oMode & Seq) {
+ attrMode |= FILE_FLAG_SEQUENTIAL_SCAN;
+ }
+ if (oMode & Temp) {
+ // we use TTempFile instead of FILE_FLAG_DELETE_ON_CLOSE
+ attrMode |= FILE_ATTRIBUTE_TEMPORARY;
+ }
+ if (oMode & Transient) {
+ attrMode |= FILE_FLAG_DELETE_ON_CLOSE;
+ }
+ if ((oMode & (Direct | DirectAligned)) && (oMode & WrOnly)) {
+ // WrOnly or RdWr
+ attrMode |= /*FILE_FLAG_NO_BUFFERING |*/ FILE_FLAG_WRITE_THROUGH;
+ }
+
+ Fd_ = NFsPrivate::CreateFileWithUtf8Name(fName, faMode, shMode, fcMode, attrMode, inheritHandle);
+
+ if ((oMode & ::ForAppend) && (Fd_ != INVALID_FHANDLE)) {
+ ::SetFilePointer(Fd_, 0, 0, FILE_END);
+ }
+
+#elif defined(_unix_)
+
+ switch (createMode) {
+ case OpenExisting:
+ fcMode = 0;
+ break;
+ case TruncExisting:
+ fcMode = O_TRUNC;
+ break;
+ case OpenAlways:
+ fcMode = O_CREAT;
+ break;
+ case CreateNew:
+ fcMode = O_CREAT | O_EXCL;
+ break;
+ case CreateAlways:
+ fcMode = O_CREAT | O_TRUNC;
+ break;
+ default:
+ abort();
+ break;
+ }
+
+ if ((oMode & RdOnly) && (oMode & WrOnly)) {
+ fcMode |= O_RDWR;
+ } else if (oMode & RdOnly) {
+ fcMode |= O_RDONLY;
+ } else if (oMode & WrOnly) {
+ fcMode |= O_WRONLY;
+ }
+
+ if (oMode & ::ForAppend) {
+ fcMode |= O_APPEND;
+ }
+
+ if (oMode & CloseOnExec) {
+ fcMode |= O_CLOEXEC;
+ }
+
+ /* I don't now about this for unix...
+ if (oMode & Temp) {
+ }
+ */
+ #if defined(_freebsd_)
+ if (oMode & (Direct | DirectAligned)) {
+ fcMode |= O_DIRECT;
+ }
+
+ if (oMode & Sync) {
+ fcMode |= O_SYNC;
+ }
+ #elif defined(_linux_)
+ if (oMode & DirectAligned) {
+ /*
+ * O_DIRECT in Linux requires aligning request size and buffer address
+ * to size of hardware sector (see hw_sector_size or ioctl BLKSSZGET).
+ * Usually 512 bytes, but modern hardware works better with 4096 bytes.
+ */
+ fcMode |= O_DIRECT;
+ }
+ if (oMode & Sync) {
+ fcMode |= O_SYNC;
+ }
+ #endif
+
+ #if defined(_linux_)
+ fcMode |= O_LARGEFILE;
+ #endif
+
+ ui32 permMode = 0;
+ if (oMode & AXOther) {
+ permMode |= S_IXOTH;
+ }
+ if (oMode & AWOther) {
+ permMode |= S_IWOTH;
+ }
+ if (oMode & AROther) {
+ permMode |= S_IROTH;
+ }
+ if (oMode & AXGroup) {
+ permMode |= S_IXGRP;
+ }
+ if (oMode & AWGroup) {
+ permMode |= S_IWGRP;
+ }
+ if (oMode & ARGroup) {
+ permMode |= S_IRGRP;
+ }
+ if (oMode & AXUser) {
+ permMode |= S_IXUSR;
+ }
+ if (oMode & AWUser) {
+ permMode |= S_IWUSR;
+ }
+ if (oMode & ARUser) {
+ permMode |= S_IRUSR;
+ }
+
+ do {
+ Fd_ = ::open(fName.data(), fcMode, permMode);
+ } while (Fd_ == -1 && errno == EINTR);
+
+ #if HAVE_POSIX_FADVISE
+ if (Fd_ >= 0) {
+ if (oMode & NoReuse) {
+ ::posix_fadvise(Fd_, 0, 0, POSIX_FADV_NOREUSE);
+ }
+
+ if (oMode & Seq) {
+ ::posix_fadvise(Fd_, 0, 0, POSIX_FADV_SEQUENTIAL);
+ }
+
+ if (oMode & NoReadAhead) {
+ ::posix_fadvise(Fd_, 0, 0, POSIX_FADV_RANDOM);
+ }
+ }
+ #endif
+
+ //temp file
+ if (Fd_ >= 0 && (oMode & Transient)) {
+ unlink(fName.data());
+ }
+#else
+ #error unsupported platform
+#endif
+}
+
+bool TFileHandle::Close() noexcept {
+ bool isOk = true;
+#ifdef _win_
+ if (Fd_ != INVALID_FHANDLE) {
+ isOk = (::CloseHandle(Fd_) != 0);
+ }
+ if (!isOk) {
+ Y_VERIFY(GetLastError() != ERROR_INVALID_HANDLE,
+ "must not quietly close invalid handle");
+ }
+#elif defined(_unix_)
+ if (Fd_ != INVALID_FHANDLE) {
+ isOk = (::close(Fd_) == 0 || errno == EINTR);
+ }
+ if (!isOk) {
+ // Do not quietly close bad descriptor,
+ // because often it means double close
+ // that is disasterous
+ Y_VERIFY(errno != EBADF, "must not quietly close bad descriptor: fd=%d", int(Fd_));
+ }
+#else
+ #error unsupported platform
+#endif
+ Fd_ = INVALID_FHANDLE;
+ return isOk;
+}
+
+static inline i64 DoSeek(FHANDLE h, i64 offset, SeekDir origin) noexcept {
+ if (h == INVALID_FHANDLE) {
+ return -1L;
+ }
+#if defined(_win_)
+ static ui32 dir[] = {FILE_BEGIN, FILE_CURRENT, FILE_END};
+ LARGE_INTEGER pos;
+ pos.QuadPart = offset;
+ pos.LowPart = ::SetFilePointer(h, pos.LowPart, &pos.HighPart, dir[origin]);
+ if (pos.LowPart == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) {
+ pos.QuadPart = -1;
+ }
+ return pos.QuadPart;
+#elif defined(_unix_)
+ static int dir[] = {SEEK_SET, SEEK_CUR, SEEK_END};
+ #if defined(_sun_)
+ return ::llseek(h, (offset_t)offset, dir[origin]);
+ #else
+ return ::lseek(h, (off_t)offset, dir[origin]);
+ #endif
+#else
+ #error unsupported platform
+#endif
+}
+
+i64 TFileHandle::GetPosition() const noexcept {
+ return DoSeek(Fd_, 0, sCur);
+}
+
+i64 TFileHandle::Seek(i64 offset, SeekDir origin) noexcept {
+ return DoSeek(Fd_, offset, origin);
+}
+
+i64 TFileHandle::GetLength() const noexcept {
+ // XXX: returns error code, but does not set errno
+ if (!IsOpen()) {
+ return -1L;
+ }
+ return GetFileLength(Fd_);
+}
+
+bool TFileHandle::Resize(i64 length) noexcept {
+ if (!IsOpen()) {
+ return false;
+ }
+ i64 currentLength = GetLength();
+ if (length == currentLength) {
+ return true;
+ }
+#if defined(_win_)
+ i64 currentPosition = GetPosition();
+ if (currentPosition == -1L) {
+ return false;
+ }
+ Seek(length, sSet);
+ if (!::SetEndOfFile(Fd_)) {
+ return false;
+ }
+ if (currentPosition < length) {
+ Seek(currentPosition, sSet);
+ }
+ return true;
+#elif defined(_unix_)
+ return (0 == ftruncate(Fd_, (off_t)length));
+#else
+ #error unsupported platform
+#endif
+}
+
+bool TFileHandle::Reserve(i64 length) noexcept {
+ // FIXME this should reserve disk space with fallocate
+ if (!IsOpen()) {
+ return false;
+ }
+ i64 currentLength = GetLength();
+ if (length <= currentLength) {
+ return true;
+ }
+ if (!Resize(length)) {
+ return false;
+ }
+#if defined(_win_)
+ if (!::SetFileValidData(Fd_, length)) {
+ Resize(currentLength);
+ return false;
+ }
+#elif defined(_unix_)
+// No way to implement this under FreeBSD. Just do nothing
+#else
+ #error unsupported platform
+#endif
+ return true;
+}
+
+bool TFileHandle::FallocateNoResize(i64 length) noexcept {
+ if (!IsOpen()) {
+ return false;
+ }
+#if defined(_linux_) && (!defined(_android_) || __ANDROID_API__ >= 21)
+ return !fallocate(Fd_, FALLOC_FL_KEEP_SIZE, 0, length);
+#else
+ Y_UNUSED(length);
+ return true;
+#endif
+}
+
+// Pair for FallocateNoResize
+bool TFileHandle::ShrinkToFit() noexcept {
+ if (!IsOpen()) {
+ return false;
+ }
+#if defined(_linux_) && (!defined(_android_) || __ANDROID_API__ >= 21)
+ return !ftruncate(Fd_, (off_t)GetLength());
+#else
+ return true;
+#endif
+}
+
+bool TFileHandle::Flush() noexcept {
+ if (!IsOpen()) {
+ return false;
+ }
+#if defined(_win_)
+ bool ok = ::FlushFileBuffers(Fd_) != 0;
+ /*
+ * FlushFileBuffers fails if hFile is a handle to the console output.
+ * That is because the console output is not buffered.
+ * The function returns FALSE, and GetLastError returns ERROR_INVALID_HANDLE.
+ */
+ return ok || GetLastError() == ERROR_INVALID_HANDLE;
+#elif defined(_unix_)
+ int ret = ::fsync(Fd_);
+
+ /*
+ * Ignore EROFS, EINVAL - fd is bound to a special file
+ * (PIPE, FIFO, or socket) which does not support synchronization.
+ * Fail in case of EIO, ENOSPC, EDQUOT - data might be lost.
+ */
+ return ret == 0 || errno == EROFS || errno == EINVAL
+ #if defined(_darwin_)
+ // ENOTSUP fd does not refer to a vnode
+ || errno == ENOTSUP
+ #endif
+ ;
+#else
+ #error unsupported platform
+#endif
+}
+
+bool TFileHandle::FlushData() noexcept {
+#if defined(_linux_)
+ if (!IsOpen()) {
+ return false;
+ }
+
+ int ret = ::fdatasync(Fd_);
+
+ // Same loginc in error handling as for fsync above.
+ return ret == 0 || errno == EROFS || errno == EINVAL;
+#else
+ return Flush();
+#endif
+}
+
+i32 TFileHandle::Read(void* buffer, ui32 byteCount) noexcept {
+ // FIXME size and return must be 64-bit
+ if (!IsOpen()) {
+ return -1;
+ }
+#if defined(_win_)
+ DWORD bytesRead = 0;
+ if (::ReadFile(Fd_, buffer, byteCount, &bytesRead, nullptr)) {
+ return bytesRead;
+ }
+ return -1;
+#elif defined(_unix_)
+ i32 ret;
+ do {
+ ret = ::read(Fd_, buffer, byteCount);
+ } while (ret == -1 && errno == EINTR);
+ return ret;
+#else
+ #error unsupported platform
+#endif
+}
+
+i32 TFileHandle::Write(const void* buffer, ui32 byteCount) noexcept {
+ if (!IsOpen()) {
+ return -1;
+ }
+#if defined(_win_)
+ DWORD bytesWritten = 0;
+ if (::WriteFile(Fd_, buffer, byteCount, &bytesWritten, nullptr)) {
+ return bytesWritten;
+ }
+ return -1;
+#elif defined(_unix_)
+ i32 ret;
+ do {
+ ret = ::write(Fd_, buffer, byteCount);
+ } while (ret == -1 && errno == EINTR);
+ return ret;
+#else
+ #error unsupported platform
+#endif
+}
+
+i32 TFileHandle::Pread(void* buffer, ui32 byteCount, i64 offset) const noexcept {
+#if defined(_win_)
+ OVERLAPPED io;
+ Zero(io);
+ DWORD bytesRead = 0;
+ io.Offset = (ui32)offset;
+ io.OffsetHigh = (ui32)(offset >> 32);
+ if (::ReadFile(Fd_, buffer, byteCount, &bytesRead, &io)) {
+ return bytesRead;
+ }
+ if (::GetLastError() == ERROR_HANDLE_EOF) {
+ return 0;
+ }
+ return -1;
+#elif defined(_unix_)
+ i32 ret;
+ do {
+ ret = ::pread(Fd_, buffer, byteCount, offset);
+ } while (ret == -1 && errno == EINTR);
+ return ret;
+#else
+ #error unsupported platform
+#endif
+}
+
+i32 TFileHandle::Pwrite(const void* buffer, ui32 byteCount, i64 offset) const noexcept {
+#if defined(_win_)
+ OVERLAPPED io;
+ Zero(io);
+ DWORD bytesWritten = 0;
+ io.Offset = (ui32)offset;
+ io.OffsetHigh = (ui32)(offset >> 32);
+ if (::WriteFile(Fd_, buffer, byteCount, &bytesWritten, &io)) {
+ return bytesWritten;
+ }
+ return -1;
+#elif defined(_unix_)
+ i32 ret;
+ do {
+ ret = ::pwrite(Fd_, buffer, byteCount, offset);
+ } while (ret == -1 && errno == EINTR);
+ return ret;
+#else
+ #error unsupported platform
+#endif
+}
+
+FHANDLE TFileHandle::Duplicate() const noexcept {
+ if (!IsOpen()) {
+ return INVALID_FHANDLE;
+ }
+#if defined(_win_)
+ FHANDLE dupHandle;
+ if (!::DuplicateHandle(GetCurrentProcess(), Fd_, GetCurrentProcess(), &dupHandle, 0, TRUE, DUPLICATE_SAME_ACCESS)) {
+ return INVALID_FHANDLE;
+ }
+ return dupHandle;
+#elif defined(_unix_)
+ return ::dup(Fd_);
+#else
+ #error unsupported platform
+#endif
+}
+
+int TFileHandle::Duplicate2Posix(int dstHandle) const noexcept {
+ if (!IsOpen()) {
+ return -1;
+ }
+#if defined(_win_)
+ FHANDLE dupHandle = Duplicate();
+ if (dupHandle == INVALID_FHANDLE) {
+ _set_errno(EMFILE);
+ return -1;
+ }
+ int posixHandle = _open_osfhandle((intptr_t)dupHandle, 0);
+ if (posixHandle == -1) {
+ CloseHandle(dupHandle);
+ return -1;
+ }
+ if (dup2(posixHandle, dstHandle) == -1) {
+ dstHandle = -1;
+ }
+ _close(posixHandle);
+ return dstHandle;
+#elif defined(_unix_)
+ while (dup2(Fd_, dstHandle) == -1) {
+ if (errno != EINTR) {
+ return -1;
+ }
+ }
+ return dstHandle;
+#else
+ #error unsupported platform
+#endif
+}
+
+bool TFileHandle::LinkTo(const TFileHandle& fh) const noexcept {
+#if defined(_unix_)
+ while (dup2(fh.Fd_, Fd_) == -1) {
+ if (errno != EINTR) {
+ return false;
+ }
+ }
+ return true;
+#elif defined(_win_)
+ TFileHandle nh(fh.Duplicate());
+
+ if (!nh.IsOpen()) {
+ return false;
+ }
+
+ //not thread-safe
+ nh.Swap(*const_cast<TFileHandle*>(this));
+
+ return true;
+#else
+ #error unsupported
+#endif
+}
+
+int TFileHandle::Flock(int op) noexcept {
+ return ::Flock(Fd_, op);
+}
+
+bool TFileHandle::SetDirect() {
+#ifdef _linux_
+ const long flags = fcntl(Fd_, F_GETFL);
+ const int r = fcntl(Fd_, F_SETFL, flags | O_DIRECT);
+
+ return !r;
+#endif
+
+ return false;
+}
+
+void TFileHandle::ResetDirect() {
+#ifdef _linux_
+ long flags = fcntl(Fd_, F_GETFL);
+ fcntl(Fd_, F_SETFL, flags & ~O_DIRECT);
+#endif
+}
+
+i64 TFileHandle::CountCache(i64 offset, i64 length) const noexcept {
+#ifdef _linux_
+ const i64 pageSize = NSystemInfo::GetPageSize();
+ constexpr size_t vecSize = 512; // Fetch up to 2MiB at once
+ const i64 batchSize = vecSize * pageSize;
+ std::array<ui8, vecSize> vec;
+ void* ptr = nullptr;
+ i64 res = 0;
+
+ if (!IsOpen()) {
+ return -1;
+ }
+
+ if (!length) {
+ length = GetLength();
+ length -= Min(length, offset);
+ }
+
+ if (!length) {
+ return 0;
+ }
+
+ const i64 begin = AlignDown(offset, pageSize);
+ const i64 end = AlignUp(offset + length, pageSize);
+ const i64 size = end - begin;
+
+ /*
+ * Since fincode is not implemented yet use mmap and mincore.
+ * This is not so effective and scalable for frequent usage.
+ */
+ ptr = ::mmap(
+ (caddr_t) nullptr,
+ size,
+ PROT_READ,
+ MAP_SHARED | MAP_NORESERVE,
+ Fd_,
+ begin);
+ if (MAP_FAILED == ptr) {
+ return -1;
+ }
+
+ for (i64 base = begin; base < end; base += batchSize) {
+ const size_t batch = Min(vecSize, size_t((end - base) / pageSize));
+ void* batchPtr = static_cast<caddr_t>(ptr) + (base - begin);
+
+ if (::mincore(batchPtr, batch * pageSize, vec.data())) {
+ res = -1;
+ break;
+ }
+
+ for (size_t i = 0; i < batch; i++) {
+ // count uptodate complete pages in cache
+ if (vec[i] & 1) {
+ res += pageSize;
+ }
+ }
+
+ if (base == begin && (vec[0] & 1)) {
+ // cut head of first page
+ res -= offset - begin;
+ }
+
+ if ((end - base) <= batchSize && (vec[batch - 1] & 1)) {
+ // cut tail of last page
+ res -= size - (offset - begin) - length;
+ }
+ }
+
+ ::munmap(ptr, size);
+
+ return res;
+#else
+ Y_UNUSED(offset);
+ Y_UNUSED(length);
+ return -1;
+#endif
+}
+
+void TFileHandle::PrefetchCache(i64 offset, i64 length, bool wait) const noexcept {
+#ifdef _linux_
+ #if HAVE_POSIX_FADVISE
+ // POSIX_FADV_WILLNEED starts reading upto read_ahead_kb in background
+ ::posix_fadvise(Fd_, offset, length, POSIX_FADV_WILLNEED);
+ #endif
+
+ if (wait) {
+ TFileHandle devnull("/dev/null", OpenExisting | WrOnly | CloseOnExec);
+ off_t end = length ? (offset + length) : GetLength();
+ off_t pos = offset;
+ ssize_t ret;
+
+ do {
+ ret = ::sendfile((FHANDLE)devnull, Fd_, &pos, end - pos);
+ } while (pos < end && (ret > 0 || errno == EINTR));
+ }
+#else
+ Y_UNUSED(offset);
+ Y_UNUSED(length);
+ Y_UNUSED(wait);
+#endif
+}
+
+void TFileHandle::EvictCache(i64 offset, i64 length) const noexcept {
+#if HAVE_POSIX_FADVISE
+ /*
+ * This tries to evicts only unmaped, clean, complete pages.
+ */
+ ::posix_fadvise(Fd_, offset, length, POSIX_FADV_DONTNEED);
+#else
+ Y_UNUSED(offset);
+ Y_UNUSED(length);
+#endif
+}
+
+bool TFileHandle::FlushCache(i64 offset, i64 length, bool wait) noexcept {
+#if HAVE_SYNC_FILE_RANGE
+ int flags = SYNC_FILE_RANGE_WRITE;
+ if (wait) {
+ flags |= SYNC_FILE_RANGE_WAIT_AFTER;
+ }
+ int ret = ::sync_file_range(Fd_, offset, length, flags);
+ return ret == 0 || errno == EROFS;
+#else
+ Y_UNUSED(offset);
+ Y_UNUSED(length);
+ if (wait) {
+ return FlushData();
+ }
+ return true;
+#endif
+}
+
+TString DecodeOpenMode(ui32 mode0) {
+ ui32 mode = mode0;
+
+ TStringBuilder r;
+
+#define F(flag) \
+ if ((mode & flag) == flag) { \
+ mode &= ~flag; \
+ if (r) { \
+ r << TStringBuf("|"); \
+ } \
+ r << TStringBuf(#flag); \
+ }
+
+ F(RdWr)
+ F(RdOnly)
+ F(WrOnly)
+
+ F(CreateAlways)
+ F(CreateNew)
+ F(OpenAlways)
+ F(TruncExisting)
+ F(ForAppend)
+ F(Transient)
+ F(CloseOnExec)
+
+ F(Temp)
+ F(Sync)
+ F(Direct)
+ F(DirectAligned)
+ F(Seq)
+ F(NoReuse)
+ F(NoReadAhead)
+
+ F(AX)
+ F(AR)
+ F(AW)
+ F(ARW)
+
+ F(AXOther)
+ F(AWOther)
+ F(AROther)
+ F(AXGroup)
+ F(AWGroup)
+ F(ARGroup)
+ F(AXUser)
+ F(AWUser)
+ F(ARUser)
+
+#undef F
+
+ if (mode != 0) {
+ if (r) {
+ r << TStringBuf("|");
+ }
+
+ r << Hex(mode);
+ }
+
+ if (!r) {
+ return "0";
+ }
+
+ return r;
+}
+
+class TFile::TImpl: public TAtomicRefCount<TImpl> {
+public:
+ inline TImpl(FHANDLE fd, const TString& fname = TString())
+ : Handle_(fd)
+ , FileName_(fname)
+ {
+ }
+
+ inline TImpl(const TString& fName, EOpenMode oMode)
+ : Handle_(fName, oMode)
+ , FileName_(fName)
+ {
+ if (!Handle_.IsOpen()) {
+ ythrow TFileError() << "can't open " << fName.Quote() << " with mode " << DecodeOpenMode(oMode) << " (" << Hex(oMode.ToBaseType()) << ")";
+ }
+ }
+
+ inline ~TImpl() = default;
+
+ inline void Close() {
+ if (!Handle_.Close()) {
+ ythrow TFileError() << "can't close " << FileName_.Quote();
+ }
+ }
+
+ const TString& GetName() const noexcept {
+ return FileName_;
+ }
+
+ void SetName(const TString& newName) {
+ FileName_ = newName;
+ }
+
+ const TFileHandle& GetHandle() const noexcept {
+ return Handle_;
+ }
+
+ i64 Seek(i64 offset, SeekDir origin) {
+ i64 pos = Handle_.Seek(offset, origin);
+ if (pos == -1L) {
+ ythrow TFileError() << "can't seek " << offset << " bytes in " << FileName_.Quote();
+ }
+ return pos;
+ }
+
+ void Resize(i64 length) {
+ if (!Handle_.Resize(length)) {
+ ythrow TFileError() << "can't resize " << FileName_.Quote() << " to size " << length;
+ }
+ }
+
+ void Reserve(i64 length) {
+ if (!Handle_.Reserve(length)) {
+ ythrow TFileError() << "can't reserve " << length << " for file " << FileName_.Quote();
+ }
+ }
+
+ void FallocateNoResize(i64 length) {
+ if (!Handle_.FallocateNoResize(length)) {
+ ythrow TFileError() << "can't allocate " << length << "bytes of space for file " << FileName_.Quote();
+ }
+ }
+
+ void ShrinkToFit() {
+ if (!Handle_.ShrinkToFit()) {
+ ythrow TFileError() << "can't shrink " << FileName_.Quote() << " to logical size";
+ }
+ }
+
+ void Flush() {
+ if (!Handle_.Flush()) {
+ ythrow TFileError() << "can't flush " << FileName_.Quote();
+ }
+ }
+
+ void FlushData() {
+ if (!Handle_.FlushData()) {
+ ythrow TFileError() << "can't flush data " << FileName_.Quote();
+ }
+ }
+
+ TFile Duplicate() const {
+ TFileHandle dupH(Handle_.Duplicate());
+ if (!dupH.IsOpen()) {
+ ythrow TFileError() << "can't duplicate the handle of " << FileName_.Quote();
+ }
+ TFile res(dupH);
+ dupH.Release();
+ return res;
+ }
+
+ // Maximum amount of bytes to be read via single system call.
+ // Some libraries fail when it is greater than max int.
+ // Syscalls can cause contention if they operate on very large data blocks.
+ static constexpr size_t MaxReadPortion = 1_GB;
+
+ i32 RawRead(void* bufferIn, size_t numBytes) {
+ const size_t toRead = Min(MaxReadPortion, numBytes);
+ return Handle_.Read(bufferIn, toRead);
+ }
+
+ size_t ReadOrFail(void* buf, size_t numBytes) {
+ const i32 reallyRead = RawRead(buf, numBytes);
+
+ if (reallyRead < 0) {
+ ythrow TFileError() << "can not read data from " << FileName_.Quote();
+ }
+
+ return reallyRead;
+ }
+
+ size_t Read(void* bufferIn, size_t numBytes) {
+ ui8* buf = (ui8*)bufferIn;
+
+ while (numBytes) {
+ const size_t reallyRead = ReadOrFail(buf, numBytes);
+
+ if (reallyRead == 0) {
+ // file exhausted
+ break;
+ }
+
+ buf += reallyRead;
+ numBytes -= reallyRead;
+ }
+
+ return buf - (ui8*)bufferIn;
+ }
+
+ void Load(void* buf, size_t len) {
+ if (Read(buf, len) != len) {
+ ythrow TFileError() << "can't read " << len << " bytes from " << FileName_.Quote();
+ }
+ }
+
+ // Maximum amount of bytes to be written via single system call.
+ // Some libraries fail when it is greater than max int.
+ // Syscalls can cause contention if they operate on very large data blocks.
+ static constexpr size_t MaxWritePortion = 1_GB;
+
+ void Write(const void* buffer, size_t numBytes) {
+ const ui8* buf = (const ui8*)buffer;
+
+ while (numBytes) {
+ const i32 toWrite = (i32)Min(MaxWritePortion, numBytes);
+ const i32 reallyWritten = Handle_.Write(buf, toWrite);
+
+ if (reallyWritten < 0) {
+ ythrow TFileError() << "can't write " << toWrite << " bytes to " << FileName_.Quote();
+ }
+
+ buf += reallyWritten;
+ numBytes -= reallyWritten;
+ }
+ }
+
+ size_t Pread(void* bufferIn, size_t numBytes, i64 offset) const {
+ ui8* buf = (ui8*)bufferIn;
+
+ while (numBytes) {
+ const i32 toRead = (i32)Min(MaxReadPortion, numBytes);
+ const i32 reallyRead = RawPread(buf, toRead, offset);
+
+ if (reallyRead < 0) {
+ ythrow TFileError() << "can not read data from " << FileName_.Quote();
+ }
+
+ if (reallyRead == 0) {
+ // file exausted
+ break;
+ }
+
+ buf += reallyRead;
+ offset += reallyRead;
+ numBytes -= reallyRead;
+ }
+
+ return buf - (ui8*)bufferIn;
+ }
+
+ i32 RawPread(void* buf, ui32 len, i64 offset) const {
+ return Handle_.Pread(buf, len, offset);
+ }
+
+ void Pload(void* buf, size_t len, i64 offset) const {
+ if (Pread(buf, len, offset) != len) {
+ ythrow TFileError() << "can't read " << len << " bytes at offset " << offset << " from " << FileName_.Quote();
+ }
+ }
+
+ void Pwrite(const void* buffer, size_t numBytes, i64 offset) const {
+ const ui8* buf = (const ui8*)buffer;
+
+ while (numBytes) {
+ const i32 toWrite = (i32)Min(MaxWritePortion, numBytes);
+ const i32 reallyWritten = Handle_.Pwrite(buf, toWrite, offset);
+
+ if (reallyWritten < 0) {
+ ythrow TFileError() << "can't write " << toWrite << " bytes to " << FileName_.Quote();
+ }
+
+ buf += reallyWritten;
+ offset += reallyWritten;
+ numBytes -= reallyWritten;
+ }
+ }
+
+ void Flock(int op) {
+ if (0 != Handle_.Flock(op)) {
+ ythrow TFileError() << "can't flock " << FileName_.Quote();
+ }
+ }
+
+ void SetDirect() {
+ if (!Handle_.SetDirect()) {
+ ythrow TFileError() << "can't set direct mode for " << FileName_.Quote();
+ }
+ }
+
+ void ResetDirect() {
+ Handle_.ResetDirect();
+ }
+
+ i64 CountCache(i64 offset, i64 length) const noexcept {
+ return Handle_.CountCache(offset, length);
+ }
+
+ void PrefetchCache(i64 offset, i64 length, bool wait) const noexcept {
+ Handle_.PrefetchCache(offset, length, wait);
+ }
+
+ void EvictCache(i64 offset, i64 length) const noexcept {
+ Handle_.EvictCache(offset, length);
+ }
+
+ void FlushCache(i64 offset, i64 length, bool wait) {
+ if (!Handle_.FlushCache(offset, length, wait)) {
+ ythrow TFileError() << "can't flush data " << FileName_.Quote();
+ }
+ }
+
+private:
+ TFileHandle Handle_;
+ TString FileName_;
+};
+
+TFile::TFile()
+ : Impl_(new TImpl(INVALID_FHANDLE))
+{
+}
+
+TFile::TFile(FHANDLE fd)
+ : Impl_(new TImpl(fd))
+{
+}
+
+TFile::TFile(FHANDLE fd, const TString& name)
+ : Impl_(new TImpl(fd, name))
+{
+}
+
+TFile::TFile(const TString& fName, EOpenMode oMode)
+ : Impl_(new TImpl(fName, oMode))
+{
+}
+
+TFile::~TFile() = default;
+
+void TFile::Close() {
+ Impl_->Close();
+}
+
+const TString& TFile::GetName() const noexcept {
+ return Impl_->GetName();
+}
+
+i64 TFile::GetPosition() const noexcept {
+ return Impl_->GetHandle().GetPosition();
+}
+
+i64 TFile::GetLength() const noexcept {
+ return Impl_->GetHandle().GetLength();
+}
+
+bool TFile::IsOpen() const noexcept {
+ return Impl_->GetHandle().IsOpen();
+}
+
+FHANDLE TFile::GetHandle() const noexcept {
+ return Impl_->GetHandle();
+}
+
+i64 TFile::Seek(i64 offset, SeekDir origin) {
+ return Impl_->Seek(offset, origin);
+}
+
+void TFile::Resize(i64 length) {
+ Impl_->Resize(length);
+}
+
+void TFile::Reserve(i64 length) {
+ Impl_->Reserve(length);
+}
+
+void TFile::FallocateNoResize(i64 length) {
+ Impl_->FallocateNoResize(length);
+}
+
+void TFile::ShrinkToFit() {
+ Impl_->ShrinkToFit();
+}
+
+void TFile::Flush() {
+ Impl_->Flush();
+}
+
+void TFile::FlushData() {
+ Impl_->FlushData();
+}
+
+TFile TFile::Duplicate() const {
+ TFile res = Impl_->Duplicate();
+ res.Impl_->SetName(Impl_->GetName());
+ return res;
+}
+
+size_t TFile::Read(void* buf, size_t len) {
+ return Impl_->Read(buf, len);
+}
+
+i32 TFile::RawRead(void* buf, size_t len) {
+ return Impl_->RawRead(buf, len);
+}
+
+size_t TFile::ReadOrFail(void* buf, size_t len) {
+ return Impl_->ReadOrFail(buf, len);
+}
+
+void TFile::Load(void* buf, size_t len) {
+ Impl_->Load(buf, len);
+}
+
+void TFile::Write(const void* buf, size_t len) {
+ Impl_->Write(buf, len);
+}
+
+size_t TFile::Pread(void* buf, size_t len, i64 offset) const {
+ return Impl_->Pread(buf, len, offset);
+}
+
+i32 TFile::RawPread(void* buf, ui32 len, i64 offset) const {
+ return Impl_->RawPread(buf, len, offset);
+}
+
+void TFile::Pload(void* buf, size_t len, i64 offset) const {
+ Impl_->Pload(buf, len, offset);
+}
+
+void TFile::Pwrite(const void* buf, size_t len, i64 offset) const {
+ Impl_->Pwrite(buf, len, offset);
+}
+
+void TFile::Flock(int op) {
+ Impl_->Flock(op);
+}
+
+void TFile::SetDirect() {
+ Impl_->SetDirect();
+}
+
+void TFile::ResetDirect() {
+ Impl_->ResetDirect();
+}
+
+i64 TFile::CountCache(i64 offset, i64 length) const noexcept {
+ return Impl_->CountCache(offset, length);
+}
+
+void TFile::PrefetchCache(i64 offset, i64 length, bool wait) const noexcept {
+ Impl_->PrefetchCache(offset, length, wait);
+}
+
+void TFile::EvictCache(i64 offset, i64 length) const noexcept {
+ Impl_->EvictCache(offset, length);
+}
+
+void TFile::FlushCache(i64 offset, i64 length, bool wait) {
+ Impl_->FlushCache(offset, length, wait);
+}
+
+void TFile::LinkTo(const TFile& f) const {
+ if (!Impl_->GetHandle().LinkTo(f.Impl_->GetHandle())) {
+ ythrow TFileError() << "can not link fd(" << GetName() << " -> " << f.GetName() << ")";
+ }
+}
+
+TFile TFile::Temporary(const TString& prefix) {
+ //TODO - handle impossible case of name collision
+ return TFile(prefix + ToString(MicroSeconds()) + "-" + ToString(RandomNumber<ui64>()), CreateNew | RdWr | Seq | Temp | Transient);
+}
+
+TFile TFile::ForAppend(const TString& path) {
+ return TFile(path, OpenAlways | WrOnly | Seq | ::ForAppend);
+}
+
+TFile Duplicate(FILE* f) {
+ return Duplicate(fileno(f));
+}
+
+TFile Duplicate(int fd) {
+#if defined(_win_)
+ /* There are two options of how to duplicate a file descriptor on Windows:
+ *
+ * 1:
+ * - Call dup.
+ * - Call _get_osfhandle on the result.
+ * - Use returned handle.
+ * - Call _close on file descriptor returned by dup. This will also close
+ * the handle.
+ *
+ * 2:
+ * - Call _get_osfhandle.
+ * - Call DuplicateHandle on the result.
+ * - Use returned handle.
+ * - Call CloseHandle.
+ *
+ * TFileHandle calls CloseHandle when destroyed, leaving us with option #2. */
+ FHANDLE handle = reinterpret_cast<FHANDLE>(::_get_osfhandle(fd));
+
+ FHANDLE dupHandle;
+ if (!::DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), &dupHandle, 0, TRUE, DUPLICATE_SAME_ACCESS)) {
+ ythrow TFileError() << "can not duplicate file descriptor " << LastSystemError() << Endl;
+ }
+
+ return TFile(dupHandle);
+#elif defined(_unix_)
+ return TFile(::dup(fd));
+#else
+ #error unsupported platform
+#endif
+}
+
+bool PosixDisableReadAhead(FHANDLE fileHandle, void* addr) noexcept {
+ int ret = -1;
+
+#if HAVE_POSIX_FADVISE
+ #if defined(_linux_)
+ Y_UNUSED(fileHandle);
+ ret = madvise(addr, 0, MADV_RANDOM); // according to klamm@ posix_fadvise does not work under linux, madvise does work
+ #else
+ Y_UNUSED(addr);
+ ret = ::posix_fadvise(fileHandle, 0, 0, POSIX_FADV_RANDOM);
+ #endif
+#else
+ Y_UNUSED(fileHandle);
+ Y_UNUSED(addr);
+#endif
+ return ret == 0;
+}
diff --git a/util/system/file.h b/util/system/file.h
new file mode 100644
index 0000000000..9502e159b6
--- /dev/null
+++ b/util/system/file.h
@@ -0,0 +1,225 @@
+#pragma once
+
+#include "fhandle.h"
+#include "flock.h"
+
+#include <util/generic/flags.h>
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+
+#include <cstdio>
+
+enum EOpenModeFlag {
+ OpenExisting = 0, // Opens a file. It fails if the file does not exist.
+ TruncExisting = 1, // Opens a file and truncates it to zero size. It fails if the file does not exist.
+ OpenAlways = 2, // Opens a file, always. If a file does not exist, it creates a file.
+ CreateNew = 3, // Creates a new file. It fails if a specified file exists.
+ CreateAlways = 4, // Creates a new file, always. If a file exists, it overwrites the file.
+ MaskCreation = 7,
+
+ RdOnly = 8, // open for reading only
+ WrOnly = 16, // open for writing only
+ RdWr = 24, // open for reading and writing
+ MaskRW = 24,
+
+ Seq = 0x20, // file access is primarily sequential (POSIX_FADV_SEQUENTIAL)
+ Direct = 0x40, // file is being opened with no system caching (Does not work as intended! See implementation)
+ Temp = 0x80, // avoid writing data back to disk if sufficient cache memory is available (no op for linux)
+ ForAppend = 0x100, // write appends data to the end of file (O_APPEND)
+ Transient = 0x200, // actually, temporary file - 'delete on close' for windows, unlink after creation for unix
+ NoReuse = 0x400, // no second access expected (POSIX_FADV_NOREUSE)
+ CloseOnExec = 0x800, // set close-on-exec right at open (O_CLOEXEC)
+ DirectAligned = 0x1000, // file is actually being opened with no system caching (may require buffer alignment) (O_DIRECT)
+ Sync = 0x2000, // no write call will return before the data is transferred to the disk (O_SYNC)
+ NoReadAhead = 0x4000, // no sequential access expected, opposite for Seq (POSIX_FADV_RANDOM)
+
+ AXOther = 0x00010000,
+ AWOther = 0x00020000,
+ AROther = 0x00040000,
+ AXGroup = 0x00100000,
+ AWGroup = 0x00200000,
+ ARGroup = 0x00400000,
+ AXUser = 0x01000000,
+ AWUser = 0x02000000,
+ ARUser = 0x04000000,
+ AX = AXUser | AXGroup | AXOther,
+ AW = AWUser | AWGroup,
+ AR = ARUser | ARGroup | AROther,
+ ARW = AR | AW,
+ AMask = 0x0FFF0000,
+};
+
+Y_DECLARE_FLAGS(EOpenMode, EOpenModeFlag)
+Y_DECLARE_OPERATORS_FOR_FLAGS(EOpenMode)
+
+TString DecodeOpenMode(ui32 openMode);
+
+enum SeekDir {
+ sSet = 0,
+ sCur = 1,
+ sEnd = 2,
+};
+
+class TFileHandle: public TNonCopyable {
+public:
+ constexpr TFileHandle() = default;
+
+ /// Warning: takes ownership of fd, so closes it in destructor.
+ inline TFileHandle(FHANDLE fd) noexcept
+ : Fd_(fd)
+ {
+ }
+
+ inline TFileHandle(TFileHandle&& other) noexcept
+ : Fd_(other.Fd_)
+ {
+ other.Fd_ = INVALID_FHANDLE;
+ }
+
+ TFileHandle(const TString& fName, EOpenMode oMode) noexcept;
+
+ inline ~TFileHandle() {
+ Close();
+ }
+
+ bool Close() noexcept;
+
+ inline FHANDLE Release() noexcept {
+ FHANDLE ret = Fd_;
+ Fd_ = INVALID_FHANDLE;
+ return ret;
+ }
+
+ inline void Swap(TFileHandle& r) noexcept {
+ DoSwap(Fd_, r.Fd_);
+ }
+
+ inline operator FHANDLE() const noexcept {
+ return Fd_;
+ }
+
+ inline bool IsOpen() const noexcept {
+ return Fd_ != INVALID_FHANDLE;
+ }
+
+ i64 GetPosition() const noexcept;
+ i64 GetLength() const noexcept;
+
+ i64 Seek(i64 offset, SeekDir origin) noexcept;
+ bool Resize(i64 length) noexcept;
+ bool Reserve(i64 length) noexcept;
+ bool FallocateNoResize(i64 length) noexcept;
+ bool ShrinkToFit() noexcept;
+ bool Flush() noexcept;
+ //flush data only, without file metadata
+ bool FlushData() noexcept;
+ i32 Read(void* buffer, ui32 byteCount) noexcept;
+ i32 Write(const void* buffer, ui32 byteCount) noexcept;
+ i32 Pread(void* buffer, ui32 byteCount, i64 offset) const noexcept;
+ i32 Pwrite(const void* buffer, ui32 byteCount, i64 offset) const noexcept;
+ int Flock(int op) noexcept;
+
+ FHANDLE Duplicate() const noexcept;
+ int Duplicate2Posix(int dstHandle) const noexcept;
+
+ //dup2 - like semantics, return true on success
+ bool LinkTo(const TFileHandle& fh) const noexcept;
+
+ //very low-level methods
+ bool SetDirect();
+ void ResetDirect();
+
+ /* Manual file cache management, length = 0 means "as much as possible" */
+
+ //measure amount of cached data in bytes, returns -1 if failed
+ i64 CountCache(i64 offset = 0, i64 length = 0) const noexcept;
+ //read data into cache and optionally wait for completion
+ void PrefetchCache(i64 offset = 0, i64 length = 0, bool wait = true) const noexcept;
+ //remove clean and unused data from cache
+ void EvictCache(i64 offset = 0, i64 length = 0) const noexcept;
+ //flush unwritten data in this range and optionally wait for completion
+ bool FlushCache(i64 offset = 0, i64 length = 0, bool wait = true) noexcept;
+
+private:
+ FHANDLE Fd_ = INVALID_FHANDLE;
+};
+
+class TFile {
+public:
+ TFile();
+ /// Takes ownership of handle, so closes it when the last holder of descriptor dies.
+ explicit TFile(FHANDLE fd);
+ TFile(FHANDLE fd, const TString& fname);
+ TFile(const TString& fName, EOpenMode oMode);
+ ~TFile();
+
+ void Close();
+
+ const TString& GetName() const noexcept;
+ i64 GetPosition() const noexcept;
+ i64 GetLength() const noexcept;
+ bool IsOpen() const noexcept;
+ FHANDLE GetHandle() const noexcept;
+
+ i64 Seek(i64 offset, SeekDir origin);
+ void Resize(i64 length);
+ void Reserve(i64 length);
+ void FallocateNoResize(i64 length);
+ void ShrinkToFit();
+ void Flush();
+ void FlushData();
+
+ void LinkTo(const TFile& f) const;
+ TFile Duplicate() const;
+
+ // Reads up to 1 GB without retrying, returns -1 on error
+ i32 RawRead(void* buf, size_t len);
+ // Reads up to 1 GB without retrying, throws on error
+ size_t ReadOrFail(void* buf, size_t len);
+ // Retries incomplete reads until EOF, throws on error
+ size_t Read(void* buf, size_t len);
+ // Reads exactly len bytes, throws on premature EOF or error
+ void Load(void* buf, size_t len);
+
+ // Retries incomplete writes, will either write len bytes or throw
+ void Write(const void* buf, size_t len);
+
+ // Retries incomplete reads until EOF, throws on error
+ size_t Pread(void* buf, size_t len, i64 offset) const;
+ // Single pread call
+ i32 RawPread(void* buf, ui32 len, i64 offset) const;
+ // Reads exactly len bytes, throws on premature EOF or error
+ void Pload(void* buf, size_t len, i64 offset) const;
+
+ // Retries incomplete writes, will either write len bytes or throw
+ void Pwrite(const void* buf, size_t len, i64 offset) const;
+
+ void Flock(int op);
+
+ //do not use, their meaning very platform-dependant
+ void SetDirect();
+ void ResetDirect();
+
+ /* Manual file cache management, length = 0 means "as much as possible" */
+
+ //measure amount of cached data in bytes, returns -1 if failed
+ i64 CountCache(i64 offset = 0, i64 length = 0) const noexcept;
+ //read data into cache and optionally wait for completion
+ void PrefetchCache(i64 offset = 0, i64 length = 0, bool wait = true) const noexcept;
+ //remove clean and unused data from cache, incomplete pages could stay
+ void EvictCache(i64 offset = 0, i64 length = 0) const noexcept;
+ //flush unwritten data in this range and optionally wait for completion
+ void FlushCache(i64 offset = 0, i64 length = 0, bool wait = true);
+
+ static TFile Temporary(const TString& prefix);
+ static TFile ForAppend(const TString& path);
+
+private:
+ class TImpl;
+ TSimpleIntrusivePtr<TImpl> Impl_;
+};
+
+TFile Duplicate(FILE*);
+TFile Duplicate(int);
+
+bool PosixDisableReadAhead(FHANDLE fileHandle, void* addr) noexcept;
diff --git a/util/system/file_lock.cpp b/util/system/file_lock.cpp
new file mode 100644
index 0000000000..45d91282c5
--- /dev/null
+++ b/util/system/file_lock.cpp
@@ -0,0 +1,46 @@
+#include "file_lock.h"
+#include "flock.h"
+
+#include <util/generic/yexception.h>
+
+#include <cerrno>
+
+namespace {
+ int GetMode(const EFileLockType type) {
+ switch (type) {
+ case EFileLockType::Exclusive:
+ return LOCK_EX;
+ case EFileLockType::Shared:
+ return LOCK_SH;
+ default:
+ Y_UNREACHABLE();
+ }
+ Y_UNREACHABLE();
+ }
+}
+
+TFileLock::TFileLock(const TString& filename, const EFileLockType type)
+ : TFile(filename, OpenAlways | RdOnly)
+ , Type(type)
+{
+}
+
+void TFileLock::Acquire() {
+ Flock(GetMode(Type));
+}
+
+bool TFileLock::TryAcquire() {
+ try {
+ Flock(GetMode(Type) | LOCK_NB);
+ return true;
+ } catch (const TSystemError& e) {
+ if (e.Status() != EWOULDBLOCK) {
+ throw;
+ }
+ return false;
+ }
+}
+
+void TFileLock::Release() {
+ Flock(LOCK_UN);
+}
diff --git a/util/system/file_lock.h b/util/system/file_lock.h
new file mode 100644
index 0000000000..b2aaff5baf
--- /dev/null
+++ b/util/system/file_lock.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+#include <util/generic/noncopyable.h>
+#include <util/system/file.h>
+
+enum class EFileLockType {
+ Exclusive,
+ Shared
+};
+
+class TFileLock: public TFile {
+public:
+ TFileLock(const TString& filename, const EFileLockType type = EFileLockType::Exclusive);
+
+ void Acquire();
+ bool TryAcquire();
+ void Release();
+
+ inline void lock() {
+ Acquire();
+ }
+
+ inline bool try_lock() {
+ return TryAcquire();
+ }
+
+ inline void unlock() {
+ Release();
+ }
+
+private:
+ EFileLockType Type;
+};
diff --git a/util/system/file_ut.cpp b/util/system/file_ut.cpp
new file mode 100644
index 0000000000..941e6a50f3
--- /dev/null
+++ b/util/system/file_ut.cpp
@@ -0,0 +1,416 @@
+#include "file.h"
+#include "fs.h"
+#include "tempfile.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/file.h>
+#include <util/generic/yexception.h>
+
+class TFileTest: public TTestBase {
+ UNIT_TEST_SUITE(TFileTest);
+ UNIT_TEST(TestOpen);
+ UNIT_TEST(TestOpenSync);
+ UNIT_TEST(TestRW);
+ UNIT_TEST(TestReWrite);
+ UNIT_TEST(TestAppend);
+ UNIT_TEST(TestLinkTo);
+ UNIT_TEST(TestResize);
+ UNIT_TEST(TestLocale);
+ UNIT_TEST(TestFlush);
+ UNIT_TEST(TestFlushSpecialFile);
+ UNIT_TEST(TestRawRead);
+ UNIT_TEST(TestRead);
+ UNIT_TEST(TestRawPread);
+ UNIT_TEST(TestPread);
+ UNIT_TEST(TestCache);
+ UNIT_TEST_SUITE_END();
+
+public:
+ void TestOpen();
+ void TestOpenSync();
+ void TestRW();
+ void TestLocale();
+ void TestFlush();
+ void TestFlushSpecialFile();
+ void TestRawRead();
+ void TestRead();
+ void TestRawPread();
+ void TestPread();
+ void TestCache();
+
+ inline void TestLinkTo() {
+ TTempFile tmp1("tmp1");
+ TTempFile tmp2("tmp2");
+
+ {
+ TFile f1(tmp1.Name(), OpenAlways | WrOnly);
+ TFile f2(tmp2.Name(), OpenAlways | WrOnly);
+
+ f1.LinkTo(f2);
+
+ f1.Write("12345", 5);
+ f2.Write("67890", 5);
+ }
+
+ UNIT_ASSERT_EQUAL(TUnbufferedFileInput(tmp2.Name()).ReadAll(), "1234567890");
+ }
+
+ inline void TestAppend() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile f(tmp.Name(), OpenAlways | WrOnly);
+
+ f.Write("12345678", 8);
+ }
+
+ {
+ TFile f(tmp.Name(), OpenAlways | WrOnly | ForAppend);
+
+ f.Write("67", 2);
+ f.Write("89", 2);
+ }
+
+ UNIT_ASSERT_EQUAL(TUnbufferedFileInput(tmp.Name()).ReadAll(), "123456786789");
+ }
+
+ inline void TestReWrite() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile f(tmp.Name(), OpenAlways | WrOnly);
+
+ f.Write("12345678", 8);
+ }
+
+ {
+ TFile f(tmp.Name(), OpenAlways | WrOnly);
+
+ f.Write("6789", 4);
+ }
+
+ UNIT_ASSERT_EQUAL(TUnbufferedFileInput(tmp.Name()).ReadAll(), "67895678");
+ }
+
+ inline void TestResize() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile file(tmp.Name(), OpenAlways | WrOnly);
+
+ file.Write("1234567", 7);
+ file.Seek(3, sSet);
+
+ file.Resize(5);
+ UNIT_ASSERT_EQUAL(file.GetLength(), 5);
+ UNIT_ASSERT_EQUAL(file.GetPosition(), 3);
+
+ file.Resize(12);
+ UNIT_ASSERT_EQUAL(file.GetLength(), 12);
+ UNIT_ASSERT_EQUAL(file.GetPosition(), 3);
+ }
+
+ const TString data = TUnbufferedFileInput(tmp.Name()).ReadAll();
+ UNIT_ASSERT_EQUAL(data.length(), 12);
+ UNIT_ASSERT(data.StartsWith("12345"));
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TFileTest);
+
+void TFileTest::TestOpen() {
+ TString res;
+ TFile f1;
+
+ try {
+ TFile f2("f1.txt", OpenExisting);
+ } catch (const yexception& e) {
+ res = e.what();
+ }
+ UNIT_ASSERT(!res.empty());
+ res.remove();
+
+ try {
+ TFile f2("f1.txt", OpenAlways);
+ f1 = f2;
+ } catch (const yexception& e) {
+ res = e.what();
+ }
+ UNIT_ASSERT(res.empty());
+ UNIT_ASSERT(f1.IsOpen());
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetName(), "f1.txt");
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetLength(), 0);
+
+ try {
+ TFile f2("f1.txt", CreateNew);
+ } catch (const yexception& e) {
+ res = e.what();
+ }
+ UNIT_ASSERT(!res.empty());
+ res.remove();
+
+ f1.Close();
+ UNIT_ASSERT(unlink("f1.txt") == 0);
+}
+
+void TFileTest::TestOpenSync() {
+ TFile f1("f1.txt", CreateNew | Sync);
+ UNIT_ASSERT(f1.IsOpen());
+ f1.Close();
+ UNIT_ASSERT(!f1.IsOpen());
+ UNIT_ASSERT(unlink("f1.txt") == 0);
+}
+
+void TFileTest::TestRW() {
+ TFile f1("f1.txt", CreateNew);
+ UNIT_ASSERT(f1.IsOpen());
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetName(), "f1.txt");
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetLength(), 0);
+ ui32 d[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ f1.Write(&d, sizeof(ui32) * 10);
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetLength(), 40);
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetPosition(), 40);
+ UNIT_ASSERT_VALUES_EQUAL(f1.Seek(12, sSet), 12);
+ f1.Flush();
+ ui32 v;
+ f1.Load(&v, sizeof(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 3u);
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetPosition(), 16);
+
+ TFile f2 = f1;
+ UNIT_ASSERT(f2.IsOpen());
+ UNIT_ASSERT_VALUES_EQUAL(f2.GetName(), "f1.txt");
+ UNIT_ASSERT_VALUES_EQUAL(f2.GetPosition(), 16);
+ UNIT_ASSERT_VALUES_EQUAL(f2.GetLength(), 40);
+ f2.Write(&v, sizeof(v));
+
+ UNIT_ASSERT_VALUES_EQUAL(f1.GetPosition(), 20);
+ UNIT_ASSERT_VALUES_EQUAL(f1.Seek(-4, sCur), 16);
+ v = 0;
+ f1.Load(&v, sizeof(v));
+ UNIT_ASSERT_VALUES_EQUAL(v, 3u);
+ f1.Close();
+ UNIT_ASSERT(!f1.IsOpen());
+ UNIT_ASSERT(!f2.IsOpen());
+ UNIT_ASSERT(unlink("f1.txt") == 0);
+}
+
+#ifdef _unix_
+ #include <locale.h>
+#endif
+
+void TFileTest::TestLocale() {
+#ifdef _unix_
+ const char* loc = setlocale(LC_CTYPE, nullptr);
+ setlocale(LC_CTYPE, "ru_RU.UTF-8");
+#endif
+ TFile f("Имя.txt", CreateNew);
+ UNIT_ASSERT(f.IsOpen());
+ UNIT_ASSERT_VALUES_EQUAL(f.GetName(), "Имя.txt");
+ UNIT_ASSERT_VALUES_EQUAL(f.GetLength(), 0);
+ f.Close();
+ UNIT_ASSERT(NFs::Remove("Имя.txt"));
+#ifdef _unix_
+ setlocale(LC_CTYPE, loc);
+#endif
+}
+
+void TFileTest::TestFlush() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile f(tmp.Name(), OpenAlways | WrOnly);
+ f.Flush();
+ f.FlushData();
+ f.Close();
+
+ UNIT_ASSERT_EXCEPTION(f.Flush(), TFileError);
+ UNIT_ASSERT_EXCEPTION(f.FlushData(), TFileError);
+ }
+}
+
+void TFileTest::TestFlushSpecialFile() {
+#ifdef _unix_
+ TFile devNull("/dev/null", WrOnly);
+ devNull.FlushData();
+ devNull.Flush();
+ devNull.Close();
+#endif
+}
+
+void TFileTest::TestRawRead() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile file(tmp.Name(), OpenAlways | WrOnly);
+ file.Write("1234567", 7);
+ file.Flush();
+ file.Close();
+ }
+
+ {
+ TFile file(tmp.Name(), OpenExisting | RdOnly);
+ char buf[7];
+ i32 reallyRead = file.RawRead(buf, 7);
+ Y_ENSURE(0 <= reallyRead && reallyRead <= 7);
+ Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("1234567").Head(reallyRead));
+ }
+}
+
+void TFileTest::TestRead() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile file(tmp.Name(), OpenAlways | WrOnly);
+ file.Write("1234567", 7);
+ file.Flush();
+ file.Close();
+ }
+
+ {
+ TFile file(tmp.Name(), OpenExisting | RdOnly);
+ char buf[7];
+ Y_ENSURE(file.Read(buf, 7) == 7);
+ Y_ENSURE(TStringBuf(buf, 7) == "1234567");
+
+ memset(buf, 0, sizeof(buf));
+ file.Seek(0, sSet);
+ Y_ENSURE(file.Read(buf, 123) == 7);
+ Y_ENSURE(TStringBuf(buf, 7) == "1234567");
+ }
+}
+
+void TFileTest::TestRawPread() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile file(tmp.Name(), OpenAlways | WrOnly);
+ file.Write("1234567", 7);
+ file.Flush();
+ file.Close();
+ }
+
+ {
+ TFile file(tmp.Name(), OpenExisting | RdOnly);
+ char buf[7];
+ i32 reallyRead = file.RawPread(buf, 3, 1);
+ Y_ENSURE(0 <= reallyRead && reallyRead <= 3);
+ Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("234").Head(reallyRead));
+
+ memset(buf, 0, sizeof(buf));
+ reallyRead = file.RawPread(buf, 2, 5);
+ Y_ENSURE(0 <= reallyRead && reallyRead <= 2);
+ Y_ENSURE(TStringBuf(buf, reallyRead) == TStringBuf("67").Head(reallyRead));
+ }
+}
+
+void TFileTest::TestPread() {
+ TTempFile tmp("tmp");
+
+ {
+ TFile file(tmp.Name(), OpenAlways | WrOnly);
+ file.Write("1234567", 7);
+ file.Flush();
+ file.Close();
+ }
+
+ {
+ TFile file(tmp.Name(), OpenExisting | RdOnly);
+ char buf[7];
+ Y_ENSURE(file.Pread(buf, 3, 1) == 3);
+ Y_ENSURE(TStringBuf(buf, 3) == "234");
+
+ memset(buf, 0, sizeof(buf));
+ Y_ENSURE(file.Pread(buf, 2, 5) == 2);
+ Y_ENSURE(TStringBuf(buf, 2) == "67");
+ }
+}
+
+#ifdef _linux_
+ #include <sys/statfs.h>
+#endif
+
+#ifndef TMPFS_MAGIC
+ #define TMPFS_MAGIC 0x01021994
+#endif
+
+void TFileTest::TestCache(){
+#ifdef _linux_
+ {// create file in /tmp, current dir could be tmpfs which does not support fadvise
+ TFile file(MakeTempName("/tmp"), OpenAlways | Transient | RdWr | NoReadAhead);
+
+struct statfs fs;
+if (!fstatfs(file.GetHandle(), &fs) && fs.f_type == TMPFS_MAGIC) {
+ return;
+}
+
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 0);
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(0, 0), 0);
+
+file.Resize(7);
+file.PrefetchCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 7);
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(3, 2), 2);
+
+file.FlushCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 7);
+
+file.EvictCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 0);
+
+file.PrefetchCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 7);
+
+file.Resize(12345);
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 4096);
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(4096, 0), 0);
+
+file.PrefetchCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 12345);
+
+file.FlushCache();
+file.EvictCache();
+UNIT_ASSERT_LE(file.CountCache(), 0);
+
+file.Resize(33333333);
+file.PrefetchCache(11111111, 11111111);
+UNIT_ASSERT_GE(file.CountCache(), 11111111);
+
+UNIT_ASSERT_LE(file.CountCache(0, 11111111), 1111111);
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(11111111, 11111111), 11111111);
+UNIT_ASSERT_LE(file.CountCache(22222222, 11111111), 1111111);
+
+file.FlushCache(11111111, 11111111);
+UNIT_ASSERT_GE(file.CountCache(), 11111111);
+
+// first and last incomplete pages could stay in cache
+file.EvictCache(11111111, 11111111);
+UNIT_ASSERT_LT(file.CountCache(11111111, 11111111), 4096 * 2);
+
+file.EvictCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), 0);
+}
+#else
+ {TFile file(MakeTempName(), OpenAlways | Transient | RdWr);
+
+file.Resize(12345);
+
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(), -1);
+file.PrefetchCache();
+file.FlushCache();
+file.EvictCache();
+UNIT_ASSERT_VALUES_EQUAL(file.CountCache(0, 12345), -1);
+}
+#endif
+}
+
+Y_UNIT_TEST_SUITE(TTestDecodeOpenMode) {
+ Y_UNIT_TEST(It) {
+ UNIT_ASSERT_VALUES_EQUAL("0", DecodeOpenMode(0));
+ UNIT_ASSERT_VALUES_EQUAL("RdOnly", DecodeOpenMode(RdOnly));
+ UNIT_ASSERT_VALUES_EQUAL("RdWr", DecodeOpenMode(RdWr));
+ UNIT_ASSERT_VALUES_EQUAL("WrOnly|ForAppend", DecodeOpenMode(WrOnly | ForAppend));
+ UNIT_ASSERT_VALUES_EQUAL("RdWr|CreateAlways|CreateNew|ForAppend|Transient|CloseOnExec|Temp|Sync|Direct|DirectAligned|Seq|NoReuse|NoReadAhead|AX|AR|AW|AWOther|0xF8888000", DecodeOpenMode(0xFFFFFFFF));
+ }
+}
diff --git a/util/system/filemap.cpp b/util/system/filemap.cpp
new file mode 100644
index 0000000000..7454a4cb94
--- /dev/null
+++ b/util/system/filemap.cpp
@@ -0,0 +1,585 @@
+#include "info.h"
+#include "madvise.h"
+#include "defaults.h"
+#include "hi_lo.h"
+
+#include <util/generic/buffer.h>
+#include <util/generic/yexception.h>
+#include <util/generic/singleton.h>
+
+#if defined(_win_)
+ #include "winint.h"
+#elif defined(_unix_)
+ #include <sys/types.h>
+ #include <sys/mman.h>
+
+ #if !defined(_linux_)
+ #ifdef MAP_POPULATE
+ #error unlisted platform supporting MAP_POPULATE
+ #endif
+ #define MAP_POPULATE 0
+ #endif
+
+ #if !defined(_freebsd_)
+ #ifdef MAP_NOCORE
+ #error unlisted platform supporting MAP_NOCORE
+ #endif
+ #define MAP_NOCORE 0
+ #endif
+#else
+ #error todo
+#endif
+
+#include <util/generic/utility.h>
+#include <util/system/sanitizers.h>
+#include "filemap.h"
+
+#undef PAGE_SIZE
+#undef GRANULARITY
+
+#ifdef _win_
+ #define MAP_FAILED ((void*)(LONG_PTR)-1)
+#endif
+
+namespace {
+ struct TSysInfo {
+ inline TSysInfo()
+ : GRANULARITY_(CalcGranularity())
+ , PAGE_SIZE_(NSystemInfo::GetPageSize())
+ {
+ }
+
+ static inline const TSysInfo& Instance() {
+ return *Singleton<TSysInfo>();
+ }
+
+ static inline size_t CalcGranularity() noexcept {
+#if defined(_win_)
+ SYSTEM_INFO sysInfo;
+ GetSystemInfo(&sysInfo);
+ return sysInfo.dwAllocationGranularity;
+#else
+ return NSystemInfo::GetPageSize();
+#endif
+ }
+
+ const size_t GRANULARITY_;
+ const size_t PAGE_SIZE_;
+ };
+}
+
+#define GRANULARITY (TSysInfo::Instance().GRANULARITY_)
+#define PAGE_SIZE (TSysInfo::Instance().PAGE_SIZE_)
+
+TString TMemoryMapCommon::UnknownFileName() {
+ return "Unknown_file_name";
+}
+
+static inline i64 DownToGranularity(i64 offset) noexcept {
+ return offset & ~((i64)(GRANULARITY - 1));
+}
+
+#if defined(_unix_)
+static int ModeToMmapFlags(TMemoryMapCommon::EOpenMode mode) {
+ int flags = MAP_NOCORE;
+ if ((mode & TMemoryMapCommon::oAccessMask) == TMemoryMapCommon::oCopyOnWr) {
+ flags |= MAP_PRIVATE;
+ } else {
+ flags |= MAP_SHARED;
+ }
+ if (mode & TMemoryMapCommon::oPopulate) {
+ flags |= MAP_POPULATE;
+ }
+ return flags;
+}
+
+static int ModeToMmapProt(TMemoryMapCommon::EOpenMode mode) {
+ int prot = PROT_READ;
+ if ((mode & TMemoryMapCommon::oAccessMask) != TMemoryMapCommon::oRdOnly) {
+ prot |= PROT_WRITE;
+ }
+ return prot;
+}
+#endif
+
+// maybe we should move this function to another .cpp file to avoid unwanted optimization?
+void NPrivate::Precharge(const void* data, size_t dataSize, size_t off, size_t size) {
+ if (off > dataSize) {
+ assert(false);
+ return;
+ }
+ size_t endOff = (size == (size_t)-1 ? dataSize : off + size);
+ if (endOff > dataSize) {
+ assert(false);
+ endOff = dataSize;
+ }
+ size = endOff - off;
+ if (dataSize == 0 || size == 0) {
+ return;
+ }
+
+ volatile const char *c = (const char*)data + off, *e = c + size;
+ for (; c < e; c += 512) {
+ *c;
+ }
+}
+
+class TMemoryMap::TImpl: public TAtomicRefCount<TImpl> {
+public:
+ inline void CreateMapping() {
+#if defined(_win_)
+ Mapping_ = nullptr;
+ if (Length_) {
+ Mapping_ = CreateFileMapping(File_.GetHandle(), nullptr,
+ (Mode_ & oAccessMask) == TFileMap::oRdWr ? PAGE_READWRITE : PAGE_READONLY,
+ (DWORD)(Length_ >> 32), (DWORD)(Length_ & 0xFFFFFFFF), nullptr);
+ if (Mapping_ == nullptr) {
+ ythrow yexception() << "Can't create file mapping of '" << DbgName_ << "': " << LastSystemErrorText();
+ }
+ } else {
+ Mapping_ = MAP_FAILED;
+ }
+#elif defined(_unix_)
+ if (!(Mode_ & oNotGreedy)) {
+ PtrStart_ = mmap((caddr_t) nullptr, Length_, ModeToMmapProt(Mode_), ModeToMmapFlags(Mode_), File_.GetHandle(), 0);
+
+ if ((MAP_FAILED == PtrStart_) && Length_) {
+ ythrow yexception() << "Can't map " << (unsigned long)Length_ << " bytes of file '" << DbgName_ << "' at offset 0: " << LastSystemErrorText();
+ }
+ } else {
+ PtrStart_ = nullptr;
+ }
+#endif
+ }
+
+ void CheckFile() const {
+ if (!File_.IsOpen()) {
+ ythrow yexception() << "TMemoryMap: FILE '" << DbgName_ << "' is not open, " << LastSystemErrorText();
+ }
+ if (Length_ < 0) {
+ ythrow yexception() << "'" << DbgName_ << "' is not a regular file";
+ }
+ }
+
+ inline TImpl(FILE* f, EOpenMode om, TString dbgName)
+ : File_(Duplicate(f))
+ , DbgName_(std::move(dbgName))
+ , Length_(File_.GetLength())
+ , Mode_(om)
+ {
+ CheckFile();
+ CreateMapping();
+ }
+
+ inline TImpl(const TString& name, EOpenMode om)
+ : File_(name, (om & oRdWr) ? OpenExisting | RdWr : OpenExisting | RdOnly)
+ , DbgName_(name)
+ , Length_(File_.GetLength())
+ , Mode_(om)
+ {
+ CheckFile();
+ CreateMapping();
+ }
+
+ inline TImpl(const TString& name, i64 length, EOpenMode om)
+ : File_(name, (om & oRdWr) ? OpenExisting | RdWr : OpenExisting | RdOnly)
+ , DbgName_(name)
+ , Length_(length)
+ , Mode_(om)
+ {
+ CheckFile();
+
+ if (File_.GetLength() < Length_) {
+ File_.Resize(Length_);
+ }
+
+ CreateMapping();
+ }
+
+ inline TImpl(const TFile& file, EOpenMode om, TString dbgName)
+ : File_(file)
+ , DbgName_(File_.GetName() ? File_.GetName() : std::move(dbgName))
+ , Length_(File_.GetLength())
+ , Mode_(om)
+ {
+ CheckFile();
+ CreateMapping();
+ }
+
+ inline bool IsOpen() const noexcept {
+ return File_.IsOpen()
+#if defined(_win_)
+ && Mapping_ != nullptr
+#endif
+ ;
+ }
+
+ inline bool IsWritable() const noexcept {
+ return (Mode_ & oRdWr || Mode_ & oCopyOnWr);
+ }
+
+ inline TMapResult Map(i64 offset, size_t size) {
+ assert(File_.IsOpen());
+
+ if (offset > Length_) {
+ ythrow yexception() << "Can't map something at offset " << offset << " of '" << DbgName_ << "' with length " << Length_;
+ }
+
+ if (offset + (i64)size > Length_) {
+ ythrow yexception() << "Can't map " << (unsigned long)size << " bytes at offset " << offset << " of '" << DbgName_ << "' with length " << Length_;
+ }
+
+ TMapResult result;
+
+ i64 base = DownToGranularity(offset);
+ result.Head = (i32)(offset - base);
+ size += result.Head;
+
+#if defined(_win_)
+ result.Ptr = MapViewOfFile(Mapping_,
+ (Mode_ & oAccessMask) == oRdOnly ? FILE_MAP_READ : (Mode_ & oAccessMask) == oCopyOnWr ? FILE_MAP_COPY
+ : FILE_MAP_WRITE,
+ Hi32(base), Lo32(base), size);
+#else
+ #if defined(_unix_)
+ if (Mode_ & oNotGreedy) {
+ #endif
+ result.Ptr = mmap((caddr_t) nullptr, size, ModeToMmapProt(Mode_), ModeToMmapFlags(Mode_), File_.GetHandle(), base);
+
+ if (result.Ptr == (char*)(-1)) {
+ result.Ptr = nullptr;
+ }
+ #if defined(_unix_)
+ } else {
+ result.Ptr = PtrStart_ ? static_cast<caddr_t>(PtrStart_) + base : nullptr;
+ }
+ #endif
+#endif
+ if (result.Ptr != nullptr || size == 0) { // allow map of size 0
+ result.Size = size;
+ } else {
+ ythrow yexception() << "Can't map " << (unsigned long)size << " bytes at offset " << offset << " of '" << DbgName_ << "': " << LastSystemErrorText();
+ }
+ NSan::Unpoison(result.Ptr, result.Size);
+ if (Mode_ & oPrecharge) {
+ NPrivate::Precharge(result.Ptr, result.Size, 0, result.Size);
+ }
+
+ return result;
+ }
+
+#if defined(_win_)
+ inline bool Unmap(void* ptr, size_t) {
+ return ::UnmapViewOfFile(ptr) != FALSE;
+ }
+#else
+ inline bool Unmap(void* ptr, size_t size) {
+ #if defined(_unix_)
+ if (Mode_ & oNotGreedy) {
+ #endif
+ return size == 0 || ::munmap(static_cast<caddr_t>(ptr), size) == 0;
+ #if defined(_unix_)
+ } else {
+ return true;
+ }
+ #endif
+ }
+#endif
+
+ void SetSequential() {
+#if defined(_unix_)
+ if (!(Mode_ & oNotGreedy) && Length_) {
+ MadviseSequentialAccess(PtrStart_, Length_);
+ }
+#endif
+ }
+
+ void Evict(void* ptr, size_t len) {
+ MadviseEvict(ptr, len);
+ }
+
+ void Evict() {
+#if defined(_unix_)
+// Evict(PtrStart_, Length_);
+#endif
+ }
+
+ inline ~TImpl() {
+#if defined(_win_)
+ if (Mapping_) {
+ ::CloseHandle(Mapping_); // != FALSE
+ Mapping_ = nullptr;
+ }
+#elif defined(_unix_)
+ if (PtrStart_) {
+ munmap((caddr_t)PtrStart_, Length_);
+ }
+#endif
+ }
+
+ inline i64 Length() const noexcept {
+ return Length_;
+ }
+
+ inline TFile GetFile() const noexcept {
+ return File_;
+ }
+
+ inline TString GetDbgName() const {
+ return DbgName_;
+ }
+
+ inline EOpenMode GetMode() const noexcept {
+ return Mode_;
+ }
+
+private:
+ TFile File_;
+ TString DbgName_; // This string is never used to actually open a file, only in exceptions
+ i64 Length_;
+ EOpenMode Mode_;
+
+#if defined(_win_)
+ void* Mapping_;
+#elif defined(_unix_)
+ void* PtrStart_;
+#endif
+};
+
+TMemoryMap::TMemoryMap(const TString& name)
+ : Impl_(new TImpl(name, EOpenModeFlag::oRdOnly))
+{
+}
+
+TMemoryMap::TMemoryMap(const TString& name, EOpenMode om)
+ : Impl_(new TImpl(name, om))
+{
+}
+
+TMemoryMap::TMemoryMap(const TString& name, i64 length, EOpenMode om)
+ : Impl_(new TImpl(name, length, om))
+{
+}
+
+TMemoryMap::TMemoryMap(FILE* f, TString dbgName)
+ : Impl_(new TImpl(f, EOpenModeFlag::oRdOnly, std::move(dbgName)))
+{
+}
+
+TMemoryMap::TMemoryMap(FILE* f, EOpenMode om, TString dbgName)
+ : Impl_(new TImpl(f, om, std::move(dbgName)))
+{
+}
+
+TMemoryMap::TMemoryMap(const TFile& file, TString dbgName)
+ : Impl_(new TImpl(file, EOpenModeFlag::oRdOnly, std::move(dbgName)))
+{
+}
+
+TMemoryMap::TMemoryMap(const TFile& file, EOpenMode om, TString dbgName)
+ : Impl_(new TImpl(file, om, std::move(dbgName)))
+{
+}
+
+TMemoryMap::~TMemoryMap() = default;
+
+TMemoryMap::TMapResult TMemoryMap::Map(i64 offset, size_t size) {
+ return Impl_->Map(offset, size);
+}
+
+bool TMemoryMap::Unmap(void* ptr, size_t size) {
+ return Impl_->Unmap(ptr, size);
+}
+
+bool TMemoryMap::Unmap(TMapResult region) {
+ return Unmap(region.Ptr, region.Size);
+}
+
+void TMemoryMap::ResizeAndReset(i64 size) {
+ EOpenMode om = Impl_->GetMode();
+ TFile file = GetFile();
+ file.Resize(size);
+ Impl_.Reset(new TImpl(file, om, Impl_->GetDbgName()));
+}
+
+TMemoryMap::TMapResult TMemoryMap::ResizeAndRemap(i64 offset, size_t size) {
+ ResizeAndReset(offset + (i64)size);
+ return Map(offset, size);
+}
+
+void TMemoryMap::SetSequential() {
+ Impl_->SetSequential();
+}
+
+void TMemoryMap::Evict(void* ptr, size_t len) {
+ Impl_->Evict(ptr, len);
+}
+
+void TMemoryMap::Evict() {
+ Impl_->Evict();
+}
+
+i64 TMemoryMap::Length() const noexcept {
+ return Impl_->Length();
+}
+
+bool TMemoryMap::IsOpen() const noexcept {
+ return Impl_->IsOpen();
+}
+
+bool TMemoryMap::IsWritable() const noexcept {
+ return Impl_->IsWritable();
+}
+
+TMemoryMap::EOpenMode TMemoryMap::GetMode() const noexcept {
+ return Impl_->GetMode();
+}
+
+TFile TMemoryMap::GetFile() const noexcept {
+ return Impl_->GetFile();
+}
+
+TFileMap::TFileMap(const TMemoryMap& map) noexcept
+ : Map_(map)
+{
+}
+
+TFileMap::TFileMap(const TString& name)
+ : Map_(name)
+{
+}
+
+TFileMap::TFileMap(const TString& name, EOpenMode om)
+ : Map_(name, om)
+{
+}
+
+TFileMap::TFileMap(const TString& name, i64 length, EOpenMode om)
+ : Map_(name, length, om)
+{
+}
+
+TFileMap::TFileMap(FILE* f, EOpenMode om, TString dbgName)
+ : Map_(f, om, dbgName)
+{
+}
+
+TFileMap::TFileMap(const TFile& file, EOpenMode om, TString dbgName)
+ : Map_(file, om, dbgName)
+{
+}
+
+TFileMap::TFileMap(const TFileMap& fm) noexcept
+ : Map_(fm.Map_)
+{
+}
+
+void TFileMap::Flush(void* ptr, size_t size, bool sync) {
+ Y_ASSERT(ptr >= Ptr());
+ Y_ASSERT(static_cast<char*>(ptr) + size <= static_cast<char*>(Ptr()) + MappedSize());
+
+ if (!Region_.IsMapped()) {
+ return;
+ }
+
+#if defined(_win_)
+ if (sync) {
+ FlushViewOfFile(ptr, size);
+ }
+#else
+ msync(ptr, size, sync ? MS_SYNC : MS_ASYNC);
+#endif
+}
+
+TFileMap::TMapResult TFileMap::Map(i64 offset, size_t size) {
+ Unmap();
+ Region_ = Map_.Map(offset, size);
+ return Region_;
+}
+
+TFileMap::TMapResult TFileMap::ResizeAndRemap(i64 offset, size_t size) {
+ // explicit Unmap() is required because in oNotGreedy mode the Map_ object doesn't own the mapped area
+ Unmap();
+ Region_ = Map_.ResizeAndRemap(offset, size);
+ return Region_;
+}
+
+void TFileMap::Unmap() {
+ if (!Region_.IsMapped()) {
+ return;
+ }
+
+ if (Map_.Unmap(Region_)) {
+ Region_.Reset();
+ } else {
+ ythrow yexception() << "can't unmap file";
+ }
+}
+
+TFileMap::~TFileMap() {
+ try {
+ // explicit Unmap() is required because in oNotGreedy mode the Map_ object doesn't own the mapped area
+ Unmap();
+ } catch (...) {
+ // ¯\_(ツ)_/¯
+ }
+}
+
+void TFileMap::Precharge(size_t pos, size_t size) const {
+ NPrivate::Precharge(Ptr(), MappedSize(), pos, size);
+}
+
+TMappedAllocation::TMappedAllocation(size_t size, bool shared, void* addr)
+ : Ptr_(nullptr)
+ , Size_(0)
+ , Shared_(shared)
+#if defined(_win_)
+ , Mapping_(nullptr)
+#endif
+{
+ if (size != 0) {
+ Alloc(size, addr);
+ }
+}
+
+void* TMappedAllocation::Alloc(size_t size, void* addr) {
+ assert(Ptr_ == nullptr);
+#if defined(_win_)
+ (void)addr;
+ Mapping_ = CreateFileMapping((HANDLE)-1, nullptr, PAGE_READWRITE, 0, size ? size : 1, nullptr);
+ Ptr_ = MapViewOfFile(Mapping_, FILE_MAP_WRITE, 0, 0, size ? size : 1);
+#else
+ Ptr_ = mmap(addr, size, PROT_READ | PROT_WRITE, (Shared_ ? MAP_SHARED : MAP_PRIVATE) | MAP_ANON, -1, 0);
+
+ if (Ptr_ == (void*)MAP_FAILED) {
+ Ptr_ = nullptr;
+ }
+#endif
+ if (Ptr_ != nullptr) {
+ Size_ = size;
+ }
+ return Ptr_;
+}
+
+void TMappedAllocation::Dealloc() {
+ if (Ptr_ == nullptr) {
+ return;
+ }
+#if defined(_win_)
+ UnmapViewOfFile(Ptr_);
+ CloseHandle(Mapping_);
+ Mapping_ = nullptr;
+#else
+ munmap((caddr_t)Ptr_, Size_);
+#endif
+ Ptr_ = nullptr;
+ Size_ = 0;
+}
+
+void TMappedAllocation::swap(TMappedAllocation& with) {
+ DoSwap(Ptr_, with.Ptr_);
+ DoSwap(Size_, with.Size_);
+#if defined(_win_)
+ DoSwap(Mapping_, with.Mapping_);
+#endif
+}
diff --git a/util/system/filemap.h b/util/system/filemap.h
new file mode 100644
index 0000000000..11be64bff4
--- /dev/null
+++ b/util/system/filemap.h
@@ -0,0 +1,381 @@
+#pragma once
+
+#include "file.h"
+#include "align.h"
+#include "yassert.h"
+
+#include <util/generic/noncopyable.h>
+#include <util/generic/ptr.h>
+#include <util/generic/utility.h>
+#include <util/generic/yexception.h>
+#include <util/generic/flags.h>
+#include <util/generic/string.h>
+
+#include <new>
+#include <cstdio>
+
+namespace NPrivate {
+ // NB: use TFileMap::Precharge() and TFileMappedArray::Prechage()
+ void Precharge(const void* data, size_t dataSize, size_t offset, size_t size);
+}
+
+struct TMemoryMapCommon {
+ struct TMapResult {
+ inline size_t MappedSize() const noexcept {
+ return Size - Head;
+ }
+
+ inline void* MappedData() const noexcept {
+ return Ptr ? (void*)((char*)Ptr + Head) : nullptr;
+ }
+
+ inline bool IsMapped() const noexcept {
+ return Ptr != nullptr;
+ }
+
+ inline void Reset() noexcept {
+ Ptr = nullptr;
+ Size = 0;
+ Head = 0;
+ }
+
+ void* Ptr;
+ size_t Size;
+ i32 Head;
+
+ TMapResult(void) noexcept {
+ Reset();
+ }
+ };
+
+ enum EOpenModeFlag {
+ oRdOnly = 1,
+ oRdWr = 2,
+ oCopyOnWr = 4,
+
+ oAccessMask = 7,
+ oNotGreedy = 8,
+ oPrecharge = 16,
+ oPopulate = 32, // Populate page table entries (see mmap's MAP_POPULATE)
+ };
+ Y_DECLARE_FLAGS(EOpenMode, EOpenModeFlag)
+
+ /**
+ * Name that will be printed in exceptions if not specified.
+ * Overridden by name obtained from `TFile` if it's not empty.
+ */
+ static TString UnknownFileName();
+};
+Y_DECLARE_OPERATORS_FOR_FLAGS(TMemoryMapCommon::EOpenMode)
+
+class TMemoryMap: public TMemoryMapCommon {
+public:
+ explicit TMemoryMap(const TString& name);
+ explicit TMemoryMap(const TString& name, EOpenMode om);
+ TMemoryMap(const TString& name, i64 length, EOpenMode om);
+ TMemoryMap(FILE* f, TString dbgName = UnknownFileName());
+ TMemoryMap(FILE* f, EOpenMode om, TString dbgName = UnknownFileName());
+ TMemoryMap(const TFile& file, TString dbgName = UnknownFileName());
+ TMemoryMap(const TFile& file, EOpenMode om, TString dbgName = UnknownFileName());
+
+ ~TMemoryMap();
+
+ TMapResult Map(i64 offset, size_t size);
+ bool Unmap(TMapResult region);
+
+ void ResizeAndReset(i64 size);
+ TMapResult ResizeAndRemap(i64 offset, size_t size);
+
+ i64 Length() const noexcept;
+ bool IsOpen() const noexcept;
+ bool IsWritable() const noexcept;
+ EOpenMode GetMode() const noexcept;
+ TFile GetFile() const noexcept;
+
+ void SetSequential();
+ void Evict(void* ptr, size_t len);
+ void Evict();
+
+ /*
+ * deprecated
+ */
+ bool Unmap(void* ptr, size_t size);
+
+private:
+ class TImpl;
+ TSimpleIntrusivePtr<TImpl> Impl_;
+};
+
+class TFileMap: public TMemoryMapCommon {
+public:
+ TFileMap(const TMemoryMap& map) noexcept;
+ TFileMap(const TString& name);
+ TFileMap(const TString& name, EOpenMode om);
+ TFileMap(const TString& name, i64 length, EOpenMode om);
+ TFileMap(FILE* f, EOpenMode om = oRdOnly, TString dbgName = UnknownFileName());
+ TFileMap(const TFile& file, EOpenMode om = oRdOnly, TString dbgName = UnknownFileName());
+ TFileMap(const TFileMap& fm) noexcept;
+
+ ~TFileMap();
+
+ TMapResult Map(i64 offset, size_t size);
+ TMapResult ResizeAndRemap(i64 offset, size_t size);
+ void Unmap();
+
+ void Flush(void* ptr, size_t size) {
+ Flush(ptr, size, true);
+ }
+
+ void Flush() {
+ Flush(Ptr(), MappedSize());
+ }
+
+ void FlushAsync(void* ptr, size_t size) {
+ Flush(ptr, size, false);
+ }
+
+ void FlushAsync() {
+ FlushAsync(Ptr(), MappedSize());
+ }
+
+ inline i64 Length() const noexcept {
+ return Map_.Length();
+ }
+
+ inline bool IsOpen() const noexcept {
+ return Map_.IsOpen();
+ }
+
+ inline bool IsWritable() const noexcept {
+ return Map_.IsWritable();
+ }
+
+ EOpenMode GetMode() const noexcept {
+ return Map_.GetMode();
+ }
+
+ inline void* Ptr() const noexcept {
+ return Region_.MappedData();
+ }
+
+ inline size_t MappedSize() const noexcept {
+ return Region_.MappedSize();
+ }
+
+ TFile GetFile() const noexcept {
+ return Map_.GetFile();
+ }
+
+ void Precharge(size_t pos = 0, size_t size = (size_t)-1) const;
+
+ void SetSequential() {
+ Map_.SetSequential();
+ }
+
+ void Evict() {
+ Map_.Evict();
+ }
+
+private:
+ void Flush(void* ptr, size_t size, bool sync);
+
+ TMemoryMap Map_;
+ TMapResult Region_;
+};
+
+template <class T>
+class TFileMappedArray {
+private:
+ const T* Ptr_;
+ const T* End_;
+ size_t Size_;
+ char DummyData_[sizeof(T) + PLATFORM_DATA_ALIGN];
+ mutable THolder<T, TDestructor> Dummy_;
+ THolder<TFileMap> DataHolder_;
+
+public:
+ TFileMappedArray()
+ : Ptr_(nullptr)
+ , End_(nullptr)
+ , Size_(0)
+ {
+ }
+ ~TFileMappedArray() {
+ Ptr_ = nullptr;
+ End_ = nullptr;
+ }
+ void Init(const char* name) {
+ DataHolder_.Reset(new TFileMap(name));
+ DoInit(name);
+ }
+ void Init(const TFileMap& fileMap) {
+ DataHolder_.Reset(new TFileMap(fileMap));
+ DoInit(fileMap.GetFile().GetName());
+ }
+ void Term() {
+ DataHolder_.Destroy();
+ Ptr_ = nullptr;
+ Size_ = 0;
+ End_ = nullptr;
+ }
+ void Precharge() {
+ DataHolder_->Precharge();
+ }
+ const T& operator[](size_t pos) const {
+ Y_ASSERT(pos < size());
+ return Ptr_[pos];
+ }
+ /// for STL compatibility only, Size() usage is recommended
+ size_t size() const {
+ return Size_;
+ }
+ size_t Size() const {
+ return Size_;
+ }
+ const T& GetAt(size_t pos) const {
+ if (pos < Size_)
+ return Ptr_[pos];
+ return Dummy();
+ }
+ void SetDummy(const T& n_Dummy) {
+ Dummy_.Destroy();
+ Dummy_.Reset(new (DummyData()) T(n_Dummy));
+ }
+ inline char* DummyData() const noexcept {
+ return AlignUp((char*)DummyData_);
+ }
+ inline const T& Dummy() const {
+ if (!Dummy_) {
+ Dummy_.Reset(new (DummyData()) T());
+ }
+
+ return *Dummy_;
+ }
+ /// for STL compatibility only, Empty() usage is recommended
+ Y_PURE_FUNCTION bool empty() const noexcept {
+ return Empty();
+ }
+
+ Y_PURE_FUNCTION bool Empty() const noexcept {
+ return 0 == Size_;
+ }
+ /// for STL compatibility only, Begin() usage is recommended
+ const T* begin() const noexcept {
+ return Begin();
+ }
+ const T* Begin() const noexcept {
+ return Ptr_;
+ }
+ /// for STL compatibility only, End() usage is recommended
+ const T* end() const noexcept {
+ return End_;
+ }
+ const T* End() const noexcept {
+ return End_;
+ }
+
+private:
+ void DoInit(const TString& fileName) {
+ DataHolder_->Map(0, DataHolder_->Length());
+ if (DataHolder_->Length() % sizeof(T)) {
+ Term();
+ ythrow yexception() << "Incorrect size of file " << fileName.Quote();
+ }
+ Ptr_ = (const T*)DataHolder_->Ptr();
+ Size_ = DataHolder_->Length() / sizeof(T);
+ End_ = Ptr_ + Size_;
+ }
+};
+
+class TMappedAllocation: TMoveOnly {
+public:
+ TMappedAllocation(size_t size = 0, bool shared = false, void* addr = nullptr);
+ ~TMappedAllocation() {
+ Dealloc();
+ }
+ TMappedAllocation(TMappedAllocation&& other) {
+ this->swap(other);
+ }
+ TMappedAllocation& operator=(TMappedAllocation&& other) {
+ this->swap(other);
+ return *this;
+ }
+ void* Alloc(size_t size, void* addr = nullptr);
+ void Dealloc();
+ void* Ptr() const {
+ return Ptr_;
+ }
+ char* Data(ui32 pos = 0) const {
+ return (char*)(Ptr_ ? ((char*)Ptr_ + pos) : nullptr);
+ }
+ char* Begin() const noexcept {
+ return (char*)Ptr();
+ }
+ char* End() const noexcept {
+ return Begin() + MappedSize();
+ }
+ size_t MappedSize() const {
+ return Size_;
+ }
+ void swap(TMappedAllocation& with);
+
+private:
+ void* Ptr_ = nullptr;
+ size_t Size_ = 0;
+ bool Shared_ = false;
+#ifdef _win_
+ void* Mapping_ = nullptr;
+#endif
+};
+
+template <class T>
+class TMappedArray: private TMappedAllocation {
+public:
+ TMappedArray(size_t siz = 0)
+ : TMappedAllocation(0)
+ {
+ if (siz)
+ Create(siz);
+ }
+ ~TMappedArray() {
+ Destroy();
+ }
+ T* Create(size_t siz) {
+ Y_ASSERT(MappedSize() == 0 && Ptr() == nullptr);
+ T* arr = (T*)Alloc((sizeof(T) * siz));
+ if (!arr)
+ return nullptr;
+ Y_ASSERT(MappedSize() == sizeof(T) * siz);
+ for (size_t n = 0; n < siz; n++)
+ new (&arr[n]) T();
+ return arr;
+ }
+ void Destroy() {
+ T* arr = (T*)Ptr();
+ if (arr) {
+ for (size_t n = 0; n < size(); n++)
+ arr[n].~T();
+ Dealloc();
+ }
+ }
+ T& operator[](size_t pos) {
+ Y_ASSERT(pos < size());
+ return ((T*)Ptr())[pos];
+ }
+ const T& operator[](size_t pos) const {
+ Y_ASSERT(pos < size());
+ return ((T*)Ptr())[pos];
+ }
+ T* begin() {
+ return (T*)Ptr();
+ }
+ T* end() {
+ return (T*)((char*)Ptr() + MappedSize());
+ }
+ size_t size() const {
+ return MappedSize() / sizeof(T);
+ }
+ void swap(TMappedArray<T>& with) {
+ TMappedAllocation::swap(with);
+ }
+};
diff --git a/util/system/filemap_ut.cpp b/util/system/filemap_ut.cpp
new file mode 100644
index 0000000000..73f109dc88
--- /dev/null
+++ b/util/system/filemap_ut.cpp
@@ -0,0 +1,359 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#ifdef _unix_
+ #include <sys/resource.h>
+#endif
+
+#include "filemap.h"
+
+#include <util/system/fs.h>
+
+#include <cstring>
+#include <cstdio>
+
+Y_UNIT_TEST_SUITE(TFileMapTest) {
+ static const char* FileName_("./mappped_file");
+
+ void BasicTest(TMemoryMapCommon::EOpenMode mode) {
+ char data[] = "abcdefgh";
+
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Write(static_cast<void*>(data), sizeof(data));
+ file.Close();
+
+ {
+ TFileMap mappedFile(FileName_, mode);
+ mappedFile.Map(0, mappedFile.Length());
+ UNIT_ASSERT(mappedFile.MappedSize() == sizeof(data) && mappedFile.Length() == sizeof(data));
+ UNIT_ASSERT(mappedFile.IsOpen());
+ for (size_t i = 0; i < sizeof(data); ++i) {
+ UNIT_ASSERT(static_cast<char*>(mappedFile.Ptr())[i] == data[i]);
+ static_cast<char*>(mappedFile.Ptr())[i] = data[i] + 1;
+ }
+ mappedFile.Flush();
+
+ TFileMap::TMapResult mapResult = mappedFile.Map(2, 2);
+ UNIT_ASSERT(mapResult.MappedSize() == 2);
+ UNIT_ASSERT(mapResult.MappedData() == mappedFile.Ptr());
+ UNIT_ASSERT(mappedFile.MappedSize() == 2);
+ UNIT_ASSERT(static_cast<char*>(mappedFile.Ptr())[0] == 'd' && static_cast<char*>(mappedFile.Ptr())[1] == 'e');
+
+ mappedFile.Unmap();
+ UNIT_ASSERT(mappedFile.MappedSize() == 0);
+
+ FILE* f = fopen(FileName_, "rb");
+ TFileMap mappedFile2(f);
+ mappedFile2.Map(0, mappedFile2.Length());
+ UNIT_ASSERT(mappedFile2.MappedSize() == sizeof(data));
+ UNIT_ASSERT(static_cast<char*>(mappedFile2.Ptr())[0] == data[0] + 1);
+ fclose(f);
+ }
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(TestFileMap) {
+ BasicTest(TMemoryMapCommon::oRdWr);
+ }
+
+ Y_UNIT_TEST(TestFileMapPopulate) {
+ BasicTest(TMemoryMapCommon::oRdWr | TMemoryMapCommon::oPopulate);
+ }
+
+ Y_UNIT_TEST(TestFileRemap) {
+ const char data1[] = "01234";
+ const char data2[] = "abcdefg";
+ const char data3[] = "COPY";
+ const char dataFinal[] = "012abcdefg";
+ const size_t data2Shift = 3;
+
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Write(static_cast<const void*>(data1), sizeof(data1));
+ file.Close();
+
+ {
+ TFileMap mappedFile(FileName_, TMemoryMapCommon::oRdWr);
+ mappedFile.Map(0, mappedFile.Length());
+ UNIT_ASSERT(mappedFile.MappedSize() == sizeof(data1) &&
+ mappedFile.Length() == sizeof(data1));
+
+ mappedFile.ResizeAndRemap(data2Shift, sizeof(data2));
+ memcpy(mappedFile.Ptr(), data2, sizeof(data2));
+ }
+
+ {
+ TFileMap mappedFile(FileName_, TMemoryMapCommon::oCopyOnWr);
+ mappedFile.Map(0, mappedFile.Length());
+ UNIT_ASSERT(mappedFile.MappedSize() == sizeof(dataFinal) &&
+ mappedFile.Length() == sizeof(dataFinal));
+
+ char* data = static_cast<char*>(mappedFile.Ptr());
+ UNIT_ASSERT(data[0] == '0');
+ UNIT_ASSERT(data[3] == 'a');
+ memcpy(data, data3, sizeof(data3));
+ UNIT_ASSERT(data[0] == 'C');
+ UNIT_ASSERT(data[3] == 'Y');
+ }
+
+ TFile resFile(FileName_, RdOnly);
+ UNIT_ASSERT(resFile.GetLength() == sizeof(dataFinal));
+ char buf[sizeof(dataFinal)];
+ resFile.Read(buf, sizeof(dataFinal));
+ UNIT_ASSERT(0 == memcmp(buf, dataFinal, sizeof(dataFinal)));
+ resFile.Close();
+
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(TestFileMapDbgName) {
+ // This test checks that dbgName passed to the TFileMap constructor is saved inside the object and appears
+ // in subsequent error messages.
+ const char* const dbgName = "THIS_IS_A_TEST";
+ FILE* f = fopen(FileName_, "w+");
+ UNIT_ASSERT(f);
+ {
+ TFileMap mappedFile(f, TFileMap::oRdWr, dbgName);
+ bool gotException = false;
+ try {
+ // trying to map an empty file to force an exception and check the message
+ mappedFile.Map(0, 1000);
+ } catch (const yexception& e) {
+ gotException = true;
+ UNIT_ASSERT_STRING_CONTAINS(e.what(), dbgName);
+ }
+ UNIT_ASSERT(gotException);
+ }
+ fclose(f);
+ NFs::Remove(FileName_);
+ }
+
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+//setrlimit incompatible with asan runtime
+#elif defined(_cygwin_)
+//cygwin is not real unix :(
+#else
+ Y_UNIT_TEST(TestNotGreedy) {
+ unsigned page[4096 / sizeof(unsigned)];
+
+ #if defined(_unix_)
+ // Temporary limit allowed virtual memory size to 1Gb
+ struct rlimit rlim;
+
+ if (getrlimit(RLIMIT_AS, &rlim)) {
+ throw TSystemError() << "Cannot get rlimit for virtual memory";
+ }
+
+ rlim_t Limit = 1 * 1024 * 1024 * 1024;
+
+ if (rlim.rlim_cur > Limit) {
+ rlim.rlim_cur = Limit;
+
+ if (setrlimit(RLIMIT_AS, &rlim)) {
+ throw TSystemError() << "Cannot set rlimit for virtual memory to 1Gb";
+ }
+ }
+ #endif
+ // Make a 128M test file
+ try {
+ TFile file(FileName_, CreateAlways | WrOnly);
+
+ for (unsigned pages = 128 * 1024 * 1024 / sizeof(page), i = 0; pages--; i++) {
+ std::fill(page, page + sizeof(page) / sizeof(*page), i);
+ file.Write(page, sizeof(page));
+ }
+
+ file.Close();
+
+ // Make 16 maps of our file, which would require 16*128M = 2Gb and exceed our 1Gb limit
+ TVector<THolder<TFileMap>> maps;
+
+ for (int i = 0; i < 16; ++i) {
+ maps.emplace_back(MakeHolder<TFileMap>(FileName_, TMemoryMapCommon::oRdOnly | TMemoryMapCommon::oNotGreedy));
+ maps.back()->Map(i * sizeof(page), sizeof(page));
+ }
+
+ // Oh, good, we're not dead yet
+ for (int i = 0; i < 16; ++i) {
+ TFileMap& map = *maps[i];
+
+ UNIT_ASSERT_EQUAL(map.Length(), 128 * 1024 * 1024);
+ UNIT_ASSERT_EQUAL(map.MappedSize(), sizeof(page));
+
+ const int* mappedPage = (const int*)map.Ptr();
+
+ for (size_t j = 0; j < sizeof(page) / sizeof(*page); ++j) {
+ UNIT_ASSERT_EQUAL(mappedPage[j], i);
+ }
+ }
+
+ #if defined(_unix_)
+ // Restore limits and cleanup
+ rlim.rlim_cur = rlim.rlim_max;
+
+ if (setrlimit(RLIMIT_AS, &rlim)) {
+ throw TSystemError() << "Cannot restore rlimit for virtual memory";
+ }
+ #endif
+ maps.clear();
+ NFs::Remove(FileName_);
+ } catch (...) {
+ // TODO: RAII'ize all this stuff
+ #if defined(_unix_)
+ rlim.rlim_cur = rlim.rlim_max;
+
+ if (setrlimit(RLIMIT_AS, &rlim)) {
+ throw TSystemError() << "Cannot restore rlimit for virtual memory";
+ }
+ #endif
+ NFs::Remove(FileName_);
+
+ throw;
+ }
+ }
+#endif
+
+ Y_UNIT_TEST(TestFileMappedArray) {
+ {
+ TFileMappedArray<ui32> mappedArray;
+ ui32 data[] = {123, 456, 789, 10};
+ size_t sz = sizeof(data) / sizeof(data[0]);
+
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Write(static_cast<void*>(data), sizeof(data));
+ file.Close();
+
+ mappedArray.Init(FileName_);
+ // actual test begin
+ UNIT_ASSERT(mappedArray.Size() == sz);
+ for (size_t i = 0; i < sz; ++i) {
+ UNIT_ASSERT(mappedArray[i] == data[i]);
+ }
+
+ UNIT_ASSERT(mappedArray.GetAt(mappedArray.Size()) == 0);
+ UNIT_ASSERT(*mappedArray.Begin() == data[0]);
+ UNIT_ASSERT(size_t(mappedArray.End() - mappedArray.Begin()) == sz);
+ UNIT_ASSERT(!mappedArray.Empty());
+ // actual test end
+ mappedArray.Term();
+
+ // Init array via file mapping
+ TFileMap fileMap(FileName_);
+ fileMap.Map(0, fileMap.Length());
+ mappedArray.Init(fileMap);
+
+ // actual test begin
+ UNIT_ASSERT(mappedArray.Size() == sz);
+ for (size_t i = 0; i < sz; ++i) {
+ UNIT_ASSERT(mappedArray[i] == data[i]);
+ }
+
+ UNIT_ASSERT(mappedArray.GetAt(mappedArray.Size()) == 0);
+ UNIT_ASSERT(*mappedArray.Begin() == data[0]);
+ UNIT_ASSERT(size_t(mappedArray.End() - mappedArray.Begin()) == sz);
+ UNIT_ASSERT(!mappedArray.Empty());
+ // actual test end
+
+ file = TFile(FileName_, WrOnly);
+ file.Seek(0, sEnd);
+ file.Write("x", 1);
+ file.Close();
+
+ bool caught = false;
+ try {
+ mappedArray.Init(FileName_);
+ } catch (const yexception&) {
+ caught = true;
+ }
+ UNIT_ASSERT(caught);
+ }
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(TestMappedArray) {
+ ui32 sz = 10;
+
+ TMappedArray<ui32> mappedArray;
+
+ ui32* ptr = mappedArray.Create(sz);
+ UNIT_ASSERT(ptr != nullptr);
+ UNIT_ASSERT(mappedArray.size() == sz);
+ UNIT_ASSERT(mappedArray.begin() + sz == mappedArray.end());
+
+ for (size_t i = 0; i < sz; ++i) {
+ mappedArray[i] = (ui32)i;
+ }
+ for (size_t i = 0; i < sz; ++i) {
+ UNIT_ASSERT(mappedArray[i] == i);
+ }
+
+ TMappedArray<ui32> mappedArray2(1000);
+ mappedArray.swap(mappedArray2);
+ UNIT_ASSERT(mappedArray.size() == 1000 && mappedArray2.size() == sz);
+ }
+
+ Y_UNIT_TEST(TestMemoryMap) {
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Close();
+
+ FILE* f = fopen(FileName_, "rb");
+ UNIT_ASSERT(f != nullptr);
+ try {
+ TMemoryMap mappedMem(f);
+ mappedMem.Map(mappedMem.Length() / 2, mappedMem.Length() + 100); // overflow
+ UNIT_ASSERT(0); // should not go here
+ } catch (yexception& exc) {
+ TString text = exc.what(); // exception should contain failed file name
+ UNIT_ASSERT(text.find(TMemoryMapCommon::UnknownFileName()) != TString::npos);
+ fclose(f);
+ }
+
+ TFile fileForMap(FileName_, OpenExisting);
+ try {
+ TMemoryMap mappedMem(fileForMap);
+ mappedMem.Map(mappedMem.Length() / 2, mappedMem.Length() + 100); // overflow
+ UNIT_ASSERT(0); // should not go here
+ } catch (yexception& exc) {
+ TString text = exc.what(); // exception should contain failed file name
+ UNIT_ASSERT(text.find(FileName_) != TString::npos);
+ }
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(TestMemoryMapIsWritable) {
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Close();
+
+ {
+ TMemoryMap mappedMem(FileName_, TMemoryMap::oRdOnly);
+ UNIT_ASSERT(!mappedMem.IsWritable());
+ }
+ {
+ TMemoryMap mappedMem(FileName_, TMemoryMap::oRdWr);
+ UNIT_ASSERT(mappedMem.IsWritable());
+ }
+ NFs::Remove(FileName_);
+ }
+
+ Y_UNIT_TEST(TestFileMapIsWritable) {
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Close();
+ {
+ TMemoryMap mappedMem(FileName_, TMemoryMap::oRdOnly);
+ TFileMap fileMap(mappedMem);
+ UNIT_ASSERT(!fileMap.IsWritable());
+ }
+ {
+ TMemoryMap mappedMem(FileName_, TMemoryMap::oRdWr);
+ TFileMap fileMap(mappedMem);
+ UNIT_ASSERT(fileMap.IsWritable());
+ }
+ {
+ TFileMap fileMap(FileName_, TFileMap::oRdOnly);
+ UNIT_ASSERT(!fileMap.IsWritable());
+ }
+ {
+ TFileMap fileMap(FileName_, TFileMap::oRdWr);
+ UNIT_ASSERT(fileMap.IsWritable());
+ }
+ NFs::Remove(FileName_);
+ }
+};
diff --git a/util/system/flock.cpp b/util/system/flock.cpp
new file mode 100644
index 0000000000..fe88fecaff
--- /dev/null
+++ b/util/system/flock.cpp
@@ -0,0 +1,71 @@
+#include "flock.h"
+
+#ifndef _unix_
+
+ #include <util/generic/utility.h>
+
+ #include "winint.h"
+ #include <io.h>
+ #include <errno.h>
+
+ #ifdef __cplusplus
+extern "C" {
+ #endif
+
+ int flock(int fd, int op) {
+ return Flock((HANDLE)_get_osfhandle(fd), op);
+ }
+
+ int Flock(void* hdl, int op) {
+ errno = 0;
+
+ if (hdl == INVALID_HANDLE_VALUE) {
+ errno = EBADF;
+ return -1;
+ }
+
+ DWORD low = 1, high = 0;
+ OVERLAPPED io;
+
+ Zero(io);
+
+ UnlockFileEx(hdl, 0, low, high, &io);
+
+ switch (op & ~LOCK_NB) {
+ case LOCK_EX:
+ case LOCK_SH: {
+ auto mode = ((op & ~LOCK_NB) == LOCK_EX) ? LOCKFILE_EXCLUSIVE_LOCK : 0;
+ if (op & LOCK_NB) {
+ if (LockFileEx(hdl, mode | LOCKFILE_FAIL_IMMEDIATELY, 0, low, high, &io)) {
+ return 0;
+ } else if (GetLastError() == ERROR_LOCK_VIOLATION) {
+ ClearLastSystemError();
+ errno = EWOULDBLOCK;
+ return -1;
+ }
+ } else {
+ if (LockFileEx(hdl, mode, 0, low, high, &io)) {
+ return 0;
+ }
+ }
+ break;
+ }
+ case LOCK_UN:
+ return 0;
+ break;
+ default:
+ break;
+ }
+ errno = EINVAL;
+ return -1;
+ }
+
+ int fsync(int fd) {
+ return _commit(fd);
+ }
+
+ #ifdef __cplusplus
+}
+ #endif
+
+#endif
diff --git a/util/system/flock.h b/util/system/flock.h
new file mode 100644
index 0000000000..797b1970a1
--- /dev/null
+++ b/util/system/flock.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include "error.h"
+#include "defaults.h"
+#include "file.h"
+
+#if defined(_unix_)
+
+ #include <sys/file.h>
+ #include <fcntl.h>
+
+static inline int Flock(int fd, int op) {
+ return flock(fd, op);
+}
+
+#else // not _unix_
+
+ #ifdef __cplusplus
+extern "C" {
+ #endif
+
+ #define LOCK_SH 1 /* shared lock */
+ #define LOCK_EX 2 /* exclusive lock */
+ #define LOCK_NB 4 /* don't block when locking */
+ #define LOCK_UN 8 /* unlock */
+
+ int Flock(void* hndl, int operation);
+ int flock(int fd, int operation);
+ int fsync(int fd);
+
+ #ifdef __cplusplus
+}
+ #endif
+
+#endif // not _unix_
diff --git a/util/system/flock_ut.cpp b/util/system/flock_ut.cpp
new file mode 100644
index 0000000000..b5f6cb5328
--- /dev/null
+++ b/util/system/flock_ut.cpp
@@ -0,0 +1,57 @@
+#include "flock.h"
+#include "file_lock.h"
+#include "guard.h"
+#include "tempfile.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TFileLockTest) {
+ Y_UNIT_TEST(TestFlock) {
+ TTempFileHandle tmp("./file");
+
+ UNIT_ASSERT_EQUAL(Flock(tmp.GetHandle(), LOCK_EX), 0);
+ UNIT_ASSERT_EQUAL(Flock(tmp.GetHandle(), LOCK_UN), 0);
+ }
+
+ Y_UNIT_TEST(TestFileLocker) {
+ TTempFileHandle tmp("./file.locker");
+ TFileLock fileLockExclusive1("./file.locker");
+ TFileLock fileLockExclusive2("./file.locker");
+ TFileLock fileLockShared1("./file.locker", EFileLockType::Shared);
+ TFileLock fileLockShared2("./file.locker", EFileLockType::Shared);
+ TFileLock fileLockShared3("./file.locker", EFileLockType::Shared);
+ {
+ TGuard<TFileLock> guard(fileLockExclusive1);
+ }
+ {
+ TTryGuard<TFileLock> tryGuard(fileLockExclusive1);
+ UNIT_ASSERT(tryGuard.WasAcquired());
+ }
+ {
+ TGuard<TFileLock> guard1(fileLockExclusive1);
+ TTryGuard<TFileLock> guard2(fileLockExclusive2);
+ UNIT_ASSERT(!guard2.WasAcquired());
+ }
+ {
+ TGuard<TFileLock> guard1(fileLockShared1);
+ TTryGuard<TFileLock> guard2(fileLockShared2);
+ TTryGuard<TFileLock> guard3(fileLockShared3);
+ UNIT_ASSERT(guard2.WasAcquired());
+ UNIT_ASSERT(guard3.WasAcquired());
+ }
+ {
+ TGuard<TFileLock> guard1(fileLockExclusive1);
+ TTryGuard<TFileLock> guard2(fileLockShared1);
+ TTryGuard<TFileLock> guard3(fileLockShared2);
+ UNIT_ASSERT(!guard2.WasAcquired());
+ UNIT_ASSERT(!guard3.WasAcquired());
+ }
+ {
+ TGuard<TFileLock> guard1(fileLockShared1);
+ TTryGuard<TFileLock> guard2(fileLockExclusive1);
+ TTryGuard<TFileLock> guard3(fileLockShared2);
+ UNIT_ASSERT(!guard2.WasAcquired());
+ UNIT_ASSERT(guard3.WasAcquired());
+ }
+ }
+}
diff --git a/util/system/fs.cpp b/util/system/fs.cpp
new file mode 100644
index 0000000000..d2611a8ccc
--- /dev/null
+++ b/util/system/fs.cpp
@@ -0,0 +1,179 @@
+#include "fs.h"
+#include "defaults.h"
+
+#if defined(_win_)
+ #include "fs_win.h"
+#else
+ #include <unistd.h>
+ #include <errno.h>
+#endif
+
+#include <util/generic/yexception.h>
+#include <util/memory/tempbuf.h>
+#include <util/stream/file.h>
+#include <util/charset/wide.h>
+#include <util/folder/iterator.h>
+#include <util/system/fstat.h>
+#include <util/folder/path.h>
+
+bool NFs::Remove(const TString& path) {
+#if defined(_win_)
+ return NFsPrivate::WinRemove(path);
+#else
+ return ::remove(path.data()) == 0;
+#endif
+}
+
+void NFs::RemoveRecursive(const TString& path) {
+ static const TStringBuf errStr = "error while removing ";
+
+ if (!NFs::Exists(path)) {
+ return;
+ }
+
+ if (!TFileStat(path).IsDir()) {
+ if (!NFs::Remove(path)) {
+ ythrow TSystemError() << errStr << path << " with cwd (" << NFs::CurrentWorkingDirectory() << ")";
+ }
+ }
+
+ TDirIterator dir(path);
+
+ for (auto it = dir.begin(); it != dir.end(); ++it) {
+ switch (it->fts_info) {
+ case FTS_DOT:
+ case FTS_D:
+ break;
+ default:
+ if (!NFs::Remove(it->fts_path)) {
+ ythrow TSystemError() << errStr << it->fts_path << " with cwd (" << NFs::CurrentWorkingDirectory() << ")";
+ }
+ break;
+ }
+ }
+}
+
+bool NFs::MakeDirectory(const TString& path, EFilePermissions mode) {
+#if defined(_win_)
+ Y_UNUSED(mode);
+ return NFsPrivate::WinMakeDirectory(path);
+#else
+ return mkdir(path.data(), mode) == 0;
+#endif
+}
+
+bool NFs::MakeDirectoryRecursive(const TString& path, EFilePermissions mode, bool alwaysCreate) {
+ if (NFs::Exists(path) && TFileStat(path).IsDir()) {
+ if (alwaysCreate) {
+ ythrow TIoException() << "path " << path << " already exists"
+ << " with cwd (" << NFs::CurrentWorkingDirectory() << ")";
+ }
+ return true;
+ } else {
+ //NOTE: recursion is finite due to existence of "." and "/"
+ if (!NFs::MakeDirectoryRecursive(TFsPath(path).Parent(), mode, false)) {
+ return false;
+ }
+
+ bool isDirMade = NFs::MakeDirectory(path, mode);
+ if (!isDirMade && alwaysCreate) {
+ ythrow TIoException() << "failed to create " << path << " with cwd (" << NFs::CurrentWorkingDirectory() << ")";
+ }
+
+ return TFileStat(path).IsDir();
+ }
+}
+
+bool NFs::Rename(const TString& oldPath, const TString& newPath) {
+#if defined(_win_)
+ return NFsPrivate::WinRename(oldPath, newPath);
+#else
+ return ::rename(oldPath.data(), newPath.data()) == 0;
+#endif
+}
+
+void NFs::HardLinkOrCopy(const TString& existingPath, const TString& newPath) {
+ if (!NFs::HardLink(existingPath, newPath)) {
+ Copy(existingPath, newPath);
+ }
+}
+
+bool NFs::HardLink(const TString& existingPath, const TString& newPath) {
+#if defined(_win_)
+ return NFsPrivate::WinHardLink(existingPath, newPath);
+#elif defined(_unix_)
+ return (0 == link(existingPath.data(), newPath.data()));
+#endif
+}
+
+bool NFs::SymLink(const TString& targetPath, const TString& linkPath) {
+#if defined(_win_)
+ return NFsPrivate::WinSymLink(targetPath, linkPath);
+#elif defined(_unix_)
+ return 0 == symlink(targetPath.data(), linkPath.data());
+#endif
+}
+
+TString NFs::ReadLink(const TString& path) {
+#if defined(_win_)
+ return NFsPrivate::WinReadLink(path);
+#elif defined(_unix_)
+ TTempBuf buf;
+ while (true) {
+ ssize_t r = readlink(path.data(), buf.Data(), buf.Size());
+ if (r < 0) {
+ ythrow yexception() << "can't read link " << path << ", errno = " << errno;
+ }
+ if (r < (ssize_t)buf.Size()) {
+ return TString(buf.Data(), r);
+ }
+ buf = TTempBuf(buf.Size() * 2);
+ }
+#endif
+}
+
+void NFs::Cat(const TString& dstPath, const TString& srcPath) {
+ TUnbufferedFileInput src(srcPath);
+ TUnbufferedFileOutput dst(TFile(dstPath, ForAppend | WrOnly | Seq));
+
+ TransferData(&src, &dst);
+}
+
+void NFs::Copy(const TString& existingPath, const TString& newPath) {
+ TUnbufferedFileInput src(existingPath);
+ TUnbufferedFileOutput dst(TFile(newPath, CreateAlways | WrOnly | Seq));
+
+ TransferData(&src, &dst);
+}
+
+bool NFs::Exists(const TString& path) {
+#if defined(_win_)
+ return NFsPrivate::WinExists(path);
+#elif defined(_unix_)
+ return access(path.data(), F_OK) == 0;
+#endif
+}
+
+TString NFs::CurrentWorkingDirectory() {
+#if defined(_win_)
+ return NFsPrivate::WinCurrentWorkingDirectory();
+#elif defined(_unix_)
+ TTempBuf result;
+ char* r = getcwd(result.Data(), result.Size());
+ if (r == nullptr) {
+ throw TIoSystemError() << "failed to getcwd";
+ }
+ return result.Data();
+#endif
+}
+
+void NFs::SetCurrentWorkingDirectory(TString path) {
+#ifdef _win_
+ bool ok = NFsPrivate::WinSetCurrentWorkingDirectory(path);
+#else
+ bool ok = !chdir(path.data());
+#endif
+ if (!ok) {
+ ythrow TSystemError() << "failed to change directory to " << path.Quote();
+ }
+}
diff --git a/util/system/fs.h b/util/system/fs.h
new file mode 100644
index 0000000000..237daf2d2d
--- /dev/null
+++ b/util/system/fs.h
@@ -0,0 +1,156 @@
+#pragma once
+
+#include <util/generic/flags.h>
+#include <util/generic/string.h>
+#include <util/generic/yexception.h>
+
+namespace NFs {
+ enum EFilePermission {
+ FP_ALL_EXEC = 01,
+ FP_ALL_WRITE = 02,
+ FP_ALL_READ = 04,
+ FP_GROUP_READ = 040,
+ FP_GROUP_WRITE = 020,
+ FP_GROUP_EXEC = 010,
+ FP_OWNER_READ = 0400,
+ FP_OWNER_WRITE = 0200,
+ FP_OWNER_EXEC = 0100,
+
+ FP_COMMON_FILE = 0777,
+ FP_SECRET_FILE = 0700,
+ FP_NONSECRET_FILE = 0744,
+ };
+
+ Y_DECLARE_FLAGS(EFilePermissions, EFilePermission);
+
+ /// Remove a file or empty directory
+ ///
+ /// @param[in] path Path to file or directory
+ /// @returns true on success or false otherwise
+ /// LastSystemError() is set in case of failure
+ bool Remove(const TString& path);
+
+ /// Remove a file or directory with contents
+ ///
+ /// @param[in] path Path to file or directory
+ /// @throws
+ void RemoveRecursive(const TString& path);
+
+ /// Creates directory
+ ///
+ /// @param[in] path Path to the directory
+ /// @param[in] mode Access permissions field; NOTE: ignored on win
+ /// @returns true on success or false otherwise
+ bool MakeDirectory(const TString& path, EFilePermissions mode);
+
+ /// Creates directory
+ ///
+ /// @param[in] path Path to the directory
+ /// @returns true on success or false otherwise
+ /// NOTE: access permissions is set to 0777
+ inline bool MakeDirectory(const TString& path) {
+ return MakeDirectory(path, FP_COMMON_FILE);
+ }
+
+ /// Creates directory and all non-existings parents
+ ///
+ /// @param[in] path Path to be created
+ /// @param[in] mode Access permissions field; NOTE: ignored on win
+ /// @param[in] alwaysCreate Throw if path already exists or failed to create
+ /// @returns true if target path created or exists (and directory)
+ bool MakeDirectoryRecursive(const TString& path, EFilePermissions mode, bool alwaysCreate);
+
+ /// Creates directory and all non-existings parents
+ ///
+ /// @param[in] path Path to be created
+ /// @param[in] mode Access permissions field; NOTE: ignored on win
+ /// @returns true if target path created or exists (and directory)
+ inline bool MakeDirectoryRecursive(const TString& path, EFilePermissions mode) {
+ return MakeDirectoryRecursive(path, mode, false);
+ }
+
+ /// Creates directory and all non-existings parents
+ ///
+ /// @param[in] path Path to be created
+ /// @returns true if target path created or exists (and directory)
+ inline bool MakeDirectoryRecursive(const TString& path) {
+ return MakeDirectoryRecursive(path, FP_COMMON_FILE, false);
+ }
+
+ /// Rename a file or directory.
+ /// Removes newPath if it exists
+ ///
+ /// @param[in] oldPath Path to file or directory to rename
+ /// @param[in] newPath New path of file or directory
+ /// @returns true on success or false otherwise
+ /// LastSystemError() is set in case of failure
+ bool Rename(const TString& oldPath, const TString& newPath);
+
+ /// Creates a new directory entry for a file
+ /// or creates a new one with the same content
+ ///
+ /// @param[in] existingPath Path to an existing file
+ /// @param[in] newPath New path of file
+ void HardLinkOrCopy(const TString& existingPath, const TString& newPath);
+
+ /// Creates a new directory entry for a file
+ ///
+ /// @param[in] existingPath Path to an existing file
+ /// @param[in] newPath New path of file
+ /// @returns true if new link was created or false otherwise
+ /// LastSystemError() is set in case of failure
+ bool HardLink(const TString& existingPath, const TString& newPath);
+
+ /// Creates a symlink to a file
+ ///
+ /// @param[in] targetPath Path to a target file
+ /// @param[in] linkPath Path of symlink
+ /// @returns true if new link was created or false otherwise
+ /// LastSystemError() is set in case of failure
+ bool SymLink(const TString& targetPath, const TString& linkPath);
+
+ /// Reads value of a symbolic link
+ ///
+ /// @param[in] path Path to a symlink
+ /// @returns File path that a symlink points to
+ TString ReadLink(const TString& path);
+
+ /// Append contents of a file to a new file
+ ///
+ /// @param[in] dstPath Path to a destination file
+ /// @param[in] srcPath Path to a source file
+ void Cat(const TString& dstPath, const TString& srcPath);
+
+ /// Copy contents of a file to a new file
+ ///
+ /// @param[in] existingPath Path to an existing file
+ /// @param[in] newPath New path of file
+ void Copy(const TString& existingPath, const TString& newPath);
+
+ /// Returns path to the current working directory
+ ///
+ /// Note: is not threadsafe
+ TString CurrentWorkingDirectory();
+
+ /// Changes current working directory
+ ///
+ /// @param[in] path Path for new cwd
+ /// Note: is not threadsafe
+ void SetCurrentWorkingDirectory(TString path);
+
+ /// Checks if file exists
+ ///
+ /// @param[in] path Path to check
+ bool Exists(const TString& path);
+
+ /// Ensures that file exists
+ ///
+ /// @param[in] path Path to check
+ /// @returns input argument
+ inline const TString& EnsureExists(const TString& path) {
+ Y_ENSURE_EX(Exists(path), TFileError{} << "Path " << path << " does not exists (checked from cwd:" << NFs::CurrentWorkingDirectory() << ")");
+ return path;
+ }
+}
+
+Y_DECLARE_OPERATORS_FOR_FLAGS(NFs::EFilePermissions)
diff --git a/util/system/fs_ut.cpp b/util/system/fs_ut.cpp
new file mode 100644
index 0000000000..de071ebf55
--- /dev/null
+++ b/util/system/fs_ut.cpp
@@ -0,0 +1,325 @@
+#include "fs.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "file.h"
+#include "sysstat.h"
+#include "fstat.h"
+#include <util/folder/dirut.h>
+#include <util/folder/path.h>
+
+//WARNING: on windows the test must be run with administative rules
+
+class TFsTest: public TTestBase {
+ UNIT_TEST_SUITE(TFsTest);
+ UNIT_TEST(TestCreateRemove);
+ UNIT_TEST(TestRename);
+ UNIT_TEST(TestSymlink);
+ UNIT_TEST(TestHardlink);
+ UNIT_TEST(TestCwdOpts);
+ UNIT_TEST(TestEnsureExists);
+ UNIT_TEST_SUITE_END();
+
+public:
+ void TestCreateRemove();
+ void TestRename();
+ void TestSymlink();
+ void TestHardlink();
+ void TestCwdOpts();
+ void TestEnsureExists();
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TFsTest);
+
+static void Touch(const TFsPath& path) {
+ TFile file(path, CreateAlways | WrOnly);
+ file.Write("123", 3);
+}
+
+void TFsTest::TestCreateRemove() {
+ TFsPath dir1 = "dir_aбвг";
+ NFs::RemoveRecursive(dir1);
+ UNIT_ASSERT(!NFs::Exists(dir1));
+ UNIT_ASSERT(NFs::MakeDirectory(dir1));
+
+ UNIT_ASSERT(TFileStat(dir1).IsDir());
+ UNIT_ASSERT(!NFs::MakeDirectory(dir1));
+
+ UNIT_ASSERT(NFs::Exists(dir1));
+ TFsPath subdir1 = dir1 / "a" / "b";
+ //TFsPath link = dir1 / "link";
+
+ UNIT_ASSERT(NFs::MakeDirectoryRecursive(subdir1, NFs::FP_COMMON_FILE, true));
+ UNIT_ASSERT(NFs::Exists(subdir1));
+ UNIT_ASSERT(NFs::MakeDirectoryRecursive(subdir1, NFs::FP_COMMON_FILE, false));
+ UNIT_ASSERT(NFs::MakeDirectoryRecursive(subdir1, NFs::FP_COMMON_FILE));
+ UNIT_ASSERT_EXCEPTION(NFs::MakeDirectoryRecursive(subdir1, NFs::FP_COMMON_FILE, true), TIoException);
+
+ TFsPath file1 = dir1 / "f1.txt";
+ TFsPath file2 = subdir1 + TString("_f2.txt");
+ TFsPath file3 = subdir1 / "f2.txt";
+ Touch(file1);
+ Touch(file2);
+ Touch(file3);
+ //UNIT_ASSERT(NFs::SymLink(file3.RealPath(), link));
+
+ UNIT_ASSERT(NFs::MakeDirectoryRecursive(dir1 / "subdir1" / "subdir2" / "subdir3" / "subdir4", NFs::FP_COMMON_FILE, false));
+ UNIT_ASSERT(NFs::MakeDirectoryRecursive(dir1 / "subdir1" / "subdir2", NFs::FP_COMMON_FILE, false));
+
+ // the target path is a file or "subdirectory" of a file
+ UNIT_ASSERT(!NFs::MakeDirectoryRecursive(file1 / "subdir1" / "subdir2", NFs::FP_COMMON_FILE, false));
+ UNIT_ASSERT(!NFs::MakeDirectoryRecursive(file1, NFs::FP_COMMON_FILE, false));
+
+ TString longUtf8Name = "";
+ while (longUtf8Name.size() < 255) {
+ longUtf8Name = longUtf8Name + "fф";
+ }
+ UNIT_ASSERT_EQUAL(longUtf8Name.size(), 255);
+ TFsPath longfile = dir1 / longUtf8Name;
+ Touch(longfile);
+
+ UNIT_ASSERT(NFs::Exists(longfile));
+ UNIT_ASSERT(NFs::Exists(file1));
+ UNIT_ASSERT(NFs::Exists(file2));
+ UNIT_ASSERT(NFs::Exists(file3));
+ //UNIT_ASSERT(NFs::Exists(link));
+
+ UNIT_ASSERT(!NFs::Remove(dir1));
+ NFs::RemoveRecursive(dir1);
+
+ UNIT_ASSERT(!NFs::Exists(file1));
+ UNIT_ASSERT(!NFs::Exists(file2));
+ UNIT_ASSERT(!NFs::Exists(file3));
+ //UNIT_ASSERT(!NFs::Exists(link));
+ UNIT_ASSERT(!NFs::Exists(subdir1));
+ UNIT_ASSERT(!NFs::Exists(longfile));
+ UNIT_ASSERT(!NFs::Exists(dir1));
+}
+
+void RunRenameTest(TFsPath src, TFsPath dst) {
+ // if previous running was failed
+ TFsPath dir1 = "dir";
+ TFsPath dir2 = "dst_dir";
+
+ NFs::Remove(src);
+ NFs::Remove(dst);
+
+ NFs::Remove(dir1 / src);
+ NFs::Remove(dir1);
+ NFs::Remove(dir2 / src);
+ NFs::Remove(dir2);
+
+ {
+ TFile file(src, CreateNew | WrOnly);
+ file.Write("123", 3);
+ }
+
+ UNIT_ASSERT(NFs::Rename(src, dst));
+ UNIT_ASSERT(NFs::Exists(dst));
+ UNIT_ASSERT(!NFs::Exists(src));
+
+ {
+ TFile file(dst, OpenExisting);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 3);
+ }
+
+ NFs::MakeDirectory(dir1);
+ {
+ TFile file(dir1 / src, CreateNew | WrOnly);
+ file.Write("123", 3);
+ }
+ UNIT_ASSERT(NFs::Rename(dir1, dir2));
+ UNIT_ASSERT(NFs::Exists(dir2 / src));
+ UNIT_ASSERT(!NFs::Exists(dir1));
+
+ {
+ TFile file(dir2 / src, OpenExisting);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 3);
+ }
+
+ UNIT_ASSERT(!NFs::Remove(src));
+ UNIT_ASSERT(NFs::Remove(dst));
+ UNIT_ASSERT(!NFs::Remove(dir1));
+ UNIT_ASSERT(NFs::Remove(dir2 / src));
+ UNIT_ASSERT(NFs::Remove(dir2));
+}
+
+void TFsTest::TestRename() {
+ RunRenameTest("src.txt", "dst.txt");
+ RunRenameTest("src_абвг.txt", "dst_абвг.txt");
+}
+
+static void RunHardlinkTest(const TFsPath& src, const TFsPath& dst) {
+ NFs::Remove(src);
+ NFs::Remove(dst);
+
+ {
+ TFile file(src, CreateNew | WrOnly);
+ file.Write("123", 3);
+ }
+
+ UNIT_ASSERT(NFs::HardLink(src, dst));
+
+ {
+ TFile file(dst, OpenExisting | RdOnly);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 3);
+ }
+ {
+ TFile file(src, OpenExisting | WrOnly);
+ file.Write("1234", 4);
+ }
+ {
+ TFile file(dst, OpenExisting | RdOnly);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 4);
+ }
+ {
+ TFile file(dst, OpenExisting | WrOnly);
+ file.Write("12345", 5);
+ }
+
+ {
+ TFile file(src, OpenExisting | RdOnly);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 5);
+ }
+
+ UNIT_ASSERT(NFs::Remove(dst));
+ UNIT_ASSERT(NFs::Remove(src));
+}
+
+void TFsTest::TestHardlink() {
+ RunHardlinkTest("tempfile", "hardlinkfile");
+ RunHardlinkTest("tempfile_абвг", "hardlinkfile_абвг"); //utf-8 names
+}
+
+static void RunSymLinkTest(TString fileLocalName, TString symLinkName) {
+ // if previous running was failed
+ TFsPath subDir = "tempsubdir";
+ TFsPath srcFile = subDir / fileLocalName;
+
+ TFsPath subsubDir1 = subDir / "dir1";
+ TFsPath subsubDir2 = subDir / "dir2";
+
+ TFsPath linkD1 = "symlinkdir";
+ TFsPath linkD2 = subsubDir1 / "linkd2";
+ TFsPath dangling = subsubDir1 / "dangling";
+
+ NFs::Remove(srcFile);
+ NFs::Remove(symLinkName);
+ NFs::Remove(linkD2);
+ NFs::Remove(dangling);
+ NFs::Remove(subsubDir1);
+ NFs::Remove(subsubDir2);
+ NFs::Remove(subDir);
+ NFs::Remove(linkD1);
+
+ NFs::MakeDirectory(subDir);
+ NFs::MakeDirectory(subsubDir1, NFs::FP_NONSECRET_FILE);
+ NFs::MakeDirectory(subsubDir2, NFs::FP_SECRET_FILE);
+ {
+ TFile file(srcFile, CreateNew | WrOnly);
+ file.Write("1234567", 7);
+ }
+ UNIT_ASSERT(NFs::SymLink(subDir, linkD1));
+ UNIT_ASSERT(NFs::SymLink("../dir2", linkD2));
+ UNIT_ASSERT(NFs::SymLink("../dir3", dangling));
+ UNIT_ASSERT_STRINGS_EQUAL(NFs::ReadLink(linkD2), TString("..") + LOCSLASH_S "dir2");
+ UNIT_ASSERT_STRINGS_EQUAL(NFs::ReadLink(dangling), TString("..") + LOCSLASH_S "dir3");
+ {
+ TFile file(linkD1 / fileLocalName, OpenExisting | RdOnly);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 7);
+ }
+ UNIT_ASSERT(NFs::SymLink(srcFile, symLinkName));
+ {
+ TFile file(symLinkName, OpenExisting | RdOnly);
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 7);
+ }
+ {
+ TFileStat fs(linkD1);
+ UNIT_ASSERT(!fs.IsFile());
+ UNIT_ASSERT(fs.IsDir());
+ UNIT_ASSERT(!fs.IsSymlink());
+ }
+ {
+ TFileStat fs(linkD1, true);
+ UNIT_ASSERT(!fs.IsFile());
+ //UNIT_ASSERT(fs.IsDir()); // failed on unix
+ UNIT_ASSERT(fs.IsSymlink());
+ }
+ {
+ TFileStat fs(symLinkName);
+ UNIT_ASSERT(fs.IsFile());
+ UNIT_ASSERT(!fs.IsDir());
+ UNIT_ASSERT(!fs.IsSymlink());
+ UNIT_ASSERT_VALUES_EQUAL(fs.Size, 7u);
+ }
+
+ {
+ TFileStat fs(symLinkName, true);
+ //UNIT_ASSERT(fs.IsFile()); // no evidence that symlink has to be a file as well
+ UNIT_ASSERT(!fs.IsDir());
+ UNIT_ASSERT(fs.IsSymlink());
+ }
+
+ UNIT_ASSERT(NFs::Remove(symLinkName));
+ UNIT_ASSERT(NFs::Exists(srcFile));
+
+ UNIT_ASSERT(NFs::Remove(linkD1));
+ UNIT_ASSERT(NFs::Exists(srcFile));
+
+ UNIT_ASSERT(!NFs::Remove(subDir));
+
+ UNIT_ASSERT(NFs::Remove(srcFile));
+ UNIT_ASSERT(NFs::Remove(linkD2));
+ UNIT_ASSERT(NFs::Remove(dangling));
+ UNIT_ASSERT(NFs::Remove(subsubDir1));
+ UNIT_ASSERT(NFs::Remove(subsubDir2));
+ UNIT_ASSERT(NFs::Remove(subDir));
+}
+
+void TFsTest::TestSymlink() {
+ // if previous running was failed
+ RunSymLinkTest("f.txt", "fl.txt");
+ RunSymLinkTest("f_абвг.txt", "fl_абвг.txt"); //utf-8 names
+}
+
+void TFsTest::TestCwdOpts() {
+ TFsPath initialCwd = NFs::CurrentWorkingDirectory();
+ TFsPath subdir = "dir_forcwd_абвг";
+ NFs::MakeDirectory(subdir, NFs::FP_SECRET_FILE | NFs::FP_ALL_READ);
+ NFs::SetCurrentWorkingDirectory(subdir);
+ TFsPath newCwd = NFs::CurrentWorkingDirectory();
+
+ UNIT_ASSERT_EQUAL(newCwd.Fix(), (initialCwd / subdir).Fix());
+
+ NFs::SetCurrentWorkingDirectory("..");
+ TFsPath newCwd2 = NFs::CurrentWorkingDirectory();
+ UNIT_ASSERT_EQUAL(newCwd2.Fix(), initialCwd.Fix());
+ UNIT_ASSERT(NFs::Remove(subdir));
+}
+
+void TFsTest::TestEnsureExists() {
+ TFsPath fileExists = "tmp_file_абвг.txt";
+ TFsPath nonExists = "tmp2_file_абвг.txt";
+ {
+ NFs::Remove(fileExists);
+ NFs::Remove(nonExists);
+ TFile file(fileExists, CreateNew | WrOnly);
+ file.Write("1234567", 7);
+ }
+
+ UNIT_ASSERT_NO_EXCEPTION(NFs::EnsureExists(fileExists));
+ UNIT_ASSERT_EXCEPTION(NFs::EnsureExists(nonExists), TFileError);
+
+ TStringBuilder expected;
+ TString got;
+ try {
+ NFs::EnsureExists(nonExists);
+ expected << __LOCATION__;
+ } catch (const TFileError& err) {
+ got = err.what();
+ }
+ UNIT_ASSERT(got.Contains(expected));
+ UNIT_ASSERT(got.Contains(NFs::CurrentWorkingDirectory()));
+
+ UNIT_ASSERT(NFs::Remove(fileExists));
+}
diff --git a/util/system/fs_win.cpp b/util/system/fs_win.cpp
new file mode 100644
index 0000000000..a410ccac06
--- /dev/null
+++ b/util/system/fs_win.cpp
@@ -0,0 +1,233 @@
+#include "fs_win.h"
+#include "defaults.h"
+#include "maxlen.h"
+
+#include <util/folder/dirut.h>
+#include <util/charset/wide.h>
+#include "file.h"
+
+#include <winioctl.h>
+
+namespace NFsPrivate {
+ static LPCWSTR UTF8ToWCHAR(const TStringBuf str, TUtf16String& wstr) {
+ wstr.resize(str.size());
+ size_t written = 0;
+ if (!UTF8ToWide(str.data(), str.size(), wstr.begin(), written))
+ return nullptr;
+ wstr.erase(written);
+ static_assert(sizeof(WCHAR) == sizeof(wchar16), "expect sizeof(WCHAR) == sizeof(wchar16)");
+ return (const WCHAR*)wstr.data();
+ }
+
+ static TString WCHARToUTF8(const LPWSTR wstr, size_t len) {
+ static_assert(sizeof(WCHAR) == sizeof(wchar16), "expect sizeof(WCHAR) == sizeof(wchar16)");
+
+ return WideToUTF8((wchar16*)wstr, len);
+ }
+
+ HANDLE CreateFileWithUtf8Name(const TStringBuf fName, ui32 accessMode, ui32 shareMode, ui32 createMode, ui32 attributes, bool inheritHandle) {
+ TUtf16String wstr;
+ LPCWSTR wname = UTF8ToWCHAR(fName, wstr);
+ if (!wname) {
+ ::SetLastError(ERROR_INVALID_NAME);
+ return INVALID_HANDLE_VALUE;
+ }
+ SECURITY_ATTRIBUTES secAttrs;
+ secAttrs.bInheritHandle = inheritHandle ? TRUE : FALSE;
+ secAttrs.lpSecurityDescriptor = nullptr;
+ secAttrs.nLength = sizeof(secAttrs);
+ return ::CreateFileW(wname, accessMode, shareMode, &secAttrs, createMode, attributes, nullptr);
+ }
+
+ bool WinRename(const TString& oldPath, const TString& newPath) {
+ TUtf16String op, np;
+ LPCWSTR opPtr = UTF8ToWCHAR(oldPath, op);
+ LPCWSTR npPtr = UTF8ToWCHAR(newPath, np);
+ if (!opPtr || !npPtr) {
+ ::SetLastError(ERROR_INVALID_NAME);
+ return false;
+ }
+
+ return MoveFileExW(opPtr, npPtr, MOVEFILE_REPLACE_EXISTING) != 0;
+ }
+
+ bool WinRemove(const TString& path) {
+ TUtf16String wstr;
+ LPCWSTR wname = UTF8ToWCHAR(path, wstr);
+ if (!wname) {
+ ::SetLastError(ERROR_INVALID_NAME);
+ return false;
+ }
+ WIN32_FILE_ATTRIBUTE_DATA fad;
+ if (::GetFileAttributesExW(wname, GetFileExInfoStandard, &fad)) {
+ if (fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ return ::RemoveDirectoryW(wname) != 0;
+ return ::DeleteFileW(wname) != 0;
+ }
+
+ return false;
+ }
+
+ bool WinSymLink(const TString& targetName, const TString& linkName) {
+ TString tName(targetName);
+ {
+ size_t pos;
+ while ((pos = tName.find('/')) != TString::npos)
+ tName.replace(pos, 1, LOCSLASH_S);
+ }
+ TUtf16String tstr;
+ LPCWSTR wname = UTF8ToWCHAR(tName, tstr);
+ TUtf16String lstr;
+ LPCWSTR lname = UTF8ToWCHAR(linkName, lstr);
+
+ // we can't create a dangling link to a dir in this way
+ ui32 attr = ::GetFileAttributesW(wname);
+ if (attr == INVALID_FILE_ATTRIBUTES) {
+ TTempBuf result;
+ if (GetFullPathNameW(lname, result.Size(), (LPWSTR)result.Data(), 0) != 0) {
+ TString fullPath = WideToUTF8(TWtringBuf((const wchar16*)result.Data()));
+ TStringBuf linkDir(fullPath);
+ linkDir.RNextTok('\\');
+
+ if (linkDir) {
+ TString fullTarget(tName);
+ resolvepath(fullTarget, TString{linkDir});
+ TUtf16String fullTargetW;
+ LPCWSTR ptrFullTarget = UTF8ToWCHAR(fullTarget, fullTargetW);
+ attr = ::GetFileAttributesW(ptrFullTarget);
+ }
+ }
+ }
+ return 0 != CreateSymbolicLinkW(lname, wname, attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY) ? SYMBOLIC_LINK_FLAG_DIRECTORY : 0);
+ }
+
+ bool WinHardLink(const TString& existingPath, const TString& newPath) {
+ TUtf16String ep, np;
+ LPCWSTR epPtr = UTF8ToWCHAR(existingPath, ep);
+ LPCWSTR npPtr = UTF8ToWCHAR(newPath, np);
+ if (!epPtr || !npPtr) {
+ ::SetLastError(ERROR_INVALID_NAME);
+ return false;
+ }
+
+ return (CreateHardLinkW(npPtr, epPtr, nullptr) != 0);
+ }
+
+ bool WinExists(const TString& path) {
+ TUtf16String buf;
+ LPCWSTR ptr = UTF8ToWCHAR(path, buf);
+ return ::GetFileAttributesW(ptr) != INVALID_FILE_ATTRIBUTES;
+ }
+
+ TString WinCurrentWorkingDirectory() {
+ TTempBuf result;
+ LPWSTR buf = reinterpret_cast<LPWSTR>(result.Data());
+ int r = GetCurrentDirectoryW(result.Size() / sizeof(WCHAR), buf);
+ if (r == 0)
+ throw TIoSystemError() << "failed to GetCurrentDirectory";
+ return WCHARToUTF8(buf, r);
+ }
+
+ bool WinSetCurrentWorkingDirectory(const TString& path) {
+ TUtf16String wstr;
+ LPCWSTR wname = UTF8ToWCHAR(path, wstr);
+ if (!wname) {
+ ::SetLastError(ERROR_INVALID_NAME);
+ return false;
+ }
+ return SetCurrentDirectoryW(wname);
+ }
+
+ bool WinMakeDirectory(const TString path) {
+ TUtf16String buf;
+ LPCWSTR ptr = UTF8ToWCHAR(path, buf);
+ return CreateDirectoryW(ptr, (LPSECURITY_ATTRIBUTES) nullptr);
+ }
+ // edited part of <Ntifs.h> from Windows DDK
+
+#define SYMLINK_FLAG_RELATIVE 1
+
+ struct TReparseBufferHeader {
+ USHORT SubstituteNameOffset;
+ USHORT SubstituteNameLength;
+ USHORT PrintNameOffset;
+ USHORT PrintNameLength;
+ };
+
+ struct TSymbolicLinkReparseBuffer: public TReparseBufferHeader {
+ ULONG Flags; // 0 or SYMLINK_FLAG_RELATIVE
+ wchar16 PathBuffer[1];
+ };
+
+ struct TMountPointReparseBuffer: public TReparseBufferHeader {
+ wchar16 PathBuffer[1];
+ };
+
+ struct TGenericReparseBuffer {
+ wchar16 DataBuffer[1];
+ };
+
+ struct REPARSE_DATA_BUFFER {
+ ULONG ReparseTag;
+ USHORT ReparseDataLength;
+ USHORT Reserved;
+ union {
+ TSymbolicLinkReparseBuffer SymbolicLinkReparseBuffer;
+ TMountPointReparseBuffer MountPointReparseBuffer;
+ TGenericReparseBuffer GenericReparseBuffer;
+ };
+ };
+
+ // the end of edited part of <Ntifs.h>
+
+ TString WinReadLink(const TString& name) {
+ TFileHandle h = CreateFileWithUtf8Name(name, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING,
+ FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, true);
+ TTempBuf buf;
+ while (true) {
+ DWORD bytesReturned = 0;
+ BOOL res = DeviceIoControl(h, FSCTL_GET_REPARSE_POINT, nullptr, 0, buf.Data(), buf.Size(), &bytesReturned, nullptr);
+ if (res) {
+ REPARSE_DATA_BUFFER* rdb = (REPARSE_DATA_BUFFER*)buf.Data();
+ if (rdb->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
+ wchar16* str = (wchar16*)&rdb->SymbolicLinkReparseBuffer.PathBuffer[rdb->SymbolicLinkReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
+ size_t len = rdb->SymbolicLinkReparseBuffer.SubstituteNameLength / sizeof(wchar16);
+ return WideToUTF8(str, len);
+ } else if (rdb->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
+ wchar16* str = (wchar16*)&rdb->MountPointReparseBuffer.PathBuffer[rdb->MountPointReparseBuffer.SubstituteNameOffset / sizeof(wchar16)];
+ size_t len = rdb->MountPointReparseBuffer.SubstituteNameLength / sizeof(wchar16);
+ return WideToUTF8(str, len);
+ }
+ //this reparse point is unsupported in arcadia
+ return TString();
+ } else {
+ if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
+ buf = TTempBuf(buf.Size() * 2);
+ } else {
+ ythrow yexception() << "can't read link " << name;
+ }
+ }
+ }
+ }
+
+ // we can't use this function to get an analog of unix inode due to a lot of NTFS folders do not have this GUID
+ //(it will be 'create' case really)
+ /*
+bool GetObjectId(const char* path, GUID* id) {
+ TFileHandle h = CreateFileWithUtf8Name(path, 0, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
+ OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT|FILE_FLAG_BACKUP_SEMANTICS, true);
+ if (h.IsOpen()) {
+ FILE_OBJECTID_BUFFER fob;
+ DWORD resSize = 0;
+ if (DeviceIoControl(h, FSCTL_CREATE_OR_GET_OBJECT_ID, nullptr, 0, &fob, sizeof(fob), &resSize, nullptr)) {
+ Y_ASSERT(resSize == sizeof(fob));
+ memcpy(id, &fob.ObjectId, sizeof(GUID));
+ return true;
+ }
+ }
+ memset(id, 0, sizeof(GUID));
+ return false;
+}
+*/
+
+}
diff --git a/util/system/fs_win.h b/util/system/fs_win.h
new file mode 100644
index 0000000000..8086129828
--- /dev/null
+++ b/util/system/fs_win.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "winint.h"
+#include "defaults.h"
+
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+
+namespace NFsPrivate {
+ bool WinRename(const TString& oldPath, const TString& newPath);
+
+ bool WinSymLink(const TString& targetName, const TString& linkName);
+
+ bool WinHardLink(const TString& existingPath, const TString& newPath);
+
+ TString WinReadLink(const TString& path);
+
+ HANDLE CreateFileWithUtf8Name(const TStringBuf fName, ui32 accessMode, ui32 shareMode, ui32 createMode, ui32 attributes, bool inheritHandle);
+
+ bool WinRemove(const TString& path);
+
+ bool WinExists(const TString& path);
+
+ TString WinCurrentWorkingDirectory();
+
+ bool WinSetCurrentWorkingDirectory(const TString& path);
+
+ bool WinMakeDirectory(const TString path);
+}
diff --git a/util/system/fstat.cpp b/util/system/fstat.cpp
new file mode 100644
index 0000000000..81e98cbc6b
--- /dev/null
+++ b/util/system/fstat.cpp
@@ -0,0 +1,215 @@
+#include "fstat.h"
+#include "file.h"
+
+#include <sys/stat.h>
+
+#include <util/folder/path.h>
+
+#include <cerrno>
+
+#if defined(_win_)
+ #include "fs_win.h"
+
+ #ifdef _S_IFLNK
+ #undef _S_IFLNK
+ #endif
+ #define _S_IFLNK 0x80000000
+
+ui32 GetFileMode(DWORD fileAttributes) {
+ ui32 mode = 0;
+ if (fileAttributes == 0xFFFFFFFF)
+ return mode;
+ if (fileAttributes & FILE_ATTRIBUTE_DEVICE)
+ mode |= _S_IFCHR;
+ if (fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+ mode |= _S_IFLNK; // todo: was undefined by the moment of writing this code
+ if (fileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ mode |= _S_IFDIR;
+ if (fileAttributes & (FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE))
+ mode |= _S_IFREG;
+ if ((fileAttributes & FILE_ATTRIBUTE_READONLY) == 0)
+ mode |= _S_IWRITE;
+ return mode;
+}
+
+ #define S_ISDIR(st_mode) (st_mode & _S_IFDIR)
+ #define S_ISREG(st_mode) (st_mode & _S_IFREG)
+ #define S_ISLNK(st_mode) (st_mode & _S_IFLNK)
+
+using TSystemFStat = BY_HANDLE_FILE_INFORMATION;
+
+#else
+
+using TSystemFStat = struct stat;
+
+#endif
+
+static void MakeStat(TFileStat& st, const TSystemFStat& fs) {
+#ifdef _unix_
+ st.Mode = fs.st_mode;
+ st.NLinks = fs.st_nlink;
+ st.Uid = fs.st_uid;
+ st.Gid = fs.st_gid;
+ st.Size = fs.st_size;
+ st.AllocationSize = fs.st_blocks * 512;
+ st.ATime = fs.st_atime;
+ st.MTime = fs.st_mtime;
+ st.CTime = fs.st_ctime;
+ st.INode = fs.st_ino;
+#else
+ timeval tv;
+ FileTimeToTimeval(&fs.ftCreationTime, &tv);
+ st.CTime = tv.tv_sec;
+ FileTimeToTimeval(&fs.ftLastAccessTime, &tv);
+ st.ATime = tv.tv_sec;
+ FileTimeToTimeval(&fs.ftLastWriteTime, &tv);
+ st.MTime = tv.tv_sec;
+ st.NLinks = fs.nNumberOfLinks;
+ st.Mode = GetFileMode(fs.dwFileAttributes);
+ st.Uid = 0;
+ st.Gid = 0;
+ st.Size = ((ui64)fs.nFileSizeHigh << 32) | fs.nFileSizeLow;
+ st.AllocationSize = st.Size; // FIXME
+ st.INode = ((ui64)fs.nFileIndexHigh << 32) | fs.nFileIndexLow;
+#endif
+}
+
+static bool GetStatByHandle(TSystemFStat& fs, FHANDLE f) {
+#ifdef _win_
+ return GetFileInformationByHandle(f, &fs);
+#else
+ return !fstat(f, &fs);
+#endif
+}
+
+static bool GetStatByName(TSystemFStat& fs, const char* fileName, bool nofollow) {
+#ifdef _win_
+ TFileHandle h = NFsPrivate::CreateFileWithUtf8Name(fileName, FILE_READ_ATTRIBUTES | FILE_READ_EA, FILE_SHARE_READ | FILE_SHARE_WRITE,
+ OPEN_EXISTING,
+ (nofollow ? FILE_FLAG_OPEN_REPARSE_POINT : 0) | FILE_FLAG_BACKUP_SEMANTICS, true);
+ if (!h.IsOpen()) {
+ return false;
+ }
+ return GetStatByHandle(fs, h);
+#else
+ return !(nofollow ? lstat : stat)(fileName, &fs);
+#endif
+}
+
+TFileStat::TFileStat() = default;
+
+TFileStat::TFileStat(const TFile& f) {
+ *this = TFileStat(f.GetHandle());
+}
+
+TFileStat::TFileStat(FHANDLE f) {
+ TSystemFStat st;
+ if (GetStatByHandle(st, f)) {
+ MakeStat(*this, st);
+ } else {
+ *this = TFileStat();
+ }
+}
+
+void TFileStat::MakeFromFileName(const char* fileName, bool nofollow) {
+ TSystemFStat st;
+ if (GetStatByName(st, fileName, nofollow)) {
+ MakeStat(*this, st);
+ } else {
+ *this = TFileStat();
+ }
+}
+
+TFileStat::TFileStat(const TFsPath& fileName, bool nofollow) {
+ MakeFromFileName(fileName.GetPath().data(), nofollow);
+}
+
+TFileStat::TFileStat(const TString& fileName, bool nofollow) {
+ MakeFromFileName(fileName.data(), nofollow);
+}
+
+TFileStat::TFileStat(const char* fileName, bool nofollow) {
+ MakeFromFileName(fileName, nofollow);
+}
+
+bool TFileStat::IsNull() const noexcept {
+ return *this == TFileStat();
+}
+
+bool TFileStat::IsFile() const noexcept {
+ return S_ISREG(Mode);
+}
+
+bool TFileStat::IsDir() const noexcept {
+ return S_ISDIR(Mode);
+}
+
+bool TFileStat::IsSymlink() const noexcept {
+ return S_ISLNK(Mode);
+}
+
+bool operator==(const TFileStat& l, const TFileStat& r) noexcept {
+ return l.Mode == r.Mode &&
+ l.Uid == r.Uid &&
+ l.Gid == r.Gid &&
+ l.NLinks == r.NLinks &&
+ l.Size == r.Size &&
+ l.ATime == r.ATime &&
+ l.MTime == r.MTime &&
+ l.CTime == r.CTime;
+}
+
+bool operator!=(const TFileStat& l, const TFileStat& r) noexcept {
+ return !(l == r);
+}
+
+i64 GetFileLength(FHANDLE fd) {
+#if defined(_win_)
+ LARGE_INTEGER pos;
+ if (!::GetFileSizeEx(fd, &pos))
+ return -1L;
+ return pos.QuadPart;
+#elif defined(_unix_)
+ struct stat statbuf;
+ if (::fstat(fd, &statbuf) != 0) {
+ return -1L;
+ }
+ if (!(statbuf.st_mode & (S_IFREG | S_IFBLK | S_IFCHR))) {
+ // st_size only makes sense for regular files or devices
+ errno = EINVAL;
+ return -1L;
+ }
+ return statbuf.st_size;
+#else
+ #error unsupported platform
+#endif
+}
+
+i64 GetFileLength(const char* name) {
+#if defined(_win_)
+ WIN32_FIND_DATA fData;
+ HANDLE h = FindFirstFileA(name, &fData);
+ if (h == INVALID_HANDLE_VALUE)
+ return -1;
+ FindClose(h);
+ return (((i64)fData.nFileSizeHigh) * (i64(MAXDWORD) + 1)) + (i64)fData.nFileSizeLow;
+#elif defined(_unix_)
+ struct stat buf;
+ int r = ::stat(name, &buf);
+ if (r == -1) {
+ return -1;
+ }
+ if (!(buf.st_mode & (S_IFREG | S_IFBLK | S_IFCHR))) {
+ // st_size only makes sense for regular files or devices
+ errno = EINVAL;
+ return -1;
+ }
+ return (i64)buf.st_size;
+#else
+ #error unsupported platform
+#endif
+}
+
+i64 GetFileLength(const TString& name) {
+ return GetFileLength(name.data());
+}
diff --git a/util/system/fstat.h b/util/system/fstat.h
new file mode 100644
index 0000000000..64e79e1b55
--- /dev/null
+++ b/util/system/fstat.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+#include <util/system/fhandle.h>
+
+class TFile;
+class TFsPath;
+
+struct TFileStat {
+ ui32 Mode = 0; /* protection */
+ ui32 Uid = 0; /* user ID of owner */
+ ui32 Gid = 0; /* group ID of owner */
+
+ ui64 NLinks = 0; /* number of hard links */
+ ui64 Size = 0; /* total size, in bytes */
+ ui64 INode = 0; /* inode number */
+ ui64 AllocationSize = 0; /* number of bytes allocated on the disk */
+
+ time_t ATime = 0; /* time of last access */
+ time_t MTime = 0; /* time of last modification */
+ time_t CTime = 0; /* time of last status change */
+
+public:
+ TFileStat();
+
+ bool IsNull() const noexcept;
+
+ bool IsFile() const noexcept;
+ bool IsDir() const noexcept;
+ bool IsSymlink() const noexcept;
+
+ explicit TFileStat(const TFile& f);
+ explicit TFileStat(FHANDLE f);
+ TFileStat(const TFsPath& fileName, bool nofollow = false);
+ TFileStat(const TString& fileName, bool nofollow = false);
+ TFileStat(const char* fileName, bool nofollow = false);
+
+ friend bool operator==(const TFileStat& l, const TFileStat& r) noexcept;
+ friend bool operator!=(const TFileStat& l, const TFileStat& r) noexcept;
+
+private:
+ void MakeFromFileName(const char* fileName, bool nofollow);
+};
+
+i64 GetFileLength(FHANDLE fd);
+i64 GetFileLength(const char* name);
+i64 GetFileLength(const TString& name);
diff --git a/util/system/fstat_ut.cpp b/util/system/fstat_ut.cpp
new file mode 100644
index 0000000000..160ecd936e
--- /dev/null
+++ b/util/system/fstat_ut.cpp
@@ -0,0 +1,157 @@
+#include "fstat.h"
+#include "file.h"
+#include "sysstat.h"
+#include "fs.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/tests_data.h>
+
+#include <util/folder/path.h>
+
+Y_UNIT_TEST_SUITE(TestFileStat) {
+ Y_UNIT_TEST(FileTest) {
+ TString fileName = "f1.txt";
+ TFileStat oFs;
+ {
+ TFile file(fileName.data(), OpenAlways | WrOnly);
+ file.Write("1234567", 7);
+
+ {
+ TFileStat fs(file);
+ UNIT_ASSERT(fs.IsFile());
+ UNIT_ASSERT(!fs.IsDir());
+ UNIT_ASSERT(!fs.IsSymlink());
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), (i64)fs.Size);
+ UNIT_ASSERT(fs.MTime >= fs.CTime);
+ UNIT_ASSERT(fs.NLinks == 1);
+ oFs = fs;
+ }
+
+ UNIT_ASSERT(file.IsOpen());
+ UNIT_ASSERT_VALUES_EQUAL(file.GetLength(), 7);
+ file.Close();
+ }
+ TFileStat cFs(fileName);
+ UNIT_ASSERT(cFs.IsFile());
+ UNIT_ASSERT(!cFs.IsDir());
+ UNIT_ASSERT(!cFs.IsSymlink());
+ UNIT_ASSERT_VALUES_EQUAL(cFs.Size, oFs.Size);
+ UNIT_ASSERT(cFs.MTime >= oFs.MTime);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.CTime, oFs.CTime);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.NLinks, oFs.NLinks);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.Mode, oFs.Mode);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.Uid, oFs.Uid);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.Gid, oFs.Gid);
+ UNIT_ASSERT_VALUES_EQUAL(cFs.INode, oFs.INode);
+ UNIT_ASSERT(unlink(fileName.data()) == 0);
+ }
+
+ Y_UNIT_TEST(DirTest) {
+ Mkdir("tmpd", MODE0777);
+ TFileStat fs("tmpd");
+ UNIT_ASSERT(!fs.IsFile());
+ UNIT_ASSERT(fs.IsDir());
+ UNIT_ASSERT(!fs.IsSymlink());
+ //UNIT_ASSERT(fs.Size == 0); // it fails under unix
+ UNIT_ASSERT(NFs::Remove("tmpd"));
+ fs = TFileStat("tmpd");
+ UNIT_ASSERT(!fs.IsFile());
+ UNIT_ASSERT(!fs.IsDir());
+ UNIT_ASSERT(!fs.IsSymlink());
+ UNIT_ASSERT(fs.Size == 0);
+ UNIT_ASSERT(fs.CTime == 0);
+ }
+
+ Y_UNIT_TEST(SymlinkToExistingFileTest) {
+ const auto path = GetOutputPath() / "file_1";
+ const auto link = GetOutputPath() / "symlink_1";
+ TFile(path, EOpenModeFlag::CreateNew | EOpenModeFlag::RdWr);
+ UNIT_ASSERT(NFs::SymLink(path, link));
+
+ const TFileStat statNoFollow(link, false);
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsFile(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsSymlink(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsDir(), ToString(statNoFollow.Mode));
+
+ const TFileStat statFollow(link, true);
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsNull(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsFile(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statFollow.IsSymlink(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsDir(), ToString(statFollow.Mode));
+ }
+
+ Y_UNIT_TEST(SymlinkToNonExistingFileTest) {
+ const auto path = GetOutputPath() / "file_2";
+ const auto link = GetOutputPath() / "symlink_2";
+ UNIT_ASSERT(NFs::SymLink(path, link));
+
+ const TFileStat statNoFollow(link, false);
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsFile(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsSymlink(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsDir(), ToString(statNoFollow.Mode));
+
+ const TFileStat statFollow(link, true);
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsNull(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsFile(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statFollow.IsSymlink(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsDir(), ToString(statFollow.Mode));
+ }
+
+ Y_UNIT_TEST(SymlinkToFileThatCantExistTest) {
+ const auto path = TFsPath("/path") / "that" / "does" / "not" / "exists";
+ const auto link = GetOutputPath() / "symlink_3";
+ UNIT_ASSERT(NFs::SymLink(path, link));
+
+ const TFileStat statNoFollow(link, false);
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsFile(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsSymlink(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsDir(), ToString(statNoFollow.Mode));
+
+ const TFileStat statFollow(link, true);
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsNull(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsFile(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statFollow.IsSymlink(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsDir(), ToString(statFollow.Mode));
+ }
+
+ Y_UNIT_TEST(FileDoesNotExistTest) {
+ const auto path = TFsPath("/path") / "that" / "does" / "not" / "exists";
+
+ const TFileStat statNoFollow(path, false);
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statNoFollow.IsNull(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsFile(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsSymlink(), ToString(statNoFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statNoFollow.IsDir(), ToString(statNoFollow.Mode));
+
+ const TFileStat statFollow(path, true);
+ UNIT_ASSERT_VALUES_EQUAL_C(true, statFollow.IsNull(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsFile(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsSymlink(), ToString(statFollow.Mode));
+ UNIT_ASSERT_VALUES_EQUAL_C(false, statFollow.IsDir(), ToString(statFollow.Mode));
+ }
+
+ Y_UNIT_TEST(ChmodTest) {
+ const TString fileName = "m.txt";
+ TFile file(fileName.c_str(), OpenAlways | WrOnly);
+ file.Write("1", 1);
+ file.Close();
+
+ const TFileStat statDefault(fileName);
+ UNIT_ASSERT(Chmod(fileName.c_str(), statDefault.Mode) == 0);
+ const TFileStat statUnchanged(fileName);
+ UNIT_ASSERT_VALUES_EQUAL(statDefault.Mode, statUnchanged.Mode);
+
+ UNIT_ASSERT(Chmod(fileName.c_str(), S_IRUSR | S_IRGRP | S_IROTH) == 0);
+ const TFileStat statReadOnly(fileName);
+ UNIT_ASSERT_VALUES_UNEQUAL(statDefault.Mode, statReadOnly.Mode);
+ UNIT_ASSERT(Chmod(fileName.c_str(), statReadOnly.Mode) == 0);
+ UNIT_ASSERT_VALUES_EQUAL(statReadOnly.Mode, TFileStat(fileName).Mode);
+
+ UNIT_ASSERT(Chmod(fileName.c_str(), statDefault.Mode) == 0);
+ UNIT_ASSERT(unlink(fileName.c_str()) == 0);
+ }
+
+}
diff --git a/util/system/getpid.cpp b/util/system/getpid.cpp
new file mode 100644
index 0000000000..b9615f0dfa
--- /dev/null
+++ b/util/system/getpid.cpp
@@ -0,0 +1,23 @@
+#include "getpid.h"
+
+#ifdef _win_
+ // The include file should be Windows.h for Windows <=7, Processthreadsapi.h for Windows >=8 and Server 2012,
+ // see http://msdn.microsoft.com/en-us/library/windows/desktop/ms683180%28v=vs.85%29.aspx
+ // The way to determine windows version is described in http://msdn.microsoft.com/en-us/library/windows/desktop/aa383745%28v=vs.85%29.aspx
+ // with additions about Windows Server 2012 in https://social.msdn.microsoft.com/forums/vstudio/en-US/8d76d1d7-d078-4c55-963b-77e060845d0c/what-is-ntddiversion-value-for-ws-2012
+ #include <Windows.h>
+ #if defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8)
+ #include <processthreadsapi.h>
+ #endif
+#else
+ #include <sys/types.h>
+ #include <unistd.h>
+#endif
+
+TProcessId GetPID() {
+#ifdef _win_
+ return GetCurrentProcessId();
+#else
+ return getpid();
+#endif
+}
diff --git a/util/system/getpid.h b/util/system/getpid.h
new file mode 100644
index 0000000000..60e243266f
--- /dev/null
+++ b/util/system/getpid.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "platform.h"
+#include "types.h"
+
+#if defined(_win_)
+using TProcessId = ui32; // DWORD
+#else
+using TProcessId = pid_t;
+#endif
+
+TProcessId GetPID();
diff --git a/util/system/getpid_ut.cpp b/util/system/getpid_ut.cpp
new file mode 100644
index 0000000000..e7122a2971
--- /dev/null
+++ b/util/system/getpid_ut.cpp
@@ -0,0 +1,19 @@
+#include "getpid.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+class TGetPidTest: public TTestBase {
+ UNIT_TEST_SUITE(TGetPidTest);
+ UNIT_TEST(Test);
+ UNIT_TEST_SUITE_END();
+
+public:
+ void Test();
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TGetPidTest);
+
+void TGetPidTest::Test() {
+ const TProcessId pid = GetPID();
+ UNIT_ASSERT(pid != 0);
+}
diff --git a/util/system/guard.cpp b/util/system/guard.cpp
new file mode 100644
index 0000000000..29d14dd227
--- /dev/null
+++ b/util/system/guard.cpp
@@ -0,0 +1 @@
+#include "guard.h"
diff --git a/util/system/guard.h b/util/system/guard.h
new file mode 100644
index 0000000000..efc091d5f8
--- /dev/null
+++ b/util/system/guard.h
@@ -0,0 +1,176 @@
+#pragma once
+
+#include <util/generic/noncopyable.h>
+
+template <class T>
+struct TCommonLockOps {
+ static inline void Acquire(T* t) noexcept {
+ t->Acquire();
+ }
+
+ static inline void Release(T* t) noexcept {
+ t->Release();
+ }
+};
+
+template <class T>
+struct TTryLockOps: public TCommonLockOps<T> {
+ static inline bool TryAcquire(T* t) noexcept {
+ return t->TryAcquire();
+ }
+};
+
+//must be used with great care
+template <class TOps>
+struct TInverseLockOps: public TOps {
+ template <class T>
+ static inline void Acquire(T* t) noexcept {
+ TOps::Release(t);
+ }
+
+ template <class T>
+ static inline void Release(T* t) noexcept {
+ TOps::Acquire(t);
+ }
+};
+
+template <class T, class TOps = TCommonLockOps<T>>
+class TGuard: public TNonCopyable {
+public:
+ inline TGuard(const T& t) noexcept {
+ Init(&t);
+ }
+
+ inline TGuard(const T* t) noexcept {
+ Init(t);
+ }
+
+ inline TGuard(TGuard&& g) noexcept
+ : T_(g.T_)
+ {
+ g.T_ = nullptr;
+ }
+
+ inline ~TGuard() {
+ Release();
+ }
+
+ inline void Release() noexcept {
+ if (WasAcquired()) {
+ TOps::Release(T_);
+ T_ = nullptr;
+ }
+ }
+
+ explicit inline operator bool() const noexcept {
+ return WasAcquired();
+ }
+
+ inline bool WasAcquired() const noexcept {
+ return T_ != nullptr;
+ }
+
+ inline T* GetMutex() const noexcept {
+ return T_;
+ }
+
+private:
+ inline void Init(const T* t) noexcept {
+ T_ = const_cast<T*>(t);
+ TOps::Acquire(T_);
+ }
+
+private:
+ T* T_;
+};
+
+/*
+ * {
+ * auto guard = Guard(Lock_);
+ * some code under guard
+ * }
+ */
+template <class T>
+static inline TGuard<T> Guard(const T& t) {
+ return {&t};
+}
+
+/*
+ * with_lock (Lock_) {
+ * some code under guard
+ * }
+ */
+#define with_lock(X) \
+ if (auto Y_GENERATE_UNIQUE_ID(__guard) = ::Guard(X); false) { \
+ } else
+
+/*
+ * auto guard = Guard(Lock_);
+ * ... some code under lock
+ * {
+ * auto unguard = Unguard(guard);
+ * ... some code not under lock
+ * }
+ * ... some code under lock
+ */
+template <class T, class TOps = TCommonLockOps<T>>
+using TInverseGuard = TGuard<T, TInverseLockOps<TOps>>;
+
+template <class T, class TOps>
+static inline TInverseGuard<T, TOps> Unguard(const TGuard<T, TOps>& guard) {
+ return {guard.GetMutex()};
+}
+
+template <class T>
+static inline TInverseGuard<T> Unguard(const T& mutex) {
+ return {&mutex};
+}
+
+template <class T, class TOps = TTryLockOps<T>>
+class TTryGuard: public TNonCopyable {
+public:
+ inline TTryGuard(const T& t) noexcept {
+ Init(&t);
+ }
+
+ inline TTryGuard(const T* t) noexcept {
+ Init(t);
+ }
+
+ inline TTryGuard(TTryGuard&& g) noexcept
+ : T_(g.T_)
+ {
+ g.T_ = nullptr;
+ }
+
+ inline ~TTryGuard() {
+ Release();
+ }
+
+ inline void Release() noexcept {
+ if (WasAcquired()) {
+ TOps::Release(T_);
+ T_ = nullptr;
+ }
+ }
+
+ inline bool WasAcquired() const noexcept {
+ return T_ != nullptr;
+ }
+
+ explicit inline operator bool() const noexcept {
+ return WasAcquired();
+ }
+
+private:
+ inline void Init(const T* t) noexcept {
+ T_ = nullptr;
+ T* tMutable = const_cast<T*>(t);
+ if (TOps::TryAcquire(tMutable)) {
+ T_ = tMutable;
+ }
+ }
+
+private:
+ T* T_;
+};
diff --git a/util/system/guard_ut.cpp b/util/system/guard_ut.cpp
new file mode 100644
index 0000000000..404ede99ab
--- /dev/null
+++ b/util/system/guard_ut.cpp
@@ -0,0 +1,180 @@
+#include "guard.h"
+#include "rwlock.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/thread/pool.h>
+
+struct TTestGuard: public TTestBase {
+ UNIT_TEST_SUITE(TTestGuard);
+ UNIT_TEST(TestGuard)
+ UNIT_TEST(TestTryGuard)
+ UNIT_TEST(TestMove)
+ UNIT_TEST(TestSync)
+ UNIT_TEST(TestUnguard)
+ UNIT_TEST(TestTryReadGuard)
+ UNIT_TEST(TestWithLock)
+ UNIT_TEST(TestWithLockScope);
+ UNIT_TEST_SUITE_END();
+
+ struct TGuardChecker {
+ TGuardChecker()
+ : guarded(false)
+ {
+ }
+
+ void Acquire() {
+ guarded = true;
+ }
+ void Release() {
+ guarded = false;
+ }
+ bool TryAcquire() {
+ if (guarded) {
+ return false;
+ } else {
+ guarded = true;
+ return true;
+ }
+ }
+
+ bool guarded;
+ };
+
+ void TestUnguard() {
+ TGuardChecker m;
+
+ {
+ auto guard = Guard(m);
+
+ UNIT_ASSERT(m.guarded);
+
+ {
+ auto unguard = Unguard(guard);
+
+ UNIT_ASSERT(!m.guarded);
+ }
+
+ UNIT_ASSERT(m.guarded);
+ }
+
+ {
+ auto guard = Guard(m);
+
+ UNIT_ASSERT(m.guarded);
+
+ {
+ auto unguard = Unguard(m);
+
+ UNIT_ASSERT(!m.guarded);
+ }
+
+ UNIT_ASSERT(m.guarded);
+ }
+ }
+
+ void TestMove() {
+ TGuardChecker m;
+ size_t n = 0;
+
+ {
+ auto guard = Guard(m);
+
+ UNIT_ASSERT(m.guarded);
+ ++n;
+ }
+
+ UNIT_ASSERT(!m.guarded);
+ UNIT_ASSERT_VALUES_EQUAL(n, 1);
+ }
+
+ void TestSync() {
+ TGuardChecker m;
+ size_t n = 0;
+
+ with_lock (m) {
+ UNIT_ASSERT(m.guarded);
+ ++n;
+ }
+
+ UNIT_ASSERT(!m.guarded);
+ UNIT_ASSERT_VALUES_EQUAL(n, 1);
+ }
+
+ void TestGuard() {
+ TGuardChecker checker;
+
+ UNIT_ASSERT(!checker.guarded);
+ {
+ TGuard<TGuardChecker> guard(checker);
+ UNIT_ASSERT(checker.guarded);
+ }
+ UNIT_ASSERT(!checker.guarded);
+ }
+
+ void TestTryGuard() {
+ TGuardChecker checker;
+
+ UNIT_ASSERT(!checker.guarded);
+ {
+ TTryGuard<TGuardChecker> guard(checker);
+ UNIT_ASSERT(checker.guarded);
+ UNIT_ASSERT(guard.WasAcquired());
+ {
+ TTryGuard<TGuardChecker> guard2(checker);
+ UNIT_ASSERT(checker.guarded);
+ UNIT_ASSERT(!guard2.WasAcquired());
+ }
+ UNIT_ASSERT(checker.guarded);
+ }
+ UNIT_ASSERT(!checker.guarded);
+ }
+
+ void TestTryReadGuard() {
+ TRWMutex mutex;
+ {
+ TTryReadGuard tryGuard(mutex);
+ UNIT_ASSERT(tryGuard.WasAcquired());
+ TReadGuard readGuard(mutex);
+ TTryReadGuard anotherTryGuard(mutex);
+ UNIT_ASSERT(tryGuard.WasAcquired());
+ }
+ {
+ TReadGuard readGuard(mutex);
+ TTryReadGuard tryGuard(mutex);
+ UNIT_ASSERT(tryGuard.WasAcquired());
+ }
+ {
+ TWriteGuard writeGuard(mutex);
+ TTryReadGuard tryGuard(mutex);
+ UNIT_ASSERT(!tryGuard.WasAcquired());
+ }
+ TTryReadGuard tryGuard(mutex);
+ UNIT_ASSERT(tryGuard.WasAcquired());
+ }
+
+ int WithLockIncrement(TGuardChecker& m, int n) {
+ with_lock (m) {
+ UNIT_ASSERT(m.guarded);
+ return n + 1;
+ }
+ }
+
+ void TestWithLock() {
+ TGuardChecker m;
+ int n = 42;
+ n = WithLockIncrement(m, n);
+ UNIT_ASSERT(!m.guarded);
+ UNIT_ASSERT_EQUAL(n, 43);
+ }
+
+ void TestWithLockScope() {
+ auto Guard = [](auto) { UNIT_FAIL("Non global Guard used"); return 0; };
+ TGuardChecker m;
+ with_lock (m) {
+ Y_UNUSED(Guard);
+ }
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TTestGuard)
diff --git a/util/system/hi_lo.cpp b/util/system/hi_lo.cpp
new file mode 100644
index 0000000000..97c3eba8be
--- /dev/null
+++ b/util/system/hi_lo.cpp
@@ -0,0 +1 @@
+#include "hi_lo.h"
diff --git a/util/system/hi_lo.h b/util/system/hi_lo.h
new file mode 100644
index 0000000000..f86870534f
--- /dev/null
+++ b/util/system/hi_lo.h
@@ -0,0 +1,149 @@
+#pragma once
+
+#include "unaligned_mem.h"
+
+#include <utility>
+
+#ifndef _little_endian_
+ #error "Not implemented"
+#endif
+
+namespace NHiLoPrivate {
+ template <class TRepr>
+ class TConstIntRef {
+ public:
+ explicit TConstIntRef(const char* ptr)
+ : Ptr(ptr)
+ {
+ }
+
+ TRepr Get() const {
+ return ReadUnaligned<TRepr>(Ptr);
+ }
+ operator TRepr() const {
+ return Get();
+ }
+
+ const char* GetPtr() const {
+ return Ptr;
+ }
+
+ protected:
+ const char* Ptr;
+ };
+
+ template <class TRepr>
+ class TIntRef: public TConstIntRef<TRepr> {
+ public:
+ explicit TIntRef(char* ptr)
+ : TConstIntRef<TRepr>(ptr)
+ {
+ }
+
+ TIntRef& operator=(TRepr value) {
+ WriteUnaligned<TRepr>(GetPtr(), value);
+ return *this;
+ }
+
+ char* GetPtr() const {
+ return const_cast<char*>(this->Ptr);
+ }
+ };
+
+ template <class T>
+ struct TReferenceType {
+ using TType = T;
+ };
+
+ template <class T>
+ struct TReferenceType<TConstIntRef<T>> {
+ using TType = T;
+ };
+
+ template <class T>
+ struct TReferenceType<TIntRef<T>> {
+ using TType = T;
+ };
+
+ template <class TRepr>
+ auto MakeIntRef(const char* ptr) {
+ return TConstIntRef<TRepr>(ptr);
+ }
+
+ template <class TRepr>
+ auto MakeIntRef(char* ptr) {
+ return TIntRef<TRepr>(ptr);
+ }
+
+ template <class T>
+ const char* CharPtrOf(const T& value) {
+ return reinterpret_cast<const char*>(&value);
+ }
+
+ template <class T>
+ char* CharPtrOf(T& value) {
+ return reinterpret_cast<char*>(&value);
+ }
+
+ template <class T>
+ const char* CharPtrOf(TConstIntRef<T> value) {
+ return value.GetPtr();
+ }
+
+ template <class T>
+ char* CharPtrOf(TIntRef<T> value) {
+ return value.GetPtr();
+ }
+
+ template <bool IsLow, class TRepr, class T>
+ auto MakeIntRef(T&& value) {
+ using TRef = typename TReferenceType<typename std::decay<T>::type>::TType;
+ static_assert(
+ std::is_scalar<TRef>::value,
+ "Hi* and Lo* functions can be applied only to scalar values");
+ static_assert(sizeof(TRef) >= sizeof(TRepr), "Requested bit range is not within provided value");
+ constexpr size_t offset = IsLow ? 0 : sizeof(TRef) - sizeof(TRepr);
+
+ return MakeIntRef<TRepr>(CharPtrOf(std::forward<T>(value)) + offset);
+ }
+}
+
+/**
+ * Return manipulator object that allows to get and set lower or higher bits of the value.
+ *
+ * @param value Must be a scalar value of sufficient size or a manipulator object obtained by
+ * calling any of the other Hi/Lo functions.
+ *
+ * @{
+ */
+template <class T>
+auto Lo32(T&& value) {
+ return NHiLoPrivate::MakeIntRef<true, ui32>(std::forward<T>(value));
+}
+
+template <class T>
+auto Hi32(T&& value) {
+ return NHiLoPrivate::MakeIntRef<false, ui32>(std::forward<T>(value));
+}
+
+template <class T>
+auto Lo16(T&& value) {
+ return NHiLoPrivate::MakeIntRef<true, ui16>(std::forward<T>(value));
+}
+
+template <class T>
+auto Hi16(T&& value) {
+ return NHiLoPrivate::MakeIntRef<false, ui16>(std::forward<T>(value));
+}
+
+template <class T>
+auto Lo8(T&& value) {
+ return NHiLoPrivate::MakeIntRef<true, ui8>(std::forward<T>(value));
+}
+
+template <class T>
+auto Hi8(T&& value) {
+ return NHiLoPrivate::MakeIntRef<false, ui8>(std::forward<T>(value));
+}
+
+/** @} */
diff --git a/util/system/hi_lo_ut.cpp b/util/system/hi_lo_ut.cpp
new file mode 100644
index 0000000000..850c12327d
--- /dev/null
+++ b/util/system/hi_lo_ut.cpp
@@ -0,0 +1,69 @@
+#include <util/system/hi_lo.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "defaults.h"
+
+Y_UNIT_TEST_SUITE(HiLo) {
+ Y_UNIT_TEST(HiLo32) {
+ ui64 x = 0;
+ Lo32(x) = 18;
+ UNIT_ASSERT_VALUES_EQUAL(x, 18);
+
+ Hi32(x) = 33;
+ UNIT_ASSERT_VALUES_EQUAL(x, 141733920786);
+
+ const ui64 y = 0x33c06196e94c03ab;
+ UNIT_ASSERT_VALUES_EQUAL(Lo32(y).Get(), 0xe94c03ab);
+ UNIT_ASSERT_VALUES_EQUAL(Hi32(y).Get(), 0x33c06196);
+ }
+
+ Y_UNIT_TEST(HiLo16) {
+ ui32 x = 0;
+ Lo16(x) = 18;
+ UNIT_ASSERT_VALUES_EQUAL(x, 18);
+
+ Hi16(x) = 33;
+ UNIT_ASSERT_VALUES_EQUAL(x, 2162706);
+
+ const ui32 y = 0xe94c03ab;
+ UNIT_ASSERT_VALUES_EQUAL(Lo16(y).Get(), 0x03ab);
+ UNIT_ASSERT_VALUES_EQUAL(Hi16(y).Get(), 0xe94c);
+ }
+
+ Y_UNIT_TEST(HiLo8) {
+ ui16 x = 0;
+ Lo8(x) = 18;
+ UNIT_ASSERT_VALUES_EQUAL(x, 18);
+
+ Hi8(x) = 33;
+ UNIT_ASSERT_VALUES_EQUAL(x, 8466);
+
+ const ui16 y = 0x03ab;
+ UNIT_ASSERT_VALUES_EQUAL(Lo8(y).Get(), 0xab);
+ UNIT_ASSERT_VALUES_EQUAL(Hi8(y).Get(), 0x03);
+ }
+
+ Y_UNIT_TEST(Combined) {
+ ui32 x = 0;
+ Lo8(Lo16(x)) = 18;
+ UNIT_ASSERT_VALUES_EQUAL(x, 18);
+
+ Hi8(Lo16(x)) = 33;
+ UNIT_ASSERT_VALUES_EQUAL(x, 8466);
+
+ const ui32 y = 0xe94c03ab;
+ UNIT_ASSERT_VALUES_EQUAL(Lo8(Lo16(y)).Get(), 0xab);
+ UNIT_ASSERT_VALUES_EQUAL(Hi8(Lo16(y)).Get(), 0x03);
+ }
+
+ Y_UNIT_TEST(NarrowFromWide) {
+ const ui64 x = 0x1122334455667788ull;
+ UNIT_ASSERT_VALUES_EQUAL(Lo8(x).Get(), 0x88);
+ UNIT_ASSERT_VALUES_EQUAL(Hi8(x).Get(), 0x11);
+ UNIT_ASSERT_VALUES_EQUAL(Lo16(x).Get(), 0x7788);
+ UNIT_ASSERT_VALUES_EQUAL(Hi16(x).Get(), 0x1122);
+ UNIT_ASSERT_VALUES_EQUAL(Lo32(x).Get(), 0x55667788);
+ UNIT_ASSERT_VALUES_EQUAL(Hi32(x).Get(), 0x11223344);
+ }
+}
diff --git a/util/system/hostname.cpp b/util/system/hostname.cpp
new file mode 100644
index 0000000000..386f646d6b
--- /dev/null
+++ b/util/system/hostname.cpp
@@ -0,0 +1,96 @@
+#include <util/memory/tempbuf.h>
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+#include <util/network/ip.h>
+
+#if defined(_unix_)
+ #include <unistd.h>
+ #include <ifaddrs.h>
+ #include <netdb.h>
+#endif
+
+#if defined(_win_)
+ #include <WinSock2.h>
+#endif
+
+#include "defaults.h"
+#include "yassert.h"
+#include "hostname.h"
+
+namespace {
+ struct THostNameHolder {
+ inline THostNameHolder() {
+ TTempBuf hostNameBuf;
+
+ if (gethostname(hostNameBuf.Data(), hostNameBuf.Size() - 1)) {
+ ythrow TSystemError() << "can not get host name";
+ }
+
+ HostName = hostNameBuf.Data();
+ }
+
+ TString HostName;
+ };
+
+ struct TFQDNHostNameHolder {
+ inline TFQDNHostNameHolder() {
+ struct addrinfo hints;
+ struct addrinfo* ais{nullptr};
+ char buf[1024];
+
+ memset(buf, 0, sizeof(buf));
+ int res = gethostname(buf, sizeof(buf) - 1);
+ if (res) {
+ ythrow TSystemError() << "can not get hostname";
+ }
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = AF_UNSPEC;
+ hints.ai_flags = AI_CANONNAME;
+ res = getaddrinfo(buf, nullptr, &hints, &ais);
+ if (res) {
+ if (res != EAI_NONAME) {
+ ythrow TSystemError() << "can not get FQDN (return code is " << res << ", hostname is \"" << buf << "\")";
+ }
+ FQDNHostName = buf;
+ } else {
+ FQDNHostName = ais->ai_canonname;
+ freeaddrinfo(ais);
+ }
+ FQDNHostName.to_lower();
+ }
+
+ TString FQDNHostName;
+ };
+}
+
+const TString& HostName() {
+ return (Singleton<THostNameHolder>())->HostName;
+}
+
+const char* GetHostName() {
+ return HostName().data();
+}
+
+const TString& FQDNHostName() {
+ return (Singleton<TFQDNHostNameHolder>())->FQDNHostName;
+}
+
+const char* GetFQDNHostName() {
+ return FQDNHostName().data();
+}
+
+bool IsFQDN(const TString& name) {
+ TString absName = name;
+ if (!absName.EndsWith('.')) {
+ absName.append(".");
+ }
+
+ try {
+ // ResolveHost() can't be used since it is ipv4-only, port is not important
+ TNetworkAddress addr(absName, 0);
+ } catch (const TNetworkResolutionError&) {
+ return false;
+ }
+ return true;
+}
diff --git a/util/system/hostname.h b/util/system/hostname.h
new file mode 100644
index 0000000000..0839ee2b59
--- /dev/null
+++ b/util/system/hostname.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+
+const char* GetHostName();
+const TString& HostName();
+
+const char* GetFQDNHostName();
+const TString& FQDNHostName();
+bool IsFQDN(const TString& name);
diff --git a/util/system/hostname_ut.cpp b/util/system/hostname_ut.cpp
new file mode 100644
index 0000000000..fb1cb7dde4
--- /dev/null
+++ b/util/system/hostname_ut.cpp
@@ -0,0 +1,25 @@
+#include "hostname.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(THostNameTest) {
+ Y_UNIT_TEST(Test1) {
+ UNIT_ASSERT(*GetHostName() != '?');
+ }
+
+ Y_UNIT_TEST(TestFQDN) {
+ UNIT_ASSERT(*GetFQDNHostName() != '?');
+ }
+
+ Y_UNIT_TEST(TestIsFQDN) {
+ const auto x = GetFQDNHostName();
+
+ try {
+ UNIT_ASSERT(IsFQDN(x));
+ } catch (...) {
+ Cerr << x << Endl;
+
+ throw;
+ }
+ }
+};
diff --git a/util/system/hp_timer.cpp b/util/system/hp_timer.cpp
new file mode 100644
index 0000000000..e4c3f21e6b
--- /dev/null
+++ b/util/system/hp_timer.cpp
@@ -0,0 +1,118 @@
+#include "hp_timer.h"
+
+#include <util/generic/algorithm.h>
+#include <util/generic/singleton.h>
+#include <util/datetime/cputimer.h>
+
+using namespace NHPTimer;
+
+namespace {
+ struct TFreq {
+ inline TFreq()
+ : Freq(InitHPTimer())
+ , Rate(1.0 / Freq)
+ , CyclesPerSecond(static_cast<ui64>(Rate))
+ {
+ }
+
+ static inline const TFreq& Instance() {
+ return *SingletonWithPriority<TFreq, 1>();
+ }
+
+ static double EstimateCPUClock() {
+ for (;;) {
+ ui64 startCycle = 0;
+ ui64 startMS = 0;
+
+ for (;;) {
+ startMS = MicroSeconds();
+ startCycle = GetCycleCount();
+
+ ui64 n = MicroSeconds();
+
+ if (n - startMS < 100) {
+ break;
+ }
+ }
+
+ Sleep(TDuration::MicroSeconds(5000));
+
+ ui64 finishCycle = 0;
+ ui64 finishMS = 0;
+
+ for (;;) {
+ finishMS = MicroSeconds();
+
+ if (finishMS - startMS < 100) {
+ continue;
+ }
+
+ finishCycle = GetCycleCount();
+
+ ui64 n = MicroSeconds();
+
+ if (n - finishMS < 100) {
+ break;
+ }
+ }
+ if (startMS < finishMS && startCycle < finishCycle) {
+ return (finishCycle - startCycle) * 1000000.0 / (finishMS - startMS);
+ }
+ }
+ }
+
+ static double InitHPTimer() {
+ const size_t N_VEC = 9;
+
+ double vec[N_VEC];
+
+ for (auto& i : vec) {
+ i = EstimateCPUClock();
+ }
+
+ Sort(vec, vec + N_VEC);
+
+ return 1.0 / vec[N_VEC / 2];
+ }
+
+ inline double GetSeconds(const STime& a) const {
+ return static_cast<double>(a) * Freq;
+ }
+
+ inline double GetClockRate() const {
+ return Rate;
+ }
+
+ inline ui64 GetCyclesPerSecond() const {
+ return CyclesPerSecond;
+ }
+
+ const double Freq;
+ const double Rate;
+ const ui64 CyclesPerSecond;
+ };
+}
+
+double NHPTimer::GetSeconds(const STime& a) noexcept {
+ return TFreq::Instance().GetSeconds(a);
+}
+
+double NHPTimer::GetClockRate() noexcept {
+ return TFreq::Instance().GetClockRate();
+}
+
+ui64 NHPTimer::GetCyclesPerSecond() noexcept {
+ return TFreq::Instance().GetCyclesPerSecond();
+}
+
+void NHPTimer::GetTime(STime* pTime) noexcept {
+ *pTime = GetCycleCount();
+}
+
+double NHPTimer::GetTimePassed(STime* pTime) noexcept {
+ STime old(*pTime);
+
+ *pTime = GetCycleCount();
+
+ return GetSeconds(*pTime - old);
+}
diff --git a/util/system/hp_timer.h b/util/system/hp_timer.h
new file mode 100644
index 0000000000..0a4c252ec2
--- /dev/null
+++ b/util/system/hp_timer.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include "defaults.h"
+
+namespace NHPTimer {
+ using STime = i64;
+ // May delay for ~50ms to compute frequency
+ double GetSeconds(const STime& a) noexcept;
+ // Returns the current time
+ void GetTime(STime* pTime) noexcept;
+ // Returns the time passed since *pTime, and writes the current time into *pTime.
+ double GetTimePassed(STime* pTime) noexcept;
+ // Get TSC frequency, may delay for ~50ms to compute frequency
+ double GetClockRate() noexcept;
+ // same as GetClockRate, but in integer
+ ui64 GetCyclesPerSecond() noexcept;
+}
+
+struct THPTimer {
+ THPTimer() noexcept {
+ Reset();
+ }
+ void Reset() noexcept {
+ NHPTimer::GetTime(&Start);
+ }
+ double Passed() const noexcept {
+ NHPTimer::STime tmp = Start;
+ return NHPTimer::GetTimePassed(&tmp);
+ }
+ double PassedReset() noexcept {
+ return NHPTimer::GetTimePassed(&Start);
+ }
+
+private:
+ NHPTimer::STime Start;
+};
diff --git a/util/system/info.cpp b/util/system/info.cpp
new file mode 100644
index 0000000000..cf6681e89a
--- /dev/null
+++ b/util/system/info.cpp
@@ -0,0 +1,239 @@
+#include "info.h"
+
+#include "error.h"
+
+#include <cstdlib>
+
+#if defined(_linux_) || defined(_cygwin_)
+ #include <fcntl.h>
+ #include <sys/sysinfo.h>
+#endif
+
+#if defined(_win_)
+ #include "winint.h"
+ #include <stdio.h>
+#else
+ #include <unistd.h>
+#endif
+
+#if defined(_bionic_)
+//TODO
+#elif defined(_cygwin_)
+static int getloadavg(double* loadavg, int nelem) {
+ for (int i = 0; i < nelem; ++i) {
+ loadavg[i] = 0.0;
+ }
+
+ return nelem;
+}
+#elif defined(_unix_) || defined(_darwin_)
+ #include <sys/types.h>
+#endif
+
+#if defined(_freebsd_) || defined(_darwin_)
+ #include <sys/sysctl.h>
+#endif
+
+#include <util/string/ascii.h>
+#include <util/string/cast.h>
+#include <util/string/strip.h>
+#include <util/stream/file.h>
+#include <util/generic/yexception.h>
+
+#if defined(_linux_)
+static inline size_t CgroupCpus() {
+ try {
+ auto q = FromString<ssize_t>(StripString(TFileInput("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").ReadAll()));
+
+ if (q <= 0) {
+ return 0;
+ }
+
+ auto p = FromString<ssize_t>(StripString(TFileInput("/sys/fs/cgroup/cpu/cpu.cfs_period_us").ReadAll()));
+
+ if (p <= 0) {
+ return 0;
+ }
+
+ return Max<ssize_t>(1, (q + p / 2) / p);
+ } catch (...) {
+ return 0;
+ }
+}
+#endif
+
+size_t NSystemInfo::NumberOfCpus() {
+#if defined(_linux_)
+ if (auto res = CgroupCpus(); res) {
+ return res;
+ }
+#endif
+
+#if defined(_win_)
+ SYSTEM_INFO info;
+
+ GetSystemInfo(&info);
+
+ return info.dwNumberOfProcessors;
+#elif defined(_SC_NPROCESSORS_ONLN)
+ return sysconf(_SC_NPROCESSORS_ONLN);
+#elif defined(_linux_)
+ unsigned ret;
+ int fd, nread, column;
+ char buf[512];
+ static const char matchstr[] = "processor\t:";
+
+ fd = open("/proc/cpuinfo", O_RDONLY);
+
+ if (fd == -1) {
+ abort();
+ }
+
+ column = 0;
+ ret = 0;
+
+ while (true) {
+ nread = read(fd, buf, sizeof(buf));
+
+ if (nread <= 0) {
+ break;
+ }
+
+ for (int i = 0; i < nread; ++i) {
+ const char ch = buf[i];
+
+ if (ch == '\n') {
+ column = 0;
+ } else if (column != -1) {
+ if (AsciiToLower(ch) == matchstr[column]) {
+ ++column;
+
+ if (column == sizeof(matchstr) - 1) {
+ column = -1;
+ ++ret;
+ }
+ } else {
+ column = -1;
+ }
+ }
+ }
+ }
+
+ if (ret == 0) {
+ abort();
+ }
+
+ close(fd);
+
+ return ret;
+#elif defined(_freebsd_) || defined(_darwin_)
+ int mib[2];
+ size_t len;
+ unsigned ncpus = 1;
+
+ mib[0] = CTL_HW;
+ mib[1] = HW_NCPU;
+ len = sizeof(ncpus);
+ if (sysctl(mib, 2, &ncpus, &len, nullptr, 0) == -1) {
+ abort();
+ }
+
+ return ncpus;
+#else
+ #error todo
+#endif
+}
+
+size_t NSystemInfo::LoadAverage(double* la, size_t len) {
+#if defined(_win_) || defined(_musl_) || defined(_bionic_)
+ int ret = -1;
+#else
+ for (size_t i = 0; i < len; ++i) {
+ la[i] = 0;
+ }
+
+ int ret = getloadavg(la, len);
+#endif
+
+ if (ret < 0) {
+ for (size_t i = 0; i < len; ++i) {
+ la[i] = 0;
+ }
+
+ ret = len;
+ }
+
+ return (size_t)ret;
+}
+
+static size_t NCpus;
+
+size_t NSystemInfo::CachedNumberOfCpus() {
+ if (!NCpus) {
+ NCpus = NumberOfCpus();
+ }
+
+ return NCpus;
+}
+
+size_t NSystemInfo::GetPageSize() noexcept {
+#if defined(_win_)
+ SYSTEM_INFO sysInfo;
+ GetSystemInfo(&sysInfo);
+
+ return sysInfo.dwPageSize;
+#else
+ return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+size_t NSystemInfo::TotalMemorySize() {
+#if defined(_linux_) && defined(_64_)
+ try {
+ auto q = FromString<size_t>(StripString(TFileInput("/sys/fs/cgroup/memory/memory.limit_in_bytes").ReadAll()));
+
+ if (q < (((size_t)1) << 60)) {
+ return q;
+ }
+ } catch (...) {
+ }
+#endif
+
+#if defined(_linux_) || defined(_cygwin_)
+ struct sysinfo info;
+ sysinfo(&info);
+ return info.totalram;
+#elif defined(_darwin_)
+ int mib[2];
+ int64_t memSize;
+ size_t length;
+
+ // Get the Physical memory size
+ mib[0] = CTL_HW;
+ mib[1] = HW_MEMSIZE;
+ length = sizeof(int64_t);
+ if (sysctl(mib, 2, &memSize, &length, NULL, 0) != 0) {
+ ythrow yexception() << "sysctl failed: " << LastSystemErrorText();
+ }
+ return (size_t)memSize;
+#elif defined(_win_)
+ MEMORYSTATUSEX memoryStatusEx;
+ memoryStatusEx.dwLength = sizeof(memoryStatusEx);
+ if (!GlobalMemoryStatusEx(&memoryStatusEx)) {
+ ythrow yexception() << "GlobalMemoryStatusEx failed: " << LastSystemErrorText();
+ }
+ return (size_t)memoryStatusEx.ullTotalPhys;
+#else
+ return 0;
+#endif
+}
+
+size_t NSystemInfo::MaxOpenFiles() {
+#if defined(ANDROID) || defined(__ANDROID__)
+ return sysconf(_SC_OPEN_MAX);
+#elif defined(_win_)
+ return _getmaxstdio();
+#else
+ return getdtablesize();
+#endif
+}
diff --git a/util/system/info.h b/util/system/info.h
new file mode 100644
index 0000000000..73ebe48a9a
--- /dev/null
+++ b/util/system/info.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "defaults.h"
+
+namespace NSystemInfo {
+ size_t NumberOfCpus();
+ size_t CachedNumberOfCpus();
+ size_t LoadAverage(double* la, size_t len);
+ size_t GetPageSize() noexcept;
+ size_t TotalMemorySize();
+ size_t MaxOpenFiles();
+}
diff --git a/util/system/info_ut.cpp b/util/system/info_ut.cpp
new file mode 100644
index 0000000000..ad7449f8f4
--- /dev/null
+++ b/util/system/info_ut.cpp
@@ -0,0 +1,22 @@
+#include "info.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+class TSysInfoTest: public TTestBase {
+ UNIT_TEST_SUITE(TSysInfoTest);
+ UNIT_TEST(TestNumberOfCpus)
+ UNIT_TEST(TestGetPageSize)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestNumberOfCpus() {
+ UNIT_ASSERT(NSystemInfo::NumberOfCpus() > 0);
+ UNIT_ASSERT_EQUAL(NSystemInfo::NumberOfCpus(), NSystemInfo::CachedNumberOfCpus());
+ }
+
+ inline void TestGetPageSize() {
+ UNIT_ASSERT(NSystemInfo::GetPageSize() >= 4096);
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TSysInfoTest);
diff --git a/util/system/interrupt_signals.cpp b/util/system/interrupt_signals.cpp
new file mode 100644
index 0000000000..96b723f2b8
--- /dev/null
+++ b/util/system/interrupt_signals.cpp
@@ -0,0 +1,60 @@
+#include "interrupt_signals.h"
+
+#include "compiler.h"
+#include "error.h"
+
+#include <util/generic/yexception.h>
+
+#include <csignal>
+
+static void (*InterruptSignalsHandler)(int signum) = nullptr;
+
+#ifdef _win_
+
+ #include <windows.h>
+
+static BOOL WINAPI WindowsSignalsHandler(_In_ DWORD dwCtrlType) {
+ if (!InterruptSignalsHandler) {
+ return FALSE;
+ }
+
+ switch (dwCtrlType) {
+ case CTRL_C_EVENT:
+ InterruptSignalsHandler(SIGINT);
+ return TRUE;
+ case CTRL_BREAK_EVENT:
+ InterruptSignalsHandler(SIGTERM);
+ return TRUE;
+ case CTRL_CLOSE_EVENT:
+ InterruptSignalsHandler(SIGHUP);
+ return TRUE;
+ default:
+ return FALSE;
+ }
+ Y_UNREACHABLE();
+}
+
+#endif
+
+// separate function is to enforce 'extern "C"' linkage
+extern "C" void CppSignalsHandler(int signum) {
+ if (InterruptSignalsHandler) {
+ InterruptSignalsHandler(signum);
+ }
+}
+
+void SetInterruptSignalsHandler(void (*handler)(int signum)) {
+ InterruptSignalsHandler = handler;
+#ifdef _win_
+ if (!SetConsoleCtrlHandler(WindowsSignalsHandler, TRUE)) {
+ ythrow TSystemError() << "SetConsoleCtrlHandler failed: " << LastSystemErrorText();
+ }
+ for (int signum : {SIGINT, SIGTERM}) {
+#else
+ for (int signum : {SIGINT, SIGTERM, SIGHUP}) {
+#endif
+ if (std::signal(signum, CppSignalsHandler) == SIG_ERR) {
+ ythrow TSystemError() << "std::signal failed to set handler for signal with id " << signum;
+ }
+ }
+}
diff --git a/util/system/interrupt_signals.h b/util/system/interrupt_signals.h
new file mode 100644
index 0000000000..9f9c2427f1
--- /dev/null
+++ b/util/system/interrupt_signals.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "platform.h"
+
+#include <csignal>
+
+#ifdef _win_
+ #ifndef SIGHUP
+ #define SIGHUP 1 /* Hangup (POSIX). */
+ #endif
+#endif
+
+/**
+ * Set handler for interrupt signals.
+ *
+ * All OSes: SIGINT, SIGTERM (defined by C++ standard)
+ * UNIX variants: Also SIGHUP
+ * Windows: CTRL_C_EVENT handled as SIGINT, CTRL_BREAK_EVENT as SIGTERM, CTRL_CLOSE_EVENT as SIGHUP
+ *
+ * \param handler Signal handler to use. Pass nullptr to clear currently set handler.
+ */
+void SetInterruptSignalsHandler(void (*handler)(int signum));
diff --git a/util/system/interrupt_signals_ut.cpp b/util/system/interrupt_signals_ut.cpp
new file mode 100644
index 0000000000..bec54fec61
--- /dev/null
+++ b/util/system/interrupt_signals_ut.cpp
@@ -0,0 +1,46 @@
+#include "interrupt_signals.h"
+
+#include "atomic.h"
+
+#include <util/datetime/base.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#ifdef _win_
+ #include <windows.h>
+#endif
+
+Y_UNIT_TEST_SUITE(TTestInterruptSignals) {
+ static TAtomic HandledSigNum = 0;
+
+ static void Handler(int signum) {
+ AtomicSet(HandledSigNum, signum);
+ }
+
+ Y_UNIT_TEST(Test1) {
+ SetInterruptSignalsHandler(Handler);
+#ifdef _win_
+ // TODO: unfortunately GenerateConsoleCtrlEvent fails under Wine
+ /*
+ for (auto [winEvent, posixSigNum] : {
+ std::make_pair(CTRL_C_EVENT, SIGINT),
+ std::make_pair(CTRL_BREAK_EVENT, SIGTERM)
+ })
+ {
+ if (!GenerateConsoleCtrlEvent(winEvent, 0)) {
+ UNIT_FAIL("GenerateConsoleCtrlEvent failed: " << LastSystemErrorText());
+ }
+ Sleep(TDuration::MilliSeconds(100));
+ UNIT_ASSERT_VALUES_EQUAL(HandledSigNum, posixSigNum);
+ }
+ */
+ for (int signum : {SIGINT, SIGTERM}) {
+#else
+ for (int signum : {SIGINT, SIGTERM, SIGHUP}) {
+#endif
+ std::raise(signum);
+ Sleep(TDuration::MilliSeconds(100)); // give it time to handle an async signal
+ UNIT_ASSERT_VALUES_EQUAL(HandledSigNum, signum);
+ }
+ }
+}
diff --git a/util/system/madvise.cpp b/util/system/madvise.cpp
new file mode 100644
index 0000000000..58c894e3ef
--- /dev/null
+++ b/util/system/madvise.cpp
@@ -0,0 +1,126 @@
+#include "madvise.h"
+#include "align.h"
+#include "info.h"
+
+#include <util/generic/yexception.h>
+
+#if defined(_win_)
+ #include <util/system/winint.h>
+#else
+ #include <sys/types.h>
+ #include <sys/mman.h>
+#endif
+
+#ifndef MADV_DONTDUMP /* This flag is defined in sys/mman.h since Linux 3.4, but currently old libc header is in use \
+ for capability with Ubuntu 12.04, so we need to define it here manually */
+ #define MADV_DONTDUMP 16 /* Explicity exclude from the core dump, overrides the coredump filter bits */
+#endif
+
+#ifndef MADV_DODUMP /* This flag is defined in sys/mman.h since Linux 3.4, but currently old libc header is in use \
+ for capability with Ubuntu 12.04, so we need to define it here manually */
+ #define MADV_DODUMP 17 /* Undo the effect of an earlier MADV_DONTDUMP */
+#endif
+
+namespace {
+ void Madvise(int flag, const void* cbegin, size_t size) {
+ static const size_t pageSize = NSystemInfo::GetPageSize();
+ void* begin = AlignDown(const_cast<void*>(cbegin), pageSize);
+ size = AlignUp(size, pageSize);
+
+#if defined(_win_)
+ if (!VirtualFree((LPVOID)begin, size, flag)) {
+ TString err(LastSystemErrorText());
+ ythrow yexception() << "VirtualFree(" << begin << ", " << size << ", " << flag << ")"
+ << " returned error: " << err;
+ }
+#else
+ if (-1 == madvise(begin, size, flag)) {
+ TString err(LastSystemErrorText());
+ ythrow yexception() << "madvise(" << begin << ", " << size << ", " << flag << ")"
+ << " returned error: " << err;
+ }
+#endif
+ }
+}
+
+void MadviseSequentialAccess(const void* begin, size_t size) {
+#if !defined(_win_)
+ Madvise(MADV_SEQUENTIAL, begin, size);
+#endif
+}
+
+void MadviseSequentialAccess(TArrayRef<const char> data) {
+ MadviseSequentialAccess(data.data(), data.size());
+}
+
+void MadviseSequentialAccess(TArrayRef<const ui8> data) {
+ MadviseSequentialAccess(data.data(), data.size());
+}
+
+void MadviseRandomAccess(const void* begin, size_t size) {
+#if !defined(_win_)
+ Madvise(MADV_RANDOM, begin, size);
+#endif
+}
+
+void MadviseRandomAccess(TArrayRef<const char> data) {
+ MadviseRandomAccess(data.data(), data.size());
+}
+
+void MadviseRandomAccess(TArrayRef<const ui8> data) {
+ MadviseRandomAccess(data.data(), data.size());
+}
+
+void MadviseEvict(const void* begin, size_t size) {
+#if defined(_win_)
+ Madvise(MEM_DECOMMIT, begin, size);
+#elif defined(_linux_) || defined(_cygwin_)
+ Madvise(MADV_DONTNEED, begin, size);
+#else // freebsd, osx
+ Madvise(MADV_FREE, begin, size);
+#endif
+}
+
+void MadviseEvict(TArrayRef<const char> data) {
+ MadviseEvict(data.data(), data.size());
+}
+
+void MadviseEvict(TArrayRef<const ui8> data) {
+ MadviseEvict(data.data(), data.size());
+}
+
+void MadviseExcludeFromCoreDump(const void* begin, size_t size) {
+#if defined(_darwin_)
+ // Don't try to call function with flag which doesn't work
+ // https://st.yandex-team.ru/PASSP-31755#6050bbafc68f501f2c22caab
+ Y_UNUSED(begin);
+ Y_UNUSED(size);
+#elif !defined(_win_)
+ Madvise(MADV_DONTDUMP, begin, size);
+#endif
+}
+
+void MadviseExcludeFromCoreDump(TArrayRef<const char> data) {
+ MadviseExcludeFromCoreDump(data.data(), data.size());
+}
+
+void MadviseExcludeFromCoreDump(TArrayRef<const ui8> data) {
+ MadviseExcludeFromCoreDump(data.data(), data.size());
+}
+
+void MadviseIncludeIntoCoreDump(const void* begin, size_t size) {
+#if defined(_darwin_)
+ Y_UNUSED(begin);
+ Y_UNUSED(size);
+#elif !defined(_win_)
+ Madvise(MADV_DODUMP, begin, size);
+#endif
+}
+
+void MadviseIncludeIntoCoreDump(TArrayRef<const char> data) {
+ MadviseIncludeIntoCoreDump(data.data(), data.size());
+}
+
+void MadviseIncludeIntoCoreDump(TArrayRef<const ui8> data) {
+ MadviseIncludeIntoCoreDump(data.data(), data.size());
+}
diff --git a/util/system/madvise.h b/util/system/madvise.h
new file mode 100644
index 0000000000..606733152e
--- /dev/null
+++ b/util/system/madvise.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/array_ref.h>
+
+/// see linux madvise(MADV_SEQUENTIAL)
+void MadviseSequentialAccess(const void* begin, size_t size);
+void MadviseSequentialAccess(TArrayRef<const char> data);
+void MadviseSequentialAccess(TArrayRef<const ui8> data);
+
+/// see linux madvise(MADV_RANDOM)
+void MadviseRandomAccess(const void* begin, size_t size);
+void MadviseRandomAccess(TArrayRef<const char> data);
+void MadviseRandomAccess(TArrayRef<const ui8> data);
+
+/// see linux madvise(MADV_DONTNEED)
+void MadviseEvict(const void* begin, size_t size);
+void MadviseEvict(TArrayRef<const char> data);
+void MadviseEvict(TArrayRef<const ui8> data);
+
+/// see linux madvise(MADV_DONTDUMP)
+void MadviseExcludeFromCoreDump(const void* begin, size_t size);
+void MadviseExcludeFromCoreDump(TArrayRef<const char> data);
+void MadviseExcludeFromCoreDump(TArrayRef<const ui8> data);
+
+/// see linux madvise(MADV_DODUMP)
+void MadviseIncludeIntoCoreDump(const void* begin, size_t size);
+void MadviseIncludeIntoCoreDump(TArrayRef<const char> data);
+void MadviseIncludeIntoCoreDump(TArrayRef<const ui8> data);
diff --git a/util/system/maxlen.cpp b/util/system/maxlen.cpp
new file mode 100644
index 0000000000..bcc9d72a4f
--- /dev/null
+++ b/util/system/maxlen.cpp
@@ -0,0 +1 @@
+#include "maxlen.h"
diff --git a/util/system/maxlen.h b/util/system/maxlen.h
new file mode 100644
index 0000000000..e1ff7f5008
--- /dev/null
+++ b/util/system/maxlen.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <cstdlib>
+
+// http://support.microsoft.com/kb/208427
+#ifndef URL_MAXLEN
+ #define URL_MAXLEN 2083
+#endif
+
+#define HOST_MAX 260
+#ifndef URL_MAX
+ #define URL_MAX 1024
+#endif
+#define FULLURL_MAX (URL_MAX + HOST_MAX)
+
+#define LINKTEXT_MAX 1024
+
+#ifdef WIN32
+ #ifndef PATH_MAX
+ #define PATH_MAX _MAX_PATH
+ #endif
+#else
+
+ #ifndef MAX_PATH
+ #define MAX_PATH PATH_MAX
+ #endif
+
+ #ifndef _MAX_PATH
+ #define _MAX_PATH PATH_MAX
+ #endif
+
+#endif
diff --git a/util/system/mem_info.cpp b/util/system/mem_info.cpp
new file mode 100644
index 0000000000..aa51ae3b16
--- /dev/null
+++ b/util/system/mem_info.cpp
@@ -0,0 +1,219 @@
+#include "mem_info.h"
+
+#include <util/generic/strbuf.h>
+#include <util/generic/utility.h>
+#include <util/generic/yexception.h>
+#include <util/stream/file.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+#include "error.h"
+#include "info.h"
+
+#if defined(_unix_)
+ #include <errno.h>
+ #include <unistd.h>
+ #if defined(_freebsd_)
+ #include <sys/sysctl.h>
+ #include <sys/types.h>
+ #include <sys/user.h>
+ #elif defined(_darwin_) && !defined(_arm_) && !defined(__IOS__)
+ #include <libproc.h>
+ #elif defined(__MACH__) && defined(__APPLE__)
+ #include <mach/mach.h>
+ #endif
+#elif defined(_win_)
+ #include <Windows.h>
+ #include <util/generic/ptr.h>
+
+using NTSTATUS = LONG;
+ #define STATUS_INFO_LENGTH_MISMATCH 0xC0000004
+ #define STATUS_BUFFER_TOO_SMALL 0xC0000023
+
+typedef struct _UNICODE_STRING {
+ USHORT Length;
+ USHORT MaximumLength;
+ PWSTR Buffer;
+} UNICODE_STRING, *PUNICODE_STRING;
+typedef struct _CLIENT_ID {
+ HANDLE UniqueProcess;
+ HANDLE UniqueThread;
+} CLIENT_ID, *PCLIENT_ID;
+using KWAIT_REASON = ULONG;
+typedef struct _SYSTEM_THREAD_INFORMATION {
+ LARGE_INTEGER KernelTime;
+ LARGE_INTEGER UserTime;
+ LARGE_INTEGER CreateTime;
+ ULONG WaitTime;
+ PVOID StartAddress;
+ CLIENT_ID ClientId;
+ LONG Priority;
+ LONG BasePriority;
+ ULONG ContextSwitches;
+ ULONG ThreadState;
+ KWAIT_REASON WaitReason;
+} SYSTEM_THREAD_INFORMATION, *PSYSTEM_THREAD_INFORMATION;
+typedef struct _SYSTEM_PROCESS_INFORMATION {
+ ULONG NextEntryOffset;
+ ULONG NumberOfThreads;
+ LARGE_INTEGER SpareLi1;
+ LARGE_INTEGER SpareLi2;
+ LARGE_INTEGER SpareLi3;
+ LARGE_INTEGER CreateTime;
+ LARGE_INTEGER UserTime;
+ LARGE_INTEGER KernelTime;
+ UNICODE_STRING ImageName;
+ LONG BasePriority;
+ HANDLE UniqueProcessId;
+ HANDLE InheritedFromUniqueProcessId;
+ ULONG HandleCount;
+ ULONG SessionId;
+ ULONG_PTR PageDirectoryBase;
+ SIZE_T PeakVirtualSize;
+ SIZE_T VirtualSize;
+ DWORD PageFaultCount;
+ SIZE_T PeakWorkingSetSize;
+ SIZE_T WorkingSetSize;
+ SIZE_T QuotaPeakPagedPoolUsage;
+ SIZE_T QuotaPagedPoolUsage;
+ SIZE_T QuotaPeakNonPagedPoolUsage;
+ SIZE_T QuotaNonPagedPoolUsage;
+ SIZE_T PagefileUsage;
+ SIZE_T PeakPagefileUsage;
+ SIZE_T PrivatePageCount;
+ LARGE_INTEGER ReadOperationCount;
+ LARGE_INTEGER WriteOperationCount;
+ LARGE_INTEGER OtherOperationCount;
+ LARGE_INTEGER ReadTransferCount;
+ LARGE_INTEGER WriteTransferCount;
+ LARGE_INTEGER OtherTransferCount;
+ SYSTEM_THREAD_INFORMATION Threads[1];
+} SYSTEM_PROCESS_INFORMATION, *PSYSTEM_PROCESS_INFORMATION;
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+ SystemBasicInformation = 0,
+ SystemProcessInformation = 5,
+} SYSTEM_INFORMATION_CLASS;
+
+#else
+
+#endif
+
+namespace NMemInfo {
+ TMemInfo GetMemInfo(pid_t pid) {
+ TMemInfo result;
+
+#if defined(_unix_)
+
+ #if defined(_linux_) || defined(_freebsd_) || defined(_cygwin_)
+ const ui32 pagesize = NSystemInfo::GetPageSize();
+ #endif
+
+ #if defined(_linux_) || defined(_cygwin_)
+ TString path;
+ if (!pid) {
+ path = "/proc/self/statm";
+ } else {
+ path = TStringBuilder() << TStringBuf("/proc/") << pid << TStringBuf("/statm");
+ }
+ const TString stats = TUnbufferedFileInput(path).ReadAll();
+
+ TStringBuf statsiter(stats);
+
+ result.VMS = FromString<ui64>(statsiter.NextTok(' ')) * pagesize;
+ result.RSS = FromString<ui64>(statsiter.NextTok(' ')) * pagesize;
+
+ #if defined(_cygwin_)
+ //cygwin not very accurate
+ result.VMS = Max(result.VMS, result.RSS);
+ #endif
+ #elif defined(_freebsd_)
+ int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, pid};
+ size_t size = sizeof(struct kinfo_proc);
+
+ struct kinfo_proc proc;
+ Zero(proc);
+
+ errno = 0;
+ if (sysctl((int*)mib, 4, &proc, &size, nullptr, 0) == -1) {
+ int err = errno;
+ TString errtxt = LastSystemErrorText(err);
+ ythrow yexception() << "sysctl({CTL_KERN,KERN_PROC,KERN_PROC_PID,pid},4,proc,&size,NULL,0) returned -1, errno: " << err << " (" << errtxt << ")" << Endl;
+ }
+
+ result.VMS = proc.ki_size;
+ result.RSS = proc.ki_rssize * pagesize;
+ #elif defined(_darwin_) && !defined(_arm_) && !defined(__IOS__)
+ if (!pid) {
+ pid = getpid();
+ }
+ struct proc_taskinfo taskInfo;
+ const int r = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &taskInfo, sizeof(taskInfo));
+
+ if (r != sizeof(taskInfo)) {
+ int err = errno;
+ TString errtxt = LastSystemErrorText(err);
+ ythrow yexception() << "proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &taskInfo, sizeof(taskInfo)) returned " << r << ", errno: " << err << " (" << errtxt << ")" << Endl;
+ }
+ result.VMS = taskInfo.pti_virtual_size;
+ result.RSS = taskInfo.pti_resident_size;
+ #elif defined(__MACH__) && defined(__APPLE__)
+ Y_UNUSED(pid);
+ struct mach_task_basic_info taskInfo;
+ mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+
+ const int r = task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&taskInfo, &infoCount);
+ if (r != KERN_SUCCESS) {
+ int err = errno;
+ TString errtxt = LastSystemErrorText(err);
+ ythrow yexception() << "task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) returned" << r << ", errno: " << err << " (" << errtxt << ")" << Endl;
+ }
+ result.VMS = taskInfo.virtual_size;
+ result.RSS = taskInfo.resident_size;
+ #elif defined(_arm_)
+ Y_UNUSED(pid);
+ ythrow yexception() << "arm is not supported";
+ #endif
+#elif defined(_win_)
+ if (!pid) {
+ pid = GetCurrentProcessId();
+ }
+
+ NTSTATUS status;
+ TArrayHolder<char> buffer;
+ ULONG bufferSize;
+
+ // Query data for all processes and threads in the system.
+ // This is probably an overkill if the target process is normal not-privileged one,
+ // but allows to obtain information even about system processes that are not open-able directly.
+ typedef NTSTATUS(_stdcall * NTQSI_PROC)(SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG);
+ NTQSI_PROC NtQuerySystemInformation = (NTQSI_PROC)GetProcAddress(GetModuleHandle(TEXT("ntdll.dll")), "NtQuerySystemInformation");
+ bufferSize = 0x4000;
+
+ for (;;) {
+ buffer.Reset(new char[bufferSize]);
+ status = NtQuerySystemInformation(SystemProcessInformation, buffer.Get(), bufferSize, &bufferSize);
+
+ if (!status) {
+ break;
+ }
+
+ if (status != STATUS_BUFFER_TOO_SMALL && status != STATUS_INFO_LENGTH_MISMATCH) {
+ ythrow yexception() << "NtQuerySystemInformation failed with status code " << status;
+ }
+ }
+
+ SYSTEM_PROCESS_INFORMATION* process = (SYSTEM_PROCESS_INFORMATION*)buffer.Get();
+ while (process->UniqueProcessId != (HANDLE)(size_t)(pid)) {
+ if (!process->NextEntryOffset) {
+ ythrow yexception() << "GetMemInfo: invalid PID";
+ }
+
+ process = (SYSTEM_PROCESS_INFORMATION*)((char*)process + process->NextEntryOffset);
+ }
+
+ result.VMS = process->VirtualSize;
+ result.RSS = process->WorkingSetSize;
+#endif
+ return result;
+ }
+}
diff --git a/util/system/mem_info.h b/util/system/mem_info.h
new file mode 100644
index 0000000000..f303d49197
--- /dev/null
+++ b/util/system/mem_info.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include "compat.h"
+
+namespace NMemInfo {
+ struct TMemInfo {
+ ui64 RSS; // current RAM size of the process
+ ui64 VMS; // current VM size of the process
+
+ TMemInfo()
+ : RSS()
+ , VMS()
+ {
+ }
+ };
+
+ TMemInfo GetMemInfo(pid_t = 0);
+}
diff --git a/util/system/mem_info_ut.cpp b/util/system/mem_info_ut.cpp
new file mode 100644
index 0000000000..a1100ba646
--- /dev/null
+++ b/util/system/mem_info_ut.cpp
@@ -0,0 +1,21 @@
+#include "mem_info.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "info.h"
+
+class TMemInfoTest: public NUnitTest::TTestBase {
+ UNIT_TEST_SUITE(TMemInfoTest)
+ UNIT_TEST(TestMemInfo)
+ UNIT_TEST_SUITE_END();
+ void TestMemInfo() {
+ using namespace NMemInfo;
+
+ TMemInfo stats = GetMemInfo();
+
+ UNIT_ASSERT(stats.RSS >= NSystemInfo::GetPageSize());
+ UNIT_ASSERT(stats.VMS >= stats.RSS);
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TMemInfoTest)
diff --git a/util/system/mincore.cpp b/util/system/mincore.cpp
new file mode 100644
index 0000000000..8cbae72586
--- /dev/null
+++ b/util/system/mincore.cpp
@@ -0,0 +1,35 @@
+#include "align.h"
+#include "compiler.h"
+#include "info.h"
+#include "mincore.h"
+
+#include <util/generic/yexception.h>
+
+#include <cstring>
+
+#if defined(_unix_)
+ #include <sys/unistd.h>
+ #include <sys/mman.h>
+ #if defined(_android_)
+ #include <sys/syscall.h>
+ #endif
+#endif
+
+void InCoreMemory(const void* addr, size_t len, unsigned char* vec, size_t vecLen) {
+#if defined(_linux_)
+ const size_t pageSize = NSystemInfo::GetPageSize();
+ void* maddr = const_cast<void*>(AlignDown(addr, pageSize));
+ len = AlignUp(len, pageSize);
+ if (vecLen * pageSize < len) {
+ ythrow yexception() << "vector argument for mincore is too small: " << vecLen * pageSize << " < " << len;
+ }
+ if (::mincore(maddr, len, vec)) {
+ ythrow yexception() << LastSystemErrorText();
+ }
+#else
+ // pessimistic assumption: nothing is in core
+ Y_UNUSED(addr);
+ Y_UNUSED(len);
+ ::memset(vec, 0, vecLen);
+#endif
+}
diff --git a/util/system/mincore.h b/util/system/mincore.h
new file mode 100644
index 0000000000..0f5d8b78c7
--- /dev/null
+++ b/util/system/mincore.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "defaults.h"
+/**
+ * Fills a vector that indicates whether pages of the calling process's virtual memory are resident in RAM. Each byte
+ * in the vector contains the status of a single page. The page size can be obtained via the NSystemInfo::GetPageSize()
+ * function. Use the IsPageInCore function to interpret the page status byte.
+ *
+ * Can be overly pessimistic:
+ * - Assumes nothing is in RAM on platforms other than Linux
+ * - Recent Linux kernels (4.21 and some backports) may return zeroes if the process doesn't have writing permissions
+ * for the given file. See CVE-2019-5489.
+ *
+ * @param[in] addr starting address of the memory range to be examined
+ * @param[in] len length (bytes) of the memory range to be examined
+ * @param[out] vec vector of bytes to store statuses of memory pages
+ * @param[in] vecLen length (bytes) of the vec, should be large enough to hold the requested pages count
+ * @throws yexception if there was a system error or if the vecLen is too small
+ *
+ * @note this is only a snapshot, results may be stale by the time they're used
+ * @see man 2 mincore
+ */
+void InCoreMemory(const void* addr, size_t len, unsigned char* vec, size_t vecLen);
+
+/**
+ * Takes as an argument an element of the vector previously filled by InCoreMemory.
+ *
+ * @param[in] byte corresponding to the status of a single page
+ *
+ * @returns true if this page was resident in memory at the time out the InCoreMemory execution
+ */
+inline bool IsPageInCore(unsigned char s) {
+ /* From mincore(2): On return, the least significant bit of each byte will be set if the corresponding page is
+ * currently resident in memory, and be clear otherwise. (The settings of the other bits in each byte are
+ * undefined; these bits are reserved for possible later use.)
+ */
+ return s & 1;
+}
diff --git a/util/system/mincore_ut.cpp b/util/system/mincore_ut.cpp
new file mode 100644
index 0000000000..fc46cb1632
--- /dev/null
+++ b/util/system/mincore_ut.cpp
@@ -0,0 +1,47 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#ifdef _unix_
+ #include <sys/resource.h>
+#endif
+
+#include "filemap.h"
+#include "info.h"
+#include "mincore.h"
+#include "mlock.h"
+#include "tempfile.h"
+
+#include <util/generic/size_literals.h>
+#include <util/system/fs.h>
+
+#include <cstring>
+#include <cstdio>
+
+Y_UNIT_TEST_SUITE(MincoreSuite) {
+ static const char* FileName_("./mappped_file");
+
+ Y_UNIT_TEST(TestLockAndInCore) {
+ TVector<char> content(2_MB);
+
+ TTempFile cleanup(FileName_);
+ TFile file(FileName_, CreateAlways | WrOnly);
+ file.Write(content.data(), content.size());
+ file.Close();
+
+ TFileMap mappedFile(FileName_, TMemoryMapCommon::oRdWr);
+ mappedFile.Map(0, mappedFile.Length());
+ UNIT_ASSERT_EQUAL(mappedFile.MappedSize(), content.size());
+ UNIT_ASSERT_EQUAL(mappedFile.Length(), static_cast<i64>(content.size()));
+
+ LockMemory(mappedFile.Ptr(), mappedFile.Length());
+
+ TVector<unsigned char> incore(mappedFile.Length() / NSystemInfo::GetPageSize());
+ InCoreMemory(mappedFile.Ptr(), mappedFile.Length(), incore.data(), incore.size());
+
+ // compile and run on all platforms, but assume non-zero results only on Linux
+#if defined(_linux_)
+ for (const auto& flag : incore) {
+ UNIT_ASSERT(IsPageInCore(flag));
+ }
+#endif
+ }
+}
diff --git a/util/system/mktemp.cpp b/util/system/mktemp.cpp
new file mode 100644
index 0000000000..505b7b4a4b
--- /dev/null
+++ b/util/system/mktemp.cpp
@@ -0,0 +1,73 @@
+#include "tempfile.h"
+
+#include <util/folder/dirut.h>
+#include <util/generic/yexception.h>
+#include <util/stream/file.h>
+
+#include <cerrno>
+#include <cstring>
+
+#ifdef _win32_
+ #include "winint.h"
+ #include <io.h>
+#else
+ #include <unistd.h>
+ #include <stdlib.h>
+#endif
+
+extern "C" int mkstemps(char* path, int slen);
+
+TString MakeTempName(const char* wrkDir, const char* prefix, const char* extension) {
+#ifndef _win32_
+ TString filePath;
+
+ if (wrkDir && *wrkDir) {
+ filePath += wrkDir;
+ } else {
+ filePath += GetSystemTempDir();
+ }
+
+ if (filePath.back() != '/') {
+ filePath += '/';
+ }
+
+ if (prefix) {
+ filePath += prefix;
+ }
+
+ filePath += "XXXXXX"; // mkstemps requirement
+
+ size_t extensionPartLength = 0;
+ if (extension && *extension) {
+ if (extension[0] != '.') {
+ filePath += '.';
+ extensionPartLength += 1;
+ }
+ filePath += extension;
+ extensionPartLength += strlen(extension);
+ }
+
+ int fd = mkstemps(const_cast<char*>(filePath.data()), extensionPartLength);
+ if (fd >= 0) {
+ close(fd);
+ return filePath;
+ }
+#else
+ char tmpDir[MAX_PATH + 1]; // +1 -- for terminating null character
+ char filePath[MAX_PATH];
+ const char* pDir = 0;
+
+ if (wrkDir && *wrkDir) {
+ pDir = wrkDir;
+ } else if (GetTempPath(MAX_PATH + 1, tmpDir)) {
+ pDir = tmpDir;
+ }
+
+ // it always takes up to 3 characters, no more
+ if (GetTempFileName(pDir, (prefix) ? (prefix) : "yan", 0, filePath)) {
+ return filePath;
+ }
+#endif
+
+ ythrow TSystemError() << "can not create temp name(" << wrkDir << ", " << prefix << ", " << extension << ")";
+}
diff --git a/util/system/mktemp_system.cpp b/util/system/mktemp_system.cpp
new file mode 100644
index 0000000000..32bea2987c
--- /dev/null
+++ b/util/system/mktemp_system.cpp
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 1987, 1993
+ * The Regents of the University of California. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the University of
+ * California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "defaults.h"
+
+#include <sys/types.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+#ifdef _win32_
+ #include "winint.h"
+ #include <util/folder/dirut.h>
+#else
+ #include <unistd.h>
+#endif
+
+#include <util/random/random.h>
+#include "sysstat.h"
+
+static const unsigned char padchar[] =
+ "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+static int
+GetTemp(char* path, int* doopen, int domkdir, int slen)
+{
+ char *start, *trv, *suffp;
+ char* pad;
+#ifndef _win32_
+ struct stat sbuf;
+ int rval;
+#endif
+ ui32 rand;
+
+ if (doopen != nullptr && domkdir) {
+ errno = EINVAL;
+ return (0);
+ }
+
+ for (trv = path; *trv != '\0'; ++trv) {
+ ;
+ }
+ trv -= slen;
+ suffp = trv;
+ --trv;
+ if (trv < path) {
+ errno = EINVAL;
+ return (0);
+ }
+
+ /* Fill space with random characters */
+ while (trv >= path && *trv == 'X') {
+ rand = (RandomNumber<ui32>()) % (sizeof(padchar) - 1);
+ *trv-- = padchar[rand];
+ }
+ start = trv + 1;
+
+ /*
+ * check the target directory.
+ */
+ if (doopen != nullptr || domkdir) {
+ for (; trv > path; --trv) {
+ if (*trv == '/') {
+ *trv = '\0';
+#ifdef _win32_
+ ui32 attr = ::GetFileAttributesA(path);
+ *trv = '/';
+ if (attr == 0xFFFFFFFF)
+ return (0);
+ if (!(attr & FILE_ATTRIBUTE_DIRECTORY)) {
+ errno = ENOTDIR;
+ return (0);
+ }
+#else
+ rval = stat(path, &sbuf);
+ *trv = '/';
+ if (rval != 0) {
+ return (0);
+ }
+ if (!S_ISDIR(sbuf.st_mode)) {
+ errno = ENOTDIR;
+ return (0);
+ }
+#endif
+ break;
+ }
+ }
+ }
+
+ for (;;) {
+ if (doopen) {
+ if ((*doopen =
+ open(path, O_CREAT | O_EXCL | O_RDWR, 0600)) >= 0) {
+ return (1);
+ }
+ if (errno != EEXIST) {
+ return (0);
+ }
+ } else if (domkdir) {
+ if (Mkdir(path, S_IRWXU) == 0) {
+ return (1);
+ }
+ if (errno != EEXIST) {
+ return (0);
+ }
+ } else
+#ifdef _win32_
+ if (::GetFileAttributesA(path) == INVALID_FILE_ATTRIBUTES)
+ return (errno == ENOENT);
+#else
+ if (lstat(path, &sbuf)) {
+ return (errno == ENOENT);
+ }
+#endif
+ /* If we have a collision, cycle through the space of filenames */
+ for (trv = start;;) {
+ if (*trv == '\0' || trv == suffp) {
+ return (0);
+ }
+ pad = strchr((char*)padchar, *trv);
+ if (pad == nullptr || *++pad == '\0') {
+ *trv++ = padchar[0];
+ } else {
+ *trv++ = *pad;
+ break;
+ }
+ }
+ }
+ /*NOTREACHED*/
+}
+
+extern "C" int mkstemps(char* path, int slen) {
+ int fd;
+
+ return (GetTemp(path, &fd, 0, slen) ? fd : -1);
+}
+
+#if defined(_win_)
+char* mkdtemp(char* path) {
+ return (GetTemp(path, (int*)nullptr, 1, 0) ? path : (char*)nullptr);
+}
+#endif
diff --git a/util/system/mlock.cpp b/util/system/mlock.cpp
new file mode 100644
index 0000000000..435338c98f
--- /dev/null
+++ b/util/system/mlock.cpp
@@ -0,0 +1,86 @@
+#include <util/generic/yexception.h>
+#include "align.h"
+#include "error.h"
+#include "info.h"
+#include "mlock.h"
+
+#if defined(_unix_)
+ #include <sys/mman.h>
+ #if !defined(MCL_ONFAULT) && defined(MCL_FUTURE) // Old glibc.
+ #define MCL_ONFAULT (MCL_FUTURE << 1)
+ #endif
+ #if defined(_android_)
+ #include <sys/syscall.h>
+ #define munlockall() syscall(__NR_munlockall)
+ #endif
+#else
+ #include "winint.h"
+#endif
+
+void LockMemory(const void* addr, size_t len) {
+#if defined(_unix_)
+ const size_t pageSize = NSystemInfo::GetPageSize();
+ if (mlock(AlignDown(addr, pageSize), AlignUp(len, pageSize))) {
+ ythrow yexception() << LastSystemErrorText();
+ }
+#elif defined(_win_)
+ HANDLE hndl = GetCurrentProcess();
+ SIZE_T min, max;
+ if (!GetProcessWorkingSetSize(hndl, &min, &max))
+ ythrow yexception() << LastSystemErrorText();
+ if (!SetProcessWorkingSetSize(hndl, min + len, max + len))
+ ythrow yexception() << LastSystemErrorText();
+ if (!VirtualLock((LPVOID)addr, len))
+ ythrow yexception() << LastSystemErrorText();
+#endif
+}
+
+void UnlockMemory(const void* addr, size_t len) {
+#if defined(_unix_)
+ if (munlock(addr, len)) {
+ ythrow yexception() << LastSystemErrorText();
+ }
+#elif defined(_win_)
+ HANDLE hndl = GetCurrentProcess();
+ SIZE_T min, max;
+ if (!GetProcessWorkingSetSize(hndl, &min, &max))
+ ythrow yexception() << LastSystemErrorText();
+ if (!SetProcessWorkingSetSize(hndl, min - len, max - len))
+ ythrow yexception() << LastSystemErrorText();
+ if (!VirtualUnlock((LPVOID)addr, len))
+ ythrow yexception() << LastSystemErrorText();
+#endif
+}
+
+void LockAllMemory(ELockAllMemoryFlags flags) {
+ Y_UNUSED(flags);
+#if defined(_android_)
+// unimplemented
+#elif defined(_cygwin_)
+// unimplemented
+#elif defined(_unix_)
+ int sys_flags = 0;
+ if (flags & LockCurrentMemory) {
+ sys_flags |= MCL_CURRENT;
+ }
+ if (flags & LockFutureMemory) {
+ sys_flags |= MCL_FUTURE;
+ }
+ if (flags & LockMemoryOnFault) {
+ sys_flags |= MCL_ONFAULT;
+ }
+ if (mlockall(sys_flags)) {
+ ythrow yexception() << LastSystemErrorText();
+ }
+#endif
+}
+
+void UnlockAllMemory() {
+#if defined(_cygwin_)
+// unimplemented
+#elif defined(_unix_)
+ if (munlockall()) {
+ ythrow yexception() << LastSystemErrorText();
+ }
+#endif
+}
diff --git a/util/system/mlock.h b/util/system/mlock.h
new file mode 100644
index 0000000000..f021c0fe67
--- /dev/null
+++ b/util/system/mlock.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/flags.h>
+
+//on some systems (not win, freebd, linux, but darwin (Mac OS X)
+//multiple mlock calls on the same address range
+//require the corresponding number of munlock calls to actually unlock the pages
+
+//on some systems you must have privilege and resource limit
+
+void LockMemory(const void* addr, size_t len);
+void UnlockMemory(const void* addr, size_t len);
+
+enum ELockAllMemoryFlag {
+ /** Lock all pages which are currently mapped into the address space of the process. */
+ LockCurrentMemory = 1,
+
+ /** Lock all pages which will become mapped into the address space of the process in the future. */
+ LockFutureMemory = 2,
+
+ /** Since Linux 4.4, with LockCurrentMemory or LockFutureMemory or both, lock only pages that are or once they are present in memory. */
+ LockMemoryOnFault = 4,
+};
+Y_DECLARE_FLAGS(ELockAllMemoryFlags, ELockAllMemoryFlag)
+Y_DECLARE_OPERATORS_FOR_FLAGS(ELockAllMemoryFlags)
+
+/**
+ * Performs provided locking operation.
+ *
+ * Does nothing on windows.
+ *
+ * \param flags Locking operation to perform.
+ */
+void LockAllMemory(ELockAllMemoryFlags flags);
+
+/**
+ * Unlocks whatever was locked with a previous call to `LockAllMemory`.
+ *
+ * Does nothing on windows.
+ */
+void UnlockAllMemory();
diff --git a/util/system/mutex.cpp b/util/system/mutex.cpp
new file mode 100644
index 0000000000..4041402db9
--- /dev/null
+++ b/util/system/mutex.cpp
@@ -0,0 +1,146 @@
+#include "mutex.h"
+
+#include <util/generic/yexception.h>
+#include <errno.h>
+
+#if defined(_win_)
+ #include "winint.h"
+#else
+ #include <pthread.h>
+#endif
+
+class TMutex::TImpl {
+public:
+ inline TImpl() {
+#if defined(_win_)
+ InitializeCriticalSection(&Obj);
+#else
+ struct T {
+ pthread_mutexattr_t Attr;
+
+ inline T() {
+ int result;
+
+ memset(&Attr, 0, sizeof(Attr));
+ result = pthread_mutexattr_init(&Attr);
+ if (result != 0) {
+ ythrow yexception() << "mutexattr init failed(" << LastSystemErrorText(result) << ")";
+ }
+
+ result = pthread_mutexattr_settype(&Attr, PTHREAD_MUTEX_RECURSIVE);
+ if (result != 0) {
+ ythrow yexception() << "mutexattr set type failed(" << LastSystemErrorText(result) << ")";
+ }
+ }
+
+ inline ~T() {
+ int result = pthread_mutexattr_destroy(&Attr);
+ Y_VERIFY(result == 0, "mutexattr destroy(%s)", LastSystemErrorText(result));
+ }
+ } pma;
+
+ int result = pthread_mutex_init(&Obj, &pma.Attr);
+ if (result != 0) {
+ ythrow yexception() << "mutex init failed(" << LastSystemErrorText(result) << ")";
+ }
+#endif
+ }
+
+ inline ~TImpl() {
+#if defined(_win_)
+ DeleteCriticalSection(&Obj);
+#else
+ int result = pthread_mutex_destroy(&Obj);
+ Y_VERIFY(result == 0, "mutex destroy failure (%s)", LastSystemErrorText(result));
+#endif
+ }
+
+ inline void Acquire() noexcept {
+#if defined(_win_)
+ EnterCriticalSection(&Obj);
+#else
+ int result = pthread_mutex_lock(&Obj);
+ Y_VERIFY(result == 0, "mutex lock failure (%s)", LastSystemErrorText(result));
+#endif
+ }
+
+#if defined(_win_)
+ static bool TryEnterCriticalSectionInt(CRITICAL_SECTION* obj) {
+ #if (_WIN32_WINNT < 0x0400)
+ if (-1L == ::InterlockedCompareExchange(&obj->LockCount, 0, -1)) {
+ obj->OwningThread = (HANDLE)(DWORD_PTR)::GetCurrentThreadId();
+ obj->RecursionCount = 1;
+
+ return true;
+ }
+
+ if (obj->OwningThread == (HANDLE)(DWORD_PTR)::GetCurrentThreadId()) {
+ ::InterlockedIncrement(&obj->LockCount);
+ ++obj->RecursionCount;
+ return true;
+ }
+
+ return false;
+ #else // _WIN32_WINNT < 0x0400
+ return TryEnterCriticalSection(obj);
+ #endif // _WIN32_WINNT < 0x0400
+ }
+#endif // _win_
+
+ inline bool TryAcquire() noexcept {
+#if defined(_win_)
+ return TryEnterCriticalSectionInt(&Obj);
+#else
+ int result = pthread_mutex_trylock(&Obj);
+ if (result == 0 || result == EBUSY) {
+ return result == 0;
+ }
+ Y_FAIL("mutex trylock failure (%s)", LastSystemErrorText(result));
+#endif
+ }
+
+ inline void Release() noexcept {
+#if defined(_win_)
+ LeaveCriticalSection(&Obj);
+#else
+ int result = pthread_mutex_unlock(&Obj);
+ Y_VERIFY(result == 0, "mutex unlock failure (%s)", LastSystemErrorText(result));
+#endif
+ }
+
+ inline void* Handle() const noexcept {
+ return (void*)&Obj;
+ }
+
+private:
+#ifdef _win_
+ CRITICAL_SECTION Obj;
+#else
+ pthread_mutex_t Obj;
+#endif
+};
+
+TMutex::TMutex()
+ : Impl_(new TImpl())
+{
+}
+
+TMutex::TMutex(TMutex&&) = default;
+
+TMutex::~TMutex() = default;
+
+void TMutex::Acquire() noexcept {
+ Impl_->Acquire();
+}
+
+bool TMutex::TryAcquire() noexcept {
+ return Impl_->TryAcquire();
+}
+
+void TMutex::Release() noexcept {
+ Impl_->Release();
+}
+
+void* TMutex::Handle() const noexcept {
+ return Impl_->Handle();
+}
diff --git a/util/system/mutex.h b/util/system/mutex.h
new file mode 100644
index 0000000000..032630d134
--- /dev/null
+++ b/util/system/mutex.h
@@ -0,0 +1,64 @@
+#pragma once
+
+#include "guard.h"
+#include "defaults.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+
+class TFakeMutex: public TNonCopyable {
+public:
+ inline void Acquire() noexcept {
+ }
+
+ inline bool TryAcquire() noexcept {
+ return true;
+ }
+
+ inline void Release() noexcept {
+ }
+
+ inline void lock() noexcept {
+ Acquire();
+ }
+
+ inline bool try_lock() noexcept {
+ return TryAcquire();
+ }
+
+ inline void unlock() noexcept {
+ Release();
+ }
+
+ ~TFakeMutex() = default;
+};
+
+class TMutex {
+public:
+ TMutex();
+ TMutex(TMutex&&);
+ ~TMutex();
+
+ void Acquire() noexcept;
+ bool TryAcquire() noexcept;
+ void Release() noexcept;
+
+ inline void lock() noexcept {
+ Acquire();
+ }
+
+ inline bool try_lock() noexcept {
+ return TryAcquire();
+ }
+
+ inline void unlock() noexcept {
+ Release();
+ }
+
+ //return opaque pointer to real handler
+ void* Handle() const noexcept;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
diff --git a/util/system/mutex_ut.cpp b/util/system/mutex_ut.cpp
new file mode 100644
index 0000000000..c8d7caafa1
--- /dev/null
+++ b/util/system/mutex_ut.cpp
@@ -0,0 +1,129 @@
+#include "mutex.h"
+#include "atomic.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/thread/pool.h>
+#include <util/random/random.h>
+
+class TMutexTest: public TTestBase {
+ UNIT_TEST_SUITE(TMutexTest);
+ UNIT_TEST(TestBasics)
+ UNIT_TEST(TestFake)
+ UNIT_TEST(TestRecursive)
+ UNIT_TEST_SUITE_END();
+
+ struct TSharedData {
+ TSharedData()
+ : sharedCounter(0)
+ , failed(false)
+ {
+ }
+
+ volatile ui32 sharedCounter;
+ TMutex mutex;
+ TFakeMutex fakeMutex;
+
+ bool failed;
+ };
+
+ class TThreadTask: public IObjectInQueue {
+ public:
+ using PFunc = void (TThreadTask::*)(void);
+
+ TThreadTask(PFunc func, TSharedData& data, size_t id)
+ : Func_(func)
+ , Data_(data)
+ , Id_(id)
+ {
+ }
+
+ void Process(void*) override {
+ THolder<TThreadTask> This(this);
+
+ (this->*Func_)();
+ }
+
+#define FAIL_ASSERT(cond) \
+ if (!(cond)) { \
+ Data_.failed = true; \
+ }
+ void RunBasics() {
+ Data_.mutex.Acquire();
+
+ ui32 oldCounter = ui32(Data_.sharedCounter + Id_);
+ Data_.sharedCounter = oldCounter;
+ usleep(10 + RandomNumber<ui32>() % 10);
+ FAIL_ASSERT(Data_.sharedCounter == oldCounter);
+
+ Data_.mutex.Release();
+ }
+
+ void RunFakeMutex() {
+ bool res = Data_.fakeMutex.TryAcquire();
+ FAIL_ASSERT(res);
+ }
+
+ void RunRecursiveMutex() {
+ for (size_t i = 0; i < Id_ + 1; ++i) {
+ Data_.mutex.Acquire();
+ ++Data_.sharedCounter;
+ usleep(1);
+ }
+ FAIL_ASSERT(Data_.sharedCounter == Id_ + 1);
+
+ bool res = Data_.mutex.TryAcquire();
+ FAIL_ASSERT(res);
+ Data_.mutex.Release();
+
+ for (size_t i = 0; i < Id_; ++i) {
+ --Data_.sharedCounter;
+ Data_.mutex.Release();
+ }
+ FAIL_ASSERT(Data_.sharedCounter == 1);
+ --Data_.sharedCounter;
+ Data_.mutex.Release();
+ }
+
+#undef FAIL_ASSERT
+
+ private:
+ PFunc Func_;
+ TSharedData& Data_;
+ size_t Id_;
+ };
+
+private:
+#define RUN_CYCLE(what, count) \
+ Q_.Start(count); \
+ for (size_t i = 0; i < count; ++i) { \
+ UNIT_ASSERT(Q_.Add(new TThreadTask(&TThreadTask::what, Data_, i))); \
+ } \
+ Q_.Stop(); \
+ bool b = Data_.failed; \
+ Data_.failed = false; \
+ UNIT_ASSERT(!b);
+
+ void TestBasics() {
+ RUN_CYCLE(RunBasics, 5);
+
+ UNIT_ASSERT(Data_.sharedCounter == 10);
+ Data_.sharedCounter = 0;
+ }
+
+ void TestFake() {
+ RUN_CYCLE(RunFakeMutex, 3);
+ }
+
+ void TestRecursive() {
+ RUN_CYCLE(RunRecursiveMutex, 4);
+ }
+
+#undef RUN_CYCLE
+
+private:
+ TSharedData Data_;
+ TThreadPool Q_;
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TMutexTest)
diff --git a/util/system/nice.cpp b/util/system/nice.cpp
new file mode 100644
index 0000000000..20fe926054
--- /dev/null
+++ b/util/system/nice.cpp
@@ -0,0 +1,15 @@
+#include "nice.h"
+
+#include "platform.h"
+
+#if defined(_unix_)
+ #include <unistd.h>
+#endif
+
+bool Nice(int prioDelta) {
+#if defined(_unix_)
+ return nice(prioDelta) != -1;
+#else
+ return prioDelta == 0;
+#endif
+}
diff --git a/util/system/nice.h b/util/system/nice.h
new file mode 100644
index 0000000000..86b8bc5f3c
--- /dev/null
+++ b/util/system/nice.h
@@ -0,0 +1,3 @@
+#pragma once
+
+bool Nice(int prioDelta);
diff --git a/util/system/nice_ut.cpp b/util/system/nice_ut.cpp
new file mode 100644
index 0000000000..ae0051f285
--- /dev/null
+++ b/util/system/nice_ut.cpp
@@ -0,0 +1,42 @@
+#include "nice.h"
+
+#include "platform.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#ifdef _unix_
+ #include <sys/resource.h>
+
+static int GetPriority() {
+ return getpriority(PRIO_PROCESS, 0);
+}
+#endif
+
+Y_UNIT_TEST_SUITE(NiceTest) {
+ Y_UNIT_TEST(TestNiceZero) {
+ UNIT_ASSERT(Nice(0));
+ UNIT_ASSERT(Nice(0));
+ }
+#ifdef _unix_
+ Y_UNIT_TEST(TestNice) {
+ int prio = GetPriority();
+
+ if (prio >= 10) {
+ return;
+ }
+
+ if (Nice(-2)) {
+ UNIT_ASSERT_VALUES_EQUAL(GetPriority(), prio - 2);
+ prio -= 2;
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(GetPriority(), prio);
+ }
+ UNIT_ASSERT(Nice(1));
+ UNIT_ASSERT_VALUES_EQUAL(GetPriority(), prio + 1);
+ UNIT_ASSERT(Nice(0));
+ UNIT_ASSERT_VALUES_EQUAL(GetPriority(), prio + 1);
+ UNIT_ASSERT(Nice(2));
+ UNIT_ASSERT_VALUES_EQUAL(GetPriority(), prio + 3);
+ }
+#endif
+}
diff --git a/util/system/pipe.cpp b/util/system/pipe.cpp
new file mode 100644
index 0000000000..a543bd7472
--- /dev/null
+++ b/util/system/pipe.cpp
@@ -0,0 +1,161 @@
+#include "pipe.h"
+
+#include <util/stream/output.h>
+#include <util/generic/yexception.h>
+
+ssize_t TPipeHandle::Read(void* buffer, size_t byteCount) const noexcept {
+#ifdef _win_
+ return recv(Fd_, (char*)buffer, byteCount, 0);
+#else
+ return read(Fd_, buffer, byteCount);
+#endif
+}
+
+ssize_t TPipeHandle::Write(const void* buffer, size_t byteCount) const noexcept {
+#ifdef _win_
+ return send(Fd_, (const char*)buffer, byteCount, 0);
+#else
+ return write(Fd_, buffer, byteCount);
+#endif
+}
+
+bool TPipeHandle::Close() noexcept {
+ bool ok = true;
+ if (Fd_ != INVALID_PIPEHANDLE) {
+#ifdef _win_
+ ok = closesocket(Fd_) == 0;
+#else
+ ok = close(Fd_) == 0;
+#endif
+ }
+ Fd_ = INVALID_PIPEHANDLE;
+ return ok;
+}
+
+void TPipeHandle::Pipe(TPipeHandle& reader, TPipeHandle& writer, EOpenMode mode) {
+ PIPEHANDLE fds[2];
+#ifdef _win_
+ int r = SocketPair(fds, false /* non-overlapped */, mode & CloseOnExec /* cloexec */);
+#elif defined(_linux_)
+ int r = pipe2(fds, mode & CloseOnExec ? O_CLOEXEC : 0);
+#else
+ int r = pipe(fds);
+#endif
+ if (r < 0) {
+ ythrow TFileError() << "failed to create a pipe";
+ }
+
+#if !defined(_win_) && !defined(_linux_)
+ // Non-atomic wrt exec
+ if (mode & CloseOnExec) {
+ for (int i = 0; i < 2; ++i) {
+ int flags = fcntl(fds[i], F_GETFD, 0);
+ if (flags < 0) {
+ ythrow TFileError() << "failed to get flags";
+ }
+ int r = fcntl(fds[i], F_SETFD, flags | FD_CLOEXEC);
+ if (r < 0) {
+ ythrow TFileError() << "failed to set flags";
+ }
+ }
+ }
+#endif
+
+ TPipeHandle(fds[0]).Swap(reader);
+ TPipeHandle(fds[1]).Swap(writer);
+}
+
+class TPipe::TImpl: public TAtomicRefCount<TImpl> {
+public:
+ TImpl()
+ : Handle_(INVALID_PIPEHANDLE)
+ {
+ }
+
+ TImpl(PIPEHANDLE fd)
+ : Handle_(fd)
+ {
+ }
+
+ inline ~TImpl() {
+ Close();
+ }
+
+ bool IsOpen() {
+ return Handle_.IsOpen();
+ }
+
+ inline void Close() {
+ if (!Handle_.IsOpen()) {
+ return;
+ }
+ if (!Handle_.Close()) {
+ ythrow TFileError() << "failed to close pipe";
+ }
+ }
+
+ TPipeHandle& GetHandle() noexcept {
+ return Handle_;
+ }
+
+ size_t Read(void* buffer, size_t count) const {
+ ssize_t r = Handle_.Read(buffer, count);
+ if (r < 0) {
+ ythrow TFileError() << "failed to read from pipe";
+ }
+ return r;
+ }
+
+ size_t Write(const void* buffer, size_t count) const {
+ ssize_t r = Handle_.Write(buffer, count);
+ if (r < 0) {
+ ythrow TFileError() << "failed to write to pipe";
+ }
+ return r;
+ }
+
+private:
+ TPipeHandle Handle_;
+};
+
+TPipe::TPipe()
+ : Impl_(new TImpl)
+{
+}
+
+TPipe::TPipe(PIPEHANDLE fd)
+ : Impl_(new TImpl(fd))
+{
+}
+
+TPipe::~TPipe() = default;
+
+void TPipe::Close() {
+ Impl_->Close();
+}
+
+PIPEHANDLE TPipe::GetHandle() const noexcept {
+ return Impl_->GetHandle();
+}
+
+bool TPipe::IsOpen() const noexcept {
+ return Impl_->IsOpen();
+}
+
+size_t TPipe::Read(void* buf, size_t len) const {
+ return Impl_->Read(buf, len);
+}
+
+size_t TPipe::Write(const void* buf, size_t len) const {
+ return Impl_->Write(buf, len);
+}
+
+void TPipe::Pipe(TPipe& reader, TPipe& writer, EOpenMode mode) {
+ TImplRef r(new TImpl());
+ TImplRef w(new TImpl());
+
+ TPipeHandle::Pipe(r->GetHandle(), w->GetHandle(), mode);
+
+ r.Swap(reader.Impl_);
+ w.Swap(writer.Impl_);
+}
diff --git a/util/system/pipe.h b/util/system/pipe.h
new file mode 100644
index 0000000000..75d0360049
--- /dev/null
+++ b/util/system/pipe.h
@@ -0,0 +1,90 @@
+#pragma once
+
+#ifdef __GNUC__
+ #pragma GCC diagnostic push
+ #pragma GCC diagnostic ignored "-Warray-bounds" //need because of bug in gcc4.9.2
+#endif
+
+#include "defaults.h"
+#include "file.h"
+#include <util/generic/ptr.h>
+#include <util/network/pair.h>
+#include <util/generic/noncopyable.h>
+
+using PIPEHANDLE = SOCKET;
+#define INVALID_PIPEHANDLE INVALID_SOCKET
+
+/// Pipe-like object: pipe on POSIX and socket on windows
+class TPipeHandle: public TNonCopyable {
+public:
+ inline TPipeHandle() noexcept
+ : Fd_(INVALID_PIPEHANDLE)
+ {
+ }
+
+ inline TPipeHandle(PIPEHANDLE fd) noexcept
+ : Fd_(fd)
+ {
+ }
+
+ inline ~TPipeHandle() {
+ Close();
+ }
+
+ bool Close() noexcept;
+
+ inline PIPEHANDLE Release() noexcept {
+ PIPEHANDLE ret = Fd_;
+ Fd_ = INVALID_PIPEHANDLE;
+ return ret;
+ }
+
+ inline void Swap(TPipeHandle& r) noexcept {
+ DoSwap(Fd_, r.Fd_);
+ }
+
+ inline operator PIPEHANDLE() const noexcept {
+ return Fd_;
+ }
+
+ inline bool IsOpen() const noexcept {
+ return Fd_ != INVALID_PIPEHANDLE;
+ }
+
+ ssize_t Read(void* buffer, size_t byteCount) const noexcept;
+ ssize_t Write(const void* buffer, size_t byteCount) const noexcept;
+
+ // Only CloseOnExec is supported
+ static void Pipe(TPipeHandle& reader, TPipeHandle& writer, EOpenMode mode = 0);
+
+private:
+ PIPEHANDLE Fd_;
+};
+
+class TPipe {
+public:
+ TPipe();
+ /// Takes ownership of handle, so closes it when the last holder of descriptor dies.
+ explicit TPipe(PIPEHANDLE fd);
+ ~TPipe();
+
+ void Close();
+
+ bool IsOpen() const noexcept;
+ PIPEHANDLE GetHandle() const noexcept;
+
+ size_t Read(void* buf, size_t len) const;
+ size_t Write(const void* buf, size_t len) const;
+
+ // Only CloseOnExec is supported
+ static void Pipe(TPipe& reader, TPipe& writer, EOpenMode mode = 0);
+
+private:
+ class TImpl;
+ using TImplRef = TSimpleIntrusivePtr<TImpl>;
+ TImplRef Impl_;
+};
+
+#ifdef __GNUC__
+ #pragma GCC diagnostic pop
+#endif
diff --git a/util/system/pipe_ut.cpp b/util/system/pipe_ut.cpp
new file mode 100644
index 0000000000..6d53432de8
--- /dev/null
+++ b/util/system/pipe_ut.cpp
@@ -0,0 +1,15 @@
+#include "pipe.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TPipeTest) {
+ Y_UNIT_TEST(TestPipe) {
+ TPipe r;
+ TPipe w;
+ TPipe::Pipe(r, w);
+ char c = 'a';
+ UNIT_ASSERT(1 == w.Write(&c, 1));
+ UNIT_ASSERT(1 == r.Read(&c, 1));
+ UNIT_ASSERT_VALUES_EQUAL('a', c);
+ }
+}
diff --git a/util/system/platform.cpp b/util/system/platform.cpp
new file mode 100644
index 0000000000..9421235bb4
--- /dev/null
+++ b/util/system/platform.cpp
@@ -0,0 +1,18 @@
+#include "platform.h"
+
+#include <cstddef>
+
+static_assert(sizeof(char) == SIZEOF_CHAR, "incorrect SIZEOF_CHAR");
+static_assert(sizeof(short) == SIZEOF_SHORT, "incorrect SIZEOF_SHORT");
+static_assert(sizeof(int) == SIZEOF_INT, "incorrect SIZEOF_INT");
+static_assert(sizeof(long) == SIZEOF_LONG, "incorrect SIZEOF_LONG");
+static_assert(sizeof(long long) == SIZEOF_LONG_LONG, "incorrect SIZEOF_LONG_LONG");
+
+static_assert(sizeof(unsigned char) == SIZEOF_UNSIGNED_CHAR, "incorrect SIZEOF_UNSIGNED_CHAR");
+static_assert(sizeof(unsigned short) == SIZEOF_UNSIGNED_SHORT, "incorrect SIZEOF_UNSIGNED_SHORT");
+static_assert(sizeof(unsigned int) == SIZEOF_UNSIGNED_INT, "incorrect SIZEOF_UNSIGNED_INT");
+static_assert(sizeof(unsigned long) == SIZEOF_UNSIGNED_LONG, "incorrect SIZEOF_UNSIGNED_LONG");
+static_assert(sizeof(unsigned long long) == SIZEOF_UNSIGNED_LONG_LONG, "incorrect SIZEOF_UNSIGNED_LONG_LONG");
+
+static_assert(sizeof(void*) == SIZEOF_PTR, "incorrect SIZEOF_PTR");
+static_assert(sizeof(void*) == sizeof(size_t), "unsupported platform");
diff --git a/util/system/platform.h b/util/system/platform.h
new file mode 100644
index 0000000000..58f310ab34
--- /dev/null
+++ b/util/system/platform.h
@@ -0,0 +1,246 @@
+#pragma once
+
+// ya style breaks indentation in ifdef's and code becomes unreadable
+// clang-format off
+
+// What OS ?
+// our definition has the form _{osname}_
+
+#if defined(_WIN64)
+ #define _win64_
+ #define _win32_
+#elif defined(__WIN32__) || defined(_WIN32) // _WIN32 is also defined by the 64-bit compiler for backward compatibility
+ #define _win32_
+#else
+ #define _unix_
+
+ #if defined(__sun__) || defined(sun) || defined(sparc) || defined(__sparc)
+ #define _sun_
+ #endif
+
+ #if defined(__hpux__)
+ #define _hpux_
+ #endif
+
+ #if defined(__linux__)
+ // Stands for "Linux" in the means of Linux kernel (i. e. Android is included)
+ #define _linux_
+ #endif
+
+ #if defined(__FreeBSD__)
+ #define _freebsd_
+ #endif
+
+ #if defined(__CYGWIN__)
+ #define _cygwin_
+ #endif
+
+ #if defined(__APPLE__)
+ #define _darwin_
+ #endif
+
+ #if defined(__ANDROID__)
+ #define _android_
+ #endif
+#endif
+
+#if defined(__IOS__)
+ #define _ios_
+#endif
+
+#if defined(_linux_)
+ #if defined(_musl_)
+ // nothing to do
+ #elif defined(_android_)
+ // Please do not mix with android-based systems.
+ // This definition describes Standard Library (libc) type.
+ #define _bionic_
+ #else
+ #define _glibc_
+ #endif
+#endif
+
+#if defined(_darwin_)
+ #define unix
+ #define __unix__
+#endif
+
+#if defined(_win32_) || defined(_win64_)
+ #define _win_
+#endif
+
+#if defined(__arm__) || defined(__ARM__) || defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM)
+ #if defined(__arm64) || defined(__arm64__) || defined(__aarch64__)
+ #define _arm64_
+ #else
+ #define _arm32_
+ #endif
+#endif
+
+#if defined(_arm64_) || defined(_arm32_)
+ #define _arm_
+#endif
+
+/* __ia64__ and __x86_64__ - defined by GNU C.
+ * _M_IA64, _M_X64, _M_AMD64 - defined by Visual Studio.
+ *
+ * Microsoft can define _M_IX86, _M_AMD64 (before Visual Studio 8)
+ * or _M_X64 (starting in Visual Studio 8).
+ */
+#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64)
+ #define _x86_64_
+#endif
+
+#if defined(__i386__) || defined(_M_IX86)
+ #define _i386_
+#endif
+
+#if defined(__ia64__) || defined(_M_IA64)
+ #define _ia64_
+#endif
+
+#if defined(__powerpc__)
+ #define _ppc_
+#endif
+
+#if defined(__powerpc64__)
+ #define _ppc64_
+#endif
+
+#if !defined(sparc) && !defined(__sparc) && !defined(__hpux__) && !defined(__alpha__) && !defined(_ia64_) && !defined(_x86_64_) && !defined(_arm_) && !defined(_i386_) && !defined(_ppc_) && !defined(_ppc64_)
+ #error "platform not defined, please, define one"
+#endif
+
+#if defined(_x86_64_) || defined(_i386_)
+ #define _x86_
+#endif
+
+#if defined(__MIC__)
+ #define _mic_
+ #define _k1om_
+#endif
+
+// stdio or MessageBox
+#if defined(__CONSOLE__) || defined(_CONSOLE)
+ #define _console_
+#endif
+#if (defined(_win_) && !defined(_console_))
+ #define _windows_
+#elif !defined(_console_)
+ #define _console_
+#endif
+
+#if defined(__SSE__) || defined(SSE_ENABLED)
+ #define _sse_
+#endif
+
+#if defined(__SSE2__) || defined(SSE2_ENABLED)
+ #define _sse2_
+#endif
+
+#if defined(__SSE3__) || defined(SSE3_ENABLED)
+ #define _sse3_
+#endif
+
+#if defined(__SSSE3__) || defined(SSSE3_ENABLED)
+ #define _ssse3_
+#endif
+
+#if defined(__SSE4_1__) || defined(SSE41_ENABLED)
+ #define _sse4_1_
+#endif
+
+#if defined(__SSE4_2__) || defined(SSE42_ENABLED)
+ #define _sse4_2_
+#endif
+
+#if defined(__POPCNT__) || defined(POPCNT_ENABLED)
+ #define _popcnt_
+#endif
+
+#if defined(__PCLMUL__) || defined(PCLMUL_ENABLED)
+ #define _pclmul_
+#endif
+
+#if defined(__AES__) || defined(AES_ENABLED)
+ #define _aes_
+#endif
+
+#if defined(__AVX__) || defined(AVX_ENABLED)
+ #define _avx_
+#endif
+
+#if defined(__AVX2__) || defined(AVX2_ENABLED)
+ #define _avx2_
+#endif
+
+#if defined(__FMA__) || defined(FMA_ENABLED)
+ #define _fma_
+#endif
+
+#if defined(__DLL__) || defined(_DLL)
+ #define _dll_
+#endif
+
+// 16, 32 or 64
+#if defined(__sparc_v9__) || defined(_x86_64_) || defined(_ia64_) || defined(_arm64_) || defined(_ppc64_)
+ #define _64_
+#else
+ #define _32_
+#endif
+
+/* All modern 64-bit Unix systems use scheme LP64 (long, pointers are 64-bit).
+ * Microsoft uses a different scheme: LLP64 (long long, pointers are 64-bit).
+ *
+ * Scheme LP64 LLP64
+ * char 8 8
+ * short 16 16
+ * int 32 32
+ * long 64 32
+ * long long 64 64
+ * pointer 64 64
+ */
+
+#if defined(_32_)
+ #define SIZEOF_PTR 4
+#elif defined(_64_)
+ #define SIZEOF_PTR 8
+#endif
+
+#define PLATFORM_DATA_ALIGN SIZEOF_PTR
+
+#if !defined(SIZEOF_PTR)
+ #error todo
+#endif
+
+#define SIZEOF_CHAR 1
+#define SIZEOF_UNSIGNED_CHAR 1
+#define SIZEOF_SHORT 2
+#define SIZEOF_UNSIGNED_SHORT 2
+#define SIZEOF_INT 4
+#define SIZEOF_UNSIGNED_INT 4
+
+#if defined(_32_)
+ #define SIZEOF_LONG 4
+ #define SIZEOF_UNSIGNED_LONG 4
+#elif defined(_64_)
+ #if defined(_win_)
+ #define SIZEOF_LONG 4
+ #define SIZEOF_UNSIGNED_LONG 4
+ #else
+ #define SIZEOF_LONG 8
+ #define SIZEOF_UNSIGNED_LONG 8
+ #endif // _win_
+#endif // _32_
+
+#if !defined(SIZEOF_LONG)
+ #error todo
+#endif
+
+#define SIZEOF_LONG_LONG 8
+#define SIZEOF_UNSIGNED_LONG_LONG 8
+
+#undef SIZEOF_SIZE_T // in case we include <Python.h> which defines it, too
+#define SIZEOF_SIZE_T SIZEOF_PTR
+
+// clang-format on
diff --git a/util/system/platform_ut.cpp b/util/system/platform_ut.cpp
new file mode 100644
index 0000000000..9bfbce8315
--- /dev/null
+++ b/util/system/platform_ut.cpp
@@ -0,0 +1,27 @@
+#include "platform.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+class TPlatformTest: public TTestBase {
+ UNIT_TEST_SUITE(TPlatformTest);
+ UNIT_TEST(TestSizeOf)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline void TestSizeOf() {
+ UNIT_ASSERT_EQUAL(SIZEOF_PTR, sizeof(void*));
+ UNIT_ASSERT_EQUAL(SIZEOF_CHAR, sizeof(char));
+ UNIT_ASSERT_EQUAL(SIZEOF_SHORT, sizeof(short));
+ UNIT_ASSERT_EQUAL(SIZEOF_INT, sizeof(int));
+ UNIT_ASSERT_EQUAL(SIZEOF_LONG, sizeof(long));
+ UNIT_ASSERT_EQUAL(SIZEOF_LONG_LONG, sizeof(long long));
+ UNIT_ASSERT_EQUAL(SIZEOF_SIZE_T, sizeof(size_t));
+ UNIT_ASSERT_EQUAL(SIZEOF_UNSIGNED_CHAR, sizeof(unsigned char));
+ UNIT_ASSERT_EQUAL(SIZEOF_UNSIGNED_INT, sizeof(unsigned int));
+ UNIT_ASSERT_EQUAL(SIZEOF_UNSIGNED_LONG, sizeof(unsigned long));
+ UNIT_ASSERT_EQUAL(SIZEOF_UNSIGNED_LONG_LONG, sizeof(unsigned long long));
+ UNIT_ASSERT_EQUAL(SIZEOF_UNSIGNED_SHORT, sizeof(unsigned short));
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TPlatformTest);
diff --git a/util/system/progname.cpp b/util/system/progname.cpp
new file mode 100644
index 0000000000..2c29119320
--- /dev/null
+++ b/util/system/progname.cpp
@@ -0,0 +1,26 @@
+#include "execpath.h"
+#include "progname.h"
+
+#include <util/folder/dirut.h>
+#include <util/generic/singleton.h>
+
+static const char* Argv0;
+
+namespace {
+ struct TProgramNameHolder {
+ inline TProgramNameHolder()
+ : ProgName(GetFileNameComponent(Argv0 ? Argv0 : GetExecPath().data()))
+ {
+ }
+
+ TString ProgName;
+ };
+}
+
+const TString& GetProgramName() {
+ return Singleton<TProgramNameHolder>()->ProgName;
+}
+
+void SetProgramName(const char* argv0) {
+ Argv0 = argv0;
+}
diff --git a/util/system/progname.h b/util/system/progname.h
new file mode 100644
index 0000000000..e5e2a0eee2
--- /dev/null
+++ b/util/system/progname.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+
+void SetProgramName(const char* argv0);
+
+#define SAVE_PROGRAM_NAME \
+ do { \
+ SetProgramName(argv[0]); \
+ } while (0)
+
+/// guaranted return the same immutable instance of TString
+const TString& GetProgramName();
diff --git a/util/system/progname_ut.cpp b/util/system/progname_ut.cpp
new file mode 100644
index 0000000000..11f3d9308b
--- /dev/null
+++ b/util/system/progname_ut.cpp
@@ -0,0 +1,18 @@
+#include "progname.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TProgramNameTest) {
+ Y_UNIT_TEST(TestIt) {
+ TString progName = GetProgramName();
+
+ try {
+ UNIT_ASSERT(
+ progName.find("ut_util") != TString::npos || progName.find("util-system_ut") != TString::npos || progName.find("util-system-ut") != TString::npos);
+ } catch (...) {
+ Cerr << progName << Endl;
+
+ throw;
+ }
+ }
+}
diff --git a/util/system/protect.cpp b/util/system/protect.cpp
new file mode 100644
index 0000000000..bbb8d410df
--- /dev/null
+++ b/util/system/protect.cpp
@@ -0,0 +1,95 @@
+#include "protect.h"
+
+#include <util/generic/yexception.h>
+#include <util/generic/string.h>
+#include <util/stream/output.h>
+
+#include "yassert.h"
+
+#if defined(_unix_) || defined(_darwin_)
+ #include <sys/mman.h>
+#endif
+
+#ifdef _win_
+ #include <Windows.h>
+#endif
+
+static TString ModeToString(const EProtectMemory mode) {
+ TString strMode;
+ if (mode == PM_NONE) {
+ return "PM_NONE";
+ }
+
+ if (mode & PM_READ) {
+ strMode += "PM_READ|";
+ }
+ if (mode & PM_WRITE) {
+ strMode += "PM_WRITE|";
+ }
+ if (mode & PM_EXEC) {
+ strMode += "PM_EXEC|";
+ }
+ return strMode.substr(0, strMode.size() - 1);
+}
+
+void ProtectMemory(void* addr, const size_t length, const EProtectMemory mode) {
+ Y_VERIFY(!(mode & ~(PM_READ | PM_WRITE | PM_EXEC)), "Invalid memory protection flag combination. ");
+
+#if defined(_unix_) || defined(_darwin_)
+ int mpMode = PROT_NONE;
+ if (mode & PM_READ) {
+ mpMode |= PROT_READ;
+ }
+ if (mode & PM_WRITE) {
+ mpMode |= PROT_WRITE;
+ }
+ if (mode & PM_EXEC) {
+ mpMode |= PROT_EXEC;
+ }
+ // some old manpages for mprotect say 'const void* addr', but that's wrong
+ if (mprotect(addr, length, mpMode) == -1) {
+ ythrow TSystemError() << "Memory protection failed for mode " << ModeToString(mode) << ". ";
+ }
+#endif
+
+#ifdef _win_
+ DWORD mpMode = PAGE_NOACCESS;
+ // windows developers are not aware of bit flags :(
+
+ /*
+ * It's unclear that we should NOT fail on Windows that does not support write-only
+ * memory protection. As we don't know, what behavior is more correct, we choose
+ * one of them. A discussion was here: REVIEW: 39725
+ */
+ switch (mode.ToBaseType()) {
+ case PM_READ:
+ mpMode = PAGE_READONLY;
+ break;
+ case PM_WRITE:
+ mpMode = PAGE_READWRITE;
+ break; // BUG: no write-only support
+ /*case PM_WRITE:
+ ythrow TSystemError() << "Write-only protection mode is not supported under Windows. ";*/
+ case PM_READ | PM_WRITE:
+ mpMode = PAGE_READWRITE;
+ break;
+ case PM_EXEC:
+ mpMode = PAGE_EXECUTE;
+ break;
+ case PM_READ | PM_EXEC:
+ mpMode = PAGE_EXECUTE_READ;
+ break;
+ case PM_WRITE | PM_EXEC:
+ mpMode = PAGE_EXECUTE_READWRITE;
+ break; // BUG: no write-only support
+ /*case PM_WRITE | PM_EXEC:
+ ythrow TSystemError() << "Write-execute-only protection mode is not supported under Windows. ";*/
+ case PM_READ | PM_WRITE | PM_EXEC:
+ mpMode = PAGE_EXECUTE_READWRITE;
+ break;
+ }
+ DWORD oldMode = 0;
+ if (!VirtualProtect(addr, length, mpMode, &oldMode))
+ ythrow TSystemError() << "Memory protection failed for mode " << ModeToString(mode) << ". ";
+#endif
+}
diff --git a/util/system/protect.h b/util/system/protect.h
new file mode 100644
index 0000000000..26714f3e92
--- /dev/null
+++ b/util/system/protect.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/flags.h>
+
+enum EProtectMemoryMode {
+ PM_NONE = 0x00, // no access allowed
+ PM_READ = 0x01, // read access allowed
+ PM_WRITE = 0x02, // write access allowed
+ PM_EXEC = 0x04 // execute access allowed
+};
+
+Y_DECLARE_FLAGS(EProtectMemory, EProtectMemoryMode)
+Y_DECLARE_OPERATORS_FOR_FLAGS(EProtectMemory)
+
+/**
+ * Set protection mode on memory block
+ * @param addr Block address to be protected
+ * @param length Block size in bytes
+ * @param mode A bitwise combination of @c EProtectMemoryMode flags
+ * @note On Windows there is no write-only protection mode,
+ * so PM_WRITE will be translated to (PM_READ | PM_WRITE) on Windows.
+ **/
+void ProtectMemory(void* addr, const size_t length, const EProtectMemory mode);
diff --git a/util/system/rusage.cpp b/util/system/rusage.cpp
new file mode 100644
index 0000000000..2befeca875
--- /dev/null
+++ b/util/system/rusage.cpp
@@ -0,0 +1,121 @@
+#include "platform.h"
+
+#if defined(__APPLE__) && defined(__MACH__)
+
+ #include <mach/mach.h>
+
+#endif
+
+#ifdef _win_
+
+ #include "winint.h"
+ #include <psapi.h>
+
+#else
+
+ #include <sys/time.h>
+ #include <sys/resource.h>
+
+#endif
+
+#include <util/generic/yexception.h>
+
+#include "info.h"
+
+#include "rusage.h"
+
+#ifdef _win_
+TDuration FiletimeToDuration(const FILETIME& ft) {
+ union {
+ ui64 ft_scalar;
+ FILETIME ft_struct;
+ } nt_time;
+ nt_time.ft_struct = ft;
+ return TDuration::MicroSeconds(nt_time.ft_scalar / 10);
+}
+#endif
+
+size_t TRusage::GetCurrentRSS() {
+/*
+ * Author: David Robert Nadeau
+ * Site: http://NadeauSoftware.com/
+ * License: Creative Commons Attribution 3.0 Unported License
+ * http://creativecommons.org/licenses/by/3.0/deed.en_US
+ */
+#if defined(_WIN32)
+ /* Windows -------------------------------------------------- */
+ PROCESS_MEMORY_COUNTERS info;
+ GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
+ return (size_t)info.WorkingSetSize;
+#elif defined(__APPLE__) && defined(__MACH__)
+ /* OSX ------------------------------------------------------ */
+ struct mach_task_basic_info info;
+ mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+ if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO,
+ (task_info_t)&info, &infoCount) != KERN_SUCCESS)
+ return (size_t)0L; /* Can't access? */
+ return (size_t)info.resident_size;
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+ /* Linux ---------------------------------------------------- */
+ long rss = 0L;
+ FILE* fp = nullptr;
+ if ((fp = fopen("/proc/self/statm", "r")) == nullptr) {
+ return (size_t)0L; /* Can't open? */
+ }
+ if (fscanf(fp, "%*s%ld", &rss) != 1) {
+ fclose(fp);
+ return (size_t)0L; /* Can't read? */
+ }
+ fclose(fp);
+ return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE);
+#else
+ /* AIX, BSD, Solaris, and Unknown OS ------------------------ */
+ return (size_t)0L; /* Unsupported. */
+#endif
+}
+
+void TRusage::Fill() {
+ *this = TRusage();
+
+#ifdef _win_
+ // copy-paste from PostgreSQL getrusage.c
+
+ FILETIME starttime;
+ FILETIME exittime;
+ FILETIME kerneltime;
+ FILETIME usertime;
+
+ if (GetProcessTimes(GetCurrentProcess(), &starttime, &exittime, &kerneltime, &usertime) == 0) {
+ ythrow TSystemError() << "GetProcessTimes failed";
+ }
+
+ Utime = FiletimeToDuration(usertime);
+ Stime = FiletimeToDuration(kerneltime);
+
+ PROCESS_MEMORY_COUNTERS pmc;
+
+ if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) == 0) {
+ ythrow TSystemError() << "GetProcessMemoryInfo failed";
+ }
+
+ MaxRss = pmc.PeakWorkingSetSize;
+ MajorPageFaults = pmc.PageFaultCount;
+
+#else
+ struct rusage ru;
+ int r = getrusage(RUSAGE_SELF, &ru);
+ if (r < 0) {
+ ythrow TSystemError() << "rusage failed";
+ }
+
+ #if defined(_darwin_)
+ // see https://lists.apple.com/archives/darwin-kernel/2009/Mar/msg00005.html
+ MaxRss = ru.ru_maxrss;
+ #else
+ MaxRss = ru.ru_maxrss * 1024LL;
+ #endif
+ MajorPageFaults = ru.ru_majflt;
+ Utime = ru.ru_utime;
+ Stime = ru.ru_stime;
+#endif
+}
diff --git a/util/system/rusage.h b/util/system/rusage.h
new file mode 100644
index 0000000000..61aeca83f2
--- /dev/null
+++ b/util/system/rusage.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include "defaults.h"
+#include <util/generic/utility.h>
+#include <util/datetime/base.h>
+
+/// portable getrusage
+
+struct TRusage {
+ // some fields may be zero if unsupported
+
+ ui64 MaxRss = 0;
+ ui64 MajorPageFaults = 0;
+ TDuration Utime;
+ TDuration Stime;
+
+ void Fill();
+
+ static size_t GetCurrentRSS();
+
+ static TRusage Get() {
+ TRusage r;
+ r.Fill();
+ return r;
+ }
+};
diff --git a/util/system/rusage_ut.cpp b/util/system/rusage_ut.cpp
new file mode 100644
index 0000000000..0d4e0fe54b
--- /dev/null
+++ b/util/system/rusage_ut.cpp
@@ -0,0 +1,11 @@
+#include "rusage.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TRusageTest) {
+ Y_UNIT_TEST(TestRusage) {
+ TRusage r;
+ // just check it returns something
+ r.Fill();
+ }
+}
diff --git a/util/system/rwlock.cpp b/util/system/rwlock.cpp
new file mode 100644
index 0000000000..bb3dcbf188
--- /dev/null
+++ b/util/system/rwlock.cpp
@@ -0,0 +1,250 @@
+#include "rwlock.h"
+#include "mutex.h"
+#include "condvar.h"
+
+#include <util/generic/yexception.h>
+
+#if defined(_unix_)
+ #include <errno.h>
+ #include <pthread.h>
+#endif
+
+#if defined(_win_) || defined(_darwin_)
+//darwin rwlocks not recursive
+class TRWMutex::TImpl {
+public:
+ TImpl();
+ ~TImpl();
+
+ void AcquireRead() noexcept;
+ bool TryAcquireRead() noexcept;
+ void ReleaseRead() noexcept;
+
+ void AcquireWrite() noexcept;
+ bool TryAcquireWrite() noexcept;
+ void ReleaseWrite() noexcept;
+
+ void Release() noexcept;
+
+private:
+ TMutex Lock_;
+ int State_;
+ TCondVar ReadCond_;
+ TCondVar WriteCond_;
+ int BlockedWriters_;
+};
+
+TRWMutex::TImpl::TImpl()
+ : State_(0)
+ , BlockedWriters_(0)
+{
+}
+
+TRWMutex::TImpl::~TImpl() {
+ Y_VERIFY(State_ == 0, "failure, State_ != 0");
+ Y_VERIFY(BlockedWriters_ == 0, "failure, BlockedWriters_ != 0");
+}
+
+void TRWMutex::TImpl::AcquireRead() noexcept {
+ with_lock (Lock_) {
+ while (BlockedWriters_ || State_ < 0) {
+ ReadCond_.Wait(Lock_);
+ }
+
+ ++State_;
+ }
+
+ ReadCond_.Signal();
+}
+
+bool TRWMutex::TImpl::TryAcquireRead() noexcept {
+ with_lock (Lock_) {
+ if (BlockedWriters_ || State_ < 0) {
+ return false;
+ }
+
+ ++State_;
+ }
+
+ return true;
+}
+
+void TRWMutex::TImpl::ReleaseRead() noexcept {
+ Lock_.Acquire();
+
+ if (--State_ > 0) {
+ Lock_.Release();
+ } else if (BlockedWriters_) {
+ Lock_.Release();
+ WriteCond_.Signal();
+ } else {
+ Lock_.Release();
+ }
+}
+
+void TRWMutex::TImpl::AcquireWrite() noexcept {
+ with_lock (Lock_) {
+ while (State_ != 0) {
+ ++BlockedWriters_;
+ WriteCond_.Wait(Lock_);
+ --BlockedWriters_;
+ }
+
+ State_ = -1;
+ }
+}
+
+bool TRWMutex::TImpl::TryAcquireWrite() noexcept {
+ with_lock (Lock_) {
+ if (State_ != 0) {
+ return false;
+ }
+
+ State_ = -1;
+ }
+
+ return true;
+}
+
+void TRWMutex::TImpl::ReleaseWrite() noexcept {
+ Lock_.Acquire();
+ State_ = 0;
+
+ if (BlockedWriters_) {
+ Lock_.Release();
+ WriteCond_.Signal();
+ } else {
+ Lock_.Release();
+ ReadCond_.Signal();
+ }
+}
+
+void TRWMutex::TImpl::Release() noexcept {
+ Lock_.Acquire();
+
+ if (State_ > 0) {
+ if (--State_ > 0) {
+ Lock_.Release();
+ } else if (BlockedWriters_) {
+ Lock_.Release();
+ WriteCond_.Signal();
+ }
+ } else {
+ State_ = 0;
+
+ if (BlockedWriters_) {
+ Lock_.Release();
+ WriteCond_.Signal();
+ } else {
+ Lock_.Release();
+ ReadCond_.Signal();
+ }
+ }
+}
+
+#else /* POSIX threads */
+
+class TRWMutex::TImpl {
+public:
+ TImpl();
+ ~TImpl();
+
+ void AcquireRead() noexcept;
+ bool TryAcquireRead() noexcept;
+ void ReleaseRead() noexcept;
+
+ void AcquireWrite() noexcept;
+ bool TryAcquireWrite() noexcept;
+ void ReleaseWrite() noexcept;
+
+ void Release() noexcept;
+
+private:
+ pthread_rwlock_t Lock_;
+};
+
+TRWMutex::TImpl::TImpl() {
+ int result = pthread_rwlock_init(&Lock_, nullptr);
+ if (result != 0) {
+ ythrow yexception() << "rwlock init failed (" << LastSystemErrorText(result) << ")";
+ }
+}
+
+TRWMutex::TImpl::~TImpl() {
+ const int result = pthread_rwlock_destroy(&Lock_);
+ Y_VERIFY(result == 0, "rwlock destroy failed (%s)", LastSystemErrorText(result));
+}
+
+void TRWMutex::TImpl::AcquireRead() noexcept {
+ const int result = pthread_rwlock_rdlock(&Lock_);
+ Y_VERIFY(result == 0, "rwlock rdlock failed (%s)", LastSystemErrorText(result));
+}
+
+bool TRWMutex::TImpl::TryAcquireRead() noexcept {
+ const int result = pthread_rwlock_tryrdlock(&Lock_);
+ Y_VERIFY(result == 0 || result == EBUSY, "rwlock tryrdlock failed (%s)", LastSystemErrorText(result));
+ return result == 0;
+}
+
+void TRWMutex::TImpl::ReleaseRead() noexcept {
+ const int result = pthread_rwlock_unlock(&Lock_);
+ Y_VERIFY(result == 0, "rwlock (read) unlock failed (%s)", LastSystemErrorText(result));
+}
+
+void TRWMutex::TImpl::AcquireWrite() noexcept {
+ const int result = pthread_rwlock_wrlock(&Lock_);
+ Y_VERIFY(result == 0, "rwlock wrlock failed (%s)", LastSystemErrorText(result));
+}
+
+bool TRWMutex::TImpl::TryAcquireWrite() noexcept {
+ const int result = pthread_rwlock_trywrlock(&Lock_);
+ Y_VERIFY(result == 0 || result == EBUSY, "rwlock trywrlock failed (%s)", LastSystemErrorText(result));
+ return result == 0;
+}
+
+void TRWMutex::TImpl::ReleaseWrite() noexcept {
+ const int result = pthread_rwlock_unlock(&Lock_);
+ Y_VERIFY(result == 0, "rwlock (write) unlock failed (%s)", LastSystemErrorText(result));
+}
+
+void TRWMutex::TImpl::Release() noexcept {
+ const int result = pthread_rwlock_unlock(&Lock_);
+ Y_VERIFY(result == 0, "rwlock unlock failed (%s)", LastSystemErrorText(result));
+}
+
+#endif
+
+TRWMutex::TRWMutex()
+ : Impl_(new TImpl())
+{
+}
+
+TRWMutex::~TRWMutex() = default;
+
+void TRWMutex::AcquireRead() noexcept {
+ Impl_->AcquireRead();
+}
+
+bool TRWMutex::TryAcquireRead() noexcept {
+ return Impl_->TryAcquireRead();
+}
+
+void TRWMutex::ReleaseRead() noexcept {
+ Impl_->ReleaseRead();
+}
+
+void TRWMutex::AcquireWrite() noexcept {
+ Impl_->AcquireWrite();
+}
+
+bool TRWMutex::TryAcquireWrite() noexcept {
+ return Impl_->TryAcquireWrite();
+}
+
+void TRWMutex::ReleaseWrite() noexcept {
+ Impl_->ReleaseWrite();
+}
+
+void TRWMutex::Release() noexcept {
+ Impl_->Release();
+}
diff --git a/util/system/rwlock.h b/util/system/rwlock.h
new file mode 100644
index 0000000000..0bb9b3fe1c
--- /dev/null
+++ b/util/system/rwlock.h
@@ -0,0 +1,78 @@
+#pragma once
+
+#include "guard.h"
+#include "defaults.h"
+
+#include <util/generic/ptr.h>
+
+class TRWMutex {
+public:
+ TRWMutex();
+ ~TRWMutex();
+
+ void AcquireRead() noexcept;
+ bool TryAcquireRead() noexcept;
+ void ReleaseRead() noexcept;
+
+ void AcquireWrite() noexcept;
+ bool TryAcquireWrite() noexcept;
+ void ReleaseWrite() noexcept;
+
+ void Release() noexcept;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+template <class T>
+struct TReadGuardOps {
+ static inline void Acquire(T* t) noexcept {
+ t->AcquireRead();
+ }
+
+ static inline void Release(T* t) noexcept {
+ t->ReleaseRead();
+ }
+};
+
+template <class T>
+struct TTryReadGuardOps: public TReadGuardOps<T> {
+ static inline bool TryAcquire(T* t) noexcept {
+ return t->TryAcquireRead();
+ }
+};
+
+template <class T>
+struct TWriteGuardOps {
+ static inline void Acquire(T* t) noexcept {
+ t->AcquireWrite();
+ }
+
+ static inline void Release(T* t) noexcept {
+ t->ReleaseWrite();
+ }
+};
+
+template <class T>
+struct TTryWriteGuardOps: public TWriteGuardOps<T> {
+ static inline bool TryAcquire(T* t) noexcept {
+ return t->TryAcquireWrite();
+ }
+};
+
+template <class T>
+using TReadGuardBase = TGuard<T, TReadGuardOps<T>>;
+template <class T>
+using TTryReadGuardBase = TTryGuard<T, TTryReadGuardOps<T>>;
+
+template <class T>
+using TWriteGuardBase = TGuard<T, TWriteGuardOps<T>>;
+template <class T>
+using TTryWriteGuardBase = TTryGuard<T, TTryWriteGuardOps<T>>;
+
+using TReadGuard = TReadGuardBase<TRWMutex>;
+using TTryReadGuard = TTryReadGuardBase<TRWMutex>;
+
+using TWriteGuard = TWriteGuardBase<TRWMutex>;
+using TTryWriteGuard = TTryWriteGuardBase<TRWMutex>;
diff --git a/util/system/rwlock_ut.cpp b/util/system/rwlock_ut.cpp
new file mode 100644
index 0000000000..2b384c05b3
--- /dev/null
+++ b/util/system/rwlock_ut.cpp
@@ -0,0 +1,124 @@
+#include "rwlock.h"
+#include "atomic.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/thread/pool.h>
+#include <util/random/random.h>
+
+class TRWMutexTest: public TTestBase {
+ UNIT_TEST_SUITE(TRWMutexTest);
+ UNIT_TEST(TestReaders)
+ UNIT_TEST(TestReadersWriters)
+ UNIT_TEST_SUITE_END();
+
+ struct TSharedData {
+ TSharedData()
+ : writersIn(0)
+ , readersIn(0)
+ , failed(false)
+ {
+ }
+
+ TAtomic writersIn;
+ TAtomic readersIn;
+
+ bool failed;
+
+ TRWMutex mutex;
+ };
+
+ class TThreadTask: public IObjectInQueue {
+ public:
+ using PFunc = void (TThreadTask::*)(void);
+
+ TThreadTask(PFunc func, TSharedData& data, size_t id, size_t total)
+ : Func_(func)
+ , Data_(data)
+ , Id_(id)
+ , Total_(total)
+ {
+ }
+
+ void Process(void*) override {
+ THolder<TThreadTask> This(this);
+
+ (this->*Func_)();
+ }
+
+#define FAIL_ASSERT(cond) \
+ if (!(cond)) { \
+ Data_.failed = true; \
+ }
+ void RunReaders() {
+ Data_.mutex.AcquireRead();
+
+ AtomicIncrement(Data_.readersIn);
+ usleep(100);
+ FAIL_ASSERT(Data_.readersIn == long(Total_));
+ usleep(100);
+ AtomicDecrement(Data_.readersIn);
+
+ Data_.mutex.ReleaseRead();
+ }
+
+ void RunReadersWriters() {
+ if (Id_ % 2 == 0) {
+ for (size_t i = 0; i < 10; ++i) {
+ Data_.mutex.AcquireRead();
+
+ AtomicIncrement(Data_.readersIn);
+ FAIL_ASSERT(Data_.writersIn == 0);
+ usleep(RandomNumber<ui32>() % 5);
+ AtomicDecrement(Data_.readersIn);
+
+ Data_.mutex.ReleaseRead();
+ }
+ } else {
+ for (size_t i = 0; i < 10; ++i) {
+ Data_.mutex.AcquireWrite();
+
+ AtomicIncrement(Data_.writersIn);
+ FAIL_ASSERT(Data_.readersIn == 0 && Data_.writersIn == 1);
+ usleep(RandomNumber<ui32>() % 5);
+ AtomicDecrement(Data_.writersIn);
+
+ Data_.mutex.ReleaseWrite();
+ }
+ }
+ }
+#undef FAIL_ASSERT
+
+ private:
+ PFunc Func_;
+ TSharedData& Data_;
+ size_t Id_;
+ size_t Total_;
+ };
+
+private:
+#define RUN_CYCLE(what, count) \
+ Q_.Start(count); \
+ for (size_t i = 0; i < count; ++i) { \
+ UNIT_ASSERT(Q_.Add(new TThreadTask(&TThreadTask::what, Data_, i, count))); \
+ } \
+ Q_.Stop(); \
+ bool b = Data_.failed; \
+ Data_.failed = false; \
+ UNIT_ASSERT(!b);
+
+ void TestReaders() {
+ RUN_CYCLE(RunReaders, 1);
+ }
+
+ void TestReadersWriters() {
+ RUN_CYCLE(RunReadersWriters, 1);
+ }
+
+#undef RUN_CYCLE
+private:
+ TSharedData Data_;
+ TThreadPool Q_;
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TRWMutexTest)
diff --git a/util/system/sanitizers.cpp b/util/system/sanitizers.cpp
new file mode 100644
index 0000000000..bb799a9e2e
--- /dev/null
+++ b/util/system/sanitizers.cpp
@@ -0,0 +1,142 @@
+#include "sanitizers.h"
+#include "thread.h"
+
+#if defined(_asan_enabled_)
+extern "C" {
+ void __sanitizer_start_switch_fiber(void** fake_stack_save, const void* bottom, size_t size);
+ void __sanitizer_finish_switch_fiber(void* fake_stack_save, const void** old_bottom, size_t* old_size);
+}
+#endif
+
+#if defined(_tsan_enabled_)
+ #if defined(__clang_major__) && (__clang_major__ >= 9)
+extern "C" {
+ void* __tsan_get_current_fiber(void);
+ void* __tsan_create_fiber(unsigned flags);
+ void __tsan_destroy_fiber(void* fiber);
+ void __tsan_switch_to_fiber(void* fiber, unsigned flags);
+ void __tsan_set_fiber_name(void* fiber, const char* name);
+}
+ #else
+namespace {
+ void* __tsan_get_current_fiber(void) {
+ return nullptr;
+ }
+ void* __tsan_create_fiber(unsigned) {
+ return nullptr;
+ }
+ void __tsan_destroy_fiber(void*) {
+ }
+ void __tsan_switch_to_fiber(void*, unsigned) {
+ }
+ void __tsan_set_fiber_name(void*, const char*) {
+ }
+}
+ #endif
+#endif
+
+using namespace NSan;
+
+TFiberContext::TFiberContext() noexcept
+ : Token_(nullptr)
+ , IsMainFiber_(true)
+#if defined(_tsan_enabled_)
+ , CurrentTSanFiberContext_(__tsan_get_current_fiber())
+#endif
+{
+ TCurrentThreadLimits sl;
+
+ Stack_ = sl.StackBegin;
+ Len_ = sl.StackLength;
+
+#if defined(_tsan_enabled_)
+ static constexpr char MainFiberName[] = "main_fiber";
+ __tsan_set_fiber_name(CurrentTSanFiberContext_, MainFiberName);
+#endif
+}
+
+TFiberContext::TFiberContext(const void* stack, size_t len, const char* contName) noexcept
+ : Token_(nullptr)
+ , Stack_(stack)
+ , Len_(len)
+ , IsMainFiber_(false)
+#if defined(_tsan_enabled_)
+ , CurrentTSanFiberContext_(__tsan_create_fiber(/*flags =*/0))
+#endif
+{
+ (void)contName;
+#if defined(_tsan_enabled_)
+ __tsan_set_fiber_name(CurrentTSanFiberContext_, contName);
+#endif
+}
+
+TFiberContext::~TFiberContext() noexcept {
+ if (!IsMainFiber_) {
+#if defined(_asan_enabled_)
+ if (Token_) {
+ // destroy saved FakeStack
+ void* activeFakeStack = nullptr;
+ const void* activeStack = nullptr;
+ size_t activeStackSize = 0;
+ __sanitizer_start_switch_fiber(&activeFakeStack, (char*)Stack_, Len_);
+ __sanitizer_finish_switch_fiber(Token_, &activeStack, &activeStackSize);
+ __sanitizer_start_switch_fiber(nullptr, activeStack, activeStackSize);
+ __sanitizer_finish_switch_fiber(activeFakeStack, nullptr, nullptr);
+ }
+#endif
+#if defined(_tsan_enabled_)
+ __tsan_destroy_fiber(CurrentTSanFiberContext_);
+#endif
+ }
+}
+
+void TFiberContext::BeforeFinish() noexcept {
+#if defined(_asan_enabled_)
+ __sanitizer_start_switch_fiber(nullptr, nullptr, 0);
+#else
+ (void)Token_;
+ (void)Stack_;
+ (void)Len_;
+#endif
+}
+
+void TFiberContext::BeforeSwitch(TFiberContext* old) noexcept {
+#if defined(_asan_enabled_)
+ __sanitizer_start_switch_fiber(old ? &old->Token_ : nullptr, (char*)Stack_, Len_);
+#else
+ (void)old;
+#endif
+
+#if defined(_tsan_enabled_)
+ __tsan_switch_to_fiber(CurrentTSanFiberContext_, /*flags =*/0);
+#endif
+}
+
+void TFiberContext::AfterSwitch() noexcept {
+#if defined(_asan_enabled_)
+ __sanitizer_finish_switch_fiber(Token_, nullptr, nullptr);
+#endif
+}
+
+void TFiberContext::AfterStart() noexcept {
+#if defined(_asan_enabled_)
+ __sanitizer_finish_switch_fiber(nullptr, nullptr, nullptr);
+#endif
+}
+
+#if defined(_tsan_enabled_)
+extern "C" {
+ // This function should not be directly exposed in headers
+ // due to signature variations among contrib headers.
+ void AnnotateBenignRaceSized(const char* file, int line,
+ const volatile void* address,
+ size_t size,
+ const char* description);
+}
+void NSan::AnnotateBenignRaceSized(const char* file, int line,
+ const volatile void* address,
+ size_t size,
+ const char* description) noexcept {
+ ::AnnotateBenignRaceSized(file, line, address, size, description);
+}
+#endif
diff --git a/util/system/sanitizers.h b/util/system/sanitizers.h
new file mode 100644
index 0000000000..965e5c751e
--- /dev/null
+++ b/util/system/sanitizers.h
@@ -0,0 +1,141 @@
+#pragma once
+
+#include "defaults.h"
+
+extern "C" { // sanitizers API
+
+#if defined(_asan_enabled_)
+ void __lsan_ignore_object(const void* p);
+#endif
+
+#if defined(_msan_enabled_)
+ void __msan_unpoison(const volatile void* a, size_t size);
+ void __msan_poison(const volatile void* a, size_t size);
+ void __msan_check_mem_is_initialized(const volatile void* x, size_t size);
+#endif
+
+}; // sanitizers API
+
+namespace NSan {
+ class TFiberContext {
+ public:
+ TFiberContext() noexcept;
+ TFiberContext(const void* stack, size_t len, const char* contName) noexcept;
+
+ ~TFiberContext() noexcept;
+
+ void BeforeFinish() noexcept;
+ void BeforeSwitch(TFiberContext* old) noexcept;
+ void AfterSwitch() noexcept;
+
+ static void AfterStart() noexcept;
+
+ private:
+ void* Token_;
+ const void* Stack_;
+ size_t Len_;
+
+ const bool IsMainFiber_;
+#if defined(_tsan_enabled_)
+ void* const CurrentTSanFiberContext_;
+#endif
+ };
+
+ // Returns plain if no sanitizer enabled or sanitized otherwise
+ // Ment to be used in test code for constants (timeouts, etc)
+ template <typename T>
+ inline constexpr static T PlainOrUnderSanitizer(T plain, T sanitized) noexcept {
+#if defined(_tsan_enabled_) || defined(_msan_enabled_) || defined(_asan_enabled_)
+ Y_UNUSED(plain);
+ return sanitized;
+#else
+ Y_UNUSED(sanitized);
+ return plain;
+#endif
+ }
+
+ // Determines if asan present
+ inline constexpr static bool ASanIsOn() noexcept {
+#if defined(_asan_enabled_)
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Determines if tsan present
+ inline constexpr static bool TSanIsOn() noexcept {
+#if defined(_tsan_enabled_)
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Determines if msan present
+ inline constexpr static bool MSanIsOn() noexcept {
+#if defined(_msan_enabled_)
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Make memory region fully initialized (without changing its contents).
+ inline static void Unpoison(const volatile void* a, size_t size) noexcept {
+#if defined(_msan_enabled_)
+ __msan_unpoison(a, size);
+#else
+ Y_UNUSED(a);
+ Y_UNUSED(size);
+#endif
+ }
+
+ // Make memory region fully uninitialized (without changing its contents).
+ // This is a legacy interface that does not update origin information. Use __msan_allocated_memory() instead.
+ inline static void Poison(const volatile void* a, size_t size) noexcept {
+#if defined(_msan_enabled_)
+ __msan_poison(a, size);
+#else
+ Y_UNUSED(a);
+ Y_UNUSED(size);
+#endif
+ }
+
+ // Checks that memory range is fully initialized, and reports an error if it is not.
+ inline static void CheckMemIsInitialized(const volatile void* a, size_t size) noexcept {
+#if defined(_msan_enabled_)
+ __msan_check_mem_is_initialized(a, size);
+#else
+ Y_UNUSED(a);
+ Y_UNUSED(size);
+#endif
+ }
+
+ inline static void MarkAsIntentionallyLeaked(const void* ptr) noexcept {
+#if defined(_asan_enabled_)
+ __lsan_ignore_object(ptr);
+#else
+ Y_UNUSED(ptr);
+#endif
+ }
+
+#if defined(_tsan_enabled_)
+ // defined in .cpp to avoid exposing problematic C-linkage version of AnnotateBenignRaceSized(...)
+ void AnnotateBenignRaceSized(const char* file, int line,
+ const volatile void* address,
+ size_t size,
+ const char* description) noexcept;
+#else
+ inline static void AnnotateBenignRaceSized(const char* file, int line,
+ const volatile void* address,
+ size_t size,
+ const char* description) noexcept {
+ Y_UNUSED(file);
+ Y_UNUSED(line);
+ Y_UNUSED(address);
+ Y_UNUSED(size);
+ Y_UNUSED(description);
+ }
+#endif
+}
diff --git a/util/system/sanitizers_ut.cpp b/util/system/sanitizers_ut.cpp
new file mode 100644
index 0000000000..5de096e14b
--- /dev/null
+++ b/util/system/sanitizers_ut.cpp
@@ -0,0 +1,15 @@
+#include "sanitizers.h"
+#include "sys_alloc.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(Sanitizers) {
+ Y_UNIT_TEST(MarkAsIntentionallyLeaked) {
+ auto* p1 = new i32[100];
+ NSan::MarkAsIntentionallyLeaked(p1);
+
+ auto* p2 = y_allocate(123);
+ NSan::MarkAsIntentionallyLeaked(p2);
+ }
+
+} // Y_UNIT_TEST_SUITE(Sanitizers)
diff --git a/util/system/sem.cpp b/util/system/sem.cpp
new file mode 100644
index 0000000000..4a93b903b5
--- /dev/null
+++ b/util/system/sem.cpp
@@ -0,0 +1,278 @@
+#include "sem.h"
+
+#ifdef _win_
+ #include <malloc.h>
+#elif defined(_sun)
+ #include <alloca.h>
+#endif
+
+#include <cerrno>
+#include <cstring>
+
+#ifdef _win_
+ #include "winint.h"
+#else
+ #include <signal.h>
+ #include <unistd.h>
+ #include <semaphore.h>
+
+ #if defined(_bionic_) || defined(_darwin_) && defined(_arm_)
+ #include <fcntl.h>
+ #else
+ #define USE_SYSV_SEMAPHORES //unixoids declared the standard but not implemented it...
+ #endif
+#endif
+
+#ifdef USE_SYSV_SEMAPHORES
+ #include <errno.h>
+ #include <sys/types.h>
+ #include <sys/ipc.h>
+ #include <sys/sem.h>
+
+ #if defined(_linux_) || defined(_sun_) || defined(_cygwin_)
+union semun {
+ int val;
+ struct semid_ds* buf;
+ unsigned short* array;
+} arg;
+ #else
+union semun arg;
+ #endif
+#endif
+
+#include <util/digest/city.h>
+#include <util/string/cast.h>
+#include <util/random/random.h>
+#include <util/random/fast.h>
+
+namespace {
+ class TSemaphoreImpl {
+ private:
+#ifdef _win_
+ using SEMHANDLE = HANDLE;
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ using SEMHANDLE = int;
+ #else
+ using SEMHANDLE = sem_t*;
+ #endif
+#endif
+
+ SEMHANDLE Handle;
+
+ public:
+ inline TSemaphoreImpl(const char* name, ui32 max_free_count)
+ : Handle(0)
+ {
+#ifdef _win_
+ char* key = (char*)name;
+ if (name) {
+ size_t len = strlen(name);
+ key = (char*)alloca(len + 1);
+ strcpy(key, name);
+ if (len > MAX_PATH)
+ *(key + MAX_PATH) = 0;
+ char* p = key;
+ while (*p) {
+ if (*p == '\\')
+ *p = '/';
+ ++p;
+ }
+ }
+ // non-blocking on init
+ Handle = ::CreateSemaphore(0, max_free_count, max_free_count, key);
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ key_t key = TPCGMixer::Mix(CityHash64(name, strlen(name))); //32 bit hash
+ Handle = semget(key, 0, 0); // try to open exist semaphore
+ if (Handle == -1) { // create new semaphore
+ Handle = semget(key, 1, 0666 | IPC_CREAT);
+ if (Handle != -1) {
+ union semun arg;
+ arg.val = max_free_count;
+ semctl(Handle, 0, SETVAL, arg);
+ } else {
+ ythrow TSystemError() << "can not init sempahore";
+ }
+ }
+ #else
+ Handle = sem_open(name, O_CREAT, 0666, max_free_count);
+ if (Handle == SEM_FAILED) {
+ ythrow TSystemError() << "can not init sempahore";
+ }
+ #endif
+#endif
+ }
+
+ inline ~TSemaphoreImpl() {
+#ifdef _win_
+ ::CloseHandle(Handle);
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ // we DO NOT want 'semctl(Handle, 0, IPC_RMID)' for multiprocess tasks;
+ //struct sembuf ops[] = {{0, 0, IPC_NOWAIT}};
+ //if (semop(Handle, ops, 1) != 0) // close only if semaphore's value is zero
+ // semctl(Handle, 0, IPC_RMID);
+ #else
+ sem_close(Handle); // we DO NOT want sem_unlink(...)
+ #endif
+#endif
+ }
+
+ inline void Release() noexcept {
+#ifdef _win_
+ ::ReleaseSemaphore(Handle, 1, 0);
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ struct sembuf ops[] = {{0, 1, SEM_UNDO}};
+ int ret = semop(Handle, ops, 1);
+ #else
+ int ret = sem_post(Handle);
+ #endif
+ Y_VERIFY(ret == 0, "can not release semaphore");
+#endif
+ }
+
+ //The UNIX semaphore object does not support a timed "wait", and
+ //hence to maintain consistancy, for win32 case we use INFINITE or 0 timeout.
+ inline void Acquire() noexcept {
+#ifdef _win_
+ Y_VERIFY(::WaitForSingleObject(Handle, INFINITE) == WAIT_OBJECT_0, "can not acquire semaphore");
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ struct sembuf ops[] = {{0, -1, SEM_UNDO}};
+ int ret = semop(Handle, ops, 1);
+ #else
+ int ret = sem_wait(Handle);
+ #endif
+ Y_VERIFY(ret == 0, "can not acquire semaphore");
+#endif
+ }
+
+ inline bool TryAcquire() noexcept {
+#ifdef _win_
+ // zero-second time-out interval
+ // WAIT_OBJECT_0: current free count > 0
+ // WAIT_TIMEOUT: current free count == 0
+ return ::WaitForSingleObject(Handle, 0) == WAIT_OBJECT_0;
+#else
+ #ifdef USE_SYSV_SEMAPHORES
+ struct sembuf ops[] = {{0, -1, SEM_UNDO | IPC_NOWAIT}};
+ int ret = semop(Handle, ops, 1);
+ #else
+ int ret = sem_trywait(Handle);
+ #endif
+ return ret == 0;
+#endif
+ }
+ };
+
+#if defined(_unix_)
+ /*
+ Disable errors/warnings about deprecated sem_* in Darwin
+*/
+ #ifdef _darwin_
+ Y_PRAGMA_DIAGNOSTIC_PUSH
+ Y_PRAGMA_NO_DEPRECATED
+ #endif
+ struct TPosixSemaphore {
+ inline TPosixSemaphore(ui32 maxFreeCount) {
+ if (sem_init(&S_, 0, maxFreeCount)) {
+ ythrow TSystemError() << "can not init semaphore";
+ }
+ }
+
+ inline ~TPosixSemaphore() {
+ Y_VERIFY(sem_destroy(&S_) == 0, "semaphore destroy failed");
+ }
+
+ inline void Acquire() noexcept {
+ Y_VERIFY(sem_wait(&S_) == 0, "semaphore acquire failed");
+ }
+
+ inline void Release() noexcept {
+ Y_VERIFY(sem_post(&S_) == 0, "semaphore release failed");
+ }
+
+ inline bool TryAcquire() noexcept {
+ if (sem_trywait(&S_)) {
+ Y_VERIFY(errno == EAGAIN, "semaphore try wait failed");
+
+ return false;
+ }
+
+ return true;
+ }
+
+ sem_t S_;
+ };
+ #ifdef _darwin_
+ Y_PRAGMA_DIAGNOSTIC_POP
+ #endif
+#endif
+}
+
+class TSemaphore::TImpl: public TSemaphoreImpl {
+public:
+ inline TImpl(const char* name, ui32 maxFreeCount)
+ : TSemaphoreImpl(name, maxFreeCount)
+ {
+ }
+};
+
+TSemaphore::TSemaphore(const char* name, ui32 maxFreeCount)
+ : Impl_(new TImpl(name, maxFreeCount))
+{
+}
+
+TSemaphore::~TSemaphore() = default;
+
+void TSemaphore::Release() noexcept {
+ Impl_->Release();
+}
+
+void TSemaphore::Acquire() noexcept {
+ Impl_->Acquire();
+}
+
+bool TSemaphore::TryAcquire() noexcept {
+ return Impl_->TryAcquire();
+}
+
+#if defined(_unix_) && !defined(_darwin_)
+class TFastSemaphore::TImpl: public TPosixSemaphore {
+public:
+ inline TImpl(ui32 n)
+ : TPosixSemaphore(n)
+ {
+ }
+};
+#else
+class TFastSemaphore::TImpl: public TString, public TSemaphoreImpl {
+public:
+ inline TImpl(ui32 n)
+ : TString(ToString(RandomNumber<ui64>()))
+ , TSemaphoreImpl(c_str(), n)
+ {
+ }
+};
+#endif
+
+TFastSemaphore::TFastSemaphore(ui32 maxFreeCount)
+ : Impl_(new TImpl(maxFreeCount))
+{
+}
+
+TFastSemaphore::~TFastSemaphore() = default;
+
+void TFastSemaphore::Release() noexcept {
+ Impl_->Release();
+}
+
+void TFastSemaphore::Acquire() noexcept {
+ Impl_->Acquire();
+}
+
+bool TFastSemaphore::TryAcquire() noexcept {
+ return Impl_->TryAcquire();
+}
diff --git a/util/system/sem.h b/util/system/sem.h
new file mode 100644
index 0000000000..0c964ad6b6
--- /dev/null
+++ b/util/system/sem.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/ptr.h>
+
+//named sempahore
+class TSemaphore {
+public:
+ TSemaphore(const char* name, ui32 maxFreeCount);
+ ~TSemaphore();
+
+ //Increase the semaphore counter.
+ void Release() noexcept;
+
+ //Keep a thread held while the semaphore counter is equal 0.
+ void Acquire() noexcept;
+
+ //Try to enter the semaphore gate. A non-blocking variant of Acquire.
+ //Returns 'true' if the semaphore counter decreased
+ bool TryAcquire() noexcept;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+//unnamed semaphore, faster, than previous
+class TFastSemaphore {
+public:
+ TFastSemaphore(ui32 maxFreeCount);
+ ~TFastSemaphore();
+
+ void Release() noexcept;
+ void Acquire() noexcept;
+ bool TryAcquire() noexcept;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
diff --git a/util/system/shellcommand.cpp b/util/system/shellcommand.cpp
new file mode 100644
index 0000000000..b1989b5c8c
--- /dev/null
+++ b/util/system/shellcommand.cpp
@@ -0,0 +1,1200 @@
+#include "shellcommand.h"
+#include "user.h"
+#include "nice.h"
+#include "sigset.h"
+#include "atomic.h"
+
+#include <util/folder/dirut.h>
+#include <util/generic/algorithm.h>
+#include <util/generic/buffer.h>
+#include <util/generic/vector.h>
+#include <util/generic/yexception.h>
+#include <util/memory/tempbuf.h>
+#include <util/network/socket.h>
+#include <util/stream/pipe.h>
+#include <util/stream/str.h>
+#include <util/string/cast.h>
+#include <util/system/info.h>
+
+#include <errno.h>
+
+#if defined(_unix_)
+ #include <unistd.h>
+ #include <fcntl.h>
+ #include <grp.h>
+ #include <sys/wait.h>
+
+using TPid = pid_t;
+using TWaitResult = pid_t;
+using TExitStatus = int;
+ #define WAIT_PROCEED 0
+
+ #if defined(_darwin_)
+using TGetGroupListGid = int;
+ #else
+using TGetGroupListGid = gid_t;
+ #endif
+#elif defined(_win_)
+ #include <string>
+
+ #include "winint.h"
+
+using TPid = HANDLE;
+using TWaitResult = DWORD;
+using TExitStatus = DWORD;
+ #define WAIT_PROCEED WAIT_TIMEOUT
+
+ #pragma warning(disable : 4296) // 'wait_result >= WAIT_OBJECT_0' : expression is always tru
+#else
+ #error("unknown os, shell command is not implemented")
+#endif
+
+#define DBG(stmt) \
+ {}
+// #define DBG(stmt) stmt
+
+namespace {
+ constexpr static size_t DATA_BUFFER_SIZE = 128 * 1024;
+
+#if defined(_unix_)
+ void SetUserGroups(const passwd* pw) {
+ int ngroups = 1;
+ THolder<gid_t, TFree> groups = THolder<gid_t, TFree>(static_cast<gid_t*>(malloc(ngroups * sizeof(gid_t))));
+ if (getgrouplist(pw->pw_name, pw->pw_gid, reinterpret_cast<TGetGroupListGid*>(groups.Get()), &ngroups) == -1) {
+ groups.Reset(static_cast<gid_t*>(malloc(ngroups * sizeof(gid_t))));
+ if (getgrouplist(pw->pw_name, pw->pw_gid, reinterpret_cast<TGetGroupListGid*>(groups.Get()), &ngroups) == -1) {
+ ythrow TSystemError() << "getgrouplist failed: user " << pw->pw_name << " (" << pw->pw_uid << ")";
+ }
+ }
+ if (setgroups(ngroups, groups.Get()) == -1) {
+ ythrow TSystemError(errno) << "Unable to set groups for user " << pw->pw_name << Endl;
+ }
+ }
+
+ void ImpersonateUser(const TShellCommandOptions::TUserOptions& userOpts) {
+ if (GetUsername() == userOpts.Name) {
+ return;
+ }
+ const passwd* newUser = getpwnam(userOpts.Name.c_str());
+ if (!newUser) {
+ ythrow TSystemError(errno) << "getpwnam failed";
+ }
+ if (userOpts.UseUserGroups) {
+ SetUserGroups(newUser);
+ }
+ if (setuid(newUser->pw_uid)) {
+ ythrow TSystemError(errno) << "setuid failed";
+ }
+ }
+#elif defined(_win_)
+ constexpr static size_t MAX_COMMAND_LINE = 32 * 1024;
+
+ std::wstring GetWString(const char* astring) {
+ if (!astring)
+ return std::wstring();
+
+ std::string str(astring);
+ return std::wstring(str.begin(), str.end());
+ }
+
+ std::string GetAString(const wchar_t* wstring) {
+ if (!wstring)
+ return std::string();
+
+ std::wstring str(wstring);
+ return std::string(str.begin(), str.end());
+ }
+#endif
+}
+
+// temporary measure to avoid rewriting all poll calls on win TPipeHandle
+#if defined(_win_)
+using REALPIPEHANDLE = HANDLE;
+ #define INVALID_REALPIPEHANDLE INVALID_HANDLE_VALUE
+
+class TRealPipeHandle
+ : public TNonCopyable {
+public:
+ inline TRealPipeHandle() noexcept
+ : Fd_(INVALID_REALPIPEHANDLE)
+ {
+ }
+
+ inline TRealPipeHandle(REALPIPEHANDLE fd) noexcept
+ : Fd_(fd)
+ {
+ }
+
+ inline ~TRealPipeHandle() {
+ Close();
+ }
+
+ bool Close() noexcept {
+ bool ok = true;
+ if (Fd_ != INVALID_REALPIPEHANDLE)
+ ok = CloseHandle(Fd_);
+ Fd_ = INVALID_REALPIPEHANDLE;
+ return ok;
+ }
+
+ inline REALPIPEHANDLE Release() noexcept {
+ REALPIPEHANDLE ret = Fd_;
+ Fd_ = INVALID_REALPIPEHANDLE;
+ return ret;
+ }
+
+ inline void Swap(TRealPipeHandle& r) noexcept {
+ DoSwap(Fd_, r.Fd_);
+ }
+
+ inline operator REALPIPEHANDLE() const noexcept {
+ return Fd_;
+ }
+
+ inline bool IsOpen() const noexcept {
+ return Fd_ != INVALID_REALPIPEHANDLE;
+ }
+
+ ssize_t Read(void* buffer, size_t byteCount) const noexcept {
+ DWORD doneBytes;
+ if (!ReadFile(Fd_, buffer, byteCount, &doneBytes, nullptr))
+ return -1;
+ return doneBytes;
+ }
+ ssize_t Write(const void* buffer, size_t byteCount) const noexcept {
+ DWORD doneBytes;
+ if (!WriteFile(Fd_, buffer, byteCount, &doneBytes, nullptr))
+ return -1;
+ return doneBytes;
+ }
+
+ static void Pipe(TRealPipeHandle& reader, TRealPipeHandle& writer, EOpenMode mode) {
+ (void)mode;
+ REALPIPEHANDLE fds[2];
+ if (!CreatePipe(&fds[0], &fds[1], nullptr /* handles are not inherited */, 0))
+ ythrow TFileError() << "failed to create a pipe";
+ TRealPipeHandle(fds[0]).Swap(reader);
+ TRealPipeHandle(fds[1]).Swap(writer);
+ }
+
+private:
+ REALPIPEHANDLE Fd_;
+};
+
+#else
+using TRealPipeHandle = TPipeHandle;
+using REALPIPEHANDLE = PIPEHANDLE;
+ #define INVALID_REALPIPEHANDLE INVALID_PIPEHANDLE
+#endif
+
+class TShellCommand::TImpl
+ : public TAtomicRefCount<TShellCommand::TImpl> {
+private:
+ TPid Pid;
+ TString Command;
+ TList<TString> Arguments;
+ TString WorkDir;
+ TAtomic ExecutionStatus; // TShellCommand::ECommandStatus
+ TMaybe<int> ExitCode;
+ IInputStream* InputStream;
+ IOutputStream* OutputStream;
+ IOutputStream* ErrorStream;
+ TString CollectedOutput;
+ TString CollectedError;
+ TString InternalError;
+ TThread* WatchThread;
+ TMutex TerminateMutex;
+ TFileHandle InputHandle;
+ TFileHandle OutputHandle;
+ TFileHandle ErrorHandle;
+
+ /// @todo: store const TShellCommandOptions, no need for so many vars
+ bool TerminateFlag = false;
+ bool ClearSignalMask = false;
+ bool CloseAllFdsOnExec = false;
+ bool AsyncMode = false;
+ size_t PollDelayMs = 0;
+ bool UseShell = false;
+ bool QuoteArguments = false;
+ bool DetachSession = false;
+ bool CloseStreams = false;
+ TAtomic ShouldCloseInput;
+ TShellCommandOptions::EHandleMode InputMode = TShellCommandOptions::HANDLE_STREAM;
+ TShellCommandOptions::EHandleMode OutputMode = TShellCommandOptions::HANDLE_STREAM;
+ TShellCommandOptions::EHandleMode ErrorMode = TShellCommandOptions::HANDLE_STREAM;
+
+ TShellCommandOptions::TUserOptions User;
+ THashMap<TString, TString> Environment;
+ int Nice = 0;
+ std::function<void()> FuncAfterFork = {};
+
+ struct TProcessInfo {
+ TImpl* Parent;
+ TRealPipeHandle InputFd;
+ TRealPipeHandle OutputFd;
+ TRealPipeHandle ErrorFd;
+ TProcessInfo(TImpl* parent, REALPIPEHANDLE inputFd, REALPIPEHANDLE outputFd, REALPIPEHANDLE errorFd)
+ : Parent(parent)
+ , InputFd(inputFd)
+ , OutputFd(outputFd)
+ , ErrorFd(errorFd)
+ {
+ }
+ };
+
+ struct TPipes {
+ TRealPipeHandle OutputPipeFd[2];
+ TRealPipeHandle ErrorPipeFd[2];
+ TRealPipeHandle InputPipeFd[2];
+ // pipes are closed by automatic dtor
+ void PrepareParents() {
+ if (OutputPipeFd[1].IsOpen()) {
+ OutputPipeFd[1].Close();
+ }
+ if (ErrorPipeFd[1].IsOpen()) {
+ ErrorPipeFd[1].Close();
+ }
+ if (InputPipeFd[1].IsOpen()) {
+ InputPipeFd[0].Close();
+ }
+ }
+ void ReleaseParents() {
+ InputPipeFd[1].Release();
+ OutputPipeFd[0].Release();
+ ErrorPipeFd[0].Release();
+ }
+ };
+
+ struct TPipePump {
+ TRealPipeHandle* Pipe;
+ IOutputStream* OutputStream;
+ IInputStream* InputStream;
+ TAtomic* ShouldClosePipe;
+ TString InternalError;
+ };
+
+#if defined(_unix_)
+ void OnFork(TPipes& pipes, sigset_t oldmask, char* const* argv, char* const* envp, const std::function<void()>& afterFork) const;
+#else
+ void StartProcess(TPipes& pipes);
+#endif
+
+public:
+ inline TImpl(const TStringBuf cmd, const TList<TString>& args, const TShellCommandOptions& options, const TString& workdir)
+ : Pid(0)
+ , Command(ToString(cmd))
+ , Arguments(args)
+ , WorkDir(workdir)
+ , ExecutionStatus(SHELL_NONE)
+ , InputStream(options.InputStream)
+ , OutputStream(options.OutputStream)
+ , ErrorStream(options.ErrorStream)
+ , WatchThread(nullptr)
+ , TerminateFlag(false)
+ , ClearSignalMask(options.ClearSignalMask)
+ , CloseAllFdsOnExec(options.CloseAllFdsOnExec)
+ , AsyncMode(options.AsyncMode)
+ , PollDelayMs(options.PollDelayMs)
+ , UseShell(options.UseShell)
+ , QuoteArguments(options.QuoteArguments)
+ , DetachSession(options.DetachSession)
+ , CloseStreams(options.CloseStreams)
+ , ShouldCloseInput(options.ShouldCloseInput)
+ , InputMode(options.InputMode)
+ , OutputMode(options.OutputMode)
+ , ErrorMode(options.ErrorMode)
+ , User(options.User)
+ , Environment(options.Environment)
+ , Nice(options.Nice)
+ , FuncAfterFork(options.FuncAfterFork)
+ {
+ if (InputStream) {
+ // TODO change usages to call SetInputStream instead of directly assigning to InputStream
+ InputMode = TShellCommandOptions::HANDLE_STREAM;
+ }
+ }
+
+ inline ~TImpl() {
+ if (WatchThread) {
+ with_lock (TerminateMutex) {
+ TerminateFlag = true;
+ }
+
+ delete WatchThread;
+ }
+
+#if defined(_win_)
+ if (Pid) {
+ CloseHandle(Pid);
+ }
+#endif
+ }
+
+ inline void AppendArgument(const TStringBuf argument) {
+ if (AtomicGet(ExecutionStatus) == SHELL_RUNNING) {
+ ythrow yexception() << "You cannot change command parameters while process is running";
+ }
+ Arguments.push_back(ToString(argument));
+ }
+
+ inline const TString& GetOutput() const {
+ if (AtomicGet(ExecutionStatus) == SHELL_RUNNING) {
+ ythrow yexception() << "You cannot retrieve output while process is running.";
+ }
+ return CollectedOutput;
+ }
+
+ inline const TString& GetError() const {
+ if (AtomicGet(ExecutionStatus) == SHELL_RUNNING) {
+ ythrow yexception() << "You cannot retrieve output while process is running.";
+ }
+ return CollectedError;
+ }
+
+ inline const TString& GetInternalError() const {
+ if (AtomicGet(ExecutionStatus) != SHELL_INTERNAL_ERROR) {
+ ythrow yexception() << "Internal error hasn't occured so can't be retrieved.";
+ }
+ return InternalError;
+ }
+
+ inline ECommandStatus GetStatus() const {
+ return static_cast<ECommandStatus>(AtomicGet(ExecutionStatus));
+ }
+
+ inline TMaybe<int> GetExitCode() const {
+ return ExitCode;
+ }
+
+ inline TProcessId GetPid() const {
+#if defined(_win_)
+ return GetProcessId(Pid);
+#else
+ return Pid;
+#endif
+ }
+
+ inline TFileHandle& GetInputHandle() {
+ return InputHandle;
+ }
+
+ inline TFileHandle& GetOutputHandle() {
+ return OutputHandle;
+ }
+
+ inline TFileHandle& GetErrorHandle() {
+ return ErrorHandle;
+ }
+
+ // start child process
+ void Run();
+
+ inline void Terminate() {
+ if (!!Pid && (AtomicGet(ExecutionStatus) == SHELL_RUNNING)) {
+ bool ok =
+#if defined(_unix_)
+ kill(DetachSession ? -1 * Pid : Pid, SIGTERM) == 0;
+ if (!ok && (errno == ESRCH) && DetachSession) {
+ // this could fail when called before child proc completes setsid().
+ ok = kill(Pid, SIGTERM) == 0;
+ kill(-Pid, SIGTERM); // between a failed kill(-Pid) and a successful kill(Pid) a grandchild could have been spawned
+ }
+#else
+ TerminateProcess(Pid, 1 /* exit code */);
+#endif
+ if (!ok) {
+ ythrow TSystemError() << "cannot terminate " << Pid;
+ }
+ }
+ }
+
+ inline void Wait() {
+ if (WatchThread) {
+ WatchThread->Join();
+ }
+ }
+
+ inline void CloseInput() {
+ AtomicSet(ShouldCloseInput, true);
+ }
+
+ inline static bool TerminateIsRequired(void* processInfo) {
+ TProcessInfo* pi = reinterpret_cast<TProcessInfo*>(processInfo);
+ if (!pi->Parent->TerminateFlag) {
+ return false;
+ }
+ pi->InputFd.Close();
+ pi->ErrorFd.Close();
+ pi->OutputFd.Close();
+
+ if (pi->Parent->CloseStreams) {
+ if (pi->Parent->ErrorStream) {
+ pi->Parent->ErrorStream->Finish();
+ }
+ if (pi->Parent->OutputStream) {
+ pi->Parent->OutputStream->Finish();
+ }
+ }
+
+ delete pi;
+ return true;
+ }
+
+ // interchange io while process is alive
+ inline static void Communicate(TProcessInfo* pi);
+
+ inline static void* WatchProcess(void* data) {
+ TProcessInfo* pi = reinterpret_cast<TProcessInfo*>(data);
+ Communicate(pi);
+ return nullptr;
+ }
+
+ inline static void* ReadStream(void* data) noexcept {
+ TPipePump* pump = reinterpret_cast<TPipePump*>(data);
+ try {
+ int bytes = 0;
+ TBuffer buffer(DATA_BUFFER_SIZE);
+
+ while (true) {
+ bytes = pump->Pipe->Read(buffer.Data(), buffer.Capacity());
+ if (bytes > 0) {
+ pump->OutputStream->Write(buffer.Data(), bytes);
+ } else {
+ break;
+ }
+ }
+ if (pump->Pipe->IsOpen()) {
+ pump->Pipe->Close();
+ }
+ } catch (...) {
+ pump->InternalError = CurrentExceptionMessage();
+ }
+ return nullptr;
+ }
+
+ inline static void* WriteStream(void* data) noexcept {
+ TPipePump* pump = reinterpret_cast<TPipePump*>(data);
+ try {
+ int bytes = 0;
+ int bytesToWrite = 0;
+ char* bufPos = nullptr;
+ TBuffer buffer(DATA_BUFFER_SIZE);
+
+ while (true) {
+ if (!bytesToWrite) {
+ bytesToWrite = pump->InputStream->Read(buffer.Data(), buffer.Capacity());
+ if (bytesToWrite == 0) {
+ if (AtomicGet(pump->ShouldClosePipe)) {
+ break;
+ }
+ continue;
+ }
+ bufPos = buffer.Data();
+ }
+
+ bytes = pump->Pipe->Write(bufPos, bytesToWrite);
+ if (bytes > 0) {
+ bytesToWrite -= bytes;
+ bufPos += bytes;
+ } else {
+ break;
+ }
+ }
+ if (pump->Pipe->IsOpen()) {
+ pump->Pipe->Close();
+ }
+ } catch (...) {
+ pump->InternalError = CurrentExceptionMessage();
+ }
+ return nullptr;
+ }
+
+ TString GetQuotedCommand() const;
+};
+
+#if defined(_win_)
+void TShellCommand::TImpl::StartProcess(TShellCommand::TImpl::TPipes& pipes) {
+ // Setup STARTUPINFO to redirect handles.
+ STARTUPINFOW startup_info;
+ ZeroMemory(&startup_info, sizeof(startup_info));
+ startup_info.cb = sizeof(startup_info);
+ startup_info.dwFlags = STARTF_USESTDHANDLES;
+
+ if (OutputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ if (!SetHandleInformation(pipes.OutputPipeFd[1], HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
+ ythrow TSystemError() << "cannot set handle info";
+ }
+ }
+ if (ErrorMode != TShellCommandOptions::HANDLE_INHERIT) {
+ if (!SetHandleInformation(pipes.ErrorPipeFd[1], HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
+ ythrow TSystemError() << "cannot set handle info";
+ }
+ }
+ if (InputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ if (!SetHandleInformation(pipes.InputPipeFd[0], HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT))
+ ythrow TSystemError() << "cannot set handle info";
+ }
+
+ // A sockets do not work as std streams for some reason
+ if (OutputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ startup_info.hStdOutput = pipes.OutputPipeFd[1];
+ } else {
+ startup_info.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
+ }
+ if (ErrorMode != TShellCommandOptions::HANDLE_INHERIT) {
+ startup_info.hStdError = pipes.ErrorPipeFd[1];
+ } else {
+ startup_info.hStdError = GetStdHandle(STD_ERROR_HANDLE);
+ }
+ if (InputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ startup_info.hStdInput = pipes.InputPipeFd[0];
+ } else {
+ // Don't leave hStdInput unfilled, otherwise any attempt to retrieve the operating-system file handle
+ // that is associated with the specified file descriptor will led to errors.
+ startup_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
+ }
+
+ PROCESS_INFORMATION process_info;
+ // TString cmd = "cmd /U" + TUtf16String can be used to read unicode messages from cmd
+ // /A - ansi charset /Q - echo off, /C - command, /Q - special quotes
+ TString qcmd = GetQuotedCommand();
+ TString cmd = UseShell ? "cmd /A /Q /S /C \"" + qcmd + "\"" : qcmd;
+ // winapi can modify command text, copy it
+
+ Y_ENSURE_EX(cmd.size() < MAX_COMMAND_LINE, yexception() << "Command is too long (length=" << cmd.size() << ")");
+ TTempArray<wchar_t> cmdcopy(MAX_COMMAND_LINE);
+ Copy(cmd.data(), cmd.data() + cmd.size(), cmdcopy.Data());
+ *(cmdcopy.Data() + cmd.size()) = 0;
+
+ const wchar_t* cwd = NULL;
+ std::wstring cwdBuff;
+ if (WorkDir.size()) {
+ cwdBuff = GetWString(WorkDir.data());
+ cwd = cwdBuff.c_str();
+ }
+
+ void* lpEnvironment = nullptr;
+ TString env;
+ if (!Environment.empty()) {
+ for (auto e = Environment.begin(); e != Environment.end(); ++e) {
+ env += e->first + '=' + e->second + '\0';
+ }
+ env += '\0';
+ lpEnvironment = const_cast<char*>(env.data());
+ }
+
+ // disable messagebox (may be in debug too)
+ #ifndef NDEBUG
+ SetErrorMode(GetErrorMode() | SEM_NOGPFAULTERRORBOX);
+ #endif
+ BOOL res = 0;
+ if (User.Name.empty() || GetUsername() == User.Name) {
+ res = CreateProcessW(
+ nullptr, // image name
+ cmdcopy.Data(),
+ nullptr, // process security attributes
+ nullptr, // thread security attributes
+ TRUE, // inherit handles - needed for IO, CloseAllFdsOnExec not respected
+ 0, // obscure creation flags
+ lpEnvironment, // environment
+ cwd, // current directory
+ &startup_info,
+ &process_info);
+ } else {
+ res = CreateProcessWithLogonW(
+ GetWString(User.Name.data()).c_str(),
+ nullptr, // domain (if this parameter is NULL, the user name must be specified in UPN format)
+ GetWString(User.Password.data()).c_str(),
+ 0, // logon flags
+ NULL, // image name
+ cmdcopy.Data(),
+ 0, // obscure creation flags
+ lpEnvironment, // environment
+ cwd, // current directory
+ &startup_info,
+ &process_info);
+ }
+
+ if (!res) {
+ AtomicSet(ExecutionStatus, SHELL_ERROR);
+ /// @todo: write to error stream if set
+ TStringOutput out(CollectedError);
+ out << "Process was not created: " << LastSystemErrorText() << " command text was: '" << GetAString(cmdcopy.Data()) << "'";
+ }
+ Pid = process_info.hProcess;
+ CloseHandle(process_info.hThread);
+ DBG(Cerr << "created process id " << Pid << " in dir: " << cwd << ", cmd: " << cmdcopy.Data() << Endl);
+}
+#endif
+
+void ShellQuoteArg(TString& dst, TStringBuf argument) {
+ dst.append("\"");
+ TStringBuf l, r;
+ while (argument.TrySplit('"', l, r)) {
+ dst.append(l);
+ dst.append("\\\"");
+ argument = r;
+ }
+ dst.append(argument);
+ dst.append("\"");
+}
+
+void ShellQuoteArgSp(TString& dst, TStringBuf argument) {
+ dst.append(' ');
+ ShellQuoteArg(dst, argument);
+}
+
+bool ArgNeedsQuotes(TStringBuf arg) noexcept {
+ if (arg.empty()) {
+ return true;
+ }
+ return arg.find_first_of(" \"\'\t&()*<>\\`^|") != TString::npos;
+}
+
+TString TShellCommand::TImpl::GetQuotedCommand() const {
+ TString quoted = Command; /// @todo command itself should be quoted too
+ for (const auto& argument : Arguments) {
+ // Don't add unnecessary quotes. It's especially important for the windows with a 32k command line length limit.
+ if (QuoteArguments && ArgNeedsQuotes(argument)) {
+ ::ShellQuoteArgSp(quoted, argument);
+ } else {
+ quoted.append(" ").append(argument);
+ }
+ }
+ return quoted;
+}
+
+#if defined(_unix_)
+void TShellCommand::TImpl::OnFork(TPipes& pipes, sigset_t oldmask, char* const* argv, char* const* envp, const std::function<void()>& afterFork) const {
+ try {
+ if (DetachSession) {
+ setsid();
+ }
+
+ // reset signal handlers from parent
+ struct sigaction sa;
+ sa.sa_handler = SIG_DFL;
+ sa.sa_flags = 0;
+ SigEmptySet(&sa.sa_mask);
+ for (int i = 0; i < NSIG; ++i) {
+ // some signals cannot be caught, so just ignore return value
+ sigaction(i, &sa, nullptr);
+ }
+ if (ClearSignalMask) {
+ SigEmptySet(&oldmask);
+ }
+ // clear / restore signal mask
+ if (SigProcMask(SIG_SETMASK, &oldmask, nullptr) != 0) {
+ ythrow TSystemError() << "Cannot " << (ClearSignalMask ? "clear" : "restore") << " signal mask in child";
+ }
+
+ TFileHandle sIn(0);
+ TFileHandle sOut(1);
+ TFileHandle sErr(2);
+ if (InputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ pipes.InputPipeFd[1].Close();
+ TFileHandle sInNew(pipes.InputPipeFd[0]);
+ sIn.LinkTo(sInNew);
+ sIn.Release();
+ sInNew.Release();
+ } else {
+ // do not close fd 0 - next open will return it and confuse all readers
+ /// @todo in case of real need - reopen /dev/null
+ }
+ if (OutputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ pipes.OutputPipeFd[0].Close();
+ TFileHandle sOutNew(pipes.OutputPipeFd[1]);
+ sOut.LinkTo(sOutNew);
+ sOut.Release();
+ sOutNew.Release();
+ }
+ if (ErrorMode != TShellCommandOptions::HANDLE_INHERIT) {
+ pipes.ErrorPipeFd[0].Close();
+ TFileHandle sErrNew(pipes.ErrorPipeFd[1]);
+ sErr.LinkTo(sErrNew);
+ sErr.Release();
+ sErrNew.Release();
+ }
+
+ if (WorkDir.size()) {
+ NFs::SetCurrentWorkingDirectory(WorkDir);
+ }
+
+ if (CloseAllFdsOnExec) {
+ for (int fd = NSystemInfo::MaxOpenFiles(); fd > STDERR_FILENO; --fd) {
+ fcntl(fd, F_SETFD, FD_CLOEXEC);
+ }
+ }
+
+ if (!User.Name.empty()) {
+ ImpersonateUser(User);
+ }
+
+ if (Nice) {
+ // Don't verify Nice() call - it does not work properly with WSL https://github.com/Microsoft/WSL/issues/1838
+ ::Nice(Nice);
+ }
+ if (afterFork) {
+ afterFork();
+ }
+
+ if (envp == nullptr) {
+ execvp(argv[0], argv);
+ } else {
+ execve(argv[0], argv, envp);
+ }
+ Cerr << "Process was not created: " << LastSystemErrorText() << Endl;
+ } catch (const std::exception& error) {
+ Cerr << "Process was not created: " << error.what() << Endl;
+ } catch (...) {
+ Cerr << "Process was not created: "
+ << "unknown error" << Endl;
+ }
+
+ _exit(-1);
+}
+#endif
+
+void TShellCommand::TImpl::Run() {
+ Y_ENSURE(AtomicGet(ExecutionStatus) != SHELL_RUNNING, TStringBuf("Process is already running"));
+ // Prepare I/O streams
+ CollectedOutput.clear();
+ CollectedError.clear();
+ TPipes pipes;
+
+ if (OutputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ TRealPipeHandle::Pipe(pipes.OutputPipeFd[0], pipes.OutputPipeFd[1], CloseOnExec);
+ }
+ if (ErrorMode != TShellCommandOptions::HANDLE_INHERIT) {
+ TRealPipeHandle::Pipe(pipes.ErrorPipeFd[0], pipes.ErrorPipeFd[1], CloseOnExec);
+ }
+ if (InputMode != TShellCommandOptions::HANDLE_INHERIT) {
+ TRealPipeHandle::Pipe(pipes.InputPipeFd[0], pipes.InputPipeFd[1], CloseOnExec);
+ }
+
+ AtomicSet(ExecutionStatus, SHELL_RUNNING);
+
+#if defined(_unix_)
+ // block all signals to avoid signal handler race after fork()
+ sigset_t oldmask, newmask;
+ SigFillSet(&newmask);
+ if (SigProcMask(SIG_SETMASK, &newmask, &oldmask) != 0) {
+ ythrow TSystemError() << "Cannot block all signals in parent";
+ }
+
+ /* arguments holders */
+ TString shellArg;
+ TVector<char*> qargv;
+ /*
+ Following "const_cast"s are safe:
+ http://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html
+ */
+ if (UseShell) {
+ shellArg = GetQuotedCommand();
+ qargv.reserve(4);
+ qargv.push_back(const_cast<char*>("/bin/sh"));
+ qargv.push_back(const_cast<char*>("-c"));
+ // two args for 'sh -c -- ',
+ // one for program name, and one for NULL at the end
+ qargv.push_back(const_cast<char*>(shellArg.data()));
+ } else {
+ qargv.reserve(Arguments.size() + 2);
+ qargv.push_back(const_cast<char*>(Command.data()));
+ for (auto& i : Arguments) {
+ qargv.push_back(const_cast<char*>(i.data()));
+ }
+ }
+
+ qargv.push_back(nullptr);
+
+ TVector<TString> envHolder;
+ TVector<char*> envp;
+ if (!Environment.empty()) {
+ for (auto& env : Environment) {
+ envHolder.emplace_back(env.first + '=' + env.second);
+ envp.push_back(const_cast<char*>(envHolder.back().data()));
+ }
+ envp.push_back(nullptr);
+ }
+
+ pid_t pid = fork();
+ if (pid == -1) {
+ AtomicSet(ExecutionStatus, SHELL_ERROR);
+ /// @todo check if pipes are still open
+ ythrow TSystemError() << "Cannot fork";
+ } else if (pid == 0) { // child
+ if (envp.size() != 0) {
+ OnFork(pipes, oldmask, qargv.data(), envp.data(), FuncAfterFork);
+ } else {
+ OnFork(pipes, oldmask, qargv.data(), nullptr, FuncAfterFork);
+ }
+ } else { // parent
+ // restore signal mask
+ if (SigProcMask(SIG_SETMASK, &oldmask, nullptr) != 0) {
+ ythrow TSystemError() << "Cannot restore signal mask in parent";
+ }
+ }
+ Pid = pid;
+#else
+ StartProcess(pipes);
+#endif
+ pipes.PrepareParents();
+
+ if (AtomicGet(ExecutionStatus) != SHELL_RUNNING) {
+ return;
+ }
+
+ if (InputMode == TShellCommandOptions::HANDLE_PIPE) {
+ TFileHandle inputHandle(pipes.InputPipeFd[1].Release());
+ InputHandle.Swap(inputHandle);
+ }
+
+ if (OutputMode == TShellCommandOptions::HANDLE_PIPE) {
+ TFileHandle outputHandle(pipes.OutputPipeFd[0].Release());
+ OutputHandle.Swap(outputHandle);
+ }
+
+ if (ErrorMode == TShellCommandOptions::HANDLE_PIPE) {
+ TFileHandle errorHandle(pipes.ErrorPipeFd[0].Release());
+ ErrorHandle.Swap(errorHandle);
+ }
+
+ TProcessInfo* processInfo = new TProcessInfo(this,
+ pipes.InputPipeFd[1].Release(), pipes.OutputPipeFd[0].Release(), pipes.ErrorPipeFd[0].Release());
+ if (AsyncMode) {
+ WatchThread = new TThread(&TImpl::WatchProcess, processInfo);
+ WatchThread->Start();
+ /// @todo wait for child to start its process session (if options.Detach)
+ } else {
+ Communicate(processInfo);
+ }
+
+ pipes.ReleaseParents(); // not needed
+}
+
+void TShellCommand::TImpl::Communicate(TProcessInfo* pi) {
+ THolder<IOutputStream> outputHolder;
+ IOutputStream* output = pi->Parent->OutputStream;
+ if (!output) {
+ outputHolder.Reset(output = new TStringOutput(pi->Parent->CollectedOutput));
+ }
+
+ THolder<IOutputStream> errorHolder;
+ IOutputStream* error = pi->Parent->ErrorStream;
+ if (!error) {
+ errorHolder.Reset(error = new TStringOutput(pi->Parent->CollectedError));
+ }
+
+ IInputStream*& input = pi->Parent->InputStream;
+
+#if defined(_unix_)
+ // not really needed, io is done via poll
+ if (pi->OutputFd.IsOpen()) {
+ SetNonBlock(pi->OutputFd);
+ }
+ if (pi->ErrorFd.IsOpen()) {
+ SetNonBlock(pi->ErrorFd);
+ }
+ if (pi->InputFd.IsOpen()) {
+ SetNonBlock(pi->InputFd);
+ }
+#endif
+
+ try {
+#if defined(_win_)
+ TPipePump pumps[3] = {0};
+ pumps[0] = {&pi->ErrorFd, error};
+ pumps[1] = {&pi->OutputFd, output};
+
+ TVector<THolder<TThread>> streamThreads;
+ streamThreads.emplace_back(new TThread(&TImpl::ReadStream, &pumps[0]));
+ streamThreads.emplace_back(new TThread(&TImpl::ReadStream, &pumps[1]));
+
+ if (input) {
+ pumps[2] = {&pi->InputFd, nullptr, input, &pi->Parent->ShouldCloseInput};
+ streamThreads.emplace_back(new TThread(&TImpl::WriteStream, &pumps[2]));
+ }
+
+ for (auto& threadHolder : streamThreads)
+ threadHolder->Start();
+#else
+ TBuffer buffer(DATA_BUFFER_SIZE);
+ TBuffer inputBuffer(DATA_BUFFER_SIZE);
+ int bytes;
+ int bytesToWrite = 0;
+ char* bufPos = nullptr;
+#endif
+ TWaitResult waitPidResult;
+ TExitStatus status = 0;
+
+ while (true) {
+ {
+ with_lock (pi->Parent->TerminateMutex) {
+ if (TerminateIsRequired(pi)) {
+ return;
+ }
+ }
+
+ waitPidResult =
+#if defined(_unix_)
+ waitpid(pi->Parent->Pid, &status, WNOHANG);
+#else
+ WaitForSingleObject(pi->Parent->Pid /* process_info.hProcess */, pi->Parent->PollDelayMs /* ms */);
+ Y_UNUSED(status);
+#endif
+ // DBG(Cerr << "wait result: " << waitPidResult << Endl);
+ if (waitPidResult != WAIT_PROCEED) {
+ break;
+ }
+ }
+/// @todo factor out (poll + wfmo)
+#if defined(_unix_)
+ bool haveIn = false;
+ bool haveOut = false;
+ bool haveErr = false;
+
+ if (!input && pi->InputFd.IsOpen()) {
+ DBG(Cerr << "closing input stream..." << Endl);
+ pi->InputFd.Close();
+ }
+ if (!output && pi->OutputFd.IsOpen()) {
+ DBG(Cerr << "closing output stream..." << Endl);
+ pi->OutputFd.Close();
+ }
+ if (!error && pi->ErrorFd.IsOpen()) {
+ DBG(Cerr << "closing error stream..." << Endl);
+ pi->ErrorFd.Close();
+ }
+
+ if (!input && !output && !error) {
+ continue;
+ }
+
+ struct pollfd fds[] = {
+ {REALPIPEHANDLE(pi->InputFd), POLLOUT, 0},
+ {REALPIPEHANDLE(pi->OutputFd), POLLIN, 0},
+ {REALPIPEHANDLE(pi->ErrorFd), POLLIN, 0}};
+ int res;
+
+ if (!input) {
+ fds[0].events = 0;
+ }
+ if (!output) {
+ fds[1].events = 0;
+ }
+ if (!error) {
+ fds[2].events = 0;
+ }
+
+ res = PollD(fds, 3, TInstant::Now() + TDuration::MilliSeconds(pi->Parent->PollDelayMs));
+ // DBG(Cerr << "poll result: " << res << Endl);
+ if (-res == ETIMEDOUT || res == 0) {
+ // DBG(Cerr << "poll again..." << Endl);
+ continue;
+ }
+ if (res < 0) {
+ ythrow yexception() << "poll failed: " << LastSystemErrorText();
+ }
+
+ if ((fds[1].revents & POLLIN) == POLLIN) {
+ haveOut = true;
+ } else if (fds[1].revents & (POLLERR | POLLHUP)) {
+ output = nullptr;
+ }
+
+ if ((fds[2].revents & POLLIN) == POLLIN) {
+ haveErr = true;
+ } else if (fds[2].revents & (POLLERR | POLLHUP)) {
+ error = nullptr;
+ }
+
+ if (input && ((fds[0].revents & POLLOUT) == POLLOUT)) {
+ haveIn = true;
+ }
+
+ if (haveOut) {
+ bytes = pi->OutputFd.Read(buffer.Data(), buffer.Capacity());
+ DBG(Cerr << "transferred " << bytes << " bytes of output" << Endl);
+ if (bytes > 0) {
+ output->Write(buffer.Data(), bytes);
+ } else {
+ output = nullptr;
+ }
+ }
+ if (haveErr) {
+ bytes = pi->ErrorFd.Read(buffer.Data(), buffer.Capacity());
+ DBG(Cerr << "transferred " << bytes << " bytes of error" << Endl);
+ if (bytes > 0) {
+ error->Write(buffer.Data(), bytes);
+ } else {
+ error = nullptr;
+ }
+ }
+
+ if (haveIn) {
+ if (!bytesToWrite) {
+ bytesToWrite = input->Read(inputBuffer.Data(), inputBuffer.Capacity());
+ if (bytesToWrite == 0) {
+ if (AtomicGet(pi->Parent->ShouldCloseInput)) {
+ input = nullptr;
+ }
+ continue;
+ }
+ bufPos = inputBuffer.Data();
+ }
+
+ bytes = pi->InputFd.Write(bufPos, bytesToWrite);
+ if (bytes > 0) {
+ bytesToWrite -= bytes;
+ bufPos += bytes;
+ } else {
+ input = nullptr;
+ }
+
+ DBG(Cerr << "transferred " << bytes << " bytes of input" << Endl);
+ }
+#endif
+ }
+ DBG(Cerr << "process finished" << Endl);
+
+ // What's the reason of process exit.
+ // We need to set exit code before waiting for input thread
+ // Otherwise there is no way for input stream provider to discover
+ // that process has exited and stream shouldn't wait for new data.
+ bool cleanExit = false;
+ TMaybe<int> processExitCode;
+#if defined(_unix_)
+ processExitCode = WEXITSTATUS(status);
+ if (WIFEXITED(status) && processExitCode == 0) {
+ cleanExit = true;
+ } else if (WIFSIGNALED(status)) {
+ processExitCode = -WTERMSIG(status);
+ }
+#else
+ if (waitPidResult == WAIT_OBJECT_0) {
+ DWORD exitCode = STILL_ACTIVE;
+ if (!GetExitCodeProcess(pi->Parent->Pid, &exitCode)) {
+ ythrow yexception() << "GetExitCodeProcess: " << LastSystemErrorText();
+ }
+ if (exitCode == 0)
+ cleanExit = true;
+ processExitCode = static_cast<int>(exitCode);
+ DBG(Cerr << "exit code: " << exitCode << Endl);
+ }
+#endif
+ pi->Parent->ExitCode = processExitCode;
+ if (cleanExit) {
+ AtomicSet(pi->Parent->ExecutionStatus, SHELL_FINISHED);
+ } else {
+ AtomicSet(pi->Parent->ExecutionStatus, SHELL_ERROR);
+ }
+
+#if defined(_win_)
+ for (auto& threadHolder : streamThreads)
+ threadHolder->Join();
+ for (const auto pump : pumps) {
+ if (!pump.InternalError.empty())
+ throw yexception() << pump.InternalError;
+ }
+#else
+ // Now let's read remaining stdout/stderr
+ while (output && (bytes = pi->OutputFd.Read(buffer.Data(), buffer.Capacity())) > 0) {
+ DBG(Cerr << bytes << " more bytes of output: " << Endl);
+ output->Write(buffer.Data(), bytes);
+ }
+ while (error && (bytes = pi->ErrorFd.Read(buffer.Data(), buffer.Capacity())) > 0) {
+ DBG(Cerr << bytes << " more bytes of error" << Endl);
+ error->Write(buffer.Data(), bytes);
+ }
+#endif
+ } catch (const yexception& e) {
+ // Some error in watch occured, set result to error
+ AtomicSet(pi->Parent->ExecutionStatus, SHELL_INTERNAL_ERROR);
+ pi->Parent->InternalError = e.what();
+ if (input) {
+ pi->InputFd.Close();
+ }
+ Cdbg << "shell command internal error: " << pi->Parent->InternalError << Endl;
+ }
+ // Now we can safely delete process info struct and other data
+ pi->Parent->TerminateFlag = true;
+ TerminateIsRequired(pi);
+}
+
+TShellCommand::TShellCommand(const TStringBuf cmd, const TList<TString>& args, const TShellCommandOptions& options,
+ const TString& workdir)
+ : Impl(new TImpl(cmd, args, options, workdir))
+{
+}
+
+TShellCommand::TShellCommand(const TStringBuf cmd, const TShellCommandOptions& options, const TString& workdir)
+ : Impl(new TImpl(cmd, TList<TString>(), options, workdir))
+{
+}
+
+TShellCommand::~TShellCommand() = default;
+
+TShellCommand& TShellCommand::operator<<(const TStringBuf argument) {
+ Impl->AppendArgument(argument);
+ return *this;
+}
+
+const TString& TShellCommand::GetOutput() const {
+ return Impl->GetOutput();
+}
+
+const TString& TShellCommand::GetError() const {
+ return Impl->GetError();
+}
+
+const TString& TShellCommand::GetInternalError() const {
+ return Impl->GetInternalError();
+}
+
+TShellCommand::ECommandStatus TShellCommand::GetStatus() const {
+ return Impl->GetStatus();
+}
+
+TMaybe<int> TShellCommand::GetExitCode() const {
+ return Impl->GetExitCode();
+}
+
+TProcessId TShellCommand::GetPid() const {
+ return Impl->GetPid();
+}
+
+TFileHandle& TShellCommand::GetInputHandle() {
+ return Impl->GetInputHandle();
+}
+
+TFileHandle& TShellCommand::GetOutputHandle() {
+ return Impl->GetOutputHandle();
+}
+
+TFileHandle& TShellCommand::GetErrorHandle() {
+ return Impl->GetErrorHandle();
+}
+
+TShellCommand& TShellCommand::Run() {
+ Impl->Run();
+ return *this;
+}
+
+TShellCommand& TShellCommand::Terminate() {
+ Impl->Terminate();
+ return *this;
+}
+
+TShellCommand& TShellCommand::Wait() {
+ Impl->Wait();
+ return *this;
+}
+
+TShellCommand& TShellCommand::CloseInput() {
+ Impl->CloseInput();
+ return *this;
+}
+
+TString TShellCommand::GetQuotedCommand() const {
+ return Impl->GetQuotedCommand();
+}
diff --git a/util/system/shellcommand.h b/util/system/shellcommand.h
new file mode 100644
index 0000000000..8730627fe5
--- /dev/null
+++ b/util/system/shellcommand.h
@@ -0,0 +1,485 @@
+#pragma once
+
+#include <util/generic/noncopyable.h>
+#include <util/generic/string.h>
+#include <util/generic/list.h>
+#include <util/generic/hash.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/maybe.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+#include "file.h"
+#include "getpid.h"
+#include "thread.h"
+#include "mutex.h"
+#include <sys/types.h>
+
+class TShellCommandOptions {
+public:
+ struct TUserOptions {
+ TString Name;
+#if defined(_win_)
+ TString Password;
+#endif
+#if defined(_unix_)
+ /**
+ * Run child process with the user supplementary groups.
+ * If true, the user supplementary groups will be set in the child process upon exec().
+ * If false, the supplementary groups of the parent process will be used.
+ */
+ bool UseUserGroups = false;
+#endif
+ };
+
+ enum EHandleMode {
+ HANDLE_INHERIT,
+ HANDLE_PIPE,
+ HANDLE_STREAM
+ };
+
+public:
+ inline TShellCommandOptions() noexcept
+ : ClearSignalMask(false)
+ , CloseAllFdsOnExec(false)
+ , AsyncMode(false)
+ , PollDelayMs(DefaultSyncPollDelay)
+ , UseShell(true)
+ , QuoteArguments(true)
+ , DetachSession(true)
+ , CloseStreams(false)
+ , ShouldCloseInput(true)
+ , InputMode(HANDLE_INHERIT)
+ , OutputMode(HANDLE_STREAM)
+ , ErrorMode(HANDLE_STREAM)
+ , InputStream(nullptr)
+ , OutputStream(nullptr)
+ , ErrorStream(nullptr)
+ , Nice(0)
+ , FuncAfterFork(std::function<void()>())
+ {
+ }
+
+ inline TShellCommandOptions& SetNice(int value) noexcept {
+ Nice = value;
+
+ return *this;
+ }
+
+ /**
+ * @brief clear signal mask from parent process. If true, child process
+ * clears the signal mask inherited from the parent process; otherwise
+ * child process retains the signal mask of the parent process.
+ *
+ * @param clearSignalMask true if child process should clear signal mask
+ * @note in default child process inherits signal mask.
+ * @return self
+ */
+ inline TShellCommandOptions& SetClearSignalMask(bool clearSignalMask) {
+ ClearSignalMask = clearSignalMask;
+ return *this;
+ }
+
+ /**
+ * @brief set close-on-exec mode. If true, all file descriptors
+ * from the parent process, except stdin, stdout, stderr, will be closed
+ * in the child process upon exec().
+ *
+ * @param closeAllFdsOnExec true if close-on-exec mode is needed
+ * @note in default close-on-exec mode is off.
+ * @return self
+ */
+ inline TShellCommandOptions& SetCloseAllFdsOnExec(bool closeAllFdsOnExec) {
+ CloseAllFdsOnExec = closeAllFdsOnExec;
+ return *this;
+ }
+
+ /**
+ * @brief set asynchronous mode. If true, task will be run
+ * in separate thread, and control will be returned immediately
+ *
+ * @param async true if asynchonous mode is needed
+ * @note in default async mode launcher will need 100% cpu for rapid process termination
+ * @return self
+ */
+ inline TShellCommandOptions& SetAsync(bool async) {
+ AsyncMode = async;
+ if (AsyncMode)
+ PollDelayMs = 0;
+ return *this;
+ }
+
+ /**
+ * @brief specify delay for process controlling loop
+ * @param ms number of milliseconds to poll for
+ * @note for synchronous process default of 1s should generally fit
+ * for async process default is no latency and that consumes 100% one cpu
+ * SetAsync(true) will reset this delay to 0, so call this method after
+ * @return self
+ */
+ inline TShellCommandOptions& SetLatency(size_t ms) {
+ PollDelayMs = ms;
+ return *this;
+ }
+
+ /**
+ * @brief set the stream, which is input fetched from
+ *
+ * @param stream Pointer to stream.
+ * If stream is NULL or not set, input channel will be closed.
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetInputStream(IInputStream* stream) {
+ InputStream = stream;
+ if (InputStream == nullptr) {
+ InputMode = HANDLE_INHERIT;
+ } else {
+ InputMode = HANDLE_STREAM;
+ }
+ return *this;
+ }
+
+ /**
+ * @brief set the stream, collecting the command output
+ *
+ * @param stream Pointer to stream.
+ * If stream is NULL or not set, output will be collected to the
+ * internal variable
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetOutputStream(IOutputStream* stream) {
+ OutputStream = stream;
+ return *this;
+ }
+
+ /**
+ * @brief set the stream, collecting the command error output
+ *
+ * @param stream Pointer to stream.
+ * If stream is NULL or not set, errors will be collected to the
+ * internal variable
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetErrorStream(IOutputStream* stream) {
+ ErrorStream = stream;
+ return *this;
+ }
+
+ /**
+ * @brief set if Finish() should be called on user-supplied streams
+ * if process is run in async mode Finish will be called in process' thread
+ * @param val if Finish() should be called
+ * @return self
+ */
+ inline TShellCommandOptions& SetCloseStreams(bool val) {
+ CloseStreams = val;
+ return *this;
+ }
+
+ /**
+ * @brief set if input stream should be closed after all data is read
+ * call SetCloseInput(false) for interactive process
+ * @param val if input stream should be closed
+ * @return self
+ */
+ inline TShellCommandOptions& SetCloseInput(bool val) {
+ ShouldCloseInput = val;
+ return *this;
+ }
+
+ /**
+ * @brief set if command should be interpreted by OS shell (/bin/sh or cmd.exe)
+ * shell is enabled by default
+ * call SetUseShell(false) for command to be sent to OS verbatim
+ * @note shell operators > < | && || will not work if this option is off
+ * @param useShell if command should be run in shell
+ * @return self
+ */
+ inline TShellCommandOptions& SetUseShell(bool useShell) {
+ UseShell = useShell;
+ if (!useShell)
+ QuoteArguments = false;
+ return *this;
+ }
+
+ /**
+ * @brief set if the arguments should be wrapped in quotes.
+ * Please, note that this option makes no difference between
+ * real arguments and shell syntax, so if you execute something
+ * like \b TShellCommand("sleep") << "3" << "&&" << "ls", your
+ * command will look like:
+ * sleep "3" "&&" "ls"
+ * which will never end successfully.
+ * By default, this option is turned on.
+ *
+ * @note arguments will only be quoted if shell is used
+ * @param quote if the arguments should be quoted
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetQuoteArguments(bool quote) {
+ QuoteArguments = quote;
+ return *this;
+ }
+
+ /**
+ * @brief set to run command in new session
+ * @note set this option to off to deliver parent's signals to command as well
+ * @note currently ignored on windows
+ * @param detach if command should be run in new session
+ * @return self
+ */
+ inline TShellCommandOptions& SetDetachSession(bool detach) {
+ DetachSession = detach;
+ return *this;
+ }
+
+ /**
+ * @brief specifies pure function to be called in the child process after fork, before calling execve
+ * @note currently ignored on windows
+ * @param function function to be called after fork
+ * @return self
+ */
+ inline TShellCommandOptions& SetFuncAfterFork(const std::function<void()>& function) {
+ FuncAfterFork = function;
+ return *this;
+ }
+
+ /**
+ * @brief create a pipe for child input
+ * Write end of the pipe will be accessible via TShellCommand::GetInputHandle
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& PipeInput() {
+ InputMode = HANDLE_PIPE;
+ InputStream = nullptr;
+ return *this;
+ }
+
+ inline TShellCommandOptions& PipeOutput() {
+ OutputMode = HANDLE_PIPE;
+ OutputStream = nullptr;
+ return *this;
+ }
+
+ inline TShellCommandOptions& PipeError() {
+ ErrorMode = HANDLE_PIPE;
+ ErrorStream = nullptr;
+ return *this;
+ }
+
+ /**
+ * @brief set if child should inherit output handle
+ *
+ * @param inherit if child should inherit output handle
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetInheritOutput(bool inherit) {
+ OutputMode = inherit ? HANDLE_INHERIT : HANDLE_STREAM;
+ return *this;
+ }
+
+ /**
+ * @brief set if child should inherit stderr handle
+ *
+ * @param inherit if child should inherit error output handle
+ *
+ * @return self
+ */
+ inline TShellCommandOptions& SetInheritError(bool inherit) {
+ ErrorMode = inherit ? HANDLE_INHERIT : HANDLE_STREAM;
+ return *this;
+ }
+
+public:
+ bool ClearSignalMask = false;
+ bool CloseAllFdsOnExec = false;
+ bool AsyncMode = false;
+ size_t PollDelayMs = 0;
+ bool UseShell = false;
+ bool QuoteArguments = false;
+ bool DetachSession = false;
+ bool CloseStreams = false;
+ bool ShouldCloseInput = false;
+ EHandleMode InputMode = HANDLE_STREAM;
+ EHandleMode OutputMode = HANDLE_STREAM;
+ EHandleMode ErrorMode = HANDLE_STREAM;
+
+ /// @todo more options
+ // bool SearchPath // search exe name in $PATH
+ // bool UnicodeConsole
+ // bool EmulateConsole // provide isatty == true
+ /// @todo command's stdin should be exposet as IOutputStream to support dialogue
+ IInputStream* InputStream;
+ IOutputStream* OutputStream;
+ IOutputStream* ErrorStream;
+ TUserOptions User;
+ THashMap<TString, TString> Environment;
+ int Nice = 0;
+
+ static const size_t DefaultSyncPollDelay = 1000; // ms
+ std::function<void()> FuncAfterFork = {};
+};
+
+/**
+ * @brief Execute command in shell and provide its results
+ * @attention Not thread-safe
+ */
+class TShellCommand: public TNonCopyable {
+private:
+ TShellCommand();
+
+public:
+ enum ECommandStatus {
+ SHELL_NONE,
+ SHELL_RUNNING,
+ SHELL_FINISHED,
+ SHELL_INTERNAL_ERROR,
+ SHELL_ERROR
+ };
+
+public:
+ /**
+ * @brief create the command with initial arguments list
+ *
+ * @param cmd binary name
+ * @param args arguments list
+ * @param options execution options
+ * @todo store entire options structure
+ */
+ TShellCommand(const TStringBuf cmd, const TList<TString>& args, const TShellCommandOptions& options = TShellCommandOptions(),
+ const TString& workdir = TString());
+ TShellCommand(const TStringBuf cmd, const TShellCommandOptions& options = TShellCommandOptions(), const TString& workdir = TString());
+ ~TShellCommand();
+
+public:
+ /**
+ * @brief append argument to the args list
+ *
+ * @param argument string argument
+ *
+ * @return self
+ */
+ TShellCommand& operator<<(const TStringBuf argument);
+
+ /**
+ * @brief return the collected output from the command.
+ * If the output stream is set, empty string will be returned
+ *
+ * @return collected output
+ */
+ const TString& GetOutput() const;
+
+ /**
+ * @brief return the collected error output from the command.
+ * If the error stream is set, empty string will be returned
+ *
+ * @return collected error output
+ */
+ const TString& GetError() const;
+
+ /**
+ * @brief return the internal error occured while watching
+ * the command execution. Should be called if execution
+ * status is SHELL_INTERNAL_ERROR
+ *
+ * @return error text
+ */
+ const TString& GetInternalError() const;
+
+ /**
+ * @brief get current status of command execution
+ *
+ * @return current status
+ */
+ ECommandStatus GetStatus() const;
+
+ /**
+ * @brief return exit code of finished process
+ * The value is unspecified in case of internal errors or if the process is running
+ *
+ * @return exit code
+ */
+ TMaybe<int> GetExitCode() const;
+
+ /**
+ * @brief get id of underlying process
+ * @note depends on os: pid_t on UNIX, HANDLE on win
+ *
+ * @return pid or handle
+ */
+ TProcessId GetPid() const;
+
+ /**
+ * @brief return the file handle that provides input to the child process
+ *
+ * @return input file handle
+ */
+ TFileHandle& GetInputHandle();
+
+ /**
+ * @brief return the file handle that provides output from the child process
+ *
+ * @return output file handle
+ */
+ TFileHandle& GetOutputHandle();
+
+ /**
+ * @brief return the file handle that provides error output from the child process
+ *
+ * @return error file handle
+ */
+ TFileHandle& GetErrorHandle();
+
+ /**
+ * @brief run the execution
+ *
+ * @return self
+ */
+ TShellCommand& Run();
+
+ /**
+ * @brief terminate the execution
+ * @note if DetachSession is set, it terminates all procs in command's new process group
+ *
+ * @return self
+ */
+ TShellCommand& Terminate();
+
+ /**
+ * @brief wait until the execution is finished
+ *
+ * @return self
+ */
+ TShellCommand& Wait();
+
+ /**
+ * @brief close process' stdin
+ *
+ * @return self
+ */
+ TShellCommand& CloseInput();
+
+ /**
+ * @brief Get quoted command (for debug/view purposes only!)
+ **/
+ TString GetQuotedCommand() const;
+
+private:
+ class TImpl;
+ using TImplRef = TSimpleIntrusivePtr<TImpl>;
+ TImplRef Impl;
+};
+
+/// Appends to dst: quoted arg
+void ShellQuoteArg(TString& dst, TStringBuf arg);
+
+/// Appends to dst: space, quoted arg
+void ShellQuoteArgSp(TString& dst, TStringBuf arg);
+
+/// Returns true if arg should be quoted
+bool ArgNeedsQuotes(TStringBuf arg) noexcept;
diff --git a/util/system/shellcommand_ut.cpp b/util/system/shellcommand_ut.cpp
new file mode 100644
index 0000000000..9d849279d2
--- /dev/null
+++ b/util/system/shellcommand_ut.cpp
@@ -0,0 +1,493 @@
+#include "shellcommand.h"
+
+#include "compat.h"
+#include "defaults.h"
+#include "fs.h"
+#include "sigset.h"
+#include "spinlock.h"
+
+#include <library/cpp/testing/unittest/env.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/folder/dirut.h>
+#include <util/random/random.h>
+#include <util/stream/file.h>
+#include <util/stream/str.h>
+#include <util/stream/mem.h>
+#include <util/string/strip.h>
+#include <util/folder/tempdir.h>
+
+#if defined(_win_)
+ #define NL "\r\n"
+const char catCommand[] = "sort"; // not really cat but ok
+const size_t textSize = 1;
+#else
+ #define NL "\n"
+const char catCommand[] = "/bin/cat";
+const size_t textSize = 20000;
+#endif
+
+class TGuardedStringStream: public IInputStream, public IOutputStream {
+public:
+ TGuardedStringStream() {
+ Stream_.Reserve(100);
+ }
+
+ TString Str() const {
+ with_lock (Lock_) {
+ return Stream_.Str();
+ }
+ return TString(); // line for compiler
+ }
+
+protected:
+ size_t DoRead(void* buf, size_t len) override {
+ with_lock (Lock_) {
+ return Stream_.Read(buf, len);
+ }
+ return 0; // line for compiler
+ }
+
+ void DoWrite(const void* buf, size_t len) override {
+ with_lock (Lock_) {
+ return Stream_.Write(buf, len);
+ }
+ }
+
+private:
+ TAdaptiveLock Lock_;
+ TStringStream Stream_;
+};
+
+Y_UNIT_TEST_SUITE(TShellQuoteTest) {
+ Y_UNIT_TEST(TestQuoteArg) {
+ TString cmd;
+ ShellQuoteArg(cmd, "/pr f/krev/prev.exe");
+ ShellQuoteArgSp(cmd, "-DVal=\"W Quotes\"");
+ ShellQuoteArgSp(cmd, "-DVal=W Space");
+ ShellQuoteArgSp(cmd, "-DVal=Blah");
+ UNIT_ASSERT_STRINGS_EQUAL(cmd, "\"/pr f/krev/prev.exe\" \"-DVal=\\\"W Quotes\\\"\" \"-DVal=W Space\" \"-DVal=Blah\"");
+ }
+}
+
+Y_UNIT_TEST_SUITE(TShellCommandTest) {
+ Y_UNIT_TEST(TestNoQuotes) {
+ TShellCommandOptions options;
+ options.SetQuoteArguments(false);
+ TShellCommand cmd("echo hello");
+ cmd.Run();
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError(), "");
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput(), "hello" NL);
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetQuotedCommand(), "echo hello");
+ }
+
+ Y_UNIT_TEST(TestOnlyNecessaryQuotes) {
+ TShellCommandOptions options;
+ options.SetQuoteArguments(true);
+ TShellCommand cmd("echo");
+ cmd << "hey"
+ << "hello&world";
+ cmd.Run();
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError(), "");
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput(), "hey hello&world" NL);
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+ }
+
+ Y_UNIT_TEST(TestRun) {
+ TShellCommand cmd("echo");
+ cmd << "hello";
+ cmd.Run();
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError(), "");
+#if defined(_win_)
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput(), "\"hello\"\r\n");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput(), "hello\n");
+#endif
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+ }
+ // running with no shell is not implemented for win
+ // there should be no problem with it as long as SearchPath is on
+ Y_UNIT_TEST(TestNoShell) {
+#if defined(_win_)
+ const char dir[] = "dir";
+#else
+ const char dir[] = "ls";
+#endif
+
+ TShellCommandOptions options;
+ options.SetQuoteArguments(false);
+
+ {
+ options.SetUseShell(false);
+ TShellCommand cmd(dir, options);
+ cmd << "|"
+ << "sort";
+
+ cmd.Run();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 != cmd.GetExitCode());
+ }
+ {
+ options.SetUseShell(true);
+ TShellCommand cmd(dir, options);
+ cmd << "|"
+ << "sort";
+ cmd.Run();
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError().size(), 0u);
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+ }
+ }
+ Y_UNIT_TEST(TestAsyncRun) {
+ TShellCommandOptions options;
+ options.SetAsync(true);
+#if defined(_win_)
+ // fails with weird error "Input redirection is not supported"
+ // TShellCommand cmd("sleep", options);
+ // cmd << "3";
+ TShellCommand cmd("ping 1.1.1.1 -n 1 -w 2000", options);
+#else
+ TShellCommand cmd("sleep", options);
+ cmd << "2";
+#endif
+ UNIT_ASSERT(TShellCommand::SHELL_NONE == cmd.GetStatus());
+ cmd.Run();
+ sleep(1);
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING == cmd.GetStatus());
+ cmd.Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING != cmd.GetStatus());
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError(), "");
+#if !defined(_win_)
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput().size(), 0u);
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+#endif
+ }
+ Y_UNIT_TEST(TestQuotes) {
+ TShellCommandOptions options;
+ TString input = TString("a\"a a");
+ TString output;
+ TStringOutput outputStream(output);
+ options.SetOutputStream(&outputStream);
+ TShellCommand cmd("echo", options);
+ cmd << input;
+ cmd.Run().Wait();
+ output = StripString(output);
+#if defined(_win_)
+ UNIT_ASSERT_VALUES_EQUAL("\"a\\\"a a\"", output);
+#else
+ UNIT_ASSERT_VALUES_EQUAL(input, output);
+#endif
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError().size(), 0u);
+ }
+ Y_UNIT_TEST(TestRunNonexistent) {
+ TShellCommand cmd("iwerognweiofnewio"); // some nonexistent command name
+ cmd.Run().Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_UNEQUAL(cmd.GetError().size(), 0u);
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 != cmd.GetExitCode());
+ }
+ Y_UNIT_TEST(TestExitCode) {
+ TShellCommand cmd("grep qwerty qwerty"); // some nonexistent file name
+ cmd.Run().Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_UNEQUAL(cmd.GetError().size(), 0u);
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 2 == cmd.GetExitCode());
+ }
+ // 'type con' and 'copy con con' want real console, not stdin, use sort
+ Y_UNIT_TEST(TestInput) {
+ TShellCommandOptions options;
+ TString input = (TString("a") * 2000).append(NL) * textSize;
+ TStringInput inputStream(input);
+ options.SetInputStream(&inputStream);
+ TShellCommand cmd(catCommand, options);
+ cmd.Run().Wait();
+ UNIT_ASSERT_VALUES_EQUAL(input, cmd.GetOutput());
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError().size(), 0u);
+ }
+ Y_UNIT_TEST(TestOutput) {
+ TShellCommandOptions options;
+ TString input = (TString("a") * 2000).append(NL) * textSize;
+ TStringInput inputStream(input);
+ options.SetInputStream(&inputStream);
+ TString output;
+ TStringOutput outputStream(output);
+ options.SetOutputStream(&outputStream);
+ TShellCommand cmd(catCommand, options);
+ cmd.Run().Wait();
+ UNIT_ASSERT_VALUES_EQUAL(input, output);
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError().size(), 0u);
+ }
+ Y_UNIT_TEST(TestIO) {
+ // descriptive test: use all options
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.SetQuoteArguments(false);
+ options.SetLatency(10);
+ options.SetClearSignalMask(true);
+ options.SetCloseAllFdsOnExec(true);
+ options.SetCloseInput(false);
+ TGuardedStringStream write;
+ options.SetInputStream(&write);
+ TGuardedStringStream read;
+ options.SetOutputStream(&read);
+ options.SetUseShell(true);
+
+ TShellCommand cmd("cat", options);
+ cmd.Run();
+
+ write << "alpha" << NL;
+ while (read.Str() != "alpha" NL) {
+ Sleep(TDuration::MilliSeconds(10));
+ }
+
+ write << "omega" << NL;
+ while (read.Str() != "alpha" NL "omega" NL) {
+ Sleep(TDuration::MilliSeconds(10));
+ }
+
+ write << "zeta" << NL;
+ cmd.CloseInput();
+ cmd.Wait();
+
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError(), "");
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_EQUAL(read.Str(), "alpha" NL "omega" NL "zeta" NL);
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+ }
+ Y_UNIT_TEST(TestStreamClose) {
+ struct TStream: public IOutputStream {
+ size_t NumCloses = 0;
+ void DoWrite(const void* buf, size_t len) override {
+ Y_UNUSED(buf);
+ Y_UNUSED(len);
+ }
+ void DoFinish() override {
+ ++NumCloses;
+ }
+ } stream;
+
+ auto options1 = TShellCommandOptions().SetCloseStreams(false).SetOutputStream(&stream).SetErrorStream(&stream);
+ TShellCommand("echo hello", options1).Run().Wait();
+ UNIT_ASSERT_VALUES_EQUAL(stream.NumCloses, 0);
+
+ auto options = TShellCommandOptions().SetCloseStreams(true).SetOutputStream(&stream).SetErrorStream(&stream);
+ TShellCommand("echo hello", options).Run().Wait();
+ UNIT_ASSERT_VALUES_EQUAL(stream.NumCloses, 2);
+ }
+ Y_UNIT_TEST(TestInterruptSimple) {
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.SetCloseInput(false);
+ TGuardedStringStream write;
+ options.SetInputStream(&write); // set input stream that will be waited by cat
+ TShellCommand cmd(catCommand, options);
+ cmd.Run();
+ sleep(1);
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING == cmd.GetStatus());
+ cmd.Terminate();
+ cmd.Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING != cmd.GetStatus());
+ }
+#if !defined(_win_)
+ // this ut is unix-only, port to win using %TEMP%
+ Y_UNIT_TEST(TestInterrupt) {
+ TString tmpfile = TString("shellcommand_ut.interrupt.") + ToString(RandomNumber<ui32>());
+
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.SetQuoteArguments(false);
+ {
+ TShellCommand cmd("/bin/sleep", options);
+ cmd << " 1300 & wait; /usr/bin/touch " << tmpfile;
+ cmd.Run();
+ sleep(1);
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING == cmd.GetStatus());
+ // Async mode requires Terminate() + Wait() to send kill to child proc!
+ cmd.Terminate();
+ cmd.Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && -15 == cmd.GetExitCode());
+ }
+ sleep(1);
+ UNIT_ASSERT(!NFs::Exists(tmpfile));
+ }
+ // this ut is unix-only (win has no signal mask)
+ Y_UNIT_TEST(TestSignalMask) {
+ // block SIGTERM
+ int rc;
+ sigset_t newmask, oldmask;
+ SigEmptySet(&newmask);
+ SigAddSet(&newmask, SIGTERM);
+ rc = SigProcMask(SIG_SETMASK, &newmask, &oldmask);
+ UNIT_ASSERT(rc == 0);
+
+ TString tmpfile = TString("shellcommand_ut.interrupt.") + ToString(RandomNumber<ui32>());
+
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.SetQuoteArguments(false);
+
+ // child proc should not receive SIGTERM anymore
+ {
+ TShellCommand cmd("/bin/sleep", options);
+ // touch file only if sleep not interrupted by SIGTERM
+ cmd << " 10 & wait; [ $? == 0 ] || /usr/bin/touch " << tmpfile;
+ cmd.Run();
+ sleep(1);
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING == cmd.GetStatus());
+ cmd.Terminate();
+ cmd.Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus() || TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ }
+ sleep(1);
+ UNIT_ASSERT(!NFs::Exists(tmpfile));
+
+ // child proc should receive SIGTERM
+ options.SetClearSignalMask(true);
+ {
+ TShellCommand cmd("/bin/sleep", options);
+ // touch file regardless -- it will be interrupted
+ cmd << " 10 & wait; /usr/bin/touch " << tmpfile;
+ cmd.Run();
+ sleep(1);
+ UNIT_ASSERT(TShellCommand::SHELL_RUNNING == cmd.GetStatus());
+ cmd.Terminate();
+ cmd.Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_ERROR == cmd.GetStatus());
+ }
+ sleep(1);
+ UNIT_ASSERT(!NFs::Exists(tmpfile));
+
+ // restore signal mask
+ rc = SigProcMask(SIG_SETMASK, &oldmask, nullptr);
+ UNIT_ASSERT(rc == 0);
+ }
+#else
+ // This ut is windows-only
+ Y_UNIT_TEST(TestStdinProperlyConstructed) {
+ TShellCommandOptions options;
+ options.SetErrorStream(&Cerr);
+
+ TShellCommand cmd(BinaryPath("util/system/ut/stdin_osfhandle/stdin_osfhandle"), options);
+ cmd.Run().Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_FINISHED == cmd.GetStatus());
+ UNIT_ASSERT(cmd.GetExitCode().Defined() && 0 == cmd.GetExitCode());
+ }
+#endif
+ Y_UNIT_TEST(TestInternalError) {
+ TString input = (TString("a") * 2000).append("\n");
+ TStringInput inputStream(input);
+ TMemoryOutput outputStream(nullptr, 0);
+ TShellCommandOptions options;
+ options.SetInputStream(&inputStream);
+ options.SetOutputStream(&outputStream);
+ TShellCommand cmd(catCommand, options);
+ cmd.Run().Wait();
+ UNIT_ASSERT(TShellCommand::SHELL_INTERNAL_ERROR == cmd.GetStatus());
+ UNIT_ASSERT_VALUES_UNEQUAL(cmd.GetInternalError().size(), 0u);
+ }
+ Y_UNIT_TEST(TestHugeOutput) {
+ TShellCommandOptions options;
+ TGuardedStringStream stream;
+ options.SetOutputStream(&stream);
+ options.SetUseShell(true);
+
+ TString input = TString(7000, 'a');
+ TString command = TStringBuilder{} << "echo " << input;
+ TShellCommand cmd(command, options);
+ cmd.Run().Wait();
+
+ UNIT_ASSERT_VALUES_EQUAL(stream.Str(), input + NL);
+ }
+ Y_UNIT_TEST(TestHugeError) {
+ TShellCommandOptions options;
+ TGuardedStringStream stream;
+ options.SetErrorStream(&stream);
+ options.SetUseShell(true);
+
+ TString input = TString(7000, 'a');
+ TString command = TStringBuilder{} << "echo " << input << ">&2";
+ TShellCommand cmd(command, options);
+ cmd.Run().Wait();
+
+ UNIT_ASSERT_VALUES_EQUAL(stream.Str(), input + NL);
+ }
+ Y_UNIT_TEST(TestPipeInput) {
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.PipeInput();
+
+ TShellCommand cmd(catCommand, options);
+ cmd.Run();
+
+ {
+ TFile file(cmd.GetInputHandle().Release());
+ TUnbufferedFileOutput fo(file);
+ fo << "hello" << Endl;
+ }
+
+ cmd.Wait();
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetOutput(), "hello" NL);
+ UNIT_ASSERT_VALUES_EQUAL(cmd.GetError().size(), 0u);
+ }
+ Y_UNIT_TEST(TestPipeOutput) {
+ TShellCommandOptions options;
+ options.SetAsync(true);
+ options.PipeOutput();
+ constexpr TStringBuf firstMessage = "first message";
+ constexpr TStringBuf secondMessage = "second message";
+ const TString command = TStringBuilder() << "echo '" << firstMessage << "' && sleep 10 && echo '" << secondMessage << "'";
+ TShellCommand cmd(command, options);
+ cmd.Run();
+ TUnbufferedFileInput cmdOutput(TFile(cmd.GetOutputHandle().Release()));
+ TString firstLineOutput, secondLineOutput;
+ {
+ Sleep(TDuration::Seconds(5));
+ firstLineOutput = cmdOutput.ReadLine();
+ cmd.Wait();
+ secondLineOutput = cmdOutput.ReadLine();
+ }
+ UNIT_ASSERT_VALUES_EQUAL(firstLineOutput, firstMessage);
+ UNIT_ASSERT_VALUES_EQUAL(secondLineOutput, secondLineOutput);
+ }
+ Y_UNIT_TEST(TestOptionsConsistency) {
+ TShellCommandOptions options;
+
+ options.SetInheritOutput(false).SetInheritError(false);
+ options.SetOutputStream(nullptr).SetErrorStream(nullptr);
+
+ UNIT_ASSERT(options.OutputMode == TShellCommandOptions::HANDLE_STREAM);
+ UNIT_ASSERT(options.ErrorMode == TShellCommandOptions::HANDLE_STREAM);
+ }
+ Y_UNIT_TEST(TestForkCallback) {
+ TString tmpFile = TString("shellcommand_ut.test_for_callback.txt");
+ TFsPath cwd(::NFs::CurrentWorkingDirectory());
+ const TString tmpFilePath = cwd.Child(tmpFile);
+
+ const TString text = "test output";
+ auto afterForkCallback = [&tmpFilePath, &text]() -> void {
+ TFixedBufferFileOutput out(tmpFilePath);
+ out << text;
+ };
+
+ TShellCommandOptions options;
+ options.SetFuncAfterFork(afterForkCallback);
+
+ const TString command = "ls";
+ TShellCommand cmd(command, options);
+ cmd.Run();
+
+ UNIT_ASSERT(NFs::Exists(tmpFilePath));
+
+ TUnbufferedFileInput fileOutput(tmpFilePath);
+ TString firstLine = fileOutput.ReadLine();
+
+ UNIT_ASSERT_VALUES_EQUAL(firstLine, text);
+ }
+}
diff --git a/util/system/shmat.cpp b/util/system/shmat.cpp
new file mode 100644
index 0000000000..07ff0d6caa
--- /dev/null
+++ b/util/system/shmat.cpp
@@ -0,0 +1,231 @@
+#include "shmat.h"
+
+#include <util/generic/guid.h>
+
+#if defined(_win_)
+ #include <stdio.h>
+ #include "winint.h"
+#elif defined(_bionic_)
+ #include <sys/types.h>
+ #include <sys/ipc.h>
+ #include <sys/syscall.h>
+#elif defined(_unix_)
+ #include <sys/types.h>
+ #include <sys/ipc.h>
+ #include <sys/shm.h>
+#endif
+
+#if defined(_cygwin_)
+ #define WINAPI __stdcall
+ #define FILE_MAP_ALL_ACCESS ((long)983071)
+ #define PAGE_READWRITE 4
+ #define FALSE 0
+
+extern "C" {
+ using HANDLE = OS_HANDLE;
+ using BOOL = int;
+ using DWORD = ui32;
+ using LPCTSTR = const char*;
+ using LPVOID = void*;
+ using LPCVOID = void const*;
+ using SIZE_T = size_t;
+
+ BOOL WINAPI CloseHandle(HANDLE hObject);
+ HANDLE WINAPI OpenFileMappingA(DWORD dwDesiredAccess, BOOL bInheritHandle, LPCTSTR lpName);
+ LPVOID WINAPI MapViewOfFile(HANDLE hFileMappingObject, DWORD DesiredAccess, DWORD FileOffsetHigh, DWORD FileOffsetLow, SIZE_T NumberOfBytesToMap);
+ HANDLE WINAPI CreateFileMappingA(HANDLE hFile, LPVOID lpAttributes, DWORD flProtect, DWORD MaximumSizeHigh, DWORD MaximumSizeLow, LPCTSTR lpName);
+ BOOL WINAPI UnmapViewOfFile(LPCVOID lpBaseAddress);
+ DWORD WINAPI GetLastError(void);
+}
+#endif
+
+#if defined(_bionic_)
+namespace {
+ #if !defined(__i386__)
+ static int shmget(key_t key, size_t size, int flag) {
+ if (size > PTRDIFF_MAX) {
+ size = SIZE_MAX;
+ }
+
+ return syscall(__NR_shmget, key, size, flag);
+ }
+
+ static void* shmat(int id, const void* addr, int flag) {
+ return (void*)syscall(__NR_shmat, id, addr, flag);
+ }
+
+ static int shmctl(int id, int cmd, void* buf) {
+ return syscall(__NR_shmctl, id, cmd | IPC_64, buf);
+ }
+
+ static int shmdt(const void* addr) {
+ return syscall(__NR_shmdt, addr);
+ }
+
+ #else
+ #define IPCOP_shmat 21
+ #define IPCOP_shmdt 22
+ #define IPCOP_shmget 23
+ #define IPCOP_shmctl 24
+
+ static int shmget(key_t key, size_t size, int flag) {
+ return syscall(__NR_ipc, IPCOP_shmget, key, size, flag, 0);
+ }
+
+ static void* shmat(int id, const void* addr, int flag) {
+ void* retval;
+ long res = syscall(__NR_ipc, IPCOP_shmat, id, flag, (long)&retval, addr);
+ return (res >= 0) ? retval : (void*)-1;
+ }
+
+ static int shmctl(int id, int cmd, void* buf) {
+ return syscall(__NR_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf);
+ }
+
+ static int shmdt(const void* addr) {
+ return syscall(__NR_ipc, IPCOP_shmdt, 0, 0, 0, addr);
+ }
+ #endif
+}
+#endif
+
+TSharedMemory::TSharedMemory()
+ : Handle(INVALID_FHANDLE)
+ , Data(nullptr)
+ , Size(0)
+{
+}
+
+#if defined(_win_)
+static void FormatName(char* buf, const TGUID& id) {
+ sprintf(buf, "Global\\shmat-%s", GetGuidAsString(id).c_str());
+}
+
+bool TSharedMemory::Open(const TGUID& id, int size) {
+ //Y_ASSERT(Data == 0);
+ Id = id;
+ Size = size;
+
+ char name[100];
+ FormatName(name, Id);
+ Handle = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, name);
+
+ if (Handle == 0) {
+ return false;
+ }
+
+ Data = MapViewOfFile(Handle, FILE_MAP_ALL_ACCESS, 0, 0, size);
+
+ if (Data == 0) {
+ //Y_ASSERT(0);
+ CloseHandle(Handle);
+ Handle = INVALID_OS_HANDLE;
+
+ return false;
+ }
+
+ return true;
+}
+
+bool TSharedMemory::Create(int size) {
+ //Y_ASSERT(Data == 0);
+ Size = size;
+
+ CreateGuid(&Id);
+
+ char name[100];
+ FormatName(name, Id);
+ Handle = CreateFileMappingA(INVALID_OS_HANDLE, nullptr, PAGE_READWRITE, 0, size, name);
+
+ if (Handle == 0) {
+ //Y_ASSERT(0);
+ return false;
+ }
+
+ Data = MapViewOfFile(Handle, FILE_MAP_ALL_ACCESS, 0, 0, size);
+
+ if (Data == 0) {
+ //Y_ASSERT(0);
+ CloseHandle(Handle);
+ Handle = INVALID_OS_HANDLE;
+
+ return false;
+ }
+
+ return true;
+}
+
+TSharedMemory::~TSharedMemory() {
+ if (Data) {
+ UnmapViewOfFile(Handle);
+ }
+
+ CloseHandle(Handle);
+}
+#else
+static key_t GetKey(const TGUID& id) {
+ i64 id64 = (ui64)(((ui64)id.dw[0] + (ui64)id.dw[2]) << 32) + (ui64)id.dw[1] + (ui64)id.dw[3];
+
+ return id64;
+}
+
+bool TSharedMemory::Open(const TGUID& id, int size) {
+ Y_VERIFY(id, "invalid shared memory guid: %s", GetGuidAsString(id).data());
+
+ //Y_ASSERT(Data == 0);
+ Size = size;
+
+ key_t k = GetKey(id);
+ int shmId = shmget(k, Size, 0777); // do not fill Handle, since IPC_RMID should be called by owner
+
+ if (shmId < 0) {
+ return false;
+ }
+
+ Data = shmat(shmId, nullptr, 0);
+
+ if (Data == nullptr) {
+ //Y_ASSERT(0);
+ return false;
+ }
+
+ return true;
+}
+
+bool TSharedMemory::Create(int size) {
+ //Y_ASSERT(Data == 0);
+ Size = size;
+
+ CreateGuid(&Id);
+
+ key_t k = GetKey(Id);
+ Handle = shmget(k, Size, IPC_CREAT | IPC_EXCL | 0777);
+
+ if (Handle < 0) {
+ //Y_ASSERT(0);
+ return false;
+ }
+
+ Data = shmat(Handle, nullptr, 0);
+
+ if (Data == (void*)-1) {
+ //Y_ASSERT(0);
+ shmctl(Handle, IPC_RMID, nullptr);
+ Handle = -1;
+
+ return false;
+ }
+
+ return true;
+}
+
+TSharedMemory::~TSharedMemory() {
+ if (Data) {
+ shmdt(Data);
+ }
+
+ if (Handle >= 0) {
+ shmctl(Handle, IPC_RMID, nullptr);
+ }
+}
+#endif
diff --git a/util/system/shmat.h b/util/system/shmat.h
new file mode 100644
index 0000000000..d9da3c151a
--- /dev/null
+++ b/util/system/shmat.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "fhandle.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/guid.h>
+
+class TSharedMemory: public TThrRefBase {
+ TGUID Id;
+ FHANDLE Handle;
+ void* Data;
+ int Size;
+
+public:
+ TSharedMemory();
+ ~TSharedMemory() override;
+
+ bool Create(int Size);
+ bool Open(const TGUID& id, int size);
+
+ const TGUID& GetId() {
+ return Id;
+ }
+
+ void* GetPtr() {
+ return Data;
+ }
+
+ int GetSize() const {
+ return Size;
+ }
+};
diff --git a/util/system/shmat_ut.cpp b/util/system/shmat_ut.cpp
new file mode 100644
index 0000000000..9d92243ae7
--- /dev/null
+++ b/util/system/shmat_ut.cpp
@@ -0,0 +1,17 @@
+#include "shmat.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TTestSharedMemory) {
+ Y_UNIT_TEST(TestInProc) {
+ TSharedMemory m1;
+ TSharedMemory m2;
+
+ UNIT_ASSERT(m1.Create(128));
+ UNIT_ASSERT(m2.Open(m1.GetId(), m1.GetSize()));
+
+ *(ui32*)m1.GetPtr() = 123;
+
+ UNIT_ASSERT_VALUES_EQUAL(*(ui32*)m2.GetPtr(), 123);
+ }
+}
diff --git a/util/system/sigset.cpp b/util/system/sigset.cpp
new file mode 100644
index 0000000000..4d91538191
--- /dev/null
+++ b/util/system/sigset.cpp
@@ -0,0 +1 @@
+#include "sigset.h"
diff --git a/util/system/sigset.h b/util/system/sigset.h
new file mode 100644
index 0000000000..8dd02fd817
--- /dev/null
+++ b/util/system/sigset.h
@@ -0,0 +1,78 @@
+#pragma once
+
+// Functions for manipulating signal sets
+
+#include "compat.h"
+
+#if defined _unix_
+ #include <pthread.h>
+#elif defined _win_
+ // Flags for sigprocmask:
+ #define SIG_BLOCK 1
+ #define SIG_UNBLOCK 2
+ #define SIG_SETMASK 3
+
+using sigset_t = ui32;
+
+#else
+ #error not supported yet
+#endif
+
+inline int SigEmptySet(sigset_t* set) {
+#if defined _unix_
+ return sigemptyset(set);
+#else
+ Y_UNUSED(set);
+ return 0;
+#endif
+}
+
+inline int SigFillSet(sigset_t* set) {
+#if defined _unix_
+ return sigfillset(set);
+#else
+ Y_UNUSED(set);
+ return 0;
+#endif
+}
+
+inline int SigAddSet(sigset_t* set, int signo) {
+#if defined _unix_
+ return sigaddset(set, signo);
+#else
+ Y_UNUSED(set);
+ Y_UNUSED(signo);
+ return 0;
+#endif
+}
+
+inline int SigDelSet(sigset_t* set, int signo) {
+#if defined _unix_
+ return sigdelset(set, signo);
+#else
+ Y_UNUSED(set);
+ Y_UNUSED(signo);
+ return 0;
+#endif
+}
+
+inline int SigIsMember(const sigset_t* set, int signo) {
+#if defined _unix_
+ return sigismember(const_cast<sigset_t*>(set), signo);
+#else
+ Y_UNUSED(set);
+ Y_UNUSED(signo);
+ return 0;
+#endif
+}
+
+inline int SigProcMask(int how, const sigset_t* set, sigset_t* oset) {
+#if defined _unix_
+ return pthread_sigmask(how, set, oset);
+#else
+ Y_UNUSED(set);
+ Y_UNUSED(oset);
+ Y_UNUSED(how);
+ return 0;
+#endif
+}
diff --git a/util/system/spin_wait.cpp b/util/system/spin_wait.cpp
new file mode 100644
index 0000000000..e27045e74f
--- /dev/null
+++ b/util/system/spin_wait.cpp
@@ -0,0 +1,40 @@
+#include "spin_wait.h"
+#include "yield.h"
+#include "compat.h"
+#include "thread.h"
+#include "spinlock.h"
+
+#include <util/digest/numeric.h>
+#include <util/generic/utility.h>
+
+template <class T>
+static inline T RandomizeSleepTime(T t) noexcept {
+ static TAtomic counter = 0;
+ const T rndNum = IntHash((T)AtomicIncrement(counter));
+
+ return (t * (T)4 + (rndNum % t) * (T)2) / (T)5;
+}
+
+//arbitrary values
+#define MIN_SLEEP_TIME 500
+#define MAX_SPIN_COUNT 0x7FF
+
+TSpinWait::TSpinWait() noexcept
+ : T(MIN_SLEEP_TIME)
+ , C(0)
+{
+}
+
+void TSpinWait::Sleep() noexcept {
+ ++C;
+
+ if (C == MAX_SPIN_COUNT) {
+ ThreadYield();
+ } else if ((C & MAX_SPIN_COUNT) == 0) {
+ usleep(RandomizeSleepTime(T));
+
+ T = Min<unsigned>((T * 3) / 2, 20000);
+ } else {
+ SpinLockPause();
+ }
+}
diff --git a/util/system/spin_wait.h b/util/system/spin_wait.h
new file mode 100644
index 0000000000..91dd423e33
--- /dev/null
+++ b/util/system/spin_wait.h
@@ -0,0 +1,10 @@
+#pragma once
+
+struct TSpinWait {
+ TSpinWait() noexcept;
+
+ void Sleep() noexcept;
+
+ unsigned T;
+ unsigned C;
+};
diff --git a/util/system/spinlock.cpp b/util/system/spinlock.cpp
new file mode 100644
index 0000000000..63a803a30e
--- /dev/null
+++ b/util/system/spinlock.cpp
@@ -0,0 +1 @@
+#include "spinlock.h"
diff --git a/util/system/spinlock.h b/util/system/spinlock.h
new file mode 100644
index 0000000000..aa81d1cf2e
--- /dev/null
+++ b/util/system/spinlock.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include "atomic.h"
+#include "spin_wait.h"
+
+class TSpinLockBase {
+protected:
+ inline TSpinLockBase() noexcept {
+ AtomicSet(Val_, 0);
+ }
+
+public:
+ inline bool IsLocked() const noexcept {
+ return AtomicGet(Val_);
+ }
+
+ inline bool TryAcquire() noexcept {
+ return AtomicTryLock(&Val_);
+ }
+
+ inline bool try_lock() noexcept {
+ return TryAcquire();
+ }
+
+protected:
+ TAtomic Val_;
+};
+
+static inline void SpinLockPause() {
+#if defined(__GNUC__) && (defined(_i386_) || defined(_x86_64_))
+ __asm __volatile("pause");
+#endif
+}
+
+static inline void AcquireSpinLock(TAtomic* l) {
+ if (!AtomicTryLock(l)) {
+ do {
+ SpinLockPause();
+ } while (!AtomicTryAndTryLock(l));
+ }
+}
+
+static inline void ReleaseSpinLock(TAtomic* l) {
+ AtomicUnlock(l);
+}
+
+/*
+ * You should almost always use TAdaptiveLock instead of TSpinLock
+ */
+class TSpinLock: public TSpinLockBase {
+public:
+ using TSpinLockBase::TSpinLockBase;
+
+ inline void Release() noexcept {
+ ReleaseSpinLock(&Val_);
+ }
+
+ inline void Acquire() noexcept {
+ AcquireSpinLock(&Val_);
+ }
+
+ inline void unlock() noexcept {
+ Release();
+ }
+
+ inline void lock() noexcept {
+ Acquire();
+ }
+};
+
+static inline void AcquireAdaptiveLock(TAtomic* l) {
+ if (!AtomicTryLock(l)) {
+ TSpinWait sw;
+
+ while (!AtomicTryAndTryLock(l)) {
+ sw.Sleep();
+ }
+ }
+}
+
+static inline void ReleaseAdaptiveLock(TAtomic* l) {
+ AtomicUnlock(l);
+}
+
+class TAdaptiveLock: public TSpinLockBase {
+public:
+ using TSpinLockBase::TSpinLockBase;
+
+ inline void Release() noexcept {
+ ReleaseAdaptiveLock(&Val_);
+ }
+
+ inline void Acquire() noexcept {
+ AcquireAdaptiveLock(&Val_);
+ }
+
+ inline void unlock() noexcept {
+ Release();
+ }
+
+ inline void lock() noexcept {
+ Acquire();
+ }
+};
+
+#include "guard.h"
+
+template <>
+struct TCommonLockOps<TAtomic> {
+ static inline void Acquire(TAtomic* v) noexcept {
+ AcquireAdaptiveLock(v);
+ }
+
+ static inline bool TryAcquire(TAtomic* v) noexcept {
+ return AtomicTryLock(v);
+ }
+
+ static inline void Release(TAtomic* v) noexcept {
+ ReleaseAdaptiveLock(v);
+ }
+};
diff --git a/util/system/spinlock_ut.cpp b/util/system/spinlock_ut.cpp
new file mode 100644
index 0000000000..e8639a6404
--- /dev/null
+++ b/util/system/spinlock_ut.cpp
@@ -0,0 +1,37 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "spinlock.h"
+
+Y_UNIT_TEST_SUITE(TSpinLock) {
+ template <typename TLock>
+ void TestLock() {
+ TLock lock;
+ UNIT_ASSERT(!lock.IsLocked());
+ lock.Acquire();
+ UNIT_ASSERT(lock.IsLocked());
+ lock.Release();
+ UNIT_ASSERT(!lock.IsLocked());
+
+ UNIT_ASSERT(lock.TryAcquire());
+ UNIT_ASSERT(lock.IsLocked());
+ UNIT_ASSERT(!lock.TryAcquire());
+ UNIT_ASSERT(lock.IsLocked());
+ lock.Release();
+ UNIT_ASSERT(!lock.IsLocked());
+
+ // Lockable requirements
+ lock.lock();
+ UNIT_ASSERT(lock.IsLocked());
+ UNIT_ASSERT(!lock.try_lock());
+ lock.unlock();
+ UNIT_ASSERT(!lock.IsLocked());
+ }
+
+ Y_UNIT_TEST(TSpinLock_IsLocked) {
+ TestLock<TSpinLock>();
+ }
+
+ Y_UNIT_TEST(TAdaptiveLock_IsLocked) {
+ TestLock<TAdaptiveLock>();
+ }
+}
diff --git a/util/system/src_location.cpp b/util/system/src_location.cpp
new file mode 100644
index 0000000000..13ff1f24b3
--- /dev/null
+++ b/util/system/src_location.cpp
@@ -0,0 +1,17 @@
+#include "src_location.h"
+
+#include <util/stream/output.h>
+
+#include <algorithm>
+
+template <>
+void Out<TSourceLocation>(IOutputStream& o, const TSourceLocation& t) {
+#if defined(_win_)
+ TString file(t.File);
+ std::replace(file.begin(), file.vend(), '\\', '/');
+ o << file;
+#else
+ o << t.File;
+#endif
+ o << ':' << t.Line;
+}
diff --git a/util/system/src_location.h b/util/system/src_location.h
new file mode 100644
index 0000000000..12ba6e063e
--- /dev/null
+++ b/util/system/src_location.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "src_root.h"
+
+#include <util/generic/strbuf.h>
+
+struct TSourceLocation {
+ constexpr TSourceLocation(const TStringBuf f, int l) noexcept
+ : File(f)
+ , Line(l)
+ {
+ }
+
+ TStringBuf File;
+ int Line;
+};
+
+// __SOURCE_FILE__ should be used instead of __FILE__
+#if !defined(__NVCC__)
+ #define __SOURCE_FILE__ (__SOURCE_FILE_IMPL__.As<TStringBuf>())
+#else
+ #define __SOURCE_FILE__ (__SOURCE_FILE_IMPL__.template As<TStringBuf>())
+#endif
+
+#define __LOCATION__ ::TSourceLocation(__SOURCE_FILE__, __LINE__)
diff --git a/util/system/src_location_ut.cpp b/util/system/src_location_ut.cpp
new file mode 100644
index 0000000000..5b86cb86ef
--- /dev/null
+++ b/util/system/src_location_ut.cpp
@@ -0,0 +1,18 @@
+#include "src_location.h"
+
+#include <util/string/builder.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+static inline TString GenLoc() {
+ return TStringBuilder() << __LOCATION__;
+}
+
+Y_UNIT_TEST_SUITE(TestLocation) {
+ Y_UNIT_TEST(Test1) {
+ UNIT_ASSERT_VALUES_EQUAL(GenLoc(), "util/system/src_location_ut.cpp:8");
+
+ static constexpr TSourceLocation location = __LOCATION__;
+ static_assert(location.Line >= 0, "__LOCATION__ can be used at compile time expressions");
+ }
+}
diff --git a/util/system/src_root.h b/util/system/src_root.h
new file mode 100644
index 0000000000..4f2d9f5ee6
--- /dev/null
+++ b/util/system/src_root.h
@@ -0,0 +1,68 @@
+#pragma once
+
+#include "compiler.h"
+#include "defaults.h"
+
+#include <type_traits>
+
+namespace NPrivate {
+ struct TStaticBuf {
+ constexpr TStaticBuf(const char* data, unsigned len) noexcept
+ : Data(data)
+ , Len(len)
+ {
+ }
+
+ template <class T>
+ constexpr T As() const noexcept {
+ return T(Data, Len);
+ }
+
+ template <class T>
+ constexpr operator T() const noexcept {
+ return this->As<T>();
+ }
+
+ const char* Data;
+ unsigned Len;
+ };
+
+#define STATIC_BUF(x) ::NPrivate::TStaticBuf(x, sizeof(x) - 1)
+
+ constexpr TStaticBuf ArcRoot = STATIC_BUF(Y_STRINGIZE(ARCADIA_ROOT));
+ constexpr TStaticBuf BuildRoot = STATIC_BUF(Y_STRINGIZE(ARCADIA_BUILD_ROOT));
+
+ constexpr Y_FORCE_INLINE bool IsProperPrefix(const TStaticBuf prefix, const TStaticBuf string) noexcept {
+ if (prefix.Len < string.Len) {
+ for (unsigned i = prefix.Len; i-- > 0;) {
+ if (prefix.Data[i] != string.Data[i]) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ constexpr unsigned RootPrefixLength(const TStaticBuf& f) noexcept {
+ if (IsProperPrefix(ArcRoot, f)) {
+ return ArcRoot.Len + 1;
+ }
+ if (IsProperPrefix(BuildRoot, f)) {
+ return BuildRoot.Len + 1;
+ }
+ return 0;
+ }
+
+ constexpr Y_FORCE_INLINE TStaticBuf StripRoot(const TStaticBuf& f, unsigned prefixLength) noexcept {
+ return TStaticBuf(f.Data + prefixLength, f.Len - prefixLength);
+ }
+
+ //$(SRC_ROOT)/prj/blah.cpp -> prj/blah.cpp
+ constexpr Y_FORCE_INLINE TStaticBuf StripRoot(const TStaticBuf& f) noexcept {
+ return StripRoot(f, RootPrefixLength(f));
+ }
+}
+
+#define __SOURCE_FILE_IMPL__ ::NPrivate::StripRoot(STATIC_BUF(__FILE__), std::integral_constant<unsigned, ::NPrivate::RootPrefixLength(STATIC_BUF(__FILE__))>::value)
diff --git a/util/system/src_root_ut.cpp b/util/system/src_root_ut.cpp
new file mode 100644
index 0000000000..e9a675eb9a
--- /dev/null
+++ b/util/system/src_root_ut.cpp
@@ -0,0 +1,27 @@
+#include "src_root.h"
+
+#include <util/folder/pathsplit.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TestSourceRoot) {
+ Y_UNIT_TEST(TestStrip) {
+ // Reconstruct() converts "\" -> "/" on Windows
+ const TString path = TPathSplit(__SOURCE_FILE_IMPL__.As<TStringBuf>()).Reconstruct();
+ UNIT_ASSERT_EQUAL(path, "util" LOCSLASH_S "system" LOCSLASH_S "src_root_ut.cpp");
+ }
+
+ Y_UNIT_TEST(TestPrivateChopPrefixRoutine) {
+ static constexpr const char str[] = ":\0:\0: It's unlikely that this string has an ARCADIA_ROOT as its prefix :\0:\0:";
+ static constexpr const auto strStaticBuf = STATIC_BUF(str);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(str, sizeof(str) - 1), ::NPrivate::StripRoot(strStaticBuf).As<TStringBuf>());
+ UNIT_ASSERT_VALUES_EQUAL(0, ::NPrivate::RootPrefixLength(strStaticBuf));
+
+ static_assert(::NPrivate::IsProperPrefix(STATIC_BUF("foo"), STATIC_BUF("foobar")), R"(IsProperPrefix("foo", "foobar") failed)");
+ static_assert(!::NPrivate::IsProperPrefix(STATIC_BUF("foobar"), STATIC_BUF("foo")), R"(IsProperPrefix("foobar", "foo") failed)");
+ static_assert(!::NPrivate::IsProperPrefix(STATIC_BUF("name"), STATIC_BUF("name")), R"(IsProperPrefix("name", "name") failed)");
+ static_assert(::NPrivate::IsProperPrefix(STATIC_BUF("name"), STATIC_BUF("name/")), R"(IsProperPrefix("name", "name/") failed)");
+ static_assert(::NPrivate::IsProperPrefix(STATIC_BUF(""), STATIC_BUF("foobar")), R"(IsProperPrefix("", "foobar") failed)");
+ static_assert(!::NPrivate::IsProperPrefix(STATIC_BUF(""), STATIC_BUF("")), R"(IsProperPrefix("", "") failed)");
+ static_assert(::NPrivate::IsProperPrefix(STATIC_BUF("dir"), STATIC_BUF("dir/file")), R"(IsProperPrefix("dir", "dir/file") failed)");
+ }
+}
diff --git a/util/system/sys_alloc.cpp b/util/system/sys_alloc.cpp
new file mode 100644
index 0000000000..45ed522667
--- /dev/null
+++ b/util/system/sys_alloc.cpp
@@ -0,0 +1 @@
+#include "sys_alloc.h"
diff --git a/util/system/sys_alloc.h b/util/system/sys_alloc.h
new file mode 100644
index 0000000000..4221a28f8c
--- /dev/null
+++ b/util/system/sys_alloc.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <util/system/compiler.h>
+
+#include <cstdlib>
+#include <new>
+
+inline void* y_allocate(size_t n) {
+ void* r = malloc(n);
+
+ if (r == nullptr) {
+ throw std::bad_alloc();
+ }
+
+ return r;
+}
+
+inline void y_deallocate(void* p) {
+ free(p);
+}
+
+/**
+ * Behavior of realloc from C++99 to C++11 changed (http://www.cplusplus.com/reference/cstdlib/realloc/).
+ *
+ * Our implementation work as C++99: if new_sz == 0 free will be called on 'p' and nullptr returned.
+ */
+inline void* y_reallocate(void* p, size_t new_sz) {
+ if (!new_sz) {
+ if (p) {
+ free(p);
+ }
+
+ return nullptr;
+ }
+
+ void* r = realloc(p, new_sz);
+
+ if (r == nullptr) {
+ throw std::bad_alloc();
+ }
+
+ return r;
+}
diff --git a/util/system/sysstat.cpp b/util/system/sysstat.cpp
new file mode 100644
index 0000000000..db3338b02e
--- /dev/null
+++ b/util/system/sysstat.cpp
@@ -0,0 +1,47 @@
+#include "sysstat.h"
+
+#ifdef _win_
+
+ #include "winint.h"
+ #include <errno.h>
+
+int Chmod(const char* fname, int mode) {
+ if (!fname) {
+ errno = EINVAL;
+ return -1;
+ }
+ ui32 fAttr = ::GetFileAttributesA(fname);
+ if (fAttr == 0xffffffff)
+ return -1;
+ if (mode & _S_IWRITE) {
+ fAttr &= ~FILE_ATTRIBUTE_READONLY;
+ } else {
+ fAttr |= FILE_ATTRIBUTE_READONLY;
+ }
+ if (!::SetFileAttributesA(fname, fAttr)) {
+ return -1;
+ }
+ return 0;
+}
+
+int Mkdir(const char* path, int /*mode*/) {
+ errno = 0;
+ if (!path) {
+ errno = EINVAL;
+ return -1;
+ }
+ if (!CreateDirectoryA(path, (LPSECURITY_ATTRIBUTES) nullptr)) {
+ ui32 errCode = GetLastError();
+ if (errCode == ERROR_ALREADY_EXISTS) {
+ errno = EEXIST;
+ } else if (errCode == ERROR_PATH_NOT_FOUND) {
+ errno = ENOENT;
+ } else {
+ errno = EINVAL;
+ }
+ return -1;
+ }
+ return 0;
+}
+
+#endif
diff --git a/util/system/sysstat.h b/util/system/sysstat.h
new file mode 100644
index 0000000000..b7c424c11b
--- /dev/null
+++ b/util/system/sysstat.h
@@ -0,0 +1,52 @@
+#pragma once
+
+#include "defaults.h"
+#include <sys/stat.h>
+
+#ifdef _win_
+ #define S_IRUSR _S_IREAD
+ #define S_IWUSR _S_IWRITE
+ #define S_IXUSR _S_IEXEC
+ #define S_IRWXU (S_IRUSR | S_IWUSR | S_IXUSR)
+
+ #define S_IRGRP _S_IREAD
+ #define S_IWGRP _S_IWRITE
+ #define S_IXGRP _S_IEXEC
+ #define S_IRWXG (S_IRGRP | S_IWGRP | S_IXGRP)
+
+ #define S_IROTH _S_IREAD
+ #define S_IWOTH _S_IWRITE
+ #define S_IXOTH _S_IEXEC
+ #define S_IRWXO (S_IROTH | S_IWOTH | S_IXOTH)
+#endif
+
+int Chmod(const char* fname, int mode);
+int Umask(int mode);
+
+static constexpr int MODE0777 = (S_IRWXU | S_IRWXG | S_IRWXO);
+static constexpr int MODE0775 = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+static constexpr int MODE0755 = (S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+
+int Mkdir(const char* path, int mode);
+
+/* uff... mkfifo(...) is not used now */
+
+#ifdef _unix_
+inline int Chmod(const char* fname, int mode) {
+ return ::chmod(fname, mode);
+}
+inline int Umask(int mode) {
+ return ::umask(mode);
+}
+inline int Mkdir(const char* path, int mode) {
+ return ::mkdir(path, mode);
+}
+#endif
+
+#ifdef _win_
+inline int Umask(int /*mode*/) {
+ /* The only thing this method could make is to set FILE_ATTRIBUTE_READONLY on a handle from 'int open(...)',
+ but open() is deprecated. */
+ return 0;
+}
+#endif
diff --git a/util/system/tempfile.cpp b/util/system/tempfile.cpp
new file mode 100644
index 0000000000..a2e9f49eb1
--- /dev/null
+++ b/util/system/tempfile.cpp
@@ -0,0 +1,25 @@
+#include "tempfile.h"
+
+TTempFileHandle::TTempFileHandle()
+ : TTempFile(MakeTempName())
+ , TFile(CreateFile())
+{
+}
+
+TTempFileHandle::TTempFileHandle(const TString& fname)
+ : TTempFile(fname)
+ , TFile(CreateFile())
+{
+}
+
+TTempFileHandle TTempFileHandle::InCurrentDir(const TString& filePrefix, const TString& extension) {
+ return TTempFileHandle(MakeTempName(".", filePrefix.c_str(), extension.c_str()));
+}
+
+TTempFileHandle TTempFileHandle::InDir(const TFsPath& dirPath, const TString& filePrefix, const TString& extension) {
+ return TTempFileHandle(MakeTempName(dirPath.c_str(), filePrefix.c_str(), extension.c_str()));
+}
+
+TFile TTempFileHandle::CreateFile() const {
+ return TFile(Name(), CreateAlways | RdWr);
+}
diff --git a/util/system/tempfile.h b/util/system/tempfile.h
new file mode 100644
index 0000000000..de249c129d
--- /dev/null
+++ b/util/system/tempfile.h
@@ -0,0 +1,50 @@
+#pragma once
+
+#include "fs.h"
+#include "file.h"
+
+#include <util/folder/path.h>
+#include <util/generic/string.h>
+
+class TTempFile {
+public:
+ inline TTempFile(const TString& fname)
+ : Name_(fname)
+ {
+ }
+
+ inline ~TTempFile() {
+ NFs::Remove(Name());
+ }
+
+ inline const TString& Name() const noexcept {
+ return Name_;
+ }
+
+private:
+ const TString Name_;
+};
+
+class TTempFileHandle: public TTempFile, public TFile {
+public:
+ TTempFileHandle();
+ TTempFileHandle(const TString& fname);
+
+ static TTempFileHandle InCurrentDir(const TString& filePrefix = "yandex", const TString& extension = "tmp");
+ static TTempFileHandle InDir(const TFsPath& dirPath, const TString& filePrefix = "yandex", const TString& extension = "tmp");
+
+private:
+ TFile CreateFile() const;
+};
+
+/*
+ * Creates a unique temporary filename in specified directory.
+ * If specified directory is NULL or empty, then system temporary directory is used.
+ *
+ * Note, that the function is not race-free, the file is guaranteed to exist at the time the function returns, but not at the time the returned name is first used.
+ * Throws TSystemError on error.
+ *
+ * Returned filepath has such format: dir/prefixXXXXXX.extension or dir/prefixXXXXXX
+ * But win32: dir/preXXXX.tmp (prefix is up to 3 characters, extension is always tmp).
+ */
+TString MakeTempName(const char* wrkDir = nullptr, const char* prefix = "yandex", const char* extension = "tmp");
diff --git a/util/system/tempfile_ut.cpp b/util/system/tempfile_ut.cpp
new file mode 100644
index 0000000000..e4a0923d0b
--- /dev/null
+++ b/util/system/tempfile_ut.cpp
@@ -0,0 +1,151 @@
+#include "tempfile.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/folder/dirut.h>
+#include <util/generic/yexception.h>
+#include <util/stream/file.h>
+
+#include <algorithm>
+
+Y_UNIT_TEST_SUITE(TTempFileHandle) {
+ Y_UNIT_TEST(Create) {
+ TString path;
+ {
+ TTempFileHandle tmp;
+ path = tmp.Name();
+ tmp.Write("hello world\n", 12);
+ tmp.FlushData();
+ UNIT_ASSERT_STRINGS_EQUAL(TUnbufferedFileInput(tmp.Name()).ReadAll(), "hello world\n");
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(InCurrentDir) {
+#ifndef _win32_
+ static const TString TEST_PREFIX = "unique_prefix";
+#else
+ static const TString TEST_PREFIX = "uni";
+#endif
+
+ TString path;
+ {
+ TTempFileHandle tmp = TTempFileHandle::InCurrentDir(TEST_PREFIX);
+ path = tmp.Name();
+ UNIT_ASSERT(NFs::Exists(path));
+
+ TVector<TString> names;
+ TFsPath(".").ListNames(names);
+ bool containsFileWithPrefix = std::any_of(names.begin(), names.end(), [&](const TString& name) {
+ return name.Contains(TEST_PREFIX);
+ });
+ UNIT_ASSERT(containsFileWithPrefix);
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(UseExtensionWithoutDot) {
+ TString path;
+ {
+ TTempFileHandle tmp = TTempFileHandle::InCurrentDir("hello", "world");
+ path = tmp.Name();
+ UNIT_ASSERT(NFs::Exists(path));
+
+#ifndef _win32_
+ UNIT_ASSERT(path.Contains("hello"));
+ UNIT_ASSERT(path.EndsWith(".world"));
+ UNIT_ASSERT(!path.EndsWith("..world"));
+#else
+ UNIT_ASSERT(path.Contains("hel"));
+ UNIT_ASSERT(path.EndsWith(".tmp"));
+#endif
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(UseExtensionWithDot) {
+ TString path;
+ {
+ TTempFileHandle tmp = TTempFileHandle::InCurrentDir("lorem", ".ipsum");
+ path = tmp.Name();
+ UNIT_ASSERT(NFs::Exists(path));
+
+#ifndef _win32_
+ UNIT_ASSERT(path.Contains("lorem"));
+ UNIT_ASSERT(path.EndsWith(".ipsum"));
+ UNIT_ASSERT(!path.EndsWith("..ipsum"));
+#else
+ UNIT_ASSERT(path.Contains("lor"));
+ UNIT_ASSERT(path.EndsWith(".tmp"));
+#endif
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(SafeDestructor) {
+ TString path;
+ {
+ path = MakeTempName();
+ UNIT_ASSERT(NFs::Exists(path));
+
+ TTempFileHandle tmp(path);
+ Y_UNUSED(tmp);
+ UNIT_ASSERT(NFs::Exists(path));
+
+ TTempFileHandle anotherTmp(path);
+ Y_UNUSED(anotherTmp);
+ UNIT_ASSERT(NFs::Exists(path));
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(RemovesOpen) {
+ TString path;
+ {
+ TTempFileHandle tmp;
+ path = tmp.Name();
+ tmp.Write("hello world\n", 12);
+ tmp.FlushData();
+ UNIT_ASSERT(NFs::Exists(path));
+ UNIT_ASSERT(tmp.IsOpen());
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(NonExistingDirectory) {
+ UNIT_ASSERT_EXCEPTION(TTempFileHandle::InDir("nonexsistingdirname"), TSystemError);
+ }
+}
+
+Y_UNIT_TEST_SUITE(MakeTempName) {
+ Y_UNIT_TEST(Default) {
+ TString path;
+ {
+ TTempFile tmp(MakeTempName());
+ path = tmp.Name();
+
+ UNIT_ASSERT(!path.Contains('\0'));
+ UNIT_ASSERT(NFs::Exists(path));
+ UNIT_ASSERT(path.EndsWith(".tmp"));
+
+#ifndef _win32_
+ UNIT_ASSERT(path.Contains("yandex"));
+#else
+ UNIT_ASSERT(path.Contains("yan"));
+#endif
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+
+ Y_UNIT_TEST(UseNullptr) {
+ TString path;
+ {
+ TTempFile tmp(MakeTempName(nullptr, nullptr, nullptr));
+ path = tmp.Name();
+
+ UNIT_ASSERT(!path.Contains('\0'));
+ UNIT_ASSERT(NFs::Exists(path));
+ }
+ UNIT_ASSERT(!NFs::Exists(path));
+ }
+}
diff --git a/util/system/thread.cpp b/util/system/thread.cpp
new file mode 100644
index 0000000000..6236746c2d
--- /dev/null
+++ b/util/system/thread.cpp
@@ -0,0 +1,557 @@
+#include "tls.h"
+#include "thread.h"
+#include "thread.i"
+
+#include <util/generic/ptr.h>
+#include <util/generic/ymath.h>
+#include <util/generic/ylimits.h>
+#include <util/generic/yexception.h>
+#include "yassert.h"
+#include <utility>
+
+#if defined(_glibc_)
+ #if !__GLIBC_PREREQ(2, 30)
+ #include <sys/syscall.h>
+ #endif
+#endif
+
+#if defined(_unix_)
+ #include <pthread.h>
+ #include <sys/types.h>
+#elif defined(_win_)
+ #include "dynlib.h"
+ #include <util/charset/wide.h>
+ #include <util/generic/scope.h>
+#else
+ #error "FIXME"
+#endif
+
+bool SetHighestThreadPriority() {
+#ifdef _win_
+ return SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST);
+#else
+ struct sched_param sch;
+ memset(&sch, 0, sizeof(sch));
+ sch.sched_priority = 31;
+ return pthread_setschedparam(pthread_self(), SCHED_RR, &sch) == 0;
+#endif
+}
+
+namespace {
+ using TParams = TThread::TParams;
+ using TId = TThread::TId;
+
+ inline void SetThrName(const TParams& p) {
+ try {
+ if (p.Name) {
+ TThread::SetCurrentThreadName(p.Name.data());
+ }
+ } catch (...) {
+ // ¯\_(ツ)_/¯
+ }
+ }
+
+ inline size_t StackSize(const TParams& p) noexcept {
+ if (p.StackSize) {
+ return FastClp2(p.StackSize);
+ }
+
+ return 0;
+ }
+
+#if defined(_win_)
+ class TWinThread {
+ struct TMyParams: public TParams, public TThrRefBase {
+ inline TMyParams(const TParams& p)
+ : TParams(p)
+ , Result(0)
+ {
+ }
+
+ void* Result;
+ };
+
+ using TParamsRef = TIntrusivePtr<TMyParams>;
+
+ public:
+ inline TWinThread(const TParams& params)
+ : P_(new TMyParams(params))
+ , Handle(0)
+ #if _WIN32_WINNT < 0x0502
+ , ThreadId(0)
+ #endif
+ {
+ }
+
+ inline bool Running() const noexcept {
+ return Handle != 0;
+ }
+
+ inline TId SystemThreadId() const noexcept {
+ #if _WIN32_WINNT < 0x0502
+ return (TId)ThreadId;
+ #else
+ return (TId)GetThreadId(Handle);
+ #endif
+ }
+
+ inline void* Join() {
+ ::WaitForSingleObject(Handle, INFINITE);
+ ::CloseHandle(Handle);
+
+ return P_->Result;
+ }
+
+ inline void Detach() {
+ ::CloseHandle(Handle);
+ }
+
+ static ui32 __stdcall Proxy(void* ptr) {
+ NTls::TCleaner cleaner;
+
+ (void)cleaner;
+
+ {
+ TParamsRef p((TMyParams*)(ptr));
+
+ //drop counter, gotten in Start()
+ p->UnRef();
+
+ SetThrName(*p);
+ p->Result = p->Proc(p->Data);
+ }
+
+ return 0;
+ }
+
+ inline void Start() {
+ //do not do this, kids, at home
+ P_->Ref();
+ #if _WIN32_WINNT < 0x0502
+ Handle = reinterpret_cast<HANDLE>(::_beginthreadex(nullptr, (unsigned)StackSize(*P_), Proxy, (void*)P_.Get(), 0, &ThreadId));
+ #else
+ Handle = reinterpret_cast<HANDLE>(::_beginthreadex(nullptr, (unsigned)StackSize(*P_), Proxy, (void*)P_.Get(), 0, nullptr));
+ #endif
+
+ if (!Handle) {
+ P_->UnRef();
+ ythrow yexception() << "failed to create a thread";
+ }
+ }
+
+ private:
+ TParamsRef P_;
+ HANDLE Handle;
+ #if _WIN32_WINNT < 0x0502
+ ui32 ThreadId;
+ #endif
+ };
+
+ using TThreadBase = TWinThread;
+#else
+ //unix
+
+ #define PCHECK(x, y) \
+ { \
+ const int err_ = x; \
+ if (err_) { \
+ ythrow TSystemError(err_) << TStringBuf(y); \
+ } \
+ }
+
+ class TPosixThread {
+ public:
+ inline TPosixThread(const TParams& params)
+ : P_(new TParams(params))
+ , H_()
+ {
+ static_assert(sizeof(H_) == sizeof(TId), "expect sizeof(H_) == sizeof(TId)");
+ }
+
+ inline TId SystemThreadId() const noexcept {
+ return (TId)H_;
+ }
+
+ inline void* Join() {
+ void* tec = nullptr;
+ PCHECK(pthread_join(H_, &tec), "can not join thread");
+
+ return tec;
+ }
+
+ inline void Detach() {
+ PCHECK(pthread_detach(H_), "can not detach thread");
+ }
+
+ inline bool Running() const noexcept {
+ return (bool)H_;
+ }
+
+ inline void Start() {
+ pthread_attr_t* pattrs = nullptr;
+ pthread_attr_t attrs;
+
+ if (P_->StackSize > 0) {
+ Zero(attrs);
+ pthread_attr_init(&attrs);
+ pattrs = &attrs;
+
+ if (P_->StackPointer) {
+ pthread_attr_setstack(pattrs, P_->StackPointer, P_->StackSize);
+ } else {
+ pthread_attr_setstacksize(pattrs, StackSize(*P_));
+ }
+ }
+
+ {
+ TParams* holdP = P_.Release();
+ int err = pthread_create(&H_, pattrs, ThreadProxy, holdP);
+ if (err) {
+ H_ = {};
+ P_.Reset(holdP);
+ PCHECK(err, "failed to create thread");
+ }
+ }
+ }
+
+ private:
+ static void* ThreadProxy(void* arg) {
+ THolder<TParams> p((TParams*)arg);
+
+ SetThrName(*p);
+
+ return p->Proc(p->Data);
+ }
+
+ private:
+ THolder<TParams> P_;
+ pthread_t H_;
+ };
+
+ #undef PCHECK
+
+ using TThreadBase = TPosixThread;
+#endif
+
+ template <class T>
+ static inline typename T::TValueType* Impl(T& t, const char* op, bool check = true) {
+ if (!t) {
+ ythrow yexception() << "can not " << op << " dead thread";
+ }
+
+ if (t->Running() != check) {
+ static const char* const msg[] = {"running", "not running"};
+
+ ythrow yexception() << "can not " << op << " " << msg[check] << " thread";
+ }
+
+ return t.Get();
+ }
+}
+
+class TThread::TImpl: public TThreadBase {
+public:
+ inline TImpl(const TParams& params, THolder<TCallableBase> callable = {})
+ : TThreadBase(params)
+ , Callable_(std::move(callable))
+ {
+ }
+
+ inline TId Id() const noexcept {
+ return ThreadIdHashFunction(SystemThreadId());
+ }
+
+ static THolder<TImpl> Create(THolder<TCallableBase> callable) {
+ TParams params(TCallableBase::ThreadWorker, callable.Get());
+ return MakeHolder<TImpl>(std::move(params), std::move(callable));
+ }
+
+private:
+ THolder<TCallableBase> Callable_;
+};
+
+TThread::TThread(const TParams& p)
+ : Impl_(new TImpl(p))
+{
+}
+
+TThread::TThread(TThreadProc threadProc, void* param)
+ : Impl_(new TImpl(TParams(threadProc, param)))
+{
+}
+
+TThread::TThread(TPrivateCtor, THolder<TCallableBase> callable)
+ : Impl_(TImpl::Create(std::move(callable)))
+{
+}
+
+TThread::~TThread() {
+ Join();
+}
+
+void TThread::Start() {
+ Impl(Impl_, "start", false)->Start();
+}
+
+void* TThread::Join() {
+ if (Running()) {
+ void* ret = Impl_->Join();
+
+ Impl_.Destroy();
+
+ return ret;
+ }
+
+ return nullptr;
+}
+
+void TThread::Detach() {
+ if (Running()) {
+ Impl_->Detach();
+ Impl_.Destroy();
+ }
+}
+
+bool TThread::Running() const noexcept {
+ return Impl_ && Impl_->Running();
+}
+
+TThread::TId TThread::Id() const noexcept {
+ if (Running()) {
+ return Impl_->Id();
+ }
+
+ return ImpossibleThreadId();
+}
+
+TThread::TId TThread::CurrentThreadId() noexcept {
+ return SystemCurrentThreadId();
+}
+
+TThread::TId TThread::CurrentThreadNumericId() noexcept {
+#if defined(_win_)
+ return GetCurrentThreadId();
+#elif defined(_darwin_)
+ // There is no gettid() on MacOS and SYS_gettid returns completely unrelated numbers.
+ // See: http://elliotth.blogspot.com/2012/04/gettid-on-mac-os.html
+ uint64_t threadId;
+ pthread_threadid_np(nullptr, &threadId);
+ return threadId;
+#elif defined(_musl_) || defined(_bionic_)
+ // both musl and android libc provide gettid() function
+ return gettid();
+#elif defined(_glibc_)
+ #if __GLIBC_PREREQ(2, 30)
+ return gettid();
+ #else
+ // gettid() was introduced in glibc=2.30, previous versions lack neat syscall wrapper
+ return syscall(SYS_gettid);
+ #endif
+#else
+ #error "Implement me"
+#endif
+}
+
+TThread::TId TThread::ImpossibleThreadId() noexcept {
+ return Max<TThread::TId>();
+}
+
+namespace {
+ template <class T>
+ static void* ThreadProcWrapper(void* param) {
+ return reinterpret_cast<T*>(param)->ThreadProc();
+ }
+}
+
+ISimpleThread::ISimpleThread(size_t stackSize)
+ : TThread(TParams(ThreadProcWrapper<ISimpleThread>, reinterpret_cast<void*>(this), stackSize))
+{
+}
+
+#if defined(_MSC_VER)
+ // This beautiful piece of code is borrowed from
+ // http://msdn.microsoft.com/en-us/library/xcb2z8hs.aspx
+
+ //
+ // Usage: WindowsCurrentSetThreadName (-1, "MainThread");
+ //
+ #include <windows.h>
+ #include <processthreadsapi.h>
+
+const DWORD MS_VC_EXCEPTION = 0x406D1388;
+
+ #pragma pack(push, 8)
+typedef struct tagTHREADNAME_INFO {
+ DWORD dwType; // Must be 0x1000.
+ LPCSTR szName; // Pointer to name (in user addr space).
+ DWORD dwThreadID; // Thread ID (-1=caller thread).
+ DWORD dwFlags; // Reserved for future use, must be zero.
+} THREADNAME_INFO;
+ #pragma pack(pop)
+
+static void WindowsCurrentSetThreadName(DWORD dwThreadID, const char* threadName) {
+ THREADNAME_INFO info;
+ info.dwType = 0x1000;
+ info.szName = threadName;
+ info.dwThreadID = dwThreadID;
+ info.dwFlags = 0;
+
+ __try {
+ RaiseException(MS_VC_EXCEPTION, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR*)&info);
+ } __except (EXCEPTION_EXECUTE_HANDLER) {
+ }
+}
+#endif
+
+#if defined(_win_)
+namespace {
+ struct TWinThreadDescrAPI {
+ TWinThreadDescrAPI()
+ : Kernel32Dll("kernel32.dll")
+ , SetThreadDescription((TSetThreadDescription)Kernel32Dll.SymOptional("SetThreadDescription"))
+ , GetThreadDescription((TGetThreadDescription)Kernel32Dll.SymOptional("GetThreadDescription"))
+ {
+ }
+
+ // This API is for Windows 10+ only:
+ // https://msdn.microsoft.com/en-us/library/windows/desktop/mt774972(v=vs.85).aspx
+ bool HasAPI() noexcept {
+ return SetThreadDescription && GetThreadDescription;
+ }
+
+ // Should always succeed, unless something very strange is passed in `descr'
+ void SetDescr(const char* descr) {
+ auto hr = SetThreadDescription(GetCurrentThread(), (const WCHAR*)UTF8ToWide(descr).data());
+ Y_VERIFY(SUCCEEDED(hr), "SetThreadDescription failed");
+ }
+
+ TString GetDescr() {
+ PWSTR wideName;
+ auto hr = GetThreadDescription(GetCurrentThread(), &wideName);
+ Y_VERIFY(SUCCEEDED(hr), "GetThreadDescription failed");
+ Y_DEFER {
+ LocalFree(wideName);
+ };
+ return WideToUTF8((const wchar16*)wideName);
+ }
+
+ typedef HRESULT(__cdecl* TSetThreadDescription)(HANDLE hThread, PCWSTR lpThreadDescription);
+ typedef HRESULT(__cdecl* TGetThreadDescription)(HANDLE hThread, PWSTR* ppszThreadDescription);
+
+ TDynamicLibrary Kernel32Dll;
+ TSetThreadDescription SetThreadDescription;
+ TGetThreadDescription GetThreadDescription;
+ };
+}
+#endif // _win_
+
+void TThread::SetCurrentThreadName(const char* name) {
+ (void)name;
+
+#if defined(_freebsd_)
+ pthread_t thread = pthread_self();
+ pthread_set_name_np(thread, name);
+#elif defined(_linux_)
+ prctl(PR_SET_NAME, name, 0, 0, 0);
+#elif defined(_darwin_)
+ pthread_setname_np(name);
+#elif defined(_win_)
+ auto api = Singleton<TWinThreadDescrAPI>();
+ if (api->HasAPI()) {
+ api->SetDescr(name);
+ } else {
+ #if defined(_MSC_VER)
+ WindowsCurrentSetThreadName(DWORD(-1), name);
+ #endif
+ }
+#else
+// no idea
+#endif // OS
+}
+
+TString TThread::CurrentThreadName() {
+#if defined(_freebsd_)
+// TODO: check pthread_get_name_np API availability
+#elif defined(_linux_)
+ // > The buffer should allow space for up to 16 bytes; the returned string will be
+ // > null-terminated.
+ // via `man prctl`
+ char name[16];
+ memset(name, 0, sizeof(name));
+ Y_VERIFY(prctl(PR_GET_NAME, name, 0, 0, 0) == 0, "pctl failed: %s", strerror(errno));
+ return name;
+#elif defined(_darwin_)
+ // available on Mac OS 10.6+
+ const auto thread = pthread_self();
+ char name[256];
+ memset(name, 0, sizeof(name));
+ Y_VERIFY(pthread_getname_np(thread, name, sizeof(name)) == 0, "pthread_getname_np failed: %s", strerror(errno));
+ return name;
+#elif defined(_win_)
+ auto api = Singleton<TWinThreadDescrAPI>();
+ if (api->HasAPI()) {
+ return api->GetDescr();
+ }
+ return {};
+#else
+// no idea
+#endif // OS
+
+ return {};
+}
+
+bool TThread::CanGetCurrentThreadName() {
+#if defined(_linux_) || defined(_darwin_)
+ return true;
+#elif defined(_win_)
+ return Singleton<TWinThreadDescrAPI>()->HasAPI();
+#else
+ return false;
+#endif // OS
+}
+
+TCurrentThreadLimits::TCurrentThreadLimits() noexcept
+ : StackBegin(nullptr)
+ , StackLength(0)
+{
+#if defined(_linux_) || defined(_cygwin_) || defined(_freebsd_)
+ pthread_attr_t attr;
+ pthread_attr_init(&attr);
+
+ #if defined(_linux_) || defined(_cygwin_)
+ Y_VERIFY(pthread_getattr_np(pthread_self(), &attr) == 0, "pthread_getattr failed");
+ #else
+ Y_VERIFY(pthread_attr_get_np(pthread_self(), &attr) == 0, "pthread_attr_get_np failed");
+ #endif
+ pthread_attr_getstack(&attr, (void**)&StackBegin, &StackLength);
+ pthread_attr_destroy(&attr);
+
+#elif defined(_darwin_)
+ StackBegin = pthread_get_stackaddr_np(pthread_self());
+ StackLength = pthread_get_stacksize_np(pthread_self());
+#elif defined(_MSC_VER)
+
+ #if _WIN32_WINNT >= _WIN32_WINNT_WIN8
+ ULONG_PTR b = 0;
+ ULONG_PTR e = 0;
+
+ GetCurrentThreadStackLimits(&b, &e);
+
+ StackBegin = (const void*)b;
+ StackLength = e - b;
+
+ #else
+ // Copied from https://github.com/llvm-mirror/compiler-rt/blob/release_40/lib/sanitizer_common/sanitizer_win.cc#L91
+ void* place_on_stack = alloca(16);
+ MEMORY_BASIC_INFORMATION memory_info;
+ Y_VERIFY(VirtualQuery(place_on_stack, &memory_info, sizeof(memory_info)));
+
+ StackBegin = memory_info.AllocationBase;
+ StackLength = static_cast<const char*>(memory_info.BaseAddress) + memory_info.RegionSize - static_cast<const char*>(StackBegin);
+
+ #endif
+
+#else
+ #error port me
+#endif
+}
diff --git a/util/system/thread.h b/util/system/thread.h
new file mode 100644
index 0000000000..a6e8abdb5b
--- /dev/null
+++ b/util/system/thread.h
@@ -0,0 +1,174 @@
+#pragma once
+
+/// This code should not be used directly unless you really understand what you do.
+/// If you need threads, use thread pool functionality in <util/thread/factory.h>
+/// @see SystemThreadFactory()
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
+#include "defaults.h"
+#include "progname.h"
+
+bool SetHighestThreadPriority();
+
+class TThread {
+ template <typename Callable>
+ struct TCallableParams;
+ struct TPrivateCtor {};
+
+public:
+ using TThreadProc = void* (*)(void*);
+ using TId = size_t;
+
+ struct TParams {
+ TThreadProc Proc;
+ void* Data;
+ size_t StackSize;
+ void* StackPointer;
+ // See comments for `SetCurrentThreadName`
+ TString Name = GetProgramName();
+
+ inline TParams()
+ : Proc(nullptr)
+ , Data(nullptr)
+ , StackSize(0)
+ , StackPointer(nullptr)
+ {
+ }
+
+ inline TParams(TThreadProc proc, void* data)
+ : Proc(proc)
+ , Data(data)
+ , StackSize(0)
+ , StackPointer(nullptr)
+ {
+ }
+
+ inline TParams(TThreadProc proc, void* data, size_t stackSize)
+ : Proc(proc)
+ , Data(data)
+ , StackSize(stackSize)
+ , StackPointer(nullptr)
+ {
+ }
+
+ inline TParams& SetName(const TString& name) noexcept {
+ Name = name;
+
+ return *this;
+ }
+
+ inline TParams& SetStackSize(size_t size) noexcept {
+ StackSize = size;
+
+ return *this;
+ }
+
+ inline TParams& SetStackPointer(void* ptr) noexcept {
+ StackPointer = ptr;
+
+ return *this;
+ }
+ };
+
+ TThread(const TParams& params);
+ TThread(TThreadProc threadProc, void* param);
+
+ template <typename Callable>
+ TThread(Callable&& callable)
+ : TThread(TPrivateCtor{},
+ MakeHolder<TCallableParams<Callable>>(std::forward<Callable>(callable)))
+ {
+ }
+
+ TThread(TParams&& params)
+ : TThread((const TParams&)params)
+ {
+ }
+
+ TThread(TParams& params)
+ : TThread((const TParams&)params)
+ {
+ }
+
+ ~TThread();
+
+ void Start();
+
+ void* Join();
+ void Detach();
+ bool Running() const noexcept;
+ TId Id() const noexcept;
+
+ static TId ImpossibleThreadId() noexcept;
+ static TId CurrentThreadId() noexcept;
+
+ /*
+ * Returns numeric thread id, as visible in e. g. htop.
+ * Consider using this value for logging.
+ */
+ static TId CurrentThreadNumericId() noexcept;
+
+ // NOTE: Content of `name` will be copied.
+ //
+ // NOTE: On Linux thread name is limited to 15 symbols which is probably the smallest one among
+ // all platforms. If you provide a name longer than 15 symbols it will be cut. So if you expect
+ // `CurrentThreadName` to return the same name as `name` make sure it's not longer than 15
+ // symbols.
+ static void SetCurrentThreadName(const char* name);
+
+ // NOTE: Will return empty string where CanGetCurrentThreadName() returns false.
+ static TString CurrentThreadName();
+
+ // NOTE: Depends on a platform version.
+ // Will return true for Darwin, Linux or fresh Windows 10.
+ static bool CanGetCurrentThreadName();
+
+private:
+ struct TCallableBase {
+ virtual ~TCallableBase() = default;
+ virtual void run() = 0;
+
+ static void* ThreadWorker(void* arg) {
+ static_cast<TCallableBase*>(arg)->run();
+ return nullptr;
+ }
+ };
+
+ template <typename Callable>
+ struct TCallableParams: public TCallableBase {
+ TCallableParams(Callable&& callable)
+ : Callable_(std::forward<Callable>(callable))
+ {
+ }
+
+ Callable Callable_;
+
+ void run() override {
+ Callable_();
+ }
+ };
+
+ TThread(TPrivateCtor, THolder<TCallableBase> callable);
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+class ISimpleThread: public TThread {
+public:
+ ISimpleThread(size_t stackSize = 0);
+
+ virtual ~ISimpleThread() = default;
+
+ virtual void* ThreadProc() = 0;
+};
+
+struct TCurrentThreadLimits {
+ TCurrentThreadLimits() noexcept;
+
+ const void* StackBegin;
+ size_t StackLength;
+};
diff --git a/util/system/thread.i b/util/system/thread.i
new file mode 100644
index 0000000000..8cba505473
--- /dev/null
+++ b/util/system/thread.i
@@ -0,0 +1,52 @@
+//do not use directly
+#pragma once
+#include "platform.h"
+
+#if defined(_win_)
+ #include "winint.h"
+ #include <process.h>
+
+ typedef HANDLE THREADHANDLE;
+#else
+ #include <pthread.h>
+ #include <sched.h>
+ #include <errno.h>
+ #include <string.h>
+
+ typedef pthread_t THREADHANDLE;
+#endif
+
+#if defined(_freebsd_)
+ #include <pthread_np.h>
+#elif defined(_linux_)
+ #include <sys/prctl.h>
+#endif
+
+#include <util/digest/numeric.h>
+
+static inline size_t SystemCurrentThreadIdImpl() noexcept {
+ #if defined(_unix_)
+ return (size_t)pthread_self();
+ #elif defined(_win_)
+ return (size_t)GetCurrentThreadId();
+ #else
+ #error todo
+ #endif
+}
+
+template <class T>
+static inline T ThreadIdHashFunction(T t) noexcept {
+ /*
+ * we must permute threadid bits, because some strange platforms(such Linux)
+ * have strange threadid numeric properties
+ *
+ * Because they are alligned pointers to pthread_t rather that tid.
+ * Currently there is no way to get tid without syscall (slightly slower)
+ * (pthread_getthreadid_np is not implemeted in glibc/musl for some reason).
+ */
+ return IntHash(t);
+}
+
+static inline size_t SystemCurrentThreadId() noexcept {
+ return ThreadIdHashFunction(SystemCurrentThreadIdImpl());
+}
diff --git a/util/system/thread_ut.cpp b/util/system/thread_ut.cpp
new file mode 100644
index 0000000000..d820898fd5
--- /dev/null
+++ b/util/system/thread_ut.cpp
@@ -0,0 +1,229 @@
+#include "thread.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <atomic>
+
+Y_UNIT_TEST_SUITE(TSysThreadTest) {
+ struct TIdTester {
+ inline TIdTester()
+ : Thr(nullptr)
+ , Cur(0)
+ , Real(0)
+ {
+ }
+
+ static inline void* DoRun(void* ptr) {
+ ((TIdTester*)ptr)->Run();
+
+ return nullptr;
+ }
+
+ inline void Run() {
+ Cur = TThread::CurrentThreadId();
+ Real = Thr->Id();
+ Numeric = TThread::CurrentThreadNumericId();
+ }
+
+ TThread* Thr;
+ TThread::TId Cur;
+ TThread::TId Real;
+ TThread::TId Numeric;
+ };
+
+ Y_UNIT_TEST(TestThreadId) {
+ TIdTester tst;
+ TThread thr(tst.DoRun, &tst);
+
+ tst.Thr = &thr;
+
+ thr.Start();
+ thr.Join();
+
+ UNIT_ASSERT_EQUAL(tst.Cur, tst.Real);
+ UNIT_ASSERT(tst.Cur != 0);
+ UNIT_ASSERT(tst.Numeric != 0);
+ UNIT_ASSERT(tst.Numeric != tst.Real);
+ }
+
+ void* ThreadProc(void*) {
+ TThread::SetCurrentThreadName("CurrentThreadSetNameTest");
+ return nullptr;
+ }
+
+ void* ThreadProc2(void*) {
+ return nullptr;
+ }
+
+ void* ThreadProc3(void*) {
+ const auto name = TThread::CurrentThreadName();
+ Y_FAKE_READ(name);
+ return nullptr;
+ }
+
+ void* ThreadProc4(void*) {
+ const TString setName = "ThreadName";
+ TThread::SetCurrentThreadName(setName.data());
+
+ const auto getName = TThread::CurrentThreadName();
+ if (TThread::CanGetCurrentThreadName()) {
+ UNIT_ASSERT_VALUES_EQUAL(setName, getName);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL("", getName);
+ }
+ return nullptr;
+ }
+
+ void* ThreadProcChild(void*) {
+ const auto name = TThread::CurrentThreadName();
+ const auto defaultName = GetProgramName();
+
+ (void)name;
+ (void)defaultName;
+
+#if defined(_darwin_) || defined(_linux_)
+ UNIT_ASSERT_VALUES_EQUAL(name, defaultName);
+#endif
+ return nullptr;
+ }
+
+ void* ThreadProcParent(void*) {
+ const TString setName = "Parent";
+ TThread::SetCurrentThreadName(setName.data());
+
+ TThread thread(&ThreadProcChild, nullptr);
+
+ thread.Start();
+ thread.Join();
+
+ const auto getName = TThread::CurrentThreadName();
+ if (TThread::CanGetCurrentThreadName()) {
+ UNIT_ASSERT_VALUES_EQUAL(setName, getName);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL("", getName);
+ }
+ return nullptr;
+ }
+
+ Y_UNIT_TEST(TestSetThreadName) {
+ TThread thread(&ThreadProc, nullptr);
+ // just check it doesn't crash
+ thread.Start();
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestSetThreadName2) {
+ TThread thread(TThread::TParams(&ThreadProc, nullptr, 0).SetName("XXX"));
+
+ thread.Start();
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestGetThreadName) {
+ TThread thread(&ThreadProc3, nullptr);
+ thread.Start();
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestSetGetThreadName) {
+ TThread thread(&ThreadProc4, nullptr);
+ thread.Start();
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestSetGetThreadNameInChildThread) {
+ TThread thread(&ThreadProcParent, nullptr);
+ thread.Start();
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestDoubleJoin) {
+ TThread thread(&ThreadProc, nullptr);
+
+ thread.Start();
+ thread.Join();
+
+ UNIT_ASSERT_EQUAL(thread.Join(), nullptr);
+ }
+
+ Y_UNIT_TEST(TestDoubleStart) {
+ TThread thread(&ThreadProc, nullptr);
+
+ thread.Start();
+ UNIT_ASSERT_EXCEPTION(thread.Start(), yexception);
+ thread.Join();
+ }
+
+ Y_UNIT_TEST(TestNoStart) {
+ TThread thread(&ThreadProc, nullptr);
+ }
+
+ Y_UNIT_TEST(TestNoStartJoin) {
+ TThread thread(&ThreadProc, nullptr);
+
+ UNIT_ASSERT_EQUAL(thread.Join(), nullptr);
+ }
+
+ Y_UNIT_TEST(TestStackPointer) {
+ TArrayHolder<char> buf(new char[64000]);
+ TThread thr(TThread::TParams(ThreadProc2, nullptr).SetStackPointer(buf.Get()).SetStackSize(64000));
+
+ thr.Start();
+ UNIT_ASSERT_VALUES_EQUAL(thr.Join(), nullptr);
+ }
+
+ Y_UNIT_TEST(TestStackLimits) {
+ TCurrentThreadLimits sl;
+
+ UNIT_ASSERT(sl.StackBegin);
+ UNIT_ASSERT(sl.StackLength > 0);
+ }
+
+ Y_UNIT_TEST(TestFunc) {
+ std::atomic_bool flag = {false};
+ TThread thread([&flag]() { flag = true; });
+
+ thread.Start();
+ UNIT_ASSERT_VALUES_EQUAL(thread.Join(), nullptr);
+ UNIT_ASSERT(flag);
+ }
+
+ Y_UNIT_TEST(TestCopyFunc) {
+ std::atomic_bool flag = {false};
+ auto func = [&flag]() { flag = true; };
+
+ TThread thread(func);
+ thread.Start();
+ UNIT_ASSERT_VALUES_EQUAL(thread.Join(), nullptr);
+
+ TThread thread2(func);
+ thread2.Start();
+ UNIT_ASSERT_VALUES_EQUAL(thread2.Join(), nullptr);
+
+ UNIT_ASSERT(flag);
+ }
+
+ Y_UNIT_TEST(TestCallable) {
+ std::atomic_bool flag = {false};
+
+ struct TCallable: TMoveOnly {
+ std::atomic_bool* Flag_;
+
+ TCallable(std::atomic_bool* flag)
+ : Flag_(flag)
+ {
+ }
+
+ void operator()() {
+ *Flag_ = true;
+ }
+ };
+
+ TCallable foo(&flag);
+ TThread thread(std::move(foo));
+
+ thread.Start();
+ UNIT_ASSERT_VALUES_EQUAL(thread.Join(), nullptr);
+ UNIT_ASSERT(flag);
+ }
+};
diff --git a/util/system/tls.cpp b/util/system/tls.cpp
new file mode 100644
index 0000000000..c2f1a04a14
--- /dev/null
+++ b/util/system/tls.cpp
@@ -0,0 +1,260 @@
+#include "tls.h"
+#include "mutex.h"
+#include "thread.h"
+
+#include <util/generic/set.h>
+#include <util/generic/hash.h>
+#include <util/generic/intrlist.h>
+#include <util/generic/singleton.h>
+#include <util/generic/vector.h>
+
+#if defined(_unix_)
+ #include <pthread.h>
+#endif
+
+using namespace NTls;
+
+namespace {
+ static inline TAtomicBase AcquireKey() {
+ static TAtomic cur;
+
+ return AtomicIncrement(cur) - (TAtomicBase)1;
+ }
+
+ class TGenericTlsBase {
+ public:
+ using TSmallKey = size_t;
+
+ class TPerThreadStorage {
+ public:
+ struct TKey: public TNonCopyable {
+ inline TKey(TDtor dtor)
+ : Key(AcquireKey())
+ , Dtor(dtor)
+ {
+ }
+
+ TSmallKey Key;
+ TDtor Dtor;
+ };
+
+ class TStoredValue: public TIntrusiveListItem<TStoredValue> {
+ public:
+ inline TStoredValue(const TKey* key)
+ : Data_(nullptr)
+ , Dtor_(key->Dtor)
+ {
+ }
+
+ inline ~TStoredValue() {
+ if (Dtor_ && Data_) {
+ Dtor_(Data_);
+ }
+ }
+
+ inline void Set(void* ptr) noexcept {
+ Data_ = ptr;
+ }
+
+ inline void* Get() const noexcept {
+ return Data_;
+ }
+
+ private:
+ void* Data_;
+ TDtor Dtor_;
+ };
+
+ inline TStoredValue* Value(const TKey* key) {
+ TStoredValue*& ret = *ValuePtr((size_t)key->Key);
+
+ if (!ret) {
+ THolder<TStoredValue> sv(new TStoredValue(key));
+
+ Storage_.PushFront(sv.Get());
+ ret = sv.Release();
+ }
+
+ return ret;
+ }
+
+ inline TStoredValue** ValuePtr(size_t idx) {
+ // do not grow vector too much
+ if (idx < 10000) {
+ if (idx >= Values_.size()) {
+ Values_.resize(idx + 1);
+ }
+
+ return &Values_[idx];
+ }
+
+ return &FarValues_[idx];
+ }
+
+ private:
+ TVector<TStoredValue*> Values_;
+ THashMap<size_t, TStoredValue*> FarValues_;
+ TIntrusiveListWithAutoDelete<TStoredValue, TDelete> Storage_;
+ };
+
+ inline TPerThreadStorage* MyStorage() {
+#if defined(Y_HAVE_FAST_POD_TLS)
+ Y_POD_STATIC_THREAD(TPerThreadStorage*)
+ my(nullptr);
+
+ if (!my) {
+ my = MyStorageSlow();
+ }
+
+ return my;
+#else
+ return MyStorageSlow();
+#endif
+ }
+
+ virtual TPerThreadStorage* MyStorageSlow() = 0;
+
+ virtual ~TGenericTlsBase() = default;
+ };
+}
+
+#if defined(_unix_)
+namespace {
+ class TMasterTls: public TGenericTlsBase {
+ public:
+ inline TMasterTls() {
+ Y_VERIFY(!pthread_key_create(&Key_, Dtor), "pthread_key_create failed");
+ }
+
+ inline ~TMasterTls() override {
+ //explicitly call dtor for main thread
+ Dtor(pthread_getspecific(Key_));
+
+ Y_VERIFY(!pthread_key_delete(Key_), "pthread_key_delete failed");
+ }
+
+ static inline TMasterTls* Instance() {
+ return SingletonWithPriority<TMasterTls, 1>();
+ }
+
+ private:
+ TPerThreadStorage* MyStorageSlow() override {
+ void* ret = pthread_getspecific(Key_);
+
+ if (!ret) {
+ ret = new TPerThreadStorage();
+
+ Y_VERIFY(!pthread_setspecific(Key_, ret), "pthread_setspecific failed");
+ }
+
+ return (TPerThreadStorage*)ret;
+ }
+
+ static void Dtor(void* ptr) {
+ delete (TPerThreadStorage*)ptr;
+ }
+
+ private:
+ pthread_key_t Key_;
+ };
+
+ using TKeyDescriptor = TMasterTls::TPerThreadStorage::TKey;
+}
+
+class TKey::TImpl: public TKeyDescriptor {
+public:
+ inline TImpl(TDtor dtor)
+ : TKeyDescriptor(dtor)
+ {
+ }
+
+ inline void* Get() const {
+ return TMasterTls::Instance()->MyStorage()->Value(this)->Get();
+ }
+
+ inline void Set(void* val) const {
+ TMasterTls::Instance()->MyStorage()->Value(this)->Set(val);
+ }
+
+ static inline void Cleanup() {
+ }
+};
+#else
+namespace {
+ class TGenericTls: public TGenericTlsBase {
+ public:
+ virtual TPerThreadStorage* MyStorageSlow() {
+ auto lock = Guard(Lock_);
+
+ {
+ TPTSRef& ret = Datas_[TThread::CurrentThreadId()];
+
+ if (!ret) {
+ ret.Reset(new TPerThreadStorage());
+ }
+
+ return ret.Get();
+ }
+ }
+
+ inline void Cleanup() noexcept {
+ with_lock (Lock_) {
+ Datas_.erase(TThread::CurrentThreadId());
+ }
+ }
+
+ static inline TGenericTls* Instance() {
+ return SingletonWithPriority<TGenericTls, 1>();
+ }
+
+ private:
+ using TPTSRef = THolder<TPerThreadStorage>;
+ TMutex Lock_;
+ THashMap<TThread::TId, TPTSRef> Datas_;
+ };
+}
+
+class TKey::TImpl {
+public:
+ inline TImpl(TDtor dtor)
+ : Key_(dtor)
+ {
+ }
+
+ inline void* Get() {
+ return TGenericTls::Instance()->MyStorage()->Value(&Key_)->Get();
+ }
+
+ inline void Set(void* ptr) {
+ TGenericTls::Instance()->MyStorage()->Value(&Key_)->Set(ptr);
+ }
+
+ static inline void Cleanup() {
+ TGenericTls::Instance()->Cleanup();
+ }
+
+private:
+ TGenericTls::TPerThreadStorage::TKey Key_;
+};
+#endif
+
+TKey::TKey(TDtor dtor)
+ : Impl_(new TImpl(dtor))
+{
+}
+
+TKey::TKey(TKey&&) noexcept = default;
+
+TKey::~TKey() = default;
+
+void* TKey::Get() const {
+ return Impl_->Get();
+}
+
+void TKey::Set(void* ptr) const {
+ Impl_->Set(ptr);
+}
+
+void TKey::Cleanup() noexcept {
+ TImpl::Cleanup();
+}
diff --git a/util/system/tls.h b/util/system/tls.h
new file mode 100644
index 0000000000..3c4f56dbeb
--- /dev/null
+++ b/util/system/tls.h
@@ -0,0 +1,307 @@
+#pragma once
+
+#include "defaults.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+
+#include <new>
+
+#if defined(_darwin_)
+ #define Y_DISABLE_THRKEY_OPTIMIZATION
+#endif
+
+#if defined(_arm_) && defined(_linux_)
+ #define Y_DISABLE_THRKEY_OPTIMIZATION
+#endif
+
+#if defined(__GNUC__) && defined(__ANDROID__) && defined(__i686__) // https://st.yandex-team.ru/DEVTOOLS-3352
+ #define Y_DISABLE_THRKEY_OPTIMIZATION
+#endif
+
+/**
+ @def Y_THREAD(TType)
+
+ A thread-local wrapper for a given class. Suitable for POD and classes with a constructor with a single argument.
+
+ The wrapper can be treated as the original class in many cases, as it has the same signature for the constructor and an implicit cast to the origianl class.
+
+ Has methods :
+ - implicit caster to TType
+ - TType& Get()
+ - TType* GetPtr()
+
+ Time complexity: getting a variable takes O(number of threads where the variable has been constructed)
+
+ Memory usage: O(number of threads where the variable has been constructed)
+
+ Best practices:
+ - storing singletons won't result in heavy memory overheads
+ - storing pointers allows complex constructors as well as lazy constructions
+ - storing static variables won't result in heavy memory overheads
+
+ Possibly bad practices:
+ - field in a class with numerous instances and numerous threads will result in slow working and memory overheads
+
+ Example:
+ @code
+ //the field declaration in header
+ Y_THREAD(TBuffer) TmpBuffer;
+ //...later somewhere in cpp...
+ TmpBuffer.Clear();
+ for (size_t i = 0; i < sz && TrieCursor[i].second.IsFork(); ++i) {
+ TmpBuffer.Append(TrieCursor[i].second.Char);
+ }
+ @endcode
+
+ Example:
+ @code
+ //the field decalrataion in header
+ Y_THREAD(TMyWriter*) ThreadLocalWriter;
+ //...later somewhere in cpp...
+ TMyWriter*& writerRef = ThreadLocalWriter.Get();
+ if (writerRef == nullptr) {
+ THolder<TMyWriter> threadLocalWriter( new TMyWriter(
+ *Session,
+ MinLogError,
+ MaxRps,
+ LogFraction,
+ WriteCounters,
+ Log));
+ writerRef = threadLocalWriter.Get();
+ }
+ @endcode
+
+ Example:
+ @code
+ //in header
+ namespace TMorph {
+ Y_THREAD(ELanguage) ThreadLocalMainLanguage;
+ }
+ //in cpp
+ Y_THREAD(ELanguage) TMorph::ThreadLocalMainLanguage(LANG_RUS);
+ @endcode
+
+ Example:
+ @code
+ Y_THREAD(TScoreCalcer*) ScoreCalcerPtr;
+ static TScoreCalcer* GetScoreCalcer(yint maxElemCount) {
+ if (ScoreCalcerPtr == 0) {
+ ScoreCalcerPtr = new TScoreCalcer();
+ ScoreCalcerPtr->Alloc(maxElemCount);
+ }
+ return ScoreCalcerPtr;
+ }
+ @endcode
+
+ @param TType POD or a class with a constructor taking 1 argument
+**/
+
+/**
+ @def Y_STATIC_THREAD(TType)
+
+ Equivalent to "static Y_THREAD(TType)"
+
+ @see Y_THREAD(TType)
+**/
+
+/**
+ @def Y_POD_THREAD(TType)
+
+ Same interface as Y_THREAD(TType), but TType must be a POD.
+ Implemented (based on the compiler) as Y_THREAD(TType) or as native tls.
+
+ @see Y_THREAD(TType)
+**/
+
+/**
+ @def STATIC_POD_THREAD(TType)
+
+ Equivalent to "static Y_POD_THREAD(TType)"
+
+ @see Y_POD_THREAD(TType)
+**/
+
+#define Y_THREAD(T) ::NTls::TValue<T>
+#define Y_STATIC_THREAD(T) static Y_THREAD(T)
+
+// gcc and msvc support automatic tls for POD types
+#if defined(Y_DISABLE_THRKEY_OPTIMIZATION)
+// nothing to do
+#elif defined(__clang__)
+ #define Y_POD_THREAD(T) thread_local T
+ #define Y_POD_STATIC_THREAD(T) static thread_local T
+#elif defined(__GNUC__) && !defined(_cygwin_) && !defined(_arm_) && !defined(__IOS_SIMULATOR__)
+ #define Y_POD_THREAD(T) __thread T
+ #define Y_POD_STATIC_THREAD(T) static __thread T
+// msvc doesn't support __declspec(thread) in dlls, loaded manually (via LoadLibrary)
+#elif (defined(_MSC_VER) && !defined(_WINDLL)) || defined(_arm_)
+ #define Y_POD_THREAD(T) __declspec(thread) T
+ #define Y_POD_STATIC_THREAD(T) __declspec(thread) static T
+#endif
+
+#if !defined(Y_POD_THREAD) || !defined(Y_POD_STATIC_THREAD)
+ #define Y_POD_THREAD(T) Y_THREAD(T)
+ #define Y_POD_STATIC_THREAD(T) Y_STATIC_THREAD(T)
+#else
+ #define Y_HAVE_FAST_POD_TLS
+#endif
+
+namespace NPrivate {
+ void FillWithTrash(void* ptr, size_t len);
+}
+
+namespace NTls {
+ using TDtor = void (*)(void*);
+
+ class TKey {
+ public:
+ TKey(TDtor dtor);
+ TKey(TKey&&) noexcept;
+ ~TKey();
+
+ void* Get() const;
+ void Set(void* ptr) const;
+
+ static void Cleanup() noexcept;
+
+ private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+ };
+
+ struct TCleaner {
+ inline ~TCleaner() {
+ TKey::Cleanup();
+ }
+ };
+
+ template <class T>
+ class TValue: public TMoveOnly {
+ class TConstructor {
+ public:
+ TConstructor() noexcept = default;
+
+ virtual ~TConstructor() = default;
+
+ virtual T* Construct(void* ptr) const = 0;
+ };
+
+ class TDefaultConstructor: public TConstructor {
+ public:
+ ~TDefaultConstructor() override = default;
+
+ T* Construct(void* ptr) const override {
+ //memset(ptr, 0, sizeof(T));
+ return ::new (ptr) T();
+ }
+ };
+
+ template <class T1>
+ class TCopyConstructor: public TConstructor {
+ public:
+ inline TCopyConstructor(const T1& value)
+ : Value(value)
+ {
+ }
+
+ ~TCopyConstructor() override = default;
+
+ T* Construct(void* ptr) const override {
+ return ::new (ptr) T(Value);
+ }
+
+ private:
+ T1 Value;
+ };
+
+ public:
+ inline TValue()
+ : Constructor_(new TDefaultConstructor())
+ , Key_(Dtor)
+ {
+ }
+
+ template <class T1>
+ inline TValue(const T1& value)
+ : Constructor_(new TCopyConstructor<T1>(value))
+ , Key_(Dtor)
+ {
+ }
+
+ template <class T1>
+ inline T& operator=(const T1& val) {
+ return Get() = val;
+ }
+
+ inline operator const T&() const {
+ return Get();
+ }
+
+ inline operator T&() {
+ return Get();
+ }
+
+ inline const T& operator->() const {
+ return Get();
+ }
+
+ inline T& operator->() {
+ return Get();
+ }
+
+ inline const T* operator&() const {
+ return GetPtr();
+ }
+
+ inline T* operator&() {
+ return GetPtr();
+ }
+
+ inline T& Get() const {
+ return *GetPtr();
+ }
+
+ inline T* GetPtr() const {
+ T* val = static_cast<T*>(Key_.Get());
+
+ if (!val) {
+ THolder<void> mem(::operator new(sizeof(T)));
+ THolder<T> newval(Constructor_->Construct(mem.Get()));
+
+ Y_UNUSED(mem.Release());
+ Key_.Set((void*)newval.Get());
+ val = newval.Release();
+ }
+
+ return val;
+ }
+
+ private:
+ static void Dtor(void* ptr) {
+ THolder<void> mem(ptr);
+
+ ((T*)ptr)->~T();
+ ::NPrivate::FillWithTrash(ptr, sizeof(T));
+ }
+
+ private:
+ THolder<TConstructor> Constructor_;
+ TKey Key_;
+ };
+}
+
+template <class T>
+static inline T& TlsRef(NTls::TValue<T>& v) noexcept {
+ return v;
+}
+
+template <class T>
+static inline const T& TlsRef(const NTls::TValue<T>& v) noexcept {
+ return v;
+}
+
+template <class T>
+static inline T& TlsRef(T& v) noexcept {
+ return v;
+}
diff --git a/util/system/tls_ut.cpp b/util/system/tls_ut.cpp
new file mode 100644
index 0000000000..e84d34b42a
--- /dev/null
+++ b/util/system/tls_ut.cpp
@@ -0,0 +1,58 @@
+#include "tls.h"
+#include "thread.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TTestTLS) {
+ struct X {
+ inline X()
+ : V(0)
+ {
+ }
+
+ inline void Do() {
+ ++TlsRef(V);
+ }
+
+ inline int Get() {
+ return TlsRef(V);
+ }
+
+ Y_THREAD(int)
+ V;
+ };
+
+ Y_UNIT_TEST(TestHugeSetup) {
+ TArrayHolder<X> x(new X[100000]);
+
+ struct TThr: public ISimpleThread {
+ inline TThr(X* ptr)
+ : P(ptr)
+ {
+ }
+
+ void* ThreadProc() noexcept override {
+ for (size_t i = 0; i < 100000; ++i) {
+ P[i].Do();
+ }
+
+ return nullptr;
+ }
+
+ X* P;
+ };
+
+ TThr thr1(x.Get());
+ TThr thr2(x.Get());
+
+ thr1.Start();
+ thr2.Start();
+
+ thr1.Join();
+ thr2.Join();
+
+ for (size_t i = 0; i < 100000; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(x.Get()[i].Get(), 0);
+ }
+ }
+}
diff --git a/util/system/type_name.cpp b/util/system/type_name.cpp
new file mode 100644
index 0000000000..0377da4212
--- /dev/null
+++ b/util/system/type_name.cpp
@@ -0,0 +1,60 @@
+#include "platform.h"
+#include "demangle_impl.h"
+
+#ifdef __GNUC__
+ #include <stdexcept>
+ #include <cxxabi.h>
+#endif
+
+#include "type_name.h"
+
+namespace {
+
+#if defined(_LIBCPP_VERSION)
+ // libc++ is nested under std::__y1
+ constexpr std::string_view STD_ABI_PREFIX = "std::__y1::";
+#elif defined(_linux_)
+ // libstdc++ is nested under std::__cxx11
+ // FIXME: is there any way to test if we are building against libstdc++?
+ constexpr std::string_view STD_ABI_PREFIX = "std::__cxx11::";
+#else
+ // There is no need to cutoff ABI prefix on Windows
+#endif
+ constexpr std::string_view STD_PREFIX = "std::";
+
+} // anonymous namespace
+
+const char* NPrivate::TCppDemangler::Demangle(const char* name) {
+#ifndef __GNUC__
+ return name;
+#else
+ int status;
+ TmpBuf_.Reset(__cxxabiv1::__cxa_demangle(name, nullptr, nullptr, &status));
+
+ if (!TmpBuf_) {
+ return name;
+ }
+
+ return TmpBuf_.Get();
+#endif
+}
+
+TString CppDemangle(const TString& name) {
+ return NPrivate::TCppDemangler().Demangle(name.data());
+}
+
+TString TypeName(const std::type_info& typeInfo) {
+ TString demangled = CppDemangle(typeInfo.name()); // NOLINT(arcadia-typeid-name-restriction)
+#if defined(_linux_) || defined(_darwin_)
+ SubstGlobal(demangled, STD_ABI_PREFIX, STD_PREFIX);
+#endif
+ return demangled;
+}
+
+TString TypeName(const std::type_index& typeIndex) {
+ TString demangled = CppDemangle(typeIndex.name());
+#if defined(_linux_) || defined(_darwin_)
+ SubstGlobal(demangled, STD_ABI_PREFIX, STD_PREFIX);
+#endif
+ return demangled;
+}
diff --git a/util/system/type_name.h b/util/system/type_name.h
new file mode 100644
index 0000000000..b6619aba3f
--- /dev/null
+++ b/util/system/type_name.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <util/generic/string.h>
+#include <util/string/subst.h>
+
+#include <typeindex>
+#include <typeinfo>
+
+// Consider using TypeName function family.
+TString CppDemangle(const TString& name);
+
+// TypeName function family return human readable type name.
+
+TString TypeName(const std::type_info& typeInfo);
+TString TypeName(const std::type_index& typeInfo);
+
+// Works for types known at compile-time
+// (thus, does not take any inheritance into account)
+template <class T>
+inline TString TypeName() {
+ return TypeName(typeid(T));
+}
+
+// Works for dynamic type, including complex class hierarchies.
+// Also, distinguishes between T, T*, T const*, T volatile*, T const volatile*,
+// but not T and T const.
+template <class T>
+inline TString TypeName(const T& t) {
+ return TypeName(typeid(t));
+}
diff --git a/util/system/type_name_ut.cpp b/util/system/type_name_ut.cpp
new file mode 100644
index 0000000000..86597f4232
--- /dev/null
+++ b/util/system/type_name_ut.cpp
@@ -0,0 +1,184 @@
+#include "type_name.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/yexception.h>
+#include <util/generic/fwd.h>
+
+#include <stdexcept>
+#include <string>
+
+Y_UNIT_TEST_SUITE(TDemangleTest) {
+ Y_UNIT_TEST(SimpleTest) {
+ // just check it does not crash or leak
+ CppDemangle("hello");
+ CppDemangle("");
+ CppDemangle("Sfsdf$dfsdfTTSFSDF23234::SDFS:FSDFSDF#$%");
+ }
+}
+
+namespace NUtil::NTypeNameTest {
+
+ class TSonde {
+ // intentionally left empty
+ };
+
+ class TRombicHead {
+ public:
+ virtual ~TRombicHead() = default;
+ };
+
+ class TRombicLeftArc: public virtual TRombicHead {
+ public:
+ int x;
+ virtual ~TRombicLeftArc() = default;
+ };
+
+ class TRombicRightArc: public virtual TRombicHead {
+ public:
+ int y;
+ virtual ~TRombicRightArc() = default;
+ };
+
+ class TRombicTail: public virtual TRombicRightArc, TRombicLeftArc {
+ public:
+ virtual ~TRombicTail() = default;
+ };
+
+ class TFromThis {
+ public:
+ TString GetTypeName() const {
+ return TypeName(*this);
+ }
+ };
+}
+
+using namespace NUtil::NTypeNameTest;
+
+Y_UNIT_TEST_SUITE(TypeName) {
+ Y_UNIT_TEST(FromWellKnownTypes) {
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<void>(), "void");
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void * __ptr64");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void*");
+#endif
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<int>(), "int");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<double>(), "double");
+
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<std::string>(), "class std::basic_string<char,struct std::char_traits<char>,class std::allocator<char> >");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "class std::runtime_error");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<std::string>(), "std::basic_string<char, std::char_traits<char>, std::allocator<char> >");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "std::runtime_error");
+#endif
+ }
+
+ Y_UNIT_TEST(FromArcadiaTypes) {
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "class yexception");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TString>(), "class TBasicString<char,struct std::char_traits<char> >");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "yexception");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TString>(), "TBasicString<char, std::char_traits<char> >");
+#endif
+ }
+
+ Y_UNIT_TEST(FromUserTypes) {
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "class NUtil::NTypeNameTest::TSonde");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "class NUtil::NTypeNameTest::TRombicTail");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "NUtil::NTypeNameTest::TSonde");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "NUtil::NTypeNameTest::TRombicTail");
+#endif
+ }
+
+ Y_UNIT_TEST(FromWellKnownValues) {
+ void* value = (void*)"123";
+ const void* constValue = (const void*)"456";
+
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void * __ptr64");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void * __ptr64 * __ptr64");
+
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const * __ptr64");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const * __ptr64 * __ptr64");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void*");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void**");
+
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const*");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const**");
+#endif
+
+ int zero = 0;
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(zero), "int");
+
+ double pi = M_PI;
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(pi), "double");
+
+ std::string string;
+ std::runtime_error err("This is awful");
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(string), "class std::basic_string<char,struct std::char_traits<char>,class std::allocator<char> >");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "class std::runtime_error");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(string), "std::basic_string<char, std::char_traits<char>, std::allocator<char> >");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "std::runtime_error");
+#endif
+ }
+
+ Y_UNIT_TEST(FromUserValues) {
+ TSonde sonde;
+ const TSonde constSonde;
+
+ TRombicTail rombicTail;
+ TRombicHead& castedTail = rombicTail;
+
+ TFromThis obj;
+
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "class NUtil::NTypeNameTest::TSonde");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "class NUtil::NTypeNameTest::TSonde");
+
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "class NUtil::NTypeNameTest::TRombicTail");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "class NUtil::NTypeNameTest::TRombicTail");
+
+ UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "class NUtil::NTypeNameTest::TFromThis");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "NUtil::NTypeNameTest::TSonde");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "NUtil::NTypeNameTest::TSonde");
+
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "NUtil::NTypeNameTest::TRombicTail");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "NUtil::NTypeNameTest::TRombicTail");
+
+ UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "NUtil::NTypeNameTest::TFromThis");
+#endif
+ }
+
+ Y_UNIT_TEST(FromTypeInfo) {
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(typeid(int)), "int");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(std::type_index(typeid(int))), "int");
+ }
+
+ Y_UNIT_TEST(DistinguishPointerQualifiers) {
+ char* simplePtr = nullptr;
+ const char* constPtr = nullptr;
+ volatile char* volatilePtr = nullptr;
+ const volatile char* cvPtr = nullptr;
+
+#ifdef _MSC_VER
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char * __ptr64");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const * __ptr64");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile * __ptr64");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile * __ptr64");
+#else
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char*");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const*");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile*");
+ UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile*");
+#endif
+ }
+}
diff --git a/util/system/types.cpp b/util/system/types.cpp
new file mode 100644
index 0000000000..11cc72aee3
--- /dev/null
+++ b/util/system/types.cpp
@@ -0,0 +1,18 @@
+#include "types.h"
+
+#include <util/generic/typetraits.h>
+#include <util/generic/typelist.h>
+
+static_assert(sizeof(ui8) == 1, "incorrect ui8 type");
+static_assert(sizeof(ui16) == 2, "incorrect ui16 type");
+static_assert(sizeof(ui32) == 4, "incorrect ui32 type");
+static_assert(sizeof(ui64) == 8, "incorrect ui64 type");
+
+static_assert(sizeof(i8) == 1, "incorrect i8 type");
+static_assert(sizeof(i16) == 2, "incorrect i16 type");
+static_assert(sizeof(i32) == 4, "incorrect i32 type");
+static_assert(sizeof(i64) == 8, "incorrect i64 type");
+
+static_assert(sizeof(size_t) == sizeof(ssize_t), "incorrect ssize_t");
+
+static_assert(TTypeList<ui32, ui64>::THave<size_t>::value, "incorrect size_t");
diff --git a/util/system/types.h b/util/system/types.h
new file mode 100644
index 0000000000..12e68a6060
--- /dev/null
+++ b/util/system/types.h
@@ -0,0 +1,119 @@
+#pragma once
+
+// DO_NOT_STYLE
+
+#include "platform.h"
+
+#include <inttypes.h>
+
+typedef int8_t i8;
+typedef int16_t i16;
+typedef uint8_t ui8;
+typedef uint16_t ui16;
+
+typedef int yssize_t;
+#define PRIYSZT "d"
+
+#if defined(_darwin_) && defined(_32_)
+typedef unsigned long ui32;
+typedef long i32;
+#else
+typedef uint32_t ui32;
+typedef int32_t i32;
+#endif
+
+#if defined(_darwin_) && defined(_64_)
+typedef unsigned long ui64;
+typedef long i64;
+#else
+typedef uint64_t ui64;
+typedef int64_t i64;
+#endif
+
+#define LL(number) INT64_C(number)
+#define ULL(number) UINT64_C(number)
+
+// Macro for size_t and ptrdiff_t types
+#if defined(_32_)
+ #if defined(_darwin_)
+ #define PRISZT "lu"
+ #undef PRIi32
+ #define PRIi32 "li"
+ #undef SCNi32
+ #define SCNi32 "li"
+ #undef PRId32
+ #define PRId32 "li"
+ #undef SCNd32
+ #define SCNd32 "li"
+ #undef PRIu32
+ #define PRIu32 "lu"
+ #undef SCNu32
+ #define SCNu32 "lu"
+ #undef PRIx32
+ #define PRIx32 "lx"
+ #undef SCNx32
+ #define SCNx32 "lx"
+ #elif !defined(_cygwin_)
+ #define PRISZT PRIu32
+ #else
+ #define PRISZT "u"
+ #endif
+ #define SCNSZT SCNu32
+ #define PRIPDT PRIi32
+ #define SCNPDT SCNi32
+ #define PRITMT PRIi32
+ #define SCNTMT SCNi32
+#elif defined(_64_)
+ #if defined(_darwin_)
+ #define PRISZT "lu"
+ #undef PRIu64
+ #define PRIu64 PRISZT
+ #undef PRIx64
+ #define PRIx64 "lx"
+ #undef PRIX64
+ #define PRIX64 "lX"
+ #undef PRId64
+ #define PRId64 "ld"
+ #undef PRIi64
+ #define PRIi64 "li"
+ #undef SCNi64
+ #define SCNi64 "li"
+ #undef SCNu64
+ #define SCNu64 "lu"
+ #undef SCNx64
+ #define SCNx64 "lx"
+ #else
+ #define PRISZT PRIu64
+ #endif
+ #define SCNSZT SCNu64
+ #define PRIPDT PRIi64
+ #define SCNPDT SCNi64
+ #define PRITMT PRIi64
+ #define SCNTMT SCNi64
+#else
+ #error "Unsupported platform"
+#endif
+
+// SUPERLONG
+#if !defined(DONT_USE_SUPERLONG) && !defined(SUPERLONG_MAX)
+ #define SUPERLONG_MAX ~LL(0)
+typedef i64 SUPERLONG;
+#endif
+
+// UNICODE
+#ifdef __cplusplus
+// UCS-2, native byteorder
+typedef char16_t wchar16;
+// internal symbol type: UTF-16LE
+typedef wchar16 TChar;
+typedef char32_t wchar32;
+#endif
+
+#if defined(_MSC_VER)
+ #include <basetsd.h>
+typedef SSIZE_T ssize_t;
+ #define HAVE_SSIZE_T 1
+ #include <wchar.h>
+#endif
+
+#include <sys/types.h>
diff --git a/util/system/types.pxd b/util/system/types.pxd
new file mode 100644
index 0000000000..cc916f6e70
--- /dev/null
+++ b/util/system/types.pxd
@@ -0,0 +1,13 @@
+from libc.stdint cimport int8_t, int16_t, int32_t, int64_t
+from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t
+
+cdef extern from "<util/system/types.h>" nogil:
+ ctypedef int8_t i8
+ ctypedef int16_t i16
+ ctypedef int32_t i32
+ ctypedef int64_t i64
+
+ ctypedef uint8_t ui8
+ ctypedef uint16_t ui16
+ ctypedef uint32_t ui32
+ ctypedef uint64_t ui64
diff --git a/util/system/types_ut.cpp b/util/system/types_ut.cpp
new file mode 100644
index 0000000000..19e40cef46
--- /dev/null
+++ b/util/system/types_ut.cpp
@@ -0,0 +1,23 @@
+#include "types.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TestTypes) {
+ Y_UNIT_TEST(TestScanf) {
+ i32 val32 = 0;
+ sscanf("-123", "%" SCNi32, &val32);
+ UNIT_ASSERT(val32 == -123);
+ sscanf("234", "%" SCNu32, &val32);
+ UNIT_ASSERT(val32 == 234);
+ sscanf("159", "%" SCNx32, &val32);
+ UNIT_ASSERT(val32 == 345);
+
+ i64 val64 = 0;
+ sscanf("-123", "%" SCNi64, &val64);
+ UNIT_ASSERT(val64 == -123);
+ sscanf("234", "%" SCNu64, &val64);
+ UNIT_ASSERT(val64 == 234);
+ sscanf("159", "%" SCNx64, &val64);
+ UNIT_ASSERT(val64 == 345);
+ }
+}
diff --git a/util/system/types_ut.pyx b/util/system/types_ut.pyx
new file mode 100644
index 0000000000..bb93ac8566
--- /dev/null
+++ b/util/system/types_ut.pyx
@@ -0,0 +1,40 @@
+from util.system.types cimport i8, i16, i32, i64
+from util.system.types cimport ui8, ui16, ui32, ui64
+
+import pytest
+import unittest
+
+
+class TestTypes(unittest.TestCase):
+ def test_i8(self):
+ cdef i8 value = 42
+ self.assertEqual(sizeof(value), 1)
+
+ def test_ui8(self):
+ cdef ui8 value = 42
+ self.assertEqual(sizeof(value), 1)
+
+ def test_i16(self):
+ cdef i16 value = 42
+ self.assertEqual(sizeof(value), 2)
+
+ def test_ui16(self):
+ cdef ui16 value = 42
+ self.assertEqual(sizeof(value), 2)
+
+ def test_i32(self):
+ cdef i32 value = 42
+ self.assertEqual(sizeof(value), 4)
+
+ def test_ui32(self):
+ cdef ui32 value = 42
+ self.assertEqual(sizeof(value), 4)
+
+ def test_i64(self):
+ cdef i64 value = 42
+ self.assertEqual(sizeof(value), 8)
+
+ def test_ui64(self):
+ cdef ui64 value = 42
+ self.assertEqual(sizeof(value), 8)
+
diff --git a/util/system/unaligned_mem.cpp b/util/system/unaligned_mem.cpp
new file mode 100644
index 0000000000..b683ea9aae
--- /dev/null
+++ b/util/system/unaligned_mem.cpp
@@ -0,0 +1 @@
+#include "unaligned_mem.h"
diff --git a/util/system/unaligned_mem.h b/util/system/unaligned_mem.h
new file mode 100644
index 0000000000..4b84686f2f
--- /dev/null
+++ b/util/system/unaligned_mem.h
@@ -0,0 +1,67 @@
+#pragma once
+
+#include "defaults.h"
+#include "yassert.h"
+
+#include <string.h>
+#include <type_traits>
+
+// The following code used to have smart tricks assuming that unaligned reads and writes are OK on x86. This assumption
+// is wrong because compiler may emit alignment-sensitive x86 instructions e.g. movaps. See IGNIETFERRO-735.
+
+template <class T>
+inline T ReadUnaligned(const void* from) noexcept {
+ T ret;
+ memcpy(&ret, from, sizeof(T));
+ return ret;
+}
+
+// std::remove_reference_t for non-deduced context to prevent such code to blow below:
+// ui8 first = f(); ui8 second = g();
+// WriteUnaligned(to, first - second) (int will be deduced)
+template <class T>
+inline void WriteUnaligned(void* to, const std::remove_reference_t<T>& t) noexcept {
+ memcpy(to, &t, sizeof(T));
+}
+
+template <class T, unsigned Align = sizeof(T)>
+class TUnalignedMemoryIterator {
+public:
+ inline TUnalignedMemoryIterator(const void* buf, size_t len)
+ : C_((const unsigned char*)buf)
+ , E_(C_ + len)
+ , L_(E_ - (len % Align))
+ {
+ Y_FAKE_READ(buf);
+ }
+
+ inline bool AtEnd() const noexcept {
+ return C_ == L_;
+ }
+
+ inline T Cur() const noexcept {
+ Y_ASSERT(C_ < L_ || sizeof(T) < Align);
+ return ::ReadUnaligned<T>(C_);
+ }
+
+ inline T Next() noexcept {
+ T ret(Cur());
+
+ C_ += sizeof(T);
+
+ return ret;
+ }
+
+ inline const unsigned char* Last() const noexcept {
+ return C_;
+ }
+
+ inline size_t Left() const noexcept {
+ return E_ - C_;
+ }
+
+private:
+ const unsigned char* C_;
+ const unsigned char* E_;
+ const unsigned char* L_;
+};
diff --git a/util/system/unaligned_mem_ut.cpp b/util/system/unaligned_mem_ut.cpp
new file mode 100644
index 0000000000..9de3f3e931
--- /dev/null
+++ b/util/system/unaligned_mem_ut.cpp
@@ -0,0 +1,96 @@
+#include "unaligned_mem.h"
+
+#include <library/cpp/testing/benchmark/bench.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/system/compiler.h>
+
+#ifdef Y_HAVE_INT128
+namespace {
+ struct TUInt128 {
+ bool operator==(const TUInt128& other) const {
+ return x == other.x;
+ }
+
+ ui64 Low() const {
+ return (ui64)x;
+ }
+
+ ui64 High() const {
+ return (ui64)(x >> 64);
+ }
+
+ static TUInt128 Max() {
+ return {~(__uint128_t)0};
+ }
+
+ __uint128_t x;
+ };
+}
+#endif
+
+Y_UNIT_TEST_SUITE(UnalignedMem) {
+ Y_UNIT_TEST(TestReadWrite) {
+ alignas(ui64) char buf[100];
+
+ WriteUnaligned<ui16>(buf + 1, (ui16)1);
+ WriteUnaligned<ui32>(buf + 1 + 2, (ui32)2);
+ WriteUnaligned<ui64>(buf + 1 + 2 + 4, (ui64)3);
+
+ UNIT_ASSERT_VALUES_EQUAL(ReadUnaligned<ui16>(buf + 1), 1);
+ UNIT_ASSERT_VALUES_EQUAL(ReadUnaligned<ui32>(buf + 1 + 2), 2);
+ UNIT_ASSERT_VALUES_EQUAL(ReadUnaligned<ui64>(buf + 1 + 2 + 4), 3);
+ }
+
+ Y_UNIT_TEST(TestReadWriteRuntime) {
+ // Unlike the test above, this test avoids compile-time execution by a smart compiler.
+ // It is required to catch the SIGSEGV in case compiler emits an alignment-sensitive instruction.
+
+ alignas(ui64) static char buf[100] = {0}; // static is required for Clobber to work
+
+ WriteUnaligned<ui16>(buf + 1, (ui16)1);
+ WriteUnaligned<ui32>(buf + 1 + 2, (ui32)2);
+ WriteUnaligned<ui64>(buf + 1 + 2 + 4, (ui64)3);
+ NBench::Clobber();
+
+ auto val1 = ReadUnaligned<ui16>(buf + 1);
+ auto val2 = ReadUnaligned<ui32>(buf + 1 + 2);
+ auto val3 = ReadUnaligned<ui64>(buf + 1 + 2 + 4);
+
+ Y_DO_NOT_OPTIMIZE_AWAY(&val1);
+ Y_DO_NOT_OPTIMIZE_AWAY(&val2);
+ Y_DO_NOT_OPTIMIZE_AWAY(&val3);
+ Y_DO_NOT_OPTIMIZE_AWAY(val1);
+ Y_DO_NOT_OPTIMIZE_AWAY(val2);
+ Y_DO_NOT_OPTIMIZE_AWAY(val3);
+
+ UNIT_ASSERT_VALUES_EQUAL(val1, 1);
+ UNIT_ASSERT_VALUES_EQUAL(val2, 2);
+ UNIT_ASSERT_VALUES_EQUAL(val3, 3);
+ }
+#ifdef Y_HAVE_INT128
+ Y_UNIT_TEST(TestReadWrite128) {
+ alignas(TUInt128) char buf[100] = {0};
+
+ WriteUnaligned<TUInt128>(buf + 1, TUInt128::Max());
+ auto val = ReadUnaligned<TUInt128>(buf + 1);
+ UNIT_ASSERT(val == TUInt128::Max());
+ }
+ Y_UNIT_TEST(TestReadWriteRuntime128) {
+ // Unlike the test above, this test avoids compile-time execution by a smart compiler.
+ // It is required to catch the SIGSEGV in case compiler emits an alignment-sensitive instruction.
+
+ alignas(TUInt128) static char buf[100] = {0}; // static is required for Clobber to work
+
+ WriteUnaligned<TUInt128>(buf + 1, TUInt128::Max());
+ NBench::Clobber();
+
+ auto val = ReadUnaligned<TUInt128>(buf + 1);
+ Y_DO_NOT_OPTIMIZE_AWAY(&val);
+ Y_DO_NOT_OPTIMIZE_AWAY(val.Low());
+ Y_DO_NOT_OPTIMIZE_AWAY(val.High());
+
+ UNIT_ASSERT(val == TUInt128::Max());
+ }
+#endif
+}
diff --git a/util/system/user.cpp b/util/system/user.cpp
new file mode 100644
index 0000000000..83e89ea0a8
--- /dev/null
+++ b/util/system/user.cpp
@@ -0,0 +1,58 @@
+#include "user.h"
+#include "platform.h"
+#include "defaults.h"
+#include "env.h"
+
+#include <util/generic/yexception.h>
+
+#ifdef _win_
+ #include "winint.h"
+#else
+ #include <errno.h>
+ #include <pwd.h>
+ #include <unistd.h>
+#endif
+
+TString GetUsername() {
+ for (const auto& var : {"LOGNAME", "USER", "LNAME", "USERNAME"}) {
+ TString val = GetEnv(var);
+ if (val) {
+ return val;
+ }
+ }
+
+ TTempBuf nameBuf;
+ for (;;) {
+#if defined(_win_)
+ DWORD len = (DWORD)Min(nameBuf.Size(), size_t(32767));
+ if (!GetUserNameA(nameBuf.Data(), &len)) {
+ DWORD err = GetLastError();
+ if ((err == ERROR_INSUFFICIENT_BUFFER) && (nameBuf.Size() <= 32767))
+ nameBuf = TTempBuf((size_t)len);
+ else
+ ythrow TSystemError(err) << " GetUserName failed";
+ } else {
+ return TString(nameBuf.Data(), (size_t)(len - 1));
+ }
+#elif defined(_bionic_)
+ const passwd* pwd = getpwuid(geteuid());
+
+ if (pwd) {
+ return TString(pwd->pw_name);
+ }
+
+ ythrow TSystemError() << TStringBuf(" getpwuid failed");
+#else
+ passwd pwd;
+ passwd* tmpPwd;
+ int err = getpwuid_r(geteuid(), &pwd, nameBuf.Data(), nameBuf.Size(), &tmpPwd);
+ if (err == 0 && tmpPwd) {
+ return TString(pwd.pw_name);
+ } else if (err == ERANGE) {
+ nameBuf = TTempBuf(nameBuf.Size() * 2);
+ } else {
+ ythrow TSystemError(err) << " getpwuid_r failed";
+ }
+#endif
+ }
+}
diff --git a/util/system/user.h b/util/system/user.h
new file mode 100644
index 0000000000..be348d1cee
--- /dev/null
+++ b/util/system/user.h
@@ -0,0 +1,5 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+
+TString GetUsername();
diff --git a/util/system/user_ut.cpp b/util/system/user_ut.cpp
new file mode 100644
index 0000000000..4f8a5ce2ef
--- /dev/null
+++ b/util/system/user_ut.cpp
@@ -0,0 +1,9 @@
+#include "user.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TestUser) {
+ Y_UNIT_TEST(TestNotEmpty) {
+ UNIT_ASSERT(GetUsername());
+ }
+}
diff --git a/util/system/ut/stdin_osfhandle/main.cpp b/util/system/ut/stdin_osfhandle/main.cpp
new file mode 100644
index 0000000000..fe2ea836a9
--- /dev/null
+++ b/util/system/ut/stdin_osfhandle/main.cpp
@@ -0,0 +1,15 @@
+#include <io.h>
+#include <stdio.h>
+#include <windows.h>
+
+int main() {
+ auto handle = (unsigned long long)_get_osfhandle(0);
+ fprintf(stderr, "_get_osfhandle(0)=%llu\n", handle);
+ // It look's like classic windows undocumented behaviour
+ // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/get-osfhandle
+ // _get_osfhandle returns INVALID_HANDLE_VALUE - 1 without any sign of error if specified fd was closed.
+ // Working with such handle will lead to future various errors.
+ if (handle + 1 == (unsigned long long)INVALID_HANDLE_VALUE)
+ return 1;
+ return 0;
+}
diff --git a/util/system/ut/stdin_osfhandle/ya.make b/util/system/ut/stdin_osfhandle/ya.make
new file mode 100644
index 0000000000..d71ab22e69
--- /dev/null
+++ b/util/system/ut/stdin_osfhandle/ya.make
@@ -0,0 +1,14 @@
+PROGRAM()
+
+OWNER(g:util)
+SUBSCRIBER(g:util-subscribers)
+
+SRCS(
+ main.cpp
+)
+
+NO_UTIL()
+
+BUILD_ONLY_IF(OS_WINDOWS)
+
+END()
diff --git a/util/system/ut/ya.make b/util/system/ut/ya.make
new file mode 100644
index 0000000000..127e7c261e
--- /dev/null
+++ b/util/system/ut/ya.make
@@ -0,0 +1,88 @@
+UNITTEST_FOR(util)
+
+OWNER(g:util)
+SUBSCRIBER(g:util-subscribers)
+
+FORK_TESTS()
+
+FORK_SUBTESTS()
+
+SPLIT_FACTOR(40)
+
+TIMEOUT(300)
+
+SIZE(MEDIUM)
+
+IF (OS_DARWIN)
+ SIZE(LARGE)
+ TAG(ya:fat ya:force_sandbox ya:exotic_platform)
+ TIMEOUT(3600)
+ENDIF()
+
+PEERDIR(
+ library/cpp/testing/benchmark
+)
+
+SRCS(
+ system/align_ut.cpp
+ system/atexit_ut.cpp
+ system/atomic_ut.cpp
+ system/backtrace_ut.cpp
+ system/byteorder_ut.cpp
+ system/compat_ut.cpp
+ system/compiler_ut.cpp
+ system/context_ut.cpp
+ system/condvar_ut.cpp
+ system/cpu_id_ut.cpp
+ system/datetime_ut.cpp
+ system/daemon_ut.cpp
+ system/direct_io_ut.cpp
+ system/env_ut.cpp
+ system/error_ut.cpp
+ system/event_ut.cpp
+ system/execpath_ut.cpp
+ system/file_ut.cpp
+ system/filemap_ut.cpp
+ system/flock_ut.cpp
+ system/fs_ut.cpp
+ system/fstat_ut.cpp
+ system/getpid_ut.cpp
+ system/guard_ut.cpp
+ system/hi_lo_ut.cpp
+ system/hostname_ut.cpp
+ system/info_ut.cpp
+ system/interrupt_signals_ut.cpp
+ system/mem_info_ut.cpp
+ system/mincore_ut.cpp
+ system/mutex_ut.cpp
+ system/nice_ut.cpp
+ system/pipe_ut.cpp
+ system/platform_ut.cpp
+ system/progname_ut.cpp
+ system/rusage_ut.cpp
+ system/rwlock_ut.cpp
+ system/sanitizers_ut.cpp
+ system/shellcommand_ut.cpp
+ system/spinlock_ut.cpp
+ system/src_root_ut.cpp
+ system/src_location_ut.cpp
+ system/shmat_ut.cpp
+ system/tempfile_ut.cpp
+ system/thread_ut.cpp
+ system/tls_ut.cpp
+ system/types_ut.cpp
+ system/type_name_ut.cpp
+ system/user_ut.cpp
+ system/unaligned_mem_ut.cpp
+ system/yassert_ut.cpp
+)
+
+IF (OS_WINDOWS)
+ DEPENDS(
+ util/system/ut/stdin_osfhandle
+ )
+ENDIF()
+
+REQUIREMENTS(ram:12)
+
+END()
diff --git a/util/system/utime.cpp b/util/system/utime.cpp
new file mode 100644
index 0000000000..c7bfc4bf3d
--- /dev/null
+++ b/util/system/utime.cpp
@@ -0,0 +1,20 @@
+#include "../system/utime.h"
+
+#ifdef _MSC_VER
+ #include <sys/utime.h>
+#else
+ #define HDR <../include/utime.h>
+ #include <sys/types.h>
+ #include HDR
+#endif
+
+int TouchFile(const char* filePath) {
+ return utime(filePath, nullptr);
+}
+
+int SetModTime(const char* filePath, time_t modtime, time_t actime) {
+ struct utimbuf buf;
+ buf.modtime = modtime;
+ buf.actime = actime;
+ return utime(filePath, &buf);
+}
diff --git a/util/system/utime.h b/util/system/utime.h
new file mode 100644
index 0000000000..1c52e6614d
--- /dev/null
+++ b/util/system/utime.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#include "defaults.h"
+
+int TouchFile(const char* filePath);
+int SetModTime(const char* filePath, time_t modtime, time_t actime);
diff --git a/util/system/valgrind.cpp b/util/system/valgrind.cpp
new file mode 100644
index 0000000000..8b2f172a6c
--- /dev/null
+++ b/util/system/valgrind.cpp
@@ -0,0 +1 @@
+#include "valgrind.h"
diff --git a/util/system/valgrind.h b/util/system/valgrind.h
new file mode 100644
index 0000000000..2ec4ed927c
--- /dev/null
+++ b/util/system/valgrind.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#if defined(WITH_VALGRIND) && defined(HAVE_VALGRIND)
+ #include <valgrind/valgrind.h>
+ #include <valgrind/memcheck.h>
+
+ #if !defined(VALGRIND_CHECK_READABLE)
+ #define VALGRIND_CHECK_READABLE(s, l) VALGRIND_CHECK_MEM_IS_DEFINED(s, l)
+ #endif
+
+ #if !defined(VALGRIND_MAKE_READABLE)
+ #define VALGRIND_MAKE_READABLE(a, b) VALGRIND_MAKE_MEM_DEFINED(a, b)
+ #endif
+#else
+ #define RUNNING_ON_VALGRIND 0
+ #define VALGRIND_CHECK_READABLE(s, l)
+ #define VALGRIND_MAKE_READABLE(a, b) 0
+ #define VALGRIND_STACK_REGISTER(start, end) 0
+ #define VALGRIND_STACK_DEREGISTER(id)
+ #define VALGRIND_DISCARD(v) ((void)v)
+static inline int VALGRIND_PRINTF(...) {
+ return 0;
+}
+ #define VALGRIND_DO_LEAK_CHECK
+#endif
+
+namespace NValgrind {
+ inline constexpr static bool ValgrindIsOn() noexcept {
+#if defined(WITH_VALGRIND)
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Returns valgrinded if running under Valgrind and plain otherwise
+ // Ment to be used in test code for constants (timeouts, etc)
+ template <typename T>
+ inline constexpr static T PlainOrUnderValgrind(T plain, T valgrinded) noexcept {
+#if defined(WITH_VALGRIND)
+ Y_UNUSED(plain);
+ return valgrinded;
+#else
+ Y_UNUSED(valgrinded);
+ return plain;
+#endif
+ }
+}
diff --git a/util/system/winint.cpp b/util/system/winint.cpp
new file mode 100644
index 0000000000..b13033bdee
--- /dev/null
+++ b/util/system/winint.cpp
@@ -0,0 +1 @@
+#include "winint.h"
diff --git a/util/system/winint.h b/util/system/winint.h
new file mode 100644
index 0000000000..ebeaefb3d2
--- /dev/null
+++ b/util/system/winint.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "platform.h"
+
+#if defined(_win_)
+
+ #ifndef WIN32_LEAN_AND_MEAN
+ #define WIN32_LEAN_AND_MEAN
+ #endif
+
+ #ifndef NOMINMAX
+ #define NOMINMAX
+ #endif
+
+ #include <windows.h>
+
+ #undef GetFreeSpace
+ #undef LoadImage
+ #undef GetMessage
+ #undef SendMessage
+ #undef DeleteFile
+ #undef OPTIONAL
+ #undef GetUserName
+ #undef CreateMutex
+ #undef GetObject
+ #undef GetGeoInfo
+ #undef GetClassName
+ #undef LANG_LAO
+ #undef GetKValue
+ #undef StartDoc
+ #undef UpdateResource
+ #undef GetNameInfo
+ #undef GetProp
+ #undef SetProp
+ #undef RemoveProp
+
+ #undef IGNORE
+ #undef ERROR
+ #undef TRANSPARENT
+
+ #undef CM_NONE
+
+#endif
diff --git a/util/system/ya.make b/util/system/ya.make
new file mode 100644
index 0000000000..79c9498ddd
--- /dev/null
+++ b/util/system/ya.make
@@ -0,0 +1,6 @@
+OWNER(g:util)
+SUBSCRIBER(g:util-subscribers)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/util/system/yassert.cpp b/util/system/yassert.cpp
new file mode 100644
index 0000000000..0f586648b7
--- /dev/null
+++ b/util/system/yassert.cpp
@@ -0,0 +1,94 @@
+#include "yassert.h"
+
+#include "backtrace.h"
+#include "guard.h"
+#include "spinlock.h"
+#include "src_root.h"
+
+#include <util/datetime/base.h>
+#include <util/generic/singleton.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+#include <util/stream/output.h>
+#include <util/stream/str.h>
+#include <util/string/printf.h>
+
+#include <cstdlib>
+#include <stdarg.h>
+#include <stdio.h>
+
+#ifdef CLANG_COVERAGE
+extern "C" {
+ // __llvm_profile_write_file may not be provided if the executable target uses NO_CLANG_COVERAGE() macro and
+ // arrives as test's dependency via DEPENDS() macro.
+ // That's why we provide a weak no-op implementation for __llvm_profile_write_file,
+ // which is used below in the code, to correctly save codecoverage profile before program exits using abort().
+ Y_WEAK int __llvm_profile_write_file(void) {
+ return 0;
+ }
+}
+
+#endif
+
+namespace {
+ struct TPanicLockHolder: public TAdaptiveLock {
+ };
+}
+namespace NPrivate {
+ [[noreturn]] Y_NO_INLINE void InternalPanicImpl(int line, const char* function, const char* expr, int, int, int, const TStringBuf file, const char* errorMessage, size_t errorMessageSize) noexcept;
+}
+
+void ::NPrivate::Panic(const TStaticBuf& file, int line, const char* function, const char* expr, const char* format, ...) noexcept {
+ try {
+ // We care of panic of first failed thread only
+ // Otherwise stderr could contain multiple messages and stack traces shuffled
+ auto guard = Guard(*Singleton<TPanicLockHolder>());
+
+ TString errorMsg;
+ va_list args;
+ va_start(args, format);
+ // format has " " prefix to mute GCC warning on empty format
+ vsprintf(errorMsg, format[0] == ' ' ? format + 1 : format, args);
+ va_end(args);
+
+ constexpr int abiPlaceholder = 0;
+ ::NPrivate::InternalPanicImpl(line, function, expr, abiPlaceholder, abiPlaceholder, abiPlaceholder, file.As<TStringBuf>(), errorMsg.c_str(), errorMsg.size());
+ } catch (...) {
+ // ¯\_(ツ)_/¯
+ }
+
+ abort();
+}
+
+namespace NPrivate {
+ [[noreturn]] Y_NO_INLINE void InternalPanicImpl(int line, const char* function, const char* expr, int, int, int, const TStringBuf file, const char* errorMessage, size_t errorMessageSize) noexcept try {
+ TStringBuf errorMsg{errorMessage, errorMessageSize};
+ const TString now = TInstant::Now().ToStringLocal();
+
+ TString r;
+ TStringOutput o(r);
+ if (expr) {
+ o << "VERIFY failed (" << now << "): " << errorMsg << Endl;
+ } else {
+ o << "FAIL (" << now << "): " << errorMsg << Endl;
+ }
+ o << " " << file << ":" << line << Endl;
+ if (expr) {
+ o << " " << function << "(): requirement " << expr << " failed" << Endl;
+ } else {
+ o << " " << function << "() failed" << Endl;
+ }
+ Cerr << r << Flush;
+#ifndef WITH_VALGRIND
+ PrintBackTrace();
+#endif
+#ifdef CLANG_COVERAGE
+ if (__llvm_profile_write_file()) {
+ Cerr << "Failed to dump clang coverage" << Endl;
+ }
+#endif
+ abort();
+ } catch (...) {
+ abort();
+ }
+}
diff --git a/util/system/yassert.h b/util/system/yassert.h
new file mode 100644
index 0000000000..529823440c
--- /dev/null
+++ b/util/system/yassert.h
@@ -0,0 +1,126 @@
+#pragma once
+
+#include "defaults.h"
+#include "src_root.h"
+#include "backtrace.h"
+
+#if defined(_MSC_VER)
+ #include <new>
+ #if defined(_DEBUG)
+ #if defined(_CRTDBG_MAP_ALLOC)
+ #include <cstdlib> /* definitions for malloc/calloc */
+ #include <malloc.h> /* must be before their redefinitions as _*_dbg() */
+ #endif
+ #include <crtdbg.h>
+ #else
+ #endif
+ #include <cassert>
+#elif defined(__GNUC__)
+ #ifdef _sun_
+ #include <alloca.h>
+ #endif
+ #include <cassert>
+#endif
+
+#if !defined(_MSC_VER)
+ #if defined(__has_builtin) && __has_builtin(__debugbreak)
+ // Do nothing, use __debugbreak builtin
+ #else
+inline void __debugbreak() {
+ #if defined(__x86_64__) || defined(__i386__)
+ __asm__ volatile("int $3\n");
+ #else
+ assert(0);
+ #endif
+}
+ #endif
+
+inline bool YaIsDebuggerPresent() {
+ return false;
+}
+#else
+// __debugbreak is intrinsic in MSVC
+
+extern "C" {
+ __declspec(dllimport) int __stdcall IsDebuggerPresent();
+}
+
+inline bool YaIsDebuggerPresent() {
+ return IsDebuggerPresent() != 0;
+}
+#endif
+
+inline void YaDebugBreak() {
+ __debugbreak();
+}
+
+#undef Y_ASSERT
+
+#if !defined(NDEBUG) && !defined(__GCCXML__)
+ #define Y_ASSERT(a) \
+ do { \
+ try { \
+ if (Y_UNLIKELY(!(a))) { \
+ if (YaIsDebuggerPresent()) \
+ __debugbreak(); \
+ else { \
+ PrintBackTrace(); \
+ /* NOLINTNEXTLINE */ \
+ assert(false && (a)); \
+ } \
+ } \
+ } catch (...) { \
+ if (YaIsDebuggerPresent()) \
+ __debugbreak(); \
+ else { \
+ PrintBackTrace(); \
+ /* NOLINTNEXTLINE */ \
+ assert(false && "Exception during assert"); \
+ } \
+ } \
+ } while (false)
+#else
+ #define Y_ASSERT(a) \
+ do { \
+ if (false) { \
+ auto __xxx = static_cast<bool>(a); \
+ Y_UNUSED(__xxx); \
+ } \
+ } while (false)
+#endif
+
+namespace NPrivate {
+ /// method should not be used directly
+ [[noreturn]] void Panic(const TStaticBuf& file, int line, const char* function, const char* expr, const char* format, ...) noexcept Y_PRINTF_FORMAT(5, 6);
+}
+
+/// Assert that does not depend on NDEBUG macro and outputs message like printf
+#define Y_VERIFY(expr, ...) \
+ do { \
+ if (Y_UNLIKELY(!(expr))) { \
+ ::NPrivate::Panic(__SOURCE_FILE_IMPL__, __LINE__, __FUNCTION__, #expr, " " __VA_ARGS__); \
+ } \
+ } while (false)
+
+#define Y_FAIL(...) \
+ do { \
+ ::NPrivate::Panic(__SOURCE_FILE_IMPL__, __LINE__, __FUNCTION__, nullptr, " " __VA_ARGS__); \
+ } while (false)
+
+#ifndef NDEBUG
+ /// Assert that depend on NDEBUG macro and outputs message like printf
+ #define Y_VERIFY_DEBUG(expr, ...) \
+ do { \
+ if (Y_UNLIKELY(!(expr))) { \
+ ::NPrivate::Panic(__SOURCE_FILE_IMPL__, __LINE__, __FUNCTION__, #expr, " " __VA_ARGS__); \
+ } \
+ } while (false)
+#else
+ #define Y_VERIFY_DEBUG(expr, ...) \
+ do { \
+ if (false) { \
+ bool __xxx = static_cast<bool>(expr); \
+ Y_UNUSED(__xxx); \
+ } \
+ } while (false)
+#endif
diff --git a/util/system/yassert_ut.cpp b/util/system/yassert_ut.cpp
new file mode 100644
index 0000000000..ddd392666c
--- /dev/null
+++ b/util/system/yassert_ut.cpp
@@ -0,0 +1,35 @@
+#undef NDEBUG
+// yassert.h must be included before all headers
+#include "yassert.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(YassertTest) {
+ Y_UNIT_TEST(TestAcsLikeFunctionCall) {
+ if (true) {
+ Y_ASSERT(true); // this cannot be compiled if Y_ASSERT is "if (!cond) { ... }"
+ } else {
+ Y_ASSERT(false);
+ }
+
+ bool var = false;
+ if (false) {
+ Y_ASSERT(false);
+ } else {
+ var = true; // this is unreachable if Y_ASSERT is "if (!cond) { ... }"
+ }
+ UNIT_ASSERT(var);
+ }
+
+ Y_UNIT_TEST(TestFailCompiles) {
+ if (false) {
+ Y_FAIL("%d is a lucky number", 7);
+ Y_FAIL();
+ }
+ }
+
+ Y_UNIT_TEST(TestVerify) {
+ Y_VERIFY(true, "hi %s", "there");
+ Y_VERIFY(true);
+ }
+}
diff --git a/util/system/yield.cpp b/util/system/yield.cpp
new file mode 100644
index 0000000000..b327b37b1a
--- /dev/null
+++ b/util/system/yield.cpp
@@ -0,0 +1,25 @@
+#include "platform.h"
+
+#ifdef _win_
+ #include "winint.h"
+ #include <process.h>
+#else
+ #include <pthread.h>
+ #include <sched.h>
+#endif
+
+void SchedYield() noexcept {
+#if defined(_unix_)
+ sched_yield();
+#else
+ Sleep(0);
+#endif
+}
+
+void ThreadYield() noexcept {
+#if defined(_freebsd_)
+ pthread_yield();
+#else
+ SchedYield();
+#endif
+}
diff --git a/util/system/yield.h b/util/system/yield.h
new file mode 100644
index 0000000000..9965fb52b5
--- /dev/null
+++ b/util/system/yield.h
@@ -0,0 +1,4 @@
+#pragma once
+
+void SchedYield() noexcept;
+void ThreadYield() noexcept;