intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

15 files changed, 511 insertions, 0 deletions

diff --git a/library/python/symbols/libc/syms.cpp b/library/python/symbols/libc/syms.cpp
new file mode 100644
index 00000000000..6c04a7ef6e5
--- /dev/null
+++ b/library/python/symbols/libc/syms.cpp
@@ -0,0 +1,157 @@
+#include <util/system/platform.h>
+
+#include <library/python/symbols/registry/syms.h>
+
+#if !defined(_MSC_VER)
+#if __has_include(<aio.h>)
+#include <aio.h>
+#endif
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <ifaddrs.h>
+#include <netdb.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <sched.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+#include <sys/ipc.h>
+#include <dlfcn.h>
+
+#if defined(_linux_)
+#include <sys/prctl.h>
+#include <sys/ptrace.h>
+#include <sys/sendfile.h>
+#else
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+#endif
+
+#if defined(_darwin_)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <mach/mach_error.h> // Y_IGNORE
+#include <mach/mach_time.h> // Y_IGNORE
+#endif
+
+#if defined(_linux_)
+#include <sys/inotify.h>
+#include <sys/mman.h>
+#endif
+
+namespace {
+    static inline void* ErrnoLocation() {
+        return &errno;
+    }
+
+    static int ClockGetres(clockid_t clk_id, struct timespec* res) {
+#if defined(_darwin_)
+        static auto func = (decltype(&ClockGetres))dlsym(RTLD_SELF, "_clock_getres");
+
+        if (func) {
+            return func(clk_id, res);
+        }
+
+        // https://opensource.apple.com/source/Libc/Libc-1158.1.2/gen/clock_gettime.c.auto.html
+
+        switch (clk_id){
+            case CLOCK_REALTIME:
+            case CLOCK_MONOTONIC:
+            case CLOCK_PROCESS_CPUTIME_ID:
+                res->tv_nsec = NSEC_PER_USEC;
+                res->tv_sec = 0;
+
+                return 0;
+
+            case CLOCK_MONOTONIC_RAW:
+            case CLOCK_MONOTONIC_RAW_APPROX:
+            case CLOCK_UPTIME_RAW:
+            case CLOCK_UPTIME_RAW_APPROX:
+            case CLOCK_THREAD_CPUTIME_ID: {
+                mach_timebase_info_data_t tb_info;
+
+                if (mach_timebase_info(&tb_info)) {
+                    return -1;
+                }
+
+                res->tv_nsec = tb_info.numer / tb_info.denom + (tb_info.numer % tb_info.denom != 0);
+                res->tv_sec = 0;
+
+                return 0;
+            }
+
+            default:
+                errno = EINVAL;
+                return -1;
+        }
+#else
+        return clock_getres(clk_id, res);
+#endif
+    }
+}
+
+BEGIN_SYMS("c")
+
+SYM(calloc)
+SYM(clock_gettime)
+SYM_2("clock_getres", ClockGetres)
+SYM(closedir)
+SYM(fdopen)
+SYM(fflush)
+SYM(freeifaddrs)
+SYM(ftok)
+SYM(getifaddrs)
+SYM(getnameinfo)
+SYM(getpwnam)
+SYM(inet_ntop)
+SYM(opendir)
+SYM(printf)
+SYM(pthread_kill)
+SYM(pthread_self)
+SYM(readdir_r)
+SYM(sem_close)
+SYM(sem_getvalue)
+SYM(sem_open)
+SYM(sem_post)
+SYM(sem_trywait)
+SYM(sem_unlink)
+SYM(sem_wait)
+SYM(siginterrupt)
+SYM(strdup)
+SYM(sendfile)
+SYM(strtod)
+SYM_2("__errno_location", ErrnoLocation)
+
+#if defined(_linux_)
+SYM(prctl)
+SYM(ptrace)
+SYM(sched_getaffinity)
+SYM(sched_setaffinity)
+SYM(sem_timedwait)
+SYM(inotify_init)
+SYM(inotify_add_watch)
+SYM(inotify_rm_watch)
+SYM(mlockall)
+#endif
+
+#if defined(_darwin_)
+SYM(mach_absolute_time)
+SYM(mach_timebase_info)
+SYM(sysctlbyname)
+#endif
+
+#if __has_include(<aio.h>)
+SYM(aio_error)
+SYM(aio_read)
+SYM(aio_return)
+SYM(aio_suspend)
+#endif
+
+END_SYMS()
+#endif
diff --git a/library/python/symbols/libc/ya.make b/library/python/symbols/libc/ya.make
new file mode 100644
index 00000000000..7b84cbc9617
--- /dev/null
+++ b/library/python/symbols/libc/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+OWNER(pg orivej)
+
+PEERDIR(
+    library/python/symbols/registry
+)
+
+IF (GCC OR CLANG)
+    CFLAGS(
+        -Wno-deprecated-declarations # For sem_getvalue.
+    )
+ENDIF()
+
+SRCS(
+    GLOBAL syms.cpp
+)
+
+END()
diff --git a/library/python/symbols/module/__init__.py b/library/python/symbols/module/__init__.py
new file mode 100644
index 00000000000..0061b9e598e
--- /dev/null
+++ b/library/python/symbols/module/__init__.py
@@ -0,0 +1,48 @@
+import subprocess
+
+from collections import defaultdict
+
+from .syms import syms
+
+
+def gen_builtin():
+    res = defaultdict(dict)
+
+    for k, v in syms.items():
+        mod, sym = k.split('|')
+
+        res[mod][sym] = v
+
+    return res
+
+
+builtin_symbols = gen_builtin()
+caps = builtin_symbols['_capability']
+
+
+def find_library(name, find_next):
+    subst = {
+        'rt': 'c',
+        'pthread': 'c',
+        'm': 'c',
+    }
+
+    builtin = builtin_symbols.get(subst.get(name, name))
+
+    if builtin:
+        return {
+            'name': name,
+            'symbols': builtin,
+        }
+
+    if 'musl' in caps:
+        return None
+
+    try:
+        subprocess.Popen.__patched__
+
+        return None
+    except Exception:
+        pass
+
+    return find_next(name)
diff --git a/library/python/symbols/module/module.cpp b/library/python/symbols/module/module.cpp
new file mode 100644
index 00000000000..92bc7f4d67d
--- /dev/null
+++ b/library/python/symbols/module/module.cpp
@@ -0,0 +1,85 @@
+#include <Python.h>
+
+#include <library/python/symbols/registry/syms.h>
+
+#include <util/generic/string.h>
+
+#define CAP(x) SYM_2(x, x)
+
+BEGIN_SYMS("_capability")
+#if defined(_musl_)
+CAP("musl")
+#endif
+#if defined(_linux_)
+CAP("linux")
+#endif
+#if defined(_darwin_)
+CAP("darwin")
+#endif
+CAP("_sentinel")
+END_SYMS()
+
+#undef CAP
+
+using namespace NPrivate;
+
+namespace {
+    template <class T>
+    struct TCB: public ICB {
+        inline TCB(T& t)
+            : CB(&t)
+        {
+        }
+
+        void Apply(const char* mod, const char* name, void* sym) override {
+            (*CB)(mod, name, sym);
+        }
+
+        T* CB;
+    };
+
+    template <class T>
+    static inline TCB<T> MakeTCB(T& t) {
+        return t;
+    }
+}
+
+static void DictSetStringPtr(PyObject* dict, const char* name, void* value) {
+    PyObject* p = PyLong_FromVoidPtr(value);
+    PyDict_SetItemString(dict, name, p);
+    Py_DECREF(p);
+}
+
+static PyObject* InitSyms(PyObject* m) {
+    if (!m)
+        return NULL;
+    PyObject* d = PyDict_New();
+    if (!d)
+        return NULL;
+
+    auto f = [&](const char* mod, const char* name, void* sym) {
+        DictSetStringPtr(d, (TString(mod) + "|" + TString(name)).c_str(), sym);
+    };
+
+    auto cb = MakeTCB(f);
+
+    ForEachSymbol(cb);
+
+    if (PyObject_SetAttrString(m, "syms", d))
+        m = NULL;
+    Py_DECREF(d);
+    return m;
+}
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT, "syms", NULL, -1, NULL, NULL, NULL, NULL, NULL};
+
+extern "C" PyObject* PyInit_syms() {
+    return InitSyms(PyModule_Create(&module));
+}
+#else
+extern "C" void initsyms() {
+    InitSyms(Py_InitModule("syms", NULL));
+}
+#endif
diff --git a/library/python/symbols/module/ya.make b/library/python/symbols/module/ya.make
new file mode 100644
index 00000000000..78e30f25478
--- /dev/null
+++ b/library/python/symbols/module/ya.make
@@ -0,0 +1,23 @@
+PY23_LIBRARY()
+
+OWNER(pg orivej)
+
+NO_PYTHON_INCLUDES()
+
+PEERDIR(
+    contrib/libs/python/Include
+)
+
+SRCS(
+    module.cpp
+)
+
+PY_REGISTER(
+    library.python.symbols.module.syms
+)
+
+PY_SRCS(
+    __init__.py
+)
+
+END()
diff --git a/library/python/symbols/python/syms.cpp b/library/python/symbols/python/syms.cpp
new file mode 100644
index 00000000000..9b52574cb1b
--- /dev/null
+++ b/library/python/symbols/python/syms.cpp
@@ -0,0 +1,17 @@
+#define SYM(SYM_NAME) extern "C" void SYM_NAME();
+SYM(PyObject_GetBuffer)
+SYM(PyBuffer_Release)
+SYM(PyCell_New)
+SYM(Py_DecRef)
+SYM(Py_IncRef)
+#undef SYM
+
+#include <library/python/symbols/registry/syms.h>
+
+BEGIN_SYMS("python")
+SYM(PyObject_GetBuffer)
+SYM(PyBuffer_Release)
+SYM(PyCell_New)
+SYM(Py_DecRef)
+SYM(Py_IncRef)
+END_SYMS()
diff --git a/library/python/symbols/python/ut/py2/ya.make b/library/python/symbols/python/ut/py2/ya.make
new file mode 100644
index 00000000000..214194de57b
--- /dev/null
+++ b/library/python/symbols/python/ut/py2/ya.make
@@ -0,0 +1,9 @@
+PY2TEST()
+
+OWNER(orivej)
+
+PEERDIR(
+    library/python/symbols/python/ut
+)
+
+END()
diff --git a/library/python/symbols/python/ut/py3/ya.make b/library/python/symbols/python/ut/py3/ya.make
new file mode 100644
index 00000000000..76611c6a194
--- /dev/null
+++ b/library/python/symbols/python/ut/py3/ya.make
@@ -0,0 +1,9 @@
+PY3TEST()
+
+OWNER(orivej)
+
+PEERDIR(
+    library/python/symbols/python/ut
+)
+
+END()
diff --git a/library/python/symbols/python/ut/test_ctypes.py b/library/python/symbols/python/ut/test_ctypes.py
new file mode 100644
index 00000000000..253a10d8b32
--- /dev/null
+++ b/library/python/symbols/python/ut/test_ctypes.py
@@ -0,0 +1,37 @@
+from ctypes import (
+    byref, POINTER, c_int, c_char, c_char_p,
+    c_void_p, py_object, c_ssize_t, pythonapi, Structure
+)
+
+c_ssize_p = POINTER(c_ssize_t)
+
+
+class Py_buffer(Structure):
+    _fields_ = [
+        ('buf', c_void_p),
+        ('obj', py_object),
+        ('len', c_ssize_t),
+        ('itemsize', c_ssize_t),
+        ('readonly', c_int),
+        ('ndim', c_int),
+        ('format', c_char_p),
+        ('shape', c_ssize_p),
+        ('strides', c_ssize_p),
+        ('suboffsets', c_ssize_p),
+        ('smalltable', c_ssize_t * 2),
+        ('internal', c_void_p)
+    ]
+
+
+def get_buffer(obj):
+    buf = Py_buffer()
+    pythonapi.PyObject_GetBuffer(py_object(obj), byref(buf), 0)
+    try:
+        buffer_type = c_char * buf.len
+        return buffer_type.from_address(buf.buf)
+    finally:
+        pythonapi.PyBuffer_Release(byref(buf))
+
+
+def test_buffer():
+    assert get_buffer(b'test string')
diff --git a/library/python/symbols/python/ut/ya.make b/library/python/symbols/python/ut/ya.make
new file mode 100644
index 00000000000..2849e01b1e0
--- /dev/null
+++ b/library/python/symbols/python/ut/ya.make
@@ -0,0 +1,16 @@
+PY23_LIBRARY()
+
+OWNER(orivej)
+
+TEST_SRCS(test_ctypes.py)
+
+PEERDIR(
+    library/python/symbols/python
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    py2
+    py3
+)
diff --git a/library/python/symbols/python/ya.make b/library/python/symbols/python/ya.make
new file mode 100644
index 00000000000..6bfd54f8bcc
--- /dev/null
+++ b/library/python/symbols/python/ya.make
@@ -0,0 +1,15 @@
+LIBRARY()
+
+OWNER(orivej)
+
+PEERDIR(
+    library/python/symbols/registry
+)
+
+SRCS(
+    GLOBAL syms.cpp
+)
+
+END()
+
+RECURSE_FOR_TESTS(ut)
diff --git a/library/python/symbols/registry/syms.cpp b/library/python/symbols/registry/syms.cpp
new file mode 100644
index 00000000000..ae8e98b32e7
--- /dev/null
+++ b/library/python/symbols/registry/syms.cpp
@@ -0,0 +1,31 @@
+#include "syms.h"
+
+#include <util/generic/vector.h>
+#include <util/generic/string.h>
+#include <util/generic/singleton.h>
+
+using namespace NPrivate;
+
+namespace {
+    struct TSym {
+        const char* Mod;
+        const char* Name;
+        void* Sym;
+    };
+
+    struct TSymbols: public TVector<TSym> {
+        static inline TSymbols* Instance() {
+            return Singleton<TSymbols>();
+        }
+    };
+}
+
+void NPrivate::RegisterSymbol(const char* mod, const char* name, void* sym) {
+    TSymbols::Instance()->push_back(TSym{mod, name, sym});
+}
+
+void NPrivate::ForEachSymbol(ICB& cb) {
+    for (auto& x : *TSymbols::Instance()) {
+        cb.Apply(x.Mod, x.Name, x.Sym);
+    }
+}
diff --git a/library/python/symbols/registry/syms.h b/library/python/symbols/registry/syms.h
new file mode 100644
index 00000000000..5e34d35298c
--- /dev/null
+++ b/library/python/symbols/registry/syms.h
@@ -0,0 +1,24 @@
+#pragma once
+
+namespace NPrivate {
+    struct ICB {
+        virtual void Apply(const char* mod, const char* name, void* sym) = 0;
+    };
+
+    void ForEachSymbol(ICB& cb);
+    void RegisterSymbol(const char* mod, const char* name, void* sym);
+}
+
+#define BEGIN_SYMS(name)                   \
+    namespace {                            \
+        static struct TRegister {          \
+            const char* ModuleName = name; \
+            inline TRegister() {
+#define END_SYMS() \
+    }              \
+    }              \
+    REGISTRY;      \
+    }
+
+#define SYM_2(n, s) ::NPrivate::RegisterSymbol(this->ModuleName, n, (void*)&s);
+#define SYM(s) SYM_2(#s, s);
diff --git a/library/python/symbols/registry/ya.make b/library/python/symbols/registry/ya.make
new file mode 100644
index 00000000000..2737b9fce94
--- /dev/null
+++ b/library/python/symbols/registry/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+OWNER(pg orivej)
+
+SRCS(
+    syms.cpp
+)
+
+END()
diff --git a/library/python/symbols/ya.make b/library/python/symbols/ya.make
new file mode 100644
index 00000000000..340a710c487
--- /dev/null
+++ b/library/python/symbols/ya.make
@@ -0,0 +1,12 @@
+RECURSE(
+    module
+    registry
+    tests
+
+    crypto
+    libc
+    libmagic
+    python
+    uuid
+    win_unicode_console
+)