Intermediate changes

12 files changed, 360 insertions, 0 deletions

diff --git a/library/python/mlockall/__init__.py b/library/python/mlockall/__init__.py
new file mode 100644
index 00000000000..4d867d692e6
--- /dev/null
+++ b/library/python/mlockall/__init__.py
@@ -0,0 +1,10 @@
+import sys
+
+
+def mlockall_current():
+    if not sys.platform.startswith('linux'):
+        return -1
+
+    import library.python.mlockall.mlockall as ml
+
+    return ml.mlockall_current()
diff --git a/library/python/mlockall/mlockall.pyx b/library/python/mlockall/mlockall.pyx
new file mode 100644
index 00000000000..b35d661a424
--- /dev/null
+++ b/library/python/mlockall/mlockall.pyx
@@ -0,0 +1,19 @@
+cdef extern from "<util/system/error.h>":
+    int LastSystemError()
+
+cdef extern from "<util/system/mlock.h>":
+    cdef enum ELockAllMemoryFlag:
+        LockCurrentMemory
+        LockFutureMemory
+    cppclass ELockAllMemoryFlags:
+        operator=(ELockAllMemoryFlag)
+    void LockAllMemory(ELockAllMemoryFlags flags) except+
+
+def mlockall_current():
+    cdef ELockAllMemoryFlags flags
+    try:
+        flags = LockCurrentMemory
+        LockAllMemory(flags)
+        return 0
+    except Exception:
+        return LastSystemError()
diff --git a/library/python/mlockall/ya.make b/library/python/mlockall/ya.make
new file mode 100644
index 00000000000..96eba051abf
--- /dev/null
+++ b/library/python/mlockall/ya.make
@@ -0,0 +1,14 @@
+PY23_LIBRARY()
+
+PY_SRCS(
+    __init__.py
+)
+
+IF (OS_LINUX)
+    PY_SRCS(
+        mlockall.pyx
+    )
+ENDIF()
+
+END()
+
diff --git a/library/python/nstools/__init__.py b/library/python/nstools/__init__.py
new file mode 100644
index 00000000000..34cc0f95742
--- /dev/null
+++ b/library/python/nstools/__init__.py
@@ -0,0 +1,6 @@
+from .nstools import unshare_ns, move_to_ns
+
+__all__ = [
+    'unshare_ns',
+    'move_to_ns'
+]
diff --git a/library/python/nstools/nstools.pyx b/library/python/nstools/nstools.pyx
new file mode 100644
index 00000000000..5ef30373ffd
--- /dev/null
+++ b/library/python/nstools/nstools.pyx
@@ -0,0 +1,28 @@
+from cpython.exc cimport PyErr_SetFromErrno
+
+cdef extern from "<sched.h>" nogil:
+    int setns(int fd, int mode)
+    int unshare(int flags)
+
+    cpdef enum:
+        Fs "CLONE_FS"
+        # Cgroup "CLONE_NEWCGROUP"
+        Ipc "CLONE_NEWIPC"
+        Network "CLONE_NEWNET"
+        Mount "CLONE_NEWNS"
+        Pid "CLONE_NEWPID"
+        User "CLONE_NEWUSER"
+        Uts "CLONE_NEWUTS"
+
+def unshare_ns(int flags):
+    cdef int ret = unshare(flags)
+    if ret != 0:
+        PyErr_SetFromErrno(OSError)
+
+def move_to_ns(object fileobject, int mode):
+    if not isinstance(fileobject, int):
+        fileobject = fileobject.fileno()
+
+    cdef int ret = setns(fileobject, mode)
+    if ret != 0:
+        PyErr_SetFromErrno(OSError)
diff --git a/library/python/nstools/ya.make b/library/python/nstools/ya.make
new file mode 100644
index 00000000000..49eddeb919f
--- /dev/null
+++ b/library/python/nstools/ya.make
@@ -0,0 +1,14 @@
+PY23_LIBRARY()
+
+IF(OS_LINUX)
+PY_SRCS(
+    __init__.py
+    nstools.pyx
+)
+ELSE()
+PY_SRCS(
+    nstools.py
+)
+ENDIF()
+
+END()
diff --git a/library/python/symbols/libmagic/syms.cpp b/library/python/symbols/libmagic/syms.cpp
new file mode 100644
index 00000000000..839441ae146
--- /dev/null
+++ b/library/python/symbols/libmagic/syms.cpp
@@ -0,0 +1,19 @@
+#include <contrib/libs/libmagic/src/magic.h>
+
+#include <library/python/symbols/registry/syms.h>
+
+BEGIN_SYMS("magic")
+SYM(magic_open)
+SYM(magic_close)
+SYM(magic_error)
+SYM(magic_errno)
+SYM(magic_file)
+SYM(magic_buffer)
+SYM(magic_load)
+SYM(magic_setflags)
+SYM(magic_check)
+SYM(magic_compile)
+SYM(magic_descriptor)
+SYM(magic_list)
+SYM(magic_version)
+END_SYMS()
diff --git a/library/python/symbols/libmagic/ya.make b/library/python/symbols/libmagic/ya.make
new file mode 100644
index 00000000000..a248603a410
--- /dev/null
+++ b/library/python/symbols/libmagic/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+PEERDIR(
+    contrib/libs/libmagic
+    library/python/symbols/registry
+)
+
+SRCS(
+    GLOBAL syms.cpp
+)
+
+END()
diff --git a/library/python/testing/coverage_utils/__init__.py b/library/python/testing/coverage_utils/__init__.py
new file mode 100644
index 00000000000..3313eee7b50
--- /dev/null
+++ b/library/python/testing/coverage_utils/__init__.py
@@ -0,0 +1,14 @@
+import re
+
+
+def make_filter(prefix_filter, exclude_regexp):
+    filters = []
+    if prefix_filter:
+        filters.append(lambda x: x.startswith(prefix_filter))
+    if exclude_regexp:
+        regexp = re.compile(exclude_regexp)
+        filters.append(lambda x: not regexp.search(x))
+
+    if filters:
+        return lambda x: all(pred(x) for pred in filters)
+    return lambda x: True
diff --git a/library/python/testing/coverage_utils/ya.make b/library/python/testing/coverage_utils/ya.make
new file mode 100644
index 00000000000..35821361805
--- /dev/null
+++ b/library/python/testing/coverage_utils/ya.make
@@ -0,0 +1,5 @@
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+END()
diff --git a/library/python/testing/system_info/__init__.py b/library/python/testing/system_info/__init__.py
new file mode 100644
index 00000000000..8bad854d97a
--- /dev/null
+++ b/library/python/testing/system_info/__init__.py
@@ -0,0 +1,204 @@
+import collections
+import psutil
+from functools import wraps
+
+
+def safe(name):
+    def decorator_safe(func):
+        """
+        Decorator for try-catch on string assembly
+        """
+
+        @wraps(func)
+        def wrap_safe(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                return "Failed to get {}: {}".format(name, e)
+
+        return wrap_safe
+
+    return decorator_safe
+
+
+def get_proc_attrib(attr):
+    if callable(attr):
+        try:
+            return attr()
+        except psutil.Error:
+            return None
+    else:
+        return attr
+
+
+@safe("cpu/mem info")
+def _cpu_mem_str():
+    vm = psutil.virtual_memory()
+    cpu_tp = psutil.cpu_times_percent(0.1)
+
+    str_items = []
+    str_items.append(
+        "CPU:  Idle: {}%  User: {}%  System: {}%  IOwait: {}%\n".format(
+            cpu_tp.idle, cpu_tp.user, cpu_tp.system, cpu_tp.iowait
+        )
+    )
+
+    str_items.append(
+        "MEM:  total {} Gb  available: {} Gb  used: {} Gb  free: {} Gb  active: {} Gb  inactive: {} Gb  shared: {} Gb\n".format(
+            round(vm.total / 1e9, 2),
+            round(vm.available / 1e9, 2),
+            round(vm.used / 1e9, 2),
+            round(vm.free / 1e9, 2),
+            round(vm.active / 1e9, 2),
+            round(vm.inactive / 1e9, 2),
+            round(vm.shared / 1e9, 2),
+        )
+    )
+
+    str_items.append("Used swap: {}%\n".format(psutil.swap_memory().percent))
+
+    return "".join(str_items)
+
+
+@safe("processes tree")
+def _proc_tree_str():
+    tree = collections.defaultdict(list)
+    for p in psutil.process_iter():
+        try:
+            tree[p.ppid()].append(p.pid)
+        except (psutil.NoSuchProcess, psutil.ZombieProcess):
+            pass
+    # on systems supporting PID 0, PID 0's parent is usually 0
+    if 0 in tree and 0 in tree[0]:
+        tree[0].remove(0)
+
+    return _print_proc_tree(min(tree), tree)
+
+
+def _print_proc_tree(parent_root, tree, indent_root=''):
+    stack = [(parent_root, indent_root, "")]
+    str_items = list()
+
+    while len(stack) > 0:
+        try:
+            parent, indent, prefix = stack.pop()
+            p = psutil.Process(parent)
+            name = get_proc_attrib(p.name)
+            str_items.append("{}({}, '{}'".format(prefix, parent, name if name else '?'))
+
+            exe = get_proc_attrib(p.exe)
+            if exe:
+                str_items.append(" [{}]".format(exe))
+
+            str_items.append(") ")
+            str_items.append("  st: {}".format(p.status()))
+            str_items.append("  mem: {}%".format(round(p.memory_percent(), 2)))
+
+            ndfs = get_proc_attrib(p.num_fds)
+            if ndfs and ndfs > 0:
+                str_items.append("  fds: {}".format(ndfs))
+
+            conns = get_proc_attrib(p.connections)
+            if conns and len(conns) > 1:
+                str_items.append("  num con: {}".format(len(conns)))
+
+            ths = get_proc_attrib(p.num_threads)
+            if ths and ths > 1:
+                str_items.append("  threads: {}".format(ths))
+
+            str_items.append("\n")
+        except psutil.Error:
+            name = "?"
+            str_items.append("({}, '{}')\n".format(parent, name))
+
+        if parent not in tree:
+            continue
+
+        child = tree[parent][-1]
+        stack.append((child, indent + "  ", indent + "`_ "))
+
+        children = tree[parent][:-1]
+        children.reverse()
+        for child in children:
+            stack.append((child, indent + "| ", indent + "|- "))
+
+    return "".join(str_items)
+
+
+@safe("network info")
+def _network_conn_str():
+    str_items = list()
+
+    counters = psutil.net_io_counters()
+    str_items.append(
+        "\nPackSent: {}  PackRecv: {}  ErrIn: {}  ErrOut: {}  DropIn: {}  DropOut: {}\n\n".format(
+            counters.packets_sent,
+            counters.packets_recv,
+            counters.errin,
+            counters.errout,
+            counters.dropin,
+            counters.dropout,
+        )
+    )
+
+    ifaces = psutil.net_if_addrs()
+    conns = psutil.net_connections()
+    list_ip = collections.defaultdict(list)
+    for con in conns:
+        list_ip[con.laddr.ip].append(con)
+
+    for name, addrs in ifaces.iteritems():
+        str_items.append("{}:\n".format(name))
+
+        for ip in addrs:
+            str_items.append("   {}".format(ip.address))
+            if ip.netmask:
+                str_items.append("   mask={}".format(ip.netmask))
+            if ip.broadcast:
+                str_items.append("   bc={}".format(ip.broadcast))
+            str_items.append("\n")
+
+            for con in list_ip[ip.address]:
+                str_items.append("      {}".format(con.laddr.port))
+                if con.raddr:
+                    str_items.append(" <--> {} : {}".format(con.raddr.ip, con.raddr.port))
+                str_items.append("   (stat: {}".format(con.status))
+                if con.pid:
+                    str_items.append("  proc: {} (pid={})".format(psutil.Process(con.pid).exe(), con.pid))
+                str_items.append(")\n")
+
+            del list_ip[ip.address]
+
+    str_items.append("***\n")
+    for ip, conns in list_ip.iteritems():
+        str_items.append("   {}\n".format(ip))
+
+        for con in conns:
+            str_items.append("      {}".format(con.laddr.port))
+            if con.raddr:
+                str_items.append(" <--> {} : {}".format(con.raddr.ip, con.raddr.port))
+            str_items.append("   (stat: {}".format(con.status))
+            if con.pid:
+                str_items.append("  proc: {} (pid={})".format(psutil.Process(con.pid).exe(), con.pid))
+            str_items.append(")\n")
+
+    return "".join(str_items)
+
+
+@safe("info")
+def get_system_info():
+    str_items = list()
+
+    str_items.append("\n --- CPU MEM --- \n")
+    str_items.append(_cpu_mem_str())
+    str_items.append("\n")
+
+    str_items.append("\n --- PROCESSES TREE --- \n")
+    str_items.append(_proc_tree_str())
+    str_items.append("\n")
+
+    str_items.append("\n --- NETWORK INFO --- \n")
+    str_items.append(_network_conn_str())
+    str_items.append("\n")
+
+    return "".join(str_items)
diff --git a/library/python/testing/system_info/ya.make b/library/python/testing/system_info/ya.make
new file mode 100644
index 00000000000..f655db8ebea
--- /dev/null
+++ b/library/python/testing/system_info/ya.make
@@ -0,0 +1,15 @@
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+PEERDIR(
+    contrib/python/psutil
+)
+
+STYLE_PYTHON()
+
+END()
+
+RECURSE_FOR_TESTS(
+    test
+)