intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

16 files changed, 1794 insertions, 0 deletions

diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make
new file mode 100644
index 0000000000..5662db4c5d
--- /dev/null
+++ b/library/python/testing/yatest_common/ya.make
@@ -0,0 +1,40 @@
+OWNER(g:yatest)
+
+PY23_LIBRARY()
+
+OWNER(g:yatest)
+
+NO_EXTENDED_SOURCE_SEARCH()
+
+PY_SRCS(
+    TOP_LEVEL
+    yatest/__init__.py
+    yatest/common/__init__.py
+    yatest/common/benchmark.py
+    yatest/common/canonical.py
+    yatest/common/environment.py
+    yatest/common/errors.py
+    yatest/common/legacy.py
+    yatest/common/misc.py
+    yatest/common/network.py
+    yatest/common/path.py
+    yatest/common/process.py
+    yatest/common/runtime.py
+    yatest/common/runtime_java.py
+    yatest/common/tags.py
+)
+
+PEERDIR(
+    contrib/python/six
+    library/python/cores
+    library/python/filelock
+    library/python/fs
+)
+
+IF (NOT CATBOOST_OPENSOURCE)
+    PEERDIR(
+        library/python/coredump_filter
+    )
+ENDIF()
+
+END()
diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py
new file mode 100644
index 0000000000..b846b3317a
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ["common"]
+
+from . import common
diff --git a/library/python/testing/yatest_common/yatest/common/__init__.py b/library/python/testing/yatest_common/yatest/common/__init__.py
new file mode 100644
index 0000000000..cf57779e27
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/__init__.py
@@ -0,0 +1,8 @@
+from .benchmark import *  # noqa
+from .canonical import *  # noqa
+from .errors import *  # noqa
+from .misc import *  # noqa
+from .path import *  # noqa
+from .process import *  # noqa
+from .runtime import *  # noqa
+from .tags import *  # noqa
diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py
new file mode 100644
index 0000000000..c3784cbe4c
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/benchmark.py
@@ -0,0 +1,22 @@
+import json
+
+from . import process
+from . import runtime
+
+
+def execute_benchmark(path, budget=None, threads=None):
+    """
+    Run benchmark and return values
+    :param path: path to benchmark binary
+    :param budget: time budget, sec (supported only by ybenchmark)
+    :param threads: number of threads to run benchmark (supported only by ybenchmark)
+    :return: map of benchmark values
+    """
+    benchmark_path = runtime.binary_path(path)
+    cmd = [benchmark_path, "--benchmark_format=json"]
+    if budget is not None:
+        cmd += ["-b", str(budget)]
+    if threads is not None:
+        cmd += ["-t", str(threads)]
+    res = process.execute(cmd)
+    return json.loads(res.std_out)
diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py
new file mode 100644
index 0000000000..b6a136d3e9
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/canonical.py
@@ -0,0 +1,176 @@
+import os
+import logging
+import shutil
+import tempfile
+
+import six
+
+from . import process
+from . import runtime
+from . import path
+
+yatest_logger = logging.getLogger("ya.test")
+
+
+def _copy(src, dst, universal_lines=False):
+    if universal_lines:
+        with open(dst, "wb") as f:
+            for line in open(src, "rbU"):
+                f.write(line)
+        return
+    shutil.copy(src, dst)
+
+
+def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None):
+    """
+    Create canonical file that can be returned from a test
+    :param path: path to the file
+    :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used
+    :param local: save file locally, otherwise move to sandbox
+    :param universal_lines: normalize EOL
+    :param diff_tool_timeout: timeout for running diff tool
+    :return: object that can be canonized
+    """
+    abs_path = os.path.abspath(path)
+    assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+    tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path())
+    safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+    # if the created file is in output_path, we copy it, so that it will be available when the tests finishes
+    _copy(path, safe_path, universal_lines=universal_lines)
+    if diff_tool:
+        if not isinstance(diff_tool, six.string_types):
+            try:  # check if iterable
+                if not isinstance(diff_tool[0], six.string_types):
+                    raise Exception("Invalid custom diff-tool: not cmd")
+            except:
+                raise Exception("Invalid custom diff-tool: not binary path")
+    return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None):
+    abs_path = os.path.abspath(path)
+    assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+    assert os.path.isdir(abs_path), "Path {} is not a directory".format(path)
+    if diff_file_name and not diff_tool:
+        raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir")
+    tempdir = tempfile.mkdtemp()
+    safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+    shutil.copytree(abs_path, safe_path)
+    return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_execute(
+    binary, args=None, check_exit_code=True,
+    shell=False, timeout=None, cwd=None,
+    env=None, stdin=None, stderr=None, creationflags=0,
+    file_name=None, save_locally=False, close_fds=False,
+    diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
+):
+    """
+    Shortcut to execute a binary and canonize its stdout
+    :param binary: absolute path to the binary
+    :param args: binary arguments
+    :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+    :param shell: use shell to run the command
+    :param timeout: execution timeout
+    :param cwd: working directory
+    :param env: command environment
+    :param stdin: command stdin
+    :param stderr: command stderr
+    :param creationflags: command creation flags
+    :param file_name: output file name. if not specified program name will be used
+    :param diff_tool: path to custome diff tool
+    :param diff_file_name: custom diff file name to create when diff is found
+    :param diff_tool_timeout: timeout for running diff tool
+    :return: object that can be canonized
+    """
+    if type(binary) == list:
+        command = binary
+    else:
+        command = [binary]
+    command += _prepare_args(args)
+    if shell:
+        command = " ".join(command)
+    execute_args = locals()
+    del execute_args["binary"]
+    del execute_args["args"]
+    del execute_args["file_name"]
+    del execute_args["save_locally"]
+    del execute_args["diff_tool"]
+    del execute_args["diff_file_name"]
+    del execute_args["diff_tool_timeout"]
+    if not file_name and stdin:
+        file_name = os.path.basename(stdin.name)
+    return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
+
+
+def canonical_py_execute(
+    script_path, args=None, check_exit_code=True,
+    shell=False, timeout=None, cwd=None, env=None,
+    stdin=None, stderr=None, creationflags=0,
+    file_name=None, save_locally=False, close_fds=False,
+    diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
+):
+    """
+    Shortcut to execute a python script and canonize its stdout
+    :param script_path: path to the script arcadia relative
+    :param args: script arguments
+    :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+    :param shell: use shell to run the command
+    :param timeout: execution timeout
+    :param cwd: working directory
+    :param env: command environment
+    :param stdin: command stdin
+    :param stderr: command stderr
+    :param creationflags: command creation flags
+    :param file_name: output file name. if not specified program name will be used
+    :param diff_tool: path to custome diff tool
+    :param diff_file_name: custom diff file name to create when diff is found
+    :param diff_tool_timeout: timeout for running diff tool
+    :return: object that can be canonized
+    """
+    command = [runtime.source_path(script_path)] + _prepare_args(args)
+    if shell:
+        command = " ".join(command)
+    execute_args = locals()
+    del execute_args["script_path"]
+    del execute_args["args"]
+    del execute_args["file_name"]
+    del execute_args["save_locally"]
+    del execute_args["diff_tool"]
+    del execute_args["diff_file_name"]
+    del execute_args["diff_tool_timeout"]
+    return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
+
+
+def _prepare_args(args):
+    if args is None:
+        args = []
+    if isinstance(args, six.string_types):
+        args = map(lambda a: a.strip(), args.split())
+    return args
+
+
+def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout):
+    res = excutor(**kwargs)
+    command = kwargs["command"]
+    file_name = file_name or process.get_command_name(command)
+    if file_name.endswith(".exe"):
+        file_name = os.path.splitext(file_name)[0]  # don't want to bring windows stuff in file names
+    out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name))
+    err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name))
+
+    try:
+        os.makedirs(os.path.dirname(out_file_path))
+    except OSError:
+        pass
+
+    with open(out_file_path, "wb") as out_file:
+        yatest_logger.debug("Will store file in %s", out_file_path)
+        out_file.write(res.std_out)
+
+    if res.std_err:
+        with open(err_file_path, "wb") as err_file:
+            err_file.write(res.std_err)
+
+    return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py
new file mode 100644
index 0000000000..43f48d0958
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/environment.py
@@ -0,0 +1,5 @@
+# coding: utf-8
+
+
+def extend_env_var(env, name, value, sep=":"):
+    return sep.join(filter(None, [env.get(name), value]))
diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py
new file mode 100644
index 0000000000..8c038fc381
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/errors.py
@@ -0,0 +1,20 @@
+import os
+import sys
+
+
+class RestartTestException(Exception):
+
+    def __init__(self, *args, **kwargs):
+        super(RestartTestException, self).__init__(*args, **kwargs)
+        sys.stderr.write("##restart-test##\n")
+        sys.stderr.flush()
+        os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
+
+
+class InfrastructureException(Exception):
+
+    def __init__(self, *args, **kwargs):
+        super(InfrastructureException, self).__init__(*args, **kwargs)
+        sys.stderr.write("##infrastructure-error##\n")
+        sys.stderr.flush()
+        os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py
new file mode 100644
index 0000000000..459972d253
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/legacy.py
@@ -0,0 +1,12 @@
+from . import canonical
+
+
+def old_canonical_file(output_file_name, storage_md5):
+    import yalibrary.svn
+    yalibrary.svn.run_svn([
+        'export',
+        'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5,
+        output_file_name,
+        "--force"
+    ])
+    return canonical.canonical_file(output_file_name)
diff --git a/library/python/testing/yatest_common/yatest/common/misc.py b/library/python/testing/yatest_common/yatest/common/misc.py
new file mode 100644
index 0000000000..20d3725ac9
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/misc.py
@@ -0,0 +1,19 @@
+import functools
+
+
+def first(it):
+    for d in it:
+        if d:
+            return d
+
+
+def lazy(func):
+    res = []
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if not res:
+            res.append(func(*args, **kwargs))
+        return res[0]
+
+    return wrapper
diff --git a/library/python/testing/yatest_common/yatest/common/network.py b/library/python/testing/yatest_common/yatest/common/network.py
new file mode 100644
index 0000000000..37bcb1b8e0
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/network.py
@@ -0,0 +1,271 @@
+# coding=utf-8
+
+import os
+import errno
+import socket
+import random
+import logging
+import platform
+import threading
+
+import six
+
+UI16MAXVAL = (1 << 16) - 1
+logger = logging.getLogger(__name__)
+
+
+class PortManagerException(Exception):
+    pass
+
+
+class PortManager(object):
+    """
+    See documentation here
+
+    https://wiki.yandex-team.ru/yatool/test/#python-acquire-ports
+    """
+
+    def __init__(self, sync_dir=None):
+        self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH')
+        if self._sync_dir:
+            _makedirs(self._sync_dir)
+
+        self._valid_range = get_valid_port_range()
+        self._valid_port_count = self._count_valid_ports()
+        self._filelocks = {}
+        self._lock = threading.Lock()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.release()
+
+    def get_port(self, port=0):
+        '''
+        Gets free TCP port
+        '''
+        return self.get_tcp_port(port)
+
+    def get_tcp_port(self, port=0):
+        '''
+        Gets free TCP port
+        '''
+        return self._get_port(port, socket.SOCK_STREAM)
+
+    def get_udp_port(self, port=0):
+        '''
+        Gets free UDP port
+        '''
+        return self._get_port(port, socket.SOCK_DGRAM)
+
+    def get_tcp_and_udp_port(self, port=0):
+        '''
+        Gets one free port for use in both TCP and UDP protocols
+        '''
+        if port and self._no_random_ports():
+            return port
+
+        retries = 20
+        while retries > 0:
+            retries -= 1
+
+            result_port = self.get_tcp_port()
+            if not self.is_port_free(result_port, socket.SOCK_DGRAM):
+                self.release_port(result_port)
+            # Don't try to _capture_port(), it's already captured in the get_tcp_port()
+            return result_port
+        raise Exception('Failed to find port')
+
+    def release_port(self, port):
+        with self._lock:
+            self._release_port_no_lock(port)
+
+    def _release_port_no_lock(self, port):
+        filelock = self._filelocks.pop(port, None)
+        if filelock:
+            filelock.release()
+
+    def release(self):
+        with self._lock:
+            while self._filelocks:
+                _, filelock = self._filelocks.popitem()
+                if filelock:
+                    filelock.release()
+
+    def get_port_range(self, start_port, count, random_start=True):
+        assert count > 0
+        if start_port and self._no_random_ports():
+            return start_port
+
+        candidates = []
+
+        def drop_candidates():
+            for port in candidates:
+                self._release_port_no_lock(port)
+            candidates[:] = []
+
+        with self._lock:
+            for attempts in six.moves.range(128):
+                for left, right in self._valid_range:
+                    if right - left < count:
+                        continue
+
+                    if random_start:
+                        start = random.randint(left, right - ((right - left) // 2))
+                    else:
+                        start = left
+                    for probe_port in six.moves.range(start, right):
+                        if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM):
+                            candidates.append(probe_port)
+                        else:
+                            drop_candidates()
+
+                        if len(candidates) == count:
+                            return candidates[0]
+                    # Can't find required number of ports without gap in the current range
+                    drop_candidates()
+
+            raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format(
+                start_port, count, self._valid_range, self._filelocks))
+
+    def _count_valid_ports(self):
+        res = 0
+        for left, right in self._valid_range:
+            res += right - left
+        assert res, ('There are no available valid ports', self._valid_range)
+        return res
+
+    def _get_port(self, port, sock_type):
+        if port and self._no_random_ports():
+            return port
+
+        if len(self._filelocks) >= self._valid_port_count:
+            raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks))
+
+        salt = random.randint(0, UI16MAXVAL)
+        for attempt in six.moves.range(self._valid_port_count):
+            probe_port = (salt + attempt) % self._valid_port_count
+
+            for left, right in self._valid_range:
+                if probe_port >= (right - left):
+                    probe_port -= right - left
+                else:
+                    probe_port += left
+                    break
+            if not self._capture_port(probe_port, sock_type):
+                continue
+            return probe_port
+
+        raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks))
+
+    def _capture_port(self, port, sock_type):
+        with self._lock:
+            return self._capture_port_no_lock(port, sock_type)
+
+    def is_port_free(self, port, sock_type=socket.SOCK_STREAM):
+        sock = socket.socket(socket.AF_INET6, sock_type)
+        try:
+            sock.bind(('::', port))
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        except socket.error as e:
+            if e.errno == errno.EADDRINUSE:
+                return False
+            raise
+        finally:
+            sock.close()
+        return True
+
+    def _capture_port_no_lock(self, port, sock_type):
+        if port in self._filelocks:
+            return False
+
+        filelock = None
+        if self._sync_dir:
+            # yatest.common should try to be hermetic and don't have peerdirs
+            # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work
+            import library.python.filelock
+
+            filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port)))
+            if not filelock.acquire(blocking=False):
+                return False
+            if self.is_port_free(port, sock_type):
+                self._filelocks[port] = filelock
+                return True
+            else:
+                filelock.release()
+                return False
+
+        if self.is_port_free(port, sock_type):
+            self._filelocks[port] = filelock
+            return True
+        if filelock:
+            filelock.release()
+        return False
+
+    def _no_random_ports(self):
+        return os.environ.get("NO_RANDOM_PORTS")
+
+
+def get_valid_port_range():
+    first_valid = 1025
+    last_valid = UI16MAXVAL
+
+    given_range = os.environ.get('VALID_PORT_RANGE')
+    if given_range and ':' in given_range:
+        return [list(int(x) for x in given_range.split(':', 2))]
+
+    first_eph, last_eph = get_ephemeral_range()
+    first_invalid = max(first_eph, first_valid)
+    last_invalid = min(last_eph, last_valid)
+
+    ranges = []
+    if first_invalid > first_valid:
+        ranges.append((first_valid, first_invalid - 1))
+    if last_invalid < last_valid:
+        ranges.append((last_invalid + 1, last_valid))
+    return ranges
+
+
+def get_ephemeral_range():
+    if platform.system() == 'Linux':
+        filename = "/proc/sys/net/ipv4/ip_local_port_range"
+        if os.path.exists(filename):
+            with open(filename) as afile:
+                data = afile.read(1024)  # fix for musl
+            port_range = tuple(map(int, data.strip().split()))
+            if len(port_range) == 2:
+                return port_range
+            else:
+                logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data)
+    elif platform.system() == 'Darwin':
+        first = _sysctlbyname_uint("net.inet.ip.portrange.first")
+        last = _sysctlbyname_uint("net.inet.ip.portrange.last")
+        if first and last:
+            return first, last
+    # IANA suggestion
+    return (1 << 15) + (1 << 14), UI16MAXVAL
+
+
+def _sysctlbyname_uint(name):
+    try:
+        from ctypes import CDLL, c_uint, byref
+        from ctypes.util import find_library
+    except ImportError:
+        return
+
+    libc = CDLL(find_library("c"))
+    size = c_uint(0)
+    res = c_uint(0)
+    libc.sysctlbyname(name, None, byref(size), None, 0)
+    libc.sysctlbyname(name, byref(res), byref(size), None, 0)
+    return res.value
+
+
+def _makedirs(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            return
+        raise
diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py
new file mode 100644
index 0000000000..6fed7dda8a
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/path.py
@@ -0,0 +1,90 @@
+# coding=utf-8
+
+import errno
+import os
+import shutil
+import contextlib
+
+import library.python.fs as lpf
+
+
+def replace_in_file(path, old, new):
+    """
+    Replace text occurrences in a file
+    :param path: path to the file
+    :param old: text to replace
+    :param new: replacement
+    """
+    with open(path) as fp:
+        content = fp.read()
+
+    lpf.ensure_removed(path)
+    with open(path, 'w') as fp:
+        fp.write(content.replace(old, new))
+
+
+@contextlib.contextmanager
+def change_dir(path):
+    old = os.getcwd()
+    try:
+        os.chdir(path)
+        yield path
+    finally:
+        os.chdir(old)
+
+
+def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None):
+    '''
+    Copy an entire directory of files into an existing directory
+    instead of raising Exception what shtuil.copytree does
+    '''
+    if not os.path.exists(dst) and os.path.isdir(src):
+        os.makedirs(dst)
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, symlinks, ignore)
+        else:
+            shutil.copy2(s, d)
+    if postprocessing:
+        postprocessing(dst, False)
+        for root, dirs, files in os.walk(dst):
+            for path in dirs:
+                postprocessing(os.path.join(root, path), False)
+            for path in files:
+                postprocessing(os.path.join(root, path), True)
+
+
+def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000):
+    def atomic_file_create(path):
+        try:
+            fd = os.open(path, os.O_CREAT | os.O_EXCL, 0o644)
+            os.close(fd)
+            return True
+        except OSError as e:
+            if e.errno in [errno.EEXIST, errno.EISDIR, errno.ETXTBSY]:
+                return False
+            # Access issue with file itself, not parent directory.
+            if e.errno == errno.EACCES and os.path.exists(path):
+                return False
+            raise e
+
+    def atomic_dir_create(path):
+        try:
+            os.mkdir(path)
+            return True
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                return False
+            raise e
+
+    file_path = os.path.join(dir_path, file_pattern)
+    lpf.ensure_dir(os.path.dirname(file_path))
+    file_counter = 0
+    handler = atomic_file_create if create_file else atomic_dir_create
+    while os.path.exists(file_path) or not handler(file_path):
+        file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter))
+        file_counter += 1
+        assert file_counter < max_suffix
+    return file_path
diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py
new file mode 100644
index 0000000000..a8bcc21f51
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/process.py
@@ -0,0 +1,733 @@
+# coding: utf-8
+
+import os
+import re
+import time
+import signal
+import shutil
+import logging
+import tempfile
+import subprocess
+import errno
+import distutils.version
+
+import six
+
+try:
+    # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work.
+    import library.python.cores as cores
+except ImportError:
+    cores = None
+
+from . import runtime
+from . import path
+from . import environment
+
+
+MAX_OUT_LEN = 1000 * 1000  # 1 mb
+MAX_MESSAGE_LEN = 1500
+SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)"
+GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)")
+yatest_logger = logging.getLogger("ya.test")
+
+
+def truncate(s, size):
+    if s is None:
+        return None
+    elif len(s) <= size:
+        return s
+    else:
+        return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):]
+
+
+def get_command_name(command):
+    return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0])
+
+
+class ExecutionError(Exception):
+
+    def __init__(self, execution_result):
+        if not isinstance(execution_result.command, six.string_types):
+            command = " ".join(str(arg) for arg in execution_result.command)
+        else:
+            command = execution_result.command
+        message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format(
+            command=command,
+            code=execution_result.exit_code,
+            err=_format_error(execution_result.std_err))
+        if cores:
+            if execution_result.backtrace:
+                message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
+        else:
+            message += "Backtrace is not available: module cores isn't available"
+
+        super(ExecutionError, self).__init__(message)
+        self.execution_result = execution_result
+
+
+class TimeoutError(Exception):
+    pass
+
+
+class ExecutionTimeoutError(TimeoutError):
+    def __init__(self, execution_result, *args, **kwargs):
+        super(ExecutionTimeoutError, self).__init__(args, kwargs)
+        self.execution_result = execution_result
+
+
+class InvalidExecutionStateError(Exception):
+    pass
+
+
+class SignalInterruptionError(Exception):
+    def __init__(self, message=None):
+        super(SignalInterruptionError, self).__init__(message)
+        self.res = None
+
+
+class InvalidCommandError(Exception):
+    pass
+
+
+class _Execution(object):
+
+    def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False):
+        self._command = command
+        self._process = process
+        self._out_file = out_file
+        self._err_file = err_file
+        self._std_out = None
+        self._std_err = None
+        self._elapsed = None
+        self._start = time.time()
+        self._process_progress_listener = process_progress_listener
+        self._cwd = cwd or os.getcwd()
+        self._collect_cores = collect_cores
+        self._backtrace = ''
+        self._check_sanitizer = check_sanitizer
+        self._metrics = {}
+        self._started = started
+        self._user_stdout = bool(user_stdout)
+        self._user_stderr = bool(user_stderr)
+        self._exit_code = None
+        if process_progress_listener:
+            process_progress_listener.open(command, process, out_file, err_file)
+
+    @property
+    def running(self):
+        return self._process.poll() is None
+
+    def kill(self):
+        if self.running:
+            self._save_outputs(False)
+            _kill_process_tree(self._process.pid)
+            self._clean_files()
+            # DEVTOOLS-2347
+            yatest_logger.debug("Process status before wait_for: %s", self.running)
+            try:
+                wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1)
+            except TimeoutError:
+                yatest_logger.debug("Process status after wait_for: %s", self.running)
+                yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
+                raise
+        else:
+            raise InvalidExecutionStateError("Cannot kill a stopped process")
+
+    def terminate(self):
+        if self.running:
+            self._process.terminate()
+
+    @property
+    def process(self):
+        return self._process
+
+    @property
+    def command(self):
+        return self._command
+
+    @property
+    def returncode(self):
+        return self.exit_code
+
+    @property
+    def exit_code(self):
+        """
+        Deprecated, use returncode
+        """
+        if self._exit_code is None:
+            self._exit_code = self._process.returncode
+        return self._exit_code
+
+    @property
+    def stdout(self):
+        return self.std_out
+
+    @property
+    def std_out(self):
+        """
+        Deprecated, use stdout
+        """
+        if self._std_out is not None:
+            return self._std_out
+        if self._process.stdout and not self._user_stdout:
+            self._std_out = self._process.stdout.read()
+            return self._std_out
+
+    @property
+    def stderr(self):
+        return self.std_err
+
+    @property
+    def std_err(self):
+        """
+        Deprecated, use stderr
+        """
+        if self._std_err is not None:
+            return self._std_err
+        if self._process.stderr and not self._user_stderr:
+            self._std_err = self._process.stderr.read()
+            return self._std_err
+
+    @property
+    def elapsed(self):
+        return self._elapsed
+
+    @property
+    def backtrace(self):
+        return self._backtrace
+
+    @property
+    def metrics(self):
+        return self._metrics
+
+    def _save_outputs(self, clean_files=True):
+        if self._process_progress_listener:
+            self._process_progress_listener()
+            self._process_progress_listener.close()
+        if not self._user_stdout:
+            if self._out_file is None:
+                pass
+            elif self._out_file != subprocess.PIPE:
+                self._out_file.flush()
+                self._out_file.seek(0, os.SEEK_SET)
+                self._std_out = self._out_file.read()
+            else:
+                self._std_out = self._process.stdout.read()
+        if not self._user_stderr:
+            if self._err_file is None:
+                pass
+            elif self._err_file != subprocess.PIPE:
+                self._err_file.flush()
+                self._err_file.seek(0, os.SEEK_SET)
+                self._std_err = self._err_file.read()
+            else:
+                self._std_err = self._process.stderr.read()
+
+        if clean_files:
+            self._clean_files()
+        yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code)
+        yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed)
+        if self._metrics:
+            for key, value in six.iteritems(self._metrics):
+                yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value)
+
+        # Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str.
+        printable_std_out, err = _try_convert_bytes_to_string(self._std_out)
+        if err:
+            yatest_logger.debug("Got error during parse process stdout: %s", err)
+            yatest_logger.debug("stdout will be displayed as raw bytes.")
+        printable_std_err, err = _try_convert_bytes_to_string(self._std_err)
+        if err:
+            yatest_logger.debug("Got error during parse process stderr: %s", err)
+            yatest_logger.debug("stderr will be displayed as raw bytes.")
+
+        yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN))
+        yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN))
+
+    def _clean_files(self):
+        if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE:
+            self._err_file.close()
+            self._err_file = None
+        if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE:
+            self._out_file.close()
+            self._out_file = None
+
+    def _recover_core(self):
+        core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid)
+        if core_path:
+            # Core dump file recovering may be disabled (for distbuild for example) - produce only bt
+            store_cores = runtime._get_ya_config().collect_cores
+            if store_cores:
+                new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
+                # Copy core dump file, because it may be overwritten
+                yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
+                shutil.copyfile(core_path, new_core_path)
+                core_path = new_core_path
+
+            bt_filename = None
+            pbt_filename = None
+
+            if os.path.exists(runtime.gdb_path()):
+                self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path())
+                bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid))
+                with open(bt_filename, "wb") as afile:
+                    afile.write(six.ensure_binary(self._backtrace))
+                # generate pretty html version of backtrace aka Tri Korochki
+                pbt_filename = bt_filename + ".html"
+                backtrace_to_html(bt_filename, pbt_filename)
+
+            if store_cores:
+                runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename)
+            else:
+                runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
+
+    def wait(self, check_exit_code=True, timeout=None, on_timeout=None):
+        def _wait():
+            finished = None
+            interrupted = False
+            try:
+                if hasattr(os, "wait4"):
+                    try:
+                        if hasattr(subprocess, "_eintr_retry_call"):
+                            pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0)
+                        else:
+                            # PEP 475
+                            pid, sts, rusage = os.wait4(self._process.pid, 0)
+                        finished = time.time()
+                        self._process._handle_exitstatus(sts)
+                        for field in [
+                            "ru_idrss",
+                            "ru_inblock",
+                            "ru_isrss",
+                            "ru_ixrss",
+                            "ru_majflt",
+                            "ru_maxrss",
+                            "ru_minflt",
+                            "ru_msgrcv",
+                            "ru_msgsnd",
+                            "ru_nivcsw",
+                            "ru_nsignals",
+                            "ru_nswap",
+                            "ru_nvcsw",
+                            "ru_oublock",
+                            "ru_stime",
+                            "ru_utime",
+                        ]:
+                            if hasattr(rusage, field):
+                                self._metrics[field.replace("ru_", "")] = getattr(rusage, field)
+                    except OSError as exc:
+
+                        if exc.errno == errno.ECHILD:
+                            yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
+                        else:
+                            raise
+            except SignalInterruptionError:
+                interrupted = True
+                raise
+            finally:
+                if not interrupted:
+                    self._process.wait()  # this has to be here unconditionally, so that all process properties are set
+
+            if not finished:
+                finished = time.time()
+            self._metrics["wtime"] = round(finished - self._started, 3)
+
+        try:
+            if timeout:
+                process_is_finished = lambda: not self.running
+                fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
+                try:
+                    wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
+                except TimeoutError as e:
+                    if on_timeout:
+                        yatest_logger.debug("Calling user specified on_timeout function")
+                        try:
+                            on_timeout(self, timeout)
+                        except Exception:
+                            yatest_logger.exception("Exception while calling on_timeout")
+                    raise ExecutionTimeoutError(self, str(e))
+            # Wait should be always called here, it finalizes internal states of its process and sets up return code
+            _wait()
+        except BaseException as e:
+            _kill_process_tree(self._process.pid)
+            _wait()
+            yatest_logger.debug("Command exception: %s", e)
+            raise
+        finally:
+            self._elapsed = time.time() - self._start
+            self._save_outputs()
+            self.verify_no_coredumps()
+
+        self._finalise(check_exit_code)
+
+    def _finalise(self, check_exit_code):
+        # Set the signal (negative number) which caused the process to exit
+        if check_exit_code and self.exit_code != 0:
+            yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
+                                self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
+            raise ExecutionError(self)
+
+        # Don't search for sanitize errors if stderr was redirected
+        self.verify_sanitize_errors()
+
+    def verify_no_coredumps(self):
+        """
+        Verify there is no coredump from this binary. If there is then report backtrace.
+        """
+        if self.exit_code < 0 and self._collect_cores:
+            if cores:
+                try:
+                    self._recover_core()
+                except Exception:
+                    yatest_logger.exception("Exception while recovering core")
+            else:
+                yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available")
+
+    def verify_sanitize_errors(self):
+        """
+        Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them.
+        """
+        if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks:
+            build_path = runtime.build_path()
+            if self.command[0].startswith(build_path):
+                match = re.search(SANITIZER_ERROR_PATTERN, self._std_err)
+                if match:
+                    yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
+                    raise ExecutionError(self)
+                else:
+                    yatest_logger.debug("No sanitizer errors found")
+            else:
+                yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
+
+
+def on_timeout_gen_coredump(exec_obj, _):
+    """
+    Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump)
+    to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout.
+    All files will be available in the testing_out_stuff and via links.
+    """
+    try:
+        os.kill(exec_obj.process.pid, signal.SIGQUIT)
+    except OSError:
+        # process might be already terminated
+        pass
+
+
+def execute(
+    command, check_exit_code=True,
+    shell=False, timeout=None,
+    cwd=None, env=None,
+    stdin=None, stdout=None, stderr=None,
+    creationflags=0, wait=True,
+    process_progress_listener=None, close_fds=False,
+    collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
+    executor=_Execution,
+):
+    """
+    Executes a command
+    :param command: command: can be a list of arguments or a string
+    :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+    :param shell: use shell to run the command
+    :param timeout: execution timeout
+    :param cwd: working directory
+    :param env: command environment
+    :param stdin: command stdin
+    :param stdout: command stdout
+    :param stderr: command stderr
+    :param creationflags: command creation flags
+    :param wait: should wait until the command finishes
+    :param process_progress_listener=object that is polled while execution is in progress
+    :param close_fds:  subrpocess.Popen close_fds args
+    :param collect_cores: recover core dump files if shell == False
+    :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors
+    :param preexec_fn: subrpocess.Popen preexec_fn arg
+    :param on_timeout: on_timeout(<execution object>, <timeout value>) callback
+
+    :return _Execution: Execution object
+    """
+    if env is None:
+        env = os.environ.copy()
+    else:
+        # Certain environment variables must be present for programs to work properly.
+        # For more info see DEVTOOLSSUPPORT-4907
+        mandatory_env_name = 'YA_MANDATORY_ENV_VARS'
+        mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or ''
+        if mandatory_vars:
+            env[mandatory_env_name] = mandatory_vars
+            mandatory_system_vars = filter(None, mandatory_vars.split(':'))
+        else:
+            mandatory_system_vars = ['TMPDIR']
+
+        for var in mandatory_system_vars:
+            if var not in env and var in os.environ:
+                env[var] = os.environ[var]
+
+    if not wait and timeout is not None:
+        raise ValueError("Incompatible arguments 'timeout' and wait=False")
+
+    # if subprocess.PIPE in [stdout, stderr]:
+    #     raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock")
+
+    def get_out_stream(stream, default_name):
+        if stream is None:
+            # No stream is supplied: open new temp file
+            return _get_command_output_file(command, default_name), False
+
+        if isinstance(stream, six.string_types):
+            # User filename is supplied: open file for writing
+            return open(stream, 'wb+'), stream.startswith('/dev/')
+
+        # Open file or PIPE sentinel is supplied
+        is_pipe = stream == subprocess.PIPE
+        return stream, not is_pipe
+
+    # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE,
+    # as it can cause processes hangs, but use tempfiles instead
+    out_file, user_stdout = get_out_stream(stdout, 'out')
+    err_file, user_stderr = get_out_stream(stderr, 'err')
+    in_file = stdin
+
+    if shell and type(command) == list:
+        command = " ".join(command)
+
+    if shell:
+        collect_cores = False
+        check_sanitizer = False
+    else:
+        if isinstance(command, (list, tuple)):
+            executable = command[0]
+        else:
+            executable = command
+        if os.path.isabs(executable):
+            if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"):
+                exists = os.path.exists(executable)
+                if exists:
+                    stat = os.stat(executable)
+                else:
+                    stat = None
+                raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat))
+            if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK):
+                raise InvalidCommandError("Target program is not executable: {}".format(executable))
+
+    if check_sanitizer:
+        env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100")
+
+    if stdin:
+        name = "PIPE" if stdin == subprocess.PIPE else stdin.name
+        yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd)
+    else:
+        yatest_logger.debug("Executing '%s' in '%s'", command, cwd)
+    # XXX
+
+    started = time.time()
+    process = subprocess.Popen(
+        command, shell=shell, universal_newlines=True,
+        stdout=out_file, stderr=err_file, stdin=in_file,
+        cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn,
+    )
+    yatest_logger.debug("Command pid: %s", process.pid)
+
+    res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr)
+    if wait:
+        res.wait(check_exit_code, timeout, on_timeout)
+    return res
+
+
+def _get_command_output_file(cmd, ext):
+    parts = [get_command_name(cmd)]
+    if 'YA_RETRY_INDEX' in os.environ:
+        parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX')))
+    if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1:
+        parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0')))
+
+    filename = '.'.join(parts + [ext])
+    try:
+        # if execution is performed from test, save out / err to the test logs dir
+        import yatest.common
+        import library.python.pytest.plugins.ya
+        if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None:
+            raise ImportError("not in test")
+        filename = path.get_unique_file_path(yatest.common.output_path(), filename)
+        yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename))
+        return open(filename, "wb+")
+    except ImportError:
+        return tempfile.NamedTemporaryFile(delete=False, suffix=filename)
+
+
+def _get_proc_tree_info(pids):
+    if os.name == 'nt':
+        return 'Not supported'
+    else:
+        stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        return stdout
+
+
+def py_execute(
+    command, check_exit_code=True,
+    shell=False, timeout=None,
+    cwd=None, env=None,
+    stdin=None, stdout=None, stderr=None,
+    creationflags=0, wait=True,
+    process_progress_listener=None, close_fds=False
+):
+    """
+    Executes a command with the arcadia python
+    :param command: command to pass to python
+    :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+    :param shell: use shell to run the command
+    :param timeout: execution timeout
+    :param cwd: working directory
+    :param env: command environment
+    :param stdin: command stdin
+    :param stdout: command stdout
+    :param stderr: command stderr
+    :param creationflags: command creation flags
+    :param wait: should wait until the command finishes
+    :param process_progress_listener=object that is polled while execution is in progress
+    :return _Execution: Execution object
+    """
+    if isinstance(command, six.string_types):
+        command = [command]
+    command = [runtime.python_path()] + command
+    if shell:
+        command = " ".join(command)
+    return execute(**locals())
+
+
+def _format_error(error):
+    return truncate(error, MAX_MESSAGE_LEN)
+
+
+def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None):
+    """
+    Tries to execute `check_function` for `timeout` seconds.
+    Continue until function returns nonfalse value.
+    If function doesn't return nonfalse value for `timeout` seconds
+    OperationTimeoutException is raised.
+    Return first nonfalse result returned by `checkFunction`.
+    """
+    if sleep_time <= 0:
+        raise ValueError("Incorrect sleep time value {}".format(sleep_time))
+    if timeout < 0:
+        raise ValueError("Incorrect timeout value {}".format(timeout))
+    start = time.time()
+    while start + timeout > time.time():
+        if on_check_condition:
+            on_check_condition()
+
+        res = check_function()
+        if res:
+            return res
+        time.sleep(sleep_time)
+
+    message = "{} second(s) wait timeout has expired".format(timeout)
+    if fail_message:
+        message += ": {}".format(fail_message)
+    raise TimeoutError(truncate(message, MAX_MESSAGE_LEN))
+
+
+def _kill_process_tree(process_pid, target_pid_signal=None):
+    """
+    Kills child processes, req. Note that psutil should be installed
+    @param process_pid: parent id to search for descendants
+    """
+    yatest_logger.debug("Killing process %s", process_pid)
+    if os.name == 'nt':
+        _win_kill_process_tree(process_pid)
+    else:
+        _nix_kill_process_tree(process_pid, target_pid_signal)
+
+
+def _nix_get_proc_children(pid):
+    try:
+        cmd = ["pgrep", "-P", str(pid)]
+        return [int(p) for p in subprocess.check_output(cmd).split()]
+    except Exception:
+        return []
+
+
+def _get_binname(pid):
+    try:
+        return os.path.basename(os.readlink('/proc/{}/exe'.format(pid)))
+    except Exception as e:
+        return "error({})".format(e)
+
+
+def _nix_kill_process_tree(pid, target_pid_signal=None):
+    """
+    Kills the process tree.
+    """
+    yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid)))
+
+    def try_to_send_signal(pid, sig):
+        try:
+            os.kill(pid, sig)
+            yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
+        except Exception as exc:
+            yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
+
+    try_to_send_signal(pid, signal.SIGSTOP)  # Stop the process to prevent it from starting any child processes.
+
+    # Get the child process PID list.
+    child_pids = _nix_get_proc_children(pid)
+    # Stop the child processes.
+    for child_pid in child_pids:
+        try:
+            # Kill the child recursively.
+            _kill_process_tree(int(child_pid))
+        except Exception as e:
+            # Skip the error and continue killing.
+            yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e))
+            continue
+
+    try_to_send_signal(pid, target_pid_signal or signal.SIGKILL)  # Kill the root process.
+
+    # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent
+    # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html
+    try_to_send_signal(pid, signal.SIGCONT)
+
+
+def _win_kill_process_tree(pid):
+    subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)])
+
+
+def _run_readelf(binary_path):
+    return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)]))
+
+
+def check_glibc_version(binary_path):
+    lucid_glibc_version = distutils.version.LooseVersion("2.11")
+
+    for l in _run_readelf(binary_path).split('\n'):
+        match = GLIBC_PATTERN.search(l)
+        if not match:
+            continue
+        assert distutils.version.LooseVersion(match.group(1)) <= lucid_glibc_version, match.group(0)
+
+
+def backtrace_to_html(bt_filename, output):
+    try:
+        from library.python import coredump_filter
+        with open(output, "wb") as afile:
+            coredump_filter.filter_stackdump(bt_filename, stream=afile)
+    except ImportError as e:
+        yatest_logger.debug("Failed to import coredump_filter: %s", e)
+        with open(output, "wb") as afile:
+            afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>")
+
+
+def _try_convert_bytes_to_string(source):
+    """ Function is necessary while this code Python2/3 compatible, because bytes in Python3 is a real bytes and in Python2 is not """
+    # Bit ugly typecheck, because in Python2 isinstance(str(), bytes) and "type(str()) is bytes" working as True as well
+    if 'bytes' not in str(type(source)):
+        # We already got not bytes. Nothing to do here.
+        return source, False
+
+    result = source
+    error = False
+    try:
+        result = source.decode(encoding='utf-8')
+    except ValueError as e:
+        error = e
+
+    return result, error
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
new file mode 100644
index 0000000000..e55e193446
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -0,0 +1,343 @@
+import errno
+import functools
+import json
+import os
+import threading
+
+import six
+
+
+_lock = threading.Lock()
+
+
+def _get_ya_config():
+    try:
+        import library.python.pytest.plugins.ya as ya_plugin
+        if ya_plugin.pytest_config is not None:
+            return ya_plugin.pytest_config
+        import pytest
+        return pytest.config
+    except (ImportError, AttributeError):
+        try:
+            import library.python.testing.recipe
+            if library.python.testing.recipe.ya:
+                return library.python.testing.recipe
+        except (ImportError, AttributeError):
+            pass
+        raise NotImplementedError("yatest.common.* is only available from the testing runtime")
+
+
+def _get_ya_plugin_instance():
+    return _get_ya_config().ya
+
+
+def _norm_path(path):
+    if path is None:
+        return None
+    assert isinstance(path, six.string_types)
+    if "\\" in path:
+        raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path))
+    return os.path.normpath(path)
+
+
+def _join_path(main_path, path):
+    if not path:
+        return main_path
+    return os.path.join(main_path, _norm_path(path))
+
+
+def not_test(func):
+    """
+    Mark any function as not a test for py.test
+    :param func:
+    :return:
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        return func(*args, **kwds)
+    setattr(wrapper, '__test__', False)
+    return wrapper
+
+
+def source_path(path=None):
+    """
+    Get source path inside arcadia
+    :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya')
+    :return: absolute path to the source folder
+    """
+    return _join_path(_get_ya_plugin_instance().source_root, path)
+
+
+def build_path(path=None):
+    """
+    Get path inside build directory
+    :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin')
+    :return: absolute path inside build directory
+    """
+    return _join_path(_get_ya_plugin_instance().build_root, path)
+
+
+def java_path():
+    """
+    [DEPRECATED] Get path to java
+    :return: absolute path to java
+    """
+    from . import runtime_java
+    return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk')))
+
+
+def java_home():
+    """
+    Get jdk directory path
+    """
+    from . import runtime_java
+    jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk'))
+    if not jdk_dir:
+        raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform")
+    return jdk_dir
+
+
+def java_bin():
+    """
+    Get path to the java binary
+    Requires DEPENDS(jdk)
+    """
+    return os.path.join(java_home(), "bin", "java")
+
+
+def data_path(path=None):
+    """
+    Get path inside arcadia_tests_data directory
+    :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
+    :return: absolute path inside arcadia_tests_data
+    """
+    return _join_path(_get_ya_plugin_instance().data_root, path)
+
+
+def output_path(path=None):
+    """
+    Get path inside the current test suite output dir.
+    Placing files to this dir guarantees that files will be accessible after the test suite execution.
+    :param path: path relative to the test suite output dir
+    :return: absolute path inside the test suite output dir
+    """
+    return _join_path(_get_ya_plugin_instance().output_dir, path)
+
+
+def ram_drive_path(path=None):
+    """
+    :param path: path relative to the ram drive.
+    :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment.
+    """
+    if 'YA_TEST_RAM_DRIVE_PATH' in os.environ:
+        return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path)
+
+
+def output_ram_drive_path(path=None):
+    """
+    Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing.
+    Returns None if no ram drive was provided by environment.
+    :param path: path relative to the output ram drive directory
+    """
+    if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ:
+        return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path)
+
+
+def binary_path(path=None):
+    """
+    Get path to the built binary
+    :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin')
+    :return: absolute path to the binary
+    """
+    path = _norm_path(path)
+    return _get_ya_plugin_instance().get_binary(path)
+
+
+def work_path(path=None):
+    """
+    Get path inside the current test suite working directory. Creating files in the work directory does not guarantee
+    that files will be accessible after the test suite execution
+    :param path: path relative to the test suite working dir
+    :return: absolute path inside the test suite working dir
+    """
+    return _join_path(
+        os.environ.get("TEST_WORK_PATH") or
+            _get_ya_plugin_instance().get_context("work_path") or
+            os.getcwd(),
+        path)
+
+
+def python_path():
+    """
+    Get path to the arcadia python.
+
+    Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles
+    are built in a stripped-down form that is needed for building, not running tests.
+    See comments in the file below to find out which version of python is compatible with tests.
+    https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc
+    :return: absolute path to python
+    """
+    return _get_ya_plugin_instance().python_path
+
+
+def valgrind_path():
+    """
+    Get path to valgrind
+    :return: absolute path to valgrind
+    """
+    return _get_ya_plugin_instance().valgrind_path
+
+
+def get_param(key, default=None):
+    """
+    Get arbitrary parameter passed via command line
+    :param key: key
+    :param default: default value
+    :return: parameter value or the default
+    """
+    return _get_ya_plugin_instance().get_param(key, default)
+
+
+def get_param_dict_copy():
+    """
+    Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters.
+
+    :return: copy of dictionary with all parameters
+    """
+    return _get_ya_plugin_instance().get_param_dict_copy()
+
+
+@not_test
+def test_output_path(path=None):
+    """
+    Get dir in the suite output_path for the current test case
+    """
+    test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0]
+    try:
+        os.makedirs(test_out_dir)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+    return _join_path(test_out_dir, path)
+
+
+def project_path(path=None):
+    """
+    Get path in build root relating to build_root/project path
+    """
+    return _join_path(os.path.join(build_path(), context.project_path), path)
+
+
+def gdb_path():
+    """
+    Get path to the gdb
+    """
+    return _get_ya_plugin_instance().gdb_path
+
+
+def c_compiler_path():
+    """
+    Get path to the gdb
+    """
+    return os.environ.get("YA_CC")
+
+
+def get_yt_hdd_path(path=None):
+    if 'HDD_PATH' in os.environ:
+        return _join_path(os.environ['HDD_PATH'], path)
+
+
+def cxx_compiler_path():
+    """
+    Get path to the gdb
+    """
+    return os.environ.get("YA_CXX")
+
+
+def global_resources():
+    try:
+        return json.loads(os.environ.get("YA_GLOBAL_RESOURCES"))
+    except (TypeError, ValueError):
+        return {}
+
+
+def _register_core(name, binary_path, core_path, bt_path, pbt_path):
+    config = _get_ya_config()
+
+    with _lock:
+        if not hasattr(config, 'test_cores_count'):
+            config.test_cores_count = 0
+        config.test_cores_count += 1
+        count_str = '' if config.test_cores_count == 1 else str(config.test_cores_count)
+
+    log_entry = config.test_logs[config.current_item_nodeid]
+    if binary_path:
+        log_entry['{} binary{}'.format(name, count_str)] = binary_path
+    if core_path:
+        log_entry['{} core{}'.format(name, count_str)] = core_path
+    if bt_path:
+        log_entry['{} backtrace{}'.format(name, count_str)] = bt_path
+    if pbt_path:
+        log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path
+
+
+@not_test
+def test_source_path(path=None):
+    return _join_path(os.path.join(source_path(), context.project_path), path)
+
+
+class Context(object):
+    """
+    Runtime context
+    """
+
+    @property
+    def build_type(self):
+        return _get_ya_plugin_instance().get_context("build_type")
+
+    @property
+    def project_path(self):
+        return _get_ya_plugin_instance().get_context("project_path")
+
+    @property
+    def test_stderr(self):
+        return _get_ya_plugin_instance().get_context("test_stderr")
+
+    @property
+    def test_debug(self):
+        return _get_ya_plugin_instance().get_context("test_debug")
+
+    @property
+    def test_traceback(self):
+        return _get_ya_plugin_instance().get_context("test_traceback")
+
+    @property
+    def test_name(self):
+        return _get_ya_config().current_test_name
+
+    @property
+    def sanitize(self):
+        """
+        Detect if current test run is under sanitizer
+
+        :return: one of `None`, 'address', 'memory', 'thread', 'undefined'
+        """
+        return _get_ya_plugin_instance().get_context("sanitize")
+
+    @property
+    def flags(self):
+        _flags = _get_ya_plugin_instance().get_context("flags")
+        if _flags:
+            _flags_dict = dict()
+            for f in _flags:
+                key, value = f.split('=', 1)
+                _flags_dict[key] = value
+            return _flags_dict
+        else:
+            return dict()
+
+    def get_context_key(self, key):
+        return _get_ya_plugin_instance().get_context(key)
+
+
+context = Context()
diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py
new file mode 100644
index 0000000000..39bbb45570
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py
@@ -0,0 +1,46 @@
+import os
+import tarfile
+import contextlib
+
+from . import runtime
+
+_JAVA_DIR = []
+
+
+def get_java_path(jdk_dir):
+    # deprecated - to be deleted
+    java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe'))
+
+    for p in java_paths:
+        if os.path.exists(p):
+            return p
+
+    for f in os.listdir(jdk_dir):
+        if f.endswith('.tar'):
+            with contextlib.closing(tarfile.open(os.path.join(jdk_dir, f))) as tf:
+                tf.extractall(jdk_dir)
+
+    for p in java_paths:
+        if os.path.exists(p):
+            return p
+
+    return ''
+
+
+def get_build_java_dir(jdk_dir):
+    versions = [8, 10, 11, 12, 13, 14, 15]
+
+    if not _JAVA_DIR:
+        for version in versions:
+            jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version))
+            if os.path.exists(jdk_tar_path):
+                jdk_dir = runtime.build_path('jdk4test')
+                with contextlib.closing(tarfile.open(jdk_tar_path)) as tf:
+                    tf.extractall(jdk_dir)
+                assert os.path.exists(os.path.join(jdk_dir, "bin", "java"))
+                _JAVA_DIR.append(jdk_dir)
+                break
+        else:
+            _JAVA_DIR.append(None)
+
+    return _JAVA_DIR[0]
diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py
new file mode 100644
index 0000000000..9e7a74cdf5
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/tags.py
@@ -0,0 +1,5 @@
+try:
+    import pytest
+    ya_external = getattr(pytest.mark, "ya:external")
+except ImportError:
+    ya_external = None
diff --git a/library/python/testing/yatest_common/yatest/common/ya.make b/library/python/testing/yatest_common/yatest/common/ya.make
new file mode 100644
index 0000000000..f7c50dfe64
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/ya.make
@@ -0,0 +1 @@
+OWNER(g:yatest)