[library/python/testing/yatest_common] Relaxed runtime methods for non-standalone runs

1 files changed, 132 insertions, 49 deletions

diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py
index c2dec1f7c12..66a1b9d56b4 100644
--- a/library/python/testing/yatest_common/yatest/common/process.py
+++ b/library/python/testing/yatest_common/yatest/common/process.py
@@ -38,7 +38,7 @@ def truncate(s, size):
     elif len(s) <= size:
         return s
     else:
-        return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):]
+        return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3) :]
 
 
 def get_command_name(command):
@@ -46,19 +46,19 @@ def get_command_name(command):
 
 
 class ExecutionError(Exception):
-
     def __init__(self, execution_result):
         if not isinstance(execution_result.command, six.string_types):
             command = " ".join(str(arg) for arg in execution_result.command)
         else:
             command = execution_result.command
         message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format(
-            command=command,
-            code=execution_result.exit_code,
-            err=_format_error(execution_result.std_err))
+            command=command, code=execution_result.exit_code, err=_format_error(execution_result.std_err)
+        )
         if cores:
             if execution_result.backtrace:
-                message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
+                message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(
+                    cores.colorize_backtrace(execution_result._backtrace)
+                )
         else:
             message += "Backtrace is not available: module cores isn't available"
 
@@ -91,11 +91,21 @@ class InvalidCommandError(Exception):
 
 
 class _Execution(object):
-
-    def __init__(self, command, process, out_file, err_file,
-                 process_progress_listener=None, cwd=None, collect_cores=True,
-                 check_sanitizer=True, started=0, user_stdout=False, user_stderr=False,
-                 core_pattern=None):
+    def __init__(
+        self,
+        command,
+        process,
+        out_file,
+        err_file,
+        process_progress_listener=None,
+        cwd=None,
+        collect_cores=True,
+        check_sanitizer=True,
+        started=0,
+        user_stdout=False,
+        user_stderr=False,
+        core_pattern=None,
+    ):
 
         self._command = command
         self._process = process
@@ -132,7 +142,12 @@ class _Execution(object):
             # DEVTOOLS-2347
             yatest_logger.debug("Process status before wait_for: %s", self.running)
             try:
-                wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1)
+                wait_for(
+                    lambda: not self.running,
+                    timeout=5,
+                    fail_message="Could not kill process {}".format(self._process.pid),
+                    sleep_time=0.1,
+                )
             except TimeoutError:
                 yatest_logger.debug("Process status after wait_for: %s", self.running)
                 yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
@@ -264,16 +279,14 @@ class _Execution(object):
             self._out_file = None
 
     def _recover_core(self):
-        core_path = cores.recover_core_dump_file(
-            self.command[0],
-            self._cwd,
-            self.process.pid,
-            self.core_pattern)
+        core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid, self.core_pattern)
         if core_path:
             # Core dump file recovering may be disabled (for distbuild for example) - produce only bt
             store_cores = runtime._get_ya_config().collect_cores
             if store_cores:
-                new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
+                new_core_path = path.get_unique_file_path(
+                    runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)
+                )
                 # Copy core dump file, because it may be overwritten
                 yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
                 shutil.copyfile(core_path, new_core_path)
@@ -284,7 +297,10 @@ class _Execution(object):
 
             if os.path.exists(runtime.gdb_path()):
                 self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path())
-                bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid))
+                bt_filename = path.get_unique_file_path(
+                    runtime.output_path(),
+                    "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid),
+                )
                 with open(bt_filename, "wb") as afile:
                     afile.write(six.ensure_binary(self._backtrace))
                 # generate pretty html version of backtrace aka Tri Korochki
@@ -292,7 +308,9 @@ class _Execution(object):
                 backtrace_to_html(bt_filename, pbt_filename)
 
             if store_cores:
-                runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename)
+                runtime._register_core(
+                    os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename
+                )
             else:
                 runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
 
@@ -333,7 +351,9 @@ class _Execution(object):
                     except OSError as exc:
 
                         if exc.errno == errno.ECHILD:
-                            yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
+                            yatest_logger.debug(
+                                "Process resource usage is not available as process finished before wait4 was called"
+                            )
                         else:
                             raise
             except SignalInterruptionError:
@@ -352,7 +372,13 @@ class _Execution(object):
                 process_is_finished = lambda: not self.running
                 fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
                 try:
-                    wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
+                    wait_for(
+                        process_is_finished,
+                        timeout,
+                        fail_message,
+                        sleep_time=0.1,
+                        on_check_condition=self._process_progress_listener,
+                    )
                 except TimeoutError as e:
                     if on_timeout:
                         yatest_logger.debug("Calling user specified on_timeout function")
@@ -378,8 +404,12 @@ class _Execution(object):
     def _finalise(self, check_exit_code):
         # Set the signal (negative number) which caused the process to exit
         if check_exit_code and self.exit_code != 0:
-            yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
-                                self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
+            yatest_logger.error(
+                "Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
+                self.exit_code,
+                truncate(self.std_out, MAX_OUT_LEN),
+                truncate(self.std_err, MAX_OUT_LEN),
+            )
             raise ExecutionError(self)
 
         # Don't search for sanitize errors if stderr was redirected
@@ -407,12 +437,18 @@ class _Execution(object):
             if self.command[0].startswith(build_path):
                 match = re.search(SANITIZER_ERROR_PATTERN, self._std_err)
                 if match:
-                    yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
+                    yatest_logger.error(
+                        "%s sanitizer found errors:\n\tstd_err:%s\n",
+                        match.group(1),
+                        truncate(self.std_err, MAX_OUT_LEN),
+                    )
                     raise ExecutionError(self)
                 else:
                     yatest_logger.debug("No sanitizer errors found")
             else:
-                yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
+                yatest_logger.debug(
+                    "'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path
+                )
 
 
 def on_timeout_gen_coredump(exec_obj, _):
@@ -429,13 +465,23 @@ def on_timeout_gen_coredump(exec_obj, _):
 
 
 def execute(
-    command, check_exit_code=True,
-    shell=False, timeout=None,
-    cwd=None, env=None,
-    stdin=None, stdout=None, stderr=None,
-    creationflags=0, wait=True,
-    process_progress_listener=None, close_fds=False,
-    collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
+    command,
+    check_exit_code=True,
+    shell=False,
+    timeout=None,
+    cwd=None,
+    env=None,
+    stdin=None,
+    stdout=None,
+    stderr=None,
+    creationflags=0,
+    wait=True,
+    process_progress_listener=None,
+    close_fds=False,
+    collect_cores=True,
+    check_sanitizer=True,
+    preexec_fn=None,
+    on_timeout=None,
     executor=_Execution,
     core_pattern=None,
     popen_kwargs=None,
@@ -527,7 +573,9 @@ def execute(
                     stat = os.stat(executable)
                 else:
                     stat = None
-                raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat))
+                raise InvalidCommandError(
+                    "Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)
+                )
             if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK):
                 raise InvalidCommandError("Target program is not executable: {}".format(executable))
 
@@ -543,9 +591,17 @@ def execute(
 
     started = time.time()
     process = subprocess.Popen(
-        command, shell=shell, universal_newlines=True,
-        stdout=out_file, stderr=err_file, stdin=in_file,
-        cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn,
+        command,
+        shell=shell,
+        universal_newlines=True,
+        stdout=out_file,
+        stderr=err_file,
+        stdin=in_file,
+        cwd=cwd,
+        env=env,
+        creationflags=creationflags,
+        close_fds=close_fds,
+        preexec_fn=preexec_fn,
         **popen_kwargs
     )
     yatest_logger.debug("Command pid: %s", process.pid)
@@ -563,8 +619,18 @@ def execute(
     if 'core_pattern' in executor_args:
         kwargs.update([('core_pattern', core_pattern)])
 
-    res = executor(command, process, out_file, err_file, process_progress_listener,
-                   cwd, collect_cores, check_sanitizer, started, **kwargs)
+    res = executor(
+        command,
+        process,
+        out_file,
+        err_file,
+        process_progress_listener,
+        cwd,
+        collect_cores,
+        check_sanitizer,
+        started,
+        **kwargs
+    )
     if wait:
         res.wait(check_exit_code, timeout, on_timeout)
     return res
@@ -582,6 +648,7 @@ def _get_command_output_file(cmd, ext):
         # if execution is performed from test, save out / err to the test logs dir
         import yatest.common
         import library.python.pytest.plugins.ya
+
         if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None:
             raise ImportError("not in test")
         filename = path.get_unique_file_path(yatest.common.output_path(), filename)
@@ -595,17 +662,26 @@ def _get_proc_tree_info(pids):
     if os.name == 'nt':
         return 'Not supported'
     else:
-        stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        stdout, _ = subprocess.Popen(
+            ["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        ).communicate()
         return stdout
 
 
 def py_execute(
-    command, check_exit_code=True,
-    shell=False, timeout=None,
-    cwd=None, env=None,
-    stdin=None, stdout=None, stderr=None,
-    creationflags=0, wait=True,
-    process_progress_listener=None, close_fds=False
+    command,
+    check_exit_code=True,
+    shell=False,
+    timeout=None,
+    cwd=None,
+    env=None,
+    stdin=None,
+    stdout=None,
+    stderr=None,
+    creationflags=0,
+    wait=True,
+    process_progress_listener=None,
+    close_fds=False,
 ):
     """
     Executes a command with the arcadia python
@@ -701,7 +777,9 @@ def _nix_kill_process_tree(pid, target_pid_signal=None):
             os.kill(pid, sig)
             yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
         except Exception as exc:
-            yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
+            yatest_logger.debug(
+                "Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))
+            )
 
     try_to_send_signal(pid, signal.SIGSTOP)  # Stop the process to prevent it from starting any child processes.
 
@@ -729,7 +807,11 @@ def _win_kill_process_tree(pid):
 
 
 def _run_readelf(binary_path):
-    return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)]))
+    return str(
+        subprocess.check_output(
+            [runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)]
+        )
+    )
 
 
 def check_glibc_version(binary_path):
@@ -745,6 +827,7 @@ def check_glibc_version(binary_path):
 def backtrace_to_html(bt_filename, output):
     try:
         from library.python import coredump_filter
+
         with open(output, "w") as afile:
             coredump_filter.filter_stackdump(bt_filename, stream=afile)
     except ImportError as e:
@@ -754,7 +837,7 @@ def backtrace_to_html(bt_filename, output):
 
 
 def _try_convert_bytes_to_string(source):
-    """ Function is necessary while this code Python2/3 compatible, because bytes in Python3 is a real bytes and in Python2 is not """
+    """Function is necessary while this code Python2/3 compatible, because bytes in Python3 is a real bytes and in Python2 is not"""
     # Bit ugly typecheck, because in Python2 isinstance(str(), bytes) and "type(str()) is bytes" working as True as well
     if 'bytes' not in str(type(source)):
         # We already got not bytes. Nothing to do here.