diff options
author | Aleksandr <[email protected]> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:47:52 +0300 |
commit | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (patch) | |
tree | d16cef493ac1e092b4a03ab9437ec06ffe3d188f /library/python/testing/yatest_common/yatest/common/process.py | |
parent | 37de222addabbef336dcaaea5f7c7645a629fc6d (diff) |
Restoring authorship annotation for Aleksandr <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'library/python/testing/yatest_common/yatest/common/process.py')
-rw-r--r-- | library/python/testing/yatest_common/yatest/common/process.py | 634 |
1 files changed, 317 insertions, 317 deletions
diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py index a8bcc21f518..f9eeb16525c 100644 --- a/library/python/testing/yatest_common/yatest/common/process.py +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -1,10 +1,10 @@ -# coding: utf-8 - +# coding: utf-8 + import os -import re +import re import time import signal -import shutil +import shutil import logging import tempfile import subprocess @@ -13,19 +13,19 @@ import distutils.version import six -try: - # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work. - import library.python.cores as cores -except ImportError: - cores = None - +try: + # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work. + import library.python.cores as cores +except ImportError: + cores = None + from . import runtime from . import path from . import environment MAX_OUT_LEN = 1000 * 1000 # 1 mb -MAX_MESSAGE_LEN = 1500 +MAX_MESSAGE_LEN = 1500 SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)" GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)") yatest_logger = logging.getLogger("ya.test") @@ -55,12 +55,12 @@ class ExecutionError(Exception): command=command, code=execution_result.exit_code, err=_format_error(execution_result.std_err)) - if cores: - if execution_result.backtrace: - message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) - else: - message += "Backtrace is not available: module cores isn't available" - + if cores: + if execution_result.backtrace: + message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) + else: + message += "Backtrace is not available: module cores isn't available" + super(ExecutionError, self).__init__(message) self.execution_result = execution_result @@ -85,31 +85,31 @@ class SignalInterruptionError(Exception): self.res = None -class InvalidCommandError(Exception): - pass - - +class InvalidCommandError(Exception): + pass + + class _Execution(object): - def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): + def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): self._command = command self._process = process self._out_file = out_file self._err_file = err_file self._std_out = None self._std_err = None - self._elapsed = None - self._start = time.time() + self._elapsed = None + self._start = time.time() self._process_progress_listener = process_progress_listener - self._cwd = cwd or os.getcwd() - self._collect_cores = collect_cores - self._backtrace = '' - self._check_sanitizer = check_sanitizer + self._cwd = cwd or os.getcwd() + self._collect_cores = collect_cores + self._backtrace = '' + self._check_sanitizer = check_sanitizer self._metrics = {} - self._started = started - self._user_stdout = bool(user_stdout) - self._user_stderr = bool(user_stderr) - self._exit_code = None + self._started = started + self._user_stdout = bool(user_stdout) + self._user_stderr = bool(user_stderr) + self._exit_code = None if process_progress_listener: process_progress_listener.open(command, process, out_file, err_file) @@ -122,21 +122,21 @@ class _Execution(object): self._save_outputs(False) _kill_process_tree(self._process.pid) self._clean_files() - # DEVTOOLS-2347 - yatest_logger.debug("Process status before wait_for: %s", self.running) - try: - wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1) - except TimeoutError: - yatest_logger.debug("Process status after wait_for: %s", self.running) - yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) - raise + # DEVTOOLS-2347 + yatest_logger.debug("Process status before wait_for: %s", self.running) + try: + wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1) + except TimeoutError: + yatest_logger.debug("Process status after wait_for: %s", self.running) + yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) + raise else: raise InvalidExecutionStateError("Cannot kill a stopped process") - def terminate(self): - if self.running: - self._process.terminate() - + def terminate(self): + if self.running: + self._process.terminate() + @property def process(self): return self._process @@ -146,56 +146,56 @@ class _Execution(object): return self._command @property - def returncode(self): - return self.exit_code - - @property + def returncode(self): + return self.exit_code + + @property def exit_code(self): - """ - Deprecated, use returncode - """ - if self._exit_code is None: - self._exit_code = self._process.returncode - return self._exit_code - - @property - def stdout(self): - return self.std_out + """ + Deprecated, use returncode + """ + if self._exit_code is None: + self._exit_code = self._process.returncode + return self._exit_code @property + def stdout(self): + return self.std_out + + @property def std_out(self): - """ - Deprecated, use stdout - """ + """ + Deprecated, use stdout + """ if self._std_out is not None: return self._std_out - if self._process.stdout and not self._user_stdout: - self._std_out = self._process.stdout.read() - return self._std_out - - @property - def stderr(self): - return self.std_err + if self._process.stdout and not self._user_stdout: + self._std_out = self._process.stdout.read() + return self._std_out @property + def stderr(self): + return self.std_err + + @property def std_err(self): - """ - Deprecated, use stderr - """ + """ + Deprecated, use stderr + """ if self._std_err is not None: return self._std_err - if self._process.stderr and not self._user_stderr: - self._std_err = self._process.stderr.read() - return self._std_err - - @property - def elapsed(self): - return self._elapsed - - @property - def backtrace(self): - return self._backtrace - + if self._process.stderr and not self._user_stderr: + self._std_err = self._process.stderr.read() + return self._std_err + + @property + def elapsed(self): + return self._elapsed + + @property + def backtrace(self): + return self._backtrace + @property def metrics(self): return self._metrics @@ -204,25 +204,25 @@ class _Execution(object): if self._process_progress_listener: self._process_progress_listener() self._process_progress_listener.close() - if not self._user_stdout: - if self._out_file is None: - pass - elif self._out_file != subprocess.PIPE: - self._out_file.flush() - self._out_file.seek(0, os.SEEK_SET) - self._std_out = self._out_file.read() - else: - self._std_out = self._process.stdout.read() - if not self._user_stderr: - if self._err_file is None: - pass - elif self._err_file != subprocess.PIPE: - self._err_file.flush() - self._err_file.seek(0, os.SEEK_SET) - self._std_err = self._err_file.read() - else: - self._std_err = self._process.stderr.read() - + if not self._user_stdout: + if self._out_file is None: + pass + elif self._out_file != subprocess.PIPE: + self._out_file.flush() + self._out_file.seek(0, os.SEEK_SET) + self._std_out = self._out_file.read() + else: + self._std_out = self._process.stdout.read() + if not self._user_stderr: + if self._err_file is None: + pass + elif self._err_file != subprocess.PIPE: + self._err_file.flush() + self._err_file.seek(0, os.SEEK_SET) + self._std_err = self._err_file.read() + else: + self._std_err = self._process.stderr.read() + if clean_files: self._clean_files() yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) @@ -245,45 +245,45 @@ class _Execution(object): yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN)) def _clean_files(self): - if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: - self._err_file.close() - self._err_file = None - if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: - self._out_file.close() - self._out_file = None - - def _recover_core(self): - core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) - if core_path: - # Core dump file recovering may be disabled (for distbuild for example) - produce only bt - store_cores = runtime._get_ya_config().collect_cores - if store_cores: - new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) - # Copy core dump file, because it may be overwritten + if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: + self._err_file.close() + self._err_file = None + if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: + self._out_file.close() + self._out_file = None + + def _recover_core(self): + core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) + if core_path: + # Core dump file recovering may be disabled (for distbuild for example) - produce only bt + store_cores = runtime._get_ya_config().collect_cores + if store_cores: + new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) + # Copy core dump file, because it may be overwritten yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) - shutil.copyfile(core_path, new_core_path) - core_path = new_core_path - - bt_filename = None - pbt_filename = None - - if os.path.exists(runtime.gdb_path()): - self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path()) - bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid)) + shutil.copyfile(core_path, new_core_path) + core_path = new_core_path + + bt_filename = None + pbt_filename = None + + if os.path.exists(runtime.gdb_path()): + self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path()) + bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid)) with open(bt_filename, "wb") as afile: afile.write(six.ensure_binary(self._backtrace)) - # generate pretty html version of backtrace aka Tri Korochki - pbt_filename = bt_filename + ".html" + # generate pretty html version of backtrace aka Tri Korochki + pbt_filename = bt_filename + ".html" backtrace_to_html(bt_filename, pbt_filename) - - if store_cores: - runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename) - else: - runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) - + + if store_cores: + runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename) + else: + runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) + def wait(self, check_exit_code=True, timeout=None, on_timeout=None): def _wait(): - finished = None + finished = None interrupted = False try: if hasattr(os, "wait4"): @@ -293,7 +293,7 @@ class _Execution(object): else: # PEP 475 pid, sts, rusage = os.wait4(self._process.pid, 0) - finished = time.time() + finished = time.time() self._process._handle_exitstatus(sts) for field in [ "ru_idrss", @@ -328,46 +328,46 @@ class _Execution(object): if not interrupted: self._process.wait() # this has to be here unconditionally, so that all process properties are set - if not finished: - finished = time.time() - self._metrics["wtime"] = round(finished - self._started, 3) - + if not finished: + finished = time.time() + self._metrics["wtime"] = round(finished - self._started, 3) + try: - if timeout: - process_is_finished = lambda: not self.running - fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) + if timeout: + process_is_finished = lambda: not self.running + fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) try: wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) except TimeoutError as e: if on_timeout: - yatest_logger.debug("Calling user specified on_timeout function") + yatest_logger.debug("Calling user specified on_timeout function") try: on_timeout(self, timeout) except Exception: yatest_logger.exception("Exception while calling on_timeout") raise ExecutionTimeoutError(self, str(e)) - # Wait should be always called here, it finalizes internal states of its process and sets up return code + # Wait should be always called here, it finalizes internal states of its process and sets up return code _wait() - except BaseException as e: + except BaseException as e: _kill_process_tree(self._process.pid) _wait() yatest_logger.debug("Command exception: %s", e) - raise + raise finally: - self._elapsed = time.time() - self._start - self._save_outputs() + self._elapsed = time.time() - self._start + self._save_outputs() self.verify_no_coredumps() - self._finalise(check_exit_code) - - def _finalise(self, check_exit_code): + self._finalise(check_exit_code) + + def _finalise(self, check_exit_code): # Set the signal (negative number) which caused the process to exit if check_exit_code and self.exit_code != 0: yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) raise ExecutionError(self) - # Don't search for sanitize errors if stderr was redirected + # Don't search for sanitize errors if stderr was redirected self.verify_sanitize_errors() def verify_no_coredumps(self): @@ -375,44 +375,44 @@ class _Execution(object): Verify there is no coredump from this binary. If there is then report backtrace. """ if self.exit_code < 0 and self._collect_cores: - if cores: - try: - self._recover_core() - except Exception: - yatest_logger.exception("Exception while recovering core") - else: - yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available") + if cores: + try: + self._recover_core() + except Exception: + yatest_logger.exception("Exception while recovering core") + else: + yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available") def verify_sanitize_errors(self): """ Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them. """ - if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks: - build_path = runtime.build_path() - if self.command[0].startswith(build_path): - match = re.search(SANITIZER_ERROR_PATTERN, self._std_err) - if match: - yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) - raise ExecutionError(self) + if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks: + build_path = runtime.build_path() + if self.command[0].startswith(build_path): + match = re.search(SANITIZER_ERROR_PATTERN, self._std_err) + if match: + yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) else: yatest_logger.debug("No sanitizer errors found") - else: + else: yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) - -def on_timeout_gen_coredump(exec_obj, _): - """ - Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump) - to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout. - All files will be available in the testing_out_stuff and via links. - """ - try: - os.kill(exec_obj.process.pid, signal.SIGQUIT) - except OSError: - # process might be already terminated - pass - - + +def on_timeout_gen_coredump(exec_obj, _): + """ + Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump) + to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout. + All files will be available in the testing_out_stuff and via links. + """ + try: + os.kill(exec_obj.process.pid, signal.SIGQUIT) + except OSError: + # process might be already terminated + pass + + def execute( command, check_exit_code=True, shell=False, timeout=None, @@ -421,7 +421,7 @@ def execute( creationflags=0, wait=True, process_progress_listener=None, close_fds=False, collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, - executor=_Execution, + executor=_Execution, ): """ Executes a command @@ -438,26 +438,26 @@ def execute( :param wait: should wait until the command finishes :param process_progress_listener=object that is polled while execution is in progress :param close_fds: subrpocess.Popen close_fds args - :param collect_cores: recover core dump files if shell == False - :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors + :param collect_cores: recover core dump files if shell == False + :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors :param preexec_fn: subrpocess.Popen preexec_fn arg :param on_timeout: on_timeout(<execution object>, <timeout value>) callback :return _Execution: Execution object """ - if env is None: - env = os.environ.copy() + if env is None: + env = os.environ.copy() else: - # Certain environment variables must be present for programs to work properly. - # For more info see DEVTOOLSSUPPORT-4907 - mandatory_env_name = 'YA_MANDATORY_ENV_VARS' - mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or '' - if mandatory_vars: - env[mandatory_env_name] = mandatory_vars - mandatory_system_vars = filter(None, mandatory_vars.split(':')) - else: - mandatory_system_vars = ['TMPDIR'] - + # Certain environment variables must be present for programs to work properly. + # For more info see DEVTOOLSSUPPORT-4907 + mandatory_env_name = 'YA_MANDATORY_ENV_VARS' + mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or '' + if mandatory_vars: + env[mandatory_env_name] = mandatory_vars + mandatory_system_vars = filter(None, mandatory_vars.split(':')) + else: + mandatory_system_vars = ['TMPDIR'] + for var in mandatory_system_vars: if var not in env and var in os.environ: env[var] = os.environ[var] @@ -465,13 +465,13 @@ def execute( if not wait and timeout is not None: raise ValueError("Incompatible arguments 'timeout' and wait=False") - # if subprocess.PIPE in [stdout, stderr]: - # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") - + # if subprocess.PIPE in [stdout, stderr]: + # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") + def get_out_stream(stream, default_name): if stream is None: # No stream is supplied: open new temp file - return _get_command_output_file(command, default_name), False + return _get_command_output_file(command, default_name), False if isinstance(stream, six.string_types): # User filename is supplied: open file for writing @@ -490,86 +490,86 @@ def execute( if shell and type(command) == list: command = " ".join(command) - if shell: - collect_cores = False - check_sanitizer = False - else: - if isinstance(command, (list, tuple)): - executable = command[0] - else: - executable = command - if os.path.isabs(executable): - if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"): - exists = os.path.exists(executable) - if exists: - stat = os.stat(executable) - else: - stat = None - raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)) - if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK): - raise InvalidCommandError("Target program is not executable: {}".format(executable)) - - if check_sanitizer: - env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100") - - if stdin: - name = "PIPE" if stdin == subprocess.PIPE else stdin.name - yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) - else: + if shell: + collect_cores = False + check_sanitizer = False + else: + if isinstance(command, (list, tuple)): + executable = command[0] + else: + executable = command + if os.path.isabs(executable): + if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"): + exists = os.path.exists(executable) + if exists: + stat = os.stat(executable) + else: + stat = None + raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)) + if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK): + raise InvalidCommandError("Target program is not executable: {}".format(executable)) + + if check_sanitizer: + env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100") + + if stdin: + name = "PIPE" if stdin == subprocess.PIPE else stdin.name + yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) + else: yatest_logger.debug("Executing '%s' in '%s'", command, cwd) - # XXX - - started = time.time() - process = subprocess.Popen( - command, shell=shell, universal_newlines=True, - stdout=out_file, stderr=err_file, stdin=in_file, - cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn, - ) - yatest_logger.debug("Command pid: %s", process.pid) - - res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) + # XXX + + started = time.time() + process = subprocess.Popen( + command, shell=shell, universal_newlines=True, + stdout=out_file, stderr=err_file, stdin=in_file, + cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn, + ) + yatest_logger.debug("Command pid: %s", process.pid) + + res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) if wait: res.wait(check_exit_code, timeout, on_timeout) return res -def _get_command_output_file(cmd, ext): - parts = [get_command_name(cmd)] - if 'YA_RETRY_INDEX' in os.environ: - parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX'))) - if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1: - parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0'))) - - filename = '.'.join(parts + [ext]) - try: - # if execution is performed from test, save out / err to the test logs dir - import yatest.common +def _get_command_output_file(cmd, ext): + parts = [get_command_name(cmd)] + if 'YA_RETRY_INDEX' in os.environ: + parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX'))) + if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1: + parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0'))) + + filename = '.'.join(parts + [ext]) + try: + # if execution is performed from test, save out / err to the test logs dir + import yatest.common import library.python.pytest.plugins.ya if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None: - raise ImportError("not in test") - filename = path.get_unique_file_path(yatest.common.output_path(), filename) - yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename)) - return open(filename, "wb+") - except ImportError: - return tempfile.NamedTemporaryFile(delete=False, suffix=filename) - - -def _get_proc_tree_info(pids): - if os.name == 'nt': - return 'Not supported' - else: - stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - return stdout - - -def py_execute( - command, check_exit_code=True, - shell=False, timeout=None, - cwd=None, env=None, - stdin=None, stdout=None, stderr=None, - creationflags=0, wait=True, - process_progress_listener=None, close_fds=False -): + raise ImportError("not in test") + filename = path.get_unique_file_path(yatest.common.output_path(), filename) + yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename)) + return open(filename, "wb+") + except ImportError: + return tempfile.NamedTemporaryFile(delete=False, suffix=filename) + + +def _get_proc_tree_info(pids): + if os.name == 'nt': + return 'Not supported' + else: + stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return stdout + + +def py_execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False +): """ Executes a command with the arcadia python :param command: command to pass to python @@ -635,50 +635,50 @@ def _kill_process_tree(process_pid, target_pid_signal=None): if os.name == 'nt': _win_kill_process_tree(process_pid) else: - _nix_kill_process_tree(process_pid, target_pid_signal) - - -def _nix_get_proc_children(pid): - try: - cmd = ["pgrep", "-P", str(pid)] - return [int(p) for p in subprocess.check_output(cmd).split()] - except Exception: - return [] - - -def _get_binname(pid): - try: - return os.path.basename(os.readlink('/proc/{}/exe'.format(pid))) - except Exception as e: - return "error({})".format(e) - - + _nix_kill_process_tree(process_pid, target_pid_signal) + + +def _nix_get_proc_children(pid): + try: + cmd = ["pgrep", "-P", str(pid)] + return [int(p) for p in subprocess.check_output(cmd).split()] + except Exception: + return [] + + +def _get_binname(pid): + try: + return os.path.basename(os.readlink('/proc/{}/exe'.format(pid))) + except Exception as e: + return "error({})".format(e) + + def _nix_kill_process_tree(pid, target_pid_signal=None): """ Kills the process tree. """ - yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) + yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) def try_to_send_signal(pid, sig): try: os.kill(pid, sig) - yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) + yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) except Exception as exc: - yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) + yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. # Get the child process PID list. - child_pids = _nix_get_proc_children(pid) - # Stop the child processes. - for child_pid in child_pids: - try: - # Kill the child recursively. - _kill_process_tree(int(child_pid)) - except Exception as e: - # Skip the error and continue killing. - yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) - continue + child_pids = _nix_get_proc_children(pid) + # Stop the child processes. + for child_pid in child_pids: + try: + # Kill the child recursively. + _kill_process_tree(int(child_pid)) + except Exception as e: + # Skip the error and continue killing. + yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) + continue try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process. @@ -689,16 +689,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None): def _win_kill_process_tree(pid): subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) - - -def _run_readelf(binary_path): + + +def _run_readelf(binary_path): return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)])) - - -def check_glibc_version(binary_path): + + +def check_glibc_version(binary_path): lucid_glibc_version = distutils.version.LooseVersion("2.11") - for l in _run_readelf(binary_path).split('\n'): + for l in _run_readelf(binary_path).split('\n'): match = GLIBC_PATTERN.search(l) if not match: continue @@ -706,14 +706,14 @@ def check_glibc_version(binary_path): def backtrace_to_html(bt_filename, output): - try: + try: from library.python import coredump_filter - with open(output, "wb") as afile: + with open(output, "wb") as afile: coredump_filter.filter_stackdump(bt_filename, stream=afile) - except ImportError as e: - yatest_logger.debug("Failed to import coredump_filter: %s", e) - with open(output, "wb") as afile: - afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>") + except ImportError as e: + yatest_logger.debug("Failed to import coredump_filter: %s", e) + with open(output, "wb") as afile: + afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>") def _try_convert_bytes_to_string(source): |