aboutsummaryrefslogtreecommitdiffstats
path: root/library/python/testing/yatest_common/yatest
diff options
context:
space:
mode:
authorDmitry Kopylov <kopylovd@gmail.com>2022-02-10 16:48:18 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:48:18 +0300
commitb2f5101486cc0de2e979c8ba9ada2109785bf5fd (patch)
treeaffe28b840816b505db0467f2285b01c89c04247 /library/python/testing/yatest_common/yatest
parente9b28b5aad71453a4637b70dde02e801e4147a2a (diff)
downloadydb-b2f5101486cc0de2e979c8ba9ada2109785bf5fd.tar.gz
Restoring authorship annotation for Dmitry Kopylov <kopylovd@gmail.com>. Commit 1 of 2.
Diffstat (limited to 'library/python/testing/yatest_common/yatest')
-rw-r--r--library/python/testing/yatest_common/yatest/__init__.py4
-rw-r--r--library/python/testing/yatest_common/yatest/common/benchmark.py34
-rw-r--r--library/python/testing/yatest_common/yatest/common/canonical.py268
-rw-r--r--library/python/testing/yatest_common/yatest/common/environment.py2
-rw-r--r--library/python/testing/yatest_common/yatest/common/errors.py30
-rw-r--r--library/python/testing/yatest_common/yatest/common/legacy.py22
-rw-r--r--library/python/testing/yatest_common/yatest/common/path.py44
-rw-r--r--library/python/testing/yatest_common/yatest/common/process.py576
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime.py322
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime_java.py10
-rw-r--r--library/python/testing/yatest_common/yatest/common/tags.py10
11 files changed, 661 insertions, 661 deletions
diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py
index b846b3317a..19593ff8d2 100644
--- a/library/python/testing/yatest_common/yatest/__init__.py
+++ b/library/python/testing/yatest_common/yatest/__init__.py
@@ -1,3 +1,3 @@
-__all__ = ["common"]
-
+__all__ = ["common"]
+
from . import common
diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py
index c3784cbe4c..500dfaca7e 100644
--- a/library/python/testing/yatest_common/yatest/common/benchmark.py
+++ b/library/python/testing/yatest_common/yatest/common/benchmark.py
@@ -1,22 +1,22 @@
-import json
-
+import json
+
from . import process
from . import runtime
-
-
-def execute_benchmark(path, budget=None, threads=None):
- """
- Run benchmark and return values
- :param path: path to benchmark binary
+
+
+def execute_benchmark(path, budget=None, threads=None):
+ """
+ Run benchmark and return values
+ :param path: path to benchmark binary
:param budget: time budget, sec (supported only by ybenchmark)
:param threads: number of threads to run benchmark (supported only by ybenchmark)
- :return: map of benchmark values
- """
- benchmark_path = runtime.binary_path(path)
+ :return: map of benchmark values
+ """
+ benchmark_path = runtime.binary_path(path)
cmd = [benchmark_path, "--benchmark_format=json"]
- if budget is not None:
- cmd += ["-b", str(budget)]
- if threads is not None:
- cmd += ["-t", str(threads)]
- res = process.execute(cmd)
- return json.loads(res.std_out)
+ if budget is not None:
+ cmd += ["-b", str(budget)]
+ if threads is not None:
+ cmd += ["-t", str(threads)]
+ res = process.execute(cmd)
+ return json.loads(res.std_out)
diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py
index b6a136d3e9..9c02fe1c11 100644
--- a/library/python/testing/yatest_common/yatest/common/canonical.py
+++ b/library/python/testing/yatest_common/yatest/common/canonical.py
@@ -1,17 +1,17 @@
-import os
-import logging
-import shutil
-import tempfile
-
+import os
+import logging
+import shutil
+import tempfile
+
import six
from . import process
from . import runtime
from . import path
-
-yatest_logger = logging.getLogger("ya.test")
-
-
+
+yatest_logger = logging.getLogger("ya.test")
+
+
def _copy(src, dst, universal_lines=False):
if universal_lines:
with open(dst, "wb") as f:
@@ -21,21 +21,21 @@ def _copy(src, dst, universal_lines=False):
shutil.copy(src, dst)
-def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None):
- """
- Create canonical file that can be returned from a test
- :param path: path to the file
- :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used
- :param local: save file locally, otherwise move to sandbox
+def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None):
+ """
+ Create canonical file that can be returned from a test
+ :param path: path to the file
+ :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used
+ :param local: save file locally, otherwise move to sandbox
:param universal_lines: normalize EOL
- :param diff_tool_timeout: timeout for running diff tool
- :return: object that can be canonized
- """
- abs_path = os.path.abspath(path)
- assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
- tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path())
- safe_path = os.path.join(tempdir, os.path.basename(abs_path))
- # if the created file is in output_path, we copy it, so that it will be available when the tests finishes
+ :param diff_tool_timeout: timeout for running diff tool
+ :return: object that can be canonized
+ """
+ abs_path = os.path.abspath(path)
+ assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+ tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path())
+ safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+ # if the created file is in output_path, we copy it, so that it will be available when the tests finishes
_copy(path, safe_path, universal_lines=universal_lines)
if diff_tool:
if not isinstance(diff_tool, six.string_types):
@@ -44,133 +44,133 @@ def canonical_file(path, diff_tool=None, local=False, universal_lines=False, dif
raise Exception("Invalid custom diff-tool: not cmd")
except:
raise Exception("Invalid custom diff-tool: not binary path")
- return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
-
-
-def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None):
- abs_path = os.path.abspath(path)
- assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
- assert os.path.isdir(abs_path), "Path {} is not a directory".format(path)
- if diff_file_name and not diff_tool:
- raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir")
- tempdir = tempfile.mkdtemp()
- safe_path = os.path.join(tempdir, os.path.basename(abs_path))
- shutil.copytree(abs_path, safe_path)
- return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
-
-
-def canonical_execute(
- binary, args=None, check_exit_code=True,
- shell=False, timeout=None, cwd=None,
- env=None, stdin=None, stderr=None, creationflags=0,
+ return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None):
+ abs_path = os.path.abspath(path)
+ assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+ assert os.path.isdir(abs_path), "Path {} is not a directory".format(path)
+ if diff_file_name and not diff_tool:
+ raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir")
+ tempdir = tempfile.mkdtemp()
+ safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+ shutil.copytree(abs_path, safe_path)
+ return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_execute(
+ binary, args=None, check_exit_code=True,
+ shell=False, timeout=None, cwd=None,
+ env=None, stdin=None, stderr=None, creationflags=0,
file_name=None, save_locally=False, close_fds=False,
diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
-):
- """
- Shortcut to execute a binary and canonize its stdout
+):
+ """
+ Shortcut to execute a binary and canonize its stdout
:param binary: absolute path to the binary
- :param args: binary arguments
- :param check_exit_code: will raise ExecutionError if the command exits with non zero code
- :param shell: use shell to run the command
- :param timeout: execution timeout
- :param cwd: working directory
- :param env: command environment
- :param stdin: command stdin
- :param stderr: command stderr
- :param creationflags: command creation flags
+ :param args: binary arguments
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stderr: command stderr
+ :param creationflags: command creation flags
:param file_name: output file name. if not specified program name will be used
- :param diff_tool: path to custome diff tool
- :param diff_file_name: custom diff file name to create when diff is found
+ :param diff_tool: path to custome diff tool
+ :param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
- :return: object that can be canonized
- """
- if type(binary) == list:
- command = binary
- else:
- command = [binary]
- command += _prepare_args(args)
- if shell:
- command = " ".join(command)
- execute_args = locals()
- del execute_args["binary"]
- del execute_args["args"]
+ :return: object that can be canonized
+ """
+ if type(binary) == list:
+ command = binary
+ else:
+ command = [binary]
+ command += _prepare_args(args)
+ if shell:
+ command = " ".join(command)
+ execute_args = locals()
+ del execute_args["binary"]
+ del execute_args["args"]
del execute_args["file_name"]
- del execute_args["save_locally"]
- del execute_args["diff_tool"]
- del execute_args["diff_file_name"]
+ del execute_args["save_locally"]
+ del execute_args["diff_tool"]
+ del execute_args["diff_file_name"]
del execute_args["diff_tool_timeout"]
- if not file_name and stdin:
- file_name = os.path.basename(stdin.name)
+ if not file_name and stdin:
+ file_name = os.path.basename(stdin.name)
return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
-
-
-def canonical_py_execute(
- script_path, args=None, check_exit_code=True,
- shell=False, timeout=None, cwd=None, env=None,
- stdin=None, stderr=None, creationflags=0,
- file_name=None, save_locally=False, close_fds=False,
+
+
+def canonical_py_execute(
+ script_path, args=None, check_exit_code=True,
+ shell=False, timeout=None, cwd=None, env=None,
+ stdin=None, stderr=None, creationflags=0,
+ file_name=None, save_locally=False, close_fds=False,
diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
-):
- """
- Shortcut to execute a python script and canonize its stdout
- :param script_path: path to the script arcadia relative
- :param args: script arguments
- :param check_exit_code: will raise ExecutionError if the command exits with non zero code
- :param shell: use shell to run the command
- :param timeout: execution timeout
- :param cwd: working directory
- :param env: command environment
- :param stdin: command stdin
- :param stderr: command stderr
- :param creationflags: command creation flags
+):
+ """
+ Shortcut to execute a python script and canonize its stdout
+ :param script_path: path to the script arcadia relative
+ :param args: script arguments
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stderr: command stderr
+ :param creationflags: command creation flags
:param file_name: output file name. if not specified program name will be used
- :param diff_tool: path to custome diff tool
- :param diff_file_name: custom diff file name to create when diff is found
+ :param diff_tool: path to custome diff tool
+ :param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
- :return: object that can be canonized
- """
- command = [runtime.source_path(script_path)] + _prepare_args(args)
- if shell:
- command = " ".join(command)
- execute_args = locals()
- del execute_args["script_path"]
- del execute_args["args"]
+ :return: object that can be canonized
+ """
+ command = [runtime.source_path(script_path)] + _prepare_args(args)
+ if shell:
+ command = " ".join(command)
+ execute_args = locals()
+ del execute_args["script_path"]
+ del execute_args["args"]
del execute_args["file_name"]
- del execute_args["save_locally"]
- del execute_args["diff_tool"]
- del execute_args["diff_file_name"]
+ del execute_args["save_locally"]
+ del execute_args["diff_tool"]
+ del execute_args["diff_file_name"]
del execute_args["diff_tool_timeout"]
return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
-
-
-def _prepare_args(args):
- if args is None:
- args = []
+
+
+def _prepare_args(args):
+ if args is None:
+ args = []
if isinstance(args, six.string_types):
- args = map(lambda a: a.strip(), args.split())
- return args
-
-
+ args = map(lambda a: a.strip(), args.split())
+ return args
+
+
def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout):
- res = excutor(**kwargs)
- command = kwargs["command"]
- file_name = file_name or process.get_command_name(command)
- if file_name.endswith(".exe"):
- file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names
- out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name))
- err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name))
-
- try:
- os.makedirs(os.path.dirname(out_file_path))
- except OSError:
- pass
-
+ res = excutor(**kwargs)
+ command = kwargs["command"]
+ file_name = file_name or process.get_command_name(command)
+ if file_name.endswith(".exe"):
+ file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names
+ out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name))
+ err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name))
+
+ try:
+ os.makedirs(os.path.dirname(out_file_path))
+ except OSError:
+ pass
+
with open(out_file_path, "wb") as out_file:
- yatest_logger.debug("Will store file in %s", out_file_path)
- out_file.write(res.std_out)
-
- if res.std_err:
+ yatest_logger.debug("Will store file in %s", out_file_path)
+ out_file.write(res.std_out)
+
+ if res.std_err:
with open(err_file_path, "wb") as err_file:
- err_file.write(res.std_err)
-
+ err_file.write(res.std_err)
+
return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py
index 43f48d0958..f3d55058e4 100644
--- a/library/python/testing/yatest_common/yatest/common/environment.py
+++ b/library/python/testing/yatest_common/yatest/common/environment.py
@@ -1,5 +1,5 @@
# coding: utf-8
-
+
def extend_env_var(env, name, value, sep=":"):
return sep.join(filter(None, [env.get(name), value]))
diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py
index 8c038fc381..5cdd62c6d0 100644
--- a/library/python/testing/yatest_common/yatest/common/errors.py
+++ b/library/python/testing/yatest_common/yatest/common/errors.py
@@ -1,20 +1,20 @@
import os
-import sys
-
-
-class RestartTestException(Exception):
-
- def __init__(self, *args, **kwargs):
+import sys
+
+
+class RestartTestException(Exception):
+
+ def __init__(self, *args, **kwargs):
super(RestartTestException, self).__init__(*args, **kwargs)
- sys.stderr.write("##restart-test##\n")
- sys.stderr.flush()
+ sys.stderr.write("##restart-test##\n")
+ sys.stderr.flush()
os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
-
-
-class InfrastructureException(Exception):
-
- def __init__(self, *args, **kwargs):
+
+
+class InfrastructureException(Exception):
+
+ def __init__(self, *args, **kwargs):
super(InfrastructureException, self).__init__(*args, **kwargs)
- sys.stderr.write("##infrastructure-error##\n")
- sys.stderr.flush()
+ sys.stderr.write("##infrastructure-error##\n")
+ sys.stderr.flush()
os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py
index 459972d253..cfb6975802 100644
--- a/library/python/testing/yatest_common/yatest/common/legacy.py
+++ b/library/python/testing/yatest_common/yatest/common/legacy.py
@@ -1,12 +1,12 @@
from . import canonical
-
-
-def old_canonical_file(output_file_name, storage_md5):
- import yalibrary.svn
- yalibrary.svn.run_svn([
- 'export',
- 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5,
- output_file_name,
- "--force"
- ])
- return canonical.canonical_file(output_file_name)
+
+
+def old_canonical_file(output_file_name, storage_md5):
+ import yalibrary.svn
+ yalibrary.svn.run_svn([
+ 'export',
+ 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5,
+ output_file_name,
+ "--force"
+ ])
+ return canonical.canonical_file(output_file_name)
diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py
index 6fed7dda8a..d31df2d278 100644
--- a/library/python/testing/yatest_common/yatest/common/path.py
+++ b/library/python/testing/yatest_common/yatest/common/path.py
@@ -1,5 +1,5 @@
# coding=utf-8
-
+
import errno
import os
import shutil
@@ -8,19 +8,19 @@ import contextlib
import library.python.fs as lpf
-def replace_in_file(path, old, new):
- """
- Replace text occurrences in a file
- :param path: path to the file
- :param old: text to replace
- :param new: replacement
- """
- with open(path) as fp:
- content = fp.read()
-
+def replace_in_file(path, old, new):
+ """
+ Replace text occurrences in a file
+ :param path: path to the file
+ :param old: text to replace
+ :param new: replacement
+ """
+ with open(path) as fp:
+ content = fp.read()
+
lpf.ensure_removed(path)
- with open(path, 'w') as fp:
- fp.write(content.replace(old, new))
+ with open(path, 'w') as fp:
+ fp.write(content.replace(old, new))
@contextlib.contextmanager
@@ -38,8 +38,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None):
Copy an entire directory of files into an existing directory
instead of raising Exception what shtuil.copytree does
'''
- if not os.path.exists(dst) and os.path.isdir(src):
- os.makedirs(dst)
+ if not os.path.exists(dst) and os.path.isdir(src):
+ os.makedirs(dst)
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
@@ -54,8 +54,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None):
postprocessing(os.path.join(root, path), False)
for path in files:
postprocessing(os.path.join(root, path), True)
-
-
+
+
def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000):
def atomic_file_create(path):
try:
@@ -79,12 +79,12 @@ def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10
return False
raise e
- file_path = os.path.join(dir_path, file_pattern)
+ file_path = os.path.join(dir_path, file_pattern)
lpf.ensure_dir(os.path.dirname(file_path))
- file_counter = 0
+ file_counter = 0
handler = atomic_file_create if create_file else atomic_dir_create
while os.path.exists(file_path) or not handler(file_path):
- file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter))
- file_counter += 1
+ file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter))
+ file_counter += 1
assert file_counter < max_suffix
- return file_path
+ return file_path
diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py
index a8bcc21f51..672ebe0823 100644
--- a/library/python/testing/yatest_common/yatest/common/process.py
+++ b/library/python/testing/yatest_common/yatest/common/process.py
@@ -1,16 +1,16 @@
# coding: utf-8
-import os
+import os
import re
-import time
-import signal
+import time
+import signal
import shutil
-import logging
-import tempfile
-import subprocess
-import errno
+import logging
+import tempfile
+import subprocess
+import errno
import distutils.version
-
+
import six
try:
@@ -22,13 +22,13 @@ except ImportError:
from . import runtime
from . import path
from . import environment
-
-
+
+
MAX_OUT_LEN = 1000 * 1000 # 1 mb
MAX_MESSAGE_LEN = 1500
SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)"
GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)")
-yatest_logger = logging.getLogger("ya.test")
+yatest_logger = logging.getLogger("ya.test")
def truncate(s, size):
@@ -40,45 +40,45 @@ def truncate(s, size):
return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):]
-def get_command_name(command):
+def get_command_name(command):
return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0])
-
-
-class ExecutionError(Exception):
-
- def __init__(self, execution_result):
+
+
+class ExecutionError(Exception):
+
+ def __init__(self, execution_result):
if not isinstance(execution_result.command, six.string_types):
command = " ".join(str(arg) for arg in execution_result.command)
else:
command = execution_result.command
- message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format(
+ message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format(
command=command,
- code=execution_result.exit_code,
- err=_format_error(execution_result.std_err))
+ code=execution_result.exit_code,
+ err=_format_error(execution_result.std_err))
if cores:
if execution_result.backtrace:
message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
else:
message += "Backtrace is not available: module cores isn't available"
- super(ExecutionError, self).__init__(message)
- self.execution_result = execution_result
-
-
-class TimeoutError(Exception):
- pass
-
-
-class ExecutionTimeoutError(TimeoutError):
- def __init__(self, execution_result, *args, **kwargs):
- super(ExecutionTimeoutError, self).__init__(args, kwargs)
+ super(ExecutionError, self).__init__(message)
self.execution_result = execution_result
-
-
-class InvalidExecutionStateError(Exception):
- pass
-
-
+
+
+class TimeoutError(Exception):
+ pass
+
+
+class ExecutionTimeoutError(TimeoutError):
+ def __init__(self, execution_result, *args, **kwargs):
+ super(ExecutionTimeoutError, self).__init__(args, kwargs)
+ self.execution_result = execution_result
+
+
+class InvalidExecutionStateError(Exception):
+ pass
+
+
class SignalInterruptionError(Exception):
def __init__(self, message=None):
super(SignalInterruptionError, self).__init__(message)
@@ -89,39 +89,39 @@ class InvalidCommandError(Exception):
pass
-class _Execution(object):
-
+class _Execution(object):
+
def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False):
- self._command = command
- self._process = process
- self._out_file = out_file
- self._err_file = err_file
- self._std_out = None
- self._std_err = None
+ self._command = command
+ self._process = process
+ self._out_file = out_file
+ self._err_file = err_file
+ self._std_out = None
+ self._std_err = None
self._elapsed = None
self._start = time.time()
- self._process_progress_listener = process_progress_listener
+ self._process_progress_listener = process_progress_listener
self._cwd = cwd or os.getcwd()
self._collect_cores = collect_cores
self._backtrace = ''
self._check_sanitizer = check_sanitizer
- self._metrics = {}
+ self._metrics = {}
self._started = started
self._user_stdout = bool(user_stdout)
self._user_stderr = bool(user_stderr)
self._exit_code = None
- if process_progress_listener:
- process_progress_listener.open(command, process, out_file, err_file)
-
- @property
- def running(self):
- return self._process.poll() is None
-
- def kill(self):
- if self.running:
- self._save_outputs(False)
- _kill_process_tree(self._process.pid)
- self._clean_files()
+ if process_progress_listener:
+ process_progress_listener.open(command, process, out_file, err_file)
+
+ @property
+ def running(self):
+ return self._process.poll() is None
+
+ def kill(self):
+ if self.running:
+ self._save_outputs(False)
+ _kill_process_tree(self._process.pid)
+ self._clean_files()
# DEVTOOLS-2347
yatest_logger.debug("Process status before wait_for: %s", self.running)
try:
@@ -130,64 +130,64 @@ class _Execution(object):
yatest_logger.debug("Process status after wait_for: %s", self.running)
yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
raise
- else:
- raise InvalidExecutionStateError("Cannot kill a stopped process")
-
+ else:
+ raise InvalidExecutionStateError("Cannot kill a stopped process")
+
def terminate(self):
if self.running:
self._process.terminate()
- @property
- def process(self):
- return self._process
-
- @property
- def command(self):
- return self._command
-
- @property
+ @property
+ def process(self):
+ return self._process
+
+ @property
+ def command(self):
+ return self._command
+
+ @property
def returncode(self):
return self.exit_code
@property
- def exit_code(self):
+ def exit_code(self):
"""
Deprecated, use returncode
"""
if self._exit_code is None:
self._exit_code = self._process.returncode
return self._exit_code
-
- @property
+
+ @property
def stdout(self):
return self.std_out
@property
- def std_out(self):
+ def std_out(self):
"""
Deprecated, use stdout
"""
- if self._std_out is not None:
- return self._std_out
+ if self._std_out is not None:
+ return self._std_out
if self._process.stdout and not self._user_stdout:
self._std_out = self._process.stdout.read()
return self._std_out
-
- @property
+
+ @property
def stderr(self):
return self.std_err
@property
- def std_err(self):
+ def std_err(self):
"""
Deprecated, use stderr
"""
- if self._std_err is not None:
- return self._std_err
+ if self._std_err is not None:
+ return self._std_err
if self._process.stderr and not self._user_stderr:
self._std_err = self._process.stderr.read()
return self._std_err
-
+
@property
def elapsed(self):
return self._elapsed
@@ -196,14 +196,14 @@ class _Execution(object):
def backtrace(self):
return self._backtrace
- @property
- def metrics(self):
- return self._metrics
-
- def _save_outputs(self, clean_files=True):
- if self._process_progress_listener:
- self._process_progress_listener()
- self._process_progress_listener.close()
+ @property
+ def metrics(self):
+ return self._metrics
+
+ def _save_outputs(self, clean_files=True):
+ if self._process_progress_listener:
+ self._process_progress_listener()
+ self._process_progress_listener.close()
if not self._user_stdout:
if self._out_file is None:
pass
@@ -223,14 +223,14 @@ class _Execution(object):
else:
self._std_err = self._process.stderr.read()
- if clean_files:
- self._clean_files()
- yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code)
- yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed)
- if self._metrics:
+ if clean_files:
+ self._clean_files()
+ yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code)
+ yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed)
+ if self._metrics:
for key, value in six.iteritems(self._metrics):
- yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value)
-
+ yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value)
+
# Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str.
printable_std_out, err = _try_convert_bytes_to_string(self._std_out)
if err:
@@ -244,14 +244,14 @@ class _Execution(object):
yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN))
yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN))
- def _clean_files(self):
+ def _clean_files(self):
if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE:
self._err_file.close()
self._err_file = None
if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE:
self._out_file.close()
self._out_file = None
-
+
def _recover_core(self):
core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid)
if core_path:
@@ -260,7 +260,7 @@ class _Execution(object):
if store_cores:
new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
# Copy core dump file, because it may be overwritten
- yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
+ yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
shutil.copyfile(core_path, new_core_path)
core_path = new_core_path
@@ -281,92 +281,92 @@ class _Execution(object):
else:
runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
- def wait(self, check_exit_code=True, timeout=None, on_timeout=None):
- def _wait():
+ def wait(self, check_exit_code=True, timeout=None, on_timeout=None):
+ def _wait():
finished = None
interrupted = False
- try:
- if hasattr(os, "wait4"):
- try:
+ try:
+ if hasattr(os, "wait4"):
+ try:
if hasattr(subprocess, "_eintr_retry_call"):
pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0)
else:
# PEP 475
pid, sts, rusage = os.wait4(self._process.pid, 0)
finished = time.time()
- self._process._handle_exitstatus(sts)
- for field in [
- "ru_idrss",
- "ru_inblock",
- "ru_isrss",
- "ru_ixrss",
- "ru_majflt",
- "ru_maxrss",
- "ru_minflt",
- "ru_msgrcv",
- "ru_msgsnd",
- "ru_nivcsw",
- "ru_nsignals",
- "ru_nswap",
- "ru_nvcsw",
- "ru_oublock",
- "ru_stime",
- "ru_utime",
- ]:
- if hasattr(rusage, field):
- self._metrics[field.replace("ru_", "")] = getattr(rusage, field)
- except OSError as exc:
-
- if exc.errno == errno.ECHILD:
- yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
- else:
- raise
+ self._process._handle_exitstatus(sts)
+ for field in [
+ "ru_idrss",
+ "ru_inblock",
+ "ru_isrss",
+ "ru_ixrss",
+ "ru_majflt",
+ "ru_maxrss",
+ "ru_minflt",
+ "ru_msgrcv",
+ "ru_msgsnd",
+ "ru_nivcsw",
+ "ru_nsignals",
+ "ru_nswap",
+ "ru_nvcsw",
+ "ru_oublock",
+ "ru_stime",
+ "ru_utime",
+ ]:
+ if hasattr(rusage, field):
+ self._metrics[field.replace("ru_", "")] = getattr(rusage, field)
+ except OSError as exc:
+
+ if exc.errno == errno.ECHILD:
+ yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
+ else:
+ raise
except SignalInterruptionError:
interrupted = True
raise
- finally:
+ finally:
if not interrupted:
self._process.wait() # this has to be here unconditionally, so that all process properties are set
-
+
if not finished:
finished = time.time()
self._metrics["wtime"] = round(finished - self._started, 3)
- try:
+ try:
if timeout:
process_is_finished = lambda: not self.running
fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
- try:
- wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
- except TimeoutError as e:
- if on_timeout:
+ try:
+ wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
+ except TimeoutError as e:
+ if on_timeout:
yatest_logger.debug("Calling user specified on_timeout function")
- try:
- on_timeout(self, timeout)
- except Exception:
- yatest_logger.exception("Exception while calling on_timeout")
- raise ExecutionTimeoutError(self, str(e))
+ try:
+ on_timeout(self, timeout)
+ except Exception:
+ yatest_logger.exception("Exception while calling on_timeout")
+ raise ExecutionTimeoutError(self, str(e))
# Wait should be always called here, it finalizes internal states of its process and sets up return code
- _wait()
+ _wait()
except BaseException as e:
- _kill_process_tree(self._process.pid)
- _wait()
- yatest_logger.debug("Command exception: %s", e)
+ _kill_process_tree(self._process.pid)
+ _wait()
+ yatest_logger.debug("Command exception: %s", e)
raise
- finally:
+ finally:
self._elapsed = time.time() - self._start
self._save_outputs()
self.verify_no_coredumps()
-
+
self._finalise(check_exit_code)
def _finalise(self, check_exit_code):
- # Set the signal (negative number) which caused the process to exit
- if check_exit_code and self.exit_code != 0:
- yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
- self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
- raise ExecutionError(self)
-
+ # Set the signal (negative number) which caused the process to exit
+ if check_exit_code and self.exit_code != 0:
+ yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
+ self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
+ raise ExecutionError(self)
+
# Don't search for sanitize errors if stderr was redirected
self.verify_sanitize_errors()
@@ -394,11 +394,11 @@ class _Execution(object):
if match:
yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
raise ExecutionError(self)
- else:
- yatest_logger.debug("No sanitizer errors found")
+ else:
+ yatest_logger.debug("No sanitizer errors found")
else:
- yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
-
+ yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
+
def on_timeout_gen_coredump(exec_obj, _):
"""
@@ -413,38 +413,38 @@ def on_timeout_gen_coredump(exec_obj, _):
pass
-def execute(
- command, check_exit_code=True,
- shell=False, timeout=None,
- cwd=None, env=None,
- stdin=None, stdout=None, stderr=None,
- creationflags=0, wait=True,
- process_progress_listener=None, close_fds=False,
- collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
+def execute(
+ command, check_exit_code=True,
+ shell=False, timeout=None,
+ cwd=None, env=None,
+ stdin=None, stdout=None, stderr=None,
+ creationflags=0, wait=True,
+ process_progress_listener=None, close_fds=False,
+ collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
executor=_Execution,
-):
- """
- Executes a command
- :param command: command: can be a list of arguments or a string
- :param check_exit_code: will raise ExecutionError if the command exits with non zero code
- :param shell: use shell to run the command
- :param timeout: execution timeout
- :param cwd: working directory
- :param env: command environment
- :param stdin: command stdin
- :param stdout: command stdout
- :param stderr: command stderr
- :param creationflags: command creation flags
- :param wait: should wait until the command finishes
- :param process_progress_listener=object that is polled while execution is in progress
- :param close_fds: subrpocess.Popen close_fds args
+):
+ """
+ Executes a command
+ :param command: command: can be a list of arguments or a string
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stdout: command stdout
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param wait: should wait until the command finishes
+ :param process_progress_listener=object that is polled while execution is in progress
+ :param close_fds: subrpocess.Popen close_fds args
:param collect_cores: recover core dump files if shell == False
:param check_sanitizer: raise ExecutionError if stderr contains sanitize errors
- :param preexec_fn: subrpocess.Popen preexec_fn arg
- :param on_timeout: on_timeout(<execution object>, <timeout value>) callback
-
+ :param preexec_fn: subrpocess.Popen preexec_fn arg
+ :param on_timeout: on_timeout(<execution object>, <timeout value>) callback
+
:return _Execution: Execution object
- """
+ """
if env is None:
env = os.environ.copy()
else:
@@ -462,9 +462,9 @@ def execute(
if var not in env and var in os.environ:
env[var] = os.environ[var]
- if not wait and timeout is not None:
- raise ValueError("Incompatible arguments 'timeout' and wait=False")
-
+ if not wait and timeout is not None:
+ raise ValueError("Incompatible arguments 'timeout' and wait=False")
+
# if subprocess.PIPE in [stdout, stderr]:
# raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock")
@@ -481,15 +481,15 @@ def execute(
is_pipe = stream == subprocess.PIPE
return stream, not is_pipe
- # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE,
- # as it can cause processes hangs, but use tempfiles instead
+ # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE,
+ # as it can cause processes hangs, but use tempfiles instead
out_file, user_stdout = get_out_stream(stdout, 'out')
err_file, user_stderr = get_out_stream(stderr, 'err')
- in_file = stdin
-
- if shell and type(command) == list:
- command = " ".join(command)
-
+ in_file = stdin
+
+ if shell and type(command) == list:
+ command = " ".join(command)
+
if shell:
collect_cores = False
check_sanitizer = False
@@ -516,7 +516,7 @@ def execute(
name = "PIPE" if stdin == subprocess.PIPE else stdin.name
yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd)
else:
- yatest_logger.debug("Executing '%s' in '%s'", command, cwd)
+ yatest_logger.debug("Executing '%s' in '%s'", command, cwd)
# XXX
started = time.time()
@@ -528,11 +528,11 @@ def execute(
yatest_logger.debug("Command pid: %s", process.pid)
res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr)
- if wait:
- res.wait(check_exit_code, timeout, on_timeout)
- return res
-
-
+ if wait:
+ res.wait(check_exit_code, timeout, on_timeout)
+ return res
+
+
def _get_command_output_file(cmd, ext):
parts = [get_command_name(cmd)]
if 'YA_RETRY_INDEX' in os.environ:
@@ -570,74 +570,74 @@ def py_execute(
creationflags=0, wait=True,
process_progress_listener=None, close_fds=False
):
- """
- Executes a command with the arcadia python
- :param command: command to pass to python
- :param check_exit_code: will raise ExecutionError if the command exits with non zero code
- :param shell: use shell to run the command
- :param timeout: execution timeout
- :param cwd: working directory
- :param env: command environment
- :param stdin: command stdin
- :param stdout: command stdout
- :param stderr: command stderr
- :param creationflags: command creation flags
- :param wait: should wait until the command finishes
- :param process_progress_listener=object that is polled while execution is in progress
+ """
+ Executes a command with the arcadia python
+ :param command: command to pass to python
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stdout: command stdout
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param wait: should wait until the command finishes
+ :param process_progress_listener=object that is polled while execution is in progress
:return _Execution: Execution object
- """
+ """
if isinstance(command, six.string_types):
- command = [command]
- command = [runtime.python_path()] + command
- if shell:
- command = " ".join(command)
- return execute(**locals())
-
-
-def _format_error(error):
- return truncate(error, MAX_MESSAGE_LEN)
-
-
-def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None):
- """
- Tries to execute `check_function` for `timeout` seconds.
- Continue until function returns nonfalse value.
- If function doesn't return nonfalse value for `timeout` seconds
- OperationTimeoutException is raised.
- Return first nonfalse result returned by `checkFunction`.
- """
- if sleep_time <= 0:
- raise ValueError("Incorrect sleep time value {}".format(sleep_time))
- if timeout < 0:
- raise ValueError("Incorrect timeout value {}".format(timeout))
- start = time.time()
- while start + timeout > time.time():
- if on_check_condition:
- on_check_condition()
-
- res = check_function()
- if res:
- return res
- time.sleep(sleep_time)
-
- message = "{} second(s) wait timeout has expired".format(timeout)
- if fail_message:
- message += ": {}".format(fail_message)
+ command = [command]
+ command = [runtime.python_path()] + command
+ if shell:
+ command = " ".join(command)
+ return execute(**locals())
+
+
+def _format_error(error):
+ return truncate(error, MAX_MESSAGE_LEN)
+
+
+def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None):
+ """
+ Tries to execute `check_function` for `timeout` seconds.
+ Continue until function returns nonfalse value.
+ If function doesn't return nonfalse value for `timeout` seconds
+ OperationTimeoutException is raised.
+ Return first nonfalse result returned by `checkFunction`.
+ """
+ if sleep_time <= 0:
+ raise ValueError("Incorrect sleep time value {}".format(sleep_time))
+ if timeout < 0:
+ raise ValueError("Incorrect timeout value {}".format(timeout))
+ start = time.time()
+ while start + timeout > time.time():
+ if on_check_condition:
+ on_check_condition()
+
+ res = check_function()
+ if res:
+ return res
+ time.sleep(sleep_time)
+
+ message = "{} second(s) wait timeout has expired".format(timeout)
+ if fail_message:
+ message += ": {}".format(fail_message)
raise TimeoutError(truncate(message, MAX_MESSAGE_LEN))
-
+
def _kill_process_tree(process_pid, target_pid_signal=None):
- """
- Kills child processes, req. Note that psutil should be installed
- @param process_pid: parent id to search for descendants
- """
- yatest_logger.debug("Killing process %s", process_pid)
- if os.name == 'nt':
- _win_kill_process_tree(process_pid)
- else:
+ """
+ Kills child processes, req. Note that psutil should be installed
+ @param process_pid: parent id to search for descendants
+ """
+ yatest_logger.debug("Killing process %s", process_pid)
+ if os.name == 'nt':
+ _win_kill_process_tree(process_pid)
+ else:
_nix_kill_process_tree(process_pid, target_pid_signal)
-
-
+
+
def _nix_get_proc_children(pid):
try:
cmd = ["pgrep", "-P", str(pid)]
@@ -654,21 +654,21 @@ def _get_binname(pid):
def _nix_kill_process_tree(pid, target_pid_signal=None):
- """
- Kills the process tree.
- """
+ """
+ Kills the process tree.
+ """
yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid)))
-
- def try_to_send_signal(pid, sig):
- try:
- os.kill(pid, sig)
+
+ def try_to_send_signal(pid, sig):
+ try:
+ os.kill(pid, sig)
yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
- except Exception as exc:
+ except Exception as exc:
yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
-
- try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes.
-
- # Get the child process PID list.
+
+ try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes.
+
+ # Get the child process PID list.
child_pids = _nix_get_proc_children(pid)
# Stop the child processes.
for child_pid in child_pids:
@@ -679,16 +679,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None):
# Skip the error and continue killing.
yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e))
continue
-
+
try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process.
-
- # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent
- # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html
- try_to_send_signal(pid, signal.SIGCONT)
-
-
-def _win_kill_process_tree(pid):
- subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)])
+
+ # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent
+ # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html
+ try_to_send_signal(pid, signal.SIGCONT)
+
+
+def _win_kill_process_tree(pid):
+ subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)])
def _run_readelf(binary_path):
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
index e55e193446..aa9161aead 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -3,49 +3,49 @@ import functools
import json
import os
import threading
-
+
import six
-
+
_lock = threading.Lock()
def _get_ya_config():
- try:
+ try:
import library.python.pytest.plugins.ya as ya_plugin
if ya_plugin.pytest_config is not None:
return ya_plugin.pytest_config
- import pytest
+ import pytest
return pytest.config
- except (ImportError, AttributeError):
+ except (ImportError, AttributeError):
try:
import library.python.testing.recipe
if library.python.testing.recipe.ya:
return library.python.testing.recipe
except (ImportError, AttributeError):
pass
- raise NotImplementedError("yatest.common.* is only available from the testing runtime")
-
-
+ raise NotImplementedError("yatest.common.* is only available from the testing runtime")
+
+
def _get_ya_plugin_instance():
return _get_ya_config().ya
-def _norm_path(path):
- if path is None:
- return None
+def _norm_path(path):
+ if path is None:
+ return None
assert isinstance(path, six.string_types)
- if "\\" in path:
- raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path))
- return os.path.normpath(path)
-
-
-def _join_path(main_path, path):
- if not path:
- return main_path
- return os.path.join(main_path, _norm_path(path))
-
-
+ if "\\" in path:
+ raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path))
+ return os.path.normpath(path)
+
+
+def _join_path(main_path, path):
+ if not path:
+ return main_path
+ return os.path.join(main_path, _norm_path(path))
+
+
def not_test(func):
"""
Mark any function as not a test for py.test
@@ -60,70 +60,70 @@ def not_test(func):
def source_path(path=None):
- """
- Get source path inside arcadia
+ """
+ Get source path inside arcadia
:param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya')
- :return: absolute path to the source folder
- """
+ :return: absolute path to the source folder
+ """
return _join_path(_get_ya_plugin_instance().source_root, path)
-
-
+
+
def build_path(path=None):
- """
- Get path inside build directory
+ """
+ Get path inside build directory
:param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin')
- :return: absolute path inside build directory
- """
+ :return: absolute path inside build directory
+ """
return _join_path(_get_ya_plugin_instance().build_root, path)
-
-
+
+
def java_path():
"""
- [DEPRECATED] Get path to java
+ [DEPRECATED] Get path to java
:return: absolute path to java
"""
from . import runtime_java
return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk')))
-def java_home():
- """
- Get jdk directory path
- """
+def java_home():
+ """
+ Get jdk directory path
+ """
from . import runtime_java
- jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk'))
- if not jdk_dir:
- raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform")
- return jdk_dir
-
-
-def java_bin():
- """
- Get path to the java binary
+ jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk'))
+ if not jdk_dir:
+ raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform")
+ return jdk_dir
+
+
+def java_bin():
+ """
+ Get path to the java binary
Requires DEPENDS(jdk)
- """
- return os.path.join(java_home(), "bin", "java")
-
-
+ """
+ return os.path.join(java_home(), "bin", "java")
+
+
def data_path(path=None):
- """
- Get path inside arcadia_tests_data directory
+ """
+ Get path inside arcadia_tests_data directory
:param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
- :return: absolute path inside arcadia_tests_data
- """
+ :return: absolute path inside arcadia_tests_data
+ """
return _join_path(_get_ya_plugin_instance().data_root, path)
-
-
+
+
def output_path(path=None):
- """
- Get path inside the current test suite output dir.
- Placing files to this dir guarantees that files will be accessible after the test suite execution.
- :param path: path relative to the test suite output dir
- :return: absolute path inside the test suite output dir
- """
+ """
+ Get path inside the current test suite output dir.
+ Placing files to this dir guarantees that files will be accessible after the test suite execution.
+ :param path: path relative to the test suite output dir
+ :return: absolute path inside the test suite output dir
+ """
return _join_path(_get_ya_plugin_instance().output_dir, path)
-
-
+
+
def ram_drive_path(path=None):
"""
:param path: path relative to the ram drive.
@@ -144,60 +144,60 @@ def output_ram_drive_path(path=None):
def binary_path(path=None):
- """
- Get path to the built binary
+ """
+ Get path to the built binary
:param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin')
- :return: absolute path to the binary
- """
+ :return: absolute path to the binary
+ """
path = _norm_path(path)
return _get_ya_plugin_instance().get_binary(path)
-
-
+
+
def work_path(path=None):
- """
- Get path inside the current test suite working directory. Creating files in the work directory does not guarantee
- that files will be accessible after the test suite execution
- :param path: path relative to the test suite working dir
- :return: absolute path inside the test suite working dir
- """
+ """
+ Get path inside the current test suite working directory. Creating files in the work directory does not guarantee
+ that files will be accessible after the test suite execution
+ :param path: path relative to the test suite working dir
+ :return: absolute path inside the test suite working dir
+ """
return _join_path(
os.environ.get("TEST_WORK_PATH") or
_get_ya_plugin_instance().get_context("work_path") or
os.getcwd(),
path)
-
-
-def python_path():
- """
+
+
+def python_path():
+ """
Get path to the arcadia python.
Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles
are built in a stripped-down form that is needed for building, not running tests.
See comments in the file below to find out which version of python is compatible with tests.
https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc
- :return: absolute path to python
- """
- return _get_ya_plugin_instance().python_path
-
-
-def valgrind_path():
- """
- Get path to valgrind
- :return: absolute path to valgrind
- """
- return _get_ya_plugin_instance().valgrind_path
-
-
-def get_param(key, default=None):
- """
- Get arbitrary parameter passed via command line
- :param key: key
- :param default: default value
- :return: parameter value or the default
- """
- return _get_ya_plugin_instance().get_param(key, default)
-
-
+ :return: absolute path to python
+ """
+ return _get_ya_plugin_instance().python_path
+
+
+def valgrind_path():
+ """
+ Get path to valgrind
+ :return: absolute path to valgrind
+ """
+ return _get_ya_plugin_instance().valgrind_path
+
+
+def get_param(key, default=None):
+ """
+ Get arbitrary parameter passed via command line
+ :param key: key
+ :param default: default value
+ :return: parameter value or the default
+ """
+ return _get_ya_plugin_instance().get_param(key, default)
+
+
def get_param_dict_copy():
"""
Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters.
@@ -209,25 +209,25 @@ def get_param_dict_copy():
@not_test
def test_output_path(path=None):
- """
- Get dir in the suite output_path for the current test case
- """
+ """
+ Get dir in the suite output_path for the current test case
+ """
test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0]
try:
- os.makedirs(test_out_dir)
+ os.makedirs(test_out_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return _join_path(test_out_dir, path)
-
-
-def project_path(path=None):
- """
- Get path in build root relating to build_root/project path
- """
- return _join_path(os.path.join(build_path(), context.project_path), path)
-
-
+
+
+def project_path(path=None):
+ """
+ Get path in build root relating to build_root/project path
+ """
+ return _join_path(os.path.join(build_path(), context.project_path), path)
+
+
def gdb_path():
"""
Get path to the gdb
@@ -235,25 +235,25 @@ def gdb_path():
return _get_ya_plugin_instance().gdb_path
-def c_compiler_path():
- """
- Get path to the gdb
- """
- return os.environ.get("YA_CC")
-
-
+def c_compiler_path():
+ """
+ Get path to the gdb
+ """
+ return os.environ.get("YA_CC")
+
+
def get_yt_hdd_path(path=None):
if 'HDD_PATH' in os.environ:
return _join_path(os.environ['HDD_PATH'], path)
-def cxx_compiler_path():
- """
- Get path to the gdb
- """
- return os.environ.get("YA_CXX")
-
-
+def cxx_compiler_path():
+ """
+ Get path to the gdb
+ """
+ return os.environ.get("YA_CXX")
+
+
def global_resources():
try:
return json.loads(os.environ.get("YA_GLOBAL_RESOURCES"))
@@ -282,39 +282,39 @@ def _register_core(name, binary_path, core_path, bt_path, pbt_path):
@not_test
-def test_source_path(path=None):
- return _join_path(os.path.join(source_path(), context.project_path), path)
-
-
-class Context(object):
- """
- Runtime context
- """
-
- @property
- def build_type(self):
- return _get_ya_plugin_instance().get_context("build_type")
-
- @property
- def project_path(self):
- return _get_ya_plugin_instance().get_context("project_path")
-
- @property
- def test_stderr(self):
- return _get_ya_plugin_instance().get_context("test_stderr")
-
- @property
+def test_source_path(path=None):
+ return _join_path(os.path.join(source_path(), context.project_path), path)
+
+
+class Context(object):
+ """
+ Runtime context
+ """
+
+ @property
+ def build_type(self):
+ return _get_ya_plugin_instance().get_context("build_type")
+
+ @property
+ def project_path(self):
+ return _get_ya_plugin_instance().get_context("project_path")
+
+ @property
+ def test_stderr(self):
+ return _get_ya_plugin_instance().get_context("test_stderr")
+
+ @property
def test_debug(self):
return _get_ya_plugin_instance().get_context("test_debug")
@property
- def test_traceback(self):
- return _get_ya_plugin_instance().get_context("test_traceback")
-
- @property
- def test_name(self):
+ def test_traceback(self):
+ return _get_ya_plugin_instance().get_context("test_traceback")
+
+ @property
+ def test_name(self):
return _get_ya_config().current_test_name
-
+
@property
def sanitize(self):
"""
@@ -340,4 +340,4 @@ class Context(object):
return _get_ya_plugin_instance().get_context(key)
-context = Context()
+context = Context()
diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py
index 39bbb45570..1aa7b04827 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime_java.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py
@@ -8,7 +8,7 @@ _JAVA_DIR = []
def get_java_path(jdk_dir):
- # deprecated - to be deleted
+ # deprecated - to be deleted
java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe'))
for p in java_paths:
@@ -25,11 +25,11 @@ def get_java_path(jdk_dir):
return p
return ''
-
-
-def get_build_java_dir(jdk_dir):
+
+
+def get_build_java_dir(jdk_dir):
versions = [8, 10, 11, 12, 13, 14, 15]
-
+
if not _JAVA_DIR:
for version in versions:
jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version))
diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py
index 9e7a74cdf5..c5401bd2ed 100644
--- a/library/python/testing/yatest_common/yatest/common/tags.py
+++ b/library/python/testing/yatest_common/yatest/common/tags.py
@@ -1,5 +1,5 @@
-try:
- import pytest
- ya_external = getattr(pytest.mark, "ya:external")
-except ImportError:
- ya_external = None
+try:
+ import pytest
+ ya_external = getattr(pytest.mark, "ya:external")
+except ImportError:
+ ya_external = None