diff options
author | Dmitry Kopylov <kopylovd@gmail.com> | 2022-02-10 16:48:18 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:18 +0300 |
commit | 7230275728d34873cba1ba78bb68669b0c5faa31 (patch) | |
tree | b222e5ac2e2e98872661c51ccceee5da0d291e13 /library/python/testing | |
parent | b2f5101486cc0de2e979c8ba9ada2109785bf5fd (diff) | |
download | ydb-7230275728d34873cba1ba78bb68669b0c5faa31.tar.gz |
Restoring authorship annotation for Dmitry Kopylov <kopylovd@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'library/python/testing')
18 files changed, 880 insertions, 880 deletions
diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py index 5b002f7aea..3e3b7234ef 100644 --- a/library/python/testing/import_test/import_test.py +++ b/library/python/testing/import_test/import_test.py @@ -2,7 +2,7 @@ from __future__ import print_function import os import re -import sys +import sys import time import traceback @@ -65,8 +65,8 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): s = time.time() if module == '__main__': importer.load_module('__main__', '__main__py') - elif module.endswith('.__init__'): - __import__(module[:-len('.__init__')]) + elif module.endswith('.__init__'): + __import__(module[:-len('.__init__')]) else: __import__(module) diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py index 8bc3b8a40b..5ef9c5c189 100644 --- a/library/python/testing/recipe/__init__.py +++ b/library/python/testing/recipe/__init__.py @@ -2,8 +2,8 @@ from __future__ import print_function import os import sys -import json -import logging +import json +import logging import argparse from yatest_lib.ya import Ya @@ -17,22 +17,22 @@ sanitizer_extra_checks = None def _setup_logging(level=logging.DEBUG): - root_logger = logging.getLogger() - root_logger.setLevel(level) - - log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' - - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setLevel(logging.DEBUG) - formatter = logging.Formatter(log_format) - stdout_handler.setFormatter(formatter) - root_logger.addHandler(stdout_handler) - - + root_logger = logging.getLogger() + root_logger.setLevel(level) + + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter(log_format) + stdout_handler.setFormatter(formatter) + root_logger.addHandler(stdout_handler) + + def get_options(): parser = argparse.ArgumentParser() parser.add_argument("--show-cwd", action="store_true", dest="show_cwd", default=False, help="show recipe cwd") - parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode") + parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode") parser.add_argument("--test-stderr", action="store_true", dest="test_stderr", default=False, help="test stderr") parser.add_argument("--pdb", action="store_true", dest="pdb", default=False, help="run pdb on error") parser.add_argument("--sanitizer-extra-checks", dest="sanitizer_extra_checks", action="store_true", default=False, help="enables extra checks for tests built with sanitizers") @@ -42,7 +42,7 @@ def get_options(): global ya, sanitizer_extra_checks, collect_cores _setup_logging() - + context = { "test_stderr": args.test_stderr, } diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make index 2672c6ca6e..883bc8d7ab 100644 --- a/library/python/testing/ya.make +++ b/library/python/testing/ya.make @@ -1,22 +1,22 @@ OWNER(g:yatest) - -RECURSE( - behave + +RECURSE( + behave deprecated fake_ya_package filter - gtest - gtest/test - gtest/test/gtest + gtest + gtest/test + gtest/test/gtest pyremock - pytest_runner - pytest_runner/example - pytest_runner/test + pytest_runner + pytest_runner/example + pytest_runner/test recipe system_info types_test yapackage yapackage/test - yatest_common - yatest_lib -) + yatest_common + yatest_lib +) diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make index 0cd9922c06..5662db4c5d 100644 --- a/library/python/testing/yatest_common/ya.make +++ b/library/python/testing/yatest_common/ya.make @@ -1,40 +1,40 @@ OWNER(g:yatest) - + PY23_LIBRARY() - + OWNER(g:yatest) - + NO_EXTENDED_SOURCE_SEARCH() -PY_SRCS( - TOP_LEVEL +PY_SRCS( + TOP_LEVEL yatest/__init__.py - yatest/common/__init__.py - yatest/common/benchmark.py - yatest/common/canonical.py - yatest/common/environment.py - yatest/common/errors.py - yatest/common/legacy.py + yatest/common/__init__.py + yatest/common/benchmark.py + yatest/common/canonical.py + yatest/common/environment.py + yatest/common/errors.py + yatest/common/legacy.py yatest/common/misc.py - yatest/common/network.py - yatest/common/path.py - yatest/common/process.py - yatest/common/runtime.py - yatest/common/runtime_java.py - yatest/common/tags.py + yatest/common/network.py + yatest/common/path.py + yatest/common/process.py + yatest/common/runtime.py + yatest/common/runtime_java.py + yatest/common/tags.py ) - + PEERDIR( contrib/python/six library/python/cores library/python/filelock library/python/fs -) - +) + IF (NOT CATBOOST_OPENSOURCE) PEERDIR( library/python/coredump_filter ) ENDIF() -END() +END() diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py index 19593ff8d2..b846b3317a 100644 --- a/library/python/testing/yatest_common/yatest/__init__.py +++ b/library/python/testing/yatest_common/yatest/__init__.py @@ -1,3 +1,3 @@ -__all__ = ["common"] - +__all__ = ["common"] + from . import common diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py index 500dfaca7e..c3784cbe4c 100644 --- a/library/python/testing/yatest_common/yatest/common/benchmark.py +++ b/library/python/testing/yatest_common/yatest/common/benchmark.py @@ -1,22 +1,22 @@ -import json - +import json + from . import process from . import runtime - - -def execute_benchmark(path, budget=None, threads=None): - """ - Run benchmark and return values - :param path: path to benchmark binary + + +def execute_benchmark(path, budget=None, threads=None): + """ + Run benchmark and return values + :param path: path to benchmark binary :param budget: time budget, sec (supported only by ybenchmark) :param threads: number of threads to run benchmark (supported only by ybenchmark) - :return: map of benchmark values - """ - benchmark_path = runtime.binary_path(path) + :return: map of benchmark values + """ + benchmark_path = runtime.binary_path(path) cmd = [benchmark_path, "--benchmark_format=json"] - if budget is not None: - cmd += ["-b", str(budget)] - if threads is not None: - cmd += ["-t", str(threads)] - res = process.execute(cmd) - return json.loads(res.std_out) + if budget is not None: + cmd += ["-b", str(budget)] + if threads is not None: + cmd += ["-t", str(threads)] + res = process.execute(cmd) + return json.loads(res.std_out) diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py index 9c02fe1c11..b6a136d3e9 100644 --- a/library/python/testing/yatest_common/yatest/common/canonical.py +++ b/library/python/testing/yatest_common/yatest/common/canonical.py @@ -1,17 +1,17 @@ -import os -import logging -import shutil -import tempfile - +import os +import logging +import shutil +import tempfile + import six from . import process from . import runtime from . import path - -yatest_logger = logging.getLogger("ya.test") - - + +yatest_logger = logging.getLogger("ya.test") + + def _copy(src, dst, universal_lines=False): if universal_lines: with open(dst, "wb") as f: @@ -21,21 +21,21 @@ def _copy(src, dst, universal_lines=False): shutil.copy(src, dst) -def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None): - """ - Create canonical file that can be returned from a test - :param path: path to the file - :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used - :param local: save file locally, otherwise move to sandbox +def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None): + """ + Create canonical file that can be returned from a test + :param path: path to the file + :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used + :param local: save file locally, otherwise move to sandbox :param universal_lines: normalize EOL - :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - abs_path = os.path.abspath(path) - assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) - tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path()) - safe_path = os.path.join(tempdir, os.path.basename(abs_path)) - # if the created file is in output_path, we copy it, so that it will be available when the tests finishes + :param diff_tool_timeout: timeout for running diff tool + :return: object that can be canonized + """ + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path()) + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + # if the created file is in output_path, we copy it, so that it will be available when the tests finishes _copy(path, safe_path, universal_lines=universal_lines) if diff_tool: if not isinstance(diff_tool, six.string_types): @@ -44,133 +44,133 @@ def canonical_file(path, diff_tool=None, local=False, universal_lines=False, dif raise Exception("Invalid custom diff-tool: not cmd") except: raise Exception("Invalid custom diff-tool: not binary path") - return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) - - -def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None): - abs_path = os.path.abspath(path) - assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) - assert os.path.isdir(abs_path), "Path {} is not a directory".format(path) - if diff_file_name and not diff_tool: - raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir") - tempdir = tempfile.mkdtemp() - safe_path = os.path.join(tempdir, os.path.basename(abs_path)) - shutil.copytree(abs_path, safe_path) - return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) - - -def canonical_execute( - binary, args=None, check_exit_code=True, - shell=False, timeout=None, cwd=None, - env=None, stdin=None, stderr=None, creationflags=0, + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None): + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + assert os.path.isdir(abs_path), "Path {} is not a directory".format(path) + if diff_file_name and not diff_tool: + raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir") + tempdir = tempfile.mkdtemp() + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + shutil.copytree(abs_path, safe_path) + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_execute( + binary, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, + env=None, stdin=None, stderr=None, creationflags=0, file_name=None, save_locally=False, close_fds=False, diff_tool=None, diff_file_name=None, diff_tool_timeout=None, -): - """ - Shortcut to execute a binary and canonize its stdout +): + """ + Shortcut to execute a binary and canonize its stdout :param binary: absolute path to the binary - :param args: binary arguments - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stderr: command stderr - :param creationflags: command creation flags + :param args: binary arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags :param file_name: output file name. if not specified program name will be used - :param diff_tool: path to custome diff tool - :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - if type(binary) == list: - command = binary - else: - command = [binary] - command += _prepare_args(args) - if shell: - command = " ".join(command) - execute_args = locals() - del execute_args["binary"] - del execute_args["args"] + :return: object that can be canonized + """ + if type(binary) == list: + command = binary + else: + command = [binary] + command += _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["binary"] + del execute_args["args"] del execute_args["file_name"] - del execute_args["save_locally"] - del execute_args["diff_tool"] - del execute_args["diff_file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] del execute_args["diff_tool_timeout"] - if not file_name and stdin: - file_name = os.path.basename(stdin.name) + if not file_name and stdin: + file_name = os.path.basename(stdin.name) return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) - - -def canonical_py_execute( - script_path, args=None, check_exit_code=True, - shell=False, timeout=None, cwd=None, env=None, - stdin=None, stderr=None, creationflags=0, - file_name=None, save_locally=False, close_fds=False, + + +def canonical_py_execute( + script_path, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, env=None, + stdin=None, stderr=None, creationflags=0, + file_name=None, save_locally=False, close_fds=False, diff_tool=None, diff_file_name=None, diff_tool_timeout=None, -): - """ - Shortcut to execute a python script and canonize its stdout - :param script_path: path to the script arcadia relative - :param args: script arguments - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stderr: command stderr - :param creationflags: command creation flags +): + """ + Shortcut to execute a python script and canonize its stdout + :param script_path: path to the script arcadia relative + :param args: script arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags :param file_name: output file name. if not specified program name will be used - :param diff_tool: path to custome diff tool - :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - command = [runtime.source_path(script_path)] + _prepare_args(args) - if shell: - command = " ".join(command) - execute_args = locals() - del execute_args["script_path"] - del execute_args["args"] + :return: object that can be canonized + """ + command = [runtime.source_path(script_path)] + _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["script_path"] + del execute_args["args"] del execute_args["file_name"] - del execute_args["save_locally"] - del execute_args["diff_tool"] - del execute_args["diff_file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] del execute_args["diff_tool_timeout"] return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) - - -def _prepare_args(args): - if args is None: - args = [] + + +def _prepare_args(args): + if args is None: + args = [] if isinstance(args, six.string_types): - args = map(lambda a: a.strip(), args.split()) - return args - - + args = map(lambda a: a.strip(), args.split()) + return args + + def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout): - res = excutor(**kwargs) - command = kwargs["command"] - file_name = file_name or process.get_command_name(command) - if file_name.endswith(".exe"): - file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names - out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name)) - err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name)) - - try: - os.makedirs(os.path.dirname(out_file_path)) - except OSError: - pass - + res = excutor(**kwargs) + command = kwargs["command"] + file_name = file_name or process.get_command_name(command) + if file_name.endswith(".exe"): + file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names + out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name)) + err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name)) + + try: + os.makedirs(os.path.dirname(out_file_path)) + except OSError: + pass + with open(out_file_path, "wb") as out_file: - yatest_logger.debug("Will store file in %s", out_file_path) - out_file.write(res.std_out) - - if res.std_err: + yatest_logger.debug("Will store file in %s", out_file_path) + out_file.write(res.std_out) + + if res.std_err: with open(err_file_path, "wb") as err_file: - err_file.write(res.std_err) - + err_file.write(res.std_err) + return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py index f3d55058e4..43f48d0958 100644 --- a/library/python/testing/yatest_common/yatest/common/environment.py +++ b/library/python/testing/yatest_common/yatest/common/environment.py @@ -1,5 +1,5 @@ # coding: utf-8 - + def extend_env_var(env, name, value, sep=":"): return sep.join(filter(None, [env.get(name), value])) diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py index 5cdd62c6d0..8c038fc381 100644 --- a/library/python/testing/yatest_common/yatest/common/errors.py +++ b/library/python/testing/yatest_common/yatest/common/errors.py @@ -1,20 +1,20 @@ import os -import sys - - -class RestartTestException(Exception): - - def __init__(self, *args, **kwargs): +import sys + + +class RestartTestException(Exception): + + def __init__(self, *args, **kwargs): super(RestartTestException, self).__init__(*args, **kwargs) - sys.stderr.write("##restart-test##\n") - sys.stderr.flush() + sys.stderr.write("##restart-test##\n") + sys.stderr.flush() os.environ["FORCE_EXIT_TESTSFAILED"] = "1" - - -class InfrastructureException(Exception): - - def __init__(self, *args, **kwargs): + + +class InfrastructureException(Exception): + + def __init__(self, *args, **kwargs): super(InfrastructureException, self).__init__(*args, **kwargs) - sys.stderr.write("##infrastructure-error##\n") - sys.stderr.flush() + sys.stderr.write("##infrastructure-error##\n") + sys.stderr.flush() os.environ["FORCE_EXIT_TESTSFAILED"] = "1" diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py index cfb6975802..459972d253 100644 --- a/library/python/testing/yatest_common/yatest/common/legacy.py +++ b/library/python/testing/yatest_common/yatest/common/legacy.py @@ -1,12 +1,12 @@ from . import canonical - - -def old_canonical_file(output_file_name, storage_md5): - import yalibrary.svn - yalibrary.svn.run_svn([ - 'export', - 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5, - output_file_name, - "--force" - ]) - return canonical.canonical_file(output_file_name) + + +def old_canonical_file(output_file_name, storage_md5): + import yalibrary.svn + yalibrary.svn.run_svn([ + 'export', + 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5, + output_file_name, + "--force" + ]) + return canonical.canonical_file(output_file_name) diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py index d31df2d278..6fed7dda8a 100644 --- a/library/python/testing/yatest_common/yatest/common/path.py +++ b/library/python/testing/yatest_common/yatest/common/path.py @@ -1,5 +1,5 @@ # coding=utf-8 - + import errno import os import shutil @@ -8,19 +8,19 @@ import contextlib import library.python.fs as lpf -def replace_in_file(path, old, new): - """ - Replace text occurrences in a file - :param path: path to the file - :param old: text to replace - :param new: replacement - """ - with open(path) as fp: - content = fp.read() - +def replace_in_file(path, old, new): + """ + Replace text occurrences in a file + :param path: path to the file + :param old: text to replace + :param new: replacement + """ + with open(path) as fp: + content = fp.read() + lpf.ensure_removed(path) - with open(path, 'w') as fp: - fp.write(content.replace(old, new)) + with open(path, 'w') as fp: + fp.write(content.replace(old, new)) @contextlib.contextmanager @@ -38,8 +38,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): Copy an entire directory of files into an existing directory instead of raising Exception what shtuil.copytree does ''' - if not os.path.exists(dst) and os.path.isdir(src): - os.makedirs(dst) + if not os.path.exists(dst) and os.path.isdir(src): + os.makedirs(dst) for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) @@ -54,8 +54,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): postprocessing(os.path.join(root, path), False) for path in files: postprocessing(os.path.join(root, path), True) - - + + def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000): def atomic_file_create(path): try: @@ -79,12 +79,12 @@ def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10 return False raise e - file_path = os.path.join(dir_path, file_pattern) + file_path = os.path.join(dir_path, file_pattern) lpf.ensure_dir(os.path.dirname(file_path)) - file_counter = 0 + file_counter = 0 handler = atomic_file_create if create_file else atomic_dir_create while os.path.exists(file_path) or not handler(file_path): - file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter)) - file_counter += 1 + file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter)) + file_counter += 1 assert file_counter < max_suffix - return file_path + return file_path diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py index 672ebe0823..a8bcc21f51 100644 --- a/library/python/testing/yatest_common/yatest/common/process.py +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -1,16 +1,16 @@ # coding: utf-8 -import os +import os import re -import time -import signal +import time +import signal import shutil -import logging -import tempfile -import subprocess -import errno +import logging +import tempfile +import subprocess +import errno import distutils.version - + import six try: @@ -22,13 +22,13 @@ except ImportError: from . import runtime from . import path from . import environment - - + + MAX_OUT_LEN = 1000 * 1000 # 1 mb MAX_MESSAGE_LEN = 1500 SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)" GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)") -yatest_logger = logging.getLogger("ya.test") +yatest_logger = logging.getLogger("ya.test") def truncate(s, size): @@ -40,45 +40,45 @@ def truncate(s, size): return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):] -def get_command_name(command): +def get_command_name(command): return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0]) - - -class ExecutionError(Exception): - - def __init__(self, execution_result): + + +class ExecutionError(Exception): + + def __init__(self, execution_result): if not isinstance(execution_result.command, six.string_types): command = " ".join(str(arg) for arg in execution_result.command) else: command = execution_result.command - message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format( + message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format( command=command, - code=execution_result.exit_code, - err=_format_error(execution_result.std_err)) + code=execution_result.exit_code, + err=_format_error(execution_result.std_err)) if cores: if execution_result.backtrace: message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) else: message += "Backtrace is not available: module cores isn't available" - super(ExecutionError, self).__init__(message) + super(ExecutionError, self).__init__(message) + self.execution_result = execution_result + + +class TimeoutError(Exception): + pass + + +class ExecutionTimeoutError(TimeoutError): + def __init__(self, execution_result, *args, **kwargs): + super(ExecutionTimeoutError, self).__init__(args, kwargs) self.execution_result = execution_result - - -class TimeoutError(Exception): - pass - - -class ExecutionTimeoutError(TimeoutError): - def __init__(self, execution_result, *args, **kwargs): - super(ExecutionTimeoutError, self).__init__(args, kwargs) - self.execution_result = execution_result - - -class InvalidExecutionStateError(Exception): - pass - - + + +class InvalidExecutionStateError(Exception): + pass + + class SignalInterruptionError(Exception): def __init__(self, message=None): super(SignalInterruptionError, self).__init__(message) @@ -89,39 +89,39 @@ class InvalidCommandError(Exception): pass -class _Execution(object): - +class _Execution(object): + def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): - self._command = command - self._process = process - self._out_file = out_file - self._err_file = err_file - self._std_out = None - self._std_err = None + self._command = command + self._process = process + self._out_file = out_file + self._err_file = err_file + self._std_out = None + self._std_err = None self._elapsed = None self._start = time.time() - self._process_progress_listener = process_progress_listener + self._process_progress_listener = process_progress_listener self._cwd = cwd or os.getcwd() self._collect_cores = collect_cores self._backtrace = '' self._check_sanitizer = check_sanitizer - self._metrics = {} + self._metrics = {} self._started = started self._user_stdout = bool(user_stdout) self._user_stderr = bool(user_stderr) self._exit_code = None - if process_progress_listener: - process_progress_listener.open(command, process, out_file, err_file) - - @property - def running(self): - return self._process.poll() is None - - def kill(self): - if self.running: - self._save_outputs(False) - _kill_process_tree(self._process.pid) - self._clean_files() + if process_progress_listener: + process_progress_listener.open(command, process, out_file, err_file) + + @property + def running(self): + return self._process.poll() is None + + def kill(self): + if self.running: + self._save_outputs(False) + _kill_process_tree(self._process.pid) + self._clean_files() # DEVTOOLS-2347 yatest_logger.debug("Process status before wait_for: %s", self.running) try: @@ -130,64 +130,64 @@ class _Execution(object): yatest_logger.debug("Process status after wait_for: %s", self.running) yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) raise - else: - raise InvalidExecutionStateError("Cannot kill a stopped process") - + else: + raise InvalidExecutionStateError("Cannot kill a stopped process") + def terminate(self): if self.running: self._process.terminate() - @property - def process(self): - return self._process - - @property - def command(self): - return self._command - - @property + @property + def process(self): + return self._process + + @property + def command(self): + return self._command + + @property def returncode(self): return self.exit_code @property - def exit_code(self): + def exit_code(self): """ Deprecated, use returncode """ if self._exit_code is None: self._exit_code = self._process.returncode return self._exit_code - - @property + + @property def stdout(self): return self.std_out @property - def std_out(self): + def std_out(self): """ Deprecated, use stdout """ - if self._std_out is not None: - return self._std_out + if self._std_out is not None: + return self._std_out if self._process.stdout and not self._user_stdout: self._std_out = self._process.stdout.read() return self._std_out - - @property + + @property def stderr(self): return self.std_err @property - def std_err(self): + def std_err(self): """ Deprecated, use stderr """ - if self._std_err is not None: - return self._std_err + if self._std_err is not None: + return self._std_err if self._process.stderr and not self._user_stderr: self._std_err = self._process.stderr.read() return self._std_err - + @property def elapsed(self): return self._elapsed @@ -196,14 +196,14 @@ class _Execution(object): def backtrace(self): return self._backtrace - @property - def metrics(self): - return self._metrics - - def _save_outputs(self, clean_files=True): - if self._process_progress_listener: - self._process_progress_listener() - self._process_progress_listener.close() + @property + def metrics(self): + return self._metrics + + def _save_outputs(self, clean_files=True): + if self._process_progress_listener: + self._process_progress_listener() + self._process_progress_listener.close() if not self._user_stdout: if self._out_file is None: pass @@ -223,14 +223,14 @@ class _Execution(object): else: self._std_err = self._process.stderr.read() - if clean_files: - self._clean_files() - yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) - yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed) - if self._metrics: + if clean_files: + self._clean_files() + yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) + yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed) + if self._metrics: for key, value in six.iteritems(self._metrics): - yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value) - + yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value) + # Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str. printable_std_out, err = _try_convert_bytes_to_string(self._std_out) if err: @@ -244,14 +244,14 @@ class _Execution(object): yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN)) yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN)) - def _clean_files(self): + def _clean_files(self): if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: self._err_file.close() self._err_file = None if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: self._out_file.close() self._out_file = None - + def _recover_core(self): core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) if core_path: @@ -260,7 +260,7 @@ class _Execution(object): if store_cores: new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) # Copy core dump file, because it may be overwritten - yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) + yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) shutil.copyfile(core_path, new_core_path) core_path = new_core_path @@ -281,92 +281,92 @@ class _Execution(object): else: runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) - def wait(self, check_exit_code=True, timeout=None, on_timeout=None): - def _wait(): + def wait(self, check_exit_code=True, timeout=None, on_timeout=None): + def _wait(): finished = None interrupted = False - try: - if hasattr(os, "wait4"): - try: + try: + if hasattr(os, "wait4"): + try: if hasattr(subprocess, "_eintr_retry_call"): pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0) else: # PEP 475 pid, sts, rusage = os.wait4(self._process.pid, 0) finished = time.time() - self._process._handle_exitstatus(sts) - for field in [ - "ru_idrss", - "ru_inblock", - "ru_isrss", - "ru_ixrss", - "ru_majflt", - "ru_maxrss", - "ru_minflt", - "ru_msgrcv", - "ru_msgsnd", - "ru_nivcsw", - "ru_nsignals", - "ru_nswap", - "ru_nvcsw", - "ru_oublock", - "ru_stime", - "ru_utime", - ]: - if hasattr(rusage, field): - self._metrics[field.replace("ru_", "")] = getattr(rusage, field) - except OSError as exc: - - if exc.errno == errno.ECHILD: - yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") - else: - raise + self._process._handle_exitstatus(sts) + for field in [ + "ru_idrss", + "ru_inblock", + "ru_isrss", + "ru_ixrss", + "ru_majflt", + "ru_maxrss", + "ru_minflt", + "ru_msgrcv", + "ru_msgsnd", + "ru_nivcsw", + "ru_nsignals", + "ru_nswap", + "ru_nvcsw", + "ru_oublock", + "ru_stime", + "ru_utime", + ]: + if hasattr(rusage, field): + self._metrics[field.replace("ru_", "")] = getattr(rusage, field) + except OSError as exc: + + if exc.errno == errno.ECHILD: + yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") + else: + raise except SignalInterruptionError: interrupted = True raise - finally: + finally: if not interrupted: self._process.wait() # this has to be here unconditionally, so that all process properties are set - + if not finished: finished = time.time() self._metrics["wtime"] = round(finished - self._started, 3) - try: + try: if timeout: process_is_finished = lambda: not self.running fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) - try: - wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) - except TimeoutError as e: - if on_timeout: + try: + wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) + except TimeoutError as e: + if on_timeout: yatest_logger.debug("Calling user specified on_timeout function") - try: - on_timeout(self, timeout) - except Exception: - yatest_logger.exception("Exception while calling on_timeout") - raise ExecutionTimeoutError(self, str(e)) + try: + on_timeout(self, timeout) + except Exception: + yatest_logger.exception("Exception while calling on_timeout") + raise ExecutionTimeoutError(self, str(e)) # Wait should be always called here, it finalizes internal states of its process and sets up return code - _wait() + _wait() except BaseException as e: - _kill_process_tree(self._process.pid) - _wait() - yatest_logger.debug("Command exception: %s", e) + _kill_process_tree(self._process.pid) + _wait() + yatest_logger.debug("Command exception: %s", e) raise - finally: + finally: self._elapsed = time.time() - self._start self._save_outputs() self.verify_no_coredumps() - + self._finalise(check_exit_code) def _finalise(self, check_exit_code): - # Set the signal (negative number) which caused the process to exit - if check_exit_code and self.exit_code != 0: - yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", - self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) - raise ExecutionError(self) - + # Set the signal (negative number) which caused the process to exit + if check_exit_code and self.exit_code != 0: + yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", + self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) + # Don't search for sanitize errors if stderr was redirected self.verify_sanitize_errors() @@ -394,11 +394,11 @@ class _Execution(object): if match: yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) raise ExecutionError(self) - else: - yatest_logger.debug("No sanitizer errors found") + else: + yatest_logger.debug("No sanitizer errors found") else: - yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) - + yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) + def on_timeout_gen_coredump(exec_obj, _): """ @@ -413,38 +413,38 @@ def on_timeout_gen_coredump(exec_obj, _): pass -def execute( - command, check_exit_code=True, - shell=False, timeout=None, - cwd=None, env=None, - stdin=None, stdout=None, stderr=None, - creationflags=0, wait=True, - process_progress_listener=None, close_fds=False, - collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, +def execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False, + collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, executor=_Execution, -): - """ - Executes a command - :param command: command: can be a list of arguments or a string - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stdout: command stdout - :param stderr: command stderr - :param creationflags: command creation flags - :param wait: should wait until the command finishes - :param process_progress_listener=object that is polled while execution is in progress - :param close_fds: subrpocess.Popen close_fds args +): + """ + Executes a command + :param command: command: can be a list of arguments or a string + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress + :param close_fds: subrpocess.Popen close_fds args :param collect_cores: recover core dump files if shell == False :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors - :param preexec_fn: subrpocess.Popen preexec_fn arg - :param on_timeout: on_timeout(<execution object>, <timeout value>) callback - + :param preexec_fn: subrpocess.Popen preexec_fn arg + :param on_timeout: on_timeout(<execution object>, <timeout value>) callback + :return _Execution: Execution object - """ + """ if env is None: env = os.environ.copy() else: @@ -462,9 +462,9 @@ def execute( if var not in env and var in os.environ: env[var] = os.environ[var] - if not wait and timeout is not None: - raise ValueError("Incompatible arguments 'timeout' and wait=False") - + if not wait and timeout is not None: + raise ValueError("Incompatible arguments 'timeout' and wait=False") + # if subprocess.PIPE in [stdout, stderr]: # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") @@ -481,15 +481,15 @@ def execute( is_pipe = stream == subprocess.PIPE return stream, not is_pipe - # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE, - # as it can cause processes hangs, but use tempfiles instead + # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE, + # as it can cause processes hangs, but use tempfiles instead out_file, user_stdout = get_out_stream(stdout, 'out') err_file, user_stderr = get_out_stream(stderr, 'err') - in_file = stdin - - if shell and type(command) == list: - command = " ".join(command) - + in_file = stdin + + if shell and type(command) == list: + command = " ".join(command) + if shell: collect_cores = False check_sanitizer = False @@ -516,7 +516,7 @@ def execute( name = "PIPE" if stdin == subprocess.PIPE else stdin.name yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) else: - yatest_logger.debug("Executing '%s' in '%s'", command, cwd) + yatest_logger.debug("Executing '%s' in '%s'", command, cwd) # XXX started = time.time() @@ -528,11 +528,11 @@ def execute( yatest_logger.debug("Command pid: %s", process.pid) res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) - if wait: - res.wait(check_exit_code, timeout, on_timeout) - return res - - + if wait: + res.wait(check_exit_code, timeout, on_timeout) + return res + + def _get_command_output_file(cmd, ext): parts = [get_command_name(cmd)] if 'YA_RETRY_INDEX' in os.environ: @@ -570,74 +570,74 @@ def py_execute( creationflags=0, wait=True, process_progress_listener=None, close_fds=False ): - """ - Executes a command with the arcadia python - :param command: command to pass to python - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stdout: command stdout - :param stderr: command stderr - :param creationflags: command creation flags - :param wait: should wait until the command finishes - :param process_progress_listener=object that is polled while execution is in progress + """ + Executes a command with the arcadia python + :param command: command to pass to python + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress :return _Execution: Execution object - """ + """ if isinstance(command, six.string_types): - command = [command] - command = [runtime.python_path()] + command - if shell: - command = " ".join(command) - return execute(**locals()) - - -def _format_error(error): - return truncate(error, MAX_MESSAGE_LEN) - - -def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None): - """ - Tries to execute `check_function` for `timeout` seconds. - Continue until function returns nonfalse value. - If function doesn't return nonfalse value for `timeout` seconds - OperationTimeoutException is raised. - Return first nonfalse result returned by `checkFunction`. - """ - if sleep_time <= 0: - raise ValueError("Incorrect sleep time value {}".format(sleep_time)) - if timeout < 0: - raise ValueError("Incorrect timeout value {}".format(timeout)) - start = time.time() - while start + timeout > time.time(): - if on_check_condition: - on_check_condition() - - res = check_function() - if res: - return res - time.sleep(sleep_time) - - message = "{} second(s) wait timeout has expired".format(timeout) - if fail_message: - message += ": {}".format(fail_message) + command = [command] + command = [runtime.python_path()] + command + if shell: + command = " ".join(command) + return execute(**locals()) + + +def _format_error(error): + return truncate(error, MAX_MESSAGE_LEN) + + +def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None): + """ + Tries to execute `check_function` for `timeout` seconds. + Continue until function returns nonfalse value. + If function doesn't return nonfalse value for `timeout` seconds + OperationTimeoutException is raised. + Return first nonfalse result returned by `checkFunction`. + """ + if sleep_time <= 0: + raise ValueError("Incorrect sleep time value {}".format(sleep_time)) + if timeout < 0: + raise ValueError("Incorrect timeout value {}".format(timeout)) + start = time.time() + while start + timeout > time.time(): + if on_check_condition: + on_check_condition() + + res = check_function() + if res: + return res + time.sleep(sleep_time) + + message = "{} second(s) wait timeout has expired".format(timeout) + if fail_message: + message += ": {}".format(fail_message) raise TimeoutError(truncate(message, MAX_MESSAGE_LEN)) - + def _kill_process_tree(process_pid, target_pid_signal=None): - """ - Kills child processes, req. Note that psutil should be installed - @param process_pid: parent id to search for descendants - """ - yatest_logger.debug("Killing process %s", process_pid) - if os.name == 'nt': - _win_kill_process_tree(process_pid) - else: + """ + Kills child processes, req. Note that psutil should be installed + @param process_pid: parent id to search for descendants + """ + yatest_logger.debug("Killing process %s", process_pid) + if os.name == 'nt': + _win_kill_process_tree(process_pid) + else: _nix_kill_process_tree(process_pid, target_pid_signal) - - + + def _nix_get_proc_children(pid): try: cmd = ["pgrep", "-P", str(pid)] @@ -654,21 +654,21 @@ def _get_binname(pid): def _nix_kill_process_tree(pid, target_pid_signal=None): - """ - Kills the process tree. - """ + """ + Kills the process tree. + """ yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) - - def try_to_send_signal(pid, sig): - try: - os.kill(pid, sig) + + def try_to_send_signal(pid, sig): + try: + os.kill(pid, sig) yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) - except Exception as exc: + except Exception as exc: yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) - - try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. - - # Get the child process PID list. + + try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. + + # Get the child process PID list. child_pids = _nix_get_proc_children(pid) # Stop the child processes. for child_pid in child_pids: @@ -679,16 +679,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None): # Skip the error and continue killing. yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) continue - + try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process. - - # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent - # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html - try_to_send_signal(pid, signal.SIGCONT) - - -def _win_kill_process_tree(pid): - subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) + + # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent + # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html + try_to_send_signal(pid, signal.SIGCONT) + + +def _win_kill_process_tree(pid): + subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) def _run_readelf(binary_path): diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py index aa9161aead..e55e193446 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime.py +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -3,49 +3,49 @@ import functools import json import os import threading - + import six - + _lock = threading.Lock() def _get_ya_config(): - try: + try: import library.python.pytest.plugins.ya as ya_plugin if ya_plugin.pytest_config is not None: return ya_plugin.pytest_config - import pytest + import pytest return pytest.config - except (ImportError, AttributeError): + except (ImportError, AttributeError): try: import library.python.testing.recipe if library.python.testing.recipe.ya: return library.python.testing.recipe except (ImportError, AttributeError): pass - raise NotImplementedError("yatest.common.* is only available from the testing runtime") - - + raise NotImplementedError("yatest.common.* is only available from the testing runtime") + + def _get_ya_plugin_instance(): return _get_ya_config().ya -def _norm_path(path): - if path is None: - return None +def _norm_path(path): + if path is None: + return None assert isinstance(path, six.string_types) - if "\\" in path: - raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path)) - return os.path.normpath(path) - - -def _join_path(main_path, path): - if not path: - return main_path - return os.path.join(main_path, _norm_path(path)) - - + if "\\" in path: + raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path)) + return os.path.normpath(path) + + +def _join_path(main_path, path): + if not path: + return main_path + return os.path.join(main_path, _norm_path(path)) + + def not_test(func): """ Mark any function as not a test for py.test @@ -60,70 +60,70 @@ def not_test(func): def source_path(path=None): - """ - Get source path inside arcadia + """ + Get source path inside arcadia :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya') - :return: absolute path to the source folder - """ + :return: absolute path to the source folder + """ return _join_path(_get_ya_plugin_instance().source_root, path) - - + + def build_path(path=None): - """ - Get path inside build directory + """ + Get path inside build directory :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin') - :return: absolute path inside build directory - """ + :return: absolute path inside build directory + """ return _join_path(_get_ya_plugin_instance().build_root, path) - - + + def java_path(): """ - [DEPRECATED] Get path to java + [DEPRECATED] Get path to java :return: absolute path to java """ from . import runtime_java return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk'))) -def java_home(): - """ - Get jdk directory path - """ +def java_home(): + """ + Get jdk directory path + """ from . import runtime_java - jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) - if not jdk_dir: - raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") - return jdk_dir - - -def java_bin(): - """ - Get path to the java binary + jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) + if not jdk_dir: + raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") + return jdk_dir + + +def java_bin(): + """ + Get path to the java binary Requires DEPENDS(jdk) - """ - return os.path.join(java_home(), "bin", "java") - - + """ + return os.path.join(java_home(), "bin", "java") + + def data_path(path=None): - """ - Get path inside arcadia_tests_data directory + """ + Get path inside arcadia_tests_data directory :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") - :return: absolute path inside arcadia_tests_data - """ + :return: absolute path inside arcadia_tests_data + """ return _join_path(_get_ya_plugin_instance().data_root, path) - - + + def output_path(path=None): - """ - Get path inside the current test suite output dir. - Placing files to this dir guarantees that files will be accessible after the test suite execution. - :param path: path relative to the test suite output dir - :return: absolute path inside the test suite output dir - """ + """ + Get path inside the current test suite output dir. + Placing files to this dir guarantees that files will be accessible after the test suite execution. + :param path: path relative to the test suite output dir + :return: absolute path inside the test suite output dir + """ return _join_path(_get_ya_plugin_instance().output_dir, path) - - + + def ram_drive_path(path=None): """ :param path: path relative to the ram drive. @@ -144,60 +144,60 @@ def output_ram_drive_path(path=None): def binary_path(path=None): - """ - Get path to the built binary + """ + Get path to the built binary :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin') - :return: absolute path to the binary - """ + :return: absolute path to the binary + """ path = _norm_path(path) return _get_ya_plugin_instance().get_binary(path) - - + + def work_path(path=None): - """ - Get path inside the current test suite working directory. Creating files in the work directory does not guarantee - that files will be accessible after the test suite execution - :param path: path relative to the test suite working dir - :return: absolute path inside the test suite working dir - """ + """ + Get path inside the current test suite working directory. Creating files in the work directory does not guarantee + that files will be accessible after the test suite execution + :param path: path relative to the test suite working dir + :return: absolute path inside the test suite working dir + """ return _join_path( os.environ.get("TEST_WORK_PATH") or _get_ya_plugin_instance().get_context("work_path") or os.getcwd(), path) - - -def python_path(): - """ + + +def python_path(): + """ Get path to the arcadia python. Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles are built in a stripped-down form that is needed for building, not running tests. See comments in the file below to find out which version of python is compatible with tests. https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc - :return: absolute path to python - """ - return _get_ya_plugin_instance().python_path - - -def valgrind_path(): - """ - Get path to valgrind - :return: absolute path to valgrind - """ - return _get_ya_plugin_instance().valgrind_path - - -def get_param(key, default=None): - """ - Get arbitrary parameter passed via command line - :param key: key - :param default: default value - :return: parameter value or the default - """ - return _get_ya_plugin_instance().get_param(key, default) - - + :return: absolute path to python + """ + return _get_ya_plugin_instance().python_path + + +def valgrind_path(): + """ + Get path to valgrind + :return: absolute path to valgrind + """ + return _get_ya_plugin_instance().valgrind_path + + +def get_param(key, default=None): + """ + Get arbitrary parameter passed via command line + :param key: key + :param default: default value + :return: parameter value or the default + """ + return _get_ya_plugin_instance().get_param(key, default) + + def get_param_dict_copy(): """ Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters. @@ -209,25 +209,25 @@ def get_param_dict_copy(): @not_test def test_output_path(path=None): - """ - Get dir in the suite output_path for the current test case - """ + """ + Get dir in the suite output_path for the current test case + """ test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0] try: - os.makedirs(test_out_dir) + os.makedirs(test_out_dir) except OSError as e: if e.errno != errno.EEXIST: raise return _join_path(test_out_dir, path) - - -def project_path(path=None): - """ - Get path in build root relating to build_root/project path - """ - return _join_path(os.path.join(build_path(), context.project_path), path) - - + + +def project_path(path=None): + """ + Get path in build root relating to build_root/project path + """ + return _join_path(os.path.join(build_path(), context.project_path), path) + + def gdb_path(): """ Get path to the gdb @@ -235,25 +235,25 @@ def gdb_path(): return _get_ya_plugin_instance().gdb_path -def c_compiler_path(): - """ - Get path to the gdb - """ - return os.environ.get("YA_CC") - - +def c_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CC") + + def get_yt_hdd_path(path=None): if 'HDD_PATH' in os.environ: return _join_path(os.environ['HDD_PATH'], path) -def cxx_compiler_path(): - """ - Get path to the gdb - """ - return os.environ.get("YA_CXX") - - +def cxx_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CXX") + + def global_resources(): try: return json.loads(os.environ.get("YA_GLOBAL_RESOURCES")) @@ -282,39 +282,39 @@ def _register_core(name, binary_path, core_path, bt_path, pbt_path): @not_test -def test_source_path(path=None): - return _join_path(os.path.join(source_path(), context.project_path), path) - - -class Context(object): - """ - Runtime context - """ - - @property - def build_type(self): - return _get_ya_plugin_instance().get_context("build_type") - - @property - def project_path(self): - return _get_ya_plugin_instance().get_context("project_path") - - @property - def test_stderr(self): - return _get_ya_plugin_instance().get_context("test_stderr") - - @property +def test_source_path(path=None): + return _join_path(os.path.join(source_path(), context.project_path), path) + + +class Context(object): + """ + Runtime context + """ + + @property + def build_type(self): + return _get_ya_plugin_instance().get_context("build_type") + + @property + def project_path(self): + return _get_ya_plugin_instance().get_context("project_path") + + @property + def test_stderr(self): + return _get_ya_plugin_instance().get_context("test_stderr") + + @property def test_debug(self): return _get_ya_plugin_instance().get_context("test_debug") @property - def test_traceback(self): - return _get_ya_plugin_instance().get_context("test_traceback") - - @property - def test_name(self): + def test_traceback(self): + return _get_ya_plugin_instance().get_context("test_traceback") + + @property + def test_name(self): return _get_ya_config().current_test_name - + @property def sanitize(self): """ @@ -340,4 +340,4 @@ class Context(object): return _get_ya_plugin_instance().get_context(key) -context = Context() +context = Context() diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py index 1aa7b04827..39bbb45570 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime_java.py +++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py @@ -8,7 +8,7 @@ _JAVA_DIR = [] def get_java_path(jdk_dir): - # deprecated - to be deleted + # deprecated - to be deleted java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe')) for p in java_paths: @@ -25,11 +25,11 @@ def get_java_path(jdk_dir): return p return '' - - -def get_build_java_dir(jdk_dir): + + +def get_build_java_dir(jdk_dir): versions = [8, 10, 11, 12, 13, 14, 15] - + if not _JAVA_DIR: for version in versions: jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version)) diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py index c5401bd2ed..9e7a74cdf5 100644 --- a/library/python/testing/yatest_common/yatest/common/tags.py +++ b/library/python/testing/yatest_common/yatest/common/tags.py @@ -1,5 +1,5 @@ -try: - import pytest - ya_external = getattr(pytest.mark, "ya:external") -except ImportError: - ya_external = None +try: + import pytest + ya_external = getattr(pytest.mark, "ya:external") +except ImportError: + ya_external = None diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py index 459d12c878..39113230d9 100644 --- a/library/python/testing/yatest_lib/external.py +++ b/library/python/testing/yatest_lib/external.py @@ -1,147 +1,147 @@ -from __future__ import absolute_import - +from __future__ import absolute_import + import re import sys -import copy +import copy import logging - -from . import tools + +from . import tools from datetime import date, datetime - + import enum import six logger = logging.getLogger(__name__) MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' - + def apply(func, value, apply_to_keys=False): - """ - Applies func to every possible member of value - :param value: could be either a primitive object or a complex one (list, dicts) - :param func: func to be applied - :return: - """ - def _apply(func, value, value_path): - if value_path is None: - value_path = [] - - if isinstance(value, list) or isinstance(value, tuple): - res = [] - for ind, item in enumerate(value): - path = copy.copy(value_path) - path.append(ind) - res.append(_apply(func, item, path)) - elif isinstance(value, dict): - if is_external(value): - # this is a special serialized object pointing to some external place - res = func(value, value_path) - else: - res = {} - for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]): - path = copy.copy(value_path) - path.append(key) + """ + Applies func to every possible member of value + :param value: could be either a primitive object or a complex one (list, dicts) + :param func: func to be applied + :return: + """ + def _apply(func, value, value_path): + if value_path is None: + value_path = [] + + if isinstance(value, list) or isinstance(value, tuple): + res = [] + for ind, item in enumerate(value): + path = copy.copy(value_path) + path.append(ind) + res.append(_apply(func, item, path)) + elif isinstance(value, dict): + if is_external(value): + # this is a special serialized object pointing to some external place + res = func(value, value_path) + else: + res = {} + for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]): + path = copy.copy(value_path) + path.append(key) res[_apply(func, key, path) if apply_to_keys else key] = _apply(func, val, path) - else: - res = func(value, value_path) - return res - return _apply(func, value, None) - - + else: + res = func(value, value_path) + return res + return _apply(func, value, None) + + def is_coroutine(val): if sys.version_info[0] < 3: return False else: import asyncio - return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val) - - -def serialize(value): - """ - Serialize value to json-convertible object - Ensures that all components of value can be serialized to json - :param value: object to be serialized - """ - def _serialize(val, _): - if val is None: - return val + return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val) + + +def serialize(value): + """ + Serialize value to json-convertible object + Ensures that all components of value can be serialized to json + :param value: object to be serialized + """ + def _serialize(val, _): + if val is None: + return val if isinstance(val, six.string_types) or isinstance(val, bytes): - return tools.to_utf8(val) + return tools.to_utf8(val) if isinstance(val, enum.Enum): return str(val) if isinstance(val, six.integer_types) or type(val) in [float, bool]: - return val - if is_external(val): - return dict(val) + return val + if is_external(val): + return dict(val) if isinstance(val, (date, datetime)): return repr(val) if is_coroutine(val): - return None - raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val))) + return None + raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val))) return apply(_serialize, value, apply_to_keys=True) - - -def is_external(value): - return isinstance(value, dict) and "uri" in value.keys() - - -class ExternalSchema(object): - File = "file" - SandboxResource = "sbr" - Delayed = "delayed" + + +def is_external(value): + return isinstance(value, dict) and "uri" in value.keys() + + +class ExternalSchema(object): + File = "file" + SandboxResource = "sbr" + Delayed = "delayed" HTTP = "http" - - -class CanonicalObject(dict): - def __iter__(self): - raise TypeError("Iterating canonical object is not implemented") - - -class ExternalDataInfo(object): - - def __init__(self, data): - assert is_external(data) - self._data = data - - def __str__(self): - type_str = "File" if self.is_file else "Sandbox resource" - return "{}({})".format(type_str, self.path) - - def __repr__(self): - return str(self) - - @property - def uri(self): - return self._data["uri"] - - @property - def checksum(self): - return self._data.get("checksum") - - @property - def is_file(self): - return self.uri.startswith(ExternalSchema.File) - - @property - def is_sandbox_resource(self): - return self.uri.startswith(ExternalSchema.SandboxResource) - - @property - def is_delayed(self): - return self.uri.startswith(ExternalSchema.Delayed) - - @property + + +class CanonicalObject(dict): + def __iter__(self): + raise TypeError("Iterating canonical object is not implemented") + + +class ExternalDataInfo(object): + + def __init__(self, data): + assert is_external(data) + self._data = data + + def __str__(self): + type_str = "File" if self.is_file else "Sandbox resource" + return "{}({})".format(type_str, self.path) + + def __repr__(self): + return str(self) + + @property + def uri(self): + return self._data["uri"] + + @property + def checksum(self): + return self._data.get("checksum") + + @property + def is_file(self): + return self.uri.startswith(ExternalSchema.File) + + @property + def is_sandbox_resource(self): + return self.uri.startswith(ExternalSchema.SandboxResource) + + @property + def is_delayed(self): + return self.uri.startswith(ExternalSchema.Delayed) + + @property def is_http(self): return self.uri.startswith(ExternalSchema.HTTP) @property - def path(self): + def path(self): if self.uri.count("://") != 1: logger.error("Invalid external data uri: '%s'", self.uri) return self.uri - _, path = self.uri.split("://") - return path - + _, path = self.uri.split("://") + return path + def get_mds_key(self): assert self.is_http m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri) @@ -149,44 +149,44 @@ class ExternalDataInfo(object): return m.group(1) raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri)) - @property - def size(self): - return self._data.get("size") - - def serialize(self): - return self._data - - @classmethod - def _serialize(cls, schema, path, checksum=None, attrs=None): - res = CanonicalObject({"uri": "{}://{}".format(schema, path)}) - if checksum: - res["checksum"] = checksum - if attrs: - res.update(attrs) - return res - - @classmethod - def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None): - attrs = {} - if diff_tool: - attrs["diff_tool"] = diff_tool - if local: - attrs["local"] = local - if diff_file_name: - attrs["diff_file_name"] = diff_file_name - if diff_tool_timeout: - attrs["diff_tool_timeout"] = diff_tool_timeout - if size is not None: - attrs["size"] = size - return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs) - - @classmethod - def serialize_resource(cls, id, checksum=None): - return cls._serialize(ExternalSchema.SandboxResource, id, checksum) - - @classmethod - def serialize_delayed(cls, upload_id, checksum): - return cls._serialize(ExternalSchema.Delayed, upload_id, checksum) - - def get(self, key, default=None): - return self._data.get(key, default) + @property + def size(self): + return self._data.get("size") + + def serialize(self): + return self._data + + @classmethod + def _serialize(cls, schema, path, checksum=None, attrs=None): + res = CanonicalObject({"uri": "{}://{}".format(schema, path)}) + if checksum: + res["checksum"] = checksum + if attrs: + res.update(attrs) + return res + + @classmethod + def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None): + attrs = {} + if diff_tool: + attrs["diff_tool"] = diff_tool + if local: + attrs["local"] = local + if diff_file_name: + attrs["diff_file_name"] = diff_file_name + if diff_tool_timeout: + attrs["diff_tool_timeout"] = diff_tool_timeout + if size is not None: + attrs["size"] = size + return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs) + + @classmethod + def serialize_resource(cls, id, checksum=None): + return cls._serialize(ExternalSchema.SandboxResource, id, checksum) + + @classmethod + def serialize_delayed(cls, upload_id, checksum): + return cls._serialize(ExternalSchema.Delayed, upload_id, checksum) + + def get(self, key, default=None): + return self._data.get(key, default) diff --git a/library/python/testing/yatest_lib/tools.py b/library/python/testing/yatest_lib/tools.py index 602eaeefcc..b72d79c162 100644 --- a/library/python/testing/yatest_lib/tools.py +++ b/library/python/testing/yatest_lib/tools.py @@ -1,13 +1,13 @@ import six import sys - -def to_utf8(value): - """ - Converts value to string encoded into utf-8 - :param value: - :return: - """ + +def to_utf8(value): + """ + Converts value to string encoded into utf-8 + :param value: + :return: + """ if sys.version_info[0] < 3: if not isinstance(value, basestring): # noqa value = unicode(value) # noqa diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make index d115846230..342bae82ba 100644 --- a/library/python/testing/yatest_lib/ya.make +++ b/library/python/testing/yatest_lib/ya.make @@ -1,16 +1,16 @@ OWNER(g:yatest) - + PY23_LIBRARY() - -PY_SRCS( + +PY_SRCS( NAMESPACE yatest_lib - external.py + external.py test_splitter.py - tools.py + tools.py ya.py -) - +) + PEERDIR( contrib/python/six ) @@ -21,6 +21,6 @@ IF(PYTHON2) ) ENDIF() -END() +END() RECURSE_FOR_TESTS(tests) |