diff options
author | Aleksandr <ivansduck@gmail.com> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:52 +0300 |
commit | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (patch) | |
tree | d16cef493ac1e092b4a03ab9437ec06ffe3d188f /library/python/testing | |
parent | 37de222addabbef336dcaaea5f7c7645a629fc6d (diff) | |
download | ydb-ea6c5b7f172becca389cacaff7d5f45f6adccbe6.tar.gz |
Restoring authorship annotation for Aleksandr <ivansduck@gmail.com>. Commit 1 of 2.
Diffstat (limited to 'library/python/testing')
22 files changed, 800 insertions, 800 deletions
diff --git a/library/python/testing/filter/ya.make b/library/python/testing/filter/ya.make index 22c485d258..7944b713a5 100644 --- a/library/python/testing/filter/ya.make +++ b/library/python/testing/filter/ya.make @@ -1,5 +1,5 @@ PY23_LIBRARY() -OWNER(g:yatest) +OWNER(g:yatest) PY_SRCS(filter.py) END() diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py index 3e3b7234ef..440690af59 100644 --- a/library/python/testing/import_test/import_test.py +++ b/library/python/testing/import_test/import_test.py @@ -3,8 +3,8 @@ from __future__ import print_function import os import re import sys -import time -import traceback +import time +import traceback import __res from __res import importer @@ -32,7 +32,7 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): rx = re.compile('^({})$'.format('|'.join(patterns))) failed = [] - import_times = {} + import_times = {} norm = lambda s: s[:-9] if s.endswith('.__init__') else s @@ -59,10 +59,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): try: print('TRY', module) - # XXX waiting for py3 to use print(..., flush=True) - sys.stdout.flush() - - s = time.time() + # XXX waiting for py3 to use print(..., flush=True) + sys.stdout.flush() + + s = time.time() if module == '__main__': importer.load_module('__main__', '__main__py') elif module.endswith('.__init__'): @@ -70,10 +70,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): else: __import__(module) - delay = time.time() - s - import_times[str(module)] = delay - print('OK ', module, '{:.3f}s'.format(delay)) - + delay = time.time() - s + import_times[str(module)] = delay + print('OK ', module, '{:.3f}s'.format(delay)) + except Exception as e: print('FAIL:', module, e, file=sys.stderr) print_backtrace_marked(sys.exc_info()) @@ -86,10 +86,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): failed.append('{}: {}'.format(module, e)) raise - print("Slowest imports:") - for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]: - print(' ', '{:.3f}s'.format(t), m) - + print("Slowest imports:") + for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]: + print(' ', '{:.3f}s'.format(t), m) + if failed: raise ImportError('modules not imported:\n' + '\n'.join(failed)) diff --git a/library/python/testing/import_test/ya.make b/library/python/testing/import_test/ya.make index fae36ffe8f..b996336159 100644 --- a/library/python/testing/import_test/ya.make +++ b/library/python/testing/import_test/ya.make @@ -1,5 +1,5 @@ OWNER( - g:yatest + g:yatest exprmntr ) diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py index 5ef9c5c189..3ea95ebb1d 100644 --- a/library/python/testing/recipe/__init__.py +++ b/library/python/testing/recipe/__init__.py @@ -16,7 +16,7 @@ collect_cores = None sanitizer_extra_checks = None -def _setup_logging(level=logging.DEBUG): +def _setup_logging(level=logging.DEBUG): root_logger = logging.getLogger() root_logger.setLevel(level) @@ -41,7 +41,7 @@ def get_options(): args, opts = parser.parse_known_args() global ya, sanitizer_extra_checks, collect_cores - _setup_logging() + _setup_logging() context = { "test_stderr": args.test_stderr, @@ -96,7 +96,7 @@ def declare_recipe(start, stop): except Exception: if parsed_args.pdb: tty() - import ipdb - ipdb.post_mortem() + import ipdb + ipdb.post_mortem() else: raise diff --git a/library/python/testing/recipe/ya.make b/library/python/testing/recipe/ya.make index dd323aa245..239223c789 100644 --- a/library/python/testing/recipe/ya.make +++ b/library/python/testing/recipe/ya.make @@ -1,6 +1,6 @@ OWNER( exprmntr - g:yatest + g:yatest ) PY23_LIBRARY() @@ -10,10 +10,10 @@ PY_SRCS( ports.py ) -PEERDIR( - contrib/python/ipdb +PEERDIR( + contrib/python/ipdb library/python/testing/yatest_common library/python/testing/yatest_lib -) - +) + END() diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make index 883bc8d7ab..c232bc663e 100644 --- a/library/python/testing/ya.make +++ b/library/python/testing/ya.make @@ -1,8 +1,8 @@ -OWNER(g:yatest) +OWNER(g:yatest) RECURSE( behave - deprecated + deprecated fake_ya_package filter gtest @@ -15,8 +15,8 @@ RECURSE( recipe system_info types_test - yapackage - yapackage/test + yapackage + yapackage/test yatest_common yatest_lib ) diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make index 5662db4c5d..0098e1be85 100644 --- a/library/python/testing/yatest_common/ya.make +++ b/library/python/testing/yatest_common/ya.make @@ -1,31 +1,31 @@ -OWNER(g:yatest) +OWNER(g:yatest) PY23_LIBRARY() -OWNER(g:yatest) +OWNER(g:yatest) NO_EXTENDED_SOURCE_SEARCH() PY_SRCS( TOP_LEVEL - yatest/__init__.py + yatest/__init__.py yatest/common/__init__.py yatest/common/benchmark.py yatest/common/canonical.py yatest/common/environment.py yatest/common/errors.py yatest/common/legacy.py - yatest/common/misc.py + yatest/common/misc.py yatest/common/network.py yatest/common/path.py yatest/common/process.py yatest/common/runtime.py yatest/common/runtime_java.py yatest/common/tags.py -) +) -PEERDIR( - contrib/python/six +PEERDIR( + contrib/python/six library/python/cores library/python/filelock library/python/fs diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py index b6a136d3e9..7a258d070b 100644 --- a/library/python/testing/yatest_common/yatest/common/canonical.py +++ b/library/python/testing/yatest_common/yatest/common/canonical.py @@ -68,7 +68,7 @@ def canonical_execute( ): """ Shortcut to execute a binary and canonize its stdout - :param binary: absolute path to the binary + :param binary: absolute path to the binary :param args: binary arguments :param check_exit_code: will raise ExecutionError if the command exits with non zero code :param shell: use shell to run the command @@ -78,7 +78,7 @@ def canonical_execute( :param stdin: command stdin :param stderr: command stderr :param creationflags: command creation flags - :param file_name: output file name. if not specified program name will be used + :param file_name: output file name. if not specified program name will be used :param diff_tool: path to custome diff tool :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool @@ -94,7 +94,7 @@ def canonical_execute( execute_args = locals() del execute_args["binary"] del execute_args["args"] - del execute_args["file_name"] + del execute_args["file_name"] del execute_args["save_locally"] del execute_args["diff_tool"] del execute_args["diff_file_name"] @@ -123,7 +123,7 @@ def canonical_py_execute( :param stdin: command stdin :param stderr: command stderr :param creationflags: command creation flags - :param file_name: output file name. if not specified program name will be used + :param file_name: output file name. if not specified program name will be used :param diff_tool: path to custome diff tool :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool @@ -135,7 +135,7 @@ def canonical_py_execute( execute_args = locals() del execute_args["script_path"] del execute_args["args"] - del execute_args["file_name"] + del execute_args["file_name"] del execute_args["save_locally"] del execute_args["diff_tool"] del execute_args["diff_file_name"] diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py index 43f48d0958..62fc501d52 100644 --- a/library/python/testing/yatest_common/yatest/common/environment.py +++ b/library/python/testing/yatest_common/yatest/common/environment.py @@ -1,5 +1,5 @@ -# coding: utf-8 +# coding: utf-8 + - -def extend_env_var(env, name, value, sep=":"): - return sep.join(filter(None, [env.get(name), value])) +def extend_env_var(env, name, value, sep=":"): + return sep.join(filter(None, [env.get(name), value])) diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py index 8c038fc381..8d5abc9b96 100644 --- a/library/python/testing/yatest_common/yatest/common/errors.py +++ b/library/python/testing/yatest_common/yatest/common/errors.py @@ -1,4 +1,4 @@ -import os +import os import sys @@ -8,7 +8,7 @@ class RestartTestException(Exception): super(RestartTestException, self).__init__(*args, **kwargs) sys.stderr.write("##restart-test##\n") sys.stderr.flush() - os.environ["FORCE_EXIT_TESTSFAILED"] = "1" + os.environ["FORCE_EXIT_TESTSFAILED"] = "1" class InfrastructureException(Exception): @@ -17,4 +17,4 @@ class InfrastructureException(Exception): super(InfrastructureException, self).__init__(*args, **kwargs) sys.stderr.write("##infrastructure-error##\n") sys.stderr.flush() - os.environ["FORCE_EXIT_TESTSFAILED"] = "1" + os.environ["FORCE_EXIT_TESTSFAILED"] = "1" diff --git a/library/python/testing/yatest_common/yatest/common/misc.py b/library/python/testing/yatest_common/yatest/common/misc.py index 20d3725ac9..3c8d239a40 100644 --- a/library/python/testing/yatest_common/yatest/common/misc.py +++ b/library/python/testing/yatest_common/yatest/common/misc.py @@ -1,19 +1,19 @@ -import functools - - -def first(it): - for d in it: - if d: - return d - - -def lazy(func): - res = [] - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if not res: - res.append(func(*args, **kwargs)) - return res[0] - - return wrapper +import functools + + +def first(it): + for d in it: + if d: + return d + + +def lazy(func): + res = [] + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not res: + res.append(func(*args, **kwargs)) + return res[0] + + return wrapper diff --git a/library/python/testing/yatest_common/yatest/common/network.py b/library/python/testing/yatest_common/yatest/common/network.py index 37bcb1b8e0..e3e3b0abc2 100644 --- a/library/python/testing/yatest_common/yatest/common/network.py +++ b/library/python/testing/yatest_common/yatest/common/network.py @@ -1,47 +1,47 @@ # coding=utf-8 - -import os -import errno -import socket -import random -import logging -import platform -import threading - + +import os +import errno +import socket +import random +import logging +import platform +import threading + import six -UI16MAXVAL = (1 << 16) - 1 -logger = logging.getLogger(__name__) - - -class PortManagerException(Exception): - pass - - -class PortManager(object): +UI16MAXVAL = (1 << 16) - 1 +logger = logging.getLogger(__name__) + + +class PortManagerException(Exception): + pass + + +class PortManager(object): """ See documentation here - + https://wiki.yandex-team.ru/yatool/test/#python-acquire-ports """ - def __init__(self, sync_dir=None): - self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH') - if self._sync_dir: - _makedirs(self._sync_dir) - - self._valid_range = get_valid_port_range() - self._valid_port_count = self._count_valid_ports() - self._filelocks = {} - self._lock = threading.Lock() - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.release() - - def get_port(self, port=0): + def __init__(self, sync_dir=None): + self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH') + if self._sync_dir: + _makedirs(self._sync_dir) + + self._valid_range = get_valid_port_range() + self._valid_port_count = self._count_valid_ports() + self._filelocks = {} + self._lock = threading.Lock() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.release() + + def get_port(self, port=0): ''' Gets free TCP port ''' @@ -64,8 +64,8 @@ class PortManager(object): Gets one free port for use in both TCP and UDP protocols ''' if port and self._no_random_ports(): - return port - + return port + retries = 20 while retries > 0: retries -= 1 @@ -73,199 +73,199 @@ class PortManager(object): result_port = self.get_tcp_port() if not self.is_port_free(result_port, socket.SOCK_DGRAM): self.release_port(result_port) - # Don't try to _capture_port(), it's already captured in the get_tcp_port() + # Don't try to _capture_port(), it's already captured in the get_tcp_port() return result_port raise Exception('Failed to find port') def release_port(self, port): - with self._lock: - self._release_port_no_lock(port) - - def _release_port_no_lock(self, port): - filelock = self._filelocks.pop(port, None) - if filelock: - filelock.release() - + with self._lock: + self._release_port_no_lock(port) + + def _release_port_no_lock(self, port): + filelock = self._filelocks.pop(port, None) + if filelock: + filelock.release() + def release(self): - with self._lock: - while self._filelocks: - _, filelock = self._filelocks.popitem() - if filelock: - filelock.release() - - def get_port_range(self, start_port, count, random_start=True): - assert count > 0 - if start_port and self._no_random_ports(): - return start_port - - candidates = [] - - def drop_candidates(): - for port in candidates: - self._release_port_no_lock(port) - candidates[:] = [] - - with self._lock: - for attempts in six.moves.range(128): - for left, right in self._valid_range: - if right - left < count: - continue - - if random_start: - start = random.randint(left, right - ((right - left) // 2)) - else: - start = left - for probe_port in six.moves.range(start, right): - if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM): - candidates.append(probe_port) - else: - drop_candidates() - - if len(candidates) == count: - return candidates[0] - # Can't find required number of ports without gap in the current range - drop_candidates() - - raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format( - start_port, count, self._valid_range, self._filelocks)) - - def _count_valid_ports(self): - res = 0 - for left, right in self._valid_range: - res += right - left - assert res, ('There are no available valid ports', self._valid_range) - return res - + with self._lock: + while self._filelocks: + _, filelock = self._filelocks.popitem() + if filelock: + filelock.release() + + def get_port_range(self, start_port, count, random_start=True): + assert count > 0 + if start_port and self._no_random_ports(): + return start_port + + candidates = [] + + def drop_candidates(): + for port in candidates: + self._release_port_no_lock(port) + candidates[:] = [] + + with self._lock: + for attempts in six.moves.range(128): + for left, right in self._valid_range: + if right - left < count: + continue + + if random_start: + start = random.randint(left, right - ((right - left) // 2)) + else: + start = left + for probe_port in six.moves.range(start, right): + if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM): + candidates.append(probe_port) + else: + drop_candidates() + + if len(candidates) == count: + return candidates[0] + # Can't find required number of ports without gap in the current range + drop_candidates() + + raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format( + start_port, count, self._valid_range, self._filelocks)) + + def _count_valid_ports(self): + res = 0 + for left, right in self._valid_range: + res += right - left + assert res, ('There are no available valid ports', self._valid_range) + return res + def _get_port(self, port, sock_type): if port and self._no_random_ports(): return port - if len(self._filelocks) >= self._valid_port_count: - raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks)) - - salt = random.randint(0, UI16MAXVAL) + if len(self._filelocks) >= self._valid_port_count: + raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks)) + + salt = random.randint(0, UI16MAXVAL) for attempt in six.moves.range(self._valid_port_count): - probe_port = (salt + attempt) % self._valid_port_count - - for left, right in self._valid_range: - if probe_port >= (right - left): - probe_port -= right - left - else: - probe_port += left - break - if not self._capture_port(probe_port, sock_type): - continue - return probe_port - - raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks)) - - def _capture_port(self, port, sock_type): - with self._lock: - return self._capture_port_no_lock(port, sock_type) - + probe_port = (salt + attempt) % self._valid_port_count + + for left, right in self._valid_range: + if probe_port >= (right - left): + probe_port -= right - left + else: + probe_port += left + break + if not self._capture_port(probe_port, sock_type): + continue + return probe_port + + raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks)) + + def _capture_port(self, port, sock_type): + with self._lock: + return self._capture_port_no_lock(port, sock_type) + def is_port_free(self, port, sock_type=socket.SOCK_STREAM): - sock = socket.socket(socket.AF_INET6, sock_type) - try: - sock.bind(('::', port)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except socket.error as e: - if e.errno == errno.EADDRINUSE: - return False - raise - finally: - sock.close() - return True - - def _capture_port_no_lock(self, port, sock_type): - if port in self._filelocks: - return False - - filelock = None - if self._sync_dir: - # yatest.common should try to be hermetic and don't have peerdirs - # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work - import library.python.filelock - - filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port))) - if not filelock.acquire(blocking=False): - return False + sock = socket.socket(socket.AF_INET6, sock_type) + try: + sock.bind(('::', port)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + return False + raise + finally: + sock.close() + return True + + def _capture_port_no_lock(self, port, sock_type): + if port in self._filelocks: + return False + + filelock = None + if self._sync_dir: + # yatest.common should try to be hermetic and don't have peerdirs + # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work + import library.python.filelock + + filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port))) + if not filelock.acquire(blocking=False): + return False if self.is_port_free(port, sock_type): - self._filelocks[port] = filelock - return True - else: - filelock.release() - return False - + self._filelocks[port] = filelock + return True + else: + filelock.release() + return False + if self.is_port_free(port, sock_type): - self._filelocks[port] = filelock - return True - if filelock: - filelock.release() - return False - + self._filelocks[port] = filelock + return True + if filelock: + filelock.release() + return False + def _no_random_ports(self): return os.environ.get("NO_RANDOM_PORTS") - - -def get_valid_port_range(): - first_valid = 1025 - last_valid = UI16MAXVAL - - given_range = os.environ.get('VALID_PORT_RANGE') - if given_range and ':' in given_range: - return [list(int(x) for x in given_range.split(':', 2))] - - first_eph, last_eph = get_ephemeral_range() - first_invalid = max(first_eph, first_valid) - last_invalid = min(last_eph, last_valid) - - ranges = [] - if first_invalid > first_valid: - ranges.append((first_valid, first_invalid - 1)) - if last_invalid < last_valid: - ranges.append((last_invalid + 1, last_valid)) - return ranges - - -def get_ephemeral_range(): - if platform.system() == 'Linux': - filename = "/proc/sys/net/ipv4/ip_local_port_range" - if os.path.exists(filename): - with open(filename) as afile: + + +def get_valid_port_range(): + first_valid = 1025 + last_valid = UI16MAXVAL + + given_range = os.environ.get('VALID_PORT_RANGE') + if given_range and ':' in given_range: + return [list(int(x) for x in given_range.split(':', 2))] + + first_eph, last_eph = get_ephemeral_range() + first_invalid = max(first_eph, first_valid) + last_invalid = min(last_eph, last_valid) + + ranges = [] + if first_invalid > first_valid: + ranges.append((first_valid, first_invalid - 1)) + if last_invalid < last_valid: + ranges.append((last_invalid + 1, last_valid)) + return ranges + + +def get_ephemeral_range(): + if platform.system() == 'Linux': + filename = "/proc/sys/net/ipv4/ip_local_port_range" + if os.path.exists(filename): + with open(filename) as afile: data = afile.read(1024) # fix for musl - port_range = tuple(map(int, data.strip().split())) - if len(port_range) == 2: - return port_range - else: - logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data) - elif platform.system() == 'Darwin': - first = _sysctlbyname_uint("net.inet.ip.portrange.first") - last = _sysctlbyname_uint("net.inet.ip.portrange.last") - if first and last: - return first, last - # IANA suggestion - return (1 << 15) + (1 << 14), UI16MAXVAL - - -def _sysctlbyname_uint(name): - try: - from ctypes import CDLL, c_uint, byref - from ctypes.util import find_library - except ImportError: - return - - libc = CDLL(find_library("c")) - size = c_uint(0) - res = c_uint(0) - libc.sysctlbyname(name, None, byref(size), None, 0) - libc.sysctlbyname(name, byref(res), byref(size), None, 0) - return res.value - - -def _makedirs(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST: - return - raise + port_range = tuple(map(int, data.strip().split())) + if len(port_range) == 2: + return port_range + else: + logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data) + elif platform.system() == 'Darwin': + first = _sysctlbyname_uint("net.inet.ip.portrange.first") + last = _sysctlbyname_uint("net.inet.ip.portrange.last") + if first and last: + return first, last + # IANA suggestion + return (1 << 15) + (1 << 14), UI16MAXVAL + + +def _sysctlbyname_uint(name): + try: + from ctypes import CDLL, c_uint, byref + from ctypes.util import find_library + except ImportError: + return + + libc = CDLL(find_library("c")) + size = c_uint(0) + res = c_uint(0) + libc.sysctlbyname(name, None, byref(size), None, 0) + libc.sysctlbyname(name, byref(res), byref(size), None, 0) + return res.value + + +def _makedirs(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST: + return + raise diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py index 6fed7dda8a..d4f52f6333 100644 --- a/library/python/testing/yatest_common/yatest/common/path.py +++ b/library/python/testing/yatest_common/yatest/common/path.py @@ -1,12 +1,12 @@ # coding=utf-8 import errno -import os -import shutil -import contextlib - +import os +import shutil +import contextlib + import library.python.fs as lpf - + def replace_in_file(path, old, new): """ @@ -21,32 +21,32 @@ def replace_in_file(path, old, new): lpf.ensure_removed(path) with open(path, 'w') as fp: fp.write(content.replace(old, new)) - - -@contextlib.contextmanager -def change_dir(path): - old = os.getcwd() - try: - os.chdir(path) - yield path - finally: - os.chdir(old) - - + + +@contextlib.contextmanager +def change_dir(path): + old = os.getcwd() + try: + os.chdir(path) + yield path + finally: + os.chdir(old) + + def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): - ''' - Copy an entire directory of files into an existing directory - instead of raising Exception what shtuil.copytree does - ''' + ''' + Copy an entire directory of files into an existing directory + instead of raising Exception what shtuil.copytree does + ''' if not os.path.exists(dst) and os.path.isdir(src): os.makedirs(dst) - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - shutil.copytree(s, d, symlinks, ignore) - else: - shutil.copy2(s, d) + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) if postprocessing: postprocessing(dst, False) for root, dirs, files in os.walk(dst): diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py index a8bcc21f51..f9eeb16525 100644 --- a/library/python/testing/yatest_common/yatest/common/process.py +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -1,10 +1,10 @@ -# coding: utf-8 - +# coding: utf-8 + import os -import re +import re import time import signal -import shutil +import shutil import logging import tempfile import subprocess @@ -13,19 +13,19 @@ import distutils.version import six -try: - # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work. - import library.python.cores as cores -except ImportError: - cores = None - +try: + # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work. + import library.python.cores as cores +except ImportError: + cores = None + from . import runtime from . import path from . import environment MAX_OUT_LEN = 1000 * 1000 # 1 mb -MAX_MESSAGE_LEN = 1500 +MAX_MESSAGE_LEN = 1500 SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)" GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)") yatest_logger = logging.getLogger("ya.test") @@ -55,12 +55,12 @@ class ExecutionError(Exception): command=command, code=execution_result.exit_code, err=_format_error(execution_result.std_err)) - if cores: - if execution_result.backtrace: - message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) - else: - message += "Backtrace is not available: module cores isn't available" - + if cores: + if execution_result.backtrace: + message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) + else: + message += "Backtrace is not available: module cores isn't available" + super(ExecutionError, self).__init__(message) self.execution_result = execution_result @@ -85,31 +85,31 @@ class SignalInterruptionError(Exception): self.res = None -class InvalidCommandError(Exception): - pass - - +class InvalidCommandError(Exception): + pass + + class _Execution(object): - def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): + def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): self._command = command self._process = process self._out_file = out_file self._err_file = err_file self._std_out = None self._std_err = None - self._elapsed = None - self._start = time.time() + self._elapsed = None + self._start = time.time() self._process_progress_listener = process_progress_listener - self._cwd = cwd or os.getcwd() - self._collect_cores = collect_cores - self._backtrace = '' - self._check_sanitizer = check_sanitizer + self._cwd = cwd or os.getcwd() + self._collect_cores = collect_cores + self._backtrace = '' + self._check_sanitizer = check_sanitizer self._metrics = {} - self._started = started - self._user_stdout = bool(user_stdout) - self._user_stderr = bool(user_stderr) - self._exit_code = None + self._started = started + self._user_stdout = bool(user_stdout) + self._user_stderr = bool(user_stderr) + self._exit_code = None if process_progress_listener: process_progress_listener.open(command, process, out_file, err_file) @@ -122,21 +122,21 @@ class _Execution(object): self._save_outputs(False) _kill_process_tree(self._process.pid) self._clean_files() - # DEVTOOLS-2347 - yatest_logger.debug("Process status before wait_for: %s", self.running) - try: - wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1) - except TimeoutError: - yatest_logger.debug("Process status after wait_for: %s", self.running) - yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) - raise + # DEVTOOLS-2347 + yatest_logger.debug("Process status before wait_for: %s", self.running) + try: + wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1) + except TimeoutError: + yatest_logger.debug("Process status after wait_for: %s", self.running) + yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) + raise else: raise InvalidExecutionStateError("Cannot kill a stopped process") - def terminate(self): - if self.running: - self._process.terminate() - + def terminate(self): + if self.running: + self._process.terminate() + @property def process(self): return self._process @@ -146,56 +146,56 @@ class _Execution(object): return self._command @property - def returncode(self): - return self.exit_code - - @property + def returncode(self): + return self.exit_code + + @property def exit_code(self): - """ - Deprecated, use returncode - """ - if self._exit_code is None: - self._exit_code = self._process.returncode - return self._exit_code - - @property - def stdout(self): - return self.std_out + """ + Deprecated, use returncode + """ + if self._exit_code is None: + self._exit_code = self._process.returncode + return self._exit_code @property + def stdout(self): + return self.std_out + + @property def std_out(self): - """ - Deprecated, use stdout - """ + """ + Deprecated, use stdout + """ if self._std_out is not None: return self._std_out - if self._process.stdout and not self._user_stdout: - self._std_out = self._process.stdout.read() - return self._std_out - - @property - def stderr(self): - return self.std_err + if self._process.stdout and not self._user_stdout: + self._std_out = self._process.stdout.read() + return self._std_out @property + def stderr(self): + return self.std_err + + @property def std_err(self): - """ - Deprecated, use stderr - """ + """ + Deprecated, use stderr + """ if self._std_err is not None: return self._std_err - if self._process.stderr and not self._user_stderr: - self._std_err = self._process.stderr.read() - return self._std_err - - @property - def elapsed(self): - return self._elapsed - - @property - def backtrace(self): - return self._backtrace - + if self._process.stderr and not self._user_stderr: + self._std_err = self._process.stderr.read() + return self._std_err + + @property + def elapsed(self): + return self._elapsed + + @property + def backtrace(self): + return self._backtrace + @property def metrics(self): return self._metrics @@ -204,25 +204,25 @@ class _Execution(object): if self._process_progress_listener: self._process_progress_listener() self._process_progress_listener.close() - if not self._user_stdout: - if self._out_file is None: - pass - elif self._out_file != subprocess.PIPE: - self._out_file.flush() - self._out_file.seek(0, os.SEEK_SET) - self._std_out = self._out_file.read() - else: - self._std_out = self._process.stdout.read() - if not self._user_stderr: - if self._err_file is None: - pass - elif self._err_file != subprocess.PIPE: - self._err_file.flush() - self._err_file.seek(0, os.SEEK_SET) - self._std_err = self._err_file.read() - else: - self._std_err = self._process.stderr.read() - + if not self._user_stdout: + if self._out_file is None: + pass + elif self._out_file != subprocess.PIPE: + self._out_file.flush() + self._out_file.seek(0, os.SEEK_SET) + self._std_out = self._out_file.read() + else: + self._std_out = self._process.stdout.read() + if not self._user_stderr: + if self._err_file is None: + pass + elif self._err_file != subprocess.PIPE: + self._err_file.flush() + self._err_file.seek(0, os.SEEK_SET) + self._std_err = self._err_file.read() + else: + self._std_err = self._process.stderr.read() + if clean_files: self._clean_files() yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) @@ -245,45 +245,45 @@ class _Execution(object): yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN)) def _clean_files(self): - if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: - self._err_file.close() - self._err_file = None - if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: - self._out_file.close() - self._out_file = None - - def _recover_core(self): - core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) - if core_path: - # Core dump file recovering may be disabled (for distbuild for example) - produce only bt - store_cores = runtime._get_ya_config().collect_cores - if store_cores: - new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) - # Copy core dump file, because it may be overwritten + if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: + self._err_file.close() + self._err_file = None + if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: + self._out_file.close() + self._out_file = None + + def _recover_core(self): + core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) + if core_path: + # Core dump file recovering may be disabled (for distbuild for example) - produce only bt + store_cores = runtime._get_ya_config().collect_cores + if store_cores: + new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) + # Copy core dump file, because it may be overwritten yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) - shutil.copyfile(core_path, new_core_path) - core_path = new_core_path - - bt_filename = None - pbt_filename = None - - if os.path.exists(runtime.gdb_path()): - self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path()) - bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid)) + shutil.copyfile(core_path, new_core_path) + core_path = new_core_path + + bt_filename = None + pbt_filename = None + + if os.path.exists(runtime.gdb_path()): + self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path()) + bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid)) with open(bt_filename, "wb") as afile: afile.write(six.ensure_binary(self._backtrace)) - # generate pretty html version of backtrace aka Tri Korochki - pbt_filename = bt_filename + ".html" + # generate pretty html version of backtrace aka Tri Korochki + pbt_filename = bt_filename + ".html" backtrace_to_html(bt_filename, pbt_filename) - - if store_cores: - runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename) - else: - runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) - + + if store_cores: + runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename) + else: + runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) + def wait(self, check_exit_code=True, timeout=None, on_timeout=None): def _wait(): - finished = None + finished = None interrupted = False try: if hasattr(os, "wait4"): @@ -293,7 +293,7 @@ class _Execution(object): else: # PEP 475 pid, sts, rusage = os.wait4(self._process.pid, 0) - finished = time.time() + finished = time.time() self._process._handle_exitstatus(sts) for field in [ "ru_idrss", @@ -328,46 +328,46 @@ class _Execution(object): if not interrupted: self._process.wait() # this has to be here unconditionally, so that all process properties are set - if not finished: - finished = time.time() - self._metrics["wtime"] = round(finished - self._started, 3) - + if not finished: + finished = time.time() + self._metrics["wtime"] = round(finished - self._started, 3) + try: - if timeout: - process_is_finished = lambda: not self.running - fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) + if timeout: + process_is_finished = lambda: not self.running + fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) try: wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) except TimeoutError as e: if on_timeout: - yatest_logger.debug("Calling user specified on_timeout function") + yatest_logger.debug("Calling user specified on_timeout function") try: on_timeout(self, timeout) except Exception: yatest_logger.exception("Exception while calling on_timeout") raise ExecutionTimeoutError(self, str(e)) - # Wait should be always called here, it finalizes internal states of its process and sets up return code + # Wait should be always called here, it finalizes internal states of its process and sets up return code _wait() - except BaseException as e: + except BaseException as e: _kill_process_tree(self._process.pid) _wait() yatest_logger.debug("Command exception: %s", e) - raise + raise finally: - self._elapsed = time.time() - self._start - self._save_outputs() + self._elapsed = time.time() - self._start + self._save_outputs() self.verify_no_coredumps() - self._finalise(check_exit_code) - - def _finalise(self, check_exit_code): + self._finalise(check_exit_code) + + def _finalise(self, check_exit_code): # Set the signal (negative number) which caused the process to exit if check_exit_code and self.exit_code != 0: yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) raise ExecutionError(self) - # Don't search for sanitize errors if stderr was redirected + # Don't search for sanitize errors if stderr was redirected self.verify_sanitize_errors() def verify_no_coredumps(self): @@ -375,44 +375,44 @@ class _Execution(object): Verify there is no coredump from this binary. If there is then report backtrace. """ if self.exit_code < 0 and self._collect_cores: - if cores: - try: - self._recover_core() - except Exception: - yatest_logger.exception("Exception while recovering core") - else: - yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available") + if cores: + try: + self._recover_core() + except Exception: + yatest_logger.exception("Exception while recovering core") + else: + yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available") def verify_sanitize_errors(self): """ Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them. """ - if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks: - build_path = runtime.build_path() - if self.command[0].startswith(build_path): - match = re.search(SANITIZER_ERROR_PATTERN, self._std_err) - if match: - yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) - raise ExecutionError(self) + if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks: + build_path = runtime.build_path() + if self.command[0].startswith(build_path): + match = re.search(SANITIZER_ERROR_PATTERN, self._std_err) + if match: + yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) else: yatest_logger.debug("No sanitizer errors found") - else: + else: yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) - -def on_timeout_gen_coredump(exec_obj, _): - """ - Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump) - to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout. - All files will be available in the testing_out_stuff and via links. - """ - try: - os.kill(exec_obj.process.pid, signal.SIGQUIT) - except OSError: - # process might be already terminated - pass - - + +def on_timeout_gen_coredump(exec_obj, _): + """ + Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump) + to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout. + All files will be available in the testing_out_stuff and via links. + """ + try: + os.kill(exec_obj.process.pid, signal.SIGQUIT) + except OSError: + # process might be already terminated + pass + + def execute( command, check_exit_code=True, shell=False, timeout=None, @@ -421,7 +421,7 @@ def execute( creationflags=0, wait=True, process_progress_listener=None, close_fds=False, collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, - executor=_Execution, + executor=_Execution, ): """ Executes a command @@ -438,26 +438,26 @@ def execute( :param wait: should wait until the command finishes :param process_progress_listener=object that is polled while execution is in progress :param close_fds: subrpocess.Popen close_fds args - :param collect_cores: recover core dump files if shell == False - :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors + :param collect_cores: recover core dump files if shell == False + :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors :param preexec_fn: subrpocess.Popen preexec_fn arg :param on_timeout: on_timeout(<execution object>, <timeout value>) callback :return _Execution: Execution object """ - if env is None: - env = os.environ.copy() + if env is None: + env = os.environ.copy() else: - # Certain environment variables must be present for programs to work properly. - # For more info see DEVTOOLSSUPPORT-4907 - mandatory_env_name = 'YA_MANDATORY_ENV_VARS' - mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or '' - if mandatory_vars: - env[mandatory_env_name] = mandatory_vars - mandatory_system_vars = filter(None, mandatory_vars.split(':')) - else: - mandatory_system_vars = ['TMPDIR'] - + # Certain environment variables must be present for programs to work properly. + # For more info see DEVTOOLSSUPPORT-4907 + mandatory_env_name = 'YA_MANDATORY_ENV_VARS' + mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or '' + if mandatory_vars: + env[mandatory_env_name] = mandatory_vars + mandatory_system_vars = filter(None, mandatory_vars.split(':')) + else: + mandatory_system_vars = ['TMPDIR'] + for var in mandatory_system_vars: if var not in env and var in os.environ: env[var] = os.environ[var] @@ -465,13 +465,13 @@ def execute( if not wait and timeout is not None: raise ValueError("Incompatible arguments 'timeout' and wait=False") - # if subprocess.PIPE in [stdout, stderr]: - # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") - + # if subprocess.PIPE in [stdout, stderr]: + # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") + def get_out_stream(stream, default_name): if stream is None: # No stream is supplied: open new temp file - return _get_command_output_file(command, default_name), False + return _get_command_output_file(command, default_name), False if isinstance(stream, six.string_types): # User filename is supplied: open file for writing @@ -490,86 +490,86 @@ def execute( if shell and type(command) == list: command = " ".join(command) - if shell: - collect_cores = False - check_sanitizer = False - else: - if isinstance(command, (list, tuple)): - executable = command[0] - else: - executable = command - if os.path.isabs(executable): - if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"): - exists = os.path.exists(executable) - if exists: - stat = os.stat(executable) - else: - stat = None - raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)) - if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK): - raise InvalidCommandError("Target program is not executable: {}".format(executable)) - - if check_sanitizer: - env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100") - - if stdin: - name = "PIPE" if stdin == subprocess.PIPE else stdin.name - yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) - else: + if shell: + collect_cores = False + check_sanitizer = False + else: + if isinstance(command, (list, tuple)): + executable = command[0] + else: + executable = command + if os.path.isabs(executable): + if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"): + exists = os.path.exists(executable) + if exists: + stat = os.stat(executable) + else: + stat = None + raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)) + if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK): + raise InvalidCommandError("Target program is not executable: {}".format(executable)) + + if check_sanitizer: + env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100") + + if stdin: + name = "PIPE" if stdin == subprocess.PIPE else stdin.name + yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) + else: yatest_logger.debug("Executing '%s' in '%s'", command, cwd) - # XXX - - started = time.time() - process = subprocess.Popen( - command, shell=shell, universal_newlines=True, - stdout=out_file, stderr=err_file, stdin=in_file, - cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn, - ) - yatest_logger.debug("Command pid: %s", process.pid) - - res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) + # XXX + + started = time.time() + process = subprocess.Popen( + command, shell=shell, universal_newlines=True, + stdout=out_file, stderr=err_file, stdin=in_file, + cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn, + ) + yatest_logger.debug("Command pid: %s", process.pid) + + res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) if wait: res.wait(check_exit_code, timeout, on_timeout) return res -def _get_command_output_file(cmd, ext): - parts = [get_command_name(cmd)] - if 'YA_RETRY_INDEX' in os.environ: - parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX'))) - if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1: - parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0'))) - - filename = '.'.join(parts + [ext]) - try: - # if execution is performed from test, save out / err to the test logs dir - import yatest.common +def _get_command_output_file(cmd, ext): + parts = [get_command_name(cmd)] + if 'YA_RETRY_INDEX' in os.environ: + parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX'))) + if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1: + parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0'))) + + filename = '.'.join(parts + [ext]) + try: + # if execution is performed from test, save out / err to the test logs dir + import yatest.common import library.python.pytest.plugins.ya if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None: - raise ImportError("not in test") - filename = path.get_unique_file_path(yatest.common.output_path(), filename) - yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename)) - return open(filename, "wb+") - except ImportError: - return tempfile.NamedTemporaryFile(delete=False, suffix=filename) - - -def _get_proc_tree_info(pids): - if os.name == 'nt': - return 'Not supported' - else: - stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - return stdout - - -def py_execute( - command, check_exit_code=True, - shell=False, timeout=None, - cwd=None, env=None, - stdin=None, stdout=None, stderr=None, - creationflags=0, wait=True, - process_progress_listener=None, close_fds=False -): + raise ImportError("not in test") + filename = path.get_unique_file_path(yatest.common.output_path(), filename) + yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename)) + return open(filename, "wb+") + except ImportError: + return tempfile.NamedTemporaryFile(delete=False, suffix=filename) + + +def _get_proc_tree_info(pids): + if os.name == 'nt': + return 'Not supported' + else: + stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return stdout + + +def py_execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False +): """ Executes a command with the arcadia python :param command: command to pass to python @@ -635,50 +635,50 @@ def _kill_process_tree(process_pid, target_pid_signal=None): if os.name == 'nt': _win_kill_process_tree(process_pid) else: - _nix_kill_process_tree(process_pid, target_pid_signal) - - -def _nix_get_proc_children(pid): - try: - cmd = ["pgrep", "-P", str(pid)] - return [int(p) for p in subprocess.check_output(cmd).split()] - except Exception: - return [] - - -def _get_binname(pid): - try: - return os.path.basename(os.readlink('/proc/{}/exe'.format(pid))) - except Exception as e: - return "error({})".format(e) - - + _nix_kill_process_tree(process_pid, target_pid_signal) + + +def _nix_get_proc_children(pid): + try: + cmd = ["pgrep", "-P", str(pid)] + return [int(p) for p in subprocess.check_output(cmd).split()] + except Exception: + return [] + + +def _get_binname(pid): + try: + return os.path.basename(os.readlink('/proc/{}/exe'.format(pid))) + except Exception as e: + return "error({})".format(e) + + def _nix_kill_process_tree(pid, target_pid_signal=None): """ Kills the process tree. """ - yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) + yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) def try_to_send_signal(pid, sig): try: os.kill(pid, sig) - yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) + yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) except Exception as exc: - yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) + yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. # Get the child process PID list. - child_pids = _nix_get_proc_children(pid) - # Stop the child processes. - for child_pid in child_pids: - try: - # Kill the child recursively. - _kill_process_tree(int(child_pid)) - except Exception as e: - # Skip the error and continue killing. - yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) - continue + child_pids = _nix_get_proc_children(pid) + # Stop the child processes. + for child_pid in child_pids: + try: + # Kill the child recursively. + _kill_process_tree(int(child_pid)) + except Exception as e: + # Skip the error and continue killing. + yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) + continue try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process. @@ -689,16 +689,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None): def _win_kill_process_tree(pid): subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) - - -def _run_readelf(binary_path): + + +def _run_readelf(binary_path): return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)])) - - -def check_glibc_version(binary_path): + + +def check_glibc_version(binary_path): lucid_glibc_version = distutils.version.LooseVersion("2.11") - for l in _run_readelf(binary_path).split('\n'): + for l in _run_readelf(binary_path).split('\n'): match = GLIBC_PATTERN.search(l) if not match: continue @@ -706,14 +706,14 @@ def check_glibc_version(binary_path): def backtrace_to_html(bt_filename, output): - try: + try: from library.python import coredump_filter - with open(output, "wb") as afile: + with open(output, "wb") as afile: coredump_filter.filter_stackdump(bt_filename, stream=afile) - except ImportError as e: - yatest_logger.debug("Failed to import coredump_filter: %s", e) - with open(output, "wb") as afile: - afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>") + except ImportError as e: + yatest_logger.debug("Failed to import coredump_filter: %s", e) + with open(output, "wb") as afile: + afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>") def _try_convert_bytes_to_string(source): diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py index e55e193446..07a41ebbbc 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime.py +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -1,7 +1,7 @@ -import errno -import functools +import errno +import functools import json -import os +import os import threading import six @@ -10,13 +10,13 @@ import six _lock = threading.Lock() -def _get_ya_config(): +def _get_ya_config(): try: import library.python.pytest.plugins.ya as ya_plugin if ya_plugin.pytest_config is not None: return ya_plugin.pytest_config import pytest - return pytest.config + return pytest.config except (ImportError, AttributeError): try: import library.python.testing.recipe @@ -27,10 +27,10 @@ def _get_ya_config(): raise NotImplementedError("yatest.common.* is only available from the testing runtime") -def _get_ya_plugin_instance(): - return _get_ya_config().ya - - +def _get_ya_plugin_instance(): + return _get_ya_config().ya + + def _norm_path(path): if path is None: return None @@ -46,35 +46,35 @@ def _join_path(main_path, path): return os.path.join(main_path, _norm_path(path)) -def not_test(func): - """ - Mark any function as not a test for py.test - :param func: - :return: - """ - @functools.wraps(func) - def wrapper(*args, **kwds): - return func(*args, **kwds) - setattr(wrapper, '__test__', False) - return wrapper - - -def source_path(path=None): +def not_test(func): + """ + Mark any function as not a test for py.test + :param func: + :return: + """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return func(*args, **kwds) + setattr(wrapper, '__test__', False) + return wrapper + + +def source_path(path=None): """ Get source path inside arcadia - :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya') + :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya') :return: absolute path to the source folder """ - return _join_path(_get_ya_plugin_instance().source_root, path) + return _join_path(_get_ya_plugin_instance().source_root, path) -def build_path(path=None): +def build_path(path=None): """ Get path inside build directory - :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin') + :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin') :return: absolute path inside build directory """ - return _join_path(_get_ya_plugin_instance().build_root, path) + return _join_path(_get_ya_plugin_instance().build_root, path) def java_path(): @@ -82,7 +82,7 @@ def java_path(): [DEPRECATED] Get path to java :return: absolute path to java """ - from . import runtime_java + from . import runtime_java return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk'))) @@ -90,7 +90,7 @@ def java_home(): """ Get jdk directory path """ - from . import runtime_java + from . import runtime_java jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) if not jdk_dir: raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") @@ -100,60 +100,60 @@ def java_home(): def java_bin(): """ Get path to the java binary - Requires DEPENDS(jdk) + Requires DEPENDS(jdk) """ return os.path.join(java_home(), "bin", "java") -def data_path(path=None): +def data_path(path=None): """ Get path inside arcadia_tests_data directory - :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") + :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") :return: absolute path inside arcadia_tests_data """ - return _join_path(_get_ya_plugin_instance().data_root, path) + return _join_path(_get_ya_plugin_instance().data_root, path) -def output_path(path=None): +def output_path(path=None): """ Get path inside the current test suite output dir. Placing files to this dir guarantees that files will be accessible after the test suite execution. :param path: path relative to the test suite output dir :return: absolute path inside the test suite output dir """ - return _join_path(_get_ya_plugin_instance().output_dir, path) - - -def ram_drive_path(path=None): - """ - :param path: path relative to the ram drive. - :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment. - """ - if 'YA_TEST_RAM_DRIVE_PATH' in os.environ: - return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path) - - -def output_ram_drive_path(path=None): - """ - Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing. - Returns None if no ram drive was provided by environment. - :param path: path relative to the output ram drive directory - """ - if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ: - return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path) - - -def binary_path(path=None): + return _join_path(_get_ya_plugin_instance().output_dir, path) + + +def ram_drive_path(path=None): + """ + :param path: path relative to the ram drive. + :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment. + """ + if 'YA_TEST_RAM_DRIVE_PATH' in os.environ: + return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path) + + +def output_ram_drive_path(path=None): + """ + Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing. + Returns None if no ram drive was provided by environment. + :param path: path relative to the output ram drive directory + """ + if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ: + return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path) + + +def binary_path(path=None): """ Get path to the built binary - :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin') + :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin') :return: absolute path to the binary """ - path = _norm_path(path) - return _get_ya_plugin_instance().get_binary(path) + path = _norm_path(path) + return _get_ya_plugin_instance().get_binary(path) -def work_path(path=None): +def work_path(path=None): """ Get path inside the current test suite working directory. Creating files in the work directory does not guarantee that files will be accessible after the test suite execution @@ -169,12 +169,12 @@ def work_path(path=None): def python_path(): """ - Get path to the arcadia python. - - Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles - are built in a stripped-down form that is needed for building, not running tests. - See comments in the file below to find out which version of python is compatible with tests. - https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc + Get path to the arcadia python. + + Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles + are built in a stripped-down form that is needed for building, not running tests. + See comments in the file below to find out which version of python is compatible with tests. + https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc :return: absolute path to python """ return _get_ya_plugin_instance().python_path @@ -207,18 +207,18 @@ def get_param_dict_copy(): return _get_ya_plugin_instance().get_param_dict_copy() -@not_test -def test_output_path(path=None): +@not_test +def test_output_path(path=None): """ Get dir in the suite output_path for the current test case """ test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0] - try: + try: os.makedirs(test_out_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - return _join_path(test_out_dir, path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return _join_path(test_out_dir, path) def project_path(path=None): @@ -228,13 +228,13 @@ def project_path(path=None): return _join_path(os.path.join(build_path(), context.project_path), path) -def gdb_path(): - """ - Get path to the gdb - """ - return _get_ya_plugin_instance().gdb_path - - +def gdb_path(): + """ + Get path to the gdb + """ + return _get_ya_plugin_instance().gdb_path + + def c_compiler_path(): """ Get path to the gdb @@ -261,27 +261,27 @@ def global_resources(): return {} -def _register_core(name, binary_path, core_path, bt_path, pbt_path): - config = _get_ya_config() +def _register_core(name, binary_path, core_path, bt_path, pbt_path): + config = _get_ya_config() with _lock: - if not hasattr(config, 'test_cores_count'): - config.test_cores_count = 0 + if not hasattr(config, 'test_cores_count'): + config.test_cores_count = 0 config.test_cores_count += 1 count_str = '' if config.test_cores_count == 1 else str(config.test_cores_count) - log_entry = config.test_logs[config.current_item_nodeid] - if binary_path: - log_entry['{} binary{}'.format(name, count_str)] = binary_path - if core_path: - log_entry['{} core{}'.format(name, count_str)] = core_path - if bt_path: - log_entry['{} backtrace{}'.format(name, count_str)] = bt_path - if pbt_path: - log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path - - -@not_test + log_entry = config.test_logs[config.current_item_nodeid] + if binary_path: + log_entry['{} binary{}'.format(name, count_str)] = binary_path + if core_path: + log_entry['{} core{}'.format(name, count_str)] = core_path + if bt_path: + log_entry['{} backtrace{}'.format(name, count_str)] = bt_path + if pbt_path: + log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path + + +@not_test def test_source_path(path=None): return _join_path(os.path.join(source_path(), context.project_path), path) diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py index 39bbb45570..5ed678a457 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime_java.py +++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py @@ -2,11 +2,11 @@ import os import tarfile import contextlib -from . import runtime - -_JAVA_DIR = [] - +from . import runtime +_JAVA_DIR = [] + + def get_java_path(jdk_dir): # deprecated - to be deleted java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe')) @@ -30,17 +30,17 @@ def get_java_path(jdk_dir): def get_build_java_dir(jdk_dir): versions = [8, 10, 11, 12, 13, 14, 15] - if not _JAVA_DIR: - for version in versions: - jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version)) - if os.path.exists(jdk_tar_path): - jdk_dir = runtime.build_path('jdk4test') - with contextlib.closing(tarfile.open(jdk_tar_path)) as tf: - tf.extractall(jdk_dir) - assert os.path.exists(os.path.join(jdk_dir, "bin", "java")) - _JAVA_DIR.append(jdk_dir) - break - else: - _JAVA_DIR.append(None) - - return _JAVA_DIR[0] + if not _JAVA_DIR: + for version in versions: + jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version)) + if os.path.exists(jdk_tar_path): + jdk_dir = runtime.build_path('jdk4test') + with contextlib.closing(tarfile.open(jdk_tar_path)) as tf: + tf.extractall(jdk_dir) + assert os.path.exists(os.path.join(jdk_dir, "bin", "java")) + _JAVA_DIR.append(jdk_dir) + break + else: + _JAVA_DIR.append(None) + + return _JAVA_DIR[0] diff --git a/library/python/testing/yatest_common/yatest/common/ya.make b/library/python/testing/yatest_common/yatest/common/ya.make index f7c50dfe64..fe4ea86d2d 100644 --- a/library/python/testing/yatest_common/yatest/common/ya.make +++ b/library/python/testing/yatest_common/yatest/common/ya.make @@ -1 +1 @@ -OWNER(g:yatest) +OWNER(g:yatest) diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py index 39113230d9..69874dece4 100644 --- a/library/python/testing/yatest_lib/external.py +++ b/library/python/testing/yatest_lib/external.py @@ -1,20 +1,20 @@ from __future__ import absolute_import -import re +import re import sys import copy -import logging +import logging from . import tools from datetime import date, datetime -import enum +import enum import six -logger = logging.getLogger(__name__) -MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' - +logger = logging.getLogger(__name__) +MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' + def apply(func, value, apply_to_keys=False): """ Applies func to every possible member of value @@ -67,8 +67,8 @@ def serialize(value): return val if isinstance(val, six.string_types) or isinstance(val, bytes): return tools.to_utf8(val) - if isinstance(val, enum.Enum): - return str(val) + if isinstance(val, enum.Enum): + return str(val) if isinstance(val, six.integer_types) or type(val) in [float, bool]: return val if is_external(val): @@ -136,19 +136,19 @@ class ExternalDataInfo(object): @property def path(self): - if self.uri.count("://") != 1: - logger.error("Invalid external data uri: '%s'", self.uri) - return self.uri + if self.uri.count("://") != 1: + logger.error("Invalid external data uri: '%s'", self.uri) + return self.uri _, path = self.uri.split("://") return path - def get_mds_key(self): - assert self.is_http - m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri) - if m: - return m.group(1) - raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri)) - + def get_mds_key(self): + assert self.is_http + m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri) + if m: + return m.group(1) + raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri)) + @property def size(self): return self._data.get("size") diff --git a/library/python/testing/yatest_lib/test_splitter.py b/library/python/testing/yatest_lib/test_splitter.py index acbcd4300e..bc7beba568 100644 --- a/library/python/testing/yatest_lib/test_splitter.py +++ b/library/python/testing/yatest_lib/test_splitter.py @@ -1,15 +1,15 @@ -# coding: utf-8 +# coding: utf-8 import collections -def flatten_tests(test_classes): +def flatten_tests(test_classes): """ >>> test_classes = {x: [x] for x in range(5)} - >>> flatten_tests(test_classes) + >>> flatten_tests(test_classes) [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] >>> test_classes = {x: [x + 1, x + 2] for x in range(2)} - >>> flatten_tests(test_classes) + >>> flatten_tests(test_classes) [(0, 1), (0, 2), (1, 2), (1, 3)] """ tests = [] diff --git a/library/python/testing/yatest_lib/tests/test_external.py b/library/python/testing/yatest_lib/tests/test_external.py index 18cb560b17..ea5ebf97a3 100644 --- a/library/python/testing/yatest_lib/tests/test_external.py +++ b/library/python/testing/yatest_lib/tests/test_external.py @@ -1,20 +1,20 @@ -import enum -import pytest - -from yatest_lib import external - - -class MyEnum(enum.Enum): - VAL1 = 1 - VAL2 = 2 - - -@pytest.mark.parametrize("data, expected_val, expected_type", [ - ({}, {}, dict), - (MyEnum.VAL1, "MyEnum.VAL1", str), - ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict), -]) -def test_serialize(data, expected_val, expected_type): - data = external.serialize(data) - assert expected_type == type(data), data - assert expected_val == data +import enum +import pytest + +from yatest_lib import external + + +class MyEnum(enum.Enum): + VAL1 = 1 + VAL2 = 2 + + +@pytest.mark.parametrize("data, expected_val, expected_type", [ + ({}, {}, dict), + (MyEnum.VAL1, "MyEnum.VAL1", str), + ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict), +]) +def test_serialize(data, expected_val, expected_type): + data = external.serialize(data) + assert expected_type == type(data), data + assert expected_val == data diff --git a/library/python/testing/yatest_lib/tests/ya.make b/library/python/testing/yatest_lib/tests/ya.make index 8586c6ef7d..89396b733e 100644 --- a/library/python/testing/yatest_lib/tests/ya.make +++ b/library/python/testing/yatest_lib/tests/ya.make @@ -1,13 +1,13 @@ -OWNER(g:yatest) +OWNER(g:yatest) -PY23_TEST() +PY23_TEST() PEERDIR( library/python/testing/yatest_lib ) TEST_SRCS( - test_external.py + test_external.py test_testsplitter.py ) diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make index 342bae82ba..1b9d7aa8c2 100644 --- a/library/python/testing/yatest_lib/ya.make +++ b/library/python/testing/yatest_lib/ya.make @@ -1,4 +1,4 @@ -OWNER(g:yatest) +OWNER(g:yatest) PY23_LIBRARY() @@ -15,12 +15,12 @@ PEERDIR( contrib/python/six ) -IF(PYTHON2) - PEERDIR( - contrib/python/enum34 - ) -ENDIF() - +IF(PYTHON2) + PEERDIR( + contrib/python/enum34 + ) +ENDIF() + END() RECURSE_FOR_TESTS(tests) |