aboutsummaryrefslogtreecommitdiffstats
path: root/library/python/testing
diff options
context:
space:
mode:
authorAleksandr <ivansduck@gmail.com>2022-02-10 16:47:52 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:47:52 +0300
commitea6c5b7f172becca389cacaff7d5f45f6adccbe6 (patch)
treed16cef493ac1e092b4a03ab9437ec06ffe3d188f /library/python/testing
parent37de222addabbef336dcaaea5f7c7645a629fc6d (diff)
downloadydb-ea6c5b7f172becca389cacaff7d5f45f6adccbe6.tar.gz
Restoring authorship annotation for Aleksandr <ivansduck@gmail.com>. Commit 1 of 2.
Diffstat (limited to 'library/python/testing')
-rw-r--r--library/python/testing/filter/ya.make2
-rw-r--r--library/python/testing/import_test/import_test.py30
-rw-r--r--library/python/testing/import_test/ya.make2
-rw-r--r--library/python/testing/recipe/__init__.py8
-rw-r--r--library/python/testing/recipe/ya.make10
-rw-r--r--library/python/testing/ya.make8
-rw-r--r--library/python/testing/yatest_common/ya.make14
-rw-r--r--library/python/testing/yatest_common/yatest/common/canonical.py10
-rw-r--r--library/python/testing/yatest_common/yatest/common/environment.py8
-rw-r--r--library/python/testing/yatest_common/yatest/common/errors.py6
-rw-r--r--library/python/testing/yatest_common/yatest/common/misc.py38
-rw-r--r--library/python/testing/yatest_common/yatest/common/network.py436
-rw-r--r--library/python/testing/yatest_common/yatest/common/path.py56
-rw-r--r--library/python/testing/yatest_common/yatest/common/process.py634
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime.py196
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime_java.py36
-rw-r--r--library/python/testing/yatest_common/yatest/common/ya.make2
-rw-r--r--library/python/testing/yatest_lib/external.py36
-rw-r--r--library/python/testing/yatest_lib/test_splitter.py8
-rw-r--r--library/python/testing/yatest_lib/tests/test_external.py40
-rw-r--r--library/python/testing/yatest_lib/tests/ya.make6
-rw-r--r--library/python/testing/yatest_lib/ya.make14
22 files changed, 800 insertions, 800 deletions
diff --git a/library/python/testing/filter/ya.make b/library/python/testing/filter/ya.make
index 22c485d258..7944b713a5 100644
--- a/library/python/testing/filter/ya.make
+++ b/library/python/testing/filter/ya.make
@@ -1,5 +1,5 @@
PY23_LIBRARY()
-OWNER(g:yatest)
+OWNER(g:yatest)
PY_SRCS(filter.py)
END()
diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py
index 3e3b7234ef..440690af59 100644
--- a/library/python/testing/import_test/import_test.py
+++ b/library/python/testing/import_test/import_test.py
@@ -3,8 +3,8 @@ from __future__ import print_function
import os
import re
import sys
-import time
-import traceback
+import time
+import traceback
import __res
from __res import importer
@@ -32,7 +32,7 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
- import_times = {}
+ import_times = {}
norm = lambda s: s[:-9] if s.endswith('.__init__') else s
@@ -59,10 +59,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
try:
print('TRY', module)
- # XXX waiting for py3 to use print(..., flush=True)
- sys.stdout.flush()
-
- s = time.time()
+ # XXX waiting for py3 to use print(..., flush=True)
+ sys.stdout.flush()
+
+ s = time.time()
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
@@ -70,10 +70,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
else:
__import__(module)
- delay = time.time() - s
- import_times[str(module)] = delay
- print('OK ', module, '{:.3f}s'.format(delay))
-
+ delay = time.time() - s
+ import_times[str(module)] = delay
+ print('OK ', module, '{:.3f}s'.format(delay))
+
except Exception as e:
print('FAIL:', module, e, file=sys.stderr)
print_backtrace_marked(sys.exc_info())
@@ -86,10 +86,10 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
failed.append('{}: {}'.format(module, e))
raise
- print("Slowest imports:")
- for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]:
- print(' ', '{:.3f}s'.format(t), m)
-
+ print("Slowest imports:")
+ for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]:
+ print(' ', '{:.3f}s'.format(t), m)
+
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
diff --git a/library/python/testing/import_test/ya.make b/library/python/testing/import_test/ya.make
index fae36ffe8f..b996336159 100644
--- a/library/python/testing/import_test/ya.make
+++ b/library/python/testing/import_test/ya.make
@@ -1,5 +1,5 @@
OWNER(
- g:yatest
+ g:yatest
exprmntr
)
diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py
index 5ef9c5c189..3ea95ebb1d 100644
--- a/library/python/testing/recipe/__init__.py
+++ b/library/python/testing/recipe/__init__.py
@@ -16,7 +16,7 @@ collect_cores = None
sanitizer_extra_checks = None
-def _setup_logging(level=logging.DEBUG):
+def _setup_logging(level=logging.DEBUG):
root_logger = logging.getLogger()
root_logger.setLevel(level)
@@ -41,7 +41,7 @@ def get_options():
args, opts = parser.parse_known_args()
global ya, sanitizer_extra_checks, collect_cores
- _setup_logging()
+ _setup_logging()
context = {
"test_stderr": args.test_stderr,
@@ -96,7 +96,7 @@ def declare_recipe(start, stop):
except Exception:
if parsed_args.pdb:
tty()
- import ipdb
- ipdb.post_mortem()
+ import ipdb
+ ipdb.post_mortem()
else:
raise
diff --git a/library/python/testing/recipe/ya.make b/library/python/testing/recipe/ya.make
index dd323aa245..239223c789 100644
--- a/library/python/testing/recipe/ya.make
+++ b/library/python/testing/recipe/ya.make
@@ -1,6 +1,6 @@
OWNER(
exprmntr
- g:yatest
+ g:yatest
)
PY23_LIBRARY()
@@ -10,10 +10,10 @@ PY_SRCS(
ports.py
)
-PEERDIR(
- contrib/python/ipdb
+PEERDIR(
+ contrib/python/ipdb
library/python/testing/yatest_common
library/python/testing/yatest_lib
-)
-
+)
+
END()
diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make
index 883bc8d7ab..c232bc663e 100644
--- a/library/python/testing/ya.make
+++ b/library/python/testing/ya.make
@@ -1,8 +1,8 @@
-OWNER(g:yatest)
+OWNER(g:yatest)
RECURSE(
behave
- deprecated
+ deprecated
fake_ya_package
filter
gtest
@@ -15,8 +15,8 @@ RECURSE(
recipe
system_info
types_test
- yapackage
- yapackage/test
+ yapackage
+ yapackage/test
yatest_common
yatest_lib
)
diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make
index 5662db4c5d..0098e1be85 100644
--- a/library/python/testing/yatest_common/ya.make
+++ b/library/python/testing/yatest_common/ya.make
@@ -1,31 +1,31 @@
-OWNER(g:yatest)
+OWNER(g:yatest)
PY23_LIBRARY()
-OWNER(g:yatest)
+OWNER(g:yatest)
NO_EXTENDED_SOURCE_SEARCH()
PY_SRCS(
TOP_LEVEL
- yatest/__init__.py
+ yatest/__init__.py
yatest/common/__init__.py
yatest/common/benchmark.py
yatest/common/canonical.py
yatest/common/environment.py
yatest/common/errors.py
yatest/common/legacy.py
- yatest/common/misc.py
+ yatest/common/misc.py
yatest/common/network.py
yatest/common/path.py
yatest/common/process.py
yatest/common/runtime.py
yatest/common/runtime_java.py
yatest/common/tags.py
-)
+)
-PEERDIR(
- contrib/python/six
+PEERDIR(
+ contrib/python/six
library/python/cores
library/python/filelock
library/python/fs
diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py
index b6a136d3e9..7a258d070b 100644
--- a/library/python/testing/yatest_common/yatest/common/canonical.py
+++ b/library/python/testing/yatest_common/yatest/common/canonical.py
@@ -68,7 +68,7 @@ def canonical_execute(
):
"""
Shortcut to execute a binary and canonize its stdout
- :param binary: absolute path to the binary
+ :param binary: absolute path to the binary
:param args: binary arguments
:param check_exit_code: will raise ExecutionError if the command exits with non zero code
:param shell: use shell to run the command
@@ -78,7 +78,7 @@ def canonical_execute(
:param stdin: command stdin
:param stderr: command stderr
:param creationflags: command creation flags
- :param file_name: output file name. if not specified program name will be used
+ :param file_name: output file name. if not specified program name will be used
:param diff_tool: path to custome diff tool
:param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
@@ -94,7 +94,7 @@ def canonical_execute(
execute_args = locals()
del execute_args["binary"]
del execute_args["args"]
- del execute_args["file_name"]
+ del execute_args["file_name"]
del execute_args["save_locally"]
del execute_args["diff_tool"]
del execute_args["diff_file_name"]
@@ -123,7 +123,7 @@ def canonical_py_execute(
:param stdin: command stdin
:param stderr: command stderr
:param creationflags: command creation flags
- :param file_name: output file name. if not specified program name will be used
+ :param file_name: output file name. if not specified program name will be used
:param diff_tool: path to custome diff tool
:param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
@@ -135,7 +135,7 @@ def canonical_py_execute(
execute_args = locals()
del execute_args["script_path"]
del execute_args["args"]
- del execute_args["file_name"]
+ del execute_args["file_name"]
del execute_args["save_locally"]
del execute_args["diff_tool"]
del execute_args["diff_file_name"]
diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py
index 43f48d0958..62fc501d52 100644
--- a/library/python/testing/yatest_common/yatest/common/environment.py
+++ b/library/python/testing/yatest_common/yatest/common/environment.py
@@ -1,5 +1,5 @@
-# coding: utf-8
+# coding: utf-8
+
-
-def extend_env_var(env, name, value, sep=":"):
- return sep.join(filter(None, [env.get(name), value]))
+def extend_env_var(env, name, value, sep=":"):
+ return sep.join(filter(None, [env.get(name), value]))
diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py
index 8c038fc381..8d5abc9b96 100644
--- a/library/python/testing/yatest_common/yatest/common/errors.py
+++ b/library/python/testing/yatest_common/yatest/common/errors.py
@@ -1,4 +1,4 @@
-import os
+import os
import sys
@@ -8,7 +8,7 @@ class RestartTestException(Exception):
super(RestartTestException, self).__init__(*args, **kwargs)
sys.stderr.write("##restart-test##\n")
sys.stderr.flush()
- os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
+ os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
class InfrastructureException(Exception):
@@ -17,4 +17,4 @@ class InfrastructureException(Exception):
super(InfrastructureException, self).__init__(*args, **kwargs)
sys.stderr.write("##infrastructure-error##\n")
sys.stderr.flush()
- os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
+ os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
diff --git a/library/python/testing/yatest_common/yatest/common/misc.py b/library/python/testing/yatest_common/yatest/common/misc.py
index 20d3725ac9..3c8d239a40 100644
--- a/library/python/testing/yatest_common/yatest/common/misc.py
+++ b/library/python/testing/yatest_common/yatest/common/misc.py
@@ -1,19 +1,19 @@
-import functools
-
-
-def first(it):
- for d in it:
- if d:
- return d
-
-
-def lazy(func):
- res = []
-
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- if not res:
- res.append(func(*args, **kwargs))
- return res[0]
-
- return wrapper
+import functools
+
+
+def first(it):
+ for d in it:
+ if d:
+ return d
+
+
+def lazy(func):
+ res = []
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ if not res:
+ res.append(func(*args, **kwargs))
+ return res[0]
+
+ return wrapper
diff --git a/library/python/testing/yatest_common/yatest/common/network.py b/library/python/testing/yatest_common/yatest/common/network.py
index 37bcb1b8e0..e3e3b0abc2 100644
--- a/library/python/testing/yatest_common/yatest/common/network.py
+++ b/library/python/testing/yatest_common/yatest/common/network.py
@@ -1,47 +1,47 @@
# coding=utf-8
-
-import os
-import errno
-import socket
-import random
-import logging
-import platform
-import threading
-
+
+import os
+import errno
+import socket
+import random
+import logging
+import platform
+import threading
+
import six
-UI16MAXVAL = (1 << 16) - 1
-logger = logging.getLogger(__name__)
-
-
-class PortManagerException(Exception):
- pass
-
-
-class PortManager(object):
+UI16MAXVAL = (1 << 16) - 1
+logger = logging.getLogger(__name__)
+
+
+class PortManagerException(Exception):
+ pass
+
+
+class PortManager(object):
"""
See documentation here
-
+
https://wiki.yandex-team.ru/yatool/test/#python-acquire-ports
"""
- def __init__(self, sync_dir=None):
- self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH')
- if self._sync_dir:
- _makedirs(self._sync_dir)
-
- self._valid_range = get_valid_port_range()
- self._valid_port_count = self._count_valid_ports()
- self._filelocks = {}
- self._lock = threading.Lock()
-
- def __enter__(self):
- return self
-
- def __exit__(self, type, value, traceback):
- self.release()
-
- def get_port(self, port=0):
+ def __init__(self, sync_dir=None):
+ self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH')
+ if self._sync_dir:
+ _makedirs(self._sync_dir)
+
+ self._valid_range = get_valid_port_range()
+ self._valid_port_count = self._count_valid_ports()
+ self._filelocks = {}
+ self._lock = threading.Lock()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.release()
+
+ def get_port(self, port=0):
'''
Gets free TCP port
'''
@@ -64,8 +64,8 @@ class PortManager(object):
Gets one free port for use in both TCP and UDP protocols
'''
if port and self._no_random_ports():
- return port
-
+ return port
+
retries = 20
while retries > 0:
retries -= 1
@@ -73,199 +73,199 @@ class PortManager(object):
result_port = self.get_tcp_port()
if not self.is_port_free(result_port, socket.SOCK_DGRAM):
self.release_port(result_port)
- # Don't try to _capture_port(), it's already captured in the get_tcp_port()
+ # Don't try to _capture_port(), it's already captured in the get_tcp_port()
return result_port
raise Exception('Failed to find port')
def release_port(self, port):
- with self._lock:
- self._release_port_no_lock(port)
-
- def _release_port_no_lock(self, port):
- filelock = self._filelocks.pop(port, None)
- if filelock:
- filelock.release()
-
+ with self._lock:
+ self._release_port_no_lock(port)
+
+ def _release_port_no_lock(self, port):
+ filelock = self._filelocks.pop(port, None)
+ if filelock:
+ filelock.release()
+
def release(self):
- with self._lock:
- while self._filelocks:
- _, filelock = self._filelocks.popitem()
- if filelock:
- filelock.release()
-
- def get_port_range(self, start_port, count, random_start=True):
- assert count > 0
- if start_port and self._no_random_ports():
- return start_port
-
- candidates = []
-
- def drop_candidates():
- for port in candidates:
- self._release_port_no_lock(port)
- candidates[:] = []
-
- with self._lock:
- for attempts in six.moves.range(128):
- for left, right in self._valid_range:
- if right - left < count:
- continue
-
- if random_start:
- start = random.randint(left, right - ((right - left) // 2))
- else:
- start = left
- for probe_port in six.moves.range(start, right):
- if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM):
- candidates.append(probe_port)
- else:
- drop_candidates()
-
- if len(candidates) == count:
- return candidates[0]
- # Can't find required number of ports without gap in the current range
- drop_candidates()
-
- raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format(
- start_port, count, self._valid_range, self._filelocks))
-
- def _count_valid_ports(self):
- res = 0
- for left, right in self._valid_range:
- res += right - left
- assert res, ('There are no available valid ports', self._valid_range)
- return res
-
+ with self._lock:
+ while self._filelocks:
+ _, filelock = self._filelocks.popitem()
+ if filelock:
+ filelock.release()
+
+ def get_port_range(self, start_port, count, random_start=True):
+ assert count > 0
+ if start_port and self._no_random_ports():
+ return start_port
+
+ candidates = []
+
+ def drop_candidates():
+ for port in candidates:
+ self._release_port_no_lock(port)
+ candidates[:] = []
+
+ with self._lock:
+ for attempts in six.moves.range(128):
+ for left, right in self._valid_range:
+ if right - left < count:
+ continue
+
+ if random_start:
+ start = random.randint(left, right - ((right - left) // 2))
+ else:
+ start = left
+ for probe_port in six.moves.range(start, right):
+ if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM):
+ candidates.append(probe_port)
+ else:
+ drop_candidates()
+
+ if len(candidates) == count:
+ return candidates[0]
+ # Can't find required number of ports without gap in the current range
+ drop_candidates()
+
+ raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format(
+ start_port, count, self._valid_range, self._filelocks))
+
+ def _count_valid_ports(self):
+ res = 0
+ for left, right in self._valid_range:
+ res += right - left
+ assert res, ('There are no available valid ports', self._valid_range)
+ return res
+
def _get_port(self, port, sock_type):
if port and self._no_random_ports():
return port
- if len(self._filelocks) >= self._valid_port_count:
- raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks))
-
- salt = random.randint(0, UI16MAXVAL)
+ if len(self._filelocks) >= self._valid_port_count:
+ raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks))
+
+ salt = random.randint(0, UI16MAXVAL)
for attempt in six.moves.range(self._valid_port_count):
- probe_port = (salt + attempt) % self._valid_port_count
-
- for left, right in self._valid_range:
- if probe_port >= (right - left):
- probe_port -= right - left
- else:
- probe_port += left
- break
- if not self._capture_port(probe_port, sock_type):
- continue
- return probe_port
-
- raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks))
-
- def _capture_port(self, port, sock_type):
- with self._lock:
- return self._capture_port_no_lock(port, sock_type)
-
+ probe_port = (salt + attempt) % self._valid_port_count
+
+ for left, right in self._valid_range:
+ if probe_port >= (right - left):
+ probe_port -= right - left
+ else:
+ probe_port += left
+ break
+ if not self._capture_port(probe_port, sock_type):
+ continue
+ return probe_port
+
+ raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks))
+
+ def _capture_port(self, port, sock_type):
+ with self._lock:
+ return self._capture_port_no_lock(port, sock_type)
+
def is_port_free(self, port, sock_type=socket.SOCK_STREAM):
- sock = socket.socket(socket.AF_INET6, sock_type)
- try:
- sock.bind(('::', port))
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- except socket.error as e:
- if e.errno == errno.EADDRINUSE:
- return False
- raise
- finally:
- sock.close()
- return True
-
- def _capture_port_no_lock(self, port, sock_type):
- if port in self._filelocks:
- return False
-
- filelock = None
- if self._sync_dir:
- # yatest.common should try to be hermetic and don't have peerdirs
- # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work
- import library.python.filelock
-
- filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port)))
- if not filelock.acquire(blocking=False):
- return False
+ sock = socket.socket(socket.AF_INET6, sock_type)
+ try:
+ sock.bind(('::', port))
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ except socket.error as e:
+ if e.errno == errno.EADDRINUSE:
+ return False
+ raise
+ finally:
+ sock.close()
+ return True
+
+ def _capture_port_no_lock(self, port, sock_type):
+ if port in self._filelocks:
+ return False
+
+ filelock = None
+ if self._sync_dir:
+ # yatest.common should try to be hermetic and don't have peerdirs
+ # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work
+ import library.python.filelock
+
+ filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port)))
+ if not filelock.acquire(blocking=False):
+ return False
if self.is_port_free(port, sock_type):
- self._filelocks[port] = filelock
- return True
- else:
- filelock.release()
- return False
-
+ self._filelocks[port] = filelock
+ return True
+ else:
+ filelock.release()
+ return False
+
if self.is_port_free(port, sock_type):
- self._filelocks[port] = filelock
- return True
- if filelock:
- filelock.release()
- return False
-
+ self._filelocks[port] = filelock
+ return True
+ if filelock:
+ filelock.release()
+ return False
+
def _no_random_ports(self):
return os.environ.get("NO_RANDOM_PORTS")
-
-
-def get_valid_port_range():
- first_valid = 1025
- last_valid = UI16MAXVAL
-
- given_range = os.environ.get('VALID_PORT_RANGE')
- if given_range and ':' in given_range:
- return [list(int(x) for x in given_range.split(':', 2))]
-
- first_eph, last_eph = get_ephemeral_range()
- first_invalid = max(first_eph, first_valid)
- last_invalid = min(last_eph, last_valid)
-
- ranges = []
- if first_invalid > first_valid:
- ranges.append((first_valid, first_invalid - 1))
- if last_invalid < last_valid:
- ranges.append((last_invalid + 1, last_valid))
- return ranges
-
-
-def get_ephemeral_range():
- if platform.system() == 'Linux':
- filename = "/proc/sys/net/ipv4/ip_local_port_range"
- if os.path.exists(filename):
- with open(filename) as afile:
+
+
+def get_valid_port_range():
+ first_valid = 1025
+ last_valid = UI16MAXVAL
+
+ given_range = os.environ.get('VALID_PORT_RANGE')
+ if given_range and ':' in given_range:
+ return [list(int(x) for x in given_range.split(':', 2))]
+
+ first_eph, last_eph = get_ephemeral_range()
+ first_invalid = max(first_eph, first_valid)
+ last_invalid = min(last_eph, last_valid)
+
+ ranges = []
+ if first_invalid > first_valid:
+ ranges.append((first_valid, first_invalid - 1))
+ if last_invalid < last_valid:
+ ranges.append((last_invalid + 1, last_valid))
+ return ranges
+
+
+def get_ephemeral_range():
+ if platform.system() == 'Linux':
+ filename = "/proc/sys/net/ipv4/ip_local_port_range"
+ if os.path.exists(filename):
+ with open(filename) as afile:
data = afile.read(1024) # fix for musl
- port_range = tuple(map(int, data.strip().split()))
- if len(port_range) == 2:
- return port_range
- else:
- logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data)
- elif platform.system() == 'Darwin':
- first = _sysctlbyname_uint("net.inet.ip.portrange.first")
- last = _sysctlbyname_uint("net.inet.ip.portrange.last")
- if first and last:
- return first, last
- # IANA suggestion
- return (1 << 15) + (1 << 14), UI16MAXVAL
-
-
-def _sysctlbyname_uint(name):
- try:
- from ctypes import CDLL, c_uint, byref
- from ctypes.util import find_library
- except ImportError:
- return
-
- libc = CDLL(find_library("c"))
- size = c_uint(0)
- res = c_uint(0)
- libc.sysctlbyname(name, None, byref(size), None, 0)
- libc.sysctlbyname(name, byref(res), byref(size), None, 0)
- return res.value
-
-
-def _makedirs(path):
- try:
- os.makedirs(path)
- except OSError as e:
- if e.errno == errno.EEXIST:
- return
- raise
+ port_range = tuple(map(int, data.strip().split()))
+ if len(port_range) == 2:
+ return port_range
+ else:
+ logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data)
+ elif platform.system() == 'Darwin':
+ first = _sysctlbyname_uint("net.inet.ip.portrange.first")
+ last = _sysctlbyname_uint("net.inet.ip.portrange.last")
+ if first and last:
+ return first, last
+ # IANA suggestion
+ return (1 << 15) + (1 << 14), UI16MAXVAL
+
+
+def _sysctlbyname_uint(name):
+ try:
+ from ctypes import CDLL, c_uint, byref
+ from ctypes.util import find_library
+ except ImportError:
+ return
+
+ libc = CDLL(find_library("c"))
+ size = c_uint(0)
+ res = c_uint(0)
+ libc.sysctlbyname(name, None, byref(size), None, 0)
+ libc.sysctlbyname(name, byref(res), byref(size), None, 0)
+ return res.value
+
+
+def _makedirs(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ return
+ raise
diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py
index 6fed7dda8a..d4f52f6333 100644
--- a/library/python/testing/yatest_common/yatest/common/path.py
+++ b/library/python/testing/yatest_common/yatest/common/path.py
@@ -1,12 +1,12 @@
# coding=utf-8
import errno
-import os
-import shutil
-import contextlib
-
+import os
+import shutil
+import contextlib
+
import library.python.fs as lpf
-
+
def replace_in_file(path, old, new):
"""
@@ -21,32 +21,32 @@ def replace_in_file(path, old, new):
lpf.ensure_removed(path)
with open(path, 'w') as fp:
fp.write(content.replace(old, new))
-
-
-@contextlib.contextmanager
-def change_dir(path):
- old = os.getcwd()
- try:
- os.chdir(path)
- yield path
- finally:
- os.chdir(old)
-
-
+
+
+@contextlib.contextmanager
+def change_dir(path):
+ old = os.getcwd()
+ try:
+ os.chdir(path)
+ yield path
+ finally:
+ os.chdir(old)
+
+
def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None):
- '''
- Copy an entire directory of files into an existing directory
- instead of raising Exception what shtuil.copytree does
- '''
+ '''
+ Copy an entire directory of files into an existing directory
+ instead of raising Exception what shtuil.copytree does
+ '''
if not os.path.exists(dst) and os.path.isdir(src):
os.makedirs(dst)
- for item in os.listdir(src):
- s = os.path.join(src, item)
- d = os.path.join(dst, item)
- if os.path.isdir(s):
- shutil.copytree(s, d, symlinks, ignore)
- else:
- shutil.copy2(s, d)
+ for item in os.listdir(src):
+ s = os.path.join(src, item)
+ d = os.path.join(dst, item)
+ if os.path.isdir(s):
+ shutil.copytree(s, d, symlinks, ignore)
+ else:
+ shutil.copy2(s, d)
if postprocessing:
postprocessing(dst, False)
for root, dirs, files in os.walk(dst):
diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py
index a8bcc21f51..f9eeb16525 100644
--- a/library/python/testing/yatest_common/yatest/common/process.py
+++ b/library/python/testing/yatest_common/yatest/common/process.py
@@ -1,10 +1,10 @@
-# coding: utf-8
-
+# coding: utf-8
+
import os
-import re
+import re
import time
import signal
-import shutil
+import shutil
import logging
import tempfile
import subprocess
@@ -13,19 +13,19 @@ import distutils.version
import six
-try:
- # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work.
- import library.python.cores as cores
-except ImportError:
- cores = None
-
+try:
+ # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work.
+ import library.python.cores as cores
+except ImportError:
+ cores = None
+
from . import runtime
from . import path
from . import environment
MAX_OUT_LEN = 1000 * 1000 # 1 mb
-MAX_MESSAGE_LEN = 1500
+MAX_MESSAGE_LEN = 1500
SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)"
GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)")
yatest_logger = logging.getLogger("ya.test")
@@ -55,12 +55,12 @@ class ExecutionError(Exception):
command=command,
code=execution_result.exit_code,
err=_format_error(execution_result.std_err))
- if cores:
- if execution_result.backtrace:
- message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
- else:
- message += "Backtrace is not available: module cores isn't available"
-
+ if cores:
+ if execution_result.backtrace:
+ message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
+ else:
+ message += "Backtrace is not available: module cores isn't available"
+
super(ExecutionError, self).__init__(message)
self.execution_result = execution_result
@@ -85,31 +85,31 @@ class SignalInterruptionError(Exception):
self.res = None
-class InvalidCommandError(Exception):
- pass
-
-
+class InvalidCommandError(Exception):
+ pass
+
+
class _Execution(object):
- def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False):
+ def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False):
self._command = command
self._process = process
self._out_file = out_file
self._err_file = err_file
self._std_out = None
self._std_err = None
- self._elapsed = None
- self._start = time.time()
+ self._elapsed = None
+ self._start = time.time()
self._process_progress_listener = process_progress_listener
- self._cwd = cwd or os.getcwd()
- self._collect_cores = collect_cores
- self._backtrace = ''
- self._check_sanitizer = check_sanitizer
+ self._cwd = cwd or os.getcwd()
+ self._collect_cores = collect_cores
+ self._backtrace = ''
+ self._check_sanitizer = check_sanitizer
self._metrics = {}
- self._started = started
- self._user_stdout = bool(user_stdout)
- self._user_stderr = bool(user_stderr)
- self._exit_code = None
+ self._started = started
+ self._user_stdout = bool(user_stdout)
+ self._user_stderr = bool(user_stderr)
+ self._exit_code = None
if process_progress_listener:
process_progress_listener.open(command, process, out_file, err_file)
@@ -122,21 +122,21 @@ class _Execution(object):
self._save_outputs(False)
_kill_process_tree(self._process.pid)
self._clean_files()
- # DEVTOOLS-2347
- yatest_logger.debug("Process status before wait_for: %s", self.running)
- try:
- wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1)
- except TimeoutError:
- yatest_logger.debug("Process status after wait_for: %s", self.running)
- yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
- raise
+ # DEVTOOLS-2347
+ yatest_logger.debug("Process status before wait_for: %s", self.running)
+ try:
+ wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1)
+ except TimeoutError:
+ yatest_logger.debug("Process status after wait_for: %s", self.running)
+ yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
+ raise
else:
raise InvalidExecutionStateError("Cannot kill a stopped process")
- def terminate(self):
- if self.running:
- self._process.terminate()
-
+ def terminate(self):
+ if self.running:
+ self._process.terminate()
+
@property
def process(self):
return self._process
@@ -146,56 +146,56 @@ class _Execution(object):
return self._command
@property
- def returncode(self):
- return self.exit_code
-
- @property
+ def returncode(self):
+ return self.exit_code
+
+ @property
def exit_code(self):
- """
- Deprecated, use returncode
- """
- if self._exit_code is None:
- self._exit_code = self._process.returncode
- return self._exit_code
-
- @property
- def stdout(self):
- return self.std_out
+ """
+ Deprecated, use returncode
+ """
+ if self._exit_code is None:
+ self._exit_code = self._process.returncode
+ return self._exit_code
@property
+ def stdout(self):
+ return self.std_out
+
+ @property
def std_out(self):
- """
- Deprecated, use stdout
- """
+ """
+ Deprecated, use stdout
+ """
if self._std_out is not None:
return self._std_out
- if self._process.stdout and not self._user_stdout:
- self._std_out = self._process.stdout.read()
- return self._std_out
-
- @property
- def stderr(self):
- return self.std_err
+ if self._process.stdout and not self._user_stdout:
+ self._std_out = self._process.stdout.read()
+ return self._std_out
@property
+ def stderr(self):
+ return self.std_err
+
+ @property
def std_err(self):
- """
- Deprecated, use stderr
- """
+ """
+ Deprecated, use stderr
+ """
if self._std_err is not None:
return self._std_err
- if self._process.stderr and not self._user_stderr:
- self._std_err = self._process.stderr.read()
- return self._std_err
-
- @property
- def elapsed(self):
- return self._elapsed
-
- @property
- def backtrace(self):
- return self._backtrace
-
+ if self._process.stderr and not self._user_stderr:
+ self._std_err = self._process.stderr.read()
+ return self._std_err
+
+ @property
+ def elapsed(self):
+ return self._elapsed
+
+ @property
+ def backtrace(self):
+ return self._backtrace
+
@property
def metrics(self):
return self._metrics
@@ -204,25 +204,25 @@ class _Execution(object):
if self._process_progress_listener:
self._process_progress_listener()
self._process_progress_listener.close()
- if not self._user_stdout:
- if self._out_file is None:
- pass
- elif self._out_file != subprocess.PIPE:
- self._out_file.flush()
- self._out_file.seek(0, os.SEEK_SET)
- self._std_out = self._out_file.read()
- else:
- self._std_out = self._process.stdout.read()
- if not self._user_stderr:
- if self._err_file is None:
- pass
- elif self._err_file != subprocess.PIPE:
- self._err_file.flush()
- self._err_file.seek(0, os.SEEK_SET)
- self._std_err = self._err_file.read()
- else:
- self._std_err = self._process.stderr.read()
-
+ if not self._user_stdout:
+ if self._out_file is None:
+ pass
+ elif self._out_file != subprocess.PIPE:
+ self._out_file.flush()
+ self._out_file.seek(0, os.SEEK_SET)
+ self._std_out = self._out_file.read()
+ else:
+ self._std_out = self._process.stdout.read()
+ if not self._user_stderr:
+ if self._err_file is None:
+ pass
+ elif self._err_file != subprocess.PIPE:
+ self._err_file.flush()
+ self._err_file.seek(0, os.SEEK_SET)
+ self._std_err = self._err_file.read()
+ else:
+ self._std_err = self._process.stderr.read()
+
if clean_files:
self._clean_files()
yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code)
@@ -245,45 +245,45 @@ class _Execution(object):
yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN))
def _clean_files(self):
- if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE:
- self._err_file.close()
- self._err_file = None
- if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE:
- self._out_file.close()
- self._out_file = None
-
- def _recover_core(self):
- core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid)
- if core_path:
- # Core dump file recovering may be disabled (for distbuild for example) - produce only bt
- store_cores = runtime._get_ya_config().collect_cores
- if store_cores:
- new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
- # Copy core dump file, because it may be overwritten
+ if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE:
+ self._err_file.close()
+ self._err_file = None
+ if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE:
+ self._out_file.close()
+ self._out_file = None
+
+ def _recover_core(self):
+ core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid)
+ if core_path:
+ # Core dump file recovering may be disabled (for distbuild for example) - produce only bt
+ store_cores = runtime._get_ya_config().collect_cores
+ if store_cores:
+ new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
+ # Copy core dump file, because it may be overwritten
yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
- shutil.copyfile(core_path, new_core_path)
- core_path = new_core_path
-
- bt_filename = None
- pbt_filename = None
-
- if os.path.exists(runtime.gdb_path()):
- self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path())
- bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid))
+ shutil.copyfile(core_path, new_core_path)
+ core_path = new_core_path
+
+ bt_filename = None
+ pbt_filename = None
+
+ if os.path.exists(runtime.gdb_path()):
+ self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path())
+ bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid))
with open(bt_filename, "wb") as afile:
afile.write(six.ensure_binary(self._backtrace))
- # generate pretty html version of backtrace aka Tri Korochki
- pbt_filename = bt_filename + ".html"
+ # generate pretty html version of backtrace aka Tri Korochki
+ pbt_filename = bt_filename + ".html"
backtrace_to_html(bt_filename, pbt_filename)
-
- if store_cores:
- runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename)
- else:
- runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
-
+
+ if store_cores:
+ runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename)
+ else:
+ runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
+
def wait(self, check_exit_code=True, timeout=None, on_timeout=None):
def _wait():
- finished = None
+ finished = None
interrupted = False
try:
if hasattr(os, "wait4"):
@@ -293,7 +293,7 @@ class _Execution(object):
else:
# PEP 475
pid, sts, rusage = os.wait4(self._process.pid, 0)
- finished = time.time()
+ finished = time.time()
self._process._handle_exitstatus(sts)
for field in [
"ru_idrss",
@@ -328,46 +328,46 @@ class _Execution(object):
if not interrupted:
self._process.wait() # this has to be here unconditionally, so that all process properties are set
- if not finished:
- finished = time.time()
- self._metrics["wtime"] = round(finished - self._started, 3)
-
+ if not finished:
+ finished = time.time()
+ self._metrics["wtime"] = round(finished - self._started, 3)
+
try:
- if timeout:
- process_is_finished = lambda: not self.running
- fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
+ if timeout:
+ process_is_finished = lambda: not self.running
+ fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
try:
wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
except TimeoutError as e:
if on_timeout:
- yatest_logger.debug("Calling user specified on_timeout function")
+ yatest_logger.debug("Calling user specified on_timeout function")
try:
on_timeout(self, timeout)
except Exception:
yatest_logger.exception("Exception while calling on_timeout")
raise ExecutionTimeoutError(self, str(e))
- # Wait should be always called here, it finalizes internal states of its process and sets up return code
+ # Wait should be always called here, it finalizes internal states of its process and sets up return code
_wait()
- except BaseException as e:
+ except BaseException as e:
_kill_process_tree(self._process.pid)
_wait()
yatest_logger.debug("Command exception: %s", e)
- raise
+ raise
finally:
- self._elapsed = time.time() - self._start
- self._save_outputs()
+ self._elapsed = time.time() - self._start
+ self._save_outputs()
self.verify_no_coredumps()
- self._finalise(check_exit_code)
-
- def _finalise(self, check_exit_code):
+ self._finalise(check_exit_code)
+
+ def _finalise(self, check_exit_code):
# Set the signal (negative number) which caused the process to exit
if check_exit_code and self.exit_code != 0:
yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
raise ExecutionError(self)
- # Don't search for sanitize errors if stderr was redirected
+ # Don't search for sanitize errors if stderr was redirected
self.verify_sanitize_errors()
def verify_no_coredumps(self):
@@ -375,44 +375,44 @@ class _Execution(object):
Verify there is no coredump from this binary. If there is then report backtrace.
"""
if self.exit_code < 0 and self._collect_cores:
- if cores:
- try:
- self._recover_core()
- except Exception:
- yatest_logger.exception("Exception while recovering core")
- else:
- yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available")
+ if cores:
+ try:
+ self._recover_core()
+ except Exception:
+ yatest_logger.exception("Exception while recovering core")
+ else:
+ yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available")
def verify_sanitize_errors(self):
"""
Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them.
"""
- if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks:
- build_path = runtime.build_path()
- if self.command[0].startswith(build_path):
- match = re.search(SANITIZER_ERROR_PATTERN, self._std_err)
- if match:
- yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
- raise ExecutionError(self)
+ if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks:
+ build_path = runtime.build_path()
+ if self.command[0].startswith(build_path):
+ match = re.search(SANITIZER_ERROR_PATTERN, self._std_err)
+ if match:
+ yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
+ raise ExecutionError(self)
else:
yatest_logger.debug("No sanitizer errors found")
- else:
+ else:
yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
-
-def on_timeout_gen_coredump(exec_obj, _):
- """
- Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump)
- to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout.
- All files will be available in the testing_out_stuff and via links.
- """
- try:
- os.kill(exec_obj.process.pid, signal.SIGQUIT)
- except OSError:
- # process might be already terminated
- pass
-
-
+
+def on_timeout_gen_coredump(exec_obj, _):
+ """
+ Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump)
+ to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout.
+ All files will be available in the testing_out_stuff and via links.
+ """
+ try:
+ os.kill(exec_obj.process.pid, signal.SIGQUIT)
+ except OSError:
+ # process might be already terminated
+ pass
+
+
def execute(
command, check_exit_code=True,
shell=False, timeout=None,
@@ -421,7 +421,7 @@ def execute(
creationflags=0, wait=True,
process_progress_listener=None, close_fds=False,
collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
- executor=_Execution,
+ executor=_Execution,
):
"""
Executes a command
@@ -438,26 +438,26 @@ def execute(
:param wait: should wait until the command finishes
:param process_progress_listener=object that is polled while execution is in progress
:param close_fds: subrpocess.Popen close_fds args
- :param collect_cores: recover core dump files if shell == False
- :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors
+ :param collect_cores: recover core dump files if shell == False
+ :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors
:param preexec_fn: subrpocess.Popen preexec_fn arg
:param on_timeout: on_timeout(<execution object>, <timeout value>) callback
:return _Execution: Execution object
"""
- if env is None:
- env = os.environ.copy()
+ if env is None:
+ env = os.environ.copy()
else:
- # Certain environment variables must be present for programs to work properly.
- # For more info see DEVTOOLSSUPPORT-4907
- mandatory_env_name = 'YA_MANDATORY_ENV_VARS'
- mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or ''
- if mandatory_vars:
- env[mandatory_env_name] = mandatory_vars
- mandatory_system_vars = filter(None, mandatory_vars.split(':'))
- else:
- mandatory_system_vars = ['TMPDIR']
-
+ # Certain environment variables must be present for programs to work properly.
+ # For more info see DEVTOOLSSUPPORT-4907
+ mandatory_env_name = 'YA_MANDATORY_ENV_VARS'
+ mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or ''
+ if mandatory_vars:
+ env[mandatory_env_name] = mandatory_vars
+ mandatory_system_vars = filter(None, mandatory_vars.split(':'))
+ else:
+ mandatory_system_vars = ['TMPDIR']
+
for var in mandatory_system_vars:
if var not in env and var in os.environ:
env[var] = os.environ[var]
@@ -465,13 +465,13 @@ def execute(
if not wait and timeout is not None:
raise ValueError("Incompatible arguments 'timeout' and wait=False")
- # if subprocess.PIPE in [stdout, stderr]:
- # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock")
-
+ # if subprocess.PIPE in [stdout, stderr]:
+ # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock")
+
def get_out_stream(stream, default_name):
if stream is None:
# No stream is supplied: open new temp file
- return _get_command_output_file(command, default_name), False
+ return _get_command_output_file(command, default_name), False
if isinstance(stream, six.string_types):
# User filename is supplied: open file for writing
@@ -490,86 +490,86 @@ def execute(
if shell and type(command) == list:
command = " ".join(command)
- if shell:
- collect_cores = False
- check_sanitizer = False
- else:
- if isinstance(command, (list, tuple)):
- executable = command[0]
- else:
- executable = command
- if os.path.isabs(executable):
- if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"):
- exists = os.path.exists(executable)
- if exists:
- stat = os.stat(executable)
- else:
- stat = None
- raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat))
- if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK):
- raise InvalidCommandError("Target program is not executable: {}".format(executable))
-
- if check_sanitizer:
- env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100")
-
- if stdin:
- name = "PIPE" if stdin == subprocess.PIPE else stdin.name
- yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd)
- else:
+ if shell:
+ collect_cores = False
+ check_sanitizer = False
+ else:
+ if isinstance(command, (list, tuple)):
+ executable = command[0]
+ else:
+ executable = command
+ if os.path.isabs(executable):
+ if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"):
+ exists = os.path.exists(executable)
+ if exists:
+ stat = os.stat(executable)
+ else:
+ stat = None
+ raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat))
+ if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK):
+ raise InvalidCommandError("Target program is not executable: {}".format(executable))
+
+ if check_sanitizer:
+ env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100")
+
+ if stdin:
+ name = "PIPE" if stdin == subprocess.PIPE else stdin.name
+ yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd)
+ else:
yatest_logger.debug("Executing '%s' in '%s'", command, cwd)
- # XXX
-
- started = time.time()
- process = subprocess.Popen(
- command, shell=shell, universal_newlines=True,
- stdout=out_file, stderr=err_file, stdin=in_file,
- cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn,
- )
- yatest_logger.debug("Command pid: %s", process.pid)
-
- res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr)
+ # XXX
+
+ started = time.time()
+ process = subprocess.Popen(
+ command, shell=shell, universal_newlines=True,
+ stdout=out_file, stderr=err_file, stdin=in_file,
+ cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn,
+ )
+ yatest_logger.debug("Command pid: %s", process.pid)
+
+ res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr)
if wait:
res.wait(check_exit_code, timeout, on_timeout)
return res
-def _get_command_output_file(cmd, ext):
- parts = [get_command_name(cmd)]
- if 'YA_RETRY_INDEX' in os.environ:
- parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX')))
- if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1:
- parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0')))
-
- filename = '.'.join(parts + [ext])
- try:
- # if execution is performed from test, save out / err to the test logs dir
- import yatest.common
+def _get_command_output_file(cmd, ext):
+ parts = [get_command_name(cmd)]
+ if 'YA_RETRY_INDEX' in os.environ:
+ parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX')))
+ if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1:
+ parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0')))
+
+ filename = '.'.join(parts + [ext])
+ try:
+ # if execution is performed from test, save out / err to the test logs dir
+ import yatest.common
import library.python.pytest.plugins.ya
if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None:
- raise ImportError("not in test")
- filename = path.get_unique_file_path(yatest.common.output_path(), filename)
- yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename))
- return open(filename, "wb+")
- except ImportError:
- return tempfile.NamedTemporaryFile(delete=False, suffix=filename)
-
-
-def _get_proc_tree_info(pids):
- if os.name == 'nt':
- return 'Not supported'
- else:
- stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
- return stdout
-
-
-def py_execute(
- command, check_exit_code=True,
- shell=False, timeout=None,
- cwd=None, env=None,
- stdin=None, stdout=None, stderr=None,
- creationflags=0, wait=True,
- process_progress_listener=None, close_fds=False
-):
+ raise ImportError("not in test")
+ filename = path.get_unique_file_path(yatest.common.output_path(), filename)
+ yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename))
+ return open(filename, "wb+")
+ except ImportError:
+ return tempfile.NamedTemporaryFile(delete=False, suffix=filename)
+
+
+def _get_proc_tree_info(pids):
+ if os.name == 'nt':
+ return 'Not supported'
+ else:
+ stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+ return stdout
+
+
+def py_execute(
+ command, check_exit_code=True,
+ shell=False, timeout=None,
+ cwd=None, env=None,
+ stdin=None, stdout=None, stderr=None,
+ creationflags=0, wait=True,
+ process_progress_listener=None, close_fds=False
+):
"""
Executes a command with the arcadia python
:param command: command to pass to python
@@ -635,50 +635,50 @@ def _kill_process_tree(process_pid, target_pid_signal=None):
if os.name == 'nt':
_win_kill_process_tree(process_pid)
else:
- _nix_kill_process_tree(process_pid, target_pid_signal)
-
-
-def _nix_get_proc_children(pid):
- try:
- cmd = ["pgrep", "-P", str(pid)]
- return [int(p) for p in subprocess.check_output(cmd).split()]
- except Exception:
- return []
-
-
-def _get_binname(pid):
- try:
- return os.path.basename(os.readlink('/proc/{}/exe'.format(pid)))
- except Exception as e:
- return "error({})".format(e)
-
-
+ _nix_kill_process_tree(process_pid, target_pid_signal)
+
+
+def _nix_get_proc_children(pid):
+ try:
+ cmd = ["pgrep", "-P", str(pid)]
+ return [int(p) for p in subprocess.check_output(cmd).split()]
+ except Exception:
+ return []
+
+
+def _get_binname(pid):
+ try:
+ return os.path.basename(os.readlink('/proc/{}/exe'.format(pid)))
+ except Exception as e:
+ return "error({})".format(e)
+
+
def _nix_kill_process_tree(pid, target_pid_signal=None):
"""
Kills the process tree.
"""
- yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid)))
+ yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid)))
def try_to_send_signal(pid, sig):
try:
os.kill(pid, sig)
- yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
+ yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
except Exception as exc:
- yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
+ yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes.
# Get the child process PID list.
- child_pids = _nix_get_proc_children(pid)
- # Stop the child processes.
- for child_pid in child_pids:
- try:
- # Kill the child recursively.
- _kill_process_tree(int(child_pid))
- except Exception as e:
- # Skip the error and continue killing.
- yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e))
- continue
+ child_pids = _nix_get_proc_children(pid)
+ # Stop the child processes.
+ for child_pid in child_pids:
+ try:
+ # Kill the child recursively.
+ _kill_process_tree(int(child_pid))
+ except Exception as e:
+ # Skip the error and continue killing.
+ yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e))
+ continue
try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process.
@@ -689,16 +689,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None):
def _win_kill_process_tree(pid):
subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)])
-
-
-def _run_readelf(binary_path):
+
+
+def _run_readelf(binary_path):
return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)]))
-
-
-def check_glibc_version(binary_path):
+
+
+def check_glibc_version(binary_path):
lucid_glibc_version = distutils.version.LooseVersion("2.11")
- for l in _run_readelf(binary_path).split('\n'):
+ for l in _run_readelf(binary_path).split('\n'):
match = GLIBC_PATTERN.search(l)
if not match:
continue
@@ -706,14 +706,14 @@ def check_glibc_version(binary_path):
def backtrace_to_html(bt_filename, output):
- try:
+ try:
from library.python import coredump_filter
- with open(output, "wb") as afile:
+ with open(output, "wb") as afile:
coredump_filter.filter_stackdump(bt_filename, stream=afile)
- except ImportError as e:
- yatest_logger.debug("Failed to import coredump_filter: %s", e)
- with open(output, "wb") as afile:
- afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>")
+ except ImportError as e:
+ yatest_logger.debug("Failed to import coredump_filter: %s", e)
+ with open(output, "wb") as afile:
+ afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>")
def _try_convert_bytes_to_string(source):
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
index e55e193446..07a41ebbbc 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -1,7 +1,7 @@
-import errno
-import functools
+import errno
+import functools
import json
-import os
+import os
import threading
import six
@@ -10,13 +10,13 @@ import six
_lock = threading.Lock()
-def _get_ya_config():
+def _get_ya_config():
try:
import library.python.pytest.plugins.ya as ya_plugin
if ya_plugin.pytest_config is not None:
return ya_plugin.pytest_config
import pytest
- return pytest.config
+ return pytest.config
except (ImportError, AttributeError):
try:
import library.python.testing.recipe
@@ -27,10 +27,10 @@ def _get_ya_config():
raise NotImplementedError("yatest.common.* is only available from the testing runtime")
-def _get_ya_plugin_instance():
- return _get_ya_config().ya
-
-
+def _get_ya_plugin_instance():
+ return _get_ya_config().ya
+
+
def _norm_path(path):
if path is None:
return None
@@ -46,35 +46,35 @@ def _join_path(main_path, path):
return os.path.join(main_path, _norm_path(path))
-def not_test(func):
- """
- Mark any function as not a test for py.test
- :param func:
- :return:
- """
- @functools.wraps(func)
- def wrapper(*args, **kwds):
- return func(*args, **kwds)
- setattr(wrapper, '__test__', False)
- return wrapper
-
-
-def source_path(path=None):
+def not_test(func):
+ """
+ Mark any function as not a test for py.test
+ :param func:
+ :return:
+ """
+ @functools.wraps(func)
+ def wrapper(*args, **kwds):
+ return func(*args, **kwds)
+ setattr(wrapper, '__test__', False)
+ return wrapper
+
+
+def source_path(path=None):
"""
Get source path inside arcadia
- :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya')
+ :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya')
:return: absolute path to the source folder
"""
- return _join_path(_get_ya_plugin_instance().source_root, path)
+ return _join_path(_get_ya_plugin_instance().source_root, path)
-def build_path(path=None):
+def build_path(path=None):
"""
Get path inside build directory
- :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin')
+ :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin')
:return: absolute path inside build directory
"""
- return _join_path(_get_ya_plugin_instance().build_root, path)
+ return _join_path(_get_ya_plugin_instance().build_root, path)
def java_path():
@@ -82,7 +82,7 @@ def java_path():
[DEPRECATED] Get path to java
:return: absolute path to java
"""
- from . import runtime_java
+ from . import runtime_java
return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk')))
@@ -90,7 +90,7 @@ def java_home():
"""
Get jdk directory path
"""
- from . import runtime_java
+ from . import runtime_java
jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk'))
if not jdk_dir:
raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform")
@@ -100,60 +100,60 @@ def java_home():
def java_bin():
"""
Get path to the java binary
- Requires DEPENDS(jdk)
+ Requires DEPENDS(jdk)
"""
return os.path.join(java_home(), "bin", "java")
-def data_path(path=None):
+def data_path(path=None):
"""
Get path inside arcadia_tests_data directory
- :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
+ :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
:return: absolute path inside arcadia_tests_data
"""
- return _join_path(_get_ya_plugin_instance().data_root, path)
+ return _join_path(_get_ya_plugin_instance().data_root, path)
-def output_path(path=None):
+def output_path(path=None):
"""
Get path inside the current test suite output dir.
Placing files to this dir guarantees that files will be accessible after the test suite execution.
:param path: path relative to the test suite output dir
:return: absolute path inside the test suite output dir
"""
- return _join_path(_get_ya_plugin_instance().output_dir, path)
-
-
-def ram_drive_path(path=None):
- """
- :param path: path relative to the ram drive.
- :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment.
- """
- if 'YA_TEST_RAM_DRIVE_PATH' in os.environ:
- return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path)
-
-
-def output_ram_drive_path(path=None):
- """
- Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing.
- Returns None if no ram drive was provided by environment.
- :param path: path relative to the output ram drive directory
- """
- if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ:
- return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path)
-
-
-def binary_path(path=None):
+ return _join_path(_get_ya_plugin_instance().output_dir, path)
+
+
+def ram_drive_path(path=None):
+ """
+ :param path: path relative to the ram drive.
+ :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment.
+ """
+ if 'YA_TEST_RAM_DRIVE_PATH' in os.environ:
+ return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path)
+
+
+def output_ram_drive_path(path=None):
+ """
+ Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing.
+ Returns None if no ram drive was provided by environment.
+ :param path: path relative to the output ram drive directory
+ """
+ if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ:
+ return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path)
+
+
+def binary_path(path=None):
"""
Get path to the built binary
- :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin')
+ :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin')
:return: absolute path to the binary
"""
- path = _norm_path(path)
- return _get_ya_plugin_instance().get_binary(path)
+ path = _norm_path(path)
+ return _get_ya_plugin_instance().get_binary(path)
-def work_path(path=None):
+def work_path(path=None):
"""
Get path inside the current test suite working directory. Creating files in the work directory does not guarantee
that files will be accessible after the test suite execution
@@ -169,12 +169,12 @@ def work_path(path=None):
def python_path():
"""
- Get path to the arcadia python.
-
- Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles
- are built in a stripped-down form that is needed for building, not running tests.
- See comments in the file below to find out which version of python is compatible with tests.
- https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc
+ Get path to the arcadia python.
+
+ Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles
+ are built in a stripped-down form that is needed for building, not running tests.
+ See comments in the file below to find out which version of python is compatible with tests.
+ https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc
:return: absolute path to python
"""
return _get_ya_plugin_instance().python_path
@@ -207,18 +207,18 @@ def get_param_dict_copy():
return _get_ya_plugin_instance().get_param_dict_copy()
-@not_test
-def test_output_path(path=None):
+@not_test
+def test_output_path(path=None):
"""
Get dir in the suite output_path for the current test case
"""
test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0]
- try:
+ try:
os.makedirs(test_out_dir)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- return _join_path(test_out_dir, path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ return _join_path(test_out_dir, path)
def project_path(path=None):
@@ -228,13 +228,13 @@ def project_path(path=None):
return _join_path(os.path.join(build_path(), context.project_path), path)
-def gdb_path():
- """
- Get path to the gdb
- """
- return _get_ya_plugin_instance().gdb_path
-
-
+def gdb_path():
+ """
+ Get path to the gdb
+ """
+ return _get_ya_plugin_instance().gdb_path
+
+
def c_compiler_path():
"""
Get path to the gdb
@@ -261,27 +261,27 @@ def global_resources():
return {}
-def _register_core(name, binary_path, core_path, bt_path, pbt_path):
- config = _get_ya_config()
+def _register_core(name, binary_path, core_path, bt_path, pbt_path):
+ config = _get_ya_config()
with _lock:
- if not hasattr(config, 'test_cores_count'):
- config.test_cores_count = 0
+ if not hasattr(config, 'test_cores_count'):
+ config.test_cores_count = 0
config.test_cores_count += 1
count_str = '' if config.test_cores_count == 1 else str(config.test_cores_count)
- log_entry = config.test_logs[config.current_item_nodeid]
- if binary_path:
- log_entry['{} binary{}'.format(name, count_str)] = binary_path
- if core_path:
- log_entry['{} core{}'.format(name, count_str)] = core_path
- if bt_path:
- log_entry['{} backtrace{}'.format(name, count_str)] = bt_path
- if pbt_path:
- log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path
-
-
-@not_test
+ log_entry = config.test_logs[config.current_item_nodeid]
+ if binary_path:
+ log_entry['{} binary{}'.format(name, count_str)] = binary_path
+ if core_path:
+ log_entry['{} core{}'.format(name, count_str)] = core_path
+ if bt_path:
+ log_entry['{} backtrace{}'.format(name, count_str)] = bt_path
+ if pbt_path:
+ log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path
+
+
+@not_test
def test_source_path(path=None):
return _join_path(os.path.join(source_path(), context.project_path), path)
diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py
index 39bbb45570..5ed678a457 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime_java.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py
@@ -2,11 +2,11 @@ import os
import tarfile
import contextlib
-from . import runtime
-
-_JAVA_DIR = []
-
+from . import runtime
+_JAVA_DIR = []
+
+
def get_java_path(jdk_dir):
# deprecated - to be deleted
java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe'))
@@ -30,17 +30,17 @@ def get_java_path(jdk_dir):
def get_build_java_dir(jdk_dir):
versions = [8, 10, 11, 12, 13, 14, 15]
- if not _JAVA_DIR:
- for version in versions:
- jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version))
- if os.path.exists(jdk_tar_path):
- jdk_dir = runtime.build_path('jdk4test')
- with contextlib.closing(tarfile.open(jdk_tar_path)) as tf:
- tf.extractall(jdk_dir)
- assert os.path.exists(os.path.join(jdk_dir, "bin", "java"))
- _JAVA_DIR.append(jdk_dir)
- break
- else:
- _JAVA_DIR.append(None)
-
- return _JAVA_DIR[0]
+ if not _JAVA_DIR:
+ for version in versions:
+ jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version))
+ if os.path.exists(jdk_tar_path):
+ jdk_dir = runtime.build_path('jdk4test')
+ with contextlib.closing(tarfile.open(jdk_tar_path)) as tf:
+ tf.extractall(jdk_dir)
+ assert os.path.exists(os.path.join(jdk_dir, "bin", "java"))
+ _JAVA_DIR.append(jdk_dir)
+ break
+ else:
+ _JAVA_DIR.append(None)
+
+ return _JAVA_DIR[0]
diff --git a/library/python/testing/yatest_common/yatest/common/ya.make b/library/python/testing/yatest_common/yatest/common/ya.make
index f7c50dfe64..fe4ea86d2d 100644
--- a/library/python/testing/yatest_common/yatest/common/ya.make
+++ b/library/python/testing/yatest_common/yatest/common/ya.make
@@ -1 +1 @@
-OWNER(g:yatest)
+OWNER(g:yatest)
diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py
index 39113230d9..69874dece4 100644
--- a/library/python/testing/yatest_lib/external.py
+++ b/library/python/testing/yatest_lib/external.py
@@ -1,20 +1,20 @@
from __future__ import absolute_import
-import re
+import re
import sys
import copy
-import logging
+import logging
from . import tools
from datetime import date, datetime
-import enum
+import enum
import six
-logger = logging.getLogger(__name__)
-MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/'
-
+logger = logging.getLogger(__name__)
+MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/'
+
def apply(func, value, apply_to_keys=False):
"""
Applies func to every possible member of value
@@ -67,8 +67,8 @@ def serialize(value):
return val
if isinstance(val, six.string_types) or isinstance(val, bytes):
return tools.to_utf8(val)
- if isinstance(val, enum.Enum):
- return str(val)
+ if isinstance(val, enum.Enum):
+ return str(val)
if isinstance(val, six.integer_types) or type(val) in [float, bool]:
return val
if is_external(val):
@@ -136,19 +136,19 @@ class ExternalDataInfo(object):
@property
def path(self):
- if self.uri.count("://") != 1:
- logger.error("Invalid external data uri: '%s'", self.uri)
- return self.uri
+ if self.uri.count("://") != 1:
+ logger.error("Invalid external data uri: '%s'", self.uri)
+ return self.uri
_, path = self.uri.split("://")
return path
- def get_mds_key(self):
- assert self.is_http
- m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri)
- if m:
- return m.group(1)
- raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri))
-
+ def get_mds_key(self):
+ assert self.is_http
+ m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri)
+ if m:
+ return m.group(1)
+ raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri))
+
@property
def size(self):
return self._data.get("size")
diff --git a/library/python/testing/yatest_lib/test_splitter.py b/library/python/testing/yatest_lib/test_splitter.py
index acbcd4300e..bc7beba568 100644
--- a/library/python/testing/yatest_lib/test_splitter.py
+++ b/library/python/testing/yatest_lib/test_splitter.py
@@ -1,15 +1,15 @@
-# coding: utf-8
+# coding: utf-8
import collections
-def flatten_tests(test_classes):
+def flatten_tests(test_classes):
"""
>>> test_classes = {x: [x] for x in range(5)}
- >>> flatten_tests(test_classes)
+ >>> flatten_tests(test_classes)
[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]
>>> test_classes = {x: [x + 1, x + 2] for x in range(2)}
- >>> flatten_tests(test_classes)
+ >>> flatten_tests(test_classes)
[(0, 1), (0, 2), (1, 2), (1, 3)]
"""
tests = []
diff --git a/library/python/testing/yatest_lib/tests/test_external.py b/library/python/testing/yatest_lib/tests/test_external.py
index 18cb560b17..ea5ebf97a3 100644
--- a/library/python/testing/yatest_lib/tests/test_external.py
+++ b/library/python/testing/yatest_lib/tests/test_external.py
@@ -1,20 +1,20 @@
-import enum
-import pytest
-
-from yatest_lib import external
-
-
-class MyEnum(enum.Enum):
- VAL1 = 1
- VAL2 = 2
-
-
-@pytest.mark.parametrize("data, expected_val, expected_type", [
- ({}, {}, dict),
- (MyEnum.VAL1, "MyEnum.VAL1", str),
- ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict),
-])
-def test_serialize(data, expected_val, expected_type):
- data = external.serialize(data)
- assert expected_type == type(data), data
- assert expected_val == data
+import enum
+import pytest
+
+from yatest_lib import external
+
+
+class MyEnum(enum.Enum):
+ VAL1 = 1
+ VAL2 = 2
+
+
+@pytest.mark.parametrize("data, expected_val, expected_type", [
+ ({}, {}, dict),
+ (MyEnum.VAL1, "MyEnum.VAL1", str),
+ ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict),
+])
+def test_serialize(data, expected_val, expected_type):
+ data = external.serialize(data)
+ assert expected_type == type(data), data
+ assert expected_val == data
diff --git a/library/python/testing/yatest_lib/tests/ya.make b/library/python/testing/yatest_lib/tests/ya.make
index 8586c6ef7d..89396b733e 100644
--- a/library/python/testing/yatest_lib/tests/ya.make
+++ b/library/python/testing/yatest_lib/tests/ya.make
@@ -1,13 +1,13 @@
-OWNER(g:yatest)
+OWNER(g:yatest)
-PY23_TEST()
+PY23_TEST()
PEERDIR(
library/python/testing/yatest_lib
)
TEST_SRCS(
- test_external.py
+ test_external.py
test_testsplitter.py
)
diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make
index 342bae82ba..1b9d7aa8c2 100644
--- a/library/python/testing/yatest_lib/ya.make
+++ b/library/python/testing/yatest_lib/ya.make
@@ -1,4 +1,4 @@
-OWNER(g:yatest)
+OWNER(g:yatest)
PY23_LIBRARY()
@@ -15,12 +15,12 @@ PEERDIR(
contrib/python/six
)
-IF(PYTHON2)
- PEERDIR(
- contrib/python/enum34
- )
-ENDIF()
-
+IF(PYTHON2)
+ PEERDIR(
+ contrib/python/enum34
+ )
+ENDIF()
+
END()
RECURSE_FOR_TESTS(tests)