diff options
| author | Dmitry Kopylov <kopylovd@gmail.com> | 2022-02-10 16:48:18 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:18 +0300 |
| commit | 7230275728d34873cba1ba78bb68669b0c5faa31 (patch) | |
| tree | b222e5ac2e2e98872661c51ccceee5da0d291e13 /library/python | |
| parent | b2f5101486cc0de2e979c8ba9ada2109785bf5fd (diff) | |
| download | ydb-7230275728d34873cba1ba78bb68669b0c5faa31.tar.gz | |
Restoring authorship annotation for Dmitry Kopylov <kopylovd@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'library/python')
43 files changed, 2021 insertions, 2021 deletions
diff --git a/library/python/certifi/certifi/__init__.py b/library/python/certifi/certifi/__init__.py index b025fc3510..5270d206cd 100644 --- a/library/python/certifi/certifi/__init__.py +++ b/library/python/certifi/certifi/__init__.py @@ -4,7 +4,7 @@ if hasattr(ssl, 'builtin_cadata'): from .binary import where else: from .source import where - + __all__ = ['where', '__version__'] __version__ = '2020.04.05.2' diff --git a/library/python/certifi/certifi/binary.py b/library/python/certifi/certifi/binary.py index 7cd6c020d2..1050e733a3 100644 --- a/library/python/certifi/certifi/binary.py +++ b/library/python/certifi/certifi/binary.py @@ -1,6 +1,6 @@ import ssl - - + + def builtin_ca(): return None, None, ssl.builtin_cadata() diff --git a/library/python/filelock/__init__.py b/library/python/filelock/__init__.py index aea171410d..f81ff67f37 100644 --- a/library/python/filelock/__init__.py +++ b/library/python/filelock/__init__.py @@ -5,9 +5,9 @@ import sys import library.python.windows -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) + - def set_close_on_exec(stream): if library.python.windows.on_win(): library.python.windows.set_handle_information(stream, inherit=False) @@ -16,16 +16,16 @@ def set_close_on_exec(stream): fcntl.fcntl(stream, fcntl.F_SETFD, fcntl.FD_CLOEXEC) -class AbstractFileLock(object): - +class AbstractFileLock(object): + def __init__(self, path): self.path = path def acquire(self, blocking=True): - raise NotImplementedError + raise NotImplementedError def release(self): - raise NotImplementedError + raise NotImplementedError def __enter__(self): self.acquire() @@ -34,17 +34,17 @@ class AbstractFileLock(object): def __exit__(self, type, value, traceback): self.release() - -class _NixFileLock(AbstractFileLock): - - def __init__(self, path): - super(_NixFileLock, self).__init__(path) + +class _NixFileLock(AbstractFileLock): + + def __init__(self, path): + super(_NixFileLock, self).__init__(path) from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB self._locker = lambda lock, blocking: flock(lock, LOCK_EX if blocking else LOCK_EX | LOCK_NB) self._unlocker = lambda lock: flock(lock, LOCK_UN) self._lock = open(self.path, 'a') set_close_on_exec(self._lock) - + def acquire(self, blocking=True): import errno try: @@ -54,69 +54,69 @@ class _NixFileLock(AbstractFileLock): return False raise return True - - def release(self): + + def release(self): self._unlocker(self._lock) - + def __del__(self): if hasattr(self, "_lock"): self._lock.close() - - -class _WinFileLock(AbstractFileLock): - """ - Based on LockFile / UnlockFile from win32 API - https://msdn.microsoft.com/en-us/library/windows/desktop/aa365202(v=vs.85).aspx - """ - - _LOCKED_BYTES_NUM = 1 - - def __init__(self, path): - super(_WinFileLock, self).__init__(path) - self._lock = None + + +class _WinFileLock(AbstractFileLock): + """ + Based on LockFile / UnlockFile from win32 API + https://msdn.microsoft.com/en-us/library/windows/desktop/aa365202(v=vs.85).aspx + """ + + _LOCKED_BYTES_NUM = 1 + + def __init__(self, path): + super(_WinFileLock, self).__init__(path) + self._lock = None try: with file(path, 'w') as lock_file: lock_file.write(" " * self._LOCKED_BYTES_NUM) except IOError as e: if e.errno != errno.EACCES or not os.path.isfile(path): raise - + def acquire(self, blocking=True): - self._lock = open(self.path) + self._lock = open(self.path) set_close_on_exec(self._lock) - import time + import time locked = False - while not locked: + while not locked: locked = library.python.windows.lock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False) if locked: return True if blocking: - time.sleep(.5) + time.sleep(.5) else: return False - - def release(self): + + def release(self): if self._lock: library.python.windows.unlock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False) self._lock.close() self._lock = None - - -class FileLock(AbstractFileLock): - - def __init__(self, path): - super(FileLock, self).__init__(path) - - if sys.platform.startswith('win'): - self._lock = _WinFileLock(path) - else: - self._lock = _NixFileLock(path) - + + +class FileLock(AbstractFileLock): + + def __init__(self, path): + super(FileLock, self).__init__(path) + + if sys.platform.startswith('win'): + self._lock = _WinFileLock(path) + else: + self._lock = _NixFileLock(path) + def acquire(self, blocking=True): logger.debug('Acquiring filelock (blocking=%s): %s', blocking, self.path) return self._lock.acquire(blocking) - - def release(self): - logger.debug('Ensuring filelock released: %s', self.path) - return self._lock.release() + + def release(self): + logger.debug('Ensuring filelock released: %s', self.path) + return self._lock.release() diff --git a/library/python/filelock/ut/lib/test_filelock.py b/library/python/filelock/ut/lib/test_filelock.py index 4c985022d4..1b11d89123 100644 --- a/library/python/filelock/ut/lib/test_filelock.py +++ b/library/python/filelock/ut/lib/test_filelock.py @@ -1,13 +1,13 @@ -import os -import time -import logging -import multiprocessing +import os +import time +import logging +import multiprocessing import tempfile import threading - + import library.python.filelock - - + + def _acquire_lock(lock_path, out_file_path): with library.python.filelock.FileLock(lock_path): with open(out_file_path, "a") as out: @@ -15,21 +15,21 @@ def _acquire_lock(lock_path, out_file_path): time.sleep(2) -def test_filelock(): +def test_filelock(): temp_dir = tempfile.mkdtemp() lock_path = os.path.join(temp_dir, "file.lock") out_file_path = os.path.join(temp_dir, "out.txt") - + process_count = 5 processes = [] for i in range(process_count): process = multiprocessing.Process(target=_acquire_lock, args=(lock_path, out_file_path)) process.start() processes.append(process) - + for process in processes: process.join() - + pids = [] times = [] with open(out_file_path) as out: @@ -39,7 +39,7 @@ def test_filelock(): pid, time_val = line.split(":") pids.append(pid) times.append(float(time_val)) - + assert len(set(pids)) == process_count time1 = times.pop() while times: diff --git a/library/python/find_root/__init__.py b/library/python/find_root/__init__.py index a53252c6e7..6da604d62e 100644 --- a/library/python/find_root/__init__.py +++ b/library/python/find_root/__init__.py @@ -1,20 +1,20 @@ -import os - - -def is_root(path): +import os + + +def is_root(path): return os.path.exists(os.path.join(path, ".arcadia.root")) or os.path.exists(os.path.join(path, 'devtools', 'ya', 'ya.conf.json')) - - -def detect_root(path, detector=is_root): - return _find_path(path, detector) - - -def _find_path(starts_from, check): - p = os.path.realpath(starts_from) - while True: - if check(p): - return p - next_p = os.path.dirname(p) - if next_p == p: - return None - p = next_p + + +def detect_root(path, detector=is_root): + return _find_path(path, detector) + + +def _find_path(starts_from, check): + p = os.path.realpath(starts_from) + while True: + if check(p): + return p + next_p = os.path.dirname(p) + if next_p == p: + return None + p = next_p diff --git a/library/python/find_root/ya.make b/library/python/find_root/ya.make index a672093d96..beaa8e3c52 100644 --- a/library/python/find_root/ya.make +++ b/library/python/find_root/ya.make @@ -1,7 +1,7 @@ PY23_LIBRARY() - + OWNER(g:yatool) - + PY_SRCS(__init__.py) - -END() + +END() diff --git a/library/python/fs/__init__.py b/library/python/fs/__init__.py index 06dced90d8..b1b7cde079 100644 --- a/library/python/fs/__init__.py +++ b/library/python/fs/__init__.py @@ -14,15 +14,15 @@ import library.python.func import library.python.strings import library.python.windows -logger = logging.getLogger(__name__) - - -try: - WindowsError -except NameError: - WindowsError = None - - +logger = logging.getLogger(__name__) + + +try: + WindowsError +except NameError: + WindowsError = None + + _diehard_win_tries = 10 errorfix_win = library.python.windows.errorfix @@ -127,10 +127,10 @@ def remove_dir(path): os.rmdir(path) -def fix_path_encoding(path): +def fix_path_encoding(path): return library.python.strings.to_str(path, library.python.strings.fs_encoding()) - - + + # File/directory remove # Non-atomic # Throws OSError, AssertionError @@ -140,8 +140,8 @@ def remove_tree(path): def rmtree(path): if library.python.windows.on_win(): library.python.windows.rmtree(path) - else: - shutil.rmtree(fix_path_encoding(path)) + else: + shutil.rmtree(fix_path_encoding(path)) st = os.lstat(path) if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode): @@ -161,7 +161,7 @@ def remove_tree_safe(path): if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode): os.remove(path) elif stat.S_ISDIR(st.st_mode): - shutil.rmtree(fix_path_encoding(path), ignore_errors=True) + shutil.rmtree(fix_path_encoding(path), ignore_errors=True) # XXX except UnicodeDecodeError as e: logging.exception(u'remove_tree_safe with argument %s raise exception: %s', path, e) @@ -196,11 +196,11 @@ def hardlink(src, lnk): os.link(src, lnk) -@errorfix_win -def hardlink_or_copy(src, lnk): - def should_fallback_to_copy(exc): - if WindowsError is not None and isinstance(exc, WindowsError) and exc.winerror == 1142: # too many hardlinks - return True +@errorfix_win +def hardlink_or_copy(src, lnk): + def should_fallback_to_copy(exc): + if WindowsError is not None and isinstance(exc, WindowsError) and exc.winerror == 1142: # too many hardlinks + return True # cross-device hardlink or too many hardlinks, or some known WSL error if isinstance(exc, OSError) and exc.errno in ( errno.EXDEV, @@ -209,19 +209,19 @@ def hardlink_or_copy(src, lnk): errno.EACCES, errno.EPERM, ): - return True - return False - - try: - hardlink(src, lnk) - except Exception as e: + return True + return False + + try: + hardlink(src, lnk) + except Exception as e: logger.debug('Failed to hardlink %s to %s with error %s, will copy it', src, lnk, repr(e)) - if should_fallback_to_copy(e): + if should_fallback_to_copy(e): copy2(src, lnk, follow_symlinks=False) - else: - raise - - + else: + raise + + # Atomic file/directory symlink (Unix only) # Dst must not exist # Throws OSError @@ -259,7 +259,7 @@ def hardlink_tree(src, dst): if os.path.isfile(src): hardlink(src, dst) return - for dirpath, _, filenames in walk_relative(src): + for dirpath, _, filenames in walk_relative(src): src_dirpath = os.path.join(src, dirpath) if dirpath != '.' else src dst_dirpath = os.path.join(dst, dirpath) if dirpath != '.' else dst os.mkdir(dst_dirpath) @@ -270,20 +270,20 @@ def hardlink_tree(src, dst): # File copy # throws EnvironmentError (OSError, IOError) @errorfix_win -def copy_file(src, dst, copy_function=shutil.copy2): +def copy_file(src, dst, copy_function=shutil.copy2): if os.path.isdir(dst): raise CustomFsError(errno.EISDIR, filename=dst) - copy_function(src, dst) + copy_function(src, dst) # File/directory copy # throws EnvironmentError (OSError, IOError, shutil.Error) @errorfix_win -def copy_tree(src, dst, copy_function=shutil.copy2): +def copy_tree(src, dst, copy_function=shutil.copy2): if os.path.isfile(src): - copy_file(src, dst, copy_function=copy_function) + copy_file(src, dst, copy_function=copy_function) return - copytree3(src, dst, copy_function=copy_function) + copytree3(src, dst, copy_function=copy_function) # File read @@ -356,7 +356,7 @@ def get_tree_size(path, recursive=False, raise_all_errors=False): except OSError as e: if raise_all_errors: raise - logger.debug("Cannot calculate file size: %s", e) + logger.debug("Cannot calculate file size: %s", e) if not recursive: break return total_size @@ -447,25 +447,25 @@ def copytree3( copy_function(srcname, dstname) # catch the Error from the recursive copytree3 so that we can # continue with other files - except shutil.Error as err: + except shutil.Error as err: errors.extend(err.args[0]) - except EnvironmentError as why: + except EnvironmentError as why: errors.append((srcname, dstname, str(why))) try: shutil.copystat(src, dst) - except OSError as why: + except OSError as why: if WindowsError is not None and isinstance(why, WindowsError): # Copying file access times may fail on Windows pass else: errors.extend((src, dst, str(why))) if errors: - raise shutil.Error(errors) - - -def walk_relative(path, topdown=True, onerror=None, followlinks=False): - for dirpath, dirnames, filenames in os.walk(path, topdown=topdown, onerror=onerror, followlinks=followlinks): - yield os.path.relpath(dirpath, path), dirnames, filenames + raise shutil.Error(errors) + + +def walk_relative(path, topdown=True, onerror=None, followlinks=False): + for dirpath, dirnames, filenames in os.walk(path, topdown=topdown, onerror=onerror, followlinks=followlinks): + yield os.path.relpath(dirpath, path), dirnames, filenames def supports_clone(): diff --git a/library/python/fs/test/test_fs.py b/library/python/fs/test/test_fs.py index 6bbd761c04..9e2c70c069 100644 --- a/library/python/fs/test/test_fs.py +++ b/library/python/fs/test/test_fs.py @@ -3,20 +3,20 @@ import errno import os import pytest -import shutil +import shutil import six -import library.python.fs +import library.python.fs import library.python.strings -import library.python.tmp +import library.python.tmp import library.python.windows -import yatest.common +import yatest.common + - def in_env(case): def wrapped_case(*args, **kwargs): - with library.python.tmp.temp_dir() as temp_dir: + with library.python.tmp.temp_dir() as temp_dir: case(lambda path: os.path.join(temp_dir, path)) return wrapped_case @@ -84,7 +84,7 @@ def gen_error_access_denied(): def test_errorfix_win(): - @library.python.fs.errorfix_win + @library.python.fs.errorfix_win def erroneous_func(): gen_error_access_denied() @@ -99,7 +99,7 @@ def test_errorfix_win(): def test_custom_fs_error(): with pytest.raises(OSError) as errinfo: - raise library.python.fs.CustomFsError(errno.EACCES, filename='some/file') + raise library.python.fs.CustomFsError(errno.EACCES, filename='some/file') assert errinfo.value.errno == errno.EACCES # See transcode_error, which encodes strerror, in library/python/windows/__init__.py assert isinstance(errinfo.value.strerror, (six.binary_type, six.text_type)) @@ -108,7 +108,7 @@ def test_custom_fs_error(): @in_env def test_ensure_dir(path): - library.python.fs.ensure_dir(path('dir/subdir')) + library.python.fs.ensure_dir(path('dir/subdir')) assert os.path.isdir(path('dir')) assert os.path.isdir(path('dir/subdir')) @@ -116,7 +116,7 @@ def test_ensure_dir(path): @in_env def test_ensure_dir_exists(path): os.makedirs(path('dir/subdir')) - library.python.fs.ensure_dir(path('dir/subdir')) + library.python.fs.ensure_dir(path('dir/subdir')) assert os.path.isdir(path('dir')) assert os.path.isdir(path('dir/subdir')) @@ -124,7 +124,7 @@ def test_ensure_dir_exists(path): @in_env def test_ensure_dir_exists_partly(path): os.mkdir(path('dir')) - library.python.fs.ensure_dir(path('dir/subdir')) + library.python.fs.ensure_dir(path('dir/subdir')) assert os.path.isdir(path('dir')) assert os.path.isdir(path('dir/subdir')) @@ -133,7 +133,7 @@ def test_ensure_dir_exists_partly(path): def test_ensure_dir_exists_file(path): mkfile(path('dir')) with pytest.raises(OSError) as errinfo: - library.python.fs.ensure_dir(path('dir/subdir')) + library.python.fs.ensure_dir(path('dir/subdir')) # ENOENT on Windows! assert errinfo.value.errno in (errno.ENOTDIR, errno.ENOENT) assert os.path.isfile(path('dir')) @@ -141,7 +141,7 @@ def test_ensure_dir_exists_file(path): @in_env def test_create_dirs(path): - assert library.python.fs.create_dirs(path('dir/subdir')) == path('dir/subdir') + assert library.python.fs.create_dirs(path('dir/subdir')) == path('dir/subdir') assert os.path.isdir(path('dir')) assert os.path.isdir(path('dir/subdir')) @@ -149,7 +149,7 @@ def test_create_dirs(path): @in_env def test_move_file(path): mkfile(path('src'), 'SRC') - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -158,7 +158,7 @@ def test_move_file(path): @in_env def test_move_file_no_src(path): with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno == errno.ENOENT @@ -169,13 +169,13 @@ def test_move_file_exists(path): if library.python.windows.on_win(): # move is platform-dependent, use replace_file for dst replacement on all platforms with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'DST' else: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -186,7 +186,7 @@ def test_move_file_exists_dir_empty(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) @@ -199,7 +199,7 @@ def test_move_file_exists_dir_nonempty(path): os.mkdir(path('dst')) mkfile(path('dst/dst_file')) with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) @@ -211,7 +211,7 @@ def test_move_file_exists_dir_nonempty(path): def test_move_dir(path): os.mkdir(path('src')) mkfile(path('src/src_file')) - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -225,13 +225,13 @@ def test_move_dir_exists_empty(path): if library.python.windows.on_win(): # move is platform-dependent, use non-atomic replace for directory replacement with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert not os.path.isfile(path('dst/src_file')) else: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -244,7 +244,7 @@ def test_move_dir_exists_nonempty(path): os.mkdir(path('dst')) mkfile(path('dst/dst_file')) with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno in (errno.EEXIST, errno.ENOTEMPTY) assert os.path.isdir(path('src')) assert os.path.isfile(path('src/src_file')) @@ -259,7 +259,7 @@ def test_move_dir_exists_file(path): mkfile(path('src/src_file')) mkfile(path('dst'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.move(path('src'), path('dst')) + library.python.fs.move(path('src'), path('dst')) assert errinfo.value.errno in (errno.EEXIST, errno.ENOTDIR) assert os.path.isdir(path('src')) assert os.path.isfile(path('dst')) @@ -269,13 +269,13 @@ def test_move_dir_exists_file(path): @in_env def test_replace_file(path): mkfile(path('src'), 'SRC') - library.python.fs.replace_file(path('src'), path('dst')) + library.python.fs.replace_file(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' mkfile(path('src'), 'SRC') - library.python.fs.replace(path('src'), path('dst2')) + library.python.fs.replace(path('src'), path('dst2')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst2')) assert file_data(path('dst2')) == 'SRC' @@ -284,11 +284,11 @@ def test_replace_file(path): @in_env def test_replace_file_no_src(path): with pytest.raises(OSError) as errinfo: - library.python.fs.replace_file(path('src'), path('dst')) + library.python.fs.replace_file(path('src'), path('dst')) assert errinfo.value.errno == errno.ENOENT with pytest.raises(OSError) as errinfo2: - library.python.fs.replace(path('src'), path('dst2')) + library.python.fs.replace(path('src'), path('dst2')) assert errinfo2.value.errno == errno.ENOENT @@ -296,14 +296,14 @@ def test_replace_file_no_src(path): def test_replace_file_exists(path): mkfile(path('src'), 'SRC') mkfile(path('dst'), 'DST') - library.python.fs.replace_file(path('src'), path('dst')) + library.python.fs.replace_file(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' mkfile(path('src'), 'SRC') mkfile(path('dst2'), 'DST') - library.python.fs.replace(path('src'), path('dst2')) + library.python.fs.replace(path('src'), path('dst2')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst2')) assert file_data(path('dst2')) == 'SRC' @@ -314,7 +314,7 @@ def test_replace_file_exists_dir_empty(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) with pytest.raises(OSError) as errinfo: - library.python.fs.replace_file(path('src'), path('dst')) + library.python.fs.replace_file(path('src'), path('dst')) assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) @@ -325,7 +325,7 @@ def test_replace_file_exists_dir_empty(path): def test_replace_file_exists_dir_empty_overwrite(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -337,20 +337,20 @@ def test_replace_file_exists_dir_nonempty(path): os.mkdir(path('dst')) mkfile(path('dst/dst_file')) with pytest.raises(OSError) as errinfo: - library.python.fs.replace_file(path('src'), path('dst')) + library.python.fs.replace_file(path('src'), path('dst')) assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/dst_file')) assert not os.path.isfile(path('dst/src')) - - + + @in_env def test_replace_file_exists_dir_nonempty_overwrite(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) mkfile(path('dst/dst_file')) - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -360,7 +360,7 @@ def test_replace_file_exists_dir_nonempty_overwrite(path): def test_replace_dir(path): os.mkdir(path('src')) mkfile(path('src/src_file')) - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -371,7 +371,7 @@ def test_replace_dir_exists_empty(path): os.mkdir(path('src')) mkfile(path('src/src_file')) os.mkdir(path('dst')) - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -383,7 +383,7 @@ def test_replace_dir_exists_nonempty(path): mkfile(path('src/src_file')) os.mkdir(path('dst')) mkfile(path('dst/dst_file')) - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -395,7 +395,7 @@ def test_replace_dir_exists_file(path): os.mkdir(path('src')) mkfile(path('src/src_file')) mkfile(path('dst'), 'DST') - library.python.fs.replace(path('src'), path('dst')) + library.python.fs.replace(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/src_file')) @@ -404,14 +404,14 @@ def test_replace_dir_exists_file(path): @in_env def test_remove_file(path): mkfile(path('path')) - library.python.fs.remove_file(path('path')) + library.python.fs.remove_file(path('path')) assert not os.path.exists(path('path')) @in_env def test_remove_file_no(path): with pytest.raises(OSError) as errinfo: - library.python.fs.remove_file(path('path')) + library.python.fs.remove_file(path('path')) assert errinfo.value.errno == errno.ENOENT @@ -419,7 +419,7 @@ def test_remove_file_no(path): def test_remove_file_exists_dir(path): os.mkdir(path('path')) with pytest.raises(OSError) as errinfo: - library.python.fs.remove_file(path('path')) + library.python.fs.remove_file(path('path')) assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) assert os.path.isdir(path('path')) @@ -427,14 +427,14 @@ def test_remove_file_exists_dir(path): @in_env def test_remove_dir(path): os.mkdir(path('path')) - library.python.fs.remove_dir(path('path')) + library.python.fs.remove_dir(path('path')) assert not os.path.exists(path('path')) @in_env def test_remove_dir_no(path): with pytest.raises(OSError) as errinfo: - library.python.fs.remove_dir(path('path')) + library.python.fs.remove_dir(path('path')) assert errinfo.value.errno == errno.ENOENT @@ -442,7 +442,7 @@ def test_remove_dir_no(path): def test_remove_dir_exists_file(path): mkfile(path('path')) with pytest.raises(OSError) as errinfo: - library.python.fs.remove_dir(path('path')) + library.python.fs.remove_dir(path('path')) assert errinfo.value.errno in (errno.ENOTDIR, errno.EINVAL) assert os.path.isfile(path('path')) @@ -450,52 +450,52 @@ def test_remove_dir_exists_file(path): @in_env def test_remove_tree(path): mktree_example(path, 'path') - library.python.fs.remove_tree(path('path')) + library.python.fs.remove_tree(path('path')) assert not os.path.exists(path('path')) @in_env def test_remove_tree_empty(path): os.mkdir(path('path')) - library.python.fs.remove_tree(path('path')) + library.python.fs.remove_tree(path('path')) assert not os.path.exists(path('path')) @in_env def test_remove_tree_file(path): mkfile(path('path')) - library.python.fs.remove_tree(path('path')) + library.python.fs.remove_tree(path('path')) assert not os.path.exists(path('path')) @in_env def test_remove_tree_no(path): with pytest.raises(OSError) as errinfo: - library.python.fs.remove_tree(path('path')) + library.python.fs.remove_tree(path('path')) assert errinfo.value.errno == errno.ENOENT @in_env def test_remove_tree_safe(path): - library.python.fs.remove_tree_safe(path('path')) + library.python.fs.remove_tree_safe(path('path')) @in_env def test_ensure_removed(path): - library.python.fs.ensure_removed(path('path')) + library.python.fs.ensure_removed(path('path')) @in_env def test_ensure_removed_exists(path): os.makedirs(path('dir/subdir')) - library.python.fs.ensure_removed(path('dir')) + library.python.fs.ensure_removed(path('dir')) assert not os.path.exists(path('dir')) @in_env def test_ensure_removed_exists_precise(path): os.makedirs(path('dir/subdir')) - library.python.fs.ensure_removed(path('dir/subdir')) + library.python.fs.ensure_removed(path('dir/subdir')) assert os.path.exists(path('dir')) assert not os.path.exists(path('dir/subdir')) @@ -503,7 +503,7 @@ def test_ensure_removed_exists_precise(path): @in_env def test_hardlink_file(path): mkfile(path('src'), 'SRC') - library.python.fs.hardlink(path('src'), path('dst')) + library.python.fs.hardlink(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -513,7 +513,7 @@ def test_hardlink_file(path): @in_env def test_hardlink_file_no_src(path): with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink(path('src'), path('dst')) + library.python.fs.hardlink(path('src'), path('dst')) assert errinfo.value.errno == errno.ENOENT @@ -522,7 +522,7 @@ def test_hardlink_file_exists(path): mkfile(path('src'), 'SRC') mkfile(path('dst'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink(path('src'), path('dst')) + library.python.fs.hardlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) @@ -535,7 +535,7 @@ def test_hardlink_file_exists_dir(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink(path('src'), path('dst')) + library.python.fs.hardlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) @@ -547,7 +547,7 @@ def test_hardlink_dir(path): os.mkdir(path('src')) mkfile(path('src/src_file')) with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink(path('src'), path('dst')) + library.python.fs.hardlink(path('src'), path('dst')) assert errinfo.value.errno in (errno.EPERM, errno.EACCES) assert os.path.isdir(path('src')) assert not os.path.isdir(path('dst')) @@ -557,7 +557,7 @@ def test_hardlink_dir(path): @in_env def test_symlink_file(path): mkfile(path('src'), 'SRC') - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert os.path.islink(path('dst')) @@ -567,7 +567,7 @@ def test_symlink_file(path): @pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') @in_env def test_symlink_file_no_src(path): - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert not os.path.isfile(path('src')) assert not os.path.isfile(path('dst')) assert os.path.islink(path('dst')) @@ -579,7 +579,7 @@ def test_symlink_file_exists(path): mkfile(path('src'), 'SRC') mkfile(path('dst'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) @@ -593,7 +593,7 @@ def test_symlink_file_exists_dir(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) with pytest.raises(OSError) as errinfo: - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) @@ -606,7 +606,7 @@ def test_symlink_file_exists_dir(path): def test_symlink_dir(path): os.mkdir(path('src')) mkfile(path('src/src_file')) - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert os.path.isdir(path('src')) assert os.path.isdir(path('dst')) assert os.path.islink(path('dst')) @@ -616,7 +616,7 @@ def test_symlink_dir(path): @pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') @in_env def test_symlink_dir_no_src(path): - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert not os.path.isdir(path('src')) assert not os.path.isdir(path('dst')) assert os.path.islink(path('dst')) @@ -629,7 +629,7 @@ def test_symlink_dir_exists(path): mkfile(path('src/src_file')) os.mkdir(path('dst')) with pytest.raises(OSError) as errinfo: - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isdir(path('src')) assert os.path.isdir(path('dst')) @@ -644,7 +644,7 @@ def test_symlink_dir_exists_file(path): mkfile(path('src/src_file')) mkfile(path('dst'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.symlink(path('src'), path('dst')) + library.python.fs.symlink(path('src'), path('dst')) assert errinfo.value.errno == errno.EEXIST assert os.path.isdir(path('src')) assert os.path.isfile(path('dst')) @@ -654,28 +654,28 @@ def test_symlink_dir_exists_file(path): @in_env def test_hardlink_tree(path): mktree_example(path, 'src') - library.python.fs.hardlink_tree(path('src'), path('dst')) + library.python.fs.hardlink_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_hardlink_tree_empty(path): os.mkdir(path('src')) - library.python.fs.hardlink_tree(path('src'), path('dst')) + library.python.fs.hardlink_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_hardlink_tree_file(path): mkfile(path('src'), 'SRC') - library.python.fs.hardlink_tree(path('src'), path('dst')) + library.python.fs.hardlink_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_hardlink_tree_no_src(path): with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink_tree(path('src'), path('dst')) + library.python.fs.hardlink_tree(path('src'), path('dst')) assert errinfo.value.errno == errno.ENOENT @@ -684,11 +684,11 @@ def test_hardlink_tree_exists(path): mktree_example(path, 'src') os.mkdir(path('dst_dir')) with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink_tree(path('src'), path('dst_dir')) + library.python.fs.hardlink_tree(path('src'), path('dst_dir')) assert errinfo.value.errno == errno.EEXIST mkfile(path('dst_file'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink_tree(path('src'), path('dst_file')) + library.python.fs.hardlink_tree(path('src'), path('dst_file')) assert errinfo.value.errno == errno.EEXIST @@ -697,18 +697,18 @@ def test_hardlink_tree_file_exists(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst_dir')) with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink_tree(path('src'), path('dst_dir')) + library.python.fs.hardlink_tree(path('src'), path('dst_dir')) assert errinfo.value.errno == errno.EEXIST mkfile(path('dst_file'), 'DST') with pytest.raises(OSError) as errinfo: - library.python.fs.hardlink_tree(path('src'), path('dst_file')) + library.python.fs.hardlink_tree(path('src'), path('dst_file')) assert errinfo.value.errno == errno.EEXIST @in_env def test_copy_file(path): mkfile(path('src'), 'SRC') - library.python.fs.copy_file(path('src'), path('dst')) + library.python.fs.copy_file(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -717,14 +717,14 @@ def test_copy_file(path): @in_env def test_copy_file_no_src(path): with pytest.raises(EnvironmentError): - library.python.fs.copy_file(path('src'), path('dst')) + library.python.fs.copy_file(path('src'), path('dst')) @in_env def test_copy_file_exists(path): mkfile(path('src'), 'SRC') mkfile(path('dst'), 'DST') - library.python.fs.copy_file(path('src'), path('dst')) + library.python.fs.copy_file(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isfile(path('dst')) assert file_data(path('dst')) == 'SRC' @@ -735,7 +735,7 @@ def test_copy_file_exists_dir_empty(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst')) with pytest.raises(EnvironmentError): - library.python.fs.copy_file(path('src'), path('dst')) + library.python.fs.copy_file(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) assert not os.path.isfile(path('dst/src')) @@ -747,7 +747,7 @@ def test_copy_file_exists_dir_nonempty(path): os.mkdir(path('dst')) mkfile(path('dst/dst_file')) with pytest.raises(EnvironmentError): - library.python.fs.copy_file(path('src'), path('dst')) + library.python.fs.copy_file(path('src'), path('dst')) assert os.path.isfile(path('src')) assert os.path.isdir(path('dst')) assert os.path.isfile(path('dst/dst_file')) @@ -757,28 +757,28 @@ def test_copy_file_exists_dir_nonempty(path): @in_env def test_copy_tree(path): mktree_example(path, 'src') - library.python.fs.copy_tree(path('src'), path('dst')) + library.python.fs.copy_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_copy_tree_empty(path): os.mkdir(path('src')) - library.python.fs.copy_tree(path('src'), path('dst')) + library.python.fs.copy_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_copy_tree_file(path): mkfile(path('src'), 'SRC') - library.python.fs.copy_tree(path('src'), path('dst')) + library.python.fs.copy_tree(path('src'), path('dst')) assert trees_equal(path('src'), path('dst')) @in_env def test_copy_tree_no_src(path): with pytest.raises(EnvironmentError): - library.python.fs.copy_tree(path('src'), path('dst')) + library.python.fs.copy_tree(path('src'), path('dst')) @in_env @@ -786,10 +786,10 @@ def test_copy_tree_exists(path): mktree_example(path, 'src') os.mkdir(path('dst_dir')) with pytest.raises(EnvironmentError): - library.python.fs.copy_tree(path('src'), path('dst_dir')) + library.python.fs.copy_tree(path('src'), path('dst_dir')) mkfile(path('dst_file'), 'DST') with pytest.raises(EnvironmentError): - library.python.fs.copy_tree(path('src'), path('dst_file')) + library.python.fs.copy_tree(path('src'), path('dst_file')) @in_env @@ -797,9 +797,9 @@ def test_copy_tree_file_exists(path): mkfile(path('src'), 'SRC') os.mkdir(path('dst_dir')) with pytest.raises(EnvironmentError): - library.python.fs.copy_tree(path('src'), path('dst_dir')) + library.python.fs.copy_tree(path('src'), path('dst_dir')) mkfile(path('dst_file'), 'DST') - library.python.fs.copy_tree(path('src'), path('dst_file')) + library.python.fs.copy_tree(path('src'), path('dst_file')) assert trees_equal(path('src'), path('dst_file')) @@ -807,14 +807,14 @@ def test_copy_tree_file_exists(path): def test_read_file(path): mkfile(path('src'), 'SRC') assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == 'SRC' - assert library.python.fs.read_file(path('src'), binary=False) == 'SRC' + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC' @in_env def test_read_file_empty(path): mkfile(path('src')) assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == '' - assert library.python.fs.read_file(path('src'), binary=False) == '' + assert library.python.fs.read_file(path('src'), binary=False) == '' @in_env @@ -824,7 +824,7 @@ def test_read_file_multiline(path): library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == 'SRC line 1\nSRC line 2\n' ) - assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' @in_env @@ -835,9 +835,9 @@ def test_read_file_multiline_crlf(path): == 'SRC line 1\r\nSRC line 2\r\n' ) if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3 - assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' else: - assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\r\nSRC line 2\r\n' + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\r\nSRC line 2\r\n' @in_env @@ -845,20 +845,20 @@ def test_read_file_unicode(path): s = u'АБВ' mkfile(path('src'), s.encode('utf-8')) mkfile(path('src_cp1251'), s.encode('cp1251')) - assert library.python.fs.read_file_unicode(path('src')) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s - assert library.python.fs.read_file_unicode(path('src'), binary=False) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s @in_env def test_read_file_unicode_empty(path): mkfile(path('src')) mkfile(path('src_cp1251')) - assert library.python.fs.read_file_unicode(path('src')) == '' - assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == '' - assert library.python.fs.read_file_unicode(path('src'), binary=False) == '' - assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == '' + assert library.python.fs.read_file_unicode(path('src')) == '' + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == '' + assert library.python.fs.read_file_unicode(path('src'), binary=False) == '' + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == '' @in_env @@ -866,10 +866,10 @@ def test_read_file_unicode_multiline(path): s = u'АБВ\nИ еще\n' mkfile(path('src'), s.encode('utf-8')) mkfile(path('src_cp1251'), s.encode('cp1251')) - assert library.python.fs.read_file_unicode(path('src')) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s - assert library.python.fs.read_file_unicode(path('src'), binary=False) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s @in_env @@ -877,37 +877,37 @@ def test_read_file_unicode_multiline_crlf(path): s = u'АБВ\r\nИ еще\r\n' mkfile(path('src'), s.encode('utf-8')) mkfile(path('src_cp1251'), s.encode('cp1251')) - assert library.python.fs.read_file_unicode(path('src')) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3 - assert library.python.fs.read_file_unicode(path('src'), binary=False) == u'АБВ\nИ еще\n' - assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == u'АБВ\nИ еще\n' + assert library.python.fs.read_file_unicode(path('src'), binary=False) == u'АБВ\nИ еще\n' + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == u'АБВ\nИ еще\n' else: - assert library.python.fs.read_file_unicode(path('src'), binary=False) == s - assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s @in_env def test_write_file(path): - library.python.fs.write_file(path('src'), 'SRC') + library.python.fs.write_file(path('src'), 'SRC') assert file_data(path('src')) == 'SRC' - library.python.fs.write_file(path('src2'), 'SRC', binary=False) + library.python.fs.write_file(path('src2'), 'SRC', binary=False) assert file_data(path('src2')) == 'SRC' @in_env def test_write_file_empty(path): - library.python.fs.write_file(path('src'), '') + library.python.fs.write_file(path('src'), '') assert file_data(path('src')) == '' - library.python.fs.write_file(path('src2'), '', binary=False) + library.python.fs.write_file(path('src2'), '', binary=False) assert file_data(path('src2')) == '' @in_env def test_write_file_multiline(path): - library.python.fs.write_file(path('src'), 'SRC line 1\nSRC line 2\n') + library.python.fs.write_file(path('src'), 'SRC line 1\nSRC line 2\n') assert file_data(path('src')) == 'SRC line 1\nSRC line 2\n' - library.python.fs.write_file(path('src2'), 'SRC line 1\nSRC line 2\n', binary=False) + library.python.fs.write_file(path('src2'), 'SRC line 1\nSRC line 2\n', binary=False) if library.python.windows.on_win(): assert file_data(path('src2')) == 'SRC line 1\r\nSRC line 2\r\n' else: @@ -916,9 +916,9 @@ def test_write_file_multiline(path): @in_env def test_write_file_multiline_crlf(path): - library.python.fs.write_file(path('src'), 'SRC line 1\r\nSRC line 2\r\n') + library.python.fs.write_file(path('src'), 'SRC line 1\r\nSRC line 2\r\n') assert file_data(path('src')) == 'SRC line 1\r\nSRC line 2\r\n' - library.python.fs.write_file(path('src2'), 'SRC line 1\r\nSRC line 2\r\n', binary=False) + library.python.fs.write_file(path('src2'), 'SRC line 1\r\nSRC line 2\r\n', binary=False) if library.python.windows.on_win(): assert file_data(path('src2')) == 'SRC line 1\r\r\nSRC line 2\r\r\n' else: @@ -928,13 +928,13 @@ def test_write_file_multiline_crlf(path): @in_env def test_get_file_size(path): mkfile(path('one.txt'), '22') - assert library.python.fs.get_file_size(path('one.txt')) == 2 - - + assert library.python.fs.get_file_size(path('one.txt')) == 2 + + @in_env def test_get_file_size_empty(path): mkfile(path('one.txt')) - assert library.python.fs.get_file_size(path('one.txt')) == 0 + assert library.python.fs.get_file_size(path('one.txt')) == 0 @in_env @@ -942,9 +942,9 @@ def test_get_tree_size(path): os.makedirs(path('deeper')) mkfile(path('one.txt'), '1') mkfile(path('deeper/two.txt'), '22') - assert library.python.fs.get_tree_size(path('one.txt')) == 1 - assert library.python.fs.get_tree_size(path('')) == 1 - assert library.python.fs.get_tree_size(path(''), recursive=True) == 3 + assert library.python.fs.get_tree_size(path('one.txt')) == 1 + assert library.python.fs.get_tree_size(path('')) == 1 + assert library.python.fs.get_tree_size(path(''), recursive=True) == 3 @pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') @@ -956,61 +956,61 @@ def test_get_tree_size_dangling_symlink(path): os.symlink(path('deeper/two.txt'), path("deeper/link.txt")) os.remove(path('deeper/two.txt')) # does not fail - assert library.python.fs.get_tree_size(path(''), recursive=True) == 1 - - + assert library.python.fs.get_tree_size(path(''), recursive=True) == 1 + + @pytest.mark.skipif(not library.python.windows.on_win(), reason='Test hardlinks on windows') -def test_hardlink_or_copy(): - max_allowed_hard_links = 1023 - - def run(hardlink_function, dir): - src = r"test.txt" - with open(src, "w") as f: - f.write("test") - for i in range(max_allowed_hard_links + 1): - hardlink_function(src, os.path.join(dir, "{}.txt".format(i))) - - dir1 = library.python.fs.create_dirs("one") - with pytest.raises(WindowsError) as e: - run(library.python.fs.hardlink, dir1) - assert e.value.winerror == 1142 - assert len(os.listdir(dir1)) == max_allowed_hard_links - - dir2 = library.python.fs.create_dirs("two") - run(library.python.fs.hardlink_or_copy, dir2) - assert len(os.listdir(dir2)) == max_allowed_hard_links + 1 - - -def test_remove_tree_unicode(): - path = u"test_remove_tree_unicode/русский".encode("utf-8") - os.makedirs(path) +def test_hardlink_or_copy(): + max_allowed_hard_links = 1023 + + def run(hardlink_function, dir): + src = r"test.txt" + with open(src, "w") as f: + f.write("test") + for i in range(max_allowed_hard_links + 1): + hardlink_function(src, os.path.join(dir, "{}.txt".format(i))) + + dir1 = library.python.fs.create_dirs("one") + with pytest.raises(WindowsError) as e: + run(library.python.fs.hardlink, dir1) + assert e.value.winerror == 1142 + assert len(os.listdir(dir1)) == max_allowed_hard_links + + dir2 = library.python.fs.create_dirs("two") + run(library.python.fs.hardlink_or_copy, dir2) + assert len(os.listdir(dir2)) == max_allowed_hard_links + 1 + + +def test_remove_tree_unicode(): + path = u"test_remove_tree_unicode/русский".encode("utf-8") + os.makedirs(path) library.python.fs.remove_tree(six.text_type("test_remove_tree_unicode")) - assert not os.path.exists("test_remove_tree_unicode") - - -def test_remove_tree_safe_unicode(): - path = u"test_remove_tree_safe_unicode/русский".encode("utf-8") - os.makedirs(path) + assert not os.path.exists("test_remove_tree_unicode") + + +def test_remove_tree_safe_unicode(): + path = u"test_remove_tree_safe_unicode/русский".encode("utf-8") + os.makedirs(path) library.python.fs.remove_tree_safe(six.text_type("test_remove_tree_safe_unicode")) - assert not os.path.exists("test_remove_tree_safe_unicode") - - -def test_copy_tree_custom_copy_function(): - library.python.fs.create_dirs("test_copy_tree_src/deepper/inner") - library.python.fs.write_file("test_copy_tree_src/deepper/deepper.txt", "deepper.txt") - library.python.fs.write_file("test_copy_tree_src/deepper/inner/inner.txt", "inner.txt") - copied = [] - - def copy_function(src, dst): - shutil.copy2(src, dst) - copied.append(dst) - + assert not os.path.exists("test_remove_tree_safe_unicode") + + +def test_copy_tree_custom_copy_function(): + library.python.fs.create_dirs("test_copy_tree_src/deepper/inner") + library.python.fs.write_file("test_copy_tree_src/deepper/deepper.txt", "deepper.txt") + library.python.fs.write_file("test_copy_tree_src/deepper/inner/inner.txt", "inner.txt") + copied = [] + + def copy_function(src, dst): + shutil.copy2(src, dst) + copied.append(dst) + library.python.fs.copy_tree( "test_copy_tree_src", yatest.common.work_path("test_copy_tree_dst"), copy_function=copy_function ) - assert len(copied) == 2 - assert yatest.common.work_path("test_copy_tree_dst/deepper/deepper.txt") in copied - assert yatest.common.work_path("test_copy_tree_dst/deepper/inner/inner.txt") in copied + assert len(copied) == 2 + assert yatest.common.work_path("test_copy_tree_dst/deepper/deepper.txt") in copied + assert yatest.common.work_path("test_copy_tree_dst/deepper/inner/inner.txt") in copied def test_copy2(): diff --git a/library/python/fs/test/ya.make b/library/python/fs/test/ya.make index c58182bfba..33e3f5b4ff 100644 --- a/library/python/fs/test/ya.make +++ b/library/python/fs/test/ya.make @@ -1,14 +1,14 @@ -OWNER(g:yatool) - +OWNER(g:yatool) + PY23_TEST() - -TEST_SRCS( - test_fs.py -) - -PEERDIR( - library/python/fs - library/python/tmp -) - -END() + +TEST_SRCS( + test_fs.py +) + +PEERDIR( + library/python/fs + library/python/tmp +) + +END() diff --git a/library/python/fs/ya.make b/library/python/fs/ya.make index 2597e4fb8e..b3c5092c71 100644 --- a/library/python/fs/ya.make +++ b/library/python/fs/ya.make @@ -1,23 +1,23 @@ -OWNER(g:yatool) - -PY23_LIBRARY() - +OWNER(g:yatool) + +PY23_LIBRARY() + PY_SRCS( __init__.py ) - + IF (OS_DARWIN) PY_SRCS( clonefile.pyx ) ENDIF() -PEERDIR( - library/python/func - library/python/strings - library/python/windows -) - -END() - -RECURSE_FOR_TESTS(test) +PEERDIR( + library/python/func + library/python/strings + library/python/windows +) + +END() + +RECURSE_FOR_TESTS(test) diff --git a/library/python/pytest/allure/conftest.py b/library/python/pytest/allure/conftest.py index 451e377cfc..0d5cfda1e5 100644 --- a/library/python/pytest/allure/conftest.py +++ b/library/python/pytest/allure/conftest.py @@ -1,8 +1,8 @@ -import os -import pytest - - -@pytest.mark.tryfirst -def pytest_configure(config): - if "ALLURE_REPORT_DIR" in os.environ: - config.option.allurereportdir = os.environ["ALLURE_REPORT_DIR"] +import os +import pytest + + +@pytest.mark.tryfirst +def pytest_configure(config): + if "ALLURE_REPORT_DIR" in os.environ: + config.option.allurereportdir = os.environ["ALLURE_REPORT_DIR"] diff --git a/library/python/pytest/empty/main.c b/library/python/pytest/empty/main.c index d49fcc7031..9efa08162a 100644 --- a/library/python/pytest/empty/main.c +++ b/library/python/pytest/empty/main.c @@ -1,7 +1,7 @@ -/* -to be used for build python tests in a stub binary for the case of using system python -*/ - -int main(void) { - return 0; -} +/* +to be used for build python tests in a stub binary for the case of using system python +*/ + +int main(void) { + return 0; +} diff --git a/library/python/pytest/empty/ya.make b/library/python/pytest/empty/ya.make index 4394568f15..8f0fa37e2a 100644 --- a/library/python/pytest/empty/ya.make +++ b/library/python/pytest/empty/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( g:yatool dmitko ) - + SRCS( main.c ) - -END() + +END() diff --git a/library/python/pytest/main.py b/library/python/pytest/main.py index df3513031f..6296bd6f0f 100644 --- a/library/python/pytest/main.py +++ b/library/python/pytest/main.py @@ -1,25 +1,25 @@ import os -import sys +import sys import time import __res - + FORCE_EXIT_TESTSFAILED_ENV = 'FORCE_EXIT_TESTSFAILED' -def main(): +def main(): import library.python.pytest.context as context context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() profile = None if '--profile-pytest' in sys.argv: sys.argv.remove('--profile-pytest') - + import pstats import cProfile profile = cProfile.Profile() profile.enable() - + # Reset influencing env. vars # For more info see library/python/testing/yatest_common/yatest/common/errors.py if FORCE_EXIT_TESTSFAILED_ENV in os.environ: @@ -46,12 +46,12 @@ def main(): m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook", rewrite.AssertionRewritingHook) prefix = '__tests__.' - + test_modules = [ name[len(prefix):] for name in sys.extra_modules if name.startswith(prefix) and not name.endswith('.conftest') ] - + doctest_packages = __res.find("PY_DOCTEST_PACKAGES") or "" if isinstance(doctest_packages, bytes): doctest_packages = doctest_packages.decode('utf-8') @@ -70,20 +70,20 @@ def main(): def remove_user_site(paths): site_paths = ('site-packages', 'site-python') - + def is_site_path(path): for p in site_paths: if path.find(p) != -1: return True return False - + new_paths = list(paths) for p in paths: if is_site_path(p): new_paths.remove(p) - + return new_paths - + sys.path = remove_user_site(sys.path) rc = pytest.main(plugins=[ collection.CollectionPlugin(test_modules, doctest_modules), @@ -91,10 +91,10 @@ def main(): conftests, ]) - if rc == 5: - # don't care about EXIT_NOTESTSCOLLECTED - rc = 0 - + if rc == 5: + # don't care about EXIT_NOTESTSCOLLECTED + rc = 0 + if rc == 1 and yatest_runner and not listing_mode and not os.environ.get(FORCE_EXIT_TESTSFAILED_ENV) == '1': # XXX it's place for future improvements # Test wrapper should terminate with 0 exit code if there are common test failures @@ -110,7 +110,7 @@ def main(): ps.print_stats() sys.exit(rc) - - -if __name__ == '__main__': - main() + + +if __name__ == '__main__': + main() diff --git a/library/python/pytest/plugins/collection.py b/library/python/pytest/plugins/collection.py index 1535da686c..e36f47a78f 100644 --- a/library/python/pytest/plugins/collection.py +++ b/library/python/pytest/plugins/collection.py @@ -1,26 +1,26 @@ import os -import sys +import sys from six import reraise - -import py + +import py import pytest # noqa -import _pytest.python -import _pytest.doctest +import _pytest.python +import _pytest.doctest import json import library.python.testing.filter.filter as test_filter - - -class LoadedModule(_pytest.python.Module): + + +class LoadedModule(_pytest.python.Module): def __init__(self, parent, name, **kwargs): self.name = name + '.py' self.session = parent self.parent = parent self.config = parent.config - self.keywords = {} + self.keywords = {} self.own_markers = [] self.fspath = py.path.local() - + @classmethod def from_parent(cls, **kwargs): namespace = kwargs.pop('namespace', True) @@ -31,7 +31,7 @@ class LoadedModule(_pytest.python.Module): return loaded_module - @property + @property def _nodeid(self): if os.getenv('CONFTEST_LOAD_POLICY') == 'LOCAL': return self._getobj().__file__ @@ -41,25 +41,25 @@ class LoadedModule(_pytest.python.Module): @property def nodeid(self): return self._nodeid - - def _getobj(self): + + def _getobj(self): module_name = self.name[:-len('.py')] if self.namespace: module_name = '__tests__.' + module_name - __import__(module_name) - return sys.modules[module_name] - - -class DoctestModule(LoadedModule): - - def collect(self): - import doctest + __import__(module_name) + return sys.modules[module_name] + + +class DoctestModule(LoadedModule): + + def collect(self): + import doctest module = self._getobj() # uses internal doctest module parsing mechanism - finder = doctest.DocTestFinder() - optionflags = _pytest.doctest.get_optionflags(self) - runner = doctest.DebugRunner(verbose=0, optionflags=optionflags) + finder = doctest.DocTestFinder() + optionflags = _pytest.doctest.get_optionflags(self) + runner = doctest.DebugRunner(verbose=0, optionflags=optionflags) try: for test in finder.find(module, self.name[:-len('.py')]): @@ -75,8 +75,8 @@ class DoctestModule(LoadedModule): etype, exc, tb = sys.exc_info() msg = 'DoctestModule failed, probably you can add NO_DOCTESTS() macro to ya.make' reraise(etype, type(exc)('{}\n{}'.format(exc, msg)), tb) - - + + # NOTE: Since we are overriding collect method of pytest session, pytest hooks are not invoked during collection. def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_filename_predicate): if session.config.option.mode == 'list': @@ -93,14 +93,14 @@ def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_f return False -class CollectionPlugin(object): +class CollectionPlugin(object): def __init__(self, test_modules, doctest_modules): - self._test_modules = test_modules + self._test_modules = test_modules self._doctest_modules = doctest_modules - - def pytest_sessionstart(self, session): - - def collect(*args, **kwargs): + + def pytest_sessionstart(self, session): + + def collect(*args, **kwargs): accept_filename_predicate = test_filter.make_py_file_filter(session.config.option.test_filter) full_test_names_file_path = session.config.option.test_list_path filenames_filter = None @@ -111,7 +111,7 @@ class CollectionPlugin(object): full_names_filter = set(json.load(afile)[int(session.config.option.modulo_index)]) filenames_filter = set(map(lambda x: x.split('::')[0], full_names_filter)) - for test_module in self._test_modules: + for test_module in self._test_modules: module = LoadedModule.from_parent(name=test_module, parent=session) if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): yield module @@ -120,9 +120,9 @@ class CollectionPlugin(object): module = DoctestModule.from_parent(name=test_module, parent=session) if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): yield module - + if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no': for doctest_module in self._doctest_modules: yield DoctestModule.from_parent(name=doctest_module, parent=session, namespace=False) - session.collect = collect + session.collect = collect diff --git a/library/python/pytest/plugins/conftests.py b/library/python/pytest/plugins/conftests.py index dfae771ef8..522041f5a7 100644 --- a/library/python/pytest/plugins/conftests.py +++ b/library/python/pytest/plugins/conftests.py @@ -1,26 +1,26 @@ import os -import importlib +import importlib import sys -import inspect +import inspect from pytest import hookimpl from .fixtures import metrics, links # noqa - -orig_getfile = inspect.getfile - - -def getfile(object): - res = orig_getfile(object) - if inspect.ismodule(object): - if not res and getattr(object, '__orig_file__'): - res = object.__orig_file__ - return res - -inspect.getfile = getfile + +orig_getfile = inspect.getfile + + +def getfile(object): + res = orig_getfile(object) + if inspect.ismodule(object): + if not res and getattr(object, '__orig_file__'): + res = object.__orig_file__ + return res + +inspect.getfile = getfile conftest_modules = [] - - + + @hookimpl(trylast=True) def pytest_load_initial_conftests(early_config, parser, args): conftests = filter(lambda name: name.endswith(".conftest"), sys.extra_modules) @@ -45,6 +45,6 @@ def getconftestmodules(*args, **kwargs): def pytest_sessionstart(session): - # Override filesystem based relevant conftest discovery on the call path - assert session.config.pluginmanager - session.config.pluginmanager._getconftestmodules = getconftestmodules + # Override filesystem based relevant conftest discovery on the call path + assert session.config.pluginmanager + session.config.pluginmanager._getconftestmodules = getconftestmodules diff --git a/library/python/pytest/plugins/fixtures.py b/library/python/pytest/plugins/fixtures.py index 9f5fd6ccf1..6f7e0a27e4 100644 --- a/library/python/pytest/plugins/fixtures.py +++ b/library/python/pytest/plugins/fixtures.py @@ -1,26 +1,26 @@ -import os -import pytest +import os +import pytest import six - - -MAX_ALLOWED_LINKS_COUNT = 10 - - -@pytest.fixture -def metrics(request): - - class Metrics(object): - @classmethod - def set(cls, name, value): - assert len(name) <= 128, "Length of the metric name must less than 128" - assert type(value) in [int, float], "Metric value must be of type int or float" - test_name = request.node.nodeid - if test_name not in request.config.test_metrics: - request.config.test_metrics[test_name] = {} - request.config.test_metrics[test_name][name] = value - - @classmethod - def set_benchmark(cls, benchmark_values): + + +MAX_ALLOWED_LINKS_COUNT = 10 + + +@pytest.fixture +def metrics(request): + + class Metrics(object): + @classmethod + def set(cls, name, value): + assert len(name) <= 128, "Length of the metric name must less than 128" + assert type(value) in [int, float], "Metric value must be of type int or float" + test_name = request.node.nodeid + if test_name not in request.config.test_metrics: + request.config.test_metrics[test_name] = {} + request.config.test_metrics[test_name][name] = value + + @classmethod + def set_benchmark(cls, benchmark_values): # report of google has key 'benchmarks' which is a list of benchmark results # yandex benchmark has key 'benchmark', which is a list of benchmark results # use this to differentiate which kind of result it is @@ -31,12 +31,12 @@ def metrics(request): @classmethod def set_ybenchmark(cls, benchmark_values): - for benchmark in benchmark_values["benchmark"]: - name = benchmark["name"] + for benchmark in benchmark_values["benchmark"]: + name = benchmark["name"] for key, value in six.iteritems(benchmark): - if key != "name": - cls.set("{}_{}".format(name, key), value) - + if key != "name": + cls.set("{}_{}".format(name, key), value) + @classmethod def set_gbenchmark(cls, benchmark_values): time_unit_multipliers = {"ns": 1, "us": 1000, "ms": 1000000} @@ -50,36 +50,36 @@ def metrics(request): cls.set("{}_{}".format(name, k), v * time_unit_mult) elif k not in ignore_keys and isinstance(v, (float, int)): cls.set("{}_{}".format(name, k), v) - return Metrics - - -@pytest.fixture -def links(request): - - class Links(object): - @classmethod - def set(cls, name, path): - - if len(request.config.test_logs[request.node.nodeid]) >= MAX_ALLOWED_LINKS_COUNT: - raise Exception("Cannot add more than {} links to test".format(MAX_ALLOWED_LINKS_COUNT)) - - reserved_names = ["log", "logsdir", "stdout", "stderr"] - if name in reserved_names: - raise Exception("Attachment name should not belong to the reserved list: {}".format(", ".join(reserved_names))) - output_dir = request.config.ya.output_dir - - if not os.path.exists(path): - raise Exception("Path to be attached does not exist: {}".format(path)) - - if os.path.isabs(path) and ".." in os.path.relpath(path, output_dir): - raise Exception("Test attachment must be inside yatest.common.output_path()") - - request.config.test_logs[request.node.nodeid][name] = path - - @classmethod - def get(cls, name): - if name not in request.config.test_logs[request.node.nodeid]: - raise KeyError("Attachment with name '{}' does not exist".format(name)) - return request.config.test_logs[request.node.nodeid][name] - - return Links + return Metrics + + +@pytest.fixture +def links(request): + + class Links(object): + @classmethod + def set(cls, name, path): + + if len(request.config.test_logs[request.node.nodeid]) >= MAX_ALLOWED_LINKS_COUNT: + raise Exception("Cannot add more than {} links to test".format(MAX_ALLOWED_LINKS_COUNT)) + + reserved_names = ["log", "logsdir", "stdout", "stderr"] + if name in reserved_names: + raise Exception("Attachment name should not belong to the reserved list: {}".format(", ".join(reserved_names))) + output_dir = request.config.ya.output_dir + + if not os.path.exists(path): + raise Exception("Path to be attached does not exist: {}".format(path)) + + if os.path.isabs(path) and ".." in os.path.relpath(path, output_dir): + raise Exception("Test attachment must be inside yatest.common.output_path()") + + request.config.test_logs[request.node.nodeid][name] = path + + @classmethod + def get(cls, name): + if name not in request.config.test_logs[request.node.nodeid]: + raise KeyError("Attachment with name '{}' does not exist".format(name)) + return request.config.test_logs[request.node.nodeid][name] + + return Links diff --git a/library/python/pytest/plugins/ya.make b/library/python/pytest/plugins/ya.make index 07914cf4d3..c15d6f759d 100644 --- a/library/python/pytest/plugins/ya.make +++ b/library/python/pytest/plugins/ya.make @@ -1,20 +1,20 @@ OWNER(g:yatest) - + PY23_LIBRARY() - -PY_SRCS( - ya.py - collection.py - conftests.py - fixtures.py -) - -PEERDIR( + +PY_SRCS( + ya.py + collection.py + conftests.py + fixtures.py +) + +PEERDIR( library/python/filelock - library/python/find_root + library/python/find_root library/python/testing/filter -) - +) + IF (PYTHON2) PY_SRCS( fakeid_py2.py @@ -29,4 +29,4 @@ ELSE() ) ENDIF() -END() +END() diff --git a/library/python/pytest/plugins/ya.py b/library/python/pytest/plugins/ya.py index c6b06478d9..1bde03042d 100644 --- a/library/python/pytest/plugins/ya.py +++ b/library/python/pytest/plugins/ya.py @@ -3,101 +3,101 @@ import base64 import errno import re -import sys -import os -import logging -import fnmatch -import json -import time +import sys +import os +import logging +import fnmatch +import json +import time import traceback -import collections +import collections import signal import inspect import warnings import attr import faulthandler -import py -import pytest +import py +import pytest import six -import _pytest +import _pytest import _pytest._io import _pytest.mark import _pytest.outcomes import _pytest.skipping - + from _pytest.warning_types import PytestUnhandledCoroutineWarning from yatest_lib import test_splitter -try: - import resource -except ImportError: - resource = None - -try: - import library.python.pytest.yatest_tools as tools -except ImportError: - # fallback for pytest script mode - import yatest_tools as tools - +try: + import resource +except ImportError: + resource = None + +try: + import library.python.pytest.yatest_tools as tools +except ImportError: + # fallback for pytest script mode + import yatest_tools as tools + try: from library.python import filelock except ImportError: filelock = None -import yatest_lib.tools - -import yatest_lib.external as canon - +import yatest_lib.tools + +import yatest_lib.external as canon + import yatest_lib.ya from library.python.pytest import context -console_logger = logging.getLogger("console") -yatest_logger = logging.getLogger("ya.test") - - -_pytest.main.EXIT_NOTESTSCOLLECTED = 0 +console_logger = logging.getLogger("console") +yatest_logger = logging.getLogger("ya.test") + + +_pytest.main.EXIT_NOTESTSCOLLECTED = 0 SHUTDOWN_REQUESTED = False - + pytest_config = None - -def configure_pdb_on_demand(): - import signal - - if hasattr(signal, "SIGUSR1"): - def on_signal(*args): + +def configure_pdb_on_demand(): + import signal + + if hasattr(signal, "SIGUSR1"): + def on_signal(*args): import ipdb ipdb.set_trace() - - signal.signal(signal.SIGUSR1, on_signal) - - -class CustomImporter(object): - def __init__(self, roots): - self._roots = roots - - def find_module(self, fullname, package_path=None): - for path in self._roots: - full_path = self._get_module_path(path, fullname) - - if os.path.exists(full_path) and os.path.isdir(full_path) and not os.path.exists(os.path.join(full_path, "__init__.py")): - open(os.path.join(full_path, "__init__.py"), "w").close() - - return None - - def _get_module_path(self, path, fullname): - return os.path.join(path, *fullname.split('.')) - - -class YaTestLoggingFileHandler(logging.FileHandler): - pass - - + + signal.signal(signal.SIGUSR1, on_signal) + + +class CustomImporter(object): + def __init__(self, roots): + self._roots = roots + + def find_module(self, fullname, package_path=None): + for path in self._roots: + full_path = self._get_module_path(path, fullname) + + if os.path.exists(full_path) and os.path.isdir(full_path) and not os.path.exists(os.path.join(full_path, "__init__.py")): + open(os.path.join(full_path, "__init__.py"), "w").close() + + return None + + def _get_module_path(self, path, fullname): + return os.path.join(path, *fullname.split('.')) + + +class YaTestLoggingFileHandler(logging.FileHandler): + pass + + class _TokenFilterFormatter(logging.Formatter): def __init__(self, fmt): super(_TokenFilterFormatter, self).__init__(fmt) @@ -123,141 +123,141 @@ class _TokenFilterFormatter(logging.Formatter): return self._filter(super(_TokenFilterFormatter, self).format(record)) -def setup_logging(log_path, level=logging.DEBUG, *other_logs): - logs = [log_path] + list(other_logs) - root_logger = logging.getLogger() - for i in range(len(root_logger.handlers) - 1, -1, -1): - if isinstance(root_logger.handlers[i], YaTestLoggingFileHandler): +def setup_logging(log_path, level=logging.DEBUG, *other_logs): + logs = [log_path] + list(other_logs) + root_logger = logging.getLogger() + for i in range(len(root_logger.handlers) - 1, -1, -1): + if isinstance(root_logger.handlers[i], YaTestLoggingFileHandler): root_logger.handlers.pop(i).close() - root_logger.setLevel(level) - for log_file in logs: - file_handler = YaTestLoggingFileHandler(log_file) - log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' + root_logger.setLevel(level) + for log_file in logs: + file_handler = YaTestLoggingFileHandler(log_file) + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' file_handler.setFormatter(_TokenFilterFormatter(log_format)) - file_handler.setLevel(level) - root_logger.addHandler(file_handler) - - -def pytest_addoption(parser): - parser.addoption("--build-root", action="store", dest="build_root", default="", help="path to the build root") - parser.addoption("--dep-root", action="append", dest="dep_roots", default=[], help="path to the dep build roots") - parser.addoption("--source-root", action="store", dest="source_root", default="", help="path to the source root") - parser.addoption("--data-root", action="store", dest="data_root", default="", help="path to the arcadia_tests_data root") - parser.addoption("--output-dir", action="store", dest="output_dir", default="", help="path to the test output dir") - parser.addoption("--python-path", action="store", dest="python_path", default="", help="path the canonical python binary") - parser.addoption("--valgrind-path", action="store", dest="valgrind_path", default="", help="path the canonical valgring binary") - parser.addoption("--test-filter", action="append", dest="test_filter", default=None, help="test filter") + file_handler.setLevel(level) + root_logger.addHandler(file_handler) + + +def pytest_addoption(parser): + parser.addoption("--build-root", action="store", dest="build_root", default="", help="path to the build root") + parser.addoption("--dep-root", action="append", dest="dep_roots", default=[], help="path to the dep build roots") + parser.addoption("--source-root", action="store", dest="source_root", default="", help="path to the source root") + parser.addoption("--data-root", action="store", dest="data_root", default="", help="path to the arcadia_tests_data root") + parser.addoption("--output-dir", action="store", dest="output_dir", default="", help="path to the test output dir") + parser.addoption("--python-path", action="store", dest="python_path", default="", help="path the canonical python binary") + parser.addoption("--valgrind-path", action="store", dest="valgrind_path", default="", help="path the canonical valgring binary") + parser.addoption("--test-filter", action="append", dest="test_filter", default=None, help="test filter") parser.addoption("--test-file-filter", action="store", dest="test_file_filter", default=None, help="test file filter") - parser.addoption("--test-param", action="append", dest="test_params", default=None, help="test parameters") - parser.addoption("--test-log-level", action="store", dest="test_log_level", choices=["critical", "error", "warning", "info", "debug"], default="debug", help="test log level") + parser.addoption("--test-param", action="append", dest="test_params", default=None, help="test parameters") + parser.addoption("--test-log-level", action="store", dest="test_log_level", choices=["critical", "error", "warning", "info", "debug"], default="debug", help="test log level") parser.addoption("--mode", action="store", choices=[yatest_lib.ya.RunMode.List, yatest_lib.ya.RunMode.Run], dest="mode", default=yatest_lib.ya.RunMode.Run, help="testing mode") parser.addoption("--test-list-file", action="store", dest="test_list_file") - parser.addoption("--modulo", default=1, type=int) - parser.addoption("--modulo-index", default=0, type=int) + parser.addoption("--modulo", default=1, type=int) + parser.addoption("--modulo-index", default=0, type=int) parser.addoption("--partition-mode", default='SEQUENTIAL', help="Split tests according to partitoin mode") - parser.addoption("--split-by-tests", action='store_true', help="Split test execution by tests instead of suites", default=False) - parser.addoption("--project-path", action="store", default="", help="path to CMakeList where test is declared") - parser.addoption("--build-type", action="store", default="", help="build type") + parser.addoption("--split-by-tests", action='store_true', help="Split test execution by tests instead of suites", default=False) + parser.addoption("--project-path", action="store", default="", help="path to CMakeList where test is declared") + parser.addoption("--build-type", action="store", default="", help="build type") parser.addoption("--flags", action="append", dest="flags", default=[], help="build flags (-D)") parser.addoption("--sanitize", action="store", default="", help="sanitize mode") - parser.addoption("--test-stderr", action="store_true", default=False, help="test stderr") + parser.addoption("--test-stderr", action="store_true", default=False, help="test stderr") parser.addoption("--test-debug", action="store_true", default=False, help="test debug mode") - parser.addoption("--root-dir", action="store", default=None) - parser.addoption("--ya-trace", action="store", dest="ya_trace_path", default=None, help="path to ya trace report") + parser.addoption("--root-dir", action="store", default=None) + parser.addoption("--ya-trace", action="store", dest="ya_trace_path", default=None, help="path to ya trace report") parser.addoption("--ya-version", action="store", dest="ya_version", default=0, type=int, help="allows to be compatible with ya and the new changes in ya-dev") - parser.addoption( - "--test-suffix", action="store", dest="test_suffix", default=None, help="add suffix to every test name" - ) - parser.addoption("--gdb-path", action="store", dest="gdb_path", default="", help="path the canonical gdb binary") - parser.addoption("--collect-cores", action="store_true", dest="collect_cores", default=False, help="allows core dump file recovering during test") + parser.addoption( + "--test-suffix", action="store", dest="test_suffix", default=None, help="add suffix to every test name" + ) + parser.addoption("--gdb-path", action="store", dest="gdb_path", default="", help="path the canonical gdb binary") + parser.addoption("--collect-cores", action="store_true", dest="collect_cores", default=False, help="allows core dump file recovering during test") parser.addoption("--sanitizer-extra-checks", action="store_true", dest="sanitizer_extra_checks", default=False, help="enables extra checks for tests built with sanitizers") - parser.addoption("--report-deselected", action="store_true", dest="report_deselected", default=False, help="report deselected tests to the trace file") - parser.addoption("--pdb-on-sigusr1", action="store_true", default=False, help="setup pdb.set_trace on SIGUSR1") + parser.addoption("--report-deselected", action="store_true", dest="report_deselected", default=False, help="report deselected tests to the trace file") + parser.addoption("--pdb-on-sigusr1", action="store_true", default=False, help="setup pdb.set_trace on SIGUSR1") parser.addoption("--test-tool-bin", help="Path to test_tool") parser.addoption("--test-list-path", dest="test_list_path", action="store", help="path to test list", default="") - - + + def from_ya_test(): return "YA_TEST_RUNNER" in os.environ -def pytest_configure(config): +def pytest_configure(config): global pytest_config pytest_config = config config.option.continue_on_collection_errors = True - + config.addinivalue_line("markers", "ya:external") config.from_ya_test = from_ya_test() - config.test_logs = collections.defaultdict(dict) - config.test_metrics = {} + config.test_logs = collections.defaultdict(dict) + config.test_metrics = {} config.suite_metrics = {} config.configure_timestamp = time.time() - context = { - "project_path": config.option.project_path, - "test_stderr": config.option.test_stderr, + context = { + "project_path": config.option.project_path, + "test_stderr": config.option.test_stderr, "test_debug": config.option.test_debug, - "build_type": config.option.build_type, - "test_traceback": config.option.tbstyle, + "build_type": config.option.build_type, + "test_traceback": config.option.tbstyle, "flags": config.option.flags, "sanitize": config.option.sanitize, - } + } if config.option.collectonly: config.option.mode = yatest_lib.ya.RunMode.List config.ya = yatest_lib.ya.Ya( - config.option.mode, - config.option.source_root, - config.option.build_root, - config.option.dep_roots, - config.option.output_dir, - config.option.test_params, - context, - config.option.python_path, - config.option.valgrind_path, - config.option.gdb_path, - config.option.data_root, - ) - config.option.test_log_level = { - "critical": logging.CRITICAL, - "error": logging.ERROR, - "warning": logging.WARN, - "info": logging.INFO, - "debug": logging.DEBUG, - }[config.option.test_log_level] - - if not config.option.collectonly: - setup_logging(os.path.join(config.ya.output_dir, "run.log"), config.option.test_log_level) - config.current_item_nodeid = None - config.current_test_name = None - config.test_cores_count = 0 - config.collect_cores = config.option.collect_cores + config.option.mode, + config.option.source_root, + config.option.build_root, + config.option.dep_roots, + config.option.output_dir, + config.option.test_params, + context, + config.option.python_path, + config.option.valgrind_path, + config.option.gdb_path, + config.option.data_root, + ) + config.option.test_log_level = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warning": logging.WARN, + "info": logging.INFO, + "debug": logging.DEBUG, + }[config.option.test_log_level] + + if not config.option.collectonly: + setup_logging(os.path.join(config.ya.output_dir, "run.log"), config.option.test_log_level) + config.current_item_nodeid = None + config.current_test_name = None + config.test_cores_count = 0 + config.collect_cores = config.option.collect_cores config.sanitizer_extra_checks = config.option.sanitizer_extra_checks try: config.test_tool_bin = config.option.test_tool_bin except AttributeError: logging.info("test_tool_bin not specified") - - if config.sanitizer_extra_checks: + + if config.sanitizer_extra_checks: for envvar in ['LSAN_OPTIONS', 'ASAN_OPTIONS']: if envvar in os.environ: os.environ.pop(envvar) if envvar + '_ORIGINAL' in os.environ: os.environ[envvar] = os.environ[envvar + '_ORIGINAL'] - if config.option.root_dir: + if config.option.root_dir: config.rootdir = py.path.local(config.option.root_dir) config.invocation_params = attr.evolve(config.invocation_params, dir=config.rootdir) - + extra_sys_path = [] # Arcadia paths from the test DEPENDS section of ya.make extra_sys_path.append(os.path.join(config.option.source_root, config.option.project_path)) - # Build root is required for correct import of protobufs, because imports are related to the root - # (like import devtools.dummy_arcadia.protos.lib.my_proto_pb2) + # Build root is required for correct import of protobufs, because imports are related to the root + # (like import devtools.dummy_arcadia.protos.lib.my_proto_pb2) extra_sys_path.append(config.option.build_root) - + for path in config.option.dep_roots: if os.path.isabs(path): extra_sys_path.append(path) @@ -272,17 +272,17 @@ def pytest_configure(config): os.environ["PYTHONPATH"] = os.pathsep.join(sys.path) - if not config.option.collectonly: - if config.option.ya_trace_path: - config.ya_trace_reporter = TraceReportGenerator(config.option.ya_trace_path) - else: - config.ya_trace_reporter = DryTraceReportGenerator(config.option.ya_trace_path) + if not config.option.collectonly: + if config.option.ya_trace_path: + config.ya_trace_reporter = TraceReportGenerator(config.option.ya_trace_path) + else: + config.ya_trace_reporter = DryTraceReportGenerator(config.option.ya_trace_path) config.ya_version = config.option.ya_version - - sys.meta_path.append(CustomImporter([config.option.build_root] + [os.path.join(config.option.build_root, dep) for dep in config.option.dep_roots])) - if config.option.pdb_on_sigusr1: - configure_pdb_on_demand() - + + sys.meta_path.append(CustomImporter([config.option.build_root] + [os.path.join(config.option.build_root, dep) for dep in config.option.dep_roots])) + if config.option.pdb_on_sigusr1: + configure_pdb_on_demand() + # Dump python backtrace in case of any errors faulthandler.enable() if hasattr(signal, "SIGQUIT"): @@ -291,7 +291,7 @@ def pytest_configure(config): if hasattr(signal, "SIGUSR2"): signal.signal(signal.SIGUSR2, _graceful_shutdown) - + session_should_exit = False @@ -327,122 +327,122 @@ def _graceful_shutdown(*args): _graceful_shutdown_on_log(not capman.is_globally_capturing()) -def _get_rusage(): - return resource and resource.getrusage(resource.RUSAGE_SELF) - - -def _collect_test_rusage(item): - if resource and hasattr(item, "rusage"): - finish_rusage = _get_rusage() +def _get_rusage(): + return resource and resource.getrusage(resource.RUSAGE_SELF) + + +def _collect_test_rusage(item): + if resource and hasattr(item, "rusage"): + finish_rusage = _get_rusage() ya_inst = pytest_config.ya - - def add_metric(attr_name, metric_name=None, modifier=None): - if not metric_name: - metric_name = attr_name - if not modifier: - modifier = lambda x: x - if hasattr(item.rusage, attr_name): + + def add_metric(attr_name, metric_name=None, modifier=None): + if not metric_name: + metric_name = attr_name + if not modifier: + modifier = lambda x: x + if hasattr(item.rusage, attr_name): ya_inst.set_metric_value(metric_name, modifier(getattr(finish_rusage, attr_name) - getattr(item.rusage, attr_name))) - - for args in [ - ("ru_maxrss", "ru_rss", lambda x: x*1024), # to be the same as in util/system/rusage.cpp - ("ru_utime",), - ("ru_stime",), - ("ru_ixrss", None, lambda x: x*1024), - ("ru_idrss", None, lambda x: x*1024), - ("ru_isrss", None, lambda x: x*1024), - ("ru_majflt", "ru_major_pagefaults"), - ("ru_minflt", "ru_minor_pagefaults"), - ("ru_nswap",), - ("ru_inblock",), - ("ru_oublock",), - ("ru_msgsnd",), - ("ru_msgrcv",), - ("ru_nsignals",), - ("ru_nvcsw",), - ("ru_nivcsw",), - ]: - add_metric(*args) - - -def _get_item_tags(item): - tags = [] - for key, value in item.keywords.items(): + + for args in [ + ("ru_maxrss", "ru_rss", lambda x: x*1024), # to be the same as in util/system/rusage.cpp + ("ru_utime",), + ("ru_stime",), + ("ru_ixrss", None, lambda x: x*1024), + ("ru_idrss", None, lambda x: x*1024), + ("ru_isrss", None, lambda x: x*1024), + ("ru_majflt", "ru_major_pagefaults"), + ("ru_minflt", "ru_minor_pagefaults"), + ("ru_nswap",), + ("ru_inblock",), + ("ru_oublock",), + ("ru_msgsnd",), + ("ru_msgrcv",), + ("ru_nsignals",), + ("ru_nvcsw",), + ("ru_nivcsw",), + ]: + add_metric(*args) + + +def _get_item_tags(item): + tags = [] + for key, value in item.keywords.items(): if key == 'pytestmark' and isinstance(value, list): for mark in value: tags.append(mark.name) elif isinstance(value, _pytest.mark.MarkDecorator): - tags.append(key) - return tags - - -def pytest_runtest_setup(item): - item.rusage = _get_rusage() + tags.append(key) + return tags + + +def pytest_runtest_setup(item): + item.rusage = _get_rusage() pytest_config.test_cores_count = 0 pytest_config.current_item_nodeid = item.nodeid - class_name, test_name = tools.split_node_id(item.nodeid) + class_name, test_name = tools.split_node_id(item.nodeid) test_log_path = tools.get_test_log_file_path(pytest_config.ya.output_dir, class_name, test_name) - setup_logging( + setup_logging( os.path.join(pytest_config.ya.output_dir, "run.log"), pytest_config.option.test_log_level, - test_log_path - ) + test_log_path + ) pytest_config.test_logs[item.nodeid]['log'] = test_log_path pytest_config.test_logs[item.nodeid]['logsdir'] = pytest_config.ya.output_dir pytest_config.current_test_log_path = test_log_path pytest_config.current_test_name = "{}::{}".format(class_name, test_name) - separator = "#" * 100 - yatest_logger.info(separator) - yatest_logger.info(test_name) - yatest_logger.info(separator) - yatest_logger.info("Test setup") - + separator = "#" * 100 + yatest_logger.info(separator) + yatest_logger.info(test_name) + yatest_logger.info(separator) + yatest_logger.info("Test setup") + test_item = CrashedTestItem(item.nodeid, pytest_config.option.test_suffix) pytest_config.ya_trace_reporter.on_start_test_class(test_item) pytest_config.ya_trace_reporter.on_start_test_case(test_item) - - -def pytest_runtest_teardown(item, nextitem): - yatest_logger.info("Test teardown") - - -def pytest_runtest_call(item): + + +def pytest_runtest_teardown(item, nextitem): + yatest_logger.info("Test teardown") + + +def pytest_runtest_call(item): class_name, test_name = tools.split_node_id(item.nodeid) yatest_logger.info("Test call (class_name: %s, test_name: %s)", class_name, test_name) - - -def pytest_deselected(items): + + +def pytest_deselected(items): config = pytest_config - if config.option.report_deselected: - for item in items: - deselected_item = DeselectedTestItem(item.nodeid, config.option.test_suffix) - config.ya_trace_reporter.on_start_test_class(deselected_item) - config.ya_trace_reporter.on_finish_test_case(deselected_item) - config.ya_trace_reporter.on_finish_test_class(deselected_item) - - -@pytest.mark.trylast -def pytest_collection_modifyitems(items, config): - - def filter_items(filters): - filtered_items = [] - deselected_items = [] - for item in items: + if config.option.report_deselected: + for item in items: + deselected_item = DeselectedTestItem(item.nodeid, config.option.test_suffix) + config.ya_trace_reporter.on_start_test_class(deselected_item) + config.ya_trace_reporter.on_finish_test_case(deselected_item) + config.ya_trace_reporter.on_finish_test_class(deselected_item) + + +@pytest.mark.trylast +def pytest_collection_modifyitems(items, config): + + def filter_items(filters): + filtered_items = [] + deselected_items = [] + for item in items: canonical_node_id = str(CustomTestItem(item.nodeid, pytest_config.option.test_suffix)) - matched = False - for flt in filters: + matched = False + for flt in filters: if "::" not in flt and "*" not in flt: - flt += "*" # add support for filtering by module name - if canonical_node_id.endswith(flt) or fnmatch.fnmatch(tools.escape_for_fnmatch(canonical_node_id), tools.escape_for_fnmatch(flt)): - matched = True - if matched: - filtered_items.append(item) - else: - deselected_items.append(item) - - config.hook.pytest_deselected(items=deselected_items) - items[:] = filtered_items - + flt += "*" # add support for filtering by module name + if canonical_node_id.endswith(flt) or fnmatch.fnmatch(tools.escape_for_fnmatch(canonical_node_id), tools.escape_for_fnmatch(flt)): + matched = True + if matched: + filtered_items.append(item) + else: + deselected_items.append(item) + + config.hook.pytest_deselected(items=deselected_items) + items[:] = filtered_items + def filter_by_full_name(filters): filter_set = {flt for flt in filters} filtered_items = [] @@ -456,10 +456,10 @@ def pytest_collection_modifyitems(items, config): config.hook.pytest_deselected(items=deselected_items) items[:] = filtered_items - # XXX - check to be removed when tests for peerdirs don't run - for item in items: - if not item.nodeid: - item._nodeid = os.path.basename(item.location[0]) + # XXX - check to be removed when tests for peerdirs don't run + for item in items: + if not item.nodeid: + item._nodeid = os.path.basename(item.location[0]) if os.path.exists(config.option.test_list_path): with open(config.option.test_list_path, 'r') as afile: chunks = json.load(afile) @@ -490,39 +490,39 @@ def pytest_collection_modifyitems(items, config): for item in chunk_items: items.extend(item) yatest_logger.info("Modulo %s tests are: %s", modulo_index, chunk_items) - + if config.option.mode == yatest_lib.ya.RunMode.Run: - for item in items: - test_item = NotLaunchedTestItem(item.nodeid, config.option.test_suffix) - config.ya_trace_reporter.on_start_test_class(test_item) - config.ya_trace_reporter.on_finish_test_case(test_item) - config.ya_trace_reporter.on_finish_test_class(test_item) + for item in items: + test_item = NotLaunchedTestItem(item.nodeid, config.option.test_suffix) + config.ya_trace_reporter.on_start_test_class(test_item) + config.ya_trace_reporter.on_finish_test_case(test_item) + config.ya_trace_reporter.on_finish_test_class(test_item) elif config.option.mode == yatest_lib.ya.RunMode.List: - tests = [] - for item in items: + tests = [] + for item in items: item = CustomTestItem(item.nodeid, pytest_config.option.test_suffix, item.keywords) - record = { - "class": item.class_name, - "test": item.test_name, - "tags": _get_item_tags(item), - } - tests.append(record) + record = { + "class": item.class_name, + "test": item.test_name, + "tags": _get_item_tags(item), + } + tests.append(record) if config.option.test_list_file: with open(config.option.test_list_file, 'w') as afile: json.dump(tests, afile) # TODO prettyboy remove after test_tool release - currently it's required for backward compatibility - sys.stderr.write(json.dumps(tests)) - - -def pytest_collectreport(report): - if not report.passed: + sys.stderr.write(json.dumps(tests)) + + +def pytest_collectreport(report): + if not report.passed: if hasattr(pytest_config, 'ya_trace_reporter'): test_item = TestItem(report, None, pytest_config.option.test_suffix) pytest_config.ya_trace_reporter.on_error(test_item) - else: - sys.stderr.write(yatest_lib.tools.to_utf8(report.longrepr)) - - + else: + sys.stderr.write(yatest_lib.tools.to_utf8(report.longrepr)) + + @pytest.mark.tryfirst def pytest_pyfunc_call(pyfuncitem): testfunction = pyfuncitem.obj @@ -542,7 +542,7 @@ def pytest_pyfunc_call(pyfuncitem): @pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): +def pytest_runtest_makereport(item, call): def logreport(report, result, call): test_item = TestItem(report, result, pytest_config.option.test_suffix) if not pytest_config.suite_metrics and context.Ctx.get("YA_PYTEST_START_TIMESTAMP"): @@ -554,24 +554,24 @@ def pytest_runtest_makereport(item, call): if report.outcome == "failed": yatest_logger.error(report.longrepr) - if report.when == "call": - _collect_test_rusage(item) + if report.when == "call": + _collect_test_rusage(item) pytest_config.ya_trace_reporter.on_finish_test_case(test_item) - elif report.when == "setup": + elif report.when == "setup": pytest_config.ya_trace_reporter.on_start_test_class(test_item) - if report.outcome != "passed": + if report.outcome != "passed": pytest_config.ya_trace_reporter.on_start_test_case(test_item) pytest_config.ya_trace_reporter.on_finish_test_case(test_item) - else: + else: pytest_config.ya_trace_reporter.on_start_test_case(test_item) - elif report.when == "teardown": - if report.outcome == "failed": + elif report.when == "teardown": + if report.outcome == "failed": pytest_config.ya_trace_reporter.on_start_test_case(test_item) pytest_config.ya_trace_reporter.on_finish_test_case(test_item) else: pytest_config.ya_trace_reporter.on_finish_test_case(test_item, duration_only=True) pytest_config.ya_trace_reporter.on_finish_test_class(test_item) - + outcome = yield rep = outcome.get_result() result = None @@ -580,10 +580,10 @@ def pytest_runtest_makereport(item, call): if not pytest_config.from_ya_test: ti = TestItem(rep, result, pytest_config.option.test_suffix) tr = pytest_config.pluginmanager.getplugin('terminalreporter') - tr.write_line("{} - Validating canonical data is not supported when running standalone binary".format(ti), yellow=True, bold=True) + tr.write_line("{} - Validating canonical data is not supported when running standalone binary".format(ti), yellow=True, bold=True) logreport(rep, result, call) - - + + def pytest_make_parametrize_id(config, val, argname): # Avoid <, > symbols in canondata file names if inspect.isfunction(val) and val.__name__ == "<lambda>": @@ -598,7 +598,7 @@ def get_formatted_error(report): text += colorize(entry) else: text = colorize(report.longrepr) - text = yatest_lib.tools.to_utf8(text) + text = yatest_lib.tools.to_utf8(text) return text @@ -616,9 +616,9 @@ def colorize(longrepr): if hasattr(longrepr, 'reprtraceback') and hasattr(longrepr.reprtraceback, 'toterminal'): longrepr.reprtraceback.toterminal(writer) return io.getvalue().strip() - return yatest_lib.tools.to_utf8(longrepr) + return yatest_lib.tools.to_utf8(longrepr) - text = yatest_lib.tools.to_utf8(longrepr) + text = yatest_lib.tools.to_utf8(longrepr) pos = text.find("E ") if pos == -1: return text @@ -633,25 +633,25 @@ def colorize(longrepr): return "{}[[bad]]{}".format(bt, error) -class TestItem(object): - - def __init__(self, report, result, test_suffix): - self._result = result - self.nodeid = report.nodeid - self._class_name, self._test_name = tools.split_node_id(self.nodeid, test_suffix) - self._error = None - self._status = None - self._process_report(report) - self._duration = hasattr(report, 'duration') and report.duration or 0 - self._keywords = getattr(report, "keywords", {}) - - def _process_report(self, report): - if report.longrepr: - self.set_error(report) - if hasattr(report, 'when') and report.when != "call": - self.set_error(report.when + " failed:\n" + self._error) - else: - self.set_error("") +class TestItem(object): + + def __init__(self, report, result, test_suffix): + self._result = result + self.nodeid = report.nodeid + self._class_name, self._test_name = tools.split_node_id(self.nodeid, test_suffix) + self._error = None + self._status = None + self._process_report(report) + self._duration = hasattr(report, 'duration') and report.duration or 0 + self._keywords = getattr(report, "keywords", {}) + + def _process_report(self, report): + if report.longrepr: + self.set_error(report) + if hasattr(report, 'when') and report.when != "call": + self.set_error(report.when + " failed:\n" + self._error) + else: + self.set_error("") report_teststatus = _pytest.skipping.pytest_report_teststatus(report) if report_teststatus is not None: @@ -667,89 +667,89 @@ class TestItem(object): self._status = 'skipped' self.set_error(yatest_lib.tools.to_utf8(report.longrepr[-1])) elif report.passed: - self._status = 'good' - self.set_error("") + self._status = 'good' + self.set_error("") else: self._status = 'fail' - - @property - def status(self): - return self._status - - def set_status(self, status): - self._status = status - - @property - def test_name(self): - return tools.normalize_name(self._test_name) - - @property - def class_name(self): - return tools.normalize_name(self._class_name) - - @property - def error(self): - return self._error - + + @property + def status(self): + return self._status + + def set_status(self, status): + self._status = status + + @property + def test_name(self): + return tools.normalize_name(self._test_name) + + @property + def class_name(self): + return tools.normalize_name(self._class_name) + + @property + def error(self): + return self._error + def set_error(self, entry, marker='bad'): if isinstance(entry, _pytest.reports.BaseReport): - self._error = get_formatted_error(entry) - else: + self._error = get_formatted_error(entry) + else: self._error = "[[{}]]{}".format(yatest_lib.tools.to_str(marker), yatest_lib.tools.to_str(entry)) - - @property - def duration(self): - return self._duration - - @property - def result(self): - if 'not_canonize' in self._keywords: - return None - return self._result - - @property - def keywords(self): - return self._keywords - - def __str__(self): - return "{}::{}".format(self.class_name, self.test_name) - - -class CustomTestItem(TestItem): - - def __init__(self, nodeid, test_suffix, keywords=None): - self._result = None - self.nodeid = nodeid - self._class_name, self._test_name = tools.split_node_id(nodeid, test_suffix) - self._duration = 0 - self._error = "" - self._keywords = keywords if keywords is not None else {} - - -class NotLaunchedTestItem(CustomTestItem): - - def __init__(self, nodeid, test_suffix): - super(NotLaunchedTestItem, self).__init__(nodeid, test_suffix) - self._status = "not_launched" - - -class CrashedTestItem(CustomTestItem): - - def __init__(self, nodeid, test_suffix): - super(CrashedTestItem, self).__init__(nodeid, test_suffix) - self._status = "crashed" - - -class DeselectedTestItem(CustomTestItem): - - def __init__(self, nodeid, test_suffix): - super(DeselectedTestItem, self).__init__(nodeid, test_suffix) - self._status = "deselected" - - -class TraceReportGenerator(object): - - def __init__(self, out_file_path): + + @property + def duration(self): + return self._duration + + @property + def result(self): + if 'not_canonize' in self._keywords: + return None + return self._result + + @property + def keywords(self): + return self._keywords + + def __str__(self): + return "{}::{}".format(self.class_name, self.test_name) + + +class CustomTestItem(TestItem): + + def __init__(self, nodeid, test_suffix, keywords=None): + self._result = None + self.nodeid = nodeid + self._class_name, self._test_name = tools.split_node_id(nodeid, test_suffix) + self._duration = 0 + self._error = "" + self._keywords = keywords if keywords is not None else {} + + +class NotLaunchedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(NotLaunchedTestItem, self).__init__(nodeid, test_suffix) + self._status = "not_launched" + + +class CrashedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(CrashedTestItem, self).__init__(nodeid, test_suffix) + self._status = "crashed" + + +class DeselectedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(DeselectedTestItem, self).__init__(nodeid, test_suffix) + self._status = "deselected" + + +class TraceReportGenerator(object): + + def __init__(self, out_file_path): self._filename = out_file_path self._file = open(out_file_path, 'w') self._wreckage_filename = out_file_path + '.wreckage' @@ -759,7 +759,7 @@ class TraceReportGenerator(object): self._current_test = (None, None) self._pid = os.getpid() self._check_intricate_respawn() - + def _check_intricate_respawn(self): pid_file = self._filename + '.pid' try: @@ -803,40 +803,40 @@ class TraceReportGenerator(object): # Test binary is launched without `ya make -t`'s testing machinery - don't rely on clean environment pass - def on_start_test_class(self, test_item): + def on_start_test_class(self, test_item): pytest_config.ya.set_test_item_node_id(test_item.nodeid) class_name = test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name self._current_test = (class_name, None) self.trace('test-started', {'class': class_name}) - - def on_finish_test_class(self, test_item): + + def on_finish_test_class(self, test_item): pytest_config.ya.set_test_item_node_id(test_item.nodeid) self.trace('test-finished', {'class': test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name}) - - def on_start_test_case(self, test_item): + + def on_start_test_case(self, test_item): class_name = yatest_lib.tools.to_utf8(test_item.class_name) subtest_name = yatest_lib.tools.to_utf8(test_item.test_name) - message = { + message = { 'class': class_name, 'subtest': subtest_name, - } + } if test_item.nodeid in pytest_config.test_logs: message['logs'] = pytest_config.test_logs[test_item.nodeid] pytest_config.ya.set_test_item_node_id(test_item.nodeid) self._current_test = (class_name, subtest_name) - self.trace('subtest-started', message) - + self.trace('subtest-started', message) + def on_finish_test_case(self, test_item, duration_only=False): if test_item.result is not None: - try: + try: result = canon.serialize(test_item.result) - except Exception as e: - yatest_logger.exception("Error while serializing test results") - test_item.set_error("Invalid test result: {}".format(e)) - test_item.set_status("fail") - result = None - else: - result = None + except Exception as e: + yatest_logger.exception("Error while serializing test results") + test_item.set_error("Invalid test result: {}".format(e)) + test_item.set_status("fail") + result = None + else: + result = None if duration_only and test_item.nodeid in self._test_messages: # add teardown time message = self._test_messages[test_item.nodeid] @@ -860,7 +860,7 @@ class TraceReportGenerator(object): self.trace('subtest-finished', message) self._test_messages[test_item.nodeid] = message - + def dump_suite_metrics(self): message = {"metrics": pytest_config.suite_metrics} self.trace("suite-event", message) @@ -874,28 +874,28 @@ class TraceReportGenerator(object): else: self._test_duration[test_item.nodeid] = test_item._duration - @staticmethod - def _get_comment(test_item): - msg = yatest_lib.tools.to_utf8(test_item.error) - if not msg: - return "" + @staticmethod + def _get_comment(test_item): + msg = yatest_lib.tools.to_utf8(test_item.error) + if not msg: + return "" return msg + "[[rst]]" - + def _dump_trace(self, name, value): - event = { - 'timestamp': time.time(), - 'value': value, - 'name': name - } + event = { + 'timestamp': time.time(), + 'value': value, + 'name': name + } data = yatest_lib.tools.to_str(json.dumps(event, ensure_ascii=False)) self._file.write(data + '\n') self._file.flush() - + def _check_sloppy_fork(self, name, value): if self._pid == os.getpid(): return - + yatest_logger.error("Skip tracing to avoid data corruption, name = %s, value = %s", name, value) try: @@ -950,14 +950,14 @@ class TraceReportGenerator(object): self._dump_trace(name, value) -class DryTraceReportGenerator(TraceReportGenerator): - """ - Generator does not write any information. - """ - - def __init__(self, *args, **kwargs): +class DryTraceReportGenerator(TraceReportGenerator): + """ + Generator does not write any information. + """ + + def __init__(self, *args, **kwargs): self._test_messages = {} self._test_duration = {} - - def trace(self, name, value): - pass + + def trace(self, name, value): + pass diff --git a/library/python/pytest/pytest.yatest.ini b/library/python/pytest/pytest.yatest.ini index 554a1eb84f..70d6c98516 100644 --- a/library/python/pytest/pytest.yatest.ini +++ b/library/python/pytest/pytest.yatest.ini @@ -1,7 +1,7 @@ -[pytest] +[pytest] pep8maxlinelength = 200 norecursedirs = * -pep8ignore = E127 E123 E226 E24 +pep8ignore = E127 E123 E226 E24 filterwarnings = ignore::pytest.RemovedInPytest4Warning addopts = -p no:warnings diff --git a/library/python/pytest/ya.make b/library/python/pytest/ya.make index ee3c47dccb..060c92c313 100644 --- a/library/python/pytest/ya.make +++ b/library/python/pytest/ya.make @@ -1,32 +1,32 @@ PY23_LIBRARY() - + OWNER( g:yatool dmitko ) - -PY_SRCS( + +PY_SRCS( __init__.py - main.py + main.py rewrite.py - yatest_tools.py + yatest_tools.py context.py -) - -PEERDIR( +) + +PEERDIR( contrib/python/dateutil contrib/python/ipdb contrib/python/py contrib/python/pytest contrib/python/requests - library/python/pytest/plugins - library/python/testing/yatest_common - library/python/testing/yatest_lib -) - + library/python/pytest/plugins + library/python/testing/yatest_common + library/python/testing/yatest_lib +) + RESOURCE_FILES( PREFIX library/python/pytest/ pytest.yatest.ini ) -END() +END() diff --git a/library/python/pytest/yatest_tools.py b/library/python/pytest/yatest_tools.py index c618f8ff07..6b8b896394 100644 --- a/library/python/pytest/yatest_tools.py +++ b/library/python/pytest/yatest_tools.py @@ -3,13 +3,13 @@ import collections import functools import math -import os -import re +import os +import re import sys - -import yatest_lib.tools - - + +import yatest_lib.tools + + class Subtest(object): def __init__(self, name, test_name, status, comment, elapsed, result=None, test_type=None, logs=None, cwd=None, metrics=None): self._name = name @@ -17,103 +17,103 @@ class Subtest(object): self.status = status self.elapsed = elapsed self.comment = comment - self.result = result - self.test_type = test_type + self.result = result + self.test_type = test_type self.logs = logs or {} - self.cwd = cwd - self.metrics = metrics + self.cwd = cwd + self.metrics = metrics - def __eq__(self, other): - if not isinstance(other, Subtest): - return False - return self.name == other.name and self.test_name == other.test_name + def __eq__(self, other): + if not isinstance(other, Subtest): + return False + return self.name == other.name and self.test_name == other.test_name + + def __str__(self): + return yatest_lib.tools.to_utf8(unicode(self)) - def __str__(self): - return yatest_lib.tools.to_utf8(unicode(self)) - def __unicode__(self): return u"{}::{}".format(self.test_name, self.test_name) @property def name(self): - return yatest_lib.tools.to_utf8(self._name) + return yatest_lib.tools.to_utf8(self._name) @property def test_name(self): - return yatest_lib.tools.to_utf8(self._test_name) - - def __repr__(self): - return "Subtest [{}::{} - {}[{}]: {}]".format(self.name, self.test_name, self.status, self.elapsed, self.comment) - - def __hash__(self): - return hash(str(self)) - - -class SubtestInfo(object): - + return yatest_lib.tools.to_utf8(self._test_name) + + def __repr__(self): + return "Subtest [{}::{} - {}[{}]: {}]".format(self.name, self.test_name, self.status, self.elapsed, self.comment) + + def __hash__(self): + return hash(str(self)) + + +class SubtestInfo(object): + skipped_prefix = '[SKIPPED] ' - @classmethod - def from_str(cls, s): + @classmethod + def from_str(cls, s): if s.startswith(SubtestInfo.skipped_prefix): s = s[len(SubtestInfo.skipped_prefix):] skipped = True - + else: skipped = False return SubtestInfo(*s.rsplit(TEST_SUBTEST_SEPARATOR, 1), skipped=skipped) def __init__(self, test, subtest="", skipped=False, **kwargs): - self.test = test - self.subtest = subtest + self.test = test + self.subtest = subtest self.skipped = skipped - for key, value in kwargs.iteritems(): - setattr(self, key, value) - - def __str__(self): + for key, value in kwargs.iteritems(): + setattr(self, key, value) + + def __str__(self): s = '' - + if self.skipped: s += SubtestInfo.skipped_prefix return s + TEST_SUBTEST_SEPARATOR.join([self.test, self.subtest]) - def __repr__(self): - return str(self) - - + def __repr__(self): + return str(self) + + class Status(object): GOOD, XFAIL, FAIL, XPASS, MISSING, CRASHED, TIMEOUT = range(7) - SKIPPED = -100 - NOT_LAUNCHED = -200 - CANON_DIFF = -300 - FLAKY = -1 - BY_NAME = {'good': GOOD, 'fail': FAIL, 'xfail': XFAIL, 'xpass': XPASS, 'missing': MISSING, 'crashed': CRASHED, - 'skipped': SKIPPED, 'flaky': FLAKY, 'not_launched': NOT_LAUNCHED, 'timeout': TIMEOUT, 'diff': CANON_DIFF} - TO_STR = {GOOD: 'good', FAIL: 'fail', XFAIL: 'xfail', XPASS: 'xpass', MISSING: 'missing', CRASHED: 'crashed', - SKIPPED: 'skipped', FLAKY: 'flaky', NOT_LAUNCHED: 'not_launched', TIMEOUT: 'timeout', CANON_DIFF: 'diff'} + SKIPPED = -100 + NOT_LAUNCHED = -200 + CANON_DIFF = -300 + FLAKY = -1 + BY_NAME = {'good': GOOD, 'fail': FAIL, 'xfail': XFAIL, 'xpass': XPASS, 'missing': MISSING, 'crashed': CRASHED, + 'skipped': SKIPPED, 'flaky': FLAKY, 'not_launched': NOT_LAUNCHED, 'timeout': TIMEOUT, 'diff': CANON_DIFF} + TO_STR = {GOOD: 'good', FAIL: 'fail', XFAIL: 'xfail', XPASS: 'xpass', MISSING: 'missing', CRASHED: 'crashed', + SKIPPED: 'skipped', FLAKY: 'flaky', NOT_LAUNCHED: 'not_launched', TIMEOUT: 'timeout', CANON_DIFF: 'diff'} class Test(object): - def __init__(self, name, path, status=None, comment=None, subtests=None): + def __init__(self, name, path, status=None, comment=None, subtests=None): self.name = name self.path = path - self.status = status - self.comment = comment - self.subtests = subtests or [] - - def __eq__(self, other): - if not isinstance(other, Test): - return False - return self.name == other.name and self.path == other.path - - def __str__(self): - return "Test [{} {}] - {} - {}".format(self.name, self.path, self.status, self.comment) - - def __repr__(self): - return str(self) - + self.status = status + self.comment = comment + self.subtests = subtests or [] + + def __eq__(self, other): + if not isinstance(other, Test): + return False + return self.name == other.name and self.path == other.path + + def __str__(self): + return "Test [{} {}] - {} - {}".format(self.name, self.path, self.status, self.comment) + + def __repr__(self): + return str(self) + def add_subtest(self, subtest): self.subtests.append(subtest) @@ -148,10 +148,10 @@ class YaCtx(object): pass ya_ctx = YaCtx() - -TRACE_FILE_NAME = "ytest.report.trace" - - + +TRACE_FILE_NAME = "ytest.report.trace" + + def lazy(func): mem = {} @@ -174,7 +174,7 @@ def _get_mtab(): def get_max_filename_length(dirname): - """ + """ Return maximum filename length for the filesystem :return: """ @@ -194,10 +194,10 @@ def get_unique_file_path(dir_path, filename, cache=collections.defaultdict(set)) """ Get unique filename in dir with proper filename length, using given filename/dir. File/dir won't be created (thread nonsafe) - :param dir_path: path to dir + :param dir_path: path to dir :param filename: original filename :return: unique filename - """ + """ max_suffix = 10000 # + 1 symbol for dot before suffix tail_length = int(round(math.log(max_suffix, 10))) + 1 @@ -222,83 +222,83 @@ def get_unique_file_path(dir_path, filename, cache=collections.defaultdict(set)) assert counter < max_suffix candidate = os.path.join(dir_path, filename + ".{}".format(counter)) return candidate - - -def escape_for_fnmatch(s): - return s.replace("[", "[").replace("]", "]") - - -def get_python_cmd(opts=None, use_huge=True, suite=None): + + +def escape_for_fnmatch(s): + return s.replace("[", "[").replace("]", "]") + + +def get_python_cmd(opts=None, use_huge=True, suite=None): if opts and getattr(opts, 'flags', {}).get("USE_ARCADIA_PYTHON") == "no": - return ["python"] - if suite and not suite._use_arcadia_python: - return ["python"] + return ["python"] + if suite and not suite._use_arcadia_python: + return ["python"] if use_huge: return ["$(PYTHON)/python"] ymake_path = opts.ymake_bin if opts and getattr(opts, 'ymake_bin', None) else "$(YMAKE)/ymake" return [ymake_path, "--python"] - - -def normalize_name(name): - replacements = [ - ("\\", "\\\\"), - ("\n", "\\n"), - ("\t", "\\t"), - ("\r", "\\r"), - ] - for l, r in replacements: - name = name.replace(l, r) - return name - - + + +def normalize_name(name): + replacements = [ + ("\\", "\\\\"), + ("\n", "\\n"), + ("\t", "\\t"), + ("\r", "\\r"), + ] + for l, r in replacements: + name = name.replace(l, r) + return name + + def normalize_filename(filename): - """ - Replace invalid for file names characters with string equivalents - :param some_string: string to be converted to a valid file name - :return: valid file name - """ + """ + Replace invalid for file names characters with string equivalents + :param some_string: string to be converted to a valid file name + :return: valid file name + """ not_allowed_pattern = r"[\[\]\/:*?\"\'<>|+\0\\\s\x0b\x0c]" filename = re.sub(not_allowed_pattern, ".", filename) return re.sub(r"\.{2,}", ".", filename) - - + + def get_test_log_file_path(output_dir, class_name, test_name, extension="log"): - """ - get test log file path, platform dependant - :param output_dir: dir where log file should be placed - :param class_name: test class name - :param test_name: test name - :return: test log file name - """ - if os.name == "nt": + """ + get test log file path, platform dependant + :param output_dir: dir where log file should be placed + :param class_name: test class name + :param test_name: test name + :return: test log file name + """ + if os.name == "nt": # don't add class name to the log's filename # to reduce it's length on windows filename = test_name - else: + else: filename = "{}.{}".format(class_name, test_name) if not filename: filename = "test" filename += "." + extension filename = normalize_filename(filename) return get_unique_file_path(output_dir, filename) - - -def split_node_id(nodeid, test_suffix=None): + + +def split_node_id(nodeid, test_suffix=None): path, possible_open_bracket, params = nodeid.partition('[') - separator = "::" + separator = "::" if separator in path: path, test_name = path.split(separator, 1) - else: - test_name = os.path.basename(path) - if test_suffix: - test_name += "::" + test_suffix - class_name = os.path.basename(path.strip()) - if separator in test_name: - klass_name, test_name = test_name.split(separator, 1) - if not test_suffix: - # test suffix is used for flakes and pep8, no need to add class_name as it's === class_name - class_name += separator + klass_name - if separator in test_name: - test_name = test_name.split(separator)[-1] + else: + test_name = os.path.basename(path) + if test_suffix: + test_name += "::" + test_suffix + class_name = os.path.basename(path.strip()) + if separator in test_name: + klass_name, test_name = test_name.split(separator, 1) + if not test_suffix: + # test suffix is used for flakes and pep8, no need to add class_name as it's === class_name + class_name += separator + klass_name + if separator in test_name: + test_name = test_name.split(separator)[-1] test_name += possible_open_bracket + params - return yatest_lib.tools.to_utf8(class_name), yatest_lib.tools.to_utf8(test_name) + return yatest_lib.tools.to_utf8(class_name), yatest_lib.tools.to_utf8(test_name) diff --git a/library/python/strings/strings.py b/library/python/strings/strings.py index 83856db4c4..5bfddfe78a 100644 --- a/library/python/strings/strings.py +++ b/library/python/strings/strings.py @@ -2,24 +2,24 @@ import locale import logging import six import sys -import codecs +import codecs import library.python.func -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) + - DEFAULT_ENCODING = 'utf-8' ENCODING_ERRORS_POLICY = 'replace' -def left_strip(el, prefix): - """ - Strips prefix at the left of el - """ - if el.startswith(prefix): - return el[len(prefix):] - return el +def left_strip(el, prefix): + """ + Strips prefix at the left of el + """ + if el.startswith(prefix): + return el[len(prefix):] + return el # Explicit to-text conversion @@ -49,8 +49,8 @@ def to_unicode(value, from_enc=DEFAULT_ENCODING): else: return value.decode(from_enc, errors=ENCODING_ERRORS_POLICY) return six.text_type(value) - - + + # Optional from_enc enables transcoding def to_str(value, to_enc=DEFAULT_ENCODING, from_enc=None): if isinstance(value, six.binary_type): @@ -68,7 +68,7 @@ def _convert_deep(x, enc, convert, relaxed=True): return None if isinstance(x, (six.text_type, six.binary_type)): return convert(x, enc) - if isinstance(x, dict): + if isinstance(x, dict): return {convert(k, enc): _convert_deep(v, enc, convert, relaxed) for k, v in six.iteritems(x)} if isinstance(x, list): return [_convert_deep(e, enc, convert, relaxed) for e in x] @@ -77,9 +77,9 @@ def _convert_deep(x, enc, convert, relaxed=True): if relaxed: return x - raise TypeError('unsupported type') - - + raise TypeError('unsupported type') + + def unicodize_deep(x, enc=DEFAULT_ENCODING, relaxed=True): return _convert_deep(x, enc, to_unicode, relaxed) @@ -99,7 +99,7 @@ def locale_encoding(): logger.debug('Cannot get system locale: %s', e) return None except ValueError as e: - logger.warn('Cannot get system locale: %s', e) + logger.warn('Cannot get system locale: %s', e) return None @@ -110,16 +110,16 @@ def fs_encoding(): def guess_default_encoding(): enc = locale_encoding() return enc if enc else DEFAULT_ENCODING - - + + @library.python.func.memoize() -def get_stream_encoding(stream): - if stream.encoding: - try: - codecs.lookup(stream.encoding) - return stream.encoding - except LookupError: - pass +def get_stream_encoding(stream): + if stream.encoding: + try: + codecs.lookup(stream.encoding) + return stream.encoding + except LookupError: + pass return DEFAULT_ENCODING diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py index 5b002f7aea..3e3b7234ef 100644 --- a/library/python/testing/import_test/import_test.py +++ b/library/python/testing/import_test/import_test.py @@ -2,7 +2,7 @@ from __future__ import print_function import os import re -import sys +import sys import time import traceback @@ -65,8 +65,8 @@ def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): s = time.time() if module == '__main__': importer.load_module('__main__', '__main__py') - elif module.endswith('.__init__'): - __import__(module[:-len('.__init__')]) + elif module.endswith('.__init__'): + __import__(module[:-len('.__init__')]) else: __import__(module) diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py index 8bc3b8a40b..5ef9c5c189 100644 --- a/library/python/testing/recipe/__init__.py +++ b/library/python/testing/recipe/__init__.py @@ -2,8 +2,8 @@ from __future__ import print_function import os import sys -import json -import logging +import json +import logging import argparse from yatest_lib.ya import Ya @@ -17,22 +17,22 @@ sanitizer_extra_checks = None def _setup_logging(level=logging.DEBUG): - root_logger = logging.getLogger() - root_logger.setLevel(level) - - log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' - - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setLevel(logging.DEBUG) - formatter = logging.Formatter(log_format) - stdout_handler.setFormatter(formatter) - root_logger.addHandler(stdout_handler) - - + root_logger = logging.getLogger() + root_logger.setLevel(level) + + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter(log_format) + stdout_handler.setFormatter(formatter) + root_logger.addHandler(stdout_handler) + + def get_options(): parser = argparse.ArgumentParser() parser.add_argument("--show-cwd", action="store_true", dest="show_cwd", default=False, help="show recipe cwd") - parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode") + parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode") parser.add_argument("--test-stderr", action="store_true", dest="test_stderr", default=False, help="test stderr") parser.add_argument("--pdb", action="store_true", dest="pdb", default=False, help="run pdb on error") parser.add_argument("--sanitizer-extra-checks", dest="sanitizer_extra_checks", action="store_true", default=False, help="enables extra checks for tests built with sanitizers") @@ -42,7 +42,7 @@ def get_options(): global ya, sanitizer_extra_checks, collect_cores _setup_logging() - + context = { "test_stderr": args.test_stderr, } diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make index 2672c6ca6e..883bc8d7ab 100644 --- a/library/python/testing/ya.make +++ b/library/python/testing/ya.make @@ -1,22 +1,22 @@ OWNER(g:yatest) - -RECURSE( - behave + +RECURSE( + behave deprecated fake_ya_package filter - gtest - gtest/test - gtest/test/gtest + gtest + gtest/test + gtest/test/gtest pyremock - pytest_runner - pytest_runner/example - pytest_runner/test + pytest_runner + pytest_runner/example + pytest_runner/test recipe system_info types_test yapackage yapackage/test - yatest_common - yatest_lib -) + yatest_common + yatest_lib +) diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make index 0cd9922c06..5662db4c5d 100644 --- a/library/python/testing/yatest_common/ya.make +++ b/library/python/testing/yatest_common/ya.make @@ -1,40 +1,40 @@ OWNER(g:yatest) - + PY23_LIBRARY() - + OWNER(g:yatest) - + NO_EXTENDED_SOURCE_SEARCH() -PY_SRCS( - TOP_LEVEL +PY_SRCS( + TOP_LEVEL yatest/__init__.py - yatest/common/__init__.py - yatest/common/benchmark.py - yatest/common/canonical.py - yatest/common/environment.py - yatest/common/errors.py - yatest/common/legacy.py + yatest/common/__init__.py + yatest/common/benchmark.py + yatest/common/canonical.py + yatest/common/environment.py + yatest/common/errors.py + yatest/common/legacy.py yatest/common/misc.py - yatest/common/network.py - yatest/common/path.py - yatest/common/process.py - yatest/common/runtime.py - yatest/common/runtime_java.py - yatest/common/tags.py + yatest/common/network.py + yatest/common/path.py + yatest/common/process.py + yatest/common/runtime.py + yatest/common/runtime_java.py + yatest/common/tags.py ) - + PEERDIR( contrib/python/six library/python/cores library/python/filelock library/python/fs -) - +) + IF (NOT CATBOOST_OPENSOURCE) PEERDIR( library/python/coredump_filter ) ENDIF() -END() +END() diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py index 19593ff8d2..b846b3317a 100644 --- a/library/python/testing/yatest_common/yatest/__init__.py +++ b/library/python/testing/yatest_common/yatest/__init__.py @@ -1,3 +1,3 @@ -__all__ = ["common"] - +__all__ = ["common"] + from . import common diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py index 500dfaca7e..c3784cbe4c 100644 --- a/library/python/testing/yatest_common/yatest/common/benchmark.py +++ b/library/python/testing/yatest_common/yatest/common/benchmark.py @@ -1,22 +1,22 @@ -import json - +import json + from . import process from . import runtime - - -def execute_benchmark(path, budget=None, threads=None): - """ - Run benchmark and return values - :param path: path to benchmark binary + + +def execute_benchmark(path, budget=None, threads=None): + """ + Run benchmark and return values + :param path: path to benchmark binary :param budget: time budget, sec (supported only by ybenchmark) :param threads: number of threads to run benchmark (supported only by ybenchmark) - :return: map of benchmark values - """ - benchmark_path = runtime.binary_path(path) + :return: map of benchmark values + """ + benchmark_path = runtime.binary_path(path) cmd = [benchmark_path, "--benchmark_format=json"] - if budget is not None: - cmd += ["-b", str(budget)] - if threads is not None: - cmd += ["-t", str(threads)] - res = process.execute(cmd) - return json.loads(res.std_out) + if budget is not None: + cmd += ["-b", str(budget)] + if threads is not None: + cmd += ["-t", str(threads)] + res = process.execute(cmd) + return json.loads(res.std_out) diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py index 9c02fe1c11..b6a136d3e9 100644 --- a/library/python/testing/yatest_common/yatest/common/canonical.py +++ b/library/python/testing/yatest_common/yatest/common/canonical.py @@ -1,17 +1,17 @@ -import os -import logging -import shutil -import tempfile - +import os +import logging +import shutil +import tempfile + import six from . import process from . import runtime from . import path - -yatest_logger = logging.getLogger("ya.test") - - + +yatest_logger = logging.getLogger("ya.test") + + def _copy(src, dst, universal_lines=False): if universal_lines: with open(dst, "wb") as f: @@ -21,21 +21,21 @@ def _copy(src, dst, universal_lines=False): shutil.copy(src, dst) -def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None): - """ - Create canonical file that can be returned from a test - :param path: path to the file - :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used - :param local: save file locally, otherwise move to sandbox +def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None): + """ + Create canonical file that can be returned from a test + :param path: path to the file + :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used + :param local: save file locally, otherwise move to sandbox :param universal_lines: normalize EOL - :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - abs_path = os.path.abspath(path) - assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) - tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path()) - safe_path = os.path.join(tempdir, os.path.basename(abs_path)) - # if the created file is in output_path, we copy it, so that it will be available when the tests finishes + :param diff_tool_timeout: timeout for running diff tool + :return: object that can be canonized + """ + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path()) + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + # if the created file is in output_path, we copy it, so that it will be available when the tests finishes _copy(path, safe_path, universal_lines=universal_lines) if diff_tool: if not isinstance(diff_tool, six.string_types): @@ -44,133 +44,133 @@ def canonical_file(path, diff_tool=None, local=False, universal_lines=False, dif raise Exception("Invalid custom diff-tool: not cmd") except: raise Exception("Invalid custom diff-tool: not binary path") - return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) - - -def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None): - abs_path = os.path.abspath(path) - assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) - assert os.path.isdir(abs_path), "Path {} is not a directory".format(path) - if diff_file_name and not diff_tool: - raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir") - tempdir = tempfile.mkdtemp() - safe_path = os.path.join(tempdir, os.path.basename(abs_path)) - shutil.copytree(abs_path, safe_path) - return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) - - -def canonical_execute( - binary, args=None, check_exit_code=True, - shell=False, timeout=None, cwd=None, - env=None, stdin=None, stderr=None, creationflags=0, + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None): + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + assert os.path.isdir(abs_path), "Path {} is not a directory".format(path) + if diff_file_name and not diff_tool: + raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir") + tempdir = tempfile.mkdtemp() + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + shutil.copytree(abs_path, safe_path) + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_execute( + binary, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, + env=None, stdin=None, stderr=None, creationflags=0, file_name=None, save_locally=False, close_fds=False, diff_tool=None, diff_file_name=None, diff_tool_timeout=None, -): - """ - Shortcut to execute a binary and canonize its stdout +): + """ + Shortcut to execute a binary and canonize its stdout :param binary: absolute path to the binary - :param args: binary arguments - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stderr: command stderr - :param creationflags: command creation flags + :param args: binary arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags :param file_name: output file name. if not specified program name will be used - :param diff_tool: path to custome diff tool - :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - if type(binary) == list: - command = binary - else: - command = [binary] - command += _prepare_args(args) - if shell: - command = " ".join(command) - execute_args = locals() - del execute_args["binary"] - del execute_args["args"] + :return: object that can be canonized + """ + if type(binary) == list: + command = binary + else: + command = [binary] + command += _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["binary"] + del execute_args["args"] del execute_args["file_name"] - del execute_args["save_locally"] - del execute_args["diff_tool"] - del execute_args["diff_file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] del execute_args["diff_tool_timeout"] - if not file_name and stdin: - file_name = os.path.basename(stdin.name) + if not file_name and stdin: + file_name = os.path.basename(stdin.name) return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) - - -def canonical_py_execute( - script_path, args=None, check_exit_code=True, - shell=False, timeout=None, cwd=None, env=None, - stdin=None, stderr=None, creationflags=0, - file_name=None, save_locally=False, close_fds=False, + + +def canonical_py_execute( + script_path, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, env=None, + stdin=None, stderr=None, creationflags=0, + file_name=None, save_locally=False, close_fds=False, diff_tool=None, diff_file_name=None, diff_tool_timeout=None, -): - """ - Shortcut to execute a python script and canonize its stdout - :param script_path: path to the script arcadia relative - :param args: script arguments - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stderr: command stderr - :param creationflags: command creation flags +): + """ + Shortcut to execute a python script and canonize its stdout + :param script_path: path to the script arcadia relative + :param args: script arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags :param file_name: output file name. if not specified program name will be used - :param diff_tool: path to custome diff tool - :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found :param diff_tool_timeout: timeout for running diff tool - :return: object that can be canonized - """ - command = [runtime.source_path(script_path)] + _prepare_args(args) - if shell: - command = " ".join(command) - execute_args = locals() - del execute_args["script_path"] - del execute_args["args"] + :return: object that can be canonized + """ + command = [runtime.source_path(script_path)] + _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["script_path"] + del execute_args["args"] del execute_args["file_name"] - del execute_args["save_locally"] - del execute_args["diff_tool"] - del execute_args["diff_file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] del execute_args["diff_tool_timeout"] return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) - - -def _prepare_args(args): - if args is None: - args = [] + + +def _prepare_args(args): + if args is None: + args = [] if isinstance(args, six.string_types): - args = map(lambda a: a.strip(), args.split()) - return args - - + args = map(lambda a: a.strip(), args.split()) + return args + + def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout): - res = excutor(**kwargs) - command = kwargs["command"] - file_name = file_name or process.get_command_name(command) - if file_name.endswith(".exe"): - file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names - out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name)) - err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name)) - - try: - os.makedirs(os.path.dirname(out_file_path)) - except OSError: - pass - + res = excutor(**kwargs) + command = kwargs["command"] + file_name = file_name or process.get_command_name(command) + if file_name.endswith(".exe"): + file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names + out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name)) + err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name)) + + try: + os.makedirs(os.path.dirname(out_file_path)) + except OSError: + pass + with open(out_file_path, "wb") as out_file: - yatest_logger.debug("Will store file in %s", out_file_path) - out_file.write(res.std_out) - - if res.std_err: + yatest_logger.debug("Will store file in %s", out_file_path) + out_file.write(res.std_out) + + if res.std_err: with open(err_file_path, "wb") as err_file: - err_file.write(res.std_err) - + err_file.write(res.std_err) + return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py index f3d55058e4..43f48d0958 100644 --- a/library/python/testing/yatest_common/yatest/common/environment.py +++ b/library/python/testing/yatest_common/yatest/common/environment.py @@ -1,5 +1,5 @@ # coding: utf-8 - + def extend_env_var(env, name, value, sep=":"): return sep.join(filter(None, [env.get(name), value])) diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py index 5cdd62c6d0..8c038fc381 100644 --- a/library/python/testing/yatest_common/yatest/common/errors.py +++ b/library/python/testing/yatest_common/yatest/common/errors.py @@ -1,20 +1,20 @@ import os -import sys - - -class RestartTestException(Exception): - - def __init__(self, *args, **kwargs): +import sys + + +class RestartTestException(Exception): + + def __init__(self, *args, **kwargs): super(RestartTestException, self).__init__(*args, **kwargs) - sys.stderr.write("##restart-test##\n") - sys.stderr.flush() + sys.stderr.write("##restart-test##\n") + sys.stderr.flush() os.environ["FORCE_EXIT_TESTSFAILED"] = "1" - - -class InfrastructureException(Exception): - - def __init__(self, *args, **kwargs): + + +class InfrastructureException(Exception): + + def __init__(self, *args, **kwargs): super(InfrastructureException, self).__init__(*args, **kwargs) - sys.stderr.write("##infrastructure-error##\n") - sys.stderr.flush() + sys.stderr.write("##infrastructure-error##\n") + sys.stderr.flush() os.environ["FORCE_EXIT_TESTSFAILED"] = "1" diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py index cfb6975802..459972d253 100644 --- a/library/python/testing/yatest_common/yatest/common/legacy.py +++ b/library/python/testing/yatest_common/yatest/common/legacy.py @@ -1,12 +1,12 @@ from . import canonical - - -def old_canonical_file(output_file_name, storage_md5): - import yalibrary.svn - yalibrary.svn.run_svn([ - 'export', - 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5, - output_file_name, - "--force" - ]) - return canonical.canonical_file(output_file_name) + + +def old_canonical_file(output_file_name, storage_md5): + import yalibrary.svn + yalibrary.svn.run_svn([ + 'export', + 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5, + output_file_name, + "--force" + ]) + return canonical.canonical_file(output_file_name) diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py index d31df2d278..6fed7dda8a 100644 --- a/library/python/testing/yatest_common/yatest/common/path.py +++ b/library/python/testing/yatest_common/yatest/common/path.py @@ -1,5 +1,5 @@ # coding=utf-8 - + import errno import os import shutil @@ -8,19 +8,19 @@ import contextlib import library.python.fs as lpf -def replace_in_file(path, old, new): - """ - Replace text occurrences in a file - :param path: path to the file - :param old: text to replace - :param new: replacement - """ - with open(path) as fp: - content = fp.read() - +def replace_in_file(path, old, new): + """ + Replace text occurrences in a file + :param path: path to the file + :param old: text to replace + :param new: replacement + """ + with open(path) as fp: + content = fp.read() + lpf.ensure_removed(path) - with open(path, 'w') as fp: - fp.write(content.replace(old, new)) + with open(path, 'w') as fp: + fp.write(content.replace(old, new)) @contextlib.contextmanager @@ -38,8 +38,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): Copy an entire directory of files into an existing directory instead of raising Exception what shtuil.copytree does ''' - if not os.path.exists(dst) and os.path.isdir(src): - os.makedirs(dst) + if not os.path.exists(dst) and os.path.isdir(src): + os.makedirs(dst) for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) @@ -54,8 +54,8 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): postprocessing(os.path.join(root, path), False) for path in files: postprocessing(os.path.join(root, path), True) - - + + def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000): def atomic_file_create(path): try: @@ -79,12 +79,12 @@ def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10 return False raise e - file_path = os.path.join(dir_path, file_pattern) + file_path = os.path.join(dir_path, file_pattern) lpf.ensure_dir(os.path.dirname(file_path)) - file_counter = 0 + file_counter = 0 handler = atomic_file_create if create_file else atomic_dir_create while os.path.exists(file_path) or not handler(file_path): - file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter)) - file_counter += 1 + file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter)) + file_counter += 1 assert file_counter < max_suffix - return file_path + return file_path diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py index 672ebe0823..a8bcc21f51 100644 --- a/library/python/testing/yatest_common/yatest/common/process.py +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -1,16 +1,16 @@ # coding: utf-8 -import os +import os import re -import time -import signal +import time +import signal import shutil -import logging -import tempfile -import subprocess -import errno +import logging +import tempfile +import subprocess +import errno import distutils.version - + import six try: @@ -22,13 +22,13 @@ except ImportError: from . import runtime from . import path from . import environment - - + + MAX_OUT_LEN = 1000 * 1000 # 1 mb MAX_MESSAGE_LEN = 1500 SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)" GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)") -yatest_logger = logging.getLogger("ya.test") +yatest_logger = logging.getLogger("ya.test") def truncate(s, size): @@ -40,45 +40,45 @@ def truncate(s, size): return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):] -def get_command_name(command): +def get_command_name(command): return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0]) - - -class ExecutionError(Exception): - - def __init__(self, execution_result): + + +class ExecutionError(Exception): + + def __init__(self, execution_result): if not isinstance(execution_result.command, six.string_types): command = " ".join(str(arg) for arg in execution_result.command) else: command = execution_result.command - message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format( + message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format( command=command, - code=execution_result.exit_code, - err=_format_error(execution_result.std_err)) + code=execution_result.exit_code, + err=_format_error(execution_result.std_err)) if cores: if execution_result.backtrace: message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) else: message += "Backtrace is not available: module cores isn't available" - super(ExecutionError, self).__init__(message) + super(ExecutionError, self).__init__(message) + self.execution_result = execution_result + + +class TimeoutError(Exception): + pass + + +class ExecutionTimeoutError(TimeoutError): + def __init__(self, execution_result, *args, **kwargs): + super(ExecutionTimeoutError, self).__init__(args, kwargs) self.execution_result = execution_result - - -class TimeoutError(Exception): - pass - - -class ExecutionTimeoutError(TimeoutError): - def __init__(self, execution_result, *args, **kwargs): - super(ExecutionTimeoutError, self).__init__(args, kwargs) - self.execution_result = execution_result - - -class InvalidExecutionStateError(Exception): - pass - - + + +class InvalidExecutionStateError(Exception): + pass + + class SignalInterruptionError(Exception): def __init__(self, message=None): super(SignalInterruptionError, self).__init__(message) @@ -89,39 +89,39 @@ class InvalidCommandError(Exception): pass -class _Execution(object): - +class _Execution(object): + def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): - self._command = command - self._process = process - self._out_file = out_file - self._err_file = err_file - self._std_out = None - self._std_err = None + self._command = command + self._process = process + self._out_file = out_file + self._err_file = err_file + self._std_out = None + self._std_err = None self._elapsed = None self._start = time.time() - self._process_progress_listener = process_progress_listener + self._process_progress_listener = process_progress_listener self._cwd = cwd or os.getcwd() self._collect_cores = collect_cores self._backtrace = '' self._check_sanitizer = check_sanitizer - self._metrics = {} + self._metrics = {} self._started = started self._user_stdout = bool(user_stdout) self._user_stderr = bool(user_stderr) self._exit_code = None - if process_progress_listener: - process_progress_listener.open(command, process, out_file, err_file) - - @property - def running(self): - return self._process.poll() is None - - def kill(self): - if self.running: - self._save_outputs(False) - _kill_process_tree(self._process.pid) - self._clean_files() + if process_progress_listener: + process_progress_listener.open(command, process, out_file, err_file) + + @property + def running(self): + return self._process.poll() is None + + def kill(self): + if self.running: + self._save_outputs(False) + _kill_process_tree(self._process.pid) + self._clean_files() # DEVTOOLS-2347 yatest_logger.debug("Process status before wait_for: %s", self.running) try: @@ -130,64 +130,64 @@ class _Execution(object): yatest_logger.debug("Process status after wait_for: %s", self.running) yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) raise - else: - raise InvalidExecutionStateError("Cannot kill a stopped process") - + else: + raise InvalidExecutionStateError("Cannot kill a stopped process") + def terminate(self): if self.running: self._process.terminate() - @property - def process(self): - return self._process - - @property - def command(self): - return self._command - - @property + @property + def process(self): + return self._process + + @property + def command(self): + return self._command + + @property def returncode(self): return self.exit_code @property - def exit_code(self): + def exit_code(self): """ Deprecated, use returncode """ if self._exit_code is None: self._exit_code = self._process.returncode return self._exit_code - - @property + + @property def stdout(self): return self.std_out @property - def std_out(self): + def std_out(self): """ Deprecated, use stdout """ - if self._std_out is not None: - return self._std_out + if self._std_out is not None: + return self._std_out if self._process.stdout and not self._user_stdout: self._std_out = self._process.stdout.read() return self._std_out - - @property + + @property def stderr(self): return self.std_err @property - def std_err(self): + def std_err(self): """ Deprecated, use stderr """ - if self._std_err is not None: - return self._std_err + if self._std_err is not None: + return self._std_err if self._process.stderr and not self._user_stderr: self._std_err = self._process.stderr.read() return self._std_err - + @property def elapsed(self): return self._elapsed @@ -196,14 +196,14 @@ class _Execution(object): def backtrace(self): return self._backtrace - @property - def metrics(self): - return self._metrics - - def _save_outputs(self, clean_files=True): - if self._process_progress_listener: - self._process_progress_listener() - self._process_progress_listener.close() + @property + def metrics(self): + return self._metrics + + def _save_outputs(self, clean_files=True): + if self._process_progress_listener: + self._process_progress_listener() + self._process_progress_listener.close() if not self._user_stdout: if self._out_file is None: pass @@ -223,14 +223,14 @@ class _Execution(object): else: self._std_err = self._process.stderr.read() - if clean_files: - self._clean_files() - yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) - yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed) - if self._metrics: + if clean_files: + self._clean_files() + yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) + yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed) + if self._metrics: for key, value in six.iteritems(self._metrics): - yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value) - + yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value) + # Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str. printable_std_out, err = _try_convert_bytes_to_string(self._std_out) if err: @@ -244,14 +244,14 @@ class _Execution(object): yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN)) yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN)) - def _clean_files(self): + def _clean_files(self): if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: self._err_file.close() self._err_file = None if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: self._out_file.close() self._out_file = None - + def _recover_core(self): core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) if core_path: @@ -260,7 +260,7 @@ class _Execution(object): if store_cores: new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) # Copy core dump file, because it may be overwritten - yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) + yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) shutil.copyfile(core_path, new_core_path) core_path = new_core_path @@ -281,92 +281,92 @@ class _Execution(object): else: runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) - def wait(self, check_exit_code=True, timeout=None, on_timeout=None): - def _wait(): + def wait(self, check_exit_code=True, timeout=None, on_timeout=None): + def _wait(): finished = None interrupted = False - try: - if hasattr(os, "wait4"): - try: + try: + if hasattr(os, "wait4"): + try: if hasattr(subprocess, "_eintr_retry_call"): pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0) else: # PEP 475 pid, sts, rusage = os.wait4(self._process.pid, 0) finished = time.time() - self._process._handle_exitstatus(sts) - for field in [ - "ru_idrss", - "ru_inblock", - "ru_isrss", - "ru_ixrss", - "ru_majflt", - "ru_maxrss", - "ru_minflt", - "ru_msgrcv", - "ru_msgsnd", - "ru_nivcsw", - "ru_nsignals", - "ru_nswap", - "ru_nvcsw", - "ru_oublock", - "ru_stime", - "ru_utime", - ]: - if hasattr(rusage, field): - self._metrics[field.replace("ru_", "")] = getattr(rusage, field) - except OSError as exc: - - if exc.errno == errno.ECHILD: - yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") - else: - raise + self._process._handle_exitstatus(sts) + for field in [ + "ru_idrss", + "ru_inblock", + "ru_isrss", + "ru_ixrss", + "ru_majflt", + "ru_maxrss", + "ru_minflt", + "ru_msgrcv", + "ru_msgsnd", + "ru_nivcsw", + "ru_nsignals", + "ru_nswap", + "ru_nvcsw", + "ru_oublock", + "ru_stime", + "ru_utime", + ]: + if hasattr(rusage, field): + self._metrics[field.replace("ru_", "")] = getattr(rusage, field) + except OSError as exc: + + if exc.errno == errno.ECHILD: + yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") + else: + raise except SignalInterruptionError: interrupted = True raise - finally: + finally: if not interrupted: self._process.wait() # this has to be here unconditionally, so that all process properties are set - + if not finished: finished = time.time() self._metrics["wtime"] = round(finished - self._started, 3) - try: + try: if timeout: process_is_finished = lambda: not self.running fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) - try: - wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) - except TimeoutError as e: - if on_timeout: + try: + wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) + except TimeoutError as e: + if on_timeout: yatest_logger.debug("Calling user specified on_timeout function") - try: - on_timeout(self, timeout) - except Exception: - yatest_logger.exception("Exception while calling on_timeout") - raise ExecutionTimeoutError(self, str(e)) + try: + on_timeout(self, timeout) + except Exception: + yatest_logger.exception("Exception while calling on_timeout") + raise ExecutionTimeoutError(self, str(e)) # Wait should be always called here, it finalizes internal states of its process and sets up return code - _wait() + _wait() except BaseException as e: - _kill_process_tree(self._process.pid) - _wait() - yatest_logger.debug("Command exception: %s", e) + _kill_process_tree(self._process.pid) + _wait() + yatest_logger.debug("Command exception: %s", e) raise - finally: + finally: self._elapsed = time.time() - self._start self._save_outputs() self.verify_no_coredumps() - + self._finalise(check_exit_code) def _finalise(self, check_exit_code): - # Set the signal (negative number) which caused the process to exit - if check_exit_code and self.exit_code != 0: - yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", - self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) - raise ExecutionError(self) - + # Set the signal (negative number) which caused the process to exit + if check_exit_code and self.exit_code != 0: + yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", + self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) + # Don't search for sanitize errors if stderr was redirected self.verify_sanitize_errors() @@ -394,11 +394,11 @@ class _Execution(object): if match: yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) raise ExecutionError(self) - else: - yatest_logger.debug("No sanitizer errors found") + else: + yatest_logger.debug("No sanitizer errors found") else: - yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) - + yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) + def on_timeout_gen_coredump(exec_obj, _): """ @@ -413,38 +413,38 @@ def on_timeout_gen_coredump(exec_obj, _): pass -def execute( - command, check_exit_code=True, - shell=False, timeout=None, - cwd=None, env=None, - stdin=None, stdout=None, stderr=None, - creationflags=0, wait=True, - process_progress_listener=None, close_fds=False, - collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, +def execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False, + collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, executor=_Execution, -): - """ - Executes a command - :param command: command: can be a list of arguments or a string - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stdout: command stdout - :param stderr: command stderr - :param creationflags: command creation flags - :param wait: should wait until the command finishes - :param process_progress_listener=object that is polled while execution is in progress - :param close_fds: subrpocess.Popen close_fds args +): + """ + Executes a command + :param command: command: can be a list of arguments or a string + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress + :param close_fds: subrpocess.Popen close_fds args :param collect_cores: recover core dump files if shell == False :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors - :param preexec_fn: subrpocess.Popen preexec_fn arg - :param on_timeout: on_timeout(<execution object>, <timeout value>) callback - + :param preexec_fn: subrpocess.Popen preexec_fn arg + :param on_timeout: on_timeout(<execution object>, <timeout value>) callback + :return _Execution: Execution object - """ + """ if env is None: env = os.environ.copy() else: @@ -462,9 +462,9 @@ def execute( if var not in env and var in os.environ: env[var] = os.environ[var] - if not wait and timeout is not None: - raise ValueError("Incompatible arguments 'timeout' and wait=False") - + if not wait and timeout is not None: + raise ValueError("Incompatible arguments 'timeout' and wait=False") + # if subprocess.PIPE in [stdout, stderr]: # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") @@ -481,15 +481,15 @@ def execute( is_pipe = stream == subprocess.PIPE return stream, not is_pipe - # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE, - # as it can cause processes hangs, but use tempfiles instead + # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE, + # as it can cause processes hangs, but use tempfiles instead out_file, user_stdout = get_out_stream(stdout, 'out') err_file, user_stderr = get_out_stream(stderr, 'err') - in_file = stdin - - if shell and type(command) == list: - command = " ".join(command) - + in_file = stdin + + if shell and type(command) == list: + command = " ".join(command) + if shell: collect_cores = False check_sanitizer = False @@ -516,7 +516,7 @@ def execute( name = "PIPE" if stdin == subprocess.PIPE else stdin.name yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) else: - yatest_logger.debug("Executing '%s' in '%s'", command, cwd) + yatest_logger.debug("Executing '%s' in '%s'", command, cwd) # XXX started = time.time() @@ -528,11 +528,11 @@ def execute( yatest_logger.debug("Command pid: %s", process.pid) res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) - if wait: - res.wait(check_exit_code, timeout, on_timeout) - return res - - + if wait: + res.wait(check_exit_code, timeout, on_timeout) + return res + + def _get_command_output_file(cmd, ext): parts = [get_command_name(cmd)] if 'YA_RETRY_INDEX' in os.environ: @@ -570,74 +570,74 @@ def py_execute( creationflags=0, wait=True, process_progress_listener=None, close_fds=False ): - """ - Executes a command with the arcadia python - :param command: command to pass to python - :param check_exit_code: will raise ExecutionError if the command exits with non zero code - :param shell: use shell to run the command - :param timeout: execution timeout - :param cwd: working directory - :param env: command environment - :param stdin: command stdin - :param stdout: command stdout - :param stderr: command stderr - :param creationflags: command creation flags - :param wait: should wait until the command finishes - :param process_progress_listener=object that is polled while execution is in progress + """ + Executes a command with the arcadia python + :param command: command to pass to python + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress :return _Execution: Execution object - """ + """ if isinstance(command, six.string_types): - command = [command] - command = [runtime.python_path()] + command - if shell: - command = " ".join(command) - return execute(**locals()) - - -def _format_error(error): - return truncate(error, MAX_MESSAGE_LEN) - - -def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None): - """ - Tries to execute `check_function` for `timeout` seconds. - Continue until function returns nonfalse value. - If function doesn't return nonfalse value for `timeout` seconds - OperationTimeoutException is raised. - Return first nonfalse result returned by `checkFunction`. - """ - if sleep_time <= 0: - raise ValueError("Incorrect sleep time value {}".format(sleep_time)) - if timeout < 0: - raise ValueError("Incorrect timeout value {}".format(timeout)) - start = time.time() - while start + timeout > time.time(): - if on_check_condition: - on_check_condition() - - res = check_function() - if res: - return res - time.sleep(sleep_time) - - message = "{} second(s) wait timeout has expired".format(timeout) - if fail_message: - message += ": {}".format(fail_message) + command = [command] + command = [runtime.python_path()] + command + if shell: + command = " ".join(command) + return execute(**locals()) + + +def _format_error(error): + return truncate(error, MAX_MESSAGE_LEN) + + +def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None): + """ + Tries to execute `check_function` for `timeout` seconds. + Continue until function returns nonfalse value. + If function doesn't return nonfalse value for `timeout` seconds + OperationTimeoutException is raised. + Return first nonfalse result returned by `checkFunction`. + """ + if sleep_time <= 0: + raise ValueError("Incorrect sleep time value {}".format(sleep_time)) + if timeout < 0: + raise ValueError("Incorrect timeout value {}".format(timeout)) + start = time.time() + while start + timeout > time.time(): + if on_check_condition: + on_check_condition() + + res = check_function() + if res: + return res + time.sleep(sleep_time) + + message = "{} second(s) wait timeout has expired".format(timeout) + if fail_message: + message += ": {}".format(fail_message) raise TimeoutError(truncate(message, MAX_MESSAGE_LEN)) - + def _kill_process_tree(process_pid, target_pid_signal=None): - """ - Kills child processes, req. Note that psutil should be installed - @param process_pid: parent id to search for descendants - """ - yatest_logger.debug("Killing process %s", process_pid) - if os.name == 'nt': - _win_kill_process_tree(process_pid) - else: + """ + Kills child processes, req. Note that psutil should be installed + @param process_pid: parent id to search for descendants + """ + yatest_logger.debug("Killing process %s", process_pid) + if os.name == 'nt': + _win_kill_process_tree(process_pid) + else: _nix_kill_process_tree(process_pid, target_pid_signal) - - + + def _nix_get_proc_children(pid): try: cmd = ["pgrep", "-P", str(pid)] @@ -654,21 +654,21 @@ def _get_binname(pid): def _nix_kill_process_tree(pid, target_pid_signal=None): - """ - Kills the process tree. - """ + """ + Kills the process tree. + """ yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) - - def try_to_send_signal(pid, sig): - try: - os.kill(pid, sig) + + def try_to_send_signal(pid, sig): + try: + os.kill(pid, sig) yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) - except Exception as exc: + except Exception as exc: yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) - - try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. - - # Get the child process PID list. + + try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. + + # Get the child process PID list. child_pids = _nix_get_proc_children(pid) # Stop the child processes. for child_pid in child_pids: @@ -679,16 +679,16 @@ def _nix_kill_process_tree(pid, target_pid_signal=None): # Skip the error and continue killing. yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) continue - + try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process. - - # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent - # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html - try_to_send_signal(pid, signal.SIGCONT) - - -def _win_kill_process_tree(pid): - subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) + + # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent + # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html + try_to_send_signal(pid, signal.SIGCONT) + + +def _win_kill_process_tree(pid): + subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) def _run_readelf(binary_path): diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py index aa9161aead..e55e193446 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime.py +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -3,49 +3,49 @@ import functools import json import os import threading - + import six - + _lock = threading.Lock() def _get_ya_config(): - try: + try: import library.python.pytest.plugins.ya as ya_plugin if ya_plugin.pytest_config is not None: return ya_plugin.pytest_config - import pytest + import pytest return pytest.config - except (ImportError, AttributeError): + except (ImportError, AttributeError): try: import library.python.testing.recipe if library.python.testing.recipe.ya: return library.python.testing.recipe except (ImportError, AttributeError): pass - raise NotImplementedError("yatest.common.* is only available from the testing runtime") - - + raise NotImplementedError("yatest.common.* is only available from the testing runtime") + + def _get_ya_plugin_instance(): return _get_ya_config().ya -def _norm_path(path): - if path is None: - return None +def _norm_path(path): + if path is None: + return None assert isinstance(path, six.string_types) - if "\\" in path: - raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path)) - return os.path.normpath(path) - - -def _join_path(main_path, path): - if not path: - return main_path - return os.path.join(main_path, _norm_path(path)) - - + if "\\" in path: + raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path)) + return os.path.normpath(path) + + +def _join_path(main_path, path): + if not path: + return main_path + return os.path.join(main_path, _norm_path(path)) + + def not_test(func): """ Mark any function as not a test for py.test @@ -60,70 +60,70 @@ def not_test(func): def source_path(path=None): - """ - Get source path inside arcadia + """ + Get source path inside arcadia :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya') - :return: absolute path to the source folder - """ + :return: absolute path to the source folder + """ return _join_path(_get_ya_plugin_instance().source_root, path) - - + + def build_path(path=None): - """ - Get path inside build directory + """ + Get path inside build directory :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin') - :return: absolute path inside build directory - """ + :return: absolute path inside build directory + """ return _join_path(_get_ya_plugin_instance().build_root, path) - - + + def java_path(): """ - [DEPRECATED] Get path to java + [DEPRECATED] Get path to java :return: absolute path to java """ from . import runtime_java return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk'))) -def java_home(): - """ - Get jdk directory path - """ +def java_home(): + """ + Get jdk directory path + """ from . import runtime_java - jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) - if not jdk_dir: - raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") - return jdk_dir - - -def java_bin(): - """ - Get path to the java binary + jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) + if not jdk_dir: + raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") + return jdk_dir + + +def java_bin(): + """ + Get path to the java binary Requires DEPENDS(jdk) - """ - return os.path.join(java_home(), "bin", "java") - - + """ + return os.path.join(java_home(), "bin", "java") + + def data_path(path=None): - """ - Get path inside arcadia_tests_data directory + """ + Get path inside arcadia_tests_data directory :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") - :return: absolute path inside arcadia_tests_data - """ + :return: absolute path inside arcadia_tests_data + """ return _join_path(_get_ya_plugin_instance().data_root, path) - - + + def output_path(path=None): - """ - Get path inside the current test suite output dir. - Placing files to this dir guarantees that files will be accessible after the test suite execution. - :param path: path relative to the test suite output dir - :return: absolute path inside the test suite output dir - """ + """ + Get path inside the current test suite output dir. + Placing files to this dir guarantees that files will be accessible after the test suite execution. + :param path: path relative to the test suite output dir + :return: absolute path inside the test suite output dir + """ return _join_path(_get_ya_plugin_instance().output_dir, path) - - + + def ram_drive_path(path=None): """ :param path: path relative to the ram drive. @@ -144,60 +144,60 @@ def output_ram_drive_path(path=None): def binary_path(path=None): - """ - Get path to the built binary + """ + Get path to the built binary :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin') - :return: absolute path to the binary - """ + :return: absolute path to the binary + """ path = _norm_path(path) return _get_ya_plugin_instance().get_binary(path) - - + + def work_path(path=None): - """ - Get path inside the current test suite working directory. Creating files in the work directory does not guarantee - that files will be accessible after the test suite execution - :param path: path relative to the test suite working dir - :return: absolute path inside the test suite working dir - """ + """ + Get path inside the current test suite working directory. Creating files in the work directory does not guarantee + that files will be accessible after the test suite execution + :param path: path relative to the test suite working dir + :return: absolute path inside the test suite working dir + """ return _join_path( os.environ.get("TEST_WORK_PATH") or _get_ya_plugin_instance().get_context("work_path") or os.getcwd(), path) - - -def python_path(): - """ + + +def python_path(): + """ Get path to the arcadia python. Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles are built in a stripped-down form that is needed for building, not running tests. See comments in the file below to find out which version of python is compatible with tests. https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc - :return: absolute path to python - """ - return _get_ya_plugin_instance().python_path - - -def valgrind_path(): - """ - Get path to valgrind - :return: absolute path to valgrind - """ - return _get_ya_plugin_instance().valgrind_path - - -def get_param(key, default=None): - """ - Get arbitrary parameter passed via command line - :param key: key - :param default: default value - :return: parameter value or the default - """ - return _get_ya_plugin_instance().get_param(key, default) - - + :return: absolute path to python + """ + return _get_ya_plugin_instance().python_path + + +def valgrind_path(): + """ + Get path to valgrind + :return: absolute path to valgrind + """ + return _get_ya_plugin_instance().valgrind_path + + +def get_param(key, default=None): + """ + Get arbitrary parameter passed via command line + :param key: key + :param default: default value + :return: parameter value or the default + """ + return _get_ya_plugin_instance().get_param(key, default) + + def get_param_dict_copy(): """ Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters. @@ -209,25 +209,25 @@ def get_param_dict_copy(): @not_test def test_output_path(path=None): - """ - Get dir in the suite output_path for the current test case - """ + """ + Get dir in the suite output_path for the current test case + """ test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0] try: - os.makedirs(test_out_dir) + os.makedirs(test_out_dir) except OSError as e: if e.errno != errno.EEXIST: raise return _join_path(test_out_dir, path) - - -def project_path(path=None): - """ - Get path in build root relating to build_root/project path - """ - return _join_path(os.path.join(build_path(), context.project_path), path) - - + + +def project_path(path=None): + """ + Get path in build root relating to build_root/project path + """ + return _join_path(os.path.join(build_path(), context.project_path), path) + + def gdb_path(): """ Get path to the gdb @@ -235,25 +235,25 @@ def gdb_path(): return _get_ya_plugin_instance().gdb_path -def c_compiler_path(): - """ - Get path to the gdb - """ - return os.environ.get("YA_CC") - - +def c_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CC") + + def get_yt_hdd_path(path=None): if 'HDD_PATH' in os.environ: return _join_path(os.environ['HDD_PATH'], path) -def cxx_compiler_path(): - """ - Get path to the gdb - """ - return os.environ.get("YA_CXX") - - +def cxx_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CXX") + + def global_resources(): try: return json.loads(os.environ.get("YA_GLOBAL_RESOURCES")) @@ -282,39 +282,39 @@ def _register_core(name, binary_path, core_path, bt_path, pbt_path): @not_test -def test_source_path(path=None): - return _join_path(os.path.join(source_path(), context.project_path), path) - - -class Context(object): - """ - Runtime context - """ - - @property - def build_type(self): - return _get_ya_plugin_instance().get_context("build_type") - - @property - def project_path(self): - return _get_ya_plugin_instance().get_context("project_path") - - @property - def test_stderr(self): - return _get_ya_plugin_instance().get_context("test_stderr") - - @property +def test_source_path(path=None): + return _join_path(os.path.join(source_path(), context.project_path), path) + + +class Context(object): + """ + Runtime context + """ + + @property + def build_type(self): + return _get_ya_plugin_instance().get_context("build_type") + + @property + def project_path(self): + return _get_ya_plugin_instance().get_context("project_path") + + @property + def test_stderr(self): + return _get_ya_plugin_instance().get_context("test_stderr") + + @property def test_debug(self): return _get_ya_plugin_instance().get_context("test_debug") @property - def test_traceback(self): - return _get_ya_plugin_instance().get_context("test_traceback") - - @property - def test_name(self): + def test_traceback(self): + return _get_ya_plugin_instance().get_context("test_traceback") + + @property + def test_name(self): return _get_ya_config().current_test_name - + @property def sanitize(self): """ @@ -340,4 +340,4 @@ class Context(object): return _get_ya_plugin_instance().get_context(key) -context = Context() +context = Context() diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py index 1aa7b04827..39bbb45570 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime_java.py +++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py @@ -8,7 +8,7 @@ _JAVA_DIR = [] def get_java_path(jdk_dir): - # deprecated - to be deleted + # deprecated - to be deleted java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe')) for p in java_paths: @@ -25,11 +25,11 @@ def get_java_path(jdk_dir): return p return '' - - -def get_build_java_dir(jdk_dir): + + +def get_build_java_dir(jdk_dir): versions = [8, 10, 11, 12, 13, 14, 15] - + if not _JAVA_DIR: for version in versions: jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version)) diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py index c5401bd2ed..9e7a74cdf5 100644 --- a/library/python/testing/yatest_common/yatest/common/tags.py +++ b/library/python/testing/yatest_common/yatest/common/tags.py @@ -1,5 +1,5 @@ -try: - import pytest - ya_external = getattr(pytest.mark, "ya:external") -except ImportError: - ya_external = None +try: + import pytest + ya_external = getattr(pytest.mark, "ya:external") +except ImportError: + ya_external = None diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py index 459d12c878..39113230d9 100644 --- a/library/python/testing/yatest_lib/external.py +++ b/library/python/testing/yatest_lib/external.py @@ -1,147 +1,147 @@ -from __future__ import absolute_import - +from __future__ import absolute_import + import re import sys -import copy +import copy import logging - -from . import tools + +from . import tools from datetime import date, datetime - + import enum import six logger = logging.getLogger(__name__) MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' - + def apply(func, value, apply_to_keys=False): - """ - Applies func to every possible member of value - :param value: could be either a primitive object or a complex one (list, dicts) - :param func: func to be applied - :return: - """ - def _apply(func, value, value_path): - if value_path is None: - value_path = [] - - if isinstance(value, list) or isinstance(value, tuple): - res = [] - for ind, item in enumerate(value): - path = copy.copy(value_path) - path.append(ind) - res.append(_apply(func, item, path)) - elif isinstance(value, dict): - if is_external(value): - # this is a special serialized object pointing to some external place - res = func(value, value_path) - else: - res = {} - for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]): - path = copy.copy(value_path) - path.append(key) + """ + Applies func to every possible member of value + :param value: could be either a primitive object or a complex one (list, dicts) + :param func: func to be applied + :return: + """ + def _apply(func, value, value_path): + if value_path is None: + value_path = [] + + if isinstance(value, list) or isinstance(value, tuple): + res = [] + for ind, item in enumerate(value): + path = copy.copy(value_path) + path.append(ind) + res.append(_apply(func, item, path)) + elif isinstance(value, dict): + if is_external(value): + # this is a special serialized object pointing to some external place + res = func(value, value_path) + else: + res = {} + for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]): + path = copy.copy(value_path) + path.append(key) res[_apply(func, key, path) if apply_to_keys else key] = _apply(func, val, path) - else: - res = func(value, value_path) - return res - return _apply(func, value, None) - - + else: + res = func(value, value_path) + return res + return _apply(func, value, None) + + def is_coroutine(val): if sys.version_info[0] < 3: return False else: import asyncio - return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val) - - -def serialize(value): - """ - Serialize value to json-convertible object - Ensures that all components of value can be serialized to json - :param value: object to be serialized - """ - def _serialize(val, _): - if val is None: - return val + return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val) + + +def serialize(value): + """ + Serialize value to json-convertible object + Ensures that all components of value can be serialized to json + :param value: object to be serialized + """ + def _serialize(val, _): + if val is None: + return val if isinstance(val, six.string_types) or isinstance(val, bytes): - return tools.to_utf8(val) + return tools.to_utf8(val) if isinstance(val, enum.Enum): return str(val) if isinstance(val, six.integer_types) or type(val) in [float, bool]: - return val - if is_external(val): - return dict(val) + return val + if is_external(val): + return dict(val) if isinstance(val, (date, datetime)): return repr(val) if is_coroutine(val): - return None - raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val))) + return None + raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val))) return apply(_serialize, value, apply_to_keys=True) - - -def is_external(value): - return isinstance(value, dict) and "uri" in value.keys() - - -class ExternalSchema(object): - File = "file" - SandboxResource = "sbr" - Delayed = "delayed" + + +def is_external(value): + return isinstance(value, dict) and "uri" in value.keys() + + +class ExternalSchema(object): + File = "file" + SandboxResource = "sbr" + Delayed = "delayed" HTTP = "http" - - -class CanonicalObject(dict): - def __iter__(self): - raise TypeError("Iterating canonical object is not implemented") - - -class ExternalDataInfo(object): - - def __init__(self, data): - assert is_external(data) - self._data = data - - def __str__(self): - type_str = "File" if self.is_file else "Sandbox resource" - return "{}({})".format(type_str, self.path) - - def __repr__(self): - return str(self) - - @property - def uri(self): - return self._data["uri"] - - @property - def checksum(self): - return self._data.get("checksum") - - @property - def is_file(self): - return self.uri.startswith(ExternalSchema.File) - - @property - def is_sandbox_resource(self): - return self.uri.startswith(ExternalSchema.SandboxResource) - - @property - def is_delayed(self): - return self.uri.startswith(ExternalSchema.Delayed) - - @property + + +class CanonicalObject(dict): + def __iter__(self): + raise TypeError("Iterating canonical object is not implemented") + + +class ExternalDataInfo(object): + + def __init__(self, data): + assert is_external(data) + self._data = data + + def __str__(self): + type_str = "File" if self.is_file else "Sandbox resource" + return "{}({})".format(type_str, self.path) + + def __repr__(self): + return str(self) + + @property + def uri(self): + return self._data["uri"] + + @property + def checksum(self): + return self._data.get("checksum") + + @property + def is_file(self): + return self.uri.startswith(ExternalSchema.File) + + @property + def is_sandbox_resource(self): + return self.uri.startswith(ExternalSchema.SandboxResource) + + @property + def is_delayed(self): + return self.uri.startswith(ExternalSchema.Delayed) + + @property def is_http(self): return self.uri.startswith(ExternalSchema.HTTP) @property - def path(self): + def path(self): if self.uri.count("://") != 1: logger.error("Invalid external data uri: '%s'", self.uri) return self.uri - _, path = self.uri.split("://") - return path - + _, path = self.uri.split("://") + return path + def get_mds_key(self): assert self.is_http m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri) @@ -149,44 +149,44 @@ class ExternalDataInfo(object): return m.group(1) raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri)) - @property - def size(self): - return self._data.get("size") - - def serialize(self): - return self._data - - @classmethod - def _serialize(cls, schema, path, checksum=None, attrs=None): - res = CanonicalObject({"uri": "{}://{}".format(schema, path)}) - if checksum: - res["checksum"] = checksum - if attrs: - res.update(attrs) - return res - - @classmethod - def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None): - attrs = {} - if diff_tool: - attrs["diff_tool"] = diff_tool - if local: - attrs["local"] = local - if diff_file_name: - attrs["diff_file_name"] = diff_file_name - if diff_tool_timeout: - attrs["diff_tool_timeout"] = diff_tool_timeout - if size is not None: - attrs["size"] = size - return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs) - - @classmethod - def serialize_resource(cls, id, checksum=None): - return cls._serialize(ExternalSchema.SandboxResource, id, checksum) - - @classmethod - def serialize_delayed(cls, upload_id, checksum): - return cls._serialize(ExternalSchema.Delayed, upload_id, checksum) - - def get(self, key, default=None): - return self._data.get(key, default) + @property + def size(self): + return self._data.get("size") + + def serialize(self): + return self._data + + @classmethod + def _serialize(cls, schema, path, checksum=None, attrs=None): + res = CanonicalObject({"uri": "{}://{}".format(schema, path)}) + if checksum: + res["checksum"] = checksum + if attrs: + res.update(attrs) + return res + + @classmethod + def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None): + attrs = {} + if diff_tool: + attrs["diff_tool"] = diff_tool + if local: + attrs["local"] = local + if diff_file_name: + attrs["diff_file_name"] = diff_file_name + if diff_tool_timeout: + attrs["diff_tool_timeout"] = diff_tool_timeout + if size is not None: + attrs["size"] = size + return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs) + + @classmethod + def serialize_resource(cls, id, checksum=None): + return cls._serialize(ExternalSchema.SandboxResource, id, checksum) + + @classmethod + def serialize_delayed(cls, upload_id, checksum): + return cls._serialize(ExternalSchema.Delayed, upload_id, checksum) + + def get(self, key, default=None): + return self._data.get(key, default) diff --git a/library/python/testing/yatest_lib/tools.py b/library/python/testing/yatest_lib/tools.py index 602eaeefcc..b72d79c162 100644 --- a/library/python/testing/yatest_lib/tools.py +++ b/library/python/testing/yatest_lib/tools.py @@ -1,13 +1,13 @@ import six import sys - -def to_utf8(value): - """ - Converts value to string encoded into utf-8 - :param value: - :return: - """ + +def to_utf8(value): + """ + Converts value to string encoded into utf-8 + :param value: + :return: + """ if sys.version_info[0] < 3: if not isinstance(value, basestring): # noqa value = unicode(value) # noqa diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make index d115846230..342bae82ba 100644 --- a/library/python/testing/yatest_lib/ya.make +++ b/library/python/testing/yatest_lib/ya.make @@ -1,16 +1,16 @@ OWNER(g:yatest) - + PY23_LIBRARY() - -PY_SRCS( + +PY_SRCS( NAMESPACE yatest_lib - external.py + external.py test_splitter.py - tools.py + tools.py ya.py -) - +) + PEERDIR( contrib/python/six ) @@ -21,6 +21,6 @@ IF(PYTHON2) ) ENDIF() -END() +END() RECURSE_FOR_TESTS(tests) diff --git a/library/python/windows/__init__.py b/library/python/windows/__init__.py index 9966cb8c83..62861b3309 100644 --- a/library/python/windows/__init__.py +++ b/library/python/windows/__init__.py @@ -3,7 +3,7 @@ import os import stat import sys -import shutil +import shutil import logging from six import reraise @@ -11,22 +11,22 @@ from six import reraise import library.python.func import library.python.strings -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) + - ERRORS = { 'SUCCESS': 0, - 'PATH_NOT_FOUND': 3, - 'ACCESS_DENIED': 5, - 'SHARING_VIOLATION': 32, + 'PATH_NOT_FOUND': 3, + 'ACCESS_DENIED': 5, + 'SHARING_VIOLATION': 32, 'INSUFFICIENT_BUFFER': 122, 'DIR_NOT_EMPTY': 145, } -RETRIABLE_FILE_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['SHARING_VIOLATION']) -RETRIABLE_DIR_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['DIR_NOT_EMPTY'], ERRORS['SHARING_VIOLATION']) +RETRIABLE_FILE_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['SHARING_VIOLATION']) +RETRIABLE_DIR_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['DIR_NOT_EMPTY'], ERRORS['SHARING_VIOLATION']) + - # Check if on Windows @library.python.func.lazy def on_win(): @@ -145,7 +145,7 @@ if on_win(): ei = None for t in xrange(tries): if t: - logger.debug('Diehard [errs %s]: try #%d in %s', ','.join(str(x) for x in winerrors), t, f) + logger.debug('Diehard [errs %s]: try #%d in %s', ','.join(str(x) for x in winerrors), t, f) try: return f(*args, **kwargs) except WindowsError as e: @@ -285,14 +285,14 @@ if on_win(): def set_error_mode(mode): return ctypes.windll.kernel32.SetErrorMode(mode) - @win_only - def rmtree(path): - def error_handler(func, handling_path, execinfo): - e = execinfo[1] - if e.winerror == ERRORS['PATH_NOT_FOUND']: - handling_path = "\\\\?\\" + handling_path # handle path over 256 symbols - if os.path.exists(path): - return func(handling_path) + @win_only + def rmtree(path): + def error_handler(func, handling_path, execinfo): + e = execinfo[1] + if e.winerror == ERRORS['PATH_NOT_FOUND']: + handling_path = "\\\\?\\" + handling_path # handle path over 256 symbols + if os.path.exists(path): + return func(handling_path) if e.winerror == ERRORS['ACCESS_DENIED']: try: # removing of r/w directory with read-only files in it yields ACCESS_DENIED @@ -303,9 +303,9 @@ if on_win(): else: # propagate true last error if this attempt fails return func(handling_path) - raise e - shutil.rmtree(path, onerror=error_handler) - + raise e + shutil.rmtree(path, onerror=error_handler) + # Don't display the Windows GPF dialog if the invoked program dies. # http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx @win_only diff --git a/library/python/ya.make b/library/python/ya.make index f0f7ebc6a8..2e1eb6e0e1 100644 --- a/library/python/ya.make +++ b/library/python/ya.make @@ -3,10 +3,10 @@ OWNER(g:python-contrib) RECURSE( aho_corasick aho_corasick/ut - archive - archive/benchmark - archive/test - archive/test/data + archive + archive/benchmark + archive/test + archive/test/data asgi_yauth async_clients auth_client_parser @@ -19,14 +19,14 @@ RECURSE( blackbox/tests blackbox/tvm2 bloom - boost_test + boost_test bstr build_info build_info/ut capabilities celery_dashboard certifi - cgroups + cgroups charset charts_notes charts_notes/example @@ -72,7 +72,7 @@ RECURSE( filelock/ut filesys filesys/ut - find_root + find_root flask flask_passport fnvhash @@ -81,7 +81,7 @@ RECURSE( framing/ut func func/ut - fs + fs geolocation geolocation/ut geohash @@ -184,8 +184,8 @@ RECURSE( svn_version svn_version/ut symbols - testing - tmp + testing + tmp toloka_client toloka-kit toloka-airflow @@ -199,7 +199,7 @@ RECURSE( tvm2/tests type_info type_info/test - unique_id + unique_id vault_client watch_dog watch_dog/example |