diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/python | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/python')
161 files changed, 11051 insertions, 0 deletions
diff --git a/library/python/certifi/.dist-info/METADATA b/library/python/certifi/.dist-info/METADATA new file mode 100644 index 0000000000..4849f81ed4 --- /dev/null +++ b/library/python/certifi/.dist-info/METADATA @@ -0,0 +1,2 @@ +Name: certifi +Version: 2019.7.1 diff --git a/library/python/certifi/.dist-info/top_level.txt b/library/python/certifi/.dist-info/top_level.txt new file mode 100644 index 0000000000..963eac530b --- /dev/null +++ b/library/python/certifi/.dist-info/top_level.txt @@ -0,0 +1 @@ +certifi diff --git a/library/python/certifi/README.md b/library/python/certifi/README.md new file mode 100644 index 0000000000..75a812733c --- /dev/null +++ b/library/python/certifi/README.md @@ -0,0 +1,7 @@ +This library provides arcadia certs via certifi-compatible API. + +In binary (single executable) mode it patches python ssl module to +support loading certs from memory. + +.dist-info/METADATA version reflects the last update time of the +arcadia certs. diff --git a/library/python/certifi/certifi/__init__.py b/library/python/certifi/certifi/__init__.py new file mode 100644 index 0000000000..5270d206cd --- /dev/null +++ b/library/python/certifi/certifi/__init__.py @@ -0,0 +1,10 @@ +import ssl + +if hasattr(ssl, 'builtin_cadata'): + from .binary import where +else: + from .source import where + +__all__ = ['where', '__version__'] + +__version__ = '2020.04.05.2' diff --git a/library/python/certifi/certifi/binary.py b/library/python/certifi/certifi/binary.py new file mode 100644 index 0000000000..1050e733a3 --- /dev/null +++ b/library/python/certifi/certifi/binary.py @@ -0,0 +1,25 @@ +import ssl + + +def builtin_ca(): + return None, None, ssl.builtin_cadata() + + +# Normally certifi.where() returns a path to a certificate file; +# here it returns a callable. +def where(): + return builtin_ca + + +# Patch ssl module to accept a callable cafile. +load_verify_locations = ssl.SSLContext.load_verify_locations + + +def load_verify_locations__callable(self, cafile=None, capath=None, cadata=None): + if callable(cafile): + cafile, capath, cadata = cafile() + + return load_verify_locations(self, cafile, capath, cadata) + + +ssl.SSLContext.load_verify_locations = load_verify_locations__callable diff --git a/library/python/certifi/certifi/source.py b/library/python/certifi/certifi/source.py new file mode 100644 index 0000000000..f539b67002 --- /dev/null +++ b/library/python/certifi/certifi/source.py @@ -0,0 +1,13 @@ +import os.path + +pem = os.path.abspath(__file__) +pem = os.path.dirname(pem) +pem = os.path.dirname(pem) +pem = os.path.dirname(pem) +pem = os.path.dirname(pem) +pem = os.path.dirname(pem) +pem = os.path.join(pem, 'certs', 'cacert.pem') + + +def where(): + return pem diff --git a/library/python/certifi/ya.make b/library/python/certifi/ya.make new file mode 100644 index 0000000000..64fefe2833 --- /dev/null +++ b/library/python/certifi/ya.make @@ -0,0 +1,18 @@ +PY23_LIBRARY() + +OWNER(orivej g:python-contrib) + +RESOURCE_FILES( + PREFIX library/python/certifi/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +PY_SRCS( + TOP_LEVEL + certifi/__init__.py + certifi/binary.py + certifi/source.py +) + +END() diff --git a/library/python/cores/__init__.py b/library/python/cores/__init__.py new file mode 100644 index 0000000000..fdb1f82a46 --- /dev/null +++ b/library/python/cores/__init__.py @@ -0,0 +1,193 @@ +# coding: utf-8 + +import os +import re +import glob +import socket +import logging +import platform +import subprocess + +import six + +from library.python.reservoir_sampling import reservoir_sampling + + +logger = logging.getLogger(__name__) + + +def _read_file(filename): + with open(filename) as afile: + return afile.read().strip("\n") + + +def recover_core_dump_file(binary_path, cwd, pid): + + class CoreFilePattern(object): + def __init__(self, path, mask): + self.path = path + self.mask = mask + + cwd = cwd or os.getcwd() + system = platform.system().lower() + if system.startswith("linux"): + import stat + import resource + + logger.debug("hostname = '%s'", socket.gethostname()) + logger.debug("rlimit_core = '%s'", str(resource.getrlimit(resource.RLIMIT_CORE))) + core_pattern = _read_file("/proc/sys/kernel/core_pattern") + logger.debug("core_pattern = '%s'", core_pattern) + if core_pattern.startswith("/"): + default_pattern = CoreFilePattern(os.path.dirname(core_pattern), '*') + else: + default_pattern = CoreFilePattern(cwd, '*') + + def resolve_core_mask(core_mask): + def resolve(text): + if text == "%p": + return str(pid) + elif text == "%e": + # https://github.com/torvalds/linux/blob/7876320f88802b22d4e2daf7eb027dd14175a0f8/include/linux/sched.h#L847 + # https://github.com/torvalds/linux/blob/7876320f88802b22d4e2daf7eb027dd14175a0f8/fs/coredump.c#L278 + return os.path.basename(binary_path)[:15] + elif text == "%E": + return binary_path.replace("/", "!") + elif text == "%%": + return "%" + elif text.startswith("%"): + return "*" + return text + + parts = filter(None, re.split(r"(%.)", core_mask)) + return "".join([resolve(p) for p in parts]) + + # don't interpret a program for piping core dumps as a pattern + if core_pattern and not core_pattern.startswith("|"): + default_pattern.mask = os.path.basename(core_pattern) + else: + core_uses_pid = int(_read_file("/proc/sys/kernel/core_uses_pid")) + logger.debug("core_uses_pid = '%d'", core_uses_pid) + if core_uses_pid == 0: + default_pattern.mask = "core" + else: + default_pattern.mask = "core.%p" + + # widely distributed core dump dir and mask (see DEVTOOLS-4408) + yandex_pattern = CoreFilePattern('/coredumps', '%e.%p.%s') + yandex_market_pattern = CoreFilePattern('/var/tmp/cores', 'core.%..%e.%s.%p.*') + + for pattern in [default_pattern, yandex_pattern, yandex_market_pattern]: + pattern.mask = resolve_core_mask(pattern.mask) + + if not os.path.exists(pattern.path): + logger.warning("Core dump dir doesn't exist: %s", pattern.path) + continue + + logger.debug( + "Core dump dir (%s) permission mask: %s (expected: %s (%s-dir, %s-sticky bit))", + pattern.path, + oct(os.stat(pattern.path)[stat.ST_MODE]), + oct(stat.S_IFDIR | stat.S_ISVTX | 0o777), + oct(stat.S_IFDIR), + oct(stat.S_ISVTX), + ) + logger.debug("Search for core dump files match pattern '%s' in '%s'", pattern.mask, pattern.path) + cores = glob.glob(os.path.join(pattern.path, pattern.mask)) + files = os.listdir(pattern.path) + logger.debug( + "Matched core dump files (%d/%d): [%s] (mismatched samples: %s)", + len(cores), + len(files), + ", ".join(cores), + ", ".join(reservoir_sampling(files, 5)), + ) + + if len(cores) == 1: + return cores[0] + elif len(cores) > 1: + stat = [(filename, os.stat(filename).st_mtime) for filename in cores] + entry = sorted(stat, key=lambda x: x[1])[-1] + logger.debug("Latest core dump file: '%s' with %d mtime", entry[0], entry[1]) + return entry[0] + else: + logger.debug("Core dump file recovering is not supported on '%s'", system) + return None + + +def get_gdb_full_backtrace(binary, core, gdb_path): + cmd = [ + gdb_path, binary, core, + "--eval-command", "set print thread-events off", + "--eval-command", "thread apply all backtrace full", + "--batch", + "--quiet", + ] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, stderr = proc.communicate() + output = six.ensure_str(output) + if stderr: + output += "\nstderr >>\n" + six.ensure_str(stderr) + return output + + +def get_problem_stack(backtrace): + stack = [] + found_thread1 = False + regex = re.compile(r'[Tt]hread (\d+)') + + for line in backtrace.split("\n"): + match = regex.search(line) + if match: + if found_thread1: + break + if int(match.group(1)) == 1: + found_thread1 = True + if found_thread1: + stack.append(line) + + if not stack: + return backtrace + return "\n".join(stack) + + +# XXX +def colorize_backtrace(text): + filters = [ + # Function names and the class they belong to + (re.compile(r"^(#[0-9]+ .*?)([a-zA-Z0-9_:\.@]+)(\s?\()", flags=re.MULTILINE), r"\1[[c:cyan]]\2[[rst]]\3"), + # Function argument names + (re.compile(r"([a-zA-Z0-9_#]*)(\s?=\s?)"), r"[[c:green]]\1[[rst]]\2"), + # Stack frame number + (re.compile(r"^(#[0-9]+)", flags=re.MULTILINE), r"[[c:red]]\1[[rst]]"), + # Thread id colorization + (re.compile(r"^([ \*]) ([0-9]+)", flags=re.MULTILINE), r"[[c:light-cyan]]\1 [[c:red]]\2[[rst]]"), + # File path and line number + (re.compile(r"(\.*[/A-Za-z0-9\+_\.\-]*):(([0-9]+)(:[0-9]+)?)$", flags=re.MULTILINE), r"[[c:light-grey]]\1[[rst]]:[[c:magenta]]\2[[rst]]"), + # Addresses + (re.compile(r"\b(0x[a-f0-9]{6,})\b"), r"[[c:light-grey]]\1[[rst]]"), + ] + + for regex, substitution in filters: + text = regex.sub(substitution, text) + return text + + +def resolve_addresses(addresses, symbolizer, binary): + addresses = list(set(addresses)) + cmd = [ + symbolizer, + "-demangle", + "-obj", + binary, + ] + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate(input="\n".join(addresses)) + if proc.returncode: + raise Exception("Symbolizer failed with rc:{}\nstderr: {}".format(proc.returncode, err)) + + resolved = filter(None, out.split("\n\n")) + if len(addresses) != len(resolved): + raise Exception("llvm-symbolizer can not extract lines from addresses (count mismatch: {}-{})".format(len(addresses), len(resolved))) + + return {k: v.strip(" \n") for k, v in zip(addresses, resolved)} diff --git a/library/python/cores/ya.make b/library/python/cores/ya.make new file mode 100644 index 0000000000..76264e9cce --- /dev/null +++ b/library/python/cores/ya.make @@ -0,0 +1,15 @@ +OWNER( + prettyboy + g:yatest +) + +PY23_LIBRARY() + +PY_SRCS(__init__.py) + +PEERDIR( + contrib/python/six + library/python/reservoir_sampling +) + +END() diff --git a/library/python/filelock/__init__.py b/library/python/filelock/__init__.py new file mode 100644 index 0000000000..f81ff67f37 --- /dev/null +++ b/library/python/filelock/__init__.py @@ -0,0 +1,122 @@ +import errno +import logging +import os +import sys + +import library.python.windows + +logger = logging.getLogger(__name__) + + +def set_close_on_exec(stream): + if library.python.windows.on_win(): + library.python.windows.set_handle_information(stream, inherit=False) + else: + import fcntl + fcntl.fcntl(stream, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + + +class AbstractFileLock(object): + + def __init__(self, path): + self.path = path + + def acquire(self, blocking=True): + raise NotImplementedError + + def release(self): + raise NotImplementedError + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, type, value, traceback): + self.release() + + +class _NixFileLock(AbstractFileLock): + + def __init__(self, path): + super(_NixFileLock, self).__init__(path) + from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB + self._locker = lambda lock, blocking: flock(lock, LOCK_EX if blocking else LOCK_EX | LOCK_NB) + self._unlocker = lambda lock: flock(lock, LOCK_UN) + self._lock = open(self.path, 'a') + set_close_on_exec(self._lock) + + def acquire(self, blocking=True): + import errno + try: + self._locker(self._lock, blocking) + except IOError as e: + if e.errno in (errno.EAGAIN, errno.EACCES) and not blocking: + return False + raise + return True + + def release(self): + self._unlocker(self._lock) + + def __del__(self): + if hasattr(self, "_lock"): + self._lock.close() + + +class _WinFileLock(AbstractFileLock): + """ + Based on LockFile / UnlockFile from win32 API + https://msdn.microsoft.com/en-us/library/windows/desktop/aa365202(v=vs.85).aspx + """ + + _LOCKED_BYTES_NUM = 1 + + def __init__(self, path): + super(_WinFileLock, self).__init__(path) + self._lock = None + try: + with file(path, 'w') as lock_file: + lock_file.write(" " * self._LOCKED_BYTES_NUM) + except IOError as e: + if e.errno != errno.EACCES or not os.path.isfile(path): + raise + + def acquire(self, blocking=True): + self._lock = open(self.path) + set_close_on_exec(self._lock) + + import time + locked = False + while not locked: + locked = library.python.windows.lock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False) + if locked: + return True + if blocking: + time.sleep(.5) + else: + return False + + def release(self): + if self._lock: + library.python.windows.unlock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False) + self._lock.close() + self._lock = None + + +class FileLock(AbstractFileLock): + + def __init__(self, path): + super(FileLock, self).__init__(path) + + if sys.platform.startswith('win'): + self._lock = _WinFileLock(path) + else: + self._lock = _NixFileLock(path) + + def acquire(self, blocking=True): + logger.debug('Acquiring filelock (blocking=%s): %s', blocking, self.path) + return self._lock.acquire(blocking) + + def release(self): + logger.debug('Ensuring filelock released: %s', self.path) + return self._lock.release() diff --git a/library/python/filelock/ut/lib/test_filelock.py b/library/python/filelock/ut/lib/test_filelock.py new file mode 100644 index 0000000000..1b11d89123 --- /dev/null +++ b/library/python/filelock/ut/lib/test_filelock.py @@ -0,0 +1,83 @@ +import os +import time +import logging +import multiprocessing +import tempfile +import threading + +import library.python.filelock + + +def _acquire_lock(lock_path, out_file_path): + with library.python.filelock.FileLock(lock_path): + with open(out_file_path, "a") as out: + out.write("{}:{}\n".format(os.getpid(), time.time())) + time.sleep(2) + + +def test_filelock(): + temp_dir = tempfile.mkdtemp() + lock_path = os.path.join(temp_dir, "file.lock") + out_file_path = os.path.join(temp_dir, "out.txt") + + process_count = 5 + processes = [] + for i in range(process_count): + process = multiprocessing.Process(target=_acquire_lock, args=(lock_path, out_file_path)) + process.start() + processes.append(process) + + for process in processes: + process.join() + + pids = [] + times = [] + with open(out_file_path) as out: + content = out.read() + logging.info("Times:\n%s", content) + for line in content.strip().split("\n"): + pid, time_val = line.split(":") + pids.append(pid) + times.append(float(time_val)) + + assert len(set(pids)) == process_count + time1 = times.pop() + while times: + time2 = times.pop() + assert int(time1) - int(time2) >= 2 + time1 = time2 + + +def test_filelock_init_acquired(): + temp_dir = tempfile.mkdtemp() + lock_path = os.path.join(temp_dir, "file.lock") + + with library.python.filelock.FileLock(lock_path): + sublock = library.python.filelock.FileLock(lock_path) + del sublock + + +def test_concurrent_lock(): + filename = 'con.lock' + + def lock(): + l = library.python.filelock.FileLock(filename) + time.sleep(1) + l.acquire() + l.release() + try: + os.unlink(filename) + except OSError: + pass + + threads = [] + for i in range(100): + t = threading.Thread(target=lock) + t.daemon = True + threads.append(t) + + for t in threads: + t.start() + + for t in threads: + t.join() diff --git a/library/python/filelock/ut/lib/ya.make b/library/python/filelock/ut/lib/ya.make new file mode 100644 index 0000000000..f3f9da5a67 --- /dev/null +++ b/library/python/filelock/ut/lib/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +TEST_SRCS(test_filelock.py) + +PEERDIR( + library/python/filelock +) + +END() diff --git a/library/python/filelock/ut/py2/ya.make b/library/python/filelock/ut/py2/ya.make new file mode 100644 index 0000000000..30b54e0232 --- /dev/null +++ b/library/python/filelock/ut/py2/ya.make @@ -0,0 +1,9 @@ +OWNER(g:yatool) + +PY2TEST() + +PEERDIR( + library/python/filelock/ut/lib +) + +END() diff --git a/library/python/filelock/ut/py3/ya.make b/library/python/filelock/ut/py3/ya.make new file mode 100644 index 0000000000..05a856a19a --- /dev/null +++ b/library/python/filelock/ut/py3/ya.make @@ -0,0 +1,9 @@ +OWNER(g:yatool) + +PY3TEST() + +PEERDIR( + library/python/filelock/ut/lib +) + +END() diff --git a/library/python/filelock/ut/ya.make b/library/python/filelock/ut/ya.make new file mode 100644 index 0000000000..9280ea415e --- /dev/null +++ b/library/python/filelock/ut/ya.make @@ -0,0 +1,7 @@ +OWNER(g:yatool) + +RECURSE( + lib + py2 + py3 +) diff --git a/library/python/filelock/ya.make b/library/python/filelock/ya.make new file mode 100644 index 0000000000..958cc1866f --- /dev/null +++ b/library/python/filelock/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +PY_SRCS(__init__.py) + +PEERDIR( + library/python/windows +) + +END() diff --git a/library/python/find_root/__init__.py b/library/python/find_root/__init__.py new file mode 100644 index 0000000000..6da604d62e --- /dev/null +++ b/library/python/find_root/__init__.py @@ -0,0 +1,20 @@ +import os + + +def is_root(path): + return os.path.exists(os.path.join(path, ".arcadia.root")) or os.path.exists(os.path.join(path, 'devtools', 'ya', 'ya.conf.json')) + + +def detect_root(path, detector=is_root): + return _find_path(path, detector) + + +def _find_path(starts_from, check): + p = os.path.realpath(starts_from) + while True: + if check(p): + return p + next_p = os.path.dirname(p) + if next_p == p: + return None + p = next_p diff --git a/library/python/find_root/ya.make b/library/python/find_root/ya.make new file mode 100644 index 0000000000..beaa8e3c52 --- /dev/null +++ b/library/python/find_root/ya.make @@ -0,0 +1,7 @@ +PY23_LIBRARY() + +OWNER(g:yatool) + +PY_SRCS(__init__.py) + +END() diff --git a/library/python/fs/__init__.py b/library/python/fs/__init__.py new file mode 100644 index 0000000000..b1b7cde079 --- /dev/null +++ b/library/python/fs/__init__.py @@ -0,0 +1,501 @@ +# coding: utf-8 + +import codecs +import errno +import logging +import os +import random +import shutil +import six +import stat +import sys + +import library.python.func +import library.python.strings +import library.python.windows + +logger = logging.getLogger(__name__) + + +try: + WindowsError +except NameError: + WindowsError = None + + +_diehard_win_tries = 10 +errorfix_win = library.python.windows.errorfix + + +class CustomFsError(OSError): + def __init__(self, errno, message='', filename=None): + super(CustomFsError, self).__init__(message) + self.errno = errno + self.strerror = os.strerror(errno) + self.filename = filename + + +# Directories creation +# If dst is already exists and is a directory - does nothing +# Throws OSError +@errorfix_win +def ensure_dir(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST or not os.path.isdir(path): + raise + + +# Directories creation +# If dst is already exists and is a directory - does nothing +# Returns path +# Throws OSError +@errorfix_win +def create_dirs(path): + ensure_dir(path) + return path + + +# Atomic file/directory move (rename) +# Doesn't guarantee dst replacement +# Atomic if no device boundaries are crossed +# Depends on ctypes on Windows +# Throws OSError +# On Unix, if dst exists: +# if dst is file or empty dir - replaces it +# if src is dir and dst is not dir - throws OSError (errno ENOTDIR) +# if src is dir and dst is non-empty dir - throws OSError (errno ENOTEMPTY) +# if src is file and dst is dir - throws OSError (errno EISDIR) +# On Windows, if dst exists - throws OSError (errno EEXIST) +@errorfix_win +@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries) +def move(src, dst): + os.rename(src, dst) + + +# Atomic replacing file move (rename) +# Replaces dst if exists and not a dir +# Doesn't guarantee dst dir replacement +# Atomic if no device boundaries are crossed +# Depends on ctypes on Windows +# Throws OSError +# On Unix, if dst exists: +# if dst is file - replaces it +# if dst is dir - throws OSError (errno EISDIR) +# On Windows, if dst exists: +# if dst is file - replaces it +# if dst is dir - throws OSError (errno EACCES) +@errorfix_win +@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries) +def replace_file(src, dst): + if library.python.windows.on_win(): + library.python.windows.replace_file(src, dst) + else: + os.rename(src, dst) + + +# File/directory replacing move (rename) +# Removes dst if exists +# Non-atomic +# Depends on ctypes on Windows +# Throws OSError +@errorfix_win +def replace(src, dst): + try: + move(src, dst) + except OSError as e: + if e.errno not in (errno.EEXIST, errno.EISDIR, errno.ENOTDIR, errno.ENOTEMPTY): + raise + remove_tree(dst) + move(src, dst) + + +# Atomic file remove +# Throws OSError +@errorfix_win +@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries) +def remove_file(path): + os.remove(path) + + +# Atomic empty directory remove +# Throws OSError +@errorfix_win +@library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries) +def remove_dir(path): + os.rmdir(path) + + +def fix_path_encoding(path): + return library.python.strings.to_str(path, library.python.strings.fs_encoding()) + + +# File/directory remove +# Non-atomic +# Throws OSError, AssertionError +@errorfix_win +def remove_tree(path): + @library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries) + def rmtree(path): + if library.python.windows.on_win(): + library.python.windows.rmtree(path) + else: + shutil.rmtree(fix_path_encoding(path)) + + st = os.lstat(path) + if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode): + remove_file(path) + elif stat.S_ISDIR(st.st_mode): + rmtree(path) + else: + assert False + + +# File/directory remove ignoring errors +# Non-atomic +@errorfix_win +def remove_tree_safe(path): + try: + st = os.lstat(path) + if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode): + os.remove(path) + elif stat.S_ISDIR(st.st_mode): + shutil.rmtree(fix_path_encoding(path), ignore_errors=True) + # XXX + except UnicodeDecodeError as e: + logging.exception(u'remove_tree_safe with argument %s raise exception: %s', path, e) + raise + except OSError: + pass + + +# File/directory remove +# If path doesn't exist - does nothing +# Non-atomic +# Throws OSError, AssertionError +@errorfix_win +def ensure_removed(path): + try: + remove_tree(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +# Atomic file hardlink +# Dst must not exist +# Depends on ctypes on Windows +# Throws OSError +# If dst exists - throws OSError (errno EEXIST) +@errorfix_win +def hardlink(src, lnk): + if library.python.windows.on_win(): + library.python.windows.hardlink(src, lnk) + else: + os.link(src, lnk) + + +@errorfix_win +def hardlink_or_copy(src, lnk): + def should_fallback_to_copy(exc): + if WindowsError is not None and isinstance(exc, WindowsError) and exc.winerror == 1142: # too many hardlinks + return True + # cross-device hardlink or too many hardlinks, or some known WSL error + if isinstance(exc, OSError) and exc.errno in ( + errno.EXDEV, + errno.EMLINK, + errno.EINVAL, + errno.EACCES, + errno.EPERM, + ): + return True + return False + + try: + hardlink(src, lnk) + except Exception as e: + logger.debug('Failed to hardlink %s to %s with error %s, will copy it', src, lnk, repr(e)) + if should_fallback_to_copy(e): + copy2(src, lnk, follow_symlinks=False) + else: + raise + + +# Atomic file/directory symlink (Unix only) +# Dst must not exist +# Throws OSError +# If dst exists - throws OSError (errno EEXIST) +@errorfix_win +def symlink(src, lnk): + if library.python.windows.on_win(): + library.python.windows.run_disabled(src, lnk) + else: + os.symlink(src, lnk) + + +# shutil.copy2 with follow_symlinks=False parameter (Unix only) +def copy2(src, lnk, follow_symlinks=True): + if six.PY3: + shutil.copy2(src, lnk, follow_symlinks=follow_symlinks) + return + + if follow_symlinks or not os.path.islink(src): + shutil.copy2(src, lnk) + return + + symlink(os.readlink(src), lnk) + + +# Recursively hardlink directory +# Uses plain hardlink for files +# Dst must not exist +# Non-atomic +# Throws OSError +@errorfix_win +def hardlink_tree(src, dst): + if not os.path.exists(src): + raise CustomFsError(errno.ENOENT, filename=src) + if os.path.isfile(src): + hardlink(src, dst) + return + for dirpath, _, filenames in walk_relative(src): + src_dirpath = os.path.join(src, dirpath) if dirpath != '.' else src + dst_dirpath = os.path.join(dst, dirpath) if dirpath != '.' else dst + os.mkdir(dst_dirpath) + for filename in filenames: + hardlink(os.path.join(src_dirpath, filename), os.path.join(dst_dirpath, filename)) + + +# File copy +# throws EnvironmentError (OSError, IOError) +@errorfix_win +def copy_file(src, dst, copy_function=shutil.copy2): + if os.path.isdir(dst): + raise CustomFsError(errno.EISDIR, filename=dst) + copy_function(src, dst) + + +# File/directory copy +# throws EnvironmentError (OSError, IOError, shutil.Error) +@errorfix_win +def copy_tree(src, dst, copy_function=shutil.copy2): + if os.path.isfile(src): + copy_file(src, dst, copy_function=copy_function) + return + copytree3(src, dst, copy_function=copy_function) + + +# File read +# Throws OSError +@errorfix_win +def read_file(path, binary=True): + with open(path, 'r' + ('b' if binary else '')) as f: + return f.read() + + +# Decoding file read +# Throws OSError +@errorfix_win +def read_file_unicode(path, binary=True, enc='utf-8'): + if not binary: + if six.PY2: + with open(path, 'r') as f: + return library.python.strings.to_unicode(f.read(), enc) + else: + with open(path, 'r', encoding=enc) as f: + return f.read() + # codecs.open is always binary + with codecs.open(path, 'r', encoding=enc, errors=library.python.strings.ENCODING_ERRORS_POLICY) as f: + return f.read() + + +@errorfix_win +def open_file(*args, **kwargs): + return ( + library.python.windows.open_file(*args, **kwargs) if library.python.windows.on_win() else open(*args, **kwargs) + ) + + +# Atomic file write +# Throws OSError +@errorfix_win +def write_file(path, data, binary=True): + dir_path = os.path.dirname(path) + if dir_path: + ensure_dir(dir_path) + tmp_path = path + '.tmp.' + str(random.random()) + with open_file(tmp_path, 'w' + ('b' if binary else '')) as f: + if not isinstance(data, bytes) and binary: + data = data.encode('UTF-8') + f.write(data) + replace_file(tmp_path, path) + + +# File size +# Throws OSError +@errorfix_win +def get_file_size(path): + return os.path.getsize(path) + + +# File/directory size +# Non-recursive mode for directory counts size for immediates +# While raise_all_errors is set to False, file size fallbacks to zero in case of getsize errors +# Throws OSError +@errorfix_win +def get_tree_size(path, recursive=False, raise_all_errors=False): + if os.path.isfile(path): + return get_file_size(path) + total_size = 0 + for dir_path, _, files in os.walk(path): + for f in files: + fp = os.path.join(dir_path, f) + try: + total_size += get_file_size(fp) + except OSError as e: + if raise_all_errors: + raise + logger.debug("Cannot calculate file size: %s", e) + if not recursive: + break + return total_size + + +# Directory copy ported from Python 3 +def copytree3( + src, + dst, + symlinks=False, + ignore=None, + copy_function=shutil.copy2, + ignore_dangling_symlinks=False, + dirs_exist_ok=False, +): + """Recursively copy a directory tree. + + The copytree3 is a port of shutil.copytree function from python-3.2. + It has additional useful parameters and may be helpful while we are + on python-2.x. It has to be removed as soon as we have moved to + python-3.2 or higher. + + The destination directory must not already exist. + If exception(s) occur, an Error is raised with a list of reasons. + + If the optional symlinks flag is true, symbolic links in the + source tree result in symbolic links in the destination tree; if + it is false, the contents of the files pointed to by symbolic + links are copied. If the file pointed by the symlink doesn't + exist, an exception will be added in the list of errors raised in + an Error exception at the end of the copy process. + + You can set the optional ignore_dangling_symlinks flag to true if you + want to silence this exception. Notice that this has no effect on + platforms that don't support os.symlink. + + The optional ignore argument is a callable. If given, it + is called with the `src` parameter, which is the directory + being visited by copytree3(), and `names` which is the list of + `src` contents, as returned by os.listdir(): + + callable(src, names) -> ignored_names + + Since copytree3() is called recursively, the callable will be + called once for each directory that is copied. It returns a + list of names relative to the `src` directory that should + not be copied. + + The optional copy_function argument is a callable that will be used + to copy each file. It will be called with the source path and the + destination path as arguments. By default, copy2() is used, but any + function that supports the same signature (like copy()) can be used. + + """ + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + if not (dirs_exist_ok and os.path.isdir(dst)): + os.makedirs(dst) + + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if os.path.islink(srcname): + linkto = os.readlink(srcname) + if symlinks: + # We can't just leave it to `copy_function` because legacy + # code with a custom `copy_function` may rely on copytree3 + # doing the right thing. + os.symlink(linkto, dstname) + else: + # ignore dangling symlink if the flag is on + if not os.path.exists(linkto) and ignore_dangling_symlinks: + continue + # otherwise let the copy occurs. copy2 will raise an error + copy_function(srcname, dstname) + elif os.path.isdir(srcname): + copytree3(srcname, dstname, symlinks, ignore, copy_function, dirs_exist_ok=dirs_exist_ok) + else: + # Will raise a SpecialFileError for unsupported file types + copy_function(srcname, dstname) + # catch the Error from the recursive copytree3 so that we can + # continue with other files + except shutil.Error as err: + errors.extend(err.args[0]) + except EnvironmentError as why: + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) + except OSError as why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: + raise shutil.Error(errors) + + +def walk_relative(path, topdown=True, onerror=None, followlinks=False): + for dirpath, dirnames, filenames in os.walk(path, topdown=topdown, onerror=onerror, followlinks=followlinks): + yield os.path.relpath(dirpath, path), dirnames, filenames + + +def supports_clone(): + if 'darwin' in sys.platform: + import platform + + return list(map(int, platform.mac_ver()[0].split('.'))) >= [10, 13] + return False + + +def commonpath(paths): + assert paths + if len(paths) == 1: + return next(iter(paths)) + + split_paths = [path.split(os.sep) for path in paths] + smin = min(split_paths) + smax = max(split_paths) + + common = smin + for i, c in enumerate(smin): + if c != smax[i]: + common = smin[:i] + break + + return os.path.sep.join(common) + + +def set_execute_bits(filename): + stm = os.stat(filename).st_mode + exe = stm | 0o111 + if stm != exe: + os.chmod(filename, exe) diff --git a/library/python/fs/clonefile.pyx b/library/python/fs/clonefile.pyx new file mode 100644 index 0000000000..830bb894f2 --- /dev/null +++ b/library/python/fs/clonefile.pyx @@ -0,0 +1,18 @@ +import six + +cdef extern from "sys/clonefile.h" nogil: + int clonefile(const char * src, const char * dst, int flags) + +cdef extern from "Python.h": + ctypedef struct PyObject + cdef PyObject *PyExc_OSError + PyObject *PyErr_SetFromErrno(PyObject *) + +cdef int _macos_clone_file(const char* src, const char* dst) except? 0: + if clonefile(src, dst, 0) == -1: + PyErr_SetFromErrno(PyExc_OSError) + return 0 + return 1 + +def macos_clone_file(src, dst): + return _macos_clone_file(six.ensure_binary(src), six.ensure_binary(dst)) != 0 diff --git a/library/python/fs/test/test_fs.py b/library/python/fs/test/test_fs.py new file mode 100644 index 0000000000..9e2c70c069 --- /dev/null +++ b/library/python/fs/test/test_fs.py @@ -0,0 +1,1037 @@ +# coding=utf-8 + +import errno +import os +import pytest +import shutil +import six + +import library.python.fs +import library.python.strings +import library.python.tmp +import library.python.windows + +import yatest.common + + +def in_env(case): + def wrapped_case(*args, **kwargs): + with library.python.tmp.temp_dir() as temp_dir: + case(lambda path: os.path.join(temp_dir, path)) + + return wrapped_case + + +def mkfile(path, data=''): + with open(path, 'wb') as f: + if data: + f.write(data) if isinstance(data, six.binary_type) else f.write( + data.encode(library.python.strings.fs_encoding()) + ) + + +def mktree_example(path, name): + os.mkdir(path(name)) + mkfile(path(name + '/file1'), 'FILE1') + os.mkdir(path(name + '/dir1')) + os.mkdir(path(name + '/dir2')) + mkfile(path(name + '/dir2/file2'), 'FILE2') + mkfile(path(name + '/dir2/file3'), 'FILE3') + + +def file_data(path): + with open(path, 'rb') as f: + return f.read().decode('utf-8') + + +def serialize_tree(path): + if os.path.isfile(path): + return file_data(path) + data = {'dirs': set(), 'files': {}} + for dirpath, dirnames, filenames in os.walk(path): + dirpath_rel = os.path.relpath(dirpath, path) + if dirpath_rel == '.': + dirpath_rel = '' + data['dirs'].update(set(os.path.join(dirpath_rel, x) for x in dirnames)) + data['files'].update({os.path.join(dirpath_rel, x): file_data(os.path.join(dirpath, x)) for x in filenames}) + return data + + +def trees_equal(dir1, dir2): + return serialize_tree(dir1) == serialize_tree(dir2) + + +def inodes_unsupported(): + return library.python.windows.on_win() + + +def inodes_equal(path1, path2): + return os.stat(path1).st_ino == os.stat(path2).st_ino + + +def gen_error_access_denied(): + if library.python.windows.on_win(): + err = WindowsError() + err.errno = errno.EACCES + err.strerror = '' + err.winerror = library.python.windows.ERRORS['ACCESS_DENIED'] + else: + err = OSError() + err.errno = errno.EACCES + err.strerror = os.strerror(err.errno) + err.filename = 'unknown/file' + raise err + + +def test_errorfix_win(): + @library.python.fs.errorfix_win + def erroneous_func(): + gen_error_access_denied() + + with pytest.raises(OSError) as errinfo: + erroneous_func() + assert errinfo.value.errno == errno.EACCES + assert errinfo.value.filename == 'unknown/file' + # See transcode_error, which encodes strerror, in library/python/windows/__init__.py + assert isinstance(errinfo.value.strerror, (six.binary_type, six.text_type)) + assert errinfo.value.strerror + + +def test_custom_fs_error(): + with pytest.raises(OSError) as errinfo: + raise library.python.fs.CustomFsError(errno.EACCES, filename='some/file') + assert errinfo.value.errno == errno.EACCES + # See transcode_error, which encodes strerror, in library/python/windows/__init__.py + assert isinstance(errinfo.value.strerror, (six.binary_type, six.text_type)) + assert errinfo.value.filename == 'some/file' + + +@in_env +def test_ensure_dir(path): + library.python.fs.ensure_dir(path('dir/subdir')) + assert os.path.isdir(path('dir')) + assert os.path.isdir(path('dir/subdir')) + + +@in_env +def test_ensure_dir_exists(path): + os.makedirs(path('dir/subdir')) + library.python.fs.ensure_dir(path('dir/subdir')) + assert os.path.isdir(path('dir')) + assert os.path.isdir(path('dir/subdir')) + + +@in_env +def test_ensure_dir_exists_partly(path): + os.mkdir(path('dir')) + library.python.fs.ensure_dir(path('dir/subdir')) + assert os.path.isdir(path('dir')) + assert os.path.isdir(path('dir/subdir')) + + +@in_env +def test_ensure_dir_exists_file(path): + mkfile(path('dir')) + with pytest.raises(OSError) as errinfo: + library.python.fs.ensure_dir(path('dir/subdir')) + # ENOENT on Windows! + assert errinfo.value.errno in (errno.ENOTDIR, errno.ENOENT) + assert os.path.isfile(path('dir')) + + +@in_env +def test_create_dirs(path): + assert library.python.fs.create_dirs(path('dir/subdir')) == path('dir/subdir') + assert os.path.isdir(path('dir')) + assert os.path.isdir(path('dir/subdir')) + + +@in_env +def test_move_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.move(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_move_file_no_src(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_move_file_exists(path): + mkfile(path('src'), 'SRC') + mkfile(path('dst'), 'DST') + if library.python.windows.on_win(): + # move is platform-dependent, use replace_file for dst replacement on all platforms + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'DST' + else: + library.python.fs.move(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_move_file_exists_dir_empty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_move_file_exists_dir_nonempty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/dst_file')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_move_dir(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + library.python.fs.move(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@in_env +def test_move_dir_exists_empty(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + os.mkdir(path('dst')) + if library.python.windows.on_win(): + # move is platform-dependent, use non-atomic replace for directory replacement + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src_file')) + else: + library.python.fs.move(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@in_env +def test_move_dir_exists_nonempty(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EEXIST, errno.ENOTEMPTY) + assert os.path.isdir(path('src')) + assert os.path.isfile(path('src/src_file')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src_file')) + assert os.path.isfile(path('dst/dst_file')) + + +@in_env +def test_move_dir_exists_file(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + mkfile(path('dst'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.move(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EEXIST, errno.ENOTDIR) + assert os.path.isdir(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'DST' + + +@in_env +def test_replace_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.replace_file(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + mkfile(path('src'), 'SRC') + library.python.fs.replace(path('src'), path('dst2')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst2')) + assert file_data(path('dst2')) == 'SRC' + + +@in_env +def test_replace_file_no_src(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.replace_file(path('src'), path('dst')) + assert errinfo.value.errno == errno.ENOENT + + with pytest.raises(OSError) as errinfo2: + library.python.fs.replace(path('src'), path('dst2')) + assert errinfo2.value.errno == errno.ENOENT + + +@in_env +def test_replace_file_exists(path): + mkfile(path('src'), 'SRC') + mkfile(path('dst'), 'DST') + library.python.fs.replace_file(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + mkfile(path('src'), 'SRC') + mkfile(path('dst2'), 'DST') + library.python.fs.replace(path('src'), path('dst2')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst2')) + assert file_data(path('dst2')) == 'SRC' + + +@in_env +def test_replace_file_exists_dir_empty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + with pytest.raises(OSError) as errinfo: + library.python.fs.replace_file(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_replace_file_exists_dir_empty_overwrite(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_replace_file_exists_dir_nonempty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + with pytest.raises(OSError) as errinfo: + library.python.fs.replace_file(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/dst_file')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_replace_file_exists_dir_nonempty_overwrite(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_replace_dir(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@in_env +def test_replace_dir_exists_empty(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + os.mkdir(path('dst')) + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@in_env +def test_replace_dir_exists_nonempty(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + assert not os.path.isfile(path('dst/dst_file')) + + +@in_env +def test_replace_dir_exists_file(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + mkfile(path('dst'), 'DST') + library.python.fs.replace(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@in_env +def test_remove_file(path): + mkfile(path('path')) + library.python.fs.remove_file(path('path')) + assert not os.path.exists(path('path')) + + +@in_env +def test_remove_file_no(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.remove_file(path('path')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_remove_file_exists_dir(path): + os.mkdir(path('path')) + with pytest.raises(OSError) as errinfo: + library.python.fs.remove_file(path('path')) + assert errinfo.value.errno in (errno.EISDIR, errno.EACCES) + assert os.path.isdir(path('path')) + + +@in_env +def test_remove_dir(path): + os.mkdir(path('path')) + library.python.fs.remove_dir(path('path')) + assert not os.path.exists(path('path')) + + +@in_env +def test_remove_dir_no(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.remove_dir(path('path')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_remove_dir_exists_file(path): + mkfile(path('path')) + with pytest.raises(OSError) as errinfo: + library.python.fs.remove_dir(path('path')) + assert errinfo.value.errno in (errno.ENOTDIR, errno.EINVAL) + assert os.path.isfile(path('path')) + + +@in_env +def test_remove_tree(path): + mktree_example(path, 'path') + library.python.fs.remove_tree(path('path')) + assert not os.path.exists(path('path')) + + +@in_env +def test_remove_tree_empty(path): + os.mkdir(path('path')) + library.python.fs.remove_tree(path('path')) + assert not os.path.exists(path('path')) + + +@in_env +def test_remove_tree_file(path): + mkfile(path('path')) + library.python.fs.remove_tree(path('path')) + assert not os.path.exists(path('path')) + + +@in_env +def test_remove_tree_no(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.remove_tree(path('path')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_remove_tree_safe(path): + library.python.fs.remove_tree_safe(path('path')) + + +@in_env +def test_ensure_removed(path): + library.python.fs.ensure_removed(path('path')) + + +@in_env +def test_ensure_removed_exists(path): + os.makedirs(path('dir/subdir')) + library.python.fs.ensure_removed(path('dir')) + assert not os.path.exists(path('dir')) + + +@in_env +def test_ensure_removed_exists_precise(path): + os.makedirs(path('dir/subdir')) + library.python.fs.ensure_removed(path('dir/subdir')) + assert os.path.exists(path('dir')) + assert not os.path.exists(path('dir/subdir')) + + +@in_env +def test_hardlink_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.hardlink(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + assert inodes_unsupported() or inodes_equal(path('src'), path('dst')) + + +@in_env +def test_hardlink_file_no_src(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_hardlink_file_exists(path): + mkfile(path('src'), 'SRC') + mkfile(path('dst'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'DST' + assert inodes_unsupported() or not inodes_equal(path('src'), path('dst')) + + +@in_env +def test_hardlink_file_exists_dir(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_hardlink_dir(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink(path('src'), path('dst')) + assert errinfo.value.errno in (errno.EPERM, errno.EACCES) + assert os.path.isdir(path('src')) + assert not os.path.isdir(path('dst')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.symlink(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert os.path.islink(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_file_no_src(path): + library.python.fs.symlink(path('src'), path('dst')) + assert not os.path.isfile(path('src')) + assert not os.path.isfile(path('dst')) + assert os.path.islink(path('dst')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_file_exists(path): + mkfile(path('src'), 'SRC') + mkfile(path('dst'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.symlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert not os.path.islink(path('dst')) + assert file_data(path('dst')) == 'DST' + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_file_exists_dir(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + with pytest.raises(OSError) as errinfo: + library.python.fs.symlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.islink(path('dst')) + assert not os.path.isfile(path('dst/src')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_dir(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + library.python.fs.symlink(path('src'), path('dst')) + assert os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.islink(path('dst')) + assert os.path.isfile(path('dst/src_file')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_dir_no_src(path): + library.python.fs.symlink(path('src'), path('dst')) + assert not os.path.isdir(path('src')) + assert not os.path.isdir(path('dst')) + assert os.path.islink(path('dst')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_dir_exists(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + os.mkdir(path('dst')) + with pytest.raises(OSError) as errinfo: + library.python.fs.symlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isdir(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.islink(path('dst')) + assert not os.path.isfile(path('dst/src_file')) + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_symlink_dir_exists_file(path): + os.mkdir(path('src')) + mkfile(path('src/src_file')) + mkfile(path('dst'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.symlink(path('src'), path('dst')) + assert errinfo.value.errno == errno.EEXIST + assert os.path.isdir(path('src')) + assert os.path.isfile(path('dst')) + assert not os.path.islink(path('dst')) + + +@in_env +def test_hardlink_tree(path): + mktree_example(path, 'src') + library.python.fs.hardlink_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_hardlink_tree_empty(path): + os.mkdir(path('src')) + library.python.fs.hardlink_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_hardlink_tree_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.hardlink_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_hardlink_tree_no_src(path): + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink_tree(path('src'), path('dst')) + assert errinfo.value.errno == errno.ENOENT + + +@in_env +def test_hardlink_tree_exists(path): + mktree_example(path, 'src') + os.mkdir(path('dst_dir')) + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink_tree(path('src'), path('dst_dir')) + assert errinfo.value.errno == errno.EEXIST + mkfile(path('dst_file'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink_tree(path('src'), path('dst_file')) + assert errinfo.value.errno == errno.EEXIST + + +@in_env +def test_hardlink_tree_file_exists(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst_dir')) + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink_tree(path('src'), path('dst_dir')) + assert errinfo.value.errno == errno.EEXIST + mkfile(path('dst_file'), 'DST') + with pytest.raises(OSError) as errinfo: + library.python.fs.hardlink_tree(path('src'), path('dst_file')) + assert errinfo.value.errno == errno.EEXIST + + +@in_env +def test_copy_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.copy_file(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_copy_file_no_src(path): + with pytest.raises(EnvironmentError): + library.python.fs.copy_file(path('src'), path('dst')) + + +@in_env +def test_copy_file_exists(path): + mkfile(path('src'), 'SRC') + mkfile(path('dst'), 'DST') + library.python.fs.copy_file(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isfile(path('dst')) + assert file_data(path('dst')) == 'SRC' + + +@in_env +def test_copy_file_exists_dir_empty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + with pytest.raises(EnvironmentError): + library.python.fs.copy_file(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_copy_file_exists_dir_nonempty(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst')) + mkfile(path('dst/dst_file')) + with pytest.raises(EnvironmentError): + library.python.fs.copy_file(path('src'), path('dst')) + assert os.path.isfile(path('src')) + assert os.path.isdir(path('dst')) + assert os.path.isfile(path('dst/dst_file')) + assert not os.path.isfile(path('dst/src')) + + +@in_env +def test_copy_tree(path): + mktree_example(path, 'src') + library.python.fs.copy_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_copy_tree_empty(path): + os.mkdir(path('src')) + library.python.fs.copy_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_copy_tree_file(path): + mkfile(path('src'), 'SRC') + library.python.fs.copy_tree(path('src'), path('dst')) + assert trees_equal(path('src'), path('dst')) + + +@in_env +def test_copy_tree_no_src(path): + with pytest.raises(EnvironmentError): + library.python.fs.copy_tree(path('src'), path('dst')) + + +@in_env +def test_copy_tree_exists(path): + mktree_example(path, 'src') + os.mkdir(path('dst_dir')) + with pytest.raises(EnvironmentError): + library.python.fs.copy_tree(path('src'), path('dst_dir')) + mkfile(path('dst_file'), 'DST') + with pytest.raises(EnvironmentError): + library.python.fs.copy_tree(path('src'), path('dst_file')) + + +@in_env +def test_copy_tree_file_exists(path): + mkfile(path('src'), 'SRC') + os.mkdir(path('dst_dir')) + with pytest.raises(EnvironmentError): + library.python.fs.copy_tree(path('src'), path('dst_dir')) + mkfile(path('dst_file'), 'DST') + library.python.fs.copy_tree(path('src'), path('dst_file')) + assert trees_equal(path('src'), path('dst_file')) + + +@in_env +def test_read_file(path): + mkfile(path('src'), 'SRC') + assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == 'SRC' + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC' + + +@in_env +def test_read_file_empty(path): + mkfile(path('src')) + assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == '' + assert library.python.fs.read_file(path('src'), binary=False) == '' + + +@in_env +def test_read_file_multiline(path): + mkfile(path('src'), 'SRC line 1\nSRC line 2\n') + assert ( + library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) + == 'SRC line 1\nSRC line 2\n' + ) + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' + + +@in_env +def test_read_file_multiline_crlf(path): + mkfile(path('src'), 'SRC line 1\r\nSRC line 2\r\n') + assert ( + library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) + == 'SRC line 1\r\nSRC line 2\r\n' + ) + if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3 + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n' + else: + assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\r\nSRC line 2\r\n' + + +@in_env +def test_read_file_unicode(path): + s = u'АБВ' + mkfile(path('src'), s.encode('utf-8')) + mkfile(path('src_cp1251'), s.encode('cp1251')) + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + + +@in_env +def test_read_file_unicode_empty(path): + mkfile(path('src')) + mkfile(path('src_cp1251')) + assert library.python.fs.read_file_unicode(path('src')) == '' + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == '' + assert library.python.fs.read_file_unicode(path('src'), binary=False) == '' + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == '' + + +@in_env +def test_read_file_unicode_multiline(path): + s = u'АБВ\nИ еще\n' + mkfile(path('src'), s.encode('utf-8')) + mkfile(path('src_cp1251'), s.encode('cp1251')) + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + + +@in_env +def test_read_file_unicode_multiline_crlf(path): + s = u'АБВ\r\nИ еще\r\n' + mkfile(path('src'), s.encode('utf-8')) + mkfile(path('src_cp1251'), s.encode('cp1251')) + assert library.python.fs.read_file_unicode(path('src')) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s + if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3 + assert library.python.fs.read_file_unicode(path('src'), binary=False) == u'АБВ\nИ еще\n' + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == u'АБВ\nИ еще\n' + else: + assert library.python.fs.read_file_unicode(path('src'), binary=False) == s + assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s + + +@in_env +def test_write_file(path): + library.python.fs.write_file(path('src'), 'SRC') + assert file_data(path('src')) == 'SRC' + library.python.fs.write_file(path('src2'), 'SRC', binary=False) + assert file_data(path('src2')) == 'SRC' + + +@in_env +def test_write_file_empty(path): + library.python.fs.write_file(path('src'), '') + assert file_data(path('src')) == '' + library.python.fs.write_file(path('src2'), '', binary=False) + assert file_data(path('src2')) == '' + + +@in_env +def test_write_file_multiline(path): + library.python.fs.write_file(path('src'), 'SRC line 1\nSRC line 2\n') + assert file_data(path('src')) == 'SRC line 1\nSRC line 2\n' + library.python.fs.write_file(path('src2'), 'SRC line 1\nSRC line 2\n', binary=False) + if library.python.windows.on_win(): + assert file_data(path('src2')) == 'SRC line 1\r\nSRC line 2\r\n' + else: + assert file_data(path('src2')) == 'SRC line 1\nSRC line 2\n' + + +@in_env +def test_write_file_multiline_crlf(path): + library.python.fs.write_file(path('src'), 'SRC line 1\r\nSRC line 2\r\n') + assert file_data(path('src')) == 'SRC line 1\r\nSRC line 2\r\n' + library.python.fs.write_file(path('src2'), 'SRC line 1\r\nSRC line 2\r\n', binary=False) + if library.python.windows.on_win(): + assert file_data(path('src2')) == 'SRC line 1\r\r\nSRC line 2\r\r\n' + else: + assert file_data(path('src2')) == 'SRC line 1\r\nSRC line 2\r\n' + + +@in_env +def test_get_file_size(path): + mkfile(path('one.txt'), '22') + assert library.python.fs.get_file_size(path('one.txt')) == 2 + + +@in_env +def test_get_file_size_empty(path): + mkfile(path('one.txt')) + assert library.python.fs.get_file_size(path('one.txt')) == 0 + + +@in_env +def test_get_tree_size(path): + os.makedirs(path('deeper')) + mkfile(path('one.txt'), '1') + mkfile(path('deeper/two.txt'), '22') + assert library.python.fs.get_tree_size(path('one.txt')) == 1 + assert library.python.fs.get_tree_size(path('')) == 1 + assert library.python.fs.get_tree_size(path(''), recursive=True) == 3 + + +@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows') +@in_env +def test_get_tree_size_dangling_symlink(path): + os.makedirs(path('deeper')) + mkfile(path('one.txt'), '1') + mkfile(path('deeper/two.txt'), '22') + os.symlink(path('deeper/two.txt'), path("deeper/link.txt")) + os.remove(path('deeper/two.txt')) + # does not fail + assert library.python.fs.get_tree_size(path(''), recursive=True) == 1 + + +@pytest.mark.skipif(not library.python.windows.on_win(), reason='Test hardlinks on windows') +def test_hardlink_or_copy(): + max_allowed_hard_links = 1023 + + def run(hardlink_function, dir): + src = r"test.txt" + with open(src, "w") as f: + f.write("test") + for i in range(max_allowed_hard_links + 1): + hardlink_function(src, os.path.join(dir, "{}.txt".format(i))) + + dir1 = library.python.fs.create_dirs("one") + with pytest.raises(WindowsError) as e: + run(library.python.fs.hardlink, dir1) + assert e.value.winerror == 1142 + assert len(os.listdir(dir1)) == max_allowed_hard_links + + dir2 = library.python.fs.create_dirs("two") + run(library.python.fs.hardlink_or_copy, dir2) + assert len(os.listdir(dir2)) == max_allowed_hard_links + 1 + + +def test_remove_tree_unicode(): + path = u"test_remove_tree_unicode/русский".encode("utf-8") + os.makedirs(path) + library.python.fs.remove_tree(six.text_type("test_remove_tree_unicode")) + assert not os.path.exists("test_remove_tree_unicode") + + +def test_remove_tree_safe_unicode(): + path = u"test_remove_tree_safe_unicode/русский".encode("utf-8") + os.makedirs(path) + library.python.fs.remove_tree_safe(six.text_type("test_remove_tree_safe_unicode")) + assert not os.path.exists("test_remove_tree_safe_unicode") + + +def test_copy_tree_custom_copy_function(): + library.python.fs.create_dirs("test_copy_tree_src/deepper/inner") + library.python.fs.write_file("test_copy_tree_src/deepper/deepper.txt", "deepper.txt") + library.python.fs.write_file("test_copy_tree_src/deepper/inner/inner.txt", "inner.txt") + copied = [] + + def copy_function(src, dst): + shutil.copy2(src, dst) + copied.append(dst) + + library.python.fs.copy_tree( + "test_copy_tree_src", yatest.common.work_path("test_copy_tree_dst"), copy_function=copy_function + ) + assert len(copied) == 2 + assert yatest.common.work_path("test_copy_tree_dst/deepper/deepper.txt") in copied + assert yatest.common.work_path("test_copy_tree_dst/deepper/inner/inner.txt") in copied + + +def test_copy2(): + library.python.fs.symlink("non-existent", "link") + library.python.fs.copy2("link", "link2", follow_symlinks=False) + + assert os.path.islink("link2") + assert os.readlink("link2") == "non-existent" + + +def test_commonpath(): + pj = os.path.join + pja = lambda *x: os.path.abspath(pj(*x)) + + assert library.python.fs.commonpath(['a', 'b']) == '' + assert library.python.fs.commonpath([pj('t', '1')]) == pj('t', '1') + assert library.python.fs.commonpath([pj('t', '1'), pj('t', '2')]) == pj('t') + assert library.python.fs.commonpath([pj('t', '1', '2'), pj('t', '1', '2')]) == pj('t', '1', '2') + assert library.python.fs.commonpath([pj('t', '1', '1'), pj('t', '1', '2')]) == pj('t', '1') + assert library.python.fs.commonpath([pj('t', '1', '1'), pj('t', '1', '2'), pj('t', '1', '3')]) == pj('t', '1') + + assert library.python.fs.commonpath([pja('t', '1', '1'), pja('t', '1', '2')]) == pja('t', '1') + + assert library.python.fs.commonpath({pj('t', '1'), pj('t', '2')}) == pj('t') diff --git a/library/python/fs/test/ya.make b/library/python/fs/test/ya.make new file mode 100644 index 0000000000..33e3f5b4ff --- /dev/null +++ b/library/python/fs/test/ya.make @@ -0,0 +1,14 @@ +OWNER(g:yatool) + +PY23_TEST() + +TEST_SRCS( + test_fs.py +) + +PEERDIR( + library/python/fs + library/python/tmp +) + +END() diff --git a/library/python/fs/ya.make b/library/python/fs/ya.make new file mode 100644 index 0000000000..b3c5092c71 --- /dev/null +++ b/library/python/fs/ya.make @@ -0,0 +1,23 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +PY_SRCS( + __init__.py +) + +IF (OS_DARWIN) + PY_SRCS( + clonefile.pyx + ) +ENDIF() + +PEERDIR( + library/python/func + library/python/strings + library/python/windows +) + +END() + +RECURSE_FOR_TESTS(test) diff --git a/library/python/func/__init__.py b/library/python/func/__init__.py new file mode 100644 index 0000000000..7424361635 --- /dev/null +++ b/library/python/func/__init__.py @@ -0,0 +1,170 @@ +import functools +import threading +import collections + + +def map0(func, value): + return func(value) if value is not None else value + + +def single(x): + if len(x) != 1: + raise Exception('Length of {} is not equal to 1'.format(x)) + return x[0] + + +class _Result(object): + pass + + +def lazy(func): + result = _Result() + + @functools.wraps(func) + def wrapper(*args): + try: + return result.result + except AttributeError: + result.result = func(*args) + + return result.result + + return wrapper + + +def lazy_property(fn): + attr_name = '_lazy_' + fn.__name__ + + @property + def _lazy_property(self): + if not hasattr(self, attr_name): + setattr(self, attr_name, fn(self)) + return getattr(self, attr_name) + + return _lazy_property + + +class classproperty(object): + def __init__(self, func): + self.func = func + + def __get__(self, _, owner): + return self.func(owner) + + +class lazy_classproperty(object): + def __init__(self, func): + self.func = func + + def __get__(self, _, owner): + attr_name = '_lazy_' + self.func.__name__ + + if not hasattr(owner, attr_name): + setattr(owner, attr_name, self.func(owner)) + return getattr(owner, attr_name) + + +def memoize(limit=0, thread_local=False): + assert limit >= 0 + + def decorator(func): + memory = {} + lock = threading.Lock() + + if limit: + keys = collections.deque() + + def get(args): + try: + return memory[args] + except KeyError: + with lock: + if args not in memory: + fargs = args[-1] + memory[args] = func(*fargs) + keys.append(args) + if len(keys) > limit: + del memory[keys.popleft()] + return memory[args] + + else: + + def get(args): + if args not in memory: + with lock: + if args not in memory: + fargs = args[-1] + memory[args] = func(*fargs) + return memory[args] + + if thread_local: + + @functools.wraps(func) + def wrapper(*args): + th = threading.current_thread() + return get((th.ident, th.name, args)) + + else: + + @functools.wraps(func) + def wrapper(*args): + return get(('', '', args)) + + return wrapper + + return decorator + + +# XXX: add test +def compose(*functions): + def compose2(f, g): + return lambda x: f(g(x)) + + return functools.reduce(compose2, functions, lambda x: x) + + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + + +def stable_uniq(it): + seen = set() + res = [] + for e in it: + if e not in seen: + res.append(e) + seen.add(e) + return res + + +def first(it): + for d in it: + if d: + return d + + +def split(data, func): + l, r = [], [] + for e in data: + if func(e): + l.append(e) + else: + r.append(e) + return l, r + + +def flatten_dict(dd, separator='.', prefix=''): + return ( + { + prefix + separator + k if prefix else k: v + for kk, vv in dd.items() + for k, v in flatten_dict(vv, separator, kk).items() + } + if isinstance(dd, dict) + else {prefix: dd} + ) diff --git a/library/python/func/ut/test_func.py b/library/python/func/ut/test_func.py new file mode 100644 index 0000000000..3c4fad1a07 --- /dev/null +++ b/library/python/func/ut/test_func.py @@ -0,0 +1,162 @@ +import pytest +import threading + +import library.python.func as func + + +def test_map0(): + assert None is func.map0(lambda x: x + 1, None) + assert 3 == func.map0(lambda x: x + 1, 2) + assert None is func.map0(len, None) + assert 2 == func.map0(len, [1, 2]) + + +def test_single(): + assert 1 == func.single([1]) + with pytest.raises(Exception): + assert 1 == func.single([]) + with pytest.raises(Exception): + assert 1 == func.single([1, 2]) + + +def test_memoize(): + class Counter(object): + @staticmethod + def inc(): + Counter._qty = getattr(Counter, '_qty', 0) + 1 + return Counter._qty + + @func.memoize() + def t1(a): + return a, Counter.inc() + + @func.memoize() + def t2(a): + return a, Counter.inc() + + @func.memoize() + def t3(a): + return a, Counter.inc() + + @func.memoize() + def t4(a): + return a, Counter.inc() + + @func.memoize() + def t5(a, b, c): + return a + b + c, Counter.inc() + + @func.memoize() + def t6(): + return Counter.inc() + + @func.memoize(limit=2) + def t7(a, _b): + return a, Counter.inc() + + assert (1, 1) == t1(1) + assert (1, 1) == t1(1) + assert (2, 2) == t1(2) + assert (2, 2) == t1(2) + + assert (1, 3) == t2(1) + assert (1, 3) == t2(1) + assert (2, 4) == t2(2) + assert (2, 4) == t2(2) + + assert (1, 5) == t3(1) + assert (1, 5) == t3(1) + assert (2, 6) == t3(2) + assert (2, 6) == t3(2) + + assert (1, 7) == t4(1) + assert (1, 7) == t4(1) + assert (2, 8) == t4(2) + assert (2, 8) == t4(2) + + assert (6, 9) == t5(1, 2, 3) + assert (6, 9) == t5(1, 2, 3) + assert (7, 10) == t5(1, 2, 4) + assert (7, 10) == t5(1, 2, 4) + + assert 11 == t6() + assert 11 == t6() + + assert (1, 12) == t7(1, None) + assert (2, 13) == t7(2, None) + assert (1, 12) == t7(1, None) + assert (2, 13) == t7(2, None) + # removed result for (1, None) + assert (3, 14) == t7(3, None) + assert (1, 15) == t7(1, None) + + class ClassWithMemoizedMethod(object): + def __init__(self): + self.a = 0 + + @func.memoize(True) + def t(self, i): + self.a += i + return i + + obj = ClassWithMemoizedMethod() + assert 10 == obj.t(10) + assert 10 == obj.a + assert 10 == obj.t(10) + assert 10 == obj.a + + assert 20 == obj.t(20) + assert 30 == obj.a + assert 20 == obj.t(20) + assert 30 == obj.a + + +def test_first(): + assert func.first([0, [], (), None, False, {}, 0.0, '1', 0]) == '1' + assert func.first([]) is None + assert func.first([0]) is None + + +def test_split(): + assert func.split([1, 1], lambda x: x) == ([1, 1], []) + assert func.split([0, 0], lambda x: x) == ([], [0, 0]) + assert func.split([], lambda x: x) == ([], []) + assert func.split([1, 0, 1], lambda x: x) == ([1, 1], [0]) + + +def test_flatten_dict(): + assert func.flatten_dict({"a": 1, "b": 2}) == {"a": 1, "b": 2} + assert func.flatten_dict({"a": 1}) == {"a": 1} + assert func.flatten_dict({}) == {} + assert func.flatten_dict({"a": 1, "b": {"c": {"d": 2}}}) == {"a": 1, "b.c.d": 2} + assert func.flatten_dict({"a": 1, "b": {"c": {"d": 2}}}, separator="/") == {"a": 1, "b/c/d": 2} + + +def test_memoize_thread_local(): + class Counter(object): + def __init__(self, s): + self.val = s + + def inc(self): + self.val += 1 + return self.val + + @func.memoize(thread_local=True) + def get_counter(start): + return Counter(start) + + def th_inc(): + assert get_counter(0).inc() == 1 + assert get_counter(0).inc() == 2 + assert get_counter(10).inc() == 11 + assert get_counter(10).inc() == 12 + + th_inc() + + th = threading.Thread(target=th_inc) + th.start() + th.join() + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/library/python/func/ut/ya.make b/library/python/func/ut/ya.make new file mode 100644 index 0000000000..5ec6c1225e --- /dev/null +++ b/library/python/func/ut/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY23_TEST() + +TEST_SRCS(test_func.py) + +PEERDIR( + library/python/func +) + +END() diff --git a/library/python/func/ya.make b/library/python/func/ya.make new file mode 100644 index 0000000000..9d414a976e --- /dev/null +++ b/library/python/func/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +PY_SRCS(__init__.py) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/library/python/pytest/__init__.py b/library/python/pytest/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/library/python/pytest/__init__.py diff --git a/library/python/pytest/allure/conftest.py b/library/python/pytest/allure/conftest.py new file mode 100644 index 0000000000..0d5cfda1e5 --- /dev/null +++ b/library/python/pytest/allure/conftest.py @@ -0,0 +1,8 @@ +import os +import pytest + + +@pytest.mark.tryfirst +def pytest_configure(config): + if "ALLURE_REPORT_DIR" in os.environ: + config.option.allurereportdir = os.environ["ALLURE_REPORT_DIR"] diff --git a/library/python/pytest/allure/ya.make b/library/python/pytest/allure/ya.make new file mode 100644 index 0000000000..ab3f449c7f --- /dev/null +++ b/library/python/pytest/allure/ya.make @@ -0,0 +1,11 @@ +PY23_LIBRARY() + +OWNER(exprmntr) + +PY_SRCS(conftest.py) + +PEERDIR( + contrib/python/pytest-allure-adaptor +) + +END() diff --git a/library/python/pytest/context.py b/library/python/pytest/context.py new file mode 100644 index 0000000000..bfcdae50b5 --- /dev/null +++ b/library/python/pytest/context.py @@ -0,0 +1 @@ +Ctx = {} diff --git a/library/python/pytest/empty/main.c b/library/python/pytest/empty/main.c new file mode 100644 index 0000000000..9efa08162a --- /dev/null +++ b/library/python/pytest/empty/main.c @@ -0,0 +1,7 @@ +/* +to be used for build python tests in a stub binary for the case of using system python +*/ + +int main(void) { + return 0; +} diff --git a/library/python/pytest/empty/ya.make b/library/python/pytest/empty/ya.make new file mode 100644 index 0000000000..8f0fa37e2a --- /dev/null +++ b/library/python/pytest/empty/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +OWNER( + g:yatool + dmitko +) + +SRCS( + main.c +) + +END() diff --git a/library/python/pytest/main.py b/library/python/pytest/main.py new file mode 100644 index 0000000000..6296bd6f0f --- /dev/null +++ b/library/python/pytest/main.py @@ -0,0 +1,116 @@ +import os +import sys +import time + +import __res + +FORCE_EXIT_TESTSFAILED_ENV = 'FORCE_EXIT_TESTSFAILED' + + +def main(): + import library.python.pytest.context as context + context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() + + profile = None + if '--profile-pytest' in sys.argv: + sys.argv.remove('--profile-pytest') + + import pstats + import cProfile + profile = cProfile.Profile() + profile.enable() + + # Reset influencing env. vars + # For more info see library/python/testing/yatest_common/yatest/common/errors.py + if FORCE_EXIT_TESTSFAILED_ENV in os.environ: + del os.environ[FORCE_EXIT_TESTSFAILED_ENV] + + if "Y_PYTHON_CLEAR_ENTRY_POINT" in os.environ: + if "Y_PYTHON_ENTRY_POINT" in os.environ: + del os.environ["Y_PYTHON_ENTRY_POINT"] + del os.environ["Y_PYTHON_CLEAR_ENTRY_POINT"] + + listing_mode = '--collect-only' in sys.argv + yatest_runner = os.environ.get('YA_TEST_RUNNER') == '1' + + import pytest + + import library.python.pytest.plugins.collection as collection + import library.python.pytest.plugins.ya as ya + import library.python.pytest.plugins.conftests as conftests + + import _pytest.assertion + from _pytest.monkeypatch import MonkeyPatch + from . import rewrite + m = MonkeyPatch() + m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook", rewrite.AssertionRewritingHook) + + prefix = '__tests__.' + + test_modules = [ + name[len(prefix):] for name in sys.extra_modules + if name.startswith(prefix) and not name.endswith('.conftest') + ] + + doctest_packages = __res.find("PY_DOCTEST_PACKAGES") or "" + if isinstance(doctest_packages, bytes): + doctest_packages = doctest_packages.decode('utf-8') + doctest_packages = doctest_packages.split() + + def is_doctest_module(name): + for package in doctest_packages: + if name == package or name.startswith(str(package) + "."): + return True + return False + + doctest_modules = [ + name for name in sys.extra_modules + if is_doctest_module(name) + ] + + def remove_user_site(paths): + site_paths = ('site-packages', 'site-python') + + def is_site_path(path): + for p in site_paths: + if path.find(p) != -1: + return True + return False + + new_paths = list(paths) + for p in paths: + if is_site_path(p): + new_paths.remove(p) + + return new_paths + + sys.path = remove_user_site(sys.path) + rc = pytest.main(plugins=[ + collection.CollectionPlugin(test_modules, doctest_modules), + ya, + conftests, + ]) + + if rc == 5: + # don't care about EXIT_NOTESTSCOLLECTED + rc = 0 + + if rc == 1 and yatest_runner and not listing_mode and not os.environ.get(FORCE_EXIT_TESTSFAILED_ENV) == '1': + # XXX it's place for future improvements + # Test wrapper should terminate with 0 exit code if there are common test failures + # and report it with trace-file machinery. + # However, there are several case when we don't want to suppress exit_code: + # - listing machinery doesn't use trace-file currently and rely on stdout and exit_code + # - RestartTestException and InfrastructureException required non-zero exit_code to be processes correctly + rc = 0 + + if profile: + profile.disable() + ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative') + ps.print_stats() + + sys.exit(rc) + + +if __name__ == '__main__': + main() diff --git a/library/python/pytest/plugins/collection.py b/library/python/pytest/plugins/collection.py new file mode 100644 index 0000000000..e36f47a78f --- /dev/null +++ b/library/python/pytest/plugins/collection.py @@ -0,0 +1,128 @@ +import os +import sys +from six import reraise + +import py + +import pytest # noqa +import _pytest.python +import _pytest.doctest +import json +import library.python.testing.filter.filter as test_filter + + +class LoadedModule(_pytest.python.Module): + def __init__(self, parent, name, **kwargs): + self.name = name + '.py' + self.session = parent + self.parent = parent + self.config = parent.config + self.keywords = {} + self.own_markers = [] + self.fspath = py.path.local() + + @classmethod + def from_parent(cls, **kwargs): + namespace = kwargs.pop('namespace', True) + kwargs.setdefault('fspath', py.path.local()) + + loaded_module = getattr(super(LoadedModule, cls), 'from_parent', cls)(**kwargs) + loaded_module.namespace = namespace + + return loaded_module + + @property + def _nodeid(self): + if os.getenv('CONFTEST_LOAD_POLICY') == 'LOCAL': + return self._getobj().__file__ + else: + return self.name + + @property + def nodeid(self): + return self._nodeid + + def _getobj(self): + module_name = self.name[:-len('.py')] + if self.namespace: + module_name = '__tests__.' + module_name + __import__(module_name) + return sys.modules[module_name] + + +class DoctestModule(LoadedModule): + + def collect(self): + import doctest + + module = self._getobj() + # uses internal doctest module parsing mechanism + finder = doctest.DocTestFinder() + optionflags = _pytest.doctest.get_optionflags(self) + runner = doctest.DebugRunner(verbose=0, optionflags=optionflags) + + try: + for test in finder.find(module, self.name[:-len('.py')]): + if test.examples: # skip empty doctests + yield getattr(_pytest.doctest.DoctestItem, 'from_parent', _pytest.doctest.DoctestItem)( + name=test.name, + parent=self, + runner=runner, + dtest=test) + except Exception: + import logging + logging.exception('DoctestModule failed, probably you can add NO_DOCTESTS() macro to ya.make') + etype, exc, tb = sys.exc_info() + msg = 'DoctestModule failed, probably you can add NO_DOCTESTS() macro to ya.make' + reraise(etype, type(exc)('{}\n{}'.format(exc, msg)), tb) + + +# NOTE: Since we are overriding collect method of pytest session, pytest hooks are not invoked during collection. +def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_filename_predicate): + if session.config.option.mode == 'list': + return not accept_filename_predicate(module.name) + + if filenames_from_full_filters is not None and module.name not in filenames_from_full_filters: + return True + + test_file_filter = getattr(session.config.option, 'test_file_filter', None) + if test_file_filter is None: + return False + if module.name != test_file_filter.replace('/', '.'): + return True + return False + + +class CollectionPlugin(object): + def __init__(self, test_modules, doctest_modules): + self._test_modules = test_modules + self._doctest_modules = doctest_modules + + def pytest_sessionstart(self, session): + + def collect(*args, **kwargs): + accept_filename_predicate = test_filter.make_py_file_filter(session.config.option.test_filter) + full_test_names_file_path = session.config.option.test_list_path + filenames_filter = None + + if full_test_names_file_path and os.path.exists(full_test_names_file_path): + with open(full_test_names_file_path, 'r') as afile: + # in afile stored 2 dimensional array such that array[modulo_index] contains tests which should be run in this test suite + full_names_filter = set(json.load(afile)[int(session.config.option.modulo_index)]) + filenames_filter = set(map(lambda x: x.split('::')[0], full_names_filter)) + + for test_module in self._test_modules: + module = LoadedModule.from_parent(name=test_module, parent=session) + if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): + yield module + + if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no': + module = DoctestModule.from_parent(name=test_module, parent=session) + if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): + yield module + + if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no': + for doctest_module in self._doctest_modules: + yield DoctestModule.from_parent(name=doctest_module, parent=session, namespace=False) + + session.collect = collect diff --git a/library/python/pytest/plugins/conftests.py b/library/python/pytest/plugins/conftests.py new file mode 100644 index 0000000000..522041f5a7 --- /dev/null +++ b/library/python/pytest/plugins/conftests.py @@ -0,0 +1,50 @@ +import os +import importlib +import sys +import inspect + +from pytest import hookimpl + +from .fixtures import metrics, links # noqa + +orig_getfile = inspect.getfile + + +def getfile(object): + res = orig_getfile(object) + if inspect.ismodule(object): + if not res and getattr(object, '__orig_file__'): + res = object.__orig_file__ + return res + +inspect.getfile = getfile +conftest_modules = [] + + +@hookimpl(trylast=True) +def pytest_load_initial_conftests(early_config, parser, args): + conftests = filter(lambda name: name.endswith(".conftest"), sys.extra_modules) + + def conftest_key(name): + if not name.startswith("__tests__."): + # Make __tests__ come last + return "_." + name + return name + + for name in sorted(conftests, key=conftest_key): + mod = importlib.import_module(name) + if os.getenv("CONFTEST_LOAD_POLICY") != "LOCAL": + mod.__orig_file__ = mod.__file__ + mod.__file__ = "" + conftest_modules.append(mod) + early_config.pluginmanager.consider_conftest(mod) + + +def getconftestmodules(*args, **kwargs): + return conftest_modules + + +def pytest_sessionstart(session): + # Override filesystem based relevant conftest discovery on the call path + assert session.config.pluginmanager + session.config.pluginmanager._getconftestmodules = getconftestmodules diff --git a/library/python/pytest/plugins/fakeid_py2.py b/library/python/pytest/plugins/fakeid_py2.py new file mode 100644 index 0000000000..8b26148e2e --- /dev/null +++ b/library/python/pytest/plugins/fakeid_py2.py @@ -0,0 +1,2 @@ +# Inc this number to change uid for every PYTEST() target +fake_id = 0 diff --git a/library/python/pytest/plugins/fakeid_py3.py b/library/python/pytest/plugins/fakeid_py3.py new file mode 100644 index 0000000000..247cc8b29d --- /dev/null +++ b/library/python/pytest/plugins/fakeid_py3.py @@ -0,0 +1,2 @@ +# Inc this number to change uid for every PY3TEST() target +fake_id = 10 diff --git a/library/python/pytest/plugins/fixtures.py b/library/python/pytest/plugins/fixtures.py new file mode 100644 index 0000000000..6f7e0a27e4 --- /dev/null +++ b/library/python/pytest/plugins/fixtures.py @@ -0,0 +1,85 @@ +import os +import pytest +import six + + +MAX_ALLOWED_LINKS_COUNT = 10 + + +@pytest.fixture +def metrics(request): + + class Metrics(object): + @classmethod + def set(cls, name, value): + assert len(name) <= 128, "Length of the metric name must less than 128" + assert type(value) in [int, float], "Metric value must be of type int or float" + test_name = request.node.nodeid + if test_name not in request.config.test_metrics: + request.config.test_metrics[test_name] = {} + request.config.test_metrics[test_name][name] = value + + @classmethod + def set_benchmark(cls, benchmark_values): + # report of google has key 'benchmarks' which is a list of benchmark results + # yandex benchmark has key 'benchmark', which is a list of benchmark results + # use this to differentiate which kind of result it is + if 'benchmarks' in benchmark_values: + cls.set_gbenchmark(benchmark_values) + else: + cls.set_ybenchmark(benchmark_values) + + @classmethod + def set_ybenchmark(cls, benchmark_values): + for benchmark in benchmark_values["benchmark"]: + name = benchmark["name"] + for key, value in six.iteritems(benchmark): + if key != "name": + cls.set("{}_{}".format(name, key), value) + + @classmethod + def set_gbenchmark(cls, benchmark_values): + time_unit_multipliers = {"ns": 1, "us": 1000, "ms": 1000000} + time_keys = {"real_time", "cpu_time"} + ignore_keys = {"name", "run_name", "time_unit", "run_type", "repetition_index"} + for benchmark in benchmark_values["benchmarks"]: + name = benchmark["name"].replace('/', '_') # ci does not work properly with '/' in metric name + time_unit_mult = time_unit_multipliers[benchmark.get("time_unit", "ns")] + for k, v in six.iteritems(benchmark): + if k in time_keys: + cls.set("{}_{}".format(name, k), v * time_unit_mult) + elif k not in ignore_keys and isinstance(v, (float, int)): + cls.set("{}_{}".format(name, k), v) + return Metrics + + +@pytest.fixture +def links(request): + + class Links(object): + @classmethod + def set(cls, name, path): + + if len(request.config.test_logs[request.node.nodeid]) >= MAX_ALLOWED_LINKS_COUNT: + raise Exception("Cannot add more than {} links to test".format(MAX_ALLOWED_LINKS_COUNT)) + + reserved_names = ["log", "logsdir", "stdout", "stderr"] + if name in reserved_names: + raise Exception("Attachment name should not belong to the reserved list: {}".format(", ".join(reserved_names))) + output_dir = request.config.ya.output_dir + + if not os.path.exists(path): + raise Exception("Path to be attached does not exist: {}".format(path)) + + if os.path.isabs(path) and ".." in os.path.relpath(path, output_dir): + raise Exception("Test attachment must be inside yatest.common.output_path()") + + request.config.test_logs[request.node.nodeid][name] = path + + @classmethod + def get(cls, name): + if name not in request.config.test_logs[request.node.nodeid]: + raise KeyError("Attachment with name '{}' does not exist".format(name)) + return request.config.test_logs[request.node.nodeid][name] + + return Links diff --git a/library/python/pytest/plugins/ya.make b/library/python/pytest/plugins/ya.make new file mode 100644 index 0000000000..c15d6f759d --- /dev/null +++ b/library/python/pytest/plugins/ya.make @@ -0,0 +1,32 @@ +OWNER(g:yatest) + +PY23_LIBRARY() + +PY_SRCS( + ya.py + collection.py + conftests.py + fixtures.py +) + +PEERDIR( + library/python/filelock + library/python/find_root + library/python/testing/filter +) + +IF (PYTHON2) + PY_SRCS( + fakeid_py2.py + ) + + PEERDIR( + contrib/python/faulthandler + ) +ELSE() + PY_SRCS( + fakeid_py3.py + ) +ENDIF() + +END() diff --git a/library/python/pytest/plugins/ya.py b/library/python/pytest/plugins/ya.py new file mode 100644 index 0000000000..1bde03042d --- /dev/null +++ b/library/python/pytest/plugins/ya.py @@ -0,0 +1,963 @@ +# coding: utf-8 + +import base64 +import errno +import re +import sys +import os +import logging +import fnmatch +import json +import time +import traceback +import collections +import signal +import inspect +import warnings + +import attr +import faulthandler +import py +import pytest +import six + +import _pytest +import _pytest._io +import _pytest.mark +import _pytest.outcomes +import _pytest.skipping + +from _pytest.warning_types import PytestUnhandledCoroutineWarning + +from yatest_lib import test_splitter + +try: + import resource +except ImportError: + resource = None + +try: + import library.python.pytest.yatest_tools as tools +except ImportError: + # fallback for pytest script mode + import yatest_tools as tools + +try: + from library.python import filelock +except ImportError: + filelock = None + + +import yatest_lib.tools + +import yatest_lib.external as canon + +import yatest_lib.ya + +from library.python.pytest import context + +console_logger = logging.getLogger("console") +yatest_logger = logging.getLogger("ya.test") + + +_pytest.main.EXIT_NOTESTSCOLLECTED = 0 +SHUTDOWN_REQUESTED = False + +pytest_config = None + + +def configure_pdb_on_demand(): + import signal + + if hasattr(signal, "SIGUSR1"): + def on_signal(*args): + import ipdb + ipdb.set_trace() + + signal.signal(signal.SIGUSR1, on_signal) + + +class CustomImporter(object): + def __init__(self, roots): + self._roots = roots + + def find_module(self, fullname, package_path=None): + for path in self._roots: + full_path = self._get_module_path(path, fullname) + + if os.path.exists(full_path) and os.path.isdir(full_path) and not os.path.exists(os.path.join(full_path, "__init__.py")): + open(os.path.join(full_path, "__init__.py"), "w").close() + + return None + + def _get_module_path(self, path, fullname): + return os.path.join(path, *fullname.split('.')) + + +class YaTestLoggingFileHandler(logging.FileHandler): + pass + + +class _TokenFilterFormatter(logging.Formatter): + def __init__(self, fmt): + super(_TokenFilterFormatter, self).__init__(fmt) + self._replacements = [] + if not self._replacements: + if six.PY2: + for k, v in os.environ.iteritems(): + if k.endswith('TOKEN') and v: + self._replacements.append(v) + elif six.PY3: + for k, v in os.environ.items(): + if k.endswith('TOKEN') and v: + self._replacements.append(v) + self._replacements = sorted(self._replacements) + + def _filter(self, s): + for r in self._replacements: + s = s.replace(r, "[SECRET]") + + return s + + def format(self, record): + return self._filter(super(_TokenFilterFormatter, self).format(record)) + + +def setup_logging(log_path, level=logging.DEBUG, *other_logs): + logs = [log_path] + list(other_logs) + root_logger = logging.getLogger() + for i in range(len(root_logger.handlers) - 1, -1, -1): + if isinstance(root_logger.handlers[i], YaTestLoggingFileHandler): + root_logger.handlers.pop(i).close() + root_logger.setLevel(level) + for log_file in logs: + file_handler = YaTestLoggingFileHandler(log_file) + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' + file_handler.setFormatter(_TokenFilterFormatter(log_format)) + file_handler.setLevel(level) + root_logger.addHandler(file_handler) + + +def pytest_addoption(parser): + parser.addoption("--build-root", action="store", dest="build_root", default="", help="path to the build root") + parser.addoption("--dep-root", action="append", dest="dep_roots", default=[], help="path to the dep build roots") + parser.addoption("--source-root", action="store", dest="source_root", default="", help="path to the source root") + parser.addoption("--data-root", action="store", dest="data_root", default="", help="path to the arcadia_tests_data root") + parser.addoption("--output-dir", action="store", dest="output_dir", default="", help="path to the test output dir") + parser.addoption("--python-path", action="store", dest="python_path", default="", help="path the canonical python binary") + parser.addoption("--valgrind-path", action="store", dest="valgrind_path", default="", help="path the canonical valgring binary") + parser.addoption("--test-filter", action="append", dest="test_filter", default=None, help="test filter") + parser.addoption("--test-file-filter", action="store", dest="test_file_filter", default=None, help="test file filter") + parser.addoption("--test-param", action="append", dest="test_params", default=None, help="test parameters") + parser.addoption("--test-log-level", action="store", dest="test_log_level", choices=["critical", "error", "warning", "info", "debug"], default="debug", help="test log level") + parser.addoption("--mode", action="store", choices=[yatest_lib.ya.RunMode.List, yatest_lib.ya.RunMode.Run], dest="mode", default=yatest_lib.ya.RunMode.Run, help="testing mode") + parser.addoption("--test-list-file", action="store", dest="test_list_file") + parser.addoption("--modulo", default=1, type=int) + parser.addoption("--modulo-index", default=0, type=int) + parser.addoption("--partition-mode", default='SEQUENTIAL', help="Split tests according to partitoin mode") + parser.addoption("--split-by-tests", action='store_true', help="Split test execution by tests instead of suites", default=False) + parser.addoption("--project-path", action="store", default="", help="path to CMakeList where test is declared") + parser.addoption("--build-type", action="store", default="", help="build type") + parser.addoption("--flags", action="append", dest="flags", default=[], help="build flags (-D)") + parser.addoption("--sanitize", action="store", default="", help="sanitize mode") + parser.addoption("--test-stderr", action="store_true", default=False, help="test stderr") + parser.addoption("--test-debug", action="store_true", default=False, help="test debug mode") + parser.addoption("--root-dir", action="store", default=None) + parser.addoption("--ya-trace", action="store", dest="ya_trace_path", default=None, help="path to ya trace report") + parser.addoption("--ya-version", action="store", dest="ya_version", default=0, type=int, help="allows to be compatible with ya and the new changes in ya-dev") + parser.addoption( + "--test-suffix", action="store", dest="test_suffix", default=None, help="add suffix to every test name" + ) + parser.addoption("--gdb-path", action="store", dest="gdb_path", default="", help="path the canonical gdb binary") + parser.addoption("--collect-cores", action="store_true", dest="collect_cores", default=False, help="allows core dump file recovering during test") + parser.addoption("--sanitizer-extra-checks", action="store_true", dest="sanitizer_extra_checks", default=False, help="enables extra checks for tests built with sanitizers") + parser.addoption("--report-deselected", action="store_true", dest="report_deselected", default=False, help="report deselected tests to the trace file") + parser.addoption("--pdb-on-sigusr1", action="store_true", default=False, help="setup pdb.set_trace on SIGUSR1") + parser.addoption("--test-tool-bin", help="Path to test_tool") + parser.addoption("--test-list-path", dest="test_list_path", action="store", help="path to test list", default="") + + +def from_ya_test(): + return "YA_TEST_RUNNER" in os.environ + + +def pytest_configure(config): + global pytest_config + pytest_config = config + + config.option.continue_on_collection_errors = True + + config.addinivalue_line("markers", "ya:external") + + config.from_ya_test = from_ya_test() + config.test_logs = collections.defaultdict(dict) + config.test_metrics = {} + config.suite_metrics = {} + config.configure_timestamp = time.time() + context = { + "project_path": config.option.project_path, + "test_stderr": config.option.test_stderr, + "test_debug": config.option.test_debug, + "build_type": config.option.build_type, + "test_traceback": config.option.tbstyle, + "flags": config.option.flags, + "sanitize": config.option.sanitize, + } + + if config.option.collectonly: + config.option.mode = yatest_lib.ya.RunMode.List + + config.ya = yatest_lib.ya.Ya( + config.option.mode, + config.option.source_root, + config.option.build_root, + config.option.dep_roots, + config.option.output_dir, + config.option.test_params, + context, + config.option.python_path, + config.option.valgrind_path, + config.option.gdb_path, + config.option.data_root, + ) + config.option.test_log_level = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warning": logging.WARN, + "info": logging.INFO, + "debug": logging.DEBUG, + }[config.option.test_log_level] + + if not config.option.collectonly: + setup_logging(os.path.join(config.ya.output_dir, "run.log"), config.option.test_log_level) + config.current_item_nodeid = None + config.current_test_name = None + config.test_cores_count = 0 + config.collect_cores = config.option.collect_cores + config.sanitizer_extra_checks = config.option.sanitizer_extra_checks + try: + config.test_tool_bin = config.option.test_tool_bin + except AttributeError: + logging.info("test_tool_bin not specified") + + if config.sanitizer_extra_checks: + for envvar in ['LSAN_OPTIONS', 'ASAN_OPTIONS']: + if envvar in os.environ: + os.environ.pop(envvar) + if envvar + '_ORIGINAL' in os.environ: + os.environ[envvar] = os.environ[envvar + '_ORIGINAL'] + + if config.option.root_dir: + config.rootdir = py.path.local(config.option.root_dir) + config.invocation_params = attr.evolve(config.invocation_params, dir=config.rootdir) + + extra_sys_path = [] + # Arcadia paths from the test DEPENDS section of ya.make + extra_sys_path.append(os.path.join(config.option.source_root, config.option.project_path)) + # Build root is required for correct import of protobufs, because imports are related to the root + # (like import devtools.dummy_arcadia.protos.lib.my_proto_pb2) + extra_sys_path.append(config.option.build_root) + + for path in config.option.dep_roots: + if os.path.isabs(path): + extra_sys_path.append(path) + else: + extra_sys_path.append(os.path.join(config.option.source_root, path)) + + sys_path_set = set(sys.path) + for path in extra_sys_path: + if path not in sys_path_set: + sys.path.append(path) + sys_path_set.add(path) + + os.environ["PYTHONPATH"] = os.pathsep.join(sys.path) + + if not config.option.collectonly: + if config.option.ya_trace_path: + config.ya_trace_reporter = TraceReportGenerator(config.option.ya_trace_path) + else: + config.ya_trace_reporter = DryTraceReportGenerator(config.option.ya_trace_path) + config.ya_version = config.option.ya_version + + sys.meta_path.append(CustomImporter([config.option.build_root] + [os.path.join(config.option.build_root, dep) for dep in config.option.dep_roots])) + if config.option.pdb_on_sigusr1: + configure_pdb_on_demand() + + # Dump python backtrace in case of any errors + faulthandler.enable() + if hasattr(signal, "SIGQUIT"): + # SIGQUIT is used by test_tool to teardown tests which overruns timeout + faulthandler.register(signal.SIGQUIT, chain=True) + + if hasattr(signal, "SIGUSR2"): + signal.signal(signal.SIGUSR2, _graceful_shutdown) + + +session_should_exit = False + + +def _graceful_shutdown_on_log(should_exit): + if should_exit: + pytest.exit("Graceful shutdown requested") + + +def pytest_runtest_logreport(report): + _graceful_shutdown_on_log(session_should_exit) + + +def pytest_runtest_logstart(nodeid, location): + _graceful_shutdown_on_log(session_should_exit) + + +def pytest_runtest_logfinish(nodeid, location): + _graceful_shutdown_on_log(session_should_exit) + + +def _graceful_shutdown(*args): + global session_should_exit + session_should_exit = True + try: + import library.python.coverage + library.python.coverage.stop_coverage_tracing() + except ImportError: + pass + traceback.print_stack(file=sys.stderr) + capman = pytest_config.pluginmanager.getplugin("capturemanager") + capman.suspend(in_=True) + _graceful_shutdown_on_log(not capman.is_globally_capturing()) + + +def _get_rusage(): + return resource and resource.getrusage(resource.RUSAGE_SELF) + + +def _collect_test_rusage(item): + if resource and hasattr(item, "rusage"): + finish_rusage = _get_rusage() + ya_inst = pytest_config.ya + + def add_metric(attr_name, metric_name=None, modifier=None): + if not metric_name: + metric_name = attr_name + if not modifier: + modifier = lambda x: x + if hasattr(item.rusage, attr_name): + ya_inst.set_metric_value(metric_name, modifier(getattr(finish_rusage, attr_name) - getattr(item.rusage, attr_name))) + + for args in [ + ("ru_maxrss", "ru_rss", lambda x: x*1024), # to be the same as in util/system/rusage.cpp + ("ru_utime",), + ("ru_stime",), + ("ru_ixrss", None, lambda x: x*1024), + ("ru_idrss", None, lambda x: x*1024), + ("ru_isrss", None, lambda x: x*1024), + ("ru_majflt", "ru_major_pagefaults"), + ("ru_minflt", "ru_minor_pagefaults"), + ("ru_nswap",), + ("ru_inblock",), + ("ru_oublock",), + ("ru_msgsnd",), + ("ru_msgrcv",), + ("ru_nsignals",), + ("ru_nvcsw",), + ("ru_nivcsw",), + ]: + add_metric(*args) + + +def _get_item_tags(item): + tags = [] + for key, value in item.keywords.items(): + if key == 'pytestmark' and isinstance(value, list): + for mark in value: + tags.append(mark.name) + elif isinstance(value, _pytest.mark.MarkDecorator): + tags.append(key) + return tags + + +def pytest_runtest_setup(item): + item.rusage = _get_rusage() + pytest_config.test_cores_count = 0 + pytest_config.current_item_nodeid = item.nodeid + class_name, test_name = tools.split_node_id(item.nodeid) + test_log_path = tools.get_test_log_file_path(pytest_config.ya.output_dir, class_name, test_name) + setup_logging( + os.path.join(pytest_config.ya.output_dir, "run.log"), + pytest_config.option.test_log_level, + test_log_path + ) + pytest_config.test_logs[item.nodeid]['log'] = test_log_path + pytest_config.test_logs[item.nodeid]['logsdir'] = pytest_config.ya.output_dir + pytest_config.current_test_log_path = test_log_path + pytest_config.current_test_name = "{}::{}".format(class_name, test_name) + separator = "#" * 100 + yatest_logger.info(separator) + yatest_logger.info(test_name) + yatest_logger.info(separator) + yatest_logger.info("Test setup") + + test_item = CrashedTestItem(item.nodeid, pytest_config.option.test_suffix) + pytest_config.ya_trace_reporter.on_start_test_class(test_item) + pytest_config.ya_trace_reporter.on_start_test_case(test_item) + + +def pytest_runtest_teardown(item, nextitem): + yatest_logger.info("Test teardown") + + +def pytest_runtest_call(item): + class_name, test_name = tools.split_node_id(item.nodeid) + yatest_logger.info("Test call (class_name: %s, test_name: %s)", class_name, test_name) + + +def pytest_deselected(items): + config = pytest_config + if config.option.report_deselected: + for item in items: + deselected_item = DeselectedTestItem(item.nodeid, config.option.test_suffix) + config.ya_trace_reporter.on_start_test_class(deselected_item) + config.ya_trace_reporter.on_finish_test_case(deselected_item) + config.ya_trace_reporter.on_finish_test_class(deselected_item) + + +@pytest.mark.trylast +def pytest_collection_modifyitems(items, config): + + def filter_items(filters): + filtered_items = [] + deselected_items = [] + for item in items: + canonical_node_id = str(CustomTestItem(item.nodeid, pytest_config.option.test_suffix)) + matched = False + for flt in filters: + if "::" not in flt and "*" not in flt: + flt += "*" # add support for filtering by module name + if canonical_node_id.endswith(flt) or fnmatch.fnmatch(tools.escape_for_fnmatch(canonical_node_id), tools.escape_for_fnmatch(flt)): + matched = True + if matched: + filtered_items.append(item) + else: + deselected_items.append(item) + + config.hook.pytest_deselected(items=deselected_items) + items[:] = filtered_items + + def filter_by_full_name(filters): + filter_set = {flt for flt in filters} + filtered_items = [] + deselected_items = [] + for item in items: + if item.nodeid in filter_set: + filtered_items.append(item) + else: + deselected_items.append(item) + + config.hook.pytest_deselected(items=deselected_items) + items[:] = filtered_items + + # XXX - check to be removed when tests for peerdirs don't run + for item in items: + if not item.nodeid: + item._nodeid = os.path.basename(item.location[0]) + if os.path.exists(config.option.test_list_path): + with open(config.option.test_list_path, 'r') as afile: + chunks = json.load(afile) + filters = chunks[config.option.modulo_index] + filter_by_full_name(filters) + else: + if config.option.test_filter: + filter_items(config.option.test_filter) + partition_mode = config.option.partition_mode + modulo = config.option.modulo + if modulo > 1: + items[:] = sorted(items, key=lambda item: item.nodeid) + modulo_index = config.option.modulo_index + split_by_tests = config.option.split_by_tests + items_by_classes = {} + res = [] + for item in items: + if item.nodeid.count("::") == 2 and not split_by_tests: + class_name = item.nodeid.rsplit("::", 1)[0] + if class_name not in items_by_classes: + items_by_classes[class_name] = [] + res.append(items_by_classes[class_name]) + items_by_classes[class_name].append(item) + else: + res.append([item]) + chunk_items = test_splitter.get_splitted_tests(res, modulo, modulo_index, partition_mode, is_sorted=True) + items[:] = [] + for item in chunk_items: + items.extend(item) + yatest_logger.info("Modulo %s tests are: %s", modulo_index, chunk_items) + + if config.option.mode == yatest_lib.ya.RunMode.Run: + for item in items: + test_item = NotLaunchedTestItem(item.nodeid, config.option.test_suffix) + config.ya_trace_reporter.on_start_test_class(test_item) + config.ya_trace_reporter.on_finish_test_case(test_item) + config.ya_trace_reporter.on_finish_test_class(test_item) + elif config.option.mode == yatest_lib.ya.RunMode.List: + tests = [] + for item in items: + item = CustomTestItem(item.nodeid, pytest_config.option.test_suffix, item.keywords) + record = { + "class": item.class_name, + "test": item.test_name, + "tags": _get_item_tags(item), + } + tests.append(record) + if config.option.test_list_file: + with open(config.option.test_list_file, 'w') as afile: + json.dump(tests, afile) + # TODO prettyboy remove after test_tool release - currently it's required for backward compatibility + sys.stderr.write(json.dumps(tests)) + + +def pytest_collectreport(report): + if not report.passed: + if hasattr(pytest_config, 'ya_trace_reporter'): + test_item = TestItem(report, None, pytest_config.option.test_suffix) + pytest_config.ya_trace_reporter.on_error(test_item) + else: + sys.stderr.write(yatest_lib.tools.to_utf8(report.longrepr)) + + +@pytest.mark.tryfirst +def pytest_pyfunc_call(pyfuncitem): + testfunction = pyfuncitem.obj + iscoroutinefunction = getattr(inspect, "iscoroutinefunction", None) + if iscoroutinefunction is not None and iscoroutinefunction(testfunction): + msg = "Coroutine functions are not natively supported and have been skipped.\n" + msg += "You need to install a suitable plugin for your async framework, for example:\n" + msg += " - pytest-asyncio\n" + msg += " - pytest-trio\n" + msg += " - pytest-tornasync" + warnings.warn(PytestUnhandledCoroutineWarning(msg.format(pyfuncitem.nodeid))) + _pytest.outcomes.skip(msg="coroutine function and no async plugin installed (see warnings)") + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + pyfuncitem.retval = testfunction(**testargs) + return True + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + def logreport(report, result, call): + test_item = TestItem(report, result, pytest_config.option.test_suffix) + if not pytest_config.suite_metrics and context.Ctx.get("YA_PYTEST_START_TIMESTAMP"): + pytest_config.suite_metrics["pytest_startup_duration"] = call.start - context.Ctx["YA_PYTEST_START_TIMESTAMP"] + pytest_config.ya_trace_reporter.dump_suite_metrics() + + pytest_config.ya_trace_reporter.on_log_report(test_item) + + if report.outcome == "failed": + yatest_logger.error(report.longrepr) + + if report.when == "call": + _collect_test_rusage(item) + pytest_config.ya_trace_reporter.on_finish_test_case(test_item) + elif report.when == "setup": + pytest_config.ya_trace_reporter.on_start_test_class(test_item) + if report.outcome != "passed": + pytest_config.ya_trace_reporter.on_start_test_case(test_item) + pytest_config.ya_trace_reporter.on_finish_test_case(test_item) + else: + pytest_config.ya_trace_reporter.on_start_test_case(test_item) + elif report.when == "teardown": + if report.outcome == "failed": + pytest_config.ya_trace_reporter.on_start_test_case(test_item) + pytest_config.ya_trace_reporter.on_finish_test_case(test_item) + else: + pytest_config.ya_trace_reporter.on_finish_test_case(test_item, duration_only=True) + pytest_config.ya_trace_reporter.on_finish_test_class(test_item) + + outcome = yield + rep = outcome.get_result() + result = None + if hasattr(item, 'retval') and item.retval is not None: + result = item.retval + if not pytest_config.from_ya_test: + ti = TestItem(rep, result, pytest_config.option.test_suffix) + tr = pytest_config.pluginmanager.getplugin('terminalreporter') + tr.write_line("{} - Validating canonical data is not supported when running standalone binary".format(ti), yellow=True, bold=True) + logreport(rep, result, call) + + +def pytest_make_parametrize_id(config, val, argname): + # Avoid <, > symbols in canondata file names + if inspect.isfunction(val) and val.__name__ == "<lambda>": + return str(argname) + return None + + +def get_formatted_error(report): + if isinstance(report.longrepr, tuple): + text = "" + for entry in report.longrepr: + text += colorize(entry) + else: + text = colorize(report.longrepr) + text = yatest_lib.tools.to_utf8(text) + return text + + +def colorize(longrepr): + # use default pytest colorization + if pytest_config.option.tbstyle != "short": + io = py.io.TextIO() + if six.PY2: + writer = py.io.TerminalWriter(file=io) + else: + writer = _pytest._io.TerminalWriter(file=io) + # enable colorization + writer.hasmarkup = True + + if hasattr(longrepr, 'reprtraceback') and hasattr(longrepr.reprtraceback, 'toterminal'): + longrepr.reprtraceback.toterminal(writer) + return io.getvalue().strip() + return yatest_lib.tools.to_utf8(longrepr) + + text = yatest_lib.tools.to_utf8(longrepr) + pos = text.find("E ") + if pos == -1: + return text + + bt, error = text[:pos], text[pos:] + filters = [ + # File path, line number and function name + (re.compile(r"^(.*?):(\d+): in (\S+)", flags=re.MULTILINE), r"[[unimp]]\1[[rst]]:[[alt2]]\2[[rst]]: in [[alt1]]\3[[rst]]"), + ] + for regex, substitution in filters: + bt = regex.sub(substitution, bt) + return "{}[[bad]]{}".format(bt, error) + + +class TestItem(object): + + def __init__(self, report, result, test_suffix): + self._result = result + self.nodeid = report.nodeid + self._class_name, self._test_name = tools.split_node_id(self.nodeid, test_suffix) + self._error = None + self._status = None + self._process_report(report) + self._duration = hasattr(report, 'duration') and report.duration or 0 + self._keywords = getattr(report, "keywords", {}) + + def _process_report(self, report): + if report.longrepr: + self.set_error(report) + if hasattr(report, 'when') and report.when != "call": + self.set_error(report.when + " failed:\n" + self._error) + else: + self.set_error("") + + report_teststatus = _pytest.skipping.pytest_report_teststatus(report) + if report_teststatus is not None: + report_teststatus = report_teststatus[0] + + if report_teststatus == 'xfailed': + self._status = 'xfail' + self.set_error(report.wasxfail, 'imp') + elif report_teststatus == 'xpassed': + self._status = 'xpass' + self.set_error("Test unexpectedly passed") + elif report.skipped: + self._status = 'skipped' + self.set_error(yatest_lib.tools.to_utf8(report.longrepr[-1])) + elif report.passed: + self._status = 'good' + self.set_error("") + else: + self._status = 'fail' + + @property + def status(self): + return self._status + + def set_status(self, status): + self._status = status + + @property + def test_name(self): + return tools.normalize_name(self._test_name) + + @property + def class_name(self): + return tools.normalize_name(self._class_name) + + @property + def error(self): + return self._error + + def set_error(self, entry, marker='bad'): + if isinstance(entry, _pytest.reports.BaseReport): + self._error = get_formatted_error(entry) + else: + self._error = "[[{}]]{}".format(yatest_lib.tools.to_str(marker), yatest_lib.tools.to_str(entry)) + + @property + def duration(self): + return self._duration + + @property + def result(self): + if 'not_canonize' in self._keywords: + return None + return self._result + + @property + def keywords(self): + return self._keywords + + def __str__(self): + return "{}::{}".format(self.class_name, self.test_name) + + +class CustomTestItem(TestItem): + + def __init__(self, nodeid, test_suffix, keywords=None): + self._result = None + self.nodeid = nodeid + self._class_name, self._test_name = tools.split_node_id(nodeid, test_suffix) + self._duration = 0 + self._error = "" + self._keywords = keywords if keywords is not None else {} + + +class NotLaunchedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(NotLaunchedTestItem, self).__init__(nodeid, test_suffix) + self._status = "not_launched" + + +class CrashedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(CrashedTestItem, self).__init__(nodeid, test_suffix) + self._status = "crashed" + + +class DeselectedTestItem(CustomTestItem): + + def __init__(self, nodeid, test_suffix): + super(DeselectedTestItem, self).__init__(nodeid, test_suffix) + self._status = "deselected" + + +class TraceReportGenerator(object): + + def __init__(self, out_file_path): + self._filename = out_file_path + self._file = open(out_file_path, 'w') + self._wreckage_filename = out_file_path + '.wreckage' + self._test_messages = {} + self._test_duration = {} + # Some machinery to avoid data corruption due sloppy fork() + self._current_test = (None, None) + self._pid = os.getpid() + self._check_intricate_respawn() + + def _check_intricate_respawn(self): + pid_file = self._filename + '.pid' + try: + # python2 doesn't support open(f, 'x') + afile = os.fdopen(os.open(pid_file, os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'w') + afile.write(str(self._pid)) + afile.close() + return + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Looks like the test binary was respawned + if from_ya_test(): + try: + with open(pid_file) as afile: + prev_pid = afile.read() + except Exception as e: + prev_pid = '(failed to obtain previous pid: {})'.format(e) + + parts = [ + "Aborting test run: test machinery found that the test binary {} has already been run before.".format(sys.executable), + "Looks like test has incorrect respawn/relaunch logic within test binary.", + "Test should not try to restart itself - this is a poorly designed test case that leads to errors and could corrupt internal test machinery files.", + "Debug info: previous pid:{} current:{}".format(prev_pid, self._pid), + ] + msg = '\n'.join(parts) + yatest_logger.error(msg) + + if filelock: + lock = filelock.FileLock(self._wreckage_filename + '.lock') + lock.acquire() + + with open(self._wreckage_filename, 'a') as afile: + self._file = afile + + self._dump_trace('chunk_event', {"errors": [('fail', '[[bad]]' + msg)]}) + + raise Exception(msg) + else: + # Test binary is launched without `ya make -t`'s testing machinery - don't rely on clean environment + pass + + def on_start_test_class(self, test_item): + pytest_config.ya.set_test_item_node_id(test_item.nodeid) + class_name = test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name + self._current_test = (class_name, None) + self.trace('test-started', {'class': class_name}) + + def on_finish_test_class(self, test_item): + pytest_config.ya.set_test_item_node_id(test_item.nodeid) + self.trace('test-finished', {'class': test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name}) + + def on_start_test_case(self, test_item): + class_name = yatest_lib.tools.to_utf8(test_item.class_name) + subtest_name = yatest_lib.tools.to_utf8(test_item.test_name) + message = { + 'class': class_name, + 'subtest': subtest_name, + } + if test_item.nodeid in pytest_config.test_logs: + message['logs'] = pytest_config.test_logs[test_item.nodeid] + pytest_config.ya.set_test_item_node_id(test_item.nodeid) + self._current_test = (class_name, subtest_name) + self.trace('subtest-started', message) + + def on_finish_test_case(self, test_item, duration_only=False): + if test_item.result is not None: + try: + result = canon.serialize(test_item.result) + except Exception as e: + yatest_logger.exception("Error while serializing test results") + test_item.set_error("Invalid test result: {}".format(e)) + test_item.set_status("fail") + result = None + else: + result = None + + if duration_only and test_item.nodeid in self._test_messages: # add teardown time + message = self._test_messages[test_item.nodeid] + else: + comment = self._test_messages[test_item.nodeid]['comment'] if test_item.nodeid in self._test_messages else '' + comment += self._get_comment(test_item) + message = { + 'class': yatest_lib.tools.to_utf8(test_item.class_name), + 'subtest': yatest_lib.tools.to_utf8(test_item.test_name), + 'status': test_item.status, + 'comment': comment, + 'result': result, + 'metrics': pytest_config.test_metrics.get(test_item.nodeid), + 'is_diff_test': 'diff_test' in test_item.keywords, + 'tags': _get_item_tags(test_item), + } + if test_item.nodeid in pytest_config.test_logs: + message['logs'] = pytest_config.test_logs[test_item.nodeid] + + message['time'] = self._test_duration.get(test_item.nodeid, test_item.duration) + + self.trace('subtest-finished', message) + self._test_messages[test_item.nodeid] = message + + def dump_suite_metrics(self): + message = {"metrics": pytest_config.suite_metrics} + self.trace("suite-event", message) + + def on_error(self, test_item): + self.trace('chunk_event', {"errors": [(test_item.status, self._get_comment(test_item))]}) + + def on_log_report(self, test_item): + if test_item.nodeid in self._test_duration: + self._test_duration[test_item.nodeid] += test_item._duration + else: + self._test_duration[test_item.nodeid] = test_item._duration + + @staticmethod + def _get_comment(test_item): + msg = yatest_lib.tools.to_utf8(test_item.error) + if not msg: + return "" + return msg + "[[rst]]" + + def _dump_trace(self, name, value): + event = { + 'timestamp': time.time(), + 'value': value, + 'name': name + } + + data = yatest_lib.tools.to_str(json.dumps(event, ensure_ascii=False)) + self._file.write(data + '\n') + self._file.flush() + + def _check_sloppy_fork(self, name, value): + if self._pid == os.getpid(): + return + + yatest_logger.error("Skip tracing to avoid data corruption, name = %s, value = %s", name, value) + + try: + # Lock wreckage tracefile to avoid race if multiple tests use fork sloppily + if filelock: + lock = filelock.FileLock(self._wreckage_filename + '.lock') + lock.acquire() + + with open(self._wreckage_filename, 'a') as afile: + self._file = afile + + parts = [ + "It looks like you have leaked process - it could corrupt internal test machinery files.", + "Usually it happens when you casually use fork() without os._exit(),", + "which results in two pytest processes running at the same time.", + "Pid of the original pytest's process is {}, however current process has {} pid.".format(self._pid, os.getpid()), + ] + if self._current_test[1]: + parts.append("Most likely the problem is in '{}' test.".format(self._current_test)) + else: + parts.append("Most likely new process was created before any test was launched (during the import stage?).") + + if value.get('comment'): + comment = value.get('comment', '').strip() + # multiline comment + newline_required = '\n' if '\n' in comment else '' + parts.append("Debug info: name = '{}' comment:{}{}".format(name, newline_required, comment)) + else: + val_str = json.dumps(value, ensure_ascii=False).encode('utf-8') + parts.append("Debug info: name = '{}' value = '{}'".format(name, base64.b64encode(val_str))) + + msg = "[[bad]]{}".format('\n'.join(parts)) + class_name, subtest_name = self._current_test + if subtest_name: + data = { + 'class': class_name, + 'subtest': subtest_name, + 'status': 'fail', + 'comment': msg, + } + # overwrite original status + self._dump_trace('subtest-finished', data) + else: + self._dump_trace('chunk_event', {"errors": [('fail', msg)]}) + except Exception as e: + yatest_logger.exception(e) + finally: + os._exit(38) + + def trace(self, name, value): + self._check_sloppy_fork(name, value) + self._dump_trace(name, value) + + +class DryTraceReportGenerator(TraceReportGenerator): + """ + Generator does not write any information. + """ + + def __init__(self, *args, **kwargs): + self._test_messages = {} + self._test_duration = {} + + def trace(self, name, value): + pass diff --git a/library/python/pytest/pytest.yatest.ini b/library/python/pytest/pytest.yatest.ini new file mode 100644 index 0000000000..70d6c98516 --- /dev/null +++ b/library/python/pytest/pytest.yatest.ini @@ -0,0 +1,7 @@ +[pytest] +pep8maxlinelength = 200 +norecursedirs = * +pep8ignore = E127 E123 E226 E24 +filterwarnings = + ignore::pytest.RemovedInPytest4Warning +addopts = -p no:warnings diff --git a/library/python/pytest/rewrite.py b/library/python/pytest/rewrite.py new file mode 100644 index 0000000000..ec188d847f --- /dev/null +++ b/library/python/pytest/rewrite.py @@ -0,0 +1,123 @@ +from __future__ import absolute_import +from __future__ import print_function + +import ast + +import py + +from _pytest.assertion import rewrite +try: + import importlib.util +except ImportError: + pass +from __res import importer +import sys +import six + + +def _get_state(config): + if hasattr(config, '_assertstate'): + return config._assertstate + return config._store[rewrite.assertstate_key] + + +class AssertionRewritingHook(rewrite.AssertionRewritingHook): + def __init__(self, *args, **kwargs): + self.modules = {} + super(AssertionRewritingHook, self).__init__(*args, **kwargs) + + def find_module(self, name, path=None): + co = self._find_module(name, path) + if co is not None: + return self + + def _find_module(self, name, path=None): + state = _get_state(self.config) + if not self._should_rewrite(name, None, state): + return None + state.trace("find_module called for: %s" % name) + + try: + if self.is_package(name): + return None + except ImportError: + return None + + self._rewritten_names.add(name) + + state.trace("rewriting %s" % name) + co = _rewrite_test(self.config, name) + if co is None: + # Probably a SyntaxError in the test. + return None + self.modules[name] = co, None + return co + + def find_spec(self, name, path=None, target=None): + co = self._find_module(name, path) + if co is not None: + return importlib.util.spec_from_file_location( + name, + co.co_filename, + loader=self, + ) + + def _should_rewrite(self, name, fn, state): + if name.startswith("__tests__.") or name.endswith(".conftest"): + return True + + return self._is_marked_for_rewrite(name, state) + + def is_package(self, name): + return importer.is_package(name) + + def get_source(self, name): + return importer.get_source(name) + + if six.PY3: + def load_module(self, module): + co, _ = self.modules.pop(module.__name__) + try: + module.__file__ = co.co_filename + module.__cached__ = None + module.__loader__ = self + module.__spec__ = importlib.util.spec_from_file_location(module.__name__, co.co_filename, loader=self) + exec(co, module.__dict__) + except: # noqa + if module.__name__ in sys.modules: + del sys.modules[module.__name__] + raise + return sys.modules[module.__name__] + + def exec_module(self, module): + if module.__name__ in self.modules: + self.load_module(module) + else: + super(AssertionRewritingHook, self).exec_module(module) + + +def _rewrite_test(config, name): + """Try to read and rewrite *fn* and return the code object.""" + state = _get_state(config) + + source = importer.get_source(name) + if source is None: + return None + + path = importer.get_filename(name) + + try: + tree = ast.parse(source, filename=path) + except SyntaxError: + # Let this pop up again in the real import. + state.trace("failed to parse: %r" % (path,)) + return None + rewrite.rewrite_asserts(tree, py.path.local(path), config) + try: + co = compile(tree, path, "exec", dont_inherit=True) + except SyntaxError: + # It's possible that this error is from some bug in the + # assertion rewriting, but I don't know of a fast way to tell. + state.trace("failed to compile: %r" % (path,)) + return None + return co diff --git a/library/python/pytest/ya.make b/library/python/pytest/ya.make new file mode 100644 index 0000000000..060c92c313 --- /dev/null +++ b/library/python/pytest/ya.make @@ -0,0 +1,32 @@ +PY23_LIBRARY() + +OWNER( + g:yatool + dmitko +) + +PY_SRCS( + __init__.py + main.py + rewrite.py + yatest_tools.py + context.py +) + +PEERDIR( + contrib/python/dateutil + contrib/python/ipdb + contrib/python/py + contrib/python/pytest + contrib/python/requests + library/python/pytest/plugins + library/python/testing/yatest_common + library/python/testing/yatest_lib +) + +RESOURCE_FILES( + PREFIX library/python/pytest/ + pytest.yatest.ini +) + +END() diff --git a/library/python/pytest/yatest_tools.py b/library/python/pytest/yatest_tools.py new file mode 100644 index 0000000000..6b8b896394 --- /dev/null +++ b/library/python/pytest/yatest_tools.py @@ -0,0 +1,304 @@ +# coding: utf-8 + +import collections +import functools +import math +import os +import re +import sys + +import yatest_lib.tools + + +class Subtest(object): + def __init__(self, name, test_name, status, comment, elapsed, result=None, test_type=None, logs=None, cwd=None, metrics=None): + self._name = name + self._test_name = test_name + self.status = status + self.elapsed = elapsed + self.comment = comment + self.result = result + self.test_type = test_type + self.logs = logs or {} + self.cwd = cwd + self.metrics = metrics + + def __eq__(self, other): + if not isinstance(other, Subtest): + return False + return self.name == other.name and self.test_name == other.test_name + + def __str__(self): + return yatest_lib.tools.to_utf8(unicode(self)) + + def __unicode__(self): + return u"{}::{}".format(self.test_name, self.test_name) + + @property + def name(self): + return yatest_lib.tools.to_utf8(self._name) + + @property + def test_name(self): + return yatest_lib.tools.to_utf8(self._test_name) + + def __repr__(self): + return "Subtest [{}::{} - {}[{}]: {}]".format(self.name, self.test_name, self.status, self.elapsed, self.comment) + + def __hash__(self): + return hash(str(self)) + + +class SubtestInfo(object): + + skipped_prefix = '[SKIPPED] ' + + @classmethod + def from_str(cls, s): + if s.startswith(SubtestInfo.skipped_prefix): + s = s[len(SubtestInfo.skipped_prefix):] + skipped = True + + else: + skipped = False + + return SubtestInfo(*s.rsplit(TEST_SUBTEST_SEPARATOR, 1), skipped=skipped) + + def __init__(self, test, subtest="", skipped=False, **kwargs): + self.test = test + self.subtest = subtest + self.skipped = skipped + for key, value in kwargs.iteritems(): + setattr(self, key, value) + + def __str__(self): + s = '' + + if self.skipped: + s += SubtestInfo.skipped_prefix + + return s + TEST_SUBTEST_SEPARATOR.join([self.test, self.subtest]) + + def __repr__(self): + return str(self) + + +class Status(object): + GOOD, XFAIL, FAIL, XPASS, MISSING, CRASHED, TIMEOUT = range(7) + SKIPPED = -100 + NOT_LAUNCHED = -200 + CANON_DIFF = -300 + FLAKY = -1 + BY_NAME = {'good': GOOD, 'fail': FAIL, 'xfail': XFAIL, 'xpass': XPASS, 'missing': MISSING, 'crashed': CRASHED, + 'skipped': SKIPPED, 'flaky': FLAKY, 'not_launched': NOT_LAUNCHED, 'timeout': TIMEOUT, 'diff': CANON_DIFF} + TO_STR = {GOOD: 'good', FAIL: 'fail', XFAIL: 'xfail', XPASS: 'xpass', MISSING: 'missing', CRASHED: 'crashed', + SKIPPED: 'skipped', FLAKY: 'flaky', NOT_LAUNCHED: 'not_launched', TIMEOUT: 'timeout', CANON_DIFF: 'diff'} + + +class Test(object): + def __init__(self, name, path, status=None, comment=None, subtests=None): + self.name = name + self.path = path + self.status = status + self.comment = comment + self.subtests = subtests or [] + + def __eq__(self, other): + if not isinstance(other, Test): + return False + return self.name == other.name and self.path == other.path + + def __str__(self): + return "Test [{} {}] - {} - {}".format(self.name, self.path, self.status, self.comment) + + def __repr__(self): + return str(self) + + def add_subtest(self, subtest): + self.subtests.append(subtest) + + def setup_status(self, status, comment): + self.status = Status.BY_NAME[status or 'good'] + if len(self.subtests) != 0: + self.status = max(self.status, max(s.status for s in self.subtests)) + self.comment = comment + + def subtests_by_status(self, status): + return [x.status for x in self.subtests].count(status) + + +class NoMd5FileException(Exception): + pass + + +TEST_SUBTEST_SEPARATOR = '::' + + +# TODO: extract color theme logic from ya +COLOR_THEME = { + 'test_name': 'light-blue', + 'test_project_path': 'dark-blue', + 'test_dir_desc': 'dark-magenta', + 'test_binary_path': 'light-gray', +} + + +# XXX: remove me +class YaCtx(object): + pass + +ya_ctx = YaCtx() + +TRACE_FILE_NAME = "ytest.report.trace" + + +def lazy(func): + mem = {} + + @functools.wraps(func) + def wrapper(): + if "results" not in mem: + mem["results"] = func() + return mem["results"] + + return wrapper + + +@lazy +def _get_mtab(): + if os.path.exists("/etc/mtab"): + with open("/etc/mtab") as afile: + data = afile.read() + return [line.split(" ") for line in data.split("\n") if line] + return [] + + +def get_max_filename_length(dirname): + """ + Return maximum filename length for the filesystem + :return: + """ + if sys.platform.startswith("linux"): + # Linux user's may work on mounted ecryptfs filesystem + # which has filename length limitations + for entry in _get_mtab(): + mounted_dir, filesystem = entry[1], entry[2] + # http://unix.stackexchange.com/questions/32795/what-is-the-maximum-allowed-filename-and-folder-size-with-ecryptfs + if filesystem == "ecryptfs" and dirname and dirname.startswith(mounted_dir): + return 140 + # default maximum filename length for most filesystems + return 255 + + +def get_unique_file_path(dir_path, filename, cache=collections.defaultdict(set)): + """ + Get unique filename in dir with proper filename length, using given filename/dir. + File/dir won't be created (thread nonsafe) + :param dir_path: path to dir + :param filename: original filename + :return: unique filename + """ + max_suffix = 10000 + # + 1 symbol for dot before suffix + tail_length = int(round(math.log(max_suffix, 10))) + 1 + # truncate filename length in accordance with filesystem limitations + filename, extension = os.path.splitext(filename) + # XXX + if sys.platform.startswith("win"): + # Trying to fit into MAX_PATH if it's possible. + # Remove after DEVTOOLS-1646 + max_path = 260 + filename_len = len(dir_path) + len(extension) + tail_length + len(os.sep) + if filename_len < max_path: + filename = yatest_lib.tools.trim_string(filename, max_path - filename_len) + filename = yatest_lib.tools.trim_string(filename, get_max_filename_length(dir_path) - tail_length - len(extension)) + extension + candidate = os.path.join(dir_path, filename) + + key = dir_path + filename + counter = sorted(cache.get(key, {0, }))[-1] + while os.path.exists(candidate): + cache[key].add(counter) + counter += 1 + assert counter < max_suffix + candidate = os.path.join(dir_path, filename + ".{}".format(counter)) + return candidate + + +def escape_for_fnmatch(s): + return s.replace("[", "[").replace("]", "]") + + +def get_python_cmd(opts=None, use_huge=True, suite=None): + if opts and getattr(opts, 'flags', {}).get("USE_ARCADIA_PYTHON") == "no": + return ["python"] + if suite and not suite._use_arcadia_python: + return ["python"] + if use_huge: + return ["$(PYTHON)/python"] + ymake_path = opts.ymake_bin if opts and getattr(opts, 'ymake_bin', None) else "$(YMAKE)/ymake" + return [ymake_path, "--python"] + + +def normalize_name(name): + replacements = [ + ("\\", "\\\\"), + ("\n", "\\n"), + ("\t", "\\t"), + ("\r", "\\r"), + ] + for l, r in replacements: + name = name.replace(l, r) + return name + + +def normalize_filename(filename): + """ + Replace invalid for file names characters with string equivalents + :param some_string: string to be converted to a valid file name + :return: valid file name + """ + not_allowed_pattern = r"[\[\]\/:*?\"\'<>|+\0\\\s\x0b\x0c]" + filename = re.sub(not_allowed_pattern, ".", filename) + return re.sub(r"\.{2,}", ".", filename) + + +def get_test_log_file_path(output_dir, class_name, test_name, extension="log"): + """ + get test log file path, platform dependant + :param output_dir: dir where log file should be placed + :param class_name: test class name + :param test_name: test name + :return: test log file name + """ + if os.name == "nt": + # don't add class name to the log's filename + # to reduce it's length on windows + filename = test_name + else: + filename = "{}.{}".format(class_name, test_name) + if not filename: + filename = "test" + filename += "." + extension + filename = normalize_filename(filename) + return get_unique_file_path(output_dir, filename) + + +def split_node_id(nodeid, test_suffix=None): + path, possible_open_bracket, params = nodeid.partition('[') + separator = "::" + if separator in path: + path, test_name = path.split(separator, 1) + else: + test_name = os.path.basename(path) + if test_suffix: + test_name += "::" + test_suffix + class_name = os.path.basename(path.strip()) + if separator in test_name: + klass_name, test_name = test_name.split(separator, 1) + if not test_suffix: + # test suffix is used for flakes and pep8, no need to add class_name as it's === class_name + class_name += separator + klass_name + if separator in test_name: + test_name = test_name.split(separator)[-1] + test_name += possible_open_bracket + params + return yatest_lib.tools.to_utf8(class_name), yatest_lib.tools.to_utf8(test_name) diff --git a/library/python/reservoir_sampling/README.md b/library/python/reservoir_sampling/README.md new file mode 100644 index 0000000000..27674ba4f0 --- /dev/null +++ b/library/python/reservoir_sampling/README.md @@ -0,0 +1,11 @@ +### Overview +Reservoir sampling is a family of randomized algorithms for choosing a simple random sample, without replacement, of k items from a population of unknown size n in a single pass over the items. + +### Example + +```jupyter +In [1]: from library.python import reservoir_sampling + +In [2]: reservoir_sampling.reservoir_sampling(data=range(100), nsamples=10) +Out[2]: [27, 19, 81, 45, 89, 78, 13, 36, 29, 9] +``` diff --git a/library/python/reservoir_sampling/__init__.py b/library/python/reservoir_sampling/__init__.py new file mode 100644 index 0000000000..4ee46ee5e1 --- /dev/null +++ b/library/python/reservoir_sampling/__init__.py @@ -0,0 +1,16 @@ +import random + + +def reservoir_sampling(data, nsamples, prng=None): + if prng is None: + prng = random + + result = [] + for i, entry in enumerate(data): + if i < nsamples: + result.append(entry) + else: + j = prng.randint(0, i) + if j < nsamples: + result[j] = entry + return result diff --git a/library/python/reservoir_sampling/ya.make b/library/python/reservoir_sampling/ya.make new file mode 100644 index 0000000000..24cac20157 --- /dev/null +++ b/library/python/reservoir_sampling/ya.make @@ -0,0 +1,10 @@ +OWNER( + prettyboy + g:yatool +) + +PY23_LIBRARY() + +PY_SRCS(__init__.py) + +END() diff --git a/library/python/resource/__init__.py b/library/python/resource/__init__.py new file mode 100644 index 0000000000..26503ef7fc --- /dev/null +++ b/library/python/resource/__init__.py @@ -0,0 +1,49 @@ +from __res import find as __find, count, key_by_index, resfs_files as __resfs_files +from __res import resfs_read, resfs_resolve, resfs_src # noqa + +import six + + +def iterkeys(prefix='', strip_prefix=False): + decode = lambda s: s + if isinstance(prefix, six.text_type): + prefix = prefix.encode('utf-8') + decode = lambda s: s.decode('utf-8') + + for i in six.moves.range(count()): + key = key_by_index(i) + if key.startswith(prefix): + if strip_prefix: + key = key[len(prefix):] + yield decode(key) + + +def itervalues(prefix=b''): + for key in iterkeys(prefix=prefix): + value = find(key) + yield value + + +def iteritems(prefix='', strip_prefix=False): + for key in iterkeys(prefix=prefix): + value = find(key) + if strip_prefix: + key = key[len(prefix):] + yield key, value + + +def resfs_file_exists(path): + return resfs_src(path, resfs_file=True) is not None + + +def resfs_files(prefix=''): + decode = lambda s: s + if isinstance(prefix, six.text_type): + decode = lambda s: s.decode('utf-8') + return [decode(s) for s in __resfs_files(prefix=prefix)] + + +def find(path): + if isinstance(path, six.text_type): + path = path.encode('utf-8') + return __find(path) diff --git a/library/python/resource/ut/lib/qw.txt b/library/python/resource/ut/lib/qw.txt new file mode 100644 index 0000000000..50e37d5cac --- /dev/null +++ b/library/python/resource/ut/lib/qw.txt @@ -0,0 +1 @@ +na gorshke sidel korol diff --git a/library/python/resource/ut/lib/test_simple.py b/library/python/resource/ut/lib/test_simple.py new file mode 100644 index 0000000000..52f006ff91 --- /dev/null +++ b/library/python/resource/ut/lib/test_simple.py @@ -0,0 +1,31 @@ +import six # noqa + +import library.python.resource as rs + +text = b'na gorshke sidel korol\n' + + +def test_find(): + assert rs.find('/qw.txt') == text + + +def test_iter(): + assert set(rs.iterkeys()).issuperset({'/qw.txt', '/prefix/1.txt', '/prefix/2.txt'}) + assert set(rs.iterkeys(prefix='/prefix/')) == {'/prefix/1.txt', '/prefix/2.txt'} + assert set(rs.iterkeys(prefix='/prefix/', strip_prefix=True)) == {'1.txt', '2.txt'} + assert set(rs.iteritems(prefix='/prefix')) == { + ('/prefix/1.txt', text), + ('/prefix/2.txt', text), + } + assert set(rs.iteritems(prefix='/prefix', strip_prefix=True)) == { + ('/1.txt', text), + ('/2.txt', text), + } + + +def test_resfs_files(): + assert 'contrib/python/py/.dist-info/METADATA' in set(rs.resfs_files()) + + +def test_resfs_read(): + assert 'Metadata-Version' in rs.resfs_read('contrib/python/py/.dist-info/METADATA').decode('utf-8') diff --git a/library/python/resource/ut/lib/ya.make b/library/python/resource/ut/lib/ya.make new file mode 100644 index 0000000000..693e388878 --- /dev/null +++ b/library/python/resource/ut/lib/ya.make @@ -0,0 +1,17 @@ +PY23_LIBRARY() + +OWNER(pg) + +TEST_SRCS(test_simple.py) + +PEERDIR( + library/python/resource +) + +RESOURCE( + qw.txt /qw.txt + qw.txt /prefix/1.txt + qw.txt /prefix/2.txt +) + +END() diff --git a/library/python/resource/ut/py2/ya.make b/library/python/resource/ut/py2/ya.make new file mode 100644 index 0000000000..5085610faf --- /dev/null +++ b/library/python/resource/ut/py2/ya.make @@ -0,0 +1,9 @@ +PY2TEST() + +OWNER(pg) + +PEERDIR( + library/python/resource/ut/lib +) + +END() diff --git a/library/python/resource/ut/py3/ya.make b/library/python/resource/ut/py3/ya.make new file mode 100644 index 0000000000..64eb2e83ce --- /dev/null +++ b/library/python/resource/ut/py3/ya.make @@ -0,0 +1,9 @@ +PY3TEST() + +OWNER(pg) + +PEERDIR( + library/python/resource/ut/lib +) + +END() diff --git a/library/python/resource/ut/ya.make b/library/python/resource/ut/ya.make new file mode 100644 index 0000000000..a5ec192d74 --- /dev/null +++ b/library/python/resource/ut/ya.make @@ -0,0 +1,6 @@ +OWNER(pg) + +RECURSE( + py2 + py3 +) diff --git a/library/python/resource/ya.make b/library/python/resource/ya.make new file mode 100644 index 0000000000..989329fa4b --- /dev/null +++ b/library/python/resource/ya.make @@ -0,0 +1,13 @@ +PY23_LIBRARY() + +OWNER(pg) + +PEERDIR( + contrib/python/six +) + +PY_SRCS(__init__.py) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/python/runtime_py3/__res.pyx b/library/python/runtime_py3/__res.pyx new file mode 100644 index 0000000000..97190d9f29 --- /dev/null +++ b/library/python/runtime_py3/__res.pyx @@ -0,0 +1,36 @@ +from _codecs import utf_8_decode, utf_8_encode + +from libcpp cimport bool + +from util.generic.string cimport TString, TStringBuf + + +cdef extern from "library/cpp/resource/resource.h" namespace "NResource": + cdef size_t Count() except + + cdef TStringBuf KeyByIndex(size_t idx) except + + cdef bool FindExact(const TStringBuf key, TString* result) nogil except + + + +def count(): + return Count() + + +def key_by_index(idx): + cdef TStringBuf ret = KeyByIndex(idx) + + return ret.Data()[:ret.Size()] + + +def find(s): + cdef TString res + + if isinstance(s, str): + s = utf_8_encode(s)[0] + + if FindExact(TStringBuf(s, len(s)), &res): + return res.c_str()[:res.length()] + + return None + + +include "importer.pxi" diff --git a/library/python/runtime_py3/entry_points.py b/library/python/runtime_py3/entry_points.py new file mode 100644 index 0000000000..05098723cb --- /dev/null +++ b/library/python/runtime_py3/entry_points.py @@ -0,0 +1,52 @@ +import sys + +import __res + + +def repl(): + user_ns = {} + py_main = __res.find('PY_MAIN') + + if py_main: + mod_name, func_name = (py_main.split(b':', 1) + [None])[:2] + try: + import importlib + mod = importlib.import_module(mod_name.decode('UTF-8')) + user_ns = mod.__dict__ + except: + import traceback + traceback.print_exc() + + if func_name and '__main__' not in user_ns: + def run(args): + if isinstance(args, str): + import shlex + args = shlex.split(args) + + import sys + sys.argv = [sys.argv[0]] + args + getattr(mod, func_name)() + + user_ns['__main__'] = run + + try: + import IPython + except ModuleNotFoundError: + pass + else: + return IPython.start_ipython(user_ns=user_ns) + + import code + code.interact(local=user_ns) + + +def resource_files(): + sys.stdout.buffer.write(b'\n'.join(sorted(__res.resfs_files()) + [b''])) + + +def run_constructors(): + for key, module_name in __res.iter_keys(b'py/constructors/'): + import importlib + module = importlib.import_module(module_name.decode()) + init_func = getattr(module, __res.find(key).decode()) + init_func() diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi new file mode 100644 index 0000000000..904f94dea2 --- /dev/null +++ b/library/python/runtime_py3/importer.pxi @@ -0,0 +1,571 @@ +import marshal +import sys +from _codecs import utf_8_decode, utf_8_encode +from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter +from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split +from _io import FileIO + +import __res as __resource + +_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] +_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] +env_entry_point = b'Y_PYTHON_ENTRY_POINT' +env_source_root = b'Y_PYTHON_SOURCE_ROOT' +cfg_source_root = b'arcadia-source-root' +env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' +res_ya_ide_venv = b'YA_IDE_VENV' +executable = sys.executable or 'Y_PYTHON' +sys.modules['run_import_hook'] = __resource + +# This is the prefix in contrib/tools/python3/src/Lib/ya.make. +py_prefix = b'py/' +py_prefix_len = len(py_prefix) + +YA_IDE_VENV = __resource.find(res_ya_ide_venv) +Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV + + +def _init_venv(): + if not _path_isabs(executable): + raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) + + # Creative copy-paste from site.py + exe_dir, _ = _path_split(executable) + site_prefix, _ = _path_split(exe_dir) + libpath = _path_join(site_prefix, 'lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + sys.path.insert(0, libpath) + + # emulate site.venv() + sys.prefix = site_prefix + sys.exec_prefix = site_prefix + + conf_basename = 'pyvenv.cfg' + candidate_confs = [ + conffile for conffile in ( + _path_join(exe_dir, conf_basename), + _path_join(site_prefix, conf_basename) + ) + if _path_isfile(conffile) + ] + if not candidate_confs: + raise RuntimeError('{} not found'.format(conf_basename)) + virtual_conf = candidate_confs[0] + with FileIO(virtual_conf, 'r') as f: + for line in f: + if b'=' in line: + key, _, value = line.partition(b'=') + key = key.strip().lower() + value = value.strip() + if key == cfg_source_root: + return value + raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf)) + + +def _get_source_root(): + env_value = _os.environ.get(env_source_root) + if env_value or not YA_IDE_VENV: + return env_value + + return _init_venv() + + +Y_PYTHON_SOURCE_ROOT = _get_source_root() + + +def _print(*xs): + """ + This is helpful for debugging, since automatic bytes to str conversion is + not available yet. It is also possible to debug with GDB by breaking on + __Pyx_AddTraceback (with Python GDB pretty printers enabled). + """ + parts = [] + for s in xs: + if not isinstance(s, (bytes, str)): + s = str(s) + parts.append(_s(s)) + sys.stderr.write(' '.join(parts) + '\n') + + +def file_bytes(path): + # 'open' is not avaiable yet. + with FileIO(path, 'r') as f: + return f.read() + + +def iter_keys(prefix): + l = len(prefix) + for idx in range(__resource.count()): + key = __resource.key_by_index(idx) + if key.startswith(prefix): + yield key, key[l:] + + +def iter_py_modules(with_keys=False): + for key, path in iter_keys(b'resfs/file/' + py_prefix): + if path.endswith(b'.py'): # It may also end with '.pyc'. + mod = _s(path[:-3].replace(b'/', b'.')) + if with_keys: + yield key, mod + else: + yield mod + + +def iter_prefixes(s): + i = s.find('.') + while i >= 0: + yield s[:i] + i = s.find('.', i + 1) + + +def resfs_resolve(path): + """ + Return the absolute path of a root-relative path if it exists. + """ + path = _b(path) + if Y_PYTHON_SOURCE_ROOT: + if not path.startswith(Y_PYTHON_SOURCE_ROOT): + path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path)) + if _path_isfile(path): + return path + + +def resfs_src(key, resfs_file=False): + """ + Return the root-relative file path of a resource key. + """ + if resfs_file: + key = b'resfs/file/' + _b(key) + return __resource.find(b'resfs/src/' + _b(key)) + + +def resfs_read(path, builtin=None): + """ + Return the bytes of the resource file at path, or None. + If builtin is True, do not look for it on the filesystem. + If builtin is False, do not look in the builtin resources. + """ + if builtin is not True: + arcpath = resfs_src(path, resfs_file=True) + if arcpath: + fspath = resfs_resolve(arcpath) + if fspath: + return file_bytes(fspath) + + if builtin is not False: + return __resource.find(b'resfs/file/' + _b(path)) + + +def resfs_files(prefix=b''): + """ + List builtin resource file paths. + """ + return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))] + + +def mod_path(mod): + """ + Return the resfs path to the source code of the module with the given name. + """ + return py_prefix + _b(mod).replace(b'.', b'/') + b'.py' + + +class ResourceImporter(object): + + """ A meta_path importer that loads code from built-in resources. + """ + + def __init__(self): + self.memory = set(iter_py_modules()) # Set of importable module names. + self.source_map = {} # Map from file names to module names. + self._source_name = {} # Map from original to altered module names. + self._package_prefix = '' + if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: + self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) + else: + self.arcadia_source_finder = None + + for p in list(self.memory) + list(sys.builtin_module_names): + for pp in iter_prefixes(p): + k = pp + '.__init__' + if k not in self.memory: + self.memory.add(k) + + def for_package(self, name): + import copy + importer = copy.copy(self) + importer._package_prefix = name + '.' + return importer + + def _find_mod_path(self, fullname): + """Find arcadia relative path by module name""" + relpath = resfs_src(mod_path(fullname), resfs_file=True) + if relpath or not self.arcadia_source_finder: + return relpath + return self.arcadia_source_finder.get_module_path(fullname) + + def find_spec(self, fullname, path=None, target=None): + try: + is_package = self.is_package(fullname) + except ImportError: + return None + return spec_from_loader(fullname, self, is_package=is_package) + + def find_module(self, fullname, path=None): + """For backward compatibility.""" + spec = self.find_spec(fullname, path) + return spec.loader if spec is not None else None + + def create_module(self, spec): + """Use default semantics for module creation.""" + + def exec_module(self, module): + code = self.get_code(module.__name__) + module.__file__ = code.co_filename + if self.is_package(module.__name__): + module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)] + # exec(code, module.__dict__) + _call_with_frames_removed(exec, code, module.__dict__) + + # PEP-302 extension 1 of 3: data loader. + def get_data(self, path): + path = _b(path) + abspath = resfs_resolve(path) + if abspath: + return file_bytes(abspath) + path = path.replace(_b('\\'), _b('/')) + data = resfs_read(path, builtin=True) + if data is None: + raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files + return data + + # PEP-302 extension 2 of 3: get __file__ without importing. + def get_filename(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' + relpath = self._find_mod_path(fullname) + if isinstance(relpath, bytes): + relpath = _s(relpath) + return relpath or modname + + # PEP-302 extension 3 of 3: packaging introspection. + # Used by `linecache` (while printing tracebacks) unless module filename + # exists on the filesystem. + def get_source(self, fullname): + fullname = self._source_name.get(fullname) or fullname + if self.is_package(fullname): + fullname += '.__init__' + + relpath = self.get_filename(fullname) + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + return _s(file_bytes(abspath)) + data = resfs_read(mod_path(fullname)) + return _s(data) if data else '' + + def get_code(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' + + path = mod_path(fullname) + relpath = self._find_mod_path(fullname) + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + data = file_bytes(abspath) + return compile(data, _s(abspath), 'exec', dont_inherit=True) + + yapyc_path = path + b'.yapyc3' + yapyc_data = resfs_read(yapyc_path, builtin=True) + if yapyc_data: + return marshal.loads(yapyc_data) + else: + py_data = resfs_read(path, builtin=True) + if py_data: + return compile(py_data, _s(relpath), 'exec', dont_inherit=True) + else: + # This covers packages with no __init__.py in resources. + return compile('', modname, 'exec', dont_inherit=True) + + def is_package(self, fullname): + if fullname in self.memory: + return False + + if fullname + '.__init__' in self.memory: + return True + + if self.arcadia_source_finder: + return self.arcadia_source_finder.is_package(fullname) + + raise ImportError(fullname) + + # Extension for contrib/python/coverage. + def file_source(self, filename): + """ + Return the key of the module source by its resource path. + """ + if not self.source_map: + for key, mod in iter_py_modules(with_keys=True): + path = self.get_filename(mod) + self.source_map[path] = key + + if filename in self.source_map: + return self.source_map[filename] + + if resfs_read(filename, builtin=True) is not None: + return b'resfs/file/' + _b(filename) + + return b'' + + # Extension for pkgutil.iter_modules. + def iter_modules(self, prefix=''): + import re + rx = re.compile(re.escape(self._package_prefix) + r'([^.]+)(\.__init__)?$') + for p in self.memory: + m = rx.match(p) + if m: + yield prefix + m.group(1), m.group(2) is not None + if self.arcadia_source_finder: + for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix): + yield m + + def get_resource_reader(self, fullname): + try: + if not self.is_package(fullname): + return None + except ImportError: + return None + return _ResfsResourceReader(self, fullname) + + +class _ResfsResourceReader: + + def __init__(self, importer, fullname): + self.importer = importer + self.fullname = fullname + + import os + self.prefix = "{}/".format(os.path.dirname(self.importer.get_filename(self.fullname))) + + def open_resource(self, resource): + path = f'{self.prefix}{resource}' + from io import BytesIO + try: + return BytesIO(self.importer.get_data(path)) + except OSError: + raise FileNotFoundError(path) + + def resource_path(self, resource): + # All resources are in the binary file, so there is no path to the file. + # Raising FileNotFoundError tells the higher level API to extract the + # binary data and create a temporary file. + raise FileNotFoundError + + def is_resource(self, name): + path = f'{self.prefix}{name}' + try: + self.importer.get_data(path) + except OSError: + return False + return True + + def contents(self): + subdirs_seen = set() + for key in resfs_files(self.prefix): + relative = key[len(self.prefix):] + res_or_subdir, *other = relative.split(b'/') + if not other: + yield _s(res_or_subdir) + elif res_or_subdir not in subdirs_seen: + subdirs_seen.add(res_or_subdir) + yield _s(res_or_subdir) + + +class BuiltinSubmoduleImporter(BuiltinImporter): + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if path is not None: + return super().find_spec(fullname, None, target) + else: + return None + + +class ArcadiaSourceFinder: + """ + Search modules and packages in arcadia source tree. + See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details + """ + NAMESPACE_PREFIX = b'py/namespace/' + PY_EXT = '.py' + YA_MAKE = 'ya.make' + + def __init__(self, source_root): + self.source_root = source_root + self.module_path_cache = {'': set()} + for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX): + # dirty_path contains unique prefix to prevent repeatable keys in the resource storage + path = dirty_path.split(b'/', 1)[1] + namespaces = __resource.find(key).split(b':') + for n in namespaces: + package_name = _s(n.rstrip(b'.')) + self.module_path_cache.setdefault(package_name, set()).add(_s(path)) + # Fill parents with default empty path set if parent doesn't exist in the cache yet + while package_name: + package_name = package_name.rpartition('.')[0] + if package_name in self.module_path_cache: + break + self.module_path_cache.setdefault(package_name, set()) + for package_name in self.module_path_cache.keys(): + self._add_parent_dirs(package_name, visited=set()) + + def get_module_path(self, fullname): + """ + Find file path for module 'fullname'. + For packages caller pass fullname as 'package.__init__'. + Return None if nothing is found. + """ + try: + if not self.is_package(fullname): + return _b(self._cache_module_path(fullname)) + except ImportError: + pass + + def is_package(self, fullname): + """Check if fullname is a package. Raise ImportError if fullname is not found""" + path = self._cache_module_path(fullname) + if isinstance(path, set): + return True + if isinstance(path, str): + return False + raise ImportError(fullname) + + def iter_modules(self, package_prefix, prefix): + paths = self._cache_module_path(package_prefix.rstrip('.')) + if paths is not None: + # Note: it's ok to yield duplicates because pkgutil discards them + + # Yield from cache + import re + rx = re.compile(re.escape(package_prefix) + r'([^.]+)$') + for mod, path in self.module_path_cache.items(): + if path is not None: + m = rx.match(mod) + if m: + yield prefix + m.group(1), self.is_package(mod) + + # Yield from file system + for path in paths: + abs_path = _path_join(self.source_root, path) + for dir_item in _os.listdir(abs_path): + if self._path_is_simple_dir(_path_join(abs_path, dir_item)): + yield prefix + dir_item, True + elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)): + yield prefix + dir_item[:-len(self.PY_EXT)], False + + def _path_is_simple_dir(self, abs_path): + """ + Check if path is a directory but doesn't contain ya.make file. + We don't want to steal directory from nested project and treat it as a package + """ + return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE)) + + def _find_module_in_paths(self, find_package_only, paths, module): + """Auxiliary method. See _cache_module_path() for details""" + if paths: + package_paths = set() + for path in paths: + rel_path = _path_join(path, module) + if not find_package_only: + # Check if file_path is a module + module_path = rel_path + self.PY_EXT + if _path_isfile(_path_join(self.source_root, module_path)): + return module_path + # Check if file_path is a package + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + package_paths.add(rel_path) + if package_paths: + return package_paths + + def _cache_module_path(self, fullname, find_package_only=False): + """ + Find module path or package directory paths and save result in the cache + + find_package_only=True - don't try to find module + + Returns: + List of relative package paths - for a package + Relative module path - for a module + None - module or package is not found + """ + if fullname not in self.module_path_cache: + parent, _, tail = fullname.rpartition('.') + parent_paths = self._cache_module_path(parent, find_package_only=True) + self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail) + return self.module_path_cache[fullname] + + def _add_parent_dirs(self, package_name, visited): + if not package_name or package_name in visited: + return + visited.add(package_name) + + parent, _, tail = package_name.rpartition('.') + self._add_parent_dirs(parent, visited) + + paths = self.module_path_cache[package_name] + for parent_path in self.module_path_cache[parent]: + rel_path = _path_join(parent_path, tail) + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + paths.add(rel_path) + + +def excepthook(*args, **kws): + # traceback module cannot be imported at module level, because interpreter + # is not fully initialized yet + + import traceback + + return traceback.print_exception(*args, **kws) + + +importer = ResourceImporter() + + +def executable_path_hook(path): + if path == executable: + return importer + + if path.startswith(executable + path_sep): + return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.')) + + raise ImportError(path) + + +if YA_IDE_VENV: + sys.meta_path.append(importer) + sys.meta_path.append(BuiltinSubmoduleImporter) + if executable not in sys.path: + sys.path.append(executable) + sys.path_hooks.append(executable_path_hook) +else: + sys.meta_path.insert(0, BuiltinSubmoduleImporter) + sys.meta_path.insert(0, importer) + if executable not in sys.path: + sys.path.insert(0, executable) + sys.path_hooks.insert(0, executable_path_hook) + +sys.path_importer_cache[executable] = importer + +# Indicator that modules and resources are built-in rather than on the file system. +sys.is_standalone_binary = True +sys.frozen = True + +# Set of names of importable modules. +sys.extra_modules = importer.memory + +# Use custom implementation of traceback printer. +# Built-in printer (PyTraceBack_Print) does not support custom module loaders +sys.excepthook = excepthook diff --git a/library/python/runtime_py3/main/get_py_main.cpp b/library/python/runtime_py3/main/get_py_main.cpp new file mode 100644 index 0000000000..67c400d4f4 --- /dev/null +++ b/library/python/runtime_py3/main/get_py_main.cpp @@ -0,0 +1,8 @@ +#include <library/cpp/resource/resource.h> + +#include <stdlib.h> + +extern "C" char* GetPyMain() { + TString res = NResource::Find("PY_MAIN"); + return strdup(res.c_str()); +} diff --git a/library/python/runtime_py3/main/main.c b/library/python/runtime_py3/main/main.c new file mode 100644 index 0000000000..3159800615 --- /dev/null +++ b/library/python/runtime_py3/main/main.c @@ -0,0 +1,231 @@ +#include <Python.h> +#include <contrib/tools/python3/src/Include/internal/pycore_runtime.h> // _PyRuntime_Initialize() + +#include <stdlib.h> +#include <string.h> +#include <locale.h> + +void Py_InitArgcArgv(int argc, wchar_t **argv); +char* GetPyMain(); + +static const char* env_entry_point = "Y_PYTHON_ENTRY_POINT"; +static const char* env_bytes_warning = "Y_PYTHON_BYTES_WARNING"; + +#ifdef _MSC_VER +extern char** environ; + +void unsetenv(const char* name) { + const int n = strlen(name); + char** dst = environ; + for (char** src = environ; *src; src++) + if (strncmp(*src, name, n) || (*src)[n] != '=') + *dst++ = *src; + *dst = NULL; +} +#endif + +static int RunModule(const char *modname) +{ + PyObject *module, *runpy, *runmodule, *runargs, *result; + runpy = PyImport_ImportModule("runpy"); + if (runpy == NULL) { + fprintf(stderr, "Could not import runpy module\n"); + PyErr_Print(); + return -1; + } + runmodule = PyObject_GetAttrString(runpy, "_run_module_as_main"); + if (runmodule == NULL) { + fprintf(stderr, "Could not access runpy._run_module_as_main\n"); + PyErr_Print(); + Py_DECREF(runpy); + return -1; + } + module = PyUnicode_FromString(modname); + if (module == NULL) { + fprintf(stderr, "Could not convert module name to unicode\n"); + PyErr_Print(); + Py_DECREF(runpy); + Py_DECREF(runmodule); + return -1; + } + runargs = Py_BuildValue("(Oi)", module, 0); + if (runargs == NULL) { + fprintf(stderr, + "Could not create arguments for runpy._run_module_as_main\n"); + PyErr_Print(); + Py_DECREF(runpy); + Py_DECREF(runmodule); + Py_DECREF(module); + return -1; + } + result = PyObject_Call(runmodule, runargs, NULL); + if (result == NULL) { + PyErr_Print(); + } + Py_DECREF(runpy); + Py_DECREF(runmodule); + Py_DECREF(module); + Py_DECREF(runargs); + if (result == NULL) { + return -1; + } + Py_DECREF(result); + return 0; +} + +static int pymain(int argc, char** argv) { + PyStatus status = _PyRuntime_Initialize(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + int i, sts = 1; + char* oldloc = NULL; + wchar_t** argv_copy = NULL; + /* We need a second copies, as Python might modify the first one. */ + wchar_t** argv_copy2 = NULL; + char* entry_point_copy = NULL; + + if (argc > 0) { + argv_copy = PyMem_RawMalloc(sizeof(wchar_t*) * argc); + argv_copy2 = PyMem_RawMalloc(sizeof(wchar_t*) * argc); + if (!argv_copy || !argv_copy2) { + fprintf(stderr, "out of memory\n"); + goto error; + } + } + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.pathconfig_warnings = 0; /* Suppress errors from getpath.c */ + + const char* bytes_warning = getenv(env_bytes_warning); + if (bytes_warning) { + config.bytes_warning = atoi(bytes_warning); + } + + oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL)); + if (!oldloc) { + fprintf(stderr, "out of memory\n"); + goto error; + } + + setlocale(LC_ALL, ""); + for (i = 0; i < argc; i++) { + argv_copy[i] = Py_DecodeLocale(argv[i], NULL); + argv_copy2[i] = argv_copy[i]; + if (!argv_copy[i]) { + fprintf(stderr, "Unable to decode the command line argument #%i\n", + i + 1); + argc = i; + goto error; + } + } + setlocale(LC_ALL, oldloc); + PyMem_RawFree(oldloc); + oldloc = NULL; + + if (argc >= 1) + Py_SetProgramName(argv_copy[0]); + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + const char* entry_point = getenv(env_entry_point); + if (entry_point) { + entry_point_copy = strdup(entry_point); + if (!entry_point_copy) { + fprintf(stderr, "out of memory\n"); + goto error; + } + } else { + entry_point_copy = GetPyMain(); + } + + if (entry_point_copy == NULL) { + fprintf(stderr, "No entry point, did you forget PY_MAIN?\n"); + goto error; + } + + if (entry_point_copy && !strcmp(entry_point_copy, ":main")) { + unsetenv(env_entry_point); + sts = Py_Main(argc, argv_copy); + free(entry_point_copy); + return sts; + } + + Py_InitArgcArgv(argc, argv_copy); + PySys_SetArgv(argc, argv_copy); + + { + PyObject* module = PyImport_ImportModule("library.python.runtime_py3.entry_points"); + if (module == NULL) { + PyErr_Print(); + } else { + PyObject* res = PyObject_CallMethod(module, "run_constructors", NULL); + if (res == NULL) { + PyErr_Print(); + } else { + Py_DECREF(res); + } + Py_DECREF(module); + } + } + + const char* module_name = entry_point_copy; + const char* func_name = NULL; + + char *colon = strchr(entry_point_copy, ':'); + if (colon != NULL) { + colon[0] = '\0'; + func_name = colon + 1; + } + if (module_name[0] == '\0') { + module_name = "library.python.runtime_py3.entry_points"; + } + + if (!func_name) { + sts = RunModule(module_name); + } else { + PyObject* module = PyImport_ImportModule(module_name); + + if (module == NULL) { + PyErr_Print(); + } else { + PyObject* value = PyObject_CallMethod(module, func_name, NULL); + + if (value == NULL) { + PyErr_Print(); + } else { + Py_DECREF(value); + sts = 0; + } + + Py_DECREF(module); + } + } + + if (Py_FinalizeEx() < 0) { + sts = 120; + } + +error: + free(entry_point_copy); + PyMem_RawFree(argv_copy); + if (argv_copy2) { + for (i = 0; i < argc; i++) + PyMem_RawFree(argv_copy2[i]); + PyMem_RawFree(argv_copy2); + } + PyMem_RawFree(oldloc); + return sts; +} + +int (*mainptr)(int argc, char** argv) = pymain; + +int main(int argc, char** argv) { + return mainptr(argc, argv); +} diff --git a/library/python/runtime_py3/main/ya.make b/library/python/runtime_py3/main/ya.make new file mode 100644 index 0000000000..f308a93b28 --- /dev/null +++ b/library/python/runtime_py3/main/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +OWNER( + pg + orivej +) + +PEERDIR( + contrib/tools/python3/src + library/cpp/resource +) + +ADDINCL( + contrib/tools/python3/src/Include +) + +CFLAGS( + -DPy_BUILD_CORE +) + +SRCS( + main.c + get_py_main.cpp +) + +END() diff --git a/library/python/runtime_py3/sitecustomize.pyx b/library/python/runtime_py3/sitecustomize.pyx new file mode 100644 index 0000000000..966bbe8ba6 --- /dev/null +++ b/library/python/runtime_py3/sitecustomize.pyx @@ -0,0 +1,69 @@ +import re +import sys + +import __res + +from importlib.abc import ResourceReader +from importlib.metadata import Distribution, DistributionFinder, PackageNotFoundError, Prepared + +ResourceReader.register(__res._ResfsResourceReader) + +METADATA_NAME = re.compile('^Name: (.*)$', re.MULTILINE) + + +class ArcadiaDistribution(Distribution): + + def __init__(self, prefix): + self.prefix = prefix + + def read_text(self, filename): + data = __res.resfs_read(f'{self.prefix}{filename}') + if data: + return data.decode('utf-8') + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return f'{self.prefix}{path}' + + +class ArcadiaMetadataFinder(DistributionFinder): + + prefixes = {} + + @classmethod + def find_distributions(cls, context=DistributionFinder.Context()): + found = cls._search_prefixes(context.name) + return map(ArcadiaDistribution, found) + + @classmethod + def _init_prefixes(cls): + cls.prefixes.clear() + + for resource in __res.resfs_files(): + resource = resource.decode('utf-8') + if not resource.endswith('METADATA'): + continue + data = __res.resfs_read(resource).decode('utf-8') + metadata_name = METADATA_NAME.search(data) + if metadata_name: + metadata_name = Prepared(metadata_name.group(1)) + cls.prefixes[metadata_name.normalized] = resource[:-len('METADATA')] + + @classmethod + def _search_prefixes(cls, name): + if not cls.prefixes: + cls._init_prefixes() + + if name: + try: + yield cls.prefixes[Prepared(name).normalized] + except KeyError: + raise PackageNotFoundError(name) + else: + for prefix in sorted(cls.prefixes.values()): + yield prefix + + +# monkeypatch standart library +import importlib.metadata +importlib.metadata.MetadataPathFinder = ArcadiaMetadataFinder diff --git a/library/python/runtime_py3/test/.dist-info/METADATA b/library/python/runtime_py3/test/.dist-info/METADATA new file mode 100644 index 0000000000..bb36162199 --- /dev/null +++ b/library/python/runtime_py3/test/.dist-info/METADATA @@ -0,0 +1,14 @@ +Metadata-Version: 2.1 +Name: foo-bar +Version: 1.2.3 +Summary: +Home-page: https://foo.org/ +Author: Foo +Author-email: foo@ya.com +License: UNKNOWN +Platform: any +Classifier: Development Status :: 4 - Beta +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=3.8 +Requires-Dist: Werkzeug (>=0.15) +Requires-Dist: Jinja2 (>=2.10.1) diff --git a/library/python/runtime_py3/test/.dist-info/RECORD b/library/python/runtime_py3/test/.dist-info/RECORD new file mode 100644 index 0000000000..dabbbff80d --- /dev/null +++ b/library/python/runtime_py3/test/.dist-info/RECORD @@ -0,0 +1 @@ +foo_bar.py,sha256=0000000000000000000000000000000000000000000,20 diff --git a/library/python/runtime_py3/test/.dist-info/entry_points.txt b/library/python/runtime_py3/test/.dist-info/entry_points.txt new file mode 100644 index 0000000000..f5e2fd2657 --- /dev/null +++ b/library/python/runtime_py3/test/.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +foo_cli = foo_bar:cli diff --git a/library/python/runtime_py3/test/.dist-info/top_level.txt b/library/python/runtime_py3/test/.dist-info/top_level.txt new file mode 100644 index 0000000000..d2c068bc6b --- /dev/null +++ b/library/python/runtime_py3/test/.dist-info/top_level.txt @@ -0,0 +1 @@ +foo_bar diff --git a/library/python/runtime_py3/test/canondata/result.json b/library/python/runtime_py3/test/canondata/result.json new file mode 100644 index 0000000000..a7d045fc9c --- /dev/null +++ b/library/python/runtime_py3/test/canondata/result.json @@ -0,0 +1,62 @@ +{ + "test_traceback.test_traceback[custom-default]": { + "stderr": { + "checksum": "6c1a9b47baa51cc6903b85fd43c529b5", + "uri": "file://test_traceback.test_traceback_custom-default_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_custom-default_/stdout.txt" + } + }, + "test_traceback.test_traceback[custom-ultratb_color]": { + "stderr": { + "checksum": "048e27049fb8db64bd295b17f505b0ad", + "uri": "file://test_traceback.test_traceback_custom-ultratb_color_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_custom-ultratb_color_/stdout.txt" + } + }, + "test_traceback.test_traceback[custom-ultratb_verbose]": { + "stderr": { + "checksum": "e9af42aa3736141d9b67a1652eea412e", + "uri": "file://test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt" + } + }, + "test_traceback.test_traceback[main-default]": { + "stderr": { + "checksum": "6c1a9b47baa51cc6903b85fd43c529b5", + "uri": "file://test_traceback.test_traceback_main-default_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_main-default_/stdout.txt" + } + }, + "test_traceback.test_traceback[main-ultratb_color]": { + "stderr": { + "checksum": "048e27049fb8db64bd295b17f505b0ad", + "uri": "file://test_traceback.test_traceback_main-ultratb_color_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_main-ultratb_color_/stdout.txt" + } + }, + "test_traceback.test_traceback[main-ultratb_verbose]": { + "stderr": { + "checksum": "e9af42aa3736141d9b67a1652eea412e", + "uri": "file://test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt" + }, + "stdout": { + "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89", + "uri": "file://test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt" + } + } +}
\ No newline at end of file diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt new file mode 100644 index 0000000000..5eb7da170a --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main + one() + File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one + modfunc(two) # aaa + File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc + f() # call back to caller + File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two + three(42) + File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three + raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) +RuntimeError: Kaboom! I'm dead: 42 diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt new file mode 100644 index 0000000000..9e5a474cbd --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt @@ -0,0 +1,13 @@ +Traceback (most recent call last): + File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main + one() + File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one + modfunc(two) # aaa + File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc + f() # call back to caller + File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two + three(42) + File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three + raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) +RuntimeError: Kaboom! I'm dead: 42 + diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt new file mode 100644 index 0000000000..b0b299ebe6 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------- +RuntimeError Traceback (most recent call last) +library/python/runtime_py3/test/traceback/crash.py in main() + 42 sys.executable = '<traceback test>' + 43 +---> 44 one() + global one = <function one> + +library/python/runtime_py3/test/traceback/crash.py in one() + 10 + 11 def one(): +---> 12 modfunc(two) # aaa + global modfunc = <function modfunc> + global two = <function two> + 13 + 14 + +library/python/runtime_py3/test/traceback/mod/__init__.py in modfunc(f=<function two>) + 1 def modfunc(f): + 2 # lalala +----> 3 f() # call back to caller + f = <function two> + +library/python/runtime_py3/test/traceback/crash.py in two() + 14 + 15 def two(): +---> 16 three(42) + global three = <function three> + 17 + 18 + +library/python/runtime_py3/test/traceback/crash.py in three(x=42) + 18 + 19 def three(x): +---> 20 raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) + global RuntimeError.format = undefined + x = 42 + 21 + 22 + +RuntimeError: Kaboom! I'm dead: 42 diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt new file mode 100644 index 0000000000..5eb7da170a --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main + one() + File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one + modfunc(two) # aaa + File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc + f() # call back to caller + File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two + three(42) + File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three + raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) +RuntimeError: Kaboom! I'm dead: 42 diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt new file mode 100644 index 0000000000..9e5a474cbd --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt @@ -0,0 +1,13 @@ +Traceback (most recent call last): + File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main + one() + File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one + modfunc(two) # aaa + File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc + f() # call back to caller + File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two + three(42) + File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three + raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) +RuntimeError: Kaboom! I'm dead: 42 + diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt new file mode 100644 index 0000000000..b0b299ebe6 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------- +RuntimeError Traceback (most recent call last) +library/python/runtime_py3/test/traceback/crash.py in main() + 42 sys.executable = '<traceback test>' + 43 +---> 44 one() + global one = <function one> + +library/python/runtime_py3/test/traceback/crash.py in one() + 10 + 11 def one(): +---> 12 modfunc(two) # aaa + global modfunc = <function modfunc> + global two = <function two> + 13 + 14 + +library/python/runtime_py3/test/traceback/mod/__init__.py in modfunc(f=<function two>) + 1 def modfunc(f): + 2 # lalala +----> 3 f() # call back to caller + f = <function two> + +library/python/runtime_py3/test/traceback/crash.py in two() + 14 + 15 def two(): +---> 16 three(42) + global three = <function three> + 17 + 18 + +library/python/runtime_py3/test/traceback/crash.py in three(x=42) + 18 + 19 def three(x): +---> 20 raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) + global RuntimeError.format = undefined + x = 42 + 21 + 22 + +RuntimeError: Kaboom! I'm dead: 42 diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt new file mode 100644 index 0000000000..2c9793eb14 --- /dev/null +++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt @@ -0,0 +1,2 @@ +__name__ = library.python.runtime_py3.test.traceback.crash +__file__ = library/python/runtime_py3/test/traceback/crash.py diff --git a/library/python/runtime_py3/test/resources/__init__.py b/library/python/runtime_py3/test/resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/library/python/runtime_py3/test/resources/__init__.py diff --git a/library/python/runtime_py3/test/resources/foo.txt b/library/python/runtime_py3/test/resources/foo.txt new file mode 100644 index 0000000000..ba0e162e1c --- /dev/null +++ b/library/python/runtime_py3/test/resources/foo.txt @@ -0,0 +1 @@ +bar
\ No newline at end of file diff --git a/library/python/runtime_py3/test/resources/submodule/__init__.py b/library/python/runtime_py3/test/resources/submodule/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/library/python/runtime_py3/test/resources/submodule/__init__.py diff --git a/library/python/runtime_py3/test/resources/submodule/bar.txt b/library/python/runtime_py3/test/resources/submodule/bar.txt new file mode 100644 index 0000000000..1910281566 --- /dev/null +++ b/library/python/runtime_py3/test/resources/submodule/bar.txt @@ -0,0 +1 @@ +foo
\ No newline at end of file diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py new file mode 100644 index 0000000000..ff80d0a0a2 --- /dev/null +++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py @@ -0,0 +1,317 @@ +import unittest +import yaml +from unittest.mock import patch +from parameterized import parameterized + +import __res as res + + +NAMESPACE_PREFIX = b'py/namespace/' +TEST_SOURCE_ROOT = '/home/arcadia' + + +class ImporterMocks(object): + def __init__(self, mock_fs, mock_resources): + self._mock_fs = mock_fs + self._mock_resources = mock_resources + self._patchers = [ + patch('__res.iter_keys', wraps=self._iter_keys), + patch('__res.__resource.find', wraps=self._resource_find), + patch('__res._path_isdir', wraps=self._path_isdir), + patch('__res._path_isfile', wraps=self._path_isfile), + patch('__res._os.listdir', wraps=self._os_listdir), + ] + for patcher in self._patchers: + patcher.start() + + def stop(self): + for patcher in self._patchers: + patcher.stop() + + def _iter_keys(self, prefix): + assert prefix == NAMESPACE_PREFIX + l = len(prefix) + for k in self._mock_resources.keys(): + yield k, k[l:] + + def _resource_find(self, key): + return self._mock_resources.get(key) + + def _lookup_mock_fs(self, filename): + path = filename.lstrip('/').split('/') + curdir = self._mock_fs + for item in path: + if item in curdir: + curdir = curdir[item] + else: + return None + return curdir + + def _path_isfile(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, str) + + def _path_isdir(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, dict) + + def _os_listdir(self, dirname): + f = self._lookup_mock_fs(dirname) + if isinstance(f, dict): + return f.keys() + else: + return [] + + +class ArcadiaSourceFinderTestCase(unittest.TestCase): + def setUp(self): + self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources()) + self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT) + + def tearDown(self): + self.import_mock.stop() + + def _get_mock_fs(self): + raise NotImplementedError() + + def _get_mock_resources(self): + raise NotImplementedError() + + +class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: "" + package1: + mod2.py: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'), + ('project.lib.unknown_module', None), + ('project.lib', None), # package + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.mod1', False), + ('project.lib.package1.mod2', False), + ('project', True), + ('project.lib', True), + ('project.lib.package1', True), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + def test_is_package_for_unknown_module(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2')) + + @parameterized.expand([ + ('', { + ('PFX.project', True), + }), + ('project.', { + ('PFX.lib', True), + }), + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.package1', True), + }), + ('project.lib.package1.', { + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + # Check iter_modules() don't crash and return correct result after not existing module was requested + def test_iter_modules_after_unknown_module_import(self): + self.arcadia_source_finder.get_module_path('project.unknown_module') + assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', '')) + + +class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + lib_extension: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.mod2', b'project/lib_extension/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + ns_lib: + mod1.py: '' + top_level_lib: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/ns_lib': b'ns.', + b'py/namespace/unique_prefix2/project/top_level_lib': b'.', + } + + @parameterized.expand([ + ('ns.mod1', b'project/ns_lib/mod1.py'), + ('mod2', b'project/top_level_lib/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('ns', True), + ('ns.mod1', False), + ('mod2', False), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + @parameterized.expand([ + 'project', + 'project.ns_lib', + 'project.top_level_lib', + ]) + def test_is_package_for_unknown_modules(self, module): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module)) + + @parameterized.expand([ + ('', { + ('PFX.ns', True), + ('PFX.mod2', False), + }), + ('ns.', { + ('PFX.mod1', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase): + ''' Packages and modules from tests should not be part of pylib namespace ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylib: + mod1.py: "" + tests: + conftest.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.', + } + + def test_get_module_path_for_lib(self): + assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1') + + def test_get_module_for_tests(self): + assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None + + def test_is_package_for_tests(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests')) + + +class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase): + ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylint: + ya.make: "" + pylint: + __init__.py: "" + patcher: + patch.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylint': b'.', + b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.', + } + + @parameterized.expand([ + ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'), + ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + +class TestEmptyResources(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + ''' + + def _get_mock_resources(self): + return {} + + def test_get_module_path(self): + assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None + + def test_is_package(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project')) + + def test_iter_modules(self): + assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.')) diff --git a/library/python/runtime_py3/test/test_metadata.py b/library/python/runtime_py3/test/test_metadata.py new file mode 100644 index 0000000000..686c176468 --- /dev/null +++ b/library/python/runtime_py3/test/test_metadata.py @@ -0,0 +1,44 @@ +import importlib.metadata as im + +import pytest + + +@pytest.mark.parametrize("name", ("foo-bar", "foo_bar", "Foo-Bar")) +def test_distribution(name): + assert im.distribution(name) is not None + + +def test_unknown_package(): + with pytest.raises(im.PackageNotFoundError): + im.distribution("bar") + + +def test_version(): + assert im.version("foo-bar") == "1.2.3" + + +def test_metadata(): + assert im.metadata("foo-bar") is not None + + +def test_files(): + files = im.files("foo-bar") + assert len(files) == 1 + assert files[0].name == "foo_bar.py" + assert files[0].size == 20 + + +def test_requires(): + assert im.requires("foo-bar") == ["Werkzeug (>=0.15)", "Jinja2 (>=2.10.1)"] + + +def test_entry_points(): + entry_points = im.entry_points() + assert "console_scripts" in entry_points + + flg_found = False + for entry_point in entry_points["console_scripts"]: + if entry_point.name == "foo_cli" and entry_point.value == "foo_bar:cli": + flg_found = True + + assert flg_found diff --git a/library/python/runtime_py3/test/test_resources.py b/library/python/runtime_py3/test/test_resources.py new file mode 100644 index 0000000000..a269329f42 --- /dev/null +++ b/library/python/runtime_py3/test/test_resources.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import importlib.resources as ir + +import pytest + + +@pytest.mark.parametrize("package, resource", ( + ("resources", "foo.txt"), + ("resources.submodule", "bar.txt") +)) +def test_is_resource_good_path(package, resource): + assert ir.is_resource(package, resource) + + +@pytest.mark.parametrize("package, resource", ( + ("resources", "111.txt"), + ("resources.submodule", "222.txt") +)) +def test_is_resource_missing(package, resource): + assert not ir.is_resource(package, resource) + + +def test_is_resource_subresource_directory(): + # Directories are not resources. + assert not ir.is_resource("resources", "submodule") + + +@pytest.mark.parametrize("package, resource, expected", ( + ("resources", "foo.txt", b"bar"), + ("resources.submodule", "bar.txt", b"foo") +)) +def test_read_binary_good_path(package, resource, expected): + assert ir.read_binary(package, resource) == expected + + +def test_read_binary_missing(): + with pytest.raises(FileNotFoundError): + ir.read_binary("resources", "111.txt") + + +@pytest.mark.parametrize("package, resource, expected", ( + ("resources", "foo.txt", "bar"), + ("resources.submodule", "bar.txt", "foo") +)) +def test_read_text_good_path(package, resource, expected): + assert ir.read_text(package, resource) == expected + + +def test_read_text_missing(): + with pytest.raises(FileNotFoundError): + ir.read_text("resources", "111.txt") + + +@pytest.mark.parametrize("package, expected", ( + ("resources", ["submodule", "foo.txt"]), + ("resources.submodule", ["bar.txt"]) +)) +def test_contents_good_path(package, expected): + assert sorted(ir.contents(package)) == sorted(expected) diff --git a/library/python/runtime_py3/test/test_traceback.py b/library/python/runtime_py3/test/test_traceback.py new file mode 100644 index 0000000000..82087ce98a --- /dev/null +++ b/library/python/runtime_py3/test/test_traceback.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +from __future__ import print_function, absolute_import, division + +import os +import re + +import pytest + +import yatest.common as yc + + +def clean_traceback(traceback): + traceback = re.sub(br'\033\[(\d|;)+?m', b'', traceback) # strip ANSI codes + traceback = re.sub(br' at 0x[0-9a-fA-F]+', b'', traceback) # remove object ids + return traceback + + +@pytest.mark.parametrize('mode', [ + 'default', + 'ultratb_color', + 'ultratb_verbose', +]) +@pytest.mark.parametrize('entry_point', [ + 'main', + 'custom', +]) +def test_traceback(mode, entry_point): + tb_tool = yc.build_path('library/python/runtime_py3/test/traceback/traceback') + stdout_path = yc.test_output_path('stdout_raw.txt') + stderr_path = yc.test_output_path('stderr_raw.txt') + filtered_stdout_path = yc.test_output_path('stdout.txt') + filtered_stderr_path = yc.test_output_path('stderr.txt') + + env = os.environ.copy() + env.pop('PYTHONPATH', None) # Do not let program peek into its sources on filesystem + if entry_point == 'custom': + env['Y_PYTHON_ENTRY_POINT'] = 'library.python.runtime_py3.test.traceback.crash:main' + + proc = yc.execute( + command=[tb_tool, mode], + env=env, + stdout=stdout_path, + stderr=stderr_path, + check_exit_code=False, + ) + + with open(filtered_stdout_path, 'wb') as f: + f.write(clean_traceback(proc.std_out)) + + with open(filtered_stderr_path, 'wb') as f: + f.write(clean_traceback(proc.std_err)) + + return { + 'stdout': yc.canonical_file( + filtered_stdout_path, + local=True, + ), + 'stderr': yc.canonical_file( + filtered_stderr_path, + local=True, + ), + } diff --git a/library/python/runtime_py3/test/traceback/__main__.py b/library/python/runtime_py3/test/traceback/__main__.py new file mode 100644 index 0000000000..364db169f0 --- /dev/null +++ b/library/python/runtime_py3/test/traceback/__main__.py @@ -0,0 +1,4 @@ +from library.python.runtime_py3.test.traceback.crash import main + +if __name__ == "__main__": + main() diff --git a/library/python/runtime_py3/test/traceback/crash.py b/library/python/runtime_py3/test/traceback/crash.py new file mode 100644 index 0000000000..b5e36b3dd4 --- /dev/null +++ b/library/python/runtime_py3/test/traceback/crash.py @@ -0,0 +1,44 @@ +import argparse +import cgitb +import sys +import time + +from IPython.core import ultratb + +from .mod import modfunc + + +def one(): + modfunc(two) # aaa + + +def two(): + three(42) + + +def three(x): + raise RuntimeError('Kaboom! I\'m dead: {}'.format(x)) + + +def main(): + hooks = { + 'default': lambda: sys.excepthook, + 'cgitb': lambda: cgitb.Hook(format='text'), + 'ultratb_color': lambda: ultratb.ColorTB(ostream=sys.stderr), + 'ultratb_verbose': lambda: ultratb.VerboseTB(ostream=sys.stderr), + } + + parser = argparse.ArgumentParser() + parser.add_argument('hook', choices=sorted(hooks), default='default') + + args = parser.parse_args() + + sys.excepthook = hooks[args.hook]() + + print('__name__ =', __name__) + print('__file__ =', __file__) + + time.time = lambda: 1531996624.0 # Freeze time + sys.executable = '<traceback test>' + + one() diff --git a/library/python/runtime_py3/test/traceback/mod/__init__.py b/library/python/runtime_py3/test/traceback/mod/__init__.py new file mode 100644 index 0000000000..f00843d786 --- /dev/null +++ b/library/python/runtime_py3/test/traceback/mod/__init__.py @@ -0,0 +1,3 @@ +def modfunc(f): + # lalala + f() # call back to caller diff --git a/library/python/runtime_py3/test/traceback/ya.make b/library/python/runtime_py3/test/traceback/ya.make new file mode 100644 index 0000000000..b61fe9550b --- /dev/null +++ b/library/python/runtime_py3/test/traceback/ya.make @@ -0,0 +1,19 @@ +PY3_PROGRAM() + +OWNER( + abodrov + borman +) + +PEERDIR( + contrib/python/ipython +) + +PY_SRCS( + MAIN + __main__.py=main + crash.py + mod/__init__.py +) + +END() diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make new file mode 100644 index 0000000000..4ec3db74f5 --- /dev/null +++ b/library/python/runtime_py3/test/ya.make @@ -0,0 +1,40 @@ +PY3TEST() + +OWNER( + abodrov + borman +) + +DEPENDS(library/python/runtime_py3/test/traceback) + +PEERDIR( + contrib/python/parameterized + contrib/python/PyYAML +) + +PY_SRCS( + TOP_LEVEL + resources/__init__.py + resources/submodule/__init__.py +) + +TEST_SRCS( + test_metadata.py + test_resources.py + test_traceback.py + test_arcadia_source_finder.py +) + +RESOURCE_FILES( + PREFIX library/python/runtime_py3/test/ + .dist-info/METADATA + .dist-info/RECORD + .dist-info/entry_points.txt + .dist-info/top_level.txt + resources/foo.txt + resources/submodule/bar.txt +) + +END() + +RECURSE_FOR_TESTS(traceback) diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make new file mode 100644 index 0000000000..fa5c11341a --- /dev/null +++ b/library/python/runtime_py3/ya.make @@ -0,0 +1,49 @@ +PY3_LIBRARY() + +OWNER( + borman + orivej + pg +) + +NO_WSHADOW() + +PEERDIR( + contrib/tools/python3/src + contrib/tools/python3/lib/py + library/cpp/resource +) + +CFLAGS(-DCYTHON_REGISTER_ABCS=0) + +NO_PYTHON_INCLUDES() + +ENABLE(PYBUILD_NO_PYC) + +PY_SRCS( + entry_points.py + TOP_LEVEL + + CYTHON_DIRECTIVE + language_level=3 + + __res.pyx + sitecustomize.pyx +) + +IF (CYTHON_COVERAGE) + # Let covarage support add all needed files to resources +ELSE() + RESOURCE_FILES( + PREFIX ${MODDIR}/ + __res.pyx + importer.pxi + sitecustomize.pyx + ) +ENDIF() + +END() + +RECURSE_FOR_TESTS( + test +) diff --git a/library/python/strings/__init__.py b/library/python/strings/__init__.py new file mode 100644 index 0000000000..bd6bf6e7ce --- /dev/null +++ b/library/python/strings/__init__.py @@ -0,0 +1,17 @@ +# flake8 noqa: F401 + +from .strings import ( + DEFAULT_ENCODING, + ENCODING_ERRORS_POLICY, + encode, + fs_encoding, + get_stream_encoding, + guess_default_encoding, + left_strip, + locale_encoding, + stringize_deep, + to_basestring, + to_str, + to_unicode, + unicodize_deep, +) diff --git a/library/python/strings/strings.py b/library/python/strings/strings.py new file mode 100644 index 0000000000..5bfddfe78a --- /dev/null +++ b/library/python/strings/strings.py @@ -0,0 +1,129 @@ +import locale +import logging +import six +import sys +import codecs + +import library.python.func + +logger = logging.getLogger(__name__) + + +DEFAULT_ENCODING = 'utf-8' +ENCODING_ERRORS_POLICY = 'replace' + + +def left_strip(el, prefix): + """ + Strips prefix at the left of el + """ + if el.startswith(prefix): + return el[len(prefix):] + return el + + +# Explicit to-text conversion +# Chooses between str/unicode, i.e. six.binary_type/six.text_type +def to_basestring(value): + if isinstance(value, (six.binary_type, six.text_type)): + return value + try: + if six.PY2: + return unicode(value) + else: + return str(value) + except UnicodeDecodeError: + try: + return str(value) + except UnicodeEncodeError: + return repr(value) +to_text = to_basestring + + +def to_unicode(value, from_enc=DEFAULT_ENCODING): + if isinstance(value, six.text_type): + return value + if isinstance(value, six.binary_type): + if six.PY2: + return unicode(value, from_enc, ENCODING_ERRORS_POLICY) + else: + return value.decode(from_enc, errors=ENCODING_ERRORS_POLICY) + return six.text_type(value) + + +# Optional from_enc enables transcoding +def to_str(value, to_enc=DEFAULT_ENCODING, from_enc=None): + if isinstance(value, six.binary_type): + if from_enc is None or to_enc == from_enc: + # Unknown input encoding or input and output encoding are the same + return value + value = to_unicode(value, from_enc=from_enc) + if isinstance(value, six.text_type): + return value.encode(to_enc, ENCODING_ERRORS_POLICY) + return six.binary_type(value) + + +def _convert_deep(x, enc, convert, relaxed=True): + if x is None: + return None + if isinstance(x, (six.text_type, six.binary_type)): + return convert(x, enc) + if isinstance(x, dict): + return {convert(k, enc): _convert_deep(v, enc, convert, relaxed) for k, v in six.iteritems(x)} + if isinstance(x, list): + return [_convert_deep(e, enc, convert, relaxed) for e in x] + if isinstance(x, tuple): + return tuple([_convert_deep(e, enc, convert, relaxed) for e in x]) + + if relaxed: + return x + raise TypeError('unsupported type') + + +def unicodize_deep(x, enc=DEFAULT_ENCODING, relaxed=True): + return _convert_deep(x, enc, to_unicode, relaxed) + + +def stringize_deep(x, enc=DEFAULT_ENCODING, relaxed=True): + return _convert_deep(x, enc, to_str, relaxed) + + +@library.python.func.memoize() +def locale_encoding(): + try: + loc = locale.getdefaultlocale()[1] + if loc: + codecs.lookup(loc) + return loc + except LookupError as e: + logger.debug('Cannot get system locale: %s', e) + return None + except ValueError as e: + logger.warn('Cannot get system locale: %s', e) + return None + + +def fs_encoding(): + return sys.getfilesystemencoding() + + +def guess_default_encoding(): + enc = locale_encoding() + return enc if enc else DEFAULT_ENCODING + + +@library.python.func.memoize() +def get_stream_encoding(stream): + if stream.encoding: + try: + codecs.lookup(stream.encoding) + return stream.encoding + except LookupError: + pass + return DEFAULT_ENCODING + + +def encode(value, encoding=DEFAULT_ENCODING): + if isinstance(value, six.binary_type): + value = value.decode(encoding, errors='ignore') + return value.encode(encoding) diff --git a/library/python/strings/ut/test_strings.py b/library/python/strings/ut/test_strings.py new file mode 100644 index 0000000000..dd0c694ee1 --- /dev/null +++ b/library/python/strings/ut/test_strings.py @@ -0,0 +1,205 @@ +# coding=utf-8 + +import pytest +import six + +import library.python.strings + + +class Convertible(object): + text = u'текст' + text_utf8 = text.encode('utf-8') + + def __unicode__(self): + return self.text + + def __str__(self): + return self.text_utf8 + + +class ConvertibleToUnicodeOnly(Convertible): + def __str__(self): + return self.text.encode('ascii') + + +class ConvertibleToStrOnly(Convertible): + def __unicode__(self): + return self.text_utf8.decode('ascii') + + +class NonConvertible(ConvertibleToUnicodeOnly, ConvertibleToStrOnly): + pass + + +def test_to_basestring(): + assert library.python.strings.to_basestring('str') == 'str' + assert library.python.strings.to_basestring(u'юникод') == u'юникод' + if six.PY2: # __str__ should return str not bytes in Python3 + assert library.python.strings.to_basestring(Convertible()) == Convertible.text + assert library.python.strings.to_basestring(ConvertibleToUnicodeOnly()) == Convertible.text + assert library.python.strings.to_basestring(ConvertibleToStrOnly()) == Convertible.text_utf8 + assert library.python.strings.to_basestring(NonConvertible()) + + +def test_to_unicode(): + assert library.python.strings.to_unicode(u'юникод') == u'юникод' + assert library.python.strings.to_unicode('str') == u'str' + assert library.python.strings.to_unicode(u'строка'.encode('utf-8')) == u'строка' + assert library.python.strings.to_unicode(u'строка'.encode('cp1251'), 'cp1251') == u'строка' + if six.PY2: # __str__ should return str not bytes in Python3 + assert library.python.strings.to_unicode(Convertible()) == Convertible.text + assert library.python.strings.to_unicode(ConvertibleToUnicodeOnly()) == Convertible.text + with pytest.raises(UnicodeDecodeError): + library.python.strings.to_unicode(ConvertibleToStrOnly()) + with pytest.raises(UnicodeDecodeError): + library.python.strings.to_unicode(NonConvertible()) + + +def test_to_unicode_errors_replace(): + assert library.python.strings.to_unicode(u'abcабв'.encode('utf-8'), 'ascii') + assert library.python.strings.to_unicode(u'абв'.encode('utf-8'), 'ascii') + + +def test_to_str(): + assert library.python.strings.to_str('str') == 'str' if six.PY2 else b'str' + assert library.python.strings.to_str(u'unicode') == 'unicode' if six.PY2 else b'unicode' + assert library.python.strings.to_str(u'юникод') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод', 'cp1251') == u'юникод'.encode('cp1251') + if six.PY2: + assert library.python.strings.to_str(Convertible()) == Convertible.text_utf8 + with pytest.raises(UnicodeEncodeError): + library.python.strings.to_str(ConvertibleToUnicodeOnly()) + assert library.python.strings.to_str(ConvertibleToStrOnly()) == Convertible.text_utf8 + with pytest.raises(UnicodeEncodeError): + library.python.strings.to_str(NonConvertible()) + + +def test_to_str_errors_replace(): + assert library.python.strings.to_str(u'abcабв', 'ascii') + assert library.python.strings.to_str(u'абв', 'ascii') + + +def test_to_str_transcode(): + assert library.python.strings.to_str('str', from_enc='ascii') == 'str' if six.PY2 else b'str' + assert library.python.strings.to_str('str', from_enc='utf-8') == 'str' if six.PY2 else b'str' + + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), from_enc='utf-8') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='utf-8', from_enc='utf-8') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='cp1251', from_enc='utf-8') == u'юникод'.encode('cp1251') + + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), from_enc='cp1251') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='cp1251', from_enc='cp1251') == u'юникод'.encode('cp1251') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='utf-8', from_enc='cp1251') == u'юникод'.encode('utf-8') + + assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), from_enc='koi8-r') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='koi8-r', from_enc='koi8-r') == u'юникод'.encode('koi8-r') + assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='cp1251', from_enc='koi8-r') == u'юникод'.encode('cp1251') + + +def test_to_str_transcode_wrong(): + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), from_enc='cp1251') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), from_enc='utf-8') + + +def test_to_str_transcode_disabled(): + # No transcoding enabled, set from_enc to enable + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='utf-8') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='cp1251') == u'юникод'.encode('utf-8') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='utf-8') == u'юникод'.encode('cp1251') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='cp1251') == u'юникод'.encode('cp1251') + assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='koi8-r') == u'юникод'.encode('cp1251') + assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='cp1251') == u'юникод'.encode('koi8-r') + + +def test_stringize_deep(): + assert library.python.strings.stringize_deep({ + 'key 1': 'value 1', + u'ключ 2': u'значение 2', + 'list': [u'ключ 2', 'key 1', (u'к', 2)] + }) == { + 'key 1' if six.PY2 else b'key 1': 'value 1' if six.PY2 else b'value 1', + u'ключ 2'.encode('utf-8'): u'значение 2'.encode('utf-8'), + 'list' if six.PY2 else b'list': [u'ключ 2'.encode('utf-8'), 'key 1' if six.PY2 else b'key 1', (u'к'.encode('utf-8'), 2)] + } + + +def test_stringize_deep_doesnt_transcode(): + assert library.python.strings.stringize_deep({ + u'ключ 1'.encode('utf-8'): u'значение 1'.encode('utf-8'), + u'ключ 2'.encode('cp1251'): u'значение 2'.encode('cp1251'), + }) == { + u'ключ 1'.encode('utf-8'): u'значение 1'.encode('utf-8'), + u'ключ 2'.encode('cp1251'): u'значение 2'.encode('cp1251'), + } + + +def test_stringize_deep_nested(): + assert library.python.strings.stringize_deep({ + 'key 1': 'value 1', + u'ключ 2': { + 'subkey 1': 'value 1', + u'подключ 2': u'value 2', + }, + }) == { + 'key 1' if six.PY2 else b'key 1': 'value 1' if six.PY2 else b'value 1', + u'ключ 2'.encode('utf-8'): { + 'subkey 1' if six.PY2 else b'subkey 1': 'value 1' if six.PY2 else b'value 1', + u'подключ 2'.encode('utf-8'): u'value 2'.encode('utf-8'), + }, + } + + +def test_stringize_deep_plain(): + assert library.python.strings.stringize_deep('str') == 'str' if six.PY2 else b'str' + assert library.python.strings.stringize_deep(u'юникод') == u'юникод'.encode('utf-8') + assert library.python.strings.stringize_deep(u'юникод'.encode('utf-8')) == u'юникод'.encode('utf-8') + + +def test_stringize_deep_nonstr(): + with pytest.raises(TypeError): + library.python.strings.stringize_deep(Convertible(), relaxed=False) + x = Convertible() + assert x == library.python.strings.stringize_deep(x) + + +def test_unicodize_deep(): + assert library.python.strings.unicodize_deep({ + 'key 1': 'value 1', + u'ключ 2': u'значение 2', + u'ключ 3'.encode('utf-8'): u'значение 3'.encode('utf-8'), + }) == { + u'key 1': u'value 1', + u'ключ 2': u'значение 2', + u'ключ 3': u'значение 3', + } + + +def test_unicodize_deep_nested(): + assert library.python.strings.unicodize_deep({ + 'key 1': 'value 1', + u'ключ 2': { + 'subkey 1': 'value 1', + u'подключ 2': u'значение 2', + u'подключ 3'.encode('utf-8'): u'значение 3'.encode('utf-8'), + }, + }) == { + u'key 1': u'value 1', + u'ключ 2': { + u'subkey 1': u'value 1', + u'подключ 2': u'значение 2', + u'подключ 3': u'значение 3', + }, + } + + +def test_unicodize_deep_plain(): + assert library.python.strings.unicodize_deep('str') == u'str' + assert library.python.strings.unicodize_deep(u'юникод') == u'юникод' + assert library.python.strings.unicodize_deep(u'юникод'.encode('utf-8')) == u'юникод' + + +def test_unicodize_deep_nonstr(): + with pytest.raises(TypeError): + library.python.strings.unicodize_deep(Convertible(), relaxed=False) + x = Convertible() + assert x == library.python.strings.stringize_deep(x) diff --git a/library/python/strings/ut/ya.make b/library/python/strings/ut/ya.make new file mode 100644 index 0000000000..dfacb226c7 --- /dev/null +++ b/library/python/strings/ut/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY23_TEST() + +TEST_SRCS(test_strings.py) + +PEERDIR( + library/python/strings +) + +END() diff --git a/library/python/strings/ya.make b/library/python/strings/ya.make new file mode 100644 index 0000000000..7e0b033717 --- /dev/null +++ b/library/python/strings/ya.make @@ -0,0 +1,16 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +PY_SRCS( + __init__.py + CYTHONIZE_PY + strings.py +) + +PEERDIR( + library/python/func + contrib/python/six +) + +END() diff --git a/library/python/svn_version/__init__.py b/library/python/svn_version/__init__.py new file mode 100644 index 0000000000..7ef7d6dac7 --- /dev/null +++ b/library/python/svn_version/__init__.py @@ -0,0 +1 @@ +from .__svn_version import svn_version, commit_id, svn_revision, svn_last_revision, hash, svn_branch, svn_tag, patch_number # noqa diff --git a/library/python/svn_version/__svn_version.pyx b/library/python/svn_version/__svn_version.pyx new file mode 100644 index 0000000000..d66bc09d24 --- /dev/null +++ b/library/python/svn_version/__svn_version.pyx @@ -0,0 +1,35 @@ +import future.utils as fu + +cdef extern from "library/cpp/svnversion/svnversion.h": + cdef const char* GetProgramSvnVersion() except +; + cdef int GetProgramSvnRevision() except +; + cdef int GetArcadiaLastChangeNum() except +; + cdef const char* GetProgramCommitId() except +; + cdef const char* GetProgramHash() except +; + cdef const char* GetBranch() except +; + cdef const char* GetTag() except +; + cdef int GetArcadiaPatchNumber() except +; + +def svn_version(): + return fu.bytes_to_native_str(GetProgramSvnVersion()) + +def svn_revision(): + return GetProgramSvnRevision() + +def svn_last_revision(): + return GetArcadiaLastChangeNum() + +def commit_id(): + return fu.bytes_to_native_str(GetProgramCommitId()) + +def hash(): + return fu.bytes_to_native_str(GetProgramHash()) + +def svn_branch(): + return fu.bytes_to_native_str(GetBranch()) + +def svn_tag(): + return fu.bytes_to_native_str(GetTag()) + +def patch_number(): + return GetArcadiaPatchNumber() diff --git a/library/python/svn_version/ut/lib/test_simple.py b/library/python/svn_version/ut/lib/test_simple.py new file mode 100644 index 0000000000..2c27af6c68 --- /dev/null +++ b/library/python/svn_version/ut/lib/test_simple.py @@ -0,0 +1,16 @@ +import library.python.svn_version as sv + + +def test_simple(): + assert sv.svn_version() + assert isinstance(sv.svn_version(), str) + assert sv.svn_revision() + assert isinstance(sv.svn_revision(), int) + assert sv.svn_last_revision() + assert isinstance(sv.svn_last_revision(), int) + assert sv.svn_last_revision() > 0 + assert sv.commit_id() + assert isinstance(sv.commit_id(), str) + assert len(sv.commit_id()) > 0 + assert isinstance(sv.hash(), str) + assert isinstance(sv.patch_number(), int) diff --git a/library/python/svn_version/ut/lib/ya.make b/library/python/svn_version/ut/lib/ya.make new file mode 100644 index 0000000000..fb50caf125 --- /dev/null +++ b/library/python/svn_version/ut/lib/ya.make @@ -0,0 +1,13 @@ +PY23_LIBRARY() + +OWNER(pg) + +PEERDIR( + library/python/svn_version +) + +TEST_SRCS( + test_simple.py +) + +END() diff --git a/library/python/svn_version/ut/py2/ya.make b/library/python/svn_version/ut/py2/ya.make new file mode 100644 index 0000000000..c860e16536 --- /dev/null +++ b/library/python/svn_version/ut/py2/ya.make @@ -0,0 +1,9 @@ +PY2TEST() + +OWNER(pg) + +PEERDIR( + library/python/svn_version/ut/lib +) + +END() diff --git a/library/python/svn_version/ut/py3/ya.make b/library/python/svn_version/ut/py3/ya.make new file mode 100644 index 0000000000..4231565142 --- /dev/null +++ b/library/python/svn_version/ut/py3/ya.make @@ -0,0 +1,9 @@ +PY3TEST() + +OWNER(pg) + +PEERDIR( + library/python/svn_version/ut/lib +) + +END() diff --git a/library/python/svn_version/ut/ya.make b/library/python/svn_version/ut/ya.make new file mode 100644 index 0000000000..1363b76581 --- /dev/null +++ b/library/python/svn_version/ut/ya.make @@ -0,0 +1,5 @@ +RECURSE( + lib + py2 + py3 +) diff --git a/library/python/svn_version/ya.make b/library/python/svn_version/ya.make new file mode 100644 index 0000000000..747a663f00 --- /dev/null +++ b/library/python/svn_version/ya.make @@ -0,0 +1,15 @@ +PY23_LIBRARY() + +OWNER(pg) + +PEERDIR( + library/cpp/svnversion + contrib/python/future +) + +PY_SRCS( + __init__.py + __svn_version.pyx +) + +END() diff --git a/library/python/symbols/libc/syms.cpp b/library/python/symbols/libc/syms.cpp new file mode 100644 index 0000000000..6c04a7ef6e --- /dev/null +++ b/library/python/symbols/libc/syms.cpp @@ -0,0 +1,157 @@ +#include <util/system/platform.h> + +#include <library/python/symbols/registry/syms.h> + +#if !defined(_MSC_VER) +#if __has_include(<aio.h>) +#include <aio.h> +#endif +#include <arpa/inet.h> +#include <dirent.h> +#include <ifaddrs.h> +#include <netdb.h> +#include <pthread.h> +#include <pwd.h> +#include <sched.h> +#include <semaphore.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <errno.h> +#include <sys/ipc.h> +#include <dlfcn.h> + +#if defined(_linux_) +#include <sys/prctl.h> +#include <sys/ptrace.h> +#include <sys/sendfile.h> +#else +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/uio.h> +#endif + +#if defined(_darwin_) +#include <sys/types.h> +#include <sys/sysctl.h> +#include <mach/mach_error.h> // Y_IGNORE +#include <mach/mach_time.h> // Y_IGNORE +#endif + +#if defined(_linux_) +#include <sys/inotify.h> +#include <sys/mman.h> +#endif + +namespace { + static inline void* ErrnoLocation() { + return &errno; + } + + static int ClockGetres(clockid_t clk_id, struct timespec* res) { +#if defined(_darwin_) + static auto func = (decltype(&ClockGetres))dlsym(RTLD_SELF, "_clock_getres"); + + if (func) { + return func(clk_id, res); + } + + // https://opensource.apple.com/source/Libc/Libc-1158.1.2/gen/clock_gettime.c.auto.html + + switch (clk_id){ + case CLOCK_REALTIME: + case CLOCK_MONOTONIC: + case CLOCK_PROCESS_CPUTIME_ID: + res->tv_nsec = NSEC_PER_USEC; + res->tv_sec = 0; + + return 0; + + case CLOCK_MONOTONIC_RAW: + case CLOCK_MONOTONIC_RAW_APPROX: + case CLOCK_UPTIME_RAW: + case CLOCK_UPTIME_RAW_APPROX: + case CLOCK_THREAD_CPUTIME_ID: { + mach_timebase_info_data_t tb_info; + + if (mach_timebase_info(&tb_info)) { + return -1; + } + + res->tv_nsec = tb_info.numer / tb_info.denom + (tb_info.numer % tb_info.denom != 0); + res->tv_sec = 0; + + return 0; + } + + default: + errno = EINVAL; + return -1; + } +#else + return clock_getres(clk_id, res); +#endif + } +} + +BEGIN_SYMS("c") + +SYM(calloc) +SYM(clock_gettime) +SYM_2("clock_getres", ClockGetres) +SYM(closedir) +SYM(fdopen) +SYM(fflush) +SYM(freeifaddrs) +SYM(ftok) +SYM(getifaddrs) +SYM(getnameinfo) +SYM(getpwnam) +SYM(inet_ntop) +SYM(opendir) +SYM(printf) +SYM(pthread_kill) +SYM(pthread_self) +SYM(readdir_r) +SYM(sem_close) +SYM(sem_getvalue) +SYM(sem_open) +SYM(sem_post) +SYM(sem_trywait) +SYM(sem_unlink) +SYM(sem_wait) +SYM(siginterrupt) +SYM(strdup) +SYM(sendfile) +SYM(strtod) +SYM_2("__errno_location", ErrnoLocation) + +#if defined(_linux_) +SYM(prctl) +SYM(ptrace) +SYM(sched_getaffinity) +SYM(sched_setaffinity) +SYM(sem_timedwait) +SYM(inotify_init) +SYM(inotify_add_watch) +SYM(inotify_rm_watch) +SYM(mlockall) +#endif + +#if defined(_darwin_) +SYM(mach_absolute_time) +SYM(mach_timebase_info) +SYM(sysctlbyname) +#endif + +#if __has_include(<aio.h>) +SYM(aio_error) +SYM(aio_read) +SYM(aio_return) +SYM(aio_suspend) +#endif + +END_SYMS() +#endif diff --git a/library/python/symbols/libc/ya.make b/library/python/symbols/libc/ya.make new file mode 100644 index 0000000000..7b84cbc961 --- /dev/null +++ b/library/python/symbols/libc/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +OWNER(pg orivej) + +PEERDIR( + library/python/symbols/registry +) + +IF (GCC OR CLANG) + CFLAGS( + -Wno-deprecated-declarations # For sem_getvalue. + ) +ENDIF() + +SRCS( + GLOBAL syms.cpp +) + +END() diff --git a/library/python/symbols/module/__init__.py b/library/python/symbols/module/__init__.py new file mode 100644 index 0000000000..0061b9e598 --- /dev/null +++ b/library/python/symbols/module/__init__.py @@ -0,0 +1,48 @@ +import subprocess + +from collections import defaultdict + +from .syms import syms + + +def gen_builtin(): + res = defaultdict(dict) + + for k, v in syms.items(): + mod, sym = k.split('|') + + res[mod][sym] = v + + return res + + +builtin_symbols = gen_builtin() +caps = builtin_symbols['_capability'] + + +def find_library(name, find_next): + subst = { + 'rt': 'c', + 'pthread': 'c', + 'm': 'c', + } + + builtin = builtin_symbols.get(subst.get(name, name)) + + if builtin: + return { + 'name': name, + 'symbols': builtin, + } + + if 'musl' in caps: + return None + + try: + subprocess.Popen.__patched__ + + return None + except Exception: + pass + + return find_next(name) diff --git a/library/python/symbols/module/module.cpp b/library/python/symbols/module/module.cpp new file mode 100644 index 0000000000..92bc7f4d67 --- /dev/null +++ b/library/python/symbols/module/module.cpp @@ -0,0 +1,85 @@ +#include <Python.h> + +#include <library/python/symbols/registry/syms.h> + +#include <util/generic/string.h> + +#define CAP(x) SYM_2(x, x) + +BEGIN_SYMS("_capability") +#if defined(_musl_) +CAP("musl") +#endif +#if defined(_linux_) +CAP("linux") +#endif +#if defined(_darwin_) +CAP("darwin") +#endif +CAP("_sentinel") +END_SYMS() + +#undef CAP + +using namespace NPrivate; + +namespace { + template <class T> + struct TCB: public ICB { + inline TCB(T& t) + : CB(&t) + { + } + + void Apply(const char* mod, const char* name, void* sym) override { + (*CB)(mod, name, sym); + } + + T* CB; + }; + + template <class T> + static inline TCB<T> MakeTCB(T& t) { + return t; + } +} + +static void DictSetStringPtr(PyObject* dict, const char* name, void* value) { + PyObject* p = PyLong_FromVoidPtr(value); + PyDict_SetItemString(dict, name, p); + Py_DECREF(p); +} + +static PyObject* InitSyms(PyObject* m) { + if (!m) + return NULL; + PyObject* d = PyDict_New(); + if (!d) + return NULL; + + auto f = [&](const char* mod, const char* name, void* sym) { + DictSetStringPtr(d, (TString(mod) + "|" + TString(name)).c_str(), sym); + }; + + auto cb = MakeTCB(f); + + ForEachSymbol(cb); + + if (PyObject_SetAttrString(m, "syms", d)) + m = NULL; + Py_DECREF(d); + return m; +} + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, "syms", NULL, -1, NULL, NULL, NULL, NULL, NULL}; + +extern "C" PyObject* PyInit_syms() { + return InitSyms(PyModule_Create(&module)); +} +#else +extern "C" void initsyms() { + InitSyms(Py_InitModule("syms", NULL)); +} +#endif diff --git a/library/python/symbols/module/ya.make b/library/python/symbols/module/ya.make new file mode 100644 index 0000000000..78e30f2547 --- /dev/null +++ b/library/python/symbols/module/ya.make @@ -0,0 +1,23 @@ +PY23_LIBRARY() + +OWNER(pg orivej) + +NO_PYTHON_INCLUDES() + +PEERDIR( + contrib/libs/python/Include +) + +SRCS( + module.cpp +) + +PY_REGISTER( + library.python.symbols.module.syms +) + +PY_SRCS( + __init__.py +) + +END() diff --git a/library/python/symbols/python/syms.cpp b/library/python/symbols/python/syms.cpp new file mode 100644 index 0000000000..9b52574cb1 --- /dev/null +++ b/library/python/symbols/python/syms.cpp @@ -0,0 +1,17 @@ +#define SYM(SYM_NAME) extern "C" void SYM_NAME(); +SYM(PyObject_GetBuffer) +SYM(PyBuffer_Release) +SYM(PyCell_New) +SYM(Py_DecRef) +SYM(Py_IncRef) +#undef SYM + +#include <library/python/symbols/registry/syms.h> + +BEGIN_SYMS("python") +SYM(PyObject_GetBuffer) +SYM(PyBuffer_Release) +SYM(PyCell_New) +SYM(Py_DecRef) +SYM(Py_IncRef) +END_SYMS() diff --git a/library/python/symbols/python/ut/py2/ya.make b/library/python/symbols/python/ut/py2/ya.make new file mode 100644 index 0000000000..214194de57 --- /dev/null +++ b/library/python/symbols/python/ut/py2/ya.make @@ -0,0 +1,9 @@ +PY2TEST() + +OWNER(orivej) + +PEERDIR( + library/python/symbols/python/ut +) + +END() diff --git a/library/python/symbols/python/ut/py3/ya.make b/library/python/symbols/python/ut/py3/ya.make new file mode 100644 index 0000000000..76611c6a19 --- /dev/null +++ b/library/python/symbols/python/ut/py3/ya.make @@ -0,0 +1,9 @@ +PY3TEST() + +OWNER(orivej) + +PEERDIR( + library/python/symbols/python/ut +) + +END() diff --git a/library/python/symbols/python/ut/test_ctypes.py b/library/python/symbols/python/ut/test_ctypes.py new file mode 100644 index 0000000000..253a10d8b3 --- /dev/null +++ b/library/python/symbols/python/ut/test_ctypes.py @@ -0,0 +1,37 @@ +from ctypes import ( + byref, POINTER, c_int, c_char, c_char_p, + c_void_p, py_object, c_ssize_t, pythonapi, Structure +) + +c_ssize_p = POINTER(c_ssize_t) + + +class Py_buffer(Structure): + _fields_ = [ + ('buf', c_void_p), + ('obj', py_object), + ('len', c_ssize_t), + ('itemsize', c_ssize_t), + ('readonly', c_int), + ('ndim', c_int), + ('format', c_char_p), + ('shape', c_ssize_p), + ('strides', c_ssize_p), + ('suboffsets', c_ssize_p), + ('smalltable', c_ssize_t * 2), + ('internal', c_void_p) + ] + + +def get_buffer(obj): + buf = Py_buffer() + pythonapi.PyObject_GetBuffer(py_object(obj), byref(buf), 0) + try: + buffer_type = c_char * buf.len + return buffer_type.from_address(buf.buf) + finally: + pythonapi.PyBuffer_Release(byref(buf)) + + +def test_buffer(): + assert get_buffer(b'test string') diff --git a/library/python/symbols/python/ut/ya.make b/library/python/symbols/python/ut/ya.make new file mode 100644 index 0000000000..2849e01b1e --- /dev/null +++ b/library/python/symbols/python/ut/ya.make @@ -0,0 +1,16 @@ +PY23_LIBRARY() + +OWNER(orivej) + +TEST_SRCS(test_ctypes.py) + +PEERDIR( + library/python/symbols/python +) + +END() + +RECURSE_FOR_TESTS( + py2 + py3 +) diff --git a/library/python/symbols/python/ya.make b/library/python/symbols/python/ya.make new file mode 100644 index 0000000000..6bfd54f8bc --- /dev/null +++ b/library/python/symbols/python/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +OWNER(orivej) + +PEERDIR( + library/python/symbols/registry +) + +SRCS( + GLOBAL syms.cpp +) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/python/symbols/registry/syms.cpp b/library/python/symbols/registry/syms.cpp new file mode 100644 index 0000000000..ae8e98b32e --- /dev/null +++ b/library/python/symbols/registry/syms.cpp @@ -0,0 +1,31 @@ +#include "syms.h" + +#include <util/generic/vector.h> +#include <util/generic/string.h> +#include <util/generic/singleton.h> + +using namespace NPrivate; + +namespace { + struct TSym { + const char* Mod; + const char* Name; + void* Sym; + }; + + struct TSymbols: public TVector<TSym> { + static inline TSymbols* Instance() { + return Singleton<TSymbols>(); + } + }; +} + +void NPrivate::RegisterSymbol(const char* mod, const char* name, void* sym) { + TSymbols::Instance()->push_back(TSym{mod, name, sym}); +} + +void NPrivate::ForEachSymbol(ICB& cb) { + for (auto& x : *TSymbols::Instance()) { + cb.Apply(x.Mod, x.Name, x.Sym); + } +} diff --git a/library/python/symbols/registry/syms.h b/library/python/symbols/registry/syms.h new file mode 100644 index 0000000000..5e34d35298 --- /dev/null +++ b/library/python/symbols/registry/syms.h @@ -0,0 +1,24 @@ +#pragma once + +namespace NPrivate { + struct ICB { + virtual void Apply(const char* mod, const char* name, void* sym) = 0; + }; + + void ForEachSymbol(ICB& cb); + void RegisterSymbol(const char* mod, const char* name, void* sym); +} + +#define BEGIN_SYMS(name) \ + namespace { \ + static struct TRegister { \ + const char* ModuleName = name; \ + inline TRegister() { +#define END_SYMS() \ + } \ + } \ + REGISTRY; \ + } + +#define SYM_2(n, s) ::NPrivate::RegisterSymbol(this->ModuleName, n, (void*)&s); +#define SYM(s) SYM_2(#s, s); diff --git a/library/python/symbols/registry/ya.make b/library/python/symbols/registry/ya.make new file mode 100644 index 0000000000..2737b9fce9 --- /dev/null +++ b/library/python/symbols/registry/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +OWNER(pg orivej) + +SRCS( + syms.cpp +) + +END() diff --git a/library/python/symbols/ya.make b/library/python/symbols/ya.make new file mode 100644 index 0000000000..340a710c48 --- /dev/null +++ b/library/python/symbols/ya.make @@ -0,0 +1,12 @@ +RECURSE( + module + registry + tests + + crypto + libc + libmagic + python + uuid + win_unicode_console +) diff --git a/library/python/testing/__init__.py b/library/python/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/library/python/testing/__init__.py diff --git a/library/python/testing/filter/filter.py b/library/python/testing/filter/filter.py new file mode 100644 index 0000000000..a1642bd052 --- /dev/null +++ b/library/python/testing/filter/filter.py @@ -0,0 +1,57 @@ +# coding: utf-8 +# TODO move devtools/ya/test/filter.py to library/python/testing/filter/filter.py +import re +import fnmatch +import logging + +logger = logging.getLogger(__name__) +TEST_SUBTEST_SEPARATOR = '::' + +PARSE_TAG_RE = re.compile("([+-]?[\w:]*)") + + +class FilterException(Exception): + mute = True + + +def fix_filter(flt): + if TEST_SUBTEST_SEPARATOR not in flt and "*" not in flt: + # user wants to filter by test module name + flt = flt + TEST_SUBTEST_SEPARATOR + "*" + return flt + + +def escape_for_fnmatch(s): + return s.replace("[", "[").replace("]", "]") + + +def make_py_file_filter(filter_names): + if filter_names is not None: + with_star = [] + wo_star = set() + for flt in filter_names: + flt = flt.split(':')[0] + if '*' in flt: + with_star.append(flt.split('*')[0] + '*') + else: + wo_star.add(flt) + + def predicate(filename): + if filter_names is None: + return True + return filename in wo_star or any([fnmatch.fnmatch(escape_for_fnmatch(filename), escape_for_fnmatch(filter_name)) for filter_name in with_star]) + + return predicate + + +def make_name_filter(filter_names): + filter_names = map(fix_filter, filter_names) + filter_full_names = set() + for name in filter_names: + if '*' not in name: + filter_full_names.add(name) + + def predicate(testname): + return testname in filter_full_names or any([fnmatch.fnmatch(escape_for_fnmatch(testname), escape_for_fnmatch(filter_name)) for filter_name in filter_names]) + + return predicate diff --git a/library/python/testing/filter/ya.make b/library/python/testing/filter/ya.make new file mode 100644 index 0000000000..22c485d258 --- /dev/null +++ b/library/python/testing/filter/ya.make @@ -0,0 +1,5 @@ +PY23_LIBRARY() +OWNER(g:yatest) +PY_SRCS(filter.py) + +END() diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py new file mode 100644 index 0000000000..3e3b7234ef --- /dev/null +++ b/library/python/testing/import_test/import_test.py @@ -0,0 +1,124 @@ +from __future__ import print_function + +import os +import re +import sys +import time +import traceback + +import __res +from __res import importer + + +def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): + """ + tests all bundled modules are importable + just add + "PEERDIR(library/python/import_test)" to your CMakeLists.txt and + "from import_test import test_imports" to your python test source file. + """ + str_ = lambda s: s + if not isinstance(b'', str): + str_ = lambda s: s.decode('UTF-8') + + exceptions = list(no_check) + for key, _ in __res.iter_keys(b'py/no_check_imports/'): + exceptions += str_(__res.find(key)).split() + if exceptions: + exceptions.sort() + print('NO_CHECK_IMPORTS', ' '.join(exceptions)) + + patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions] + rx = re.compile('^({})$'.format('|'.join(patterns))) + + failed = [] + import_times = {} + + norm = lambda s: s[:-9] if s.endswith('.__init__') else s + + modules = sys.extra_modules | set(extra) + modules = sorted(modules, key=norm) + if py_main: + modules = [py_main] + modules + + for module in modules: + if module not in extra and (rx.search(module) or skip_func and skip_func(module)): + print('SKIP', module) + continue + + name = module.rsplit('.', 1)[-1] + if name == '__main__' and 'if __name__ ==' not in importer.get_source(module): + print('SKIP', module, '''without "if __name__ == '__main__'" check''') + continue + + def print_backtrace_marked(e): + tb_exc = traceback.format_exception(*e) + for item in tb_exc: + for l in item.splitlines(): + print('FAIL:', l, file=sys.stderr) + + try: + print('TRY', module) + # XXX waiting for py3 to use print(..., flush=True) + sys.stdout.flush() + + s = time.time() + if module == '__main__': + importer.load_module('__main__', '__main__py') + elif module.endswith('.__init__'): + __import__(module[:-len('.__init__')]) + else: + __import__(module) + + delay = time.time() - s + import_times[str(module)] = delay + print('OK ', module, '{:.3f}s'.format(delay)) + + except Exception as e: + print('FAIL:', module, e, file=sys.stderr) + print_backtrace_marked(sys.exc_info()) + failed.append('{}: {}'.format(module, e)) + + except: + e = sys.exc_info() + print('FAIL:', module, e, file=sys.stderr) + print_backtrace_marked(e) + failed.append('{}: {}'.format(module, e)) + raise + + print("Slowest imports:") + for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]: + print(' ', '{:.3f}s'.format(t), m) + + if failed: + raise ImportError('modules not imported:\n' + '\n'.join(failed)) + + +test_imports = check_imports + + +def main(): + skip_names = sys.argv[1:] + + os.environ['Y_PYTHON_IMPORT_TEST'] = '' + + # We should initialize Django before importing any applications + if os.getenv('DJANGO_SETTINGS_MODULE'): + try: + import django + except ImportError: + pass + else: + django.setup() + + py_main = __res.find('PY_MAIN') + + if py_main: + py_main_module = py_main.split(b':', 1)[0].decode('UTF-8') + else: + py_main_module = None + + try: + check_imports(no_check=skip_names, py_main=py_main_module) + except: + sys.exit(1) diff --git a/library/python/testing/import_test/ya.make b/library/python/testing/import_test/ya.make new file mode 100644 index 0000000000..fae36ffe8f --- /dev/null +++ b/library/python/testing/import_test/ya.make @@ -0,0 +1,10 @@ +OWNER( + g:yatest + exprmntr +) + +PY23_LIBRARY() + +PY_SRCS(import_test.py) + +END() diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py new file mode 100644 index 0000000000..5ef9c5c189 --- /dev/null +++ b/library/python/testing/recipe/__init__.py @@ -0,0 +1,102 @@ +from __future__ import print_function + +import os +import sys +import json +import logging +import argparse + +from yatest_lib.ya import Ya + +RECIPE_START_OPTION = "start" +RECIPE_STOP_OPTION = "stop" + +ya = None +collect_cores = None +sanitizer_extra_checks = None + + +def _setup_logging(level=logging.DEBUG): + root_logger = logging.getLogger() + root_logger.setLevel(level) + + log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s' + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter(log_format) + stdout_handler.setFormatter(formatter) + root_logger.addHandler(stdout_handler) + + +def get_options(): + parser = argparse.ArgumentParser() + parser.add_argument("--show-cwd", action="store_true", dest="show_cwd", default=False, help="show recipe cwd") + parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode") + parser.add_argument("--test-stderr", action="store_true", dest="test_stderr", default=False, help="test stderr") + parser.add_argument("--pdb", action="store_true", dest="pdb", default=False, help="run pdb on error") + parser.add_argument("--sanitizer-extra-checks", dest="sanitizer_extra_checks", action="store_true", default=False, help="enables extra checks for tests built with sanitizers") + parser.add_argument("--collect-cores", dest="collect_cores", action="store_true", default=False, help="allows core dump file recovering during test") + + args, opts = parser.parse_known_args() + + global ya, sanitizer_extra_checks, collect_cores + _setup_logging() + + context = { + "test_stderr": args.test_stderr, + } + + ya = Ya(context=context) + + ya._data_root = "" # XXX remove + + sanitizer_extra_checks = args.sanitizer_extra_checks + if sanitizer_extra_checks: + for envvar in ['LSAN_OPTIONS', 'ASAN_OPTIONS']: + if envvar in os.environ: + os.environ.pop(envvar) + if envvar + '_ORIGINAL' in os.environ: + os.environ[envvar] = os.environ[envvar + '_ORIGINAL'] + collect_cores = args.collect_cores + + for recipe_option in RECIPE_START_OPTION, RECIPE_STOP_OPTION: + if recipe_option in opts: + return args, opts[opts.index(recipe_option):] + + +def set_env(key, value): + with open(ya.env_file, "a") as f: + json.dump({key: value}, f) + f.write("\n") + + +def tty(): + if os.isatty(1): + return + + f = open('/dev/tty', 'w+') + fd = f.fileno() + os.dup2(fd, 0) + os.dup2(fd, 1) + os.dup2(fd, 2) + + +def declare_recipe(start, stop): + parsed_args, argv = get_options() + + if parsed_args.show_cwd: + print("Recipe \"{} {}\" working dir is {}".format(sys.argv[0], " ".join(argv), os.getcwd())) + + try: + if argv[0] == RECIPE_START_OPTION: + start(argv[1:]) + elif argv[0] == RECIPE_STOP_OPTION: + stop(argv[1:]) + except Exception: + if parsed_args.pdb: + tty() + import ipdb + ipdb.post_mortem() + else: + raise diff --git a/library/python/testing/recipe/ports.py b/library/python/testing/recipe/ports.py new file mode 100644 index 0000000000..9f7de1e767 --- /dev/null +++ b/library/python/testing/recipe/ports.py @@ -0,0 +1,33 @@ +import os +import sys +import subprocess +import time + +from yatest.common.network import PortManager + + +def __get_port_range(): + port_count = int(sys.argv[1]) + pid_filename = sys.argv[2] + port_manager = PortManager() + start_port = port_manager.get_port_range(None, port_count) + sys.stderr.write(str(start_port) + "\n") + with open(pid_filename, 'w') as afile: + afile.write(str(os.getpid())) + while 1: + time.sleep(1) + + +def get_port_range(port_count=1, pid_filename="recipe_port.pid"): + env = os.environ.copy() + env["Y_PYTHON_ENTRY_POINT"] = "library.python.testing.recipe.ports:__get_port_range" + res = subprocess.Popen([sys.argv[0], str(port_count), pid_filename], env=env, cwd=os.getcwd(), stderr=subprocess.PIPE) + while not os.path.exists(pid_filename): + time.sleep(0.01) + port_start = int(res.stderr.readline()) + return port_start + + +def release_port_range(pid_filename="recipe_port.pid"): + with open(pid_filename, 'r') as afile: + os.kill(int(afile.read()), 9) diff --git a/library/python/testing/recipe/ya.make b/library/python/testing/recipe/ya.make new file mode 100644 index 0000000000..dd323aa245 --- /dev/null +++ b/library/python/testing/recipe/ya.make @@ -0,0 +1,19 @@ +OWNER( + exprmntr + g:yatest +) + +PY23_LIBRARY() + +PY_SRCS( + __init__.py + ports.py +) + +PEERDIR( + contrib/python/ipdb + library/python/testing/yatest_common + library/python/testing/yatest_lib +) + +END() diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make new file mode 100644 index 0000000000..883bc8d7ab --- /dev/null +++ b/library/python/testing/ya.make @@ -0,0 +1,22 @@ +OWNER(g:yatest) + +RECURSE( + behave + deprecated + fake_ya_package + filter + gtest + gtest/test + gtest/test/gtest + pyremock + pytest_runner + pytest_runner/example + pytest_runner/test + recipe + system_info + types_test + yapackage + yapackage/test + yatest_common + yatest_lib +) diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make new file mode 100644 index 0000000000..5662db4c5d --- /dev/null +++ b/library/python/testing/yatest_common/ya.make @@ -0,0 +1,40 @@ +OWNER(g:yatest) + +PY23_LIBRARY() + +OWNER(g:yatest) + +NO_EXTENDED_SOURCE_SEARCH() + +PY_SRCS( + TOP_LEVEL + yatest/__init__.py + yatest/common/__init__.py + yatest/common/benchmark.py + yatest/common/canonical.py + yatest/common/environment.py + yatest/common/errors.py + yatest/common/legacy.py + yatest/common/misc.py + yatest/common/network.py + yatest/common/path.py + yatest/common/process.py + yatest/common/runtime.py + yatest/common/runtime_java.py + yatest/common/tags.py +) + +PEERDIR( + contrib/python/six + library/python/cores + library/python/filelock + library/python/fs +) + +IF (NOT CATBOOST_OPENSOURCE) + PEERDIR( + library/python/coredump_filter + ) +ENDIF() + +END() diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py new file mode 100644 index 0000000000..b846b3317a --- /dev/null +++ b/library/python/testing/yatest_common/yatest/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["common"] + +from . import common diff --git a/library/python/testing/yatest_common/yatest/common/__init__.py b/library/python/testing/yatest_common/yatest/common/__init__.py new file mode 100644 index 0000000000..cf57779e27 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/__init__.py @@ -0,0 +1,8 @@ +from .benchmark import * # noqa +from .canonical import * # noqa +from .errors import * # noqa +from .misc import * # noqa +from .path import * # noqa +from .process import * # noqa +from .runtime import * # noqa +from .tags import * # noqa diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py new file mode 100644 index 0000000000..c3784cbe4c --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/benchmark.py @@ -0,0 +1,22 @@ +import json + +from . import process +from . import runtime + + +def execute_benchmark(path, budget=None, threads=None): + """ + Run benchmark and return values + :param path: path to benchmark binary + :param budget: time budget, sec (supported only by ybenchmark) + :param threads: number of threads to run benchmark (supported only by ybenchmark) + :return: map of benchmark values + """ + benchmark_path = runtime.binary_path(path) + cmd = [benchmark_path, "--benchmark_format=json"] + if budget is not None: + cmd += ["-b", str(budget)] + if threads is not None: + cmd += ["-t", str(threads)] + res = process.execute(cmd) + return json.loads(res.std_out) diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py new file mode 100644 index 0000000000..b6a136d3e9 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/canonical.py @@ -0,0 +1,176 @@ +import os +import logging +import shutil +import tempfile + +import six + +from . import process +from . import runtime +from . import path + +yatest_logger = logging.getLogger("ya.test") + + +def _copy(src, dst, universal_lines=False): + if universal_lines: + with open(dst, "wb") as f: + for line in open(src, "rbU"): + f.write(line) + return + shutil.copy(src, dst) + + +def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None): + """ + Create canonical file that can be returned from a test + :param path: path to the file + :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used + :param local: save file locally, otherwise move to sandbox + :param universal_lines: normalize EOL + :param diff_tool_timeout: timeout for running diff tool + :return: object that can be canonized + """ + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path()) + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + # if the created file is in output_path, we copy it, so that it will be available when the tests finishes + _copy(path, safe_path, universal_lines=universal_lines) + if diff_tool: + if not isinstance(diff_tool, six.string_types): + try: # check if iterable + if not isinstance(diff_tool[0], six.string_types): + raise Exception("Invalid custom diff-tool: not cmd") + except: + raise Exception("Invalid custom diff-tool: not binary path") + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None): + abs_path = os.path.abspath(path) + assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path) + assert os.path.isdir(abs_path), "Path {} is not a directory".format(path) + if diff_file_name and not diff_tool: + raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir") + tempdir = tempfile.mkdtemp() + safe_path = os.path.join(tempdir, os.path.basename(abs_path)) + shutil.copytree(abs_path, safe_path) + return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + +def canonical_execute( + binary, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, + env=None, stdin=None, stderr=None, creationflags=0, + file_name=None, save_locally=False, close_fds=False, + diff_tool=None, diff_file_name=None, diff_tool_timeout=None, +): + """ + Shortcut to execute a binary and canonize its stdout + :param binary: absolute path to the binary + :param args: binary arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags + :param file_name: output file name. if not specified program name will be used + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool_timeout: timeout for running diff tool + :return: object that can be canonized + """ + if type(binary) == list: + command = binary + else: + command = [binary] + command += _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["binary"] + del execute_args["args"] + del execute_args["file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] + del execute_args["diff_tool_timeout"] + if not file_name and stdin: + file_name = os.path.basename(stdin.name) + return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) + + +def canonical_py_execute( + script_path, args=None, check_exit_code=True, + shell=False, timeout=None, cwd=None, env=None, + stdin=None, stderr=None, creationflags=0, + file_name=None, save_locally=False, close_fds=False, + diff_tool=None, diff_file_name=None, diff_tool_timeout=None, +): + """ + Shortcut to execute a python script and canonize its stdout + :param script_path: path to the script arcadia relative + :param args: script arguments + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stderr: command stderr + :param creationflags: command creation flags + :param file_name: output file name. if not specified program name will be used + :param diff_tool: path to custome diff tool + :param diff_file_name: custom diff file name to create when diff is found + :param diff_tool_timeout: timeout for running diff tool + :return: object that can be canonized + """ + command = [runtime.source_path(script_path)] + _prepare_args(args) + if shell: + command = " ".join(command) + execute_args = locals() + del execute_args["script_path"] + del execute_args["args"] + del execute_args["file_name"] + del execute_args["save_locally"] + del execute_args["diff_tool"] + del execute_args["diff_file_name"] + del execute_args["diff_tool_timeout"] + return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout) + + +def _prepare_args(args): + if args is None: + args = [] + if isinstance(args, six.string_types): + args = map(lambda a: a.strip(), args.split()) + return args + + +def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout): + res = excutor(**kwargs) + command = kwargs["command"] + file_name = file_name or process.get_command_name(command) + if file_name.endswith(".exe"): + file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names + out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name)) + err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name)) + + try: + os.makedirs(os.path.dirname(out_file_path)) + except OSError: + pass + + with open(out_file_path, "wb") as out_file: + yatest_logger.debug("Will store file in %s", out_file_path) + out_file.write(res.std_out) + + if res.std_err: + with open(err_file_path, "wb") as err_file: + err_file.write(res.std_err) + + return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py new file mode 100644 index 0000000000..43f48d0958 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/environment.py @@ -0,0 +1,5 @@ +# coding: utf-8 + + +def extend_env_var(env, name, value, sep=":"): + return sep.join(filter(None, [env.get(name), value])) diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py new file mode 100644 index 0000000000..8c038fc381 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/errors.py @@ -0,0 +1,20 @@ +import os +import sys + + +class RestartTestException(Exception): + + def __init__(self, *args, **kwargs): + super(RestartTestException, self).__init__(*args, **kwargs) + sys.stderr.write("##restart-test##\n") + sys.stderr.flush() + os.environ["FORCE_EXIT_TESTSFAILED"] = "1" + + +class InfrastructureException(Exception): + + def __init__(self, *args, **kwargs): + super(InfrastructureException, self).__init__(*args, **kwargs) + sys.stderr.write("##infrastructure-error##\n") + sys.stderr.flush() + os.environ["FORCE_EXIT_TESTSFAILED"] = "1" diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py new file mode 100644 index 0000000000..459972d253 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/legacy.py @@ -0,0 +1,12 @@ +from . import canonical + + +def old_canonical_file(output_file_name, storage_md5): + import yalibrary.svn + yalibrary.svn.run_svn([ + 'export', + 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5, + output_file_name, + "--force" + ]) + return canonical.canonical_file(output_file_name) diff --git a/library/python/testing/yatest_common/yatest/common/misc.py b/library/python/testing/yatest_common/yatest/common/misc.py new file mode 100644 index 0000000000..20d3725ac9 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/misc.py @@ -0,0 +1,19 @@ +import functools + + +def first(it): + for d in it: + if d: + return d + + +def lazy(func): + res = [] + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not res: + res.append(func(*args, **kwargs)) + return res[0] + + return wrapper diff --git a/library/python/testing/yatest_common/yatest/common/network.py b/library/python/testing/yatest_common/yatest/common/network.py new file mode 100644 index 0000000000..37bcb1b8e0 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/network.py @@ -0,0 +1,271 @@ +# coding=utf-8 + +import os +import errno +import socket +import random +import logging +import platform +import threading + +import six + +UI16MAXVAL = (1 << 16) - 1 +logger = logging.getLogger(__name__) + + +class PortManagerException(Exception): + pass + + +class PortManager(object): + """ + See documentation here + + https://wiki.yandex-team.ru/yatool/test/#python-acquire-ports + """ + + def __init__(self, sync_dir=None): + self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH') + if self._sync_dir: + _makedirs(self._sync_dir) + + self._valid_range = get_valid_port_range() + self._valid_port_count = self._count_valid_ports() + self._filelocks = {} + self._lock = threading.Lock() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.release() + + def get_port(self, port=0): + ''' + Gets free TCP port + ''' + return self.get_tcp_port(port) + + def get_tcp_port(self, port=0): + ''' + Gets free TCP port + ''' + return self._get_port(port, socket.SOCK_STREAM) + + def get_udp_port(self, port=0): + ''' + Gets free UDP port + ''' + return self._get_port(port, socket.SOCK_DGRAM) + + def get_tcp_and_udp_port(self, port=0): + ''' + Gets one free port for use in both TCP and UDP protocols + ''' + if port and self._no_random_ports(): + return port + + retries = 20 + while retries > 0: + retries -= 1 + + result_port = self.get_tcp_port() + if not self.is_port_free(result_port, socket.SOCK_DGRAM): + self.release_port(result_port) + # Don't try to _capture_port(), it's already captured in the get_tcp_port() + return result_port + raise Exception('Failed to find port') + + def release_port(self, port): + with self._lock: + self._release_port_no_lock(port) + + def _release_port_no_lock(self, port): + filelock = self._filelocks.pop(port, None) + if filelock: + filelock.release() + + def release(self): + with self._lock: + while self._filelocks: + _, filelock = self._filelocks.popitem() + if filelock: + filelock.release() + + def get_port_range(self, start_port, count, random_start=True): + assert count > 0 + if start_port and self._no_random_ports(): + return start_port + + candidates = [] + + def drop_candidates(): + for port in candidates: + self._release_port_no_lock(port) + candidates[:] = [] + + with self._lock: + for attempts in six.moves.range(128): + for left, right in self._valid_range: + if right - left < count: + continue + + if random_start: + start = random.randint(left, right - ((right - left) // 2)) + else: + start = left + for probe_port in six.moves.range(start, right): + if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM): + candidates.append(probe_port) + else: + drop_candidates() + + if len(candidates) == count: + return candidates[0] + # Can't find required number of ports without gap in the current range + drop_candidates() + + raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format( + start_port, count, self._valid_range, self._filelocks)) + + def _count_valid_ports(self): + res = 0 + for left, right in self._valid_range: + res += right - left + assert res, ('There are no available valid ports', self._valid_range) + return res + + def _get_port(self, port, sock_type): + if port and self._no_random_ports(): + return port + + if len(self._filelocks) >= self._valid_port_count: + raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks)) + + salt = random.randint(0, UI16MAXVAL) + for attempt in six.moves.range(self._valid_port_count): + probe_port = (salt + attempt) % self._valid_port_count + + for left, right in self._valid_range: + if probe_port >= (right - left): + probe_port -= right - left + else: + probe_port += left + break + if not self._capture_port(probe_port, sock_type): + continue + return probe_port + + raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks)) + + def _capture_port(self, port, sock_type): + with self._lock: + return self._capture_port_no_lock(port, sock_type) + + def is_port_free(self, port, sock_type=socket.SOCK_STREAM): + sock = socket.socket(socket.AF_INET6, sock_type) + try: + sock.bind(('::', port)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + return False + raise + finally: + sock.close() + return True + + def _capture_port_no_lock(self, port, sock_type): + if port in self._filelocks: + return False + + filelock = None + if self._sync_dir: + # yatest.common should try to be hermetic and don't have peerdirs + # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work + import library.python.filelock + + filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port))) + if not filelock.acquire(blocking=False): + return False + if self.is_port_free(port, sock_type): + self._filelocks[port] = filelock + return True + else: + filelock.release() + return False + + if self.is_port_free(port, sock_type): + self._filelocks[port] = filelock + return True + if filelock: + filelock.release() + return False + + def _no_random_ports(self): + return os.environ.get("NO_RANDOM_PORTS") + + +def get_valid_port_range(): + first_valid = 1025 + last_valid = UI16MAXVAL + + given_range = os.environ.get('VALID_PORT_RANGE') + if given_range and ':' in given_range: + return [list(int(x) for x in given_range.split(':', 2))] + + first_eph, last_eph = get_ephemeral_range() + first_invalid = max(first_eph, first_valid) + last_invalid = min(last_eph, last_valid) + + ranges = [] + if first_invalid > first_valid: + ranges.append((first_valid, first_invalid - 1)) + if last_invalid < last_valid: + ranges.append((last_invalid + 1, last_valid)) + return ranges + + +def get_ephemeral_range(): + if platform.system() == 'Linux': + filename = "/proc/sys/net/ipv4/ip_local_port_range" + if os.path.exists(filename): + with open(filename) as afile: + data = afile.read(1024) # fix for musl + port_range = tuple(map(int, data.strip().split())) + if len(port_range) == 2: + return port_range + else: + logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data) + elif platform.system() == 'Darwin': + first = _sysctlbyname_uint("net.inet.ip.portrange.first") + last = _sysctlbyname_uint("net.inet.ip.portrange.last") + if first and last: + return first, last + # IANA suggestion + return (1 << 15) + (1 << 14), UI16MAXVAL + + +def _sysctlbyname_uint(name): + try: + from ctypes import CDLL, c_uint, byref + from ctypes.util import find_library + except ImportError: + return + + libc = CDLL(find_library("c")) + size = c_uint(0) + res = c_uint(0) + libc.sysctlbyname(name, None, byref(size), None, 0) + libc.sysctlbyname(name, byref(res), byref(size), None, 0) + return res.value + + +def _makedirs(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST: + return + raise diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py new file mode 100644 index 0000000000..6fed7dda8a --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/path.py @@ -0,0 +1,90 @@ +# coding=utf-8 + +import errno +import os +import shutil +import contextlib + +import library.python.fs as lpf + + +def replace_in_file(path, old, new): + """ + Replace text occurrences in a file + :param path: path to the file + :param old: text to replace + :param new: replacement + """ + with open(path) as fp: + content = fp.read() + + lpf.ensure_removed(path) + with open(path, 'w') as fp: + fp.write(content.replace(old, new)) + + +@contextlib.contextmanager +def change_dir(path): + old = os.getcwd() + try: + os.chdir(path) + yield path + finally: + os.chdir(old) + + +def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): + ''' + Copy an entire directory of files into an existing directory + instead of raising Exception what shtuil.copytree does + ''' + if not os.path.exists(dst) and os.path.isdir(src): + os.makedirs(dst) + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + if postprocessing: + postprocessing(dst, False) + for root, dirs, files in os.walk(dst): + for path in dirs: + postprocessing(os.path.join(root, path), False) + for path in files: + postprocessing(os.path.join(root, path), True) + + +def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000): + def atomic_file_create(path): + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL, 0o644) + os.close(fd) + return True + except OSError as e: + if e.errno in [errno.EEXIST, errno.EISDIR, errno.ETXTBSY]: + return False + # Access issue with file itself, not parent directory. + if e.errno == errno.EACCES and os.path.exists(path): + return False + raise e + + def atomic_dir_create(path): + try: + os.mkdir(path) + return True + except OSError as e: + if e.errno == errno.EEXIST: + return False + raise e + + file_path = os.path.join(dir_path, file_pattern) + lpf.ensure_dir(os.path.dirname(file_path)) + file_counter = 0 + handler = atomic_file_create if create_file else atomic_dir_create + while os.path.exists(file_path) or not handler(file_path): + file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter)) + file_counter += 1 + assert file_counter < max_suffix + return file_path diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py new file mode 100644 index 0000000000..a8bcc21f51 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -0,0 +1,733 @@ +# coding: utf-8 + +import os +import re +import time +import signal +import shutil +import logging +import tempfile +import subprocess +import errno +import distutils.version + +import six + +try: + # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work. + import library.python.cores as cores +except ImportError: + cores = None + +from . import runtime +from . import path +from . import environment + + +MAX_OUT_LEN = 1000 * 1000 # 1 mb +MAX_MESSAGE_LEN = 1500 +SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)" +GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)") +yatest_logger = logging.getLogger("ya.test") + + +def truncate(s, size): + if s is None: + return None + elif len(s) <= size: + return s + else: + return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):] + + +def get_command_name(command): + return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0]) + + +class ExecutionError(Exception): + + def __init__(self, execution_result): + if not isinstance(execution_result.command, six.string_types): + command = " ".join(str(arg) for arg in execution_result.command) + else: + command = execution_result.command + message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format( + command=command, + code=execution_result.exit_code, + err=_format_error(execution_result.std_err)) + if cores: + if execution_result.backtrace: + message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace)) + else: + message += "Backtrace is not available: module cores isn't available" + + super(ExecutionError, self).__init__(message) + self.execution_result = execution_result + + +class TimeoutError(Exception): + pass + + +class ExecutionTimeoutError(TimeoutError): + def __init__(self, execution_result, *args, **kwargs): + super(ExecutionTimeoutError, self).__init__(args, kwargs) + self.execution_result = execution_result + + +class InvalidExecutionStateError(Exception): + pass + + +class SignalInterruptionError(Exception): + def __init__(self, message=None): + super(SignalInterruptionError, self).__init__(message) + self.res = None + + +class InvalidCommandError(Exception): + pass + + +class _Execution(object): + + def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False): + self._command = command + self._process = process + self._out_file = out_file + self._err_file = err_file + self._std_out = None + self._std_err = None + self._elapsed = None + self._start = time.time() + self._process_progress_listener = process_progress_listener + self._cwd = cwd or os.getcwd() + self._collect_cores = collect_cores + self._backtrace = '' + self._check_sanitizer = check_sanitizer + self._metrics = {} + self._started = started + self._user_stdout = bool(user_stdout) + self._user_stderr = bool(user_stderr) + self._exit_code = None + if process_progress_listener: + process_progress_listener.open(command, process, out_file, err_file) + + @property + def running(self): + return self._process.poll() is None + + def kill(self): + if self.running: + self._save_outputs(False) + _kill_process_tree(self._process.pid) + self._clean_files() + # DEVTOOLS-2347 + yatest_logger.debug("Process status before wait_for: %s", self.running) + try: + wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1) + except TimeoutError: + yatest_logger.debug("Process status after wait_for: %s", self.running) + yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid])) + raise + else: + raise InvalidExecutionStateError("Cannot kill a stopped process") + + def terminate(self): + if self.running: + self._process.terminate() + + @property + def process(self): + return self._process + + @property + def command(self): + return self._command + + @property + def returncode(self): + return self.exit_code + + @property + def exit_code(self): + """ + Deprecated, use returncode + """ + if self._exit_code is None: + self._exit_code = self._process.returncode + return self._exit_code + + @property + def stdout(self): + return self.std_out + + @property + def std_out(self): + """ + Deprecated, use stdout + """ + if self._std_out is not None: + return self._std_out + if self._process.stdout and not self._user_stdout: + self._std_out = self._process.stdout.read() + return self._std_out + + @property + def stderr(self): + return self.std_err + + @property + def std_err(self): + """ + Deprecated, use stderr + """ + if self._std_err is not None: + return self._std_err + if self._process.stderr and not self._user_stderr: + self._std_err = self._process.stderr.read() + return self._std_err + + @property + def elapsed(self): + return self._elapsed + + @property + def backtrace(self): + return self._backtrace + + @property + def metrics(self): + return self._metrics + + def _save_outputs(self, clean_files=True): + if self._process_progress_listener: + self._process_progress_listener() + self._process_progress_listener.close() + if not self._user_stdout: + if self._out_file is None: + pass + elif self._out_file != subprocess.PIPE: + self._out_file.flush() + self._out_file.seek(0, os.SEEK_SET) + self._std_out = self._out_file.read() + else: + self._std_out = self._process.stdout.read() + if not self._user_stderr: + if self._err_file is None: + pass + elif self._err_file != subprocess.PIPE: + self._err_file.flush() + self._err_file.seek(0, os.SEEK_SET) + self._std_err = self._err_file.read() + else: + self._std_err = self._process.stderr.read() + + if clean_files: + self._clean_files() + yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code) + yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed) + if self._metrics: + for key, value in six.iteritems(self._metrics): + yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value) + + # Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str. + printable_std_out, err = _try_convert_bytes_to_string(self._std_out) + if err: + yatest_logger.debug("Got error during parse process stdout: %s", err) + yatest_logger.debug("stdout will be displayed as raw bytes.") + printable_std_err, err = _try_convert_bytes_to_string(self._std_err) + if err: + yatest_logger.debug("Got error during parse process stderr: %s", err) + yatest_logger.debug("stderr will be displayed as raw bytes.") + + yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN)) + yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN)) + + def _clean_files(self): + if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE: + self._err_file.close() + self._err_file = None + if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE: + self._out_file.close() + self._out_file = None + + def _recover_core(self): + core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid) + if core_path: + # Core dump file recovering may be disabled (for distbuild for example) - produce only bt + store_cores = runtime._get_ya_config().collect_cores + if store_cores: + new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid)) + # Copy core dump file, because it may be overwritten + yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path) + shutil.copyfile(core_path, new_core_path) + core_path = new_core_path + + bt_filename = None + pbt_filename = None + + if os.path.exists(runtime.gdb_path()): + self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path()) + bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid)) + with open(bt_filename, "wb") as afile: + afile.write(six.ensure_binary(self._backtrace)) + # generate pretty html version of backtrace aka Tri Korochki + pbt_filename = bt_filename + ".html" + backtrace_to_html(bt_filename, pbt_filename) + + if store_cores: + runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename) + else: + runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename) + + def wait(self, check_exit_code=True, timeout=None, on_timeout=None): + def _wait(): + finished = None + interrupted = False + try: + if hasattr(os, "wait4"): + try: + if hasattr(subprocess, "_eintr_retry_call"): + pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0) + else: + # PEP 475 + pid, sts, rusage = os.wait4(self._process.pid, 0) + finished = time.time() + self._process._handle_exitstatus(sts) + for field in [ + "ru_idrss", + "ru_inblock", + "ru_isrss", + "ru_ixrss", + "ru_majflt", + "ru_maxrss", + "ru_minflt", + "ru_msgrcv", + "ru_msgsnd", + "ru_nivcsw", + "ru_nsignals", + "ru_nswap", + "ru_nvcsw", + "ru_oublock", + "ru_stime", + "ru_utime", + ]: + if hasattr(rusage, field): + self._metrics[field.replace("ru_", "")] = getattr(rusage, field) + except OSError as exc: + + if exc.errno == errno.ECHILD: + yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") + else: + raise + except SignalInterruptionError: + interrupted = True + raise + finally: + if not interrupted: + self._process.wait() # this has to be here unconditionally, so that all process properties are set + + if not finished: + finished = time.time() + self._metrics["wtime"] = round(finished - self._started, 3) + + try: + if timeout: + process_is_finished = lambda: not self.running + fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout) + try: + wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener) + except TimeoutError as e: + if on_timeout: + yatest_logger.debug("Calling user specified on_timeout function") + try: + on_timeout(self, timeout) + except Exception: + yatest_logger.exception("Exception while calling on_timeout") + raise ExecutionTimeoutError(self, str(e)) + # Wait should be always called here, it finalizes internal states of its process and sets up return code + _wait() + except BaseException as e: + _kill_process_tree(self._process.pid) + _wait() + yatest_logger.debug("Command exception: %s", e) + raise + finally: + self._elapsed = time.time() - self._start + self._save_outputs() + self.verify_no_coredumps() + + self._finalise(check_exit_code) + + def _finalise(self, check_exit_code): + # Set the signal (negative number) which caused the process to exit + if check_exit_code and self.exit_code != 0: + yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n", + self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) + + # Don't search for sanitize errors if stderr was redirected + self.verify_sanitize_errors() + + def verify_no_coredumps(self): + """ + Verify there is no coredump from this binary. If there is then report backtrace. + """ + if self.exit_code < 0 and self._collect_cores: + if cores: + try: + self._recover_core() + except Exception: + yatest_logger.exception("Exception while recovering core") + else: + yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available") + + def verify_sanitize_errors(self): + """ + Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them. + """ + if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks: + build_path = runtime.build_path() + if self.command[0].startswith(build_path): + match = re.search(SANITIZER_ERROR_PATTERN, self._std_err) + if match: + yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN)) + raise ExecutionError(self) + else: + yatest_logger.debug("No sanitizer errors found") + else: + yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path) + + +def on_timeout_gen_coredump(exec_obj, _): + """ + Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump) + to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout. + All files will be available in the testing_out_stuff and via links. + """ + try: + os.kill(exec_obj.process.pid, signal.SIGQUIT) + except OSError: + # process might be already terminated + pass + + +def execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False, + collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, + executor=_Execution, +): + """ + Executes a command + :param command: command: can be a list of arguments or a string + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress + :param close_fds: subrpocess.Popen close_fds args + :param collect_cores: recover core dump files if shell == False + :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors + :param preexec_fn: subrpocess.Popen preexec_fn arg + :param on_timeout: on_timeout(<execution object>, <timeout value>) callback + + :return _Execution: Execution object + """ + if env is None: + env = os.environ.copy() + else: + # Certain environment variables must be present for programs to work properly. + # For more info see DEVTOOLSSUPPORT-4907 + mandatory_env_name = 'YA_MANDATORY_ENV_VARS' + mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or '' + if mandatory_vars: + env[mandatory_env_name] = mandatory_vars + mandatory_system_vars = filter(None, mandatory_vars.split(':')) + else: + mandatory_system_vars = ['TMPDIR'] + + for var in mandatory_system_vars: + if var not in env and var in os.environ: + env[var] = os.environ[var] + + if not wait and timeout is not None: + raise ValueError("Incompatible arguments 'timeout' and wait=False") + + # if subprocess.PIPE in [stdout, stderr]: + # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock") + + def get_out_stream(stream, default_name): + if stream is None: + # No stream is supplied: open new temp file + return _get_command_output_file(command, default_name), False + + if isinstance(stream, six.string_types): + # User filename is supplied: open file for writing + return open(stream, 'wb+'), stream.startswith('/dev/') + + # Open file or PIPE sentinel is supplied + is_pipe = stream == subprocess.PIPE + return stream, not is_pipe + + # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE, + # as it can cause processes hangs, but use tempfiles instead + out_file, user_stdout = get_out_stream(stdout, 'out') + err_file, user_stderr = get_out_stream(stderr, 'err') + in_file = stdin + + if shell and type(command) == list: + command = " ".join(command) + + if shell: + collect_cores = False + check_sanitizer = False + else: + if isinstance(command, (list, tuple)): + executable = command[0] + else: + executable = command + if os.path.isabs(executable): + if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"): + exists = os.path.exists(executable) + if exists: + stat = os.stat(executable) + else: + stat = None + raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat)) + if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK): + raise InvalidCommandError("Target program is not executable: {}".format(executable)) + + if check_sanitizer: + env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100") + + if stdin: + name = "PIPE" if stdin == subprocess.PIPE else stdin.name + yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd) + else: + yatest_logger.debug("Executing '%s' in '%s'", command, cwd) + # XXX + + started = time.time() + process = subprocess.Popen( + command, shell=shell, universal_newlines=True, + stdout=out_file, stderr=err_file, stdin=in_file, + cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn, + ) + yatest_logger.debug("Command pid: %s", process.pid) + + res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr) + if wait: + res.wait(check_exit_code, timeout, on_timeout) + return res + + +def _get_command_output_file(cmd, ext): + parts = [get_command_name(cmd)] + if 'YA_RETRY_INDEX' in os.environ: + parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX'))) + if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1: + parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0'))) + + filename = '.'.join(parts + [ext]) + try: + # if execution is performed from test, save out / err to the test logs dir + import yatest.common + import library.python.pytest.plugins.ya + if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None: + raise ImportError("not in test") + filename = path.get_unique_file_path(yatest.common.output_path(), filename) + yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename)) + return open(filename, "wb+") + except ImportError: + return tempfile.NamedTemporaryFile(delete=False, suffix=filename) + + +def _get_proc_tree_info(pids): + if os.name == 'nt': + return 'Not supported' + else: + stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return stdout + + +def py_execute( + command, check_exit_code=True, + shell=False, timeout=None, + cwd=None, env=None, + stdin=None, stdout=None, stderr=None, + creationflags=0, wait=True, + process_progress_listener=None, close_fds=False +): + """ + Executes a command with the arcadia python + :param command: command to pass to python + :param check_exit_code: will raise ExecutionError if the command exits with non zero code + :param shell: use shell to run the command + :param timeout: execution timeout + :param cwd: working directory + :param env: command environment + :param stdin: command stdin + :param stdout: command stdout + :param stderr: command stderr + :param creationflags: command creation flags + :param wait: should wait until the command finishes + :param process_progress_listener=object that is polled while execution is in progress + :return _Execution: Execution object + """ + if isinstance(command, six.string_types): + command = [command] + command = [runtime.python_path()] + command + if shell: + command = " ".join(command) + return execute(**locals()) + + +def _format_error(error): + return truncate(error, MAX_MESSAGE_LEN) + + +def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None): + """ + Tries to execute `check_function` for `timeout` seconds. + Continue until function returns nonfalse value. + If function doesn't return nonfalse value for `timeout` seconds + OperationTimeoutException is raised. + Return first nonfalse result returned by `checkFunction`. + """ + if sleep_time <= 0: + raise ValueError("Incorrect sleep time value {}".format(sleep_time)) + if timeout < 0: + raise ValueError("Incorrect timeout value {}".format(timeout)) + start = time.time() + while start + timeout > time.time(): + if on_check_condition: + on_check_condition() + + res = check_function() + if res: + return res + time.sleep(sleep_time) + + message = "{} second(s) wait timeout has expired".format(timeout) + if fail_message: + message += ": {}".format(fail_message) + raise TimeoutError(truncate(message, MAX_MESSAGE_LEN)) + + +def _kill_process_tree(process_pid, target_pid_signal=None): + """ + Kills child processes, req. Note that psutil should be installed + @param process_pid: parent id to search for descendants + """ + yatest_logger.debug("Killing process %s", process_pid) + if os.name == 'nt': + _win_kill_process_tree(process_pid) + else: + _nix_kill_process_tree(process_pid, target_pid_signal) + + +def _nix_get_proc_children(pid): + try: + cmd = ["pgrep", "-P", str(pid)] + return [int(p) for p in subprocess.check_output(cmd).split()] + except Exception: + return [] + + +def _get_binname(pid): + try: + return os.path.basename(os.readlink('/proc/{}/exe'.format(pid))) + except Exception as e: + return "error({})".format(e) + + +def _nix_kill_process_tree(pid, target_pid_signal=None): + """ + Kills the process tree. + """ + yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid))) + + def try_to_send_signal(pid, sig): + try: + os.kill(pid, sig) + yatest_logger.debug("Sent signal %d to the pid %d", sig, pid) + except Exception as exc: + yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc))) + + try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes. + + # Get the child process PID list. + child_pids = _nix_get_proc_children(pid) + # Stop the child processes. + for child_pid in child_pids: + try: + # Kill the child recursively. + _kill_process_tree(int(child_pid)) + except Exception as e: + # Skip the error and continue killing. + yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e)) + continue + + try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process. + + # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent + # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html + try_to_send_signal(pid, signal.SIGCONT) + + +def _win_kill_process_tree(pid): + subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)]) + + +def _run_readelf(binary_path): + return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)])) + + +def check_glibc_version(binary_path): + lucid_glibc_version = distutils.version.LooseVersion("2.11") + + for l in _run_readelf(binary_path).split('\n'): + match = GLIBC_PATTERN.search(l) + if not match: + continue + assert distutils.version.LooseVersion(match.group(1)) <= lucid_glibc_version, match.group(0) + + +def backtrace_to_html(bt_filename, output): + try: + from library.python import coredump_filter + with open(output, "wb") as afile: + coredump_filter.filter_stackdump(bt_filename, stream=afile) + except ImportError as e: + yatest_logger.debug("Failed to import coredump_filter: %s", e) + with open(output, "wb") as afile: + afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>") + + +def _try_convert_bytes_to_string(source): + """ Function is necessary while this code Python2/3 compatible, because bytes in Python3 is a real bytes and in Python2 is not """ + # Bit ugly typecheck, because in Python2 isinstance(str(), bytes) and "type(str()) is bytes" working as True as well + if 'bytes' not in str(type(source)): + # We already got not bytes. Nothing to do here. + return source, False + + result = source + error = False + try: + result = source.decode(encoding='utf-8') + except ValueError as e: + error = e + + return result, error diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py new file mode 100644 index 0000000000..e55e193446 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -0,0 +1,343 @@ +import errno +import functools +import json +import os +import threading + +import six + + +_lock = threading.Lock() + + +def _get_ya_config(): + try: + import library.python.pytest.plugins.ya as ya_plugin + if ya_plugin.pytest_config is not None: + return ya_plugin.pytest_config + import pytest + return pytest.config + except (ImportError, AttributeError): + try: + import library.python.testing.recipe + if library.python.testing.recipe.ya: + return library.python.testing.recipe + except (ImportError, AttributeError): + pass + raise NotImplementedError("yatest.common.* is only available from the testing runtime") + + +def _get_ya_plugin_instance(): + return _get_ya_config().ya + + +def _norm_path(path): + if path is None: + return None + assert isinstance(path, six.string_types) + if "\\" in path: + raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path)) + return os.path.normpath(path) + + +def _join_path(main_path, path): + if not path: + return main_path + return os.path.join(main_path, _norm_path(path)) + + +def not_test(func): + """ + Mark any function as not a test for py.test + :param func: + :return: + """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return func(*args, **kwds) + setattr(wrapper, '__test__', False) + return wrapper + + +def source_path(path=None): + """ + Get source path inside arcadia + :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya') + :return: absolute path to the source folder + """ + return _join_path(_get_ya_plugin_instance().source_root, path) + + +def build_path(path=None): + """ + Get path inside build directory + :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin') + :return: absolute path inside build directory + """ + return _join_path(_get_ya_plugin_instance().build_root, path) + + +def java_path(): + """ + [DEPRECATED] Get path to java + :return: absolute path to java + """ + from . import runtime_java + return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk'))) + + +def java_home(): + """ + Get jdk directory path + """ + from . import runtime_java + jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk')) + if not jdk_dir: + raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform") + return jdk_dir + + +def java_bin(): + """ + Get path to the java binary + Requires DEPENDS(jdk) + """ + return os.path.join(java_home(), "bin", "java") + + +def data_path(path=None): + """ + Get path inside arcadia_tests_data directory + :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") + :return: absolute path inside arcadia_tests_data + """ + return _join_path(_get_ya_plugin_instance().data_root, path) + + +def output_path(path=None): + """ + Get path inside the current test suite output dir. + Placing files to this dir guarantees that files will be accessible after the test suite execution. + :param path: path relative to the test suite output dir + :return: absolute path inside the test suite output dir + """ + return _join_path(_get_ya_plugin_instance().output_dir, path) + + +def ram_drive_path(path=None): + """ + :param path: path relative to the ram drive. + :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment. + """ + if 'YA_TEST_RAM_DRIVE_PATH' in os.environ: + return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path) + + +def output_ram_drive_path(path=None): + """ + Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing. + Returns None if no ram drive was provided by environment. + :param path: path relative to the output ram drive directory + """ + if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ: + return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path) + + +def binary_path(path=None): + """ + Get path to the built binary + :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin') + :return: absolute path to the binary + """ + path = _norm_path(path) + return _get_ya_plugin_instance().get_binary(path) + + +def work_path(path=None): + """ + Get path inside the current test suite working directory. Creating files in the work directory does not guarantee + that files will be accessible after the test suite execution + :param path: path relative to the test suite working dir + :return: absolute path inside the test suite working dir + """ + return _join_path( + os.environ.get("TEST_WORK_PATH") or + _get_ya_plugin_instance().get_context("work_path") or + os.getcwd(), + path) + + +def python_path(): + """ + Get path to the arcadia python. + + Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles + are built in a stripped-down form that is needed for building, not running tests. + See comments in the file below to find out which version of python is compatible with tests. + https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc + :return: absolute path to python + """ + return _get_ya_plugin_instance().python_path + + +def valgrind_path(): + """ + Get path to valgrind + :return: absolute path to valgrind + """ + return _get_ya_plugin_instance().valgrind_path + + +def get_param(key, default=None): + """ + Get arbitrary parameter passed via command line + :param key: key + :param default: default value + :return: parameter value or the default + """ + return _get_ya_plugin_instance().get_param(key, default) + + +def get_param_dict_copy(): + """ + Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters. + + :return: copy of dictionary with all parameters + """ + return _get_ya_plugin_instance().get_param_dict_copy() + + +@not_test +def test_output_path(path=None): + """ + Get dir in the suite output_path for the current test case + """ + test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0] + try: + os.makedirs(test_out_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return _join_path(test_out_dir, path) + + +def project_path(path=None): + """ + Get path in build root relating to build_root/project path + """ + return _join_path(os.path.join(build_path(), context.project_path), path) + + +def gdb_path(): + """ + Get path to the gdb + """ + return _get_ya_plugin_instance().gdb_path + + +def c_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CC") + + +def get_yt_hdd_path(path=None): + if 'HDD_PATH' in os.environ: + return _join_path(os.environ['HDD_PATH'], path) + + +def cxx_compiler_path(): + """ + Get path to the gdb + """ + return os.environ.get("YA_CXX") + + +def global_resources(): + try: + return json.loads(os.environ.get("YA_GLOBAL_RESOURCES")) + except (TypeError, ValueError): + return {} + + +def _register_core(name, binary_path, core_path, bt_path, pbt_path): + config = _get_ya_config() + + with _lock: + if not hasattr(config, 'test_cores_count'): + config.test_cores_count = 0 + config.test_cores_count += 1 + count_str = '' if config.test_cores_count == 1 else str(config.test_cores_count) + + log_entry = config.test_logs[config.current_item_nodeid] + if binary_path: + log_entry['{} binary{}'.format(name, count_str)] = binary_path + if core_path: + log_entry['{} core{}'.format(name, count_str)] = core_path + if bt_path: + log_entry['{} backtrace{}'.format(name, count_str)] = bt_path + if pbt_path: + log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path + + +@not_test +def test_source_path(path=None): + return _join_path(os.path.join(source_path(), context.project_path), path) + + +class Context(object): + """ + Runtime context + """ + + @property + def build_type(self): + return _get_ya_plugin_instance().get_context("build_type") + + @property + def project_path(self): + return _get_ya_plugin_instance().get_context("project_path") + + @property + def test_stderr(self): + return _get_ya_plugin_instance().get_context("test_stderr") + + @property + def test_debug(self): + return _get_ya_plugin_instance().get_context("test_debug") + + @property + def test_traceback(self): + return _get_ya_plugin_instance().get_context("test_traceback") + + @property + def test_name(self): + return _get_ya_config().current_test_name + + @property + def sanitize(self): + """ + Detect if current test run is under sanitizer + + :return: one of `None`, 'address', 'memory', 'thread', 'undefined' + """ + return _get_ya_plugin_instance().get_context("sanitize") + + @property + def flags(self): + _flags = _get_ya_plugin_instance().get_context("flags") + if _flags: + _flags_dict = dict() + for f in _flags: + key, value = f.split('=', 1) + _flags_dict[key] = value + return _flags_dict + else: + return dict() + + def get_context_key(self, key): + return _get_ya_plugin_instance().get_context(key) + + +context = Context() diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py new file mode 100644 index 0000000000..39bbb45570 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py @@ -0,0 +1,46 @@ +import os +import tarfile +import contextlib + +from . import runtime + +_JAVA_DIR = [] + + +def get_java_path(jdk_dir): + # deprecated - to be deleted + java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe')) + + for p in java_paths: + if os.path.exists(p): + return p + + for f in os.listdir(jdk_dir): + if f.endswith('.tar'): + with contextlib.closing(tarfile.open(os.path.join(jdk_dir, f))) as tf: + tf.extractall(jdk_dir) + + for p in java_paths: + if os.path.exists(p): + return p + + return '' + + +def get_build_java_dir(jdk_dir): + versions = [8, 10, 11, 12, 13, 14, 15] + + if not _JAVA_DIR: + for version in versions: + jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version)) + if os.path.exists(jdk_tar_path): + jdk_dir = runtime.build_path('jdk4test') + with contextlib.closing(tarfile.open(jdk_tar_path)) as tf: + tf.extractall(jdk_dir) + assert os.path.exists(os.path.join(jdk_dir, "bin", "java")) + _JAVA_DIR.append(jdk_dir) + break + else: + _JAVA_DIR.append(None) + + return _JAVA_DIR[0] diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py new file mode 100644 index 0000000000..9e7a74cdf5 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/tags.py @@ -0,0 +1,5 @@ +try: + import pytest + ya_external = getattr(pytest.mark, "ya:external") +except ImportError: + ya_external = None diff --git a/library/python/testing/yatest_common/yatest/common/ya.make b/library/python/testing/yatest_common/yatest/common/ya.make new file mode 100644 index 0000000000..f7c50dfe64 --- /dev/null +++ b/library/python/testing/yatest_common/yatest/common/ya.make @@ -0,0 +1 @@ +OWNER(g:yatest) diff --git a/library/python/testing/yatest_lib/__init__.py b/library/python/testing/yatest_lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/library/python/testing/yatest_lib/__init__.py diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py new file mode 100644 index 0000000000..39113230d9 --- /dev/null +++ b/library/python/testing/yatest_lib/external.py @@ -0,0 +1,192 @@ +from __future__ import absolute_import + +import re +import sys +import copy +import logging + +from . import tools +from datetime import date, datetime + +import enum +import six + +logger = logging.getLogger(__name__) +MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' + + +def apply(func, value, apply_to_keys=False): + """ + Applies func to every possible member of value + :param value: could be either a primitive object or a complex one (list, dicts) + :param func: func to be applied + :return: + """ + def _apply(func, value, value_path): + if value_path is None: + value_path = [] + + if isinstance(value, list) or isinstance(value, tuple): + res = [] + for ind, item in enumerate(value): + path = copy.copy(value_path) + path.append(ind) + res.append(_apply(func, item, path)) + elif isinstance(value, dict): + if is_external(value): + # this is a special serialized object pointing to some external place + res = func(value, value_path) + else: + res = {} + for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]): + path = copy.copy(value_path) + path.append(key) + res[_apply(func, key, path) if apply_to_keys else key] = _apply(func, val, path) + else: + res = func(value, value_path) + return res + return _apply(func, value, None) + + +def is_coroutine(val): + if sys.version_info[0] < 3: + return False + else: + import asyncio + return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val) + + +def serialize(value): + """ + Serialize value to json-convertible object + Ensures that all components of value can be serialized to json + :param value: object to be serialized + """ + def _serialize(val, _): + if val is None: + return val + if isinstance(val, six.string_types) or isinstance(val, bytes): + return tools.to_utf8(val) + if isinstance(val, enum.Enum): + return str(val) + if isinstance(val, six.integer_types) or type(val) in [float, bool]: + return val + if is_external(val): + return dict(val) + if isinstance(val, (date, datetime)): + return repr(val) + if is_coroutine(val): + return None + raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val))) + return apply(_serialize, value, apply_to_keys=True) + + +def is_external(value): + return isinstance(value, dict) and "uri" in value.keys() + + +class ExternalSchema(object): + File = "file" + SandboxResource = "sbr" + Delayed = "delayed" + HTTP = "http" + + +class CanonicalObject(dict): + def __iter__(self): + raise TypeError("Iterating canonical object is not implemented") + + +class ExternalDataInfo(object): + + def __init__(self, data): + assert is_external(data) + self._data = data + + def __str__(self): + type_str = "File" if self.is_file else "Sandbox resource" + return "{}({})".format(type_str, self.path) + + def __repr__(self): + return str(self) + + @property + def uri(self): + return self._data["uri"] + + @property + def checksum(self): + return self._data.get("checksum") + + @property + def is_file(self): + return self.uri.startswith(ExternalSchema.File) + + @property + def is_sandbox_resource(self): + return self.uri.startswith(ExternalSchema.SandboxResource) + + @property + def is_delayed(self): + return self.uri.startswith(ExternalSchema.Delayed) + + @property + def is_http(self): + return self.uri.startswith(ExternalSchema.HTTP) + + @property + def path(self): + if self.uri.count("://") != 1: + logger.error("Invalid external data uri: '%s'", self.uri) + return self.uri + _, path = self.uri.split("://") + return path + + def get_mds_key(self): + assert self.is_http + m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri) + if m: + return m.group(1) + raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri)) + + @property + def size(self): + return self._data.get("size") + + def serialize(self): + return self._data + + @classmethod + def _serialize(cls, schema, path, checksum=None, attrs=None): + res = CanonicalObject({"uri": "{}://{}".format(schema, path)}) + if checksum: + res["checksum"] = checksum + if attrs: + res.update(attrs) + return res + + @classmethod + def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None): + attrs = {} + if diff_tool: + attrs["diff_tool"] = diff_tool + if local: + attrs["local"] = local + if diff_file_name: + attrs["diff_file_name"] = diff_file_name + if diff_tool_timeout: + attrs["diff_tool_timeout"] = diff_tool_timeout + if size is not None: + attrs["size"] = size + return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs) + + @classmethod + def serialize_resource(cls, id, checksum=None): + return cls._serialize(ExternalSchema.SandboxResource, id, checksum) + + @classmethod + def serialize_delayed(cls, upload_id, checksum): + return cls._serialize(ExternalSchema.Delayed, upload_id, checksum) + + def get(self, key, default=None): + return self._data.get(key, default) diff --git a/library/python/testing/yatest_lib/test_splitter.py b/library/python/testing/yatest_lib/test_splitter.py new file mode 100644 index 0000000000..acbcd4300e --- /dev/null +++ b/library/python/testing/yatest_lib/test_splitter.py @@ -0,0 +1,102 @@ +# coding: utf-8 + +import collections + + +def flatten_tests(test_classes): + """ + >>> test_classes = {x: [x] for x in range(5)} + >>> flatten_tests(test_classes) + [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] + >>> test_classes = {x: [x + 1, x + 2] for x in range(2)} + >>> flatten_tests(test_classes) + [(0, 1), (0, 2), (1, 2), (1, 3)] + """ + tests = [] + for class_name, test_names in test_classes.items(): + tests += [(class_name, test_name) for test_name in test_names] + return tests + + +def get_sequential_chunk(tests, modulo, modulo_index, is_sorted=False): + """ + >>> get_sequential_chunk(range(10), 4, 0) + [0, 1, 2] + >>> get_sequential_chunk(range(10), 4, 1) + [3, 4, 5] + >>> get_sequential_chunk(range(10), 4, 2) + [6, 7] + >>> get_sequential_chunk(range(10), 4, 3) + [8, 9] + >>> get_sequential_chunk(range(10), 4, 4) + [] + >>> get_sequential_chunk(range(10), 4, 5) + [] + """ + if not is_sorted: + tests = sorted(tests) + chunk_size = len(tests) // modulo + not_used = len(tests) % modulo + shift = chunk_size + (modulo_index < not_used) + start = chunk_size * modulo_index + min(modulo_index, not_used) + end = start + shift + return [] if end > len(tests) else tests[start:end] + + +def get_shuffled_chunk(tests, modulo, modulo_index, is_sorted=False): + """ + >>> get_shuffled_chunk(range(10), 4, 0) + [0, 4, 8] + >>> get_shuffled_chunk(range(10), 4, 1) + [1, 5, 9] + >>> get_shuffled_chunk(range(10), 4, 2) + [2, 6] + >>> get_shuffled_chunk(range(10), 4, 3) + [3, 7] + >>> get_shuffled_chunk(range(10), 4, 4) + [] + >>> get_shuffled_chunk(range(10), 4, 5) + [] + """ + if not is_sorted: + tests = sorted(tests) + result_tests = [] + for i, test in enumerate(tests): + if i % modulo == modulo_index: + result_tests.append(test) + return result_tests + + +def get_splitted_tests(test_entities, modulo, modulo_index, partition_mode, is_sorted=False): + if partition_mode == 'SEQUENTIAL': + return get_sequential_chunk(test_entities, modulo, modulo_index, is_sorted) + elif partition_mode == 'MODULO': + return get_shuffled_chunk(test_entities, modulo, modulo_index, is_sorted) + else: + raise ValueError("detected unknown partition mode: {}".format(partition_mode)) + + +def filter_tests_by_modulo(test_classes, modulo, modulo_index, split_by_tests, partition_mode="SEQUENTIAL"): + """ + >>> test_classes = {x: [x] for x in range(20)} + >>> filter_tests_by_modulo(test_classes, 4, 0, False) + {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} + >>> filter_tests_by_modulo(test_classes, 4, 1, False) + {8: [8], 9: [9], 5: [5], 6: [6], 7: [7]} + >>> filter_tests_by_modulo(test_classes, 4, 2, False) + {10: [10], 11: [11], 12: [12], 13: [13], 14: [14]} + + >>> dict(filter_tests_by_modulo(test_classes, 4, 0, True)) + {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} + >>> dict(filter_tests_by_modulo(test_classes, 4, 1, True)) + {8: [8], 9: [9], 5: [5], 6: [6], 7: [7]} + """ + if split_by_tests: + tests = get_splitted_tests(flatten_tests(test_classes), modulo, modulo_index, partition_mode) + test_classes = collections.defaultdict(list) + for class_name, test_name in tests: + test_classes[class_name].append(test_name) + return test_classes + else: + target_classes = get_splitted_tests(test_classes, modulo, modulo_index, partition_mode) + return {class_name: test_classes[class_name] for class_name in target_classes} diff --git a/library/python/testing/yatest_lib/tests/test_external.py b/library/python/testing/yatest_lib/tests/test_external.py new file mode 100644 index 0000000000..18cb560b17 --- /dev/null +++ b/library/python/testing/yatest_lib/tests/test_external.py @@ -0,0 +1,20 @@ +import enum +import pytest + +from yatest_lib import external + + +class MyEnum(enum.Enum): + VAL1 = 1 + VAL2 = 2 + + +@pytest.mark.parametrize("data, expected_val, expected_type", [ + ({}, {}, dict), + (MyEnum.VAL1, "MyEnum.VAL1", str), + ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict), +]) +def test_serialize(data, expected_val, expected_type): + data = external.serialize(data) + assert expected_type == type(data), data + assert expected_val == data diff --git a/library/python/testing/yatest_lib/tests/test_testsplitter.py b/library/python/testing/yatest_lib/tests/test_testsplitter.py new file mode 100644 index 0000000000..394bfe5a74 --- /dev/null +++ b/library/python/testing/yatest_lib/tests/test_testsplitter.py @@ -0,0 +1,103 @@ +# coding: utf-8 +from yatest_lib import test_splitter + + +def get_chunks(tests, modulo, mode): + chunks = [] + if mode == "MODULO": + for modulo_index in range(modulo): + chunks.append(test_splitter.get_shuffled_chunk(tests, modulo, modulo_index)) + elif mode == "SEQUENTIAL": + for modulo_index in range(modulo): + chunks.append(test_splitter.get_sequential_chunk(tests, modulo, modulo_index)) + else: + raise ValueError("no such mode") + return chunks + + +def check_not_intersect(chunk_list): + test_set = set() + total_size = 0 + for tests in chunk_list: + total_size += len(tests) + test_set.update(tests) + return total_size == len(test_set) + + +def check_max_diff(chunk_list): + return max(map(len, chunk_list)) - min(map(len, chunk_list)) + + +def test_lot_of_chunks(): + for chunk_count in range(10, 20): + for tests_count in range(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert chunks.count([]) == chunk_count - tests_count + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert chunks.count([]) == chunk_count - tests_count + assert len(chunks) == chunk_count + + +def test_lot_of_tests(): + for tests_count in range(10, 20): + for chunk_count in range(2, tests_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + + +def prime_chunk_count(): + for chunk_count in [7, 11, 13, 17, 23, 29]: + for tests_count in range(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + + +def get_divisors(number): + divisors = [] + for d in range(1, number + 1): + if number % d == 0: + divisors.append(d) + return divisors + + +def equal_chunks(): + for chunk_count in range(12, 31): + for tests_count in get_divisors(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) == 0 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) == 0 + assert len(chunks) == chunk_count + + +def chunk_count_equal_tests_count(): + for chunk_count in range(10, 20): + tests_count = chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count diff --git a/library/python/testing/yatest_lib/tests/ya.make b/library/python/testing/yatest_lib/tests/ya.make new file mode 100644 index 0000000000..8586c6ef7d --- /dev/null +++ b/library/python/testing/yatest_lib/tests/ya.make @@ -0,0 +1,14 @@ +OWNER(g:yatest) + +PY23_TEST() + +PEERDIR( + library/python/testing/yatest_lib +) + +TEST_SRCS( + test_external.py + test_testsplitter.py +) + +END() diff --git a/library/python/testing/yatest_lib/tools.py b/library/python/testing/yatest_lib/tools.py new file mode 100644 index 0000000000..b72d79c162 --- /dev/null +++ b/library/python/testing/yatest_lib/tools.py @@ -0,0 +1,64 @@ +import six +import sys + + +def to_utf8(value): + """ + Converts value to string encoded into utf-8 + :param value: + :return: + """ + if sys.version_info[0] < 3: + if not isinstance(value, basestring): # noqa + value = unicode(value) # noqa + if type(value) == str: + value = value.decode("utf-8", errors="ignore") + return value.encode('utf-8', 'ignore') + else: + return str(value) + + +def trim_string(s, max_bytes): + """ + Adjusts the length of the string s in order to fit it + into max_bytes bytes of storage after encoding as UTF-8. + Useful when cutting filesystem paths. + :param s: unicode string + :param max_bytes: number of bytes + :return the prefix of s + """ + if isinstance(s, six.text_type): + return _trim_unicode_string(s, max_bytes) + + if isinstance(s, six.binary_type): + if len(s) <= max_bytes: + return s + s = s.decode('utf-8', errors='ignore') + s = _trim_unicode_string(s, max_bytes) + s = s.encode('utf-8', errors='ignore') + return s + + raise TypeError('a string is expected') + + +def _trim_unicode_string(s, max_bytes): + if len(s) * 4 <= max_bytes: + # UTF-8 uses at most 4 bytes per character + return s + + result = [] + cur_byte_length = 0 + + for ch in s: + cur_byte_length += len(ch.encode('utf-8')) + if cur_byte_length > max_bytes: + break + result.append(ch) + + return ''.join(result) + + +def to_str(s): + if six.PY2 and isinstance(s, six.text_type): + return s.encode('utf8') + return s diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make new file mode 100644 index 0000000000..342bae82ba --- /dev/null +++ b/library/python/testing/yatest_lib/ya.make @@ -0,0 +1,26 @@ +OWNER(g:yatest) + +PY23_LIBRARY() + +PY_SRCS( + NAMESPACE + yatest_lib + external.py + test_splitter.py + tools.py + ya.py +) + +PEERDIR( + contrib/python/six +) + +IF(PYTHON2) + PEERDIR( + contrib/python/enum34 + ) +ENDIF() + +END() + +RECURSE_FOR_TESTS(tests) diff --git a/library/python/testing/yatest_lib/ya.py b/library/python/testing/yatest_lib/ya.py new file mode 100644 index 0000000000..c13b58a19f --- /dev/null +++ b/library/python/testing/yatest_lib/ya.py @@ -0,0 +1,239 @@ +import os +import sys +import logging +import json + +from .tools import to_str +from .external import ExternalDataInfo + + +TESTING_OUT_DIR_NAME = "testing_out_stuff" # XXX import from test.const + +yatest_logger = logging.getLogger("ya.test") + + +class RunMode(object): + Run = "run" + List = "list" + + +class TestMisconfigurationException(Exception): + pass + + +class Ya(object): + """ + Adds integration with ya, helps in finding dependencies + """ + + def __init__( + self, + mode=None, + source_root=None, + build_root=None, + dep_roots=None, + output_dir=None, + test_params=None, + context=None, + python_path=None, + valgrind_path=None, + gdb_path=None, + data_root=None, + ): + context_file_path = os.environ.get("YA_TEST_CONTEXT_FILE", None) + if context_file_path: + with open(context_file_path, 'r') as afile: + test_context = json.load(afile) + context_runtime = test_context["runtime"] + context_internal = test_context.get("internal", {}) + context_build = test_context.get("build", {}) + else: + context_runtime = {} + context_internal = {} + context_build = {} + self._mode = mode + self._build_root = to_str(context_runtime.get("build_root", "")) or build_root + self._source_root = to_str(context_runtime.get("source_root", "")) or source_root or self._detect_source_root() + self._output_dir = to_str(context_runtime.get("output_path", "")) or output_dir or self._detect_output_root() + if not self._output_dir: + raise Exception("Run ya make -t before running test binary") + if not self._source_root: + logging.warning("Source root was not set neither determined, use --source-root to set it explicitly") + if not self._build_root: + if self._source_root: + self._build_root = self._source_root + else: + logging.warning("Build root was not set neither determined, use --build-root to set it explicitly") + + if data_root: + self._data_root = data_root + elif self._source_root: + self._data_root = os.path.abspath(os.path.join(self._source_root, "..", "arcadia_tests_data")) + + self._dep_roots = dep_roots + + self._python_path = to_str(context_runtime.get("python_bin", "")) or python_path + self._valgrind_path = valgrind_path + self._gdb_path = to_str(context_runtime.get("gdb_bin", "")) or gdb_path + self._test_params = {} + self._context = {} + self._test_item_node_id = None + + ram_drive_path = to_str(context_runtime.get("ram_drive_path", "")) + if ram_drive_path: + self._test_params["ram_drive_path"] = ram_drive_path + if test_params: + self._test_params.update(dict(x.split('=', 1) for x in test_params)) + self._test_params.update(context_runtime.get("test_params", {})) + + self._context["project_path"] = context_runtime.get("project_path") + self._context["modulo"] = context_runtime.get("split_count", 1) + self._context["modulo_index"] = context_runtime.get("split_index", 0) + self._context["work_path"] = context_runtime.get("work_path") + + self._context["sanitize"] = context_build.get("sanitizer") + self._context["ya_trace_path"] = context_internal.get("trace_file") + + self._env_file = context_internal.get("env_file") + + if context: + self._context.update(context) + + @property + def source_root(self): + return self._source_root + + @property + def data_root(self): + return self._data_root + + @property + def build_root(self): + return self._build_root + + @property + def dep_roots(self): + return self._dep_roots + + @property + def output_dir(self): + return self._output_dir + + @property + def python_path(self): + return self._python_path or sys.executable + + @property + def valgrind_path(self): + if not self._valgrind_path: + raise ValueError("path to valgrind was not pass correctly, use --valgrind-path to fix it") + return self._valgrind_path + + @property + def gdb_path(self): + return self._gdb_path + + @property + def env_file(self): + return self._env_file + + def get_binary(self, *path): + assert self._build_root, "Build root was not set neither determined, use --build-root to set it explicitly" + path = list(path) + if os.name == "nt": + if not path[-1].endswith(".exe"): + path[-1] += ".exe" + + target_dirs = [self.build_root] + # Search for binaries within PATH dirs to be able to get path to the binaries specified by basename for exectests + if 'PATH' in os.environ: + target_dirs += os.environ['PATH'].split(':') + + for target_dir in target_dirs: + binary_path = os.path.join(target_dir, *path) + if os.path.exists(binary_path): + yatest_logger.debug("Binary was found by %s", binary_path) + return binary_path + + error_message = "Cannot find binary '{binary}': make sure it was added in the DEPENDS section".format(binary=path) + yatest_logger.debug(error_message) + if self._mode == RunMode.Run: + raise TestMisconfigurationException(error_message) + + def file(self, path, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None): + return ExternalDataInfo.serialize_file(path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout) + + def get_param(self, key, default=None): + return self._test_params.get(key, default) + + def get_param_dict_copy(self): + return dict(self._test_params) + + def get_context(self, key): + return self._context.get(key) + + def _detect_source_root(self): + root = None + try: + import library.python.find_root + # try to determine source root from cwd + cwd = os.getcwd() + root = library.python.find_root.detect_root(cwd) + + if not root: + # try to determine root pretending we are in the test work dir made from --keep-temps run + env_subdir = os.path.join("environment", "arcadia") + root = library.python.find_root.detect_root(cwd, detector=lambda p: os.path.exists(os.path.join(p, env_subdir))) + except ImportError: + logging.warning("Unable to import library.python.find_root") + + return root + + def _detect_output_root(self): + + # if run from kept test working dir + if os.path.exists(TESTING_OUT_DIR_NAME): + return TESTING_OUT_DIR_NAME + + # if run from source dir + if sys.version_info.major == 3: + test_results_dir = "py3test" + else: + test_results_dir = "pytest" + + test_results_output_path = os.path.join("test-results", test_results_dir, TESTING_OUT_DIR_NAME) + if os.path.exists(test_results_output_path): + return test_results_output_path + + if os.path.exists(os.path.dirname(test_results_output_path)): + os.mkdir(test_results_output_path) + return test_results_output_path + + return None + + def set_test_item_node_id(self, node_id): + self._test_item_node_id = node_id + + def get_test_item_node_id(self): + assert self._test_item_node_id + return self._test_item_node_id + + @property + def pytest_config(self): + if not hasattr(self, "_pytest_config"): + import library.python.pytest.plugins.ya as ya_plugin + self._pytest_config = ya_plugin.pytest_config + return self._pytest_config + + def set_metric_value(self, name, val): + node_id = self.get_test_item_node_id() + if node_id not in self.pytest_config.test_metrics: + self.pytest_config.test_metrics[node_id] = {} + + self.pytest_config.test_metrics[node_id][name] = val + + def get_metric_value(self, name, default=None): + res = self.pytest_config.test_metrics.get(self.get_test_item_node_id(), {}).get(name) + if res is None: + return default + return res diff --git a/library/python/windows/__init__.py b/library/python/windows/__init__.py new file mode 100644 index 0000000000..62861b3309 --- /dev/null +++ b/library/python/windows/__init__.py @@ -0,0 +1,364 @@ +# coding: utf-8 + +import os +import stat +import sys +import shutil +import logging + +from six import reraise + +import library.python.func +import library.python.strings + +logger = logging.getLogger(__name__) + + +ERRORS = { + 'SUCCESS': 0, + 'PATH_NOT_FOUND': 3, + 'ACCESS_DENIED': 5, + 'SHARING_VIOLATION': 32, + 'INSUFFICIENT_BUFFER': 122, + 'DIR_NOT_EMPTY': 145, +} + +RETRIABLE_FILE_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['SHARING_VIOLATION']) +RETRIABLE_DIR_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['DIR_NOT_EMPTY'], ERRORS['SHARING_VIOLATION']) + + +# Check if on Windows +@library.python.func.lazy +def on_win(): + return os.name == 'nt' + + +class NotOnWindowsError(RuntimeError): + def __init__(self, message): + super(NotOnWindowsError, self).__init__(message) + + +class DisabledOnWindowsError(RuntimeError): + def __init__(self, message): + super(DisabledOnWindowsError, self).__init__(message) + + +class NoCTypesError(RuntimeError): + def __init__(self, message): + super(NoCTypesError, self).__init__(message) + + +# Decorator for Windows-only functions +def win_only(f): + def f_wrapped(*args, **kwargs): + if not on_win(): + raise NotOnWindowsError('Windows-only function is called, but platform is not Windows') + return f(*args, **kwargs) + + return f_wrapped + + +# Decorator for functions disabled on Windows +def win_disabled(f): + def f_wrapped(*args, **kwargs): + if on_win(): + run_disabled() + return f(*args, **kwargs) + + return f_wrapped + + +def errorfix(f): + if not on_win(): + return f + + def f_wrapped(*args, **kwargs): + try: + return f(*args, **kwargs) + except WindowsError: + tp, value, tb = sys.exc_info() + fix_error(value) + reraise(tp, value, tb) + + return f_wrapped + + +# Decorator for diehard wrapper +# On Windows platform retries to run function while specific WindowsError is thrown +# On non-Windows platforms fallbacks to function itself +def diehard(winerrors, tries=100, delay=1): + def wrap(f): + if not on_win(): + return f + + return lambda *args, **kwargs: run_diehard(f, winerrors, tries, delay, *args, **kwargs) + + return wrap + + +if on_win(): + import msvcrt + import time + + import library.python.strings + + _has_ctypes = True + try: + import ctypes + from ctypes import wintypes + except ImportError: + _has_ctypes = False + + _INVALID_HANDLE_VALUE = -1 + + _MOVEFILE_REPLACE_EXISTING = 0x1 + _MOVEFILE_WRITE_THROUGH = 0x8 + + _SEM_FAILCRITICALERRORS = 0x1 + _SEM_NOGPFAULTERRORBOX = 0x2 + _SEM_NOALIGNMENTFAULTEXCEPT = 0x4 + _SEM_NOOPENFILEERRORBOX = 0x8 + + _SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1 + + _CREATE_NO_WINDOW = 0x8000000 + + _ATOMIC_RENAME_FILE_TRANSACTION_DEFAULT_TIMEOUT = 1000 + + _HANDLE_FLAG_INHERIT = 0x1 + + @win_only + def require_ctypes(f): + def f_wrapped(*args, **kwargs): + if not _has_ctypes: + raise NoCTypesError('No ctypes found') + return f(*args, **kwargs) + + return f_wrapped + + # Run function in diehard mode (see diehard decorator commentary) + @win_only + def run_diehard(f, winerrors, tries, delay, *args, **kwargs): + if isinstance(winerrors, int): + winerrors = (winerrors,) + + ei = None + for t in xrange(tries): + if t: + logger.debug('Diehard [errs %s]: try #%d in %s', ','.join(str(x) for x in winerrors), t, f) + try: + return f(*args, **kwargs) + except WindowsError as e: + if e.winerror not in winerrors: + raise + ei = sys.exc_info() + time.sleep(delay) + reraise(ei[0], ei[1], ei[2]) + + # Placeholder for disabled functions + @win_only + def run_disabled(*args, **kwargs): + raise DisabledOnWindowsError('Function called is disabled on Windows') + + class CustomWinError(WindowsError): + def __init__(self, winerror, message='', filename=None): + super(CustomWinError, self).__init__(winerror, message) + self.message = message + self.strerror = self.message if self.message else format_error(self.windows_error) + self.filename = filename + self.utf8 = True + + @win_only + def unicode_path(path): + return library.python.strings.to_unicode(path, library.python.strings.fs_encoding()) + + @win_only + @require_ctypes + def format_error(error): + if isinstance(error, WindowsError): + error = error.winerror + if not isinstance(error, int): + return 'Unknown' + return ctypes.FormatError(error) + + @win_only + def fix_error(windows_error): + if not windows_error.strerror: + windows_error.strerror = format_error(windows_error) + transcode_error(windows_error) + + @win_only + def transcode_error(windows_error, to_enc='utf-8'): + from_enc = 'utf-8' if getattr(windows_error, 'utf8', False) else library.python.strings.guess_default_encoding() + if from_enc != to_enc: + windows_error.strerror = library.python.strings.to_str(windows_error.strerror, to_enc=to_enc, from_enc=from_enc) + setattr(windows_error, 'utf8', to_enc == 'utf-8') + + class Transaction(object): + def __init__(self, timeout=None, description=''): + self.timeout = timeout + self.description = description + + @require_ctypes + def __enter__(self): + self._handle = ctypes.windll.ktmw32.CreateTransaction(None, 0, 0, 0, 0, self.timeout, self.description) + if self._handle == _INVALID_HANDLE_VALUE: + raise ctypes.WinError() + return self._handle + + @require_ctypes + def __exit__(self, t, v, tb): + try: + if not ctypes.windll.ktmw32.CommitTransaction(self._handle): + raise ctypes.WinError() + finally: + ctypes.windll.kernel32.CloseHandle(self._handle) + + @win_only + def file_handle(f): + return msvcrt.get_osfhandle(f.fileno()) + + # https://www.python.org/dev/peps/pep-0446/ + # http://mihalop.blogspot.ru/2014/05/python-subprocess-and-file-descriptors.html + @require_ctypes + @win_only + def open_file(*args, **kwargs): + f = open(*args, **kwargs) + ctypes.windll.kernel32.SetHandleInformation(file_handle(f), _HANDLE_FLAG_INHERIT, 0) + return f + + @win_only + @require_ctypes + def replace_file(src, dst): + if not ctypes.windll.kernel32.MoveFileExW(unicode_path(src), unicode_path(dst), _MOVEFILE_REPLACE_EXISTING | _MOVEFILE_WRITE_THROUGH): + raise ctypes.WinError() + + @win_only + @require_ctypes + def replace_file_across_devices(src, dst): + with Transaction(timeout=_ATOMIC_RENAME_FILE_TRANSACTION_DEFAULT_TIMEOUT, description='ya library.python.windows replace_file_across_devices') as transaction: + if not ctypes.windll.kernel32.MoveFileTransactedW(unicode_path(src), unicode_path(dst), None, None, _MOVEFILE_REPLACE_EXISTING | _MOVEFILE_WRITE_THROUGH, transaction): + raise ctypes.WinError() + + @win_only + @require_ctypes + def hardlink(src, lnk): + if not ctypes.windll.kernel32.CreateHardLinkW(unicode_path(lnk), unicode_path(src), None): + raise ctypes.WinError() + + # Requires SE_CREATE_SYMBOLIC_LINK_NAME privilege + @win_only + @win_disabled + @require_ctypes + def symlink_file(src, lnk): + if not ctypes.windll.kernel32.CreateSymbolicLinkW(unicode_path(lnk), unicode_path(src), 0): + raise ctypes.WinError() + + # Requires SE_CREATE_SYMBOLIC_LINK_NAME privilege + @win_only + @win_disabled + @require_ctypes + def symlink_dir(src, lnk): + if not ctypes.windll.kernel32.CreateSymbolicLinkW(unicode_path(lnk), unicode_path(src), _SYMBOLIC_LINK_FLAG_DIRECTORY): + raise ctypes.WinError() + + @win_only + @require_ctypes + def lock_file(f, offset, length, raises=True): + locked = ctypes.windll.kernel32.LockFile(file_handle(f), _low_dword(offset), _high_dword(offset), _low_dword(length), _high_dword(length)) + if not raises: + return bool(locked) + if not locked: + raise ctypes.WinError() + + @win_only + @require_ctypes + def unlock_file(f, offset, length, raises=True): + unlocked = ctypes.windll.kernel32.UnlockFile(file_handle(f), _low_dword(offset), _high_dword(offset), _low_dword(length), _high_dword(length)) + if not raises: + return bool(unlocked) + if not unlocked: + raise ctypes.WinError() + + @win_only + @require_ctypes + def set_error_mode(mode): + return ctypes.windll.kernel32.SetErrorMode(mode) + + @win_only + def rmtree(path): + def error_handler(func, handling_path, execinfo): + e = execinfo[1] + if e.winerror == ERRORS['PATH_NOT_FOUND']: + handling_path = "\\\\?\\" + handling_path # handle path over 256 symbols + if os.path.exists(path): + return func(handling_path) + if e.winerror == ERRORS['ACCESS_DENIED']: + try: + # removing of r/w directory with read-only files in it yields ACCESS_DENIED + # which is not an insuperable obstacle https://bugs.python.org/issue19643 + os.chmod(handling_path, stat.S_IWRITE) + except OSError: + pass + else: + # propagate true last error if this attempt fails + return func(handling_path) + raise e + shutil.rmtree(path, onerror=error_handler) + + # Don't display the Windows GPF dialog if the invoked program dies. + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx + @win_only + def disable_error_dialogs(): + set_error_mode(_SEM_NOGPFAULTERRORBOX | _SEM_FAILCRITICALERRORS) + + @win_only + def default_process_creation_flags(): + return 0 + + @require_ctypes + def _low_dword(x): + return ctypes.c_ulong(x & ((1 << 32) - 1)) + + @require_ctypes + def _high_dword(x): + return ctypes.c_ulong((x >> 32) & ((1 << 32) - 1)) + + @win_only + @require_ctypes + def get_current_process(): + handle = ctypes.windll.kernel32.GetCurrentProcess() + if not handle: + raise ctypes.WinError() + return wintypes.HANDLE(handle) + + @win_only + @require_ctypes + def get_process_handle_count(proc_handle): + assert isinstance(proc_handle, wintypes.HANDLE) + + GetProcessHandleCount = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HANDLE, wintypes.POINTER(wintypes.DWORD))(("GetProcessHandleCount", ctypes.windll.kernel32)) + hndcnt = wintypes.DWORD() + if not GetProcessHandleCount(proc_handle, ctypes.byref(hndcnt)): + raise ctypes.WinError() + return hndcnt.value + + @win_only + @require_ctypes + def set_handle_information(file, inherit=None, protect_from_close=None): + for flag, value in [(inherit, 1), (protect_from_close, 2)]: + if flag is not None: + assert isinstance(flag, bool) + if not ctypes.windll.kernel32.SetHandleInformation(file_handle(file), _low_dword(value), _low_dword(int(flag))): + raise ctypes.WinError() + + @win_only + @require_ctypes + def get_windows_directory(): + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + size = ctypes.windll.kernel32.GetWindowsDirectoryW(buf, ctypes.wintypes.MAX_PATH) + if not size: + raise ctypes.WinError() + if size > ctypes.wintypes.MAX_PATH - 1: + raise CustomWinError(ERRORS['INSUFFICIENT_BUFFER']) + return ctypes.wstring_at(buf, size) diff --git a/library/python/windows/ut/test_windows.py b/library/python/windows/ut/test_windows.py new file mode 100644 index 0000000000..bef3ec2dc5 --- /dev/null +++ b/library/python/windows/ut/test_windows.py @@ -0,0 +1,96 @@ +# coding=utf-8 + +import errno +import os +import pytest + +import library.python.strings +import library.python.windows + + +def gen_error_access_denied(): + if library.python.windows.on_win(): + err = WindowsError() + err.errno = errno.EACCES + err.strerror = '' + err.winerror = library.python.windows.ERRORS['ACCESS_DENIED'] + else: + err = OSError() + err.errno = errno.EACCES + err.strerror = os.strerror(err.errno) + err.filename = 'unknown/file' + raise err + + +def test_errorfix_buggy(): + @library.python.windows.errorfix + def erroneous_func(): + gen_error_access_denied() + + with pytest.raises(OSError) as errinfo: + erroneous_func() + assert errinfo.value.errno == errno.EACCES + assert errinfo.value.filename == 'unknown/file' + assert isinstance(errinfo.value.strerror, basestring) + assert errinfo.value.strerror + + +def test_errorfix_explicit(): + @library.python.windows.errorfix + def erroneous_func(): + if library.python.windows.on_win(): + err = WindowsError() + err.winerror = library.python.windows.ERRORS['ACCESS_DENIED'] + else: + err = OSError() + err.errno = errno.EACCES + err.strerror = 'Some error description' + err.filename = 'unknown/file' + raise err + + with pytest.raises(OSError) as errinfo: + erroneous_func() + assert errinfo.value.errno == errno.EACCES + assert errinfo.value.filename == 'unknown/file' + assert errinfo.value.strerror == 'Some error description' + + +def test_errorfix_decoding_cp1251(): + @library.python.windows.errorfix + def erroneous_func(): + model_msg = u'Какое-то описание ошибки' + if library.python.windows.on_win(): + err = WindowsError() + err.strerror = library.python.strings.to_str(model_msg, 'cp1251') + else: + err = OSError() + err.strerror = library.python.strings.to_str(model_msg) + raise err + + with pytest.raises(OSError) as errinfo: + erroneous_func() + error_msg = errinfo.value.strerror + if not isinstance(errinfo.value.strerror, unicode): + error_msg = library.python.strings.to_unicode(error_msg) + assert error_msg == u'Какое-то описание ошибки' + + +def test_diehard(): + @library.python.windows.diehard(library.python.windows.ERRORS['ACCESS_DENIED'], tries=5) + def erroneous_func(errors): + try: + gen_error_access_denied() + except Exception as e: + errors.append(e) + raise + + raised_errors = [] + with pytest.raises(OSError) as errinfo: + erroneous_func(raised_errors) + assert errinfo.value.errno == errno.EACCES + assert any(e.errno == errno.EACCES for e in raised_errors) + assert raised_errors and errinfo.value == raised_errors[-1] + if library.python.windows.on_win(): + assert len(raised_errors) == 5 + else: + assert len(raised_errors) == 1 diff --git a/library/python/windows/ut/ya.make b/library/python/windows/ut/ya.make new file mode 100644 index 0000000000..c39f1797b8 --- /dev/null +++ b/library/python/windows/ut/ya.make @@ -0,0 +1,11 @@ +OWNER(g:yatool) + +PY2TEST() + +TEST_SRCS(test_windows.py) + +PEERDIR( + library/python/windows +) + +END() diff --git a/library/python/windows/ya.make b/library/python/windows/ya.make new file mode 100644 index 0000000000..e17f86b67e --- /dev/null +++ b/library/python/windows/ya.make @@ -0,0 +1,13 @@ +OWNER(g:yatool) + +PY23_LIBRARY() + +PY_SRCS(__init__.py) + +PEERDIR( + library/python/func + library/python/strings + contrib/python/six +) + +END() diff --git a/library/python/ya.make b/library/python/ya.make new file mode 100644 index 0000000000..2e1eb6e0e1 --- /dev/null +++ b/library/python/ya.make @@ -0,0 +1,222 @@ +OWNER(g:python-contrib) + +RECURSE( + aho_corasick + aho_corasick/ut + archive + archive/benchmark + archive/test + archive/test/data + asgi_yauth + async_clients + auth_client_parser + awssdk-extensions + awssdk_async_extensions + base64 + base64/test + bclclient + blackbox + blackbox/tests + blackbox/tvm2 + bloom + boost_test + bstr + build_info + build_info/ut + capabilities + celery_dashboard + certifi + cgroups + charset + charts_notes + charts_notes/example + cityhash + cityhash/test + clickhouse_client + cmain + codecs + codecs/gen_corpus + codecs/test + compress + compress/tests + cookiemy + coredump_filter + cores + coverage + cpp_test + cppdemangle + cqueue + crowd-kit + cyson + cyson/pymodule + cyson/ut + deploy_formatter + deprecated + dir-sync + django + django/example + django-idm-api + django-multic + django-sform + django-sform/tests + django_alive + django_celery_monitoring + django_russian + django_template_common + django_tools_log_context + dssclient + dump_dict + edit_distance + errorboosterclient + filelock + filelock/ut + filesys + filesys/ut + find_root + flask + flask_passport + fnvhash + fnvhash/test + framing + framing/ut + func + func/ut + fs + geolocation + geolocation/ut + geohash + geohash/ut + golovan_stats_aggregator + granular_settings + granular_settings/tests + guid + guid/test + guid/at_fork_test + gunicorn + hnsw + ids + import_test + infected_masks + infected_masks/ut + init_log + init_log/example + intrasearch_fetcher + json + json/compare + json/perf + json/test + json/test/py2 + json/test/py3 + langdetect + langmask + langs + luigi + luigi/data + luigi/example + luigi/luigid_static + maths + messagebus + metrics_framework + mime_types + monitoring + monlib + monlib/examples + monlib/so + murmurhash + nirvana + nirvana_api + nirvana_api/test_lib + nirvana_test + nstools + nyt + oauth + oauth/example + ok_client + openssl + par_apply + par_apply/test + path + path/tests + protobuf + pymain + pyscopg2 + pytest + pytest-mongodb + pytest/allure + pytest/empty + pytest/plugins + python-blackboxer + python-django-tanker + python-django-yauth/tests + python-django-yauth + reactor + redis_utils + reservoir_sampling + refsclient + resource + retry + retry/tests + runtime + runtime/main + runtime/test + runtime_py3 + runtime_py3/main + runtime_py3/test + runtime_test + sanitizers + sdms_api + sfx + selenium_ui_test + sendmsg + stubmaker + solomon + spack + spyt + ssh_client + ssh_sign + startrek_python_client + startrek_python_client/tests_int + statface_client + step + strings + strings/ut + svn_ssh + svn_version + svn_version/ut + symbols + testing + tmp + toloka_client + toloka-kit + toloka-airflow + toloka-prefect + tools_structured_logs + thread + thread/test + tskv + tvmauth + tvm2 + tvm2/tests + type_info + type_info/test + unique_id + vault_client + watch_dog + watch_dog/example + wiki + windows + windows/ut + yandex_tracker_client + yenv + yt + yt/test + ylock + ylock/tests + zipatch +) + +IF (NOT MUSL) + RECURSE( + yt/example + ) +ENDIF() |