aboutsummaryrefslogtreecommitdiffstats
path: root/library/python
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/python
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/python')
-rw-r--r--library/python/certifi/.dist-info/METADATA2
-rw-r--r--library/python/certifi/.dist-info/top_level.txt1
-rw-r--r--library/python/certifi/README.md7
-rw-r--r--library/python/certifi/certifi/__init__.py10
-rw-r--r--library/python/certifi/certifi/binary.py25
-rw-r--r--library/python/certifi/certifi/source.py13
-rw-r--r--library/python/certifi/ya.make18
-rw-r--r--library/python/cores/__init__.py193
-rw-r--r--library/python/cores/ya.make15
-rw-r--r--library/python/filelock/__init__.py122
-rw-r--r--library/python/filelock/ut/lib/test_filelock.py83
-rw-r--r--library/python/filelock/ut/lib/ya.make11
-rw-r--r--library/python/filelock/ut/py2/ya.make9
-rw-r--r--library/python/filelock/ut/py3/ya.make9
-rw-r--r--library/python/filelock/ut/ya.make7
-rw-r--r--library/python/filelock/ya.make11
-rw-r--r--library/python/find_root/__init__.py20
-rw-r--r--library/python/find_root/ya.make7
-rw-r--r--library/python/fs/__init__.py501
-rw-r--r--library/python/fs/clonefile.pyx18
-rw-r--r--library/python/fs/test/test_fs.py1037
-rw-r--r--library/python/fs/test/ya.make14
-rw-r--r--library/python/fs/ya.make23
-rw-r--r--library/python/func/__init__.py170
-rw-r--r--library/python/func/ut/test_func.py162
-rw-r--r--library/python/func/ut/ya.make11
-rw-r--r--library/python/func/ya.make11
-rw-r--r--library/python/pytest/__init__.py0
-rw-r--r--library/python/pytest/allure/conftest.py8
-rw-r--r--library/python/pytest/allure/ya.make11
-rw-r--r--library/python/pytest/context.py1
-rw-r--r--library/python/pytest/empty/main.c7
-rw-r--r--library/python/pytest/empty/ya.make12
-rw-r--r--library/python/pytest/main.py116
-rw-r--r--library/python/pytest/plugins/collection.py128
-rw-r--r--library/python/pytest/plugins/conftests.py50
-rw-r--r--library/python/pytest/plugins/fakeid_py2.py2
-rw-r--r--library/python/pytest/plugins/fakeid_py3.py2
-rw-r--r--library/python/pytest/plugins/fixtures.py85
-rw-r--r--library/python/pytest/plugins/ya.make32
-rw-r--r--library/python/pytest/plugins/ya.py963
-rw-r--r--library/python/pytest/pytest.yatest.ini7
-rw-r--r--library/python/pytest/rewrite.py123
-rw-r--r--library/python/pytest/ya.make32
-rw-r--r--library/python/pytest/yatest_tools.py304
-rw-r--r--library/python/reservoir_sampling/README.md11
-rw-r--r--library/python/reservoir_sampling/__init__.py16
-rw-r--r--library/python/reservoir_sampling/ya.make10
-rw-r--r--library/python/resource/__init__.py49
-rw-r--r--library/python/resource/ut/lib/qw.txt1
-rw-r--r--library/python/resource/ut/lib/test_simple.py31
-rw-r--r--library/python/resource/ut/lib/ya.make17
-rw-r--r--library/python/resource/ut/py2/ya.make9
-rw-r--r--library/python/resource/ut/py3/ya.make9
-rw-r--r--library/python/resource/ut/ya.make6
-rw-r--r--library/python/resource/ya.make13
-rw-r--r--library/python/runtime_py3/__res.pyx36
-rw-r--r--library/python/runtime_py3/entry_points.py52
-rw-r--r--library/python/runtime_py3/importer.pxi571
-rw-r--r--library/python/runtime_py3/main/get_py_main.cpp8
-rw-r--r--library/python/runtime_py3/main/main.c231
-rw-r--r--library/python/runtime_py3/main/ya.make26
-rw-r--r--library/python/runtime_py3/sitecustomize.pyx69
-rw-r--r--library/python/runtime_py3/test/.dist-info/METADATA14
-rw-r--r--library/python/runtime_py3/test/.dist-info/RECORD1
-rw-r--r--library/python/runtime_py3/test/.dist-info/entry_points.txt2
-rw-r--r--library/python/runtime_py3/test/.dist-info/top_level.txt1
-rw-r--r--library/python/runtime_py3/test/canondata/result.json62
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt12
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt13
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt41
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt12
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt13
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt41
-rw-r--r--library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt2
-rw-r--r--library/python/runtime_py3/test/resources/__init__.py0
-rw-r--r--library/python/runtime_py3/test/resources/foo.txt1
-rw-r--r--library/python/runtime_py3/test/resources/submodule/__init__.py0
-rw-r--r--library/python/runtime_py3/test/resources/submodule/bar.txt1
-rw-r--r--library/python/runtime_py3/test/test_arcadia_source_finder.py317
-rw-r--r--library/python/runtime_py3/test/test_metadata.py44
-rw-r--r--library/python/runtime_py3/test/test_resources.py60
-rw-r--r--library/python/runtime_py3/test/test_traceback.py63
-rw-r--r--library/python/runtime_py3/test/traceback/__main__.py4
-rw-r--r--library/python/runtime_py3/test/traceback/crash.py44
-rw-r--r--library/python/runtime_py3/test/traceback/mod/__init__.py3
-rw-r--r--library/python/runtime_py3/test/traceback/ya.make19
-rw-r--r--library/python/runtime_py3/test/ya.make40
-rw-r--r--library/python/runtime_py3/ya.make49
-rw-r--r--library/python/strings/__init__.py17
-rw-r--r--library/python/strings/strings.py129
-rw-r--r--library/python/strings/ut/test_strings.py205
-rw-r--r--library/python/strings/ut/ya.make11
-rw-r--r--library/python/strings/ya.make16
-rw-r--r--library/python/svn_version/__init__.py1
-rw-r--r--library/python/svn_version/__svn_version.pyx35
-rw-r--r--library/python/svn_version/ut/lib/test_simple.py16
-rw-r--r--library/python/svn_version/ut/lib/ya.make13
-rw-r--r--library/python/svn_version/ut/py2/ya.make9
-rw-r--r--library/python/svn_version/ut/py3/ya.make9
-rw-r--r--library/python/svn_version/ut/ya.make5
-rw-r--r--library/python/svn_version/ya.make15
-rw-r--r--library/python/symbols/libc/syms.cpp157
-rw-r--r--library/python/symbols/libc/ya.make19
-rw-r--r--library/python/symbols/module/__init__.py48
-rw-r--r--library/python/symbols/module/module.cpp85
-rw-r--r--library/python/symbols/module/ya.make23
-rw-r--r--library/python/symbols/python/syms.cpp17
-rw-r--r--library/python/symbols/python/ut/py2/ya.make9
-rw-r--r--library/python/symbols/python/ut/py3/ya.make9
-rw-r--r--library/python/symbols/python/ut/test_ctypes.py37
-rw-r--r--library/python/symbols/python/ut/ya.make16
-rw-r--r--library/python/symbols/python/ya.make15
-rw-r--r--library/python/symbols/registry/syms.cpp31
-rw-r--r--library/python/symbols/registry/syms.h24
-rw-r--r--library/python/symbols/registry/ya.make9
-rw-r--r--library/python/symbols/ya.make12
-rw-r--r--library/python/testing/__init__.py0
-rw-r--r--library/python/testing/filter/filter.py57
-rw-r--r--library/python/testing/filter/ya.make5
-rw-r--r--library/python/testing/import_test/import_test.py124
-rw-r--r--library/python/testing/import_test/ya.make10
-rw-r--r--library/python/testing/recipe/__init__.py102
-rw-r--r--library/python/testing/recipe/ports.py33
-rw-r--r--library/python/testing/recipe/ya.make19
-rw-r--r--library/python/testing/ya.make22
-rw-r--r--library/python/testing/yatest_common/ya.make40
-rw-r--r--library/python/testing/yatest_common/yatest/__init__.py3
-rw-r--r--library/python/testing/yatest_common/yatest/common/__init__.py8
-rw-r--r--library/python/testing/yatest_common/yatest/common/benchmark.py22
-rw-r--r--library/python/testing/yatest_common/yatest/common/canonical.py176
-rw-r--r--library/python/testing/yatest_common/yatest/common/environment.py5
-rw-r--r--library/python/testing/yatest_common/yatest/common/errors.py20
-rw-r--r--library/python/testing/yatest_common/yatest/common/legacy.py12
-rw-r--r--library/python/testing/yatest_common/yatest/common/misc.py19
-rw-r--r--library/python/testing/yatest_common/yatest/common/network.py271
-rw-r--r--library/python/testing/yatest_common/yatest/common/path.py90
-rw-r--r--library/python/testing/yatest_common/yatest/common/process.py733
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime.py343
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime_java.py46
-rw-r--r--library/python/testing/yatest_common/yatest/common/tags.py5
-rw-r--r--library/python/testing/yatest_common/yatest/common/ya.make1
-rw-r--r--library/python/testing/yatest_lib/__init__.py0
-rw-r--r--library/python/testing/yatest_lib/external.py192
-rw-r--r--library/python/testing/yatest_lib/test_splitter.py102
-rw-r--r--library/python/testing/yatest_lib/tests/test_external.py20
-rw-r--r--library/python/testing/yatest_lib/tests/test_testsplitter.py103
-rw-r--r--library/python/testing/yatest_lib/tests/ya.make14
-rw-r--r--library/python/testing/yatest_lib/tools.py64
-rw-r--r--library/python/testing/yatest_lib/ya.make26
-rw-r--r--library/python/testing/yatest_lib/ya.py239
-rw-r--r--library/python/windows/__init__.py364
-rw-r--r--library/python/windows/ut/test_windows.py96
-rw-r--r--library/python/windows/ut/ya.make11
-rw-r--r--library/python/windows/ya.make13
-rw-r--r--library/python/ya.make222
161 files changed, 11051 insertions, 0 deletions
diff --git a/library/python/certifi/.dist-info/METADATA b/library/python/certifi/.dist-info/METADATA
new file mode 100644
index 0000000000..4849f81ed4
--- /dev/null
+++ b/library/python/certifi/.dist-info/METADATA
@@ -0,0 +1,2 @@
+Name: certifi
+Version: 2019.7.1
diff --git a/library/python/certifi/.dist-info/top_level.txt b/library/python/certifi/.dist-info/top_level.txt
new file mode 100644
index 0000000000..963eac530b
--- /dev/null
+++ b/library/python/certifi/.dist-info/top_level.txt
@@ -0,0 +1 @@
+certifi
diff --git a/library/python/certifi/README.md b/library/python/certifi/README.md
new file mode 100644
index 0000000000..75a812733c
--- /dev/null
+++ b/library/python/certifi/README.md
@@ -0,0 +1,7 @@
+This library provides arcadia certs via certifi-compatible API.
+
+In binary (single executable) mode it patches python ssl module to
+support loading certs from memory.
+
+.dist-info/METADATA version reflects the last update time of the
+arcadia certs.
diff --git a/library/python/certifi/certifi/__init__.py b/library/python/certifi/certifi/__init__.py
new file mode 100644
index 0000000000..5270d206cd
--- /dev/null
+++ b/library/python/certifi/certifi/__init__.py
@@ -0,0 +1,10 @@
+import ssl
+
+if hasattr(ssl, 'builtin_cadata'):
+ from .binary import where
+else:
+ from .source import where
+
+__all__ = ['where', '__version__']
+
+__version__ = '2020.04.05.2'
diff --git a/library/python/certifi/certifi/binary.py b/library/python/certifi/certifi/binary.py
new file mode 100644
index 0000000000..1050e733a3
--- /dev/null
+++ b/library/python/certifi/certifi/binary.py
@@ -0,0 +1,25 @@
+import ssl
+
+
+def builtin_ca():
+ return None, None, ssl.builtin_cadata()
+
+
+# Normally certifi.where() returns a path to a certificate file;
+# here it returns a callable.
+def where():
+ return builtin_ca
+
+
+# Patch ssl module to accept a callable cafile.
+load_verify_locations = ssl.SSLContext.load_verify_locations
+
+
+def load_verify_locations__callable(self, cafile=None, capath=None, cadata=None):
+ if callable(cafile):
+ cafile, capath, cadata = cafile()
+
+ return load_verify_locations(self, cafile, capath, cadata)
+
+
+ssl.SSLContext.load_verify_locations = load_verify_locations__callable
diff --git a/library/python/certifi/certifi/source.py b/library/python/certifi/certifi/source.py
new file mode 100644
index 0000000000..f539b67002
--- /dev/null
+++ b/library/python/certifi/certifi/source.py
@@ -0,0 +1,13 @@
+import os.path
+
+pem = os.path.abspath(__file__)
+pem = os.path.dirname(pem)
+pem = os.path.dirname(pem)
+pem = os.path.dirname(pem)
+pem = os.path.dirname(pem)
+pem = os.path.dirname(pem)
+pem = os.path.join(pem, 'certs', 'cacert.pem')
+
+
+def where():
+ return pem
diff --git a/library/python/certifi/ya.make b/library/python/certifi/ya.make
new file mode 100644
index 0000000000..64fefe2833
--- /dev/null
+++ b/library/python/certifi/ya.make
@@ -0,0 +1,18 @@
+PY23_LIBRARY()
+
+OWNER(orivej g:python-contrib)
+
+RESOURCE_FILES(
+ PREFIX library/python/certifi/
+ .dist-info/METADATA
+ .dist-info/top_level.txt
+)
+
+PY_SRCS(
+ TOP_LEVEL
+ certifi/__init__.py
+ certifi/binary.py
+ certifi/source.py
+)
+
+END()
diff --git a/library/python/cores/__init__.py b/library/python/cores/__init__.py
new file mode 100644
index 0000000000..fdb1f82a46
--- /dev/null
+++ b/library/python/cores/__init__.py
@@ -0,0 +1,193 @@
+# coding: utf-8
+
+import os
+import re
+import glob
+import socket
+import logging
+import platform
+import subprocess
+
+import six
+
+from library.python.reservoir_sampling import reservoir_sampling
+
+
+logger = logging.getLogger(__name__)
+
+
+def _read_file(filename):
+ with open(filename) as afile:
+ return afile.read().strip("\n")
+
+
+def recover_core_dump_file(binary_path, cwd, pid):
+
+ class CoreFilePattern(object):
+ def __init__(self, path, mask):
+ self.path = path
+ self.mask = mask
+
+ cwd = cwd or os.getcwd()
+ system = platform.system().lower()
+ if system.startswith("linux"):
+ import stat
+ import resource
+
+ logger.debug("hostname = '%s'", socket.gethostname())
+ logger.debug("rlimit_core = '%s'", str(resource.getrlimit(resource.RLIMIT_CORE)))
+ core_pattern = _read_file("/proc/sys/kernel/core_pattern")
+ logger.debug("core_pattern = '%s'", core_pattern)
+ if core_pattern.startswith("/"):
+ default_pattern = CoreFilePattern(os.path.dirname(core_pattern), '*')
+ else:
+ default_pattern = CoreFilePattern(cwd, '*')
+
+ def resolve_core_mask(core_mask):
+ def resolve(text):
+ if text == "%p":
+ return str(pid)
+ elif text == "%e":
+ # https://github.com/torvalds/linux/blob/7876320f88802b22d4e2daf7eb027dd14175a0f8/include/linux/sched.h#L847
+ # https://github.com/torvalds/linux/blob/7876320f88802b22d4e2daf7eb027dd14175a0f8/fs/coredump.c#L278
+ return os.path.basename(binary_path)[:15]
+ elif text == "%E":
+ return binary_path.replace("/", "!")
+ elif text == "%%":
+ return "%"
+ elif text.startswith("%"):
+ return "*"
+ return text
+
+ parts = filter(None, re.split(r"(%.)", core_mask))
+ return "".join([resolve(p) for p in parts])
+
+ # don't interpret a program for piping core dumps as a pattern
+ if core_pattern and not core_pattern.startswith("|"):
+ default_pattern.mask = os.path.basename(core_pattern)
+ else:
+ core_uses_pid = int(_read_file("/proc/sys/kernel/core_uses_pid"))
+ logger.debug("core_uses_pid = '%d'", core_uses_pid)
+ if core_uses_pid == 0:
+ default_pattern.mask = "core"
+ else:
+ default_pattern.mask = "core.%p"
+
+ # widely distributed core dump dir and mask (see DEVTOOLS-4408)
+ yandex_pattern = CoreFilePattern('/coredumps', '%e.%p.%s')
+ yandex_market_pattern = CoreFilePattern('/var/tmp/cores', 'core.%..%e.%s.%p.*')
+
+ for pattern in [default_pattern, yandex_pattern, yandex_market_pattern]:
+ pattern.mask = resolve_core_mask(pattern.mask)
+
+ if not os.path.exists(pattern.path):
+ logger.warning("Core dump dir doesn't exist: %s", pattern.path)
+ continue
+
+ logger.debug(
+ "Core dump dir (%s) permission mask: %s (expected: %s (%s-dir, %s-sticky bit))",
+ pattern.path,
+ oct(os.stat(pattern.path)[stat.ST_MODE]),
+ oct(stat.S_IFDIR | stat.S_ISVTX | 0o777),
+ oct(stat.S_IFDIR),
+ oct(stat.S_ISVTX),
+ )
+ logger.debug("Search for core dump files match pattern '%s' in '%s'", pattern.mask, pattern.path)
+ cores = glob.glob(os.path.join(pattern.path, pattern.mask))
+ files = os.listdir(pattern.path)
+ logger.debug(
+ "Matched core dump files (%d/%d): [%s] (mismatched samples: %s)",
+ len(cores),
+ len(files),
+ ", ".join(cores),
+ ", ".join(reservoir_sampling(files, 5)),
+ )
+
+ if len(cores) == 1:
+ return cores[0]
+ elif len(cores) > 1:
+ stat = [(filename, os.stat(filename).st_mtime) for filename in cores]
+ entry = sorted(stat, key=lambda x: x[1])[-1]
+ logger.debug("Latest core dump file: '%s' with %d mtime", entry[0], entry[1])
+ return entry[0]
+ else:
+ logger.debug("Core dump file recovering is not supported on '%s'", system)
+ return None
+
+
+def get_gdb_full_backtrace(binary, core, gdb_path):
+ cmd = [
+ gdb_path, binary, core,
+ "--eval-command", "set print thread-events off",
+ "--eval-command", "thread apply all backtrace full",
+ "--batch",
+ "--quiet",
+ ]
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ output, stderr = proc.communicate()
+ output = six.ensure_str(output)
+ if stderr:
+ output += "\nstderr >>\n" + six.ensure_str(stderr)
+ return output
+
+
+def get_problem_stack(backtrace):
+ stack = []
+ found_thread1 = False
+ regex = re.compile(r'[Tt]hread (\d+)')
+
+ for line in backtrace.split("\n"):
+ match = regex.search(line)
+ if match:
+ if found_thread1:
+ break
+ if int(match.group(1)) == 1:
+ found_thread1 = True
+ if found_thread1:
+ stack.append(line)
+
+ if not stack:
+ return backtrace
+ return "\n".join(stack)
+
+
+# XXX
+def colorize_backtrace(text):
+ filters = [
+ # Function names and the class they belong to
+ (re.compile(r"^(#[0-9]+ .*?)([a-zA-Z0-9_:\.@]+)(\s?\()", flags=re.MULTILINE), r"\1[[c:cyan]]\2[[rst]]\3"),
+ # Function argument names
+ (re.compile(r"([a-zA-Z0-9_#]*)(\s?=\s?)"), r"[[c:green]]\1[[rst]]\2"),
+ # Stack frame number
+ (re.compile(r"^(#[0-9]+)", flags=re.MULTILINE), r"[[c:red]]\1[[rst]]"),
+ # Thread id colorization
+ (re.compile(r"^([ \*]) ([0-9]+)", flags=re.MULTILINE), r"[[c:light-cyan]]\1 [[c:red]]\2[[rst]]"),
+ # File path and line number
+ (re.compile(r"(\.*[/A-Za-z0-9\+_\.\-]*):(([0-9]+)(:[0-9]+)?)$", flags=re.MULTILINE), r"[[c:light-grey]]\1[[rst]]:[[c:magenta]]\2[[rst]]"),
+ # Addresses
+ (re.compile(r"\b(0x[a-f0-9]{6,})\b"), r"[[c:light-grey]]\1[[rst]]"),
+ ]
+
+ for regex, substitution in filters:
+ text = regex.sub(substitution, text)
+ return text
+
+
+def resolve_addresses(addresses, symbolizer, binary):
+ addresses = list(set(addresses))
+ cmd = [
+ symbolizer,
+ "-demangle",
+ "-obj",
+ binary,
+ ]
+ proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate(input="\n".join(addresses))
+ if proc.returncode:
+ raise Exception("Symbolizer failed with rc:{}\nstderr: {}".format(proc.returncode, err))
+
+ resolved = filter(None, out.split("\n\n"))
+ if len(addresses) != len(resolved):
+ raise Exception("llvm-symbolizer can not extract lines from addresses (count mismatch: {}-{})".format(len(addresses), len(resolved)))
+
+ return {k: v.strip(" \n") for k, v in zip(addresses, resolved)}
diff --git a/library/python/cores/ya.make b/library/python/cores/ya.make
new file mode 100644
index 0000000000..76264e9cce
--- /dev/null
+++ b/library/python/cores/ya.make
@@ -0,0 +1,15 @@
+OWNER(
+ prettyboy
+ g:yatest
+)
+
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+PEERDIR(
+ contrib/python/six
+ library/python/reservoir_sampling
+)
+
+END()
diff --git a/library/python/filelock/__init__.py b/library/python/filelock/__init__.py
new file mode 100644
index 0000000000..f81ff67f37
--- /dev/null
+++ b/library/python/filelock/__init__.py
@@ -0,0 +1,122 @@
+import errno
+import logging
+import os
+import sys
+
+import library.python.windows
+
+logger = logging.getLogger(__name__)
+
+
+def set_close_on_exec(stream):
+ if library.python.windows.on_win():
+ library.python.windows.set_handle_information(stream, inherit=False)
+ else:
+ import fcntl
+ fcntl.fcntl(stream, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+
+
+class AbstractFileLock(object):
+
+ def __init__(self, path):
+ self.path = path
+
+ def acquire(self, blocking=True):
+ raise NotImplementedError
+
+ def release(self):
+ raise NotImplementedError
+
+ def __enter__(self):
+ self.acquire()
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.release()
+
+
+class _NixFileLock(AbstractFileLock):
+
+ def __init__(self, path):
+ super(_NixFileLock, self).__init__(path)
+ from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB
+ self._locker = lambda lock, blocking: flock(lock, LOCK_EX if blocking else LOCK_EX | LOCK_NB)
+ self._unlocker = lambda lock: flock(lock, LOCK_UN)
+ self._lock = open(self.path, 'a')
+ set_close_on_exec(self._lock)
+
+ def acquire(self, blocking=True):
+ import errno
+ try:
+ self._locker(self._lock, blocking)
+ except IOError as e:
+ if e.errno in (errno.EAGAIN, errno.EACCES) and not blocking:
+ return False
+ raise
+ return True
+
+ def release(self):
+ self._unlocker(self._lock)
+
+ def __del__(self):
+ if hasattr(self, "_lock"):
+ self._lock.close()
+
+
+class _WinFileLock(AbstractFileLock):
+ """
+ Based on LockFile / UnlockFile from win32 API
+ https://msdn.microsoft.com/en-us/library/windows/desktop/aa365202(v=vs.85).aspx
+ """
+
+ _LOCKED_BYTES_NUM = 1
+
+ def __init__(self, path):
+ super(_WinFileLock, self).__init__(path)
+ self._lock = None
+ try:
+ with file(path, 'w') as lock_file:
+ lock_file.write(" " * self._LOCKED_BYTES_NUM)
+ except IOError as e:
+ if e.errno != errno.EACCES or not os.path.isfile(path):
+ raise
+
+ def acquire(self, blocking=True):
+ self._lock = open(self.path)
+ set_close_on_exec(self._lock)
+
+ import time
+ locked = False
+ while not locked:
+ locked = library.python.windows.lock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False)
+ if locked:
+ return True
+ if blocking:
+ time.sleep(.5)
+ else:
+ return False
+
+ def release(self):
+ if self._lock:
+ library.python.windows.unlock_file(self._lock, 0, self._LOCKED_BYTES_NUM, raises=False)
+ self._lock.close()
+ self._lock = None
+
+
+class FileLock(AbstractFileLock):
+
+ def __init__(self, path):
+ super(FileLock, self).__init__(path)
+
+ if sys.platform.startswith('win'):
+ self._lock = _WinFileLock(path)
+ else:
+ self._lock = _NixFileLock(path)
+
+ def acquire(self, blocking=True):
+ logger.debug('Acquiring filelock (blocking=%s): %s', blocking, self.path)
+ return self._lock.acquire(blocking)
+
+ def release(self):
+ logger.debug('Ensuring filelock released: %s', self.path)
+ return self._lock.release()
diff --git a/library/python/filelock/ut/lib/test_filelock.py b/library/python/filelock/ut/lib/test_filelock.py
new file mode 100644
index 0000000000..1b11d89123
--- /dev/null
+++ b/library/python/filelock/ut/lib/test_filelock.py
@@ -0,0 +1,83 @@
+import os
+import time
+import logging
+import multiprocessing
+import tempfile
+import threading
+
+import library.python.filelock
+
+
+def _acquire_lock(lock_path, out_file_path):
+ with library.python.filelock.FileLock(lock_path):
+ with open(out_file_path, "a") as out:
+ out.write("{}:{}\n".format(os.getpid(), time.time()))
+ time.sleep(2)
+
+
+def test_filelock():
+ temp_dir = tempfile.mkdtemp()
+ lock_path = os.path.join(temp_dir, "file.lock")
+ out_file_path = os.path.join(temp_dir, "out.txt")
+
+ process_count = 5
+ processes = []
+ for i in range(process_count):
+ process = multiprocessing.Process(target=_acquire_lock, args=(lock_path, out_file_path))
+ process.start()
+ processes.append(process)
+
+ for process in processes:
+ process.join()
+
+ pids = []
+ times = []
+ with open(out_file_path) as out:
+ content = out.read()
+ logging.info("Times:\n%s", content)
+ for line in content.strip().split("\n"):
+ pid, time_val = line.split(":")
+ pids.append(pid)
+ times.append(float(time_val))
+
+ assert len(set(pids)) == process_count
+ time1 = times.pop()
+ while times:
+ time2 = times.pop()
+ assert int(time1) - int(time2) >= 2
+ time1 = time2
+
+
+def test_filelock_init_acquired():
+ temp_dir = tempfile.mkdtemp()
+ lock_path = os.path.join(temp_dir, "file.lock")
+
+ with library.python.filelock.FileLock(lock_path):
+ sublock = library.python.filelock.FileLock(lock_path)
+ del sublock
+
+
+def test_concurrent_lock():
+ filename = 'con.lock'
+
+ def lock():
+ l = library.python.filelock.FileLock(filename)
+ time.sleep(1)
+ l.acquire()
+ l.release()
+ try:
+ os.unlink(filename)
+ except OSError:
+ pass
+
+ threads = []
+ for i in range(100):
+ t = threading.Thread(target=lock)
+ t.daemon = True
+ threads.append(t)
+
+ for t in threads:
+ t.start()
+
+ for t in threads:
+ t.join()
diff --git a/library/python/filelock/ut/lib/ya.make b/library/python/filelock/ut/lib/ya.make
new file mode 100644
index 0000000000..f3f9da5a67
--- /dev/null
+++ b/library/python/filelock/ut/lib/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+TEST_SRCS(test_filelock.py)
+
+PEERDIR(
+ library/python/filelock
+)
+
+END()
diff --git a/library/python/filelock/ut/py2/ya.make b/library/python/filelock/ut/py2/ya.make
new file mode 100644
index 0000000000..30b54e0232
--- /dev/null
+++ b/library/python/filelock/ut/py2/ya.make
@@ -0,0 +1,9 @@
+OWNER(g:yatool)
+
+PY2TEST()
+
+PEERDIR(
+ library/python/filelock/ut/lib
+)
+
+END()
diff --git a/library/python/filelock/ut/py3/ya.make b/library/python/filelock/ut/py3/ya.make
new file mode 100644
index 0000000000..05a856a19a
--- /dev/null
+++ b/library/python/filelock/ut/py3/ya.make
@@ -0,0 +1,9 @@
+OWNER(g:yatool)
+
+PY3TEST()
+
+PEERDIR(
+ library/python/filelock/ut/lib
+)
+
+END()
diff --git a/library/python/filelock/ut/ya.make b/library/python/filelock/ut/ya.make
new file mode 100644
index 0000000000..9280ea415e
--- /dev/null
+++ b/library/python/filelock/ut/ya.make
@@ -0,0 +1,7 @@
+OWNER(g:yatool)
+
+RECURSE(
+ lib
+ py2
+ py3
+)
diff --git a/library/python/filelock/ya.make b/library/python/filelock/ya.make
new file mode 100644
index 0000000000..958cc1866f
--- /dev/null
+++ b/library/python/filelock/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+PEERDIR(
+ library/python/windows
+)
+
+END()
diff --git a/library/python/find_root/__init__.py b/library/python/find_root/__init__.py
new file mode 100644
index 0000000000..6da604d62e
--- /dev/null
+++ b/library/python/find_root/__init__.py
@@ -0,0 +1,20 @@
+import os
+
+
+def is_root(path):
+ return os.path.exists(os.path.join(path, ".arcadia.root")) or os.path.exists(os.path.join(path, 'devtools', 'ya', 'ya.conf.json'))
+
+
+def detect_root(path, detector=is_root):
+ return _find_path(path, detector)
+
+
+def _find_path(starts_from, check):
+ p = os.path.realpath(starts_from)
+ while True:
+ if check(p):
+ return p
+ next_p = os.path.dirname(p)
+ if next_p == p:
+ return None
+ p = next_p
diff --git a/library/python/find_root/ya.make b/library/python/find_root/ya.make
new file mode 100644
index 0000000000..beaa8e3c52
--- /dev/null
+++ b/library/python/find_root/ya.make
@@ -0,0 +1,7 @@
+PY23_LIBRARY()
+
+OWNER(g:yatool)
+
+PY_SRCS(__init__.py)
+
+END()
diff --git a/library/python/fs/__init__.py b/library/python/fs/__init__.py
new file mode 100644
index 0000000000..b1b7cde079
--- /dev/null
+++ b/library/python/fs/__init__.py
@@ -0,0 +1,501 @@
+# coding: utf-8
+
+import codecs
+import errno
+import logging
+import os
+import random
+import shutil
+import six
+import stat
+import sys
+
+import library.python.func
+import library.python.strings
+import library.python.windows
+
+logger = logging.getLogger(__name__)
+
+
+try:
+ WindowsError
+except NameError:
+ WindowsError = None
+
+
+_diehard_win_tries = 10
+errorfix_win = library.python.windows.errorfix
+
+
+class CustomFsError(OSError):
+ def __init__(self, errno, message='', filename=None):
+ super(CustomFsError, self).__init__(message)
+ self.errno = errno
+ self.strerror = os.strerror(errno)
+ self.filename = filename
+
+
+# Directories creation
+# If dst is already exists and is a directory - does nothing
+# Throws OSError
+@errorfix_win
+def ensure_dir(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST or not os.path.isdir(path):
+ raise
+
+
+# Directories creation
+# If dst is already exists and is a directory - does nothing
+# Returns path
+# Throws OSError
+@errorfix_win
+def create_dirs(path):
+ ensure_dir(path)
+ return path
+
+
+# Atomic file/directory move (rename)
+# Doesn't guarantee dst replacement
+# Atomic if no device boundaries are crossed
+# Depends on ctypes on Windows
+# Throws OSError
+# On Unix, if dst exists:
+# if dst is file or empty dir - replaces it
+# if src is dir and dst is not dir - throws OSError (errno ENOTDIR)
+# if src is dir and dst is non-empty dir - throws OSError (errno ENOTEMPTY)
+# if src is file and dst is dir - throws OSError (errno EISDIR)
+# On Windows, if dst exists - throws OSError (errno EEXIST)
+@errorfix_win
+@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
+def move(src, dst):
+ os.rename(src, dst)
+
+
+# Atomic replacing file move (rename)
+# Replaces dst if exists and not a dir
+# Doesn't guarantee dst dir replacement
+# Atomic if no device boundaries are crossed
+# Depends on ctypes on Windows
+# Throws OSError
+# On Unix, if dst exists:
+# if dst is file - replaces it
+# if dst is dir - throws OSError (errno EISDIR)
+# On Windows, if dst exists:
+# if dst is file - replaces it
+# if dst is dir - throws OSError (errno EACCES)
+@errorfix_win
+@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
+def replace_file(src, dst):
+ if library.python.windows.on_win():
+ library.python.windows.replace_file(src, dst)
+ else:
+ os.rename(src, dst)
+
+
+# File/directory replacing move (rename)
+# Removes dst if exists
+# Non-atomic
+# Depends on ctypes on Windows
+# Throws OSError
+@errorfix_win
+def replace(src, dst):
+ try:
+ move(src, dst)
+ except OSError as e:
+ if e.errno not in (errno.EEXIST, errno.EISDIR, errno.ENOTDIR, errno.ENOTEMPTY):
+ raise
+ remove_tree(dst)
+ move(src, dst)
+
+
+# Atomic file remove
+# Throws OSError
+@errorfix_win
+@library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries)
+def remove_file(path):
+ os.remove(path)
+
+
+# Atomic empty directory remove
+# Throws OSError
+@errorfix_win
+@library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries)
+def remove_dir(path):
+ os.rmdir(path)
+
+
+def fix_path_encoding(path):
+ return library.python.strings.to_str(path, library.python.strings.fs_encoding())
+
+
+# File/directory remove
+# Non-atomic
+# Throws OSError, AssertionError
+@errorfix_win
+def remove_tree(path):
+ @library.python.windows.diehard(library.python.windows.RETRIABLE_DIR_ERRORS, tries=_diehard_win_tries)
+ def rmtree(path):
+ if library.python.windows.on_win():
+ library.python.windows.rmtree(path)
+ else:
+ shutil.rmtree(fix_path_encoding(path))
+
+ st = os.lstat(path)
+ if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode):
+ remove_file(path)
+ elif stat.S_ISDIR(st.st_mode):
+ rmtree(path)
+ else:
+ assert False
+
+
+# File/directory remove ignoring errors
+# Non-atomic
+@errorfix_win
+def remove_tree_safe(path):
+ try:
+ st = os.lstat(path)
+ if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode):
+ os.remove(path)
+ elif stat.S_ISDIR(st.st_mode):
+ shutil.rmtree(fix_path_encoding(path), ignore_errors=True)
+ # XXX
+ except UnicodeDecodeError as e:
+ logging.exception(u'remove_tree_safe with argument %s raise exception: %s', path, e)
+ raise
+ except OSError:
+ pass
+
+
+# File/directory remove
+# If path doesn't exist - does nothing
+# Non-atomic
+# Throws OSError, AssertionError
+@errorfix_win
+def ensure_removed(path):
+ try:
+ remove_tree(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+
+# Atomic file hardlink
+# Dst must not exist
+# Depends on ctypes on Windows
+# Throws OSError
+# If dst exists - throws OSError (errno EEXIST)
+@errorfix_win
+def hardlink(src, lnk):
+ if library.python.windows.on_win():
+ library.python.windows.hardlink(src, lnk)
+ else:
+ os.link(src, lnk)
+
+
+@errorfix_win
+def hardlink_or_copy(src, lnk):
+ def should_fallback_to_copy(exc):
+ if WindowsError is not None and isinstance(exc, WindowsError) and exc.winerror == 1142: # too many hardlinks
+ return True
+ # cross-device hardlink or too many hardlinks, or some known WSL error
+ if isinstance(exc, OSError) and exc.errno in (
+ errno.EXDEV,
+ errno.EMLINK,
+ errno.EINVAL,
+ errno.EACCES,
+ errno.EPERM,
+ ):
+ return True
+ return False
+
+ try:
+ hardlink(src, lnk)
+ except Exception as e:
+ logger.debug('Failed to hardlink %s to %s with error %s, will copy it', src, lnk, repr(e))
+ if should_fallback_to_copy(e):
+ copy2(src, lnk, follow_symlinks=False)
+ else:
+ raise
+
+
+# Atomic file/directory symlink (Unix only)
+# Dst must not exist
+# Throws OSError
+# If dst exists - throws OSError (errno EEXIST)
+@errorfix_win
+def symlink(src, lnk):
+ if library.python.windows.on_win():
+ library.python.windows.run_disabled(src, lnk)
+ else:
+ os.symlink(src, lnk)
+
+
+# shutil.copy2 with follow_symlinks=False parameter (Unix only)
+def copy2(src, lnk, follow_symlinks=True):
+ if six.PY3:
+ shutil.copy2(src, lnk, follow_symlinks=follow_symlinks)
+ return
+
+ if follow_symlinks or not os.path.islink(src):
+ shutil.copy2(src, lnk)
+ return
+
+ symlink(os.readlink(src), lnk)
+
+
+# Recursively hardlink directory
+# Uses plain hardlink for files
+# Dst must not exist
+# Non-atomic
+# Throws OSError
+@errorfix_win
+def hardlink_tree(src, dst):
+ if not os.path.exists(src):
+ raise CustomFsError(errno.ENOENT, filename=src)
+ if os.path.isfile(src):
+ hardlink(src, dst)
+ return
+ for dirpath, _, filenames in walk_relative(src):
+ src_dirpath = os.path.join(src, dirpath) if dirpath != '.' else src
+ dst_dirpath = os.path.join(dst, dirpath) if dirpath != '.' else dst
+ os.mkdir(dst_dirpath)
+ for filename in filenames:
+ hardlink(os.path.join(src_dirpath, filename), os.path.join(dst_dirpath, filename))
+
+
+# File copy
+# throws EnvironmentError (OSError, IOError)
+@errorfix_win
+def copy_file(src, dst, copy_function=shutil.copy2):
+ if os.path.isdir(dst):
+ raise CustomFsError(errno.EISDIR, filename=dst)
+ copy_function(src, dst)
+
+
+# File/directory copy
+# throws EnvironmentError (OSError, IOError, shutil.Error)
+@errorfix_win
+def copy_tree(src, dst, copy_function=shutil.copy2):
+ if os.path.isfile(src):
+ copy_file(src, dst, copy_function=copy_function)
+ return
+ copytree3(src, dst, copy_function=copy_function)
+
+
+# File read
+# Throws OSError
+@errorfix_win
+def read_file(path, binary=True):
+ with open(path, 'r' + ('b' if binary else '')) as f:
+ return f.read()
+
+
+# Decoding file read
+# Throws OSError
+@errorfix_win
+def read_file_unicode(path, binary=True, enc='utf-8'):
+ if not binary:
+ if six.PY2:
+ with open(path, 'r') as f:
+ return library.python.strings.to_unicode(f.read(), enc)
+ else:
+ with open(path, 'r', encoding=enc) as f:
+ return f.read()
+ # codecs.open is always binary
+ with codecs.open(path, 'r', encoding=enc, errors=library.python.strings.ENCODING_ERRORS_POLICY) as f:
+ return f.read()
+
+
+@errorfix_win
+def open_file(*args, **kwargs):
+ return (
+ library.python.windows.open_file(*args, **kwargs) if library.python.windows.on_win() else open(*args, **kwargs)
+ )
+
+
+# Atomic file write
+# Throws OSError
+@errorfix_win
+def write_file(path, data, binary=True):
+ dir_path = os.path.dirname(path)
+ if dir_path:
+ ensure_dir(dir_path)
+ tmp_path = path + '.tmp.' + str(random.random())
+ with open_file(tmp_path, 'w' + ('b' if binary else '')) as f:
+ if not isinstance(data, bytes) and binary:
+ data = data.encode('UTF-8')
+ f.write(data)
+ replace_file(tmp_path, path)
+
+
+# File size
+# Throws OSError
+@errorfix_win
+def get_file_size(path):
+ return os.path.getsize(path)
+
+
+# File/directory size
+# Non-recursive mode for directory counts size for immediates
+# While raise_all_errors is set to False, file size fallbacks to zero in case of getsize errors
+# Throws OSError
+@errorfix_win
+def get_tree_size(path, recursive=False, raise_all_errors=False):
+ if os.path.isfile(path):
+ return get_file_size(path)
+ total_size = 0
+ for dir_path, _, files in os.walk(path):
+ for f in files:
+ fp = os.path.join(dir_path, f)
+ try:
+ total_size += get_file_size(fp)
+ except OSError as e:
+ if raise_all_errors:
+ raise
+ logger.debug("Cannot calculate file size: %s", e)
+ if not recursive:
+ break
+ return total_size
+
+
+# Directory copy ported from Python 3
+def copytree3(
+ src,
+ dst,
+ symlinks=False,
+ ignore=None,
+ copy_function=shutil.copy2,
+ ignore_dangling_symlinks=False,
+ dirs_exist_ok=False,
+):
+ """Recursively copy a directory tree.
+
+ The copytree3 is a port of shutil.copytree function from python-3.2.
+ It has additional useful parameters and may be helpful while we are
+ on python-2.x. It has to be removed as soon as we have moved to
+ python-3.2 or higher.
+
+ The destination directory must not already exist.
+ If exception(s) occur, an Error is raised with a list of reasons.
+
+ If the optional symlinks flag is true, symbolic links in the
+ source tree result in symbolic links in the destination tree; if
+ it is false, the contents of the files pointed to by symbolic
+ links are copied. If the file pointed by the symlink doesn't
+ exist, an exception will be added in the list of errors raised in
+ an Error exception at the end of the copy process.
+
+ You can set the optional ignore_dangling_symlinks flag to true if you
+ want to silence this exception. Notice that this has no effect on
+ platforms that don't support os.symlink.
+
+ The optional ignore argument is a callable. If given, it
+ is called with the `src` parameter, which is the directory
+ being visited by copytree3(), and `names` which is the list of
+ `src` contents, as returned by os.listdir():
+
+ callable(src, names) -> ignored_names
+
+ Since copytree3() is called recursively, the callable will be
+ called once for each directory that is copied. It returns a
+ list of names relative to the `src` directory that should
+ not be copied.
+
+ The optional copy_function argument is a callable that will be used
+ to copy each file. It will be called with the source path and the
+ destination path as arguments. By default, copy2() is used, but any
+ function that supports the same signature (like copy()) can be used.
+
+ """
+ names = os.listdir(src)
+ if ignore is not None:
+ ignored_names = ignore(src, names)
+ else:
+ ignored_names = set()
+
+ if not (dirs_exist_ok and os.path.isdir(dst)):
+ os.makedirs(dst)
+
+ errors = []
+ for name in names:
+ if name in ignored_names:
+ continue
+ srcname = os.path.join(src, name)
+ dstname = os.path.join(dst, name)
+ try:
+ if os.path.islink(srcname):
+ linkto = os.readlink(srcname)
+ if symlinks:
+ # We can't just leave it to `copy_function` because legacy
+ # code with a custom `copy_function` may rely on copytree3
+ # doing the right thing.
+ os.symlink(linkto, dstname)
+ else:
+ # ignore dangling symlink if the flag is on
+ if not os.path.exists(linkto) and ignore_dangling_symlinks:
+ continue
+ # otherwise let the copy occurs. copy2 will raise an error
+ copy_function(srcname, dstname)
+ elif os.path.isdir(srcname):
+ copytree3(srcname, dstname, symlinks, ignore, copy_function, dirs_exist_ok=dirs_exist_ok)
+ else:
+ # Will raise a SpecialFileError for unsupported file types
+ copy_function(srcname, dstname)
+ # catch the Error from the recursive copytree3 so that we can
+ # continue with other files
+ except shutil.Error as err:
+ errors.extend(err.args[0])
+ except EnvironmentError as why:
+ errors.append((srcname, dstname, str(why)))
+ try:
+ shutil.copystat(src, dst)
+ except OSError as why:
+ if WindowsError is not None and isinstance(why, WindowsError):
+ # Copying file access times may fail on Windows
+ pass
+ else:
+ errors.extend((src, dst, str(why)))
+ if errors:
+ raise shutil.Error(errors)
+
+
+def walk_relative(path, topdown=True, onerror=None, followlinks=False):
+ for dirpath, dirnames, filenames in os.walk(path, topdown=topdown, onerror=onerror, followlinks=followlinks):
+ yield os.path.relpath(dirpath, path), dirnames, filenames
+
+
+def supports_clone():
+ if 'darwin' in sys.platform:
+ import platform
+
+ return list(map(int, platform.mac_ver()[0].split('.'))) >= [10, 13]
+ return False
+
+
+def commonpath(paths):
+ assert paths
+ if len(paths) == 1:
+ return next(iter(paths))
+
+ split_paths = [path.split(os.sep) for path in paths]
+ smin = min(split_paths)
+ smax = max(split_paths)
+
+ common = smin
+ for i, c in enumerate(smin):
+ if c != smax[i]:
+ common = smin[:i]
+ break
+
+ return os.path.sep.join(common)
+
+
+def set_execute_bits(filename):
+ stm = os.stat(filename).st_mode
+ exe = stm | 0o111
+ if stm != exe:
+ os.chmod(filename, exe)
diff --git a/library/python/fs/clonefile.pyx b/library/python/fs/clonefile.pyx
new file mode 100644
index 0000000000..830bb894f2
--- /dev/null
+++ b/library/python/fs/clonefile.pyx
@@ -0,0 +1,18 @@
+import six
+
+cdef extern from "sys/clonefile.h" nogil:
+ int clonefile(const char * src, const char * dst, int flags)
+
+cdef extern from "Python.h":
+ ctypedef struct PyObject
+ cdef PyObject *PyExc_OSError
+ PyObject *PyErr_SetFromErrno(PyObject *)
+
+cdef int _macos_clone_file(const char* src, const char* dst) except? 0:
+ if clonefile(src, dst, 0) == -1:
+ PyErr_SetFromErrno(PyExc_OSError)
+ return 0
+ return 1
+
+def macos_clone_file(src, dst):
+ return _macos_clone_file(six.ensure_binary(src), six.ensure_binary(dst)) != 0
diff --git a/library/python/fs/test/test_fs.py b/library/python/fs/test/test_fs.py
new file mode 100644
index 0000000000..9e2c70c069
--- /dev/null
+++ b/library/python/fs/test/test_fs.py
@@ -0,0 +1,1037 @@
+# coding=utf-8
+
+import errno
+import os
+import pytest
+import shutil
+import six
+
+import library.python.fs
+import library.python.strings
+import library.python.tmp
+import library.python.windows
+
+import yatest.common
+
+
+def in_env(case):
+ def wrapped_case(*args, **kwargs):
+ with library.python.tmp.temp_dir() as temp_dir:
+ case(lambda path: os.path.join(temp_dir, path))
+
+ return wrapped_case
+
+
+def mkfile(path, data=''):
+ with open(path, 'wb') as f:
+ if data:
+ f.write(data) if isinstance(data, six.binary_type) else f.write(
+ data.encode(library.python.strings.fs_encoding())
+ )
+
+
+def mktree_example(path, name):
+ os.mkdir(path(name))
+ mkfile(path(name + '/file1'), 'FILE1')
+ os.mkdir(path(name + '/dir1'))
+ os.mkdir(path(name + '/dir2'))
+ mkfile(path(name + '/dir2/file2'), 'FILE2')
+ mkfile(path(name + '/dir2/file3'), 'FILE3')
+
+
+def file_data(path):
+ with open(path, 'rb') as f:
+ return f.read().decode('utf-8')
+
+
+def serialize_tree(path):
+ if os.path.isfile(path):
+ return file_data(path)
+ data = {'dirs': set(), 'files': {}}
+ for dirpath, dirnames, filenames in os.walk(path):
+ dirpath_rel = os.path.relpath(dirpath, path)
+ if dirpath_rel == '.':
+ dirpath_rel = ''
+ data['dirs'].update(set(os.path.join(dirpath_rel, x) for x in dirnames))
+ data['files'].update({os.path.join(dirpath_rel, x): file_data(os.path.join(dirpath, x)) for x in filenames})
+ return data
+
+
+def trees_equal(dir1, dir2):
+ return serialize_tree(dir1) == serialize_tree(dir2)
+
+
+def inodes_unsupported():
+ return library.python.windows.on_win()
+
+
+def inodes_equal(path1, path2):
+ return os.stat(path1).st_ino == os.stat(path2).st_ino
+
+
+def gen_error_access_denied():
+ if library.python.windows.on_win():
+ err = WindowsError()
+ err.errno = errno.EACCES
+ err.strerror = ''
+ err.winerror = library.python.windows.ERRORS['ACCESS_DENIED']
+ else:
+ err = OSError()
+ err.errno = errno.EACCES
+ err.strerror = os.strerror(err.errno)
+ err.filename = 'unknown/file'
+ raise err
+
+
+def test_errorfix_win():
+ @library.python.fs.errorfix_win
+ def erroneous_func():
+ gen_error_access_denied()
+
+ with pytest.raises(OSError) as errinfo:
+ erroneous_func()
+ assert errinfo.value.errno == errno.EACCES
+ assert errinfo.value.filename == 'unknown/file'
+ # See transcode_error, which encodes strerror, in library/python/windows/__init__.py
+ assert isinstance(errinfo.value.strerror, (six.binary_type, six.text_type))
+ assert errinfo.value.strerror
+
+
+def test_custom_fs_error():
+ with pytest.raises(OSError) as errinfo:
+ raise library.python.fs.CustomFsError(errno.EACCES, filename='some/file')
+ assert errinfo.value.errno == errno.EACCES
+ # See transcode_error, which encodes strerror, in library/python/windows/__init__.py
+ assert isinstance(errinfo.value.strerror, (six.binary_type, six.text_type))
+ assert errinfo.value.filename == 'some/file'
+
+
+@in_env
+def test_ensure_dir(path):
+ library.python.fs.ensure_dir(path('dir/subdir'))
+ assert os.path.isdir(path('dir'))
+ assert os.path.isdir(path('dir/subdir'))
+
+
+@in_env
+def test_ensure_dir_exists(path):
+ os.makedirs(path('dir/subdir'))
+ library.python.fs.ensure_dir(path('dir/subdir'))
+ assert os.path.isdir(path('dir'))
+ assert os.path.isdir(path('dir/subdir'))
+
+
+@in_env
+def test_ensure_dir_exists_partly(path):
+ os.mkdir(path('dir'))
+ library.python.fs.ensure_dir(path('dir/subdir'))
+ assert os.path.isdir(path('dir'))
+ assert os.path.isdir(path('dir/subdir'))
+
+
+@in_env
+def test_ensure_dir_exists_file(path):
+ mkfile(path('dir'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.ensure_dir(path('dir/subdir'))
+ # ENOENT on Windows!
+ assert errinfo.value.errno in (errno.ENOTDIR, errno.ENOENT)
+ assert os.path.isfile(path('dir'))
+
+
+@in_env
+def test_create_dirs(path):
+ assert library.python.fs.create_dirs(path('dir/subdir')) == path('dir/subdir')
+ assert os.path.isdir(path('dir'))
+ assert os.path.isdir(path('dir/subdir'))
+
+
+@in_env
+def test_move_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.move(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_move_file_no_src(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_move_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst'), 'DST')
+ if library.python.windows.on_win():
+ # move is platform-dependent, use replace_file for dst replacement on all platforms
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'DST'
+ else:
+ library.python.fs.move(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_move_file_exists_dir_empty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR)
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_move_file_exists_dir_nonempty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EEXIST, errno.EISDIR)
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/dst_file'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_move_dir(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ library.python.fs.move(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@in_env
+def test_move_dir_exists_empty(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ os.mkdir(path('dst'))
+ if library.python.windows.on_win():
+ # move is platform-dependent, use non-atomic replace for directory replacement
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src_file'))
+ else:
+ library.python.fs.move(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@in_env
+def test_move_dir_exists_nonempty(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EEXIST, errno.ENOTEMPTY)
+ assert os.path.isdir(path('src'))
+ assert os.path.isfile(path('src/src_file'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src_file'))
+ assert os.path.isfile(path('dst/dst_file'))
+
+
+@in_env
+def test_move_dir_exists_file(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ mkfile(path('dst'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.move(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EEXIST, errno.ENOTDIR)
+ assert os.path.isdir(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'DST'
+
+
+@in_env
+def test_replace_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.replace_file(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+ mkfile(path('src'), 'SRC')
+ library.python.fs.replace(path('src'), path('dst2'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst2'))
+ assert file_data(path('dst2')) == 'SRC'
+
+
+@in_env
+def test_replace_file_no_src(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.replace_file(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.ENOENT
+
+ with pytest.raises(OSError) as errinfo2:
+ library.python.fs.replace(path('src'), path('dst2'))
+ assert errinfo2.value.errno == errno.ENOENT
+
+
+@in_env
+def test_replace_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst'), 'DST')
+ library.python.fs.replace_file(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst2'), 'DST')
+ library.python.fs.replace(path('src'), path('dst2'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst2'))
+ assert file_data(path('dst2')) == 'SRC'
+
+
+@in_env
+def test_replace_file_exists_dir_empty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.replace_file(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EISDIR, errno.EACCES)
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_replace_file_exists_dir_empty_overwrite(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_replace_file_exists_dir_nonempty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.replace_file(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EISDIR, errno.EACCES)
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/dst_file'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_replace_file_exists_dir_nonempty_overwrite(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_replace_dir(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@in_env
+def test_replace_dir_exists_empty(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ os.mkdir(path('dst'))
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@in_env
+def test_replace_dir_exists_nonempty(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+ assert not os.path.isfile(path('dst/dst_file'))
+
+
+@in_env
+def test_replace_dir_exists_file(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ mkfile(path('dst'), 'DST')
+ library.python.fs.replace(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@in_env
+def test_remove_file(path):
+ mkfile(path('path'))
+ library.python.fs.remove_file(path('path'))
+ assert not os.path.exists(path('path'))
+
+
+@in_env
+def test_remove_file_no(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.remove_file(path('path'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_remove_file_exists_dir(path):
+ os.mkdir(path('path'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.remove_file(path('path'))
+ assert errinfo.value.errno in (errno.EISDIR, errno.EACCES)
+ assert os.path.isdir(path('path'))
+
+
+@in_env
+def test_remove_dir(path):
+ os.mkdir(path('path'))
+ library.python.fs.remove_dir(path('path'))
+ assert not os.path.exists(path('path'))
+
+
+@in_env
+def test_remove_dir_no(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.remove_dir(path('path'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_remove_dir_exists_file(path):
+ mkfile(path('path'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.remove_dir(path('path'))
+ assert errinfo.value.errno in (errno.ENOTDIR, errno.EINVAL)
+ assert os.path.isfile(path('path'))
+
+
+@in_env
+def test_remove_tree(path):
+ mktree_example(path, 'path')
+ library.python.fs.remove_tree(path('path'))
+ assert not os.path.exists(path('path'))
+
+
+@in_env
+def test_remove_tree_empty(path):
+ os.mkdir(path('path'))
+ library.python.fs.remove_tree(path('path'))
+ assert not os.path.exists(path('path'))
+
+
+@in_env
+def test_remove_tree_file(path):
+ mkfile(path('path'))
+ library.python.fs.remove_tree(path('path'))
+ assert not os.path.exists(path('path'))
+
+
+@in_env
+def test_remove_tree_no(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.remove_tree(path('path'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_remove_tree_safe(path):
+ library.python.fs.remove_tree_safe(path('path'))
+
+
+@in_env
+def test_ensure_removed(path):
+ library.python.fs.ensure_removed(path('path'))
+
+
+@in_env
+def test_ensure_removed_exists(path):
+ os.makedirs(path('dir/subdir'))
+ library.python.fs.ensure_removed(path('dir'))
+ assert not os.path.exists(path('dir'))
+
+
+@in_env
+def test_ensure_removed_exists_precise(path):
+ os.makedirs(path('dir/subdir'))
+ library.python.fs.ensure_removed(path('dir/subdir'))
+ assert os.path.exists(path('dir'))
+ assert not os.path.exists(path('dir/subdir'))
+
+
+@in_env
+def test_hardlink_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.hardlink(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+ assert inodes_unsupported() or inodes_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_hardlink_file_no_src(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_hardlink_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'DST'
+ assert inodes_unsupported() or not inodes_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_hardlink_file_exists_dir(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_hardlink_dir(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink(path('src'), path('dst'))
+ assert errinfo.value.errno in (errno.EPERM, errno.EACCES)
+ assert os.path.isdir(path('src'))
+ assert not os.path.isdir(path('dst'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert os.path.islink(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_file_no_src(path):
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert not os.path.isfile(path('src'))
+ assert not os.path.isfile(path('dst'))
+ assert os.path.islink(path('dst'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert not os.path.islink(path('dst'))
+ assert file_data(path('dst')) == 'DST'
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_file_exists_dir(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.islink(path('dst'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_dir(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.islink(path('dst'))
+ assert os.path.isfile(path('dst/src_file'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_dir_no_src(path):
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert not os.path.isdir(path('src'))
+ assert not os.path.isdir(path('dst'))
+ assert os.path.islink(path('dst'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_dir_exists(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ os.mkdir(path('dst'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isdir(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.islink(path('dst'))
+ assert not os.path.isfile(path('dst/src_file'))
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_symlink_dir_exists_file(path):
+ os.mkdir(path('src'))
+ mkfile(path('src/src_file'))
+ mkfile(path('dst'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.symlink(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.EEXIST
+ assert os.path.isdir(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert not os.path.islink(path('dst'))
+
+
+@in_env
+def test_hardlink_tree(path):
+ mktree_example(path, 'src')
+ library.python.fs.hardlink_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_hardlink_tree_empty(path):
+ os.mkdir(path('src'))
+ library.python.fs.hardlink_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_hardlink_tree_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.hardlink_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_hardlink_tree_no_src(path):
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink_tree(path('src'), path('dst'))
+ assert errinfo.value.errno == errno.ENOENT
+
+
+@in_env
+def test_hardlink_tree_exists(path):
+ mktree_example(path, 'src')
+ os.mkdir(path('dst_dir'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink_tree(path('src'), path('dst_dir'))
+ assert errinfo.value.errno == errno.EEXIST
+ mkfile(path('dst_file'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink_tree(path('src'), path('dst_file'))
+ assert errinfo.value.errno == errno.EEXIST
+
+
+@in_env
+def test_hardlink_tree_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst_dir'))
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink_tree(path('src'), path('dst_dir'))
+ assert errinfo.value.errno == errno.EEXIST
+ mkfile(path('dst_file'), 'DST')
+ with pytest.raises(OSError) as errinfo:
+ library.python.fs.hardlink_tree(path('src'), path('dst_file'))
+ assert errinfo.value.errno == errno.EEXIST
+
+
+@in_env
+def test_copy_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.copy_file(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_copy_file_no_src(path):
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_file(path('src'), path('dst'))
+
+
+@in_env
+def test_copy_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ mkfile(path('dst'), 'DST')
+ library.python.fs.copy_file(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isfile(path('dst'))
+ assert file_data(path('dst')) == 'SRC'
+
+
+@in_env
+def test_copy_file_exists_dir_empty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_file(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_copy_file_exists_dir_nonempty(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst'))
+ mkfile(path('dst/dst_file'))
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_file(path('src'), path('dst'))
+ assert os.path.isfile(path('src'))
+ assert os.path.isdir(path('dst'))
+ assert os.path.isfile(path('dst/dst_file'))
+ assert not os.path.isfile(path('dst/src'))
+
+
+@in_env
+def test_copy_tree(path):
+ mktree_example(path, 'src')
+ library.python.fs.copy_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_copy_tree_empty(path):
+ os.mkdir(path('src'))
+ library.python.fs.copy_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_copy_tree_file(path):
+ mkfile(path('src'), 'SRC')
+ library.python.fs.copy_tree(path('src'), path('dst'))
+ assert trees_equal(path('src'), path('dst'))
+
+
+@in_env
+def test_copy_tree_no_src(path):
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_tree(path('src'), path('dst'))
+
+
+@in_env
+def test_copy_tree_exists(path):
+ mktree_example(path, 'src')
+ os.mkdir(path('dst_dir'))
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_tree(path('src'), path('dst_dir'))
+ mkfile(path('dst_file'), 'DST')
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_tree(path('src'), path('dst_file'))
+
+
+@in_env
+def test_copy_tree_file_exists(path):
+ mkfile(path('src'), 'SRC')
+ os.mkdir(path('dst_dir'))
+ with pytest.raises(EnvironmentError):
+ library.python.fs.copy_tree(path('src'), path('dst_dir'))
+ mkfile(path('dst_file'), 'DST')
+ library.python.fs.copy_tree(path('src'), path('dst_file'))
+ assert trees_equal(path('src'), path('dst_file'))
+
+
+@in_env
+def test_read_file(path):
+ mkfile(path('src'), 'SRC')
+ assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == 'SRC'
+ assert library.python.fs.read_file(path('src'), binary=False) == 'SRC'
+
+
+@in_env
+def test_read_file_empty(path):
+ mkfile(path('src'))
+ assert library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding()) == ''
+ assert library.python.fs.read_file(path('src'), binary=False) == ''
+
+
+@in_env
+def test_read_file_multiline(path):
+ mkfile(path('src'), 'SRC line 1\nSRC line 2\n')
+ assert (
+ library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding())
+ == 'SRC line 1\nSRC line 2\n'
+ )
+ assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n'
+
+
+@in_env
+def test_read_file_multiline_crlf(path):
+ mkfile(path('src'), 'SRC line 1\r\nSRC line 2\r\n')
+ assert (
+ library.python.fs.read_file(path('src')).decode(library.python.strings.fs_encoding())
+ == 'SRC line 1\r\nSRC line 2\r\n'
+ )
+ if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3
+ assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\nSRC line 2\n'
+ else:
+ assert library.python.fs.read_file(path('src'), binary=False) == 'SRC line 1\r\nSRC line 2\r\n'
+
+
+@in_env
+def test_read_file_unicode(path):
+ s = u'АБВ'
+ mkfile(path('src'), s.encode('utf-8'))
+ mkfile(path('src_cp1251'), s.encode('cp1251'))
+ assert library.python.fs.read_file_unicode(path('src')) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s
+ assert library.python.fs.read_file_unicode(path('src'), binary=False) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s
+
+
+@in_env
+def test_read_file_unicode_empty(path):
+ mkfile(path('src'))
+ mkfile(path('src_cp1251'))
+ assert library.python.fs.read_file_unicode(path('src')) == ''
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == ''
+ assert library.python.fs.read_file_unicode(path('src'), binary=False) == ''
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == ''
+
+
+@in_env
+def test_read_file_unicode_multiline(path):
+ s = u'АБВ\nИ еще\n'
+ mkfile(path('src'), s.encode('utf-8'))
+ mkfile(path('src_cp1251'), s.encode('cp1251'))
+ assert library.python.fs.read_file_unicode(path('src')) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s
+ assert library.python.fs.read_file_unicode(path('src'), binary=False) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s
+
+
+@in_env
+def test_read_file_unicode_multiline_crlf(path):
+ s = u'АБВ\r\nИ еще\r\n'
+ mkfile(path('src'), s.encode('utf-8'))
+ mkfile(path('src_cp1251'), s.encode('cp1251'))
+ assert library.python.fs.read_file_unicode(path('src')) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), enc='cp1251') == s
+ if library.python.windows.on_win() or six.PY3: # universal newlines are by default in text mode in python3
+ assert library.python.fs.read_file_unicode(path('src'), binary=False) == u'АБВ\nИ еще\n'
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == u'АБВ\nИ еще\n'
+ else:
+ assert library.python.fs.read_file_unicode(path('src'), binary=False) == s
+ assert library.python.fs.read_file_unicode(path('src_cp1251'), binary=False, enc='cp1251') == s
+
+
+@in_env
+def test_write_file(path):
+ library.python.fs.write_file(path('src'), 'SRC')
+ assert file_data(path('src')) == 'SRC'
+ library.python.fs.write_file(path('src2'), 'SRC', binary=False)
+ assert file_data(path('src2')) == 'SRC'
+
+
+@in_env
+def test_write_file_empty(path):
+ library.python.fs.write_file(path('src'), '')
+ assert file_data(path('src')) == ''
+ library.python.fs.write_file(path('src2'), '', binary=False)
+ assert file_data(path('src2')) == ''
+
+
+@in_env
+def test_write_file_multiline(path):
+ library.python.fs.write_file(path('src'), 'SRC line 1\nSRC line 2\n')
+ assert file_data(path('src')) == 'SRC line 1\nSRC line 2\n'
+ library.python.fs.write_file(path('src2'), 'SRC line 1\nSRC line 2\n', binary=False)
+ if library.python.windows.on_win():
+ assert file_data(path('src2')) == 'SRC line 1\r\nSRC line 2\r\n'
+ else:
+ assert file_data(path('src2')) == 'SRC line 1\nSRC line 2\n'
+
+
+@in_env
+def test_write_file_multiline_crlf(path):
+ library.python.fs.write_file(path('src'), 'SRC line 1\r\nSRC line 2\r\n')
+ assert file_data(path('src')) == 'SRC line 1\r\nSRC line 2\r\n'
+ library.python.fs.write_file(path('src2'), 'SRC line 1\r\nSRC line 2\r\n', binary=False)
+ if library.python.windows.on_win():
+ assert file_data(path('src2')) == 'SRC line 1\r\r\nSRC line 2\r\r\n'
+ else:
+ assert file_data(path('src2')) == 'SRC line 1\r\nSRC line 2\r\n'
+
+
+@in_env
+def test_get_file_size(path):
+ mkfile(path('one.txt'), '22')
+ assert library.python.fs.get_file_size(path('one.txt')) == 2
+
+
+@in_env
+def test_get_file_size_empty(path):
+ mkfile(path('one.txt'))
+ assert library.python.fs.get_file_size(path('one.txt')) == 0
+
+
+@in_env
+def test_get_tree_size(path):
+ os.makedirs(path('deeper'))
+ mkfile(path('one.txt'), '1')
+ mkfile(path('deeper/two.txt'), '22')
+ assert library.python.fs.get_tree_size(path('one.txt')) == 1
+ assert library.python.fs.get_tree_size(path('')) == 1
+ assert library.python.fs.get_tree_size(path(''), recursive=True) == 3
+
+
+@pytest.mark.skipif(library.python.windows.on_win(), reason='Symlinks disabled on Windows')
+@in_env
+def test_get_tree_size_dangling_symlink(path):
+ os.makedirs(path('deeper'))
+ mkfile(path('one.txt'), '1')
+ mkfile(path('deeper/two.txt'), '22')
+ os.symlink(path('deeper/two.txt'), path("deeper/link.txt"))
+ os.remove(path('deeper/two.txt'))
+ # does not fail
+ assert library.python.fs.get_tree_size(path(''), recursive=True) == 1
+
+
+@pytest.mark.skipif(not library.python.windows.on_win(), reason='Test hardlinks on windows')
+def test_hardlink_or_copy():
+ max_allowed_hard_links = 1023
+
+ def run(hardlink_function, dir):
+ src = r"test.txt"
+ with open(src, "w") as f:
+ f.write("test")
+ for i in range(max_allowed_hard_links + 1):
+ hardlink_function(src, os.path.join(dir, "{}.txt".format(i)))
+
+ dir1 = library.python.fs.create_dirs("one")
+ with pytest.raises(WindowsError) as e:
+ run(library.python.fs.hardlink, dir1)
+ assert e.value.winerror == 1142
+ assert len(os.listdir(dir1)) == max_allowed_hard_links
+
+ dir2 = library.python.fs.create_dirs("two")
+ run(library.python.fs.hardlink_or_copy, dir2)
+ assert len(os.listdir(dir2)) == max_allowed_hard_links + 1
+
+
+def test_remove_tree_unicode():
+ path = u"test_remove_tree_unicode/русский".encode("utf-8")
+ os.makedirs(path)
+ library.python.fs.remove_tree(six.text_type("test_remove_tree_unicode"))
+ assert not os.path.exists("test_remove_tree_unicode")
+
+
+def test_remove_tree_safe_unicode():
+ path = u"test_remove_tree_safe_unicode/русский".encode("utf-8")
+ os.makedirs(path)
+ library.python.fs.remove_tree_safe(six.text_type("test_remove_tree_safe_unicode"))
+ assert not os.path.exists("test_remove_tree_safe_unicode")
+
+
+def test_copy_tree_custom_copy_function():
+ library.python.fs.create_dirs("test_copy_tree_src/deepper/inner")
+ library.python.fs.write_file("test_copy_tree_src/deepper/deepper.txt", "deepper.txt")
+ library.python.fs.write_file("test_copy_tree_src/deepper/inner/inner.txt", "inner.txt")
+ copied = []
+
+ def copy_function(src, dst):
+ shutil.copy2(src, dst)
+ copied.append(dst)
+
+ library.python.fs.copy_tree(
+ "test_copy_tree_src", yatest.common.work_path("test_copy_tree_dst"), copy_function=copy_function
+ )
+ assert len(copied) == 2
+ assert yatest.common.work_path("test_copy_tree_dst/deepper/deepper.txt") in copied
+ assert yatest.common.work_path("test_copy_tree_dst/deepper/inner/inner.txt") in copied
+
+
+def test_copy2():
+ library.python.fs.symlink("non-existent", "link")
+ library.python.fs.copy2("link", "link2", follow_symlinks=False)
+
+ assert os.path.islink("link2")
+ assert os.readlink("link2") == "non-existent"
+
+
+def test_commonpath():
+ pj = os.path.join
+ pja = lambda *x: os.path.abspath(pj(*x))
+
+ assert library.python.fs.commonpath(['a', 'b']) == ''
+ assert library.python.fs.commonpath([pj('t', '1')]) == pj('t', '1')
+ assert library.python.fs.commonpath([pj('t', '1'), pj('t', '2')]) == pj('t')
+ assert library.python.fs.commonpath([pj('t', '1', '2'), pj('t', '1', '2')]) == pj('t', '1', '2')
+ assert library.python.fs.commonpath([pj('t', '1', '1'), pj('t', '1', '2')]) == pj('t', '1')
+ assert library.python.fs.commonpath([pj('t', '1', '1'), pj('t', '1', '2'), pj('t', '1', '3')]) == pj('t', '1')
+
+ assert library.python.fs.commonpath([pja('t', '1', '1'), pja('t', '1', '2')]) == pja('t', '1')
+
+ assert library.python.fs.commonpath({pj('t', '1'), pj('t', '2')}) == pj('t')
diff --git a/library/python/fs/test/ya.make b/library/python/fs/test/ya.make
new file mode 100644
index 0000000000..33e3f5b4ff
--- /dev/null
+++ b/library/python/fs/test/ya.make
@@ -0,0 +1,14 @@
+OWNER(g:yatool)
+
+PY23_TEST()
+
+TEST_SRCS(
+ test_fs.py
+)
+
+PEERDIR(
+ library/python/fs
+ library/python/tmp
+)
+
+END()
diff --git a/library/python/fs/ya.make b/library/python/fs/ya.make
new file mode 100644
index 0000000000..b3c5092c71
--- /dev/null
+++ b/library/python/fs/ya.make
@@ -0,0 +1,23 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+PY_SRCS(
+ __init__.py
+)
+
+IF (OS_DARWIN)
+ PY_SRCS(
+ clonefile.pyx
+ )
+ENDIF()
+
+PEERDIR(
+ library/python/func
+ library/python/strings
+ library/python/windows
+)
+
+END()
+
+RECURSE_FOR_TESTS(test)
diff --git a/library/python/func/__init__.py b/library/python/func/__init__.py
new file mode 100644
index 0000000000..7424361635
--- /dev/null
+++ b/library/python/func/__init__.py
@@ -0,0 +1,170 @@
+import functools
+import threading
+import collections
+
+
+def map0(func, value):
+ return func(value) if value is not None else value
+
+
+def single(x):
+ if len(x) != 1:
+ raise Exception('Length of {} is not equal to 1'.format(x))
+ return x[0]
+
+
+class _Result(object):
+ pass
+
+
+def lazy(func):
+ result = _Result()
+
+ @functools.wraps(func)
+ def wrapper(*args):
+ try:
+ return result.result
+ except AttributeError:
+ result.result = func(*args)
+
+ return result.result
+
+ return wrapper
+
+
+def lazy_property(fn):
+ attr_name = '_lazy_' + fn.__name__
+
+ @property
+ def _lazy_property(self):
+ if not hasattr(self, attr_name):
+ setattr(self, attr_name, fn(self))
+ return getattr(self, attr_name)
+
+ return _lazy_property
+
+
+class classproperty(object):
+ def __init__(self, func):
+ self.func = func
+
+ def __get__(self, _, owner):
+ return self.func(owner)
+
+
+class lazy_classproperty(object):
+ def __init__(self, func):
+ self.func = func
+
+ def __get__(self, _, owner):
+ attr_name = '_lazy_' + self.func.__name__
+
+ if not hasattr(owner, attr_name):
+ setattr(owner, attr_name, self.func(owner))
+ return getattr(owner, attr_name)
+
+
+def memoize(limit=0, thread_local=False):
+ assert limit >= 0
+
+ def decorator(func):
+ memory = {}
+ lock = threading.Lock()
+
+ if limit:
+ keys = collections.deque()
+
+ def get(args):
+ try:
+ return memory[args]
+ except KeyError:
+ with lock:
+ if args not in memory:
+ fargs = args[-1]
+ memory[args] = func(*fargs)
+ keys.append(args)
+ if len(keys) > limit:
+ del memory[keys.popleft()]
+ return memory[args]
+
+ else:
+
+ def get(args):
+ if args not in memory:
+ with lock:
+ if args not in memory:
+ fargs = args[-1]
+ memory[args] = func(*fargs)
+ return memory[args]
+
+ if thread_local:
+
+ @functools.wraps(func)
+ def wrapper(*args):
+ th = threading.current_thread()
+ return get((th.ident, th.name, args))
+
+ else:
+
+ @functools.wraps(func)
+ def wrapper(*args):
+ return get(('', '', args))
+
+ return wrapper
+
+ return decorator
+
+
+# XXX: add test
+def compose(*functions):
+ def compose2(f, g):
+ return lambda x: f(g(x))
+
+ return functools.reduce(compose2, functions, lambda x: x)
+
+
+class Singleton(type):
+ _instances = {}
+
+ def __call__(cls, *args, **kwargs):
+ if cls not in cls._instances:
+ cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+ return cls._instances[cls]
+
+
+def stable_uniq(it):
+ seen = set()
+ res = []
+ for e in it:
+ if e not in seen:
+ res.append(e)
+ seen.add(e)
+ return res
+
+
+def first(it):
+ for d in it:
+ if d:
+ return d
+
+
+def split(data, func):
+ l, r = [], []
+ for e in data:
+ if func(e):
+ l.append(e)
+ else:
+ r.append(e)
+ return l, r
+
+
+def flatten_dict(dd, separator='.', prefix=''):
+ return (
+ {
+ prefix + separator + k if prefix else k: v
+ for kk, vv in dd.items()
+ for k, v in flatten_dict(vv, separator, kk).items()
+ }
+ if isinstance(dd, dict)
+ else {prefix: dd}
+ )
diff --git a/library/python/func/ut/test_func.py b/library/python/func/ut/test_func.py
new file mode 100644
index 0000000000..3c4fad1a07
--- /dev/null
+++ b/library/python/func/ut/test_func.py
@@ -0,0 +1,162 @@
+import pytest
+import threading
+
+import library.python.func as func
+
+
+def test_map0():
+ assert None is func.map0(lambda x: x + 1, None)
+ assert 3 == func.map0(lambda x: x + 1, 2)
+ assert None is func.map0(len, None)
+ assert 2 == func.map0(len, [1, 2])
+
+
+def test_single():
+ assert 1 == func.single([1])
+ with pytest.raises(Exception):
+ assert 1 == func.single([])
+ with pytest.raises(Exception):
+ assert 1 == func.single([1, 2])
+
+
+def test_memoize():
+ class Counter(object):
+ @staticmethod
+ def inc():
+ Counter._qty = getattr(Counter, '_qty', 0) + 1
+ return Counter._qty
+
+ @func.memoize()
+ def t1(a):
+ return a, Counter.inc()
+
+ @func.memoize()
+ def t2(a):
+ return a, Counter.inc()
+
+ @func.memoize()
+ def t3(a):
+ return a, Counter.inc()
+
+ @func.memoize()
+ def t4(a):
+ return a, Counter.inc()
+
+ @func.memoize()
+ def t5(a, b, c):
+ return a + b + c, Counter.inc()
+
+ @func.memoize()
+ def t6():
+ return Counter.inc()
+
+ @func.memoize(limit=2)
+ def t7(a, _b):
+ return a, Counter.inc()
+
+ assert (1, 1) == t1(1)
+ assert (1, 1) == t1(1)
+ assert (2, 2) == t1(2)
+ assert (2, 2) == t1(2)
+
+ assert (1, 3) == t2(1)
+ assert (1, 3) == t2(1)
+ assert (2, 4) == t2(2)
+ assert (2, 4) == t2(2)
+
+ assert (1, 5) == t3(1)
+ assert (1, 5) == t3(1)
+ assert (2, 6) == t3(2)
+ assert (2, 6) == t3(2)
+
+ assert (1, 7) == t4(1)
+ assert (1, 7) == t4(1)
+ assert (2, 8) == t4(2)
+ assert (2, 8) == t4(2)
+
+ assert (6, 9) == t5(1, 2, 3)
+ assert (6, 9) == t5(1, 2, 3)
+ assert (7, 10) == t5(1, 2, 4)
+ assert (7, 10) == t5(1, 2, 4)
+
+ assert 11 == t6()
+ assert 11 == t6()
+
+ assert (1, 12) == t7(1, None)
+ assert (2, 13) == t7(2, None)
+ assert (1, 12) == t7(1, None)
+ assert (2, 13) == t7(2, None)
+ # removed result for (1, None)
+ assert (3, 14) == t7(3, None)
+ assert (1, 15) == t7(1, None)
+
+ class ClassWithMemoizedMethod(object):
+ def __init__(self):
+ self.a = 0
+
+ @func.memoize(True)
+ def t(self, i):
+ self.a += i
+ return i
+
+ obj = ClassWithMemoizedMethod()
+ assert 10 == obj.t(10)
+ assert 10 == obj.a
+ assert 10 == obj.t(10)
+ assert 10 == obj.a
+
+ assert 20 == obj.t(20)
+ assert 30 == obj.a
+ assert 20 == obj.t(20)
+ assert 30 == obj.a
+
+
+def test_first():
+ assert func.first([0, [], (), None, False, {}, 0.0, '1', 0]) == '1'
+ assert func.first([]) is None
+ assert func.first([0]) is None
+
+
+def test_split():
+ assert func.split([1, 1], lambda x: x) == ([1, 1], [])
+ assert func.split([0, 0], lambda x: x) == ([], [0, 0])
+ assert func.split([], lambda x: x) == ([], [])
+ assert func.split([1, 0, 1], lambda x: x) == ([1, 1], [0])
+
+
+def test_flatten_dict():
+ assert func.flatten_dict({"a": 1, "b": 2}) == {"a": 1, "b": 2}
+ assert func.flatten_dict({"a": 1}) == {"a": 1}
+ assert func.flatten_dict({}) == {}
+ assert func.flatten_dict({"a": 1, "b": {"c": {"d": 2}}}) == {"a": 1, "b.c.d": 2}
+ assert func.flatten_dict({"a": 1, "b": {"c": {"d": 2}}}, separator="/") == {"a": 1, "b/c/d": 2}
+
+
+def test_memoize_thread_local():
+ class Counter(object):
+ def __init__(self, s):
+ self.val = s
+
+ def inc(self):
+ self.val += 1
+ return self.val
+
+ @func.memoize(thread_local=True)
+ def get_counter(start):
+ return Counter(start)
+
+ def th_inc():
+ assert get_counter(0).inc() == 1
+ assert get_counter(0).inc() == 2
+ assert get_counter(10).inc() == 11
+ assert get_counter(10).inc() == 12
+
+ th_inc()
+
+ th = threading.Thread(target=th_inc)
+ th.start()
+ th.join()
+
+
+if __name__ == '__main__':
+ pytest.main([__file__])
diff --git a/library/python/func/ut/ya.make b/library/python/func/ut/ya.make
new file mode 100644
index 0000000000..5ec6c1225e
--- /dev/null
+++ b/library/python/func/ut/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY23_TEST()
+
+TEST_SRCS(test_func.py)
+
+PEERDIR(
+ library/python/func
+)
+
+END()
diff --git a/library/python/func/ya.make b/library/python/func/ya.make
new file mode 100644
index 0000000000..9d414a976e
--- /dev/null
+++ b/library/python/func/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/library/python/pytest/__init__.py b/library/python/pytest/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/library/python/pytest/__init__.py
diff --git a/library/python/pytest/allure/conftest.py b/library/python/pytest/allure/conftest.py
new file mode 100644
index 0000000000..0d5cfda1e5
--- /dev/null
+++ b/library/python/pytest/allure/conftest.py
@@ -0,0 +1,8 @@
+import os
+import pytest
+
+
+@pytest.mark.tryfirst
+def pytest_configure(config):
+ if "ALLURE_REPORT_DIR" in os.environ:
+ config.option.allurereportdir = os.environ["ALLURE_REPORT_DIR"]
diff --git a/library/python/pytest/allure/ya.make b/library/python/pytest/allure/ya.make
new file mode 100644
index 0000000000..ab3f449c7f
--- /dev/null
+++ b/library/python/pytest/allure/ya.make
@@ -0,0 +1,11 @@
+PY23_LIBRARY()
+
+OWNER(exprmntr)
+
+PY_SRCS(conftest.py)
+
+PEERDIR(
+ contrib/python/pytest-allure-adaptor
+)
+
+END()
diff --git a/library/python/pytest/context.py b/library/python/pytest/context.py
new file mode 100644
index 0000000000..bfcdae50b5
--- /dev/null
+++ b/library/python/pytest/context.py
@@ -0,0 +1 @@
+Ctx = {}
diff --git a/library/python/pytest/empty/main.c b/library/python/pytest/empty/main.c
new file mode 100644
index 0000000000..9efa08162a
--- /dev/null
+++ b/library/python/pytest/empty/main.c
@@ -0,0 +1,7 @@
+/*
+to be used for build python tests in a stub binary for the case of using system python
+*/
+
+int main(void) {
+ return 0;
+}
diff --git a/library/python/pytest/empty/ya.make b/library/python/pytest/empty/ya.make
new file mode 100644
index 0000000000..8f0fa37e2a
--- /dev/null
+++ b/library/python/pytest/empty/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+OWNER(
+ g:yatool
+ dmitko
+)
+
+SRCS(
+ main.c
+)
+
+END()
diff --git a/library/python/pytest/main.py b/library/python/pytest/main.py
new file mode 100644
index 0000000000..6296bd6f0f
--- /dev/null
+++ b/library/python/pytest/main.py
@@ -0,0 +1,116 @@
+import os
+import sys
+import time
+
+import __res
+
+FORCE_EXIT_TESTSFAILED_ENV = 'FORCE_EXIT_TESTSFAILED'
+
+
+def main():
+ import library.python.pytest.context as context
+ context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time()
+
+ profile = None
+ if '--profile-pytest' in sys.argv:
+ sys.argv.remove('--profile-pytest')
+
+ import pstats
+ import cProfile
+ profile = cProfile.Profile()
+ profile.enable()
+
+ # Reset influencing env. vars
+ # For more info see library/python/testing/yatest_common/yatest/common/errors.py
+ if FORCE_EXIT_TESTSFAILED_ENV in os.environ:
+ del os.environ[FORCE_EXIT_TESTSFAILED_ENV]
+
+ if "Y_PYTHON_CLEAR_ENTRY_POINT" in os.environ:
+ if "Y_PYTHON_ENTRY_POINT" in os.environ:
+ del os.environ["Y_PYTHON_ENTRY_POINT"]
+ del os.environ["Y_PYTHON_CLEAR_ENTRY_POINT"]
+
+ listing_mode = '--collect-only' in sys.argv
+ yatest_runner = os.environ.get('YA_TEST_RUNNER') == '1'
+
+ import pytest
+
+ import library.python.pytest.plugins.collection as collection
+ import library.python.pytest.plugins.ya as ya
+ import library.python.pytest.plugins.conftests as conftests
+
+ import _pytest.assertion
+ from _pytest.monkeypatch import MonkeyPatch
+ from . import rewrite
+ m = MonkeyPatch()
+ m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook", rewrite.AssertionRewritingHook)
+
+ prefix = '__tests__.'
+
+ test_modules = [
+ name[len(prefix):] for name in sys.extra_modules
+ if name.startswith(prefix) and not name.endswith('.conftest')
+ ]
+
+ doctest_packages = __res.find("PY_DOCTEST_PACKAGES") or ""
+ if isinstance(doctest_packages, bytes):
+ doctest_packages = doctest_packages.decode('utf-8')
+ doctest_packages = doctest_packages.split()
+
+ def is_doctest_module(name):
+ for package in doctest_packages:
+ if name == package or name.startswith(str(package) + "."):
+ return True
+ return False
+
+ doctest_modules = [
+ name for name in sys.extra_modules
+ if is_doctest_module(name)
+ ]
+
+ def remove_user_site(paths):
+ site_paths = ('site-packages', 'site-python')
+
+ def is_site_path(path):
+ for p in site_paths:
+ if path.find(p) != -1:
+ return True
+ return False
+
+ new_paths = list(paths)
+ for p in paths:
+ if is_site_path(p):
+ new_paths.remove(p)
+
+ return new_paths
+
+ sys.path = remove_user_site(sys.path)
+ rc = pytest.main(plugins=[
+ collection.CollectionPlugin(test_modules, doctest_modules),
+ ya,
+ conftests,
+ ])
+
+ if rc == 5:
+ # don't care about EXIT_NOTESTSCOLLECTED
+ rc = 0
+
+ if rc == 1 and yatest_runner and not listing_mode and not os.environ.get(FORCE_EXIT_TESTSFAILED_ENV) == '1':
+ # XXX it's place for future improvements
+ # Test wrapper should terminate with 0 exit code if there are common test failures
+ # and report it with trace-file machinery.
+ # However, there are several case when we don't want to suppress exit_code:
+ # - listing machinery doesn't use trace-file currently and rely on stdout and exit_code
+ # - RestartTestException and InfrastructureException required non-zero exit_code to be processes correctly
+ rc = 0
+
+ if profile:
+ profile.disable()
+ ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative')
+ ps.print_stats()
+
+ sys.exit(rc)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/library/python/pytest/plugins/collection.py b/library/python/pytest/plugins/collection.py
new file mode 100644
index 0000000000..e36f47a78f
--- /dev/null
+++ b/library/python/pytest/plugins/collection.py
@@ -0,0 +1,128 @@
+import os
+import sys
+from six import reraise
+
+import py
+
+import pytest # noqa
+import _pytest.python
+import _pytest.doctest
+import json
+import library.python.testing.filter.filter as test_filter
+
+
+class LoadedModule(_pytest.python.Module):
+ def __init__(self, parent, name, **kwargs):
+ self.name = name + '.py'
+ self.session = parent
+ self.parent = parent
+ self.config = parent.config
+ self.keywords = {}
+ self.own_markers = []
+ self.fspath = py.path.local()
+
+ @classmethod
+ def from_parent(cls, **kwargs):
+ namespace = kwargs.pop('namespace', True)
+ kwargs.setdefault('fspath', py.path.local())
+
+ loaded_module = getattr(super(LoadedModule, cls), 'from_parent', cls)(**kwargs)
+ loaded_module.namespace = namespace
+
+ return loaded_module
+
+ @property
+ def _nodeid(self):
+ if os.getenv('CONFTEST_LOAD_POLICY') == 'LOCAL':
+ return self._getobj().__file__
+ else:
+ return self.name
+
+ @property
+ def nodeid(self):
+ return self._nodeid
+
+ def _getobj(self):
+ module_name = self.name[:-len('.py')]
+ if self.namespace:
+ module_name = '__tests__.' + module_name
+ __import__(module_name)
+ return sys.modules[module_name]
+
+
+class DoctestModule(LoadedModule):
+
+ def collect(self):
+ import doctest
+
+ module = self._getobj()
+ # uses internal doctest module parsing mechanism
+ finder = doctest.DocTestFinder()
+ optionflags = _pytest.doctest.get_optionflags(self)
+ runner = doctest.DebugRunner(verbose=0, optionflags=optionflags)
+
+ try:
+ for test in finder.find(module, self.name[:-len('.py')]):
+ if test.examples: # skip empty doctests
+ yield getattr(_pytest.doctest.DoctestItem, 'from_parent', _pytest.doctest.DoctestItem)(
+ name=test.name,
+ parent=self,
+ runner=runner,
+ dtest=test)
+ except Exception:
+ import logging
+ logging.exception('DoctestModule failed, probably you can add NO_DOCTESTS() macro to ya.make')
+ etype, exc, tb = sys.exc_info()
+ msg = 'DoctestModule failed, probably you can add NO_DOCTESTS() macro to ya.make'
+ reraise(etype, type(exc)('{}\n{}'.format(exc, msg)), tb)
+
+
+# NOTE: Since we are overriding collect method of pytest session, pytest hooks are not invoked during collection.
+def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_filename_predicate):
+ if session.config.option.mode == 'list':
+ return not accept_filename_predicate(module.name)
+
+ if filenames_from_full_filters is not None and module.name not in filenames_from_full_filters:
+ return True
+
+ test_file_filter = getattr(session.config.option, 'test_file_filter', None)
+ if test_file_filter is None:
+ return False
+ if module.name != test_file_filter.replace('/', '.'):
+ return True
+ return False
+
+
+class CollectionPlugin(object):
+ def __init__(self, test_modules, doctest_modules):
+ self._test_modules = test_modules
+ self._doctest_modules = doctest_modules
+
+ def pytest_sessionstart(self, session):
+
+ def collect(*args, **kwargs):
+ accept_filename_predicate = test_filter.make_py_file_filter(session.config.option.test_filter)
+ full_test_names_file_path = session.config.option.test_list_path
+ filenames_filter = None
+
+ if full_test_names_file_path and os.path.exists(full_test_names_file_path):
+ with open(full_test_names_file_path, 'r') as afile:
+ # in afile stored 2 dimensional array such that array[modulo_index] contains tests which should be run in this test suite
+ full_names_filter = set(json.load(afile)[int(session.config.option.modulo_index)])
+ filenames_filter = set(map(lambda x: x.split('::')[0], full_names_filter))
+
+ for test_module in self._test_modules:
+ module = LoadedModule.from_parent(name=test_module, parent=session)
+ if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate):
+ yield module
+
+ if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no':
+ module = DoctestModule.from_parent(name=test_module, parent=session)
+ if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate):
+ yield module
+
+ if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no':
+ for doctest_module in self._doctest_modules:
+ yield DoctestModule.from_parent(name=doctest_module, parent=session, namespace=False)
+
+ session.collect = collect
diff --git a/library/python/pytest/plugins/conftests.py b/library/python/pytest/plugins/conftests.py
new file mode 100644
index 0000000000..522041f5a7
--- /dev/null
+++ b/library/python/pytest/plugins/conftests.py
@@ -0,0 +1,50 @@
+import os
+import importlib
+import sys
+import inspect
+
+from pytest import hookimpl
+
+from .fixtures import metrics, links # noqa
+
+orig_getfile = inspect.getfile
+
+
+def getfile(object):
+ res = orig_getfile(object)
+ if inspect.ismodule(object):
+ if not res and getattr(object, '__orig_file__'):
+ res = object.__orig_file__
+ return res
+
+inspect.getfile = getfile
+conftest_modules = []
+
+
+@hookimpl(trylast=True)
+def pytest_load_initial_conftests(early_config, parser, args):
+ conftests = filter(lambda name: name.endswith(".conftest"), sys.extra_modules)
+
+ def conftest_key(name):
+ if not name.startswith("__tests__."):
+ # Make __tests__ come last
+ return "_." + name
+ return name
+
+ for name in sorted(conftests, key=conftest_key):
+ mod = importlib.import_module(name)
+ if os.getenv("CONFTEST_LOAD_POLICY") != "LOCAL":
+ mod.__orig_file__ = mod.__file__
+ mod.__file__ = ""
+ conftest_modules.append(mod)
+ early_config.pluginmanager.consider_conftest(mod)
+
+
+def getconftestmodules(*args, **kwargs):
+ return conftest_modules
+
+
+def pytest_sessionstart(session):
+ # Override filesystem based relevant conftest discovery on the call path
+ assert session.config.pluginmanager
+ session.config.pluginmanager._getconftestmodules = getconftestmodules
diff --git a/library/python/pytest/plugins/fakeid_py2.py b/library/python/pytest/plugins/fakeid_py2.py
new file mode 100644
index 0000000000..8b26148e2e
--- /dev/null
+++ b/library/python/pytest/plugins/fakeid_py2.py
@@ -0,0 +1,2 @@
+# Inc this number to change uid for every PYTEST() target
+fake_id = 0
diff --git a/library/python/pytest/plugins/fakeid_py3.py b/library/python/pytest/plugins/fakeid_py3.py
new file mode 100644
index 0000000000..247cc8b29d
--- /dev/null
+++ b/library/python/pytest/plugins/fakeid_py3.py
@@ -0,0 +1,2 @@
+# Inc this number to change uid for every PY3TEST() target
+fake_id = 10
diff --git a/library/python/pytest/plugins/fixtures.py b/library/python/pytest/plugins/fixtures.py
new file mode 100644
index 0000000000..6f7e0a27e4
--- /dev/null
+++ b/library/python/pytest/plugins/fixtures.py
@@ -0,0 +1,85 @@
+import os
+import pytest
+import six
+
+
+MAX_ALLOWED_LINKS_COUNT = 10
+
+
+@pytest.fixture
+def metrics(request):
+
+ class Metrics(object):
+ @classmethod
+ def set(cls, name, value):
+ assert len(name) <= 128, "Length of the metric name must less than 128"
+ assert type(value) in [int, float], "Metric value must be of type int or float"
+ test_name = request.node.nodeid
+ if test_name not in request.config.test_metrics:
+ request.config.test_metrics[test_name] = {}
+ request.config.test_metrics[test_name][name] = value
+
+ @classmethod
+ def set_benchmark(cls, benchmark_values):
+ # report of google has key 'benchmarks' which is a list of benchmark results
+ # yandex benchmark has key 'benchmark', which is a list of benchmark results
+ # use this to differentiate which kind of result it is
+ if 'benchmarks' in benchmark_values:
+ cls.set_gbenchmark(benchmark_values)
+ else:
+ cls.set_ybenchmark(benchmark_values)
+
+ @classmethod
+ def set_ybenchmark(cls, benchmark_values):
+ for benchmark in benchmark_values["benchmark"]:
+ name = benchmark["name"]
+ for key, value in six.iteritems(benchmark):
+ if key != "name":
+ cls.set("{}_{}".format(name, key), value)
+
+ @classmethod
+ def set_gbenchmark(cls, benchmark_values):
+ time_unit_multipliers = {"ns": 1, "us": 1000, "ms": 1000000}
+ time_keys = {"real_time", "cpu_time"}
+ ignore_keys = {"name", "run_name", "time_unit", "run_type", "repetition_index"}
+ for benchmark in benchmark_values["benchmarks"]:
+ name = benchmark["name"].replace('/', '_') # ci does not work properly with '/' in metric name
+ time_unit_mult = time_unit_multipliers[benchmark.get("time_unit", "ns")]
+ for k, v in six.iteritems(benchmark):
+ if k in time_keys:
+ cls.set("{}_{}".format(name, k), v * time_unit_mult)
+ elif k not in ignore_keys and isinstance(v, (float, int)):
+ cls.set("{}_{}".format(name, k), v)
+ return Metrics
+
+
+@pytest.fixture
+def links(request):
+
+ class Links(object):
+ @classmethod
+ def set(cls, name, path):
+
+ if len(request.config.test_logs[request.node.nodeid]) >= MAX_ALLOWED_LINKS_COUNT:
+ raise Exception("Cannot add more than {} links to test".format(MAX_ALLOWED_LINKS_COUNT))
+
+ reserved_names = ["log", "logsdir", "stdout", "stderr"]
+ if name in reserved_names:
+ raise Exception("Attachment name should not belong to the reserved list: {}".format(", ".join(reserved_names)))
+ output_dir = request.config.ya.output_dir
+
+ if not os.path.exists(path):
+ raise Exception("Path to be attached does not exist: {}".format(path))
+
+ if os.path.isabs(path) and ".." in os.path.relpath(path, output_dir):
+ raise Exception("Test attachment must be inside yatest.common.output_path()")
+
+ request.config.test_logs[request.node.nodeid][name] = path
+
+ @classmethod
+ def get(cls, name):
+ if name not in request.config.test_logs[request.node.nodeid]:
+ raise KeyError("Attachment with name '{}' does not exist".format(name))
+ return request.config.test_logs[request.node.nodeid][name]
+
+ return Links
diff --git a/library/python/pytest/plugins/ya.make b/library/python/pytest/plugins/ya.make
new file mode 100644
index 0000000000..c15d6f759d
--- /dev/null
+++ b/library/python/pytest/plugins/ya.make
@@ -0,0 +1,32 @@
+OWNER(g:yatest)
+
+PY23_LIBRARY()
+
+PY_SRCS(
+ ya.py
+ collection.py
+ conftests.py
+ fixtures.py
+)
+
+PEERDIR(
+ library/python/filelock
+ library/python/find_root
+ library/python/testing/filter
+)
+
+IF (PYTHON2)
+ PY_SRCS(
+ fakeid_py2.py
+ )
+
+ PEERDIR(
+ contrib/python/faulthandler
+ )
+ELSE()
+ PY_SRCS(
+ fakeid_py3.py
+ )
+ENDIF()
+
+END()
diff --git a/library/python/pytest/plugins/ya.py b/library/python/pytest/plugins/ya.py
new file mode 100644
index 0000000000..1bde03042d
--- /dev/null
+++ b/library/python/pytest/plugins/ya.py
@@ -0,0 +1,963 @@
+# coding: utf-8
+
+import base64
+import errno
+import re
+import sys
+import os
+import logging
+import fnmatch
+import json
+import time
+import traceback
+import collections
+import signal
+import inspect
+import warnings
+
+import attr
+import faulthandler
+import py
+import pytest
+import six
+
+import _pytest
+import _pytest._io
+import _pytest.mark
+import _pytest.outcomes
+import _pytest.skipping
+
+from _pytest.warning_types import PytestUnhandledCoroutineWarning
+
+from yatest_lib import test_splitter
+
+try:
+ import resource
+except ImportError:
+ resource = None
+
+try:
+ import library.python.pytest.yatest_tools as tools
+except ImportError:
+ # fallback for pytest script mode
+ import yatest_tools as tools
+
+try:
+ from library.python import filelock
+except ImportError:
+ filelock = None
+
+
+import yatest_lib.tools
+
+import yatest_lib.external as canon
+
+import yatest_lib.ya
+
+from library.python.pytest import context
+
+console_logger = logging.getLogger("console")
+yatest_logger = logging.getLogger("ya.test")
+
+
+_pytest.main.EXIT_NOTESTSCOLLECTED = 0
+SHUTDOWN_REQUESTED = False
+
+pytest_config = None
+
+
+def configure_pdb_on_demand():
+ import signal
+
+ if hasattr(signal, "SIGUSR1"):
+ def on_signal(*args):
+ import ipdb
+ ipdb.set_trace()
+
+ signal.signal(signal.SIGUSR1, on_signal)
+
+
+class CustomImporter(object):
+ def __init__(self, roots):
+ self._roots = roots
+
+ def find_module(self, fullname, package_path=None):
+ for path in self._roots:
+ full_path = self._get_module_path(path, fullname)
+
+ if os.path.exists(full_path) and os.path.isdir(full_path) and not os.path.exists(os.path.join(full_path, "__init__.py")):
+ open(os.path.join(full_path, "__init__.py"), "w").close()
+
+ return None
+
+ def _get_module_path(self, path, fullname):
+ return os.path.join(path, *fullname.split('.'))
+
+
+class YaTestLoggingFileHandler(logging.FileHandler):
+ pass
+
+
+class _TokenFilterFormatter(logging.Formatter):
+ def __init__(self, fmt):
+ super(_TokenFilterFormatter, self).__init__(fmt)
+ self._replacements = []
+ if not self._replacements:
+ if six.PY2:
+ for k, v in os.environ.iteritems():
+ if k.endswith('TOKEN') and v:
+ self._replacements.append(v)
+ elif six.PY3:
+ for k, v in os.environ.items():
+ if k.endswith('TOKEN') and v:
+ self._replacements.append(v)
+ self._replacements = sorted(self._replacements)
+
+ def _filter(self, s):
+ for r in self._replacements:
+ s = s.replace(r, "[SECRET]")
+
+ return s
+
+ def format(self, record):
+ return self._filter(super(_TokenFilterFormatter, self).format(record))
+
+
+def setup_logging(log_path, level=logging.DEBUG, *other_logs):
+ logs = [log_path] + list(other_logs)
+ root_logger = logging.getLogger()
+ for i in range(len(root_logger.handlers) - 1, -1, -1):
+ if isinstance(root_logger.handlers[i], YaTestLoggingFileHandler):
+ root_logger.handlers.pop(i).close()
+ root_logger.setLevel(level)
+ for log_file in logs:
+ file_handler = YaTestLoggingFileHandler(log_file)
+ log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s'
+ file_handler.setFormatter(_TokenFilterFormatter(log_format))
+ file_handler.setLevel(level)
+ root_logger.addHandler(file_handler)
+
+
+def pytest_addoption(parser):
+ parser.addoption("--build-root", action="store", dest="build_root", default="", help="path to the build root")
+ parser.addoption("--dep-root", action="append", dest="dep_roots", default=[], help="path to the dep build roots")
+ parser.addoption("--source-root", action="store", dest="source_root", default="", help="path to the source root")
+ parser.addoption("--data-root", action="store", dest="data_root", default="", help="path to the arcadia_tests_data root")
+ parser.addoption("--output-dir", action="store", dest="output_dir", default="", help="path to the test output dir")
+ parser.addoption("--python-path", action="store", dest="python_path", default="", help="path the canonical python binary")
+ parser.addoption("--valgrind-path", action="store", dest="valgrind_path", default="", help="path the canonical valgring binary")
+ parser.addoption("--test-filter", action="append", dest="test_filter", default=None, help="test filter")
+ parser.addoption("--test-file-filter", action="store", dest="test_file_filter", default=None, help="test file filter")
+ parser.addoption("--test-param", action="append", dest="test_params", default=None, help="test parameters")
+ parser.addoption("--test-log-level", action="store", dest="test_log_level", choices=["critical", "error", "warning", "info", "debug"], default="debug", help="test log level")
+ parser.addoption("--mode", action="store", choices=[yatest_lib.ya.RunMode.List, yatest_lib.ya.RunMode.Run], dest="mode", default=yatest_lib.ya.RunMode.Run, help="testing mode")
+ parser.addoption("--test-list-file", action="store", dest="test_list_file")
+ parser.addoption("--modulo", default=1, type=int)
+ parser.addoption("--modulo-index", default=0, type=int)
+ parser.addoption("--partition-mode", default='SEQUENTIAL', help="Split tests according to partitoin mode")
+ parser.addoption("--split-by-tests", action='store_true', help="Split test execution by tests instead of suites", default=False)
+ parser.addoption("--project-path", action="store", default="", help="path to CMakeList where test is declared")
+ parser.addoption("--build-type", action="store", default="", help="build type")
+ parser.addoption("--flags", action="append", dest="flags", default=[], help="build flags (-D)")
+ parser.addoption("--sanitize", action="store", default="", help="sanitize mode")
+ parser.addoption("--test-stderr", action="store_true", default=False, help="test stderr")
+ parser.addoption("--test-debug", action="store_true", default=False, help="test debug mode")
+ parser.addoption("--root-dir", action="store", default=None)
+ parser.addoption("--ya-trace", action="store", dest="ya_trace_path", default=None, help="path to ya trace report")
+ parser.addoption("--ya-version", action="store", dest="ya_version", default=0, type=int, help="allows to be compatible with ya and the new changes in ya-dev")
+ parser.addoption(
+ "--test-suffix", action="store", dest="test_suffix", default=None, help="add suffix to every test name"
+ )
+ parser.addoption("--gdb-path", action="store", dest="gdb_path", default="", help="path the canonical gdb binary")
+ parser.addoption("--collect-cores", action="store_true", dest="collect_cores", default=False, help="allows core dump file recovering during test")
+ parser.addoption("--sanitizer-extra-checks", action="store_true", dest="sanitizer_extra_checks", default=False, help="enables extra checks for tests built with sanitizers")
+ parser.addoption("--report-deselected", action="store_true", dest="report_deselected", default=False, help="report deselected tests to the trace file")
+ parser.addoption("--pdb-on-sigusr1", action="store_true", default=False, help="setup pdb.set_trace on SIGUSR1")
+ parser.addoption("--test-tool-bin", help="Path to test_tool")
+ parser.addoption("--test-list-path", dest="test_list_path", action="store", help="path to test list", default="")
+
+
+def from_ya_test():
+ return "YA_TEST_RUNNER" in os.environ
+
+
+def pytest_configure(config):
+ global pytest_config
+ pytest_config = config
+
+ config.option.continue_on_collection_errors = True
+
+ config.addinivalue_line("markers", "ya:external")
+
+ config.from_ya_test = from_ya_test()
+ config.test_logs = collections.defaultdict(dict)
+ config.test_metrics = {}
+ config.suite_metrics = {}
+ config.configure_timestamp = time.time()
+ context = {
+ "project_path": config.option.project_path,
+ "test_stderr": config.option.test_stderr,
+ "test_debug": config.option.test_debug,
+ "build_type": config.option.build_type,
+ "test_traceback": config.option.tbstyle,
+ "flags": config.option.flags,
+ "sanitize": config.option.sanitize,
+ }
+
+ if config.option.collectonly:
+ config.option.mode = yatest_lib.ya.RunMode.List
+
+ config.ya = yatest_lib.ya.Ya(
+ config.option.mode,
+ config.option.source_root,
+ config.option.build_root,
+ config.option.dep_roots,
+ config.option.output_dir,
+ config.option.test_params,
+ context,
+ config.option.python_path,
+ config.option.valgrind_path,
+ config.option.gdb_path,
+ config.option.data_root,
+ )
+ config.option.test_log_level = {
+ "critical": logging.CRITICAL,
+ "error": logging.ERROR,
+ "warning": logging.WARN,
+ "info": logging.INFO,
+ "debug": logging.DEBUG,
+ }[config.option.test_log_level]
+
+ if not config.option.collectonly:
+ setup_logging(os.path.join(config.ya.output_dir, "run.log"), config.option.test_log_level)
+ config.current_item_nodeid = None
+ config.current_test_name = None
+ config.test_cores_count = 0
+ config.collect_cores = config.option.collect_cores
+ config.sanitizer_extra_checks = config.option.sanitizer_extra_checks
+ try:
+ config.test_tool_bin = config.option.test_tool_bin
+ except AttributeError:
+ logging.info("test_tool_bin not specified")
+
+ if config.sanitizer_extra_checks:
+ for envvar in ['LSAN_OPTIONS', 'ASAN_OPTIONS']:
+ if envvar in os.environ:
+ os.environ.pop(envvar)
+ if envvar + '_ORIGINAL' in os.environ:
+ os.environ[envvar] = os.environ[envvar + '_ORIGINAL']
+
+ if config.option.root_dir:
+ config.rootdir = py.path.local(config.option.root_dir)
+ config.invocation_params = attr.evolve(config.invocation_params, dir=config.rootdir)
+
+ extra_sys_path = []
+ # Arcadia paths from the test DEPENDS section of ya.make
+ extra_sys_path.append(os.path.join(config.option.source_root, config.option.project_path))
+ # Build root is required for correct import of protobufs, because imports are related to the root
+ # (like import devtools.dummy_arcadia.protos.lib.my_proto_pb2)
+ extra_sys_path.append(config.option.build_root)
+
+ for path in config.option.dep_roots:
+ if os.path.isabs(path):
+ extra_sys_path.append(path)
+ else:
+ extra_sys_path.append(os.path.join(config.option.source_root, path))
+
+ sys_path_set = set(sys.path)
+ for path in extra_sys_path:
+ if path not in sys_path_set:
+ sys.path.append(path)
+ sys_path_set.add(path)
+
+ os.environ["PYTHONPATH"] = os.pathsep.join(sys.path)
+
+ if not config.option.collectonly:
+ if config.option.ya_trace_path:
+ config.ya_trace_reporter = TraceReportGenerator(config.option.ya_trace_path)
+ else:
+ config.ya_trace_reporter = DryTraceReportGenerator(config.option.ya_trace_path)
+ config.ya_version = config.option.ya_version
+
+ sys.meta_path.append(CustomImporter([config.option.build_root] + [os.path.join(config.option.build_root, dep) for dep in config.option.dep_roots]))
+ if config.option.pdb_on_sigusr1:
+ configure_pdb_on_demand()
+
+ # Dump python backtrace in case of any errors
+ faulthandler.enable()
+ if hasattr(signal, "SIGQUIT"):
+ # SIGQUIT is used by test_tool to teardown tests which overruns timeout
+ faulthandler.register(signal.SIGQUIT, chain=True)
+
+ if hasattr(signal, "SIGUSR2"):
+ signal.signal(signal.SIGUSR2, _graceful_shutdown)
+
+
+session_should_exit = False
+
+
+def _graceful_shutdown_on_log(should_exit):
+ if should_exit:
+ pytest.exit("Graceful shutdown requested")
+
+
+def pytest_runtest_logreport(report):
+ _graceful_shutdown_on_log(session_should_exit)
+
+
+def pytest_runtest_logstart(nodeid, location):
+ _graceful_shutdown_on_log(session_should_exit)
+
+
+def pytest_runtest_logfinish(nodeid, location):
+ _graceful_shutdown_on_log(session_should_exit)
+
+
+def _graceful_shutdown(*args):
+ global session_should_exit
+ session_should_exit = True
+ try:
+ import library.python.coverage
+ library.python.coverage.stop_coverage_tracing()
+ except ImportError:
+ pass
+ traceback.print_stack(file=sys.stderr)
+ capman = pytest_config.pluginmanager.getplugin("capturemanager")
+ capman.suspend(in_=True)
+ _graceful_shutdown_on_log(not capman.is_globally_capturing())
+
+
+def _get_rusage():
+ return resource and resource.getrusage(resource.RUSAGE_SELF)
+
+
+def _collect_test_rusage(item):
+ if resource and hasattr(item, "rusage"):
+ finish_rusage = _get_rusage()
+ ya_inst = pytest_config.ya
+
+ def add_metric(attr_name, metric_name=None, modifier=None):
+ if not metric_name:
+ metric_name = attr_name
+ if not modifier:
+ modifier = lambda x: x
+ if hasattr(item.rusage, attr_name):
+ ya_inst.set_metric_value(metric_name, modifier(getattr(finish_rusage, attr_name) - getattr(item.rusage, attr_name)))
+
+ for args in [
+ ("ru_maxrss", "ru_rss", lambda x: x*1024), # to be the same as in util/system/rusage.cpp
+ ("ru_utime",),
+ ("ru_stime",),
+ ("ru_ixrss", None, lambda x: x*1024),
+ ("ru_idrss", None, lambda x: x*1024),
+ ("ru_isrss", None, lambda x: x*1024),
+ ("ru_majflt", "ru_major_pagefaults"),
+ ("ru_minflt", "ru_minor_pagefaults"),
+ ("ru_nswap",),
+ ("ru_inblock",),
+ ("ru_oublock",),
+ ("ru_msgsnd",),
+ ("ru_msgrcv",),
+ ("ru_nsignals",),
+ ("ru_nvcsw",),
+ ("ru_nivcsw",),
+ ]:
+ add_metric(*args)
+
+
+def _get_item_tags(item):
+ tags = []
+ for key, value in item.keywords.items():
+ if key == 'pytestmark' and isinstance(value, list):
+ for mark in value:
+ tags.append(mark.name)
+ elif isinstance(value, _pytest.mark.MarkDecorator):
+ tags.append(key)
+ return tags
+
+
+def pytest_runtest_setup(item):
+ item.rusage = _get_rusage()
+ pytest_config.test_cores_count = 0
+ pytest_config.current_item_nodeid = item.nodeid
+ class_name, test_name = tools.split_node_id(item.nodeid)
+ test_log_path = tools.get_test_log_file_path(pytest_config.ya.output_dir, class_name, test_name)
+ setup_logging(
+ os.path.join(pytest_config.ya.output_dir, "run.log"),
+ pytest_config.option.test_log_level,
+ test_log_path
+ )
+ pytest_config.test_logs[item.nodeid]['log'] = test_log_path
+ pytest_config.test_logs[item.nodeid]['logsdir'] = pytest_config.ya.output_dir
+ pytest_config.current_test_log_path = test_log_path
+ pytest_config.current_test_name = "{}::{}".format(class_name, test_name)
+ separator = "#" * 100
+ yatest_logger.info(separator)
+ yatest_logger.info(test_name)
+ yatest_logger.info(separator)
+ yatest_logger.info("Test setup")
+
+ test_item = CrashedTestItem(item.nodeid, pytest_config.option.test_suffix)
+ pytest_config.ya_trace_reporter.on_start_test_class(test_item)
+ pytest_config.ya_trace_reporter.on_start_test_case(test_item)
+
+
+def pytest_runtest_teardown(item, nextitem):
+ yatest_logger.info("Test teardown")
+
+
+def pytest_runtest_call(item):
+ class_name, test_name = tools.split_node_id(item.nodeid)
+ yatest_logger.info("Test call (class_name: %s, test_name: %s)", class_name, test_name)
+
+
+def pytest_deselected(items):
+ config = pytest_config
+ if config.option.report_deselected:
+ for item in items:
+ deselected_item = DeselectedTestItem(item.nodeid, config.option.test_suffix)
+ config.ya_trace_reporter.on_start_test_class(deselected_item)
+ config.ya_trace_reporter.on_finish_test_case(deselected_item)
+ config.ya_trace_reporter.on_finish_test_class(deselected_item)
+
+
+@pytest.mark.trylast
+def pytest_collection_modifyitems(items, config):
+
+ def filter_items(filters):
+ filtered_items = []
+ deselected_items = []
+ for item in items:
+ canonical_node_id = str(CustomTestItem(item.nodeid, pytest_config.option.test_suffix))
+ matched = False
+ for flt in filters:
+ if "::" not in flt and "*" not in flt:
+ flt += "*" # add support for filtering by module name
+ if canonical_node_id.endswith(flt) or fnmatch.fnmatch(tools.escape_for_fnmatch(canonical_node_id), tools.escape_for_fnmatch(flt)):
+ matched = True
+ if matched:
+ filtered_items.append(item)
+ else:
+ deselected_items.append(item)
+
+ config.hook.pytest_deselected(items=deselected_items)
+ items[:] = filtered_items
+
+ def filter_by_full_name(filters):
+ filter_set = {flt for flt in filters}
+ filtered_items = []
+ deselected_items = []
+ for item in items:
+ if item.nodeid in filter_set:
+ filtered_items.append(item)
+ else:
+ deselected_items.append(item)
+
+ config.hook.pytest_deselected(items=deselected_items)
+ items[:] = filtered_items
+
+ # XXX - check to be removed when tests for peerdirs don't run
+ for item in items:
+ if not item.nodeid:
+ item._nodeid = os.path.basename(item.location[0])
+ if os.path.exists(config.option.test_list_path):
+ with open(config.option.test_list_path, 'r') as afile:
+ chunks = json.load(afile)
+ filters = chunks[config.option.modulo_index]
+ filter_by_full_name(filters)
+ else:
+ if config.option.test_filter:
+ filter_items(config.option.test_filter)
+ partition_mode = config.option.partition_mode
+ modulo = config.option.modulo
+ if modulo > 1:
+ items[:] = sorted(items, key=lambda item: item.nodeid)
+ modulo_index = config.option.modulo_index
+ split_by_tests = config.option.split_by_tests
+ items_by_classes = {}
+ res = []
+ for item in items:
+ if item.nodeid.count("::") == 2 and not split_by_tests:
+ class_name = item.nodeid.rsplit("::", 1)[0]
+ if class_name not in items_by_classes:
+ items_by_classes[class_name] = []
+ res.append(items_by_classes[class_name])
+ items_by_classes[class_name].append(item)
+ else:
+ res.append([item])
+ chunk_items = test_splitter.get_splitted_tests(res, modulo, modulo_index, partition_mode, is_sorted=True)
+ items[:] = []
+ for item in chunk_items:
+ items.extend(item)
+ yatest_logger.info("Modulo %s tests are: %s", modulo_index, chunk_items)
+
+ if config.option.mode == yatest_lib.ya.RunMode.Run:
+ for item in items:
+ test_item = NotLaunchedTestItem(item.nodeid, config.option.test_suffix)
+ config.ya_trace_reporter.on_start_test_class(test_item)
+ config.ya_trace_reporter.on_finish_test_case(test_item)
+ config.ya_trace_reporter.on_finish_test_class(test_item)
+ elif config.option.mode == yatest_lib.ya.RunMode.List:
+ tests = []
+ for item in items:
+ item = CustomTestItem(item.nodeid, pytest_config.option.test_suffix, item.keywords)
+ record = {
+ "class": item.class_name,
+ "test": item.test_name,
+ "tags": _get_item_tags(item),
+ }
+ tests.append(record)
+ if config.option.test_list_file:
+ with open(config.option.test_list_file, 'w') as afile:
+ json.dump(tests, afile)
+ # TODO prettyboy remove after test_tool release - currently it's required for backward compatibility
+ sys.stderr.write(json.dumps(tests))
+
+
+def pytest_collectreport(report):
+ if not report.passed:
+ if hasattr(pytest_config, 'ya_trace_reporter'):
+ test_item = TestItem(report, None, pytest_config.option.test_suffix)
+ pytest_config.ya_trace_reporter.on_error(test_item)
+ else:
+ sys.stderr.write(yatest_lib.tools.to_utf8(report.longrepr))
+
+
+@pytest.mark.tryfirst
+def pytest_pyfunc_call(pyfuncitem):
+ testfunction = pyfuncitem.obj
+ iscoroutinefunction = getattr(inspect, "iscoroutinefunction", None)
+ if iscoroutinefunction is not None and iscoroutinefunction(testfunction):
+ msg = "Coroutine functions are not natively supported and have been skipped.\n"
+ msg += "You need to install a suitable plugin for your async framework, for example:\n"
+ msg += " - pytest-asyncio\n"
+ msg += " - pytest-trio\n"
+ msg += " - pytest-tornasync"
+ warnings.warn(PytestUnhandledCoroutineWarning(msg.format(pyfuncitem.nodeid)))
+ _pytest.outcomes.skip(msg="coroutine function and no async plugin installed (see warnings)")
+ funcargs = pyfuncitem.funcargs
+ testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames}
+ pyfuncitem.retval = testfunction(**testargs)
+ return True
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+ def logreport(report, result, call):
+ test_item = TestItem(report, result, pytest_config.option.test_suffix)
+ if not pytest_config.suite_metrics and context.Ctx.get("YA_PYTEST_START_TIMESTAMP"):
+ pytest_config.suite_metrics["pytest_startup_duration"] = call.start - context.Ctx["YA_PYTEST_START_TIMESTAMP"]
+ pytest_config.ya_trace_reporter.dump_suite_metrics()
+
+ pytest_config.ya_trace_reporter.on_log_report(test_item)
+
+ if report.outcome == "failed":
+ yatest_logger.error(report.longrepr)
+
+ if report.when == "call":
+ _collect_test_rusage(item)
+ pytest_config.ya_trace_reporter.on_finish_test_case(test_item)
+ elif report.when == "setup":
+ pytest_config.ya_trace_reporter.on_start_test_class(test_item)
+ if report.outcome != "passed":
+ pytest_config.ya_trace_reporter.on_start_test_case(test_item)
+ pytest_config.ya_trace_reporter.on_finish_test_case(test_item)
+ else:
+ pytest_config.ya_trace_reporter.on_start_test_case(test_item)
+ elif report.when == "teardown":
+ if report.outcome == "failed":
+ pytest_config.ya_trace_reporter.on_start_test_case(test_item)
+ pytest_config.ya_trace_reporter.on_finish_test_case(test_item)
+ else:
+ pytest_config.ya_trace_reporter.on_finish_test_case(test_item, duration_only=True)
+ pytest_config.ya_trace_reporter.on_finish_test_class(test_item)
+
+ outcome = yield
+ rep = outcome.get_result()
+ result = None
+ if hasattr(item, 'retval') and item.retval is not None:
+ result = item.retval
+ if not pytest_config.from_ya_test:
+ ti = TestItem(rep, result, pytest_config.option.test_suffix)
+ tr = pytest_config.pluginmanager.getplugin('terminalreporter')
+ tr.write_line("{} - Validating canonical data is not supported when running standalone binary".format(ti), yellow=True, bold=True)
+ logreport(rep, result, call)
+
+
+def pytest_make_parametrize_id(config, val, argname):
+ # Avoid <, > symbols in canondata file names
+ if inspect.isfunction(val) and val.__name__ == "<lambda>":
+ return str(argname)
+ return None
+
+
+def get_formatted_error(report):
+ if isinstance(report.longrepr, tuple):
+ text = ""
+ for entry in report.longrepr:
+ text += colorize(entry)
+ else:
+ text = colorize(report.longrepr)
+ text = yatest_lib.tools.to_utf8(text)
+ return text
+
+
+def colorize(longrepr):
+ # use default pytest colorization
+ if pytest_config.option.tbstyle != "short":
+ io = py.io.TextIO()
+ if six.PY2:
+ writer = py.io.TerminalWriter(file=io)
+ else:
+ writer = _pytest._io.TerminalWriter(file=io)
+ # enable colorization
+ writer.hasmarkup = True
+
+ if hasattr(longrepr, 'reprtraceback') and hasattr(longrepr.reprtraceback, 'toterminal'):
+ longrepr.reprtraceback.toterminal(writer)
+ return io.getvalue().strip()
+ return yatest_lib.tools.to_utf8(longrepr)
+
+ text = yatest_lib.tools.to_utf8(longrepr)
+ pos = text.find("E ")
+ if pos == -1:
+ return text
+
+ bt, error = text[:pos], text[pos:]
+ filters = [
+ # File path, line number and function name
+ (re.compile(r"^(.*?):(\d+): in (\S+)", flags=re.MULTILINE), r"[[unimp]]\1[[rst]]:[[alt2]]\2[[rst]]: in [[alt1]]\3[[rst]]"),
+ ]
+ for regex, substitution in filters:
+ bt = regex.sub(substitution, bt)
+ return "{}[[bad]]{}".format(bt, error)
+
+
+class TestItem(object):
+
+ def __init__(self, report, result, test_suffix):
+ self._result = result
+ self.nodeid = report.nodeid
+ self._class_name, self._test_name = tools.split_node_id(self.nodeid, test_suffix)
+ self._error = None
+ self._status = None
+ self._process_report(report)
+ self._duration = hasattr(report, 'duration') and report.duration or 0
+ self._keywords = getattr(report, "keywords", {})
+
+ def _process_report(self, report):
+ if report.longrepr:
+ self.set_error(report)
+ if hasattr(report, 'when') and report.when != "call":
+ self.set_error(report.when + " failed:\n" + self._error)
+ else:
+ self.set_error("")
+
+ report_teststatus = _pytest.skipping.pytest_report_teststatus(report)
+ if report_teststatus is not None:
+ report_teststatus = report_teststatus[0]
+
+ if report_teststatus == 'xfailed':
+ self._status = 'xfail'
+ self.set_error(report.wasxfail, 'imp')
+ elif report_teststatus == 'xpassed':
+ self._status = 'xpass'
+ self.set_error("Test unexpectedly passed")
+ elif report.skipped:
+ self._status = 'skipped'
+ self.set_error(yatest_lib.tools.to_utf8(report.longrepr[-1]))
+ elif report.passed:
+ self._status = 'good'
+ self.set_error("")
+ else:
+ self._status = 'fail'
+
+ @property
+ def status(self):
+ return self._status
+
+ def set_status(self, status):
+ self._status = status
+
+ @property
+ def test_name(self):
+ return tools.normalize_name(self._test_name)
+
+ @property
+ def class_name(self):
+ return tools.normalize_name(self._class_name)
+
+ @property
+ def error(self):
+ return self._error
+
+ def set_error(self, entry, marker='bad'):
+ if isinstance(entry, _pytest.reports.BaseReport):
+ self._error = get_formatted_error(entry)
+ else:
+ self._error = "[[{}]]{}".format(yatest_lib.tools.to_str(marker), yatest_lib.tools.to_str(entry))
+
+ @property
+ def duration(self):
+ return self._duration
+
+ @property
+ def result(self):
+ if 'not_canonize' in self._keywords:
+ return None
+ return self._result
+
+ @property
+ def keywords(self):
+ return self._keywords
+
+ def __str__(self):
+ return "{}::{}".format(self.class_name, self.test_name)
+
+
+class CustomTestItem(TestItem):
+
+ def __init__(self, nodeid, test_suffix, keywords=None):
+ self._result = None
+ self.nodeid = nodeid
+ self._class_name, self._test_name = tools.split_node_id(nodeid, test_suffix)
+ self._duration = 0
+ self._error = ""
+ self._keywords = keywords if keywords is not None else {}
+
+
+class NotLaunchedTestItem(CustomTestItem):
+
+ def __init__(self, nodeid, test_suffix):
+ super(NotLaunchedTestItem, self).__init__(nodeid, test_suffix)
+ self._status = "not_launched"
+
+
+class CrashedTestItem(CustomTestItem):
+
+ def __init__(self, nodeid, test_suffix):
+ super(CrashedTestItem, self).__init__(nodeid, test_suffix)
+ self._status = "crashed"
+
+
+class DeselectedTestItem(CustomTestItem):
+
+ def __init__(self, nodeid, test_suffix):
+ super(DeselectedTestItem, self).__init__(nodeid, test_suffix)
+ self._status = "deselected"
+
+
+class TraceReportGenerator(object):
+
+ def __init__(self, out_file_path):
+ self._filename = out_file_path
+ self._file = open(out_file_path, 'w')
+ self._wreckage_filename = out_file_path + '.wreckage'
+ self._test_messages = {}
+ self._test_duration = {}
+ # Some machinery to avoid data corruption due sloppy fork()
+ self._current_test = (None, None)
+ self._pid = os.getpid()
+ self._check_intricate_respawn()
+
+ def _check_intricate_respawn(self):
+ pid_file = self._filename + '.pid'
+ try:
+ # python2 doesn't support open(f, 'x')
+ afile = os.fdopen(os.open(pid_file, os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'w')
+ afile.write(str(self._pid))
+ afile.close()
+ return
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ # Looks like the test binary was respawned
+ if from_ya_test():
+ try:
+ with open(pid_file) as afile:
+ prev_pid = afile.read()
+ except Exception as e:
+ prev_pid = '(failed to obtain previous pid: {})'.format(e)
+
+ parts = [
+ "Aborting test run: test machinery found that the test binary {} has already been run before.".format(sys.executable),
+ "Looks like test has incorrect respawn/relaunch logic within test binary.",
+ "Test should not try to restart itself - this is a poorly designed test case that leads to errors and could corrupt internal test machinery files.",
+ "Debug info: previous pid:{} current:{}".format(prev_pid, self._pid),
+ ]
+ msg = '\n'.join(parts)
+ yatest_logger.error(msg)
+
+ if filelock:
+ lock = filelock.FileLock(self._wreckage_filename + '.lock')
+ lock.acquire()
+
+ with open(self._wreckage_filename, 'a') as afile:
+ self._file = afile
+
+ self._dump_trace('chunk_event', {"errors": [('fail', '[[bad]]' + msg)]})
+
+ raise Exception(msg)
+ else:
+ # Test binary is launched without `ya make -t`'s testing machinery - don't rely on clean environment
+ pass
+
+ def on_start_test_class(self, test_item):
+ pytest_config.ya.set_test_item_node_id(test_item.nodeid)
+ class_name = test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name
+ self._current_test = (class_name, None)
+ self.trace('test-started', {'class': class_name})
+
+ def on_finish_test_class(self, test_item):
+ pytest_config.ya.set_test_item_node_id(test_item.nodeid)
+ self.trace('test-finished', {'class': test_item.class_name.decode('utf-8') if sys.version_info[0] < 3 else test_item.class_name})
+
+ def on_start_test_case(self, test_item):
+ class_name = yatest_lib.tools.to_utf8(test_item.class_name)
+ subtest_name = yatest_lib.tools.to_utf8(test_item.test_name)
+ message = {
+ 'class': class_name,
+ 'subtest': subtest_name,
+ }
+ if test_item.nodeid in pytest_config.test_logs:
+ message['logs'] = pytest_config.test_logs[test_item.nodeid]
+ pytest_config.ya.set_test_item_node_id(test_item.nodeid)
+ self._current_test = (class_name, subtest_name)
+ self.trace('subtest-started', message)
+
+ def on_finish_test_case(self, test_item, duration_only=False):
+ if test_item.result is not None:
+ try:
+ result = canon.serialize(test_item.result)
+ except Exception as e:
+ yatest_logger.exception("Error while serializing test results")
+ test_item.set_error("Invalid test result: {}".format(e))
+ test_item.set_status("fail")
+ result = None
+ else:
+ result = None
+
+ if duration_only and test_item.nodeid in self._test_messages: # add teardown time
+ message = self._test_messages[test_item.nodeid]
+ else:
+ comment = self._test_messages[test_item.nodeid]['comment'] if test_item.nodeid in self._test_messages else ''
+ comment += self._get_comment(test_item)
+ message = {
+ 'class': yatest_lib.tools.to_utf8(test_item.class_name),
+ 'subtest': yatest_lib.tools.to_utf8(test_item.test_name),
+ 'status': test_item.status,
+ 'comment': comment,
+ 'result': result,
+ 'metrics': pytest_config.test_metrics.get(test_item.nodeid),
+ 'is_diff_test': 'diff_test' in test_item.keywords,
+ 'tags': _get_item_tags(test_item),
+ }
+ if test_item.nodeid in pytest_config.test_logs:
+ message['logs'] = pytest_config.test_logs[test_item.nodeid]
+
+ message['time'] = self._test_duration.get(test_item.nodeid, test_item.duration)
+
+ self.trace('subtest-finished', message)
+ self._test_messages[test_item.nodeid] = message
+
+ def dump_suite_metrics(self):
+ message = {"metrics": pytest_config.suite_metrics}
+ self.trace("suite-event", message)
+
+ def on_error(self, test_item):
+ self.trace('chunk_event', {"errors": [(test_item.status, self._get_comment(test_item))]})
+
+ def on_log_report(self, test_item):
+ if test_item.nodeid in self._test_duration:
+ self._test_duration[test_item.nodeid] += test_item._duration
+ else:
+ self._test_duration[test_item.nodeid] = test_item._duration
+
+ @staticmethod
+ def _get_comment(test_item):
+ msg = yatest_lib.tools.to_utf8(test_item.error)
+ if not msg:
+ return ""
+ return msg + "[[rst]]"
+
+ def _dump_trace(self, name, value):
+ event = {
+ 'timestamp': time.time(),
+ 'value': value,
+ 'name': name
+ }
+
+ data = yatest_lib.tools.to_str(json.dumps(event, ensure_ascii=False))
+ self._file.write(data + '\n')
+ self._file.flush()
+
+ def _check_sloppy_fork(self, name, value):
+ if self._pid == os.getpid():
+ return
+
+ yatest_logger.error("Skip tracing to avoid data corruption, name = %s, value = %s", name, value)
+
+ try:
+ # Lock wreckage tracefile to avoid race if multiple tests use fork sloppily
+ if filelock:
+ lock = filelock.FileLock(self._wreckage_filename + '.lock')
+ lock.acquire()
+
+ with open(self._wreckage_filename, 'a') as afile:
+ self._file = afile
+
+ parts = [
+ "It looks like you have leaked process - it could corrupt internal test machinery files.",
+ "Usually it happens when you casually use fork() without os._exit(),",
+ "which results in two pytest processes running at the same time.",
+ "Pid of the original pytest's process is {}, however current process has {} pid.".format(self._pid, os.getpid()),
+ ]
+ if self._current_test[1]:
+ parts.append("Most likely the problem is in '{}' test.".format(self._current_test))
+ else:
+ parts.append("Most likely new process was created before any test was launched (during the import stage?).")
+
+ if value.get('comment'):
+ comment = value.get('comment', '').strip()
+ # multiline comment
+ newline_required = '\n' if '\n' in comment else ''
+ parts.append("Debug info: name = '{}' comment:{}{}".format(name, newline_required, comment))
+ else:
+ val_str = json.dumps(value, ensure_ascii=False).encode('utf-8')
+ parts.append("Debug info: name = '{}' value = '{}'".format(name, base64.b64encode(val_str)))
+
+ msg = "[[bad]]{}".format('\n'.join(parts))
+ class_name, subtest_name = self._current_test
+ if subtest_name:
+ data = {
+ 'class': class_name,
+ 'subtest': subtest_name,
+ 'status': 'fail',
+ 'comment': msg,
+ }
+ # overwrite original status
+ self._dump_trace('subtest-finished', data)
+ else:
+ self._dump_trace('chunk_event', {"errors": [('fail', msg)]})
+ except Exception as e:
+ yatest_logger.exception(e)
+ finally:
+ os._exit(38)
+
+ def trace(self, name, value):
+ self._check_sloppy_fork(name, value)
+ self._dump_trace(name, value)
+
+
+class DryTraceReportGenerator(TraceReportGenerator):
+ """
+ Generator does not write any information.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self._test_messages = {}
+ self._test_duration = {}
+
+ def trace(self, name, value):
+ pass
diff --git a/library/python/pytest/pytest.yatest.ini b/library/python/pytest/pytest.yatest.ini
new file mode 100644
index 0000000000..70d6c98516
--- /dev/null
+++ b/library/python/pytest/pytest.yatest.ini
@@ -0,0 +1,7 @@
+[pytest]
+pep8maxlinelength = 200
+norecursedirs = *
+pep8ignore = E127 E123 E226 E24
+filterwarnings =
+ ignore::pytest.RemovedInPytest4Warning
+addopts = -p no:warnings
diff --git a/library/python/pytest/rewrite.py b/library/python/pytest/rewrite.py
new file mode 100644
index 0000000000..ec188d847f
--- /dev/null
+++ b/library/python/pytest/rewrite.py
@@ -0,0 +1,123 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import ast
+
+import py
+
+from _pytest.assertion import rewrite
+try:
+ import importlib.util
+except ImportError:
+ pass
+from __res import importer
+import sys
+import six
+
+
+def _get_state(config):
+ if hasattr(config, '_assertstate'):
+ return config._assertstate
+ return config._store[rewrite.assertstate_key]
+
+
+class AssertionRewritingHook(rewrite.AssertionRewritingHook):
+ def __init__(self, *args, **kwargs):
+ self.modules = {}
+ super(AssertionRewritingHook, self).__init__(*args, **kwargs)
+
+ def find_module(self, name, path=None):
+ co = self._find_module(name, path)
+ if co is not None:
+ return self
+
+ def _find_module(self, name, path=None):
+ state = _get_state(self.config)
+ if not self._should_rewrite(name, None, state):
+ return None
+ state.trace("find_module called for: %s" % name)
+
+ try:
+ if self.is_package(name):
+ return None
+ except ImportError:
+ return None
+
+ self._rewritten_names.add(name)
+
+ state.trace("rewriting %s" % name)
+ co = _rewrite_test(self.config, name)
+ if co is None:
+ # Probably a SyntaxError in the test.
+ return None
+ self.modules[name] = co, None
+ return co
+
+ def find_spec(self, name, path=None, target=None):
+ co = self._find_module(name, path)
+ if co is not None:
+ return importlib.util.spec_from_file_location(
+ name,
+ co.co_filename,
+ loader=self,
+ )
+
+ def _should_rewrite(self, name, fn, state):
+ if name.startswith("__tests__.") or name.endswith(".conftest"):
+ return True
+
+ return self._is_marked_for_rewrite(name, state)
+
+ def is_package(self, name):
+ return importer.is_package(name)
+
+ def get_source(self, name):
+ return importer.get_source(name)
+
+ if six.PY3:
+ def load_module(self, module):
+ co, _ = self.modules.pop(module.__name__)
+ try:
+ module.__file__ = co.co_filename
+ module.__cached__ = None
+ module.__loader__ = self
+ module.__spec__ = importlib.util.spec_from_file_location(module.__name__, co.co_filename, loader=self)
+ exec(co, module.__dict__)
+ except: # noqa
+ if module.__name__ in sys.modules:
+ del sys.modules[module.__name__]
+ raise
+ return sys.modules[module.__name__]
+
+ def exec_module(self, module):
+ if module.__name__ in self.modules:
+ self.load_module(module)
+ else:
+ super(AssertionRewritingHook, self).exec_module(module)
+
+
+def _rewrite_test(config, name):
+ """Try to read and rewrite *fn* and return the code object."""
+ state = _get_state(config)
+
+ source = importer.get_source(name)
+ if source is None:
+ return None
+
+ path = importer.get_filename(name)
+
+ try:
+ tree = ast.parse(source, filename=path)
+ except SyntaxError:
+ # Let this pop up again in the real import.
+ state.trace("failed to parse: %r" % (path,))
+ return None
+ rewrite.rewrite_asserts(tree, py.path.local(path), config)
+ try:
+ co = compile(tree, path, "exec", dont_inherit=True)
+ except SyntaxError:
+ # It's possible that this error is from some bug in the
+ # assertion rewriting, but I don't know of a fast way to tell.
+ state.trace("failed to compile: %r" % (path,))
+ return None
+ return co
diff --git a/library/python/pytest/ya.make b/library/python/pytest/ya.make
new file mode 100644
index 0000000000..060c92c313
--- /dev/null
+++ b/library/python/pytest/ya.make
@@ -0,0 +1,32 @@
+PY23_LIBRARY()
+
+OWNER(
+ g:yatool
+ dmitko
+)
+
+PY_SRCS(
+ __init__.py
+ main.py
+ rewrite.py
+ yatest_tools.py
+ context.py
+)
+
+PEERDIR(
+ contrib/python/dateutil
+ contrib/python/ipdb
+ contrib/python/py
+ contrib/python/pytest
+ contrib/python/requests
+ library/python/pytest/plugins
+ library/python/testing/yatest_common
+ library/python/testing/yatest_lib
+)
+
+RESOURCE_FILES(
+ PREFIX library/python/pytest/
+ pytest.yatest.ini
+)
+
+END()
diff --git a/library/python/pytest/yatest_tools.py b/library/python/pytest/yatest_tools.py
new file mode 100644
index 0000000000..6b8b896394
--- /dev/null
+++ b/library/python/pytest/yatest_tools.py
@@ -0,0 +1,304 @@
+# coding: utf-8
+
+import collections
+import functools
+import math
+import os
+import re
+import sys
+
+import yatest_lib.tools
+
+
+class Subtest(object):
+ def __init__(self, name, test_name, status, comment, elapsed, result=None, test_type=None, logs=None, cwd=None, metrics=None):
+ self._name = name
+ self._test_name = test_name
+ self.status = status
+ self.elapsed = elapsed
+ self.comment = comment
+ self.result = result
+ self.test_type = test_type
+ self.logs = logs or {}
+ self.cwd = cwd
+ self.metrics = metrics
+
+ def __eq__(self, other):
+ if not isinstance(other, Subtest):
+ return False
+ return self.name == other.name and self.test_name == other.test_name
+
+ def __str__(self):
+ return yatest_lib.tools.to_utf8(unicode(self))
+
+ def __unicode__(self):
+ return u"{}::{}".format(self.test_name, self.test_name)
+
+ @property
+ def name(self):
+ return yatest_lib.tools.to_utf8(self._name)
+
+ @property
+ def test_name(self):
+ return yatest_lib.tools.to_utf8(self._test_name)
+
+ def __repr__(self):
+ return "Subtest [{}::{} - {}[{}]: {}]".format(self.name, self.test_name, self.status, self.elapsed, self.comment)
+
+ def __hash__(self):
+ return hash(str(self))
+
+
+class SubtestInfo(object):
+
+ skipped_prefix = '[SKIPPED] '
+
+ @classmethod
+ def from_str(cls, s):
+ if s.startswith(SubtestInfo.skipped_prefix):
+ s = s[len(SubtestInfo.skipped_prefix):]
+ skipped = True
+
+ else:
+ skipped = False
+
+ return SubtestInfo(*s.rsplit(TEST_SUBTEST_SEPARATOR, 1), skipped=skipped)
+
+ def __init__(self, test, subtest="", skipped=False, **kwargs):
+ self.test = test
+ self.subtest = subtest
+ self.skipped = skipped
+ for key, value in kwargs.iteritems():
+ setattr(self, key, value)
+
+ def __str__(self):
+ s = ''
+
+ if self.skipped:
+ s += SubtestInfo.skipped_prefix
+
+ return s + TEST_SUBTEST_SEPARATOR.join([self.test, self.subtest])
+
+ def __repr__(self):
+ return str(self)
+
+
+class Status(object):
+ GOOD, XFAIL, FAIL, XPASS, MISSING, CRASHED, TIMEOUT = range(7)
+ SKIPPED = -100
+ NOT_LAUNCHED = -200
+ CANON_DIFF = -300
+ FLAKY = -1
+ BY_NAME = {'good': GOOD, 'fail': FAIL, 'xfail': XFAIL, 'xpass': XPASS, 'missing': MISSING, 'crashed': CRASHED,
+ 'skipped': SKIPPED, 'flaky': FLAKY, 'not_launched': NOT_LAUNCHED, 'timeout': TIMEOUT, 'diff': CANON_DIFF}
+ TO_STR = {GOOD: 'good', FAIL: 'fail', XFAIL: 'xfail', XPASS: 'xpass', MISSING: 'missing', CRASHED: 'crashed',
+ SKIPPED: 'skipped', FLAKY: 'flaky', NOT_LAUNCHED: 'not_launched', TIMEOUT: 'timeout', CANON_DIFF: 'diff'}
+
+
+class Test(object):
+ def __init__(self, name, path, status=None, comment=None, subtests=None):
+ self.name = name
+ self.path = path
+ self.status = status
+ self.comment = comment
+ self.subtests = subtests or []
+
+ def __eq__(self, other):
+ if not isinstance(other, Test):
+ return False
+ return self.name == other.name and self.path == other.path
+
+ def __str__(self):
+ return "Test [{} {}] - {} - {}".format(self.name, self.path, self.status, self.comment)
+
+ def __repr__(self):
+ return str(self)
+
+ def add_subtest(self, subtest):
+ self.subtests.append(subtest)
+
+ def setup_status(self, status, comment):
+ self.status = Status.BY_NAME[status or 'good']
+ if len(self.subtests) != 0:
+ self.status = max(self.status, max(s.status for s in self.subtests))
+ self.comment = comment
+
+ def subtests_by_status(self, status):
+ return [x.status for x in self.subtests].count(status)
+
+
+class NoMd5FileException(Exception):
+ pass
+
+
+TEST_SUBTEST_SEPARATOR = '::'
+
+
+# TODO: extract color theme logic from ya
+COLOR_THEME = {
+ 'test_name': 'light-blue',
+ 'test_project_path': 'dark-blue',
+ 'test_dir_desc': 'dark-magenta',
+ 'test_binary_path': 'light-gray',
+}
+
+
+# XXX: remove me
+class YaCtx(object):
+ pass
+
+ya_ctx = YaCtx()
+
+TRACE_FILE_NAME = "ytest.report.trace"
+
+
+def lazy(func):
+ mem = {}
+
+ @functools.wraps(func)
+ def wrapper():
+ if "results" not in mem:
+ mem["results"] = func()
+ return mem["results"]
+
+ return wrapper
+
+
+@lazy
+def _get_mtab():
+ if os.path.exists("/etc/mtab"):
+ with open("/etc/mtab") as afile:
+ data = afile.read()
+ return [line.split(" ") for line in data.split("\n") if line]
+ return []
+
+
+def get_max_filename_length(dirname):
+ """
+ Return maximum filename length for the filesystem
+ :return:
+ """
+ if sys.platform.startswith("linux"):
+ # Linux user's may work on mounted ecryptfs filesystem
+ # which has filename length limitations
+ for entry in _get_mtab():
+ mounted_dir, filesystem = entry[1], entry[2]
+ # http://unix.stackexchange.com/questions/32795/what-is-the-maximum-allowed-filename-and-folder-size-with-ecryptfs
+ if filesystem == "ecryptfs" and dirname and dirname.startswith(mounted_dir):
+ return 140
+ # default maximum filename length for most filesystems
+ return 255
+
+
+def get_unique_file_path(dir_path, filename, cache=collections.defaultdict(set)):
+ """
+ Get unique filename in dir with proper filename length, using given filename/dir.
+ File/dir won't be created (thread nonsafe)
+ :param dir_path: path to dir
+ :param filename: original filename
+ :return: unique filename
+ """
+ max_suffix = 10000
+ # + 1 symbol for dot before suffix
+ tail_length = int(round(math.log(max_suffix, 10))) + 1
+ # truncate filename length in accordance with filesystem limitations
+ filename, extension = os.path.splitext(filename)
+ # XXX
+ if sys.platform.startswith("win"):
+ # Trying to fit into MAX_PATH if it's possible.
+ # Remove after DEVTOOLS-1646
+ max_path = 260
+ filename_len = len(dir_path) + len(extension) + tail_length + len(os.sep)
+ if filename_len < max_path:
+ filename = yatest_lib.tools.trim_string(filename, max_path - filename_len)
+ filename = yatest_lib.tools.trim_string(filename, get_max_filename_length(dir_path) - tail_length - len(extension)) + extension
+ candidate = os.path.join(dir_path, filename)
+
+ key = dir_path + filename
+ counter = sorted(cache.get(key, {0, }))[-1]
+ while os.path.exists(candidate):
+ cache[key].add(counter)
+ counter += 1
+ assert counter < max_suffix
+ candidate = os.path.join(dir_path, filename + ".{}".format(counter))
+ return candidate
+
+
+def escape_for_fnmatch(s):
+ return s.replace("[", "&#91;").replace("]", "&#93;")
+
+
+def get_python_cmd(opts=None, use_huge=True, suite=None):
+ if opts and getattr(opts, 'flags', {}).get("USE_ARCADIA_PYTHON") == "no":
+ return ["python"]
+ if suite and not suite._use_arcadia_python:
+ return ["python"]
+ if use_huge:
+ return ["$(PYTHON)/python"]
+ ymake_path = opts.ymake_bin if opts and getattr(opts, 'ymake_bin', None) else "$(YMAKE)/ymake"
+ return [ymake_path, "--python"]
+
+
+def normalize_name(name):
+ replacements = [
+ ("\\", "\\\\"),
+ ("\n", "\\n"),
+ ("\t", "\\t"),
+ ("\r", "\\r"),
+ ]
+ for l, r in replacements:
+ name = name.replace(l, r)
+ return name
+
+
+def normalize_filename(filename):
+ """
+ Replace invalid for file names characters with string equivalents
+ :param some_string: string to be converted to a valid file name
+ :return: valid file name
+ """
+ not_allowed_pattern = r"[\[\]\/:*?\"\'<>|+\0\\\s\x0b\x0c]"
+ filename = re.sub(not_allowed_pattern, ".", filename)
+ return re.sub(r"\.{2,}", ".", filename)
+
+
+def get_test_log_file_path(output_dir, class_name, test_name, extension="log"):
+ """
+ get test log file path, platform dependant
+ :param output_dir: dir where log file should be placed
+ :param class_name: test class name
+ :param test_name: test name
+ :return: test log file name
+ """
+ if os.name == "nt":
+ # don't add class name to the log's filename
+ # to reduce it's length on windows
+ filename = test_name
+ else:
+ filename = "{}.{}".format(class_name, test_name)
+ if not filename:
+ filename = "test"
+ filename += "." + extension
+ filename = normalize_filename(filename)
+ return get_unique_file_path(output_dir, filename)
+
+
+def split_node_id(nodeid, test_suffix=None):
+ path, possible_open_bracket, params = nodeid.partition('[')
+ separator = "::"
+ if separator in path:
+ path, test_name = path.split(separator, 1)
+ else:
+ test_name = os.path.basename(path)
+ if test_suffix:
+ test_name += "::" + test_suffix
+ class_name = os.path.basename(path.strip())
+ if separator in test_name:
+ klass_name, test_name = test_name.split(separator, 1)
+ if not test_suffix:
+ # test suffix is used for flakes and pep8, no need to add class_name as it's === class_name
+ class_name += separator + klass_name
+ if separator in test_name:
+ test_name = test_name.split(separator)[-1]
+ test_name += possible_open_bracket + params
+ return yatest_lib.tools.to_utf8(class_name), yatest_lib.tools.to_utf8(test_name)
diff --git a/library/python/reservoir_sampling/README.md b/library/python/reservoir_sampling/README.md
new file mode 100644
index 0000000000..27674ba4f0
--- /dev/null
+++ b/library/python/reservoir_sampling/README.md
@@ -0,0 +1,11 @@
+### Overview
+Reservoir sampling is a family of randomized algorithms for choosing a simple random sample, without replacement, of k items from a population of unknown size n in a single pass over the items.
+
+### Example
+
+```jupyter
+In [1]: from library.python import reservoir_sampling
+
+In [2]: reservoir_sampling.reservoir_sampling(data=range(100), nsamples=10)
+Out[2]: [27, 19, 81, 45, 89, 78, 13, 36, 29, 9]
+```
diff --git a/library/python/reservoir_sampling/__init__.py b/library/python/reservoir_sampling/__init__.py
new file mode 100644
index 0000000000..4ee46ee5e1
--- /dev/null
+++ b/library/python/reservoir_sampling/__init__.py
@@ -0,0 +1,16 @@
+import random
+
+
+def reservoir_sampling(data, nsamples, prng=None):
+ if prng is None:
+ prng = random
+
+ result = []
+ for i, entry in enumerate(data):
+ if i < nsamples:
+ result.append(entry)
+ else:
+ j = prng.randint(0, i)
+ if j < nsamples:
+ result[j] = entry
+ return result
diff --git a/library/python/reservoir_sampling/ya.make b/library/python/reservoir_sampling/ya.make
new file mode 100644
index 0000000000..24cac20157
--- /dev/null
+++ b/library/python/reservoir_sampling/ya.make
@@ -0,0 +1,10 @@
+OWNER(
+ prettyboy
+ g:yatool
+)
+
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+END()
diff --git a/library/python/resource/__init__.py b/library/python/resource/__init__.py
new file mode 100644
index 0000000000..26503ef7fc
--- /dev/null
+++ b/library/python/resource/__init__.py
@@ -0,0 +1,49 @@
+from __res import find as __find, count, key_by_index, resfs_files as __resfs_files
+from __res import resfs_read, resfs_resolve, resfs_src # noqa
+
+import six
+
+
+def iterkeys(prefix='', strip_prefix=False):
+ decode = lambda s: s
+ if isinstance(prefix, six.text_type):
+ prefix = prefix.encode('utf-8')
+ decode = lambda s: s.decode('utf-8')
+
+ for i in six.moves.range(count()):
+ key = key_by_index(i)
+ if key.startswith(prefix):
+ if strip_prefix:
+ key = key[len(prefix):]
+ yield decode(key)
+
+
+def itervalues(prefix=b''):
+ for key in iterkeys(prefix=prefix):
+ value = find(key)
+ yield value
+
+
+def iteritems(prefix='', strip_prefix=False):
+ for key in iterkeys(prefix=prefix):
+ value = find(key)
+ if strip_prefix:
+ key = key[len(prefix):]
+ yield key, value
+
+
+def resfs_file_exists(path):
+ return resfs_src(path, resfs_file=True) is not None
+
+
+def resfs_files(prefix=''):
+ decode = lambda s: s
+ if isinstance(prefix, six.text_type):
+ decode = lambda s: s.decode('utf-8')
+ return [decode(s) for s in __resfs_files(prefix=prefix)]
+
+
+def find(path):
+ if isinstance(path, six.text_type):
+ path = path.encode('utf-8')
+ return __find(path)
diff --git a/library/python/resource/ut/lib/qw.txt b/library/python/resource/ut/lib/qw.txt
new file mode 100644
index 0000000000..50e37d5cac
--- /dev/null
+++ b/library/python/resource/ut/lib/qw.txt
@@ -0,0 +1 @@
+na gorshke sidel korol
diff --git a/library/python/resource/ut/lib/test_simple.py b/library/python/resource/ut/lib/test_simple.py
new file mode 100644
index 0000000000..52f006ff91
--- /dev/null
+++ b/library/python/resource/ut/lib/test_simple.py
@@ -0,0 +1,31 @@
+import six # noqa
+
+import library.python.resource as rs
+
+text = b'na gorshke sidel korol\n'
+
+
+def test_find():
+ assert rs.find('/qw.txt') == text
+
+
+def test_iter():
+ assert set(rs.iterkeys()).issuperset({'/qw.txt', '/prefix/1.txt', '/prefix/2.txt'})
+ assert set(rs.iterkeys(prefix='/prefix/')) == {'/prefix/1.txt', '/prefix/2.txt'}
+ assert set(rs.iterkeys(prefix='/prefix/', strip_prefix=True)) == {'1.txt', '2.txt'}
+ assert set(rs.iteritems(prefix='/prefix')) == {
+ ('/prefix/1.txt', text),
+ ('/prefix/2.txt', text),
+ }
+ assert set(rs.iteritems(prefix='/prefix', strip_prefix=True)) == {
+ ('/1.txt', text),
+ ('/2.txt', text),
+ }
+
+
+def test_resfs_files():
+ assert 'contrib/python/py/.dist-info/METADATA' in set(rs.resfs_files())
+
+
+def test_resfs_read():
+ assert 'Metadata-Version' in rs.resfs_read('contrib/python/py/.dist-info/METADATA').decode('utf-8')
diff --git a/library/python/resource/ut/lib/ya.make b/library/python/resource/ut/lib/ya.make
new file mode 100644
index 0000000000..693e388878
--- /dev/null
+++ b/library/python/resource/ut/lib/ya.make
@@ -0,0 +1,17 @@
+PY23_LIBRARY()
+
+OWNER(pg)
+
+TEST_SRCS(test_simple.py)
+
+PEERDIR(
+ library/python/resource
+)
+
+RESOURCE(
+ qw.txt /qw.txt
+ qw.txt /prefix/1.txt
+ qw.txt /prefix/2.txt
+)
+
+END()
diff --git a/library/python/resource/ut/py2/ya.make b/library/python/resource/ut/py2/ya.make
new file mode 100644
index 0000000000..5085610faf
--- /dev/null
+++ b/library/python/resource/ut/py2/ya.make
@@ -0,0 +1,9 @@
+PY2TEST()
+
+OWNER(pg)
+
+PEERDIR(
+ library/python/resource/ut/lib
+)
+
+END()
diff --git a/library/python/resource/ut/py3/ya.make b/library/python/resource/ut/py3/ya.make
new file mode 100644
index 0000000000..64eb2e83ce
--- /dev/null
+++ b/library/python/resource/ut/py3/ya.make
@@ -0,0 +1,9 @@
+PY3TEST()
+
+OWNER(pg)
+
+PEERDIR(
+ library/python/resource/ut/lib
+)
+
+END()
diff --git a/library/python/resource/ut/ya.make b/library/python/resource/ut/ya.make
new file mode 100644
index 0000000000..a5ec192d74
--- /dev/null
+++ b/library/python/resource/ut/ya.make
@@ -0,0 +1,6 @@
+OWNER(pg)
+
+RECURSE(
+ py2
+ py3
+)
diff --git a/library/python/resource/ya.make b/library/python/resource/ya.make
new file mode 100644
index 0000000000..989329fa4b
--- /dev/null
+++ b/library/python/resource/ya.make
@@ -0,0 +1,13 @@
+PY23_LIBRARY()
+
+OWNER(pg)
+
+PEERDIR(
+ contrib/python/six
+)
+
+PY_SRCS(__init__.py)
+
+END()
+
+RECURSE_FOR_TESTS(ut)
diff --git a/library/python/runtime_py3/__res.pyx b/library/python/runtime_py3/__res.pyx
new file mode 100644
index 0000000000..97190d9f29
--- /dev/null
+++ b/library/python/runtime_py3/__res.pyx
@@ -0,0 +1,36 @@
+from _codecs import utf_8_decode, utf_8_encode
+
+from libcpp cimport bool
+
+from util.generic.string cimport TString, TStringBuf
+
+
+cdef extern from "library/cpp/resource/resource.h" namespace "NResource":
+ cdef size_t Count() except +
+ cdef TStringBuf KeyByIndex(size_t idx) except +
+ cdef bool FindExact(const TStringBuf key, TString* result) nogil except +
+
+
+def count():
+ return Count()
+
+
+def key_by_index(idx):
+ cdef TStringBuf ret = KeyByIndex(idx)
+
+ return ret.Data()[:ret.Size()]
+
+
+def find(s):
+ cdef TString res
+
+ if isinstance(s, str):
+ s = utf_8_encode(s)[0]
+
+ if FindExact(TStringBuf(s, len(s)), &res):
+ return res.c_str()[:res.length()]
+
+ return None
+
+
+include "importer.pxi"
diff --git a/library/python/runtime_py3/entry_points.py b/library/python/runtime_py3/entry_points.py
new file mode 100644
index 0000000000..05098723cb
--- /dev/null
+++ b/library/python/runtime_py3/entry_points.py
@@ -0,0 +1,52 @@
+import sys
+
+import __res
+
+
+def repl():
+ user_ns = {}
+ py_main = __res.find('PY_MAIN')
+
+ if py_main:
+ mod_name, func_name = (py_main.split(b':', 1) + [None])[:2]
+ try:
+ import importlib
+ mod = importlib.import_module(mod_name.decode('UTF-8'))
+ user_ns = mod.__dict__
+ except:
+ import traceback
+ traceback.print_exc()
+
+ if func_name and '__main__' not in user_ns:
+ def run(args):
+ if isinstance(args, str):
+ import shlex
+ args = shlex.split(args)
+
+ import sys
+ sys.argv = [sys.argv[0]] + args
+ getattr(mod, func_name)()
+
+ user_ns['__main__'] = run
+
+ try:
+ import IPython
+ except ModuleNotFoundError:
+ pass
+ else:
+ return IPython.start_ipython(user_ns=user_ns)
+
+ import code
+ code.interact(local=user_ns)
+
+
+def resource_files():
+ sys.stdout.buffer.write(b'\n'.join(sorted(__res.resfs_files()) + [b'']))
+
+
+def run_constructors():
+ for key, module_name in __res.iter_keys(b'py/constructors/'):
+ import importlib
+ module = importlib.import_module(module_name.decode())
+ init_func = getattr(module, __res.find(key).decode())
+ init_func()
diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi
new file mode 100644
index 0000000000..904f94dea2
--- /dev/null
+++ b/library/python/runtime_py3/importer.pxi
@@ -0,0 +1,571 @@
+import marshal
+import sys
+from _codecs import utf_8_decode, utf_8_encode
+from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter
+from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split
+from _io import FileIO
+
+import __res as __resource
+
+_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0]
+_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0]
+env_entry_point = b'Y_PYTHON_ENTRY_POINT'
+env_source_root = b'Y_PYTHON_SOURCE_ROOT'
+cfg_source_root = b'arcadia-source-root'
+env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH'
+res_ya_ide_venv = b'YA_IDE_VENV'
+executable = sys.executable or 'Y_PYTHON'
+sys.modules['run_import_hook'] = __resource
+
+# This is the prefix in contrib/tools/python3/src/Lib/ya.make.
+py_prefix = b'py/'
+py_prefix_len = len(py_prefix)
+
+YA_IDE_VENV = __resource.find(res_ya_ide_venv)
+Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV
+
+
+def _init_venv():
+ if not _path_isabs(executable):
+ raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable))
+
+ # Creative copy-paste from site.py
+ exe_dir, _ = _path_split(executable)
+ site_prefix, _ = _path_split(exe_dir)
+ libpath = _path_join(site_prefix, 'lib',
+ 'python%d.%d' % sys.version_info[:2],
+ 'site-packages')
+ sys.path.insert(0, libpath)
+
+ # emulate site.venv()
+ sys.prefix = site_prefix
+ sys.exec_prefix = site_prefix
+
+ conf_basename = 'pyvenv.cfg'
+ candidate_confs = [
+ conffile for conffile in (
+ _path_join(exe_dir, conf_basename),
+ _path_join(site_prefix, conf_basename)
+ )
+ if _path_isfile(conffile)
+ ]
+ if not candidate_confs:
+ raise RuntimeError('{} not found'.format(conf_basename))
+ virtual_conf = candidate_confs[0]
+ with FileIO(virtual_conf, 'r') as f:
+ for line in f:
+ if b'=' in line:
+ key, _, value = line.partition(b'=')
+ key = key.strip().lower()
+ value = value.strip()
+ if key == cfg_source_root:
+ return value
+ raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf))
+
+
+def _get_source_root():
+ env_value = _os.environ.get(env_source_root)
+ if env_value or not YA_IDE_VENV:
+ return env_value
+
+ return _init_venv()
+
+
+Y_PYTHON_SOURCE_ROOT = _get_source_root()
+
+
+def _print(*xs):
+ """
+ This is helpful for debugging, since automatic bytes to str conversion is
+ not available yet. It is also possible to debug with GDB by breaking on
+ __Pyx_AddTraceback (with Python GDB pretty printers enabled).
+ """
+ parts = []
+ for s in xs:
+ if not isinstance(s, (bytes, str)):
+ s = str(s)
+ parts.append(_s(s))
+ sys.stderr.write(' '.join(parts) + '\n')
+
+
+def file_bytes(path):
+ # 'open' is not avaiable yet.
+ with FileIO(path, 'r') as f:
+ return f.read()
+
+
+def iter_keys(prefix):
+ l = len(prefix)
+ for idx in range(__resource.count()):
+ key = __resource.key_by_index(idx)
+ if key.startswith(prefix):
+ yield key, key[l:]
+
+
+def iter_py_modules(with_keys=False):
+ for key, path in iter_keys(b'resfs/file/' + py_prefix):
+ if path.endswith(b'.py'): # It may also end with '.pyc'.
+ mod = _s(path[:-3].replace(b'/', b'.'))
+ if with_keys:
+ yield key, mod
+ else:
+ yield mod
+
+
+def iter_prefixes(s):
+ i = s.find('.')
+ while i >= 0:
+ yield s[:i]
+ i = s.find('.', i + 1)
+
+
+def resfs_resolve(path):
+ """
+ Return the absolute path of a root-relative path if it exists.
+ """
+ path = _b(path)
+ if Y_PYTHON_SOURCE_ROOT:
+ if not path.startswith(Y_PYTHON_SOURCE_ROOT):
+ path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path))
+ if _path_isfile(path):
+ return path
+
+
+def resfs_src(key, resfs_file=False):
+ """
+ Return the root-relative file path of a resource key.
+ """
+ if resfs_file:
+ key = b'resfs/file/' + _b(key)
+ return __resource.find(b'resfs/src/' + _b(key))
+
+
+def resfs_read(path, builtin=None):
+ """
+ Return the bytes of the resource file at path, or None.
+ If builtin is True, do not look for it on the filesystem.
+ If builtin is False, do not look in the builtin resources.
+ """
+ if builtin is not True:
+ arcpath = resfs_src(path, resfs_file=True)
+ if arcpath:
+ fspath = resfs_resolve(arcpath)
+ if fspath:
+ return file_bytes(fspath)
+
+ if builtin is not False:
+ return __resource.find(b'resfs/file/' + _b(path))
+
+
+def resfs_files(prefix=b''):
+ """
+ List builtin resource file paths.
+ """
+ return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))]
+
+
+def mod_path(mod):
+ """
+ Return the resfs path to the source code of the module with the given name.
+ """
+ return py_prefix + _b(mod).replace(b'.', b'/') + b'.py'
+
+
+class ResourceImporter(object):
+
+ """ A meta_path importer that loads code from built-in resources.
+ """
+
+ def __init__(self):
+ self.memory = set(iter_py_modules()) # Set of importable module names.
+ self.source_map = {} # Map from file names to module names.
+ self._source_name = {} # Map from original to altered module names.
+ self._package_prefix = ''
+ if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH:
+ self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT))
+ else:
+ self.arcadia_source_finder = None
+
+ for p in list(self.memory) + list(sys.builtin_module_names):
+ for pp in iter_prefixes(p):
+ k = pp + '.__init__'
+ if k not in self.memory:
+ self.memory.add(k)
+
+ def for_package(self, name):
+ import copy
+ importer = copy.copy(self)
+ importer._package_prefix = name + '.'
+ return importer
+
+ def _find_mod_path(self, fullname):
+ """Find arcadia relative path by module name"""
+ relpath = resfs_src(mod_path(fullname), resfs_file=True)
+ if relpath or not self.arcadia_source_finder:
+ return relpath
+ return self.arcadia_source_finder.get_module_path(fullname)
+
+ def find_spec(self, fullname, path=None, target=None):
+ try:
+ is_package = self.is_package(fullname)
+ except ImportError:
+ return None
+ return spec_from_loader(fullname, self, is_package=is_package)
+
+ def find_module(self, fullname, path=None):
+ """For backward compatibility."""
+ spec = self.find_spec(fullname, path)
+ return spec.loader if spec is not None else None
+
+ def create_module(self, spec):
+ """Use default semantics for module creation."""
+
+ def exec_module(self, module):
+ code = self.get_code(module.__name__)
+ module.__file__ = code.co_filename
+ if self.is_package(module.__name__):
+ module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)]
+ # exec(code, module.__dict__)
+ _call_with_frames_removed(exec, code, module.__dict__)
+
+ # PEP-302 extension 1 of 3: data loader.
+ def get_data(self, path):
+ path = _b(path)
+ abspath = resfs_resolve(path)
+ if abspath:
+ return file_bytes(abspath)
+ path = path.replace(_b('\\'), _b('/'))
+ data = resfs_read(path, builtin=True)
+ if data is None:
+ raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files
+ return data
+
+ # PEP-302 extension 2 of 3: get __file__ without importing.
+ def get_filename(self, fullname):
+ modname = fullname
+ if self.is_package(fullname):
+ fullname += '.__init__'
+ relpath = self._find_mod_path(fullname)
+ if isinstance(relpath, bytes):
+ relpath = _s(relpath)
+ return relpath or modname
+
+ # PEP-302 extension 3 of 3: packaging introspection.
+ # Used by `linecache` (while printing tracebacks) unless module filename
+ # exists on the filesystem.
+ def get_source(self, fullname):
+ fullname = self._source_name.get(fullname) or fullname
+ if self.is_package(fullname):
+ fullname += '.__init__'
+
+ relpath = self.get_filename(fullname)
+ if relpath:
+ abspath = resfs_resolve(relpath)
+ if abspath:
+ return _s(file_bytes(abspath))
+ data = resfs_read(mod_path(fullname))
+ return _s(data) if data else ''
+
+ def get_code(self, fullname):
+ modname = fullname
+ if self.is_package(fullname):
+ fullname += '.__init__'
+
+ path = mod_path(fullname)
+ relpath = self._find_mod_path(fullname)
+ if relpath:
+ abspath = resfs_resolve(relpath)
+ if abspath:
+ data = file_bytes(abspath)
+ return compile(data, _s(abspath), 'exec', dont_inherit=True)
+
+ yapyc_path = path + b'.yapyc3'
+ yapyc_data = resfs_read(yapyc_path, builtin=True)
+ if yapyc_data:
+ return marshal.loads(yapyc_data)
+ else:
+ py_data = resfs_read(path, builtin=True)
+ if py_data:
+ return compile(py_data, _s(relpath), 'exec', dont_inherit=True)
+ else:
+ # This covers packages with no __init__.py in resources.
+ return compile('', modname, 'exec', dont_inherit=True)
+
+ def is_package(self, fullname):
+ if fullname in self.memory:
+ return False
+
+ if fullname + '.__init__' in self.memory:
+ return True
+
+ if self.arcadia_source_finder:
+ return self.arcadia_source_finder.is_package(fullname)
+
+ raise ImportError(fullname)
+
+ # Extension for contrib/python/coverage.
+ def file_source(self, filename):
+ """
+ Return the key of the module source by its resource path.
+ """
+ if not self.source_map:
+ for key, mod in iter_py_modules(with_keys=True):
+ path = self.get_filename(mod)
+ self.source_map[path] = key
+
+ if filename in self.source_map:
+ return self.source_map[filename]
+
+ if resfs_read(filename, builtin=True) is not None:
+ return b'resfs/file/' + _b(filename)
+
+ return b''
+
+ # Extension for pkgutil.iter_modules.
+ def iter_modules(self, prefix=''):
+ import re
+ rx = re.compile(re.escape(self._package_prefix) + r'([^.]+)(\.__init__)?$')
+ for p in self.memory:
+ m = rx.match(p)
+ if m:
+ yield prefix + m.group(1), m.group(2) is not None
+ if self.arcadia_source_finder:
+ for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix):
+ yield m
+
+ def get_resource_reader(self, fullname):
+ try:
+ if not self.is_package(fullname):
+ return None
+ except ImportError:
+ return None
+ return _ResfsResourceReader(self, fullname)
+
+
+class _ResfsResourceReader:
+
+ def __init__(self, importer, fullname):
+ self.importer = importer
+ self.fullname = fullname
+
+ import os
+ self.prefix = "{}/".format(os.path.dirname(self.importer.get_filename(self.fullname)))
+
+ def open_resource(self, resource):
+ path = f'{self.prefix}{resource}'
+ from io import BytesIO
+ try:
+ return BytesIO(self.importer.get_data(path))
+ except OSError:
+ raise FileNotFoundError(path)
+
+ def resource_path(self, resource):
+ # All resources are in the binary file, so there is no path to the file.
+ # Raising FileNotFoundError tells the higher level API to extract the
+ # binary data and create a temporary file.
+ raise FileNotFoundError
+
+ def is_resource(self, name):
+ path = f'{self.prefix}{name}'
+ try:
+ self.importer.get_data(path)
+ except OSError:
+ return False
+ return True
+
+ def contents(self):
+ subdirs_seen = set()
+ for key in resfs_files(self.prefix):
+ relative = key[len(self.prefix):]
+ res_or_subdir, *other = relative.split(b'/')
+ if not other:
+ yield _s(res_or_subdir)
+ elif res_or_subdir not in subdirs_seen:
+ subdirs_seen.add(res_or_subdir)
+ yield _s(res_or_subdir)
+
+
+class BuiltinSubmoduleImporter(BuiltinImporter):
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ if path is not None:
+ return super().find_spec(fullname, None, target)
+ else:
+ return None
+
+
+class ArcadiaSourceFinder:
+ """
+ Search modules and packages in arcadia source tree.
+ See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details
+ """
+ NAMESPACE_PREFIX = b'py/namespace/'
+ PY_EXT = '.py'
+ YA_MAKE = 'ya.make'
+
+ def __init__(self, source_root):
+ self.source_root = source_root
+ self.module_path_cache = {'': set()}
+ for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX):
+ # dirty_path contains unique prefix to prevent repeatable keys in the resource storage
+ path = dirty_path.split(b'/', 1)[1]
+ namespaces = __resource.find(key).split(b':')
+ for n in namespaces:
+ package_name = _s(n.rstrip(b'.'))
+ self.module_path_cache.setdefault(package_name, set()).add(_s(path))
+ # Fill parents with default empty path set if parent doesn't exist in the cache yet
+ while package_name:
+ package_name = package_name.rpartition('.')[0]
+ if package_name in self.module_path_cache:
+ break
+ self.module_path_cache.setdefault(package_name, set())
+ for package_name in self.module_path_cache.keys():
+ self._add_parent_dirs(package_name, visited=set())
+
+ def get_module_path(self, fullname):
+ """
+ Find file path for module 'fullname'.
+ For packages caller pass fullname as 'package.__init__'.
+ Return None if nothing is found.
+ """
+ try:
+ if not self.is_package(fullname):
+ return _b(self._cache_module_path(fullname))
+ except ImportError:
+ pass
+
+ def is_package(self, fullname):
+ """Check if fullname is a package. Raise ImportError if fullname is not found"""
+ path = self._cache_module_path(fullname)
+ if isinstance(path, set):
+ return True
+ if isinstance(path, str):
+ return False
+ raise ImportError(fullname)
+
+ def iter_modules(self, package_prefix, prefix):
+ paths = self._cache_module_path(package_prefix.rstrip('.'))
+ if paths is not None:
+ # Note: it's ok to yield duplicates because pkgutil discards them
+
+ # Yield from cache
+ import re
+ rx = re.compile(re.escape(package_prefix) + r'([^.]+)$')
+ for mod, path in self.module_path_cache.items():
+ if path is not None:
+ m = rx.match(mod)
+ if m:
+ yield prefix + m.group(1), self.is_package(mod)
+
+ # Yield from file system
+ for path in paths:
+ abs_path = _path_join(self.source_root, path)
+ for dir_item in _os.listdir(abs_path):
+ if self._path_is_simple_dir(_path_join(abs_path, dir_item)):
+ yield prefix + dir_item, True
+ elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)):
+ yield prefix + dir_item[:-len(self.PY_EXT)], False
+
+ def _path_is_simple_dir(self, abs_path):
+ """
+ Check if path is a directory but doesn't contain ya.make file.
+ We don't want to steal directory from nested project and treat it as a package
+ """
+ return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE))
+
+ def _find_module_in_paths(self, find_package_only, paths, module):
+ """Auxiliary method. See _cache_module_path() for details"""
+ if paths:
+ package_paths = set()
+ for path in paths:
+ rel_path = _path_join(path, module)
+ if not find_package_only:
+ # Check if file_path is a module
+ module_path = rel_path + self.PY_EXT
+ if _path_isfile(_path_join(self.source_root, module_path)):
+ return module_path
+ # Check if file_path is a package
+ if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
+ package_paths.add(rel_path)
+ if package_paths:
+ return package_paths
+
+ def _cache_module_path(self, fullname, find_package_only=False):
+ """
+ Find module path or package directory paths and save result in the cache
+
+ find_package_only=True - don't try to find module
+
+ Returns:
+ List of relative package paths - for a package
+ Relative module path - for a module
+ None - module or package is not found
+ """
+ if fullname not in self.module_path_cache:
+ parent, _, tail = fullname.rpartition('.')
+ parent_paths = self._cache_module_path(parent, find_package_only=True)
+ self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail)
+ return self.module_path_cache[fullname]
+
+ def _add_parent_dirs(self, package_name, visited):
+ if not package_name or package_name in visited:
+ return
+ visited.add(package_name)
+
+ parent, _, tail = package_name.rpartition('.')
+ self._add_parent_dirs(parent, visited)
+
+ paths = self.module_path_cache[package_name]
+ for parent_path in self.module_path_cache[parent]:
+ rel_path = _path_join(parent_path, tail)
+ if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
+ paths.add(rel_path)
+
+
+def excepthook(*args, **kws):
+ # traceback module cannot be imported at module level, because interpreter
+ # is not fully initialized yet
+
+ import traceback
+
+ return traceback.print_exception(*args, **kws)
+
+
+importer = ResourceImporter()
+
+
+def executable_path_hook(path):
+ if path == executable:
+ return importer
+
+ if path.startswith(executable + path_sep):
+ return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.'))
+
+ raise ImportError(path)
+
+
+if YA_IDE_VENV:
+ sys.meta_path.append(importer)
+ sys.meta_path.append(BuiltinSubmoduleImporter)
+ if executable not in sys.path:
+ sys.path.append(executable)
+ sys.path_hooks.append(executable_path_hook)
+else:
+ sys.meta_path.insert(0, BuiltinSubmoduleImporter)
+ sys.meta_path.insert(0, importer)
+ if executable not in sys.path:
+ sys.path.insert(0, executable)
+ sys.path_hooks.insert(0, executable_path_hook)
+
+sys.path_importer_cache[executable] = importer
+
+# Indicator that modules and resources are built-in rather than on the file system.
+sys.is_standalone_binary = True
+sys.frozen = True
+
+# Set of names of importable modules.
+sys.extra_modules = importer.memory
+
+# Use custom implementation of traceback printer.
+# Built-in printer (PyTraceBack_Print) does not support custom module loaders
+sys.excepthook = excepthook
diff --git a/library/python/runtime_py3/main/get_py_main.cpp b/library/python/runtime_py3/main/get_py_main.cpp
new file mode 100644
index 0000000000..67c400d4f4
--- /dev/null
+++ b/library/python/runtime_py3/main/get_py_main.cpp
@@ -0,0 +1,8 @@
+#include <library/cpp/resource/resource.h>
+
+#include <stdlib.h>
+
+extern "C" char* GetPyMain() {
+ TString res = NResource::Find("PY_MAIN");
+ return strdup(res.c_str());
+}
diff --git a/library/python/runtime_py3/main/main.c b/library/python/runtime_py3/main/main.c
new file mode 100644
index 0000000000..3159800615
--- /dev/null
+++ b/library/python/runtime_py3/main/main.c
@@ -0,0 +1,231 @@
+#include <Python.h>
+#include <contrib/tools/python3/src/Include/internal/pycore_runtime.h> // _PyRuntime_Initialize()
+
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+
+void Py_InitArgcArgv(int argc, wchar_t **argv);
+char* GetPyMain();
+
+static const char* env_entry_point = "Y_PYTHON_ENTRY_POINT";
+static const char* env_bytes_warning = "Y_PYTHON_BYTES_WARNING";
+
+#ifdef _MSC_VER
+extern char** environ;
+
+void unsetenv(const char* name) {
+ const int n = strlen(name);
+ char** dst = environ;
+ for (char** src = environ; *src; src++)
+ if (strncmp(*src, name, n) || (*src)[n] != '=')
+ *dst++ = *src;
+ *dst = NULL;
+}
+#endif
+
+static int RunModule(const char *modname)
+{
+ PyObject *module, *runpy, *runmodule, *runargs, *result;
+ runpy = PyImport_ImportModule("runpy");
+ if (runpy == NULL) {
+ fprintf(stderr, "Could not import runpy module\n");
+ PyErr_Print();
+ return -1;
+ }
+ runmodule = PyObject_GetAttrString(runpy, "_run_module_as_main");
+ if (runmodule == NULL) {
+ fprintf(stderr, "Could not access runpy._run_module_as_main\n");
+ PyErr_Print();
+ Py_DECREF(runpy);
+ return -1;
+ }
+ module = PyUnicode_FromString(modname);
+ if (module == NULL) {
+ fprintf(stderr, "Could not convert module name to unicode\n");
+ PyErr_Print();
+ Py_DECREF(runpy);
+ Py_DECREF(runmodule);
+ return -1;
+ }
+ runargs = Py_BuildValue("(Oi)", module, 0);
+ if (runargs == NULL) {
+ fprintf(stderr,
+ "Could not create arguments for runpy._run_module_as_main\n");
+ PyErr_Print();
+ Py_DECREF(runpy);
+ Py_DECREF(runmodule);
+ Py_DECREF(module);
+ return -1;
+ }
+ result = PyObject_Call(runmodule, runargs, NULL);
+ if (result == NULL) {
+ PyErr_Print();
+ }
+ Py_DECREF(runpy);
+ Py_DECREF(runmodule);
+ Py_DECREF(module);
+ Py_DECREF(runargs);
+ if (result == NULL) {
+ return -1;
+ }
+ Py_DECREF(result);
+ return 0;
+}
+
+static int pymain(int argc, char** argv) {
+ PyStatus status = _PyRuntime_Initialize();
+ if (PyStatus_Exception(status)) {
+ Py_ExitStatusException(status);
+ }
+
+ int i, sts = 1;
+ char* oldloc = NULL;
+ wchar_t** argv_copy = NULL;
+ /* We need a second copies, as Python might modify the first one. */
+ wchar_t** argv_copy2 = NULL;
+ char* entry_point_copy = NULL;
+
+ if (argc > 0) {
+ argv_copy = PyMem_RawMalloc(sizeof(wchar_t*) * argc);
+ argv_copy2 = PyMem_RawMalloc(sizeof(wchar_t*) * argc);
+ if (!argv_copy || !argv_copy2) {
+ fprintf(stderr, "out of memory\n");
+ goto error;
+ }
+ }
+
+ PyConfig config;
+ PyConfig_InitPythonConfig(&config);
+ config.pathconfig_warnings = 0; /* Suppress errors from getpath.c */
+
+ const char* bytes_warning = getenv(env_bytes_warning);
+ if (bytes_warning) {
+ config.bytes_warning = atoi(bytes_warning);
+ }
+
+ oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
+ if (!oldloc) {
+ fprintf(stderr, "out of memory\n");
+ goto error;
+ }
+
+ setlocale(LC_ALL, "");
+ for (i = 0; i < argc; i++) {
+ argv_copy[i] = Py_DecodeLocale(argv[i], NULL);
+ argv_copy2[i] = argv_copy[i];
+ if (!argv_copy[i]) {
+ fprintf(stderr, "Unable to decode the command line argument #%i\n",
+ i + 1);
+ argc = i;
+ goto error;
+ }
+ }
+ setlocale(LC_ALL, oldloc);
+ PyMem_RawFree(oldloc);
+ oldloc = NULL;
+
+ if (argc >= 1)
+ Py_SetProgramName(argv_copy[0]);
+
+ status = Py_InitializeFromConfig(&config);
+ PyConfig_Clear(&config);
+ if (PyStatus_Exception(status)) {
+ Py_ExitStatusException(status);
+ }
+
+ const char* entry_point = getenv(env_entry_point);
+ if (entry_point) {
+ entry_point_copy = strdup(entry_point);
+ if (!entry_point_copy) {
+ fprintf(stderr, "out of memory\n");
+ goto error;
+ }
+ } else {
+ entry_point_copy = GetPyMain();
+ }
+
+ if (entry_point_copy == NULL) {
+ fprintf(stderr, "No entry point, did you forget PY_MAIN?\n");
+ goto error;
+ }
+
+ if (entry_point_copy && !strcmp(entry_point_copy, ":main")) {
+ unsetenv(env_entry_point);
+ sts = Py_Main(argc, argv_copy);
+ free(entry_point_copy);
+ return sts;
+ }
+
+ Py_InitArgcArgv(argc, argv_copy);
+ PySys_SetArgv(argc, argv_copy);
+
+ {
+ PyObject* module = PyImport_ImportModule("library.python.runtime_py3.entry_points");
+ if (module == NULL) {
+ PyErr_Print();
+ } else {
+ PyObject* res = PyObject_CallMethod(module, "run_constructors", NULL);
+ if (res == NULL) {
+ PyErr_Print();
+ } else {
+ Py_DECREF(res);
+ }
+ Py_DECREF(module);
+ }
+ }
+
+ const char* module_name = entry_point_copy;
+ const char* func_name = NULL;
+
+ char *colon = strchr(entry_point_copy, ':');
+ if (colon != NULL) {
+ colon[0] = '\0';
+ func_name = colon + 1;
+ }
+ if (module_name[0] == '\0') {
+ module_name = "library.python.runtime_py3.entry_points";
+ }
+
+ if (!func_name) {
+ sts = RunModule(module_name);
+ } else {
+ PyObject* module = PyImport_ImportModule(module_name);
+
+ if (module == NULL) {
+ PyErr_Print();
+ } else {
+ PyObject* value = PyObject_CallMethod(module, func_name, NULL);
+
+ if (value == NULL) {
+ PyErr_Print();
+ } else {
+ Py_DECREF(value);
+ sts = 0;
+ }
+
+ Py_DECREF(module);
+ }
+ }
+
+ if (Py_FinalizeEx() < 0) {
+ sts = 120;
+ }
+
+error:
+ free(entry_point_copy);
+ PyMem_RawFree(argv_copy);
+ if (argv_copy2) {
+ for (i = 0; i < argc; i++)
+ PyMem_RawFree(argv_copy2[i]);
+ PyMem_RawFree(argv_copy2);
+ }
+ PyMem_RawFree(oldloc);
+ return sts;
+}
+
+int (*mainptr)(int argc, char** argv) = pymain;
+
+int main(int argc, char** argv) {
+ return mainptr(argc, argv);
+}
diff --git a/library/python/runtime_py3/main/ya.make b/library/python/runtime_py3/main/ya.make
new file mode 100644
index 0000000000..f308a93b28
--- /dev/null
+++ b/library/python/runtime_py3/main/ya.make
@@ -0,0 +1,26 @@
+LIBRARY()
+
+OWNER(
+ pg
+ orivej
+)
+
+PEERDIR(
+ contrib/tools/python3/src
+ library/cpp/resource
+)
+
+ADDINCL(
+ contrib/tools/python3/src/Include
+)
+
+CFLAGS(
+ -DPy_BUILD_CORE
+)
+
+SRCS(
+ main.c
+ get_py_main.cpp
+)
+
+END()
diff --git a/library/python/runtime_py3/sitecustomize.pyx b/library/python/runtime_py3/sitecustomize.pyx
new file mode 100644
index 0000000000..966bbe8ba6
--- /dev/null
+++ b/library/python/runtime_py3/sitecustomize.pyx
@@ -0,0 +1,69 @@
+import re
+import sys
+
+import __res
+
+from importlib.abc import ResourceReader
+from importlib.metadata import Distribution, DistributionFinder, PackageNotFoundError, Prepared
+
+ResourceReader.register(__res._ResfsResourceReader)
+
+METADATA_NAME = re.compile('^Name: (.*)$', re.MULTILINE)
+
+
+class ArcadiaDistribution(Distribution):
+
+ def __init__(self, prefix):
+ self.prefix = prefix
+
+ def read_text(self, filename):
+ data = __res.resfs_read(f'{self.prefix}{filename}')
+ if data:
+ return data.decode('utf-8')
+ read_text.__doc__ = Distribution.read_text.__doc__
+
+ def locate_file(self, path):
+ return f'{self.prefix}{path}'
+
+
+class ArcadiaMetadataFinder(DistributionFinder):
+
+ prefixes = {}
+
+ @classmethod
+ def find_distributions(cls, context=DistributionFinder.Context()):
+ found = cls._search_prefixes(context.name)
+ return map(ArcadiaDistribution, found)
+
+ @classmethod
+ def _init_prefixes(cls):
+ cls.prefixes.clear()
+
+ for resource in __res.resfs_files():
+ resource = resource.decode('utf-8')
+ if not resource.endswith('METADATA'):
+ continue
+ data = __res.resfs_read(resource).decode('utf-8')
+ metadata_name = METADATA_NAME.search(data)
+ if metadata_name:
+ metadata_name = Prepared(metadata_name.group(1))
+ cls.prefixes[metadata_name.normalized] = resource[:-len('METADATA')]
+
+ @classmethod
+ def _search_prefixes(cls, name):
+ if not cls.prefixes:
+ cls._init_prefixes()
+
+ if name:
+ try:
+ yield cls.prefixes[Prepared(name).normalized]
+ except KeyError:
+ raise PackageNotFoundError(name)
+ else:
+ for prefix in sorted(cls.prefixes.values()):
+ yield prefix
+
+
+# monkeypatch standart library
+import importlib.metadata
+importlib.metadata.MetadataPathFinder = ArcadiaMetadataFinder
diff --git a/library/python/runtime_py3/test/.dist-info/METADATA b/library/python/runtime_py3/test/.dist-info/METADATA
new file mode 100644
index 0000000000..bb36162199
--- /dev/null
+++ b/library/python/runtime_py3/test/.dist-info/METADATA
@@ -0,0 +1,14 @@
+Metadata-Version: 2.1
+Name: foo-bar
+Version: 1.2.3
+Summary:
+Home-page: https://foo.org/
+Author: Foo
+Author-email: foo@ya.com
+License: UNKNOWN
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Requires-Dist: Werkzeug (>=0.15)
+Requires-Dist: Jinja2 (>=2.10.1)
diff --git a/library/python/runtime_py3/test/.dist-info/RECORD b/library/python/runtime_py3/test/.dist-info/RECORD
new file mode 100644
index 0000000000..dabbbff80d
--- /dev/null
+++ b/library/python/runtime_py3/test/.dist-info/RECORD
@@ -0,0 +1 @@
+foo_bar.py,sha256=0000000000000000000000000000000000000000000,20
diff --git a/library/python/runtime_py3/test/.dist-info/entry_points.txt b/library/python/runtime_py3/test/.dist-info/entry_points.txt
new file mode 100644
index 0000000000..f5e2fd2657
--- /dev/null
+++ b/library/python/runtime_py3/test/.dist-info/entry_points.txt
@@ -0,0 +1,2 @@
+[console_scripts]
+foo_cli = foo_bar:cli
diff --git a/library/python/runtime_py3/test/.dist-info/top_level.txt b/library/python/runtime_py3/test/.dist-info/top_level.txt
new file mode 100644
index 0000000000..d2c068bc6b
--- /dev/null
+++ b/library/python/runtime_py3/test/.dist-info/top_level.txt
@@ -0,0 +1 @@
+foo_bar
diff --git a/library/python/runtime_py3/test/canondata/result.json b/library/python/runtime_py3/test/canondata/result.json
new file mode 100644
index 0000000000..a7d045fc9c
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/result.json
@@ -0,0 +1,62 @@
+{
+ "test_traceback.test_traceback[custom-default]": {
+ "stderr": {
+ "checksum": "6c1a9b47baa51cc6903b85fd43c529b5",
+ "uri": "file://test_traceback.test_traceback_custom-default_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_custom-default_/stdout.txt"
+ }
+ },
+ "test_traceback.test_traceback[custom-ultratb_color]": {
+ "stderr": {
+ "checksum": "048e27049fb8db64bd295b17f505b0ad",
+ "uri": "file://test_traceback.test_traceback_custom-ultratb_color_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_custom-ultratb_color_/stdout.txt"
+ }
+ },
+ "test_traceback.test_traceback[custom-ultratb_verbose]": {
+ "stderr": {
+ "checksum": "e9af42aa3736141d9b67a1652eea412e",
+ "uri": "file://test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt"
+ }
+ },
+ "test_traceback.test_traceback[main-default]": {
+ "stderr": {
+ "checksum": "6c1a9b47baa51cc6903b85fd43c529b5",
+ "uri": "file://test_traceback.test_traceback_main-default_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_main-default_/stdout.txt"
+ }
+ },
+ "test_traceback.test_traceback[main-ultratb_color]": {
+ "stderr": {
+ "checksum": "048e27049fb8db64bd295b17f505b0ad",
+ "uri": "file://test_traceback.test_traceback_main-ultratb_color_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_main-ultratb_color_/stdout.txt"
+ }
+ },
+ "test_traceback.test_traceback[main-ultratb_verbose]": {
+ "stderr": {
+ "checksum": "e9af42aa3736141d9b67a1652eea412e",
+ "uri": "file://test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt"
+ },
+ "stdout": {
+ "checksum": "e120a1e0b7fdddc8e6b4d4b506403e89",
+ "uri": "file://test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt"
+ }
+ }
+} \ No newline at end of file
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt
new file mode 100644
index 0000000000..5eb7da170a
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stderr.txt
@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+ File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main
+ one()
+ File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one
+ modfunc(two) # aaa
+ File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc
+ f() # call back to caller
+ File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two
+ three(42)
+ File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three
+ raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+RuntimeError: Kaboom! I'm dead: 42
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-default_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt
new file mode 100644
index 0000000000..9e5a474cbd
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stderr.txt
@@ -0,0 +1,13 @@
+Traceback (most recent call last):
+ File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main
+ one()
+ File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one
+ modfunc(two) # aaa
+ File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc
+ f() # call back to caller
+ File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two
+ three(42)
+ File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three
+ raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+RuntimeError: Kaboom! I'm dead: 42
+
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_color_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt
new file mode 100644
index 0000000000..b0b299ebe6
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stderr.txt
@@ -0,0 +1,41 @@
+---------------------------------------------------------------------------
+RuntimeError Traceback (most recent call last)
+library/python/runtime_py3/test/traceback/crash.py in main()
+ 42 sys.executable = '<traceback test>'
+ 43
+---> 44 one()
+ global one = <function one>
+
+library/python/runtime_py3/test/traceback/crash.py in one()
+ 10
+ 11 def one():
+---> 12 modfunc(two) # aaa
+ global modfunc = <function modfunc>
+ global two = <function two>
+ 13
+ 14
+
+library/python/runtime_py3/test/traceback/mod/__init__.py in modfunc(f=<function two>)
+ 1 def modfunc(f):
+ 2 # lalala
+----> 3 f() # call back to caller
+ f = <function two>
+
+library/python/runtime_py3/test/traceback/crash.py in two()
+ 14
+ 15 def two():
+---> 16 three(42)
+ global three = <function three>
+ 17
+ 18
+
+library/python/runtime_py3/test/traceback/crash.py in three(x=42)
+ 18
+ 19 def three(x):
+---> 20 raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+ global RuntimeError.format = undefined
+ x = 42
+ 21
+ 22
+
+RuntimeError: Kaboom! I'm dead: 42
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_custom-ultratb_verbose_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt
new file mode 100644
index 0000000000..5eb7da170a
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stderr.txt
@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+ File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main
+ one()
+ File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one
+ modfunc(two) # aaa
+ File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc
+ f() # call back to caller
+ File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two
+ three(42)
+ File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three
+ raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+RuntimeError: Kaboom! I'm dead: 42
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-default_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt
new file mode 100644
index 0000000000..9e5a474cbd
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stderr.txt
@@ -0,0 +1,13 @@
+Traceback (most recent call last):
+ File "library/python/runtime_py3/test/traceback/crash.py", line 44, in main
+ one()
+ File "library/python/runtime_py3/test/traceback/crash.py", line 12, in one
+ modfunc(two) # aaa
+ File "library/python/runtime_py3/test/traceback/mod/__init__.py", line 3, in modfunc
+ f() # call back to caller
+ File "library/python/runtime_py3/test/traceback/crash.py", line 16, in two
+ three(42)
+ File "library/python/runtime_py3/test/traceback/crash.py", line 20, in three
+ raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+RuntimeError: Kaboom! I'm dead: 42
+
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_color_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt
new file mode 100644
index 0000000000..b0b299ebe6
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stderr.txt
@@ -0,0 +1,41 @@
+---------------------------------------------------------------------------
+RuntimeError Traceback (most recent call last)
+library/python/runtime_py3/test/traceback/crash.py in main()
+ 42 sys.executable = '<traceback test>'
+ 43
+---> 44 one()
+ global one = <function one>
+
+library/python/runtime_py3/test/traceback/crash.py in one()
+ 10
+ 11 def one():
+---> 12 modfunc(two) # aaa
+ global modfunc = <function modfunc>
+ global two = <function two>
+ 13
+ 14
+
+library/python/runtime_py3/test/traceback/mod/__init__.py in modfunc(f=<function two>)
+ 1 def modfunc(f):
+ 2 # lalala
+----> 3 f() # call back to caller
+ f = <function two>
+
+library/python/runtime_py3/test/traceback/crash.py in two()
+ 14
+ 15 def two():
+---> 16 three(42)
+ global three = <function three>
+ 17
+ 18
+
+library/python/runtime_py3/test/traceback/crash.py in three(x=42)
+ 18
+ 19 def three(x):
+---> 20 raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+ global RuntimeError.format = undefined
+ x = 42
+ 21
+ 22
+
+RuntimeError: Kaboom! I'm dead: 42
diff --git a/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt
new file mode 100644
index 0000000000..2c9793eb14
--- /dev/null
+++ b/library/python/runtime_py3/test/canondata/test_traceback.test_traceback_main-ultratb_verbose_/stdout.txt
@@ -0,0 +1,2 @@
+__name__ = library.python.runtime_py3.test.traceback.crash
+__file__ = library/python/runtime_py3/test/traceback/crash.py
diff --git a/library/python/runtime_py3/test/resources/__init__.py b/library/python/runtime_py3/test/resources/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/library/python/runtime_py3/test/resources/__init__.py
diff --git a/library/python/runtime_py3/test/resources/foo.txt b/library/python/runtime_py3/test/resources/foo.txt
new file mode 100644
index 0000000000..ba0e162e1c
--- /dev/null
+++ b/library/python/runtime_py3/test/resources/foo.txt
@@ -0,0 +1 @@
+bar \ No newline at end of file
diff --git a/library/python/runtime_py3/test/resources/submodule/__init__.py b/library/python/runtime_py3/test/resources/submodule/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/library/python/runtime_py3/test/resources/submodule/__init__.py
diff --git a/library/python/runtime_py3/test/resources/submodule/bar.txt b/library/python/runtime_py3/test/resources/submodule/bar.txt
new file mode 100644
index 0000000000..1910281566
--- /dev/null
+++ b/library/python/runtime_py3/test/resources/submodule/bar.txt
@@ -0,0 +1 @@
+foo \ No newline at end of file
diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py
new file mode 100644
index 0000000000..ff80d0a0a2
--- /dev/null
+++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py
@@ -0,0 +1,317 @@
+import unittest
+import yaml
+from unittest.mock import patch
+from parameterized import parameterized
+
+import __res as res
+
+
+NAMESPACE_PREFIX = b'py/namespace/'
+TEST_SOURCE_ROOT = '/home/arcadia'
+
+
+class ImporterMocks(object):
+ def __init__(self, mock_fs, mock_resources):
+ self._mock_fs = mock_fs
+ self._mock_resources = mock_resources
+ self._patchers = [
+ patch('__res.iter_keys', wraps=self._iter_keys),
+ patch('__res.__resource.find', wraps=self._resource_find),
+ patch('__res._path_isdir', wraps=self._path_isdir),
+ patch('__res._path_isfile', wraps=self._path_isfile),
+ patch('__res._os.listdir', wraps=self._os_listdir),
+ ]
+ for patcher in self._patchers:
+ patcher.start()
+
+ def stop(self):
+ for patcher in self._patchers:
+ patcher.stop()
+
+ def _iter_keys(self, prefix):
+ assert prefix == NAMESPACE_PREFIX
+ l = len(prefix)
+ for k in self._mock_resources.keys():
+ yield k, k[l:]
+
+ def _resource_find(self, key):
+ return self._mock_resources.get(key)
+
+ def _lookup_mock_fs(self, filename):
+ path = filename.lstrip('/').split('/')
+ curdir = self._mock_fs
+ for item in path:
+ if item in curdir:
+ curdir = curdir[item]
+ else:
+ return None
+ return curdir
+
+ def _path_isfile(self, filename):
+ f = self._lookup_mock_fs(filename)
+ return isinstance(f, str)
+
+ def _path_isdir(self, filename):
+ f = self._lookup_mock_fs(filename)
+ return isinstance(f, dict)
+
+ def _os_listdir(self, dirname):
+ f = self._lookup_mock_fs(dirname)
+ if isinstance(f, dict):
+ return f.keys()
+ else:
+ return []
+
+
+class ArcadiaSourceFinderTestCase(unittest.TestCase):
+ def setUp(self):
+ self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources())
+ self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT)
+
+ def tearDown(self):
+ self.import_mock.stop()
+
+ def _get_mock_fs(self):
+ raise NotImplementedError()
+
+ def _get_mock_resources(self):
+ raise NotImplementedError()
+
+
+class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ""
+ package1:
+ mod2.py: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
+ }
+
+ @parameterized.expand([
+ ('project.lib.mod1', b'project/lib/mod1.py'),
+ ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'),
+ ('project.lib.unknown_module', None),
+ ('project.lib', None), # package
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('project.lib.mod1', False),
+ ('project.lib.package1.mod2', False),
+ ('project', True),
+ ('project.lib', True),
+ ('project.lib.package1', True),
+ ])
+ def test_is_packages(self, module, is_package):
+ assert is_package == self.arcadia_source_finder.is_package(module)
+
+ def test_is_package_for_unknown_module(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2'))
+
+ @parameterized.expand([
+ ('', {
+ ('PFX.project', True),
+ }),
+ ('project.', {
+ ('PFX.lib', True),
+ }),
+ ('project.lib.', {
+ ('PFX.mod1', False),
+ ('PFX.package1', True),
+ }),
+ ('project.lib.package1.', {
+ ('PFX.mod2', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+ # Check iter_modules() don't crash and return correct result after not existing module was requested
+ def test_iter_modules_after_unknown_module_import(self):
+ self.arcadia_source_finder.get_module_path('project.unknown_module')
+ assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', ''))
+
+
+class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ''
+ lib_extension:
+ mod2.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
+ b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.',
+ }
+
+ @parameterized.expand([
+ ('project.lib.mod1', b'project/lib/mod1.py'),
+ ('project.lib.mod2', b'project/lib_extension/mod2.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('project.lib.', {
+ ('PFX.mod1', False),
+ ('PFX.mod2', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+
+class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ ns_lib:
+ mod1.py: ''
+ top_level_lib:
+ mod2.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/ns_lib': b'ns.',
+ b'py/namespace/unique_prefix2/project/top_level_lib': b'.',
+ }
+
+ @parameterized.expand([
+ ('ns.mod1', b'project/ns_lib/mod1.py'),
+ ('mod2', b'project/top_level_lib/mod2.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('ns', True),
+ ('ns.mod1', False),
+ ('mod2', False),
+ ])
+ def test_is_packages(self, module, is_package):
+ assert is_package == self.arcadia_source_finder.is_package(module)
+
+ @parameterized.expand([
+ 'project',
+ 'project.ns_lib',
+ 'project.top_level_lib',
+ ])
+ def test_is_package_for_unknown_modules(self, module):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module))
+
+ @parameterized.expand([
+ ('', {
+ ('PFX.ns', True),
+ ('PFX.mod2', False),
+ }),
+ ('ns.', {
+ ('PFX.mod1', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+
+class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase):
+ ''' Packages and modules from tests should not be part of pylib namespace '''
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ contrib:
+ python:
+ pylib:
+ mod1.py: ""
+ tests:
+ conftest.py: ""
+ ya.make: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.',
+ }
+
+ def test_get_module_path_for_lib(self):
+ assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1')
+
+ def test_get_module_for_tests(self):
+ assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None
+
+ def test_is_package_for_tests(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests'))
+
+
+class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase):
+ ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) '''
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ contrib:
+ python:
+ pylint:
+ ya.make: ""
+ pylint:
+ __init__.py: ""
+ patcher:
+ patch.py: ""
+ ya.make: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/contrib/python/pylint': b'.',
+ b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.',
+ }
+
+ @parameterized.expand([
+ ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'),
+ ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+
+class TestEmptyResources(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {}
+
+ def test_get_module_path(self):
+ assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None
+
+ def test_is_package(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project'))
+
+ def test_iter_modules(self):
+ assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.'))
diff --git a/library/python/runtime_py3/test/test_metadata.py b/library/python/runtime_py3/test/test_metadata.py
new file mode 100644
index 0000000000..686c176468
--- /dev/null
+++ b/library/python/runtime_py3/test/test_metadata.py
@@ -0,0 +1,44 @@
+import importlib.metadata as im
+
+import pytest
+
+
+@pytest.mark.parametrize("name", ("foo-bar", "foo_bar", "Foo-Bar"))
+def test_distribution(name):
+ assert im.distribution(name) is not None
+
+
+def test_unknown_package():
+ with pytest.raises(im.PackageNotFoundError):
+ im.distribution("bar")
+
+
+def test_version():
+ assert im.version("foo-bar") == "1.2.3"
+
+
+def test_metadata():
+ assert im.metadata("foo-bar") is not None
+
+
+def test_files():
+ files = im.files("foo-bar")
+ assert len(files) == 1
+ assert files[0].name == "foo_bar.py"
+ assert files[0].size == 20
+
+
+def test_requires():
+ assert im.requires("foo-bar") == ["Werkzeug (>=0.15)", "Jinja2 (>=2.10.1)"]
+
+
+def test_entry_points():
+ entry_points = im.entry_points()
+ assert "console_scripts" in entry_points
+
+ flg_found = False
+ for entry_point in entry_points["console_scripts"]:
+ if entry_point.name == "foo_cli" and entry_point.value == "foo_bar:cli":
+ flg_found = True
+
+ assert flg_found
diff --git a/library/python/runtime_py3/test/test_resources.py b/library/python/runtime_py3/test/test_resources.py
new file mode 100644
index 0000000000..a269329f42
--- /dev/null
+++ b/library/python/runtime_py3/test/test_resources.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+import importlib.resources as ir
+
+import pytest
+
+
+@pytest.mark.parametrize("package, resource", (
+ ("resources", "foo.txt"),
+ ("resources.submodule", "bar.txt")
+))
+def test_is_resource_good_path(package, resource):
+ assert ir.is_resource(package, resource)
+
+
+@pytest.mark.parametrize("package, resource", (
+ ("resources", "111.txt"),
+ ("resources.submodule", "222.txt")
+))
+def test_is_resource_missing(package, resource):
+ assert not ir.is_resource(package, resource)
+
+
+def test_is_resource_subresource_directory():
+ # Directories are not resources.
+ assert not ir.is_resource("resources", "submodule")
+
+
+@pytest.mark.parametrize("package, resource, expected", (
+ ("resources", "foo.txt", b"bar"),
+ ("resources.submodule", "bar.txt", b"foo")
+))
+def test_read_binary_good_path(package, resource, expected):
+ assert ir.read_binary(package, resource) == expected
+
+
+def test_read_binary_missing():
+ with pytest.raises(FileNotFoundError):
+ ir.read_binary("resources", "111.txt")
+
+
+@pytest.mark.parametrize("package, resource, expected", (
+ ("resources", "foo.txt", "bar"),
+ ("resources.submodule", "bar.txt", "foo")
+))
+def test_read_text_good_path(package, resource, expected):
+ assert ir.read_text(package, resource) == expected
+
+
+def test_read_text_missing():
+ with pytest.raises(FileNotFoundError):
+ ir.read_text("resources", "111.txt")
+
+
+@pytest.mark.parametrize("package, expected", (
+ ("resources", ["submodule", "foo.txt"]),
+ ("resources.submodule", ["bar.txt"])
+))
+def test_contents_good_path(package, expected):
+ assert sorted(ir.contents(package)) == sorted(expected)
diff --git a/library/python/runtime_py3/test/test_traceback.py b/library/python/runtime_py3/test/test_traceback.py
new file mode 100644
index 0000000000..82087ce98a
--- /dev/null
+++ b/library/python/runtime_py3/test/test_traceback.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, absolute_import, division
+
+import os
+import re
+
+import pytest
+
+import yatest.common as yc
+
+
+def clean_traceback(traceback):
+ traceback = re.sub(br'\033\[(\d|;)+?m', b'', traceback) # strip ANSI codes
+ traceback = re.sub(br' at 0x[0-9a-fA-F]+', b'', traceback) # remove object ids
+ return traceback
+
+
+@pytest.mark.parametrize('mode', [
+ 'default',
+ 'ultratb_color',
+ 'ultratb_verbose',
+])
+@pytest.mark.parametrize('entry_point', [
+ 'main',
+ 'custom',
+])
+def test_traceback(mode, entry_point):
+ tb_tool = yc.build_path('library/python/runtime_py3/test/traceback/traceback')
+ stdout_path = yc.test_output_path('stdout_raw.txt')
+ stderr_path = yc.test_output_path('stderr_raw.txt')
+ filtered_stdout_path = yc.test_output_path('stdout.txt')
+ filtered_stderr_path = yc.test_output_path('stderr.txt')
+
+ env = os.environ.copy()
+ env.pop('PYTHONPATH', None) # Do not let program peek into its sources on filesystem
+ if entry_point == 'custom':
+ env['Y_PYTHON_ENTRY_POINT'] = 'library.python.runtime_py3.test.traceback.crash:main'
+
+ proc = yc.execute(
+ command=[tb_tool, mode],
+ env=env,
+ stdout=stdout_path,
+ stderr=stderr_path,
+ check_exit_code=False,
+ )
+
+ with open(filtered_stdout_path, 'wb') as f:
+ f.write(clean_traceback(proc.std_out))
+
+ with open(filtered_stderr_path, 'wb') as f:
+ f.write(clean_traceback(proc.std_err))
+
+ return {
+ 'stdout': yc.canonical_file(
+ filtered_stdout_path,
+ local=True,
+ ),
+ 'stderr': yc.canonical_file(
+ filtered_stderr_path,
+ local=True,
+ ),
+ }
diff --git a/library/python/runtime_py3/test/traceback/__main__.py b/library/python/runtime_py3/test/traceback/__main__.py
new file mode 100644
index 0000000000..364db169f0
--- /dev/null
+++ b/library/python/runtime_py3/test/traceback/__main__.py
@@ -0,0 +1,4 @@
+from library.python.runtime_py3.test.traceback.crash import main
+
+if __name__ == "__main__":
+ main()
diff --git a/library/python/runtime_py3/test/traceback/crash.py b/library/python/runtime_py3/test/traceback/crash.py
new file mode 100644
index 0000000000..b5e36b3dd4
--- /dev/null
+++ b/library/python/runtime_py3/test/traceback/crash.py
@@ -0,0 +1,44 @@
+import argparse
+import cgitb
+import sys
+import time
+
+from IPython.core import ultratb
+
+from .mod import modfunc
+
+
+def one():
+ modfunc(two) # aaa
+
+
+def two():
+ three(42)
+
+
+def three(x):
+ raise RuntimeError('Kaboom! I\'m dead: {}'.format(x))
+
+
+def main():
+ hooks = {
+ 'default': lambda: sys.excepthook,
+ 'cgitb': lambda: cgitb.Hook(format='text'),
+ 'ultratb_color': lambda: ultratb.ColorTB(ostream=sys.stderr),
+ 'ultratb_verbose': lambda: ultratb.VerboseTB(ostream=sys.stderr),
+ }
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('hook', choices=sorted(hooks), default='default')
+
+ args = parser.parse_args()
+
+ sys.excepthook = hooks[args.hook]()
+
+ print('__name__ =', __name__)
+ print('__file__ =', __file__)
+
+ time.time = lambda: 1531996624.0 # Freeze time
+ sys.executable = '<traceback test>'
+
+ one()
diff --git a/library/python/runtime_py3/test/traceback/mod/__init__.py b/library/python/runtime_py3/test/traceback/mod/__init__.py
new file mode 100644
index 0000000000..f00843d786
--- /dev/null
+++ b/library/python/runtime_py3/test/traceback/mod/__init__.py
@@ -0,0 +1,3 @@
+def modfunc(f):
+ # lalala
+ f() # call back to caller
diff --git a/library/python/runtime_py3/test/traceback/ya.make b/library/python/runtime_py3/test/traceback/ya.make
new file mode 100644
index 0000000000..b61fe9550b
--- /dev/null
+++ b/library/python/runtime_py3/test/traceback/ya.make
@@ -0,0 +1,19 @@
+PY3_PROGRAM()
+
+OWNER(
+ abodrov
+ borman
+)
+
+PEERDIR(
+ contrib/python/ipython
+)
+
+PY_SRCS(
+ MAIN
+ __main__.py=main
+ crash.py
+ mod/__init__.py
+)
+
+END()
diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make
new file mode 100644
index 0000000000..4ec3db74f5
--- /dev/null
+++ b/library/python/runtime_py3/test/ya.make
@@ -0,0 +1,40 @@
+PY3TEST()
+
+OWNER(
+ abodrov
+ borman
+)
+
+DEPENDS(library/python/runtime_py3/test/traceback)
+
+PEERDIR(
+ contrib/python/parameterized
+ contrib/python/PyYAML
+)
+
+PY_SRCS(
+ TOP_LEVEL
+ resources/__init__.py
+ resources/submodule/__init__.py
+)
+
+TEST_SRCS(
+ test_metadata.py
+ test_resources.py
+ test_traceback.py
+ test_arcadia_source_finder.py
+)
+
+RESOURCE_FILES(
+ PREFIX library/python/runtime_py3/test/
+ .dist-info/METADATA
+ .dist-info/RECORD
+ .dist-info/entry_points.txt
+ .dist-info/top_level.txt
+ resources/foo.txt
+ resources/submodule/bar.txt
+)
+
+END()
+
+RECURSE_FOR_TESTS(traceback)
diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make
new file mode 100644
index 0000000000..fa5c11341a
--- /dev/null
+++ b/library/python/runtime_py3/ya.make
@@ -0,0 +1,49 @@
+PY3_LIBRARY()
+
+OWNER(
+ borman
+ orivej
+ pg
+)
+
+NO_WSHADOW()
+
+PEERDIR(
+ contrib/tools/python3/src
+ contrib/tools/python3/lib/py
+ library/cpp/resource
+)
+
+CFLAGS(-DCYTHON_REGISTER_ABCS=0)
+
+NO_PYTHON_INCLUDES()
+
+ENABLE(PYBUILD_NO_PYC)
+
+PY_SRCS(
+ entry_points.py
+ TOP_LEVEL
+
+ CYTHON_DIRECTIVE
+ language_level=3
+
+ __res.pyx
+ sitecustomize.pyx
+)
+
+IF (CYTHON_COVERAGE)
+ # Let covarage support add all needed files to resources
+ELSE()
+ RESOURCE_FILES(
+ PREFIX ${MODDIR}/
+ __res.pyx
+ importer.pxi
+ sitecustomize.pyx
+ )
+ENDIF()
+
+END()
+
+RECURSE_FOR_TESTS(
+ test
+)
diff --git a/library/python/strings/__init__.py b/library/python/strings/__init__.py
new file mode 100644
index 0000000000..bd6bf6e7ce
--- /dev/null
+++ b/library/python/strings/__init__.py
@@ -0,0 +1,17 @@
+# flake8 noqa: F401
+
+from .strings import (
+ DEFAULT_ENCODING,
+ ENCODING_ERRORS_POLICY,
+ encode,
+ fs_encoding,
+ get_stream_encoding,
+ guess_default_encoding,
+ left_strip,
+ locale_encoding,
+ stringize_deep,
+ to_basestring,
+ to_str,
+ to_unicode,
+ unicodize_deep,
+)
diff --git a/library/python/strings/strings.py b/library/python/strings/strings.py
new file mode 100644
index 0000000000..5bfddfe78a
--- /dev/null
+++ b/library/python/strings/strings.py
@@ -0,0 +1,129 @@
+import locale
+import logging
+import six
+import sys
+import codecs
+
+import library.python.func
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_ENCODING = 'utf-8'
+ENCODING_ERRORS_POLICY = 'replace'
+
+
+def left_strip(el, prefix):
+ """
+ Strips prefix at the left of el
+ """
+ if el.startswith(prefix):
+ return el[len(prefix):]
+ return el
+
+
+# Explicit to-text conversion
+# Chooses between str/unicode, i.e. six.binary_type/six.text_type
+def to_basestring(value):
+ if isinstance(value, (six.binary_type, six.text_type)):
+ return value
+ try:
+ if six.PY2:
+ return unicode(value)
+ else:
+ return str(value)
+ except UnicodeDecodeError:
+ try:
+ return str(value)
+ except UnicodeEncodeError:
+ return repr(value)
+to_text = to_basestring
+
+
+def to_unicode(value, from_enc=DEFAULT_ENCODING):
+ if isinstance(value, six.text_type):
+ return value
+ if isinstance(value, six.binary_type):
+ if six.PY2:
+ return unicode(value, from_enc, ENCODING_ERRORS_POLICY)
+ else:
+ return value.decode(from_enc, errors=ENCODING_ERRORS_POLICY)
+ return six.text_type(value)
+
+
+# Optional from_enc enables transcoding
+def to_str(value, to_enc=DEFAULT_ENCODING, from_enc=None):
+ if isinstance(value, six.binary_type):
+ if from_enc is None or to_enc == from_enc:
+ # Unknown input encoding or input and output encoding are the same
+ return value
+ value = to_unicode(value, from_enc=from_enc)
+ if isinstance(value, six.text_type):
+ return value.encode(to_enc, ENCODING_ERRORS_POLICY)
+ return six.binary_type(value)
+
+
+def _convert_deep(x, enc, convert, relaxed=True):
+ if x is None:
+ return None
+ if isinstance(x, (six.text_type, six.binary_type)):
+ return convert(x, enc)
+ if isinstance(x, dict):
+ return {convert(k, enc): _convert_deep(v, enc, convert, relaxed) for k, v in six.iteritems(x)}
+ if isinstance(x, list):
+ return [_convert_deep(e, enc, convert, relaxed) for e in x]
+ if isinstance(x, tuple):
+ return tuple([_convert_deep(e, enc, convert, relaxed) for e in x])
+
+ if relaxed:
+ return x
+ raise TypeError('unsupported type')
+
+
+def unicodize_deep(x, enc=DEFAULT_ENCODING, relaxed=True):
+ return _convert_deep(x, enc, to_unicode, relaxed)
+
+
+def stringize_deep(x, enc=DEFAULT_ENCODING, relaxed=True):
+ return _convert_deep(x, enc, to_str, relaxed)
+
+
+@library.python.func.memoize()
+def locale_encoding():
+ try:
+ loc = locale.getdefaultlocale()[1]
+ if loc:
+ codecs.lookup(loc)
+ return loc
+ except LookupError as e:
+ logger.debug('Cannot get system locale: %s', e)
+ return None
+ except ValueError as e:
+ logger.warn('Cannot get system locale: %s', e)
+ return None
+
+
+def fs_encoding():
+ return sys.getfilesystemencoding()
+
+
+def guess_default_encoding():
+ enc = locale_encoding()
+ return enc if enc else DEFAULT_ENCODING
+
+
+@library.python.func.memoize()
+def get_stream_encoding(stream):
+ if stream.encoding:
+ try:
+ codecs.lookup(stream.encoding)
+ return stream.encoding
+ except LookupError:
+ pass
+ return DEFAULT_ENCODING
+
+
+def encode(value, encoding=DEFAULT_ENCODING):
+ if isinstance(value, six.binary_type):
+ value = value.decode(encoding, errors='ignore')
+ return value.encode(encoding)
diff --git a/library/python/strings/ut/test_strings.py b/library/python/strings/ut/test_strings.py
new file mode 100644
index 0000000000..dd0c694ee1
--- /dev/null
+++ b/library/python/strings/ut/test_strings.py
@@ -0,0 +1,205 @@
+# coding=utf-8
+
+import pytest
+import six
+
+import library.python.strings
+
+
+class Convertible(object):
+ text = u'текст'
+ text_utf8 = text.encode('utf-8')
+
+ def __unicode__(self):
+ return self.text
+
+ def __str__(self):
+ return self.text_utf8
+
+
+class ConvertibleToUnicodeOnly(Convertible):
+ def __str__(self):
+ return self.text.encode('ascii')
+
+
+class ConvertibleToStrOnly(Convertible):
+ def __unicode__(self):
+ return self.text_utf8.decode('ascii')
+
+
+class NonConvertible(ConvertibleToUnicodeOnly, ConvertibleToStrOnly):
+ pass
+
+
+def test_to_basestring():
+ assert library.python.strings.to_basestring('str') == 'str'
+ assert library.python.strings.to_basestring(u'юникод') == u'юникод'
+ if six.PY2: # __str__ should return str not bytes in Python3
+ assert library.python.strings.to_basestring(Convertible()) == Convertible.text
+ assert library.python.strings.to_basestring(ConvertibleToUnicodeOnly()) == Convertible.text
+ assert library.python.strings.to_basestring(ConvertibleToStrOnly()) == Convertible.text_utf8
+ assert library.python.strings.to_basestring(NonConvertible())
+
+
+def test_to_unicode():
+ assert library.python.strings.to_unicode(u'юникод') == u'юникод'
+ assert library.python.strings.to_unicode('str') == u'str'
+ assert library.python.strings.to_unicode(u'строка'.encode('utf-8')) == u'строка'
+ assert library.python.strings.to_unicode(u'строка'.encode('cp1251'), 'cp1251') == u'строка'
+ if six.PY2: # __str__ should return str not bytes in Python3
+ assert library.python.strings.to_unicode(Convertible()) == Convertible.text
+ assert library.python.strings.to_unicode(ConvertibleToUnicodeOnly()) == Convertible.text
+ with pytest.raises(UnicodeDecodeError):
+ library.python.strings.to_unicode(ConvertibleToStrOnly())
+ with pytest.raises(UnicodeDecodeError):
+ library.python.strings.to_unicode(NonConvertible())
+
+
+def test_to_unicode_errors_replace():
+ assert library.python.strings.to_unicode(u'abcабв'.encode('utf-8'), 'ascii')
+ assert library.python.strings.to_unicode(u'абв'.encode('utf-8'), 'ascii')
+
+
+def test_to_str():
+ assert library.python.strings.to_str('str') == 'str' if six.PY2 else b'str'
+ assert library.python.strings.to_str(u'unicode') == 'unicode' if six.PY2 else b'unicode'
+ assert library.python.strings.to_str(u'юникод') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод', 'cp1251') == u'юникод'.encode('cp1251')
+ if six.PY2:
+ assert library.python.strings.to_str(Convertible()) == Convertible.text_utf8
+ with pytest.raises(UnicodeEncodeError):
+ library.python.strings.to_str(ConvertibleToUnicodeOnly())
+ assert library.python.strings.to_str(ConvertibleToStrOnly()) == Convertible.text_utf8
+ with pytest.raises(UnicodeEncodeError):
+ library.python.strings.to_str(NonConvertible())
+
+
+def test_to_str_errors_replace():
+ assert library.python.strings.to_str(u'abcабв', 'ascii')
+ assert library.python.strings.to_str(u'абв', 'ascii')
+
+
+def test_to_str_transcode():
+ assert library.python.strings.to_str('str', from_enc='ascii') == 'str' if six.PY2 else b'str'
+ assert library.python.strings.to_str('str', from_enc='utf-8') == 'str' if six.PY2 else b'str'
+
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), from_enc='utf-8') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='utf-8', from_enc='utf-8') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='cp1251', from_enc='utf-8') == u'юникод'.encode('cp1251')
+
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), from_enc='cp1251') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='cp1251', from_enc='cp1251') == u'юникод'.encode('cp1251')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='utf-8', from_enc='cp1251') == u'юникод'.encode('utf-8')
+
+ assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), from_enc='koi8-r') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='koi8-r', from_enc='koi8-r') == u'юникод'.encode('koi8-r')
+ assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='cp1251', from_enc='koi8-r') == u'юникод'.encode('cp1251')
+
+
+def test_to_str_transcode_wrong():
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), from_enc='cp1251')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), from_enc='utf-8')
+
+
+def test_to_str_transcode_disabled():
+ # No transcoding enabled, set from_enc to enable
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='utf-8') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('utf-8'), to_enc='cp1251') == u'юникод'.encode('utf-8')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='utf-8') == u'юникод'.encode('cp1251')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='cp1251') == u'юникод'.encode('cp1251')
+ assert library.python.strings.to_str(u'юникод'.encode('cp1251'), to_enc='koi8-r') == u'юникод'.encode('cp1251')
+ assert library.python.strings.to_str(u'юникод'.encode('koi8-r'), to_enc='cp1251') == u'юникод'.encode('koi8-r')
+
+
+def test_stringize_deep():
+ assert library.python.strings.stringize_deep({
+ 'key 1': 'value 1',
+ u'ключ 2': u'значение 2',
+ 'list': [u'ключ 2', 'key 1', (u'к', 2)]
+ }) == {
+ 'key 1' if six.PY2 else b'key 1': 'value 1' if six.PY2 else b'value 1',
+ u'ключ 2'.encode('utf-8'): u'значение 2'.encode('utf-8'),
+ 'list' if six.PY2 else b'list': [u'ключ 2'.encode('utf-8'), 'key 1' if six.PY2 else b'key 1', (u'к'.encode('utf-8'), 2)]
+ }
+
+
+def test_stringize_deep_doesnt_transcode():
+ assert library.python.strings.stringize_deep({
+ u'ключ 1'.encode('utf-8'): u'значение 1'.encode('utf-8'),
+ u'ключ 2'.encode('cp1251'): u'значение 2'.encode('cp1251'),
+ }) == {
+ u'ключ 1'.encode('utf-8'): u'значение 1'.encode('utf-8'),
+ u'ключ 2'.encode('cp1251'): u'значение 2'.encode('cp1251'),
+ }
+
+
+def test_stringize_deep_nested():
+ assert library.python.strings.stringize_deep({
+ 'key 1': 'value 1',
+ u'ключ 2': {
+ 'subkey 1': 'value 1',
+ u'подключ 2': u'value 2',
+ },
+ }) == {
+ 'key 1' if six.PY2 else b'key 1': 'value 1' if six.PY2 else b'value 1',
+ u'ключ 2'.encode('utf-8'): {
+ 'subkey 1' if six.PY2 else b'subkey 1': 'value 1' if six.PY2 else b'value 1',
+ u'подключ 2'.encode('utf-8'): u'value 2'.encode('utf-8'),
+ },
+ }
+
+
+def test_stringize_deep_plain():
+ assert library.python.strings.stringize_deep('str') == 'str' if six.PY2 else b'str'
+ assert library.python.strings.stringize_deep(u'юникод') == u'юникод'.encode('utf-8')
+ assert library.python.strings.stringize_deep(u'юникод'.encode('utf-8')) == u'юникод'.encode('utf-8')
+
+
+def test_stringize_deep_nonstr():
+ with pytest.raises(TypeError):
+ library.python.strings.stringize_deep(Convertible(), relaxed=False)
+ x = Convertible()
+ assert x == library.python.strings.stringize_deep(x)
+
+
+def test_unicodize_deep():
+ assert library.python.strings.unicodize_deep({
+ 'key 1': 'value 1',
+ u'ключ 2': u'значение 2',
+ u'ключ 3'.encode('utf-8'): u'значение 3'.encode('utf-8'),
+ }) == {
+ u'key 1': u'value 1',
+ u'ключ 2': u'значение 2',
+ u'ключ 3': u'значение 3',
+ }
+
+
+def test_unicodize_deep_nested():
+ assert library.python.strings.unicodize_deep({
+ 'key 1': 'value 1',
+ u'ключ 2': {
+ 'subkey 1': 'value 1',
+ u'подключ 2': u'значение 2',
+ u'подключ 3'.encode('utf-8'): u'значение 3'.encode('utf-8'),
+ },
+ }) == {
+ u'key 1': u'value 1',
+ u'ключ 2': {
+ u'subkey 1': u'value 1',
+ u'подключ 2': u'значение 2',
+ u'подключ 3': u'значение 3',
+ },
+ }
+
+
+def test_unicodize_deep_plain():
+ assert library.python.strings.unicodize_deep('str') == u'str'
+ assert library.python.strings.unicodize_deep(u'юникод') == u'юникод'
+ assert library.python.strings.unicodize_deep(u'юникод'.encode('utf-8')) == u'юникод'
+
+
+def test_unicodize_deep_nonstr():
+ with pytest.raises(TypeError):
+ library.python.strings.unicodize_deep(Convertible(), relaxed=False)
+ x = Convertible()
+ assert x == library.python.strings.stringize_deep(x)
diff --git a/library/python/strings/ut/ya.make b/library/python/strings/ut/ya.make
new file mode 100644
index 0000000000..dfacb226c7
--- /dev/null
+++ b/library/python/strings/ut/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY23_TEST()
+
+TEST_SRCS(test_strings.py)
+
+PEERDIR(
+ library/python/strings
+)
+
+END()
diff --git a/library/python/strings/ya.make b/library/python/strings/ya.make
new file mode 100644
index 0000000000..7e0b033717
--- /dev/null
+++ b/library/python/strings/ya.make
@@ -0,0 +1,16 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+PY_SRCS(
+ __init__.py
+ CYTHONIZE_PY
+ strings.py
+)
+
+PEERDIR(
+ library/python/func
+ contrib/python/six
+)
+
+END()
diff --git a/library/python/svn_version/__init__.py b/library/python/svn_version/__init__.py
new file mode 100644
index 0000000000..7ef7d6dac7
--- /dev/null
+++ b/library/python/svn_version/__init__.py
@@ -0,0 +1 @@
+from .__svn_version import svn_version, commit_id, svn_revision, svn_last_revision, hash, svn_branch, svn_tag, patch_number # noqa
diff --git a/library/python/svn_version/__svn_version.pyx b/library/python/svn_version/__svn_version.pyx
new file mode 100644
index 0000000000..d66bc09d24
--- /dev/null
+++ b/library/python/svn_version/__svn_version.pyx
@@ -0,0 +1,35 @@
+import future.utils as fu
+
+cdef extern from "library/cpp/svnversion/svnversion.h":
+ cdef const char* GetProgramSvnVersion() except +;
+ cdef int GetProgramSvnRevision() except +;
+ cdef int GetArcadiaLastChangeNum() except +;
+ cdef const char* GetProgramCommitId() except +;
+ cdef const char* GetProgramHash() except +;
+ cdef const char* GetBranch() except +;
+ cdef const char* GetTag() except +;
+ cdef int GetArcadiaPatchNumber() except +;
+
+def svn_version():
+ return fu.bytes_to_native_str(GetProgramSvnVersion())
+
+def svn_revision():
+ return GetProgramSvnRevision()
+
+def svn_last_revision():
+ return GetArcadiaLastChangeNum()
+
+def commit_id():
+ return fu.bytes_to_native_str(GetProgramCommitId())
+
+def hash():
+ return fu.bytes_to_native_str(GetProgramHash())
+
+def svn_branch():
+ return fu.bytes_to_native_str(GetBranch())
+
+def svn_tag():
+ return fu.bytes_to_native_str(GetTag())
+
+def patch_number():
+ return GetArcadiaPatchNumber()
diff --git a/library/python/svn_version/ut/lib/test_simple.py b/library/python/svn_version/ut/lib/test_simple.py
new file mode 100644
index 0000000000..2c27af6c68
--- /dev/null
+++ b/library/python/svn_version/ut/lib/test_simple.py
@@ -0,0 +1,16 @@
+import library.python.svn_version as sv
+
+
+def test_simple():
+ assert sv.svn_version()
+ assert isinstance(sv.svn_version(), str)
+ assert sv.svn_revision()
+ assert isinstance(sv.svn_revision(), int)
+ assert sv.svn_last_revision()
+ assert isinstance(sv.svn_last_revision(), int)
+ assert sv.svn_last_revision() > 0
+ assert sv.commit_id()
+ assert isinstance(sv.commit_id(), str)
+ assert len(sv.commit_id()) > 0
+ assert isinstance(sv.hash(), str)
+ assert isinstance(sv.patch_number(), int)
diff --git a/library/python/svn_version/ut/lib/ya.make b/library/python/svn_version/ut/lib/ya.make
new file mode 100644
index 0000000000..fb50caf125
--- /dev/null
+++ b/library/python/svn_version/ut/lib/ya.make
@@ -0,0 +1,13 @@
+PY23_LIBRARY()
+
+OWNER(pg)
+
+PEERDIR(
+ library/python/svn_version
+)
+
+TEST_SRCS(
+ test_simple.py
+)
+
+END()
diff --git a/library/python/svn_version/ut/py2/ya.make b/library/python/svn_version/ut/py2/ya.make
new file mode 100644
index 0000000000..c860e16536
--- /dev/null
+++ b/library/python/svn_version/ut/py2/ya.make
@@ -0,0 +1,9 @@
+PY2TEST()
+
+OWNER(pg)
+
+PEERDIR(
+ library/python/svn_version/ut/lib
+)
+
+END()
diff --git a/library/python/svn_version/ut/py3/ya.make b/library/python/svn_version/ut/py3/ya.make
new file mode 100644
index 0000000000..4231565142
--- /dev/null
+++ b/library/python/svn_version/ut/py3/ya.make
@@ -0,0 +1,9 @@
+PY3TEST()
+
+OWNER(pg)
+
+PEERDIR(
+ library/python/svn_version/ut/lib
+)
+
+END()
diff --git a/library/python/svn_version/ut/ya.make b/library/python/svn_version/ut/ya.make
new file mode 100644
index 0000000000..1363b76581
--- /dev/null
+++ b/library/python/svn_version/ut/ya.make
@@ -0,0 +1,5 @@
+RECURSE(
+ lib
+ py2
+ py3
+)
diff --git a/library/python/svn_version/ya.make b/library/python/svn_version/ya.make
new file mode 100644
index 0000000000..747a663f00
--- /dev/null
+++ b/library/python/svn_version/ya.make
@@ -0,0 +1,15 @@
+PY23_LIBRARY()
+
+OWNER(pg)
+
+PEERDIR(
+ library/cpp/svnversion
+ contrib/python/future
+)
+
+PY_SRCS(
+ __init__.py
+ __svn_version.pyx
+)
+
+END()
diff --git a/library/python/symbols/libc/syms.cpp b/library/python/symbols/libc/syms.cpp
new file mode 100644
index 0000000000..6c04a7ef6e
--- /dev/null
+++ b/library/python/symbols/libc/syms.cpp
@@ -0,0 +1,157 @@
+#include <util/system/platform.h>
+
+#include <library/python/symbols/registry/syms.h>
+
+#if !defined(_MSC_VER)
+#if __has_include(<aio.h>)
+#include <aio.h>
+#endif
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <ifaddrs.h>
+#include <netdb.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <sched.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+#include <sys/ipc.h>
+#include <dlfcn.h>
+
+#if defined(_linux_)
+#include <sys/prctl.h>
+#include <sys/ptrace.h>
+#include <sys/sendfile.h>
+#else
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+#endif
+
+#if defined(_darwin_)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <mach/mach_error.h> // Y_IGNORE
+#include <mach/mach_time.h> // Y_IGNORE
+#endif
+
+#if defined(_linux_)
+#include <sys/inotify.h>
+#include <sys/mman.h>
+#endif
+
+namespace {
+ static inline void* ErrnoLocation() {
+ return &errno;
+ }
+
+ static int ClockGetres(clockid_t clk_id, struct timespec* res) {
+#if defined(_darwin_)
+ static auto func = (decltype(&ClockGetres))dlsym(RTLD_SELF, "_clock_getres");
+
+ if (func) {
+ return func(clk_id, res);
+ }
+
+ // https://opensource.apple.com/source/Libc/Libc-1158.1.2/gen/clock_gettime.c.auto.html
+
+ switch (clk_id){
+ case CLOCK_REALTIME:
+ case CLOCK_MONOTONIC:
+ case CLOCK_PROCESS_CPUTIME_ID:
+ res->tv_nsec = NSEC_PER_USEC;
+ res->tv_sec = 0;
+
+ return 0;
+
+ case CLOCK_MONOTONIC_RAW:
+ case CLOCK_MONOTONIC_RAW_APPROX:
+ case CLOCK_UPTIME_RAW:
+ case CLOCK_UPTIME_RAW_APPROX:
+ case CLOCK_THREAD_CPUTIME_ID: {
+ mach_timebase_info_data_t tb_info;
+
+ if (mach_timebase_info(&tb_info)) {
+ return -1;
+ }
+
+ res->tv_nsec = tb_info.numer / tb_info.denom + (tb_info.numer % tb_info.denom != 0);
+ res->tv_sec = 0;
+
+ return 0;
+ }
+
+ default:
+ errno = EINVAL;
+ return -1;
+ }
+#else
+ return clock_getres(clk_id, res);
+#endif
+ }
+}
+
+BEGIN_SYMS("c")
+
+SYM(calloc)
+SYM(clock_gettime)
+SYM_2("clock_getres", ClockGetres)
+SYM(closedir)
+SYM(fdopen)
+SYM(fflush)
+SYM(freeifaddrs)
+SYM(ftok)
+SYM(getifaddrs)
+SYM(getnameinfo)
+SYM(getpwnam)
+SYM(inet_ntop)
+SYM(opendir)
+SYM(printf)
+SYM(pthread_kill)
+SYM(pthread_self)
+SYM(readdir_r)
+SYM(sem_close)
+SYM(sem_getvalue)
+SYM(sem_open)
+SYM(sem_post)
+SYM(sem_trywait)
+SYM(sem_unlink)
+SYM(sem_wait)
+SYM(siginterrupt)
+SYM(strdup)
+SYM(sendfile)
+SYM(strtod)
+SYM_2("__errno_location", ErrnoLocation)
+
+#if defined(_linux_)
+SYM(prctl)
+SYM(ptrace)
+SYM(sched_getaffinity)
+SYM(sched_setaffinity)
+SYM(sem_timedwait)
+SYM(inotify_init)
+SYM(inotify_add_watch)
+SYM(inotify_rm_watch)
+SYM(mlockall)
+#endif
+
+#if defined(_darwin_)
+SYM(mach_absolute_time)
+SYM(mach_timebase_info)
+SYM(sysctlbyname)
+#endif
+
+#if __has_include(<aio.h>)
+SYM(aio_error)
+SYM(aio_read)
+SYM(aio_return)
+SYM(aio_suspend)
+#endif
+
+END_SYMS()
+#endif
diff --git a/library/python/symbols/libc/ya.make b/library/python/symbols/libc/ya.make
new file mode 100644
index 0000000000..7b84cbc961
--- /dev/null
+++ b/library/python/symbols/libc/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+OWNER(pg orivej)
+
+PEERDIR(
+ library/python/symbols/registry
+)
+
+IF (GCC OR CLANG)
+ CFLAGS(
+ -Wno-deprecated-declarations # For sem_getvalue.
+ )
+ENDIF()
+
+SRCS(
+ GLOBAL syms.cpp
+)
+
+END()
diff --git a/library/python/symbols/module/__init__.py b/library/python/symbols/module/__init__.py
new file mode 100644
index 0000000000..0061b9e598
--- /dev/null
+++ b/library/python/symbols/module/__init__.py
@@ -0,0 +1,48 @@
+import subprocess
+
+from collections import defaultdict
+
+from .syms import syms
+
+
+def gen_builtin():
+ res = defaultdict(dict)
+
+ for k, v in syms.items():
+ mod, sym = k.split('|')
+
+ res[mod][sym] = v
+
+ return res
+
+
+builtin_symbols = gen_builtin()
+caps = builtin_symbols['_capability']
+
+
+def find_library(name, find_next):
+ subst = {
+ 'rt': 'c',
+ 'pthread': 'c',
+ 'm': 'c',
+ }
+
+ builtin = builtin_symbols.get(subst.get(name, name))
+
+ if builtin:
+ return {
+ 'name': name,
+ 'symbols': builtin,
+ }
+
+ if 'musl' in caps:
+ return None
+
+ try:
+ subprocess.Popen.__patched__
+
+ return None
+ except Exception:
+ pass
+
+ return find_next(name)
diff --git a/library/python/symbols/module/module.cpp b/library/python/symbols/module/module.cpp
new file mode 100644
index 0000000000..92bc7f4d67
--- /dev/null
+++ b/library/python/symbols/module/module.cpp
@@ -0,0 +1,85 @@
+#include <Python.h>
+
+#include <library/python/symbols/registry/syms.h>
+
+#include <util/generic/string.h>
+
+#define CAP(x) SYM_2(x, x)
+
+BEGIN_SYMS("_capability")
+#if defined(_musl_)
+CAP("musl")
+#endif
+#if defined(_linux_)
+CAP("linux")
+#endif
+#if defined(_darwin_)
+CAP("darwin")
+#endif
+CAP("_sentinel")
+END_SYMS()
+
+#undef CAP
+
+using namespace NPrivate;
+
+namespace {
+ template <class T>
+ struct TCB: public ICB {
+ inline TCB(T& t)
+ : CB(&t)
+ {
+ }
+
+ void Apply(const char* mod, const char* name, void* sym) override {
+ (*CB)(mod, name, sym);
+ }
+
+ T* CB;
+ };
+
+ template <class T>
+ static inline TCB<T> MakeTCB(T& t) {
+ return t;
+ }
+}
+
+static void DictSetStringPtr(PyObject* dict, const char* name, void* value) {
+ PyObject* p = PyLong_FromVoidPtr(value);
+ PyDict_SetItemString(dict, name, p);
+ Py_DECREF(p);
+}
+
+static PyObject* InitSyms(PyObject* m) {
+ if (!m)
+ return NULL;
+ PyObject* d = PyDict_New();
+ if (!d)
+ return NULL;
+
+ auto f = [&](const char* mod, const char* name, void* sym) {
+ DictSetStringPtr(d, (TString(mod) + "|" + TString(name)).c_str(), sym);
+ };
+
+ auto cb = MakeTCB(f);
+
+ ForEachSymbol(cb);
+
+ if (PyObject_SetAttrString(m, "syms", d))
+ m = NULL;
+ Py_DECREF(d);
+ return m;
+}
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef module = {
+ PyModuleDef_HEAD_INIT, "syms", NULL, -1, NULL, NULL, NULL, NULL, NULL};
+
+extern "C" PyObject* PyInit_syms() {
+ return InitSyms(PyModule_Create(&module));
+}
+#else
+extern "C" void initsyms() {
+ InitSyms(Py_InitModule("syms", NULL));
+}
+#endif
diff --git a/library/python/symbols/module/ya.make b/library/python/symbols/module/ya.make
new file mode 100644
index 0000000000..78e30f2547
--- /dev/null
+++ b/library/python/symbols/module/ya.make
@@ -0,0 +1,23 @@
+PY23_LIBRARY()
+
+OWNER(pg orivej)
+
+NO_PYTHON_INCLUDES()
+
+PEERDIR(
+ contrib/libs/python/Include
+)
+
+SRCS(
+ module.cpp
+)
+
+PY_REGISTER(
+ library.python.symbols.module.syms
+)
+
+PY_SRCS(
+ __init__.py
+)
+
+END()
diff --git a/library/python/symbols/python/syms.cpp b/library/python/symbols/python/syms.cpp
new file mode 100644
index 0000000000..9b52574cb1
--- /dev/null
+++ b/library/python/symbols/python/syms.cpp
@@ -0,0 +1,17 @@
+#define SYM(SYM_NAME) extern "C" void SYM_NAME();
+SYM(PyObject_GetBuffer)
+SYM(PyBuffer_Release)
+SYM(PyCell_New)
+SYM(Py_DecRef)
+SYM(Py_IncRef)
+#undef SYM
+
+#include <library/python/symbols/registry/syms.h>
+
+BEGIN_SYMS("python")
+SYM(PyObject_GetBuffer)
+SYM(PyBuffer_Release)
+SYM(PyCell_New)
+SYM(Py_DecRef)
+SYM(Py_IncRef)
+END_SYMS()
diff --git a/library/python/symbols/python/ut/py2/ya.make b/library/python/symbols/python/ut/py2/ya.make
new file mode 100644
index 0000000000..214194de57
--- /dev/null
+++ b/library/python/symbols/python/ut/py2/ya.make
@@ -0,0 +1,9 @@
+PY2TEST()
+
+OWNER(orivej)
+
+PEERDIR(
+ library/python/symbols/python/ut
+)
+
+END()
diff --git a/library/python/symbols/python/ut/py3/ya.make b/library/python/symbols/python/ut/py3/ya.make
new file mode 100644
index 0000000000..76611c6a19
--- /dev/null
+++ b/library/python/symbols/python/ut/py3/ya.make
@@ -0,0 +1,9 @@
+PY3TEST()
+
+OWNER(orivej)
+
+PEERDIR(
+ library/python/symbols/python/ut
+)
+
+END()
diff --git a/library/python/symbols/python/ut/test_ctypes.py b/library/python/symbols/python/ut/test_ctypes.py
new file mode 100644
index 0000000000..253a10d8b3
--- /dev/null
+++ b/library/python/symbols/python/ut/test_ctypes.py
@@ -0,0 +1,37 @@
+from ctypes import (
+ byref, POINTER, c_int, c_char, c_char_p,
+ c_void_p, py_object, c_ssize_t, pythonapi, Structure
+)
+
+c_ssize_p = POINTER(c_ssize_t)
+
+
+class Py_buffer(Structure):
+ _fields_ = [
+ ('buf', c_void_p),
+ ('obj', py_object),
+ ('len', c_ssize_t),
+ ('itemsize', c_ssize_t),
+ ('readonly', c_int),
+ ('ndim', c_int),
+ ('format', c_char_p),
+ ('shape', c_ssize_p),
+ ('strides', c_ssize_p),
+ ('suboffsets', c_ssize_p),
+ ('smalltable', c_ssize_t * 2),
+ ('internal', c_void_p)
+ ]
+
+
+def get_buffer(obj):
+ buf = Py_buffer()
+ pythonapi.PyObject_GetBuffer(py_object(obj), byref(buf), 0)
+ try:
+ buffer_type = c_char * buf.len
+ return buffer_type.from_address(buf.buf)
+ finally:
+ pythonapi.PyBuffer_Release(byref(buf))
+
+
+def test_buffer():
+ assert get_buffer(b'test string')
diff --git a/library/python/symbols/python/ut/ya.make b/library/python/symbols/python/ut/ya.make
new file mode 100644
index 0000000000..2849e01b1e
--- /dev/null
+++ b/library/python/symbols/python/ut/ya.make
@@ -0,0 +1,16 @@
+PY23_LIBRARY()
+
+OWNER(orivej)
+
+TEST_SRCS(test_ctypes.py)
+
+PEERDIR(
+ library/python/symbols/python
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ py2
+ py3
+)
diff --git a/library/python/symbols/python/ya.make b/library/python/symbols/python/ya.make
new file mode 100644
index 0000000000..6bfd54f8bc
--- /dev/null
+++ b/library/python/symbols/python/ya.make
@@ -0,0 +1,15 @@
+LIBRARY()
+
+OWNER(orivej)
+
+PEERDIR(
+ library/python/symbols/registry
+)
+
+SRCS(
+ GLOBAL syms.cpp
+)
+
+END()
+
+RECURSE_FOR_TESTS(ut)
diff --git a/library/python/symbols/registry/syms.cpp b/library/python/symbols/registry/syms.cpp
new file mode 100644
index 0000000000..ae8e98b32e
--- /dev/null
+++ b/library/python/symbols/registry/syms.cpp
@@ -0,0 +1,31 @@
+#include "syms.h"
+
+#include <util/generic/vector.h>
+#include <util/generic/string.h>
+#include <util/generic/singleton.h>
+
+using namespace NPrivate;
+
+namespace {
+ struct TSym {
+ const char* Mod;
+ const char* Name;
+ void* Sym;
+ };
+
+ struct TSymbols: public TVector<TSym> {
+ static inline TSymbols* Instance() {
+ return Singleton<TSymbols>();
+ }
+ };
+}
+
+void NPrivate::RegisterSymbol(const char* mod, const char* name, void* sym) {
+ TSymbols::Instance()->push_back(TSym{mod, name, sym});
+}
+
+void NPrivate::ForEachSymbol(ICB& cb) {
+ for (auto& x : *TSymbols::Instance()) {
+ cb.Apply(x.Mod, x.Name, x.Sym);
+ }
+}
diff --git a/library/python/symbols/registry/syms.h b/library/python/symbols/registry/syms.h
new file mode 100644
index 0000000000..5e34d35298
--- /dev/null
+++ b/library/python/symbols/registry/syms.h
@@ -0,0 +1,24 @@
+#pragma once
+
+namespace NPrivate {
+ struct ICB {
+ virtual void Apply(const char* mod, const char* name, void* sym) = 0;
+ };
+
+ void ForEachSymbol(ICB& cb);
+ void RegisterSymbol(const char* mod, const char* name, void* sym);
+}
+
+#define BEGIN_SYMS(name) \
+ namespace { \
+ static struct TRegister { \
+ const char* ModuleName = name; \
+ inline TRegister() {
+#define END_SYMS() \
+ } \
+ } \
+ REGISTRY; \
+ }
+
+#define SYM_2(n, s) ::NPrivate::RegisterSymbol(this->ModuleName, n, (void*)&s);
+#define SYM(s) SYM_2(#s, s);
diff --git a/library/python/symbols/registry/ya.make b/library/python/symbols/registry/ya.make
new file mode 100644
index 0000000000..2737b9fce9
--- /dev/null
+++ b/library/python/symbols/registry/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+OWNER(pg orivej)
+
+SRCS(
+ syms.cpp
+)
+
+END()
diff --git a/library/python/symbols/ya.make b/library/python/symbols/ya.make
new file mode 100644
index 0000000000..340a710c48
--- /dev/null
+++ b/library/python/symbols/ya.make
@@ -0,0 +1,12 @@
+RECURSE(
+ module
+ registry
+ tests
+
+ crypto
+ libc
+ libmagic
+ python
+ uuid
+ win_unicode_console
+)
diff --git a/library/python/testing/__init__.py b/library/python/testing/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/library/python/testing/__init__.py
diff --git a/library/python/testing/filter/filter.py b/library/python/testing/filter/filter.py
new file mode 100644
index 0000000000..a1642bd052
--- /dev/null
+++ b/library/python/testing/filter/filter.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+# TODO move devtools/ya/test/filter.py to library/python/testing/filter/filter.py
+import re
+import fnmatch
+import logging
+
+logger = logging.getLogger(__name__)
+TEST_SUBTEST_SEPARATOR = '::'
+
+PARSE_TAG_RE = re.compile("([+-]?[\w:]*)")
+
+
+class FilterException(Exception):
+ mute = True
+
+
+def fix_filter(flt):
+ if TEST_SUBTEST_SEPARATOR not in flt and "*" not in flt:
+ # user wants to filter by test module name
+ flt = flt + TEST_SUBTEST_SEPARATOR + "*"
+ return flt
+
+
+def escape_for_fnmatch(s):
+ return s.replace("[", "&#91;").replace("]", "&#93;")
+
+
+def make_py_file_filter(filter_names):
+ if filter_names is not None:
+ with_star = []
+ wo_star = set()
+ for flt in filter_names:
+ flt = flt.split(':')[0]
+ if '*' in flt:
+ with_star.append(flt.split('*')[0] + '*')
+ else:
+ wo_star.add(flt)
+
+ def predicate(filename):
+ if filter_names is None:
+ return True
+ return filename in wo_star or any([fnmatch.fnmatch(escape_for_fnmatch(filename), escape_for_fnmatch(filter_name)) for filter_name in with_star])
+
+ return predicate
+
+
+def make_name_filter(filter_names):
+ filter_names = map(fix_filter, filter_names)
+ filter_full_names = set()
+ for name in filter_names:
+ if '*' not in name:
+ filter_full_names.add(name)
+
+ def predicate(testname):
+ return testname in filter_full_names or any([fnmatch.fnmatch(escape_for_fnmatch(testname), escape_for_fnmatch(filter_name)) for filter_name in filter_names])
+
+ return predicate
diff --git a/library/python/testing/filter/ya.make b/library/python/testing/filter/ya.make
new file mode 100644
index 0000000000..22c485d258
--- /dev/null
+++ b/library/python/testing/filter/ya.make
@@ -0,0 +1,5 @@
+PY23_LIBRARY()
+OWNER(g:yatest)
+PY_SRCS(filter.py)
+
+END()
diff --git a/library/python/testing/import_test/import_test.py b/library/python/testing/import_test/import_test.py
new file mode 100644
index 0000000000..3e3b7234ef
--- /dev/null
+++ b/library/python/testing/import_test/import_test.py
@@ -0,0 +1,124 @@
+from __future__ import print_function
+
+import os
+import re
+import sys
+import time
+import traceback
+
+import __res
+from __res import importer
+
+
+def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
+ """
+ tests all bundled modules are importable
+ just add
+ "PEERDIR(library/python/import_test)" to your CMakeLists.txt and
+ "from import_test import test_imports" to your python test source file.
+ """
+ str_ = lambda s: s
+ if not isinstance(b'', str):
+ str_ = lambda s: s.decode('UTF-8')
+
+ exceptions = list(no_check)
+ for key, _ in __res.iter_keys(b'py/no_check_imports/'):
+ exceptions += str_(__res.find(key)).split()
+ if exceptions:
+ exceptions.sort()
+ print('NO_CHECK_IMPORTS', ' '.join(exceptions))
+
+ patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
+ rx = re.compile('^({})$'.format('|'.join(patterns)))
+
+ failed = []
+ import_times = {}
+
+ norm = lambda s: s[:-9] if s.endswith('.__init__') else s
+
+ modules = sys.extra_modules | set(extra)
+ modules = sorted(modules, key=norm)
+ if py_main:
+ modules = [py_main] + modules
+
+ for module in modules:
+ if module not in extra and (rx.search(module) or skip_func and skip_func(module)):
+ print('SKIP', module)
+ continue
+
+ name = module.rsplit('.', 1)[-1]
+ if name == '__main__' and 'if __name__ ==' not in importer.get_source(module):
+ print('SKIP', module, '''without "if __name__ == '__main__'" check''')
+ continue
+
+ def print_backtrace_marked(e):
+ tb_exc = traceback.format_exception(*e)
+ for item in tb_exc:
+ for l in item.splitlines():
+ print('FAIL:', l, file=sys.stderr)
+
+ try:
+ print('TRY', module)
+ # XXX waiting for py3 to use print(..., flush=True)
+ sys.stdout.flush()
+
+ s = time.time()
+ if module == '__main__':
+ importer.load_module('__main__', '__main__py')
+ elif module.endswith('.__init__'):
+ __import__(module[:-len('.__init__')])
+ else:
+ __import__(module)
+
+ delay = time.time() - s
+ import_times[str(module)] = delay
+ print('OK ', module, '{:.3f}s'.format(delay))
+
+ except Exception as e:
+ print('FAIL:', module, e, file=sys.stderr)
+ print_backtrace_marked(sys.exc_info())
+ failed.append('{}: {}'.format(module, e))
+
+ except:
+ e = sys.exc_info()
+ print('FAIL:', module, e, file=sys.stderr)
+ print_backtrace_marked(e)
+ failed.append('{}: {}'.format(module, e))
+ raise
+
+ print("Slowest imports:")
+ for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]:
+ print(' ', '{:.3f}s'.format(t), m)
+
+ if failed:
+ raise ImportError('modules not imported:\n' + '\n'.join(failed))
+
+
+test_imports = check_imports
+
+
+def main():
+ skip_names = sys.argv[1:]
+
+ os.environ['Y_PYTHON_IMPORT_TEST'] = ''
+
+ # We should initialize Django before importing any applications
+ if os.getenv('DJANGO_SETTINGS_MODULE'):
+ try:
+ import django
+ except ImportError:
+ pass
+ else:
+ django.setup()
+
+ py_main = __res.find('PY_MAIN')
+
+ if py_main:
+ py_main_module = py_main.split(b':', 1)[0].decode('UTF-8')
+ else:
+ py_main_module = None
+
+ try:
+ check_imports(no_check=skip_names, py_main=py_main_module)
+ except:
+ sys.exit(1)
diff --git a/library/python/testing/import_test/ya.make b/library/python/testing/import_test/ya.make
new file mode 100644
index 0000000000..fae36ffe8f
--- /dev/null
+++ b/library/python/testing/import_test/ya.make
@@ -0,0 +1,10 @@
+OWNER(
+ g:yatest
+ exprmntr
+)
+
+PY23_LIBRARY()
+
+PY_SRCS(import_test.py)
+
+END()
diff --git a/library/python/testing/recipe/__init__.py b/library/python/testing/recipe/__init__.py
new file mode 100644
index 0000000000..5ef9c5c189
--- /dev/null
+++ b/library/python/testing/recipe/__init__.py
@@ -0,0 +1,102 @@
+from __future__ import print_function
+
+import os
+import sys
+import json
+import logging
+import argparse
+
+from yatest_lib.ya import Ya
+
+RECIPE_START_OPTION = "start"
+RECIPE_STOP_OPTION = "stop"
+
+ya = None
+collect_cores = None
+sanitizer_extra_checks = None
+
+
+def _setup_logging(level=logging.DEBUG):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(level)
+
+ log_format = '%(asctime)s - %(levelname)s - %(name)s - %(funcName)s: %(message)s'
+
+ stdout_handler = logging.StreamHandler(sys.stdout)
+ stdout_handler.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(log_format)
+ stdout_handler.setFormatter(formatter)
+ root_logger.addHandler(stdout_handler)
+
+
+def get_options():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--show-cwd", action="store_true", dest="show_cwd", default=False, help="show recipe cwd")
+ parser.add_argument("--test-debug", action="store_true", dest="test_debug", default=False, help="test debug mode")
+ parser.add_argument("--test-stderr", action="store_true", dest="test_stderr", default=False, help="test stderr")
+ parser.add_argument("--pdb", action="store_true", dest="pdb", default=False, help="run pdb on error")
+ parser.add_argument("--sanitizer-extra-checks", dest="sanitizer_extra_checks", action="store_true", default=False, help="enables extra checks for tests built with sanitizers")
+ parser.add_argument("--collect-cores", dest="collect_cores", action="store_true", default=False, help="allows core dump file recovering during test")
+
+ args, opts = parser.parse_known_args()
+
+ global ya, sanitizer_extra_checks, collect_cores
+ _setup_logging()
+
+ context = {
+ "test_stderr": args.test_stderr,
+ }
+
+ ya = Ya(context=context)
+
+ ya._data_root = "" # XXX remove
+
+ sanitizer_extra_checks = args.sanitizer_extra_checks
+ if sanitizer_extra_checks:
+ for envvar in ['LSAN_OPTIONS', 'ASAN_OPTIONS']:
+ if envvar in os.environ:
+ os.environ.pop(envvar)
+ if envvar + '_ORIGINAL' in os.environ:
+ os.environ[envvar] = os.environ[envvar + '_ORIGINAL']
+ collect_cores = args.collect_cores
+
+ for recipe_option in RECIPE_START_OPTION, RECIPE_STOP_OPTION:
+ if recipe_option in opts:
+ return args, opts[opts.index(recipe_option):]
+
+
+def set_env(key, value):
+ with open(ya.env_file, "a") as f:
+ json.dump({key: value}, f)
+ f.write("\n")
+
+
+def tty():
+ if os.isatty(1):
+ return
+
+ f = open('/dev/tty', 'w+')
+ fd = f.fileno()
+ os.dup2(fd, 0)
+ os.dup2(fd, 1)
+ os.dup2(fd, 2)
+
+
+def declare_recipe(start, stop):
+ parsed_args, argv = get_options()
+
+ if parsed_args.show_cwd:
+ print("Recipe \"{} {}\" working dir is {}".format(sys.argv[0], " ".join(argv), os.getcwd()))
+
+ try:
+ if argv[0] == RECIPE_START_OPTION:
+ start(argv[1:])
+ elif argv[0] == RECIPE_STOP_OPTION:
+ stop(argv[1:])
+ except Exception:
+ if parsed_args.pdb:
+ tty()
+ import ipdb
+ ipdb.post_mortem()
+ else:
+ raise
diff --git a/library/python/testing/recipe/ports.py b/library/python/testing/recipe/ports.py
new file mode 100644
index 0000000000..9f7de1e767
--- /dev/null
+++ b/library/python/testing/recipe/ports.py
@@ -0,0 +1,33 @@
+import os
+import sys
+import subprocess
+import time
+
+from yatest.common.network import PortManager
+
+
+def __get_port_range():
+ port_count = int(sys.argv[1])
+ pid_filename = sys.argv[2]
+ port_manager = PortManager()
+ start_port = port_manager.get_port_range(None, port_count)
+ sys.stderr.write(str(start_port) + "\n")
+ with open(pid_filename, 'w') as afile:
+ afile.write(str(os.getpid()))
+ while 1:
+ time.sleep(1)
+
+
+def get_port_range(port_count=1, pid_filename="recipe_port.pid"):
+ env = os.environ.copy()
+ env["Y_PYTHON_ENTRY_POINT"] = "library.python.testing.recipe.ports:__get_port_range"
+ res = subprocess.Popen([sys.argv[0], str(port_count), pid_filename], env=env, cwd=os.getcwd(), stderr=subprocess.PIPE)
+ while not os.path.exists(pid_filename):
+ time.sleep(0.01)
+ port_start = int(res.stderr.readline())
+ return port_start
+
+
+def release_port_range(pid_filename="recipe_port.pid"):
+ with open(pid_filename, 'r') as afile:
+ os.kill(int(afile.read()), 9)
diff --git a/library/python/testing/recipe/ya.make b/library/python/testing/recipe/ya.make
new file mode 100644
index 0000000000..dd323aa245
--- /dev/null
+++ b/library/python/testing/recipe/ya.make
@@ -0,0 +1,19 @@
+OWNER(
+ exprmntr
+ g:yatest
+)
+
+PY23_LIBRARY()
+
+PY_SRCS(
+ __init__.py
+ ports.py
+)
+
+PEERDIR(
+ contrib/python/ipdb
+ library/python/testing/yatest_common
+ library/python/testing/yatest_lib
+)
+
+END()
diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make
new file mode 100644
index 0000000000..883bc8d7ab
--- /dev/null
+++ b/library/python/testing/ya.make
@@ -0,0 +1,22 @@
+OWNER(g:yatest)
+
+RECURSE(
+ behave
+ deprecated
+ fake_ya_package
+ filter
+ gtest
+ gtest/test
+ gtest/test/gtest
+ pyremock
+ pytest_runner
+ pytest_runner/example
+ pytest_runner/test
+ recipe
+ system_info
+ types_test
+ yapackage
+ yapackage/test
+ yatest_common
+ yatest_lib
+)
diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make
new file mode 100644
index 0000000000..5662db4c5d
--- /dev/null
+++ b/library/python/testing/yatest_common/ya.make
@@ -0,0 +1,40 @@
+OWNER(g:yatest)
+
+PY23_LIBRARY()
+
+OWNER(g:yatest)
+
+NO_EXTENDED_SOURCE_SEARCH()
+
+PY_SRCS(
+ TOP_LEVEL
+ yatest/__init__.py
+ yatest/common/__init__.py
+ yatest/common/benchmark.py
+ yatest/common/canonical.py
+ yatest/common/environment.py
+ yatest/common/errors.py
+ yatest/common/legacy.py
+ yatest/common/misc.py
+ yatest/common/network.py
+ yatest/common/path.py
+ yatest/common/process.py
+ yatest/common/runtime.py
+ yatest/common/runtime_java.py
+ yatest/common/tags.py
+)
+
+PEERDIR(
+ contrib/python/six
+ library/python/cores
+ library/python/filelock
+ library/python/fs
+)
+
+IF (NOT CATBOOST_OPENSOURCE)
+ PEERDIR(
+ library/python/coredump_filter
+ )
+ENDIF()
+
+END()
diff --git a/library/python/testing/yatest_common/yatest/__init__.py b/library/python/testing/yatest_common/yatest/__init__.py
new file mode 100644
index 0000000000..b846b3317a
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ["common"]
+
+from . import common
diff --git a/library/python/testing/yatest_common/yatest/common/__init__.py b/library/python/testing/yatest_common/yatest/common/__init__.py
new file mode 100644
index 0000000000..cf57779e27
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/__init__.py
@@ -0,0 +1,8 @@
+from .benchmark import * # noqa
+from .canonical import * # noqa
+from .errors import * # noqa
+from .misc import * # noqa
+from .path import * # noqa
+from .process import * # noqa
+from .runtime import * # noqa
+from .tags import * # noqa
diff --git a/library/python/testing/yatest_common/yatest/common/benchmark.py b/library/python/testing/yatest_common/yatest/common/benchmark.py
new file mode 100644
index 0000000000..c3784cbe4c
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/benchmark.py
@@ -0,0 +1,22 @@
+import json
+
+from . import process
+from . import runtime
+
+
+def execute_benchmark(path, budget=None, threads=None):
+ """
+ Run benchmark and return values
+ :param path: path to benchmark binary
+ :param budget: time budget, sec (supported only by ybenchmark)
+ :param threads: number of threads to run benchmark (supported only by ybenchmark)
+ :return: map of benchmark values
+ """
+ benchmark_path = runtime.binary_path(path)
+ cmd = [benchmark_path, "--benchmark_format=json"]
+ if budget is not None:
+ cmd += ["-b", str(budget)]
+ if threads is not None:
+ cmd += ["-t", str(threads)]
+ res = process.execute(cmd)
+ return json.loads(res.std_out)
diff --git a/library/python/testing/yatest_common/yatest/common/canonical.py b/library/python/testing/yatest_common/yatest/common/canonical.py
new file mode 100644
index 0000000000..b6a136d3e9
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/canonical.py
@@ -0,0 +1,176 @@
+import os
+import logging
+import shutil
+import tempfile
+
+import six
+
+from . import process
+from . import runtime
+from . import path
+
+yatest_logger = logging.getLogger("ya.test")
+
+
+def _copy(src, dst, universal_lines=False):
+ if universal_lines:
+ with open(dst, "wb") as f:
+ for line in open(src, "rbU"):
+ f.write(line)
+ return
+ shutil.copy(src, dst)
+
+
+def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None):
+ """
+ Create canonical file that can be returned from a test
+ :param path: path to the file
+ :param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used
+ :param local: save file locally, otherwise move to sandbox
+ :param universal_lines: normalize EOL
+ :param diff_tool_timeout: timeout for running diff tool
+ :return: object that can be canonized
+ """
+ abs_path = os.path.abspath(path)
+ assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+ tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path())
+ safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+ # if the created file is in output_path, we copy it, so that it will be available when the tests finishes
+ _copy(path, safe_path, universal_lines=universal_lines)
+ if diff_tool:
+ if not isinstance(diff_tool, six.string_types):
+ try: # check if iterable
+ if not isinstance(diff_tool[0], six.string_types):
+ raise Exception("Invalid custom diff-tool: not cmd")
+ except:
+ raise Exception("Invalid custom diff-tool: not binary path")
+ return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None):
+ abs_path = os.path.abspath(path)
+ assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
+ assert os.path.isdir(abs_path), "Path {} is not a directory".format(path)
+ if diff_file_name and not diff_tool:
+ raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir")
+ tempdir = tempfile.mkdtemp()
+ safe_path = os.path.join(tempdir, os.path.basename(abs_path))
+ shutil.copytree(abs_path, safe_path)
+ return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+
+def canonical_execute(
+ binary, args=None, check_exit_code=True,
+ shell=False, timeout=None, cwd=None,
+ env=None, stdin=None, stderr=None, creationflags=0,
+ file_name=None, save_locally=False, close_fds=False,
+ diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
+):
+ """
+ Shortcut to execute a binary and canonize its stdout
+ :param binary: absolute path to the binary
+ :param args: binary arguments
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param file_name: output file name. if not specified program name will be used
+ :param diff_tool: path to custome diff tool
+ :param diff_file_name: custom diff file name to create when diff is found
+ :param diff_tool_timeout: timeout for running diff tool
+ :return: object that can be canonized
+ """
+ if type(binary) == list:
+ command = binary
+ else:
+ command = [binary]
+ command += _prepare_args(args)
+ if shell:
+ command = " ".join(command)
+ execute_args = locals()
+ del execute_args["binary"]
+ del execute_args["args"]
+ del execute_args["file_name"]
+ del execute_args["save_locally"]
+ del execute_args["diff_tool"]
+ del execute_args["diff_file_name"]
+ del execute_args["diff_tool_timeout"]
+ if not file_name and stdin:
+ file_name = os.path.basename(stdin.name)
+ return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
+
+
+def canonical_py_execute(
+ script_path, args=None, check_exit_code=True,
+ shell=False, timeout=None, cwd=None, env=None,
+ stdin=None, stderr=None, creationflags=0,
+ file_name=None, save_locally=False, close_fds=False,
+ diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
+):
+ """
+ Shortcut to execute a python script and canonize its stdout
+ :param script_path: path to the script arcadia relative
+ :param args: script arguments
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param file_name: output file name. if not specified program name will be used
+ :param diff_tool: path to custome diff tool
+ :param diff_file_name: custom diff file name to create when diff is found
+ :param diff_tool_timeout: timeout for running diff tool
+ :return: object that can be canonized
+ """
+ command = [runtime.source_path(script_path)] + _prepare_args(args)
+ if shell:
+ command = " ".join(command)
+ execute_args = locals()
+ del execute_args["script_path"]
+ del execute_args["args"]
+ del execute_args["file_name"]
+ del execute_args["save_locally"]
+ del execute_args["diff_tool"]
+ del execute_args["diff_file_name"]
+ del execute_args["diff_tool_timeout"]
+ return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
+
+
+def _prepare_args(args):
+ if args is None:
+ args = []
+ if isinstance(args, six.string_types):
+ args = map(lambda a: a.strip(), args.split())
+ return args
+
+
+def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout):
+ res = excutor(**kwargs)
+ command = kwargs["command"]
+ file_name = file_name or process.get_command_name(command)
+ if file_name.endswith(".exe"):
+ file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names
+ out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name))
+ err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name))
+
+ try:
+ os.makedirs(os.path.dirname(out_file_path))
+ except OSError:
+ pass
+
+ with open(out_file_path, "wb") as out_file:
+ yatest_logger.debug("Will store file in %s", out_file_path)
+ out_file.write(res.std_out)
+
+ if res.std_err:
+ with open(err_file_path, "wb") as err_file:
+ err_file.write(res.std_err)
+
+ return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
diff --git a/library/python/testing/yatest_common/yatest/common/environment.py b/library/python/testing/yatest_common/yatest/common/environment.py
new file mode 100644
index 0000000000..43f48d0958
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/environment.py
@@ -0,0 +1,5 @@
+# coding: utf-8
+
+
+def extend_env_var(env, name, value, sep=":"):
+ return sep.join(filter(None, [env.get(name), value]))
diff --git a/library/python/testing/yatest_common/yatest/common/errors.py b/library/python/testing/yatest_common/yatest/common/errors.py
new file mode 100644
index 0000000000..8c038fc381
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/errors.py
@@ -0,0 +1,20 @@
+import os
+import sys
+
+
+class RestartTestException(Exception):
+
+ def __init__(self, *args, **kwargs):
+ super(RestartTestException, self).__init__(*args, **kwargs)
+ sys.stderr.write("##restart-test##\n")
+ sys.stderr.flush()
+ os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
+
+
+class InfrastructureException(Exception):
+
+ def __init__(self, *args, **kwargs):
+ super(InfrastructureException, self).__init__(*args, **kwargs)
+ sys.stderr.write("##infrastructure-error##\n")
+ sys.stderr.flush()
+ os.environ["FORCE_EXIT_TESTSFAILED"] = "1"
diff --git a/library/python/testing/yatest_common/yatest/common/legacy.py b/library/python/testing/yatest_common/yatest/common/legacy.py
new file mode 100644
index 0000000000..459972d253
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/legacy.py
@@ -0,0 +1,12 @@
+from . import canonical
+
+
+def old_canonical_file(output_file_name, storage_md5):
+ import yalibrary.svn
+ yalibrary.svn.run_svn([
+ 'export',
+ 'svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia_tests_data/tests_canonical_output/' + storage_md5,
+ output_file_name,
+ "--force"
+ ])
+ return canonical.canonical_file(output_file_name)
diff --git a/library/python/testing/yatest_common/yatest/common/misc.py b/library/python/testing/yatest_common/yatest/common/misc.py
new file mode 100644
index 0000000000..20d3725ac9
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/misc.py
@@ -0,0 +1,19 @@
+import functools
+
+
+def first(it):
+ for d in it:
+ if d:
+ return d
+
+
+def lazy(func):
+ res = []
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ if not res:
+ res.append(func(*args, **kwargs))
+ return res[0]
+
+ return wrapper
diff --git a/library/python/testing/yatest_common/yatest/common/network.py b/library/python/testing/yatest_common/yatest/common/network.py
new file mode 100644
index 0000000000..37bcb1b8e0
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/network.py
@@ -0,0 +1,271 @@
+# coding=utf-8
+
+import os
+import errno
+import socket
+import random
+import logging
+import platform
+import threading
+
+import six
+
+UI16MAXVAL = (1 << 16) - 1
+logger = logging.getLogger(__name__)
+
+
+class PortManagerException(Exception):
+ pass
+
+
+class PortManager(object):
+ """
+ See documentation here
+
+ https://wiki.yandex-team.ru/yatool/test/#python-acquire-ports
+ """
+
+ def __init__(self, sync_dir=None):
+ self._sync_dir = sync_dir or os.environ.get('PORT_SYNC_PATH')
+ if self._sync_dir:
+ _makedirs(self._sync_dir)
+
+ self._valid_range = get_valid_port_range()
+ self._valid_port_count = self._count_valid_ports()
+ self._filelocks = {}
+ self._lock = threading.Lock()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.release()
+
+ def get_port(self, port=0):
+ '''
+ Gets free TCP port
+ '''
+ return self.get_tcp_port(port)
+
+ def get_tcp_port(self, port=0):
+ '''
+ Gets free TCP port
+ '''
+ return self._get_port(port, socket.SOCK_STREAM)
+
+ def get_udp_port(self, port=0):
+ '''
+ Gets free UDP port
+ '''
+ return self._get_port(port, socket.SOCK_DGRAM)
+
+ def get_tcp_and_udp_port(self, port=0):
+ '''
+ Gets one free port for use in both TCP and UDP protocols
+ '''
+ if port and self._no_random_ports():
+ return port
+
+ retries = 20
+ while retries > 0:
+ retries -= 1
+
+ result_port = self.get_tcp_port()
+ if not self.is_port_free(result_port, socket.SOCK_DGRAM):
+ self.release_port(result_port)
+ # Don't try to _capture_port(), it's already captured in the get_tcp_port()
+ return result_port
+ raise Exception('Failed to find port')
+
+ def release_port(self, port):
+ with self._lock:
+ self._release_port_no_lock(port)
+
+ def _release_port_no_lock(self, port):
+ filelock = self._filelocks.pop(port, None)
+ if filelock:
+ filelock.release()
+
+ def release(self):
+ with self._lock:
+ while self._filelocks:
+ _, filelock = self._filelocks.popitem()
+ if filelock:
+ filelock.release()
+
+ def get_port_range(self, start_port, count, random_start=True):
+ assert count > 0
+ if start_port and self._no_random_ports():
+ return start_port
+
+ candidates = []
+
+ def drop_candidates():
+ for port in candidates:
+ self._release_port_no_lock(port)
+ candidates[:] = []
+
+ with self._lock:
+ for attempts in six.moves.range(128):
+ for left, right in self._valid_range:
+ if right - left < count:
+ continue
+
+ if random_start:
+ start = random.randint(left, right - ((right - left) // 2))
+ else:
+ start = left
+ for probe_port in six.moves.range(start, right):
+ if self._capture_port_no_lock(probe_port, socket.SOCK_STREAM):
+ candidates.append(probe_port)
+ else:
+ drop_candidates()
+
+ if len(candidates) == count:
+ return candidates[0]
+ # Can't find required number of ports without gap in the current range
+ drop_candidates()
+
+ raise PortManagerException("Failed to find valid port range (start_port: {} count: {}) (range: {} used: {})".format(
+ start_port, count, self._valid_range, self._filelocks))
+
+ def _count_valid_ports(self):
+ res = 0
+ for left, right in self._valid_range:
+ res += right - left
+ assert res, ('There are no available valid ports', self._valid_range)
+ return res
+
+ def _get_port(self, port, sock_type):
+ if port and self._no_random_ports():
+ return port
+
+ if len(self._filelocks) >= self._valid_port_count:
+ raise PortManagerException("All valid ports are taken ({}): {}".format(self._valid_range, self._filelocks))
+
+ salt = random.randint(0, UI16MAXVAL)
+ for attempt in six.moves.range(self._valid_port_count):
+ probe_port = (salt + attempt) % self._valid_port_count
+
+ for left, right in self._valid_range:
+ if probe_port >= (right - left):
+ probe_port -= right - left
+ else:
+ probe_port += left
+ break
+ if not self._capture_port(probe_port, sock_type):
+ continue
+ return probe_port
+
+ raise PortManagerException("Failed to find valid port (range: {} used: {})".format(self._valid_range, self._filelocks))
+
+ def _capture_port(self, port, sock_type):
+ with self._lock:
+ return self._capture_port_no_lock(port, sock_type)
+
+ def is_port_free(self, port, sock_type=socket.SOCK_STREAM):
+ sock = socket.socket(socket.AF_INET6, sock_type)
+ try:
+ sock.bind(('::', port))
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ except socket.error as e:
+ if e.errno == errno.EADDRINUSE:
+ return False
+ raise
+ finally:
+ sock.close()
+ return True
+
+ def _capture_port_no_lock(self, port, sock_type):
+ if port in self._filelocks:
+ return False
+
+ filelock = None
+ if self._sync_dir:
+ # yatest.common should try to be hermetic and don't have peerdirs
+ # otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work
+ import library.python.filelock
+
+ filelock = library.python.filelock.FileLock(os.path.join(self._sync_dir, str(port)))
+ if not filelock.acquire(blocking=False):
+ return False
+ if self.is_port_free(port, sock_type):
+ self._filelocks[port] = filelock
+ return True
+ else:
+ filelock.release()
+ return False
+
+ if self.is_port_free(port, sock_type):
+ self._filelocks[port] = filelock
+ return True
+ if filelock:
+ filelock.release()
+ return False
+
+ def _no_random_ports(self):
+ return os.environ.get("NO_RANDOM_PORTS")
+
+
+def get_valid_port_range():
+ first_valid = 1025
+ last_valid = UI16MAXVAL
+
+ given_range = os.environ.get('VALID_PORT_RANGE')
+ if given_range and ':' in given_range:
+ return [list(int(x) for x in given_range.split(':', 2))]
+
+ first_eph, last_eph = get_ephemeral_range()
+ first_invalid = max(first_eph, first_valid)
+ last_invalid = min(last_eph, last_valid)
+
+ ranges = []
+ if first_invalid > first_valid:
+ ranges.append((first_valid, first_invalid - 1))
+ if last_invalid < last_valid:
+ ranges.append((last_invalid + 1, last_valid))
+ return ranges
+
+
+def get_ephemeral_range():
+ if platform.system() == 'Linux':
+ filename = "/proc/sys/net/ipv4/ip_local_port_range"
+ if os.path.exists(filename):
+ with open(filename) as afile:
+ data = afile.read(1024) # fix for musl
+ port_range = tuple(map(int, data.strip().split()))
+ if len(port_range) == 2:
+ return port_range
+ else:
+ logger.warning("Bad ip_local_port_range format: '%s'. Going to use IANA suggestion", data)
+ elif platform.system() == 'Darwin':
+ first = _sysctlbyname_uint("net.inet.ip.portrange.first")
+ last = _sysctlbyname_uint("net.inet.ip.portrange.last")
+ if first and last:
+ return first, last
+ # IANA suggestion
+ return (1 << 15) + (1 << 14), UI16MAXVAL
+
+
+def _sysctlbyname_uint(name):
+ try:
+ from ctypes import CDLL, c_uint, byref
+ from ctypes.util import find_library
+ except ImportError:
+ return
+
+ libc = CDLL(find_library("c"))
+ size = c_uint(0)
+ res = c_uint(0)
+ libc.sysctlbyname(name, None, byref(size), None, 0)
+ libc.sysctlbyname(name, byref(res), byref(size), None, 0)
+ return res.value
+
+
+def _makedirs(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ return
+ raise
diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py
new file mode 100644
index 0000000000..6fed7dda8a
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/path.py
@@ -0,0 +1,90 @@
+# coding=utf-8
+
+import errno
+import os
+import shutil
+import contextlib
+
+import library.python.fs as lpf
+
+
+def replace_in_file(path, old, new):
+ """
+ Replace text occurrences in a file
+ :param path: path to the file
+ :param old: text to replace
+ :param new: replacement
+ """
+ with open(path) as fp:
+ content = fp.read()
+
+ lpf.ensure_removed(path)
+ with open(path, 'w') as fp:
+ fp.write(content.replace(old, new))
+
+
+@contextlib.contextmanager
+def change_dir(path):
+ old = os.getcwd()
+ try:
+ os.chdir(path)
+ yield path
+ finally:
+ os.chdir(old)
+
+
+def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None):
+ '''
+ Copy an entire directory of files into an existing directory
+ instead of raising Exception what shtuil.copytree does
+ '''
+ if not os.path.exists(dst) and os.path.isdir(src):
+ os.makedirs(dst)
+ for item in os.listdir(src):
+ s = os.path.join(src, item)
+ d = os.path.join(dst, item)
+ if os.path.isdir(s):
+ shutil.copytree(s, d, symlinks, ignore)
+ else:
+ shutil.copy2(s, d)
+ if postprocessing:
+ postprocessing(dst, False)
+ for root, dirs, files in os.walk(dst):
+ for path in dirs:
+ postprocessing(os.path.join(root, path), False)
+ for path in files:
+ postprocessing(os.path.join(root, path), True)
+
+
+def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000):
+ def atomic_file_create(path):
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL, 0o644)
+ os.close(fd)
+ return True
+ except OSError as e:
+ if e.errno in [errno.EEXIST, errno.EISDIR, errno.ETXTBSY]:
+ return False
+ # Access issue with file itself, not parent directory.
+ if e.errno == errno.EACCES and os.path.exists(path):
+ return False
+ raise e
+
+ def atomic_dir_create(path):
+ try:
+ os.mkdir(path)
+ return True
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ return False
+ raise e
+
+ file_path = os.path.join(dir_path, file_pattern)
+ lpf.ensure_dir(os.path.dirname(file_path))
+ file_counter = 0
+ handler = atomic_file_create if create_file else atomic_dir_create
+ while os.path.exists(file_path) or not handler(file_path):
+ file_path = os.path.join(dir_path, file_pattern + ".{}".format(file_counter))
+ file_counter += 1
+ assert file_counter < max_suffix
+ return file_path
diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py
new file mode 100644
index 0000000000..a8bcc21f51
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/process.py
@@ -0,0 +1,733 @@
+# coding: utf-8
+
+import os
+import re
+import time
+import signal
+import shutil
+import logging
+import tempfile
+import subprocess
+import errno
+import distutils.version
+
+import six
+
+try:
+ # yatest.common should try to be hermetic, otherwise, PYTEST_SCRIPT (aka USE_ARCADIA_PYTHON=no) won't work.
+ import library.python.cores as cores
+except ImportError:
+ cores = None
+
+from . import runtime
+from . import path
+from . import environment
+
+
+MAX_OUT_LEN = 1000 * 1000 # 1 mb
+MAX_MESSAGE_LEN = 1500
+SANITIZER_ERROR_PATTERN = br": ([A-Z][\w]+Sanitizer)"
+GLIBC_PATTERN = re.compile(r"\S+@GLIBC_([0-9.]+)")
+yatest_logger = logging.getLogger("ya.test")
+
+
+def truncate(s, size):
+ if s is None:
+ return None
+ elif len(s) <= size:
+ return s
+ else:
+ return (b'...' if isinstance(s, bytes) else '...') + s[-(size - 3):]
+
+
+def get_command_name(command):
+ return os.path.basename(command.split()[0] if isinstance(command, six.string_types) else command[0])
+
+
+class ExecutionError(Exception):
+
+ def __init__(self, execution_result):
+ if not isinstance(execution_result.command, six.string_types):
+ command = " ".join(str(arg) for arg in execution_result.command)
+ else:
+ command = execution_result.command
+ message = "Command '{command}' has failed with code {code}.\nErrors:\n{err}\n".format(
+ command=command,
+ code=execution_result.exit_code,
+ err=_format_error(execution_result.std_err))
+ if cores:
+ if execution_result.backtrace:
+ message += "Backtrace:\n[[rst]]{}[[bad]]\n".format(cores.colorize_backtrace(execution_result._backtrace))
+ else:
+ message += "Backtrace is not available: module cores isn't available"
+
+ super(ExecutionError, self).__init__(message)
+ self.execution_result = execution_result
+
+
+class TimeoutError(Exception):
+ pass
+
+
+class ExecutionTimeoutError(TimeoutError):
+ def __init__(self, execution_result, *args, **kwargs):
+ super(ExecutionTimeoutError, self).__init__(args, kwargs)
+ self.execution_result = execution_result
+
+
+class InvalidExecutionStateError(Exception):
+ pass
+
+
+class SignalInterruptionError(Exception):
+ def __init__(self, message=None):
+ super(SignalInterruptionError, self).__init__(message)
+ self.res = None
+
+
+class InvalidCommandError(Exception):
+ pass
+
+
+class _Execution(object):
+
+ def __init__(self, command, process, out_file, err_file, process_progress_listener=None, cwd=None, collect_cores=True, check_sanitizer=True, started=0, user_stdout=False, user_stderr=False):
+ self._command = command
+ self._process = process
+ self._out_file = out_file
+ self._err_file = err_file
+ self._std_out = None
+ self._std_err = None
+ self._elapsed = None
+ self._start = time.time()
+ self._process_progress_listener = process_progress_listener
+ self._cwd = cwd or os.getcwd()
+ self._collect_cores = collect_cores
+ self._backtrace = ''
+ self._check_sanitizer = check_sanitizer
+ self._metrics = {}
+ self._started = started
+ self._user_stdout = bool(user_stdout)
+ self._user_stderr = bool(user_stderr)
+ self._exit_code = None
+ if process_progress_listener:
+ process_progress_listener.open(command, process, out_file, err_file)
+
+ @property
+ def running(self):
+ return self._process.poll() is None
+
+ def kill(self):
+ if self.running:
+ self._save_outputs(False)
+ _kill_process_tree(self._process.pid)
+ self._clean_files()
+ # DEVTOOLS-2347
+ yatest_logger.debug("Process status before wait_for: %s", self.running)
+ try:
+ wait_for(lambda: not self.running, timeout=5, fail_message="Could not kill process {}".format(self._process.pid), sleep_time=.1)
+ except TimeoutError:
+ yatest_logger.debug("Process status after wait_for: %s", self.running)
+ yatest_logger.debug("Process %d info: %s", self._process.pid, _get_proc_tree_info([self._process.pid]))
+ raise
+ else:
+ raise InvalidExecutionStateError("Cannot kill a stopped process")
+
+ def terminate(self):
+ if self.running:
+ self._process.terminate()
+
+ @property
+ def process(self):
+ return self._process
+
+ @property
+ def command(self):
+ return self._command
+
+ @property
+ def returncode(self):
+ return self.exit_code
+
+ @property
+ def exit_code(self):
+ """
+ Deprecated, use returncode
+ """
+ if self._exit_code is None:
+ self._exit_code = self._process.returncode
+ return self._exit_code
+
+ @property
+ def stdout(self):
+ return self.std_out
+
+ @property
+ def std_out(self):
+ """
+ Deprecated, use stdout
+ """
+ if self._std_out is not None:
+ return self._std_out
+ if self._process.stdout and not self._user_stdout:
+ self._std_out = self._process.stdout.read()
+ return self._std_out
+
+ @property
+ def stderr(self):
+ return self.std_err
+
+ @property
+ def std_err(self):
+ """
+ Deprecated, use stderr
+ """
+ if self._std_err is not None:
+ return self._std_err
+ if self._process.stderr and not self._user_stderr:
+ self._std_err = self._process.stderr.read()
+ return self._std_err
+
+ @property
+ def elapsed(self):
+ return self._elapsed
+
+ @property
+ def backtrace(self):
+ return self._backtrace
+
+ @property
+ def metrics(self):
+ return self._metrics
+
+ def _save_outputs(self, clean_files=True):
+ if self._process_progress_listener:
+ self._process_progress_listener()
+ self._process_progress_listener.close()
+ if not self._user_stdout:
+ if self._out_file is None:
+ pass
+ elif self._out_file != subprocess.PIPE:
+ self._out_file.flush()
+ self._out_file.seek(0, os.SEEK_SET)
+ self._std_out = self._out_file.read()
+ else:
+ self._std_out = self._process.stdout.read()
+ if not self._user_stderr:
+ if self._err_file is None:
+ pass
+ elif self._err_file != subprocess.PIPE:
+ self._err_file.flush()
+ self._err_file.seek(0, os.SEEK_SET)
+ self._std_err = self._err_file.read()
+ else:
+ self._std_err = self._process.stderr.read()
+
+ if clean_files:
+ self._clean_files()
+ yatest_logger.debug("Command (pid %s) rc: %s", self._process.pid, self.exit_code)
+ yatest_logger.debug("Command (pid %s) elapsed time (sec): %s", self._process.pid, self.elapsed)
+ if self._metrics:
+ for key, value in six.iteritems(self._metrics):
+ yatest_logger.debug("Command (pid %s) %s: %s", self._process.pid, key, value)
+
+ # Since this code is Python2/3 compatible, we don't know is _std_out/_std_err is real bytes or bytes-str.
+ printable_std_out, err = _try_convert_bytes_to_string(self._std_out)
+ if err:
+ yatest_logger.debug("Got error during parse process stdout: %s", err)
+ yatest_logger.debug("stdout will be displayed as raw bytes.")
+ printable_std_err, err = _try_convert_bytes_to_string(self._std_err)
+ if err:
+ yatest_logger.debug("Got error during parse process stderr: %s", err)
+ yatest_logger.debug("stderr will be displayed as raw bytes.")
+
+ yatest_logger.debug("Command (pid %s) output:\n%s", self._process.pid, truncate(printable_std_out, MAX_OUT_LEN))
+ yatest_logger.debug("Command (pid %s) errors:\n%s", self._process.pid, truncate(printable_std_err, MAX_OUT_LEN))
+
+ def _clean_files(self):
+ if self._err_file and not self._user_stderr and self._err_file != subprocess.PIPE:
+ self._err_file.close()
+ self._err_file = None
+ if self._out_file and not self._user_stdout and self._out_file != subprocess.PIPE:
+ self._out_file.close()
+ self._out_file = None
+
+ def _recover_core(self):
+ core_path = cores.recover_core_dump_file(self.command[0], self._cwd, self.process.pid)
+ if core_path:
+ # Core dump file recovering may be disabled (for distbuild for example) - produce only bt
+ store_cores = runtime._get_ya_config().collect_cores
+ if store_cores:
+ new_core_path = path.get_unique_file_path(runtime.output_path(), "{}.{}.core".format(os.path.basename(self.command[0]), self._process.pid))
+ # Copy core dump file, because it may be overwritten
+ yatest_logger.debug("Coping core dump file from '%s' to the '%s'", core_path, new_core_path)
+ shutil.copyfile(core_path, new_core_path)
+ core_path = new_core_path
+
+ bt_filename = None
+ pbt_filename = None
+
+ if os.path.exists(runtime.gdb_path()):
+ self._backtrace = cores.get_gdb_full_backtrace(self.command[0], core_path, runtime.gdb_path())
+ bt_filename = path.get_unique_file_path(runtime.output_path(), "{}.{}.backtrace".format(os.path.basename(self.command[0]), self._process.pid))
+ with open(bt_filename, "wb") as afile:
+ afile.write(six.ensure_binary(self._backtrace))
+ # generate pretty html version of backtrace aka Tri Korochki
+ pbt_filename = bt_filename + ".html"
+ backtrace_to_html(bt_filename, pbt_filename)
+
+ if store_cores:
+ runtime._register_core(os.path.basename(self.command[0]), self.command[0], core_path, bt_filename, pbt_filename)
+ else:
+ runtime._register_core(os.path.basename(self.command[0]), None, None, bt_filename, pbt_filename)
+
+ def wait(self, check_exit_code=True, timeout=None, on_timeout=None):
+ def _wait():
+ finished = None
+ interrupted = False
+ try:
+ if hasattr(os, "wait4"):
+ try:
+ if hasattr(subprocess, "_eintr_retry_call"):
+ pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0)
+ else:
+ # PEP 475
+ pid, sts, rusage = os.wait4(self._process.pid, 0)
+ finished = time.time()
+ self._process._handle_exitstatus(sts)
+ for field in [
+ "ru_idrss",
+ "ru_inblock",
+ "ru_isrss",
+ "ru_ixrss",
+ "ru_majflt",
+ "ru_maxrss",
+ "ru_minflt",
+ "ru_msgrcv",
+ "ru_msgsnd",
+ "ru_nivcsw",
+ "ru_nsignals",
+ "ru_nswap",
+ "ru_nvcsw",
+ "ru_oublock",
+ "ru_stime",
+ "ru_utime",
+ ]:
+ if hasattr(rusage, field):
+ self._metrics[field.replace("ru_", "")] = getattr(rusage, field)
+ except OSError as exc:
+
+ if exc.errno == errno.ECHILD:
+ yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
+ else:
+ raise
+ except SignalInterruptionError:
+ interrupted = True
+ raise
+ finally:
+ if not interrupted:
+ self._process.wait() # this has to be here unconditionally, so that all process properties are set
+
+ if not finished:
+ finished = time.time()
+ self._metrics["wtime"] = round(finished - self._started, 3)
+
+ try:
+ if timeout:
+ process_is_finished = lambda: not self.running
+ fail_message = "Command '%s' stopped by %d seconds timeout" % (self._command, timeout)
+ try:
+ wait_for(process_is_finished, timeout, fail_message, sleep_time=0.1, on_check_condition=self._process_progress_listener)
+ except TimeoutError as e:
+ if on_timeout:
+ yatest_logger.debug("Calling user specified on_timeout function")
+ try:
+ on_timeout(self, timeout)
+ except Exception:
+ yatest_logger.exception("Exception while calling on_timeout")
+ raise ExecutionTimeoutError(self, str(e))
+ # Wait should be always called here, it finalizes internal states of its process and sets up return code
+ _wait()
+ except BaseException as e:
+ _kill_process_tree(self._process.pid)
+ _wait()
+ yatest_logger.debug("Command exception: %s", e)
+ raise
+ finally:
+ self._elapsed = time.time() - self._start
+ self._save_outputs()
+ self.verify_no_coredumps()
+
+ self._finalise(check_exit_code)
+
+ def _finalise(self, check_exit_code):
+ # Set the signal (negative number) which caused the process to exit
+ if check_exit_code and self.exit_code != 0:
+ yatest_logger.error("Execution failed with exit code: %s\n\t,std_out:%s\n\tstd_err:%s\n",
+ self.exit_code, truncate(self.std_out, MAX_OUT_LEN), truncate(self.std_err, MAX_OUT_LEN))
+ raise ExecutionError(self)
+
+ # Don't search for sanitize errors if stderr was redirected
+ self.verify_sanitize_errors()
+
+ def verify_no_coredumps(self):
+ """
+ Verify there is no coredump from this binary. If there is then report backtrace.
+ """
+ if self.exit_code < 0 and self._collect_cores:
+ if cores:
+ try:
+ self._recover_core()
+ except Exception:
+ yatest_logger.exception("Exception while recovering core")
+ else:
+ yatest_logger.warning("Core dump file recovering is skipped: module cores isn't available")
+
+ def verify_sanitize_errors(self):
+ """
+ Verify there are no sanitizer (ASAN, MSAN, TSAN, etc) errors for this binary. If there are any report them.
+ """
+ if self._std_err and self._check_sanitizer and runtime._get_ya_config().sanitizer_extra_checks:
+ build_path = runtime.build_path()
+ if self.command[0].startswith(build_path):
+ match = re.search(SANITIZER_ERROR_PATTERN, self._std_err)
+ if match:
+ yatest_logger.error("%s sanitizer found errors:\n\tstd_err:%s\n", match.group(1), truncate(self.std_err, MAX_OUT_LEN))
+ raise ExecutionError(self)
+ else:
+ yatest_logger.debug("No sanitizer errors found")
+ else:
+ yatest_logger.debug("'%s' doesn't belong to '%s' - no check for sanitize errors", self.command[0], build_path)
+
+
+def on_timeout_gen_coredump(exec_obj, _):
+ """
+ Function can be passed to the execute(..., timeout=X, on_timeout=on_timeout_gen_coredump)
+ to generate core dump file, backtrace ahd html-version of the backtrace in case of timeout.
+ All files will be available in the testing_out_stuff and via links.
+ """
+ try:
+ os.kill(exec_obj.process.pid, signal.SIGQUIT)
+ except OSError:
+ # process might be already terminated
+ pass
+
+
+def execute(
+ command, check_exit_code=True,
+ shell=False, timeout=None,
+ cwd=None, env=None,
+ stdin=None, stdout=None, stderr=None,
+ creationflags=0, wait=True,
+ process_progress_listener=None, close_fds=False,
+ collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None,
+ executor=_Execution,
+):
+ """
+ Executes a command
+ :param command: command: can be a list of arguments or a string
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stdout: command stdout
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param wait: should wait until the command finishes
+ :param process_progress_listener=object that is polled while execution is in progress
+ :param close_fds: subrpocess.Popen close_fds args
+ :param collect_cores: recover core dump files if shell == False
+ :param check_sanitizer: raise ExecutionError if stderr contains sanitize errors
+ :param preexec_fn: subrpocess.Popen preexec_fn arg
+ :param on_timeout: on_timeout(<execution object>, <timeout value>) callback
+
+ :return _Execution: Execution object
+ """
+ if env is None:
+ env = os.environ.copy()
+ else:
+ # Certain environment variables must be present for programs to work properly.
+ # For more info see DEVTOOLSSUPPORT-4907
+ mandatory_env_name = 'YA_MANDATORY_ENV_VARS'
+ mandatory_vars = env.get(mandatory_env_name, os.environ.get(mandatory_env_name)) or ''
+ if mandatory_vars:
+ env[mandatory_env_name] = mandatory_vars
+ mandatory_system_vars = filter(None, mandatory_vars.split(':'))
+ else:
+ mandatory_system_vars = ['TMPDIR']
+
+ for var in mandatory_system_vars:
+ if var not in env and var in os.environ:
+ env[var] = os.environ[var]
+
+ if not wait and timeout is not None:
+ raise ValueError("Incompatible arguments 'timeout' and wait=False")
+
+ # if subprocess.PIPE in [stdout, stderr]:
+ # raise ValueError("Don't use pipe to obtain stream data - it may leads to the deadlock")
+
+ def get_out_stream(stream, default_name):
+ if stream is None:
+ # No stream is supplied: open new temp file
+ return _get_command_output_file(command, default_name), False
+
+ if isinstance(stream, six.string_types):
+ # User filename is supplied: open file for writing
+ return open(stream, 'wb+'), stream.startswith('/dev/')
+
+ # Open file or PIPE sentinel is supplied
+ is_pipe = stream == subprocess.PIPE
+ return stream, not is_pipe
+
+ # to be able to have stdout/stderr and track the process time execution, we don't use subprocess.PIPE,
+ # as it can cause processes hangs, but use tempfiles instead
+ out_file, user_stdout = get_out_stream(stdout, 'out')
+ err_file, user_stderr = get_out_stream(stderr, 'err')
+ in_file = stdin
+
+ if shell and type(command) == list:
+ command = " ".join(command)
+
+ if shell:
+ collect_cores = False
+ check_sanitizer = False
+ else:
+ if isinstance(command, (list, tuple)):
+ executable = command[0]
+ else:
+ executable = command
+ if os.path.isabs(executable):
+ if not os.path.isfile(executable) and not os.path.isfile(executable + ".exe"):
+ exists = os.path.exists(executable)
+ if exists:
+ stat = os.stat(executable)
+ else:
+ stat = None
+ raise InvalidCommandError("Target program is not a file: {} (exists: {} stat: {})".format(executable, exists, stat))
+ if not os.access(executable, os.X_OK) and not os.access(executable + ".exe", os.X_OK):
+ raise InvalidCommandError("Target program is not executable: {}".format(executable))
+
+ if check_sanitizer:
+ env["LSAN_OPTIONS"] = environment.extend_env_var(os.environ, "LSAN_OPTIONS", "exitcode=100")
+
+ if stdin:
+ name = "PIPE" if stdin == subprocess.PIPE else stdin.name
+ yatest_logger.debug("Executing '%s' with input '%s' in '%s'", command, name, cwd)
+ else:
+ yatest_logger.debug("Executing '%s' in '%s'", command, cwd)
+ # XXX
+
+ started = time.time()
+ process = subprocess.Popen(
+ command, shell=shell, universal_newlines=True,
+ stdout=out_file, stderr=err_file, stdin=in_file,
+ cwd=cwd, env=env, creationflags=creationflags, close_fds=close_fds, preexec_fn=preexec_fn,
+ )
+ yatest_logger.debug("Command pid: %s", process.pid)
+
+ res = executor(command, process, out_file, err_file, process_progress_listener, cwd, collect_cores, check_sanitizer, started, user_stdout=user_stdout, user_stderr=user_stderr)
+ if wait:
+ res.wait(check_exit_code, timeout, on_timeout)
+ return res
+
+
+def _get_command_output_file(cmd, ext):
+ parts = [get_command_name(cmd)]
+ if 'YA_RETRY_INDEX' in os.environ:
+ parts.append('retry{}'.format(os.environ.get('YA_RETRY_INDEX')))
+ if int(os.environ.get('YA_SPLIT_COUNT', '0')) > 1:
+ parts.append('chunk{}'.format(os.environ.get('YA_SPLIT_INDEX', '0')))
+
+ filename = '.'.join(parts + [ext])
+ try:
+ # if execution is performed from test, save out / err to the test logs dir
+ import yatest.common
+ import library.python.pytest.plugins.ya
+ if getattr(library.python.pytest.plugins.ya, 'pytest_config', None) is None:
+ raise ImportError("not in test")
+ filename = path.get_unique_file_path(yatest.common.output_path(), filename)
+ yatest_logger.debug("Command %s will be placed to %s", ext, os.path.basename(filename))
+ return open(filename, "wb+")
+ except ImportError:
+ return tempfile.NamedTemporaryFile(delete=False, suffix=filename)
+
+
+def _get_proc_tree_info(pids):
+ if os.name == 'nt':
+ return 'Not supported'
+ else:
+ stdout, _ = subprocess.Popen(["/bin/ps", "-wufp"] + [str(p) for p in pids], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+ return stdout
+
+
+def py_execute(
+ command, check_exit_code=True,
+ shell=False, timeout=None,
+ cwd=None, env=None,
+ stdin=None, stdout=None, stderr=None,
+ creationflags=0, wait=True,
+ process_progress_listener=None, close_fds=False
+):
+ """
+ Executes a command with the arcadia python
+ :param command: command to pass to python
+ :param check_exit_code: will raise ExecutionError if the command exits with non zero code
+ :param shell: use shell to run the command
+ :param timeout: execution timeout
+ :param cwd: working directory
+ :param env: command environment
+ :param stdin: command stdin
+ :param stdout: command stdout
+ :param stderr: command stderr
+ :param creationflags: command creation flags
+ :param wait: should wait until the command finishes
+ :param process_progress_listener=object that is polled while execution is in progress
+ :return _Execution: Execution object
+ """
+ if isinstance(command, six.string_types):
+ command = [command]
+ command = [runtime.python_path()] + command
+ if shell:
+ command = " ".join(command)
+ return execute(**locals())
+
+
+def _format_error(error):
+ return truncate(error, MAX_MESSAGE_LEN)
+
+
+def wait_for(check_function, timeout, fail_message="", sleep_time=1.0, on_check_condition=None):
+ """
+ Tries to execute `check_function` for `timeout` seconds.
+ Continue until function returns nonfalse value.
+ If function doesn't return nonfalse value for `timeout` seconds
+ OperationTimeoutException is raised.
+ Return first nonfalse result returned by `checkFunction`.
+ """
+ if sleep_time <= 0:
+ raise ValueError("Incorrect sleep time value {}".format(sleep_time))
+ if timeout < 0:
+ raise ValueError("Incorrect timeout value {}".format(timeout))
+ start = time.time()
+ while start + timeout > time.time():
+ if on_check_condition:
+ on_check_condition()
+
+ res = check_function()
+ if res:
+ return res
+ time.sleep(sleep_time)
+
+ message = "{} second(s) wait timeout has expired".format(timeout)
+ if fail_message:
+ message += ": {}".format(fail_message)
+ raise TimeoutError(truncate(message, MAX_MESSAGE_LEN))
+
+
+def _kill_process_tree(process_pid, target_pid_signal=None):
+ """
+ Kills child processes, req. Note that psutil should be installed
+ @param process_pid: parent id to search for descendants
+ """
+ yatest_logger.debug("Killing process %s", process_pid)
+ if os.name == 'nt':
+ _win_kill_process_tree(process_pid)
+ else:
+ _nix_kill_process_tree(process_pid, target_pid_signal)
+
+
+def _nix_get_proc_children(pid):
+ try:
+ cmd = ["pgrep", "-P", str(pid)]
+ return [int(p) for p in subprocess.check_output(cmd).split()]
+ except Exception:
+ return []
+
+
+def _get_binname(pid):
+ try:
+ return os.path.basename(os.readlink('/proc/{}/exe'.format(pid)))
+ except Exception as e:
+ return "error({})".format(e)
+
+
+def _nix_kill_process_tree(pid, target_pid_signal=None):
+ """
+ Kills the process tree.
+ """
+ yatest_logger.debug("Killing process tree for pid {} (bin:'{}')".format(pid, _get_binname(pid)))
+
+ def try_to_send_signal(pid, sig):
+ try:
+ os.kill(pid, sig)
+ yatest_logger.debug("Sent signal %d to the pid %d", sig, pid)
+ except Exception as exc:
+ yatest_logger.debug("Error while sending signal {sig} to pid {pid}: {error}".format(sig=sig, pid=pid, error=str(exc)))
+
+ try_to_send_signal(pid, signal.SIGSTOP) # Stop the process to prevent it from starting any child processes.
+
+ # Get the child process PID list.
+ child_pids = _nix_get_proc_children(pid)
+ # Stop the child processes.
+ for child_pid in child_pids:
+ try:
+ # Kill the child recursively.
+ _kill_process_tree(int(child_pid))
+ except Exception as e:
+ # Skip the error and continue killing.
+ yatest_logger.debug("Killing child pid {pid} failed: {error}".format(pid=child_pid, error=e))
+ continue
+
+ try_to_send_signal(pid, target_pid_signal or signal.SIGKILL) # Kill the root process.
+
+ # sometimes on freebsd sigkill cannot kill the process and either sigkill or sigcont should be sent
+ # https://www.mail-archive.com/freebsd-hackers@freebsd.org/msg159646.html
+ try_to_send_signal(pid, signal.SIGCONT)
+
+
+def _win_kill_process_tree(pid):
+ subprocess.call(['taskkill', '/F', '/T', '/PID', str(pid)])
+
+
+def _run_readelf(binary_path):
+ return str(subprocess.check_output([runtime.binary_path('contrib/python/pyelftools/readelf/readelf'), '-s', runtime.binary_path(binary_path)]))
+
+
+def check_glibc_version(binary_path):
+ lucid_glibc_version = distutils.version.LooseVersion("2.11")
+
+ for l in _run_readelf(binary_path).split('\n'):
+ match = GLIBC_PATTERN.search(l)
+ if not match:
+ continue
+ assert distutils.version.LooseVersion(match.group(1)) <= lucid_glibc_version, match.group(0)
+
+
+def backtrace_to_html(bt_filename, output):
+ try:
+ from library.python import coredump_filter
+ with open(output, "wb") as afile:
+ coredump_filter.filter_stackdump(bt_filename, stream=afile)
+ except ImportError as e:
+ yatest_logger.debug("Failed to import coredump_filter: %s", e)
+ with open(output, "wb") as afile:
+ afile.write("<html>Failed to import coredump_filter in USE_ARCADIA_PYTHON=no mode</html>")
+
+
+def _try_convert_bytes_to_string(source):
+ """ Function is necessary while this code Python2/3 compatible, because bytes in Python3 is a real bytes and in Python2 is not """
+ # Bit ugly typecheck, because in Python2 isinstance(str(), bytes) and "type(str()) is bytes" working as True as well
+ if 'bytes' not in str(type(source)):
+ # We already got not bytes. Nothing to do here.
+ return source, False
+
+ result = source
+ error = False
+ try:
+ result = source.decode(encoding='utf-8')
+ except ValueError as e:
+ error = e
+
+ return result, error
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
new file mode 100644
index 0000000000..e55e193446
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -0,0 +1,343 @@
+import errno
+import functools
+import json
+import os
+import threading
+
+import six
+
+
+_lock = threading.Lock()
+
+
+def _get_ya_config():
+ try:
+ import library.python.pytest.plugins.ya as ya_plugin
+ if ya_plugin.pytest_config is not None:
+ return ya_plugin.pytest_config
+ import pytest
+ return pytest.config
+ except (ImportError, AttributeError):
+ try:
+ import library.python.testing.recipe
+ if library.python.testing.recipe.ya:
+ return library.python.testing.recipe
+ except (ImportError, AttributeError):
+ pass
+ raise NotImplementedError("yatest.common.* is only available from the testing runtime")
+
+
+def _get_ya_plugin_instance():
+ return _get_ya_config().ya
+
+
+def _norm_path(path):
+ if path is None:
+ return None
+ assert isinstance(path, six.string_types)
+ if "\\" in path:
+ raise AssertionError("path {} contains Windows seprators \\ - replace them with '/'".format(path))
+ return os.path.normpath(path)
+
+
+def _join_path(main_path, path):
+ if not path:
+ return main_path
+ return os.path.join(main_path, _norm_path(path))
+
+
+def not_test(func):
+ """
+ Mark any function as not a test for py.test
+ :param func:
+ :return:
+ """
+ @functools.wraps(func)
+ def wrapper(*args, **kwds):
+ return func(*args, **kwds)
+ setattr(wrapper, '__test__', False)
+ return wrapper
+
+
+def source_path(path=None):
+ """
+ Get source path inside arcadia
+ :param path: path arcadia relative, e.g. yatest.common.source_path('devtools/ya')
+ :return: absolute path to the source folder
+ """
+ return _join_path(_get_ya_plugin_instance().source_root, path)
+
+
+def build_path(path=None):
+ """
+ Get path inside build directory
+ :param path: path relative to the build directory, e.g. yatest.common.build_path('devtools/ya/bin')
+ :return: absolute path inside build directory
+ """
+ return _join_path(_get_ya_plugin_instance().build_root, path)
+
+
+def java_path():
+ """
+ [DEPRECATED] Get path to java
+ :return: absolute path to java
+ """
+ from . import runtime_java
+ return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk')))
+
+
+def java_home():
+ """
+ Get jdk directory path
+ """
+ from . import runtime_java
+ jdk_dir = runtime_java.get_build_java_dir(binary_path('jdk'))
+ if not jdk_dir:
+ raise Exception("Cannot find jdk - make sure 'jdk' is added to the DEPENDS section and exists for the current platform")
+ return jdk_dir
+
+
+def java_bin():
+ """
+ Get path to the java binary
+ Requires DEPENDS(jdk)
+ """
+ return os.path.join(java_home(), "bin", "java")
+
+
+def data_path(path=None):
+ """
+ Get path inside arcadia_tests_data directory
+ :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
+ :return: absolute path inside arcadia_tests_data
+ """
+ return _join_path(_get_ya_plugin_instance().data_root, path)
+
+
+def output_path(path=None):
+ """
+ Get path inside the current test suite output dir.
+ Placing files to this dir guarantees that files will be accessible after the test suite execution.
+ :param path: path relative to the test suite output dir
+ :return: absolute path inside the test suite output dir
+ """
+ return _join_path(_get_ya_plugin_instance().output_dir, path)
+
+
+def ram_drive_path(path=None):
+ """
+ :param path: path relative to the ram drive.
+ :return: absolute path inside the ram drive directory or None if no ram drive was provided by environment.
+ """
+ if 'YA_TEST_RAM_DRIVE_PATH' in os.environ:
+ return _join_path(os.environ['YA_TEST_RAM_DRIVE_PATH'], path)
+
+
+def output_ram_drive_path(path=None):
+ """
+ Returns path inside ram drive directory which will be saved in the testing_out_stuff directory after testing.
+ Returns None if no ram drive was provided by environment.
+ :param path: path relative to the output ram drive directory
+ """
+ if 'YA_TEST_OUTPUT_RAM_DRIVE_PATH' in os.environ:
+ return _join_path(os.environ['YA_TEST_OUTPUT_RAM_DRIVE_PATH'], path)
+
+
+def binary_path(path=None):
+ """
+ Get path to the built binary
+ :param path: path to the binary relative to the build directory e.g. yatest.common.binary_path('devtools/ya/bin/ya-bin')
+ :return: absolute path to the binary
+ """
+ path = _norm_path(path)
+ return _get_ya_plugin_instance().get_binary(path)
+
+
+def work_path(path=None):
+ """
+ Get path inside the current test suite working directory. Creating files in the work directory does not guarantee
+ that files will be accessible after the test suite execution
+ :param path: path relative to the test suite working dir
+ :return: absolute path inside the test suite working dir
+ """
+ return _join_path(
+ os.environ.get("TEST_WORK_PATH") or
+ _get_ya_plugin_instance().get_context("work_path") or
+ os.getcwd(),
+ path)
+
+
+def python_path():
+ """
+ Get path to the arcadia python.
+
+ Warn: if you are using build with system python (-DUSE_SYSTEM_PYTHON=X) beware that some python bundles
+ are built in a stripped-down form that is needed for building, not running tests.
+ See comments in the file below to find out which version of python is compatible with tests.
+ https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc
+ :return: absolute path to python
+ """
+ return _get_ya_plugin_instance().python_path
+
+
+def valgrind_path():
+ """
+ Get path to valgrind
+ :return: absolute path to valgrind
+ """
+ return _get_ya_plugin_instance().valgrind_path
+
+
+def get_param(key, default=None):
+ """
+ Get arbitrary parameter passed via command line
+ :param key: key
+ :param default: default value
+ :return: parameter value or the default
+ """
+ return _get_ya_plugin_instance().get_param(key, default)
+
+
+def get_param_dict_copy():
+ """
+ Return copy of dictionary with all parameters. Changes to this dictionary do *not* change parameters.
+
+ :return: copy of dictionary with all parameters
+ """
+ return _get_ya_plugin_instance().get_param_dict_copy()
+
+
+@not_test
+def test_output_path(path=None):
+ """
+ Get dir in the suite output_path for the current test case
+ """
+ test_out_dir = os.path.splitext(_get_ya_config().current_test_log_path)[0]
+ try:
+ os.makedirs(test_out_dir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ return _join_path(test_out_dir, path)
+
+
+def project_path(path=None):
+ """
+ Get path in build root relating to build_root/project path
+ """
+ return _join_path(os.path.join(build_path(), context.project_path), path)
+
+
+def gdb_path():
+ """
+ Get path to the gdb
+ """
+ return _get_ya_plugin_instance().gdb_path
+
+
+def c_compiler_path():
+ """
+ Get path to the gdb
+ """
+ return os.environ.get("YA_CC")
+
+
+def get_yt_hdd_path(path=None):
+ if 'HDD_PATH' in os.environ:
+ return _join_path(os.environ['HDD_PATH'], path)
+
+
+def cxx_compiler_path():
+ """
+ Get path to the gdb
+ """
+ return os.environ.get("YA_CXX")
+
+
+def global_resources():
+ try:
+ return json.loads(os.environ.get("YA_GLOBAL_RESOURCES"))
+ except (TypeError, ValueError):
+ return {}
+
+
+def _register_core(name, binary_path, core_path, bt_path, pbt_path):
+ config = _get_ya_config()
+
+ with _lock:
+ if not hasattr(config, 'test_cores_count'):
+ config.test_cores_count = 0
+ config.test_cores_count += 1
+ count_str = '' if config.test_cores_count == 1 else str(config.test_cores_count)
+
+ log_entry = config.test_logs[config.current_item_nodeid]
+ if binary_path:
+ log_entry['{} binary{}'.format(name, count_str)] = binary_path
+ if core_path:
+ log_entry['{} core{}'.format(name, count_str)] = core_path
+ if bt_path:
+ log_entry['{} backtrace{}'.format(name, count_str)] = bt_path
+ if pbt_path:
+ log_entry['{} backtrace html{}'.format(name, count_str)] = pbt_path
+
+
+@not_test
+def test_source_path(path=None):
+ return _join_path(os.path.join(source_path(), context.project_path), path)
+
+
+class Context(object):
+ """
+ Runtime context
+ """
+
+ @property
+ def build_type(self):
+ return _get_ya_plugin_instance().get_context("build_type")
+
+ @property
+ def project_path(self):
+ return _get_ya_plugin_instance().get_context("project_path")
+
+ @property
+ def test_stderr(self):
+ return _get_ya_plugin_instance().get_context("test_stderr")
+
+ @property
+ def test_debug(self):
+ return _get_ya_plugin_instance().get_context("test_debug")
+
+ @property
+ def test_traceback(self):
+ return _get_ya_plugin_instance().get_context("test_traceback")
+
+ @property
+ def test_name(self):
+ return _get_ya_config().current_test_name
+
+ @property
+ def sanitize(self):
+ """
+ Detect if current test run is under sanitizer
+
+ :return: one of `None`, 'address', 'memory', 'thread', 'undefined'
+ """
+ return _get_ya_plugin_instance().get_context("sanitize")
+
+ @property
+ def flags(self):
+ _flags = _get_ya_plugin_instance().get_context("flags")
+ if _flags:
+ _flags_dict = dict()
+ for f in _flags:
+ key, value = f.split('=', 1)
+ _flags_dict[key] = value
+ return _flags_dict
+ else:
+ return dict()
+
+ def get_context_key(self, key):
+ return _get_ya_plugin_instance().get_context(key)
+
+
+context = Context()
diff --git a/library/python/testing/yatest_common/yatest/common/runtime_java.py b/library/python/testing/yatest_common/yatest/common/runtime_java.py
new file mode 100644
index 0000000000..39bbb45570
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/runtime_java.py
@@ -0,0 +1,46 @@
+import os
+import tarfile
+import contextlib
+
+from . import runtime
+
+_JAVA_DIR = []
+
+
+def get_java_path(jdk_dir):
+ # deprecated - to be deleted
+ java_paths = (os.path.join(jdk_dir, 'bin', 'java'), os.path.join(jdk_dir, 'bin', 'java.exe'))
+
+ for p in java_paths:
+ if os.path.exists(p):
+ return p
+
+ for f in os.listdir(jdk_dir):
+ if f.endswith('.tar'):
+ with contextlib.closing(tarfile.open(os.path.join(jdk_dir, f))) as tf:
+ tf.extractall(jdk_dir)
+
+ for p in java_paths:
+ if os.path.exists(p):
+ return p
+
+ return ''
+
+
+def get_build_java_dir(jdk_dir):
+ versions = [8, 10, 11, 12, 13, 14, 15]
+
+ if not _JAVA_DIR:
+ for version in versions:
+ jdk_tar_path = os.path.join(jdk_dir, "jdk{}.tar".format(version))
+ if os.path.exists(jdk_tar_path):
+ jdk_dir = runtime.build_path('jdk4test')
+ with contextlib.closing(tarfile.open(jdk_tar_path)) as tf:
+ tf.extractall(jdk_dir)
+ assert os.path.exists(os.path.join(jdk_dir, "bin", "java"))
+ _JAVA_DIR.append(jdk_dir)
+ break
+ else:
+ _JAVA_DIR.append(None)
+
+ return _JAVA_DIR[0]
diff --git a/library/python/testing/yatest_common/yatest/common/tags.py b/library/python/testing/yatest_common/yatest/common/tags.py
new file mode 100644
index 0000000000..9e7a74cdf5
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/tags.py
@@ -0,0 +1,5 @@
+try:
+ import pytest
+ ya_external = getattr(pytest.mark, "ya:external")
+except ImportError:
+ ya_external = None
diff --git a/library/python/testing/yatest_common/yatest/common/ya.make b/library/python/testing/yatest_common/yatest/common/ya.make
new file mode 100644
index 0000000000..f7c50dfe64
--- /dev/null
+++ b/library/python/testing/yatest_common/yatest/common/ya.make
@@ -0,0 +1 @@
+OWNER(g:yatest)
diff --git a/library/python/testing/yatest_lib/__init__.py b/library/python/testing/yatest_lib/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/library/python/testing/yatest_lib/__init__.py
diff --git a/library/python/testing/yatest_lib/external.py b/library/python/testing/yatest_lib/external.py
new file mode 100644
index 0000000000..39113230d9
--- /dev/null
+++ b/library/python/testing/yatest_lib/external.py
@@ -0,0 +1,192 @@
+from __future__ import absolute_import
+
+import re
+import sys
+import copy
+import logging
+
+from . import tools
+from datetime import date, datetime
+
+import enum
+import six
+
+logger = logging.getLogger(__name__)
+MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/'
+
+
+def apply(func, value, apply_to_keys=False):
+ """
+ Applies func to every possible member of value
+ :param value: could be either a primitive object or a complex one (list, dicts)
+ :param func: func to be applied
+ :return:
+ """
+ def _apply(func, value, value_path):
+ if value_path is None:
+ value_path = []
+
+ if isinstance(value, list) or isinstance(value, tuple):
+ res = []
+ for ind, item in enumerate(value):
+ path = copy.copy(value_path)
+ path.append(ind)
+ res.append(_apply(func, item, path))
+ elif isinstance(value, dict):
+ if is_external(value):
+ # this is a special serialized object pointing to some external place
+ res = func(value, value_path)
+ else:
+ res = {}
+ for key, val in sorted(value.items(), key=lambda dict_item: dict_item[0]):
+ path = copy.copy(value_path)
+ path.append(key)
+ res[_apply(func, key, path) if apply_to_keys else key] = _apply(func, val, path)
+ else:
+ res = func(value, value_path)
+ return res
+ return _apply(func, value, None)
+
+
+def is_coroutine(val):
+ if sys.version_info[0] < 3:
+ return False
+ else:
+ import asyncio
+ return asyncio.iscoroutinefunction(val) or asyncio.iscoroutine(val)
+
+
+def serialize(value):
+ """
+ Serialize value to json-convertible object
+ Ensures that all components of value can be serialized to json
+ :param value: object to be serialized
+ """
+ def _serialize(val, _):
+ if val is None:
+ return val
+ if isinstance(val, six.string_types) or isinstance(val, bytes):
+ return tools.to_utf8(val)
+ if isinstance(val, enum.Enum):
+ return str(val)
+ if isinstance(val, six.integer_types) or type(val) in [float, bool]:
+ return val
+ if is_external(val):
+ return dict(val)
+ if isinstance(val, (date, datetime)):
+ return repr(val)
+ if is_coroutine(val):
+ return None
+ raise ValueError("Cannot serialize value '{}' of type {}".format(val, type(val)))
+ return apply(_serialize, value, apply_to_keys=True)
+
+
+def is_external(value):
+ return isinstance(value, dict) and "uri" in value.keys()
+
+
+class ExternalSchema(object):
+ File = "file"
+ SandboxResource = "sbr"
+ Delayed = "delayed"
+ HTTP = "http"
+
+
+class CanonicalObject(dict):
+ def __iter__(self):
+ raise TypeError("Iterating canonical object is not implemented")
+
+
+class ExternalDataInfo(object):
+
+ def __init__(self, data):
+ assert is_external(data)
+ self._data = data
+
+ def __str__(self):
+ type_str = "File" if self.is_file else "Sandbox resource"
+ return "{}({})".format(type_str, self.path)
+
+ def __repr__(self):
+ return str(self)
+
+ @property
+ def uri(self):
+ return self._data["uri"]
+
+ @property
+ def checksum(self):
+ return self._data.get("checksum")
+
+ @property
+ def is_file(self):
+ return self.uri.startswith(ExternalSchema.File)
+
+ @property
+ def is_sandbox_resource(self):
+ return self.uri.startswith(ExternalSchema.SandboxResource)
+
+ @property
+ def is_delayed(self):
+ return self.uri.startswith(ExternalSchema.Delayed)
+
+ @property
+ def is_http(self):
+ return self.uri.startswith(ExternalSchema.HTTP)
+
+ @property
+ def path(self):
+ if self.uri.count("://") != 1:
+ logger.error("Invalid external data uri: '%s'", self.uri)
+ return self.uri
+ _, path = self.uri.split("://")
+ return path
+
+ def get_mds_key(self):
+ assert self.is_http
+ m = re.match(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)', self.uri)
+ if m:
+ return m.group(1)
+ raise AssertionError("Failed to extract mds key properly from '{}'".format(self.uri))
+
+ @property
+ def size(self):
+ return self._data.get("size")
+
+ def serialize(self):
+ return self._data
+
+ @classmethod
+ def _serialize(cls, schema, path, checksum=None, attrs=None):
+ res = CanonicalObject({"uri": "{}://{}".format(schema, path)})
+ if checksum:
+ res["checksum"] = checksum
+ if attrs:
+ res.update(attrs)
+ return res
+
+ @classmethod
+ def serialize_file(cls, path, checksum=None, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None, size=None):
+ attrs = {}
+ if diff_tool:
+ attrs["diff_tool"] = diff_tool
+ if local:
+ attrs["local"] = local
+ if diff_file_name:
+ attrs["diff_file_name"] = diff_file_name
+ if diff_tool_timeout:
+ attrs["diff_tool_timeout"] = diff_tool_timeout
+ if size is not None:
+ attrs["size"] = size
+ return cls._serialize(ExternalSchema.File, path, checksum, attrs=attrs)
+
+ @classmethod
+ def serialize_resource(cls, id, checksum=None):
+ return cls._serialize(ExternalSchema.SandboxResource, id, checksum)
+
+ @classmethod
+ def serialize_delayed(cls, upload_id, checksum):
+ return cls._serialize(ExternalSchema.Delayed, upload_id, checksum)
+
+ def get(self, key, default=None):
+ return self._data.get(key, default)
diff --git a/library/python/testing/yatest_lib/test_splitter.py b/library/python/testing/yatest_lib/test_splitter.py
new file mode 100644
index 0000000000..acbcd4300e
--- /dev/null
+++ b/library/python/testing/yatest_lib/test_splitter.py
@@ -0,0 +1,102 @@
+# coding: utf-8
+
+import collections
+
+
+def flatten_tests(test_classes):
+ """
+ >>> test_classes = {x: [x] for x in range(5)}
+ >>> flatten_tests(test_classes)
+ [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]
+ >>> test_classes = {x: [x + 1, x + 2] for x in range(2)}
+ >>> flatten_tests(test_classes)
+ [(0, 1), (0, 2), (1, 2), (1, 3)]
+ """
+ tests = []
+ for class_name, test_names in test_classes.items():
+ tests += [(class_name, test_name) for test_name in test_names]
+ return tests
+
+
+def get_sequential_chunk(tests, modulo, modulo_index, is_sorted=False):
+ """
+ >>> get_sequential_chunk(range(10), 4, 0)
+ [0, 1, 2]
+ >>> get_sequential_chunk(range(10), 4, 1)
+ [3, 4, 5]
+ >>> get_sequential_chunk(range(10), 4, 2)
+ [6, 7]
+ >>> get_sequential_chunk(range(10), 4, 3)
+ [8, 9]
+ >>> get_sequential_chunk(range(10), 4, 4)
+ []
+ >>> get_sequential_chunk(range(10), 4, 5)
+ []
+ """
+ if not is_sorted:
+ tests = sorted(tests)
+ chunk_size = len(tests) // modulo
+ not_used = len(tests) % modulo
+ shift = chunk_size + (modulo_index < not_used)
+ start = chunk_size * modulo_index + min(modulo_index, not_used)
+ end = start + shift
+ return [] if end > len(tests) else tests[start:end]
+
+
+def get_shuffled_chunk(tests, modulo, modulo_index, is_sorted=False):
+ """
+ >>> get_shuffled_chunk(range(10), 4, 0)
+ [0, 4, 8]
+ >>> get_shuffled_chunk(range(10), 4, 1)
+ [1, 5, 9]
+ >>> get_shuffled_chunk(range(10), 4, 2)
+ [2, 6]
+ >>> get_shuffled_chunk(range(10), 4, 3)
+ [3, 7]
+ >>> get_shuffled_chunk(range(10), 4, 4)
+ []
+ >>> get_shuffled_chunk(range(10), 4, 5)
+ []
+ """
+ if not is_sorted:
+ tests = sorted(tests)
+ result_tests = []
+ for i, test in enumerate(tests):
+ if i % modulo == modulo_index:
+ result_tests.append(test)
+ return result_tests
+
+
+def get_splitted_tests(test_entities, modulo, modulo_index, partition_mode, is_sorted=False):
+ if partition_mode == 'SEQUENTIAL':
+ return get_sequential_chunk(test_entities, modulo, modulo_index, is_sorted)
+ elif partition_mode == 'MODULO':
+ return get_shuffled_chunk(test_entities, modulo, modulo_index, is_sorted)
+ else:
+ raise ValueError("detected unknown partition mode: {}".format(partition_mode))
+
+
+def filter_tests_by_modulo(test_classes, modulo, modulo_index, split_by_tests, partition_mode="SEQUENTIAL"):
+ """
+ >>> test_classes = {x: [x] for x in range(20)}
+ >>> filter_tests_by_modulo(test_classes, 4, 0, False)
+ {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]}
+ >>> filter_tests_by_modulo(test_classes, 4, 1, False)
+ {8: [8], 9: [9], 5: [5], 6: [6], 7: [7]}
+ >>> filter_tests_by_modulo(test_classes, 4, 2, False)
+ {10: [10], 11: [11], 12: [12], 13: [13], 14: [14]}
+
+ >>> dict(filter_tests_by_modulo(test_classes, 4, 0, True))
+ {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]}
+ >>> dict(filter_tests_by_modulo(test_classes, 4, 1, True))
+ {8: [8], 9: [9], 5: [5], 6: [6], 7: [7]}
+ """
+ if split_by_tests:
+ tests = get_splitted_tests(flatten_tests(test_classes), modulo, modulo_index, partition_mode)
+ test_classes = collections.defaultdict(list)
+ for class_name, test_name in tests:
+ test_classes[class_name].append(test_name)
+ return test_classes
+ else:
+ target_classes = get_splitted_tests(test_classes, modulo, modulo_index, partition_mode)
+ return {class_name: test_classes[class_name] for class_name in target_classes}
diff --git a/library/python/testing/yatest_lib/tests/test_external.py b/library/python/testing/yatest_lib/tests/test_external.py
new file mode 100644
index 0000000000..18cb560b17
--- /dev/null
+++ b/library/python/testing/yatest_lib/tests/test_external.py
@@ -0,0 +1,20 @@
+import enum
+import pytest
+
+from yatest_lib import external
+
+
+class MyEnum(enum.Enum):
+ VAL1 = 1
+ VAL2 = 2
+
+
+@pytest.mark.parametrize("data, expected_val, expected_type", [
+ ({}, {}, dict),
+ (MyEnum.VAL1, "MyEnum.VAL1", str),
+ ({MyEnum.VAL1: MyEnum.VAL2}, {"MyEnum.VAL1": "MyEnum.VAL2"}, dict),
+])
+def test_serialize(data, expected_val, expected_type):
+ data = external.serialize(data)
+ assert expected_type == type(data), data
+ assert expected_val == data
diff --git a/library/python/testing/yatest_lib/tests/test_testsplitter.py b/library/python/testing/yatest_lib/tests/test_testsplitter.py
new file mode 100644
index 0000000000..394bfe5a74
--- /dev/null
+++ b/library/python/testing/yatest_lib/tests/test_testsplitter.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from yatest_lib import test_splitter
+
+
+def get_chunks(tests, modulo, mode):
+ chunks = []
+ if mode == "MODULO":
+ for modulo_index in range(modulo):
+ chunks.append(test_splitter.get_shuffled_chunk(tests, modulo, modulo_index))
+ elif mode == "SEQUENTIAL":
+ for modulo_index in range(modulo):
+ chunks.append(test_splitter.get_sequential_chunk(tests, modulo, modulo_index))
+ else:
+ raise ValueError("no such mode")
+ return chunks
+
+
+def check_not_intersect(chunk_list):
+ test_set = set()
+ total_size = 0
+ for tests in chunk_list:
+ total_size += len(tests)
+ test_set.update(tests)
+ return total_size == len(test_set)
+
+
+def check_max_diff(chunk_list):
+ return max(map(len, chunk_list)) - min(map(len, chunk_list))
+
+
+def test_lot_of_chunks():
+ for chunk_count in range(10, 20):
+ for tests_count in range(chunk_count):
+ chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert chunks.count([]) == chunk_count - tests_count
+ assert len(chunks) == chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "MODULO")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert chunks.count([]) == chunk_count - tests_count
+ assert len(chunks) == chunk_count
+
+
+def test_lot_of_tests():
+ for tests_count in range(10, 20):
+ for chunk_count in range(2, tests_count):
+ chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "MODULO")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
+
+
+def prime_chunk_count():
+ for chunk_count in [7, 11, 13, 17, 23, 29]:
+ for tests_count in range(chunk_count):
+ chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "MODULO")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
+
+
+def get_divisors(number):
+ divisors = []
+ for d in range(1, number + 1):
+ if number % d == 0:
+ divisors.append(d)
+ return divisors
+
+
+def equal_chunks():
+ for chunk_count in range(12, 31):
+ for tests_count in get_divisors(chunk_count):
+ chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) == 0
+ assert len(chunks) == chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "MODULO")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) == 0
+ assert len(chunks) == chunk_count
+
+
+def chunk_count_equal_tests_count():
+ for chunk_count in range(10, 20):
+ tests_count = chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
+ chunks = get_chunks(range(tests_count), chunk_count, "MODULO")
+ assert check_not_intersect(chunks)
+ assert check_max_diff(chunks) <= 1
+ assert len(chunks) == chunk_count
diff --git a/library/python/testing/yatest_lib/tests/ya.make b/library/python/testing/yatest_lib/tests/ya.make
new file mode 100644
index 0000000000..8586c6ef7d
--- /dev/null
+++ b/library/python/testing/yatest_lib/tests/ya.make
@@ -0,0 +1,14 @@
+OWNER(g:yatest)
+
+PY23_TEST()
+
+PEERDIR(
+ library/python/testing/yatest_lib
+)
+
+TEST_SRCS(
+ test_external.py
+ test_testsplitter.py
+)
+
+END()
diff --git a/library/python/testing/yatest_lib/tools.py b/library/python/testing/yatest_lib/tools.py
new file mode 100644
index 0000000000..b72d79c162
--- /dev/null
+++ b/library/python/testing/yatest_lib/tools.py
@@ -0,0 +1,64 @@
+import six
+import sys
+
+
+def to_utf8(value):
+ """
+ Converts value to string encoded into utf-8
+ :param value:
+ :return:
+ """
+ if sys.version_info[0] < 3:
+ if not isinstance(value, basestring): # noqa
+ value = unicode(value) # noqa
+ if type(value) == str:
+ value = value.decode("utf-8", errors="ignore")
+ return value.encode('utf-8', 'ignore')
+ else:
+ return str(value)
+
+
+def trim_string(s, max_bytes):
+ """
+ Adjusts the length of the string s in order to fit it
+ into max_bytes bytes of storage after encoding as UTF-8.
+ Useful when cutting filesystem paths.
+ :param s: unicode string
+ :param max_bytes: number of bytes
+ :return the prefix of s
+ """
+ if isinstance(s, six.text_type):
+ return _trim_unicode_string(s, max_bytes)
+
+ if isinstance(s, six.binary_type):
+ if len(s) <= max_bytes:
+ return s
+ s = s.decode('utf-8', errors='ignore')
+ s = _trim_unicode_string(s, max_bytes)
+ s = s.encode('utf-8', errors='ignore')
+ return s
+
+ raise TypeError('a string is expected')
+
+
+def _trim_unicode_string(s, max_bytes):
+ if len(s) * 4 <= max_bytes:
+ # UTF-8 uses at most 4 bytes per character
+ return s
+
+ result = []
+ cur_byte_length = 0
+
+ for ch in s:
+ cur_byte_length += len(ch.encode('utf-8'))
+ if cur_byte_length > max_bytes:
+ break
+ result.append(ch)
+
+ return ''.join(result)
+
+
+def to_str(s):
+ if six.PY2 and isinstance(s, six.text_type):
+ return s.encode('utf8')
+ return s
diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make
new file mode 100644
index 0000000000..342bae82ba
--- /dev/null
+++ b/library/python/testing/yatest_lib/ya.make
@@ -0,0 +1,26 @@
+OWNER(g:yatest)
+
+PY23_LIBRARY()
+
+PY_SRCS(
+ NAMESPACE
+ yatest_lib
+ external.py
+ test_splitter.py
+ tools.py
+ ya.py
+)
+
+PEERDIR(
+ contrib/python/six
+)
+
+IF(PYTHON2)
+ PEERDIR(
+ contrib/python/enum34
+ )
+ENDIF()
+
+END()
+
+RECURSE_FOR_TESTS(tests)
diff --git a/library/python/testing/yatest_lib/ya.py b/library/python/testing/yatest_lib/ya.py
new file mode 100644
index 0000000000..c13b58a19f
--- /dev/null
+++ b/library/python/testing/yatest_lib/ya.py
@@ -0,0 +1,239 @@
+import os
+import sys
+import logging
+import json
+
+from .tools import to_str
+from .external import ExternalDataInfo
+
+
+TESTING_OUT_DIR_NAME = "testing_out_stuff" # XXX import from test.const
+
+yatest_logger = logging.getLogger("ya.test")
+
+
+class RunMode(object):
+ Run = "run"
+ List = "list"
+
+
+class TestMisconfigurationException(Exception):
+ pass
+
+
+class Ya(object):
+ """
+ Adds integration with ya, helps in finding dependencies
+ """
+
+ def __init__(
+ self,
+ mode=None,
+ source_root=None,
+ build_root=None,
+ dep_roots=None,
+ output_dir=None,
+ test_params=None,
+ context=None,
+ python_path=None,
+ valgrind_path=None,
+ gdb_path=None,
+ data_root=None,
+ ):
+ context_file_path = os.environ.get("YA_TEST_CONTEXT_FILE", None)
+ if context_file_path:
+ with open(context_file_path, 'r') as afile:
+ test_context = json.load(afile)
+ context_runtime = test_context["runtime"]
+ context_internal = test_context.get("internal", {})
+ context_build = test_context.get("build", {})
+ else:
+ context_runtime = {}
+ context_internal = {}
+ context_build = {}
+ self._mode = mode
+ self._build_root = to_str(context_runtime.get("build_root", "")) or build_root
+ self._source_root = to_str(context_runtime.get("source_root", "")) or source_root or self._detect_source_root()
+ self._output_dir = to_str(context_runtime.get("output_path", "")) or output_dir or self._detect_output_root()
+ if not self._output_dir:
+ raise Exception("Run ya make -t before running test binary")
+ if not self._source_root:
+ logging.warning("Source root was not set neither determined, use --source-root to set it explicitly")
+ if not self._build_root:
+ if self._source_root:
+ self._build_root = self._source_root
+ else:
+ logging.warning("Build root was not set neither determined, use --build-root to set it explicitly")
+
+ if data_root:
+ self._data_root = data_root
+ elif self._source_root:
+ self._data_root = os.path.abspath(os.path.join(self._source_root, "..", "arcadia_tests_data"))
+
+ self._dep_roots = dep_roots
+
+ self._python_path = to_str(context_runtime.get("python_bin", "")) or python_path
+ self._valgrind_path = valgrind_path
+ self._gdb_path = to_str(context_runtime.get("gdb_bin", "")) or gdb_path
+ self._test_params = {}
+ self._context = {}
+ self._test_item_node_id = None
+
+ ram_drive_path = to_str(context_runtime.get("ram_drive_path", ""))
+ if ram_drive_path:
+ self._test_params["ram_drive_path"] = ram_drive_path
+ if test_params:
+ self._test_params.update(dict(x.split('=', 1) for x in test_params))
+ self._test_params.update(context_runtime.get("test_params", {}))
+
+ self._context["project_path"] = context_runtime.get("project_path")
+ self._context["modulo"] = context_runtime.get("split_count", 1)
+ self._context["modulo_index"] = context_runtime.get("split_index", 0)
+ self._context["work_path"] = context_runtime.get("work_path")
+
+ self._context["sanitize"] = context_build.get("sanitizer")
+ self._context["ya_trace_path"] = context_internal.get("trace_file")
+
+ self._env_file = context_internal.get("env_file")
+
+ if context:
+ self._context.update(context)
+
+ @property
+ def source_root(self):
+ return self._source_root
+
+ @property
+ def data_root(self):
+ return self._data_root
+
+ @property
+ def build_root(self):
+ return self._build_root
+
+ @property
+ def dep_roots(self):
+ return self._dep_roots
+
+ @property
+ def output_dir(self):
+ return self._output_dir
+
+ @property
+ def python_path(self):
+ return self._python_path or sys.executable
+
+ @property
+ def valgrind_path(self):
+ if not self._valgrind_path:
+ raise ValueError("path to valgrind was not pass correctly, use --valgrind-path to fix it")
+ return self._valgrind_path
+
+ @property
+ def gdb_path(self):
+ return self._gdb_path
+
+ @property
+ def env_file(self):
+ return self._env_file
+
+ def get_binary(self, *path):
+ assert self._build_root, "Build root was not set neither determined, use --build-root to set it explicitly"
+ path = list(path)
+ if os.name == "nt":
+ if not path[-1].endswith(".exe"):
+ path[-1] += ".exe"
+
+ target_dirs = [self.build_root]
+ # Search for binaries within PATH dirs to be able to get path to the binaries specified by basename for exectests
+ if 'PATH' in os.environ:
+ target_dirs += os.environ['PATH'].split(':')
+
+ for target_dir in target_dirs:
+ binary_path = os.path.join(target_dir, *path)
+ if os.path.exists(binary_path):
+ yatest_logger.debug("Binary was found by %s", binary_path)
+ return binary_path
+
+ error_message = "Cannot find binary '{binary}': make sure it was added in the DEPENDS section".format(binary=path)
+ yatest_logger.debug(error_message)
+ if self._mode == RunMode.Run:
+ raise TestMisconfigurationException(error_message)
+
+ def file(self, path, diff_tool=None, local=False, diff_file_name=None, diff_tool_timeout=None):
+ return ExternalDataInfo.serialize_file(path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
+
+ def get_param(self, key, default=None):
+ return self._test_params.get(key, default)
+
+ def get_param_dict_copy(self):
+ return dict(self._test_params)
+
+ def get_context(self, key):
+ return self._context.get(key)
+
+ def _detect_source_root(self):
+ root = None
+ try:
+ import library.python.find_root
+ # try to determine source root from cwd
+ cwd = os.getcwd()
+ root = library.python.find_root.detect_root(cwd)
+
+ if not root:
+ # try to determine root pretending we are in the test work dir made from --keep-temps run
+ env_subdir = os.path.join("environment", "arcadia")
+ root = library.python.find_root.detect_root(cwd, detector=lambda p: os.path.exists(os.path.join(p, env_subdir)))
+ except ImportError:
+ logging.warning("Unable to import library.python.find_root")
+
+ return root
+
+ def _detect_output_root(self):
+
+ # if run from kept test working dir
+ if os.path.exists(TESTING_OUT_DIR_NAME):
+ return TESTING_OUT_DIR_NAME
+
+ # if run from source dir
+ if sys.version_info.major == 3:
+ test_results_dir = "py3test"
+ else:
+ test_results_dir = "pytest"
+
+ test_results_output_path = os.path.join("test-results", test_results_dir, TESTING_OUT_DIR_NAME)
+ if os.path.exists(test_results_output_path):
+ return test_results_output_path
+
+ if os.path.exists(os.path.dirname(test_results_output_path)):
+ os.mkdir(test_results_output_path)
+ return test_results_output_path
+
+ return None
+
+ def set_test_item_node_id(self, node_id):
+ self._test_item_node_id = node_id
+
+ def get_test_item_node_id(self):
+ assert self._test_item_node_id
+ return self._test_item_node_id
+
+ @property
+ def pytest_config(self):
+ if not hasattr(self, "_pytest_config"):
+ import library.python.pytest.plugins.ya as ya_plugin
+ self._pytest_config = ya_plugin.pytest_config
+ return self._pytest_config
+
+ def set_metric_value(self, name, val):
+ node_id = self.get_test_item_node_id()
+ if node_id not in self.pytest_config.test_metrics:
+ self.pytest_config.test_metrics[node_id] = {}
+
+ self.pytest_config.test_metrics[node_id][name] = val
+
+ def get_metric_value(self, name, default=None):
+ res = self.pytest_config.test_metrics.get(self.get_test_item_node_id(), {}).get(name)
+ if res is None:
+ return default
+ return res
diff --git a/library/python/windows/__init__.py b/library/python/windows/__init__.py
new file mode 100644
index 0000000000..62861b3309
--- /dev/null
+++ b/library/python/windows/__init__.py
@@ -0,0 +1,364 @@
+# coding: utf-8
+
+import os
+import stat
+import sys
+import shutil
+import logging
+
+from six import reraise
+
+import library.python.func
+import library.python.strings
+
+logger = logging.getLogger(__name__)
+
+
+ERRORS = {
+ 'SUCCESS': 0,
+ 'PATH_NOT_FOUND': 3,
+ 'ACCESS_DENIED': 5,
+ 'SHARING_VIOLATION': 32,
+ 'INSUFFICIENT_BUFFER': 122,
+ 'DIR_NOT_EMPTY': 145,
+}
+
+RETRIABLE_FILE_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['SHARING_VIOLATION'])
+RETRIABLE_DIR_ERRORS = (ERRORS['ACCESS_DENIED'], ERRORS['DIR_NOT_EMPTY'], ERRORS['SHARING_VIOLATION'])
+
+
+# Check if on Windows
+@library.python.func.lazy
+def on_win():
+ return os.name == 'nt'
+
+
+class NotOnWindowsError(RuntimeError):
+ def __init__(self, message):
+ super(NotOnWindowsError, self).__init__(message)
+
+
+class DisabledOnWindowsError(RuntimeError):
+ def __init__(self, message):
+ super(DisabledOnWindowsError, self).__init__(message)
+
+
+class NoCTypesError(RuntimeError):
+ def __init__(self, message):
+ super(NoCTypesError, self).__init__(message)
+
+
+# Decorator for Windows-only functions
+def win_only(f):
+ def f_wrapped(*args, **kwargs):
+ if not on_win():
+ raise NotOnWindowsError('Windows-only function is called, but platform is not Windows')
+ return f(*args, **kwargs)
+
+ return f_wrapped
+
+
+# Decorator for functions disabled on Windows
+def win_disabled(f):
+ def f_wrapped(*args, **kwargs):
+ if on_win():
+ run_disabled()
+ return f(*args, **kwargs)
+
+ return f_wrapped
+
+
+def errorfix(f):
+ if not on_win():
+ return f
+
+ def f_wrapped(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except WindowsError:
+ tp, value, tb = sys.exc_info()
+ fix_error(value)
+ reraise(tp, value, tb)
+
+ return f_wrapped
+
+
+# Decorator for diehard wrapper
+# On Windows platform retries to run function while specific WindowsError is thrown
+# On non-Windows platforms fallbacks to function itself
+def diehard(winerrors, tries=100, delay=1):
+ def wrap(f):
+ if not on_win():
+ return f
+
+ return lambda *args, **kwargs: run_diehard(f, winerrors, tries, delay, *args, **kwargs)
+
+ return wrap
+
+
+if on_win():
+ import msvcrt
+ import time
+
+ import library.python.strings
+
+ _has_ctypes = True
+ try:
+ import ctypes
+ from ctypes import wintypes
+ except ImportError:
+ _has_ctypes = False
+
+ _INVALID_HANDLE_VALUE = -1
+
+ _MOVEFILE_REPLACE_EXISTING = 0x1
+ _MOVEFILE_WRITE_THROUGH = 0x8
+
+ _SEM_FAILCRITICALERRORS = 0x1
+ _SEM_NOGPFAULTERRORBOX = 0x2
+ _SEM_NOALIGNMENTFAULTEXCEPT = 0x4
+ _SEM_NOOPENFILEERRORBOX = 0x8
+
+ _SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1
+
+ _CREATE_NO_WINDOW = 0x8000000
+
+ _ATOMIC_RENAME_FILE_TRANSACTION_DEFAULT_TIMEOUT = 1000
+
+ _HANDLE_FLAG_INHERIT = 0x1
+
+ @win_only
+ def require_ctypes(f):
+ def f_wrapped(*args, **kwargs):
+ if not _has_ctypes:
+ raise NoCTypesError('No ctypes found')
+ return f(*args, **kwargs)
+
+ return f_wrapped
+
+ # Run function in diehard mode (see diehard decorator commentary)
+ @win_only
+ def run_diehard(f, winerrors, tries, delay, *args, **kwargs):
+ if isinstance(winerrors, int):
+ winerrors = (winerrors,)
+
+ ei = None
+ for t in xrange(tries):
+ if t:
+ logger.debug('Diehard [errs %s]: try #%d in %s', ','.join(str(x) for x in winerrors), t, f)
+ try:
+ return f(*args, **kwargs)
+ except WindowsError as e:
+ if e.winerror not in winerrors:
+ raise
+ ei = sys.exc_info()
+ time.sleep(delay)
+ reraise(ei[0], ei[1], ei[2])
+
+ # Placeholder for disabled functions
+ @win_only
+ def run_disabled(*args, **kwargs):
+ raise DisabledOnWindowsError('Function called is disabled on Windows')
+
+ class CustomWinError(WindowsError):
+ def __init__(self, winerror, message='', filename=None):
+ super(CustomWinError, self).__init__(winerror, message)
+ self.message = message
+ self.strerror = self.message if self.message else format_error(self.windows_error)
+ self.filename = filename
+ self.utf8 = True
+
+ @win_only
+ def unicode_path(path):
+ return library.python.strings.to_unicode(path, library.python.strings.fs_encoding())
+
+ @win_only
+ @require_ctypes
+ def format_error(error):
+ if isinstance(error, WindowsError):
+ error = error.winerror
+ if not isinstance(error, int):
+ return 'Unknown'
+ return ctypes.FormatError(error)
+
+ @win_only
+ def fix_error(windows_error):
+ if not windows_error.strerror:
+ windows_error.strerror = format_error(windows_error)
+ transcode_error(windows_error)
+
+ @win_only
+ def transcode_error(windows_error, to_enc='utf-8'):
+ from_enc = 'utf-8' if getattr(windows_error, 'utf8', False) else library.python.strings.guess_default_encoding()
+ if from_enc != to_enc:
+ windows_error.strerror = library.python.strings.to_str(windows_error.strerror, to_enc=to_enc, from_enc=from_enc)
+ setattr(windows_error, 'utf8', to_enc == 'utf-8')
+
+ class Transaction(object):
+ def __init__(self, timeout=None, description=''):
+ self.timeout = timeout
+ self.description = description
+
+ @require_ctypes
+ def __enter__(self):
+ self._handle = ctypes.windll.ktmw32.CreateTransaction(None, 0, 0, 0, 0, self.timeout, self.description)
+ if self._handle == _INVALID_HANDLE_VALUE:
+ raise ctypes.WinError()
+ return self._handle
+
+ @require_ctypes
+ def __exit__(self, t, v, tb):
+ try:
+ if not ctypes.windll.ktmw32.CommitTransaction(self._handle):
+ raise ctypes.WinError()
+ finally:
+ ctypes.windll.kernel32.CloseHandle(self._handle)
+
+ @win_only
+ def file_handle(f):
+ return msvcrt.get_osfhandle(f.fileno())
+
+ # https://www.python.org/dev/peps/pep-0446/
+ # http://mihalop.blogspot.ru/2014/05/python-subprocess-and-file-descriptors.html
+ @require_ctypes
+ @win_only
+ def open_file(*args, **kwargs):
+ f = open(*args, **kwargs)
+ ctypes.windll.kernel32.SetHandleInformation(file_handle(f), _HANDLE_FLAG_INHERIT, 0)
+ return f
+
+ @win_only
+ @require_ctypes
+ def replace_file(src, dst):
+ if not ctypes.windll.kernel32.MoveFileExW(unicode_path(src), unicode_path(dst), _MOVEFILE_REPLACE_EXISTING | _MOVEFILE_WRITE_THROUGH):
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def replace_file_across_devices(src, dst):
+ with Transaction(timeout=_ATOMIC_RENAME_FILE_TRANSACTION_DEFAULT_TIMEOUT, description='ya library.python.windows replace_file_across_devices') as transaction:
+ if not ctypes.windll.kernel32.MoveFileTransactedW(unicode_path(src), unicode_path(dst), None, None, _MOVEFILE_REPLACE_EXISTING | _MOVEFILE_WRITE_THROUGH, transaction):
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def hardlink(src, lnk):
+ if not ctypes.windll.kernel32.CreateHardLinkW(unicode_path(lnk), unicode_path(src), None):
+ raise ctypes.WinError()
+
+ # Requires SE_CREATE_SYMBOLIC_LINK_NAME privilege
+ @win_only
+ @win_disabled
+ @require_ctypes
+ def symlink_file(src, lnk):
+ if not ctypes.windll.kernel32.CreateSymbolicLinkW(unicode_path(lnk), unicode_path(src), 0):
+ raise ctypes.WinError()
+
+ # Requires SE_CREATE_SYMBOLIC_LINK_NAME privilege
+ @win_only
+ @win_disabled
+ @require_ctypes
+ def symlink_dir(src, lnk):
+ if not ctypes.windll.kernel32.CreateSymbolicLinkW(unicode_path(lnk), unicode_path(src), _SYMBOLIC_LINK_FLAG_DIRECTORY):
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def lock_file(f, offset, length, raises=True):
+ locked = ctypes.windll.kernel32.LockFile(file_handle(f), _low_dword(offset), _high_dword(offset), _low_dword(length), _high_dword(length))
+ if not raises:
+ return bool(locked)
+ if not locked:
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def unlock_file(f, offset, length, raises=True):
+ unlocked = ctypes.windll.kernel32.UnlockFile(file_handle(f), _low_dword(offset), _high_dword(offset), _low_dword(length), _high_dword(length))
+ if not raises:
+ return bool(unlocked)
+ if not unlocked:
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def set_error_mode(mode):
+ return ctypes.windll.kernel32.SetErrorMode(mode)
+
+ @win_only
+ def rmtree(path):
+ def error_handler(func, handling_path, execinfo):
+ e = execinfo[1]
+ if e.winerror == ERRORS['PATH_NOT_FOUND']:
+ handling_path = "\\\\?\\" + handling_path # handle path over 256 symbols
+ if os.path.exists(path):
+ return func(handling_path)
+ if e.winerror == ERRORS['ACCESS_DENIED']:
+ try:
+ # removing of r/w directory with read-only files in it yields ACCESS_DENIED
+ # which is not an insuperable obstacle https://bugs.python.org/issue19643
+ os.chmod(handling_path, stat.S_IWRITE)
+ except OSError:
+ pass
+ else:
+ # propagate true last error if this attempt fails
+ return func(handling_path)
+ raise e
+ shutil.rmtree(path, onerror=error_handler)
+
+ # Don't display the Windows GPF dialog if the invoked program dies.
+ # http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx
+ @win_only
+ def disable_error_dialogs():
+ set_error_mode(_SEM_NOGPFAULTERRORBOX | _SEM_FAILCRITICALERRORS)
+
+ @win_only
+ def default_process_creation_flags():
+ return 0
+
+ @require_ctypes
+ def _low_dword(x):
+ return ctypes.c_ulong(x & ((1 << 32) - 1))
+
+ @require_ctypes
+ def _high_dword(x):
+ return ctypes.c_ulong((x >> 32) & ((1 << 32) - 1))
+
+ @win_only
+ @require_ctypes
+ def get_current_process():
+ handle = ctypes.windll.kernel32.GetCurrentProcess()
+ if not handle:
+ raise ctypes.WinError()
+ return wintypes.HANDLE(handle)
+
+ @win_only
+ @require_ctypes
+ def get_process_handle_count(proc_handle):
+ assert isinstance(proc_handle, wintypes.HANDLE)
+
+ GetProcessHandleCount = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HANDLE, wintypes.POINTER(wintypes.DWORD))(("GetProcessHandleCount", ctypes.windll.kernel32))
+ hndcnt = wintypes.DWORD()
+ if not GetProcessHandleCount(proc_handle, ctypes.byref(hndcnt)):
+ raise ctypes.WinError()
+ return hndcnt.value
+
+ @win_only
+ @require_ctypes
+ def set_handle_information(file, inherit=None, protect_from_close=None):
+ for flag, value in [(inherit, 1), (protect_from_close, 2)]:
+ if flag is not None:
+ assert isinstance(flag, bool)
+ if not ctypes.windll.kernel32.SetHandleInformation(file_handle(file), _low_dword(value), _low_dword(int(flag))):
+ raise ctypes.WinError()
+
+ @win_only
+ @require_ctypes
+ def get_windows_directory():
+ buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
+ size = ctypes.windll.kernel32.GetWindowsDirectoryW(buf, ctypes.wintypes.MAX_PATH)
+ if not size:
+ raise ctypes.WinError()
+ if size > ctypes.wintypes.MAX_PATH - 1:
+ raise CustomWinError(ERRORS['INSUFFICIENT_BUFFER'])
+ return ctypes.wstring_at(buf, size)
diff --git a/library/python/windows/ut/test_windows.py b/library/python/windows/ut/test_windows.py
new file mode 100644
index 0000000000..bef3ec2dc5
--- /dev/null
+++ b/library/python/windows/ut/test_windows.py
@@ -0,0 +1,96 @@
+# coding=utf-8
+
+import errno
+import os
+import pytest
+
+import library.python.strings
+import library.python.windows
+
+
+def gen_error_access_denied():
+ if library.python.windows.on_win():
+ err = WindowsError()
+ err.errno = errno.EACCES
+ err.strerror = ''
+ err.winerror = library.python.windows.ERRORS['ACCESS_DENIED']
+ else:
+ err = OSError()
+ err.errno = errno.EACCES
+ err.strerror = os.strerror(err.errno)
+ err.filename = 'unknown/file'
+ raise err
+
+
+def test_errorfix_buggy():
+ @library.python.windows.errorfix
+ def erroneous_func():
+ gen_error_access_denied()
+
+ with pytest.raises(OSError) as errinfo:
+ erroneous_func()
+ assert errinfo.value.errno == errno.EACCES
+ assert errinfo.value.filename == 'unknown/file'
+ assert isinstance(errinfo.value.strerror, basestring)
+ assert errinfo.value.strerror
+
+
+def test_errorfix_explicit():
+ @library.python.windows.errorfix
+ def erroneous_func():
+ if library.python.windows.on_win():
+ err = WindowsError()
+ err.winerror = library.python.windows.ERRORS['ACCESS_DENIED']
+ else:
+ err = OSError()
+ err.errno = errno.EACCES
+ err.strerror = 'Some error description'
+ err.filename = 'unknown/file'
+ raise err
+
+ with pytest.raises(OSError) as errinfo:
+ erroneous_func()
+ assert errinfo.value.errno == errno.EACCES
+ assert errinfo.value.filename == 'unknown/file'
+ assert errinfo.value.strerror == 'Some error description'
+
+
+def test_errorfix_decoding_cp1251():
+ @library.python.windows.errorfix
+ def erroneous_func():
+ model_msg = u'Какое-то описание ошибки'
+ if library.python.windows.on_win():
+ err = WindowsError()
+ err.strerror = library.python.strings.to_str(model_msg, 'cp1251')
+ else:
+ err = OSError()
+ err.strerror = library.python.strings.to_str(model_msg)
+ raise err
+
+ with pytest.raises(OSError) as errinfo:
+ erroneous_func()
+ error_msg = errinfo.value.strerror
+ if not isinstance(errinfo.value.strerror, unicode):
+ error_msg = library.python.strings.to_unicode(error_msg)
+ assert error_msg == u'Какое-то описание ошибки'
+
+
+def test_diehard():
+ @library.python.windows.diehard(library.python.windows.ERRORS['ACCESS_DENIED'], tries=5)
+ def erroneous_func(errors):
+ try:
+ gen_error_access_denied()
+ except Exception as e:
+ errors.append(e)
+ raise
+
+ raised_errors = []
+ with pytest.raises(OSError) as errinfo:
+ erroneous_func(raised_errors)
+ assert errinfo.value.errno == errno.EACCES
+ assert any(e.errno == errno.EACCES for e in raised_errors)
+ assert raised_errors and errinfo.value == raised_errors[-1]
+ if library.python.windows.on_win():
+ assert len(raised_errors) == 5
+ else:
+ assert len(raised_errors) == 1
diff --git a/library/python/windows/ut/ya.make b/library/python/windows/ut/ya.make
new file mode 100644
index 0000000000..c39f1797b8
--- /dev/null
+++ b/library/python/windows/ut/ya.make
@@ -0,0 +1,11 @@
+OWNER(g:yatool)
+
+PY2TEST()
+
+TEST_SRCS(test_windows.py)
+
+PEERDIR(
+ library/python/windows
+)
+
+END()
diff --git a/library/python/windows/ya.make b/library/python/windows/ya.make
new file mode 100644
index 0000000000..e17f86b67e
--- /dev/null
+++ b/library/python/windows/ya.make
@@ -0,0 +1,13 @@
+OWNER(g:yatool)
+
+PY23_LIBRARY()
+
+PY_SRCS(__init__.py)
+
+PEERDIR(
+ library/python/func
+ library/python/strings
+ contrib/python/six
+)
+
+END()
diff --git a/library/python/ya.make b/library/python/ya.make
new file mode 100644
index 0000000000..2e1eb6e0e1
--- /dev/null
+++ b/library/python/ya.make
@@ -0,0 +1,222 @@
+OWNER(g:python-contrib)
+
+RECURSE(
+ aho_corasick
+ aho_corasick/ut
+ archive
+ archive/benchmark
+ archive/test
+ archive/test/data
+ asgi_yauth
+ async_clients
+ auth_client_parser
+ awssdk-extensions
+ awssdk_async_extensions
+ base64
+ base64/test
+ bclclient
+ blackbox
+ blackbox/tests
+ blackbox/tvm2
+ bloom
+ boost_test
+ bstr
+ build_info
+ build_info/ut
+ capabilities
+ celery_dashboard
+ certifi
+ cgroups
+ charset
+ charts_notes
+ charts_notes/example
+ cityhash
+ cityhash/test
+ clickhouse_client
+ cmain
+ codecs
+ codecs/gen_corpus
+ codecs/test
+ compress
+ compress/tests
+ cookiemy
+ coredump_filter
+ cores
+ coverage
+ cpp_test
+ cppdemangle
+ cqueue
+ crowd-kit
+ cyson
+ cyson/pymodule
+ cyson/ut
+ deploy_formatter
+ deprecated
+ dir-sync
+ django
+ django/example
+ django-idm-api
+ django-multic
+ django-sform
+ django-sform/tests
+ django_alive
+ django_celery_monitoring
+ django_russian
+ django_template_common
+ django_tools_log_context
+ dssclient
+ dump_dict
+ edit_distance
+ errorboosterclient
+ filelock
+ filelock/ut
+ filesys
+ filesys/ut
+ find_root
+ flask
+ flask_passport
+ fnvhash
+ fnvhash/test
+ framing
+ framing/ut
+ func
+ func/ut
+ fs
+ geolocation
+ geolocation/ut
+ geohash
+ geohash/ut
+ golovan_stats_aggregator
+ granular_settings
+ granular_settings/tests
+ guid
+ guid/test
+ guid/at_fork_test
+ gunicorn
+ hnsw
+ ids
+ import_test
+ infected_masks
+ infected_masks/ut
+ init_log
+ init_log/example
+ intrasearch_fetcher
+ json
+ json/compare
+ json/perf
+ json/test
+ json/test/py2
+ json/test/py3
+ langdetect
+ langmask
+ langs
+ luigi
+ luigi/data
+ luigi/example
+ luigi/luigid_static
+ maths
+ messagebus
+ metrics_framework
+ mime_types
+ monitoring
+ monlib
+ monlib/examples
+ monlib/so
+ murmurhash
+ nirvana
+ nirvana_api
+ nirvana_api/test_lib
+ nirvana_test
+ nstools
+ nyt
+ oauth
+ oauth/example
+ ok_client
+ openssl
+ par_apply
+ par_apply/test
+ path
+ path/tests
+ protobuf
+ pymain
+ pyscopg2
+ pytest
+ pytest-mongodb
+ pytest/allure
+ pytest/empty
+ pytest/plugins
+ python-blackboxer
+ python-django-tanker
+ python-django-yauth/tests
+ python-django-yauth
+ reactor
+ redis_utils
+ reservoir_sampling
+ refsclient
+ resource
+ retry
+ retry/tests
+ runtime
+ runtime/main
+ runtime/test
+ runtime_py3
+ runtime_py3/main
+ runtime_py3/test
+ runtime_test
+ sanitizers
+ sdms_api
+ sfx
+ selenium_ui_test
+ sendmsg
+ stubmaker
+ solomon
+ spack
+ spyt
+ ssh_client
+ ssh_sign
+ startrek_python_client
+ startrek_python_client/tests_int
+ statface_client
+ step
+ strings
+ strings/ut
+ svn_ssh
+ svn_version
+ svn_version/ut
+ symbols
+ testing
+ tmp
+ toloka_client
+ toloka-kit
+ toloka-airflow
+ toloka-prefect
+ tools_structured_logs
+ thread
+ thread/test
+ tskv
+ tvmauth
+ tvm2
+ tvm2/tests
+ type_info
+ type_info/test
+ unique_id
+ vault_client
+ watch_dog
+ watch_dog/example
+ wiki
+ windows
+ windows/ut
+ yandex_tracker_client
+ yenv
+ yt
+ yt/test
+ ylock
+ ylock/tests
+ zipatch
+)
+
+IF (NOT MUSL)
+ RECURSE(
+ yt/example
+ )
+ENDIF()