aboutsummaryrefslogtreecommitdiffstats
path: root/library/python/runtime_py3
diff options
context:
space:
mode:
authorsay <say@yandex-team.ru>2022-02-10 16:48:19 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:48:19 +0300
commit2096e85a73bb6b3b20ae25a92943992717fe4167 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8 /library/python/runtime_py3
parenta6a6f6e1e77c7d7d0cdfad61c093e061d6fb5782 (diff)
downloadydb-2096e85a73bb6b3b20ae25a92943992717fe4167.tar.gz
Restoring authorship annotation for <say@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/python/runtime_py3')
-rw-r--r--library/python/runtime_py3/importer.pxi432
-rw-r--r--library/python/runtime_py3/test/test_arcadia_source_finder.py634
-rw-r--r--library/python/runtime_py3/test/ya.make12
3 files changed, 539 insertions, 539 deletions
diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi
index 8d8b523d4c..904f94dea2 100644
--- a/library/python/runtime_py3/importer.pxi
+++ b/library/python/runtime_py3/importer.pxi
@@ -2,7 +2,7 @@ import marshal
import sys
from _codecs import utf_8_decode, utf_8_encode
from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter
-from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split
+from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split
from _io import FileIO
import __res as __resource
@@ -11,9 +11,9 @@ _b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0]
_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0]
env_entry_point = b'Y_PYTHON_ENTRY_POINT'
env_source_root = b'Y_PYTHON_SOURCE_ROOT'
-cfg_source_root = b'arcadia-source-root'
-env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH'
-res_ya_ide_venv = b'YA_IDE_VENV'
+cfg_source_root = b'arcadia-source-root'
+env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH'
+res_ya_ide_venv = b'YA_IDE_VENV'
executable = sys.executable or 'Y_PYTHON'
sys.modules['run_import_hook'] = __resource
@@ -21,59 +21,59 @@ sys.modules['run_import_hook'] = __resource
py_prefix = b'py/'
py_prefix_len = len(py_prefix)
-YA_IDE_VENV = __resource.find(res_ya_ide_venv)
-Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV
-
-
-def _init_venv():
- if not _path_isabs(executable):
- raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable))
-
- # Creative copy-paste from site.py
- exe_dir, _ = _path_split(executable)
- site_prefix, _ = _path_split(exe_dir)
- libpath = _path_join(site_prefix, 'lib',
- 'python%d.%d' % sys.version_info[:2],
- 'site-packages')
- sys.path.insert(0, libpath)
-
- # emulate site.venv()
- sys.prefix = site_prefix
- sys.exec_prefix = site_prefix
-
- conf_basename = 'pyvenv.cfg'
- candidate_confs = [
- conffile for conffile in (
- _path_join(exe_dir, conf_basename),
- _path_join(site_prefix, conf_basename)
- )
- if _path_isfile(conffile)
- ]
- if not candidate_confs:
- raise RuntimeError('{} not found'.format(conf_basename))
- virtual_conf = candidate_confs[0]
- with FileIO(virtual_conf, 'r') as f:
- for line in f:
- if b'=' in line:
- key, _, value = line.partition(b'=')
- key = key.strip().lower()
- value = value.strip()
- if key == cfg_source_root:
- return value
- raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf))
-
-
-def _get_source_root():
- env_value = _os.environ.get(env_source_root)
- if env_value or not YA_IDE_VENV:
- return env_value
-
- return _init_venv()
-
-
-Y_PYTHON_SOURCE_ROOT = _get_source_root()
-
-
+YA_IDE_VENV = __resource.find(res_ya_ide_venv)
+Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV
+
+
+def _init_venv():
+ if not _path_isabs(executable):
+ raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable))
+
+ # Creative copy-paste from site.py
+ exe_dir, _ = _path_split(executable)
+ site_prefix, _ = _path_split(exe_dir)
+ libpath = _path_join(site_prefix, 'lib',
+ 'python%d.%d' % sys.version_info[:2],
+ 'site-packages')
+ sys.path.insert(0, libpath)
+
+ # emulate site.venv()
+ sys.prefix = site_prefix
+ sys.exec_prefix = site_prefix
+
+ conf_basename = 'pyvenv.cfg'
+ candidate_confs = [
+ conffile for conffile in (
+ _path_join(exe_dir, conf_basename),
+ _path_join(site_prefix, conf_basename)
+ )
+ if _path_isfile(conffile)
+ ]
+ if not candidate_confs:
+ raise RuntimeError('{} not found'.format(conf_basename))
+ virtual_conf = candidate_confs[0]
+ with FileIO(virtual_conf, 'r') as f:
+ for line in f:
+ if b'=' in line:
+ key, _, value = line.partition(b'=')
+ key = key.strip().lower()
+ value = value.strip()
+ if key == cfg_source_root:
+ return value
+ raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf))
+
+
+def _get_source_root():
+ env_value = _os.environ.get(env_source_root)
+ if env_value or not YA_IDE_VENV:
+ return env_value
+
+ return _init_venv()
+
+
+Y_PYTHON_SOURCE_ROOT = _get_source_root()
+
+
def _print(*xs):
"""
This is helpful for debugging, since automatic bytes to str conversion is
@@ -181,10 +181,10 @@ class ResourceImporter(object):
self.source_map = {} # Map from file names to module names.
self._source_name = {} # Map from original to altered module names.
self._package_prefix = ''
- if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH:
- self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT))
- else:
- self.arcadia_source_finder = None
+ if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH:
+ self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT))
+ else:
+ self.arcadia_source_finder = None
for p in list(self.memory) + list(sys.builtin_module_names):
for pp in iter_prefixes(p):
@@ -198,13 +198,13 @@ class ResourceImporter(object):
importer._package_prefix = name + '.'
return importer
- def _find_mod_path(self, fullname):
- """Find arcadia relative path by module name"""
- relpath = resfs_src(mod_path(fullname), resfs_file=True)
- if relpath or not self.arcadia_source_finder:
- return relpath
- return self.arcadia_source_finder.get_module_path(fullname)
-
+ def _find_mod_path(self, fullname):
+ """Find arcadia relative path by module name"""
+ relpath = resfs_src(mod_path(fullname), resfs_file=True)
+ if relpath or not self.arcadia_source_finder:
+ return relpath
+ return self.arcadia_source_finder.get_module_path(fullname)
+
def find_spec(self, fullname, path=None, target=None):
try:
is_package = self.is_package(fullname)
@@ -245,7 +245,7 @@ class ResourceImporter(object):
modname = fullname
if self.is_package(fullname):
fullname += '.__init__'
- relpath = self._find_mod_path(fullname)
+ relpath = self._find_mod_path(fullname)
if isinstance(relpath, bytes):
relpath = _s(relpath)
return relpath or modname
@@ -272,7 +272,7 @@ class ResourceImporter(object):
fullname += '.__init__'
path = mod_path(fullname)
- relpath = self._find_mod_path(fullname)
+ relpath = self._find_mod_path(fullname)
if relpath:
abspath = resfs_resolve(relpath)
if abspath:
@@ -298,9 +298,9 @@ class ResourceImporter(object):
if fullname + '.__init__' in self.memory:
return True
- if self.arcadia_source_finder:
- return self.arcadia_source_finder.is_package(fullname)
-
+ if self.arcadia_source_finder:
+ return self.arcadia_source_finder.is_package(fullname)
+
raise ImportError(fullname)
# Extension for contrib/python/coverage.
@@ -329,9 +329,9 @@ class ResourceImporter(object):
m = rx.match(p)
if m:
yield prefix + m.group(1), m.group(2) is not None
- if self.arcadia_source_finder:
- for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix):
- yield m
+ if self.arcadia_source_finder:
+ for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix):
+ yield m
def get_resource_reader(self, fullname):
try:
@@ -394,134 +394,134 @@ class BuiltinSubmoduleImporter(BuiltinImporter):
return None
-class ArcadiaSourceFinder:
- """
- Search modules and packages in arcadia source tree.
- See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details
- """
- NAMESPACE_PREFIX = b'py/namespace/'
- PY_EXT = '.py'
- YA_MAKE = 'ya.make'
-
- def __init__(self, source_root):
- self.source_root = source_root
- self.module_path_cache = {'': set()}
- for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX):
- # dirty_path contains unique prefix to prevent repeatable keys in the resource storage
- path = dirty_path.split(b'/', 1)[1]
- namespaces = __resource.find(key).split(b':')
- for n in namespaces:
- package_name = _s(n.rstrip(b'.'))
- self.module_path_cache.setdefault(package_name, set()).add(_s(path))
- # Fill parents with default empty path set if parent doesn't exist in the cache yet
- while package_name:
- package_name = package_name.rpartition('.')[0]
- if package_name in self.module_path_cache:
- break
- self.module_path_cache.setdefault(package_name, set())
- for package_name in self.module_path_cache.keys():
- self._add_parent_dirs(package_name, visited=set())
-
- def get_module_path(self, fullname):
- """
- Find file path for module 'fullname'.
- For packages caller pass fullname as 'package.__init__'.
- Return None if nothing is found.
- """
- try:
- if not self.is_package(fullname):
- return _b(self._cache_module_path(fullname))
- except ImportError:
- pass
-
- def is_package(self, fullname):
- """Check if fullname is a package. Raise ImportError if fullname is not found"""
- path = self._cache_module_path(fullname)
- if isinstance(path, set):
- return True
- if isinstance(path, str):
- return False
- raise ImportError(fullname)
-
- def iter_modules(self, package_prefix, prefix):
- paths = self._cache_module_path(package_prefix.rstrip('.'))
- if paths is not None:
- # Note: it's ok to yield duplicates because pkgutil discards them
-
- # Yield from cache
- import re
- rx = re.compile(re.escape(package_prefix) + r'([^.]+)$')
- for mod, path in self.module_path_cache.items():
- if path is not None:
- m = rx.match(mod)
- if m:
- yield prefix + m.group(1), self.is_package(mod)
-
- # Yield from file system
- for path in paths:
- abs_path = _path_join(self.source_root, path)
- for dir_item in _os.listdir(abs_path):
- if self._path_is_simple_dir(_path_join(abs_path, dir_item)):
- yield prefix + dir_item, True
- elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)):
- yield prefix + dir_item[:-len(self.PY_EXT)], False
-
- def _path_is_simple_dir(self, abs_path):
- """
- Check if path is a directory but doesn't contain ya.make file.
- We don't want to steal directory from nested project and treat it as a package
- """
- return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE))
-
- def _find_module_in_paths(self, find_package_only, paths, module):
- """Auxiliary method. See _cache_module_path() for details"""
- if paths:
- package_paths = set()
- for path in paths:
- rel_path = _path_join(path, module)
- if not find_package_only:
- # Check if file_path is a module
- module_path = rel_path + self.PY_EXT
- if _path_isfile(_path_join(self.source_root, module_path)):
- return module_path
- # Check if file_path is a package
- if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
- package_paths.add(rel_path)
- if package_paths:
- return package_paths
-
- def _cache_module_path(self, fullname, find_package_only=False):
- """
- Find module path or package directory paths and save result in the cache
-
- find_package_only=True - don't try to find module
-
- Returns:
- List of relative package paths - for a package
- Relative module path - for a module
- None - module or package is not found
+class ArcadiaSourceFinder:
+ """
+ Search modules and packages in arcadia source tree.
+ See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details
+ """
+ NAMESPACE_PREFIX = b'py/namespace/'
+ PY_EXT = '.py'
+ YA_MAKE = 'ya.make'
+
+ def __init__(self, source_root):
+ self.source_root = source_root
+ self.module_path_cache = {'': set()}
+ for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX):
+ # dirty_path contains unique prefix to prevent repeatable keys in the resource storage
+ path = dirty_path.split(b'/', 1)[1]
+ namespaces = __resource.find(key).split(b':')
+ for n in namespaces:
+ package_name = _s(n.rstrip(b'.'))
+ self.module_path_cache.setdefault(package_name, set()).add(_s(path))
+ # Fill parents with default empty path set if parent doesn't exist in the cache yet
+ while package_name:
+ package_name = package_name.rpartition('.')[0]
+ if package_name in self.module_path_cache:
+ break
+ self.module_path_cache.setdefault(package_name, set())
+ for package_name in self.module_path_cache.keys():
+ self._add_parent_dirs(package_name, visited=set())
+
+ def get_module_path(self, fullname):
+ """
+ Find file path for module 'fullname'.
+ For packages caller pass fullname as 'package.__init__'.
+ Return None if nothing is found.
+ """
+ try:
+ if not self.is_package(fullname):
+ return _b(self._cache_module_path(fullname))
+ except ImportError:
+ pass
+
+ def is_package(self, fullname):
+ """Check if fullname is a package. Raise ImportError if fullname is not found"""
+ path = self._cache_module_path(fullname)
+ if isinstance(path, set):
+ return True
+ if isinstance(path, str):
+ return False
+ raise ImportError(fullname)
+
+ def iter_modules(self, package_prefix, prefix):
+ paths = self._cache_module_path(package_prefix.rstrip('.'))
+ if paths is not None:
+ # Note: it's ok to yield duplicates because pkgutil discards them
+
+ # Yield from cache
+ import re
+ rx = re.compile(re.escape(package_prefix) + r'([^.]+)$')
+ for mod, path in self.module_path_cache.items():
+ if path is not None:
+ m = rx.match(mod)
+ if m:
+ yield prefix + m.group(1), self.is_package(mod)
+
+ # Yield from file system
+ for path in paths:
+ abs_path = _path_join(self.source_root, path)
+ for dir_item in _os.listdir(abs_path):
+ if self._path_is_simple_dir(_path_join(abs_path, dir_item)):
+ yield prefix + dir_item, True
+ elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)):
+ yield prefix + dir_item[:-len(self.PY_EXT)], False
+
+ def _path_is_simple_dir(self, abs_path):
+ """
+ Check if path is a directory but doesn't contain ya.make file.
+ We don't want to steal directory from nested project and treat it as a package
+ """
+ return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE))
+
+ def _find_module_in_paths(self, find_package_only, paths, module):
+ """Auxiliary method. See _cache_module_path() for details"""
+ if paths:
+ package_paths = set()
+ for path in paths:
+ rel_path = _path_join(path, module)
+ if not find_package_only:
+ # Check if file_path is a module
+ module_path = rel_path + self.PY_EXT
+ if _path_isfile(_path_join(self.source_root, module_path)):
+ return module_path
+ # Check if file_path is a package
+ if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
+ package_paths.add(rel_path)
+ if package_paths:
+ return package_paths
+
+ def _cache_module_path(self, fullname, find_package_only=False):
"""
- if fullname not in self.module_path_cache:
- parent, _, tail = fullname.rpartition('.')
- parent_paths = self._cache_module_path(parent, find_package_only=True)
- self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail)
- return self.module_path_cache[fullname]
-
- def _add_parent_dirs(self, package_name, visited):
- if not package_name or package_name in visited:
- return
- visited.add(package_name)
-
- parent, _, tail = package_name.rpartition('.')
- self._add_parent_dirs(parent, visited)
-
- paths = self.module_path_cache[package_name]
- for parent_path in self.module_path_cache[parent]:
- rel_path = _path_join(parent_path, tail)
- if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
- paths.add(rel_path)
-
-
+ Find module path or package directory paths and save result in the cache
+
+ find_package_only=True - don't try to find module
+
+ Returns:
+ List of relative package paths - for a package
+ Relative module path - for a module
+ None - module or package is not found
+ """
+ if fullname not in self.module_path_cache:
+ parent, _, tail = fullname.rpartition('.')
+ parent_paths = self._cache_module_path(parent, find_package_only=True)
+ self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail)
+ return self.module_path_cache[fullname]
+
+ def _add_parent_dirs(self, package_name, visited):
+ if not package_name or package_name in visited:
+ return
+ visited.add(package_name)
+
+ parent, _, tail = package_name.rpartition('.')
+ self._add_parent_dirs(parent, visited)
+
+ paths = self.module_path_cache[package_name]
+ for parent_path in self.module_path_cache[parent]:
+ rel_path = _path_join(parent_path, tail)
+ if self._path_is_simple_dir(_path_join(self.source_root, rel_path)):
+ paths.add(rel_path)
+
+
def excepthook(*args, **kws):
# traceback module cannot be imported at module level, because interpreter
# is not fully initialized yet
@@ -544,19 +544,19 @@ def executable_path_hook(path):
raise ImportError(path)
-if YA_IDE_VENV:
- sys.meta_path.append(importer)
- sys.meta_path.append(BuiltinSubmoduleImporter)
- if executable not in sys.path:
- sys.path.append(executable)
- sys.path_hooks.append(executable_path_hook)
-else:
- sys.meta_path.insert(0, BuiltinSubmoduleImporter)
- sys.meta_path.insert(0, importer)
- if executable not in sys.path:
- sys.path.insert(0, executable)
- sys.path_hooks.insert(0, executable_path_hook)
-
+if YA_IDE_VENV:
+ sys.meta_path.append(importer)
+ sys.meta_path.append(BuiltinSubmoduleImporter)
+ if executable not in sys.path:
+ sys.path.append(executable)
+ sys.path_hooks.append(executable_path_hook)
+else:
+ sys.meta_path.insert(0, BuiltinSubmoduleImporter)
+ sys.meta_path.insert(0, importer)
+ if executable not in sys.path:
+ sys.path.insert(0, executable)
+ sys.path_hooks.insert(0, executable_path_hook)
+
sys.path_importer_cache[executable] = importer
# Indicator that modules and resources are built-in rather than on the file system.
diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py
index b45f77a910..ff80d0a0a2 100644
--- a/library/python/runtime_py3/test/test_arcadia_source_finder.py
+++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py
@@ -1,317 +1,317 @@
-import unittest
-import yaml
-from unittest.mock import patch
-from parameterized import parameterized
-
-import __res as res
-
-
-NAMESPACE_PREFIX = b'py/namespace/'
-TEST_SOURCE_ROOT = '/home/arcadia'
-
-
-class ImporterMocks(object):
- def __init__(self, mock_fs, mock_resources):
- self._mock_fs = mock_fs
- self._mock_resources = mock_resources
- self._patchers = [
- patch('__res.iter_keys', wraps=self._iter_keys),
- patch('__res.__resource.find', wraps=self._resource_find),
- patch('__res._path_isdir', wraps=self._path_isdir),
- patch('__res._path_isfile', wraps=self._path_isfile),
- patch('__res._os.listdir', wraps=self._os_listdir),
- ]
- for patcher in self._patchers:
- patcher.start()
-
- def stop(self):
- for patcher in self._patchers:
- patcher.stop()
-
- def _iter_keys(self, prefix):
- assert prefix == NAMESPACE_PREFIX
- l = len(prefix)
- for k in self._mock_resources.keys():
- yield k, k[l:]
-
- def _resource_find(self, key):
- return self._mock_resources.get(key)
-
- def _lookup_mock_fs(self, filename):
- path = filename.lstrip('/').split('/')
- curdir = self._mock_fs
- for item in path:
- if item in curdir:
- curdir = curdir[item]
- else:
- return None
- return curdir
-
- def _path_isfile(self, filename):
- f = self._lookup_mock_fs(filename)
- return isinstance(f, str)
-
- def _path_isdir(self, filename):
- f = self._lookup_mock_fs(filename)
- return isinstance(f, dict)
-
- def _os_listdir(self, dirname):
- f = self._lookup_mock_fs(dirname)
- if isinstance(f, dict):
- return f.keys()
- else:
- return []
-
-
-class ArcadiaSourceFinderTestCase(unittest.TestCase):
- def setUp(self):
- self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources())
- self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT)
-
- def tearDown(self):
- self.import_mock.stop()
-
- def _get_mock_fs(self):
- raise NotImplementedError()
-
- def _get_mock_resources(self):
- raise NotImplementedError()
-
-
-class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase):
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- project:
- lib:
- mod1.py: ""
- package1:
- mod2.py: ""
- '''
-
- def _get_mock_resources(self):
- return {
- b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
- }
-
- @parameterized.expand([
- ('project.lib.mod1', b'project/lib/mod1.py'),
- ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'),
- ('project.lib.unknown_module', None),
- ('project.lib', None), # package
- ])
- def test_get_module_path(self, module, path):
- assert path == self.arcadia_source_finder.get_module_path(module)
-
- @parameterized.expand([
- ('project.lib.mod1', False),
- ('project.lib.package1.mod2', False),
- ('project', True),
- ('project.lib', True),
- ('project.lib.package1', True),
- ])
- def test_is_packages(self, module, is_package):
- assert is_package == self.arcadia_source_finder.is_package(module)
-
- def test_is_package_for_unknown_module(self):
- self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2'))
-
- @parameterized.expand([
- ('', {
- ('PFX.project', True),
- }),
- ('project.', {
- ('PFX.lib', True),
- }),
- ('project.lib.', {
- ('PFX.mod1', False),
- ('PFX.package1', True),
- }),
- ('project.lib.package1.', {
- ('PFX.mod2', False),
- }),
- ])
- def test_iter_modules(self, package_prefix, expected):
- got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
- assert expected == set(got)
-
- # Check iter_modules() don't crash and return correct result after not existing module was requested
- def test_iter_modules_after_unknown_module_import(self):
- self.arcadia_source_finder.get_module_path('project.unknown_module')
- assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', ''))
-
-
-class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase):
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- project:
- lib:
- mod1.py: ''
- lib_extension:
- mod2.py: ''
- '''
-
- def _get_mock_resources(self):
- return {
- b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
- b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.',
- }
-
- @parameterized.expand([
- ('project.lib.mod1', b'project/lib/mod1.py'),
- ('project.lib.mod2', b'project/lib_extension/mod2.py'),
- ])
- def test_get_module_path(self, module, path):
- assert path == self.arcadia_source_finder.get_module_path(module)
-
- @parameterized.expand([
- ('project.lib.', {
- ('PFX.mod1', False),
- ('PFX.mod2', False),
- }),
- ])
- def test_iter_modules(self, package_prefix, expected):
- got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
- assert expected == set(got)
-
-
-class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase):
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- project:
- ns_lib:
- mod1.py: ''
- top_level_lib:
- mod2.py: ''
- '''
-
- def _get_mock_resources(self):
- return {
- b'py/namespace/unique_prefix1/project/ns_lib': b'ns.',
- b'py/namespace/unique_prefix2/project/top_level_lib': b'.',
- }
-
- @parameterized.expand([
- ('ns.mod1', b'project/ns_lib/mod1.py'),
- ('mod2', b'project/top_level_lib/mod2.py'),
- ])
- def test_get_module_path(self, module, path):
- assert path == self.arcadia_source_finder.get_module_path(module)
-
- @parameterized.expand([
- ('ns', True),
- ('ns.mod1', False),
- ('mod2', False),
- ])
- def test_is_packages(self, module, is_package):
- assert is_package == self.arcadia_source_finder.is_package(module)
-
- @parameterized.expand([
- 'project',
- 'project.ns_lib',
- 'project.top_level_lib',
- ])
- def test_is_package_for_unknown_modules(self, module):
- self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module))
-
- @parameterized.expand([
- ('', {
- ('PFX.ns', True),
- ('PFX.mod2', False),
- }),
- ('ns.', {
- ('PFX.mod1', False),
- }),
- ])
- def test_iter_modules(self, package_prefix, expected):
- got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
- assert expected == set(got)
-
-
-class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase):
- ''' Packages and modules from tests should not be part of pylib namespace '''
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- contrib:
- python:
- pylib:
- mod1.py: ""
- tests:
- conftest.py: ""
- ya.make: ""
- '''
-
- def _get_mock_resources(self):
- return {
- b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.',
- }
-
- def test_get_module_path_for_lib(self):
- assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1')
-
- def test_get_module_for_tests(self):
- assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None
-
- def test_is_package_for_tests(self):
- self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests'))
-
-
-class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase):
- ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) '''
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- contrib:
- python:
- pylint:
- ya.make: ""
- pylint:
- __init__.py: ""
- patcher:
- patch.py: ""
- ya.make: ""
- '''
-
- def _get_mock_resources(self):
- return {
- b'py/namespace/unique_prefix1/contrib/python/pylint': b'.',
- b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.',
- }
-
- @parameterized.expand([
- ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'),
- ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'),
- ])
- def test_get_module_path(self, module, path):
- assert path == self.arcadia_source_finder.get_module_path(module)
-
-
-class TestEmptyResources(ArcadiaSourceFinderTestCase):
- def _get_mock_fs(self):
- return '''
- home:
- arcadia:
- project:
- lib:
- mod1.py: ''
- '''
-
- def _get_mock_resources(self):
- return {}
-
- def test_get_module_path(self):
- assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None
-
- def test_is_package(self):
- self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project'))
-
- def test_iter_modules(self):
- assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.'))
+import unittest
+import yaml
+from unittest.mock import patch
+from parameterized import parameterized
+
+import __res as res
+
+
+NAMESPACE_PREFIX = b'py/namespace/'
+TEST_SOURCE_ROOT = '/home/arcadia'
+
+
+class ImporterMocks(object):
+ def __init__(self, mock_fs, mock_resources):
+ self._mock_fs = mock_fs
+ self._mock_resources = mock_resources
+ self._patchers = [
+ patch('__res.iter_keys', wraps=self._iter_keys),
+ patch('__res.__resource.find', wraps=self._resource_find),
+ patch('__res._path_isdir', wraps=self._path_isdir),
+ patch('__res._path_isfile', wraps=self._path_isfile),
+ patch('__res._os.listdir', wraps=self._os_listdir),
+ ]
+ for patcher in self._patchers:
+ patcher.start()
+
+ def stop(self):
+ for patcher in self._patchers:
+ patcher.stop()
+
+ def _iter_keys(self, prefix):
+ assert prefix == NAMESPACE_PREFIX
+ l = len(prefix)
+ for k in self._mock_resources.keys():
+ yield k, k[l:]
+
+ def _resource_find(self, key):
+ return self._mock_resources.get(key)
+
+ def _lookup_mock_fs(self, filename):
+ path = filename.lstrip('/').split('/')
+ curdir = self._mock_fs
+ for item in path:
+ if item in curdir:
+ curdir = curdir[item]
+ else:
+ return None
+ return curdir
+
+ def _path_isfile(self, filename):
+ f = self._lookup_mock_fs(filename)
+ return isinstance(f, str)
+
+ def _path_isdir(self, filename):
+ f = self._lookup_mock_fs(filename)
+ return isinstance(f, dict)
+
+ def _os_listdir(self, dirname):
+ f = self._lookup_mock_fs(dirname)
+ if isinstance(f, dict):
+ return f.keys()
+ else:
+ return []
+
+
+class ArcadiaSourceFinderTestCase(unittest.TestCase):
+ def setUp(self):
+ self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources())
+ self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT)
+
+ def tearDown(self):
+ self.import_mock.stop()
+
+ def _get_mock_fs(self):
+ raise NotImplementedError()
+
+ def _get_mock_resources(self):
+ raise NotImplementedError()
+
+
+class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ""
+ package1:
+ mod2.py: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
+ }
+
+ @parameterized.expand([
+ ('project.lib.mod1', b'project/lib/mod1.py'),
+ ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'),
+ ('project.lib.unknown_module', None),
+ ('project.lib', None), # package
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('project.lib.mod1', False),
+ ('project.lib.package1.mod2', False),
+ ('project', True),
+ ('project.lib', True),
+ ('project.lib.package1', True),
+ ])
+ def test_is_packages(self, module, is_package):
+ assert is_package == self.arcadia_source_finder.is_package(module)
+
+ def test_is_package_for_unknown_module(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2'))
+
+ @parameterized.expand([
+ ('', {
+ ('PFX.project', True),
+ }),
+ ('project.', {
+ ('PFX.lib', True),
+ }),
+ ('project.lib.', {
+ ('PFX.mod1', False),
+ ('PFX.package1', True),
+ }),
+ ('project.lib.package1.', {
+ ('PFX.mod2', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+ # Check iter_modules() don't crash and return correct result after not existing module was requested
+ def test_iter_modules_after_unknown_module_import(self):
+ self.arcadia_source_finder.get_module_path('project.unknown_module')
+ assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', ''))
+
+
+class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ''
+ lib_extension:
+ mod2.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/lib': b'project.lib.',
+ b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.',
+ }
+
+ @parameterized.expand([
+ ('project.lib.mod1', b'project/lib/mod1.py'),
+ ('project.lib.mod2', b'project/lib_extension/mod2.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('project.lib.', {
+ ('PFX.mod1', False),
+ ('PFX.mod2', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+
+class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ ns_lib:
+ mod1.py: ''
+ top_level_lib:
+ mod2.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/project/ns_lib': b'ns.',
+ b'py/namespace/unique_prefix2/project/top_level_lib': b'.',
+ }
+
+ @parameterized.expand([
+ ('ns.mod1', b'project/ns_lib/mod1.py'),
+ ('mod2', b'project/top_level_lib/mod2.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+ @parameterized.expand([
+ ('ns', True),
+ ('ns.mod1', False),
+ ('mod2', False),
+ ])
+ def test_is_packages(self, module, is_package):
+ assert is_package == self.arcadia_source_finder.is_package(module)
+
+ @parameterized.expand([
+ 'project',
+ 'project.ns_lib',
+ 'project.top_level_lib',
+ ])
+ def test_is_package_for_unknown_modules(self, module):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module))
+
+ @parameterized.expand([
+ ('', {
+ ('PFX.ns', True),
+ ('PFX.mod2', False),
+ }),
+ ('ns.', {
+ ('PFX.mod1', False),
+ }),
+ ])
+ def test_iter_modules(self, package_prefix, expected):
+ got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.')
+ assert expected == set(got)
+
+
+class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase):
+ ''' Packages and modules from tests should not be part of pylib namespace '''
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ contrib:
+ python:
+ pylib:
+ mod1.py: ""
+ tests:
+ conftest.py: ""
+ ya.make: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.',
+ }
+
+ def test_get_module_path_for_lib(self):
+ assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1')
+
+ def test_get_module_for_tests(self):
+ assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None
+
+ def test_is_package_for_tests(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests'))
+
+
+class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase):
+ ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) '''
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ contrib:
+ python:
+ pylint:
+ ya.make: ""
+ pylint:
+ __init__.py: ""
+ patcher:
+ patch.py: ""
+ ya.make: ""
+ '''
+
+ def _get_mock_resources(self):
+ return {
+ b'py/namespace/unique_prefix1/contrib/python/pylint': b'.',
+ b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.',
+ }
+
+ @parameterized.expand([
+ ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'),
+ ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'),
+ ])
+ def test_get_module_path(self, module, path):
+ assert path == self.arcadia_source_finder.get_module_path(module)
+
+
+class TestEmptyResources(ArcadiaSourceFinderTestCase):
+ def _get_mock_fs(self):
+ return '''
+ home:
+ arcadia:
+ project:
+ lib:
+ mod1.py: ''
+ '''
+
+ def _get_mock_resources(self):
+ return {}
+
+ def test_get_module_path(self):
+ assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None
+
+ def test_is_package(self):
+ self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project'))
+
+ def test_iter_modules(self):
+ assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.'))
diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make
index ed0c143080..4ec3db74f5 100644
--- a/library/python/runtime_py3/test/ya.make
+++ b/library/python/runtime_py3/test/ya.make
@@ -7,11 +7,11 @@ OWNER(
DEPENDS(library/python/runtime_py3/test/traceback)
-PEERDIR(
- contrib/python/parameterized
- contrib/python/PyYAML
-)
-
+PEERDIR(
+ contrib/python/parameterized
+ contrib/python/PyYAML
+)
+
PY_SRCS(
TOP_LEVEL
resources/__init__.py
@@ -22,7 +22,7 @@ TEST_SRCS(
test_metadata.py
test_resources.py
test_traceback.py
- test_arcadia_source_finder.py
+ test_arcadia_source_finder.py
)
RESOURCE_FILES(