diff options
author | say <say@yandex-team.ru> | 2022-02-10 16:48:19 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:19 +0300 |
commit | a6a6f6e1e77c7d7d0cdfad61c093e061d6fb5782 (patch) | |
tree | 4ba72c2b9370316395657a82e1952eebbe587ade /library/python/runtime_py3 | |
parent | 24e42061bd52ed0f06dff42f3db89414ba1d186e (diff) | |
download | ydb-a6a6f6e1e77c7d7d0cdfad61c093e061d6fb5782.tar.gz |
Restoring authorship annotation for <say@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/python/runtime_py3')
-rw-r--r-- | library/python/runtime_py3/importer.pxi | 432 | ||||
-rw-r--r-- | library/python/runtime_py3/test/test_arcadia_source_finder.py | 634 | ||||
-rw-r--r-- | library/python/runtime_py3/test/ya.make | 12 |
3 files changed, 539 insertions, 539 deletions
diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi index 904f94dea2..8d8b523d4c 100644 --- a/library/python/runtime_py3/importer.pxi +++ b/library/python/runtime_py3/importer.pxi @@ -2,7 +2,7 @@ import marshal import sys from _codecs import utf_8_decode, utf_8_encode from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter -from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split +from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split from _io import FileIO import __res as __resource @@ -11,9 +11,9 @@ _b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] _s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] env_entry_point = b'Y_PYTHON_ENTRY_POINT' env_source_root = b'Y_PYTHON_SOURCE_ROOT' -cfg_source_root = b'arcadia-source-root' -env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' -res_ya_ide_venv = b'YA_IDE_VENV' +cfg_source_root = b'arcadia-source-root' +env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' +res_ya_ide_venv = b'YA_IDE_VENV' executable = sys.executable or 'Y_PYTHON' sys.modules['run_import_hook'] = __resource @@ -21,59 +21,59 @@ sys.modules['run_import_hook'] = __resource py_prefix = b'py/' py_prefix_len = len(py_prefix) -YA_IDE_VENV = __resource.find(res_ya_ide_venv) -Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV - - -def _init_venv(): - if not _path_isabs(executable): - raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) - - # Creative copy-paste from site.py - exe_dir, _ = _path_split(executable) - site_prefix, _ = _path_split(exe_dir) - libpath = _path_join(site_prefix, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') - sys.path.insert(0, libpath) - - # emulate site.venv() - sys.prefix = site_prefix - sys.exec_prefix = site_prefix - - conf_basename = 'pyvenv.cfg' - candidate_confs = [ - conffile for conffile in ( - _path_join(exe_dir, conf_basename), - _path_join(site_prefix, conf_basename) - ) - if _path_isfile(conffile) - ] - if not candidate_confs: - raise RuntimeError('{} not found'.format(conf_basename)) - virtual_conf = candidate_confs[0] - with FileIO(virtual_conf, 'r') as f: - for line in f: - if b'=' in line: - key, _, value = line.partition(b'=') - key = key.strip().lower() - value = value.strip() - if key == cfg_source_root: - return value - raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf)) - - -def _get_source_root(): - env_value = _os.environ.get(env_source_root) - if env_value or not YA_IDE_VENV: - return env_value - - return _init_venv() - - -Y_PYTHON_SOURCE_ROOT = _get_source_root() - - +YA_IDE_VENV = __resource.find(res_ya_ide_venv) +Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV + + +def _init_venv(): + if not _path_isabs(executable): + raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) + + # Creative copy-paste from site.py + exe_dir, _ = _path_split(executable) + site_prefix, _ = _path_split(exe_dir) + libpath = _path_join(site_prefix, 'lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + sys.path.insert(0, libpath) + + # emulate site.venv() + sys.prefix = site_prefix + sys.exec_prefix = site_prefix + + conf_basename = 'pyvenv.cfg' + candidate_confs = [ + conffile for conffile in ( + _path_join(exe_dir, conf_basename), + _path_join(site_prefix, conf_basename) + ) + if _path_isfile(conffile) + ] + if not candidate_confs: + raise RuntimeError('{} not found'.format(conf_basename)) + virtual_conf = candidate_confs[0] + with FileIO(virtual_conf, 'r') as f: + for line in f: + if b'=' in line: + key, _, value = line.partition(b'=') + key = key.strip().lower() + value = value.strip() + if key == cfg_source_root: + return value + raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf)) + + +def _get_source_root(): + env_value = _os.environ.get(env_source_root) + if env_value or not YA_IDE_VENV: + return env_value + + return _init_venv() + + +Y_PYTHON_SOURCE_ROOT = _get_source_root() + + def _print(*xs): """ This is helpful for debugging, since automatic bytes to str conversion is @@ -181,10 +181,10 @@ class ResourceImporter(object): self.source_map = {} # Map from file names to module names. self._source_name = {} # Map from original to altered module names. self._package_prefix = '' - if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: - self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) - else: - self.arcadia_source_finder = None + if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: + self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) + else: + self.arcadia_source_finder = None for p in list(self.memory) + list(sys.builtin_module_names): for pp in iter_prefixes(p): @@ -198,13 +198,13 @@ class ResourceImporter(object): importer._package_prefix = name + '.' return importer - def _find_mod_path(self, fullname): - """Find arcadia relative path by module name""" - relpath = resfs_src(mod_path(fullname), resfs_file=True) - if relpath or not self.arcadia_source_finder: - return relpath - return self.arcadia_source_finder.get_module_path(fullname) - + def _find_mod_path(self, fullname): + """Find arcadia relative path by module name""" + relpath = resfs_src(mod_path(fullname), resfs_file=True) + if relpath or not self.arcadia_source_finder: + return relpath + return self.arcadia_source_finder.get_module_path(fullname) + def find_spec(self, fullname, path=None, target=None): try: is_package = self.is_package(fullname) @@ -245,7 +245,7 @@ class ResourceImporter(object): modname = fullname if self.is_package(fullname): fullname += '.__init__' - relpath = self._find_mod_path(fullname) + relpath = self._find_mod_path(fullname) if isinstance(relpath, bytes): relpath = _s(relpath) return relpath or modname @@ -272,7 +272,7 @@ class ResourceImporter(object): fullname += '.__init__' path = mod_path(fullname) - relpath = self._find_mod_path(fullname) + relpath = self._find_mod_path(fullname) if relpath: abspath = resfs_resolve(relpath) if abspath: @@ -298,9 +298,9 @@ class ResourceImporter(object): if fullname + '.__init__' in self.memory: return True - if self.arcadia_source_finder: - return self.arcadia_source_finder.is_package(fullname) - + if self.arcadia_source_finder: + return self.arcadia_source_finder.is_package(fullname) + raise ImportError(fullname) # Extension for contrib/python/coverage. @@ -329,9 +329,9 @@ class ResourceImporter(object): m = rx.match(p) if m: yield prefix + m.group(1), m.group(2) is not None - if self.arcadia_source_finder: - for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix): - yield m + if self.arcadia_source_finder: + for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix): + yield m def get_resource_reader(self, fullname): try: @@ -394,134 +394,134 @@ class BuiltinSubmoduleImporter(BuiltinImporter): return None -class ArcadiaSourceFinder: - """ - Search modules and packages in arcadia source tree. - See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details - """ - NAMESPACE_PREFIX = b'py/namespace/' - PY_EXT = '.py' - YA_MAKE = 'ya.make' - - def __init__(self, source_root): - self.source_root = source_root - self.module_path_cache = {'': set()} - for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX): - # dirty_path contains unique prefix to prevent repeatable keys in the resource storage - path = dirty_path.split(b'/', 1)[1] - namespaces = __resource.find(key).split(b':') - for n in namespaces: - package_name = _s(n.rstrip(b'.')) - self.module_path_cache.setdefault(package_name, set()).add(_s(path)) - # Fill parents with default empty path set if parent doesn't exist in the cache yet - while package_name: - package_name = package_name.rpartition('.')[0] - if package_name in self.module_path_cache: - break - self.module_path_cache.setdefault(package_name, set()) - for package_name in self.module_path_cache.keys(): - self._add_parent_dirs(package_name, visited=set()) - - def get_module_path(self, fullname): - """ - Find file path for module 'fullname'. - For packages caller pass fullname as 'package.__init__'. - Return None if nothing is found. - """ - try: - if not self.is_package(fullname): - return _b(self._cache_module_path(fullname)) - except ImportError: - pass - - def is_package(self, fullname): - """Check if fullname is a package. Raise ImportError if fullname is not found""" - path = self._cache_module_path(fullname) - if isinstance(path, set): - return True - if isinstance(path, str): - return False - raise ImportError(fullname) - - def iter_modules(self, package_prefix, prefix): - paths = self._cache_module_path(package_prefix.rstrip('.')) - if paths is not None: - # Note: it's ok to yield duplicates because pkgutil discards them - - # Yield from cache - import re - rx = re.compile(re.escape(package_prefix) + r'([^.]+)$') - for mod, path in self.module_path_cache.items(): - if path is not None: - m = rx.match(mod) - if m: - yield prefix + m.group(1), self.is_package(mod) - - # Yield from file system - for path in paths: - abs_path = _path_join(self.source_root, path) - for dir_item in _os.listdir(abs_path): - if self._path_is_simple_dir(_path_join(abs_path, dir_item)): - yield prefix + dir_item, True - elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)): - yield prefix + dir_item[:-len(self.PY_EXT)], False - - def _path_is_simple_dir(self, abs_path): - """ - Check if path is a directory but doesn't contain ya.make file. - We don't want to steal directory from nested project and treat it as a package - """ - return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE)) - - def _find_module_in_paths(self, find_package_only, paths, module): - """Auxiliary method. See _cache_module_path() for details""" - if paths: - package_paths = set() - for path in paths: - rel_path = _path_join(path, module) - if not find_package_only: - # Check if file_path is a module - module_path = rel_path + self.PY_EXT - if _path_isfile(_path_join(self.source_root, module_path)): - return module_path - # Check if file_path is a package - if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): - package_paths.add(rel_path) - if package_paths: - return package_paths - - def _cache_module_path(self, fullname, find_package_only=False): +class ArcadiaSourceFinder: + """ + Search modules and packages in arcadia source tree. + See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details + """ + NAMESPACE_PREFIX = b'py/namespace/' + PY_EXT = '.py' + YA_MAKE = 'ya.make' + + def __init__(self, source_root): + self.source_root = source_root + self.module_path_cache = {'': set()} + for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX): + # dirty_path contains unique prefix to prevent repeatable keys in the resource storage + path = dirty_path.split(b'/', 1)[1] + namespaces = __resource.find(key).split(b':') + for n in namespaces: + package_name = _s(n.rstrip(b'.')) + self.module_path_cache.setdefault(package_name, set()).add(_s(path)) + # Fill parents with default empty path set if parent doesn't exist in the cache yet + while package_name: + package_name = package_name.rpartition('.')[0] + if package_name in self.module_path_cache: + break + self.module_path_cache.setdefault(package_name, set()) + for package_name in self.module_path_cache.keys(): + self._add_parent_dirs(package_name, visited=set()) + + def get_module_path(self, fullname): """ - Find module path or package directory paths and save result in the cache - - find_package_only=True - don't try to find module - - Returns: - List of relative package paths - for a package - Relative module path - for a module - None - module or package is not found - """ - if fullname not in self.module_path_cache: - parent, _, tail = fullname.rpartition('.') - parent_paths = self._cache_module_path(parent, find_package_only=True) - self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail) - return self.module_path_cache[fullname] - - def _add_parent_dirs(self, package_name, visited): - if not package_name or package_name in visited: - return - visited.add(package_name) - - parent, _, tail = package_name.rpartition('.') - self._add_parent_dirs(parent, visited) - - paths = self.module_path_cache[package_name] - for parent_path in self.module_path_cache[parent]: - rel_path = _path_join(parent_path, tail) - if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): - paths.add(rel_path) - - + Find file path for module 'fullname'. + For packages caller pass fullname as 'package.__init__'. + Return None if nothing is found. + """ + try: + if not self.is_package(fullname): + return _b(self._cache_module_path(fullname)) + except ImportError: + pass + + def is_package(self, fullname): + """Check if fullname is a package. Raise ImportError if fullname is not found""" + path = self._cache_module_path(fullname) + if isinstance(path, set): + return True + if isinstance(path, str): + return False + raise ImportError(fullname) + + def iter_modules(self, package_prefix, prefix): + paths = self._cache_module_path(package_prefix.rstrip('.')) + if paths is not None: + # Note: it's ok to yield duplicates because pkgutil discards them + + # Yield from cache + import re + rx = re.compile(re.escape(package_prefix) + r'([^.]+)$') + for mod, path in self.module_path_cache.items(): + if path is not None: + m = rx.match(mod) + if m: + yield prefix + m.group(1), self.is_package(mod) + + # Yield from file system + for path in paths: + abs_path = _path_join(self.source_root, path) + for dir_item in _os.listdir(abs_path): + if self._path_is_simple_dir(_path_join(abs_path, dir_item)): + yield prefix + dir_item, True + elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)): + yield prefix + dir_item[:-len(self.PY_EXT)], False + + def _path_is_simple_dir(self, abs_path): + """ + Check if path is a directory but doesn't contain ya.make file. + We don't want to steal directory from nested project and treat it as a package + """ + return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE)) + + def _find_module_in_paths(self, find_package_only, paths, module): + """Auxiliary method. See _cache_module_path() for details""" + if paths: + package_paths = set() + for path in paths: + rel_path = _path_join(path, module) + if not find_package_only: + # Check if file_path is a module + module_path = rel_path + self.PY_EXT + if _path_isfile(_path_join(self.source_root, module_path)): + return module_path + # Check if file_path is a package + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + package_paths.add(rel_path) + if package_paths: + return package_paths + + def _cache_module_path(self, fullname, find_package_only=False): + """ + Find module path or package directory paths and save result in the cache + + find_package_only=True - don't try to find module + + Returns: + List of relative package paths - for a package + Relative module path - for a module + None - module or package is not found + """ + if fullname not in self.module_path_cache: + parent, _, tail = fullname.rpartition('.') + parent_paths = self._cache_module_path(parent, find_package_only=True) + self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail) + return self.module_path_cache[fullname] + + def _add_parent_dirs(self, package_name, visited): + if not package_name or package_name in visited: + return + visited.add(package_name) + + parent, _, tail = package_name.rpartition('.') + self._add_parent_dirs(parent, visited) + + paths = self.module_path_cache[package_name] + for parent_path in self.module_path_cache[parent]: + rel_path = _path_join(parent_path, tail) + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + paths.add(rel_path) + + def excepthook(*args, **kws): # traceback module cannot be imported at module level, because interpreter # is not fully initialized yet @@ -544,19 +544,19 @@ def executable_path_hook(path): raise ImportError(path) -if YA_IDE_VENV: - sys.meta_path.append(importer) - sys.meta_path.append(BuiltinSubmoduleImporter) - if executable not in sys.path: - sys.path.append(executable) - sys.path_hooks.append(executable_path_hook) -else: - sys.meta_path.insert(0, BuiltinSubmoduleImporter) - sys.meta_path.insert(0, importer) - if executable not in sys.path: - sys.path.insert(0, executable) - sys.path_hooks.insert(0, executable_path_hook) - +if YA_IDE_VENV: + sys.meta_path.append(importer) + sys.meta_path.append(BuiltinSubmoduleImporter) + if executable not in sys.path: + sys.path.append(executable) + sys.path_hooks.append(executable_path_hook) +else: + sys.meta_path.insert(0, BuiltinSubmoduleImporter) + sys.meta_path.insert(0, importer) + if executable not in sys.path: + sys.path.insert(0, executable) + sys.path_hooks.insert(0, executable_path_hook) + sys.path_importer_cache[executable] = importer # Indicator that modules and resources are built-in rather than on the file system. diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py index ff80d0a0a2..b45f77a910 100644 --- a/library/python/runtime_py3/test/test_arcadia_source_finder.py +++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py @@ -1,317 +1,317 @@ -import unittest -import yaml -from unittest.mock import patch -from parameterized import parameterized - -import __res as res - - -NAMESPACE_PREFIX = b'py/namespace/' -TEST_SOURCE_ROOT = '/home/arcadia' - - -class ImporterMocks(object): - def __init__(self, mock_fs, mock_resources): - self._mock_fs = mock_fs - self._mock_resources = mock_resources - self._patchers = [ - patch('__res.iter_keys', wraps=self._iter_keys), - patch('__res.__resource.find', wraps=self._resource_find), - patch('__res._path_isdir', wraps=self._path_isdir), - patch('__res._path_isfile', wraps=self._path_isfile), - patch('__res._os.listdir', wraps=self._os_listdir), - ] - for patcher in self._patchers: - patcher.start() - - def stop(self): - for patcher in self._patchers: - patcher.stop() - - def _iter_keys(self, prefix): - assert prefix == NAMESPACE_PREFIX - l = len(prefix) - for k in self._mock_resources.keys(): - yield k, k[l:] - - def _resource_find(self, key): - return self._mock_resources.get(key) - - def _lookup_mock_fs(self, filename): - path = filename.lstrip('/').split('/') - curdir = self._mock_fs - for item in path: - if item in curdir: - curdir = curdir[item] - else: - return None - return curdir - - def _path_isfile(self, filename): - f = self._lookup_mock_fs(filename) - return isinstance(f, str) - - def _path_isdir(self, filename): - f = self._lookup_mock_fs(filename) - return isinstance(f, dict) - - def _os_listdir(self, dirname): - f = self._lookup_mock_fs(dirname) - if isinstance(f, dict): - return f.keys() - else: - return [] - - -class ArcadiaSourceFinderTestCase(unittest.TestCase): - def setUp(self): - self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources()) - self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT) - - def tearDown(self): - self.import_mock.stop() - - def _get_mock_fs(self): - raise NotImplementedError() - - def _get_mock_resources(self): - raise NotImplementedError() - - -class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: "" - package1: - mod2.py: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/lib': b'project.lib.', - } - - @parameterized.expand([ - ('project.lib.mod1', b'project/lib/mod1.py'), - ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'), - ('project.lib.unknown_module', None), - ('project.lib', None), # package - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('project.lib.mod1', False), - ('project.lib.package1.mod2', False), - ('project', True), - ('project.lib', True), - ('project.lib.package1', True), - ]) - def test_is_packages(self, module, is_package): - assert is_package == self.arcadia_source_finder.is_package(module) - - def test_is_package_for_unknown_module(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2')) - - @parameterized.expand([ - ('', { - ('PFX.project', True), - }), - ('project.', { - ('PFX.lib', True), - }), - ('project.lib.', { - ('PFX.mod1', False), - ('PFX.package1', True), - }), - ('project.lib.package1.', { - ('PFX.mod2', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - # Check iter_modules() don't crash and return correct result after not existing module was requested - def test_iter_modules_after_unknown_module_import(self): - self.arcadia_source_finder.get_module_path('project.unknown_module') - assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', '')) - - -class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: '' - lib_extension: - mod2.py: '' - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/lib': b'project.lib.', - b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.', - } - - @parameterized.expand([ - ('project.lib.mod1', b'project/lib/mod1.py'), - ('project.lib.mod2', b'project/lib_extension/mod2.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('project.lib.', { - ('PFX.mod1', False), - ('PFX.mod2', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - -class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - ns_lib: - mod1.py: '' - top_level_lib: - mod2.py: '' - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/ns_lib': b'ns.', - b'py/namespace/unique_prefix2/project/top_level_lib': b'.', - } - - @parameterized.expand([ - ('ns.mod1', b'project/ns_lib/mod1.py'), - ('mod2', b'project/top_level_lib/mod2.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('ns', True), - ('ns.mod1', False), - ('mod2', False), - ]) - def test_is_packages(self, module, is_package): - assert is_package == self.arcadia_source_finder.is_package(module) - - @parameterized.expand([ - 'project', - 'project.ns_lib', - 'project.top_level_lib', - ]) - def test_is_package_for_unknown_modules(self, module): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module)) - - @parameterized.expand([ - ('', { - ('PFX.ns', True), - ('PFX.mod2', False), - }), - ('ns.', { - ('PFX.mod1', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - -class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase): - ''' Packages and modules from tests should not be part of pylib namespace ''' - def _get_mock_fs(self): - return ''' - home: - arcadia: - contrib: - python: - pylib: - mod1.py: "" - tests: - conftest.py: "" - ya.make: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.', - } - - def test_get_module_path_for_lib(self): - assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1') - - def test_get_module_for_tests(self): - assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None - - def test_is_package_for_tests(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests')) - - -class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase): - ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) ''' - def _get_mock_fs(self): - return ''' - home: - arcadia: - contrib: - python: - pylint: - ya.make: "" - pylint: - __init__.py: "" - patcher: - patch.py: "" - ya.make: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/contrib/python/pylint': b'.', - b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.', - } - - @parameterized.expand([ - ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'), - ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - -class TestEmptyResources(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: '' - ''' - - def _get_mock_resources(self): - return {} - - def test_get_module_path(self): - assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None - - def test_is_package(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project')) - - def test_iter_modules(self): - assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.')) +import unittest +import yaml +from unittest.mock import patch +from parameterized import parameterized + +import __res as res + + +NAMESPACE_PREFIX = b'py/namespace/' +TEST_SOURCE_ROOT = '/home/arcadia' + + +class ImporterMocks(object): + def __init__(self, mock_fs, mock_resources): + self._mock_fs = mock_fs + self._mock_resources = mock_resources + self._patchers = [ + patch('__res.iter_keys', wraps=self._iter_keys), + patch('__res.__resource.find', wraps=self._resource_find), + patch('__res._path_isdir', wraps=self._path_isdir), + patch('__res._path_isfile', wraps=self._path_isfile), + patch('__res._os.listdir', wraps=self._os_listdir), + ] + for patcher in self._patchers: + patcher.start() + + def stop(self): + for patcher in self._patchers: + patcher.stop() + + def _iter_keys(self, prefix): + assert prefix == NAMESPACE_PREFIX + l = len(prefix) + for k in self._mock_resources.keys(): + yield k, k[l:] + + def _resource_find(self, key): + return self._mock_resources.get(key) + + def _lookup_mock_fs(self, filename): + path = filename.lstrip('/').split('/') + curdir = self._mock_fs + for item in path: + if item in curdir: + curdir = curdir[item] + else: + return None + return curdir + + def _path_isfile(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, str) + + def _path_isdir(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, dict) + + def _os_listdir(self, dirname): + f = self._lookup_mock_fs(dirname) + if isinstance(f, dict): + return f.keys() + else: + return [] + + +class ArcadiaSourceFinderTestCase(unittest.TestCase): + def setUp(self): + self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources()) + self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT) + + def tearDown(self): + self.import_mock.stop() + + def _get_mock_fs(self): + raise NotImplementedError() + + def _get_mock_resources(self): + raise NotImplementedError() + + +class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: "" + package1: + mod2.py: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'), + ('project.lib.unknown_module', None), + ('project.lib', None), # package + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.mod1', False), + ('project.lib.package1.mod2', False), + ('project', True), + ('project.lib', True), + ('project.lib.package1', True), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + def test_is_package_for_unknown_module(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2')) + + @parameterized.expand([ + ('', { + ('PFX.project', True), + }), + ('project.', { + ('PFX.lib', True), + }), + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.package1', True), + }), + ('project.lib.package1.', { + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + # Check iter_modules() don't crash and return correct result after not existing module was requested + def test_iter_modules_after_unknown_module_import(self): + self.arcadia_source_finder.get_module_path('project.unknown_module') + assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', '')) + + +class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + lib_extension: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.mod2', b'project/lib_extension/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + ns_lib: + mod1.py: '' + top_level_lib: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/ns_lib': b'ns.', + b'py/namespace/unique_prefix2/project/top_level_lib': b'.', + } + + @parameterized.expand([ + ('ns.mod1', b'project/ns_lib/mod1.py'), + ('mod2', b'project/top_level_lib/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('ns', True), + ('ns.mod1', False), + ('mod2', False), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + @parameterized.expand([ + 'project', + 'project.ns_lib', + 'project.top_level_lib', + ]) + def test_is_package_for_unknown_modules(self, module): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module)) + + @parameterized.expand([ + ('', { + ('PFX.ns', True), + ('PFX.mod2', False), + }), + ('ns.', { + ('PFX.mod1', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase): + ''' Packages and modules from tests should not be part of pylib namespace ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylib: + mod1.py: "" + tests: + conftest.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.', + } + + def test_get_module_path_for_lib(self): + assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1') + + def test_get_module_for_tests(self): + assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None + + def test_is_package_for_tests(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests')) + + +class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase): + ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylint: + ya.make: "" + pylint: + __init__.py: "" + patcher: + patch.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylint': b'.', + b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.', + } + + @parameterized.expand([ + ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'), + ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + +class TestEmptyResources(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + ''' + + def _get_mock_resources(self): + return {} + + def test_get_module_path(self): + assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None + + def test_is_package(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project')) + + def test_iter_modules(self): + assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.')) diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make index 4ec3db74f5..ed0c143080 100644 --- a/library/python/runtime_py3/test/ya.make +++ b/library/python/runtime_py3/test/ya.make @@ -7,11 +7,11 @@ OWNER( DEPENDS(library/python/runtime_py3/test/traceback) -PEERDIR( - contrib/python/parameterized - contrib/python/PyYAML -) - +PEERDIR( + contrib/python/parameterized + contrib/python/PyYAML +) + PY_SRCS( TOP_LEVEL resources/__init__.py @@ -22,7 +22,7 @@ TEST_SRCS( test_metadata.py test_resources.py test_traceback.py - test_arcadia_source_finder.py + test_arcadia_source_finder.py ) RESOURCE_FILES( |