diff options
author | orivej <orivej@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
commit | 718c552901d703c502ccbefdfc3c9028d608b947 (patch) | |
tree | 46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /library/python/runtime_py3/importer.pxi | |
parent | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff) | |
download | ydb-718c552901d703c502ccbefdfc3c9028d608b947.tar.gz |
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/python/runtime_py3/importer.pxi')
-rw-r--r-- | library/python/runtime_py3/importer.pxi | 390 |
1 files changed, 195 insertions, 195 deletions
diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi index 904f94dea2..7caa69e91e 100644 --- a/library/python/runtime_py3/importer.pxi +++ b/library/python/runtime_py3/importer.pxi @@ -1,30 +1,30 @@ -import marshal +import marshal import sys -from _codecs import utf_8_decode, utf_8_encode +from _codecs import utf_8_decode, utf_8_encode from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split -from _io import FileIO - +from _io import FileIO + import __res as __resource -_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] -_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] -env_entry_point = b'Y_PYTHON_ENTRY_POINT' -env_source_root = b'Y_PYTHON_SOURCE_ROOT' +_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] +_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] +env_entry_point = b'Y_PYTHON_ENTRY_POINT' +env_source_root = b'Y_PYTHON_SOURCE_ROOT' cfg_source_root = b'arcadia-source-root' env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' res_ya_ide_venv = b'YA_IDE_VENV' executable = sys.executable or 'Y_PYTHON' -sys.modules['run_import_hook'] = __resource - -# This is the prefix in contrib/tools/python3/src/Lib/ya.make. -py_prefix = b'py/' -py_prefix_len = len(py_prefix) +sys.modules['run_import_hook'] = __resource +# This is the prefix in contrib/tools/python3/src/Lib/ya.make. +py_prefix = b'py/' +py_prefix_len = len(py_prefix) + YA_IDE_VENV = __resource.find(res_ya_ide_venv) Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV - - + + def _init_venv(): if not _path_isabs(executable): raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) @@ -74,112 +74,112 @@ def _get_source_root(): Y_PYTHON_SOURCE_ROOT = _get_source_root() -def _print(*xs): - """ - This is helpful for debugging, since automatic bytes to str conversion is - not available yet. It is also possible to debug with GDB by breaking on - __Pyx_AddTraceback (with Python GDB pretty printers enabled). - """ - parts = [] - for s in xs: - if not isinstance(s, (bytes, str)): - s = str(s) - parts.append(_s(s)) - sys.stderr.write(' '.join(parts) + '\n') - - -def file_bytes(path): - # 'open' is not avaiable yet. - with FileIO(path, 'r') as f: - return f.read() - - -def iter_keys(prefix): - l = len(prefix) - for idx in range(__resource.count()): +def _print(*xs): + """ + This is helpful for debugging, since automatic bytes to str conversion is + not available yet. It is also possible to debug with GDB by breaking on + __Pyx_AddTraceback (with Python GDB pretty printers enabled). + """ + parts = [] + for s in xs: + if not isinstance(s, (bytes, str)): + s = str(s) + parts.append(_s(s)) + sys.stderr.write(' '.join(parts) + '\n') + + +def file_bytes(path): + # 'open' is not avaiable yet. + with FileIO(path, 'r') as f: + return f.read() + + +def iter_keys(prefix): + l = len(prefix) + for idx in range(__resource.count()): key = __resource.key_by_index(idx) - if key.startswith(prefix): - yield key, key[l:] - + if key.startswith(prefix): + yield key, key[l:] -def iter_py_modules(with_keys=False): - for key, path in iter_keys(b'resfs/file/' + py_prefix): - if path.endswith(b'.py'): # It may also end with '.pyc'. - mod = _s(path[:-3].replace(b'/', b'.')) - if with_keys: - yield key, mod - else: - yield mod + +def iter_py_modules(with_keys=False): + for key, path in iter_keys(b'resfs/file/' + py_prefix): + if path.endswith(b'.py'): # It may also end with '.pyc'. + mod = _s(path[:-3].replace(b'/', b'.')) + if with_keys: + yield key, mod + else: + yield mod -def iter_prefixes(s): - i = s.find('.') - while i >= 0: - yield s[:i] - i = s.find('.', i + 1) +def iter_prefixes(s): + i = s.find('.') + while i >= 0: + yield s[:i] + i = s.find('.', i + 1) def resfs_resolve(path): - """ - Return the absolute path of a root-relative path if it exists. - """ - path = _b(path) - if Y_PYTHON_SOURCE_ROOT: - if not path.startswith(Y_PYTHON_SOURCE_ROOT): - path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path)) - if _path_isfile(path): - return path - - -def resfs_src(key, resfs_file=False): - """ - Return the root-relative file path of a resource key. - """ - if resfs_file: - key = b'resfs/file/' + _b(key) - return __resource.find(b'resfs/src/' + _b(key)) - - + """ + Return the absolute path of a root-relative path if it exists. + """ + path = _b(path) + if Y_PYTHON_SOURCE_ROOT: + if not path.startswith(Y_PYTHON_SOURCE_ROOT): + path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path)) + if _path_isfile(path): + return path + + +def resfs_src(key, resfs_file=False): + """ + Return the root-relative file path of a resource key. + """ + if resfs_file: + key = b'resfs/file/' + _b(key) + return __resource.find(b'resfs/src/' + _b(key)) + + def resfs_read(path, builtin=None): + """ + Return the bytes of the resource file at path, or None. + If builtin is True, do not look for it on the filesystem. + If builtin is False, do not look in the builtin resources. + """ + if builtin is not True: + arcpath = resfs_src(path, resfs_file=True) + if arcpath: + fspath = resfs_resolve(arcpath) + if fspath: + return file_bytes(fspath) + + if builtin is not False: + return __resource.find(b'resfs/file/' + _b(path)) + + +def resfs_files(prefix=b''): + """ + List builtin resource file paths. + """ + return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))] + + +def mod_path(mod): """ - Return the bytes of the resource file at path, or None. - If builtin is True, do not look for it on the filesystem. - If builtin is False, do not look in the builtin resources. - """ - if builtin is not True: - arcpath = resfs_src(path, resfs_file=True) - if arcpath: - fspath = resfs_resolve(arcpath) - if fspath: - return file_bytes(fspath) - - if builtin is not False: - return __resource.find(b'resfs/file/' + _b(path)) - - -def resfs_files(prefix=b''): - """ - List builtin resource file paths. - """ - return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))] - - -def mod_path(mod): - """ - Return the resfs path to the source code of the module with the given name. + Return the resfs path to the source code of the module with the given name. """ - return py_prefix + _b(mod).replace(b'.', b'/') + b'.py' - + return py_prefix + _b(mod).replace(b'.', b'/') + b'.py' + class ResourceImporter(object): - """ A meta_path importer that loads code from built-in resources. + """ A meta_path importer that loads code from built-in resources. """ def __init__(self): - self.memory = set(iter_py_modules()) # Set of importable module names. - self.source_map = {} # Map from file names to module names. - self._source_name = {} # Map from original to altered module names. + self.memory = set(iter_py_modules()) # Set of importable module names. + self.source_map = {} # Map from file names to module names. + self._source_name = {} # Map from original to altered module names. self._package_prefix = '' if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) @@ -205,11 +205,11 @@ class ResourceImporter(object): return relpath return self.arcadia_source_finder.get_module_path(fullname) - def find_spec(self, fullname, path=None, target=None): - try: + def find_spec(self, fullname, path=None, target=None): + try: is_package = self.is_package(fullname) - except ImportError: - return None + except ImportError: + return None return spec_from_loader(fullname, self, is_package=is_package) def find_module(self, fullname, path=None): @@ -224,103 +224,103 @@ class ResourceImporter(object): code = self.get_code(module.__name__) module.__file__ = code.co_filename if self.is_package(module.__name__): - module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)] - # exec(code, module.__dict__) + module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)] + # exec(code, module.__dict__) _call_with_frames_removed(exec, code, module.__dict__) - # PEP-302 extension 1 of 3: data loader. - def get_data(self, path): - path = _b(path) - abspath = resfs_resolve(path) - if abspath: - return file_bytes(abspath) + # PEP-302 extension 1 of 3: data loader. + def get_data(self, path): + path = _b(path) + abspath = resfs_resolve(path) + if abspath: + return file_bytes(abspath) path = path.replace(_b('\\'), _b('/')) - data = resfs_read(path, builtin=True) - if data is None: - raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files - return data - - # PEP-302 extension 2 of 3: get __file__ without importing. - def get_filename(self, fullname): - modname = fullname - if self.is_package(fullname): - fullname += '.__init__' + data = resfs_read(path, builtin=True) + if data is None: + raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files + return data + + # PEP-302 extension 2 of 3: get __file__ without importing. + def get_filename(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' relpath = self._find_mod_path(fullname) - if isinstance(relpath, bytes): - relpath = _s(relpath) + if isinstance(relpath, bytes): + relpath = _s(relpath) return relpath or modname - # PEP-302 extension 3 of 3: packaging introspection. - # Used by `linecache` (while printing tracebacks) unless module filename - # exists on the filesystem. - def get_source(self, fullname): + # PEP-302 extension 3 of 3: packaging introspection. + # Used by `linecache` (while printing tracebacks) unless module filename + # exists on the filesystem. + def get_source(self, fullname): fullname = self._source_name.get(fullname) or fullname if self.is_package(fullname): fullname += '.__init__' relpath = self.get_filename(fullname) - if relpath: - abspath = resfs_resolve(relpath) - if abspath: - return _s(file_bytes(abspath)) + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + return _s(file_bytes(abspath)) data = resfs_read(mod_path(fullname)) - return _s(data) if data else '' - - def get_code(self, fullname): - modname = fullname - if self.is_package(fullname): - fullname += '.__init__' - - path = mod_path(fullname) + return _s(data) if data else '' + + def get_code(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' + + path = mod_path(fullname) relpath = self._find_mod_path(fullname) - if relpath: - abspath = resfs_resolve(relpath) - if abspath: - data = file_bytes(abspath) - return compile(data, _s(abspath), 'exec', dont_inherit=True) - + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + data = file_bytes(abspath) + return compile(data, _s(abspath), 'exec', dont_inherit=True) + yapyc_path = path + b'.yapyc3' yapyc_data = resfs_read(yapyc_path, builtin=True) if yapyc_data: return marshal.loads(yapyc_data) else: py_data = resfs_read(path, builtin=True) - if py_data: - return compile(py_data, _s(relpath), 'exec', dont_inherit=True) - else: - # This covers packages with no __init__.py in resources. - return compile('', modname, 'exec', dont_inherit=True) + if py_data: + return compile(py_data, _s(relpath), 'exec', dont_inherit=True) + else: + # This covers packages with no __init__.py in resources. + return compile('', modname, 'exec', dont_inherit=True) def is_package(self, fullname): if fullname in self.memory: return False - - if fullname + '.__init__' in self.memory: + + if fullname + '.__init__' in self.memory: return True if self.arcadia_source_finder: return self.arcadia_source_finder.is_package(fullname) - raise ImportError(fullname) - - # Extension for contrib/python/coverage. - def file_source(self, filename): - """ - Return the key of the module source by its resource path. - """ - if not self.source_map: - for key, mod in iter_py_modules(with_keys=True): - path = self.get_filename(mod) - self.source_map[path] = key - - if filename in self.source_map: - return self.source_map[filename] - - if resfs_read(filename, builtin=True) is not None: - return b'resfs/file/' + _b(filename) - - return b'' - + raise ImportError(fullname) + + # Extension for contrib/python/coverage. + def file_source(self, filename): + """ + Return the key of the module source by its resource path. + """ + if not self.source_map: + for key, mod in iter_py_modules(with_keys=True): + path = self.get_filename(mod) + self.source_map[path] = key + + if filename in self.source_map: + return self.source_map[filename] + + if resfs_read(filename, builtin=True) is not None: + return b'resfs/file/' + _b(filename) + + return b'' + # Extension for pkgutil.iter_modules. def iter_modules(self, prefix=''): import re @@ -531,19 +531,19 @@ def excepthook(*args, **kws): return traceback.print_exception(*args, **kws) -importer = ResourceImporter() - - -def executable_path_hook(path): - if path == executable: - return importer - - if path.startswith(executable + path_sep): - return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.')) - - raise ImportError(path) +importer = ResourceImporter() + +def executable_path_hook(path): + if path == executable: + return importer + + if path.startswith(executable + path_sep): + return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.')) + raise ImportError(path) + + if YA_IDE_VENV: sys.meta_path.append(importer) sys.meta_path.append(BuiltinSubmoduleImporter) @@ -557,13 +557,13 @@ else: sys.path.insert(0, executable) sys.path_hooks.insert(0, executable_path_hook) -sys.path_importer_cache[executable] = importer - -# Indicator that modules and resources are built-in rather than on the file system. -sys.is_standalone_binary = True -sys.frozen = True +sys.path_importer_cache[executable] = importer + +# Indicator that modules and resources are built-in rather than on the file system. +sys.is_standalone_binary = True +sys.frozen = True -# Set of names of importable modules. +# Set of names of importable modules. sys.extra_modules = importer.memory # Use custom implementation of traceback printer. |