diff options
author | orivej <orivej@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
commit | 718c552901d703c502ccbefdfc3c9028d608b947 (patch) | |
tree | 46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /library/python/runtime_py3 | |
parent | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff) | |
download | ydb-718c552901d703c502ccbefdfc3c9028d608b947.tar.gz |
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/python/runtime_py3')
-rw-r--r-- | library/python/runtime_py3/__res.pyx | 16 | ||||
-rw-r--r-- | library/python/runtime_py3/entry_points.py | 86 | ||||
-rw-r--r-- | library/python/runtime_py3/importer.pxi | 390 | ||||
-rw-r--r-- | library/python/runtime_py3/main/main.c | 64 | ||||
-rw-r--r-- | library/python/runtime_py3/main/ya.make | 14 | ||||
-rw-r--r-- | library/python/runtime_py3/sitecustomize.pyx | 2 | ||||
-rw-r--r-- | library/python/runtime_py3/ya.make | 20 |
7 files changed, 296 insertions, 296 deletions
diff --git a/library/python/runtime_py3/__res.pyx b/library/python/runtime_py3/__res.pyx index 97190d9f29..c0e17b7de8 100644 --- a/library/python/runtime_py3/__res.pyx +++ b/library/python/runtime_py3/__res.pyx @@ -1,5 +1,5 @@ -from _codecs import utf_8_decode, utf_8_encode - +from _codecs import utf_8_decode, utf_8_encode + from libcpp cimport bool from util.generic.string cimport TString, TStringBuf @@ -24,13 +24,13 @@ def key_by_index(idx): def find(s): cdef TString res - if isinstance(s, str): - s = utf_8_encode(s)[0] - + if isinstance(s, str): + s = utf_8_encode(s)[0] + if FindExact(TStringBuf(s, len(s)), &res): return res.c_str()[:res.length()] return None - - -include "importer.pxi" + + +include "importer.pxi" diff --git a/library/python/runtime_py3/entry_points.py b/library/python/runtime_py3/entry_points.py index 05098723cb..0639bc4aaf 100644 --- a/library/python/runtime_py3/entry_points.py +++ b/library/python/runtime_py3/entry_points.py @@ -1,47 +1,47 @@ -import sys - -import __res - - -def repl(): - user_ns = {} - py_main = __res.find('PY_MAIN') - - if py_main: - mod_name, func_name = (py_main.split(b':', 1) + [None])[:2] - try: - import importlib - mod = importlib.import_module(mod_name.decode('UTF-8')) - user_ns = mod.__dict__ - except: - import traceback - traceback.print_exc() - - if func_name and '__main__' not in user_ns: - def run(args): +import sys + +import __res + + +def repl(): + user_ns = {} + py_main = __res.find('PY_MAIN') + + if py_main: + mod_name, func_name = (py_main.split(b':', 1) + [None])[:2] + try: + import importlib + mod = importlib.import_module(mod_name.decode('UTF-8')) + user_ns = mod.__dict__ + except: + import traceback + traceback.print_exc() + + if func_name and '__main__' not in user_ns: + def run(args): if isinstance(args, str): - import shlex - args = shlex.split(args) - - import sys - sys.argv = [sys.argv[0]] + args - getattr(mod, func_name)() - - user_ns['__main__'] = run - - try: - import IPython - except ModuleNotFoundError: - pass - else: - return IPython.start_ipython(user_ns=user_ns) - - import code - code.interact(local=user_ns) - - -def resource_files(): - sys.stdout.buffer.write(b'\n'.join(sorted(__res.resfs_files()) + [b''])) + import shlex + args = shlex.split(args) + + import sys + sys.argv = [sys.argv[0]] + args + getattr(mod, func_name)() + + user_ns['__main__'] = run + + try: + import IPython + except ModuleNotFoundError: + pass + else: + return IPython.start_ipython(user_ns=user_ns) + + import code + code.interact(local=user_ns) + + +def resource_files(): + sys.stdout.buffer.write(b'\n'.join(sorted(__res.resfs_files()) + [b''])) def run_constructors(): diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi index 904f94dea2..7caa69e91e 100644 --- a/library/python/runtime_py3/importer.pxi +++ b/library/python/runtime_py3/importer.pxi @@ -1,30 +1,30 @@ -import marshal +import marshal import sys -from _codecs import utf_8_decode, utf_8_encode +from _codecs import utf_8_decode, utf_8_encode from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split -from _io import FileIO - +from _io import FileIO + import __res as __resource -_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] -_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] -env_entry_point = b'Y_PYTHON_ENTRY_POINT' -env_source_root = b'Y_PYTHON_SOURCE_ROOT' +_b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] +_s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] +env_entry_point = b'Y_PYTHON_ENTRY_POINT' +env_source_root = b'Y_PYTHON_SOURCE_ROOT' cfg_source_root = b'arcadia-source-root' env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' res_ya_ide_venv = b'YA_IDE_VENV' executable = sys.executable or 'Y_PYTHON' -sys.modules['run_import_hook'] = __resource - -# This is the prefix in contrib/tools/python3/src/Lib/ya.make. -py_prefix = b'py/' -py_prefix_len = len(py_prefix) +sys.modules['run_import_hook'] = __resource +# This is the prefix in contrib/tools/python3/src/Lib/ya.make. +py_prefix = b'py/' +py_prefix_len = len(py_prefix) + YA_IDE_VENV = __resource.find(res_ya_ide_venv) Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV - - + + def _init_venv(): if not _path_isabs(executable): raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) @@ -74,112 +74,112 @@ def _get_source_root(): Y_PYTHON_SOURCE_ROOT = _get_source_root() -def _print(*xs): - """ - This is helpful for debugging, since automatic bytes to str conversion is - not available yet. It is also possible to debug with GDB by breaking on - __Pyx_AddTraceback (with Python GDB pretty printers enabled). - """ - parts = [] - for s in xs: - if not isinstance(s, (bytes, str)): - s = str(s) - parts.append(_s(s)) - sys.stderr.write(' '.join(parts) + '\n') - - -def file_bytes(path): - # 'open' is not avaiable yet. - with FileIO(path, 'r') as f: - return f.read() - - -def iter_keys(prefix): - l = len(prefix) - for idx in range(__resource.count()): +def _print(*xs): + """ + This is helpful for debugging, since automatic bytes to str conversion is + not available yet. It is also possible to debug with GDB by breaking on + __Pyx_AddTraceback (with Python GDB pretty printers enabled). + """ + parts = [] + for s in xs: + if not isinstance(s, (bytes, str)): + s = str(s) + parts.append(_s(s)) + sys.stderr.write(' '.join(parts) + '\n') + + +def file_bytes(path): + # 'open' is not avaiable yet. + with FileIO(path, 'r') as f: + return f.read() + + +def iter_keys(prefix): + l = len(prefix) + for idx in range(__resource.count()): key = __resource.key_by_index(idx) - if key.startswith(prefix): - yield key, key[l:] - + if key.startswith(prefix): + yield key, key[l:] -def iter_py_modules(with_keys=False): - for key, path in iter_keys(b'resfs/file/' + py_prefix): - if path.endswith(b'.py'): # It may also end with '.pyc'. - mod = _s(path[:-3].replace(b'/', b'.')) - if with_keys: - yield key, mod - else: - yield mod + +def iter_py_modules(with_keys=False): + for key, path in iter_keys(b'resfs/file/' + py_prefix): + if path.endswith(b'.py'): # It may also end with '.pyc'. + mod = _s(path[:-3].replace(b'/', b'.')) + if with_keys: + yield key, mod + else: + yield mod -def iter_prefixes(s): - i = s.find('.') - while i >= 0: - yield s[:i] - i = s.find('.', i + 1) +def iter_prefixes(s): + i = s.find('.') + while i >= 0: + yield s[:i] + i = s.find('.', i + 1) def resfs_resolve(path): - """ - Return the absolute path of a root-relative path if it exists. - """ - path = _b(path) - if Y_PYTHON_SOURCE_ROOT: - if not path.startswith(Y_PYTHON_SOURCE_ROOT): - path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path)) - if _path_isfile(path): - return path - - -def resfs_src(key, resfs_file=False): - """ - Return the root-relative file path of a resource key. - """ - if resfs_file: - key = b'resfs/file/' + _b(key) - return __resource.find(b'resfs/src/' + _b(key)) - - + """ + Return the absolute path of a root-relative path if it exists. + """ + path = _b(path) + if Y_PYTHON_SOURCE_ROOT: + if not path.startswith(Y_PYTHON_SOURCE_ROOT): + path = _b(path_sep).join((Y_PYTHON_SOURCE_ROOT, path)) + if _path_isfile(path): + return path + + +def resfs_src(key, resfs_file=False): + """ + Return the root-relative file path of a resource key. + """ + if resfs_file: + key = b'resfs/file/' + _b(key) + return __resource.find(b'resfs/src/' + _b(key)) + + def resfs_read(path, builtin=None): + """ + Return the bytes of the resource file at path, or None. + If builtin is True, do not look for it on the filesystem. + If builtin is False, do not look in the builtin resources. + """ + if builtin is not True: + arcpath = resfs_src(path, resfs_file=True) + if arcpath: + fspath = resfs_resolve(arcpath) + if fspath: + return file_bytes(fspath) + + if builtin is not False: + return __resource.find(b'resfs/file/' + _b(path)) + + +def resfs_files(prefix=b''): + """ + List builtin resource file paths. + """ + return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))] + + +def mod_path(mod): """ - Return the bytes of the resource file at path, or None. - If builtin is True, do not look for it on the filesystem. - If builtin is False, do not look in the builtin resources. - """ - if builtin is not True: - arcpath = resfs_src(path, resfs_file=True) - if arcpath: - fspath = resfs_resolve(arcpath) - if fspath: - return file_bytes(fspath) - - if builtin is not False: - return __resource.find(b'resfs/file/' + _b(path)) - - -def resfs_files(prefix=b''): - """ - List builtin resource file paths. - """ - return [key[11:] for key, _ in iter_keys(b'resfs/file/' + _b(prefix))] - - -def mod_path(mod): - """ - Return the resfs path to the source code of the module with the given name. + Return the resfs path to the source code of the module with the given name. """ - return py_prefix + _b(mod).replace(b'.', b'/') + b'.py' - + return py_prefix + _b(mod).replace(b'.', b'/') + b'.py' + class ResourceImporter(object): - """ A meta_path importer that loads code from built-in resources. + """ A meta_path importer that loads code from built-in resources. """ def __init__(self): - self.memory = set(iter_py_modules()) # Set of importable module names. - self.source_map = {} # Map from file names to module names. - self._source_name = {} # Map from original to altered module names. + self.memory = set(iter_py_modules()) # Set of importable module names. + self.source_map = {} # Map from file names to module names. + self._source_name = {} # Map from original to altered module names. self._package_prefix = '' if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) @@ -205,11 +205,11 @@ class ResourceImporter(object): return relpath return self.arcadia_source_finder.get_module_path(fullname) - def find_spec(self, fullname, path=None, target=None): - try: + def find_spec(self, fullname, path=None, target=None): + try: is_package = self.is_package(fullname) - except ImportError: - return None + except ImportError: + return None return spec_from_loader(fullname, self, is_package=is_package) def find_module(self, fullname, path=None): @@ -224,103 +224,103 @@ class ResourceImporter(object): code = self.get_code(module.__name__) module.__file__ = code.co_filename if self.is_package(module.__name__): - module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)] - # exec(code, module.__dict__) + module.__path__= [executable + path_sep + module.__name__.replace('.', path_sep)] + # exec(code, module.__dict__) _call_with_frames_removed(exec, code, module.__dict__) - # PEP-302 extension 1 of 3: data loader. - def get_data(self, path): - path = _b(path) - abspath = resfs_resolve(path) - if abspath: - return file_bytes(abspath) + # PEP-302 extension 1 of 3: data loader. + def get_data(self, path): + path = _b(path) + abspath = resfs_resolve(path) + if abspath: + return file_bytes(abspath) path = path.replace(_b('\\'), _b('/')) - data = resfs_read(path, builtin=True) - if data is None: - raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files - return data - - # PEP-302 extension 2 of 3: get __file__ without importing. - def get_filename(self, fullname): - modname = fullname - if self.is_package(fullname): - fullname += '.__init__' + data = resfs_read(path, builtin=True) + if data is None: + raise IOError(path) # Y_PYTHON_ENTRY_POINT=:resource_files + return data + + # PEP-302 extension 2 of 3: get __file__ without importing. + def get_filename(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' relpath = self._find_mod_path(fullname) - if isinstance(relpath, bytes): - relpath = _s(relpath) + if isinstance(relpath, bytes): + relpath = _s(relpath) return relpath or modname - # PEP-302 extension 3 of 3: packaging introspection. - # Used by `linecache` (while printing tracebacks) unless module filename - # exists on the filesystem. - def get_source(self, fullname): + # PEP-302 extension 3 of 3: packaging introspection. + # Used by `linecache` (while printing tracebacks) unless module filename + # exists on the filesystem. + def get_source(self, fullname): fullname = self._source_name.get(fullname) or fullname if self.is_package(fullname): fullname += '.__init__' relpath = self.get_filename(fullname) - if relpath: - abspath = resfs_resolve(relpath) - if abspath: - return _s(file_bytes(abspath)) + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + return _s(file_bytes(abspath)) data = resfs_read(mod_path(fullname)) - return _s(data) if data else '' - - def get_code(self, fullname): - modname = fullname - if self.is_package(fullname): - fullname += '.__init__' - - path = mod_path(fullname) + return _s(data) if data else '' + + def get_code(self, fullname): + modname = fullname + if self.is_package(fullname): + fullname += '.__init__' + + path = mod_path(fullname) relpath = self._find_mod_path(fullname) - if relpath: - abspath = resfs_resolve(relpath) - if abspath: - data = file_bytes(abspath) - return compile(data, _s(abspath), 'exec', dont_inherit=True) - + if relpath: + abspath = resfs_resolve(relpath) + if abspath: + data = file_bytes(abspath) + return compile(data, _s(abspath), 'exec', dont_inherit=True) + yapyc_path = path + b'.yapyc3' yapyc_data = resfs_read(yapyc_path, builtin=True) if yapyc_data: return marshal.loads(yapyc_data) else: py_data = resfs_read(path, builtin=True) - if py_data: - return compile(py_data, _s(relpath), 'exec', dont_inherit=True) - else: - # This covers packages with no __init__.py in resources. - return compile('', modname, 'exec', dont_inherit=True) + if py_data: + return compile(py_data, _s(relpath), 'exec', dont_inherit=True) + else: + # This covers packages with no __init__.py in resources. + return compile('', modname, 'exec', dont_inherit=True) def is_package(self, fullname): if fullname in self.memory: return False - - if fullname + '.__init__' in self.memory: + + if fullname + '.__init__' in self.memory: return True if self.arcadia_source_finder: return self.arcadia_source_finder.is_package(fullname) - raise ImportError(fullname) - - # Extension for contrib/python/coverage. - def file_source(self, filename): - """ - Return the key of the module source by its resource path. - """ - if not self.source_map: - for key, mod in iter_py_modules(with_keys=True): - path = self.get_filename(mod) - self.source_map[path] = key - - if filename in self.source_map: - return self.source_map[filename] - - if resfs_read(filename, builtin=True) is not None: - return b'resfs/file/' + _b(filename) - - return b'' - + raise ImportError(fullname) + + # Extension for contrib/python/coverage. + def file_source(self, filename): + """ + Return the key of the module source by its resource path. + """ + if not self.source_map: + for key, mod in iter_py_modules(with_keys=True): + path = self.get_filename(mod) + self.source_map[path] = key + + if filename in self.source_map: + return self.source_map[filename] + + if resfs_read(filename, builtin=True) is not None: + return b'resfs/file/' + _b(filename) + + return b'' + # Extension for pkgutil.iter_modules. def iter_modules(self, prefix=''): import re @@ -531,19 +531,19 @@ def excepthook(*args, **kws): return traceback.print_exception(*args, **kws) -importer = ResourceImporter() - - -def executable_path_hook(path): - if path == executable: - return importer - - if path.startswith(executable + path_sep): - return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.')) - - raise ImportError(path) +importer = ResourceImporter() + +def executable_path_hook(path): + if path == executable: + return importer + + if path.startswith(executable + path_sep): + return importer.for_package(path[len(executable + path_sep):].replace(path_sep, '.')) + raise ImportError(path) + + if YA_IDE_VENV: sys.meta_path.append(importer) sys.meta_path.append(BuiltinSubmoduleImporter) @@ -557,13 +557,13 @@ else: sys.path.insert(0, executable) sys.path_hooks.insert(0, executable_path_hook) -sys.path_importer_cache[executable] = importer - -# Indicator that modules and resources are built-in rather than on the file system. -sys.is_standalone_binary = True -sys.frozen = True +sys.path_importer_cache[executable] = importer + +# Indicator that modules and resources are built-in rather than on the file system. +sys.is_standalone_binary = True +sys.frozen = True -# Set of names of importable modules. +# Set of names of importable modules. sys.extra_modules = importer.memory # Use custom implementation of traceback printer. diff --git a/library/python/runtime_py3/main/main.c b/library/python/runtime_py3/main/main.c index 3159800615..55afdca5f3 100644 --- a/library/python/runtime_py3/main/main.c +++ b/library/python/runtime_py3/main/main.c @@ -1,29 +1,29 @@ -#include <Python.h> +#include <Python.h> #include <contrib/tools/python3/src/Include/internal/pycore_runtime.h> // _PyRuntime_Initialize() - -#include <stdlib.h> -#include <string.h> + +#include <stdlib.h> +#include <string.h> #include <locale.h> - -void Py_InitArgcArgv(int argc, wchar_t **argv); + +void Py_InitArgcArgv(int argc, wchar_t **argv); char* GetPyMain(); - -static const char* env_entry_point = "Y_PYTHON_ENTRY_POINT"; + +static const char* env_entry_point = "Y_PYTHON_ENTRY_POINT"; static const char* env_bytes_warning = "Y_PYTHON_BYTES_WARNING"; - -#ifdef _MSC_VER -extern char** environ; - -void unsetenv(const char* name) { - const int n = strlen(name); + +#ifdef _MSC_VER +extern char** environ; + +void unsetenv(const char* name) { + const int n = strlen(name); char** dst = environ; for (char** src = environ; *src; src++) - if (strncmp(*src, name, n) || (*src)[n] != '=') - *dst++ = *src; - *dst = NULL; -} -#endif - + if (strncmp(*src, name, n) || (*src)[n] != '=') + *dst++ = *src; + *dst = NULL; +} +#endif + static int RunModule(const char *modname) { PyObject *module, *runpy, *runmodule, *runargs, *result; @@ -73,7 +73,7 @@ static int RunModule(const char *modname) return 0; } -static int pymain(int argc, char** argv) { +static int pymain(int argc, char** argv) { PyStatus status = _PyRuntime_Initialize(); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); @@ -134,7 +134,7 @@ static int pymain(int argc, char** argv) { Py_ExitStatusException(status); } - const char* entry_point = getenv(env_entry_point); + const char* entry_point = getenv(env_entry_point); if (entry_point) { entry_point_copy = strdup(entry_point); if (!entry_point_copy) { @@ -151,13 +151,13 @@ static int pymain(int argc, char** argv) { } if (entry_point_copy && !strcmp(entry_point_copy, ":main")) { - unsetenv(env_entry_point); + unsetenv(env_entry_point); sts = Py_Main(argc, argv_copy); free(entry_point_copy); return sts; - } + } - Py_InitArgcArgv(argc, argv_copy); + Py_InitArgcArgv(argc, argv_copy); PySys_SetArgv(argc, argv_copy); { @@ -222,10 +222,10 @@ error: } PyMem_RawFree(oldloc); return sts; -} - -int (*mainptr)(int argc, char** argv) = pymain; - -int main(int argc, char** argv) { - return mainptr(argc, argv); -} +} + +int (*mainptr)(int argc, char** argv) = pymain; + +int main(int argc, char** argv) { + return mainptr(argc, argv); +} diff --git a/library/python/runtime_py3/main/ya.make b/library/python/runtime_py3/main/ya.make index f308a93b28..3ffe3e0eab 100644 --- a/library/python/runtime_py3/main/ya.make +++ b/library/python/runtime_py3/main/ya.make @@ -6,21 +6,21 @@ OWNER( ) PEERDIR( - contrib/tools/python3/src + contrib/tools/python3/src library/cpp/resource ) -ADDINCL( - contrib/tools/python3/src/Include -) +ADDINCL( + contrib/tools/python3/src/Include +) CFLAGS( -DPy_BUILD_CORE ) -SRCS( - main.c +SRCS( + main.c get_py_main.cpp -) +) END() diff --git a/library/python/runtime_py3/sitecustomize.pyx b/library/python/runtime_py3/sitecustomize.pyx index 966bbe8ba6..625ccffb79 100644 --- a/library/python/runtime_py3/sitecustomize.pyx +++ b/library/python/runtime_py3/sitecustomize.pyx @@ -2,7 +2,7 @@ import re import sys import __res - + from importlib.abc import ResourceReader from importlib.metadata import Distribution, DistributionFinder, PackageNotFoundError, Prepared diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make index fa5c11341a..37e04ffd5f 100644 --- a/library/python/runtime_py3/ya.make +++ b/library/python/runtime_py3/ya.make @@ -9,19 +9,19 @@ OWNER( NO_WSHADOW() PEERDIR( - contrib/tools/python3/src - contrib/tools/python3/lib/py + contrib/tools/python3/src + contrib/tools/python3/lib/py library/cpp/resource ) CFLAGS(-DCYTHON_REGISTER_ABCS=0) - -NO_PYTHON_INCLUDES() - -ENABLE(PYBUILD_NO_PYC) - + +NO_PYTHON_INCLUDES() + +ENABLE(PYBUILD_NO_PYC) + PY_SRCS( - entry_points.py + entry_points.py TOP_LEVEL CYTHON_DIRECTIVE @@ -29,7 +29,7 @@ PY_SRCS( __res.pyx sitecustomize.pyx -) +) IF (CYTHON_COVERAGE) # Let covarage support add all needed files to resources @@ -41,7 +41,7 @@ ELSE() sitecustomize.pyx ) ENDIF() - + END() RECURSE_FOR_TESTS( |