diff options
author | Aleksandr <[email protected]> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:47:52 +0300 |
commit | b05913d1c3c02a773578bceb7285084d2933ae86 (patch) | |
tree | c0748b5dcbade83af788c0abfa89c0383d6b779c /contrib/tools/cython/Cython/Build/IpythonMagic.py | |
parent | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (diff) |
Restoring authorship annotation for Aleksandr <[email protected]>. Commit 2 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Build/IpythonMagic.py')
-rw-r--r-- | contrib/tools/cython/Cython/Build/IpythonMagic.py | 480 |
1 files changed, 240 insertions, 240 deletions
diff --git a/contrib/tools/cython/Cython/Build/IpythonMagic.py b/contrib/tools/cython/Cython/Build/IpythonMagic.py index 3b56be5525a..7abb97ec70a 100644 --- a/contrib/tools/cython/Cython/Build/IpythonMagic.py +++ b/contrib/tools/cython/Cython/Build/IpythonMagic.py @@ -52,13 +52,13 @@ import os import re import sys import time -import copy -import distutils.log -import textwrap +import copy +import distutils.log +import textwrap IO_ENCODING = sys.getfilesystemencoding() IS_PY2 = sys.version_info[0] < 3 - + try: reload except NameError: # Python 3 @@ -88,20 +88,20 @@ from .Inline import cython_inline from .Dependencies import cythonize -PGO_CONFIG = { - 'gcc': { - 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'], - 'use': ['-fprofile-use', '-fprofile-correction', '-fprofile-dir={TEMPDIR}'], - }, - # blind copy from 'configure' script in CPython 3.7 - 'icc': { - 'gen': ['-prof-gen'], - 'use': ['-prof-use'], - } -} -PGO_CONFIG['mingw32'] = PGO_CONFIG['gcc'] - - +PGO_CONFIG = { + 'gcc': { + 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'], + 'use': ['-fprofile-use', '-fprofile-correction', '-fprofile-dir={TEMPDIR}'], + }, + # blind copy from 'configure' script in CPython 3.7 + 'icc': { + 'gen': ['-prof-gen'], + 'use': ['-prof-use'], + } +} +PGO_CONFIG['mingw32'] = PGO_CONFIG['gcc'] + + if IS_PY2: def encode_fs(name): return name if isinstance(name, bytes) else name.encode(IO_ENCODING) @@ -114,25 +114,25 @@ else: class CythonMagics(Magics): def __init__(self, shell): - super(CythonMagics, self).__init__(shell) + super(CythonMagics, self).__init__(shell) self._reloads = {} self._code_cache = {} self._pyximport_installed = False def _import_all(self, module): - mdict = module.__dict__ - if '__all__' in mdict: - keys = mdict['__all__'] - else: - keys = [k for k in mdict if not k.startswith('_')] - - for k in keys: - try: - self.shell.push({k: mdict[k]}) - except KeyError: - msg = "'module' object has no attribute '%s'" % k - raise AttributeError(msg) - + mdict = module.__dict__ + if '__all__' in mdict: + keys = mdict['__all__'] + else: + keys = [k for k in mdict if not k.startswith('_')] + + for k in keys: + try: + self.shell.push({k: mdict[k]}) + except KeyError: + msg = "'module' object has no attribute '%s'" % k + raise AttributeError(msg) + @cell_magic def cython_inline(self, line, cell): """Compile and run a Cython code cell using Cython.inline. @@ -192,14 +192,14 @@ class CythonMagics(Magics): @magic_arguments.magic_arguments() @magic_arguments.argument( - '-a', '--annotate', action='store_true', default=False, - help="Produce a colorized HTML version of the source." - ) - @magic_arguments.argument( - '-+', '--cplus', action='store_true', default=False, - help="Output a C++ rather than C file." - ) - @magic_arguments.argument( + '-a', '--annotate', action='store_true', default=False, + help="Produce a colorized HTML version of the source." + ) + @magic_arguments.argument( + '-+', '--cplus', action='store_true', default=False, + help="Output a C++ rather than C file." + ) + @magic_arguments.argument( '-3', dest='language_level', action='store_const', const=3, default=None, help="Select Python 3 syntax." ) @@ -208,11 +208,11 @@ class CythonMagics(Magics): help="Select Python 2 syntax." ) @magic_arguments.argument( - '-f', '--force', action='store_true', default=False, - help="Force the compilation of a new module, even if the source has been " - "previously compiled." - ) - @magic_arguments.argument( + '-f', '--force', action='store_true', default=False, + help="Force the compilation of a new module, even if the source has been " + "previously compiled." + ) + @magic_arguments.argument( '-c', '--compile-args', action='append', default=[], help="Extra flags to pass to compiler via the `extra_compile_args` " "Extension flag (can be specified multiple times)." @@ -242,19 +242,19 @@ class CythonMagics(Magics): "multiple times)." ) @magic_arguments.argument( - '-S', '--src', action='append', default=[], - help="Add a path to the list of src files (can be specified " - "multiple times)." + '-S', '--src', action='append', default=[], + help="Add a path to the list of src files (can be specified " + "multiple times)." ) @magic_arguments.argument( - '--pgo', dest='pgo', action='store_true', default=False, - help=("Enable profile guided optimisation in the C compiler. " - "Compiles the cell twice and executes it in between to generate a runtime profile.") + '--pgo', dest='pgo', action='store_true', default=False, + help=("Enable profile guided optimisation in the C compiler. " + "Compiles the cell twice and executes it in between to generate a runtime profile.") ) @magic_arguments.argument( - '--verbose', dest='quiet', action='store_false', default=True, - help=("Print debug information like generated .c/.cpp file location " - "and exact gcc/g++ command invoked.") + '--verbose', dest='quiet', action='store_false', default=True, + help=("Print debug information like generated .c/.cpp file location " + "and exact gcc/g++ command invoked.") ) @cell_magic def cython(self, line, cell): @@ -276,78 +276,78 @@ class CythonMagics(Magics): %%cython --compile-args=-fopenmp --link-args=-fopenmp ... - - To enable profile guided optimisation, pass the ``--pgo`` option. - Note that the cell itself needs to take care of establishing a suitable - profile when executed. This can be done by implementing the functions to - optimise, and then calling them directly in the same cell on some realistic - training data like this:: - - %%cython --pgo - def critical_function(data): - for item in data: - ... - - # execute function several times to build profile - from somewhere import some_typical_data - for _ in range(100): - critical_function(some_typical_data) - - In Python 3.5 and later, you can distinguish between the profile and - non-profile runs as follows:: - - if "_pgo_" in __name__: - ... # execute critical code here + + To enable profile guided optimisation, pass the ``--pgo`` option. + Note that the cell itself needs to take care of establishing a suitable + profile when executed. This can be done by implementing the functions to + optimise, and then calling them directly in the same cell on some realistic + training data like this:: + + %%cython --pgo + def critical_function(data): + for item in data: + ... + + # execute function several times to build profile + from somewhere import some_typical_data + for _ in range(100): + critical_function(some_typical_data) + + In Python 3.5 and later, you can distinguish between the profile and + non-profile runs as follows:: + + if "_pgo_" in __name__: + ... # execute critical code here """ args = magic_arguments.parse_argstring(self.cython, line) - code = cell if cell.endswith('\n') else cell + '\n' + code = cell if cell.endswith('\n') else cell + '\n' lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') - key = (code, line, sys.version_info, sys.executable, cython_version) + key = (code, line, sys.version_info, sys.executable, cython_version) if not os.path.exists(lib_dir): os.makedirs(lib_dir) - if args.pgo: - key += ('pgo',) + if args.pgo: + key += ('pgo',) if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. - key += (time.time(),) + key += (time.time(),) if args.name: module_name = str(args.name) # no-op in Py3 else: module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() - html_file = os.path.join(lib_dir, module_name + '.html') + html_file = os.path.join(lib_dir, module_name + '.html') module_path = os.path.join(lib_dir, module_name + self.so_ext) have_module = os.path.isfile(module_path) - need_cythonize = args.pgo or not have_module + need_cythonize = args.pgo or not have_module if args.annotate: if not os.path.isfile(html_file): need_cythonize = True - extension = None + extension = None if need_cythonize: - extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet) + extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet) if extensions is None: # Compilation failed and printed error message return None - assert len(extensions) == 1 - extension = extensions[0] + assert len(extensions) == 1 + extension = extensions[0] self._code_cache[key] = module_name - if args.pgo: - self._profile_pgo_wrapper(extension, lib_dir) - + if args.pgo: + self._profile_pgo_wrapper(extension, lib_dir) + try: self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet) except distutils.errors.CompileError: # Build failed and printed error message return None - + module = imp.load_dynamic(module_name, module_path) self._import_all(module) @@ -366,129 +366,129 @@ class CythonMagics(Magics): else: return display.HTML(self.clean_annotated_html(annotated_html)) - def _profile_pgo_wrapper(self, extension, lib_dir): - """ - Generate a .c file for a separate extension module that calls the - module init function of the original module. This makes sure that the - PGO profiler sees the correct .o file of the final module, but it still - allows us to import the module under a different name for profiling, - before recompiling it into the PGO optimised module. Overwriting and - reimporting the same shared library is not portable. - """ - extension = copy.copy(extension) # shallow copy, do not modify sources in place! - module_name = extension.name - pgo_module_name = '_pgo_' + module_name - pgo_wrapper_c_file = os.path.join(lib_dir, pgo_module_name + '.c') - with io.open(pgo_wrapper_c_file, 'w', encoding='utf-8') as f: - f.write(textwrap.dedent(u""" - #include "Python.h" - #if PY_MAJOR_VERSION < 3 - extern PyMODINIT_FUNC init%(module_name)s(void); - PyMODINIT_FUNC init%(pgo_module_name)s(void); /*proto*/ - PyMODINIT_FUNC init%(pgo_module_name)s(void) { - PyObject *sys_modules; - init%(module_name)s(); if (PyErr_Occurred()) return; - sys_modules = PyImport_GetModuleDict(); /* borrowed, no exception, "never" fails */ - if (sys_modules) { - PyObject *module = PyDict_GetItemString(sys_modules, "%(module_name)s"); if (!module) return; - PyDict_SetItemString(sys_modules, "%(pgo_module_name)s", module); - Py_DECREF(module); - } - } - #else - extern PyMODINIT_FUNC PyInit_%(module_name)s(void); - PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void); /*proto*/ - PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void) { - return PyInit_%(module_name)s(); - } - #endif - """ % {'module_name': module_name, 'pgo_module_name': pgo_module_name})) - - extension.sources = extension.sources + [pgo_wrapper_c_file] # do not modify in place! - extension.name = pgo_module_name - - self._build_extension(extension, lib_dir, pgo_step_name='gen') - - # import and execute module code to generate profile - so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext) - imp.load_dynamic(pgo_module_name, so_module_path) - - def _cythonize(self, module_name, code, lib_dir, args, quiet=True): - pyx_file = os.path.join(lib_dir, module_name + '.pyx') + def _profile_pgo_wrapper(self, extension, lib_dir): + """ + Generate a .c file for a separate extension module that calls the + module init function of the original module. This makes sure that the + PGO profiler sees the correct .o file of the final module, but it still + allows us to import the module under a different name for profiling, + before recompiling it into the PGO optimised module. Overwriting and + reimporting the same shared library is not portable. + """ + extension = copy.copy(extension) # shallow copy, do not modify sources in place! + module_name = extension.name + pgo_module_name = '_pgo_' + module_name + pgo_wrapper_c_file = os.path.join(lib_dir, pgo_module_name + '.c') + with io.open(pgo_wrapper_c_file, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(u""" + #include "Python.h" + #if PY_MAJOR_VERSION < 3 + extern PyMODINIT_FUNC init%(module_name)s(void); + PyMODINIT_FUNC init%(pgo_module_name)s(void); /*proto*/ + PyMODINIT_FUNC init%(pgo_module_name)s(void) { + PyObject *sys_modules; + init%(module_name)s(); if (PyErr_Occurred()) return; + sys_modules = PyImport_GetModuleDict(); /* borrowed, no exception, "never" fails */ + if (sys_modules) { + PyObject *module = PyDict_GetItemString(sys_modules, "%(module_name)s"); if (!module) return; + PyDict_SetItemString(sys_modules, "%(pgo_module_name)s", module); + Py_DECREF(module); + } + } + #else + extern PyMODINIT_FUNC PyInit_%(module_name)s(void); + PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void); /*proto*/ + PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void) { + return PyInit_%(module_name)s(); + } + #endif + """ % {'module_name': module_name, 'pgo_module_name': pgo_module_name})) + + extension.sources = extension.sources + [pgo_wrapper_c_file] # do not modify in place! + extension.name = pgo_module_name + + self._build_extension(extension, lib_dir, pgo_step_name='gen') + + # import and execute module code to generate profile + so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext) + imp.load_dynamic(pgo_module_name, so_module_path) + + def _cythonize(self, module_name, code, lib_dir, args, quiet=True): + pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = encode_fs(pyx_file) - - c_include_dirs = args.include - c_src_files = list(map(str, args.src)) - if 'numpy' in code: - import numpy - c_include_dirs.append(numpy.get_include()) - with io.open(pyx_file, 'w', encoding='utf-8') as f: - f.write(code) - extension = Extension( - name=module_name, - sources=[pyx_file] + c_src_files, - include_dirs=c_include_dirs, - library_dirs=args.library_dirs, - extra_compile_args=args.compile_args, - extra_link_args=args.link_args, - libraries=args.lib, - language='c++' if args.cplus else 'c', - ) - try: - opts = dict( - quiet=quiet, - annotate=args.annotate, - force=True, - ) - if args.language_level is not None: - assert args.language_level in (2, 3) - opts['language_level'] = args.language_level - elif sys.version_info[0] >= 3: - opts['language_level'] = 3 - return cythonize([extension], **opts) - except CompileError: - return None - - def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True): - build_extension = self._get_build_extension( - extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name) - old_threshold = None - try: - if not quiet: - old_threshold = distutils.log.set_threshold(distutils.log.DEBUG) - build_extension.run() - finally: - if not quiet and old_threshold is not None: - distutils.log.set_threshold(old_threshold) - - def _add_pgo_flags(self, build_extension, step_name, temp_dir): - compiler_type = build_extension.compiler.compiler_type - if compiler_type == 'unix': - compiler_cmd = build_extension.compiler.compiler_so - # TODO: we could try to call "[cmd] --version" for better insights - if not compiler_cmd: - pass - elif 'clang' in compiler_cmd or 'clang' in compiler_cmd[0]: - compiler_type = 'clang' - elif 'icc' in compiler_cmd or 'icc' in compiler_cmd[0]: - compiler_type = 'icc' - elif 'gcc' in compiler_cmd or 'gcc' in compiler_cmd[0]: - compiler_type = 'gcc' - elif 'g++' in compiler_cmd or 'g++' in compiler_cmd[0]: - compiler_type = 'gcc' - config = PGO_CONFIG.get(compiler_type) - orig_flags = [] - if config and step_name in config: - flags = [f.format(TEMPDIR=temp_dir) for f in config[step_name]] - for extension in build_extension.extensions: - orig_flags.append((extension.extra_compile_args, extension.extra_link_args)) - extension.extra_compile_args = extension.extra_compile_args + flags - extension.extra_link_args = extension.extra_link_args + flags - else: - print("No PGO %s configuration known for C compiler type '%s'" % (step_name, compiler_type), - file=sys.stderr) - return orig_flags - + + c_include_dirs = args.include + c_src_files = list(map(str, args.src)) + if 'numpy' in code: + import numpy + c_include_dirs.append(numpy.get_include()) + with io.open(pyx_file, 'w', encoding='utf-8') as f: + f.write(code) + extension = Extension( + name=module_name, + sources=[pyx_file] + c_src_files, + include_dirs=c_include_dirs, + library_dirs=args.library_dirs, + extra_compile_args=args.compile_args, + extra_link_args=args.link_args, + libraries=args.lib, + language='c++' if args.cplus else 'c', + ) + try: + opts = dict( + quiet=quiet, + annotate=args.annotate, + force=True, + ) + if args.language_level is not None: + assert args.language_level in (2, 3) + opts['language_level'] = args.language_level + elif sys.version_info[0] >= 3: + opts['language_level'] = 3 + return cythonize([extension], **opts) + except CompileError: + return None + + def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True): + build_extension = self._get_build_extension( + extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name) + old_threshold = None + try: + if not quiet: + old_threshold = distutils.log.set_threshold(distutils.log.DEBUG) + build_extension.run() + finally: + if not quiet and old_threshold is not None: + distutils.log.set_threshold(old_threshold) + + def _add_pgo_flags(self, build_extension, step_name, temp_dir): + compiler_type = build_extension.compiler.compiler_type + if compiler_type == 'unix': + compiler_cmd = build_extension.compiler.compiler_so + # TODO: we could try to call "[cmd] --version" for better insights + if not compiler_cmd: + pass + elif 'clang' in compiler_cmd or 'clang' in compiler_cmd[0]: + compiler_type = 'clang' + elif 'icc' in compiler_cmd or 'icc' in compiler_cmd[0]: + compiler_type = 'icc' + elif 'gcc' in compiler_cmd or 'gcc' in compiler_cmd[0]: + compiler_type = 'gcc' + elif 'g++' in compiler_cmd or 'g++' in compiler_cmd[0]: + compiler_type = 'gcc' + config = PGO_CONFIG.get(compiler_type) + orig_flags = [] + if config and step_name in config: + flags = [f.format(TEMPDIR=temp_dir) for f in config[step_name]] + for extension in build_extension.extensions: + orig_flags.append((extension.extra_compile_args, extension.extra_link_args)) + extension.extra_compile_args = extension.extra_compile_args + flags + extension.extra_link_args = extension.extra_link_args + flags + else: + print("No PGO %s configuration known for C compiler type '%s'" % (step_name, compiler_type), + file=sys.stderr) + return orig_flags + @property def so_ext(self): """The extension suffix for compiled modules.""" @@ -510,8 +510,8 @@ class CythonMagics(Magics): else: _path_created.clear() - def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None, - pgo_step_name=None, _build_ext=build_ext): + def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None, + pgo_step_name=None, _build_ext=build_ext): self._clear_distutils_mkpath_cache() dist = Distribution() config_files = dist.find_config_files() @@ -520,28 +520,28 @@ class CythonMagics(Magics): except ValueError: pass dist.parse_config_files(config_files) - - if not temp_dir: - temp_dir = lib_dir - add_pgo_flags = self._add_pgo_flags - - if pgo_step_name: - base_build_ext = _build_ext - class _build_ext(_build_ext): - def build_extensions(self): - add_pgo_flags(self, pgo_step_name, temp_dir) - base_build_ext.build_extensions(self) - - build_extension = _build_ext(dist) + + if not temp_dir: + temp_dir = lib_dir + add_pgo_flags = self._add_pgo_flags + + if pgo_step_name: + base_build_ext = _build_ext + class _build_ext(_build_ext): + def build_extensions(self): + add_pgo_flags(self, pgo_step_name, temp_dir) + base_build_ext.build_extensions(self) + + build_extension = _build_ext(dist) build_extension.finalize_options() - if temp_dir: + if temp_dir: temp_dir = encode_fs(temp_dir) - build_extension.build_temp = temp_dir - if lib_dir: + build_extension.build_temp = temp_dir + if lib_dir: lib_dir = encode_fs(lib_dir) - build_extension.build_lib = lib_dir - if extension is not None: - build_extension.extensions = [extension] + build_extension.build_lib = lib_dir + if extension is not None: + build_extension.extensions = [extension] return build_extension @staticmethod @@ -556,10 +556,10 @@ class CythonMagics(Magics): return html __doc__ = __doc__.format( - # rST doesn't see the -+ flag as part of an option list, so we - # hide it from the module-level docstring. - CYTHON_DOC=dedent(CythonMagics.cython.__doc__\ - .replace('-+, --cplus', '--cplus ')), - CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__), - CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__), + # rST doesn't see the -+ flag as part of an option list, so we + # hide it from the module-level docstring. + CYTHON_DOC=dedent(CythonMagics.cython.__doc__\ + .replace('-+, --cplus', '--cplus ')), + CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__), + CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__), ) |