summaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Build/IpythonMagic.py
diff options
context:
space:
mode:
authorAleksandr <[email protected]>2022-02-10 16:47:52 +0300
committerDaniil Cherednik <[email protected]>2022-02-10 16:47:52 +0300
commitb05913d1c3c02a773578bceb7285084d2933ae86 (patch)
treec0748b5dcbade83af788c0abfa89c0383d6b779c /contrib/tools/cython/Cython/Build/IpythonMagic.py
parentea6c5b7f172becca389cacaff7d5f45f6adccbe6 (diff)
Restoring authorship annotation for Aleksandr <[email protected]>. Commit 2 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Build/IpythonMagic.py')
-rw-r--r--contrib/tools/cython/Cython/Build/IpythonMagic.py480
1 files changed, 240 insertions, 240 deletions
diff --git a/contrib/tools/cython/Cython/Build/IpythonMagic.py b/contrib/tools/cython/Cython/Build/IpythonMagic.py
index 3b56be5525a..7abb97ec70a 100644
--- a/contrib/tools/cython/Cython/Build/IpythonMagic.py
+++ b/contrib/tools/cython/Cython/Build/IpythonMagic.py
@@ -52,13 +52,13 @@ import os
import re
import sys
import time
-import copy
-import distutils.log
-import textwrap
+import copy
+import distutils.log
+import textwrap
IO_ENCODING = sys.getfilesystemencoding()
IS_PY2 = sys.version_info[0] < 3
-
+
try:
reload
except NameError: # Python 3
@@ -88,20 +88,20 @@ from .Inline import cython_inline
from .Dependencies import cythonize
-PGO_CONFIG = {
- 'gcc': {
- 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'],
- 'use': ['-fprofile-use', '-fprofile-correction', '-fprofile-dir={TEMPDIR}'],
- },
- # blind copy from 'configure' script in CPython 3.7
- 'icc': {
- 'gen': ['-prof-gen'],
- 'use': ['-prof-use'],
- }
-}
-PGO_CONFIG['mingw32'] = PGO_CONFIG['gcc']
-
-
+PGO_CONFIG = {
+ 'gcc': {
+ 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'],
+ 'use': ['-fprofile-use', '-fprofile-correction', '-fprofile-dir={TEMPDIR}'],
+ },
+ # blind copy from 'configure' script in CPython 3.7
+ 'icc': {
+ 'gen': ['-prof-gen'],
+ 'use': ['-prof-use'],
+ }
+}
+PGO_CONFIG['mingw32'] = PGO_CONFIG['gcc']
+
+
if IS_PY2:
def encode_fs(name):
return name if isinstance(name, bytes) else name.encode(IO_ENCODING)
@@ -114,25 +114,25 @@ else:
class CythonMagics(Magics):
def __init__(self, shell):
- super(CythonMagics, self).__init__(shell)
+ super(CythonMagics, self).__init__(shell)
self._reloads = {}
self._code_cache = {}
self._pyximport_installed = False
def _import_all(self, module):
- mdict = module.__dict__
- if '__all__' in mdict:
- keys = mdict['__all__']
- else:
- keys = [k for k in mdict if not k.startswith('_')]
-
- for k in keys:
- try:
- self.shell.push({k: mdict[k]})
- except KeyError:
- msg = "'module' object has no attribute '%s'" % k
- raise AttributeError(msg)
-
+ mdict = module.__dict__
+ if '__all__' in mdict:
+ keys = mdict['__all__']
+ else:
+ keys = [k for k in mdict if not k.startswith('_')]
+
+ for k in keys:
+ try:
+ self.shell.push({k: mdict[k]})
+ except KeyError:
+ msg = "'module' object has no attribute '%s'" % k
+ raise AttributeError(msg)
+
@cell_magic
def cython_inline(self, line, cell):
"""Compile and run a Cython code cell using Cython.inline.
@@ -192,14 +192,14 @@ class CythonMagics(Magics):
@magic_arguments.magic_arguments()
@magic_arguments.argument(
- '-a', '--annotate', action='store_true', default=False,
- help="Produce a colorized HTML version of the source."
- )
- @magic_arguments.argument(
- '-+', '--cplus', action='store_true', default=False,
- help="Output a C++ rather than C file."
- )
- @magic_arguments.argument(
+ '-a', '--annotate', action='store_true', default=False,
+ help="Produce a colorized HTML version of the source."
+ )
+ @magic_arguments.argument(
+ '-+', '--cplus', action='store_true', default=False,
+ help="Output a C++ rather than C file."
+ )
+ @magic_arguments.argument(
'-3', dest='language_level', action='store_const', const=3, default=None,
help="Select Python 3 syntax."
)
@@ -208,11 +208,11 @@ class CythonMagics(Magics):
help="Select Python 2 syntax."
)
@magic_arguments.argument(
- '-f', '--force', action='store_true', default=False,
- help="Force the compilation of a new module, even if the source has been "
- "previously compiled."
- )
- @magic_arguments.argument(
+ '-f', '--force', action='store_true', default=False,
+ help="Force the compilation of a new module, even if the source has been "
+ "previously compiled."
+ )
+ @magic_arguments.argument(
'-c', '--compile-args', action='append', default=[],
help="Extra flags to pass to compiler via the `extra_compile_args` "
"Extension flag (can be specified multiple times)."
@@ -242,19 +242,19 @@ class CythonMagics(Magics):
"multiple times)."
)
@magic_arguments.argument(
- '-S', '--src', action='append', default=[],
- help="Add a path to the list of src files (can be specified "
- "multiple times)."
+ '-S', '--src', action='append', default=[],
+ help="Add a path to the list of src files (can be specified "
+ "multiple times)."
)
@magic_arguments.argument(
- '--pgo', dest='pgo', action='store_true', default=False,
- help=("Enable profile guided optimisation in the C compiler. "
- "Compiles the cell twice and executes it in between to generate a runtime profile.")
+ '--pgo', dest='pgo', action='store_true', default=False,
+ help=("Enable profile guided optimisation in the C compiler. "
+ "Compiles the cell twice and executes it in between to generate a runtime profile.")
)
@magic_arguments.argument(
- '--verbose', dest='quiet', action='store_false', default=True,
- help=("Print debug information like generated .c/.cpp file location "
- "and exact gcc/g++ command invoked.")
+ '--verbose', dest='quiet', action='store_false', default=True,
+ help=("Print debug information like generated .c/.cpp file location "
+ "and exact gcc/g++ command invoked.")
)
@cell_magic
def cython(self, line, cell):
@@ -276,78 +276,78 @@ class CythonMagics(Magics):
%%cython --compile-args=-fopenmp --link-args=-fopenmp
...
-
- To enable profile guided optimisation, pass the ``--pgo`` option.
- Note that the cell itself needs to take care of establishing a suitable
- profile when executed. This can be done by implementing the functions to
- optimise, and then calling them directly in the same cell on some realistic
- training data like this::
-
- %%cython --pgo
- def critical_function(data):
- for item in data:
- ...
-
- # execute function several times to build profile
- from somewhere import some_typical_data
- for _ in range(100):
- critical_function(some_typical_data)
-
- In Python 3.5 and later, you can distinguish between the profile and
- non-profile runs as follows::
-
- if "_pgo_" in __name__:
- ... # execute critical code here
+
+ To enable profile guided optimisation, pass the ``--pgo`` option.
+ Note that the cell itself needs to take care of establishing a suitable
+ profile when executed. This can be done by implementing the functions to
+ optimise, and then calling them directly in the same cell on some realistic
+ training data like this::
+
+ %%cython --pgo
+ def critical_function(data):
+ for item in data:
+ ...
+
+ # execute function several times to build profile
+ from somewhere import some_typical_data
+ for _ in range(100):
+ critical_function(some_typical_data)
+
+ In Python 3.5 and later, you can distinguish between the profile and
+ non-profile runs as follows::
+
+ if "_pgo_" in __name__:
+ ... # execute critical code here
"""
args = magic_arguments.parse_argstring(self.cython, line)
- code = cell if cell.endswith('\n') else cell + '\n'
+ code = cell if cell.endswith('\n') else cell + '\n'
lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
- key = (code, line, sys.version_info, sys.executable, cython_version)
+ key = (code, line, sys.version_info, sys.executable, cython_version)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
- if args.pgo:
- key += ('pgo',)
+ if args.pgo:
+ key += ('pgo',)
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
- key += (time.time(),)
+ key += (time.time(),)
if args.name:
module_name = str(args.name) # no-op in Py3
else:
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
- html_file = os.path.join(lib_dir, module_name + '.html')
+ html_file = os.path.join(lib_dir, module_name + '.html')
module_path = os.path.join(lib_dir, module_name + self.so_ext)
have_module = os.path.isfile(module_path)
- need_cythonize = args.pgo or not have_module
+ need_cythonize = args.pgo or not have_module
if args.annotate:
if not os.path.isfile(html_file):
need_cythonize = True
- extension = None
+ extension = None
if need_cythonize:
- extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet)
+ extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet)
if extensions is None:
# Compilation failed and printed error message
return None
- assert len(extensions) == 1
- extension = extensions[0]
+ assert len(extensions) == 1
+ extension = extensions[0]
self._code_cache[key] = module_name
- if args.pgo:
- self._profile_pgo_wrapper(extension, lib_dir)
-
+ if args.pgo:
+ self._profile_pgo_wrapper(extension, lib_dir)
+
try:
self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None,
quiet=args.quiet)
except distutils.errors.CompileError:
# Build failed and printed error message
return None
-
+
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
@@ -366,129 +366,129 @@ class CythonMagics(Magics):
else:
return display.HTML(self.clean_annotated_html(annotated_html))
- def _profile_pgo_wrapper(self, extension, lib_dir):
- """
- Generate a .c file for a separate extension module that calls the
- module init function of the original module. This makes sure that the
- PGO profiler sees the correct .o file of the final module, but it still
- allows us to import the module under a different name for profiling,
- before recompiling it into the PGO optimised module. Overwriting and
- reimporting the same shared library is not portable.
- """
- extension = copy.copy(extension) # shallow copy, do not modify sources in place!
- module_name = extension.name
- pgo_module_name = '_pgo_' + module_name
- pgo_wrapper_c_file = os.path.join(lib_dir, pgo_module_name + '.c')
- with io.open(pgo_wrapper_c_file, 'w', encoding='utf-8') as f:
- f.write(textwrap.dedent(u"""
- #include "Python.h"
- #if PY_MAJOR_VERSION < 3
- extern PyMODINIT_FUNC init%(module_name)s(void);
- PyMODINIT_FUNC init%(pgo_module_name)s(void); /*proto*/
- PyMODINIT_FUNC init%(pgo_module_name)s(void) {
- PyObject *sys_modules;
- init%(module_name)s(); if (PyErr_Occurred()) return;
- sys_modules = PyImport_GetModuleDict(); /* borrowed, no exception, "never" fails */
- if (sys_modules) {
- PyObject *module = PyDict_GetItemString(sys_modules, "%(module_name)s"); if (!module) return;
- PyDict_SetItemString(sys_modules, "%(pgo_module_name)s", module);
- Py_DECREF(module);
- }
- }
- #else
- extern PyMODINIT_FUNC PyInit_%(module_name)s(void);
- PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void); /*proto*/
- PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void) {
- return PyInit_%(module_name)s();
- }
- #endif
- """ % {'module_name': module_name, 'pgo_module_name': pgo_module_name}))
-
- extension.sources = extension.sources + [pgo_wrapper_c_file] # do not modify in place!
- extension.name = pgo_module_name
-
- self._build_extension(extension, lib_dir, pgo_step_name='gen')
-
- # import and execute module code to generate profile
- so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext)
- imp.load_dynamic(pgo_module_name, so_module_path)
-
- def _cythonize(self, module_name, code, lib_dir, args, quiet=True):
- pyx_file = os.path.join(lib_dir, module_name + '.pyx')
+ def _profile_pgo_wrapper(self, extension, lib_dir):
+ """
+ Generate a .c file for a separate extension module that calls the
+ module init function of the original module. This makes sure that the
+ PGO profiler sees the correct .o file of the final module, but it still
+ allows us to import the module under a different name for profiling,
+ before recompiling it into the PGO optimised module. Overwriting and
+ reimporting the same shared library is not portable.
+ """
+ extension = copy.copy(extension) # shallow copy, do not modify sources in place!
+ module_name = extension.name
+ pgo_module_name = '_pgo_' + module_name
+ pgo_wrapper_c_file = os.path.join(lib_dir, pgo_module_name + '.c')
+ with io.open(pgo_wrapper_c_file, 'w', encoding='utf-8') as f:
+ f.write(textwrap.dedent(u"""
+ #include "Python.h"
+ #if PY_MAJOR_VERSION < 3
+ extern PyMODINIT_FUNC init%(module_name)s(void);
+ PyMODINIT_FUNC init%(pgo_module_name)s(void); /*proto*/
+ PyMODINIT_FUNC init%(pgo_module_name)s(void) {
+ PyObject *sys_modules;
+ init%(module_name)s(); if (PyErr_Occurred()) return;
+ sys_modules = PyImport_GetModuleDict(); /* borrowed, no exception, "never" fails */
+ if (sys_modules) {
+ PyObject *module = PyDict_GetItemString(sys_modules, "%(module_name)s"); if (!module) return;
+ PyDict_SetItemString(sys_modules, "%(pgo_module_name)s", module);
+ Py_DECREF(module);
+ }
+ }
+ #else
+ extern PyMODINIT_FUNC PyInit_%(module_name)s(void);
+ PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void); /*proto*/
+ PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void) {
+ return PyInit_%(module_name)s();
+ }
+ #endif
+ """ % {'module_name': module_name, 'pgo_module_name': pgo_module_name}))
+
+ extension.sources = extension.sources + [pgo_wrapper_c_file] # do not modify in place!
+ extension.name = pgo_module_name
+
+ self._build_extension(extension, lib_dir, pgo_step_name='gen')
+
+ # import and execute module code to generate profile
+ so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext)
+ imp.load_dynamic(pgo_module_name, so_module_path)
+
+ def _cythonize(self, module_name, code, lib_dir, args, quiet=True):
+ pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = encode_fs(pyx_file)
-
- c_include_dirs = args.include
- c_src_files = list(map(str, args.src))
- if 'numpy' in code:
- import numpy
- c_include_dirs.append(numpy.get_include())
- with io.open(pyx_file, 'w', encoding='utf-8') as f:
- f.write(code)
- extension = Extension(
- name=module_name,
- sources=[pyx_file] + c_src_files,
- include_dirs=c_include_dirs,
- library_dirs=args.library_dirs,
- extra_compile_args=args.compile_args,
- extra_link_args=args.link_args,
- libraries=args.lib,
- language='c++' if args.cplus else 'c',
- )
- try:
- opts = dict(
- quiet=quiet,
- annotate=args.annotate,
- force=True,
- )
- if args.language_level is not None:
- assert args.language_level in (2, 3)
- opts['language_level'] = args.language_level
- elif sys.version_info[0] >= 3:
- opts['language_level'] = 3
- return cythonize([extension], **opts)
- except CompileError:
- return None
-
- def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True):
- build_extension = self._get_build_extension(
- extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name)
- old_threshold = None
- try:
- if not quiet:
- old_threshold = distutils.log.set_threshold(distutils.log.DEBUG)
- build_extension.run()
- finally:
- if not quiet and old_threshold is not None:
- distutils.log.set_threshold(old_threshold)
-
- def _add_pgo_flags(self, build_extension, step_name, temp_dir):
- compiler_type = build_extension.compiler.compiler_type
- if compiler_type == 'unix':
- compiler_cmd = build_extension.compiler.compiler_so
- # TODO: we could try to call "[cmd] --version" for better insights
- if not compiler_cmd:
- pass
- elif 'clang' in compiler_cmd or 'clang' in compiler_cmd[0]:
- compiler_type = 'clang'
- elif 'icc' in compiler_cmd or 'icc' in compiler_cmd[0]:
- compiler_type = 'icc'
- elif 'gcc' in compiler_cmd or 'gcc' in compiler_cmd[0]:
- compiler_type = 'gcc'
- elif 'g++' in compiler_cmd or 'g++' in compiler_cmd[0]:
- compiler_type = 'gcc'
- config = PGO_CONFIG.get(compiler_type)
- orig_flags = []
- if config and step_name in config:
- flags = [f.format(TEMPDIR=temp_dir) for f in config[step_name]]
- for extension in build_extension.extensions:
- orig_flags.append((extension.extra_compile_args, extension.extra_link_args))
- extension.extra_compile_args = extension.extra_compile_args + flags
- extension.extra_link_args = extension.extra_link_args + flags
- else:
- print("No PGO %s configuration known for C compiler type '%s'" % (step_name, compiler_type),
- file=sys.stderr)
- return orig_flags
-
+
+ c_include_dirs = args.include
+ c_src_files = list(map(str, args.src))
+ if 'numpy' in code:
+ import numpy
+ c_include_dirs.append(numpy.get_include())
+ with io.open(pyx_file, 'w', encoding='utf-8') as f:
+ f.write(code)
+ extension = Extension(
+ name=module_name,
+ sources=[pyx_file] + c_src_files,
+ include_dirs=c_include_dirs,
+ library_dirs=args.library_dirs,
+ extra_compile_args=args.compile_args,
+ extra_link_args=args.link_args,
+ libraries=args.lib,
+ language='c++' if args.cplus else 'c',
+ )
+ try:
+ opts = dict(
+ quiet=quiet,
+ annotate=args.annotate,
+ force=True,
+ )
+ if args.language_level is not None:
+ assert args.language_level in (2, 3)
+ opts['language_level'] = args.language_level
+ elif sys.version_info[0] >= 3:
+ opts['language_level'] = 3
+ return cythonize([extension], **opts)
+ except CompileError:
+ return None
+
+ def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True):
+ build_extension = self._get_build_extension(
+ extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name)
+ old_threshold = None
+ try:
+ if not quiet:
+ old_threshold = distutils.log.set_threshold(distutils.log.DEBUG)
+ build_extension.run()
+ finally:
+ if not quiet and old_threshold is not None:
+ distutils.log.set_threshold(old_threshold)
+
+ def _add_pgo_flags(self, build_extension, step_name, temp_dir):
+ compiler_type = build_extension.compiler.compiler_type
+ if compiler_type == 'unix':
+ compiler_cmd = build_extension.compiler.compiler_so
+ # TODO: we could try to call "[cmd] --version" for better insights
+ if not compiler_cmd:
+ pass
+ elif 'clang' in compiler_cmd or 'clang' in compiler_cmd[0]:
+ compiler_type = 'clang'
+ elif 'icc' in compiler_cmd or 'icc' in compiler_cmd[0]:
+ compiler_type = 'icc'
+ elif 'gcc' in compiler_cmd or 'gcc' in compiler_cmd[0]:
+ compiler_type = 'gcc'
+ elif 'g++' in compiler_cmd or 'g++' in compiler_cmd[0]:
+ compiler_type = 'gcc'
+ config = PGO_CONFIG.get(compiler_type)
+ orig_flags = []
+ if config and step_name in config:
+ flags = [f.format(TEMPDIR=temp_dir) for f in config[step_name]]
+ for extension in build_extension.extensions:
+ orig_flags.append((extension.extra_compile_args, extension.extra_link_args))
+ extension.extra_compile_args = extension.extra_compile_args + flags
+ extension.extra_link_args = extension.extra_link_args + flags
+ else:
+ print("No PGO %s configuration known for C compiler type '%s'" % (step_name, compiler_type),
+ file=sys.stderr)
+ return orig_flags
+
@property
def so_ext(self):
"""The extension suffix for compiled modules."""
@@ -510,8 +510,8 @@ class CythonMagics(Magics):
else:
_path_created.clear()
- def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None,
- pgo_step_name=None, _build_ext=build_ext):
+ def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None,
+ pgo_step_name=None, _build_ext=build_ext):
self._clear_distutils_mkpath_cache()
dist = Distribution()
config_files = dist.find_config_files()
@@ -520,28 +520,28 @@ class CythonMagics(Magics):
except ValueError:
pass
dist.parse_config_files(config_files)
-
- if not temp_dir:
- temp_dir = lib_dir
- add_pgo_flags = self._add_pgo_flags
-
- if pgo_step_name:
- base_build_ext = _build_ext
- class _build_ext(_build_ext):
- def build_extensions(self):
- add_pgo_flags(self, pgo_step_name, temp_dir)
- base_build_ext.build_extensions(self)
-
- build_extension = _build_ext(dist)
+
+ if not temp_dir:
+ temp_dir = lib_dir
+ add_pgo_flags = self._add_pgo_flags
+
+ if pgo_step_name:
+ base_build_ext = _build_ext
+ class _build_ext(_build_ext):
+ def build_extensions(self):
+ add_pgo_flags(self, pgo_step_name, temp_dir)
+ base_build_ext.build_extensions(self)
+
+ build_extension = _build_ext(dist)
build_extension.finalize_options()
- if temp_dir:
+ if temp_dir:
temp_dir = encode_fs(temp_dir)
- build_extension.build_temp = temp_dir
- if lib_dir:
+ build_extension.build_temp = temp_dir
+ if lib_dir:
lib_dir = encode_fs(lib_dir)
- build_extension.build_lib = lib_dir
- if extension is not None:
- build_extension.extensions = [extension]
+ build_extension.build_lib = lib_dir
+ if extension is not None:
+ build_extension.extensions = [extension]
return build_extension
@staticmethod
@@ -556,10 +556,10 @@ class CythonMagics(Magics):
return html
__doc__ = __doc__.format(
- # rST doesn't see the -+ flag as part of an option list, so we
- # hide it from the module-level docstring.
- CYTHON_DOC=dedent(CythonMagics.cython.__doc__\
- .replace('-+, --cplus', '--cplus ')),
- CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__),
- CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__),
+ # rST doesn't see the -+ flag as part of an option list, so we
+ # hide it from the module-level docstring.
+ CYTHON_DOC=dedent(CythonMagics.cython.__doc__\
+ .replace('-+, --cplus', '--cplus ')),
+ CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__),
+ CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__),
)