diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /contrib/tools/cython/Cython/Build | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Build')
-rw-r--r-- | contrib/tools/cython/Cython/Build/BuildExecutable.py | 284 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Cythonize.py | 360 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Dependencies.py | 1596 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Inline.py | 524 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/IpythonMagic.py | 562 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Tests/TestInline.py | 112 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py | 136 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py | 110 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/Tests/__init__.py | 2 | ||||
-rw-r--r-- | contrib/tools/cython/Cython/Build/__init__.py | 2 |
10 files changed, 1844 insertions, 1844 deletions
diff --git a/contrib/tools/cython/Cython/Build/BuildExecutable.py b/contrib/tools/cython/Cython/Build/BuildExecutable.py index 2db9e5d745..7bd27c30db 100644 --- a/contrib/tools/cython/Cython/Build/BuildExecutable.py +++ b/contrib/tools/cython/Cython/Build/BuildExecutable.py @@ -1,142 +1,142 @@ -""" -Compile a Python script into an executable that embeds CPython and run it. -Requires CPython to be built as a shared library ('libpythonX.Y'). - -Basic usage: - - python cythonrun somefile.py [ARGS] -""" - -from __future__ import absolute_import - -DEBUG = True - -import sys -import os -from distutils import sysconfig - - -def get_config_var(name, default=''): - return sysconfig.get_config_var(name) or default - -INCDIR = sysconfig.get_python_inc() -LIBDIR1 = get_config_var('LIBDIR') -LIBDIR2 = get_config_var('LIBPL') -PYLIB = get_config_var('LIBRARY') -PYLIB_DYN = get_config_var('LDLIBRARY') -if PYLIB_DYN == PYLIB: - # no shared library - PYLIB_DYN = '' -else: - PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ - -CC = get_config_var('CC', os.environ.get('CC', '')) -CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') -LINKCC = get_config_var('LINKCC', os.environ.get('LINKCC', CC)) -LINKFORSHARED = get_config_var('LINKFORSHARED') -LIBS = get_config_var('LIBS') -SYSLIBS = get_config_var('SYSLIBS') -EXE_EXT = sysconfig.get_config_var('EXE') - -def _debug(msg, *args): - if DEBUG: - if args: - msg = msg % args - sys.stderr.write(msg + '\n') - -def dump_config(): - _debug('INCDIR: %s', INCDIR) - _debug('LIBDIR1: %s', LIBDIR1) - _debug('LIBDIR2: %s', LIBDIR2) - _debug('PYLIB: %s', PYLIB) - _debug('PYLIB_DYN: %s', PYLIB_DYN) - _debug('CC: %s', CC) - _debug('CFLAGS: %s', CFLAGS) - _debug('LINKCC: %s', LINKCC) - _debug('LINKFORSHARED: %s', LINKFORSHARED) - _debug('LIBS: %s', LIBS) - _debug('SYSLIBS: %s', SYSLIBS) - _debug('EXE_EXT: %s', EXE_EXT) - -def runcmd(cmd, shell=True): - if shell: - cmd = ' '.join(cmd) - _debug(cmd) - else: - _debug(' '.join(cmd)) - - try: - import subprocess - except ImportError: # Python 2.3 ... - returncode = os.system(cmd) - else: - returncode = subprocess.call(cmd, shell=shell) - - if returncode: - sys.exit(returncode) - -def clink(basename): - runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2] - + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)] - + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split()) - -def ccompile(basename): - runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split()) - -def cycompile(input_file, options=()): - from ..Compiler import Version, CmdLine, Main - options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file]) - _debug('Using Cython %s to compile %s', Version.version, input_file) - result = Main.compile(sources, options) - if result.num_errors > 0: - sys.exit(1) - -def exec_file(program_name, args=()): - runcmd([os.path.abspath(program_name)] + list(args), shell=False) - -def build(input_file, compiler_args=(), force=False): - """ - Build an executable program from a Cython module. - - Returns the name of the executable file. - """ - basename = os.path.splitext(input_file)[0] - exe_file = basename + EXE_EXT - if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): - raise ValueError("Input and output file names are the same, refusing to overwrite") - if (not force and os.path.exists(exe_file) and os.path.exists(input_file) - and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): - _debug("File is up to date, not regenerating %s", exe_file) - return exe_file - cycompile(input_file, compiler_args) - ccompile(basename) - clink(basename) - return exe_file - -def build_and_run(args): - """ - Build an executable program from a Cython module and runs it. - - Arguments after the module name will be passed verbatimely to the - program. - """ - cy_args = [] - last_arg = None - for i, arg in enumerate(args): - if arg.startswith('-'): - cy_args.append(arg) - elif last_arg in ('-X', '--directive'): - cy_args.append(arg) - else: - input_file = arg - args = args[i+1:] - break - last_arg = arg - else: - raise ValueError('no input file provided') - - program_name = build(input_file, cy_args) - exec_file(program_name, args) - -if __name__ == '__main__': - build_and_run(sys.argv[1:]) +""" +Compile a Python script into an executable that embeds CPython and run it. +Requires CPython to be built as a shared library ('libpythonX.Y'). + +Basic usage: + + python cythonrun somefile.py [ARGS] +""" + +from __future__ import absolute_import + +DEBUG = True + +import sys +import os +from distutils import sysconfig + + +def get_config_var(name, default=''): + return sysconfig.get_config_var(name) or default + +INCDIR = sysconfig.get_python_inc() +LIBDIR1 = get_config_var('LIBDIR') +LIBDIR2 = get_config_var('LIBPL') +PYLIB = get_config_var('LIBRARY') +PYLIB_DYN = get_config_var('LDLIBRARY') +if PYLIB_DYN == PYLIB: + # no shared library + PYLIB_DYN = '' +else: + PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ + +CC = get_config_var('CC', os.environ.get('CC', '')) +CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') +LINKCC = get_config_var('LINKCC', os.environ.get('LINKCC', CC)) +LINKFORSHARED = get_config_var('LINKFORSHARED') +LIBS = get_config_var('LIBS') +SYSLIBS = get_config_var('SYSLIBS') +EXE_EXT = sysconfig.get_config_var('EXE') + +def _debug(msg, *args): + if DEBUG: + if args: + msg = msg % args + sys.stderr.write(msg + '\n') + +def dump_config(): + _debug('INCDIR: %s', INCDIR) + _debug('LIBDIR1: %s', LIBDIR1) + _debug('LIBDIR2: %s', LIBDIR2) + _debug('PYLIB: %s', PYLIB) + _debug('PYLIB_DYN: %s', PYLIB_DYN) + _debug('CC: %s', CC) + _debug('CFLAGS: %s', CFLAGS) + _debug('LINKCC: %s', LINKCC) + _debug('LINKFORSHARED: %s', LINKFORSHARED) + _debug('LIBS: %s', LIBS) + _debug('SYSLIBS: %s', SYSLIBS) + _debug('EXE_EXT: %s', EXE_EXT) + +def runcmd(cmd, shell=True): + if shell: + cmd = ' '.join(cmd) + _debug(cmd) + else: + _debug(' '.join(cmd)) + + try: + import subprocess + except ImportError: # Python 2.3 ... + returncode = os.system(cmd) + else: + returncode = subprocess.call(cmd, shell=shell) + + if returncode: + sys.exit(returncode) + +def clink(basename): + runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2] + + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)] + + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split()) + +def ccompile(basename): + runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split()) + +def cycompile(input_file, options=()): + from ..Compiler import Version, CmdLine, Main + options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file]) + _debug('Using Cython %s to compile %s', Version.version, input_file) + result = Main.compile(sources, options) + if result.num_errors > 0: + sys.exit(1) + +def exec_file(program_name, args=()): + runcmd([os.path.abspath(program_name)] + list(args), shell=False) + +def build(input_file, compiler_args=(), force=False): + """ + Build an executable program from a Cython module. + + Returns the name of the executable file. + """ + basename = os.path.splitext(input_file)[0] + exe_file = basename + EXE_EXT + if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): + raise ValueError("Input and output file names are the same, refusing to overwrite") + if (not force and os.path.exists(exe_file) and os.path.exists(input_file) + and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): + _debug("File is up to date, not regenerating %s", exe_file) + return exe_file + cycompile(input_file, compiler_args) + ccompile(basename) + clink(basename) + return exe_file + +def build_and_run(args): + """ + Build an executable program from a Cython module and runs it. + + Arguments after the module name will be passed verbatimely to the + program. + """ + cy_args = [] + last_arg = None + for i, arg in enumerate(args): + if arg.startswith('-'): + cy_args.append(arg) + elif last_arg in ('-X', '--directive'): + cy_args.append(arg) + else: + input_file = arg + args = args[i+1:] + break + last_arg = arg + else: + raise ValueError('no input file provided') + + program_name = build(input_file, cy_args) + exec_file(program_name, args) + +if __name__ == '__main__': + build_and_run(sys.argv[1:]) diff --git a/contrib/tools/cython/Cython/Build/Cythonize.py b/contrib/tools/cython/Cython/Build/Cythonize.py index c85b6eabab..170961e289 100644 --- a/contrib/tools/cython/Cython/Build/Cythonize.py +++ b/contrib/tools/cython/Cython/Build/Cythonize.py @@ -1,65 +1,65 @@ -#!/usr/bin/env python - -from __future__ import absolute_import - -import os -import shutil -import tempfile -from distutils.core import setup - -from .Dependencies import cythonize, extended_iglob -from ..Utils import is_package_dir -from ..Compiler import Options - -try: - import multiprocessing - parallel_compiles = int(multiprocessing.cpu_count() * 1.5) -except ImportError: - multiprocessing = None - parallel_compiles = 0 - - -class _FakePool(object): - def map_async(self, func, args): +#!/usr/bin/env python + +from __future__ import absolute_import + +import os +import shutil +import tempfile +from distutils.core import setup + +from .Dependencies import cythonize, extended_iglob +from ..Utils import is_package_dir +from ..Compiler import Options + +try: + import multiprocessing + parallel_compiles = int(multiprocessing.cpu_count() * 1.5) +except ImportError: + multiprocessing = None + parallel_compiles = 0 + + +class _FakePool(object): + def map_async(self, func, args): try: from itertools import imap except ImportError: imap=map - for _ in imap(func, args): - pass - + for _ in imap(func, args): + pass + def close(self): pass - + def terminate(self): pass - + def join(self): pass -def parse_directives(option, name, value, parser): - dest = option.dest - old_directives = dict(getattr(parser.values, dest, +def parse_directives(option, name, value, parser): + dest = option.dest + old_directives = dict(getattr(parser.values, dest, Options.get_directive_defaults())) - directives = Options.parse_directive_list( - value, relaxed_bool=True, current_settings=old_directives) - setattr(parser.values, dest, directives) - - -def parse_options(option, name, value, parser): - dest = option.dest - options = dict(getattr(parser.values, dest, {})) - for opt in value.split(','): - if '=' in opt: - n, v = opt.split('=', 1) - v = v.lower() not in ('false', 'f', '0', 'no') - else: - n, v = opt, True - options[n] = v - setattr(parser.values, dest, options) - - + directives = Options.parse_directive_list( + value, relaxed_bool=True, current_settings=old_directives) + setattr(parser.values, dest, directives) + + +def parse_options(option, name, value, parser): + dest = option.dest + options = dict(getattr(parser.values, dest, {})) + for opt in value.split(','): + if '=' in opt: + n, v = opt.split('=', 1) + v = v.lower() not in ('false', 'f', '0', 'no') + else: + n, v = opt, True + options[n] = v + setattr(parser.values, dest, options) + + def parse_compile_time_env(option, name, value, parser): dest = option.dest old_env = dict(getattr(parser.values, dest, {})) @@ -67,96 +67,96 @@ def parse_compile_time_env(option, name, value, parser): setattr(parser.values, dest, new_env) -def find_package_base(path): - base_dir, package_path = os.path.split(path) - while os.path.isfile(os.path.join(base_dir, '__init__.py')): - base_dir, parent = os.path.split(base_dir) - package_path = '%s/%s' % (parent, package_path) - return base_dir, package_path - - -def cython_compile(path_pattern, options): - pool = None +def find_package_base(path): + base_dir, package_path = os.path.split(path) + while os.path.isfile(os.path.join(base_dir, '__init__.py')): + base_dir, parent = os.path.split(base_dir) + package_path = '%s/%s' % (parent, package_path) + return base_dir, package_path + + +def cython_compile(path_pattern, options): + pool = None all_paths = map(os.path.abspath, extended_iglob(path_pattern)) - try: + try: for path in all_paths: - if options.build_inplace: - base_dir = path - while not os.path.isdir(base_dir) or is_package_dir(base_dir): - base_dir = os.path.dirname(base_dir) - else: - base_dir = None - - if os.path.isdir(path): - # recursively compiling a package + if options.build_inplace: + base_dir = path + while not os.path.isdir(base_dir) or is_package_dir(base_dir): + base_dir = os.path.dirname(base_dir) + else: + base_dir = None + + if os.path.isdir(path): + # recursively compiling a package paths = [os.path.join(path, '**', '*.{py,pyx}')] - else: - # assume it's a file(-like thing) - paths = [path] - - ext_modules = cythonize( - paths, - nthreads=options.parallel, - exclude_failures=options.keep_going, - exclude=options.excludes, - compiler_directives=options.directives, + else: + # assume it's a file(-like thing) + paths = [path] + + ext_modules = cythonize( + paths, + nthreads=options.parallel, + exclude_failures=options.keep_going, + exclude=options.excludes, + compiler_directives=options.directives, compile_time_env=options.compile_time_env, - force=options.force, - quiet=options.quiet, + force=options.force, + quiet=options.quiet, depfile=options.depfile, - **options.options) - - if ext_modules and options.build: - if len(ext_modules) > 1 and options.parallel > 1: - if pool is None: - try: - pool = multiprocessing.Pool(options.parallel) - except OSError: - pool = _FakePool() - pool.map_async(run_distutils, [ - (base_dir, [ext]) for ext in ext_modules]) - else: - run_distutils((base_dir, ext_modules)) - except: - if pool is not None: - pool.terminate() - raise - else: - if pool is not None: - pool.close() - pool.join() - - -def run_distutils(args): - base_dir, ext_modules = args - script_args = ['build_ext', '-i'] - cwd = os.getcwd() - temp_dir = None - try: - if base_dir: - os.chdir(base_dir) - temp_dir = tempfile.mkdtemp(dir=base_dir) - script_args.extend(['--build-temp', temp_dir]) - setup( - script_name='setup.py', - script_args=script_args, - ext_modules=ext_modules, - ) - finally: - if base_dir: - os.chdir(cwd) - if temp_dir and os.path.isdir(temp_dir): - shutil.rmtree(temp_dir) - - -def parse_args(args): - from optparse import OptionParser - parser = OptionParser(usage='%prog [options] [sources and packages]+') - + **options.options) + + if ext_modules and options.build: + if len(ext_modules) > 1 and options.parallel > 1: + if pool is None: + try: + pool = multiprocessing.Pool(options.parallel) + except OSError: + pool = _FakePool() + pool.map_async(run_distutils, [ + (base_dir, [ext]) for ext in ext_modules]) + else: + run_distutils((base_dir, ext_modules)) + except: + if pool is not None: + pool.terminate() + raise + else: + if pool is not None: + pool.close() + pool.join() + + +def run_distutils(args): + base_dir, ext_modules = args + script_args = ['build_ext', '-i'] + cwd = os.getcwd() + temp_dir = None + try: + if base_dir: + os.chdir(base_dir) + temp_dir = tempfile.mkdtemp(dir=base_dir) + script_args.extend(['--build-temp', temp_dir]) + setup( + script_name='setup.py', + script_args=script_args, + ext_modules=ext_modules, + ) + finally: + if base_dir: + os.chdir(cwd) + if temp_dir and os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + + +def parse_args(args): + from optparse import OptionParser + parser = OptionParser(usage='%prog [options] [sources and packages]+') + parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', dest='directives', default={}, type="str", action='callback', callback=parse_directives, - help='set a compiler directive') + help='set a compiler directive') parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...', dest='compile_time_env', default={}, type="str", action='callback', callback=parse_compile_time_env, @@ -164,66 +164,66 @@ def parse_args(args): parser.add_option('-s', '--option', metavar='NAME=VALUE', dest='options', default={}, type="str", action='callback', callback=parse_options, - help='set a cythonize option') + help='set a cythonize option') parser.add_option('-2', dest='language_level', action='store_const', const=2, default=None, help='use Python 2 syntax mode by default') parser.add_option('-3', dest='language_level', action='store_const', const=3, - help='use Python 3 syntax mode by default') + help='use Python 3 syntax mode by default') parser.add_option('--3str', dest='language_level', action='store_const', const='3str', help='use Python 3 syntax mode by default') parser.add_option('-a', '--annotate', dest='annotate', action='store_true', help='generate annotated HTML page for source files') - - parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', - action='append', default=[], - help='exclude certain file patterns from the compilation') - - parser.add_option('-b', '--build', dest='build', action='store_true', - help='build extension modules using distutils') - parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', - help='build extension modules in place using distutils (implies -b)') - parser.add_option('-j', '--parallel', dest='parallel', metavar='N', - type=int, default=parallel_compiles, - help=('run builds in N parallel jobs (default: %d)' % - parallel_compiles or 1)) - parser.add_option('-f', '--force', dest='force', action='store_true', - help='force recompilation') - parser.add_option('-q', '--quiet', dest='quiet', action='store_true', - help='be less verbose during compilation') - - parser.add_option('--lenient', dest='lenient', action='store_true', - help='increase Python compatibility by ignoring some compile time errors') - parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', - help='compile as much as possible, ignore compilation failures') + + parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', + action='append', default=[], + help='exclude certain file patterns from the compilation') + + parser.add_option('-b', '--build', dest='build', action='store_true', + help='build extension modules using distutils') + parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', + help='build extension modules in place using distutils (implies -b)') + parser.add_option('-j', '--parallel', dest='parallel', metavar='N', + type=int, default=parallel_compiles, + help=('run builds in N parallel jobs (default: %d)' % + parallel_compiles or 1)) + parser.add_option('-f', '--force', dest='force', action='store_true', + help='force recompilation') + parser.add_option('-q', '--quiet', dest='quiet', action='store_true', + help='be less verbose during compilation') + + parser.add_option('--lenient', dest='lenient', action='store_true', + help='increase Python compatibility by ignoring some compile time errors') + parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', + help='compile as much as possible, ignore compilation failures') parser.add_option('-M', '--depfile', action='store_true', help='produce depfiles for the sources') - - options, args = parser.parse_args(args) - if not args: - parser.error("no source files provided") - if options.build_inplace: - options.build = True - if multiprocessing is None: - options.parallel = 0 + + options, args = parser.parse_args(args) + if not args: + parser.error("no source files provided") + if options.build_inplace: + options.build = True + if multiprocessing is None: + options.parallel = 0 if options.language_level: assert options.language_level in (2, 3, '3str') options.options['language_level'] = options.language_level - return options, args - - -def main(args=None): - options, paths = parse_args(args) - - if options.lenient: - # increase Python compatibility by ignoring compile time errors - Options.error_on_unknown_names = False - Options.error_on_uninitialized = False - + return options, args + + +def main(args=None): + options, paths = parse_args(args) + + if options.lenient: + # increase Python compatibility by ignoring compile time errors + Options.error_on_unknown_names = False + Options.error_on_uninitialized = False + if options.annotate: Options.annotate = True - for path in paths: - cython_compile(path, options) - - -if __name__ == '__main__': - main() + for path in paths: + cython_compile(path, options) + + +if __name__ == '__main__': + main() diff --git a/contrib/tools/cython/Cython/Build/Dependencies.py b/contrib/tools/cython/Cython/Build/Dependencies.py index 7eb55e2607..d55ee7cb88 100644 --- a/contrib/tools/cython/Cython/Build/Dependencies.py +++ b/contrib/tools/cython/Cython/Build/Dependencies.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, print_function - -import cython -from .. import __version__ - + +import cython +from .. import __version__ + import collections import contextlib import hashlib @@ -11,27 +11,27 @@ import shutil import subprocess import re, sys, time import warnings -from glob import iglob +from glob import iglob from io import open as io_open from os.path import relpath as _relpath from distutils.extension import Extension from distutils.util import strtobool import zipfile - -try: + +try: from collections.abc import Iterable except ImportError: from collections import Iterable try: - import gzip - gzip_open = gzip.open - gzip_ext = '.gz' -except ImportError: - gzip_open = open - gzip_ext = '' - -try: + import gzip + gzip_open = gzip.open + gzip_ext = '.gz' +except ImportError: + gzip_open = open + gzip_ext = '' + +try: import zlib zipfile_compression_mode = zipfile.ZIP_DEFLATED except ImportError: @@ -41,30 +41,30 @@ try: import pythran except: pythran = None - -from .. import Utils + +from .. import Utils from ..Utils import (cached_function, cached_method, path_exists, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) -from ..Compiler.Main import Context, CompilationOptions, default_options - -join_path = cached_function(os.path.join) +from ..Compiler.Main import Context, CompilationOptions, default_options + +join_path = cached_function(os.path.join) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) safe_makedirs_once = cached_function(safe_makedirs) - -if sys.version_info[0] < 3: - # stupid Py2 distutils enforces str type in list of sources - _fs_encoding = sys.getfilesystemencoding() - if _fs_encoding is None: - _fs_encoding = sys.getdefaultencoding() - def encode_filename_in_py2(filename): + +if sys.version_info[0] < 3: + # stupid Py2 distutils enforces str type in list of sources + _fs_encoding = sys.getfilesystemencoding() + if _fs_encoding is None: + _fs_encoding = sys.getdefaultencoding() + def encode_filename_in_py2(filename): if not isinstance(filename, bytes): - return filename.encode(_fs_encoding) - return filename -else: - def encode_filename_in_py2(filename): - return filename - basestring = str - + return filename.encode(_fs_encoding) + return filename +else: + def encode_filename_in_py2(filename): + return filename + basestring = str + def _make_relative(file_paths, base=None): if not base: @@ -75,7 +75,7 @@ def _make_relative(file_paths, base=None): for path in file_paths] -def extended_iglob(pattern): +def extended_iglob(pattern): if '{' in pattern: m = re.match('(.*){([^}]+)}(.*)', pattern) if m: @@ -84,26 +84,26 @@ def extended_iglob(pattern): for path in extended_iglob(before + case + after): yield path return - if '**/' in pattern: - seen = set() - first, rest = pattern.split('**/', 1) - if first: - first = iglob(first+'/') - else: - first = [''] - for root in first: - for path in extended_iglob(join_path(root, rest)): - if path not in seen: - seen.add(path) - yield path - for path in extended_iglob(join_path(root, '*', '**/' + rest)): - if path not in seen: - seen.add(path) - yield path - else: - for path in iglob(pattern): - yield path - + if '**/' in pattern: + seen = set() + first, rest = pattern.split('**/', 1) + if first: + first = iglob(first+'/') + else: + first = [''] + for root in first: + for path in extended_iglob(join_path(root, rest)): + if path not in seen: + seen.add(path) + yield path + for path in extended_iglob(join_path(root, '*', '**/' + rest)): + if path not in seen: + seen.add(path) + yield path + else: + for path in iglob(pattern): + yield path + def nonempty(it, error_msg="expected non-empty iterator"): empty = True @@ -114,18 +114,18 @@ def nonempty(it, error_msg="expected non-empty iterator"): raise ValueError(error_msg) -@cached_function -def file_hash(filename): +@cached_function +def file_hash(filename): path = os.path.normpath(filename) prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") m = hashlib.md5(prefix) with open(path, 'rb') as f: - data = f.read(65000) - while data: - m.update(data) - data = f.read(65000) - return m.hexdigest() - + data = f.read(65000) + while data: + m.update(data) + data = f.read(65000) + return m.hexdigest() + def update_pythran_extension(ext): if pythran is None: @@ -152,250 +152,250 @@ def update_pythran_extension(ext): pass -def parse_list(s): - """ +def parse_list(s): + """ >>> parse_list("") [] >>> parse_list("a") ['a'] - >>> parse_list("a b c") - ['a', 'b', 'c'] - >>> parse_list("[a, b, c]") - ['a', 'b', 'c'] - >>> parse_list('a " " b') - ['a', ' ', 'b'] - >>> parse_list('[a, ",a", "a,", ",", ]') - ['a', ',a', 'a,', ','] - """ + >>> parse_list("a b c") + ['a', 'b', 'c'] + >>> parse_list("[a, b, c]") + ['a', 'b', 'c'] + >>> parse_list('a " " b') + ['a', ' ', 'b'] + >>> parse_list('[a, ",a", "a,", ",", ]') + ['a', ',a', 'a,', ','] + """ if len(s) >= 2 and s[0] == '[' and s[-1] == ']': - s = s[1:-1] - delimiter = ',' - else: - delimiter = ' ' - s, literals = strip_string_literals(s) - def unquote(literal): - literal = literal.strip() - if literal[0] in "'\"": - return literals[literal[1:-1]] - else: - return literal - return [unquote(item) for item in s.split(delimiter) if item.strip()] - - -transitive_str = object() -transitive_list = object() + s = s[1:-1] + delimiter = ',' + else: + delimiter = ' ' + s, literals = strip_string_literals(s) + def unquote(literal): + literal = literal.strip() + if literal[0] in "'\"": + return literals[literal[1:-1]] + else: + return literal + return [unquote(item) for item in s.split(delimiter) if item.strip()] + + +transitive_str = object() +transitive_list = object() bool_or = object() - -distutils_settings = { - 'name': str, - 'sources': list, - 'define_macros': list, - 'undef_macros': list, - 'libraries': transitive_list, - 'library_dirs': transitive_list, - 'runtime_library_dirs': transitive_list, - 'include_dirs': transitive_list, - 'extra_objects': list, - 'extra_compile_args': transitive_list, - 'extra_link_args': transitive_list, - 'export_symbols': list, - 'depends': transitive_list, - 'language': transitive_str, + +distutils_settings = { + 'name': str, + 'sources': list, + 'define_macros': list, + 'undef_macros': list, + 'libraries': transitive_list, + 'library_dirs': transitive_list, + 'runtime_library_dirs': transitive_list, + 'include_dirs': transitive_list, + 'extra_objects': list, + 'extra_compile_args': transitive_list, + 'extra_link_args': transitive_list, + 'export_symbols': list, + 'depends': transitive_list, + 'language': transitive_str, 'np_pythran': bool_or -} - +} + @cython.locals(start=cython.Py_ssize_t, end=cython.Py_ssize_t) -def line_iter(source): - if isinstance(source, basestring): - start = 0 - while True: - end = source.find('\n', start) - if end == -1: - yield source[start:] - return - yield source[start:end] - start = end+1 - else: - for line in source: - yield line - - -class DistutilsInfo(object): - - def __init__(self, source=None, exn=None): - self.values = {} - if source is not None: - for line in line_iter(source): +def line_iter(source): + if isinstance(source, basestring): + start = 0 + while True: + end = source.find('\n', start) + if end == -1: + yield source[start:] + return + yield source[start:end] + start = end+1 + else: + for line in source: + yield line + + +class DistutilsInfo(object): + + def __init__(self, source=None, exn=None): + self.values = {} + if source is not None: + for line in line_iter(source): line = line.lstrip() if not line: continue if line[0] != '#': - break + break line = line[1:].lstrip() kind = next((k for k in ("distutils:","cython:") if line.startswith(k)), None) if kind is not None: key, _, value = [s.strip() for s in line[len(kind):].partition('=')] type = distutils_settings.get(key, None) if line.startswith("cython:") and type is None: continue - if type in (list, transitive_list): - value = parse_list(value) - if key == 'define_macros': + if type in (list, transitive_list): + value = parse_list(value) + if key == 'define_macros': value = [tuple(macro.split('=', 1)) if '=' in macro else (macro, None) for macro in value] if type is bool_or: value = strtobool(value) - self.values[key] = value - elif exn is not None: - for key in distutils_settings: + self.values[key] = value + elif exn is not None: + for key in distutils_settings: if key in ('name', 'sources','np_pythran'): - continue - value = getattr(exn, key, None) - if value: - self.values[key] = value - - def merge(self, other): - if other is None: - return self - for key, value in other.values.items(): - type = distutils_settings[key] - if type is transitive_str and key not in self.values: - self.values[key] = value - elif type is transitive_list: - if key in self.values: + continue + value = getattr(exn, key, None) + if value: + self.values[key] = value + + def merge(self, other): + if other is None: + return self + for key, value in other.values.items(): + type = distutils_settings[key] + if type is transitive_str and key not in self.values: + self.values[key] = value + elif type is transitive_list: + if key in self.values: # Change a *copy* of the list (Trac #845) all = self.values[key][:] - for v in value: - if v not in all: - all.append(v) + for v in value: + if v not in all: + all.append(v) value = all self.values[key] = value elif type is bool_or: self.values[key] = self.values.get(key, False) | value - return self - - def subs(self, aliases): - if aliases is None: - return self - resolved = DistutilsInfo() - for key, value in self.values.items(): - type = distutils_settings[key] - if type in [list, transitive_list]: - new_value_list = [] - for v in value: - if v in aliases: - v = aliases[v] - if isinstance(v, list): - new_value_list += v - else: - new_value_list.append(v) - value = new_value_list - else: - if value in aliases: - value = aliases[value] - resolved.values[key] = value - return resolved - - def apply(self, extension): - for key, value in self.values.items(): - type = distutils_settings[key] - if type in [list, transitive_list]: + return self + + def subs(self, aliases): + if aliases is None: + return self + resolved = DistutilsInfo() + for key, value in self.values.items(): + type = distutils_settings[key] + if type in [list, transitive_list]: + new_value_list = [] + for v in value: + if v in aliases: + v = aliases[v] + if isinstance(v, list): + new_value_list += v + else: + new_value_list.append(v) + value = new_value_list + else: + if value in aliases: + value = aliases[value] + resolved.values[key] = value + return resolved + + def apply(self, extension): + for key, value in self.values.items(): + type = distutils_settings[key] + if type in [list, transitive_list]: value = getattr(extension, key) + list(value) setattr(extension, key, value) - + @cython.locals(start=cython.Py_ssize_t, q=cython.Py_ssize_t, single_q=cython.Py_ssize_t, double_q=cython.Py_ssize_t, hash_mark=cython.Py_ssize_t, end=cython.Py_ssize_t, k=cython.Py_ssize_t, counter=cython.Py_ssize_t, quote_len=cython.Py_ssize_t) -def strip_string_literals(code, prefix='__Pyx_L'): - """ - Normalizes every string literal to be of the form '__Pyx_Lxxx', - returning the normalized code and a mapping of labels to - string literals. - """ - new_code = [] - literals = {} - counter = 0 - start = q = 0 - in_quote = False - hash_mark = single_q = double_q = -1 - code_len = len(code) +def strip_string_literals(code, prefix='__Pyx_L'): + """ + Normalizes every string literal to be of the form '__Pyx_Lxxx', + returning the normalized code and a mapping of labels to + string literals. + """ + new_code = [] + literals = {} + counter = 0 + start = q = 0 + in_quote = False + hash_mark = single_q = double_q = -1 + code_len = len(code) quote_type = None quote_len = -1 - - while True: - if hash_mark < q: - hash_mark = code.find('#', q) - if single_q < q: - single_q = code.find("'", q) - if double_q < q: - double_q = code.find('"', q) - q = min(single_q, double_q) + + while True: + if hash_mark < q: + hash_mark = code.find('#', q) + if single_q < q: + single_q = code.find("'", q) + if double_q < q: + double_q = code.find('"', q) + q = min(single_q, double_q) if q == -1: q = max(single_q, double_q) - - # We're done. - if q == -1 and hash_mark == -1: - new_code.append(code[start:]) - break - - # Try to close the quote. - elif in_quote: - if code[q-1] == u'\\': - k = 2 - while q >= k and code[q-k] == u'\\': - k += 1 - if k % 2 == 0: - q += 1 - continue + + # We're done. + if q == -1 and hash_mark == -1: + new_code.append(code[start:]) + break + + # Try to close the quote. + elif in_quote: + if code[q-1] == u'\\': + k = 2 + while q >= k and code[q-k] == u'\\': + k += 1 + if k % 2 == 0: + q += 1 + continue if code[q] == quote_type and ( quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])): - counter += 1 - label = "%s%s_" % (prefix, counter) - literals[label] = code[start+quote_len:q] - full_quote = code[q:q+quote_len] - new_code.append(full_quote) - new_code.append(label) - new_code.append(full_quote) - q += quote_len - in_quote = False - start = q - else: - q += 1 - - # Process comment. - elif -1 != hash_mark and (hash_mark < q or q == -1): - new_code.append(code[start:hash_mark+1]) - end = code.find('\n', hash_mark) - counter += 1 - label = "%s%s_" % (prefix, counter) - if end == -1: - end_or_none = None - else: - end_or_none = end - literals[label] = code[hash_mark+1:end_or_none] - new_code.append(label) - if end == -1: - break - start = q = end - - # Open the quote. - else: - if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]): - quote_len = 3 - else: - quote_len = 1 - in_quote = True - quote_type = code[q] - new_code.append(code[start:q]) - start = q - q += quote_len - - return "".join(new_code), literals - - + counter += 1 + label = "%s%s_" % (prefix, counter) + literals[label] = code[start+quote_len:q] + full_quote = code[q:q+quote_len] + new_code.append(full_quote) + new_code.append(label) + new_code.append(full_quote) + q += quote_len + in_quote = False + start = q + else: + q += 1 + + # Process comment. + elif -1 != hash_mark and (hash_mark < q or q == -1): + new_code.append(code[start:hash_mark+1]) + end = code.find('\n', hash_mark) + counter += 1 + label = "%s%s_" % (prefix, counter) + if end == -1: + end_or_none = None + else: + end_or_none = end + literals[label] = code[hash_mark+1:end_or_none] + new_code.append(label) + if end == -1: + break + start = q = end + + # Open the quote. + else: + if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]): + quote_len = 3 + else: + quote_len = 1 + in_quote = True + quote_type = code[q] + new_code.append(code[start:q]) + start = q + q += quote_len + + return "".join(new_code), literals + + # We need to allow spaces to allow for conditional compilation like # IF ...: # cimport ... @@ -407,14 +407,14 @@ dependency_after_from_regex = re.compile( r"(?:^\s+\(([0-9a-zA-Z_., ]*)\)[#\n])|" r"(?:^\s+([0-9a-zA-Z_., ]*)[#\n])", re.M) + +def normalize_existing(base_path, rel_paths): + return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths))) + -def normalize_existing(base_path, rel_paths): - return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths))) - - -@cached_function -def normalize_existing0(base_dir, rel_paths): +@cached_function +def normalize_existing0(base_dir, rel_paths): """ Given some base directory ``base_dir`` and a list of path names ``rel_paths``, normalize each relative path name ``rel`` by @@ -426,76 +426,76 @@ def normalize_existing0(base_dir, rel_paths): changed (for example, if all paths were already absolute), then ``needed_base`` is ``None``. """ - normalized = [] + normalized = [] needed_base = None - for rel in rel_paths: + for rel in rel_paths: if os.path.isabs(rel): normalized.append(rel) continue - path = join_path(base_dir, rel) - if path_exists(path): - normalized.append(os.path.normpath(path)) + path = join_path(base_dir, rel) + if path_exists(path): + normalized.append(os.path.normpath(path)) needed_base = base_dir - else: - normalized.append(rel) + else: + normalized.append(rel) return (normalized, needed_base) - - -def resolve_depends(depends, include_dirs): - include_dirs = tuple(include_dirs) - resolved = [] - for depend in depends: - path = resolve_depend(depend, include_dirs) - if path is not None: - resolved.append(path) - return resolved - - -@cached_function -def resolve_depend(depend, include_dirs): - if depend[0] == '<' and depend[-1] == '>': - return None - for dir in include_dirs: - path = join_path(dir, depend) - if path_exists(path): - return os.path.normpath(path) - return None - - -@cached_function -def package(filename): - dir = os.path.dirname(os.path.abspath(str(filename))) + + +def resolve_depends(depends, include_dirs): + include_dirs = tuple(include_dirs) + resolved = [] + for depend in depends: + path = resolve_depend(depend, include_dirs) + if path is not None: + resolved.append(path) + return resolved + + +@cached_function +def resolve_depend(depend, include_dirs): + if depend[0] == '<' and depend[-1] == '>': + return None + for dir in include_dirs: + path = join_path(dir, depend) + if path_exists(path): + return os.path.normpath(path) + return None + + +@cached_function +def package(filename): + dir = os.path.dirname(os.path.abspath(str(filename))) if dir != filename and is_package_dir(dir): - return package(dir) + (os.path.basename(dir),) - else: - return () - - -@cached_function -def fully_qualified_name(filename): - module = os.path.splitext(os.path.basename(filename))[0] - return '.'.join(package(filename) + (module,)) - - -@cached_function -def parse_dependencies(source_filename): + return package(dir) + (os.path.basename(dir),) + else: + return () + + +@cached_function +def fully_qualified_name(filename): + module = os.path.splitext(os.path.basename(filename))[0] + return '.'.join(package(filename) + (module,)) + + +@cached_function +def parse_dependencies(source_filename): # Actual parsing is way too slow, so we use regular expressions. - # The only catch is that we must strip comments and string - # literals ahead of time. + # The only catch is that we must strip comments and string + # literals ahead of time. with Utils.open_source_file(source_filename, error_handling='ignore') as fh: - source = fh.read() - distutils_info = DistutilsInfo(source) - source, literals = strip_string_literals(source) - source = source.replace('\\\n', ' ').replace('\t', ' ') - - # TODO: pure mode - cimports = [] - includes = [] - externs = [] + source = fh.read() + distutils_info = DistutilsInfo(source) + source, literals = strip_string_literals(source) + source = source.replace('\\\n', ' ').replace('\t', ' ') + + # TODO: pure mode + cimports = [] + includes = [] + externs = [] for m in dependency_regex.finditer(source): cimport_from, cimport_list, extern, include = m.groups() - if cimport_from: - cimports.append(cimport_from) + if cimport_from: + cimports.append(cimport_from) m_after_from = dependency_after_from_regex.search(source, pos=m.end()) if m_after_from: multiline, one_line = m_after_from.groups() @@ -505,130 +505,130 @@ def parse_dependencies(source_filename): elif cimport_list: cimports.extend(x.strip() for x in cimport_list.split(",")) - elif extern: - externs.append(literals[extern]) - else: - includes.append(literals[include]) - return cimports, includes, externs, distutils_info - - -class DependencyTree(object): - - def __init__(self, context, quiet=False): - self.context = context - self.quiet = quiet - self._transitive_cache = {} - - def parse_dependencies(self, source_filename): + elif extern: + externs.append(literals[extern]) + else: + includes.append(literals[include]) + return cimports, includes, externs, distutils_info + + +class DependencyTree(object): + + def __init__(self, context, quiet=False): + self.context = context + self.quiet = quiet + self._transitive_cache = {} + + def parse_dependencies(self, source_filename): if path_exists(source_filename): source_filename = os.path.normpath(source_filename) - return parse_dependencies(source_filename) - - @cached_method - def included_files(self, filename): - # This is messy because included files are textually included, resolving - # cimports (but not includes) relative to the including file. - all = set() - for include in self.parse_dependencies(filename)[1]: - include_path = join_path(os.path.dirname(filename), include) - if not path_exists(include_path): - include_path = self.context.find_include_file(include, None) - if include_path: - if '.' + os.path.sep in include_path: - include_path = os.path.normpath(include_path) - all.add(include_path) - all.update(self.included_files(include_path)) - elif not self.quiet: - print("Unable to locate '%s' referenced from '%s'" % (filename, include)) - return all - - @cached_method + return parse_dependencies(source_filename) + + @cached_method + def included_files(self, filename): + # This is messy because included files are textually included, resolving + # cimports (but not includes) relative to the including file. + all = set() + for include in self.parse_dependencies(filename)[1]: + include_path = join_path(os.path.dirname(filename), include) + if not path_exists(include_path): + include_path = self.context.find_include_file(include, None) + if include_path: + if '.' + os.path.sep in include_path: + include_path = os.path.normpath(include_path) + all.add(include_path) + all.update(self.included_files(include_path)) + elif not self.quiet: + print("Unable to locate '%s' referenced from '%s'" % (filename, include)) + return all + + @cached_method def cimports_externs_incdirs(self, filename): - # This is really ugly. Nested cimports are resolved with respect to the - # includer, but includes are resolved with respect to the includee. - cimports, includes, externs = self.parse_dependencies(filename)[:3] - cimports = set(cimports) - externs = set(externs) + # This is really ugly. Nested cimports are resolved with respect to the + # includer, but includes are resolved with respect to the includee. + cimports, includes, externs = self.parse_dependencies(filename)[:3] + cimports = set(cimports) + externs = set(externs) incdirs = set() - for include in self.included_files(filename): + for include in self.included_files(filename): included_cimports, included_externs, included_incdirs = self.cimports_externs_incdirs(include) - cimports.update(included_cimports) - externs.update(included_externs) + cimports.update(included_cimports) + externs.update(included_externs) incdirs.update(included_incdirs) externs, incdir = normalize_existing(filename, externs) if incdir: incdirs.add(incdir) return tuple(cimports), externs, incdirs - - def cimports(self, filename): + + def cimports(self, filename): return self.cimports_externs_incdirs(filename)[0] - - def package(self, filename): - return package(filename) - - def fully_qualified_name(self, filename): - return fully_qualified_name(filename) - - @cached_method - def find_pxd(self, module, filename=None): - is_relative = module[0] == '.' - if is_relative and not filename: - raise NotImplementedError("New relative imports.") - if filename is not None: - module_path = module.split('.') - if is_relative: - module_path.pop(0) # just explicitly relative - package_path = list(self.package(filename)) - while module_path and not module_path[0]: - try: - package_path.pop() - except IndexError: - return None # FIXME: error? - module_path.pop(0) - relative = '.'.join(package_path + module_path) - pxd = self.context.find_pxd_file(relative, None) - if pxd: - return pxd - if is_relative: - return None # FIXME: error? - return self.context.find_pxd_file(module, None) - - @cached_method - def cimported_files(self, filename): - if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): - pxd_list = [filename[:-4] + '.pxd'] - else: - pxd_list = [] + + def package(self, filename): + return package(filename) + + def fully_qualified_name(self, filename): + return fully_qualified_name(filename) + + @cached_method + def find_pxd(self, module, filename=None): + is_relative = module[0] == '.' + if is_relative and not filename: + raise NotImplementedError("New relative imports.") + if filename is not None: + module_path = module.split('.') + if is_relative: + module_path.pop(0) # just explicitly relative + package_path = list(self.package(filename)) + while module_path and not module_path[0]: + try: + package_path.pop() + except IndexError: + return None # FIXME: error? + module_path.pop(0) + relative = '.'.join(package_path + module_path) + pxd = self.context.find_pxd_file(relative, None) + if pxd: + return pxd + if is_relative: + return None # FIXME: error? + return self.context.find_pxd_file(module, None) + + @cached_method + def cimported_files(self, filename): + if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): + pxd_list = [filename[:-4] + '.pxd'] + else: + pxd_list = [] # Cimports generates all possible combinations package.module # when imported as from package cimport module. - for module in self.cimports(filename): - if module[:7] == 'cython.' or module == 'cython': - continue - pxd_file = self.find_pxd(module, filename) - if pxd_file is not None: - pxd_list.append(pxd_file) - return tuple(pxd_list) - - @cached_method - def immediate_dependencies(self, filename): - all = set([filename]) - all.update(self.cimported_files(filename)) - all.update(self.included_files(filename)) - return all - - def all_dependencies(self, filename): - return self.transitive_merge(filename, self.immediate_dependencies, set.union) - - @cached_method - def timestamp(self, filename): - return os.path.getmtime(filename) - - def extract_timestamp(self, filename): - return self.timestamp(filename), filename - - def newest_dependency(self, filename): - return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)]) - + for module in self.cimports(filename): + if module[:7] == 'cython.' or module == 'cython': + continue + pxd_file = self.find_pxd(module, filename) + if pxd_file is not None: + pxd_list.append(pxd_file) + return tuple(pxd_list) + + @cached_method + def immediate_dependencies(self, filename): + all = set([filename]) + all.update(self.cimported_files(filename)) + all.update(self.included_files(filename)) + return all + + def all_dependencies(self, filename): + return self.transitive_merge(filename, self.immediate_dependencies, set.union) + + @cached_method + def timestamp(self, filename): + return os.path.getmtime(filename) + + def extract_timestamp(self, filename): + return self.timestamp(filename), filename + + def newest_dependency(self, filename): + return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)]) + def transitive_fingerprint(self, filename, module, compilation_options): r""" Return a fingerprint of a cython file that is about to be cythonized. @@ -637,11 +637,11 @@ class DependencyTree(object): is found, the cythonization can be skipped. The fingerprint must incorporate everything that has an influence on the generated code. """ - try: + try: m = hashlib.md5(__version__.encode('UTF-8')) m.update(file_hash(filename).encode('UTF-8')) - for x in sorted(self.all_dependencies(filename)): - if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): + for x in sorted(self.all_dependencies(filename)): + if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): m.update(file_hash(x).encode('UTF-8')) # Include the module attributes that change the compilation result # in the fingerprint. We do not iterate over module.__dict__ and @@ -655,21 +655,21 @@ class DependencyTree(object): )).encode('UTF-8')) m.update(compilation_options.get_fingerprint().encode('UTF-8')) - return m.hexdigest() - except IOError: - return None - - def distutils_info0(self, filename): - info = self.parse_dependencies(filename)[3] + return m.hexdigest() + except IOError: + return None + + def distutils_info0(self, filename): + info = self.parse_dependencies(filename)[3] kwds = info.values cimports, externs, incdirs = self.cimports_externs_incdirs(filename) basedir = os.getcwd() # Add dependencies on "cdef extern from ..." files - if externs: + if externs: externs = _make_relative(externs, basedir) if 'depends' in kwds: kwds['depends'] = list(set(kwds['depends']).union(externs)) - else: + else: kwds['depends'] = list(externs) # Add include_dirs to ensure that the C compiler will find the # "cdef extern from ..." files @@ -679,58 +679,58 @@ class DependencyTree(object): if inc not in include_dirs: include_dirs.append(inc) kwds['include_dirs'] = include_dirs - return info - - def distutils_info(self, filename, aliases=None, base=None): - return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge) - .subs(aliases) - .merge(base)) - - def transitive_merge(self, node, extract, merge): - try: - seen = self._transitive_cache[extract, merge] - except KeyError: - seen = self._transitive_cache[extract, merge] = {} - return self.transitive_merge_helper( - node, extract, merge, seen, {}, self.cimported_files)[0] - - def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing): - if node in seen: - return seen[node], None - deps = extract(node) - if node in stack: - return deps, node - try: - stack[node] = len(stack) - loop = None - for next in outgoing(node): - sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing) - if sub_loop is not None: - if loop is not None and stack[loop] < stack[sub_loop]: - pass - else: - loop = sub_loop - deps = merge(deps, sub_deps) - if loop == node: - loop = None - if loop is None: - seen[node] = deps - return deps, loop - finally: - del stack[node] - - -_dep_tree = None - -def create_dependency_tree(ctx=None, quiet=False): - global _dep_tree - if _dep_tree is None: - if ctx is None: - ctx = Context(["."], CompilationOptions(default_options)) - _dep_tree = DependencyTree(ctx, quiet=quiet) - return _dep_tree - - + return info + + def distutils_info(self, filename, aliases=None, base=None): + return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge) + .subs(aliases) + .merge(base)) + + def transitive_merge(self, node, extract, merge): + try: + seen = self._transitive_cache[extract, merge] + except KeyError: + seen = self._transitive_cache[extract, merge] = {} + return self.transitive_merge_helper( + node, extract, merge, seen, {}, self.cimported_files)[0] + + def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing): + if node in seen: + return seen[node], None + deps = extract(node) + if node in stack: + return deps, node + try: + stack[node] = len(stack) + loop = None + for next in outgoing(node): + sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing) + if sub_loop is not None: + if loop is not None and stack[loop] < stack[sub_loop]: + pass + else: + loop = sub_loop + deps = merge(deps, sub_deps) + if loop == node: + loop = None + if loop is None: + seen[node] = deps + return deps, loop + finally: + del stack[node] + + +_dep_tree = None + +def create_dependency_tree(ctx=None, quiet=False): + global _dep_tree + if _dep_tree is None: + if ctx is None: + ctx = Context(["."], CompilationOptions(default_options)) + _dep_tree = DependencyTree(ctx, quiet=quiet) + return _dep_tree + + # If this changes, change also docs/src/reference/compilation.rst # which mentions this function def default_create_extension(template, kwds): @@ -745,7 +745,7 @@ def default_create_extension(template, kwds): return (ext, metadata) -# This may be useful for advanced users? +# This may be useful for advanced users? def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None, exclude_failures=False): if language is not None: @@ -756,17 +756,17 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= if patterns is None: return [], {} elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): - patterns = [patterns] - explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) - seen = set() - deps = create_dependency_tree(ctx, quiet=quiet) - to_exclude = set() - if not isinstance(exclude, list): - exclude = [exclude] - for pattern in exclude: - to_exclude.update(map(os.path.abspath, extended_iglob(pattern))) - - module_list = [] + patterns = [patterns] + explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) + seen = set() + deps = create_dependency_tree(ctx, quiet=quiet) + to_exclude = set() + if not isinstance(exclude, list): + exclude = [exclude] + for pattern in exclude: + to_exclude.update(map(os.path.abspath, extended_iglob(pattern))) + + module_list = [] module_metadata = {} # workaround for setuptools @@ -782,12 +782,12 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= # default function. create_extension = ctx.options.create_extension or default_create_extension - for pattern in patterns: - if isinstance(pattern, str): - filepattern = pattern + for pattern in patterns: + if isinstance(pattern, str): + filepattern = pattern template = Extension(pattern, []) # Fake Extension without sources - name = '*' - base = None + name = '*' + base = None ext_language = language elif isinstance(pattern, (Extension_distutils, Extension_setuptools)): cython_sources = [s for s in pattern.sources @@ -798,56 +798,56 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= print("Warning: Multiple cython sources found for extension '%s': %s\n" "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " "for sharing declarations among Cython files." % (pattern.name, cython_sources)) - else: - # ignore non-cython modules - module_list.append(pattern) - continue - template = pattern - name = template.name - base = DistutilsInfo(exn=template) + else: + # ignore non-cython modules + module_list.append(pattern) + continue + template = pattern + name = template.name + base = DistutilsInfo(exn=template) ext_language = None # do not override whatever the Extension says - else: + else: msg = str("pattern is not of type str nor subclass of Extension (%s)" " but of type %s and class %s" % (repr(Extension), type(pattern), pattern.__class__)) raise TypeError(msg) - + for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern): - if os.path.abspath(file) in to_exclude: - continue + if os.path.abspath(file) in to_exclude: + continue module_name = deps.fully_qualified_name(file) - if '*' in name: - if module_name in explicit_modules: - continue + if '*' in name: + if module_name in explicit_modules: + continue elif name: - module_name = name - + module_name = name + Utils.raise_error_if_module_name_forbidden(module_name) - if module_name not in seen: - try: - kwds = deps.distutils_info(file, aliases, base).values - except Exception: - if exclude_failures: - continue - raise - if base is not None: - for key, value in base.values.items(): - if key not in kwds: - kwds[key] = value - + if module_name not in seen: + try: + kwds = deps.distutils_info(file, aliases, base).values + except Exception: + if exclude_failures: + continue + raise + if base is not None: + for key, value in base.values.items(): + if key not in kwds: + kwds[key] = value + kwds['name'] = module_name sources = [file] + [m for m in template.sources if m != filepattern] - if 'sources' in kwds: - # allow users to add .c files etc. - for source in kwds['sources']: - source = encode_filename_in_py2(source) - if source not in sources: - sources.append(source) + if 'sources' in kwds: + # allow users to add .c files etc. + for source in kwds['sources']: + source = encode_filename_in_py2(source) + if source not in sources: + sources.append(source) kwds['sources'] = sources - + if ext_language and 'language' not in kwds: kwds['language'] = ext_language @@ -873,17 +873,17 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= # never seen this in the wild, but probably better to warn about this unexpected case print("Warning: Cython source file not found in sources list, adding %s" % file) m.sources.insert(0, file) - seen.add(name) + seen.add(name) return module_list, module_metadata - - -# This is the user-exposed entry point. + + +# This is the user-exposed entry point. def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, - exclude_failures=False, **options): - """ - Compile a set of source modules into C/C++ files and return a list of distutils - Extension objects for them. - + exclude_failures=False, **options): + """ + Compile a set of source modules into C/C++ files and return a list of distutils + Extension objects for them. + :param module_list: As module list, pass either a glob pattern, a list of glob patterns or a list of Extension objects. The latter allows you to configure the extensions separately @@ -892,10 +892,10 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, glob patterns as their sources. Then, cythonize will resolve the pattern and create a copy of the Extension for every matching file. - + :param exclude: When passing glob patterns as ``module_list``, you can exclude certain module names explicitly by passing them into the ``exclude`` option. - + :param nthreads: The number of concurrent builds for parallel compilation (requires the ``multiprocessing`` module). @@ -910,10 +910,10 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, find the value of ``MY_HEADERS`` in the ``setup.py``, put it in a python variable called ``foo`` as a string, and then call ``cythonize(..., aliases={'MY_HEADERS': foo})``. - + :param quiet: If True, Cython won't print error, warning, or status messages during the compilation. - + :param force: Forces the recompilation of the Cython modules, even if the timestamps don't indicate that a recompilation is necessary. @@ -946,12 +946,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, See :ref:`compiler-directives`. :param depfile: produce depfiles for the sources if True. - """ + """ if exclude is None: exclude = [] - if 'include_path' not in options: - options['include_path'] = ['.'] - if 'common_utility_include_dir' in options: + if 'include_path' not in options: + options['include_path'] = ['.'] + if 'common_utility_include_dir' in options: safe_makedirs(options['common_utility_include_dir']) depfile = options.pop('depfile', None) @@ -963,21 +963,21 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, pythran_options.cplus = True pythran_options.np_pythran = True - c_options = CompilationOptions(**options) - cpp_options = CompilationOptions(**options); cpp_options.cplus = True - ctx = c_options.create_context() - options = c_options + c_options = CompilationOptions(**options) + cpp_options = CompilationOptions(**options); cpp_options.cplus = True + ctx = c_options.create_context() + options = c_options module_list, module_metadata = create_extension_list( - module_list, - exclude=exclude, - ctx=ctx, - quiet=quiet, - exclude_failures=exclude_failures, + module_list, + exclude=exclude, + ctx=ctx, + quiet=quiet, + exclude_failures=exclude_failures, language=language, - aliases=aliases) - deps = create_dependency_tree(ctx, quiet=quiet) - build_dir = getattr(options, 'build_dir', None) - + aliases=aliases) + deps = create_dependency_tree(ctx, quiet=quiet) + build_dir = getattr(options, 'build_dir', None) + def copy_to_build_dir(filepath, root=os.getcwd()): filepath_abs = os.path.abspath(filepath) if os.path.isabs(filepath): @@ -989,12 +989,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, copy_once_if_newer(filepath_abs, mod_dir) modules_by_cfile = collections.defaultdict(list) - to_compile = [] - for m in module_list: - if build_dir: - for dep in m.depends: - copy_to_build_dir(dep) - + to_compile = [] + for m in module_list: + if build_dir: + for dep in m.depends: + copy_to_build_dir(dep) + cy_sources = [ source for source in m.sources if os.path.splitext(source)[1] in ('.pyx', '.py')] @@ -1005,28 +1005,28 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, # infer FQMN from source files full_module_name = None - new_sources = [] - for source in m.sources: - base, ext = os.path.splitext(source) - if ext in ('.pyx', '.py'): + new_sources = [] + for source in m.sources: + base, ext = os.path.splitext(source) + if ext in ('.pyx', '.py'): if m.np_pythran: - c_file = base + '.cpp' + c_file = base + '.cpp' options = pythran_options elif m.language == 'c++': c_file = base + '.cpp' - options = cpp_options - else: - c_file = base + '.c' - options = c_options - - # setup for out of place build directory if enabled - if build_dir: + options = cpp_options + else: + c_file = base + '.c' + options = c_options + + # setup for out of place build directory if enabled + if build_dir: if os.path.isabs(c_file): warnings.warn("build_dir has no effect for absolute source paths") - c_file = os.path.join(build_dir, c_file) - dir = os.path.dirname(c_file) + c_file = os.path.join(build_dir, c_file) + dir = os.path.dirname(c_file) safe_makedirs_once(dir) - + # write out the depfile, if requested if depfile: dependencies = deps.all_dependencies(source) @@ -1047,45 +1047,45 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, with open(c_file+'.dep', 'w') as outfile: outfile.write(depline) - if os.path.exists(c_file): - c_timestamp = os.path.getmtime(c_file) - else: - c_timestamp = -1 - - # Priority goes first to modified files, second to direct - # dependents, and finally to indirect dependents. - if c_timestamp < deps.timestamp(source): - dep_timestamp, dep = deps.timestamp(source), source - priority = 0 - else: - dep_timestamp, dep = deps.newest_dependency(source) - priority = 2 - (dep in deps.immediate_dependencies(source)) - if force or c_timestamp < dep_timestamp: + if os.path.exists(c_file): + c_timestamp = os.path.getmtime(c_file) + else: + c_timestamp = -1 + + # Priority goes first to modified files, second to direct + # dependents, and finally to indirect dependents. + if c_timestamp < deps.timestamp(source): + dep_timestamp, dep = deps.timestamp(source), source + priority = 0 + else: + dep_timestamp, dep = deps.newest_dependency(source) + priority = 2 - (dep in deps.immediate_dependencies(source)) + if force or c_timestamp < dep_timestamp: if not quiet and not force: - if source == dep: - print("Compiling %s because it changed." % source) - else: - print("Compiling %s because it depends on %s." % (source, dep)) + if source == dep: + print("Compiling %s because it changed." % source) + else: + print("Compiling %s because it depends on %s." % (source, dep)) if not force and options.cache: fingerprint = deps.transitive_fingerprint(source, m, options) - else: - fingerprint = None + else: + fingerprint = None to_compile.append(( priority, source, c_file, fingerprint, quiet, options, not exclude_failures, module_metadata.get(m.name), full_module_name)) - new_sources.append(c_file) + new_sources.append(c_file) modules_by_cfile[c_file].append(m) - else: - new_sources.append(source) - if build_dir: - copy_to_build_dir(source) - m.sources = new_sources - + else: + new_sources.append(source) + if build_dir: + copy_to_build_dir(source) + m.sources = new_sources + if options.cache: - if not os.path.exists(options.cache): - os.makedirs(options.cache) - to_compile.sort() + if not os.path.exists(options.cache): + os.makedirs(options.cache) + to_compile.sort() # Drop "priority" component of "to_compile" entries and add a # simple progress indicator. N = len(to_compile) @@ -1095,119 +1095,119 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, to_compile[i] = to_compile[i][1:] + (progress,) if N <= 1: - nthreads = 0 - if nthreads: - # Requires multiprocessing (or Python >= 2.6) - try: - import multiprocessing - pool = multiprocessing.Pool( - nthreads, initializer=_init_multiprocessing_helper) - except (ImportError, OSError): - print("multiprocessing required for parallel cythonization") - nthreads = 0 - else: - # This is a bit more involved than it should be, because KeyboardInterrupts - # break the multiprocessing workers when using a normal pool.map(). - # See, for example: - # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt - try: - result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) - pool.close() - while not result.ready(): - try: - result.get(99999) # seconds - except multiprocessing.TimeoutError: - pass - except KeyboardInterrupt: - pool.terminate() - raise - pool.join() - if not nthreads: - for args in to_compile: + nthreads = 0 + if nthreads: + # Requires multiprocessing (or Python >= 2.6) + try: + import multiprocessing + pool = multiprocessing.Pool( + nthreads, initializer=_init_multiprocessing_helper) + except (ImportError, OSError): + print("multiprocessing required for parallel cythonization") + nthreads = 0 + else: + # This is a bit more involved than it should be, because KeyboardInterrupts + # break the multiprocessing workers when using a normal pool.map(). + # See, for example: + # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt + try: + result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) + pool.close() + while not result.ready(): + try: + result.get(99999) # seconds + except multiprocessing.TimeoutError: + pass + except KeyboardInterrupt: + pool.terminate() + raise + pool.join() + if not nthreads: + for args in to_compile: cythonize_one(*args) - - if exclude_failures: - failed_modules = set() + + if exclude_failures: + failed_modules = set() for c_file, modules in modules_by_cfile.items(): - if not os.path.exists(c_file): - failed_modules.update(modules) - elif os.path.getsize(c_file) < 200: - f = io_open(c_file, 'r', encoding='iso8859-1') - try: - if f.read(len('#error ')) == '#error ': - # dead compilation result - failed_modules.update(modules) - finally: - f.close() - if failed_modules: - for module in failed_modules: - module_list.remove(module) - print("Failed compilations: %s" % ', '.join(sorted([ - module.name for module in failed_modules]))) - + if not os.path.exists(c_file): + failed_modules.update(modules) + elif os.path.getsize(c_file) < 200: + f = io_open(c_file, 'r', encoding='iso8859-1') + try: + if f.read(len('#error ')) == '#error ': + # dead compilation result + failed_modules.update(modules) + finally: + f.close() + if failed_modules: + for module in failed_modules: + module_list.remove(module) + print("Failed compilations: %s" % ', '.join(sorted([ + module.name for module in failed_modules]))) + if options.cache: - cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100)) - # cythonize() is often followed by the (non-Python-buffered) - # compiler output, flush now to avoid interleaving output. - sys.stdout.flush() - return module_list - - -if os.environ.get('XML_RESULTS'): - compile_result_dir = os.environ['XML_RESULTS'] - def record_results(func): - def with_record(*args): - t = time.time() - success = True - try: - try: - func(*args) - except: - success = False - finally: - t = time.time() - t - module = fully_qualified_name(args[0]) - name = "cythonize." + module - failures = 1 - success - if success: - failure_item = "" - else: - failure_item = "failure" - output = open(os.path.join(compile_result_dir, name + ".xml"), "w") - output.write(""" - <?xml version="1.0" ?> - <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s"> - <testcase classname="%(name)s" name="cythonize"> - %(failure_item)s - </testcase> - </testsuite> - """.strip() % locals()) - output.close() - return with_record -else: + cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100)) + # cythonize() is often followed by the (non-Python-buffered) + # compiler output, flush now to avoid interleaving output. + sys.stdout.flush() + return module_list + + +if os.environ.get('XML_RESULTS'): + compile_result_dir = os.environ['XML_RESULTS'] + def record_results(func): + def with_record(*args): + t = time.time() + success = True + try: + try: + func(*args) + except: + success = False + finally: + t = time.time() - t + module = fully_qualified_name(args[0]) + name = "cythonize." + module + failures = 1 - success + if success: + failure_item = "" + else: + failure_item = "failure" + output = open(os.path.join(compile_result_dir, name + ".xml"), "w") + output.write(""" + <?xml version="1.0" ?> + <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s"> + <testcase classname="%(name)s" name="cythonize"> + %(failure_item)s + </testcase> + </testsuite> + """.strip() % locals()) + output.close() + return with_record +else: def record_results(func): return func + - -# TODO: Share context? Issue: pyx processing leaks into pxd module -@record_results +# TODO: Share context? Issue: pyx processing leaks into pxd module +@record_results def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True, embedded_metadata=None, full_module_name=None, progress=""): from ..Compiler.Main import compile_single, default_options - from ..Compiler.Errors import CompileError, PyrexError - - if fingerprint: - if not os.path.exists(options.cache): + from ..Compiler.Errors import CompileError, PyrexError + + if fingerprint: + if not os.path.exists(options.cache): safe_makedirs(options.cache) - # Cython-generated c files are highly compressible. - # (E.g. a compression ratio of about 10 for Sage). + # Cython-generated c files are highly compressible. + # (E.g. a compression ratio of about 10 for Sage). fingerprint_file_base = join_path( options.cache, "%s-%s" % (os.path.basename(c_file), fingerprint)) gz_fingerprint_file = fingerprint_file_base + gzip_ext zip_fingerprint_file = fingerprint_file_base + '.zip' if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): - if not quiet: + if not quiet: print("%sFound compiled %s in cache" % (progress, pyx_file)) if os.path.exists(gz_fingerprint_file): os.utime(gz_fingerprint_file, None) @@ -1220,37 +1220,37 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, with contextlib.closing(zipfile.ZipFile(zip_fingerprint_file)) as z: for artifact in z.namelist(): z.extract(artifact, os.path.join(dirname, artifact)) - return - if not quiet: + return + if not quiet: print("%sCythonizing %s" % (progress, pyx_file)) - if options is None: - options = CompilationOptions(default_options) - options.output_file = c_file + if options is None: + options = CompilationOptions(default_options) + options.output_file = c_file options.embedded_metadata = embedded_metadata - - any_failures = 0 - try: + + any_failures = 0 + try: result = compile_single(pyx_file, options, full_module_name=full_module_name) - if result.num_errors > 0: - any_failures = 1 + if result.num_errors > 0: + any_failures = 1 except (EnvironmentError, PyrexError) as e: - sys.stderr.write('%s\n' % e) - any_failures = 1 - # XXX - import traceback - traceback.print_exc() - except Exception: - if raise_on_failure: - raise - import traceback - traceback.print_exc() - any_failures = 1 - if any_failures: - if raise_on_failure: - raise CompileError(None, pyx_file) - elif os.path.exists(c_file): - os.remove(c_file) - elif fingerprint: + sys.stderr.write('%s\n' % e) + any_failures = 1 + # XXX + import traceback + traceback.print_exc() + except Exception: + if raise_on_failure: + raise + import traceback + traceback.print_exc() + any_failures = 1 + if any_failures: + if raise_on_failure: + raise CompileError(None, pyx_file) + elif os.path.exists(c_file): + os.remove(c_file) + elif fingerprint: artifacts = list(filter(None, [ getattr(result, attr, None) for attr in ('c_file', 'h_file', 'api_file', 'i_file')])) @@ -1266,43 +1266,43 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, for artifact in artifacts: zip.write(artifact, os.path.basename(artifact)) os.rename(fingerprint_file + '.tmp', fingerprint_file) - - -def cythonize_one_helper(m): - import traceback - try: + + +def cythonize_one_helper(m): + import traceback + try: return cythonize_one(*m) - except Exception: - traceback.print_exc() - raise - - -def _init_multiprocessing_helper(): - # KeyboardInterrupt kills workers, so don't let them get it - import signal - signal.signal(signal.SIGINT, signal.SIG_IGN) - - -def cleanup_cache(cache, target_size, ratio=.85): - try: - p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) - res = p.wait() - if res == 0: - total_size = 1024 * int(p.stdout.read().strip().split()[0]) - if total_size < target_size: - return - except (OSError, ValueError): - pass - total_size = 0 - all = [] - for file in os.listdir(cache): - path = join_path(cache, file) - s = os.stat(path) - total_size += s.st_size - all.append((s.st_atime, s.st_size, path)) - if total_size > target_size: - for time, size, file in reversed(sorted(all)): - os.unlink(file) - total_size -= size - if total_size < target_size * ratio: - break + except Exception: + traceback.print_exc() + raise + + +def _init_multiprocessing_helper(): + # KeyboardInterrupt kills workers, so don't let them get it + import signal + signal.signal(signal.SIGINT, signal.SIG_IGN) + + +def cleanup_cache(cache, target_size, ratio=.85): + try: + p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) + res = p.wait() + if res == 0: + total_size = 1024 * int(p.stdout.read().strip().split()[0]) + if total_size < target_size: + return + except (OSError, ValueError): + pass + total_size = 0 + all = [] + for file in os.listdir(cache): + path = join_path(cache, file) + s = os.stat(path) + total_size += s.st_size + all.append((s.st_atime, s.st_size, path)) + if total_size > target_size: + for time, size, file in reversed(sorted(all)): + os.unlink(file) + total_size -= size + if total_size < target_size * ratio: + break diff --git a/contrib/tools/cython/Cython/Build/Inline.py b/contrib/tools/cython/Cython/Build/Inline.py index db6d2640a5..eb3c40bc77 100644 --- a/contrib/tools/cython/Cython/Build/Inline.py +++ b/contrib/tools/cython/Cython/Build/Inline.py @@ -1,39 +1,39 @@ -from __future__ import absolute_import - +from __future__ import absolute_import + import hashlib import inspect import os import re import sys - -from distutils.core import Distribution, Extension -from distutils.command.build_ext import build_ext - -import Cython + +from distutils.core import Distribution, Extension +from distutils.command.build_ext import build_ext + +import Cython from ..Compiler.Main import Context, default_options - + from ..Compiler.Visitor import CythonTransform, EnvTransform from ..Compiler.ParseTreeTransforms import SkipDeclarations -from ..Compiler.TreeFragment import parse_from_strings +from ..Compiler.TreeFragment import parse_from_strings from ..Compiler.StringEncoding import _unicode -from .Dependencies import strip_string_literals, cythonize, cached_function +from .Dependencies import strip_string_literals, cythonize, cached_function from ..Compiler import Pipeline -from ..Utils import get_cython_cache_dir -import cython as cython_module - +from ..Utils import get_cython_cache_dir +import cython as cython_module + IS_PY3 = sys.version_info >= (3,) -# A utility function to convert user-supplied ASCII strings to unicode. +# A utility function to convert user-supplied ASCII strings to unicode. if not IS_PY3: - def to_unicode(s): + def to_unicode(s): if isinstance(s, bytes): - return s.decode('ascii') - else: - return s -else: - to_unicode = lambda x: x - + return s.decode('ascii') + else: + return s +else: + to_unicode = lambda x: x + if sys.version_info < (3, 5): import imp def load_dynamic(name, module_path): @@ -46,87 +46,87 @@ else: # sys.modules[name] = module spec.loader.exec_module(module) return module - -class UnboundSymbols(EnvTransform, SkipDeclarations): - def __init__(self): - CythonTransform.__init__(self, None) - self.unbound = set() - def visit_NameNode(self, node): - if not self.current_env().lookup(node.name): - self.unbound.add(node.name) - return node - def __call__(self, node): - super(UnboundSymbols, self).__call__(node) - return self.unbound - - -@cached_function -def unbound_symbols(code, context=None): - code = to_unicode(code) - if context is None: - context = Context([], default_options) - from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform - tree = parse_from_strings('(tree fragment)', code) - for phase in Pipeline.create_pipeline(context, 'pyx'): - if phase is None: - continue - tree = phase(tree) - if isinstance(phase, AnalyseDeclarationsTransform): - break - try: - import builtins - except ImportError: - import __builtin__ as builtins + +class UnboundSymbols(EnvTransform, SkipDeclarations): + def __init__(self): + CythonTransform.__init__(self, None) + self.unbound = set() + def visit_NameNode(self, node): + if not self.current_env().lookup(node.name): + self.unbound.add(node.name) + return node + def __call__(self, node): + super(UnboundSymbols, self).__call__(node) + return self.unbound + + +@cached_function +def unbound_symbols(code, context=None): + code = to_unicode(code) + if context is None: + context = Context([], default_options) + from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform + tree = parse_from_strings('(tree fragment)', code) + for phase in Pipeline.create_pipeline(context, 'pyx'): + if phase is None: + continue + tree = phase(tree) + if isinstance(phase, AnalyseDeclarationsTransform): + break + try: + import builtins + except ImportError: + import __builtin__ as builtins return tuple(UnboundSymbols()(tree) - set(dir(builtins))) - - -def unsafe_type(arg, context=None): - py_type = type(arg) - if py_type is int: - return 'long' - else: - return safe_type(arg, context) - - -def safe_type(arg, context=None): - py_type = type(arg) + + +def unsafe_type(arg, context=None): + py_type = type(arg) + if py_type is int: + return 'long' + else: + return safe_type(arg, context) + + +def safe_type(arg, context=None): + py_type = type(arg) if py_type in (list, tuple, dict, str): - return py_type.__name__ - elif py_type is complex: - return 'double complex' - elif py_type is float: - return 'double' - elif py_type is bool: - return 'bint' - elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray): - return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim) - else: + return py_type.__name__ + elif py_type is complex: + return 'double complex' + elif py_type is float: + return 'double' + elif py_type is bool: + return 'bint' + elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray): + return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim) + else: for base_type in py_type.__mro__: - if base_type.__module__ in ('__builtin__', 'builtins'): - return 'object' - module = context.find_module(base_type.__module__, need_pxd=False) - if module: - entry = module.lookup(base_type.__name__) - if entry.is_type: - return '%s.%s' % (base_type.__module__, base_type.__name__) - return 'object' - - -def _get_build_extension(): - dist = Distribution() - # Ensure the build respects distutils configuration by parsing - # the configuration files - config_files = dist.find_config_files() - dist.parse_config_files(config_files) - build_extension = build_ext(dist) - build_extension.finalize_options() - return build_extension - - -@cached_function -def _create_context(cython_include_dirs): - return Context(list(cython_include_dirs), default_options) - + if base_type.__module__ in ('__builtin__', 'builtins'): + return 'object' + module = context.find_module(base_type.__module__, need_pxd=False) + if module: + entry = module.lookup(base_type.__name__) + if entry.is_type: + return '%s.%s' % (base_type.__module__, base_type.__name__) + return 'object' + + +def _get_build_extension(): + dist = Distribution() + # Ensure the build respects distutils configuration by parsing + # the configuration files + config_files = dist.find_config_files() + dist.parse_config_files(config_files) + build_extension = build_ext(dist) + build_extension.finalize_options() + return build_extension + + +@cached_function +def _create_context(cython_include_dirs): + return Context(list(cython_include_dirs), default_options) + _cython_inline_cache = {} _cython_inline_default_context = _create_context(('.',)) @@ -159,8 +159,8 @@ def cython_inline(code, get_type=unsafe_type, cython_include_dirs=None, cython_compiler_directives=None, force=False, quiet=False, locals=None, globals=None, language_level=None, **kwds): - if get_type is None: - get_type = lambda x: 'object' + if get_type is None: + get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context cython_compiler_directives = dict(cython_compiler_directives) if cython_compiler_directives else {} @@ -182,194 +182,194 @@ def cython_inline(code, get_type=unsafe_type, return invoke(*arg_list) orig_code = code - code = to_unicode(code) - code, literals = strip_string_literals(code) - code = strip_common_indent(code) - if locals is None: - locals = inspect.currentframe().f_back.f_back.f_locals - if globals is None: - globals = inspect.currentframe().f_back.f_back.f_globals - try: + code = to_unicode(code) + code, literals = strip_string_literals(code) + code = strip_common_indent(code) + if locals is None: + locals = inspect.currentframe().f_back.f_back.f_locals + if globals is None: + globals = inspect.currentframe().f_back.f_back.f_globals + try: _cython_inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code) _populate_unbound(kwds, _unbound_symbols, locals, globals) - except AssertionError: - if not quiet: - # Parsing from strings not fully supported (e.g. cimports). - print("Could not parse code as a string (to extract unbound symbols).") + except AssertionError: + if not quiet: + # Parsing from strings not fully supported (e.g. cimports). + print("Could not parse code as a string (to extract unbound symbols).") - cimports = [] + cimports = [] for name, arg in list(kwds.items()): - if arg is cython_module: - cimports.append('\ncimport cython as %s' % name) - del kwds[name] + if arg is cython_module: + cimports.append('\ncimport cython as %s' % name) + del kwds[name] arg_names = sorted(kwds) - arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) + arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key_hash = _inline_key(orig_code, arg_sigs, language_level) module_name = "_cython_inline_" + key_hash - - if module_name in sys.modules: - module = sys.modules[module_name] - - else: - build_extension = None - if cython_inline.so_ext is None: - # Figure out and cache current extension suffix - build_extension = _get_build_extension() - cython_inline.so_ext = build_extension.get_ext_filename('') - - module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) - - if not os.path.exists(lib_dir): - os.makedirs(lib_dir) - if force or not os.path.isfile(module_path): - cflags = [] - c_include_dirs = [] - qualified = re.compile(r'([.\w]+)[.]') - for type, _ in arg_sigs: - m = qualified.match(type) - if m: - cimports.append('\ncimport %s' % m.groups()[0]) - # one special case - if m.groups()[0] == 'numpy': - import numpy - c_include_dirs.append(numpy.get_include()) - # cflags.append('-Wno-unused') - module_body, func_body = extract_func_code(code) - params = ', '.join(['%s %s' % a for a in arg_sigs]) - module_code = """ -%(module_body)s -%(cimports)s -def __invoke(%(params)s): -%(func_body)s - return locals() - """ % {'cimports': '\n'.join(cimports), - 'module_body': module_body, - 'params': params, - 'func_body': func_body } - for key, value in literals.items(): - module_code = module_code.replace(key, value) - pyx_file = os.path.join(lib_dir, module_name + '.pyx') - fh = open(pyx_file, 'w') - try: - fh.write(module_code) - finally: - fh.close() - extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - extra_compile_args = cflags) - if build_extension is None: - build_extension = _get_build_extension() + + if module_name in sys.modules: + module = sys.modules[module_name] + + else: + build_extension = None + if cython_inline.so_ext is None: + # Figure out and cache current extension suffix + build_extension = _get_build_extension() + cython_inline.so_ext = build_extension.get_ext_filename('') + + module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) + + if not os.path.exists(lib_dir): + os.makedirs(lib_dir) + if force or not os.path.isfile(module_path): + cflags = [] + c_include_dirs = [] + qualified = re.compile(r'([.\w]+)[.]') + for type, _ in arg_sigs: + m = qualified.match(type) + if m: + cimports.append('\ncimport %s' % m.groups()[0]) + # one special case + if m.groups()[0] == 'numpy': + import numpy + c_include_dirs.append(numpy.get_include()) + # cflags.append('-Wno-unused') + module_body, func_body = extract_func_code(code) + params = ', '.join(['%s %s' % a for a in arg_sigs]) + module_code = """ +%(module_body)s +%(cimports)s +def __invoke(%(params)s): +%(func_body)s + return locals() + """ % {'cimports': '\n'.join(cimports), + 'module_body': module_body, + 'params': params, + 'func_body': func_body } + for key, value in literals.items(): + module_code = module_code.replace(key, value) + pyx_file = os.path.join(lib_dir, module_name + '.pyx') + fh = open(pyx_file, 'w') + try: + fh.write(module_code) + finally: + fh.close() + extension = Extension( + name = module_name, + sources = [pyx_file], + include_dirs = c_include_dirs, + extra_compile_args = cflags) + if build_extension is None: + build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs or ['.'], compiler_directives=cython_compiler_directives, quiet=quiet) - build_extension.build_temp = os.path.dirname(pyx_file) - build_extension.build_lib = lib_dir - build_extension.run() - + build_extension.build_temp = os.path.dirname(pyx_file) + build_extension.build_lib = lib_dir + build_extension.run() + module = load_dynamic(module_name, module_path) - + _cython_inline_cache[orig_code, arg_sigs, key_hash] = module.__invoke - arg_list = [kwds[arg] for arg in arg_names] - return module.__invoke(*arg_list) - - -# Cached suffix used by cython_inline above. None should get -# overridden with actual value upon the first cython_inline invocation -cython_inline.so_ext = None - + arg_list = [kwds[arg] for arg in arg_names] + return module.__invoke(*arg_list) + + +# Cached suffix used by cython_inline above. None should get +# overridden with actual value upon the first cython_inline invocation +cython_inline.so_ext = None + _find_non_space = re.compile('[^ ]').search -def strip_common_indent(code): - min_indent = None +def strip_common_indent(code): + min_indent = None lines = code.splitlines() - for line in lines: + for line in lines: match = _find_non_space(line) - if not match: + if not match: continue # blank - indent = match.start() - if line[indent] == '#': + indent = match.start() + if line[indent] == '#': continue # comment if min_indent is None or min_indent > indent: - min_indent = indent - for ix, line in enumerate(lines): + min_indent = indent + for ix, line in enumerate(lines): match = _find_non_space(line) if not match or not line or line[indent:indent+1] == '#': - continue + continue lines[ix] = line[min_indent:] - return '\n'.join(lines) - - -module_statement = re.compile(r'^((cdef +(extern|class))|cimport|(from .+ cimport)|(from .+ import +[*]))') -def extract_func_code(code): - module = [] - function = [] - current = function - code = code.replace('\t', ' ') - lines = code.split('\n') - for line in lines: - if not line.startswith(' '): - if module_statement.match(line): - current = module - else: - current = function - current.append(line) - return '\n'.join(module), ' ' + '\n '.join(function) - - -try: - from inspect import getcallargs -except ImportError: - def getcallargs(func, *arg_values, **kwd_values): - all = {} - args, varargs, kwds, defaults = inspect.getargspec(func) - if varargs is not None: - all[varargs] = arg_values[len(args):] - for name, value in zip(args, arg_values): - all[name] = value + return '\n'.join(lines) + + +module_statement = re.compile(r'^((cdef +(extern|class))|cimport|(from .+ cimport)|(from .+ import +[*]))') +def extract_func_code(code): + module = [] + function = [] + current = function + code = code.replace('\t', ' ') + lines = code.split('\n') + for line in lines: + if not line.startswith(' '): + if module_statement.match(line): + current = module + else: + current = function + current.append(line) + return '\n'.join(module), ' ' + '\n '.join(function) + + +try: + from inspect import getcallargs +except ImportError: + def getcallargs(func, *arg_values, **kwd_values): + all = {} + args, varargs, kwds, defaults = inspect.getargspec(func) + if varargs is not None: + all[varargs] = arg_values[len(args):] + for name, value in zip(args, arg_values): + all[name] = value for name, value in list(kwd_values.items()): - if name in args: - if name in all: - raise TypeError("Duplicate argument %s" % name) - all[name] = kwd_values.pop(name) - if kwds is not None: - all[kwds] = kwd_values - elif kwd_values: + if name in args: + if name in all: + raise TypeError("Duplicate argument %s" % name) + all[name] = kwd_values.pop(name) + if kwds is not None: + all[kwds] = kwd_values + elif kwd_values: raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values)) - if defaults is None: - defaults = () - first_default = len(args) - len(defaults) - for ix, name in enumerate(args): - if name not in all: - if ix >= first_default: - all[name] = defaults[ix - first_default] - else: - raise TypeError("Missing argument: %s" % name) - return all - - -def get_body(source): - ix = source.index(':') - if source[:5] == 'lambda': - return "return %s" % source[ix+1:] - else: - return source[ix+1:] - - -# Lots to be done here... It would be especially cool if compiled functions -# could invoke each other quickly. -class RuntimeCompiledFunction(object): - - def __init__(self, f): - self._f = f - self._body = get_body(inspect.getsource(f)) - - def __call__(self, *args, **kwds): - all = getcallargs(self._f, *args, **kwds) + if defaults is None: + defaults = () + first_default = len(args) - len(defaults) + for ix, name in enumerate(args): + if name not in all: + if ix >= first_default: + all[name] = defaults[ix - first_default] + else: + raise TypeError("Missing argument: %s" % name) + return all + + +def get_body(source): + ix = source.index(':') + if source[:5] == 'lambda': + return "return %s" % source[ix+1:] + else: + return source[ix+1:] + + +# Lots to be done here... It would be especially cool if compiled functions +# could invoke each other quickly. +class RuntimeCompiledFunction(object): + + def __init__(self, f): + self._f = f + self._body = get_body(inspect.getsource(f)) + + def __call__(self, *args, **kwds): + all = getcallargs(self._f, *args, **kwds) if IS_PY3: return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all) else: diff --git a/contrib/tools/cython/Cython/Build/IpythonMagic.py b/contrib/tools/cython/Cython/Build/IpythonMagic.py index 7abb97ec70..4b513bfd67 100644 --- a/contrib/tools/cython/Cython/Build/IpythonMagic.py +++ b/contrib/tools/cython/Cython/Build/IpythonMagic.py @@ -1,93 +1,93 @@ -# -*- coding: utf-8 -*- -""" -===================== -Cython related magics -===================== - -Magic command interface for interactive work with Cython - -.. note:: - - The ``Cython`` package needs to be installed separately. It - can be obtained using ``easy_install`` or ``pip``. - -Usage -===== - +# -*- coding: utf-8 -*- +""" +===================== +Cython related magics +===================== + +Magic command interface for interactive work with Cython + +.. note:: + + The ``Cython`` package needs to be installed separately. It + can be obtained using ``easy_install`` or ``pip``. + +Usage +===== + To enable the magics below, execute ``%load_ext cython``. - -``%%cython`` - -{CYTHON_DOC} - -``%%cython_inline`` - -{CYTHON_INLINE_DOC} - -``%%cython_pyximport`` - -{CYTHON_PYXIMPORT_DOC} - -Author: -* Brian Granger - -Code moved from IPython and adapted by: -* Martín Gaitán - -Parts of this code were taken from Cython.inline. -""" -#----------------------------------------------------------------------------- -# Copyright (C) 2010-2011, IPython Development Team. -# -# Distributed under the terms of the Modified BSD License. -# + +``%%cython`` + +{CYTHON_DOC} + +``%%cython_inline`` + +{CYTHON_INLINE_DOC} + +``%%cython_pyximport`` + +{CYTHON_PYXIMPORT_DOC} + +Author: +* Brian Granger + +Code moved from IPython and adapted by: +* Martín Gaitán + +Parts of this code were taken from Cython.inline. +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010-2011, IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# # The full license is in the file ipython-COPYING.rst, distributed with this software. -#----------------------------------------------------------------------------- - -from __future__ import absolute_import, print_function - -import imp -import io -import os -import re -import sys -import time +#----------------------------------------------------------------------------- + +from __future__ import absolute_import, print_function + +import imp +import io +import os +import re +import sys +import time import copy import distutils.log import textwrap - + IO_ENCODING = sys.getfilesystemencoding() IS_PY2 = sys.version_info[0] < 3 -try: - reload -except NameError: # Python 3 - from imp import reload - -try: - import hashlib -except ImportError: - import md5 as hashlib - -from distutils.core import Distribution, Extension -from distutils.command.build_ext import build_ext - -from IPython.core import display -from IPython.core import magic_arguments -from IPython.core.magic import Magics, magics_class, cell_magic +try: + reload +except NameError: # Python 3 + from imp import reload + +try: + import hashlib +except ImportError: + import md5 as hashlib + +from distutils.core import Distribution, Extension +from distutils.command.build_ext import build_ext + +from IPython.core import display +from IPython.core import magic_arguments +from IPython.core.magic import Magics, magics_class, cell_magic try: from IPython.paths import get_ipython_cache_dir except ImportError: # older IPython version from IPython.utils.path import get_ipython_cache_dir -from IPython.utils.text import dedent - -from ..Shadow import __version__ as cython_version -from ..Compiler.Errors import CompileError -from .Inline import cython_inline -from .Dependencies import cythonize - - +from IPython.utils.text import dedent + +from ..Shadow import __version__ as cython_version +from ..Compiler.Errors import CompileError +from .Inline import cython_inline +from .Dependencies import cythonize + + PGO_CONFIG = { 'gcc': { 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'], @@ -110,22 +110,22 @@ else: return name -@magics_class -class CythonMagics(Magics): - - def __init__(self, shell): +@magics_class +class CythonMagics(Magics): + + def __init__(self, shell): super(CythonMagics, self).__init__(shell) - self._reloads = {} - self._code_cache = {} - self._pyximport_installed = False - - def _import_all(self, module): + self._reloads = {} + self._code_cache = {} + self._pyximport_installed = False + + def _import_all(self, module): mdict = module.__dict__ if '__all__' in mdict: keys = mdict['__all__'] else: keys = [k for k in mdict if not k.startswith('_')] - + for k in keys: try: self.shell.push({k: mdict[k]}) @@ -133,65 +133,65 @@ class CythonMagics(Magics): msg = "'module' object has no attribute '%s'" % k raise AttributeError(msg) - @cell_magic - def cython_inline(self, line, cell): - """Compile and run a Cython code cell using Cython.inline. - - This magic simply passes the body of the cell to Cython.inline - and returns the result. If the variables `a` and `b` are defined - in the user's namespace, here is a simple example that returns - their sum:: - - %%cython_inline - return a+b - - For most purposes, we recommend the usage of the `%%cython` magic. - """ - locs = self.shell.user_global_ns - globs = self.shell.user_ns - return cython_inline(cell, locals=locs, globals=globs) - - @cell_magic - def cython_pyximport(self, line, cell): - """Compile and import a Cython code cell using pyximport. - - The contents of the cell are written to a `.pyx` file in the current - working directory, which is then imported using `pyximport`. This - magic requires a module name to be passed:: - - %%cython_pyximport modulename - def f(x): - return 2.0*x - - The compiled module is then imported and all of its symbols are - injected into the user's namespace. For most purposes, we recommend - the usage of the `%%cython` magic. - """ - module_name = line.strip() - if not module_name: - raise ValueError('module name must be given') - fname = module_name + '.pyx' - with io.open(fname, 'w', encoding='utf-8') as f: - f.write(cell) - if 'pyximport' not in sys.modules or not self._pyximport_installed: - import pyximport + @cell_magic + def cython_inline(self, line, cell): + """Compile and run a Cython code cell using Cython.inline. + + This magic simply passes the body of the cell to Cython.inline + and returns the result. If the variables `a` and `b` are defined + in the user's namespace, here is a simple example that returns + their sum:: + + %%cython_inline + return a+b + + For most purposes, we recommend the usage of the `%%cython` magic. + """ + locs = self.shell.user_global_ns + globs = self.shell.user_ns + return cython_inline(cell, locals=locs, globals=globs) + + @cell_magic + def cython_pyximport(self, line, cell): + """Compile and import a Cython code cell using pyximport. + + The contents of the cell are written to a `.pyx` file in the current + working directory, which is then imported using `pyximport`. This + magic requires a module name to be passed:: + + %%cython_pyximport modulename + def f(x): + return 2.0*x + + The compiled module is then imported and all of its symbols are + injected into the user's namespace. For most purposes, we recommend + the usage of the `%%cython` magic. + """ + module_name = line.strip() + if not module_name: + raise ValueError('module name must be given') + fname = module_name + '.pyx' + with io.open(fname, 'w', encoding='utf-8') as f: + f.write(cell) + if 'pyximport' not in sys.modules or not self._pyximport_installed: + import pyximport pyximport.install() - self._pyximport_installed = True - if module_name in self._reloads: - module = self._reloads[module_name] + self._pyximport_installed = True + if module_name in self._reloads: + module = self._reloads[module_name] # Note: reloading extension modules is not actually supported # (requires PEP-489 reinitialisation support). # Don't know why this should ever have worked as it reads here. # All we really need to do is to update the globals below. #reload(module) - else: - __import__(module_name) - module = sys.modules[module_name] - self._reloads[module_name] = module - self._import_all(module) - - @magic_arguments.magic_arguments() - @magic_arguments.argument( + else: + __import__(module_name) + module = sys.modules[module_name] + self._reloads[module_name] = module + self._import_all(module) + + @magic_arguments.magic_arguments() + @magic_arguments.argument( '-a', '--annotate', action='store_true', default=False, help="Produce a colorized HTML version of the source." ) @@ -213,69 +213,69 @@ class CythonMagics(Magics): "previously compiled." ) @magic_arguments.argument( - '-c', '--compile-args', action='append', default=[], - help="Extra flags to pass to compiler via the `extra_compile_args` " - "Extension flag (can be specified multiple times)." - ) - @magic_arguments.argument( - '--link-args', action='append', default=[], - help="Extra flags to pass to linker via the `extra_link_args` " - "Extension flag (can be specified multiple times)." - ) - @magic_arguments.argument( - '-l', '--lib', action='append', default=[], - help="Add a library to link the extension against (can be specified " - "multiple times)." - ) - @magic_arguments.argument( - '-n', '--name', - help="Specify a name for the Cython module." - ) - @magic_arguments.argument( - '-L', dest='library_dirs', metavar='dir', action='append', default=[], + '-c', '--compile-args', action='append', default=[], + help="Extra flags to pass to compiler via the `extra_compile_args` " + "Extension flag (can be specified multiple times)." + ) + @magic_arguments.argument( + '--link-args', action='append', default=[], + help="Extra flags to pass to linker via the `extra_link_args` " + "Extension flag (can be specified multiple times)." + ) + @magic_arguments.argument( + '-l', '--lib', action='append', default=[], + help="Add a library to link the extension against (can be specified " + "multiple times)." + ) + @magic_arguments.argument( + '-n', '--name', + help="Specify a name for the Cython module." + ) + @magic_arguments.argument( + '-L', dest='library_dirs', metavar='dir', action='append', default=[], help="Add a path to the list of library directories (can be specified " - "multiple times)." - ) - @magic_arguments.argument( - '-I', '--include', action='append', default=[], - help="Add a path to the list of include directories (can be specified " - "multiple times)." - ) - @magic_arguments.argument( + "multiple times)." + ) + @magic_arguments.argument( + '-I', '--include', action='append', default=[], + help="Add a path to the list of include directories (can be specified " + "multiple times)." + ) + @magic_arguments.argument( '-S', '--src', action='append', default=[], help="Add a path to the list of src files (can be specified " "multiple times)." - ) - @magic_arguments.argument( + ) + @magic_arguments.argument( '--pgo', dest='pgo', action='store_true', default=False, help=("Enable profile guided optimisation in the C compiler. " "Compiles the cell twice and executes it in between to generate a runtime profile.") - ) - @magic_arguments.argument( + ) + @magic_arguments.argument( '--verbose', dest='quiet', action='store_false', default=True, help=("Print debug information like generated .c/.cpp file location " "and exact gcc/g++ command invoked.") - ) - @cell_magic - def cython(self, line, cell): - """Compile and import everything from a Cython code cell. - - The contents of the cell are written to a `.pyx` file in the - directory `IPYTHONDIR/cython` using a filename with the hash of the - code. This file is then cythonized and compiled. The resulting module - is imported and all of its symbols are injected into the user's - namespace. The usage is similar to that of `%%cython_pyximport` but - you don't have to pass a module name:: - - %%cython - def f(x): - return 2.0*x - - To compile OpenMP codes, pass the required `--compile-args` - and `--link-args`. For example with gcc:: - - %%cython --compile-args=-fopenmp --link-args=-fopenmp - ... + ) + @cell_magic + def cython(self, line, cell): + """Compile and import everything from a Cython code cell. + + The contents of the cell are written to a `.pyx` file in the + directory `IPYTHONDIR/cython` using a filename with the hash of the + code. This file is then cythonized and compiled. The resulting module + is imported and all of its symbols are injected into the user's + namespace. The usage is similar to that of `%%cython_pyximport` but + you don't have to pass a module name:: + + %%cython + def f(x): + return 2.0*x + + To compile OpenMP codes, pass the required `--compile-args` + and `--link-args`. For example with gcc:: + + %%cython --compile-args=-fopenmp --link-args=-fopenmp + ... To enable profile guided optimisation, pass the ``--pgo`` option. Note that the cell itself needs to take care of establishing a suitable @@ -298,46 +298,46 @@ class CythonMagics(Magics): if "_pgo_" in __name__: ... # execute critical code here - """ - args = magic_arguments.parse_argstring(self.cython, line) + """ + args = magic_arguments.parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell + '\n' - lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') + lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') key = (code, line, sys.version_info, sys.executable, cython_version) - - if not os.path.exists(lib_dir): - os.makedirs(lib_dir) - + + if not os.path.exists(lib_dir): + os.makedirs(lib_dir) + if args.pgo: key += ('pgo',) - if args.force: - # Force a new module name by adding the current time to the - # key which is hashed to determine the module name. + if args.force: + # Force a new module name by adding the current time to the + # key which is hashed to determine the module name. key += (time.time(),) - - if args.name: + + if args.name: module_name = str(args.name) # no-op in Py3 - else: - module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() + else: + module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() html_file = os.path.join(lib_dir, module_name + '.html') - module_path = os.path.join(lib_dir, module_name + self.so_ext) - - have_module = os.path.isfile(module_path) + module_path = os.path.join(lib_dir, module_name + self.so_ext) + + have_module = os.path.isfile(module_path) need_cythonize = args.pgo or not have_module - - if args.annotate: - if not os.path.isfile(html_file): - need_cythonize = True - + + if args.annotate: + if not os.path.isfile(html_file): + need_cythonize = True + extension = None - if need_cythonize: + if need_cythonize: extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet) if extensions is None: # Compilation failed and printed error message return None assert len(extensions) == 1 extension = extensions[0] - self._code_cache[key] = module_name - + self._code_cache[key] = module_name + if args.pgo: self._profile_pgo_wrapper(extension, lib_dir) @@ -348,24 +348,24 @@ class CythonMagics(Magics): # Build failed and printed error message return None - module = imp.load_dynamic(module_name, module_path) - self._import_all(module) - - if args.annotate: - try: - with io.open(html_file, encoding='utf-8') as f: - annotated_html = f.read() - except IOError as e: - # File could not be opened. Most likely the user has a version - # of Cython before 0.15.1 (when `cythonize` learned the - # `force` keyword argument) and has already compiled this - # exact source without annotation. - print('Cython completed successfully but the annotated ' - 'source could not be read.', file=sys.stderr) - print(e, file=sys.stderr) - else: - return display.HTML(self.clean_annotated_html(annotated_html)) - + module = imp.load_dynamic(module_name, module_path) + self._import_all(module) + + if args.annotate: + try: + with io.open(html_file, encoding='utf-8') as f: + annotated_html = f.read() + except IOError as e: + # File could not be opened. Most likely the user has a version + # of Cython before 0.15.1 (when `cythonize` learned the + # `force` keyword argument) and has already compiled this + # exact source without annotation. + print('Cython completed successfully but the annotated ' + 'source could not be read.', file=sys.stderr) + print(e, file=sys.stderr) + else: + return display.HTML(self.clean_annotated_html(annotated_html)) + def _profile_pgo_wrapper(self, extension, lib_dir): """ Generate a .c file for a separate extension module that calls the @@ -489,37 +489,37 @@ class CythonMagics(Magics): file=sys.stderr) return orig_flags - @property - def so_ext(self): - """The extension suffix for compiled modules.""" - try: - return self._so_ext - except AttributeError: - self._so_ext = self._get_build_extension().get_ext_filename('') - return self._so_ext - - def _clear_distutils_mkpath_cache(self): - """clear distutils mkpath cache - - prevents distutils from skipping re-creation of dirs that have been removed - """ - try: - from distutils.dir_util import _path_created - except ImportError: - pass - else: - _path_created.clear() - + @property + def so_ext(self): + """The extension suffix for compiled modules.""" + try: + return self._so_ext + except AttributeError: + self._so_ext = self._get_build_extension().get_ext_filename('') + return self._so_ext + + def _clear_distutils_mkpath_cache(self): + """clear distutils mkpath cache + + prevents distutils from skipping re-creation of dirs that have been removed + """ + try: + from distutils.dir_util import _path_created + except ImportError: + pass + else: + _path_created.clear() + def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None, pgo_step_name=None, _build_ext=build_ext): - self._clear_distutils_mkpath_cache() - dist = Distribution() - config_files = dist.find_config_files() - try: - config_files.remove('setup.cfg') - except ValueError: - pass - dist.parse_config_files(config_files) + self._clear_distutils_mkpath_cache() + dist = Distribution() + config_files = dist.find_config_files() + try: + config_files.remove('setup.cfg') + except ValueError: + pass + dist.parse_config_files(config_files) if not temp_dir: temp_dir = lib_dir @@ -533,7 +533,7 @@ class CythonMagics(Magics): base_build_ext.build_extensions(self) build_extension = _build_ext(dist) - build_extension.finalize_options() + build_extension.finalize_options() if temp_dir: temp_dir = encode_fs(temp_dir) build_extension.build_temp = temp_dir @@ -542,24 +542,24 @@ class CythonMagics(Magics): build_extension.build_lib = lib_dir if extension is not None: build_extension.extensions = [extension] - return build_extension - - @staticmethod - def clean_annotated_html(html): - """Clean up the annotated HTML source. - - Strips the link to the generated C or C++ file, which we do not - present to the user. - """ - r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>') - html = '\n'.join(l for l in html.splitlines() if not r.match(l)) - return html - -__doc__ = __doc__.format( + return build_extension + + @staticmethod + def clean_annotated_html(html): + """Clean up the annotated HTML source. + + Strips the link to the generated C or C++ file, which we do not + present to the user. + """ + r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>') + html = '\n'.join(l for l in html.splitlines() if not r.match(l)) + return html + +__doc__ = __doc__.format( # rST doesn't see the -+ flag as part of an option list, so we # hide it from the module-level docstring. CYTHON_DOC=dedent(CythonMagics.cython.__doc__\ .replace('-+, --cplus', '--cplus ')), CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__), CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__), -) +) diff --git a/contrib/tools/cython/Cython/Build/Tests/TestInline.py b/contrib/tools/cython/Cython/Build/Tests/TestInline.py index d209488083..0877dee03f 100644 --- a/contrib/tools/cython/Cython/Build/Tests/TestInline.py +++ b/contrib/tools/cython/Cython/Build/Tests/TestInline.py @@ -1,71 +1,71 @@ -import os, tempfile -from Cython.Shadow import inline -from Cython.Build.Inline import safe_type -from Cython.TestUtils import CythonTest - -try: - import numpy - has_numpy = True -except: - has_numpy = False - -test_kwds = dict(force=True, quiet=True) - -global_value = 100 - -class TestInline(CythonTest): - def setUp(self): - CythonTest.setUp(self) - self.test_kwds = dict(test_kwds) +import os, tempfile +from Cython.Shadow import inline +from Cython.Build.Inline import safe_type +from Cython.TestUtils import CythonTest + +try: + import numpy + has_numpy = True +except: + has_numpy = False + +test_kwds = dict(force=True, quiet=True) + +global_value = 100 + +class TestInline(CythonTest): + def setUp(self): + CythonTest.setUp(self) + self.test_kwds = dict(test_kwds) if os.path.isdir('TEST_TMP'): lib_dir = os.path.join('TEST_TMP','inline') - else: - lib_dir = tempfile.mkdtemp(prefix='cython_inline_') - self.test_kwds['lib_dir'] = lib_dir - - def test_simple(self): + else: + lib_dir = tempfile.mkdtemp(prefix='cython_inline_') + self.test_kwds['lib_dir'] = lib_dir + + def test_simple(self): self.assertEqual(inline("return 1+2", **self.test_kwds), 3) - - def test_types(self): + + def test_types(self): self.assertEqual(inline(""" - cimport cython - return cython.typeof(a), cython.typeof(b) - """, a=1.0, b=[], **self.test_kwds), ('double', 'list object')) - - def test_locals(self): - a = 1 - b = 2 + cimport cython + return cython.typeof(a), cython.typeof(b) + """, a=1.0, b=[], **self.test_kwds), ('double', 'list object')) + + def test_locals(self): + a = 1 + b = 2 self.assertEqual(inline("return a+b", **self.test_kwds), 3) - - def test_globals(self): + + def test_globals(self): self.assertEqual(inline("return global_value + 1", **self.test_kwds), global_value + 1) - - def test_no_return(self): + + def test_no_return(self): self.assertEqual(inline(""" - a = 1 - cdef double b = 2 - cdef c = [] + a = 1 + cdef double b = 2 + cdef c = [] """, **self.test_kwds), dict(a=1, b=2.0, c=[])) - - def test_def_node(self): + + def test_def_node(self): foo = inline("def foo(x): return x * x", **self.test_kwds)['foo'] self.assertEqual(foo(7), 49) - + def test_class_ref(self): class Type(object): pass tp = inline("Type")['Type'] self.assertEqual(tp, Type) - def test_pure(self): - import cython as cy - b = inline(""" - b = cy.declare(float, a) - c = cy.declare(cy.pointer(cy.float), &b) - return b + def test_pure(self): + import cython as cy + b = inline(""" + b = cy.declare(float, a) + c = cy.declare(cy.pointer(cy.float), &b) + return b """, a=3, **self.test_kwds) self.assertEqual(type(b), float) - + def test_compiler_directives(self): self.assertEqual( inline('return sum(x)', @@ -86,11 +86,11 @@ class TestInline(CythonTest): 2.5 ) - if has_numpy: - - def test_numpy(self): - import numpy - a = numpy.ndarray((10, 20)) - a[0,0] = 10 + if has_numpy: + + def test_numpy(self): + import numpy + a = numpy.ndarray((10, 20)) + a[0,0] = 10 self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) diff --git a/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py b/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py index 24213091b2..1b67a970c6 100644 --- a/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py +++ b/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- -# tag: ipython - -"""Tests for the Cython magics extension.""" - +# -*- coding: utf-8 -*- +# tag: ipython + +"""Tests for the Cython magics extension.""" + from __future__ import absolute_import -import os -import sys +import os +import sys from contextlib import contextmanager from Cython.Build import IpythonMagic from Cython.TestUtils import CythonTest - -try: + +try: import IPython.testing.globalipapp except ImportError: # Disable tests and fake helpers for initialisation below. @@ -21,22 +21,22 @@ else: def skip_if_not_installed(c): return c -try: +try: # disable IPython history thread before it gets started to avoid having to clean it up - from IPython.core.history import HistoryManager - HistoryManager.enabled = False -except ImportError: - pass - + from IPython.core.history import HistoryManager + HistoryManager.enabled = False +except ImportError: + pass + code = u"""\ def f(x): - return 2*x + return 2*x """ - + cython3_code = u"""\ def f(int x): return 2 / x - + def call(x): return f(*(x,)) """ @@ -48,72 +48,72 @@ main() """ -if sys.platform == 'win32': - # not using IPython's decorators here because they depend on "nose" - try: - from unittest import skip as skip_win32 - except ImportError: - # poor dev's silent @unittest.skip() +if sys.platform == 'win32': + # not using IPython's decorators here because they depend on "nose" + try: + from unittest import skip as skip_win32 + except ImportError: + # poor dev's silent @unittest.skip() def skip_win32(dummy): def _skip_win32(func): return None return _skip_win32 -else: +else: def skip_win32(dummy): def _skip_win32(func): def wrapper(*args, **kwargs): func(*args, **kwargs) return wrapper return _skip_win32 - - + + @skip_if_not_installed -class TestIPythonMagic(CythonTest): - +class TestIPythonMagic(CythonTest): + @classmethod def setUpClass(cls): CythonTest.setUpClass() cls._ip = IPython.testing.globalipapp.get_ipython() - def setUp(self): - CythonTest.setUp(self) + def setUp(self): + CythonTest.setUp(self) self._ip.extension_manager.load_extension('cython') - - def test_cython_inline(self): + + def test_cython_inline(self): ip = self._ip - ip.ex('a=10; b=20') - result = ip.run_cell_magic('cython_inline', '', 'return a+b') - self.assertEqual(result, 30) - + ip.ex('a=10; b=20') + result = ip.run_cell_magic('cython_inline', '', 'return a+b') + self.assertEqual(result, 30) + @skip_win32('Skip on Windows') - def test_cython_pyximport(self): + def test_cython_pyximport(self): ip = self._ip - module_name = '_test_cython_pyximport' - ip.run_cell_magic('cython_pyximport', module_name, code) - ip.ex('g = f(10)') - self.assertEqual(ip.user_ns['g'], 20.0) - ip.run_cell_magic('cython_pyximport', module_name, code) - ip.ex('h = f(-10)') - self.assertEqual(ip.user_ns['h'], -20.0) - try: - os.remove(module_name + '.pyx') - except OSError: - pass - - def test_cython(self): + module_name = '_test_cython_pyximport' + ip.run_cell_magic('cython_pyximport', module_name, code) + ip.ex('g = f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + ip.run_cell_magic('cython_pyximport', module_name, code) + ip.ex('h = f(-10)') + self.assertEqual(ip.user_ns['h'], -20.0) + try: + os.remove(module_name + '.pyx') + except OSError: + pass + + def test_cython(self): ip = self._ip - ip.run_cell_magic('cython', '', code) - ip.ex('g = f(10)') - self.assertEqual(ip.user_ns['g'], 20.0) - - def test_cython_name(self): - # The Cython module named 'mymodule' defines the function f. + ip.run_cell_magic('cython', '', code) + ip.ex('g = f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + + def test_cython_name(self): + # The Cython module named 'mymodule' defines the function f. ip = self._ip - ip.run_cell_magic('cython', '--name=mymodule', code) - # This module can now be imported in the interactive namespace. - ip.ex('import mymodule; g = mymodule.f(10)') - self.assertEqual(ip.user_ns['g'], 20.0) - + ip.run_cell_magic('cython', '--name=mymodule', code) + # This module can now be imported in the interactive namespace. + ip.ex('import mymodule; g = mymodule.f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + def test_cython_language_level(self): # The Cython cell defines the functions f() and call(). ip = self._ip @@ -152,15 +152,15 @@ class TestIPythonMagic(CythonTest): self.assertEqual(ip.user_ns['h'], 2.0 / 10.0) @skip_win32('Skip on Windows') - def test_extlibs(self): + def test_extlibs(self): ip = self._ip code = u""" -from libc.math cimport sin -x = sin(0.0) +from libc.math cimport sin +x = sin(0.0) """ - ip.user_ns['x'] = 1 - ip.run_cell_magic('cython', '-l m', code) - self.assertEqual(ip.user_ns['x'], 0) + ip.user_ns['x'] = 1 + ip.run_cell_magic('cython', '-l m', code) + self.assertEqual(ip.user_ns['x'], 0) def test_cython_verbose(self): diff --git a/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py b/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py index a7572a5083..3f4261128f 100644 --- a/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py +++ b/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py @@ -1,57 +1,57 @@ -from Cython.Build.Dependencies import strip_string_literals - -from Cython.TestUtils import CythonTest - -class TestStripLiterals(CythonTest): - - def t(self, before, expected): - actual, literals = strip_string_literals(before, prefix="_L") +from Cython.Build.Dependencies import strip_string_literals + +from Cython.TestUtils import CythonTest + +class TestStripLiterals(CythonTest): + + def t(self, before, expected): + actual, literals = strip_string_literals(before, prefix="_L") self.assertEqual(expected, actual) - for key, value in literals.items(): - actual = actual.replace(key, value) + for key, value in literals.items(): + actual = actual.replace(key, value) self.assertEqual(before, actual) - - def test_empty(self): - self.t("", "") - - def test_single_quote(self): - self.t("'x'", "'_L1_'") - - def test_double_quote(self): - self.t('"x"', '"_L1_"') - - def test_nested_quotes(self): - self.t(""" '"' "'" """, """ '_L1_' "_L2_" """) - - def test_triple_quote(self): - self.t(" '''a\n''' ", " '''_L1_''' ") - - def test_backslash(self): - self.t(r"'a\'b'", "'_L1_'") - self.t(r"'a\\'", "'_L1_'") - self.t(r"'a\\\'b'", "'_L1_'") - - def test_unicode(self): - self.t("u'abc'", "u'_L1_'") - - def test_raw(self): - self.t(r"r'abc\\'", "r'_L1_'") - - def test_raw_unicode(self): - self.t(r"ru'abc\\'", "ru'_L1_'") - - def test_comment(self): - self.t("abc # foo", "abc #_L1_") - - def test_comment_and_quote(self): - self.t("abc # 'x'", "abc #_L1_") - self.t("'abc#'", "'_L1_'") - - def test_include(self): - self.t("include 'a.pxi' # something here", - "include '_L1_' #_L2_") - - def test_extern(self): - self.t("cdef extern from 'a.h': # comment", - "cdef extern from '_L1_': #_L2_") - + + def test_empty(self): + self.t("", "") + + def test_single_quote(self): + self.t("'x'", "'_L1_'") + + def test_double_quote(self): + self.t('"x"', '"_L1_"') + + def test_nested_quotes(self): + self.t(""" '"' "'" """, """ '_L1_' "_L2_" """) + + def test_triple_quote(self): + self.t(" '''a\n''' ", " '''_L1_''' ") + + def test_backslash(self): + self.t(r"'a\'b'", "'_L1_'") + self.t(r"'a\\'", "'_L1_'") + self.t(r"'a\\\'b'", "'_L1_'") + + def test_unicode(self): + self.t("u'abc'", "u'_L1_'") + + def test_raw(self): + self.t(r"r'abc\\'", "r'_L1_'") + + def test_raw_unicode(self): + self.t(r"ru'abc\\'", "ru'_L1_'") + + def test_comment(self): + self.t("abc # foo", "abc #_L1_") + + def test_comment_and_quote(self): + self.t("abc # 'x'", "abc #_L1_") + self.t("'abc#'", "'_L1_'") + + def test_include(self): + self.t("include 'a.pxi' # something here", + "include '_L1_' #_L2_") + + def test_extern(self): + self.t("cdef extern from 'a.h': # comment", + "cdef extern from '_L1_': #_L2_") + diff --git a/contrib/tools/cython/Cython/Build/Tests/__init__.py b/contrib/tools/cython/Cython/Build/Tests/__init__.py index fa81adaff6..4a2889e8e1 100644 --- a/contrib/tools/cython/Cython/Build/Tests/__init__.py +++ b/contrib/tools/cython/Cython/Build/Tests/__init__.py @@ -1 +1 @@ -# empty file +# empty file diff --git a/contrib/tools/cython/Cython/Build/__init__.py b/contrib/tools/cython/Cython/Build/__init__.py index d6f3986597..265c27a045 100644 --- a/contrib/tools/cython/Cython/Build/__init__.py +++ b/contrib/tools/cython/Cython/Build/__init__.py @@ -1,2 +1,2 @@ -from .Dependencies import cythonize +from .Dependencies import cythonize from .Distutils import build_ext |