Update Cython to 3.0.10.

commit_hash:b43c96b868cd36d636192fd2c6024d9f0d2fb6f8

12 files changed, 798 insertions, 76 deletions

diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestBuffer.py b/contrib/tools/cython/Cython/Compiler/Tests/TestBuffer.py
index 1f69d96524d..2f653d0ff4e 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestBuffer.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestBuffer.py
@@ -55,7 +55,7 @@ class TestBufferOptions(CythonTest):
         root = self.fragment(s, pipeline=[NormalizeTree(self), PostParse(self)]).root
         if not expect_error:
             vardef = root.stats[0].body.stats[0]
-            assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code
+            assert isinstance(vardef, CVarDefNode)  # use normal assert as this is to validate the test code
             buftype = vardef.base_type
             self.assertTrue(isinstance(buftype, TemplatedTypeNode))
             self.assertTrue(isinstance(buftype.base_type_node, CSimpleBaseTypeNode))
@@ -99,7 +99,7 @@ class TestBufferOptions(CythonTest):
 
     # add exotic and impossible combinations as they come along...
 
+
 if __name__ == '__main__':
     import unittest
     unittest.main()
-
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestCmdLine.py b/contrib/tools/cython/Cython/Compiler/Tests/TestCmdLine.py
index bd31da00076..290efd1d7fd 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestCmdLine.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestCmdLine.py
@@ -1,8 +1,12 @@
-
+import os
 import sys
 import re
 from unittest import TestCase
 try:
+    from unittest.mock import patch, Mock
+except ImportError:  # Py2
+    from mock import patch, Mock
+try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO  # doesn't accept 'str' in Py2
@@ -10,38 +14,39 @@ except ImportError:
 from .. import Options
 from ..CmdLine import parse_command_line
 
+from .Utils import backup_Options, restore_Options, check_global_options
 
-def check_global_options(expected_options, white_list=[]):
-    """
-    returns error message of "" if check Ok
-    """
-    no_value = object()
-    for name, orig_value in expected_options.items():
-        if name not in white_list:
-            if getattr(Options, name, no_value) != orig_value:
-                return "error in option " + name
-    return ""
+unpatched_exists = os.path.exists
 
+def patched_exists(path):
+    # avoid the Cython command raising a file not found error
+    if path in (
+        'source.pyx',
+        os.path.join('/work/dir', 'source.pyx'),
+        os.path.join('my_working_path', 'source.pyx'),
+        'file.pyx',
+        'file1.pyx',
+        'file2.pyx',
+        'file3.pyx',
+        'foo.pyx',
+        'bar.pyx',
+    ):
+        return True
+    return unpatched_exists(path)
 
+@patch('os.path.exists', new=Mock(side_effect=patched_exists))
 class CmdLineParserTest(TestCase):
     def setUp(self):
-        backup = {}
-        for name, value in vars(Options).items():
-            backup[name] = value
-        self._options_backup = backup
+        self._options_backup = backup_Options()
 
     def tearDown(self):
-        no_value = object()
-        for name, orig_value in self._options_backup.items():
-            if getattr(Options, name, no_value) != orig_value:
-                setattr(Options, name, orig_value)
+        restore_Options(self._options_backup)
 
     def check_default_global_options(self, white_list=[]):
         self.assertEqual(check_global_options(self._options_backup, white_list), "")
 
     def check_default_options(self, options, white_list=[]):
-        from ..Main import CompilationOptions, default_options
-        default_options = CompilationOptions(default_options)
+        default_options = Options.CompilationOptions(Options.default_options)
         no_value = object()
         for name in default_options.__dict__.keys():
             if name not in white_list:
@@ -121,6 +126,397 @@ class CmdLineParserTest(TestCase):
         self.assertEqual(Options.annotate_coverage_xml, 'cov.xml')
         self.assertTrue(options.gdb_debug)
         self.assertEqual(options.output_dir, '/gdb/outdir')
+        self.assertEqual(options.compiler_directives['wraparound'], False)
+
+    def test_embed_before_positional(self):
+        options, sources = parse_command_line([
+            '--embed',
+            'source.pyx',
+        ])
+        self.assertEqual(sources, ['source.pyx'])
+        self.assertEqual(Options.embed, 'main')
+
+    def test_two_embeds(self):
+        options, sources = parse_command_line([
+            '--embed', '--embed=huhu',
+            'source.pyx',
+        ])
+        self.assertEqual(sources, ['source.pyx'])
+        self.assertEqual(Options.embed, 'huhu')
+
+    def test_two_embeds2(self):
+        options, sources = parse_command_line([
+            '--embed=huhu', '--embed',
+            'source.pyx',
+        ])
+        self.assertEqual(sources, ['source.pyx'])
+        self.assertEqual(Options.embed, 'main')
+
+    def test_no_annotate(self):
+        options, sources = parse_command_line([
+            '--embed=huhu', 'source.pyx'
+        ])
+        self.assertFalse(Options.annotate)
+
+    def test_annotate_short(self):
+        options, sources = parse_command_line([
+            '-a',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'default')
+
+    def test_annotate_long(self):
+        options, sources = parse_command_line([
+            '--annotate',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'default')
+
+    def test_annotate_fullc(self):
+        options, sources = parse_command_line([
+            '--annotate-fullc',
+            'source.pyx',
+        ])
+        self.assertEqual(Options.annotate, 'fullc')
+
+    def test_short_w(self):
+        options, sources = parse_command_line([
+            '-w', 'my_working_path',
+            'source.pyx'
+        ])
+        self.assertEqual(options.working_path, 'my_working_path')
+        self.check_default_global_options()
+        self.check_default_options(options, ['working_path'])
+
+    def test_short_o(self):
+        options, sources = parse_command_line([
+            '-o', 'my_output',
+            'source.pyx'
+        ])
+        self.assertEqual(options.output_file, 'my_output')
+        self.check_default_global_options()
+        self.check_default_options(options, ['output_file'])
+
+    def test_short_z(self):
+        options, sources = parse_command_line([
+            '-z', 'my_preimport',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.pre_import, 'my_preimport')
+        self.check_default_global_options(['pre_import'])
+        self.check_default_options(options)
+
+    def test_convert_range(self):
+        options, sources = parse_command_line([
+            '--convert-range',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.convert_range, True)
+        self.check_default_global_options(['convert_range'])
+        self.check_default_options(options)
+
+    def test_line_directives(self):
+        options, sources = parse_command_line([
+            '--line-directives',
+            'source.pyx'
+        ])
+        self.assertEqual(options.emit_linenums, True)
+        self.check_default_global_options()
+        self.check_default_options(options, ['emit_linenums'])
+
+    def test_no_c_in_traceback(self):
+        options, sources = parse_command_line([
+            '--no-c-in-traceback',
+            'source.pyx'
+        ])
+        self.assertEqual(options.c_line_in_traceback, False)
+        self.check_default_global_options()
+        self.check_default_options(options, ['c_line_in_traceback'])
+
+    def test_gdb(self):
+        options, sources = parse_command_line([
+            '--gdb',
+            'source.pyx'
+        ])
+        self.assertEqual(options.gdb_debug, True)
+        self.assertEqual(options.output_dir, os.curdir)
+        self.check_default_global_options()
+        self.check_default_options(options, ['gdb_debug', 'output_dir'])
+
+    def test_3str(self):
+        options, sources = parse_command_line([
+            '--3str',
+            'source.pyx'
+        ])
+        self.assertEqual(options.language_level, '3str')
+        self.check_default_global_options()
+        self.check_default_options(options, ['language_level'])
+
+    def test_capi_reexport_cincludes(self):
+        options, sources = parse_command_line([
+            '--capi-reexport-cincludes',
+            'source.pyx'
+        ])
+        self.assertEqual(options.capi_reexport_cincludes, True)
+        self.check_default_global_options()
+        self.check_default_options(options, ['capi_reexport_cincludes'])
+
+    def test_fast_fail(self):
+        options, sources = parse_command_line([
+            '--fast-fail',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.fast_fail, True)
+        self.check_default_global_options(['fast_fail'])
+        self.check_default_options(options)
+
+    def test_cimport_from_pyx(self):
+        options, sources = parse_command_line([
+            '--cimport-from-pyx',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.cimport_from_pyx, True)
+        self.check_default_global_options(['cimport_from_pyx'])
+        self.check_default_options(options)
+
+    def test_Werror(self):
+        options, sources = parse_command_line([
+            '-Werror',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.warning_errors, True)
+        self.check_default_global_options(['warning_errors'])
+        self.check_default_options(options)
+
+    def test_warning_errors(self):
+        options, sources = parse_command_line([
+            '--warning-errors',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.warning_errors, True)
+        self.check_default_global_options(['warning_errors'])
+        self.check_default_options(options)
+
+    def test_Wextra(self):
+        options, sources = parse_command_line([
+            '-Wextra',
+            'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives, Options.extra_warnings)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_warning_extra(self):
+        options, sources = parse_command_line([
+            '--warning-extra',
+            'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives, Options.extra_warnings)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_old_style_globals(self):
+        options, sources = parse_command_line([
+            '--old-style-globals',
+            'source.pyx'
+        ])
+        self.assertEqual(Options.old_style_globals, True)
+        self.check_default_global_options(['old_style_globals'])
+        self.check_default_options(options)
+
+    def test_directive_multiple(self):
+        options, source = parse_command_line([
+               '-X', 'cdivision=True',
+               '-X', 'c_string_type=bytes',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives['cdivision'], True)
+        self.assertEqual(options.compiler_directives['c_string_type'], 'bytes')
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_directive_multiple_v2(self):
+        options, source = parse_command_line([
+               '-X', 'cdivision=True,c_string_type=bytes',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives['cdivision'], True)
+        self.assertEqual(options.compiler_directives['c_string_type'], 'bytes')
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_directive_value_yes(self):
+        options, source = parse_command_line([
+               '-X', 'cdivision=YeS',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives['cdivision'], True)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_directive_value_no(self):
+        options, source = parse_command_line([
+               '-X', 'cdivision=no',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compiler_directives['cdivision'], False)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
+
+    def test_directive_value_invalid(self):
+        self.assertRaises(ValueError, parse_command_line, [
+               '-X', 'cdivision=sadfasd',
+               'source.pyx'
+        ])
+
+    def test_directive_key_invalid(self):
+        self.assertRaises(ValueError, parse_command_line, [
+               '-X', 'abracadabra',
+               'source.pyx'
+        ])
+
+    def test_directive_no_value(self):
+        self.assertRaises(ValueError, parse_command_line, [
+               '-X', 'cdivision',
+               'source.pyx'
+        ])
+
+    def test_compile_time_env_short(self):
+        options, source = parse_command_line([
+               '-E', 'MYSIZE=10',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compile_time_env['MYSIZE'], 10)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compile_time_env'])
+
+    def test_compile_time_env_long(self):
+        options, source = parse_command_line([
+               '--compile-time-env', 'MYSIZE=10',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compile_time_env['MYSIZE'], 10)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compile_time_env'])
+
+    def test_compile_time_env_multiple(self):
+        options, source = parse_command_line([
+               '-E', 'MYSIZE=10', '-E', 'ARRSIZE=11',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compile_time_env['MYSIZE'], 10)
+        self.assertEqual(options.compile_time_env['ARRSIZE'], 11)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compile_time_env'])
+
+    def test_compile_time_env_multiple_v2(self):
+        options, source = parse_command_line([
+               '-E', 'MYSIZE=10,ARRSIZE=11',
+               'source.pyx'
+        ])
+        self.assertEqual(options.compile_time_env['MYSIZE'], 10)
+        self.assertEqual(options.compile_time_env['ARRSIZE'], 11)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compile_time_env'])
+
+    def test_option_first(self):
+        options, sources = parse_command_line(['-V', 'file.pyx'])
+        self.assertEqual(sources, ['file.pyx'])
+
+    def test_file_inbetween(self):
+        options, sources = parse_command_line(['-V', 'file.pyx', '-a'])
+        self.assertEqual(sources, ['file.pyx'])
+
+    def test_option_trailing(self):
+        options, sources = parse_command_line(['file.pyx', '-V'])
+        self.assertEqual(sources, ['file.pyx'])
+
+    def test_multiple_files(self):
+        options, sources = parse_command_line([
+             'file1.pyx', '-V',
+             'file2.pyx', '-a',
+             'file3.pyx'
+        ])
+        self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx'])
+
+    def test_debug_flags(self):
+        options, sources = parse_command_line([
+             '--debug-disposal-code', '--debug-coercion',
+             'file3.pyx'
+        ])
+        from Cython.Compiler import DebugFlags
+        for name in ['debug_disposal_code', 'debug_temp_alloc', 'debug_coercion']:
+            self.assertEqual(getattr(DebugFlags, name), name in ['debug_disposal_code', 'debug_coercion'])
+            setattr(DebugFlags, name, 0)  # restore original value
+
+    def test_gdb_overwrites_gdb_outdir(self):
+        options, sources = parse_command_line([
+            '--gdb-outdir=my_dir', '--gdb',
+            'file3.pyx'
+        ])
+        self.assertEqual(options.gdb_debug, True)
+        self.assertEqual(options.output_dir, os.curdir)
+        self.check_default_global_options()
+        self.check_default_options(options, ['gdb_debug', 'output_dir'])
+
+    def test_gdb_first(self):
+        options, sources = parse_command_line([
+            '--gdb', '--gdb-outdir=my_dir',
+            'file3.pyx'
+        ])
+        self.assertEqual(options.gdb_debug, True)
+        self.assertEqual(options.output_dir, 'my_dir')
+        self.check_default_global_options()
+        self.check_default_options(options, ['gdb_debug', 'output_dir'])
+
+    def test_coverage_overwrites_annotation(self):
+        options, sources = parse_command_line([
+            '--annotate-fullc', '--annotate-coverage=my.xml',
+            'file3.pyx'
+        ])
+        self.assertEqual(Options.annotate, True)
+        self.assertEqual(Options.annotate_coverage_xml, 'my.xml')
+        self.check_default_global_options(['annotate', 'annotate_coverage_xml'])
+        self.check_default_options(options)
+
+    def test_coverage_first(self):
+        options, sources = parse_command_line([
+            '--annotate-coverage=my.xml', '--annotate-fullc',
+            'file3.pyx'
+        ])
+        self.assertEqual(Options.annotate, 'fullc')
+        self.assertEqual(Options.annotate_coverage_xml, 'my.xml')
+        self.check_default_global_options(['annotate', 'annotate_coverage_xml'])
+        self.check_default_options(options)
+
+    def test_annotate_first_fullc_second(self):
+        options, sources = parse_command_line([
+            '--annotate', '--annotate-fullc',
+            'file3.pyx'
+        ])
+        self.assertEqual(Options.annotate, 'fullc')
+        self.check_default_global_options(['annotate'])
+        self.check_default_options(options)
+
+    def test_annotate_fullc_first(self):
+        options, sources = parse_command_line([
+            '--annotate-fullc', '--annotate',
+            'file3.pyx'
+        ])
+        self.assertEqual(Options.annotate, 'default')
+        self.check_default_global_options(['annotate'])
+        self.check_default_options(options)
+
+    def test_warning_extra_dont_overwrite(self):
+        options, sources = parse_command_line([
+            '-X', 'cdivision=True',
+            '--warning-extra',
+            '-X', 'c_string_type=bytes',
+            'source.pyx'
+        ])
+        self.assertTrue(len(options.compiler_directives), len(Options.extra_warnings) + 1)
+        self.check_default_global_options()
+        self.check_default_options(options, ['compiler_directives'])
 
     def test_module_name(self):
         options, sources = parse_command_line([
@@ -145,25 +541,38 @@ class CmdLineParserTest(TestCase):
                 self.assertRaises(SystemExit, parse_command_line, list(args))
             finally:
                 sys.stderr = old_stderr
-            msg = stderr.getvalue().strip()
-            self.assertTrue(msg)
+            msg = stderr.getvalue()
+            err_msg = 'Message "{}"'.format(msg.strip())
+            self.assertTrue(msg.startswith('usage: '),
+                            '%s does not start with "usage :"' % err_msg)
+            self.assertTrue(': error: ' in msg,
+                            '%s does not contain ": error :"' % err_msg)
             if regex:
                 self.assertTrue(re.search(regex, msg),
-                                '"%s" does not match search "%s"' %
-                                (msg, regex))
+                                '%s does not match search "%s"' %
+                                (err_msg, regex))
 
         error(['-1'],
-              'Unknown compiler flag: -1')
-        error(['-I'])
-        error(['--version=-a'])
-        error(['--version=--annotate=true'])
-        error(['--working'])
-        error(['--verbose=1'])
-        error(['--cleanup'])
+              'unknown option -1')
+        error(['-I'],
+              'argument -I/--include-dir: expected one argument')
+        error(['--version=-a'],
+              "argument -V/--version: ignored explicit argument '-a'")
+        error(['--version=--annotate=true'],
+              "argument -V/--version: ignored explicit argument "
+              "'--annotate=true'")
+        error(['--working'],
+              "argument -w/--working: expected one argument")
+        error(['--verbose=1'],
+              "argument -v/--verbose: ignored explicit argument '1'")
+        error(['--cleanup'],
+              "argument --cleanup: expected one argument")
         error(['--debug-disposal-code-wrong-name', 'file3.pyx'],
-              "Unknown debug flag: debug_disposal_code_wrong_name")
-        error(['--module-name', 'foo.pyx'])
-        error(['--module-name', 'foo.bar'])
+              "unknown option --debug-disposal-code-wrong-name")
+        error(['--module-name', 'foo.pyx'],
+              "Need at least one source file")
+        error(['--module-name', 'foo.bar'],
+              "Need at least one source file")
         error(['--module-name', 'foo.bar', 'foo.pyx', 'bar.pyx'],
               "Only one source file allowed when using --module-name")
         error(['--module-name', 'foo.bar', '--timestamps', 'foo.pyx'],
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestGrammar.py b/contrib/tools/cython/Cython/Compiler/Tests/TestGrammar.py
index 3dddc960b3a..852b48c33dd 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestGrammar.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestGrammar.py
@@ -7,9 +7,12 @@ Uses TreeFragment to test invalid syntax.
 
 from __future__ import absolute_import
 
+import ast
+import textwrap
+
 from ...TestUtils import CythonTest
-from ..Errors import CompileError
 from .. import ExprNodes
+from ..Errors import CompileError
 
 # Copied from CPython's test_grammar.py
 VALID_UNDERSCORE_LITERALS = [
@@ -27,7 +30,15 @@ VALID_UNDERSCORE_LITERALS = [
     '1e1_0',
     '.1_4',
     '.1_4e1',
+    '0b_0',
+    '0x_f',
+    '0o_5',
+    '1_00_00j',
+    '1_00_00.5j',
+    '1_00_00e5_1j',
     '.1_4j',
+    '(1_2.5+3_3j)',
+    '(.5_6j)',
 ]
 
 # Copied from CPython's test_grammar.py
@@ -36,22 +47,29 @@ INVALID_UNDERSCORE_LITERALS = [
     '0_',
     '42_',
     '1.4j_',
+    '0x_',
     '0b1_',
     '0xf_',
     '0o5_',
+    '0 if 1_Else 1',
     # Underscores in the base selector:
     '0_b0',
     '0_xf',
     '0_o5',
-    # Underscore right after the base selector:
-    '0b_0',
-    '0x_f',
-    '0o_5',
     # Old-style octal, still disallowed:
-    #'0_7',
-    #'09_99',
-    # Special case with exponent:
-    '0 if 1_Else 1',
+    # FIXME: still need to support PY_VERSION_HEX < 3
+    '0_7',
+    '09_99',
+    # Multiple consecutive underscores:
+    '4_______2',
+    '0.1__4',
+    '0.1__4j',
+    '0b1001__0100',
+    '0xffff__ffff',
+    '0x___',
+    '0o5__77',
+    '1e1__0',
+    '1e1__0j',
     # Underscore right before a dot:
     '1_.4',
     '1_.4j',
@@ -59,24 +77,24 @@ INVALID_UNDERSCORE_LITERALS = [
     '1._4',
     '1._4j',
     '._5',
+    '._5j',
     # Underscore right after a sign:
     '1.0e+_1',
-    # Multiple consecutive underscores:
-    '4_______2',
-    '0.1__4',
-    '0b1001__0100',
-    '0xffff__ffff',
-    '0o5__77',
-    '1e1__0',
+    '1.0e+_1j',
     # Underscore right before j:
     '1.4_j',
     '1.4e5_j',
     # Underscore right before e:
     '1_e1',
     '1.4_e1',
+    '1.4_e1j',
     # Underscore right after e:
     '1e_1',
     '1.4e_1',
+    '1.4e_1j',
+    # Complex cases with parens:
+    '(1+1.5_j_)',
+    '(1+1.5_j)',
     # Whitespace in literals
     '1_ 2',
     '1 _2',
@@ -88,6 +106,39 @@ INVALID_UNDERSCORE_LITERALS = [
 ]
 
 
+INVALID_ELLIPSIS = [
+    (". . .", 2, 0),
+    (". ..", 2, 0),
+    (".. .", 2, 0),
+    (". ...", 2, 0),
+    (". ... .", 2, 0),
+    (".. ... .", 2, 0),
+    (". ... ..", 2, 0),
+    ("""
+    (
+        .
+        ..
+    )
+    """, 3, 4),
+    ("""
+    [
+        ..
+        .,
+        None
+    ]
+    """, 3, 4),
+    ("""
+    {
+        None,
+        .
+        .
+
+        .
+    }
+    """, 4, 4)
+]
+
+
 class TestGrammar(CythonTest):
 
     def test_invalid_number_literals(self):
@@ -117,11 +168,34 @@ class TestGrammar(CythonTest):
                     # Add/MulNode() -> literal is first or second operand
                     literal_node = literal_node.operand2 if i % 2 else literal_node.operand1
                 if 'j' in literal or 'J' in literal:
-                    assert isinstance(literal_node, ExprNodes.ImagNode)
+                    if '+' in literal:
+                        # FIXME: tighten this test
+                        assert isinstance(literal_node, ExprNodes.AddNode), (literal, literal_node)
+                    else:
+                        assert isinstance(literal_node, ExprNodes.ImagNode), (literal, literal_node)
                 elif '.' in literal or 'e' in literal or 'E' in literal and not ('0x' in literal or '0X' in literal):
-                    assert isinstance(literal_node, ExprNodes.FloatNode)
+                    assert isinstance(literal_node, ExprNodes.FloatNode), (literal, literal_node)
                 else:
-                    assert isinstance(literal_node, ExprNodes.IntNode)
+                    assert isinstance(literal_node, ExprNodes.IntNode), (literal, literal_node)
+
+    def test_invalid_ellipsis(self):
+        ERR = ":{0}:{1}: Expected an identifier or literal"
+        for code, line, col in INVALID_ELLIPSIS:
+            try:
+                ast.parse(textwrap.dedent(code))
+            except SyntaxError as exc:
+                assert True
+            else:
+                assert False, "Invalid Python code '%s' failed to raise an exception" % code
+
+            try:
+                self.fragment(u'''\
+                # cython: language_level=3
+                ''' + code)
+            except CompileError as exc:
+                assert ERR.format(line, col) in str(exc), str(exc)
+            else:
+                assert False, "Invalid Cython code '%s' failed to raise an exception" % code
 
 
 if __name__ == "__main__":
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestMemView.py b/contrib/tools/cython/Cython/Compiler/Tests/TestMemView.py
index 3792f26e994..1d04a17fc7f 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestMemView.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestMemView.py
@@ -53,11 +53,11 @@ class TestMemviewParsing(CythonTest):
     # we also test other similar declarations (buffers, anonymous C arrays)
     # since the parsing has to distinguish between them.
 
-    def disable_test_no_buf_arg(self): # TODO
+    def disable_test_no_buf_arg(self):  # TODO
         self.not_parseable(u"Expected ']'",
                 u"cdef extern foo(object[int, ndim=2])")
 
-    def disable_test_parse_sizeof(self): # TODO
+    def disable_test_parse_sizeof(self):  # TODO
         self.parse(u"sizeof(int[NN])")
         self.parse(u"sizeof(int[])")
         self.parse(u"sizeof(int[][NN])")
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestParseTreeTransforms.py b/contrib/tools/cython/Cython/Compiler/Tests/TestParseTreeTransforms.py
index 8a16f98ccca..6e29263e55c 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestParseTreeTransforms.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestParseTreeTransforms.py
@@ -5,7 +5,7 @@ from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.ParseTreeTransforms import _calculate_pickle_checksums
 from Cython.Compiler.Nodes import *
-from Cython.Compiler import Main, Symtab
+from Cython.Compiler import Main, Symtab, Options
 
 
 class TestNormalizeTree(TransformTest):
@@ -91,7 +91,7 @@ class TestNormalizeTree(TransformTest):
         t = self.run_pipeline([NormalizeTree(None)], u"pass")
         self.assertTrue(len(t.stats) == 0)
 
-class TestWithTransform(object): # (TransformTest): # Disabled!
+class TestWithTransform(object):  # (TransformTest): # Disabled!
 
     def test_simplified(self):
         t = self.run_pipeline([WithTransform(None)], u"""
@@ -179,8 +179,8 @@ class TestInterpretCompilerDirectives(TransformTest):
     def setUp(self):
         super(TestInterpretCompilerDirectives, self).setUp()
 
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
 
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestScanning.py b/contrib/tools/cython/Cython/Compiler/Tests/TestScanning.py
new file mode 100644
index 00000000000..e9cac1b4727
--- /dev/null
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestScanning.py
@@ -0,0 +1,136 @@
+from __future__ import unicode_literals
+
+import unittest
+from io import StringIO
+import string
+
+from .. import Scanning
+from ..Symtab import ModuleScope
+from ..TreeFragment import StringParseContext
+from ..Errors import init_thread
+
+# generate some fake code - just a bunch of lines of the form "a0 a1 ..."
+code = []
+for ch in string.ascii_lowercase:
+    line = " ".join(["%s%s" % (ch, n) for n in range(10)])
+    code.append(line)
+code = "\n".join(code)
+
+init_thread()
+
+
+class TestScanning(unittest.TestCase):
+    def make_scanner(self):
+        source = Scanning.StringSourceDescriptor("fake code", code)
+        buf = StringIO(code)
+        context = StringParseContext("fake context")
+        scope = ModuleScope("fake_module", None, None)
+
+        return Scanning.PyrexScanner(buf, source, scope=scope, context=context)
+
+    def test_put_back_positions(self):
+        scanner = self.make_scanner()
+
+        self.assertEqual(scanner.sy, "IDENT")
+        self.assertEqual(scanner.systring, "a0")
+        scanner.next()
+        self.assertEqual(scanner.sy, "IDENT")
+        self.assertEqual(scanner.systring, "a1")
+        a1pos = scanner.position()
+        self.assertEqual(a1pos[1:], (1, 3))
+        a2peek = scanner.peek()  # shouldn't mess up the position
+        self.assertEqual(a1pos, scanner.position())
+        scanner.next()
+        self.assertEqual(a2peek, (scanner.sy, scanner.systring))
+
+        # find next line
+        while scanner.sy != "NEWLINE":
+            scanner.next()
+
+        line_sy = []
+        line_systring = []
+        line_pos = []
+
+        scanner.next()
+        while scanner.sy != "NEWLINE":
+            line_sy.append(scanner.sy)
+            line_systring.append(scanner.systring)
+            line_pos.append(scanner.position())
+            scanner.next()
+
+        for sy, systring, pos in zip(
+            line_sy[::-1], line_systring[::-1], line_pos[::-1]
+        ):
+            scanner.put_back(sy, systring, pos)
+
+        n = 0
+        while scanner.sy != "NEWLINE":
+            self.assertEqual(scanner.sy, line_sy[n])
+            self.assertEqual(scanner.systring, line_systring[n])
+            self.assertEqual(scanner.position(), line_pos[n])
+            scanner.next()
+            n += 1
+
+        self.assertEqual(n, len(line_pos))
+
+    def test_tentatively_scan(self):
+        scanner = self.make_scanner()
+        with Scanning.tentatively_scan(scanner) as errors:
+            while scanner.sy != "NEWLINE":
+                scanner.next()
+        self.assertFalse(errors)
+
+        scanner.next()
+        self.assertEqual(scanner.systring, "b0")
+        pos = scanner.position()
+        with Scanning.tentatively_scan(scanner) as errors:
+            while scanner.sy != "NEWLINE":
+                scanner.next()
+                if scanner.systring == "b7":
+                    scanner.error("Oh no not b7!")
+                    break
+        self.assertTrue(errors)
+        self.assertEqual(scanner.systring, "b0")  # state has been restored
+        self.assertEqual(scanner.position(), pos)
+        scanner.next()
+        self.assertEqual(scanner.systring, "b1")  # and we can keep going again
+        scanner.next()
+        self.assertEqual(scanner.systring, "b2")  # and we can keep going again
+
+        with Scanning.tentatively_scan(scanner) as error:
+            scanner.error("Something has gone wrong with the current symbol")
+        self.assertEqual(scanner.systring, "b2")
+        scanner.next()
+        self.assertEqual(scanner.systring, "b3")
+
+        # test a few combinations of nested scanning
+        sy1, systring1 = scanner.sy, scanner.systring
+        pos1 = scanner.position()
+        with Scanning.tentatively_scan(scanner):
+            scanner.next()
+            sy2, systring2 = scanner.sy, scanner.systring
+            pos2 = scanner.position()
+            with Scanning.tentatively_scan(scanner):
+                with Scanning.tentatively_scan(scanner):
+                    scanner.next()
+                    scanner.next()
+                    scanner.error("Ooops")
+                self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
+            self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
+            scanner.error("eee")
+        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
+        with Scanning.tentatively_scan(scanner):
+            scanner.next()
+            scanner.next()
+            with Scanning.tentatively_scan(scanner):
+                scanner.next()
+                # no error - but this block should be unwound by the outer block too
+            scanner.next()
+            scanner.error("Oooops")
+        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
+
+
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestSignatureMatching.py b/contrib/tools/cython/Cython/Compiler/Tests/TestSignatureMatching.py
index 166bb225b9b..57647c87363 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestSignatureMatching.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestSignatureMatching.py
@@ -47,7 +47,7 @@ class SignatureMatcherTest(unittest.TestCase):
         self.assertMatches(function_types[1], [pt.c_long_type, pt.c_int_type], functions)
 
     def test_cpp_reference_cpp_class(self):
-        classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ]
+        classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ]
         function_types = [
             cfunctype(pt.CReferenceType(classes[0])),
             cfunctype(pt.CReferenceType(classes[1])),
@@ -58,7 +58,7 @@ class SignatureMatcherTest(unittest.TestCase):
         self.assertMatches(function_types[1], [classes[1]], functions)
 
     def test_cpp_reference_cpp_class_and_int(self):
-        classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ]
+        classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ]
         function_types = [
             cfunctype(pt.CReferenceType(classes[0]), pt.c_int_type),
             cfunctype(pt.CReferenceType(classes[0]), pt.c_long_type),
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestTreeFragment.py b/contrib/tools/cython/Cython/Compiler/Tests/TestTreeFragment.py
index 9ee8da5478c..d2006b40229 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestTreeFragment.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestTreeFragment.py
@@ -2,7 +2,6 @@ from Cython.TestUtils import CythonTest
 from Cython.Compiler.TreeFragment import *
 from Cython.Compiler.Nodes import *
 from Cython.Compiler.UtilNodes import *
-import Cython.Compiler.Naming as Naming
 
 class TestTreeFragments(CythonTest):
 
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestTreePath.py b/contrib/tools/cython/Cython/Compiler/Tests/TestTreePath.py
index bee53b3d2bf..b2013086b18 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestTreePath.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestTreePath.py
@@ -1,5 +1,4 @@
 import unittest
-from Cython.Compiler.Visitor import PrintTree
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.TreePath import find_first, find_all
 from Cython.Compiler import Nodes, ExprNodes
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestTypes.py b/contrib/tools/cython/Cython/Compiler/Tests/TestTypes.py
index f2f6f3773b9..2fe7347f8de 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestTypes.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestTypes.py
@@ -17,3 +17,61 @@ class TestMethodDispatcherTransform(unittest.TestCase):
 
         cenum = PT.CEnumType("E", "cenum", typedef_flag=False)
         assert_widest(PT.c_int_type, cenum, PT.c_int_type)
+
+
+class TestTypeIdentifiers(unittest.TestCase):
+
+    TEST_DATA = [
+        ("char*", "char__ptr"),
+        ("char *", "char__ptr"),
+        ("char **", "char__ptr__ptr"),
+        ("_typedef", "_typedef"),
+        ("__typedef", "__dundertypedef"),
+        ("___typedef", "__dunder_typedef"),
+        ("____typedef", "__dunder__dundertypedef"),
+        ("_____typedef", "__dunder__dunder_typedef"),
+        ("const __typedef", "__const___dundertypedef"),
+        ("int[42]", "int__lArr42__rArr"),
+        ("int[:]", "int__lArr__D__rArr"),
+        ("int[:,:]", "int__lArr__D__comma___D__rArr"),
+        ("int[:,:,:]", "int__lArr__D__comma___D__comma___D__rArr"),
+        ("int[:,:,...]", "int__lArr__D__comma___D__comma___EL__rArr"),
+        ("std::vector", "std__in_vector"),
+        ("std::vector&&", "std__in_vector__fwref"),
+        ("const std::vector", "__const_std__in_vector"),
+        ("const std::vector&", "__const_std__in_vector__ref"),
+        ("const_std", "const_std"),
+    ]
+
+    def test_escape_special_type_characters(self):
+        test_func = PT._escape_special_type_characters  # keep test usage visible for IDEs
+        function_name = "_escape_special_type_characters"
+        self._test_escape(function_name)
+
+    def test_type_identifier_for_declaration(self):
+        test_func = PT.type_identifier_from_declaration  # keep test usage visible for IDEs
+        function_name = test_func.__name__
+        self._test_escape(function_name)
+
+        # differences due to whitespace removal
+        test_data = [
+            ("const &std::vector", "const__refstd__in_vector"),
+            ("const &std::vector<int>", "const__refstd__in_vector__lAngint__rAng"),
+            ("const &&std::vector", "const__fwrefstd__in_vector"),
+            ("const &&&std::vector", "const__fwref__refstd__in_vector"),
+            ("const &&std::vector", "const__fwrefstd__in_vector"),
+            ("void (*func)(int x, float y)",
+             "975d51__void__lParen__ptrfunc__rParen__lParenint__spac__etc"),
+            ("float ** (*func)(int x, int[:] y)",
+             "31883a__float__ptr__ptr__lParen__ptrfunc__rParen__lPar__etc"),
+        ]
+        self._test_escape(function_name, test_data)
+
+    def _test_escape(self, func_name, test_data=TEST_DATA):
+        escape = getattr(PT, func_name)
+        for declaration, expected in test_data:
+            escaped_value = escape(declaration)
+            self.assertEqual(escaped_value, expected, "%s('%s') == '%s' != '%s'" % (
+                func_name, declaration, escaped_value, expected))
+            # test that the length has been successfully capped
+            self.assertLessEqual(len(escaped_value), 64)
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/TestUtilityLoad.py b/contrib/tools/cython/Cython/Compiler/Tests/TestUtilityLoad.py
index 3d1906ca0b4..98a3853819f 100644
--- a/contrib/tools/cython/Cython/Compiler/Tests/TestUtilityLoad.py
+++ b/contrib/tools/cython/Cython/Compiler/Tests/TestUtilityLoad.py
@@ -22,14 +22,11 @@ class TestUtilityLoader(unittest.TestCase):
     cls = Code.UtilityCode
 
     def test_load_as_string(self):
-        got = strip_2tup(self.cls.load_as_string(self.name))
-        self.assertEqual(got, self.expected)
-
         got = strip_2tup(self.cls.load_as_string(self.name, self.filename))
         self.assertEqual(got, self.expected)
 
     def test_load(self):
-        utility = self.cls.load(self.name)
+        utility = self.cls.load(self.name, from_file=self.filename)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEqual(got, self.expected)
 
@@ -37,10 +34,6 @@ class TestUtilityLoader(unittest.TestCase):
         got = strip_2tup((required.proto, required.impl))
         self.assertEqual(got, self.required)
 
-        utility = self.cls.load(self.name, from_file=self.filename)
-        got = strip_2tup((utility.proto, utility.impl))
-        self.assertEqual(got, self.expected)
-
         utility = self.cls.load_cached(self.name, from_file=self.filename)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEqual(got, self.expected)
@@ -59,11 +52,11 @@ class TestTempitaUtilityLoader(TestUtilityLoader):
     cls = Code.TempitaUtilityCode
 
     def test_load_as_string(self):
-        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
+        got = strip_2tup(self.cls.load_as_string(self.name, self.filename, context=self.context))
         self.assertEqual(got, self.expected_tempita)
 
     def test_load(self):
-        utility = self.cls.load(self.name, context=self.context)
+        utility = self.cls.load(self.name, self.filename, context=self.context)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEqual(got, self.expected_tempita)
 
@@ -99,3 +92,21 @@ class TestCythonUtilityLoader(TestTempitaUtilityLoader):
 
     test_load = TestUtilityLoader.test_load
     test_load_tempita = TestTempitaUtilityLoader.test_load
+
+
+class TestUtilityCode(unittest.TestCase):
+    def test_equality(self):
+        c1 = Code.UtilityCode.load("NumpyImportUFunc", "NumpyImportArray.c")
+        c2 = Code.UtilityCode.load("NumpyImportArray", "NumpyImportArray.c")
+        c3 = Code.UtilityCode.load("pyunicode_strlen", "StringTools.c")
+        c4 = Code.UtilityCode.load("pyunicode_from_unicode", "StringTools.c")
+        c5 = Code.UtilityCode.load("IncludeStringH", "StringTools.c")
+        c6 = Code.UtilityCode.load("IncludeCppStringH", "StringTools.c")
+
+        codes = [c1, c2, c3, c4, c5, c6]
+        for m in range(len(codes)):
+            for n in range(len(codes)):
+                if n == m:
+                    self.assertEqual(codes[m], codes[n])
+                else:
+                    self.assertNotEqual(codes[m], codes[n])
diff --git a/contrib/tools/cython/Cython/Compiler/Tests/Utils.py b/contrib/tools/cython/Cython/Compiler/Tests/Utils.py
new file mode 100644
index 00000000000..a158ecc502d
--- /dev/null
+++ b/contrib/tools/cython/Cython/Compiler/Tests/Utils.py
@@ -0,0 +1,36 @@
+import copy
+
+from .. import Options
+
+
+def backup_Options():
+    backup = {}
+    for name, value in vars(Options).items():
+        # we need a deep copy of _directive_defaults, because they can be changed
+        if name == '_directive_defaults':
+            value = copy.deepcopy(value)
+        backup[name] = value
+    return backup
+
+
+def restore_Options(backup):
+    no_value = object()
+    for name, orig_value in backup.items():
+        if getattr(Options, name, no_value) != orig_value:
+            setattr(Options, name, orig_value)
+    # strip Options from new keys that might have been added:
+    for name in vars(Options).keys():
+        if name not in backup:
+            delattr(Options, name)
+
+
+def check_global_options(expected_options, white_list=[]):
+    """
+    returns error message of "" if check Ok
+    """
+    no_value = object()
+    for name, orig_value in expected_options.items():
+        if name not in white_list:
+            if getattr(Options, name, no_value) != orig_value:
+                return "error in option " + name
+    return ""