Update Cython to 3.0.10.

commit_hash:b43c96b868cd36d636192fd2c6024d9f0d2fb6f8

6 files changed, 427 insertions, 18 deletions

diff --git a/contrib/tools/cython/Cython/Tests/TestCodeWriter.py b/contrib/tools/cython/Cython/Tests/TestCodeWriter.py
index 42e457da20e..c3026cb1db3 100644
--- a/contrib/tools/cython/Cython/Tests/TestCodeWriter.py
+++ b/contrib/tools/cython/Cython/Tests/TestCodeWriter.py
@@ -19,9 +19,9 @@ class TestCodeWriter(CythonTest):
 
     def test_print(self):
         self.t(u"""
-                    print x, y
-                    print x + y ** 2
-                    print x, y, z,
+                    print(x + y ** 2)
+                    print(x, y, z)
+                    print(x + y, x + y * z, x * (y + z))
                """)
 
     def test_if(self):
@@ -47,6 +47,20 @@ class TestCodeWriter(CythonTest):
                         pass
                """)
 
+    def test_cdef(self):
+        self.t(u"""
+                    cdef f(x, y, z):
+                        pass
+                    cdef public void (x = 34, y = 54, z):
+                        pass
+                    cdef f(int *x, void *y, Value *z):
+                        pass
+                    cdef f(int **x, void **y, Value **z):
+                        pass
+                    cdef inline f(int &x, Value &z):
+                        pass
+               """)
+
     def test_longness_and_signedness(self):
         self.t(u"def f(unsigned long long long long long int y):\n    pass")
 
@@ -65,18 +79,50 @@ class TestCodeWriter(CythonTest):
     def test_for_loop(self):
         self.t(u"""
                     for x, y, z in f(g(h(34) * 2) + 23):
-                        print x, y, z
+                        print(x, y, z)
+                    else:
+                        print(43)
+                """)
+        self.t(u"""
+                    for abc in (1, 2, 3):
+                        print(x, y, z)
                     else:
-                        print 43
+                        print(43)
+                """)
+
+    def test_while_loop(self):
+        self.t(u"""
+                    while True:
+                        while True:
+                            while True:
+                                continue
                 """)
 
     def test_inplace_assignment(self):
         self.t(u"x += 43")
 
+    def test_cascaded_assignment(self):
+        self.t(u"x = y = z = abc = 43")
+
     def test_attribute(self):
         self.t(u"a.x")
 
+    def test_return_none(self):
+        self.t(u"""
+                    def f(x, y, z):
+                        return
+                    cdef f(x, y, z):
+                        return
+                    def f(x, y, z):
+                        return None
+                    cdef f(x, y, z):
+                        return None
+                    def f(x, y, z):
+                        return 1234
+                    cdef f(x, y, z):
+                        return 1234
+               """)
+
 if __name__ == "__main__":
     import unittest
     unittest.main()
-
diff --git a/contrib/tools/cython/Cython/Tests/TestCythonUtils.py b/contrib/tools/cython/Cython/Tests/TestCythonUtils.py
index 2641900c012..4c06cbe1c3c 100644
--- a/contrib/tools/cython/Cython/Tests/TestCythonUtils.py
+++ b/contrib/tools/cython/Cython/Tests/TestCythonUtils.py
@@ -1,11 +1,205 @@
+import sys
 import unittest
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO  # doesn't accept 'str' in Py2
+
+from Cython.Utils import (
+    _CACHE_NAME_PATTERN, _build_cache_name, _find_cache_attributes,
+    build_hex_version, cached_method, clear_method_caches, try_finally_contextmanager,
+    print_version, normalise_float_repr,
+)
+
+METHOD_NAME = "cached_next"
+CACHE_NAME = _build_cache_name(METHOD_NAME)
+NAMES = CACHE_NAME, METHOD_NAME
+
+class Cached(object):
+    @cached_method
+    def cached_next(self, x):
+        return next(x)
 
-from ..Utils import build_hex_version
 
 class TestCythonUtils(unittest.TestCase):
     def test_build_hex_version(self):
         self.assertEqual('0x001D00A1', build_hex_version('0.29a1'))
-        self.assertEqual('0x001D00A1', build_hex_version('0.29a1'))
         self.assertEqual('0x001D03C4', build_hex_version('0.29.3rc4'))
         self.assertEqual('0x001D00F0', build_hex_version('0.29'))
         self.assertEqual('0x040000F0', build_hex_version('4.0'))
+
+    ############################## Cached Methods ##############################
+
+    def test_cache_method_name(self):
+        method_name = "foo"
+        cache_name = _build_cache_name(method_name)
+        match = _CACHE_NAME_PATTERN.match(cache_name)
+
+        self.assertIsNot(match, None)
+        self.assertEqual(match.group(1), method_name)
+
+    def test_requirements_for_Cached(self):
+        obj = Cached()
+
+        self.assertFalse(hasattr(obj, CACHE_NAME))
+        self.assertTrue(hasattr(obj, METHOD_NAME))
+        self.set_of_names_equal(obj, set())
+
+    def set_of_names_equal(self, obj, value):
+        self.assertEqual(set(_find_cache_attributes(obj)), value)
+
+    def test_find_cache_attributes(self):
+        obj = Cached()
+        method_name = "bar"
+        cache_name = _build_cache_name(method_name)
+
+        setattr(obj, CACHE_NAME, {})
+        setattr(obj, cache_name, {})
+
+        self.assertFalse(hasattr(obj, method_name))
+        self.set_of_names_equal(obj, {NAMES, (cache_name, method_name)})
+
+    def test_cached_method(self):
+        obj = Cached()
+        value = iter(range(3))  # iter for Py2
+        cache = {(value,): 0}
+
+        # cache args
+        self.assertEqual(obj.cached_next(value), 0)
+        self.set_of_names_equal(obj, {NAMES})
+        self.assertEqual(getattr(obj, CACHE_NAME), cache)
+
+        # use cache
+        self.assertEqual(obj.cached_next(value), 0)
+        self.set_of_names_equal(obj, {NAMES})
+        self.assertEqual(getattr(obj, CACHE_NAME), cache)
+
+    def test_clear_method_caches(self):
+        obj = Cached()
+        value = iter(range(3))  # iter for Py2
+        cache = {(value,): 1}
+
+        obj.cached_next(value)  # cache args
+
+        clear_method_caches(obj)
+        self.set_of_names_equal(obj, set())
+
+        self.assertEqual(obj.cached_next(value), 1)
+        self.set_of_names_equal(obj, {NAMES})
+        self.assertEqual(getattr(obj, CACHE_NAME), cache)
+
+    def test_clear_method_caches_with_missing_method(self):
+        obj = Cached()
+        method_name = "bar"
+        cache_name = _build_cache_name(method_name)
+        names = cache_name, method_name
+
+        setattr(obj, cache_name, object())
+
+        self.assertFalse(hasattr(obj, method_name))
+        self.set_of_names_equal(obj, {names})
+
+        clear_method_caches(obj)
+        self.set_of_names_equal(obj, {names})
+
+    def test_try_finally_contextmanager(self):
+        states = []
+        @try_finally_contextmanager
+        def gen(*args, **kwargs):
+            states.append("enter")
+            yield (args, kwargs)
+            states.append("exit")
+
+        with gen(1, 2, 3, x=4) as call_args:
+            assert states == ["enter"]
+            self.assertEqual(call_args, ((1, 2, 3), {'x': 4}))
+        assert states == ["enter", "exit"]
+
+        class MyException(RuntimeError):
+            pass
+
+        del states[:]
+        with self.assertRaises(MyException):
+            with gen(1, 2, y=4) as call_args:
+                assert states == ["enter"]
+                self.assertEqual(call_args, ((1, 2), {'y': 4}))
+                raise MyException("FAIL INSIDE")
+            assert states == ["enter", "exit"]
+
+        del states[:]
+        with self.assertRaises(StopIteration):
+            with gen(1, 2, y=4) as call_args:
+                assert states == ["enter"]
+                self.assertEqual(call_args, ((1, 2), {'y': 4}))
+                raise StopIteration("STOP")
+            assert states == ["enter", "exit"]
+
+    def test_print_version(self):
+        orig_stderr = sys.stderr
+        orig_stdout = sys.stdout
+        stderr = sys.stderr = StringIO()
+        stdout = sys.stdout = StringIO()
+        try:
+            print_version()
+        finally:
+            sys.stdout = orig_stdout
+            sys.stderr = orig_stderr
+
+        stdout = stdout.getvalue()
+        stderr = stderr.getvalue()
+
+        from .. import __version__ as version
+        self.assertIn(version, stdout)
+        if stderr:  # Depends on os.fstat(1/2).
+            self.assertIn(version, stderr)
+
+    def test_print_version_stdouterr(self):
+        orig_stderr = sys.stderr
+        orig_stdout = sys.stdout
+        stdout = sys.stdout = sys.stderr = StringIO()  # same!
+        try:
+            print_version()
+        finally:
+            sys.stdout = orig_stdout
+            sys.stderr = orig_stderr
+
+        stdout = stdout.getvalue()
+
+        from .. import __version__ as version
+        self.assertIn(version, stdout)
+        self.assertEqual(stdout.count(version), 1)
+
+    def test_normalise_float_repr(self):
+        examples = [
+            ('.0', '.0'),
+            ('.000000', '.0'),
+            ('.1', '.1'),
+            ('1.', '1.'),
+            ('1.0', '1.'),
+            ('1.000000000000000000000', '1.'),
+            ('00000000000000000000001.000000000000000000000', '1.'),
+            ('12345.0025', '12345.0025'),
+            ('1E5', '100000.'),
+            ('.1E-5', '.000001'),
+            ('1.1E-5', '.000011'),
+            ('12.3E-5', '.000123'),
+            ('.1E10', '1000000000.'),
+            ('1.1E10', '11000000000.'),
+            ('123.4E10', '1234000000000.'),
+            ('123.456E0', '123.456'),
+            ('123.456E-1', '12.3456'),
+            ('123.456E-2', '1.23456'),
+            ('123.456E1', '1234.56'),
+            ('123.456E2', '12345.6'),
+            ('2.1E80', '210000000000000000000000000000000000000000000000000000000000000000000000000000000.'),
+        ]
+
+        for float_str, norm_str in examples:
+            self.assertEqual(float(float_str), float(norm_str))  # safety check for test data
+
+            result = normalise_float_repr(float_str)
+            self.assertEqual(float(float_str), float(result))
+            self.assertEqual(
+                result, norm_str,
+                "normalise_float_repr(%r) == %r != %r  (%.330f)" % (float_str, result, norm_str, float(float_str))
+            )
diff --git a/contrib/tools/cython/Cython/Tests/TestJediTyper.py b/contrib/tools/cython/Cython/Tests/TestJediTyper.py
index 253adef1715..ede99b3a8a9 100644
--- a/contrib/tools/cython/Cython/Tests/TestJediTyper.py
+++ b/contrib/tools/cython/Cython/Tests/TestJediTyper.py
@@ -11,7 +11,7 @@ from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
 
 from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives
-from Cython.Compiler import Main, Symtab, Visitor
+from Cython.Compiler import Main, Symtab, Visitor, Options
 from Cython.TestUtils import TransformTest
 
 TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools'))
@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper):
     """
     def setUp(self):
         super(TestTypeInjection, self).setUp()
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
         self.declarations_finder = DeclarationsFinder()
diff --git a/contrib/tools/cython/Cython/Tests/TestShadow.py b/contrib/tools/cython/Cython/Tests/TestShadow.py
new file mode 100644
index 00000000000..83abdcec2bf
--- /dev/null
+++ b/contrib/tools/cython/Cython/Tests/TestShadow.py
@@ -0,0 +1,79 @@
+import unittest
+
+from Cython import Shadow
+from Cython.Compiler import Options, CythonScope, PyrexTypes, Errors
+
+class TestShadow(unittest.TestCase):
+    def test_all_types_in_shadow(self):
+        cython_scope = CythonScope.create_cython_scope(None)
+        # Not doing load_cythonscope at this stage because it requires a proper context and
+        # Errors.py to be set up
+
+        missing_types = []
+        for key in cython_scope.entries.keys():
+            if key.startswith('__') and key.endswith('__'):
+                continue
+            if key in ('PyTypeObject', 'PyObject_TypeCheck'):
+                # These are declared in Shadow.py for reasons that look to
+                # be an implementation detail, but it isn't our intention for
+                # users to access them from Pure Python mode.
+                continue
+            if not hasattr(Shadow, key):
+                missing_types.append(key)
+        self.assertEqual(missing_types, [])
+
+    def test_int_types_in_shadow(self):
+        missing_types = []
+        for int_name in Shadow.int_types:
+            for sign in ['', 'u', 's']:
+                name = sign + int_name
+
+                if sign and (
+                        int_name in ['Py_UNICODE', 'Py_UCS4', 'Py_ssize_t',
+                                     'ssize_t', 'ptrdiff_t', 'Py_hash_t'] or
+                        name == "usize_t"):
+                    # size_t is special-cased here a little since ssize_t legitimate
+                    # but usize_t isn't
+                    self.assertNotIn(name, dir(Shadow))
+                    self.assertNotIn('p_' + name, dir(Shadow))
+                    continue
+
+                if not hasattr(Shadow, name):
+                    missing_types.append(name)
+
+                for ptr in range(1, 4):
+                    ptr_name = 'p' * ptr + '_' + name
+                    if not hasattr(Shadow, ptr_name):
+                        missing_types.append(ptr_name)
+        self.assertEqual(missing_types, [])
+
+    def test_most_types(self):
+        # TODO it's unfortunately hard to get a definite list of types to confirm that they're
+        # present (because they're obtained by on-the-fly string parsing in `cython_scope.lookup_type`)
+
+        cython_scope = CythonScope.create_cython_scope(None)
+        # Set up just enough of "Context" and "Errors" that CythonScope.lookup_type can fail
+        class Context:
+            cpp = False
+            language_level = 3
+            future_directives = []
+        cython_scope.context = Context
+        Errors.init_thread()
+
+        missing_types = []
+        missing_lookups = []
+        for (signed, longness, name), type_ in PyrexTypes.modifiers_and_name_to_type.items():
+            if name == 'object':
+                continue  # This probably shouldn't be in Shadow
+            if not hasattr(Shadow, name):
+                missing_types.append(name)
+            if not cython_scope.lookup_type(name):
+                missing_lookups.append(name)
+            for ptr in range(1, 4):
+                ptr_name = 'p' * ptr + '_' + name
+                if not hasattr(Shadow, ptr_name):
+                    missing_types.append(ptr_name)
+                if not cython_scope.lookup_type(ptr_name):
+                    missing_lookups.append(ptr_name)
+        self.assertEqual(missing_types, [])
+        self.assertEqual(missing_lookups, [])
diff --git a/contrib/tools/cython/Cython/Tests/TestTestUtils.py b/contrib/tools/cython/Cython/Tests/TestTestUtils.py
new file mode 100644
index 00000000000..e8329188b31
--- /dev/null
+++ b/contrib/tools/cython/Cython/Tests/TestTestUtils.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+import os.path
+import unittest
+import tempfile
+import textwrap
+import shutil
+
+from ..TestUtils import write_file, write_newer_file, _parse_pattern
+
+
+class TestTestUtils(unittest.TestCase):
+    def setUp(self):
+        super(TestTestUtils, self).setUp()
+        self.temp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        if self.temp_dir and os.path.isdir(self.temp_dir):
+            shutil.rmtree(self.temp_dir)
+        super(TestTestUtils, self).tearDown()
+
+    def _test_path(self, filename):
+        return os.path.join(self.temp_dir, filename)
+
+    def _test_write_file(self, content, expected, **kwargs):
+        file_path = self._test_path("abcfile")
+        write_file(file_path, content, **kwargs)
+        assert os.path.isfile(file_path)
+
+        with open(file_path, 'rb') as f:
+            found = f.read()
+        assert found == expected, (repr(expected), repr(found))
+
+    def test_write_file_text(self):
+        text = u"abcüöä"
+        self._test_write_file(text, text.encode('utf8'))
+
+    def test_write_file_dedent(self):
+        text = u"""
+        A horse is a horse,
+        of course, of course,
+        And no one can talk to a horse
+        of course
+        """
+        self._test_write_file(text, textwrap.dedent(text).encode('utf8'), dedent=True)
+
+    def test_write_file_bytes(self):
+        self._test_write_file(b"ab\0c", b"ab\0c")
+
+    def test_write_newer_file(self):
+        file_path_1 = self._test_path("abcfile1.txt")
+        file_path_2 = self._test_path("abcfile2.txt")
+        write_file(file_path_1, "abc")
+        assert os.path.isfile(file_path_1)
+        write_newer_file(file_path_2, file_path_1, "xyz")
+        assert os.path.isfile(file_path_2)
+        assert os.path.getmtime(file_path_2) > os.path.getmtime(file_path_1)
+
+    def test_write_newer_file_same(self):
+        file_path = self._test_path("abcfile.txt")
+        write_file(file_path, "abc")
+        mtime = os.path.getmtime(file_path)
+        write_newer_file(file_path, file_path, "xyz")
+        assert os.path.getmtime(file_path) > mtime
+
+    def test_write_newer_file_fresh(self):
+        file_path = self._test_path("abcfile.txt")
+        assert not os.path.exists(file_path)
+        write_newer_file(file_path, file_path, "xyz")
+        assert os.path.isfile(file_path)
+
+    def test_parse_pattern(self):
+        self.assertEqual(
+            _parse_pattern("pattern"),
+            (None, None, 'pattern')
+        )
+        self.assertEqual(
+            _parse_pattern("/start/:pattern"),
+            ('start', None, 'pattern')
+        )
+        self.assertEqual(
+            _parse_pattern(":/end/  pattern"),
+            (None, 'end', 'pattern')
+        )
+        self.assertEqual(
+            _parse_pattern("/start/:/end/  pattern"),
+            ('start', 'end', 'pattern')
+        )
+        self.assertEqual(
+            _parse_pattern("/start/:/end/pattern"),
+            ('start', 'end', 'pattern')
+        )
diff --git a/contrib/tools/cython/Cython/Tests/xmlrunner.py b/contrib/tools/cython/Cython/Tests/xmlrunner.py
index d6838aa22ec..eeeb4939439 100644
--- a/contrib/tools/cython/Cython/Tests/xmlrunner.py
+++ b/contrib/tools/cython/Cython/Tests/xmlrunner.py
@@ -109,8 +109,7 @@ class _XMLTestResult(TextTestResult):
         self.elapsed_times = elapsed_times
         self.output_patched = False
 
-    def _prepare_callback(self, test_info, target_list, verbose_str,
-        short_str):
+    def _prepare_callback(self, test_info, target_list, verbose_str, short_str):
         """Append a _TestInfo to the given target list and sets a callback
         method to be called by stopTest method.
         """
@@ -125,7 +124,7 @@ class _XMLTestResult(TextTestResult):
                 self.start_time = self.stop_time = 0
 
             if self.showAll:
-                self.stream.writeln('(%.3fs) %s' % \
+                self.stream.writeln('(%.3fs) %s' %
                     (test_info.get_elapsed_time(), verbose_str))
             elif self.dots:
                 self.stream.write(short_str)
@@ -300,8 +299,7 @@ class _XMLTestResult(TextTestResult):
         "Generates the XML reports to a given XMLTestRunner object."
         all_results = self._get_info_by_testcase()
 
-        if type(test_runner.output) == str and not \
-            os.path.exists(test_runner.output):
+        if isinstance(test_runner.output, str) and not os.path.exists(test_runner.output):
             os.makedirs(test_runner.output)
 
         for suite, tests in all_results.items():
@@ -321,7 +319,7 @@ class _XMLTestResult(TextTestResult):
             xml_content = doc.toprettyxml(indent='\t')
 
             if type(test_runner.output) is str:
-                report_file = open('%s%sTEST-%s.xml' % \
+                report_file = open('%s%sTEST-%s.xml' %
                     (test_runner.output, os.sep, suite), 'w')
                 try:
                     report_file.write(xml_content)
@@ -348,7 +346,7 @@ class XMLTestRunner(TextTestRunner):
         """Create the TestResult object which will be used to store
         information about the executed tests.
         """
-        return _XMLTestResult(self.stream, self.descriptions, \
+        return _XMLTestResult(self.stream, self.descriptions,
             self.verbosity, self.elapsed_times)
 
     def run(self, test):