add ymake export to ydb

5 files changed, 465 insertions, 0 deletions

diff --git a/contrib/tools/cython/Cython/Build/Tests/TestCyCache.py b/contrib/tools/cython/Cython/Build/Tests/TestCyCache.py
new file mode 100644
index 0000000000..a3224b4175
--- /dev/null
+++ b/contrib/tools/cython/Cython/Build/Tests/TestCyCache.py
@@ -0,0 +1,106 @@
+import difflib
+import glob
+import gzip
+import os
+import tempfile
+
+import Cython.Build.Dependencies
+import Cython.Utils
+from Cython.TestUtils import CythonTest
+
+
+class TestCyCache(CythonTest):
+
+    def setUp(self):
+        CythonTest.setUp(self)
+        self.temp_dir = tempfile.mkdtemp(
+            prefix='cycache-test',
+            dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None)
+        self.src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir)
+        self.cache_dir = tempfile.mkdtemp(prefix='cache', dir=self.temp_dir)
+
+    def cache_files(self, file_glob):
+        return glob.glob(os.path.join(self.cache_dir, file_glob))
+
+    def fresh_cythonize(self, *args, **kwargs):
+        Cython.Utils.clear_function_caches()
+        Cython.Build.Dependencies._dep_tree = None  # discard method caches
+        Cython.Build.Dependencies.cythonize(*args, **kwargs)
+
+    def test_cycache_switch(self):
+        content1 = 'value = 1\n'
+        content2 = 'value = 2\n'
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(1, len(self.cache_files('a.c*')))
+        a_contents1 = open(a_c).read()
+        os.unlink(a_c)
+
+        open(a_pyx, 'w').write(content2)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents2 = open(a_c).read()
+        os.unlink(a_c)
+
+        self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!')
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+        a_contents = open(a_c).read()
+        self.assertEqual(
+            a_contents, a_contents1,
+            msg='\n'.join(list(difflib.unified_diff(
+                a_contents.split('\n'), a_contents1.split('\n')))[:10]))
+
+    def test_cycache_uses_cache(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        open(a_pyx, 'w').write('pass')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0])
+        gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii'))
+        os.unlink(a_c)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents = open(a_c).read()
+        self.assertEqual(a_contents, 'fake stuff',
+                         'Unexpected contents: %s...' % a_contents[:100])
+
+    def test_multi_file_output(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        a_h = a_pyx[:-4] + '.h'
+        a_api_h = a_pyx[:-4] + '_api.h'
+        open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        expected = [a_c, a_h, a_api_h]
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+            os.unlink(output)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+
+    def test_options_invalidation(self):
+        hash_pyx = os.path.join(self.src_dir, 'options.pyx')
+        hash_c = hash_pyx[:-len('.pyx')] + '.c'
+
+        open(hash_pyx, 'w').write('pass')
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False)
+        self.assertEqual(1, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=False)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
diff --git a/contrib/tools/cython/Cython/Build/Tests/TestInline.py b/contrib/tools/cython/Cython/Build/Tests/TestInline.py
new file mode 100644
index 0000000000..d209488083
--- /dev/null
+++ b/contrib/tools/cython/Cython/Build/Tests/TestInline.py
@@ -0,0 +1,96 @@
+import os, tempfile
+from Cython.Shadow import inline
+from Cython.Build.Inline import safe_type
+from Cython.TestUtils import CythonTest
+
+try:
+    import numpy
+    has_numpy = True
+except:
+    has_numpy = False
+
+test_kwds = dict(force=True, quiet=True)
+
+global_value = 100
+
+class TestInline(CythonTest):
+    def setUp(self):
+        CythonTest.setUp(self)
+        self.test_kwds = dict(test_kwds)
+        if os.path.isdir('TEST_TMP'):
+            lib_dir = os.path.join('TEST_TMP','inline')
+        else:
+            lib_dir = tempfile.mkdtemp(prefix='cython_inline_')
+        self.test_kwds['lib_dir'] = lib_dir
+
+    def test_simple(self):
+        self.assertEqual(inline("return 1+2", **self.test_kwds), 3)
+
+    def test_types(self):
+        self.assertEqual(inline("""
+            cimport cython
+            return cython.typeof(a), cython.typeof(b)
+        """, a=1.0, b=[], **self.test_kwds), ('double', 'list object'))
+
+    def test_locals(self):
+        a = 1
+        b = 2
+        self.assertEqual(inline("return a+b", **self.test_kwds), 3)
+
+    def test_globals(self):
+        self.assertEqual(inline("return global_value + 1", **self.test_kwds), global_value + 1)
+
+    def test_no_return(self):
+        self.assertEqual(inline("""
+            a = 1
+            cdef double b = 2
+            cdef c = []
+        """, **self.test_kwds), dict(a=1, b=2.0, c=[]))
+
+    def test_def_node(self):
+        foo = inline("def foo(x): return x * x", **self.test_kwds)['foo']
+        self.assertEqual(foo(7), 49)
+
+    def test_class_ref(self):
+        class Type(object):
+            pass
+        tp = inline("Type")['Type']
+        self.assertEqual(tp, Type)
+
+    def test_pure(self):
+        import cython as cy
+        b = inline("""
+        b = cy.declare(float, a)
+        c = cy.declare(cy.pointer(cy.float), &b)
+        return b
+        """, a=3, **self.test_kwds)
+        self.assertEqual(type(b), float)
+
+    def test_compiler_directives(self):
+        self.assertEqual(
+            inline('return sum(x)',
+                   x=[1, 2, 3],
+                   cython_compiler_directives={'boundscheck': False}),
+            6
+        )
+
+    def test_lang_version(self):
+        # GH-3419. Caching for inline code didn't always respect compiler directives.
+        inline_divcode = "def f(int a, int b): return a/b"
+        self.assertEqual(
+            inline(inline_divcode, language_level=2)['f'](5,2),
+            2
+        )
+        self.assertEqual(
+            inline(inline_divcode, language_level=3)['f'](5,2),
+            2.5
+        )
+
+    if has_numpy:
+
+        def test_numpy(self):
+            import numpy
+            a = numpy.ndarray((10, 20))
+            a[0,0] = 10
+            self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]')
+            self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0)
diff --git a/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py b/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py
new file mode 100644
index 0000000000..24213091b2
--- /dev/null
+++ b/contrib/tools/cython/Cython/Build/Tests/TestIpythonMagic.py
@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+# tag: ipython
+
+"""Tests for the Cython magics extension."""
+
+from __future__ import absolute_import
+
+import os
+import sys
+from contextlib import contextmanager
+from Cython.Build import IpythonMagic
+from Cython.TestUtils import CythonTest
+
+try:
+    import IPython.testing.globalipapp
+except ImportError:
+    # Disable tests and fake helpers for initialisation below.
+    def skip_if_not_installed(_):
+        return None
+else:
+    def skip_if_not_installed(c):
+        return c
+
+try:
+    # disable IPython history thread before it gets started to avoid having to clean it up
+    from IPython.core.history import HistoryManager
+    HistoryManager.enabled = False
+except ImportError:
+    pass
+
+code = u"""\
+def f(x):
+    return 2*x
+"""
+
+cython3_code = u"""\
+def f(int x):
+    return 2 / x
+
+def call(x):
+    return f(*(x,))
+"""
+
+pgo_cython3_code = cython3_code + u"""\
+def main():
+    for _ in range(100): call(5)
+main()
+"""
+
+
+if sys.platform == 'win32':
+    # not using IPython's decorators here because they depend on "nose"
+    try:
+        from unittest import skip as skip_win32
+    except ImportError:
+        # poor dev's silent @unittest.skip()
+        def skip_win32(dummy):
+            def _skip_win32(func):
+                return None
+            return _skip_win32
+else:
+    def skip_win32(dummy):
+        def _skip_win32(func):
+            def wrapper(*args, **kwargs):
+                func(*args, **kwargs)
+            return wrapper
+        return _skip_win32
+
+
+@skip_if_not_installed
+class TestIPythonMagic(CythonTest):
+
+    @classmethod
+    def setUpClass(cls):
+        CythonTest.setUpClass()
+        cls._ip = IPython.testing.globalipapp.get_ipython()
+
+    def setUp(self):
+        CythonTest.setUp(self)
+        self._ip.extension_manager.load_extension('cython')
+
+    def test_cython_inline(self):
+        ip = self._ip
+        ip.ex('a=10; b=20')
+        result = ip.run_cell_magic('cython_inline', '', 'return a+b')
+        self.assertEqual(result, 30)
+
+    @skip_win32('Skip on Windows')
+    def test_cython_pyximport(self):
+        ip = self._ip
+        module_name = '_test_cython_pyximport'
+        ip.run_cell_magic('cython_pyximport', module_name, code)
+        ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+        ip.run_cell_magic('cython_pyximport', module_name, code)
+        ip.ex('h = f(-10)')
+        self.assertEqual(ip.user_ns['h'], -20.0)
+        try:
+            os.remove(module_name + '.pyx')
+        except OSError:
+            pass
+
+    def test_cython(self):
+        ip = self._ip
+        ip.run_cell_magic('cython', '', code)
+        ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+
+    def test_cython_name(self):
+        # The Cython module named 'mymodule' defines the function f.
+        ip = self._ip
+        ip.run_cell_magic('cython', '--name=mymodule', code)
+        # This module can now be imported in the interactive namespace.
+        ip.ex('import mymodule; g = mymodule.f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+
+    def test_cython_language_level(self):
+        # The Cython cell defines the functions f() and call().
+        ip = self._ip
+        ip.run_cell_magic('cython', '', cython3_code)
+        ip.ex('g = f(10); h = call(10)')
+        if sys.version_info[0] < 3:
+            self.assertEqual(ip.user_ns['g'], 2 // 10)
+            self.assertEqual(ip.user_ns['h'], 2 // 10)
+        else:
+            self.assertEqual(ip.user_ns['g'], 2.0 / 10.0)
+            self.assertEqual(ip.user_ns['h'], 2.0 / 10.0)
+
+    def test_cython3(self):
+        # The Cython cell defines the functions f() and call().
+        ip = self._ip
+        ip.run_cell_magic('cython', '-3', cython3_code)
+        ip.ex('g = f(10); h = call(10)')
+        self.assertEqual(ip.user_ns['g'], 2.0 / 10.0)
+        self.assertEqual(ip.user_ns['h'], 2.0 / 10.0)
+
+    def test_cython2(self):
+        # The Cython cell defines the functions f() and call().
+        ip = self._ip
+        ip.run_cell_magic('cython', '-2', cython3_code)
+        ip.ex('g = f(10); h = call(10)')
+        self.assertEqual(ip.user_ns['g'], 2 // 10)
+        self.assertEqual(ip.user_ns['h'], 2 // 10)
+
+    @skip_win32('Skip on Windows')
+    def test_cython3_pgo(self):
+        # The Cython cell defines the functions f() and call().
+        ip = self._ip
+        ip.run_cell_magic('cython', '-3 --pgo', pgo_cython3_code)
+        ip.ex('g = f(10); h = call(10); main()')
+        self.assertEqual(ip.user_ns['g'], 2.0 / 10.0)
+        self.assertEqual(ip.user_ns['h'], 2.0 / 10.0)
+
+    @skip_win32('Skip on Windows')
+    def test_extlibs(self):
+        ip = self._ip
+        code = u"""
+from libc.math cimport sin
+x = sin(0.0)
+        """
+        ip.user_ns['x'] = 1
+        ip.run_cell_magic('cython', '-l m', code)
+        self.assertEqual(ip.user_ns['x'], 0)
+
+
+    def test_cython_verbose(self):
+        ip = self._ip
+        ip.run_cell_magic('cython', '--verbose', code)
+        ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+
+    def test_cython_verbose_thresholds(self):
+        @contextmanager
+        def mock_distutils():
+            class MockLog:
+                DEBUG = 1
+                INFO = 2
+                thresholds = [INFO]
+
+                def set_threshold(self, val):
+                    self.thresholds.append(val)
+                    return self.thresholds[-2]
+
+
+            new_log = MockLog()
+            old_log = IpythonMagic.distutils.log
+            try:
+                IpythonMagic.distutils.log = new_log
+                yield new_log
+            finally:
+                IpythonMagic.distutils.log = old_log
+
+        ip = self._ip
+        with mock_distutils() as verbose_log:
+            ip.run_cell_magic('cython', '--verbose', code)
+            ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+        self.assertEqual([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO],
+                          verbose_log.thresholds)
+
+        with mock_distutils() as normal_log:
+            ip.run_cell_magic('cython', '', code)
+            ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+        self.assertEqual([normal_log.INFO], normal_log.thresholds)
diff --git a/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py b/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py
new file mode 100644
index 0000000000..a7572a5083
--- /dev/null
+++ b/contrib/tools/cython/Cython/Build/Tests/TestStripLiterals.py
@@ -0,0 +1,57 @@
+from Cython.Build.Dependencies import strip_string_literals
+
+from Cython.TestUtils import CythonTest
+
+class TestStripLiterals(CythonTest):
+
+    def t(self, before, expected):
+        actual, literals = strip_string_literals(before, prefix="_L")
+        self.assertEqual(expected, actual)
+        for key, value in literals.items():
+            actual = actual.replace(key, value)
+        self.assertEqual(before, actual)
+
+    def test_empty(self):
+        self.t("", "")
+
+    def test_single_quote(self):
+        self.t("'x'", "'_L1_'")
+
+    def test_double_quote(self):
+        self.t('"x"', '"_L1_"')
+
+    def test_nested_quotes(self):
+        self.t(""" '"' "'" """, """ '_L1_' "_L2_" """)
+
+    def test_triple_quote(self):
+        self.t(" '''a\n''' ", " '''_L1_''' ")
+
+    def test_backslash(self):
+        self.t(r"'a\'b'", "'_L1_'")
+        self.t(r"'a\\'", "'_L1_'")
+        self.t(r"'a\\\'b'", "'_L1_'")
+
+    def test_unicode(self):
+        self.t("u'abc'", "u'_L1_'")
+
+    def test_raw(self):
+        self.t(r"r'abc\\'", "r'_L1_'")
+
+    def test_raw_unicode(self):
+        self.t(r"ru'abc\\'", "ru'_L1_'")
+
+    def test_comment(self):
+        self.t("abc # foo", "abc #_L1_")
+
+    def test_comment_and_quote(self):
+        self.t("abc # 'x'", "abc #_L1_")
+        self.t("'abc#'", "'_L1_'")
+
+    def test_include(self):
+        self.t("include 'a.pxi' # something here",
+               "include '_L1_' #_L2_")
+
+    def test_extern(self):
+        self.t("cdef extern from 'a.h': # comment",
+               "cdef extern from '_L1_': #_L2_")
+
diff --git a/contrib/tools/cython/Cython/Build/Tests/__init__.py b/contrib/tools/cython/Cython/Build/Tests/__init__.py
new file mode 100644
index 0000000000..fa81adaff6
--- /dev/null
+++ b/contrib/tools/cython/Cython/Build/Tests/__init__.py
@@ -0,0 +1 @@
+# empty file