--- contrib/tools/cython_py2/Cython/Compiler/Annotate.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Annotate.py (working tree) @@ -10,7 +10,10 @@ import textwrap from datetime import datetime from functools import partial from collections import defaultdict -from xml.sax.saxutils import escape as html_escape +try: + from xml.sax.saxutils import escape as html_escape +except ImportError: + pass try: from StringIO import StringIO except ImportError: --- contrib/tools/cython_py2/Cython/Compiler/CmdLine.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/CmdLine.py (working tree) @@ -166,6 +166,9 @@ def create_cython_argparser(): parser.add_argument("--cimport-from-pyx", dest='cimport_from_pyx', action='store_true', help=SUPPRESS) parser.add_argument("--old-style-globals", dest='old_style_globals', action='store_true', help=SUPPRESS) + parser.add_argument("--init-suffix", dest='init_suffix', action='store', type=str, help=SUPPRESS) + parser.add_argument("--source-root", dest='source_root', action='store', type=str, help=SUPPRESS) + # debug stuff: from . import DebugFlags for name in vars(DebugFlags): --- contrib/tools/cython_py2/Cython/Compiler/ExprNodes.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/ExprNodes.py (working tree) @@ -10066,6 +10066,8 @@ class CodeObjectNode(ExprNode): func.name, identifier=True, is_str=False, unicode_value=func.name) # FIXME: better way to get the module file path at module init time? Encoding to use? file_path = StringEncoding.bytes_literal(func.pos[0].get_filenametable_entry().encode('utf8'), 'utf8') + # XXX Use get_description() to set arcadia root relative filename + file_path = StringEncoding.bytes_literal(func.pos[0].get_description().encode('utf8'), 'utf8') file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True) # This combination makes CPython create a new dict for "frame.f_locals" (see GH #1836). --- contrib/tools/cython_py2/Cython/Compiler/Main.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Main.py (working tree) @@ -235,9 +235,7 @@ class Context(object): if not pxd_pathname: if debug_find_module: print("...looking for pxd file") - # Only look in sys.path if we are explicitly looking - # for a .pxd file. - pxd_pathname = self.find_pxd_file(qualified_name, pos, sys_path=need_pxd and not relative_import) + pxd_pathname = self.find_pxd_file(qualified_name, pos) self._check_pxd_filename(pos, pxd_pathname, qualified_name) if debug_find_module: print("......found %s" % pxd_pathname) @@ -259,6 +257,8 @@ class Context(object): rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1] if not pxd_pathname.endswith(rel_path): rel_path = pxd_pathname # safety measure to prevent printing incorrect paths + if Options.source_root: + rel_path = os.path.relpath(pxd_pathname, Options.source_root) source_desc = FileSourceDescriptor(pxd_pathname, rel_path) err, result = self.process_pxd(source_desc, scope, qualified_name) if err: @@ -269,7 +269,7 @@ class Context(object): pass return scope - def find_pxd_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): + def find_pxd_file(self, qualified_name, pos=None, sys_path=False, source_file_path=None): # Search include path (and sys.path if sys_path is True) for # the .pxd file corresponding to the given fully-qualified # module name. @@ -498,7 +498,7 @@ def run_pipeline(source, options, full_module_name=None, context=None): # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) - full_module_name = full_module_name or context.extract_module_name(source, options) + full_module_name = full_module_name or options.module_name or context.extract_module_name(source, options) full_module_name = EncodedString(full_module_name) Utils.raise_error_if_module_name_forbidden(full_module_name) @@ -509,6 +509,8 @@ def run_pipeline(source, options, full_module_name=None, context=None): rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path + if Options.source_root: + rel_path = os.path.relpath(abs_path, Options.source_root) source_desc = FileSourceDescriptor(abs_path, rel_path) source = CompilationSource(source_desc, full_module_name, cwd) @@ -729,6 +731,22 @@ def search_include_directories(dirs, qualified_name, suffix="", pos=None, includ if path: return path + # Arcadia-specific lookup: search for packages in include paths, + # ignoring existence of __init__.py files as packages markers + # (they are not required by Arcadia build system) + module_filename = module_name + suffix + package_filename = "__init__" + suffix + + for dir in dirs: + package_dir = os.path.join(dir, *package_names) + path = os.path.join(package_dir, module_filename) + if os.path.exists(path): + return path + path = os.path.join(dir, package_dir, module_name, + package_filename) + if os.path.exists(path): + return path + return None --- contrib/tools/cython_py2/Cython/Compiler/ModuleNode.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/ModuleNode.py (working tree) @@ -302,16 +302,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") h_code_main.putln("") h_code_main.putln("#if PY_MAJOR_VERSION < 3") - if env.module_name.isascii(): - py2_mod_name = env.module_name + init_suffix = EncodedString(options.init_suffix) if options.init_suffix else env.module_name + if init_suffix.isascii(): + py2_init_suffix = init_suffix else: - py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8") - h_code_main.putln('#error "Unicode module names are not supported in Python 2";') - h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name) + py2_init_suffix = init_suffix.encode("ascii", errors="ignore").decode("utf-8") + h_code_main.putln('#error "Unicode init names are not supported in Python 2";') + h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_init_suffix) h_code_main.putln("#else") - py3_mod_func_name = self.mod_init_func_cname('PyInit', env) + py3_mod_func_name = self.mod_init_func_cname('PyInit', env, options) warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % ( - py2_mod_name, py3_mod_func_name, py3_mod_func_name)) + py2_init_suffix, py3_mod_func_name, py3_mod_func_name)) h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.')) h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name) h_code_main.putln("") @@ -530,7 +531,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_module_state_traverse(env, globalstate['module_state_traverse']) # init_globals is inserted before this - self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) + self.generate_module_init_func(modules[:-1], env, options, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) if Options.embed: self.generate_main_method(env, globalstate['main_method']) @@ -943,6 +944,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if code.globalstate.filename_list: for source_desc in code.globalstate.filename_list: file_path = source_desc.get_filenametable_entry() + if Options.source_root: + # If source root specified, dump description - it's source root relative filename + file_path = source_desc.get_description() if isabs(file_path): file_path = basename(file_path) # never include absolute paths escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"') @@ -1090,6 +1094,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): constructor = None destructor = None for attr in scope.var_entries: + if attr.type.is_cfunction: + code.put("inline ") if attr.type.is_cfunction and attr.type.is_static_method: code.put("static ") elif attr.name == "": @@ -2966,24 +2972,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): Naming.fusedfunction_type_cname) code.putln('#endif') - def generate_module_init_func(self, imported_modules, env, code): + def generate_module_init_func(self, imported_modules, env, options, code): subfunction = self.mod_init_subfunction(self.pos, self.scope, code) - self.generate_pymoduledef_struct(env, code) + self.generate_pymoduledef_struct(env, options, code) code.enter_cfunc_scope(self.scope) code.putln("") code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0]) - if env.module_name.isascii(): - py2_mod_name = env.module_name + init_suffix = EncodedString(options.init_suffix) if options.init_suffix else env.module_name + if init_suffix.isascii(): + py2_init_suffix = init_suffix fail_compilation_in_py2 = False else: fail_compilation_in_py2 = True - # at this point py2_mod_name is largely a placeholder and the value doesn't matter - py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8") + # at this point py2_init_suffix is largely a placeholder and the value doesn't matter + py2_init_suffix = init_suffix.encode("ascii", errors="ignore").decode("utf8") - header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name - header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env) + header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_init_suffix + header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env, options) header3 = EncodedString(header3) code.putln("#if PY_MAJOR_VERSION < 3") # Optimise for small code size as the module init function is only executed once. @@ -2992,7 +2999,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#error "Unicode module names are not supported in Python 2";') if self.scope.is_package: code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") - code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name) + code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_init_suffix) code.putln("#endif") code.putln(header2) code.putln("#else") @@ -3000,7 +3007,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if self.scope.is_package: code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % ( - self.mod_init_func_cname('PyInit', env))) + self.mod_init_func_cname('PyInit', env, options))) code.putln("#endif") # Hack for a distutils bug - https://bugs.python.org/issue39432 # distutils attempts to make visible a slightly wrong PyInitU module name. Just create a dummy @@ -3070,7 +3077,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) + self.generate_module_creation_code(env, options, code) if profile or linetrace: tempdecl_code.put_trace_declarations() @@ -3483,9 +3490,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): except UnicodeEncodeError: return "PyInitU" + (u"_"+name).encode('punycode').replace(b'-', b'_').decode('ascii') - def mod_init_func_cname(self, prefix, env): + def mod_init_func_cname(self, prefix, env, options=None): # from PEP483 - return self.punycode_module_name(prefix, env.module_name) + return self.punycode_module_name(prefix, options and options.init_suffix or env.module_name) # Returns the name of the C-function that corresponds to the module initialisation. # (module initialisation == the cython code outside of functions) @@ -3495,7 +3502,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): env = self.scope return self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) - def generate_pymoduledef_struct(self, env, code): + def generate_pymoduledef_struct(self, env, options, code): if env.doc: doc = "%s" % code.get_string_const(env.doc) else: @@ -3533,7 +3540,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#endif') code.putln('{') code.putln(" PyModuleDef_HEAD_INIT,") - code.putln(' %s,' % env.module_name.as_c_string_literal()) + code.putln(' %s,' % (EncodedString(options.module_name) if options.module_name else env.module_name).as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") @@ -3563,7 +3570,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#endif') code.putln("#endif") - def generate_module_creation_code(self, env, code): + def generate_module_creation_code(self, env, options, code): # Generate code to create the module object and # install the builtins. if env.doc: @@ -3581,7 +3588,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( env.module_cname, - env.module_name.as_c_string_literal(), + (EncodedString(options.module_name) if options.module_name else env.module_name).as_c_string_literal(), env.method_table_cname, doc, env.module_cname)) --- contrib/tools/cython_py2/Cython/Compiler/Nodes.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Nodes.py (working tree) @@ -4756,7 +4756,7 @@ class GeneratorBodyDefNode(DefNode): self.declare_generator_body(env) def generate_function_header(self, code, proto=False): - header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % ( + header = "static PyObject *%s(PyObject *%s_obj, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % ( self.entry.func_cname, Naming.generator_cname, Naming.local_tstate_cname, @@ -4782,6 +4782,7 @@ class GeneratorBodyDefNode(DefNode): # ----- Function header code.putln("") self.generate_function_header(code) + code.putln("__pyx_CoroutineObject *%s = (__pyx_CoroutineObject *)%s_obj;" % (Naming.generator_cname, Naming.generator_cname)) closure_init_code = code.insertion_point() # ----- Local variables code.putln("PyObject *%s = NULL;" % Naming.retval_cname) --- contrib/tools/cython_py2/Cython/Compiler/Options.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Options.py (working tree) @@ -148,6 +148,9 @@ buffer_max_dims = 8 #: Number of function closure instances to keep in a freelist (0: no freelists) closure_freelist_size = 8 +# Arcadia specific +source_root = None + def get_directive_defaults(): # To add an item to this list, all accesses should be changed to use the new @@ -787,6 +790,7 @@ default_options = dict( language_level=None, # warn but default to 2 formal_grammar=False, gdb_debug=False, + init_suffix=None, compile_time_env=None, module_name=None, common_utility_include_dir=None, --- contrib/tools/cython_py2/Cython/Compiler/Parsing.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Parsing.py (working tree) @@ -2105,7 +2105,12 @@ def p_include_statement(s, ctx): if include_file_path: s.included_files.append(include_file_name) with Utils.open_source_file(include_file_path) as f: - source_desc = FileSourceDescriptor(include_file_path) + if Options.source_root: + import os + rel_path = os.path.relpath(include_file_path, Options.source_root) + else: + rel_path = None + source_desc = FileSourceDescriptor(include_file_path, rel_path) s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments) tree = p_statement_list(s2, ctx) return tree @@ -3913,6 +3918,9 @@ def p_module(s, pxd, full_module_name, ctx=Ctx): directive_comments = p_compiler_directive_comments(s) s.parse_comments = False + if s.context.language_level is None: + s.context.set_language_level(2) # Arcadia default. + if s.context.language_level is None: s.context.set_language_level('3str') if pos[0].filename: --- contrib/tools/cython_py2/Cython/Compiler/PyrexTypes.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/PyrexTypes.py (working tree) @@ -3944,7 +3944,7 @@ class CStructOrUnionType(CType): return expr_code return super(CStructOrUnionType, self).cast_code(expr_code) -cpp_string_conversions = ("std::string",) +cpp_string_conversions = ("std::string", "TString", "TStringBuf") builtin_cpp_conversions = { # type element template params @@ -3956,6 +3956,11 @@ builtin_cpp_conversions = { "std::map": 2, "std::unordered_map": 2, "std::complex": 1, + # arcadia_cpp_conversions + "TMaybe": 1, + "TVector": 1, + "THashMap": 2, + "TMap": 2, } class CppClassType(CType): @@ -3986,7 +3991,7 @@ class CppClassType(CType): self.templates = templates self.template_type = template_type self.num_optional_templates = sum(is_optional_template_param(T) for T in templates or ()) - if templates: + if templates and False: # https://github.com/cython/cython/issues/1868 self.specializations = {tuple(zip(templates, templates)): self} else: self.specializations = {} @@ -4032,8 +4037,10 @@ class CppClassType(CType): if self.cname in cpp_string_conversions: cls = 'string' tags = type_identifier(self), - else: + elif self.cname.startswith('std::'): cls = self.cname[5:] + else: + cls = 'arcadia_' + self.cname cname = '__pyx_convert_%s_from_py_%s' % (cls, '__and_'.join(tags)) context.update({ 'cname': cname, @@ -4058,7 +4065,6 @@ class CppClassType(CType): return False return True - def create_to_py_utility_code(self, env): if self.to_py_function is not None: return True @@ -4078,9 +4084,12 @@ class CppClassType(CType): cls = 'string' prefix = 'PyObject_' # gets specialised by explicit type casts in CoerceToPyTypeNode tags = type_identifier(self), - else: + elif self.cname.startswith('std::'): cls = self.cname[5:] prefix = '' + else: + cls = 'arcadia_' + self.cname + prefix = '' cname = "__pyx_convert_%s%s_to_py_%s" % (prefix, cls, "____".join(tags)) context.update({ 'cname': cname, --- contrib/tools/cython_py2/Cython/Compiler/Scanning.py (index) +++ contrib/tools/cython_py2/Cython/Compiler/Scanning.py (working tree) @@ -229,6 +229,8 @@ class FileSourceDescriptor(SourceDescriptor): return lines def get_description(self): + # Dump path_description, it's already arcadia root relative (required for proper file matching in coverage) + return self.path_description try: return os.path.relpath(self.path_description) except ValueError: --- contrib/tools/cython_py2/Cython/Coverage.py (index) +++ contrib/tools/cython_py2/Cython/Coverage.py (working tree) @@ -56,7 +56,7 @@ from collections import defaultdict from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ from coverage.files import canonical_filename -from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file +from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file, OpenFile, generated_file_exists from . import __version__ @@ -133,10 +133,14 @@ class Plugin(CoveragePlugin): """ Try to find a C source file for a file path found by the tracer. """ + # TODO We need to pxd-files to the include map. For more info see pybuild.py + # Currently skip such files, because they are not supported in Arcadia pybuild with coverage. + if os.path.splitext(filename)[-1] not in ('.pyx', '.pxi'): + return None if filename.startswith('<') or filename.startswith('memory:'): return None c_file = py_file = None - filename = canonical_filename(os.path.abspath(filename)) + filename = canonical_filename(filename) if self._c_files_map and filename in self._c_files_map: c_file = self._c_files_map[filename][0] @@ -166,16 +170,21 @@ class Plugin(CoveragePlugin): # from coverage.python import PythonFileReporter # return PythonFileReporter(filename) - filename = canonical_filename(os.path.abspath(filename)) + filename = canonical_filename(filename) if self._c_files_map and filename in self._c_files_map: c_file, rel_file_path, code = self._c_files_map[filename] else: c_file, _ = self._find_source_files(filename) if not c_file: + if standalone(): + raise AssertionError(filename) return None # unknown file rel_file_path, code = self._read_source_lines(c_file, filename) if code is None: + if standalone(): + raise AssertionError(filename) return None # no source found + return CythonModuleReporter( c_file, filename, @@ -206,6 +215,8 @@ class Plugin(CoveragePlugin): self._find_c_source_files(os.path.dirname(filename), filename) if filename in self._c_files_map: return self._c_files_map[filename][0], None + if standalone(): + raise AssertionError(filename) else: # none of our business return None, None @@ -224,8 +235,8 @@ class Plugin(CoveragePlugin): py_source_file = os.path.splitext(c_file)[0] + '.py' if not os.path.exists(py_source_file): py_source_file = None - if not is_cython_generated_file(c_file, if_not_found=False): - if py_source_file and os.path.exists(c_file): + if not is_cython_generated_file(c_file, if_not_found=False, is_standalone=standalone()): + if py_source_file and generated_file_exists(c_file, standalone()): # if we did not generate the C file, # then we probably also shouldn't care about the .py file. py_source_file = None @@ -238,6 +249,20 @@ class Plugin(CoveragePlugin): Desperately parse all C files in the directory or its package parents (not re-descending) to find the (included) source file in one of them. """ + if standalone(): + if os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'): + broot = os.environ['PYTHON_COVERAGE_CYTHON_BUILD_ROOT'] + iter_files = lambda: (os.path.join(root, filename) for root, _, files in os.walk(broot) for filename in files) + else: + import library.python.resource + iter_files = library.python.resource.resfs_files + for c_file in iter_files(): + if os.path.splitext(c_file)[1] in C_FILE_EXTENSIONS: + self._read_source_lines(c_file, source_file) + if source_file in self._c_files_map: + return + raise AssertionError((source_file, os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'))) + if not os.path.isdir(dir_path): return splitext = os.path.splitext @@ -302,7 +327,7 @@ class Plugin(CoveragePlugin): if self._excluded_lines_map is None: self._excluded_lines_map = defaultdict(set) - with open(c_file) as lines: + with OpenFile(c_file, is_standalone=standalone()) as lines: lines = iter(lines) for line in lines: match = match_source_path_line(line) @@ -362,7 +387,10 @@ class CythonModuleTracer(FileTracer): return self._file_path_map[source_file] except KeyError: pass - abs_path = _find_dep_file_path(filename, source_file) + if standalone(): + abs_path = self.module_file + else: + abs_path = _find_dep_file_path(filename, source_file) if self.py_file and source_file[-3:].lower() == '.py': # always let coverage.py handle this case itself @@ -386,6 +414,7 @@ class CythonModuleReporter(FileReporter): self.c_file = c_file self._code = code self._excluded_lines = excluded_lines + self._abs_filename = self._find_abs_filename() def lines(self): """ @@ -412,8 +441,8 @@ class CythonModuleReporter(FileReporter): """ Return the source code of the file as a string. """ - if os.path.exists(self.filename): - with open_source_file(self.filename) as f: + if os.path.exists(self._abs_filename): + with open_source_file(self._abs_filename) as f: return f.read() else: return '\n'.join( @@ -424,16 +453,92 @@ class CythonModuleReporter(FileReporter): """ Iterate over the source code tokens. """ - if os.path.exists(self.filename): - with open_source_file(self.filename) as f: + if os.path.exists(self._abs_filename): + with open_source_file(self._abs_filename) as f: for line in f: yield [('txt', line.rstrip('\n'))] else: for line in self._iter_source_tokens(): - yield [('txt', line)] + yield line + + def _find_abs_filename(self): + for root in [ + os.environ.get('PYTHON_COVERAGE_ARCADIA_SOURCE_ROOT'), + os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'), + ]: + if root: + abs_path = os.path.join(root, self.filename) + if root and os.path.exists(abs_path): + return abs_path + return self.filename def coverage_init(reg, options): plugin = Plugin() reg.add_configurer(plugin) reg.add_file_tracer(plugin) + + +# ========================== Arcadia specific ================================= + +def standalone(): + return getattr(sys, 'is_standalone_binary', False) + +# ======================= Redefine some methods =============================== + +if standalone(): + import itertools + import json + + CYTHON_INCLUDE_MAP = {'undef': True} + + + def _find_c_source(base_path): + ''' + There are two different coverage stages when c source file might be required: + * trace - python calls c_tracefunc on every line and CythonModuleTracer needs to match + pyd and pxi files with source files. This is test's runtime and tests' clean environment might + doesn't contain required sources and generated files (c, cpp), that's why we get files from resfs_src. + * report - coverage data contains only covered data and CythonModuleReporter needs to + parse source files to obtain missing lines and branches. This is test_tool's resolve/build_report step. + test_tools doesn't have compiled in sources, however, it can extract required files + from binary and set PYTHON_COVERAGE_CYTHON_BUILD_ROOT to guide coverage. + ''' + if os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'): + # Report stage (resolve) + def exists(filename): + return os.path.exists(os.path.join(os.environ['PYTHON_COVERAGE_CYTHON_BUILD_ROOT'], filename)) + else: + # Trace stage (test's runtime) + def exists(filename): + import library.python.resource + return library.python.resource.resfs_src(filename, resfs_file=True) + + if os.environ.get('PYTHON_COVERAGE_CYTHON_INCLUDE_MAP'): + if CYTHON_INCLUDE_MAP.get('undef'): + with open(os.environ['PYTHON_COVERAGE_CYTHON_INCLUDE_MAP']) as afile: + data = json.load(afile) + data = {os.path.splitext(k)[0]: v for k, v in data.items()} + + CYTHON_INCLUDE_MAP.clear() + CYTHON_INCLUDE_MAP.update(data) + + if base_path in CYTHON_INCLUDE_MAP: + # target file was included and should be sought inside another pyx file + base_path = CYTHON_INCLUDE_MAP[base_path] + + # TODO (', '.py3', '.py2') -> ('.py3', '.py2'), when https://a.yandex-team.ru/review/3511262 is merged + suffixes = [''.join(x) for x in itertools.product(('.pyx',), ('', '.py3', '.py2'), ('.cpp', '.c'))] + suffixes += C_FILE_EXTENSIONS + + for suffix in suffixes: + if exists(base_path + suffix): + return base_path + suffix + + return None + + + def _find_dep_file_path(main_file, file_path, relative_path_search=False): + # file_path is already arcadia root relative + return canonical_filename(file_path) + --- contrib/tools/cython_py2/Cython/Utility/CppConvert.pyx (index) +++ contrib/tools/cython_py2/Cython/Utility/CppConvert.pyx (working tree) @@ -271,3 +271,151 @@ cdef object {{cname}}(const std_complex[X]& z): tmp.real = z.real() tmp.imag = z.imag() return tmp + + +#################### arcadia_TMaybe.from_py #################### + +cdef extern from *: + cdef cppclass TMaybe [T]: + TMaybe() + TMaybe(T&) + TMaybe& operator =(T&) + +@cname("{{cname}}") +cdef TMaybe[X] {{cname}}(object o) except *: + cdef TMaybe[X] result + if o is not None: + result = o + return result + +#################### arcadia_TMaybe.to_py #################### + +cdef extern from *: + cdef cppclass TMaybe [T]: + bint Defined() + T& GetRef() + +@cname("{{cname}}") +cdef object {{cname}}(const TMaybe[X]& s): + if s.Defined(): + return s.GetRef() + return None + + +#################### arcadia_TVector.from_py #################### + +cdef extern from *: + cdef cppclass TVector [T]: + void push_back(T&) + +@cname("{{cname}}") +cdef TVector[X] {{cname}}(object o) except *: + cdef TVector[X] v + for item in o: + v.push_back(item) + return v + + +#################### arcadia_TVector.to_py #################### + +cdef extern from *: + cdef cppclass TVector [T]: + size_t size() + T& operator[](size_t) + +@cname("{{cname}}") +cdef object {{cname}}(const TVector[X]& v): + return [v[i] for i in range(v.size())] + + +#################### arcadia_THashMap.from_py #################### + +cdef extern from *: + cdef cppclass pair "std::pair" [T, U]: + pair(T&, U&) + cdef cppclass THashMap [T, U]: + void insert(pair[T, U]&) + + +@cname("{{cname}}") +cdef THashMap[X,Y] {{cname}}(object o) except *: + cdef dict d = o + cdef THashMap[X,Y] m + for key, value in d.iteritems(): + m.insert(pair[X,Y](key, value)) + return m + + +#################### arcadia_THashMap.to_py #################### + +cimport cython + +cdef extern from *: + cdef cppclass THashMap [T, U]: + cppclass value_type: + T first + U second + cppclass const_iterator: + value_type& operator*() + const_iterator operator++() + bint operator!=(const_iterator) + const_iterator begin() + const_iterator end() + +@cname("{{cname}}") +cdef dict {{cname}}(const THashMap[X,Y]& s): + cdef dict result = {} + cdef const THashMap[X,Y].value_type *key_value + cdef THashMap[X,Y].const_iterator iter = s.begin() + while iter != s.end(): + key_value = &cython.operator.dereference(iter) + result[key_value.first] = key_value.second + cython.operator.preincrement(iter) + return result + + +#################### arcadia_TMap.from_py #################### + +cdef extern from *: + cdef cppclass pair "std::pair" [T, U]: + pair(T&, U&) + cdef cppclass TMap [T, U]: + void insert(pair[T, U]&) + + +@cname("{{cname}}") +cdef TMap[X,Y] {{cname}}(object o) except *: + cdef dict d = o + cdef TMap[X,Y] m + for key, value in d.iteritems(): + m.insert(pair[X,Y](key, value)) + return m + + +#################### arcadia_TMap.to_py #################### + +cimport cython + +cdef extern from *: + cdef cppclass TMap [T, U]: + cppclass value_type: + T first + U second + cppclass const_iterator: + value_type& operator*() + const_iterator operator++() + bint operator!=(const_iterator) + const_iterator begin() + const_iterator end() + +@cname("{{cname}}") +cdef dict {{cname}}(const TMap[X,Y]& s): + cdef dict result = {} + cdef const TMap[X,Y].value_type *key_value + cdef TMap[X,Y].const_iterator iter = s.begin() + while iter != s.end(): + key_value = &cython.operator.dereference(iter) + result[key_value.first] = key_value.second + cython.operator.preincrement(iter) + return result + --- contrib/tools/cython_py2/Cython/Utility/Embed.c (index) +++ contrib/tools/cython_py2/Cython/Utility/Embed.c (working tree) @@ -5,6 +5,8 @@ #endif #if PY_MAJOR_VERSION < 3 +void Py_InitArgcArgv(int argc, char **argv); + int %(main_method)s(int argc, char** argv) #elif defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) int %(wmain_method)s(int argc, wchar_t **argv) @@ -23,6 +25,10 @@ static int __Pyx_main(int argc, wchar_t **argv) m = fpgetmask(); fpsetmask(m & ~FP_X_OFL); #endif +#if PY_MAJOR_VERSION < 3 + if (argc && argv) + Py_InitArgcArgv(argc, argv); +#endif #if PY_VERSION_HEX < 0x03080000 if (argc && argv) Py_SetProgramName(argv[0]); --- contrib/tools/cython_py2/Cython/Utility/ModuleSetupCode.c (index) +++ contrib/tools/cython_py2/Cython/Utility/ModuleSetupCode.c (working tree) @@ -13,6 +13,16 @@ /////////////// CModulePreamble /////////////// +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wunused-function" +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +// Ignore tp_print initializer. Need for ya make -DUSE_SYSTEM_PYTHON=3.8 +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +#endif + #include /* For offsetof */ #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) --- contrib/tools/cython_py2/Cython/Utility/Optimize.c (index) +++ contrib/tools/cython_py2/Cython/Utility/Optimize.c (working tree) @@ -1248,7 +1248,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, } return PyInt_FromLong(x); {{elif op == 'Lshift'}} - if (likely(b < (long) (sizeof(long)*8) && a == (a << b) >> b) || !a) { + if (likely(b < (int)(sizeof(long)*8) && a == (a << b) >> b) || !a) { return PyInt_FromLong(a {{c_op}} b); } {{elif c_op == '*'}} @@ -1391,12 +1391,12 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, x = a {{c_op}} b; {{if op == 'Lshift'}} #ifdef HAVE_LONG_LONG - if (unlikely(!(b < (long) (sizeof(long)*8) && a == x >> b)) && a) { + if (unlikely(!(b < (int)(sizeof(long)*8) && a == x >> b)) && a) { ll{{ival}} = {{ival}}; goto long_long; } #else - if (likely(b < (long) (sizeof(long)*8) && a == x >> b) || !a) /* execute return statement below */ + if (likely(b < (int)(sizeof(long)*8) && a == x >> b) || !a) /* execute return statement below */ #endif {{endif}} {{endif}} @@ -1449,9 +1449,9 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, double result; {{zerodiv_check('b', 'float')}} // copied from floatobject.c in Py3.5: - PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) +// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; - PyFPE_END_PROTECT(result) +// PyFPE_END_PROTECT(result) return PyFloat_FromDouble(result); {{endif}} } @@ -1599,7 +1599,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv } {{else}} // copied from floatobject.c in Py3.5: - PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) +// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) {{if c_op == '%'}} result = fmod(a, b); if (result) @@ -1609,7 +1609,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{else}} result = a {{c_op}} b; {{endif}} - PyFPE_END_PROTECT(result) +// PyFPE_END_PROTECT(result) return PyFloat_FromDouble(result); {{endif}} } --- contrib/tools/cython_py2/Cython/Utility/StringTools.c (index) +++ contrib/tools/cython_py2/Cython/Utility/StringTools.c (working tree) @@ -505,7 +505,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_s //@requires: decode_c_bytes static CYTHON_INLINE PyObject* __Pyx_decode_cpp_string( - std::string cppstring, Py_ssize_t start, Py_ssize_t stop, + std::string_view cppstring, Py_ssize_t start, Py_ssize_t stop, const char* encoding, const char* errors, PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { return __Pyx_decode_c_bytes( --- contrib/tools/cython_py2/Cython/Utils.py (index) +++ contrib/tools/cython_py2/Cython/Utils.py (working tree) @@ -172,15 +172,15 @@ def castrate_file(path, st): os.utime(path, (st.st_atime, st.st_mtime-1)) -def is_cython_generated_file(path, allow_failed=False, if_not_found=True): - failure_marker = b"#error Do not use this file, it is the result of a failed Cython compilation." +def is_cython_generated_file(path, allow_failed=False, if_not_found=True, is_standalone=False): + failure_marker = "#error Do not use this file, it is the result of a failed Cython compilation." file_content = None - if os.path.exists(path): - try: - with open(path, "rb") as f: - file_content = f.read(len(failure_marker)) - except (OSError, IOError): - pass # Probably just doesn't exist any more + + try: + with OpenFile(path, is_standalone=is_standalone) as f: + file_content = f.read(len(failure_marker)) + except (OSError, IOError): + pass # Probably just doesn't exist any more if file_content is None: # file does not exist (yet) @@ -188,7 +188,7 @@ def is_cython_generated_file(path, allow_failed=False, if_not_found=True): return ( # Cython C file? - file_content.startswith(b"/* Generated by Cython ") or + file_content.startswith("/* Generated by Cython ") or # Cython output file after previous failures? (allow_failed and file_content == failure_marker) or # Let's allow overwriting empty files as well. They might have resulted from previous failures. @@ -719,3 +719,48 @@ def normalise_float_repr(float_str): ).rstrip('0') return result if result != '.' else '.0' + + +# ========================== Arcadia specific ================================= + +class OpenFile(object): + + def __init__(self, filename, mode='r', is_standalone=False): + assert 'r' in mode, ('Read-only', mode) + self.filename = filename + self.mode = mode + self.is_standalone = is_standalone + self.file = None + self.build_root = os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT') + + def __enter__(self): + # See redefined _find_c_source() description for more info + if self.build_root: + self.file = open(os.path.join(self.build_root, self.filename), self.mode) + return self.file + elif self.is_standalone: + import library.python.resource + from six import StringIO + + content = library.python.resource.resfs_read(self.filename, builtin=True) + assert content, (self.filename, os.environ.items()) + return StringIO(content.decode()) + else: + self.file = open(self.filename, self.mode) + return self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.file: + self.file.close() + + +def generated_file_exists(filename, is_standalone=False): + build_root = os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT') + if build_root: + filename = os.path.join(build_root, filename) + elif is_standalone: + import library.python.resource + return library.python.resource.resfs_src(filename, resfs_file=True) + + return os.path.exists(filename) + --- contrib/tools/cython_py2/cython.py (index) +++ contrib/tools/cython_py2/cython.py (working tree) @@ -1,5 +1,7 @@ #!/usr/bin/env python +# Change content of this file to change uids for cython programs - cython + # # Cython -- Main Program, generic # @@ -8,6 +10,7 @@ if __name__ == '__main__': import os import sys + sys.dont_write_bytecode = True # Make sure we import the right Cython cythonpath, _ = os.path.split(os.path.realpath(__file__))