diff options
author | orivej <orivej@yandex-team.ru> | 2022-02-10 16:45:01 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:01 +0300 |
commit | 2d37894b1b037cf24231090eda8589bbb44fb6fc (patch) | |
tree | be835aa92c6248212e705f25388ebafcf84bc7a1 /contrib/tools/cython/Cython/Coverage.py | |
parent | 718c552901d703c502ccbefdfc3c9028d608b947 (diff) | |
download | ydb-2d37894b1b037cf24231090eda8589bbb44fb6fc.tar.gz |
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Coverage.py')
-rw-r--r-- | contrib/tools/cython/Cython/Coverage.py | 630 |
1 files changed, 315 insertions, 315 deletions
diff --git a/contrib/tools/cython/Cython/Coverage.py b/contrib/tools/cython/Cython/Coverage.py index 1795790e42..5aa9df2ce0 100644 --- a/contrib/tools/cython/Cython/Coverage.py +++ b/contrib/tools/cython/Cython/Coverage.py @@ -1,181 +1,181 @@ -""" -A Cython plugin for coverage.py - -Requires the coverage package at least in version 4.0 (which added the plugin API). -""" - -from __future__ import absolute_import - -import re -import os.path -import sys -from collections import defaultdict - -from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ +""" +A Cython plugin for coverage.py + +Requires the coverage package at least in version 4.0 (which added the plugin API). +""" + +from __future__ import absolute_import + +import re +import os.path +import sys +from collections import defaultdict + +from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ from coverage.files import canonical_filename - -from .Utils import find_root_package_dir, is_package_dir, open_source_file - - -from . import __version__ - - -C_FILE_EXTENSIONS = ['.c', '.cpp', '.cc', '.cxx'] -MODULE_FILE_EXTENSIONS = set(['.py', '.pyx', '.pxd'] + C_FILE_EXTENSIONS) - - -def _find_c_source(base_path): - file_exists = os.path.exists - for ext in C_FILE_EXTENSIONS: - file_name = base_path + ext - if file_exists(file_name): - return file_name - return None - - + +from .Utils import find_root_package_dir, is_package_dir, open_source_file + + +from . import __version__ + + +C_FILE_EXTENSIONS = ['.c', '.cpp', '.cc', '.cxx'] +MODULE_FILE_EXTENSIONS = set(['.py', '.pyx', '.pxd'] + C_FILE_EXTENSIONS) + + +def _find_c_source(base_path): + file_exists = os.path.exists + for ext in C_FILE_EXTENSIONS: + file_name = base_path + ext + if file_exists(file_name): + return file_name + return None + + def _find_dep_file_path(main_file, file_path, relative_path_search=False): - abs_path = os.path.abspath(file_path) + abs_path = os.path.abspath(file_path) if not os.path.exists(abs_path) and (file_path.endswith('.pxi') or relative_path_search): # files are looked up relative to the main source file rel_file_path = os.path.join(os.path.dirname(main_file), file_path) if os.path.exists(rel_file_path): abs_path = os.path.abspath(rel_file_path) - # search sys.path for external locations if a valid file hasn't been found - if not os.path.exists(abs_path): - for sys_path in sys.path: - test_path = os.path.realpath(os.path.join(sys_path, file_path)) - if os.path.exists(test_path): + # search sys.path for external locations if a valid file hasn't been found + if not os.path.exists(abs_path): + for sys_path in sys.path: + test_path = os.path.realpath(os.path.join(sys_path, file_path)) + if os.path.exists(test_path): return canonical_filename(test_path) return canonical_filename(abs_path) - - -class Plugin(CoveragePlugin): - # map from traced file paths to absolute file paths - _file_path_map = None - # map from traced file paths to corresponding C files - _c_files_map = None - # map from parsed C files to their content - _parsed_c_files = None - - def sys_info(self): - return [('Cython version', __version__)] - - def file_tracer(self, filename): - """ - Try to find a C source file for a file path found by the tracer. - """ + + +class Plugin(CoveragePlugin): + # map from traced file paths to absolute file paths + _file_path_map = None + # map from traced file paths to corresponding C files + _c_files_map = None + # map from parsed C files to their content + _parsed_c_files = None + + def sys_info(self): + return [('Cython version', __version__)] + + def file_tracer(self, filename): + """ + Try to find a C source file for a file path found by the tracer. + """ # TODO We need to pxd-files to the include map. For more info see pybuild.py # Currently skip such files, because they are not supported in Arcadia pybuild with coverage. if os.path.splitext(filename)[-1] not in ('.pyx', '.pxi'): return None - if filename.startswith('<') or filename.startswith('memory:'): - return None - c_file = py_file = None + if filename.startswith('<') or filename.startswith('memory:'): + return None + c_file = py_file = None filename = canonical_filename(filename) - if self._c_files_map and filename in self._c_files_map: - c_file = self._c_files_map[filename][0] - - if c_file is None: - c_file, py_file = self._find_source_files(filename) - if not c_file: - return None # unknown file - - # parse all source file paths and lines from C file - # to learn about all relevant source files right away (pyx/pxi/pxd) - # FIXME: this might already be too late if the first executed line - # is not from the main .pyx file but a file with a different - # name than the .c file (which prevents us from finding the - # .c file) - _, code = self._read_source_lines(c_file, filename) - if code is None: - return None # no source found - - if self._file_path_map is None: - self._file_path_map = {} - return CythonModuleTracer(filename, py_file, c_file, self._c_files_map, self._file_path_map) - - def file_reporter(self, filename): - # TODO: let coverage.py handle .py files itself - #ext = os.path.splitext(filename)[1].lower() - #if ext == '.py': - # from coverage.python import PythonFileReporter - # return PythonFileReporter(filename) - + if self._c_files_map and filename in self._c_files_map: + c_file = self._c_files_map[filename][0] + + if c_file is None: + c_file, py_file = self._find_source_files(filename) + if not c_file: + return None # unknown file + + # parse all source file paths and lines from C file + # to learn about all relevant source files right away (pyx/pxi/pxd) + # FIXME: this might already be too late if the first executed line + # is not from the main .pyx file but a file with a different + # name than the .c file (which prevents us from finding the + # .c file) + _, code = self._read_source_lines(c_file, filename) + if code is None: + return None # no source found + + if self._file_path_map is None: + self._file_path_map = {} + return CythonModuleTracer(filename, py_file, c_file, self._c_files_map, self._file_path_map) + + def file_reporter(self, filename): + # TODO: let coverage.py handle .py files itself + #ext = os.path.splitext(filename)[1].lower() + #if ext == '.py': + # from coverage.python import PythonFileReporter + # return PythonFileReporter(filename) + filename = canonical_filename(filename) - if self._c_files_map and filename in self._c_files_map: - c_file, rel_file_path, code = self._c_files_map[filename] - else: - c_file, _ = self._find_source_files(filename) - if not c_file: + if self._c_files_map and filename in self._c_files_map: + c_file, rel_file_path, code = self._c_files_map[filename] + else: + c_file, _ = self._find_source_files(filename) + if not c_file: if standalone(): raise AssertionError(filename) - return None # unknown file - rel_file_path, code = self._read_source_lines(c_file, filename) + return None # unknown file + rel_file_path, code = self._read_source_lines(c_file, filename) if code is None: if standalone(): raise AssertionError(filename) return None # no source found - return CythonModuleReporter(c_file, filename, rel_file_path, code) - - def _find_source_files(self, filename): - basename, ext = os.path.splitext(filename) - ext = ext.lower() - if ext in MODULE_FILE_EXTENSIONS: - pass + return CythonModuleReporter(c_file, filename, rel_file_path, code) + + def _find_source_files(self, filename): + basename, ext = os.path.splitext(filename) + ext = ext.lower() + if ext in MODULE_FILE_EXTENSIONS: + pass elif ext == '.pyd': # Windows extension module platform_suffix = re.search(r'[.]cp[0-9]+-win[_a-z0-9]*$', basename, re.I) - if platform_suffix: - basename = basename[:platform_suffix.start()] + if platform_suffix: + basename = basename[:platform_suffix.start()] elif ext == '.so': # Linux/Unix/Mac extension module platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[-_a-z0-9]*$', basename, re.I) if platform_suffix: basename = basename[:platform_suffix.start()] - elif ext == '.pxi': - # if we get here, it means that the first traced line of a Cython module was - # not in the main module but in an include file, so try a little harder to - # find the main source file - self._find_c_source_files(os.path.dirname(filename), filename) - if filename in self._c_files_map: - return self._c_files_map[filename][0], None + elif ext == '.pxi': + # if we get here, it means that the first traced line of a Cython module was + # not in the main module but in an include file, so try a little harder to + # find the main source file + self._find_c_source_files(os.path.dirname(filename), filename) + if filename in self._c_files_map: + return self._c_files_map[filename][0], None if standalone(): raise AssertionError(filename) - else: - # none of our business - return None, None - - c_file = filename if ext in C_FILE_EXTENSIONS else _find_c_source(basename) - if c_file is None: - # a module "pkg/mod.so" can have a source file "pkg/pkg.mod.c" - package_root = find_root_package_dir.uncached(filename) - package_path = os.path.relpath(basename, package_root).split(os.path.sep) - if len(package_path) > 1: - test_basepath = os.path.join(os.path.dirname(filename), '.'.join(package_path)) - c_file = _find_c_source(test_basepath) - - py_source_file = None - if c_file: - py_source_file = os.path.splitext(c_file)[0] + '.py' - if not os.path.exists(py_source_file): - py_source_file = None - - try: + else: + # none of our business + return None, None + + c_file = filename if ext in C_FILE_EXTENSIONS else _find_c_source(basename) + if c_file is None: + # a module "pkg/mod.so" can have a source file "pkg/pkg.mod.c" + package_root = find_root_package_dir.uncached(filename) + package_path = os.path.relpath(basename, package_root).split(os.path.sep) + if len(package_path) > 1: + test_basepath = os.path.join(os.path.dirname(filename), '.'.join(package_path)) + c_file = _find_c_source(test_basepath) + + py_source_file = None + if c_file: + py_source_file = os.path.splitext(c_file)[0] + '.py' + if not os.path.exists(py_source_file): + py_source_file = None + + try: with OpenFile(c_file) as f: if '/* Generated by Cython ' not in f.read(30): - return None, None # not a Cython file - except (IOError, OSError): - c_file = None - - return c_file, py_source_file - - def _find_c_source_files(self, dir_path, source_file): - """ - Desperately parse all C files in the directory or its package parents - (not re-descending) to find the (included) source file in one of them. - """ + return None, None # not a Cython file + except (IOError, OSError): + c_file = None + + return c_file, py_source_file + + def _find_c_source_files(self, dir_path, source_file): + """ + Desperately parse all C files in the directory or its package parents + (not re-descending) to find the (included) source file in one of them. + """ if standalone(): if os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'): broot = os.environ['PYTHON_COVERAGE_CYTHON_BUILD_ROOT'] @@ -190,188 +190,188 @@ class Plugin(CoveragePlugin): return raise AssertionError((source_file, os.environ.get('PYTHON_COVERAGE_CYTHON_BUILD_ROOT'))) - if not os.path.isdir(dir_path): - return - splitext = os.path.splitext - for filename in os.listdir(dir_path): - ext = splitext(filename)[1].lower() - if ext in C_FILE_EXTENSIONS: - self._read_source_lines(os.path.join(dir_path, filename), source_file) - if source_file in self._c_files_map: - return - # not found? then try one package up - if is_package_dir(dir_path): - self._find_c_source_files(os.path.dirname(dir_path), source_file) - - def _read_source_lines(self, c_file, sourcefile): - """ - Parse a Cython generated C/C++ source file and find the executable lines. - Each executable line starts with a comment header that states source file - and line number, as well as the surrounding range of source code lines. - """ - if self._parsed_c_files is None: - self._parsed_c_files = {} - if c_file in self._parsed_c_files: - code_lines = self._parsed_c_files[c_file] - else: - code_lines = self._parse_cfile_lines(c_file) - self._parsed_c_files[c_file] = code_lines - - if self._c_files_map is None: - self._c_files_map = {} - - for filename, code in code_lines.items(): + if not os.path.isdir(dir_path): + return + splitext = os.path.splitext + for filename in os.listdir(dir_path): + ext = splitext(filename)[1].lower() + if ext in C_FILE_EXTENSIONS: + self._read_source_lines(os.path.join(dir_path, filename), source_file) + if source_file in self._c_files_map: + return + # not found? then try one package up + if is_package_dir(dir_path): + self._find_c_source_files(os.path.dirname(dir_path), source_file) + + def _read_source_lines(self, c_file, sourcefile): + """ + Parse a Cython generated C/C++ source file and find the executable lines. + Each executable line starts with a comment header that states source file + and line number, as well as the surrounding range of source code lines. + """ + if self._parsed_c_files is None: + self._parsed_c_files = {} + if c_file in self._parsed_c_files: + code_lines = self._parsed_c_files[c_file] + else: + code_lines = self._parse_cfile_lines(c_file) + self._parsed_c_files[c_file] = code_lines + + if self._c_files_map is None: + self._c_files_map = {} + + for filename, code in code_lines.items(): abs_path = _find_dep_file_path(c_file, filename, relative_path_search=True) - self._c_files_map[abs_path] = (c_file, filename, code) - - if sourcefile not in self._c_files_map: - return (None,) * 2 # e.g. shared library file - return self._c_files_map[sourcefile][1:] - - def _parse_cfile_lines(self, c_file): - """ - Parse a C file and extract all source file lines that generated executable code. - """ - match_source_path_line = re.compile(r' */[*] +"(.*)":([0-9]+)$').match - match_current_code_line = re.compile(r' *[*] (.*) # <<<<<<+$').match - match_comment_end = re.compile(r' *[*]/$').match - match_trace_line = re.compile(r' *__Pyx_TraceLine\(([0-9]+),').match - not_executable = re.compile( - r'\s*c(?:type)?def\s+' - r'(?:(?:public|external)\s+)?' - r'(?:struct|union|enum|class)' - r'(\s+[^:]+|)\s*:' - ).match - - code_lines = defaultdict(dict) - executable_lines = defaultdict(set) - current_filename = None - + self._c_files_map[abs_path] = (c_file, filename, code) + + if sourcefile not in self._c_files_map: + return (None,) * 2 # e.g. shared library file + return self._c_files_map[sourcefile][1:] + + def _parse_cfile_lines(self, c_file): + """ + Parse a C file and extract all source file lines that generated executable code. + """ + match_source_path_line = re.compile(r' */[*] +"(.*)":([0-9]+)$').match + match_current_code_line = re.compile(r' *[*] (.*) # <<<<<<+$').match + match_comment_end = re.compile(r' *[*]/$').match + match_trace_line = re.compile(r' *__Pyx_TraceLine\(([0-9]+),').match + not_executable = re.compile( + r'\s*c(?:type)?def\s+' + r'(?:(?:public|external)\s+)?' + r'(?:struct|union|enum|class)' + r'(\s+[^:]+|)\s*:' + ).match + + code_lines = defaultdict(dict) + executable_lines = defaultdict(set) + current_filename = None + with OpenFile(c_file) as lines: - lines = iter(lines) - for line in lines: - match = match_source_path_line(line) - if not match: - if '__Pyx_TraceLine(' in line and current_filename is not None: - trace_line = match_trace_line(line) - if trace_line: - executable_lines[current_filename].add(int(trace_line.group(1))) - continue - filename, lineno = match.groups() - current_filename = filename - lineno = int(lineno) - for comment_line in lines: - match = match_current_code_line(comment_line) - if match: - code_line = match.group(1).rstrip() - if not_executable(code_line): - break - code_lines[filename][lineno] = code_line - break - elif match_comment_end(comment_line): - # unexpected comment format - false positive? - break - - # Remove lines that generated code but are not traceable. - for filename, lines in code_lines.items(): - dead_lines = set(lines).difference(executable_lines.get(filename, ())) - for lineno in dead_lines: - del lines[lineno] - return code_lines - - -class CythonModuleTracer(FileTracer): - """ - Find the Python/Cython source file for a Cython module. - """ - def __init__(self, module_file, py_file, c_file, c_files_map, file_path_map): - super(CythonModuleTracer, self).__init__() - self.module_file = module_file - self.py_file = py_file - self.c_file = c_file - self._c_files_map = c_files_map - self._file_path_map = file_path_map - - def has_dynamic_source_filename(self): - return True - - def dynamic_source_filename(self, filename, frame): - """ - Determine source file path. Called by the function call tracer. - """ - source_file = frame.f_code.co_filename - try: - return self._file_path_map[source_file] - except KeyError: - pass + lines = iter(lines) + for line in lines: + match = match_source_path_line(line) + if not match: + if '__Pyx_TraceLine(' in line and current_filename is not None: + trace_line = match_trace_line(line) + if trace_line: + executable_lines[current_filename].add(int(trace_line.group(1))) + continue + filename, lineno = match.groups() + current_filename = filename + lineno = int(lineno) + for comment_line in lines: + match = match_current_code_line(comment_line) + if match: + code_line = match.group(1).rstrip() + if not_executable(code_line): + break + code_lines[filename][lineno] = code_line + break + elif match_comment_end(comment_line): + # unexpected comment format - false positive? + break + + # Remove lines that generated code but are not traceable. + for filename, lines in code_lines.items(): + dead_lines = set(lines).difference(executable_lines.get(filename, ())) + for lineno in dead_lines: + del lines[lineno] + return code_lines + + +class CythonModuleTracer(FileTracer): + """ + Find the Python/Cython source file for a Cython module. + """ + def __init__(self, module_file, py_file, c_file, c_files_map, file_path_map): + super(CythonModuleTracer, self).__init__() + self.module_file = module_file + self.py_file = py_file + self.c_file = c_file + self._c_files_map = c_files_map + self._file_path_map = file_path_map + + def has_dynamic_source_filename(self): + return True + + def dynamic_source_filename(self, filename, frame): + """ + Determine source file path. Called by the function call tracer. + """ + source_file = frame.f_code.co_filename + try: + return self._file_path_map[source_file] + except KeyError: + pass if standalone(): abs_path = self.module_file else: abs_path = _find_dep_file_path(filename, source_file) - - if self.py_file and source_file[-3:].lower() == '.py': - # always let coverage.py handle this case itself - self._file_path_map[source_file] = self.py_file - return self.py_file - - assert self._c_files_map is not None - if abs_path not in self._c_files_map: - self._c_files_map[abs_path] = (self.c_file, source_file, None) - self._file_path_map[source_file] = abs_path - return abs_path - - -class CythonModuleReporter(FileReporter): - """ - Provide detailed trace information for one source file to coverage.py. - """ - def __init__(self, c_file, source_file, rel_file_path, code): - super(CythonModuleReporter, self).__init__(source_file) - self.name = rel_file_path - self.c_file = c_file - self._code = code + + if self.py_file and source_file[-3:].lower() == '.py': + # always let coverage.py handle this case itself + self._file_path_map[source_file] = self.py_file + return self.py_file + + assert self._c_files_map is not None + if abs_path not in self._c_files_map: + self._c_files_map[abs_path] = (self.c_file, source_file, None) + self._file_path_map[source_file] = abs_path + return abs_path + + +class CythonModuleReporter(FileReporter): + """ + Provide detailed trace information for one source file to coverage.py. + """ + def __init__(self, c_file, source_file, rel_file_path, code): + super(CythonModuleReporter, self).__init__(source_file) + self.name = rel_file_path + self.c_file = c_file + self._code = code self._abs_filename = self._find_abs_filename() - - def lines(self): - """ - Return set of line numbers that are possibly executable. - """ - return set(self._code) - - def _iter_source_tokens(self): - current_line = 1 - for line_no, code_line in sorted(self._code.items()): - while line_no > current_line: - yield [] - current_line += 1 - yield [('txt', code_line)] - current_line += 1 - - def source(self): - """ - Return the source code of the file as a string. - """ + + def lines(self): + """ + Return set of line numbers that are possibly executable. + """ + return set(self._code) + + def _iter_source_tokens(self): + current_line = 1 + for line_no, code_line in sorted(self._code.items()): + while line_no > current_line: + yield [] + current_line += 1 + yield [('txt', code_line)] + current_line += 1 + + def source(self): + """ + Return the source code of the file as a string. + """ if os.path.exists(self._abs_filename): with open_source_file(self._abs_filename) as f: - return f.read() - else: - return '\n'.join( - (tokens[0][1] if tokens else '') - for tokens in self._iter_source_tokens()) - - def source_token_lines(self): - """ - Iterate over the source code tokens. - """ + return f.read() + else: + return '\n'.join( + (tokens[0][1] if tokens else '') + for tokens in self._iter_source_tokens()) + + def source_token_lines(self): + """ + Iterate over the source code tokens. + """ if os.path.exists(self._abs_filename): with open_source_file(self._abs_filename) as f: - for line in f: - yield [('txt', line.rstrip('\n'))] - else: - for line in self._iter_source_tokens(): + for line in f: + yield [('txt', line.rstrip('\n'))] + else: + for line in self._iter_source_tokens(): yield line - + def _find_abs_filename(self): for root in [ os.environ.get('PYTHON_COVERAGE_ARCADIA_SOURCE_ROOT'), @@ -382,10 +382,10 @@ class CythonModuleReporter(FileReporter): if root and os.path.exists(abs_path): return abs_path return self.filename - -def coverage_init(reg, options): - reg.add_file_tracer(Plugin()) + +def coverage_init(reg, options): + reg.add_file_tracer(Plugin()) # ========================== Arcadia specific ================================= |