add library/cpp/actors, ymake build to ydb oss export

1 files changed, 282 insertions, 0 deletions

diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py
new file mode 100644
index 0000000000..94491d9256
--- /dev/null
+++ b/build/scripts/coverage-info.py
@@ -0,0 +1,282 @@
+import argparse
+import os
+import sys
+import tarfile
+import collections
+import subprocess
+import re
+
+
+GCDA_EXT = '.gcda'
+GCNO_EXT = '.gcno'
+
+
+def suffixes(path):
+    """
+    >>> list(suffixes('/a/b/c'))
+    ['c', 'b/c', '/a/b/c']
+    >>> list(suffixes('/a/b/c/'))
+    ['c', 'b/c', '/a/b/c']
+    >>> list(suffixes('/a'))
+    ['/a']
+    >>> list(suffixes('/a/'))
+    ['/a']
+    >>> list(suffixes('/'))
+    []
+    """
+    path = os.path.normpath(path)
+
+    def up_dirs(cur_path):
+        while os.path.dirname(cur_path) != cur_path:
+            cur_path = os.path.dirname(cur_path)
+            yield cur_path
+
+    for x in up_dirs(path):
+        yield path.replace(x + os.path.sep, '')
+
+
+def recast(in_file, out_file, probe_path, update_stat):
+    PREFIX = 'SF:'
+
+    probed_path = None
+
+    any_payload = False
+
+    with open(in_file, 'r') as input, open(out_file, 'w') as output:
+        active = True
+        for line in input:
+            line = line.rstrip('\n')
+            if line.startswith('TN:'):
+                output.write(line + '\n')
+            elif line.startswith(PREFIX):
+                path = line[len(PREFIX):]
+                probed_path = probe_path(path)
+                if probed_path:
+                    output.write(PREFIX + probed_path + '\n')
+                active = bool(probed_path)
+            else:
+                if active:
+                    update_stat(probed_path, line)
+                    output.write(line + '\n')
+                    any_payload = True
+
+    return any_payload
+
+
+def print_stat(da, fnda, teamcity_stat_output):
+    lines_hit = sum(map(bool, da.values()))
+    lines_total = len(da.values())
+    lines_coverage = 100.0 * lines_hit / lines_total if lines_total else 0
+
+    func_hit = sum(map(bool, fnda.values()))
+    func_total = len(fnda.values())
+    func_coverage = 100.0 * func_hit / func_total if func_total else 0
+
+    print >>sys.stderr, '[[imp]]Lines[[rst]]     {: >16} {: >16} {: >16.1f}%'.format(lines_hit, lines_total, lines_coverage)
+    print >>sys.stderr, '[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format(func_hit, func_total, func_coverage)
+
+    if teamcity_stat_output:
+        with open(teamcity_stat_output, 'w') as tc_file:
+            tc_file.write("##teamcity[blockOpened name='Code Coverage Summary']\n")
+            tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLTotal\' value='{}']\n".format(lines_total))
+            tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLCovered\' value='{}']\n".format(lines_hit))
+            tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMTotal\' value='{}']\n".format(func_total))
+            tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMCovered\' value='{}']\n".format(func_hit))
+            tc_file.write("##teamcity[blockClosed name='Code Coverage Summary']\n")
+
+
+def chunks(l, n):
+    """
+    >>> list(chunks(range(10), 3))
+    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
+    >>> list(chunks(range(10), 5))
+    [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
+    """
+    for i in xrange(0, len(l), n):
+        yield l[i:i + n]
+
+
+def combine_info_files(lcov, files, out_file):
+    chunk_size = 50
+    files = list(set(files))
+
+    for chunk in chunks(files, chunk_size):
+        combine_cmd = [lcov]
+        if os.path.exists(out_file):
+            chunk.append(out_file)
+        for trace in chunk:
+            assert os.path.exists(trace), "Trace file does not exist: {} (cwd={})".format(trace, os.getcwd())
+            combine_cmd += ["-a", os.path.abspath(trace)]
+        print >>sys.stderr, '## lcov', ' '.join(combine_cmd[1:])
+        out_file_tmp = "combined.tmp"
+        with open(out_file_tmp, "w") as stdout:
+            subprocess.check_call(combine_cmd, stdout=stdout)
+        if os.path.exists(out_file):
+            os.remove(out_file)
+        os.rename(out_file_tmp, out_file)
+
+
+def probe_path_global(path, source_root, prefix_filter, exclude_files):
+    if path.endswith('_ut.cpp'):
+        return None
+
+    for suff in reversed(list(suffixes(path))):
+        if (not prefix_filter or suff.startswith(prefix_filter)) and (not exclude_files or not exclude_files.match(suff)):
+            full_path = source_root + os.sep + suff
+            if os.path.isfile(full_path):
+                return full_path
+
+    return None
+
+
+def update_stat_global(src_file, line, fnda, da):
+    if line.startswith("FNDA:"):
+        visits, func_name = line[len("FNDA:"):].split(',')
+        fnda[src_file + func_name] += int(visits)
+
+    if line.startswith("DA"):
+        line_number, visits = line[len("DA:"):].split(',')
+        if visits == '=====':
+            visits = 0
+
+        da[src_file + line_number] += int(visits)
+
+
+def gen_info_global(cmd, cov_info, probe_path, update_stat, lcov_args):
+    print >>sys.stderr, '## geninfo', ' '.join(cmd)
+    subprocess.check_call(cmd)
+    if recast(cov_info + '.tmp', cov_info, probe_path, update_stat):
+        lcov_args.append(cov_info)
+
+
+def init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info, prefix_filter, exclude_files):
+    with tarfile.open(gcno_archive) as gcno_tf:
+        for gcno_item in gcno_tf:
+            if gcno_item.isfile() and gcno_item.name.endswith(GCNO_EXT):
+                gcno_tf.extract(gcno_item)
+
+                gcno_name = gcno_item.name
+                source_fname = gcno_name[:-len(GCNO_EXT)]
+                if prefix_filter and not source_fname.startswith(prefix_filter):
+                    sys.stderr.write("Skipping {} (doesn't match prefix '{}')\n".format(source_fname, prefix_filter))
+                    continue
+                if exclude_files and exclude_files.search(source_fname):
+                    sys.stderr.write("Skipping {} (matched exclude pattern '{}')\n".format(source_fname, exclude_files.pattern))
+                    continue
+
+                fname2gcno[source_fname] = gcno_name
+
+                if os.path.getsize(gcno_name) > 0:
+                    coverage_info = source_fname + '.' + str(len(fname2info[source_fname])) + '.info'
+                    fname2info[source_fname].append(coverage_info)
+                    geninfo_cmd = [
+                        geninfo_executable,
+                        '--gcov-tool', gcov_tool,
+                        '-i', gcno_name,
+                        '-o', coverage_info + '.tmp'
+                    ]
+                    gen_info(geninfo_cmd, coverage_info)
+
+
+def process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info):
+    with tarfile.open(gcda_archive) as gcda_tf:
+        for gcda_item in gcda_tf:
+            if gcda_item.isfile() and gcda_item.name.endswith(GCDA_EXT):
+                gcda_name = gcda_item.name
+                source_fname = gcda_name[:-len(GCDA_EXT)]
+                for suff in suffixes(source_fname):
+                    if suff in fname2gcno:
+                        gcda_new_name = suff + GCDA_EXT
+                        gcda_item.name = gcda_new_name
+                        gcda_tf.extract(gcda_item)
+                        if os.path.getsize(gcda_new_name) > 0:
+                            coverage_info = suff + '.' + str(len(fname2info[suff])) + '.info'
+                            fname2info[suff].append(coverage_info)
+                            geninfo_cmd = [
+                                geninfo_executable,
+                                '--gcov-tool', gcov_tool,
+                                gcda_new_name,
+                                '-o', coverage_info + '.tmp'
+                            ]
+                            gen_info(geninfo_cmd, coverage_info)
+
+
+def gen_cobertura(tool, output, combined_info):
+    cmd = [
+        tool,
+        combined_info,
+        '-b', '#hamster#',
+        '-o', output
+    ]
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    if p.returncode:
+        raise Exception('lcov_cobertura failed with exit code {}\nstdout: {}\nstderr: {}'.format(p.returncode, out, err))
+
+
+def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filter, exclude_regexp, teamcity_stat_output, coverage_report_path, gcov_report, lcov_cobertura):
+    exclude_files = re.compile(exclude_regexp) if exclude_regexp else None
+
+    fname2gcno = {}
+    fname2info = collections.defaultdict(list)
+    lcov_args = []
+    geninfo_executable = os.path.join(source_root, 'devtools', 'lcov', 'geninfo')
+
+    def probe_path(path):
+        return probe_path_global(path, source_root, prefix_filter, exclude_files)
+
+    fnda = collections.defaultdict(int)
+    da = collections.defaultdict(int)
+
+    def update_stat(src_file, line):
+        update_stat_global(src_file, line, da, fnda)
+
+    def gen_info(cmd, cov_info):
+        gen_info_global(cmd, cov_info, probe_path, update_stat, lcov_args)
+
+    init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info, prefix_filter, exclude_files)
+    process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info)
+
+    if coverage_report_path:
+        output_dir = coverage_report_path
+    else:
+        output_dir = output + '.dir'
+
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    teamcity_stat_file = None
+    if teamcity_stat_output:
+        teamcity_stat_file = os.path.join(output_dir, 'teamcity.out')
+    print_stat(da, fnda, teamcity_stat_file)
+
+    if lcov_args:
+        output_trace = "combined.info"
+        combine_info_files(os.path.join(source_root, 'devtools', 'lcov', 'lcov'), lcov_args, output_trace)
+        cmd = [os.path.join(source_root, 'devtools', 'lcov', 'genhtml'), '-p', source_root, '--ignore-errors', 'source', '-o', output_dir, output_trace]
+        print >>sys.stderr, '## genhtml', ' '.join(cmd)
+        subprocess.check_call(cmd)
+        if lcov_cobertura:
+            gen_cobertura(lcov_cobertura, gcov_report, output_trace)
+
+    with tarfile.open(output, 'w') as tar:
+        tar.add(output_dir, arcname='.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--source-root', action='store')
+    parser.add_argument('--output', action='store')
+    parser.add_argument('--gcno-archive', action='store')
+    parser.add_argument('--gcda-archive', action='store')
+    parser.add_argument('--gcov-tool', action='store')
+    parser.add_argument('--prefix-filter', action='store')
+    parser.add_argument('--exclude-regexp', action='store')
+    parser.add_argument('--teamcity-stat-output', action='store_const', const=True)
+    parser.add_argument('--coverage-report-path', action='store')
+    parser.add_argument('--gcov-report', action='store')
+    parser.add_argument('--lcov-cobertura', action='store')
+
+    args = parser.parse_args()
+    main(**vars(args))