ci: summary extraction as independent step, add functional tests summary

ci: summary extraction as independent step, add functional tests summary

Pull Request resolved: #264

7 files changed, 435 insertions, 166 deletions

diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index 090afdadac..8fc026a808 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -136,7 +136,7 @@ runs:
       ls -la ${{steps.init.outputs.artifactsdir}}/reports.tar.gz
       echo "[Unittest/CTest XML reports archive](${{steps.init.outputs.logurlprefix}}/reports.tar.gz)" >> $GITHUB_STEP_SUMMARY
 
-  - name: postprocess junit reports
+  - name: postprocess xml reports
     if: inputs.run_unit_tests == 'true'
     shell: bash
     run: |
@@ -158,24 +158,30 @@ runs:
 
       echo "::endgroup::"
 
-  - name: extract log output
-    if: inputs.run_unit_tests == 'true'
-    shell: bash
-    run: |
+      echo "::group::extract-logs"
+
       mkdir ${{steps.init.outputs.artifactsdir}}/logs/
 
-      .github/scripts/tests/extract-logs.py \
-        --write-summary \
+      .github/scripts/tests/attach-logs.py \
         --url-prefix ${{steps.init.outputs.logurlprefix}}/logs/ \
         --filter-shard-file ${{steps.init.outputs.testshardfilterfile}} \
         --filter-test-file ${{steps.init.outputs.testfilterfile}} \
-        --patch-jsuite \
         --ctest-report $TESTREPDIR/suites/ctest_report.xml \
         --junit-reports-path $TESTREPDIR/unittests/ \
         --decompress \
         ${{steps.init.outputs.artifactsdir}}/${{steps.init.outputs.logfilename}} \
         ${{steps.init.outputs.artifactsdir}}/logs/
 
+      echo "::endgroup::"
+
+  - name: write unittests summary
+    if: inputs.run_unit_tests == 'true'
+    shell: bash
+    run: |
+      .github/scripts/tests/generate-summary.py -t "#### CTest run test shard failures" $TESTREPDIR/suites/ctest_report.xml
+      .github/scripts/tests/generate-summary.py -t "#### Unittest failures" $TESTREPDIR/unittests/
+
+
   - name: Unit test history upload results
     if: always() && inputs.run_unit_tests == 'true' && inputs.testman_token
     shell: bash
@@ -217,6 +223,12 @@ runs:
         grep -E '(FAILED|ERROR)\s*\[.*\]' | \
         tee $WORKDIR/pytest-short.log
 
+  - name: write functional tests summary
+    if: always() && inputs.run_functional_tests == 'true'
+    shell: bash
+    run: |
+      .github/scripts/tests/generate-summary.py -t "#### Functional tests failures" $PYTESTREPDIR
+
   - name: Functional tests history upload results
     if: always() && inputs.run_functional_tests == 'true' && inputs.testman_token
     shell: bash
diff --git a/.github/scripts/tests/attach-logs.py b/.github/scripts/tests/attach-logs.py
new file mode 100755
index 0000000000..fd8768c075
--- /dev/null
+++ b/.github/scripts/tests/attach-logs.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+import argparse
+import io
+import os
+import glob
+import re
+from xml.etree import ElementTree as ET
+from pathlib import Path
+from typing import List
+from log_parser import ctest_log_parser, parse_yunit_fails, parse_gtest_fails, log_reader
+from junit_utils import add_junit_log_property, create_error_testcase, create_error_testsuite, suite_case_iterator
+from ctest_utils import CTestLog
+
+fn_shard_part_re = re.compile(r"-\d+$")
+
+
+def make_filename(n, *parts):
+    fn = f'{"-".join(parts)}'
+
+    if n > 0:
+        fn = f"{fn}-{n}"
+
+    return f"{fn}.log"
+
+
+def save_log(err_lines: List[str], out_path: Path, *parts):
+    for x in range(128):
+        fn = make_filename(x, *parts)
+        print(f"save {fn} for {'::'.join(parts)}")
+        path = out_path.joinpath(fn)
+        try:
+            with open(path, "xt") as fp:
+                for line in err_lines:
+                    fp.write(f"{line}\n")
+        except FileExistsError:
+            pass
+        else:
+            return fn, path
+
+    raise Exception("Unable to create file")
+
+
+def extract_logs(log_fp: io.StringIO, out_path: Path, url_prefix):
+    # FIXME: memory inefficient because new buffer created every time
+
+    ctestlog = CTestLog()
+
+    for target, reason, ctest_buf in ctest_log_parser(log_fp):
+        fn, _ = save_log(ctest_buf, out_path, target)
+        log_url = f"{url_prefix}{fn}"
+
+        shard = ctestlog.add_shard(target, reason, log_url)
+
+        if not ctest_buf:
+            continue
+
+        first_line = ctest_buf[0]
+
+        if first_line.startswith("[==========]"):
+            for classname, method, err_lines in parse_gtest_fails(ctest_buf):
+                fn, path = save_log(err_lines, out_path, classname, method)
+                log_url = f"{url_prefix}{fn}"
+                shard.add_testcase(classname, method, path, log_url)
+        elif first_line.startswith("<-----"):
+            for classname, method, err_lines in parse_yunit_fails(ctest_buf):
+                fn, path = save_log(err_lines, out_path, classname, method)
+                log_url = f"{url_prefix}{fn}"
+                shard.add_testcase(classname, method, path, log_url)
+        else:
+            pass
+
+    return ctestlog
+
+
+def attach_to_ctest(ctest_log: CTestLog, ctest_path):
+    tree = ET.parse(ctest_path)
+    root = tree.getroot()
+    changed = False
+    for testcase in root.findall("testcase"):
+        name = testcase.attrib["classname"]
+        if ctest_log.has_error_shard(name):
+            add_junit_log_property(testcase, ctest_log.get_shard(name).log_url)
+            changed = True
+
+    if changed:
+        print(f"patch {ctest_path}")
+        tree.write(ctest_path, xml_declaration=True, encoding="UTF-8")
+
+
+def attach_to_unittests(ctest_log: CTestLog, unit_path):
+    all_found_tests = {}
+
+    for fn in glob.glob(os.path.join(unit_path, "*.xml")):
+        log_name = os.path.splitext(os.path.basename(fn))[0]
+        common_shard_name = fn_shard_part_re.sub("", log_name)
+        found_tests = all_found_tests.setdefault(common_shard_name, [])
+        tree = ET.parse(fn)
+        root = tree.getroot()
+        changed = False
+
+        for tsuite, tcase, cls, name in suite_case_iterator(root):
+            test_log = ctest_log.get_log(common_shard_name, cls, name)
+
+            if test_log is None:
+                continue
+
+            found_tests.append((cls, name))
+            add_junit_log_property(tcase, test_log.url)
+            changed = True
+
+        if changed:
+            print(f"patch {fn}")
+            tree.write(fn, xml_declaration=True, encoding="UTF-8")
+
+    for shard, found_tests in all_found_tests.items():
+        extra_logs = ctest_log.get_extra_tests(shard, found_tests)
+        if not extra_logs:
+            continue
+
+        fn = f"_{shard}_not_found.xml"
+        testcases = [create_error_testcase(t.shard.name, t.classname, t.method, t.fn, t.url) for t in extra_logs]
+
+        testsuite = create_error_testsuite(testcases)
+        testsuite.write(os.path.join(unit_path, fn), xml_declaration=True, encoding="UTF-8")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url-prefix", default="./")
+    parser.add_argument("--decompress", action="store_true", default=False, help="decompress ctest log")
+    parser.add_argument("--filter-test-file", required=False)
+    parser.add_argument("--filter-shard-file", required=False)
+    parser.add_argument("--ctest-report")
+    parser.add_argument("--junit-reports-path")
+    parser.add_argument("ctest_log")
+    parser.add_argument("out_log_dir")
+
+    args = parser.parse_args()
+
+    ctest_log = extract_logs(log_reader(args.ctest_log, args.decompress), Path(args.out_log_dir), args.url_prefix)
+
+    if ctest_log.has_logs:
+        attach_to_ctest(ctest_log, args.ctest_report)
+        attach_to_unittests(ctest_log, args.junit_reports_path)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/scripts/tests/ctest_utils.py b/.github/scripts/tests/ctest_utils.py
new file mode 100644
index 0000000000..c2f20eb9be
--- /dev/null
+++ b/.github/scripts/tests/ctest_utils.py
@@ -0,0 +1,87 @@
+import dataclasses
+import re
+from collections import defaultdict
+from typing import List, Dict, Tuple, Set
+
+
+shard_partition_re = re.compile(r"_\d+$")
+
+
+def get_common_shard_name(name):
+    return shard_partition_re.sub("", name)
+
+
+@dataclasses.dataclass
+class CTestTestcaseLog:
+    classname: str
+    method: str
+    fn: str
+    url: str
+    shard: "CTestLogShard"
+
+
+class CTestLogShard:
+    def __init__(self, name, status, log_url):
+        self.name = name
+        self.status = status
+        self.log_url = log_url
+        self.testcases: List[CTestTestcaseLog] = []
+        self.idx: Dict[Tuple[str, str], CTestTestcaseLog] = {}
+
+    def add_testcase(self, classname, method, fn, url):
+        log = CTestTestcaseLog(classname, method, fn, url, self)
+        self.testcases.append(log)
+        self.idx[(classname, method)] = log
+
+    def get_log(self, classname, method):
+        return self.idx.get((classname, method))
+
+    def get_extra_logs(self, found_testcases: Set[Tuple[str, str]]):
+        extra_keys = set(self.idx.keys()) - found_testcases
+        return [self.idx[k] for k in extra_keys]
+
+    @property
+    def logs(self):
+        return self.idx.values()
+
+    @property
+    def filename(self):
+        return f"{self.name}.xml"
+
+
+class CTestLog:
+    def __init__(self):
+        self.name_shard = {}  # type: Dict[str, CTestLogShard]
+        self.storage = defaultdict(dict)
+
+    def add_shard(self, name, status, log_url):
+        common_name = get_common_shard_name(name)
+        shard = self.storage[common_name][name] = self.name_shard[name] = CTestLogShard(name, status, log_url)
+        return shard
+
+    def has_error_shard(self, name):
+        return name in self.name_shard
+
+    def get_shard(self, name):
+        return self.name_shard[name]
+
+    def get_log(self, common_name, cls, name):
+        for shard in self.storage[common_name].values():
+            log = shard.get_log(cls, name)
+
+            if log:
+                return log
+
+        return None
+
+    def get_extra_tests(self, common_name, names):
+        result = []
+        for shard in self.storage[common_name].values():
+            extra = shard.get_extra_logs(set(names))
+            if extra:
+                result.extend(extra)
+        return result
+
+    @property
+    def has_logs(self):
+        return len(self.name_shard) > 0
diff --git a/.github/scripts/tests/extract-logs.py b/.github/scripts/tests/extract-logs.py
deleted file mode 100755
index 5493133f7e..0000000000
--- a/.github/scripts/tests/extract-logs.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import io
-import os
-import glob
-from xml.etree import ElementTree as ET
-from pathlib import Path
-from typing import List
-from log_parser import ctest_log_parser, parse_yunit_fails, parse_gtest_fails, log_reader
-from mute_utils import MutedTestCheck, MutedShardCheck
-from junit_utils import add_junit_log_property
-
-
-def make_filename(*parts):
-    return f'{"-".join(parts)}.log'
-
-
-def save_log(err_lines: List[str], out_path: Path, *parts):
-    fn = make_filename(*parts)
-    print(f"write {fn} for {'::'.join(parts)}")
-    with open(out_path.joinpath(fn), "wt") as fp:
-        for line in err_lines:
-            fp.write(f"{line}\n")
-
-    return fn
-
-
-def extract_logs(log_fp: io.StringIO, out_path: Path, url_prefix):
-    # FIXME: memory inefficient because new buffer created every time
-
-    log_urls = []
-    for target, reason, ctest_buf in ctest_log_parser(log_fp):
-        suite_summary = []
-
-        fn = save_log(ctest_buf, out_path, target)
-        log_url = f"{url_prefix}{fn}"
-
-        log_urls.append((target, reason, log_url, suite_summary))
-
-        if not ctest_buf:
-            continue
-
-        first_line = ctest_buf[0]
-        if first_line.startswith("[==========]"):
-            for classname, method, err in parse_gtest_fails(ctest_buf):
-                fn = save_log(err, out_path, classname, method)
-                log_url = f"{url_prefix}{fn}"
-                suite_summary.append((classname, method, log_url))
-        elif first_line.startswith("<-----"):
-            for classname, method, err in parse_yunit_fails(ctest_buf):
-                fn = save_log(err, out_path, classname, method)
-                log_url = f"{url_prefix}{fn}"
-                suite_summary.append((classname, method, log_url))
-        else:
-            pass
-
-    return log_urls
-
-
-def generate_summary(summary, is_mute_shard, is_mute_test):
-    icon = ":floppy_disk:"
-    mute_icon = ":white_check_mark:"
-    text = [
-        "| Test  | Status | Muted | Log |",
-        "| ----: | :----: | :---: | --: |",
-    ]
-
-    for target, reason, target_log_url, cases in summary:
-        mute_target = mute_icon if is_mute_shard(target) else ""
-        display_reason = reason if reason != "Failed" else ""
-        text.append(f"| **{target}** | {display_reason} | {mute_target} | [{icon}]({target_log_url}) |")
-        for classname, method, log_url in cases:
-            mute_class = mute_icon if is_mute_test(classname, method) else ""
-            text.append(f"| _{ classname }::{ method }_ | Failed | {mute_class} | [{icon}]({log_url}) |")
-    return text
-
-
-def write_summary(summary, is_mute_shard, is_mute_test):
-    fail_count = sum([len(s[3]) for s in summary])
-    text = generate_summary(summary, is_mute_shard, is_mute_test)
-    with open(os.environ["GITHUB_STEP_SUMMARY"], "at") as fp:
-        fp.write(f"Failed tests log files ({fail_count}):\n")
-        for line in text:
-            fp.write(f"{line}\n")
-
-
-def patch_jsuite(log_urls, ctest_path, unit_paths):
-    suite_logs = {}
-    test_logs = {}
-
-    for shard_name, _, log_url, cases in log_urls:
-        suite_logs[shard_name] = log_url
-        for classname, method, test_log_url in cases:
-            test_logs[(classname, method)] = test_log_url
-
-    if ctest_path:
-        tree = ET.parse(ctest_path)
-        root = tree.getroot()
-        changed = False
-        for testcase in root.findall("testcase"):
-            log_url = suite_logs.get(testcase.attrib["classname"])
-            if log_url:
-                add_junit_log_property(testcase, log_url)
-                changed = True
-
-        if changed:
-            print(f"patch {ctest_path}")
-            tree.write(ctest_path, xml_declaration=True, encoding="UTF-8")
-
-    for path in unit_paths:
-        for fn in glob.glob(os.path.join(path, "*.xml")):
-            tree = ET.parse(fn)
-            root = tree.getroot()
-            changed = False
-            for testsuite in root.findall("testsuite"):
-                for testcase in testsuite.findall("testcase"):
-                    cls, method = testcase.attrib["classname"], testcase.attrib["name"]
-                    log_url = test_logs.get((cls, method))
-                    if log_url:
-                        add_junit_log_property(testcase, log_url)
-                        changed = True
-            if changed:
-                print(f"patch {fn}")
-                tree.write(fn, xml_declaration=True, encoding="UTF-8")
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--url-prefix", default="./")
-    parser.add_argument("--decompress", action="store_true", default=False, help="decompress ctest log")
-    parser.add_argument("--write-summary", action="store_true", default=False, help="update github summary")
-    parser.add_argument("--filter-test-file", required=False)
-    parser.add_argument("--filter-shard-file", required=False)
-    parser.add_argument("--patch-jsuite", default=False, action="store_true")
-    parser.add_argument("--ctest-report")
-    parser.add_argument("--junit-reports-path", nargs="*")
-    parser.add_argument("ctest_log")
-    parser.add_argument("out_log_dir")
-
-    args = parser.parse_args()
-
-    log_urls = extract_logs(log_reader(args.ctest_log, args.decompress), Path(args.out_log_dir), args.url_prefix)
-
-    if args.patch_jsuite and log_urls:
-        patch_jsuite(log_urls, args.ctest_report, args.junit_reports_path)
-
-    is_mute_shard = MutedShardCheck(args.filter_shard_file)
-    is_mute_test = MutedTestCheck(args.filter_test_file)
-
-    if args.write_summary:
-        if log_urls:
-            write_summary(log_urls, is_mute_shard, is_mute_test)
-    else:
-        print("\n".join(generate_summary(log_urls, is_mute_shard, is_mute_test)))
-
-
-if __name__ == "__main__":
-    main()
diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
new file mode 100755
index 0000000000..e7dbca46eb
--- /dev/null
+++ b/.github/scripts/tests/generate-summary.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import glob
+import dataclasses
+import sys
+from typing import Optional, List
+from xml.etree import ElementTree as ET
+from junit_utils import get_property_value
+
+
+@dataclasses.dataclass
+class SummaryEntry:
+    target: str
+    log_url: Optional[str]
+    reason = ""
+    is_failure: bool = False
+    is_error: bool = False
+    is_muted: bool = False
+    is_skipped: bool = False
+
+    @property
+    def display_status(self):
+        if self.is_error:
+            return "Error"
+        elif self.is_failure:
+            return "Failure"
+        elif self.is_muted:
+            return "Muted"
+        elif self.is_skipped:
+            return "Skipped"
+
+        return "?"
+
+
+def iter_xml_files(folder_or_file):
+    if os.path.isfile(folder_or_file):
+        files = [folder_or_file]
+    else:
+        files = glob.glob(os.path.join(folder_or_file, "*.xml"))
+
+    for fn in files:
+        tree = ET.parse(fn)
+        root = tree.getroot()
+
+        if root.tag == "testsuite":
+            suites = [root]
+        elif root.tag == "testsuites":
+            suites = root.findall("testsuite")
+        else:
+            raise ValueError(f"Invalid root tag {root.tag}")
+        for suite in suites:
+            for case in suite.findall("testcase"):
+                yield fn, suite, case
+
+
+def parse_junit(folder_or_file):
+    result = []
+    for fn, suite, case in iter_xml_files(folder_or_file):
+        is_failure = case.find("failure") is not None
+        is_error = case.find("error") is not None
+        is_muted = get_property_value(case, "mute") is not None
+        is_skipped = is_muted is False and case.find("skipped") is not None
+
+        if any([is_failure, is_muted, is_skipped, is_error]):
+            cls, method = case.attrib["classname"], case.attrib["name"]
+            log_url = get_property_value(case, "url:Log")
+            target = f"{ cls }::{ method }" if cls != method else cls
+
+            result.append(
+                SummaryEntry(
+                    target=target,
+                    log_url=log_url,
+                    is_skipped=is_skipped,
+                    is_muted=is_muted,
+                    is_failure=is_failure,
+                    is_error=is_error,
+                )
+            )
+    return result
+
+
+def generate_summary(summary: List[SummaryEntry]):
+    log_icon = ":floppy_disk:"
+    mute_icon = ":white_check_mark:"
+
+    text = [
+        "| Test  | Muted | Log |",
+        "| ----: | :---: | --: |",
+    ]
+
+    for entry in summary:
+        if entry.log_url:
+            log_url = f"[{log_icon}]({entry.log_url})"
+        else:
+            log_url = ""
+
+        mute_target = mute_icon if entry.is_muted else ""
+
+        text.append(f"| {entry.target} | {mute_target} |  {log_url} |")
+
+    return text
+
+
+def write_summary(title, lines: List[str]):
+    summary_fn = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_fn:
+        fp = open(summary_fn, "at")
+    else:
+        fp = sys.stdout
+
+    if title:
+        fp.write(f"{title}\n")
+    for line in lines:
+        fp.write(f"{line}\n")
+    fp.write("\n")
+
+    if summary_fn:
+        fp.close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-t", "--title")
+    parser.add_argument("folder_or_file")
+
+    args = parser.parse_args()
+
+    summary = parse_junit(args.folder_or_file)
+
+    if summary:
+        text = generate_summary(summary)
+        write_summary(args.title, text)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/scripts/tests/junit_utils.py b/.github/scripts/tests/junit_utils.py
index c1aea12f80..232aaafa6b 100644
--- a/.github/scripts/tests/junit_utils.py
+++ b/.github/scripts/tests/junit_utils.py
@@ -20,3 +20,43 @@ def add_junit_property(testcase, name, value):
 
 def add_junit_log_property(testcase, url):
     add_junit_link_property(testcase, "Log", url)
+
+
+def get_property_value(testcase, name):
+    props = testcase.find("properties")
+    if props is None:
+        return None
+
+    for prop in props.findall("property"):
+        if prop.attrib["name"] == name:
+            return prop.attrib["value"]
+
+
+def create_error_testsuite(testcases):
+    n = str(len(testcases))
+    root = ET.Element("testsuite", dict(tests=n, failures=n))
+    root.extend(testcases)
+    return ET.ElementTree(root)
+
+
+def create_error_testcase(shardname, classname, name, log_fn=None, log_url=None):
+    testcase = ET.Element("testcase", dict(classname=classname, name=name))
+    add_junit_property(testcase, "shard", shardname)
+    if log_url:
+        add_junit_log_property(testcase, log_url)
+
+    err = ET.Element("error", dict(type="error"))
+
+    if log_fn:
+        with open(log_fn, "rt") as fp:
+            err.text = fp.read(4096)
+    testcase.append(err)
+
+    return testcase
+
+
+def suite_case_iterator(root):
+    for suite in root.findall("testsuite"):
+        for case in suite.findall("testcase"):
+            cls, method = case.attrib["classname"], case.attrib["name"]
+            yield suite, case, cls, method
diff --git a/.github/scripts/tests/mute_utils.py b/.github/scripts/tests/mute_utils.py
index ac35ff3da2..21dab3bf46 100644
--- a/.github/scripts/tests/mute_utils.py
+++ b/.github/scripts/tests/mute_utils.py
@@ -1,5 +1,6 @@
 import operator
 import xml.etree.ElementTree as ET
+from junit_utils import add_junit_property
 
 
 class MutedTestCheck:
@@ -63,6 +64,8 @@ def mute_target(node):
     node.remove(failure)
     node.append(skipped)
 
+    add_junit_property(node, "mute", "automatically muted based on rules")
+
     return True