new test summary badge

new test summary badge

Pull Request resolved: #343

3 files changed, 188 insertions, 73 deletions

diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
index 33d61b01f1..1dae6aba4b 100755
--- a/.github/scripts/tests/generate-summary.py
+++ b/.github/scripts/tests/generate-summary.py
@@ -1,93 +1,91 @@
 #!/usr/bin/env python3
 import argparse
-import os
 import dataclasses
-import sys
-from typing import Optional, List
+import os, sys
+from enum import Enum
+from itertools import groupby
+from operator import attrgetter
+from typing import List
+from jinja2 import Environment, FileSystemLoader
 from junit_utils import get_property_value, iter_xml_files
 
 
+class TestStatus(Enum):
+    PASS = 0
+    FAIL = 1
+    ERROR = 2
+    SKIP = 3
+    MUTE = 4
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+
 @dataclasses.dataclass
-class SummaryEntry:
-    target: str
-    log_url: Optional[str]
-    reason = ""
-    is_failure: bool = False
-    is_error: bool = False
-    is_muted: bool = False
-    is_skipped: bool = False
+class TestResult:
+    classname: str
+    name: str
+    status: TestStatus
 
     @property
-    def display_status(self):
-        if self.is_error:
-            return "Error"
-        elif self.is_failure:
-            return "Failure"
-        elif self.is_muted:
-            return "Muted"
-        elif self.is_skipped:
-            return "Skipped"
-
-        return "?"
-
-
-def parse_junit(folder_or_file):
-    result = []
-    for fn, suite, case in iter_xml_files(folder_or_file):
-        is_failure = case.find("failure") is not None
-        is_error = case.find("error") is not None
-        is_muted = get_property_value(case, "mute") is not None
-        is_skipped = is_muted is False and case.find("skipped") is not None
-
-        if any([is_failure, is_muted, is_skipped, is_error]):
-            cls, method = case.attrib["classname"], case.attrib["name"]
-            log_url = get_property_value(case, "url:Log")
-            target = f"{ cls }::{ method }" if cls != method else cls
-
-            result.append(
-                SummaryEntry(
-                    target=target,
-                    log_url=log_url,
-                    is_skipped=is_skipped,
-                    is_muted=is_muted,
-                    is_failure=is_failure,
-                    is_error=is_error,
-                )
-            )
-    return result
+    def status_display(self):
+        return {
+            TestStatus.PASS: "PASS",
+            TestStatus.FAIL: "FAIL",
+            TestStatus.ERROR: "ERROR",
+            TestStatus.SKIP: "SKIP",
+            TestStatus.MUTE: "MUTE",
+        }[self.status]
+
+    def __str__(self):
+        return f"{self.full_name:<138} {self.status_display}"
 
+    @property
+    def full_name(self):
+        return f"{self.classname}/{self.name}"
 
-def generate_summary(summary: List[SummaryEntry]):
-    log_icon = ":floppy_disk:"
-    mute_icon = ":white_check_mark:"
 
-    text = [
-        "| Test  | Muted | Log |",
-        "| ----: | :---: | --: |",
-    ]
+def render_pm(value, url, diff=None):
+    if value:
+        text = f"[{value}]({url})"
+    else:
+        text = str(value)
 
-    for entry in summary:
-        if entry.log_url:
-            log_url = f"[{log_icon}]({entry.log_url})"
+    if diff is not None and diff != 0:
+        if diff == 0:
+            sign = "±"
+        elif diff < 0:
+            sign = "-"
         else:
-            log_url = ""
-
-        mute_target = mute_icon if entry.is_muted else ""
+            sign = "+"
 
-        text.append(f"| {entry.target} | {mute_target} |  {log_url} |")
+        text = f"{text} {sign}{abs(diff)}"
 
     return text
 
 
-def write_summary(title, lines: List[str]):
+def render_testlist_html(rows, fn):
+    TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), "templates")
+
+    env = Environment(loader=FileSystemLoader(TEMPLATES_PATH))
+
+    rows.sort(key=attrgetter('full_name'))
+    rows.sort(key=attrgetter('status'), reverse=True)
+
+    rows = groupby(rows, key=attrgetter('status'))
+    content = env.get_template("summary.html").render(test_results=rows)
+
+    with open(fn, 'w') as fp:
+        fp.write(content)
+
+
+def write_summary(lines: List[str]):
     summary_fn = os.environ.get("GITHUB_STEP_SUMMARY")
     if summary_fn:
         fp = open(summary_fn, "at")
     else:
         fp = sys.stdout
 
-    if title:
-        fp.write(f"{title}\n")
     for line in lines:
         fp.write(f"{line}\n")
     fp.write("\n")
@@ -96,18 +94,80 @@ def write_summary(title, lines: List[str]):
         fp.close()
 
 
+def gen_summary(summary_url_prefix, summary_out_folder, paths, ):
+    summary = [
+        "|      | TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED[^1] |",
+        "| :--- | ---:  | -----: | -----: | -----: | ------: | ----: |",
+    ]
+    for title, html_fn, path in paths:
+        tests = failed = errors = muted = skipped = passed = 0
+
+        test_results = []
+
+        for fn, suite, case in iter_xml_files(path):
+            tests += 1
+            classname, name = case.get("classname"), case.get("name")
+            if case.find("failure") is not None:
+                failed += 1
+                status = TestStatus.FAIL
+            elif case.find("error") is not None:
+                errors += 1
+                status = TestStatus.ERROR
+            elif get_property_value(case, "mute") is not None:
+                muted += 1
+                status = TestStatus.MUTE
+            elif case.find("skipped") is not None:
+                skipped += 1
+                status = TestStatus.SKIP
+            else:
+                passed += 1
+                status = TestStatus.PASS
+
+            test_result = TestResult(classname=classname, name=name, status=status)
+            test_results.append(test_result)
+
+        report_url = f'{summary_url_prefix}{html_fn}'
+
+        render_testlist_html(test_results, os.path.join(summary_out_folder, html_fn))
+
+        summary.append(
+            " | ".join(
+                [
+                    title,
+                    render_pm(tests, f'{report_url}', 0),
+                    render_pm(passed, f'{report_url}#PASS', 0),
+                    render_pm(errors, f'{report_url}#ERROR', 0),
+                    render_pm(failed, f'{report_url}#FAIL', 0),
+                    render_pm(skipped, f'{report_url}#SKIP', 0),
+                    render_pm(muted, f'{report_url}#MUTE', 0),
+                ]
+            )
+        )
+
+    github_srv = os.environ.get('GITHUB_SERVER_URL', 'https://github.com')
+    repo = os.environ.get('GITHUB_REPOSITORY', 'ydb-platform/ydb')
+
+    summary.append("\n")
+    summary.append(f"[^1]: All mute rules are defined [here]({github_srv}/{repo}/tree/main/.github/config).")
+
+    write_summary(lines=summary)
+
+
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-t", "--title")
-    parser.add_argument("folder_or_file")
-
+    parser.add_argument("--summary-out-path", required=True)
+    parser.add_argument("--summary-url-prefix", required=True)
+    parser.add_argument("args", nargs="+", metavar="TITLE html_out path")
     args = parser.parse_args()
 
-    summary = parse_junit(args.folder_or_file)
+    if len(args.args) % 3 != 0:
+        print("Invalid argument count")
+        raise SystemExit(-1)
+
+    paths = iter(args.args)
+    title_path = list(zip(paths, paths, paths))
 
-    if summary:
-        text = generate_summary(summary)
-        write_summary(args.title, text)
+    gen_summary(args.summary_url_prefix, args.summary_out_path, title_path)
 
 
 if __name__ == "__main__":
diff --git a/.github/scripts/tests/junit_utils.py b/.github/scripts/tests/junit_utils.py
index fd636884cf..b4ec72e406 100644
--- a/.github/scripts/tests/junit_utils.py
+++ b/.github/scripts/tests/junit_utils.py
@@ -1,5 +1,6 @@
 import os
 import glob
+import sys
 from xml.etree import ElementTree as ET
 
 
@@ -77,7 +78,7 @@ def iter_xml_files(folder_or_file):
         try:
             tree = ET.parse(fn)
         except ET.ParseError as e:
-            print(f"Unable to parse {fn}: {e}")
+            print(f"Unable to parse {fn}: {e}", file=sys.stderr)
             continue
 
         root = tree.getroot()
diff --git a/.github/scripts/tests/templates/summary.html b/.github/scripts/tests/templates/summary.html
new file mode 100644
index 0000000000..7098edaf13
--- /dev/null
+++ b/.github/scripts/tests/templates/summary.html
@@ -0,0 +1,54 @@
+<html>
+<head>
+    <style>
+        th {
+            text-transform: uppercase;
+        }
+        
+        th, td {
+            padding: 5px;
+        }
+
+        table {
+            border-collapse: collapse;
+        }
+
+        span.test_status {
+            font-weight: bold;
+        }
+
+        span.test_fail {
+            color: red;
+        }
+
+        span.test_pass {
+            color: green;
+        }
+
+        span.test_mute {
+            color: blue;
+        }
+    </style>
+</head>
+<body>
+{% for status, rows in test_results %}
+<h1 id="{{ status.name}}">{{ status.name }}</h1>
+<table style="width:90%;" border="1">
+    <thead>
+    <tr>
+        <th>test name</th>
+        <th>status</th>
+    </tr>
+    </thead>
+    <tbody>
+    {% for t in rows %}
+    <tr>
+        <td>{{ t.full_name }}</td>
+        <td><span class="test_status test_{{ t.status_display }}">{{ t.status_display }}</span></td>
+    </tr>
+    {% endfor %}
+    </tbody>
+</table>
+{% endfor %}
+</body>
+</html>
\ No newline at end of file