Refactor upload artifacts and use SUMMARY_LINKS (#6812)

2 files changed, 47 insertions, 35 deletions

diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
index ad397760f25..66a1201f3c9 100755
--- a/.github/scripts/tests/generate-summary.py
+++ b/.github/scripts/tests/generate-summary.py
@@ -255,7 +255,7 @@ def render_testlist_html(rows, fn):
         fp.write(content)
 
 
-def write_summary(summary: TestSummary, test_log_url: str):
+def write_summary(summary: TestSummary):
     summary_fn = os.environ.get("GITHUB_STEP_SUMMARY")
     if summary_fn:
         fp = open(summary_fn, "at")
@@ -263,7 +263,7 @@ def write_summary(summary: TestSummary, test_log_url: str):
         fp = sys.stdout
 
     if summary.is_empty:
-        fp.write(f":red_circle: Test run completed, no test results found. Please check [test logs]({test_log_url}).")
+        fp.write(f":red_circle: Test run completed, no test results found. Please check ya make output.")
     else:
         for line in summary.render(add_footnote=True):
             fp.write(f"{line}\n")
@@ -274,7 +274,7 @@ def write_summary(summary: TestSummary, test_log_url: str):
         fp.close()
 
 
-def gen_summary(summary_url_prefix, summary_out_folder, paths):
+def gen_summary(public_dir, public_dir_url, paths):
     summary = TestSummary()
 
     for title, html_fn, path in paths:
@@ -287,16 +287,16 @@ def gen_summary(summary_url_prefix, summary_out_folder, paths):
         if not summary_line.tests:
             continue
 
-        report_url = f"{summary_url_prefix}{html_fn}"
+        report_url = f"{public_dir_url}/{html_fn}"
 
-        render_testlist_html(summary_line.tests, os.path.join(summary_out_folder, html_fn))
+        render_testlist_html(summary_line.tests, os.path.join(public_dir, html_fn))
         summary_line.add_report(html_fn, report_url)
         summary.add_line(summary_line)
 
     return summary
 
 
-def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: str, test_log_file_url: str):
+def get_comment_text(pr: PullRequest, summary: TestSummary, summary_links: str):
     if summary.is_empty:
         return [
             f"Test run completed, no test results found for commit {pr.head.sha}. "
@@ -309,13 +309,12 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: st
     body = [
         result
     ]
-    links = []
 
-    if test_history_url:
-        links.append(f"[Test history]({test_history_url})")
-
-    if test_log_file_url:
-        links.append(f"[Test log]({test_log_file_url})")
+    with open(summary_links) as f:
+        links = f.readlines()
+    
+    links.sort()
+    links = [line.split(" ", 1)[1].strip() for line in links]
 
     if links:
         body.append("")
@@ -328,12 +327,11 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: st
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--summary-out-path", required=True)
-    parser.add_argument("--summary-url-prefix", required=True)
-    parser.add_argument('--test-history-url', required=False)
-    parser.add_argument('--test-log-url', required=False)
-    parser.add_argument('--build-preset', default="default-linux-x86-64-relwithdebinfo", required=False)
-    parser.add_argument('--status-report-file', required=False)
+    parser.add_argument("--public_dir", required=True)
+    parser.add_argument("--public_dir_url", required=True)
+    parser.add_argument("--summary_links", required=True)
+    parser.add_argument('--build_preset', default="default-linux-x86-64-relwithdebinfo", required=False)
+    parser.add_argument('--status_report_file', required=False)
     parser.add_argument("args", nargs="+", metavar="TITLE html_out path")
     args = parser.parse_args()
 
@@ -344,8 +342,8 @@ def main():
     paths = iter(args.args)
     title_path = list(zip(paths, paths, paths))
 
-    summary = gen_summary(args.summary_url_prefix, args.summary_out_path, title_path)
-    write_summary(summary, args.test_log_url)
+    summary = gen_summary(args.public_dir, args.public_dir_url, title_path)
+    write_summary(summary)
 
     if summary.is_empty | summary.is_failed:
         color = 'red'
@@ -362,7 +360,7 @@ def main():
             event = json.load(fp)
 
         pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
-        text = get_comment_text(pr, summary, args.test_history_url, args.test_log_url)
+        text = get_comment_text(pr, summary, args.summary_links)
 
         update_pr_comment_text(pr, args.build_preset, run_number, color, text='\n'.join(text), rewrite=False)
 
diff --git a/.github/scripts/tests/transform-ya-junit.py b/.github/scripts/tests/transform-ya-junit.py
index 40dba9425b9..df79bfdace5 100755
--- a/.github/scripts/tests/transform-ya-junit.py
+++ b/.github/scripts/tests/transform-ya-junit.py
@@ -1,4 +1,10 @@
 #!/usr/bin/env python3
+
+# Tool used to transform junit report. Performs the following:
+# - adds classname with relative path to test in 'testcase' node
+# - add 'url:logsdir' and other links with in 'testcase' node
+# - mutes tests
+
 import argparse
 import re
 import json
@@ -165,7 +171,7 @@ def save_zip(suite_name, out_dir, url_prefix, logs_dir: Set[str]):
     return f"{url_prefix}{quoted_fpath}"
 
 
-def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size,
+def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_truncate_size,
               test_stuff_out, test_stuff_prefix):
     tree = ET.parse(fp)
     root = tree.getroot()
@@ -195,7 +201,7 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
                 if logs:
                     log_print(f"add {list(logs.keys())!r} properties for {test_cls}.{test_method}")
                     for name, fn in logs.items():
-                        url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_trunc_size)
+                        url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_truncate_size)
                         add_junit_link_property(case, name, url)
 
         if has_fail_tests:
@@ -221,18 +227,18 @@ def main():
         "-i", action="store_true", dest="save_inplace", default=False, help="modify input file in-place"
     )
     parser.add_argument("-m", help="muted test list")
-    parser.add_argument("--log-url-prefix", default="./", help="url prefix for logs")
-    parser.add_argument("--log-out-dir", help="symlink logs to specific directory")
+    parser.add_argument('--public_dir', help='root directory for publication')
+    parser.add_argument("--public_dir_url", help="url prefix for root directory")
+
+    parser.add_argument("--log_out_dir", help="out dir to store logs (symlinked), relative to public_dir")
     parser.add_argument(
-        "--log-truncate-size",
-        dest="log_trunc_size",
+        "--log_truncate_size",
         type=int,
         default=134217728,
         help="truncate log after specific size, 0 disables truncation",
     )
-    parser.add_argument("--ya-out", help="ya make output dir (for searching logs and artifacts)")
-    parser.add_argument('--test-stuff-out', help='output folder for archive testing_out_stuff')
-    parser.add_argument('--test-stuff-prefix', help='url prefix for testing_out_stuff')
+    parser.add_argument("--ya_out", help="ya make output dir (for searching logs and artifacts)")
+    parser.add_argument('--test_stuff_out', help='output dir for archive testing_out_stuff, relative to public_dir"')
     parser.add_argument("in_file", type=argparse.FileType("r"))
 
     args = parser.parse_args()
@@ -242,16 +248,24 @@ def main():
     if args.m:
         mute_check.load(args.m)
 
+    log_out_dir =  os.path.join(args.public_dir, args.log_out_dir)
+    os.makedirs(log_out_dir, exist_ok=True)
+    log_url_prefix = os.path.join(args.public_dir_url, args.log_out_dir)
+
+    test_stuff_out = os.path.join(args.public_dir, args.test_stuff_out)
+    os.makedirs(test_stuff_out, exist_ok=True)
+    test_stuff_prefix = os.path.join(args.public_dir_url, args.test_stuff_out)
+
     transform(
         args.in_file,
         mute_check,
         args.ya_out,
         args.save_inplace,
-        args.log_url_prefix,
-        args.log_out_dir,
-        args.log_trunc_size,
-        args.test_stuff_out,
-        args.test_stuff_prefix,
+        log_url_prefix,
+        log_out_dir,
+        args.log_truncate_size,
+        test_stuff_out,
+        test_stuff_prefix,
     )