Refactor upload artifacts and use SUMMARY_LINKS (#6812)

3 files changed, 71 insertions, 77 deletions

diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index b06ad2768c..e71aa629d0 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -79,38 +79,18 @@ runs:
         # The whole dir will be uploaded to s3 with public (=wild internet) ACL
         export PUBLIC_DIR=$TMP_DIR/results
         echo "PUBLIC_DIR=$PUBLIC_DIR" >> $GITHUB_ENV
+        export PUBLIC_DIR_URL=$S3_URL_PREFIX
+        echo "PUBLIC_DIR_URL=$PUBLIC_DIR_URL" >> $GITHUB_ENV
         mkdir -p $PUBLIC_DIR
 
         export YA_MAKE_OUT_DIR=$TMP_DIR/out
         echo "YA_MAKE_OUT_DIR=$YA_MAKE_OUT_DIR" >> $GITHUB_ENV
         mkdir -p $YA_MAKE_OUT_DIR
 
-        export ARTIFACTS_DIR=$PUBLIC_DIR/artifacts
-        echo "ARTIFACTS_DIR=$ARTIFACTS_DIR" >> $GITHUB_ENV
-        echo "ARTIFACTS_DIR_URL=$S3_URL_PREFIX/artifacts" >> $GITHUB_ENV
-        mkdir -p $ARTIFACTS_DIR
-
-        export TEST_ARTIFACTS_DIR=$PUBLIC_DIR/test_artifacts
-        echo "TEST_ARTIFACTS_DIR=$TEST_ARTIFACTS_DIR" >> $GITHUB_ENV
-        echo "TEST_ARTIFACTS_DIR_URL=$S3_URL_PREFIX/test_artifacts" >> $GITHUB_ENV
-        mkdir -p $TEST_ARTIFACTS_DIR
-
-
         echo "JUNIT_REPORT_XML=$PUBLIC_DIR/junit.xml" >> $GITHUB_ENV
         echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$PUBLIC_DIR/summary_links.txt" >> $GITHUB_ENV
         echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
-        
-        echo "YA_TEST_LOG=$PUBLIC_DIR/ya_log.log" >> $GITHUB_ENV
-        echo "YA_TEST_LOG_URL=$S3_URL_PREFIX/ya_log.log" >> $GITHUB_ENV
-
-
-        echo YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.log" >>  $GITHUB_ENV
-        export YA_MAKE_OUTPUT_URL="$S3_URL_PREFIX/ya_make_output.log"
-        echo "YA_MAKE_OUTPUT_URL=$YA_MAKE_OUTPUT_URL" >> $GITHUB_ENV
-
-        # install test mo
-        npm install -g @testmo/testmo-cli
     
 
     - name: Upload tests result to testmo
@@ -126,6 +106,9 @@ runs:
         BRANCH_TAG="$GITHUB_REF_NAME"
         ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
         
+        # install test mo
+        npm install -g @testmo/testmo-cli
+
         case "$BUILD_PRESET" in
           relwithdebinfo)
             TESTMO_SOURCE="ya-${ARCH}"
@@ -174,7 +157,8 @@ runs:
           --tags "$BRANCH_TAG" --tags "$EXTRA_TAG"
         )
         echo "runid=${RUN_ID}" >> $GITHUB_OUTPUT
-        echo "TEST_HISTORY_URL=${TESTMO_URL}/automation/runs/view/${RUN_ID}" >> $GITHUB_ENV
+        TEST_HISTORY_URL="${TESTMO_URL}/automation/runs/view/${RUN_ID}"
+        echo "TEST_HISTORY_URL=$TEST_HISTORY_URL" >> $GITHUB_ENV
 
         # Print test history link
         echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS
@@ -262,19 +246,21 @@ runs:
 
         echo "::debug::get version"
         ./ya --version
-        
+
+        YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.log"
+        YA_MAKE_OUTPUT_URL="$PUBLIC_DIR_URL/ya_make_output.log"
+        echo "10 [Ya make output]($YA_MAKE_OUTPUT_URL)" >> $SUMMARY_LINKS
+
         echo "Build+Tests **{platform_name}-${BUILD_PRESET}** is running..." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py
         set +ex
         (./ya make ${{ inputs.build_target }} "${params[@]}" \
-          --stat --log-file "$YA_TEST_LOG" -DCONSISTENT_DEBUG \
+          --stat --log-file "$PUBLIC_DIR/ya_log.log" -DCONSISTENT_DEBUG \
           --no-dir-outputs --test-failure-code 0 --build-all \
           --cache-size 2TB --force-build-depends --evlog-file "$PUBLIC_DIR/ya_evlog.jsonl" \
           --junit "$JUNIT_REPORT_XML" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& tee $YA_MAKE_OUTPUT
         set -e
         RC=`cat exit_code`
 
-        echo "10 [Ya make output]($YA_MAKE_OUTPUT_URL)" >> $SUMMARY_LINKS
-        echo "20 [Ya logs]($YA_TEST_LOG_URL)" >> $SUMMARY_LINKS
 
         if [ $RC -ne 0 ]; then
           echo "ya make returned $RC, build failed"
@@ -315,13 +301,12 @@ runs:
         # postprocess junit report
         .github/scripts/tests/transform-ya-junit.py -i \
           -m .github/config/muted_ya.txt \
-          --ya-out "$YA_MAKE_OUT_DIR" \
-          --log-url-prefix "$ARTIFACTS_DIR_URL/logs/" \
-          --log-out-dir "$ARTIFACTS_DIR/logs/" \
-          --test-stuff-out "$TEST_ARTIFACTS_DIR/" \
-          --test-stuff-prefix "$TEST_ARTIFACTS_DIR_URL/" \
+          --ya_out "$YA_MAKE_OUT_DIR" \
+          --public_dir "$PUBLIC_DIR" \
+          --public_dir_url "$PUBLIC_DIR_URL" \
+          --log_out_dir "artifacts/logs/" \
+          --test_stuff_out "test_artifacts/" \
           "$JUNIT_REPORT_XML"
-  
     
     - name: Test history upload results to YDB
       if: inputs.run_tests
@@ -403,15 +388,12 @@ runs:
           exit 0
         fi
 
-        mkdir $ARTIFACTS_DIR/summary/
-
         .github/scripts/tests/generate-summary.py \
-          --summary-out-path $ARTIFACTS_DIR/summary/ \
-          --summary-url-prefix $ARTIFACTS_DIR_URL/summary/ \
-          --test-history-url "$TEST_HISTORY_URL" \
-          --test-log-url="$YA_TEST_LOG_URL" \
-          --build-preset "$BUILD_PRESET" \
-          --status-report-file statusrep.txt \
+          --summary_links "$SUMMARY_LINKS" \
+          --public_dir "$PUBLIC_DIR" \
+          --public_dir_url "$PUBLIC_DIR_URL" \
+          --build_preset "$BUILD_PRESET" \
+          --status_report_file statusrep.txt \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
         teststatus=$(cat statusrep.txt)
@@ -441,7 +423,7 @@ runs:
         set -x
         echo "::group::s3-sync"
         .github/scripts/Indexer/indexer.py -r "$PUBLIC_DIR/"
-        echo "00 [Workflow run results](${S3_URL_PREFIX}/index.html)" >> $SUMMARY_LINKS
+        echo "00 [Workflow artifacts](${S3_URL_PREFIX}/index.html)" >> $SUMMARY_LINKS
         s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"     
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
         echo "::endgroup::"
diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
index ad397760f2..66a1201f3c 100755
--- a/.github/scripts/tests/generate-summary.py
+++ b/.github/scripts/tests/generate-summary.py
@@ -255,7 +255,7 @@ def render_testlist_html(rows, fn):
         fp.write(content)
 
 
-def write_summary(summary: TestSummary, test_log_url: str):
+def write_summary(summary: TestSummary):
     summary_fn = os.environ.get("GITHUB_STEP_SUMMARY")
     if summary_fn:
         fp = open(summary_fn, "at")
@@ -263,7 +263,7 @@ def write_summary(summary: TestSummary, test_log_url: str):
         fp = sys.stdout
 
     if summary.is_empty:
-        fp.write(f":red_circle: Test run completed, no test results found. Please check [test logs]({test_log_url}).")
+        fp.write(f":red_circle: Test run completed, no test results found. Please check ya make output.")
     else:
         for line in summary.render(add_footnote=True):
             fp.write(f"{line}\n")
@@ -274,7 +274,7 @@ def write_summary(summary: TestSummary, test_log_url: str):
         fp.close()
 
 
-def gen_summary(summary_url_prefix, summary_out_folder, paths):
+def gen_summary(public_dir, public_dir_url, paths):
     summary = TestSummary()
 
     for title, html_fn, path in paths:
@@ -287,16 +287,16 @@ def gen_summary(summary_url_prefix, summary_out_folder, paths):
         if not summary_line.tests:
             continue
 
-        report_url = f"{summary_url_prefix}{html_fn}"
+        report_url = f"{public_dir_url}/{html_fn}"
 
-        render_testlist_html(summary_line.tests, os.path.join(summary_out_folder, html_fn))
+        render_testlist_html(summary_line.tests, os.path.join(public_dir, html_fn))
         summary_line.add_report(html_fn, report_url)
         summary.add_line(summary_line)
 
     return summary
 
 
-def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: str, test_log_file_url: str):
+def get_comment_text(pr: PullRequest, summary: TestSummary, summary_links: str):
     if summary.is_empty:
         return [
             f"Test run completed, no test results found for commit {pr.head.sha}. "
@@ -309,13 +309,12 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: st
     body = [
         result
     ]
-    links = []
 
-    if test_history_url:
-        links.append(f"[Test history]({test_history_url})")
-
-    if test_log_file_url:
-        links.append(f"[Test log]({test_log_file_url})")
+    with open(summary_links) as f:
+        links = f.readlines()
+    
+    links.sort()
+    links = [line.split(" ", 1)[1].strip() for line in links]
 
     if links:
         body.append("")
@@ -328,12 +327,11 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: st
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--summary-out-path", required=True)
-    parser.add_argument("--summary-url-prefix", required=True)
-    parser.add_argument('--test-history-url', required=False)
-    parser.add_argument('--test-log-url', required=False)
-    parser.add_argument('--build-preset', default="default-linux-x86-64-relwithdebinfo", required=False)
-    parser.add_argument('--status-report-file', required=False)
+    parser.add_argument("--public_dir", required=True)
+    parser.add_argument("--public_dir_url", required=True)
+    parser.add_argument("--summary_links", required=True)
+    parser.add_argument('--build_preset', default="default-linux-x86-64-relwithdebinfo", required=False)
+    parser.add_argument('--status_report_file', required=False)
     parser.add_argument("args", nargs="+", metavar="TITLE html_out path")
     args = parser.parse_args()
 
@@ -344,8 +342,8 @@ def main():
     paths = iter(args.args)
     title_path = list(zip(paths, paths, paths))
 
-    summary = gen_summary(args.summary_url_prefix, args.summary_out_path, title_path)
-    write_summary(summary, args.test_log_url)
+    summary = gen_summary(args.public_dir, args.public_dir_url, title_path)
+    write_summary(summary)
 
     if summary.is_empty | summary.is_failed:
         color = 'red'
@@ -362,7 +360,7 @@ def main():
             event = json.load(fp)
 
         pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
-        text = get_comment_text(pr, summary, args.test_history_url, args.test_log_url)
+        text = get_comment_text(pr, summary, args.summary_links)
 
         update_pr_comment_text(pr, args.build_preset, run_number, color, text='\n'.join(text), rewrite=False)
 
diff --git a/.github/scripts/tests/transform-ya-junit.py b/.github/scripts/tests/transform-ya-junit.py
index 40dba9425b..df79bfdace 100755
--- a/.github/scripts/tests/transform-ya-junit.py
+++ b/.github/scripts/tests/transform-ya-junit.py
@@ -1,4 +1,10 @@
 #!/usr/bin/env python3
+
+# Tool used to transform junit report. Performs the following:
+# - adds classname with relative path to test in 'testcase' node
+# - add 'url:logsdir' and other links with in 'testcase' node
+# - mutes tests
+
 import argparse
 import re
 import json
@@ -165,7 +171,7 @@ def save_zip(suite_name, out_dir, url_prefix, logs_dir: Set[str]):
     return f"{url_prefix}{quoted_fpath}"
 
 
-def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size,
+def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_truncate_size,
               test_stuff_out, test_stuff_prefix):
     tree = ET.parse(fp)
     root = tree.getroot()
@@ -195,7 +201,7 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
                 if logs:
                     log_print(f"add {list(logs.keys())!r} properties for {test_cls}.{test_method}")
                     for name, fn in logs.items():
-                        url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_trunc_size)
+                        url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_truncate_size)
                         add_junit_link_property(case, name, url)
 
         if has_fail_tests:
@@ -221,18 +227,18 @@ def main():
         "-i", action="store_true", dest="save_inplace", default=False, help="modify input file in-place"
     )
     parser.add_argument("-m", help="muted test list")
-    parser.add_argument("--log-url-prefix", default="./", help="url prefix for logs")
-    parser.add_argument("--log-out-dir", help="symlink logs to specific directory")
+    parser.add_argument('--public_dir', help='root directory for publication')
+    parser.add_argument("--public_dir_url", help="url prefix for root directory")
+
+    parser.add_argument("--log_out_dir", help="out dir to store logs (symlinked), relative to public_dir")
     parser.add_argument(
-        "--log-truncate-size",
-        dest="log_trunc_size",
+        "--log_truncate_size",
         type=int,
         default=134217728,
         help="truncate log after specific size, 0 disables truncation",
     )
-    parser.add_argument("--ya-out", help="ya make output dir (for searching logs and artifacts)")
-    parser.add_argument('--test-stuff-out', help='output folder for archive testing_out_stuff')
-    parser.add_argument('--test-stuff-prefix', help='url prefix for testing_out_stuff')
+    parser.add_argument("--ya_out", help="ya make output dir (for searching logs and artifacts)")
+    parser.add_argument('--test_stuff_out', help='output dir for archive testing_out_stuff, relative to public_dir"')
     parser.add_argument("in_file", type=argparse.FileType("r"))
 
     args = parser.parse_args()
@@ -242,16 +248,24 @@ def main():
     if args.m:
         mute_check.load(args.m)
 
+    log_out_dir =  os.path.join(args.public_dir, args.log_out_dir)
+    os.makedirs(log_out_dir, exist_ok=True)
+    log_url_prefix = os.path.join(args.public_dir_url, args.log_out_dir)
+
+    test_stuff_out = os.path.join(args.public_dir, args.test_stuff_out)
+    os.makedirs(test_stuff_out, exist_ok=True)
+    test_stuff_prefix = os.path.join(args.public_dir_url, args.test_stuff_out)
+
     transform(
         args.in_file,
         mute_check,
         args.ya_out,
         args.save_inplace,
-        args.log_url_prefix,
-        args.log_out_dir,
-        args.log_trunc_size,
-        args.test_stuff_out,
-        args.test_stuff_prefix,
+        log_url_prefix,
+        log_out_dir,
+        args.log_truncate_size,
+        test_stuff_out,
+        test_stuff_prefix,
     )