ci: add testmo-proxy, split junit into multiple parts, generate summary and post comment after tests logs upload

4 files changed, 97 insertions, 23 deletions

diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index f1c4b6b4db..5401a6114c 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -57,7 +57,9 @@ runs:
         echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
         echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
         echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
+        echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV
         echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
+        echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV
         echo "TESTMO_TOKEN=${{ inputs.testman_token }}" >> $GITHUB_ENV
         echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
@@ -65,8 +67,8 @@ runs:
     - name: prepare
       shell: bash
       run: |
-        rm -rf $TMP_DIR $JUNIT_REPORT_XML
-        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR
+        rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
+        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
     
     - name: Install Node required for Testmo CLI
       uses: actions/setup-node@v3
@@ -210,6 +212,11 @@ runs:
             fi
         )
 
+    - name: archive unitest reports (orig)
+      shell: bash
+      run: |
+        gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz
+    
     - name: postprocess junit report
       shell: bash
       run: |
@@ -220,6 +227,43 @@ runs:
           --log-url-prefix "$S3_URL_PREFIX/logs/" \
           --log-out-dir "$ARTIFACTS_DIR/logs/" \
           "$JUNIT_REPORT_XML"
+    
+        .github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML"
+    
+    - name: archive unitest reports (transformed)
+      shell: bash
+      run: |
+        tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS)
+
+    - name: Unit test history upload results
+      if: inputs.testman_token
+      shell: bash
+      run: |
+        PROXY_ADDR=127.0.0.1:8888
+        openssl req -x509 -newkey rsa:2048 \
+          -keyout $TMP_DIR/key.pem -out $TMP_DIR/cert.pem \
+          -sha256 -days 1 -nodes -subj "/CN=127.0.0.1"
+        
+        ./ydb/ci/testmo-proxy/testmo-proxy.py -l $PROXY_ADDR \
+          --cert-file "$TMP_DIR/cert.pem" \
+          --cert-key "$TMP_DIR/key.pem" \
+          --target-timeout 3,10 \
+          --max-request-time 55 \
+          "$TESTMO_URL" &
+        
+        proxy_pid=$!
+        
+        NODE_TLS_REJECT_UNAUTHORIZED=0 testmo automation:run:submit-thread \
+          --instance "https://$PROXY_ADDR" --run-id ${{ steps.th.outputs.runid }} \
+          --results "$JUNIT_REPORT_PARTS/*.xml"
+        
+        kill $proxy_pid
+
+    - name: Test history run complete
+      if: always() && inputs.testman_token
+      shell: bash
+      run: |
+        testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }}
 
     - name: write tests summary
       shell: bash
@@ -227,7 +271,7 @@ runs:
         GITHUB_TOKEN: ${{ github.token }}
       run: |
         mkdir $ARTIFACTS_DIR/summary/
-
+        
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
         
         platform_name=$(uname | tr '[:upper:]' '[:lower:]')-$(arch)
@@ -239,23 +283,14 @@ runs:
           --build-preset "${platform_name}-${{ inputs.build_preset }}" \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
-
-    - name: Unit test history upload results
-      if: inputs.testman_token
-      shell: bash
-      run: |
-        testmo automation:run:submit-thread \
-          --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }} \
-          --results "$JUNIT_REPORT_XML"
-
     - name: sync test results to s3
       if: always()
       shell: bash
       run: |
         echo "::group::s3-sync"
-        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
+        s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
         echo "::endgroup::"
-
+    
     - name: sync logs results to s3
       if: always()
       shell: bash
@@ -263,14 +298,7 @@ runs:
         echo "::group::s3-sync"
         s3cmd sync --follow-symlinks --acl-private --no-progress --stats --no-check-md5 "$LOG_DIR/" "$S3_BUCKET_PATH/test_logs/"
         echo "::endgroup::"
-
-
-    - name: Test history run complete
-      if: always() && inputs.testman_token
-      shell: bash
-      run: |
-        testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }}
-
+        
     - name: check test results
       shell: bash
       run: |
diff --git a/.github/scripts/tests/split-junit.py b/.github/scripts/tests/split-junit.py
new file mode 100755
index 0000000000..bae60415de
--- /dev/null
+++ b/.github/scripts/tests/split-junit.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+
+def save_suite(suite, path):
+    root = ET.Element("testsuites")
+    root.append(suite)
+    tree = ET.ElementTree(root)
+    tree.write(path)
+
+
+def do_split(fn, out_dir):
+    try:
+        tree = ET.parse(fn)
+    except ET.ParseError as e:
+        print(f"Unable to parse {fn}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    root = tree.getroot()
+
+    for n, suite in enumerate(root.iter('testsuite')):
+        part_fn = Path(out_dir).joinpath(f"part_{n}.xml")
+        print(f"write {part_fn}")
+        save_suite(suite, part_fn)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', dest='out_dir', required=True)
+    parser.add_argument("in_file", type=argparse.FileType("r"))
+
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.out_dir):
+        os.makedirs(args.out_dir)
+
+    do_split(args.in_file, args.out_dir)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/.github/scripts/tests/transform-ya-junit.py b/.github/scripts/tests/transform-ya-junit.py
index 3521af9064..657a2f21ff 100755
--- a/.github/scripts/tests/transform-ya-junit.py
+++ b/.github/scripts/tests/transform-ya-junit.py
@@ -57,7 +57,7 @@ class YTestReportTrace:
         test_results_dir = os.path.join(self.out_root, f"{subdir}/test-results/")
 
         if not os.path.isdir(test_results_dir):
-            print(f"Directory {test_results_dir} doesn't exist")
+            log_print(f"Directory {test_results_dir} doesn't exist")
             return
 
         for folder in os.listdir(test_results_dir):
diff --git a/.mapping.json b/.mapping.json
index f6384adc5f..e1d974cff5 100644
--- a/.mapping.json
+++ b/.mapping.json
@@ -29,6 +29,7 @@
   ".github/scripts/tests/log_parser.py":"ydb/github_toplevel/.github/scripts/tests/log_parser.py",
   ".github/scripts/tests/mute_utils.py":"ydb/github_toplevel/.github/scripts/tests/mute_utils.py",
   ".github/scripts/tests/pytest-postprocess.py":"ydb/github_toplevel/.github/scripts/tests/pytest-postprocess.py",
+  ".github/scripts/tests/split-junit.py":"ydb/github_toplevel/.github/scripts/tests/split-junit.py",
   ".github/scripts/tests/templates/summary.html":"ydb/github_toplevel/.github/scripts/tests/templates/summary.html",
   ".github/scripts/tests/transform-ya-junit.py":"ydb/github_toplevel/.github/scripts/tests/transform-ya-junit.py",
   ".github/workflows/allowed_dirs.yml":"ydb/github_toplevel/.github/workflows/allowed_dirs.yml",