Publish test results to Testmo

Publish test results to Testmo

Pull Request resolved: #183

4 files changed, 118 insertions, 17 deletions

diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index bc40b2a07c1..adf0d7b685c 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -11,32 +11,80 @@ inputs:
     required: true
     type: string
   aws_key_value: 
-    required: trye
+    required: true
     type: string
+  testman_token:
+     required: false
+     type: string
+  testman_url:
+     required: false
+     type: string
+  testman_project_id:
+     required: false
+     type: string
     
 runs:
   using: "composite"
   steps:
+  - name: Init
+    id: init
+    shell: bash
+    run: |
+      mkdir -p artifacts tmp test_reports
+      rm -rf artifacts/* tmp/* test_reports/*
+      echo "WORKDIR=$(pwd)" >> $GITHUB_ENV
+      echo "TESTREPDIR=$(pwd)/test_reports" >> $GITHUB_ENV
+      echo "TESTMO_TOKEN=${{inputs.testman_token}}" >> $GITHUB_ENV
+      echo "TESTMO_URL=${{inputs.testman_url}}" >> $GITHUB_ENV
+      echo "logfilename=${{inputs.log_suffix}}-stdout.gz" >> $GITHUB_OUTPUT
+  - name: Install Node required for Testmo CLI
+    uses: actions/setup-node@v3
+    with:
+      node-version: 19
+  - name: Install Testmo CLI
+    shell: bash
+    run: npm install -g @testmo/testmo-cli
+  - name: Test history run create
+    id: th
+    if: inputs.testman_token
+    shell: bash
+    run: |
+      RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+      testmo automation:resources:add-link --name build --url $RUN_URL --resources testmo.json
+      testmo automation:resources:add-field --name git-sha --type string --value ${GITHUB_SHA:0:7} --resources testmo.json
+      testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{inputs.testman_project_id}} --name "${{ github.run_id }}" \
+        --source "${{inputs.log_suffix}}" --resources testmo.json | \
+        echo "runid=$(cat)" >> $GITHUB_OUTPUT
   - name: Test
     shell: bash
     run: |
-      cd ../build/ydb
-      rm -rf $(pwd)/../../tmp/*
-      rm -rf $(pwd)/../../test_reports/*
-      mkdir -p ../../artifacts
-      rm -rf $(pwd)/../../artifacts/*
+      cd $WORKDIR/../build/ydb
 
-      echo "Stdout log (gzip archive): https://storage.yandexcloud.net/ydb-tech-ci/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-stdout.gz" >> $GITHUB_STEP_SUMMARY
+      echo "Stdout log (gzip archive): https://storage.yandexcloud.net/ydb-tech-ci/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{steps.init.outputs.logfilename}}" >> $GITHUB_STEP_SUMMARY
 
       # Sed removes coloring from the output
-      TMPDIR=$(pwd)/../../tmp GTEST_OUTPUT="xml:$(pwd)/../../test_reports/" \
+      
+      TMPDIR=$WORKDIR/tmp GTEST_OUTPUT="xml:$TESTREPDIR/unittests/" Y_UNITTEST_OUTPUT="xml:$TESTREPDIR/unittests/" \
         ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \
-              --output-junit $(pwd)/../../test_reports/ctest_report.xml \
-              -L '${{inputs.test_label_regexp}}' | \
+              --output-junit $TESTREPDIR/suites/ctest_report.xml \
+              -L '${{inputs.test_label_regexp}}' \
+              -E ydb-core-kqp-ut-spilling | \
         sed -e 's/\x1b\[[0-9;]*m//g' | \
-        tee >(gzip --stdout > ../../artifacts/${{inputs.log_suffix}}-${{inputs.sanitizer}}-stdout.gz) | \
+        tee >(gzip --stdout > $WORKDIR/artifacts/${{steps.init.outputs.logfilename}}) | \
         grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \
-        tee -a $GITHUB_STEP_SUMMARY
+        tee $WORKDIR/short.log
+  - name: Test history upload results
+    if: always() && inputs.testman_token
+    shell: bash
+    run: |
+      testmo automation:run:submit-thread \
+        --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \
+        --results $TESTREPDIR/unittests/*.xml 
+      testmo automation:run:submit-thread \
+        --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \
+        --results $TESTREPDIR/suites/*.xml \
+        -- cat $WORKDIR/short.log
+      testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}}
   - name: Upload S3
     uses: shallwefootball/s3-upload-action@master
     if: always()
@@ -44,6 +92,6 @@ runs:
       aws_key_id: ${{inputs.AWS_KEY_ID }}
       aws_secret_access_key: ${{inputs.AWS_KEY_VALUE}}
       aws_bucket: ydb-tech-ci
-      source_dir: ../artifacts
+      source_dir: artifacts
       destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}'
       endpoint: https://storage.yandexcloud.net
diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml
index 123118cc239..eb5e54923e8 100644
--- a/.github/workflows/build_and_test_ondemand.yml
+++ b/.github/workflows/build_and_test_ondemand.yml
@@ -9,6 +9,22 @@ on:
       test_label_regexp:
         required: false
         type: string
+      image:
+        type: string
+        required: false
+        default: fd8snjpoq85qqv0mk9gi
+  workflow_dispatch:
+    inputs: 
+      sanitizer:
+        required: false
+        type: string
+      test_label_regexp:
+        required: false
+        type: string
+      image:
+        type: string
+        required: false
+        default: fd8snjpoq85qqv0mk9gi
     
 jobs:
 
@@ -28,7 +44,7 @@ jobs:
           yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           folder-id: ${{secrets.YC_FOLDER}}
-          image-id: fd8snjpoq85qqv0mk9gi
+          image-id: ${{inputs.image}}
           disk-size: 930GB
           disk-type: network-ssd-nonreplicated
           cores: 32
@@ -68,10 +84,13 @@ jobs:
     - name: Test
       uses: ./.github/actions/test
       with: 
-        log_suffix: ${{format('{0}-{1}', 'X64', inputs.sanitizer)}}
+        log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}}
         test_label_regexp: ${{ inputs.test_label_regexp }}
         aws_key_id: ${{secrets.AWS_KEY_ID}}
         aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        testman_token: ${{secrets.TESTMO_TOKEN}}
+        testman_url: ${{vars.TESTMO_URL}}
+        testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
 
   release-runner:
     name: Release self-hosted YC runner if provided on-demand
@@ -89,4 +108,4 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           label: ${{ needs.provide-runner.outputs.label }}
           instance-id: ${{ needs.provide-runner.outputs.instance-id }}
-  
+          
diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml
index abfa07ff2aa..3af5be215a6 100644
--- a/.github/workflows/build_and_test_provisioned.yml
+++ b/.github/workflows/build_and_test_provisioned.yml
@@ -6,9 +6,32 @@ on:
       runner_label:
         required: true
         type: string
+      run_build:
+        type: boolean
+        default: true
       sanitizer:
         required: false
         type: string
+      run_tests:
+        type: boolean
+        default: true
+      test_label_regexp:
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+      run_build:
+        type: boolean
+        default: true
+      sanitizer:
+        required: false
+        type: string
+      run_tests:
+        type: boolean
+        default: true
       test_label_regexp:
         required: false
         type: string
@@ -22,12 +45,17 @@ jobs:
       uses: actions/checkout@v3
     - name: Build
       uses: ./.github/actions/build
+      if: inputs.run_build
       with: 
         sanitizer: ${{ inputs.sanitizer }}
     - name: Test
       uses: ./.github/actions/test
+      if: inputs.run_tests
       with: 
-        log_suffix: ${{format('{0}-{1}', inputs.runner_label, inputs.sanitizer)}}
+        log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}}
         test_label_regexp: ${{ inputs.test_label_regexp }}
         aws_key_id: ${{secrets.AWS_KEY_ID}}
         aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        testman_token: ${{secrets.TESTMO_TOKEN}}
+        testman_url: ${{vars.TESTMO_URL}}
+        testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml
index 6804f87d7b4..bfa6d972c35 100644
--- a/.github/workflows/prepare_vm_for_build.yml
+++ b/.github/workflows/prepare_vm_for_build.yml
@@ -9,6 +9,11 @@ on:
     secrets:
       rc_auth:
         required: false
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        required: true
+        type: string  
         
 jobs:
   prepare:
@@ -25,6 +30,7 @@ jobs:
         sudo apt-get -y install git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev
         sudo pip3 install conan==1.59
     - name: Install AllureCtl
+      if: false
       shell: bash
       run: |
         wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl