Refacator workflows (#6630)

3 files changed, 47 insertions, 216 deletions

diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml
index 5d3163ef05..b3ba245b7e 100644
--- a/.github/actions/build_and_test_ya/action.yml
+++ b/.github/actions/build_and_test_ya/action.yml
@@ -14,9 +14,6 @@ inputs:
     type: boolean
     default: true
     description: "run tests"
-  run_tests_if_build_fails:  # TODO: not used, remove it
-    default: "true"
-    description: "run tests if build fails"
   test_threads:
     type: string
     default: 28
@@ -94,7 +91,7 @@ runs:
       if: inputs.increment == 'true'
       uses: ./.github/actions/graph_compare
  
-    - name: Change target in case of incremental (tests)
+    - name: Change target in case of incremental
       id: target_choice
       shell: bash
       run: |
@@ -104,7 +101,7 @@ runs:
           echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
         fi
 
-    - name: Run tests
+    - name: Run build and tests
       id: build
       uses: ./.github/actions/test_ya
       with:
diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml
deleted file mode 100644
index 937812aa4c..0000000000
--- a/.github/actions/build_ya/action.yml
+++ /dev/null
@@ -1,172 +0,0 @@
-# TODO: not used, delete it
-
-name: Build (ya make)
-description: Build targets
-inputs:
-  build_target:
-    required: false
-    description: "build target"
-  build_preset:
-    required: true
-    default: "relwithdebinfo"
-    description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14"
-  put_build_results_to_cache:
-    required: false
-    default: "true"
-  bazel_remote_uri:
-    required: false
-    description: "bazel-remote endpoint"
-  bazel_remote_username:
-    required: false
-    description: "bazel-remote username"
-  bazel_remote_password:
-    required: false
-    description: "bazel-remote password"
-  link_threads:
-    required: false
-    default: "8"
-    description: "link threads count"
-  additional_ya_make_args:
-    type: string
-    default: ""
-outputs:
-  success:
-    value: ${{ steps.build.outputs.status }}
-    description: "build success"
-  log_url:
-    value: ${{ steps.init.outputs.log_url }}
-    description: "build log url"
-runs:
-  using: "composite"
-  steps:
-    - name: Init
-      id: init
-      shell: bash
-      env:
-        build_preset: ${{ inputs.build_preset }}
-      run: |
-        export TMP_DIR=$(pwd)/tmp_build
-        echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
-        
-        export log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
-        
-        rm -rf $TMP_DIR && mkdir $TMP_DIR
-        
-        echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV
-        echo "LOG_URL=$log_url" >> $GITHUB_ENV
-        echo "log_url=$log_url" >> $GITHUB_OUTPUT
-
-    - name: build
-      id: build
-      shell: bash
-      run: |
-        set -x
-        extra_params=()
-
-        if [ ! -z "${{ inputs.build_target }}" ]; then
-          extra_params+=(--target="${{ inputs.build_target }}")
-        fi
-
-        if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
-          extra_params+=(--bazel-remote-store)
-          extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
-        fi
-        
-        if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
-          extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
-          extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
-          extra_params+=(--bazel-remote-put --dist-cache-max-file-size=209715200)
-        fi
-        
-        case "${{ inputs.build_preset }}" in
-          debug)
-            build_type=debug
-            ;;
-          relwithdebinfo)
-            build_type=relwithdebinfo
-            ;;
-          release)
-            build_type=release
-            ;;
-          release-clang14)
-            build_type=release
-            extra_params+=(--target-platform="CLANG14-LINUX-X86_64")
-            extra_params+=(-DLLD_VERSION=16)
-            extra_params+=(-DSTRIP)
-            ;;
-          release-asan)
-            build_type=release
-            extra_params+=(--sanitize="address")
-            extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
-            ;;
-          release-tsan)
-            build_type=release
-            extra_params+=(--sanitize="thread")
-            extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
-            ;;
-          release-msan)
-            build_type=release
-            extra_params+=(--sanitize="memory")
-            extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY)
-            ;;
-          *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
-            exit 1
-            ;;
-        esac
-        
-        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
-          extra_params+=(${{ inputs.additional_ya_make_args }})
-        fi
-
-        echo "::debug::get version"
-        ./ya --version
-        
-        echo "Build **{platform_name}-${BUILD_PRESET}** is running..." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py
-        
-        # to be sure
-        set -o pipefail
-        
-        ./ya make --cache-size 2TB --build "${build_type}" --force-build-depends -T --stat -DCONSISTENT_DEBUG \
-          --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
-          --link-threads "${{ inputs.link_threads }}" \
-          "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || (
-              RC=$?
-              echo "::debug::ya make RC=$RC"
-              echo "status=failed" >> $GITHUB_OUTPUT
-          )
-
-    - name: sync logs to s3
-      if: always()
-      shell: bash
-      run: |
-        set -x
-        echo "::group::s3-sync"
-        s3cmd sync --acl-private --exclude="ya_make.log" --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
-        s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/"
-        echo "::endgroup::"
-
-    - name: comment-build-status
-      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-      shell: bash
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      run: |
-        set -x
-        
-        if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
-          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
-            -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}'
-          echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
-        else
-          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
-            -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}'
-          echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
-        fi
-
-    - name: show free space
-      if: always()
-      shell: bash
-      run: df -h
diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index b7c289b9d7..ff6b40d78e 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -1,4 +1,4 @@
-name: Run tests (ya make)
+name: Run build and tests (ya make)
 description: Run test targets listed in repository root ya.make (to be created previously)
 inputs:
   build_target:
@@ -60,21 +60,47 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Install Node required for Testmo CLI
+      uses: actions/setup-node@v3
+      with:
+        node-version: 19
+    
     - name: Init
       id: init
       shell: bash
       run: |
+        set -x
+
         export TMP_DIR=$(pwd)/tmp
-        export LOG_DIR=$TMP_DIR/logs
-        
+        rm -rf $TMP_DIR
+        mkdir -p $TMP_DIR
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
+
+        export LOG_DIR=$TMP_DIR/logs
         echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV
-        echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
-        echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
-        echo "TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts" >> $GITHUB_ENV
-        echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV
+        mkdir -p $LOG_DIR
+
+        export OUT_DIR=$TMP_DIR/out
+        echo "OUT_DIR=$OUT_DIR" >> $GITHUB_ENV
+        mkdir -p $OUT_DIR
+
+        export ARTIFACTS_DIR=$TMP_DIR/artifacts
+        echo "ARTIFACTS_DIR=$ARTIFACTS_DIR" >> $GITHUB_ENV
+        mkdir -p $ARTIFACTS_DIR
+
+        export TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts
+        echo "TEST_ARTIFACTS_DIR=$TEST_ARTIFACTS_DIR" >> $GITHUB_ENV
+        mkdir -p $TEST_ARTIFACTS_DIR
+
+        export REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports
+        echo "REPORTS_ARTIFACTS_DIR=$REPORTS_ARTIFACTS_DIR" >> $GITHUB_ENV
+        mkdir -p $REPORTS_ARTIFACTS_DIR
+
+        export JUNIT_REPORT_PARTS=$TMP_DIR/junit-split
+        echo "JUNIT_REPORT_PARTS=$JUNIT_REPORT_PARTS" >> $GITHUB_ENV
+        mkdir -p $JUNIT_REPORT_PARTS
+
         echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
-        echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV
         echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
         echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
@@ -86,22 +112,10 @@ runs:
         export log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
         echo "LOG_URL=$log_url" >> $GITHUB_ENV
         echo "log_url=$log_url" >> $GITHUB_OUTPUT
+
+        # install test mo
+        npm install -g @testmo/testmo-cli
     
-    - name: prepare
-      shell: bash
-      run: |
-        set -x
-        rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
-        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $TEST_ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
-    
-    - name: Install Node required for Testmo CLI
-      uses: actions/setup-node@v3
-      with:
-        node-version: 19
-    
-    - name: Install Testmo CLI
-      shell: bash
-      run: npm install -g @testmo/testmo-cli
 
     - name: Upload tests result to testmo
       id: th
@@ -166,10 +180,7 @@ runs:
         echo "runid=${RUN_ID}" >> $GITHUB_OUTPUT
         echo "TEST_HISTORY_URL=${TESTMO_URL}/automation/runs/view/${RUN_ID}" >> $GITHUB_ENV
 
-    - name: Print test history link
-      shell: bash
-      if: inputs.testman_token && inputs.run_tests
-      run: |
+        # Print test history link
         echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS
 
     - name: set environment variables required by some tests
@@ -296,17 +307,15 @@ runs:
           echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
         fi
 
-    - name: archive unitest reports (orig)
+    - name: process reports
       if: inputs.run_tests
       shell: bash
       run: |
+        set -x
+        # archive unitest reports (orig) 
         gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz
 
-    - name: postprocess junit report
-      if: inputs.run_tests
-      shell: bash
-      run: |
-        set -x
+        # postprocess junit report
         .github/scripts/tests/transform-ya-junit.py -i \
           -m .github/config/muted_ya.txt \
           --ya-out "$OUT_DIR" \
@@ -317,6 +326,9 @@ runs:
           "$JUNIT_REPORT_XML"
     
         .github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML"
+
+        # archive unitest reports (transformed)
+        tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS)
     
     - name: Test history upload results to YDB
       if: inputs.run_tests
@@ -340,12 +352,6 @@ runs:
         
         result=`.github/scripts/upload_tests_results.py --test-results-file ${JUNIT_REPORT_XML} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull ${PR_NUMBER} --job-name ${JOB_NAME} --job-id ${JOB_ID} --branch ${GITHUB_REF_NAME}`
 
-    - name: archive unitest reports (transformed)
-      if: inputs.run_tests
-      shell: bash
-      run: |
-        tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS)
-
     - name: Unit test history upload results
       if: inputs.testman_token && inputs.run_tests
       shell: bash