Remove build stage (#6527)

4 files changed, 114 insertions, 64 deletions

diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml
index b280c4d938a..5d3163ef058 100644
--- a/.github/actions/build_and_test_ya/action.yml
+++ b/.github/actions/build_and_test_ya/action.yml
@@ -14,7 +14,7 @@ inputs:
     type: boolean
     default: true
     description: "run tests"
-  run_tests_if_build_fails:
+  run_tests_if_build_fails:  # TODO: not used, remove it
     default: "true"
     description: "run tests if build fails"
   test_threads:
@@ -104,28 +104,15 @@ runs:
           echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
         fi
 
-    - name: Build
-      uses: ./.github/actions/build_ya
-      id: build
-      if: ${{ inputs.run_build == 'true' }}
-      with:
-        build_target: ${{ steps.target_choice.outputs.target }}
-        build_preset: ${{ inputs.build_preset }}
-        bazel_remote_uri: ${{  fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
-        bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
-        bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
-        put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
-        link_threads: ${{ inputs.link_threads }}
-        additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
-
     - name: Run tests
+      id: build
       uses: ./.github/actions/test_ya
-      if: ${{ inputs.run_tests == 'true' &&  (steps.build.outputs.success == 'true' || inputs.run_tests_if_build_fails == 'true') }}
       with:
         build_target: ${{ steps.target_choice.outputs.target }}
         build_preset: ${{ inputs.build_preset }}
         test_size: ${{ inputs.test_size }}
         test_type: ${{ inputs.test_type }}
+        run_tests: ${{ inputs.run_tests }}
         testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN2 }}
         testman_url: ${{  fromJSON( inputs.vars ).TESTMO_URL }}
         testman_project_id: ${{  fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
@@ -133,21 +120,9 @@ runs:
         additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
         test_threads: ${{ inputs.test_threads }}
         bazel_remote_uri: ${{  fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
-
-    
-    - name: Notify about failed build
-      if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }}
-      shell: bash
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      run: |
-        echo 'Build failed. See the [build log](${{ steps.build.outputs.log_url }}).' >> $GITHUB_STEP_SUMMARY
-        
-        if [[ "$GITHUB_EVENT_NAME" =~ ^pull_request ]]; then
-          echo "Tests run skipped." | .github/scripts/tests/comment-pr.py --fail
-        fi
-        
-        exit 1
+        bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
+        bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
         
     - name: build_stats
       shell: bash
diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml
index a0210552523..937812aa4c2 100644
--- a/.github/actions/build_ya/action.yml
+++ b/.github/actions/build_ya/action.yml
@@ -1,3 +1,5 @@
+# TODO: not used, delete it
+
 name: Build (ya make)
 description: Build targets
 inputs:
diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index ecd54efb8bb..b7c289b9d77 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -36,9 +36,26 @@ inputs:
   testman_project_id:
     required: false
     description: "test manager project id"
+  put_build_results_to_cache:
+    required: false
+    default: "true"
   bazel_remote_uri:
     required: false
     description: "bazel-remote endpoint"
+  bazel_remote_username:
+    required: false
+    description: "bazel-remote username"
+  bazel_remote_password:
+    required: false
+    description: "bazel-remote password"
+  run_tests:
+    type: boolean
+    default: true
+    description: "run tests"
+outputs:
+  success:
+    value: ${{ steps.build.outputs.status }}
+    description: "build success"
 
 runs:
   using: "composite"
@@ -65,7 +82,10 @@ runs:
         echo "YA_TEST_LOG_FILENAME=$LOG_DIR/ya_test.log" >> $GITHUB_ENV
         echo "YA_TEST_LOG_URL=$S3_URL_PREFIX/test_logs/ya_test.log" >> $GITHUB_ENV
         echo "S3_LOG_BUCKET_PATH=$S3_BUCKET_PATH/test_logs" >> $GITHUB_ENV
-        
+
+        export log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
+        echo "LOG_URL=$log_url" >> $GITHUB_ENV
+        echo "log_url=$log_url" >> $GITHUB_OUTPUT
     
     - name: prepare
       shell: bash
@@ -85,7 +105,7 @@ runs:
 
     - name: Upload tests result to testmo
       id: th
-      if: inputs.testman_token
+      if: inputs.testman_token && inputs.run_tests
       shell: bash
       env:
         PR_NUMBER: ${{ github.event.number }}
@@ -148,16 +168,18 @@ runs:
 
     - name: Print test history link
       shell: bash
-      if: inputs.testman_token
+      if: inputs.testman_token && inputs.run_tests
       run: |
         echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS
 
     - name: set environment variables required by some tests
+      if: inputs.run_tests
       shell: bash
       run: |
         echo "PSQL_BINARY=/usr/bin/psql" >> $GITHUB_ENV
 
-    - name: ya test
+    - name: ya build and test
+      id: build
       shell: bash
       run: |
         set -x
@@ -165,14 +187,12 @@ runs:
         readarray -d ',' -t test_type < <(printf "%s" "${{ inputs.test_type }}")
 
         params=(
-          -T -k
+          -T
           ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=}
-          --do-not-output-stderrs
           --stat
           --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}"
         )
         
-        # FIXME: copy-paste from build_ya
         case "$BUILD_PRESET" in
           debug)
             params+=(--build "debug")
@@ -180,6 +200,15 @@ runs:
           relwithdebinfo)
             params+=(--build "relwithdebinfo")
             ;;
+          release)  
+            params+=(--build "release")
+            ;;
+          release-clang14)
+            params+=(--build "release")
+            params+=(--target-platform="CLANG14-LINUX-X86_64")
+            params+=(-DLLD_VERSION=16)
+            params+=(-DSTRIP)
+            ;;
           release-asan)
             params+=(
               --build "release" --sanitize="address"
@@ -213,37 +242,68 @@ runs:
           params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
         fi
 
+        if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
+          params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
+          params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
+          params+=(--bazel-remote-put --dist-cache-max-file-size=209715200)
+        fi
+
+        if [ true = ${{ inputs.run_tests }} ]; then
+          params+=(-A)
+        fi
+
         echo "::debug::get version"
         ./ya --version
         
-        echo "Tests are running..." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py
-        
-        ./ya test ${{ inputs.build_target }} "${params[@]}" \
+        echo "Build+Tests **{platform_name}-${BUILD_PRESET}** is running..." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py
+        set +e
+        (./ya make ${{ inputs.build_target }} "${params[@]}" \
           --stat --log-file "$YA_TEST_LOG_FILENAME" -DCONSISTENT_DEBUG \
-          --no-dir-outputs \
-          --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
-            RC=$?
-            if [ $RC -ne 0 ]; then
-              echo "10 [Test run log]($YA_TEST_LOG_URL)" >> $SUMMARY_LINKS
+          --no-dir-outputs --test-failure-code 0 --build-all \
+          --cache-size 2TB --force-build-depends --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
+          --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR"; echo $? > exit_code) |& tee $TMP_DIR/ya_make.log
+        set -e
+        RC=`cat exit_code`
 
-              echo "ya test returned $RC, check existence $JUNIT_REPORT_XML"
-              if [ -s "$JUNIT_REPORT_XML" ]; then
-                echo "$JUNIT_REPORT_XML exists"
-                ls -la "$JUNIT_REPORT_XML"
-              else
-                echo "$JUNIT_REPORT_XML doesn't exist or has zero size"
-                ls -la "$JUNIT_REPORT_XML" || true
-                exit $RC
-              fi
-            fi
-        )
+        echo "10 [Ya make output]($LOG_URL)" >> $SUMMARY_LINKS
+        echo "20 [Ya logs]($YA_TEST_LOG_URL)" >> $SUMMARY_LINKS
+
+        if [ $RC -ne 0 ]; then
+          echo "ya make returned $RC, build failed"
+          echo "status=failed" >> $GITHUB_OUTPUT
+        else
+          echo "status=true" >> $GITHUB_OUTPUT
+        fi
+
+
+    - name: comment-build-status
+      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        set -x
+        
+        if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}'
+          echo "Build failed. see the [logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
+        else
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}'
+          echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
+        fi
 
     - name: archive unitest reports (orig)
+      if: inputs.run_tests
       shell: bash
       run: |
         gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz
 
     - name: postprocess junit report
+      if: inputs.run_tests
       shell: bash
       run: |
         set -x
@@ -259,7 +319,7 @@ runs:
         .github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML"
     
     - name: Test history upload results to YDB
-      if: always()
+      if: inputs.run_tests
       continue-on-error: true
       shell: bash
       env:
@@ -281,12 +341,13 @@ runs:
         result=`.github/scripts/upload_tests_results.py --test-results-file ${JUNIT_REPORT_XML} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull ${PR_NUMBER} --job-name ${JOB_NAME} --job-id ${JOB_ID} --branch ${GITHUB_REF_NAME}`
 
     - name: archive unitest reports (transformed)
+      if: inputs.run_tests
       shell: bash
       run: |
         tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS)
 
     - name: Unit test history upload results
-      if: inputs.testman_token
+      if: inputs.testman_token && inputs.run_tests
       shell: bash
       env:
         TESTMO_TOKEN: ${{ inputs.testman_token }}
@@ -314,7 +375,7 @@ runs:
         kill $proxy_pid
 
     - name: Test history run complete
-      if: always() && inputs.testman_token
+      if: always() && inputs.testman_token && inputs.run_tests
       shell: bash
       env:
         TESTMO_TOKEN: ${{ inputs.testman_token }}
@@ -328,15 +389,23 @@ runs:
         set -x
         echo "::group::s3-sync"
         s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$LOG_DIR/" "$S3_LOG_BUCKET_PATH/"
+        s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/"
         echo "::endgroup::"
     
-    - name: write tests summary
+    - name: analyze tests results
       shell: bash
-      if: always()
+      if: always() && inputs.run_tests
       env:
         GITHUB_TOKEN: ${{ github.token }}
       run: |
         set -x
+
+        if [ "${{ inputs.run_tests }}" != "true" ]; then
+          # I have no idea why 'always() && inputs.run_tests' not working 
+          # Probably because of implicit bool->str cast? 
+          exit 0
+        fi
+
         mkdir $ARTIFACTS_DIR/summary/
         
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
@@ -359,9 +428,13 @@ runs:
         curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
             https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
             -d '{"state":"'$teststatus'","description":"'"$testmessage"'","context":"test_${{inputs.build_preset}}"}'
+        
+        if [[ $teststatus != "success" ]];then
+          echo "status=failed" >> $GITHUB_OUTPUT
+        fi
 
     - name: sync test results to s3
-      if: always()
+      if: always() && inputs.run_tests
       shell: bash
       run: |
         set -x
@@ -371,6 +444,7 @@ runs:
         echo "::endgroup::"
     
     - name: check test results
+      if: inputs.run_tests
       shell: bash
       run: |
         .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML"
diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
index aef7074652d..ad397760f25 100755
--- a/.github/scripts/tests/generate-summary.py
+++ b/.github/scripts/tests/generate-summary.py
@@ -300,7 +300,6 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: st
     if summary.is_empty:
         return [
             f"Test run completed, no test results found for commit {pr.head.sha}. "
-            f"Please check [test log]({test_log_file_url})."
         ]
     elif summary.is_failed:
         result = f"Some tests failed, follow the links below."