diff options
author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2024-07-15 14:51:53 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-07-15 14:51:53 +0300 |
commit | dc3a89d33ad44e5feec57e0ccd8e3dd3bea13d66 (patch) | |
tree | ea3ad7fc64ee094a525fecec7552043aa32f4352 | |
parent | 755aa56521d25e7d7f4b078e675b1d60b5fc5264 (diff) | |
download | ydb-dc3a89d33ad44e5feec57e0ccd8e3dd3bea13d66.tar.gz |
Refacator workflows (#6630)
-rw-r--r-- | .github/actions/build_and_test_ya/action.yml | 7 | ||||
-rw-r--r-- | .github/actions/build_ya/action.yml | 172 | ||||
-rw-r--r-- | .github/actions/test_ya/action.yml | 84 |
3 files changed, 47 insertions, 216 deletions
diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml index 5d3163ef05..b3ba245b7e 100644 --- a/.github/actions/build_and_test_ya/action.yml +++ b/.github/actions/build_and_test_ya/action.yml @@ -14,9 +14,6 @@ inputs: type: boolean default: true description: "run tests" - run_tests_if_build_fails: # TODO: not used, remove it - default: "true" - description: "run tests if build fails" test_threads: type: string default: 28 @@ -94,7 +91,7 @@ runs: if: inputs.increment == 'true' uses: ./.github/actions/graph_compare - - name: Change target in case of incremental (tests) + - name: Change target in case of incremental id: target_choice shell: bash run: | @@ -104,7 +101,7 @@ runs: echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT fi - - name: Run tests + - name: Run build and tests id: build uses: ./.github/actions/test_ya with: diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml deleted file mode 100644 index 937812aa4c..0000000000 --- a/.github/actions/build_ya/action.yml +++ /dev/null @@ -1,172 +0,0 @@ -# TODO: not used, delete it - -name: Build (ya make) -description: Build targets -inputs: - build_target: - required: false - description: "build target" - build_preset: - required: true - default: "relwithdebinfo" - description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14" - put_build_results_to_cache: - required: false - default: "true" - bazel_remote_uri: - required: false - description: "bazel-remote endpoint" - bazel_remote_username: - required: false - description: "bazel-remote username" - bazel_remote_password: - required: false - description: "bazel-remote password" - link_threads: - required: false - default: "8" - description: "link threads count" - additional_ya_make_args: - type: string - default: "" -outputs: - success: - value: ${{ steps.build.outputs.status }} - description: "build success" - log_url: - value: ${{ steps.init.outputs.log_url }} - description: "build log url" -runs: - using: "composite" - steps: - - name: Init - id: init - shell: bash - env: - build_preset: ${{ inputs.build_preset }} - run: | - export TMP_DIR=$(pwd)/tmp_build - echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV - - export log_url="$S3_URL_PREFIX/build_logs/ya_make.log" - - rm -rf $TMP_DIR && mkdir $TMP_DIR - - echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV - echo "LOG_URL=$log_url" >> $GITHUB_ENV - echo "log_url=$log_url" >> $GITHUB_OUTPUT - - - name: build - id: build - shell: bash - run: | - set -x - extra_params=() - - if [ ! -z "${{ inputs.build_target }}" ]; then - extra_params+=(--target="${{ inputs.build_target }}") - fi - - if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then - extra_params+=(--bazel-remote-store) - extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") - fi - - if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then - extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") - extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}") - extra_params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) - fi - - case "${{ inputs.build_preset }}" in - debug) - build_type=debug - ;; - relwithdebinfo) - build_type=relwithdebinfo - ;; - release) - build_type=release - ;; - release-clang14) - build_type=release - extra_params+=(--target-platform="CLANG14-LINUX-X86_64") - extra_params+=(-DLLD_VERSION=16) - extra_params+=(-DSTRIP) - ;; - release-asan) - build_type=release - extra_params+=(--sanitize="address") - extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY) - ;; - release-tsan) - build_type=release - extra_params+=(--sanitize="thread") - extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY) - ;; - release-msan) - build_type=release - extra_params+=(--sanitize="memory") - extra_params+=(-DSKIP_JUNK -DUSE_EAT_MY_DATA -DDEBUGINFO_LINES_ONLY) - ;; - *) - echo "Invalid preset: ${{ inputs.build_preset }}" - exit 1 - ;; - esac - - if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then - extra_params+=(${{ inputs.additional_ya_make_args }}) - fi - - echo "::debug::get version" - ./ya --version - - echo "Build **{platform_name}-${BUILD_PRESET}** is running..." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py - - # to be sure - set -o pipefail - - ./ya make --cache-size 2TB --build "${build_type}" --force-build-depends -T --stat -DCONSISTENT_DEBUG \ - --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \ - --link-threads "${{ inputs.link_threads }}" \ - "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || ( - RC=$? - echo "::debug::ya make RC=$RC" - echo "status=failed" >> $GITHUB_OUTPUT - ) - - - name: sync logs to s3 - if: always() - shell: bash - run: | - set -x - echo "::group::s3-sync" - s3cmd sync --acl-private --exclude="ya_make.log" --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/" - s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/" - echo "::endgroup::" - - - name: comment-build-status - if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -x - - if [ "${{ steps.build.outputs.status }}" == "failed" ]; then - curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ - -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}' - echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail - else - curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ - -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}' - echo "Build successful." | .github/scripts/tests/comment-pr.py --ok - fi - - - name: show free space - if: always() - shell: bash - run: df -h diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index b7c289b9d7..ff6b40d78e 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -1,4 +1,4 @@ -name: Run tests (ya make) +name: Run build and tests (ya make) description: Run test targets listed in repository root ya.make (to be created previously) inputs: build_target: @@ -60,21 +60,47 @@ outputs: runs: using: "composite" steps: + - name: Install Node required for Testmo CLI + uses: actions/setup-node@v3 + with: + node-version: 19 + - name: Init id: init shell: bash run: | + set -x + export TMP_DIR=$(pwd)/tmp - export LOG_DIR=$TMP_DIR/logs - + rm -rf $TMP_DIR + mkdir -p $TMP_DIR echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV + + export LOG_DIR=$TMP_DIR/logs echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV - echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV - echo "TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts" >> $GITHUB_ENV - echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV + mkdir -p $LOG_DIR + + export OUT_DIR=$TMP_DIR/out + echo "OUT_DIR=$OUT_DIR" >> $GITHUB_ENV + mkdir -p $OUT_DIR + + export ARTIFACTS_DIR=$TMP_DIR/artifacts + echo "ARTIFACTS_DIR=$ARTIFACTS_DIR" >> $GITHUB_ENV + mkdir -p $ARTIFACTS_DIR + + export TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts + echo "TEST_ARTIFACTS_DIR=$TEST_ARTIFACTS_DIR" >> $GITHUB_ENV + mkdir -p $TEST_ARTIFACTS_DIR + + export REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports + echo "REPORTS_ARTIFACTS_DIR=$REPORTS_ARTIFACTS_DIR" >> $GITHUB_ENV + mkdir -p $REPORTS_ARTIFACTS_DIR + + export JUNIT_REPORT_PARTS=$TMP_DIR/junit-split + echo "JUNIT_REPORT_PARTS=$JUNIT_REPORT_PARTS" >> $GITHUB_ENV + mkdir -p $JUNIT_REPORT_PARTS + echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV - echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV @@ -86,22 +112,10 @@ runs: export log_url="$S3_URL_PREFIX/build_logs/ya_make.log" echo "LOG_URL=$log_url" >> $GITHUB_ENV echo "log_url=$log_url" >> $GITHUB_OUTPUT + + # install test mo + npm install -g @testmo/testmo-cli - - name: prepare - shell: bash - run: | - set -x - rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR - mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $TEST_ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR - - - name: Install Node required for Testmo CLI - uses: actions/setup-node@v3 - with: - node-version: 19 - - - name: Install Testmo CLI - shell: bash - run: npm install -g @testmo/testmo-cli - name: Upload tests result to testmo id: th @@ -166,10 +180,7 @@ runs: echo "runid=${RUN_ID}" >> $GITHUB_OUTPUT echo "TEST_HISTORY_URL=${TESTMO_URL}/automation/runs/view/${RUN_ID}" >> $GITHUB_ENV - - name: Print test history link - shell: bash - if: inputs.testman_token && inputs.run_tests - run: | + # Print test history link echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS - name: set environment variables required by some tests @@ -296,17 +307,15 @@ runs: echo "Build successful." | .github/scripts/tests/comment-pr.py --ok fi - - name: archive unitest reports (orig) + - name: process reports if: inputs.run_tests shell: bash run: | + set -x + # archive unitest reports (orig) gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz - - name: postprocess junit report - if: inputs.run_tests - shell: bash - run: | - set -x + # postprocess junit report .github/scripts/tests/transform-ya-junit.py -i \ -m .github/config/muted_ya.txt \ --ya-out "$OUT_DIR" \ @@ -317,6 +326,9 @@ runs: "$JUNIT_REPORT_XML" .github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML" + + # archive unitest reports (transformed) + tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS) - name: Test history upload results to YDB if: inputs.run_tests @@ -340,12 +352,6 @@ runs: result=`.github/scripts/upload_tests_results.py --test-results-file ${JUNIT_REPORT_XML} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull ${PR_NUMBER} --job-name ${JOB_NAME} --job-id ${JOB_ID} --branch ${GITHUB_REF_NAME}` - - name: archive unitest reports (transformed) - if: inputs.run_tests - shell: bash - run: | - tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS) - - name: Unit test history upload results if: inputs.testman_token && inputs.run_tests shell: bash |