diff options
author | Alexander Smirnov <alexv-smirnov@yandex-team.ru> | 2023-05-31 19:05:05 +0200 |
---|---|---|
committer | Alexander Smirnov <alexv-smirnov@yandex-team.ru> | 2023-05-31 19:05:05 +0200 |
commit | 10ba5cc0c3d130ce4b33d307d265b937dd572c39 (patch) | |
tree | f5de7deac53b0989332e9e8571d75c8916239f3f | |
parent | 1db08b73476ce8a181ff6163ec91305fe7a8cc56 (diff) | |
download | ydb-10ba5cc0c3d130ce4b33d307d265b937dd572c39.tar.gz |
bring workflows from main
-rw-r--r-- | .github/actions/build/action.yml | 46 | ||||
-rw-r--r-- | .github/actions/test/action.yml | 128 | ||||
-rw-r--r-- | .github/actions/test_python/action.yml | 62 | ||||
-rw-r--r-- | .github/workflows/build_and_test_ondemand.yml | 137 | ||||
-rw-r--r-- | .github/workflows/build_and_test_provisioned.yml | 80 | ||||
-rw-r--r-- | .github/workflows/nightly_run.yaml | 34 | ||||
-rw-r--r-- | .github/workflows/pr_check.yml | 90 | ||||
-rw-r--r-- | .github/workflows/prepare_vm_for_build.yml | 55 |
8 files changed, 632 insertions, 0 deletions
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 0000000000..6b754d2b7a --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,46 @@ +name: Build +description: Build YDB +inputs: + sanitizer: + required: false + type: string + +runs: + using: "composite" + steps: + - name: Configure for sanitizer + shell: bash + if: inputs.sanitizer + run: | + mkdir -p ../build + patch -p1 < ydb/deploy/patches/0001-sanitizer-build.patch + cd ../build + rm -rf * + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \ + -DCMAKE_CXX_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \ + -DCMAKE_C_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \ + -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \ + ../ydb + - name: Configure + shell: bash + if: ${{!inputs.sanitizer}} + run: | + mkdir -p ../build + cd ../build + rm -rf * + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \ + -DCMAKE_CXX_FLAGS="-g" \ + -DCMAKE_C_FLAGS="-g" \ + -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \ + ../ydb + - name: Build + shell: bash + run: | + ccache -z + cd ../build + ninja + ccache -s diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 0000000000..0a3c848c84 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,128 @@ +name: build-and-test +description: Build YDB and run Tests +inputs: + log_suffix: + required: true + type: string + test_label_regexp: + required: false + type: string + aws_key_id: + required: true + type: string + aws_key_value: + required: true + type: string + testman_token: + required: false + type: string + testman_url: + required: false + type: string + testman_project_id: + required: false + type: string + aws_bucket: + required: true + type: string + aws_endpoint: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Init + id: init + shell: bash + run: | + mkdir -p artifacts tmp test_reports + rm -rf artifacts/* tmp/* test_reports/* + echo "WORKDIR=$(pwd)" >> $GITHUB_ENV + echo "TESTREPDIR=$(pwd)/test_reports" >> $GITHUB_ENV + echo "TESTMO_TOKEN=${{inputs.testman_token}}" >> $GITHUB_ENV + echo "TESTMO_URL=${{inputs.testman_url}}" >> $GITHUB_ENV + echo "logfilename=${{inputs.log_suffix}}-ctest-stdout.gz" >> $GITHUB_OUTPUT + - name: Install Node required for Testmo CLI + uses: actions/setup-node@v3 + with: + node-version: 19 + - name: Install Testmo CLI + shell: bash + run: npm install -g @testmo/testmo-cli + - name: Test history run create + id: th + if: inputs.testman_token + shell: bash + env: + PR_NUMBER: ${{ github.event.number }} + run: | + RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + BRANCH_TAG="$GITHUB_REF_NAME" + + case $GITHUB_EVENT_NAME in + workflow_dispatch) + TESTMO_RUN_NAME="${{ github.run_id }} manual" + EXTRA_TAG="manual" + ;; + pull_request | pull_request_target) + TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}" + EXTRA_TAG="pr" + BRANCH_TAG="" + ;; + schedule) + TESTMO_RUN_NAME="${{ github.run_id }} schedule" + EXTRA_TAG="schedule" + ;; + *) + TESTMO_RUN_NAME="${{ github.run_id }}" + EXTRA_TAG="" + ;; + esac + + testmo automation:resources:add-link --name build --url $RUN_URL --resources testmo.json + testmo automation:resources:add-field --name git-sha --type string --value ${GITHUB_SHA:0:7} --resources testmo.json + testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{inputs.testman_project_id}} --name "$TESTMO_RUN_NAME" \ + --source "${{inputs.log_suffix}}" --resources testmo.json \ + --tags "$BRANCH_TAG" --tags "$EXTRA_TAG" | \ + echo "runid=$(cat)" >> $GITHUB_OUTPUT + + - name: Test + shell: bash + run: | + cd $WORKDIR/../build/ydb + + echo "Stdout log (gzip archive): ${{inputs.aws_endpoint}}/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{steps.init.outputs.logfilename}}" >> $GITHUB_STEP_SUMMARY + + # Sed removes coloring from the output + + TMPDIR=$WORKDIR/tmp GTEST_OUTPUT="xml:$TESTREPDIR/unittests/" Y_UNITTEST_OUTPUT="xml:$TESTREPDIR/unittests/" \ + ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \ + --output-junit $TESTREPDIR/suites/ctest_report.xml \ + -L '${{inputs.test_label_regexp}}' | \ + sed -e 's/\x1b\[[0-9;]*m//g' | \ + tee >(gzip --stdout > $WORKDIR/artifacts/${{steps.init.outputs.logfilename}}) | \ + grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \ + tee $WORKDIR/short.log + - name: Test history upload results + if: always() && inputs.testman_token + shell: bash + run: | + testmo automation:run:submit-thread \ + --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \ + --results $TESTREPDIR/unittests/*.xml + testmo automation:run:submit-thread \ + --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \ + --results $TESTREPDIR/suites/*.xml \ + -- cat $WORKDIR/short.log + testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} + - name: Upload S3 + uses: shallwefootball/s3-upload-action@master + if: always() + with: + aws_key_id: ${{inputs.AWS_KEY_ID }} + aws_secret_access_key: ${{inputs.AWS_KEY_VALUE}} + aws_bucket: ${{inputs.aws_bucket}} + source_dir: artifacts + destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}' + endpoint: ${{inputs.aws_endpoint}} diff --git a/.github/actions/test_python/action.yml b/.github/actions/test_python/action.yml new file mode 100644 index 0000000000..f195e408f2 --- /dev/null +++ b/.github/actions/test_python/action.yml @@ -0,0 +1,62 @@ +name: test-python +description: Run functional Python tests +inputs: + log_suffix: + required: true + type: string + test_label_regexp: + required: false + type: string + aws_key_id: + required: true + type: string + aws_key_value: + required: true + type: string + aws_bucket: + required: true + type: string + aws_endpoint: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Test + shell: bash + run: | + export source_root=$(pwd) + export build_root=$(pwd)/../build/ + mkdir -p ../artifacts + rm -rf ../artifacts/* + + echo "Stdout log (gzip archive): https://storage.yandexcloud.net/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-pytest-stdout.gz" >> $GITHUB_STEP_SUMMARY + cd ${source_root}/ydb/tests/functional/ + bad_suites=$(grep -Eo 'ignore=[a-zA-Z_-]*' pytest.ini | sed -e 's/ignore=//g') + suites="" + for suite in $(echo */ | sed -e 's/\///g'); do + if [[ $(echo "$bad_suites" | grep -F -e $suite -) == '' ]]; then + suites+=$suite + suites+=$'\n' + fi + done + if [[ "${{inputs.test_label_regexp}}" != '' ]]; then + suites="${{inputs.test_label_regexp}}" + fi + source ${source_root}/ydb/tests/oss/launch/prepare.sh + echo -n "$suites" | parallel -j32 "pytest -o junit_logging=log -o junit_log_passing_tests=False \ + -v --junit-xml=${source_root}/ydb/tests/functional/test-results/xml/{}/res.xml {}" | \ + sed -e 's/\x1b\[[0-9;]*m//g' | \ + tee >(gzip --stdout > ${source_root}/../artifacts/${{inputs.log_suffix}}-pytest-stdout.gz) | \ + tee -a $GITHUB_STEP_SUMMARY + - name: Upload S3 + uses: shallwefootball/s3-upload-action@master + if: always() + with: + aws_key_id: ${{inputs.aws_key_id}} + aws_secret_access_key: ${{inputs.aws_key_value}} + aws_bucket: ${{inputs.aws_bucket}} + source_dir: ../artifacts + destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}' + endpoint: ${{inputs.aws_endpoint}} diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml new file mode 100644 index 0000000000..bc3a01ff34 --- /dev/null +++ b/.github/workflows/build_and_test_ondemand.yml @@ -0,0 +1,137 @@ +name: Build-and-Test-On-demand + +on: + workflow_call: + inputs: + sanitizer: + required: false + type: string + test_label_regexp: + required: false + type: string + image: + type: string + required: false + default: fd8earpjmhevh8h6ug5o + run_unit_tests: + type: boolean + default: true + run_functional_tests: + type: boolean + default: false + workflow_dispatch: + inputs: + sanitizer: + required: false + type: string + test_label_regexp: + required: false + type: string + image: + type: string + required: false + default: fd8earpjmhevh8h6ug5o + run_unit_tests: + type: boolean + default: true + run_functional_tests: + type: boolean + default: false + +jobs: + + provide-runner: + name: Start self-hosted YC runner + timeout-minutes: 5 + runs-on: ubuntu-latest + outputs: + label: ${{steps.start-yc-runner.outputs.label}} + instance-id: ${{steps.start-yc-runner.outputs.instance-id}} + steps: + - name: Start YC runner + id: start-yc-runner + uses: yc-actions/yc-github-runner@v1 + with: + mode: start + yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + folder-id: ${{secrets.YC_FOLDER}} + image-id: ${{inputs.image}} + disk-size: 930GB + disk-type: network-ssd-nonreplicated + cores: 32 + memory: 64GB + core-fraction: 100 + zone-id: ru-central1-b + subnet-id: ${{secrets.YC_SUBNET}} + + prepare-vm: + name: Prepare runner + uses: ./.github/workflows/prepare_vm_for_build.yml + needs: provide-runner + with: + runner_label: ${{ needs.provide-runner.outputs.label }} + secrets: + rc_auth: ${{ secrets.REMOTE_CACHE_AUTH }} + + main: + name: Build and test + needs: + - provide-runner + - prepare-vm + runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ] + steps: + - name: Checkout PR + uses: actions/checkout@v3 + if: github.event.pull_request.head.sha != '' + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout + uses: actions/checkout@v3 + if: github.event.pull_request.head.sha == '' + - name: Build + id: build + uses: ./.github/actions/build + with: + sanitizer: ${{ inputs.sanitizer }} + - name: Run unit tests + if: inputs.run_unit_tests + uses: ./.github/actions/test + with: + log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}} + test_label_regexp: ${{ inputs.test_label_regexp }} + aws_key_id: ${{secrets.AWS_KEY_ID}} + aws_key_value: ${{secrets.AWS_KEY_VALUE}} + aws_bucket: ${{vars.AWS_BUCKET}} + aws_endpoint: ${{vars.AWS_ENDPOINT}} + testman_token: ${{secrets.TESTMO_TOKEN}} + testman_url: ${{vars.TESTMO_URL}} + testman_project_id: ${{vars.TESTMO_PROJECT_ID}} + - name: Run functional tests + if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure' + uses: ./.github/actions/test_python + with: + log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}} + test_label_regexp: ${{ inputs.test_label_regexp }} + aws_key_id: ${{secrets.AWS_KEY_ID}} + aws_key_value: ${{secrets.AWS_KEY_VALUE}} + aws_bucket: ${{vars.AWS_BUCKET}} + aws_endpoint: ${{vars.AWS_ENDPOINT}} + + release-runner: + name: Release self-hosted YC runner if provided on-demand + needs: + - provide-runner # required to get output from the start-runner job + - main # required to wait when the main job is done + runs-on: ubuntu-latest + if: always() + steps: + - name: Stop YC runner + uses: yc-actions/yc-github-runner@v1 + with: + mode: stop + yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.provide-runner.outputs.label }} + instance-id: ${{ needs.provide-runner.outputs.instance-id }} + diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml new file mode 100644 index 0000000000..fa726a115d --- /dev/null +++ b/.github/workflows/build_and_test_provisioned.yml @@ -0,0 +1,80 @@ +name: Build-and-Test-Provisioned + +on: + workflow_call: + inputs: + runner_label: + required: true + type: string + run_build: + type: boolean + default: true + sanitizer: + required: false + type: string + run_unit_tests: + type: boolean + default: true + run_functional_tests: + type: boolean + default: false + test_label_regexp: + required: false + type: string + workflow_dispatch: + inputs: + runner_label: + required: true + type: string + run_build: + type: boolean + default: true + sanitizer: + required: false + type: string + run_unit_tests: + type: boolean + default: true + run_functional_tests: + type: boolean + default: true + test_label_regexp: + required: false + type: string + +jobs: + main: + name: Build and test + runs-on: [ self-hosted, Provisioned, "${{ inputs.runner_label }}" ] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Build + uses: ./.github/actions/build + if: inputs.run_build + with: + sanitizer: ${{ inputs.sanitizer }} + - name: Run unit tests + uses: ./.github/actions/test + if: inputs.run_unit_tests + with: + log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}} + test_label_regexp: ${{ inputs.test_label_regexp }} + aws_key_id: ${{secrets.AWS_KEY_ID}} + aws_key_value: ${{secrets.AWS_KEY_VALUE}} + aws_bucket: ${{vars.AWS_BUCKET}} + aws_endpoint: ${{vars.AWS_ENDPOINT}} + testman_token: ${{secrets.TESTMO_TOKEN}} + testman_url: ${{vars.TESTMO_URL}} + testman_project_id: ${{vars.TESTMO_PROJECT_ID}} + - name: Run functional tests + if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure' + uses: ./.github/actions/test_python + with: + log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}} + test_label_regexp: ${{ inputs.test_label_regexp }} + aws_key_id: ${{secrets.AWS_KEY_ID}} + aws_key_value: ${{secrets.AWS_KEY_VALUE}} + aws_bucket: ${{vars.AWS_BUCKET}} + aws_endpoint: ${{vars.AWS_ENDPOINT}} + diff --git a/.github/workflows/nightly_run.yaml b/.github/workflows/nightly_run.yaml new file mode 100644 index 0000000000..015b68cd05 --- /dev/null +++ b/.github/workflows/nightly_run.yaml @@ -0,0 +1,34 @@ +name: Nightly-run +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + test_label_regexp: + required: false + type: string + +jobs: + build_ondemand: + name: Build/test X64 + strategy: + matrix: ${{ fromJSON(vars.NIGHTLY_ONDEMAND_MATRIX) }} + fail-fast: false + + uses: ./.github/workflows/build_and_test_ondemand.yml + with: + sanitizer: ${{matrix.sanitizer}} + test_label_regexp: ${{inputs.test_label_regexp}} + secrets: inherit + + build_provisioned: + name: Build/test ARM64 + strategy: + matrix: ${{ fromJSON(vars.NIGHTLY_PROVISIONED_MATRIX) }} + fail-fast: false + uses: ./.github/workflows/build_and_test_provisioned.yml + with: + runner_label: ARM64 + sanitizer: ${{matrix.sanitizer}} + test_label_regexp: ${{inputs.test_label_regexp}} + secrets: inherit diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml new file mode 100644 index 0000000000..c50690b1de --- /dev/null +++ b/.github/workflows/pr_check.yml @@ -0,0 +1,90 @@ +name: PR check +on: + pull_request_target: + branches: + - 'main' + - 'stable-*' + paths-ignore: + - 'ydb/docs/**' + types: + - 'opened' + - 'synchronize' + - 'reopened' + - 'labeled' +jobs: + + check-running-allowed: + runs-on: ubuntu-latest + outputs: + result: ${{ steps.check-ownership-membership.outputs.result }} + steps: + - name: Check if running tests is allowed + id: check-ownership-membership + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + script: | + // This is used primarily in forks. Repository owner + // should be allowed to run anything. + const userLogin = context.payload.pull_request.user.login; + if (context.payload.repository.owner.login == userLogin) { + return true; + } + + const response = await github.rest.orgs.checkMembershipForUser({ + org: context.payload.organization.login, + username: userLogin, + }); + + // How to interpret membership status code: + // https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user + if (response.status == '204') { + return true; + } + + const labels = context.payload.pull_request.labels; + const okToTestLabel = labels.find( + label => label.name == 'ok-to-test' + ); + return okToTestLabel !== undefined; + - name: comment-if-waiting-on-ok + if: steps.check-ownership-membership.outputs.result == 'false' && + github.event.action == 'opened' + uses: actions/github-script@v6 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!' + }); + - name: cleanup-test-label + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + script: | + const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request.number; + const labelToRemove = 'ok-to-test'; + try { + const result = await github.rest.issues.removeLabel({ + owner, + repo, + issue_number: prNumber, + name: labelToRemove + }); + } catch(e) { + // ignore the 404 error that arises + // when the label did not exist for the + // organization member + console.log(e); + } + build_and_test: + needs: + - check-running-allowed + if: needs.check-running-allowed.outputs.result == 'true' + uses: ./.github/workflows/build_and_test_ondemand.yml + with: + test_label_regexp: '(SMALL|MEDIUM)' + secrets: inherit diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml new file mode 100644 index 0000000000..dab4270299 --- /dev/null +++ b/.github/workflows/prepare_vm_for_build.yml @@ -0,0 +1,55 @@ +name: Prepare VM for YDB build + +on: + workflow_call: + inputs: + runner_label: + required: true + type: string + secrets: + rc_auth: + required: false + workflow_dispatch: + inputs: + runner_label: + required: true + type: string + +jobs: + prepare: + runs-on: ${{ inputs.runner_label }} # run the job on a particular runner + steps: + - name: Install YDB Build dependencies + shell: bash + run: | + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - + echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null + echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null + sudo apt-get update + sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev parallel + sudo pip3 install conan==1.59 pytest==7.1.3 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow + - name: Install AllureCtl + if: false + shell: bash + run: | + wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl + chmod +x ~/allurectl + - name: Checkout ccache + uses: actions/checkout@v3 + with: + repository: ccache/ccache + path: ccache + - name: Build ccache + shell: bash + run: | + pwd + cd ccache + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j32 + make install + mkdir -p ~/.ccache + echo 'remote_storage = http://${{secrets.rc_auth}}${{vars.REMOTE_CACHE_URL}}' > /root/.ccache/ccache.conf + echo 'max_size = 50G' >> /root/.ccache/ccache.conf + ccache -p |