aboutsummaryrefslogtreecommitdiffstats
path: root/.github/workflows
diff options
context:
space:
mode:
authorDaniil Cherednik <dan.cherednik@gmail.com>2023-06-15 18:24:59 +0300
committerDaniil Cherednik <dan.cherednik@gmail.com>2023-06-15 18:26:33 +0300
commit068d4453cf9fc68c875eee73f5c637bb076f6a71 (patch)
treeda3e83fdb9488ea08faa39d8b41916744f9acad7 /.github/workflows
parent7e7de263d4acbc6eacf92b618bcb5f9049bccc9b (diff)
downloadydb-068d4453cf9fc68c875eee73f5c637bb076f6a71.tar.gz
Create stable-23-2 branch
x-stable-origin-commit: 3fd4d58117c143ed9e6b21813ccd9e507d2cd4d3
Diffstat (limited to '.github/workflows')
-rw-r--r--.github/workflows/build_and_test_ondemand.yml137
-rw-r--r--.github/workflows/build_and_test_provisioned.yml80
-rw-r--r--.github/workflows/nightly_run.yaml34
-rw-r--r--.github/workflows/pr_check.yml90
-rw-r--r--.github/workflows/prepare_vm_for_build.yml55
5 files changed, 0 insertions, 396 deletions
diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml
deleted file mode 100644
index bc3a01ff34..0000000000
--- a/.github/workflows/build_and_test_ondemand.yml
+++ /dev/null
@@ -1,137 +0,0 @@
-name: Build-and-Test-On-demand
-
-on:
- workflow_call:
- inputs:
- sanitizer:
- required: false
- type: string
- test_label_regexp:
- required: false
- type: string
- image:
- type: string
- required: false
- default: fd8earpjmhevh8h6ug5o
- run_unit_tests:
- type: boolean
- default: true
- run_functional_tests:
- type: boolean
- default: false
- workflow_dispatch:
- inputs:
- sanitizer:
- required: false
- type: string
- test_label_regexp:
- required: false
- type: string
- image:
- type: string
- required: false
- default: fd8earpjmhevh8h6ug5o
- run_unit_tests:
- type: boolean
- default: true
- run_functional_tests:
- type: boolean
- default: false
-
-jobs:
-
- provide-runner:
- name: Start self-hosted YC runner
- timeout-minutes: 5
- runs-on: ubuntu-latest
- outputs:
- label: ${{steps.start-yc-runner.outputs.label}}
- instance-id: ${{steps.start-yc-runner.outputs.instance-id}}
- steps:
- - name: Start YC runner
- id: start-yc-runner
- uses: yc-actions/yc-github-runner@v1
- with:
- mode: start
- yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
- folder-id: ${{secrets.YC_FOLDER}}
- image-id: ${{inputs.image}}
- disk-size: 930GB
- disk-type: network-ssd-nonreplicated
- cores: 32
- memory: 64GB
- core-fraction: 100
- zone-id: ru-central1-b
- subnet-id: ${{secrets.YC_SUBNET}}
-
- prepare-vm:
- name: Prepare runner
- uses: ./.github/workflows/prepare_vm_for_build.yml
- needs: provide-runner
- with:
- runner_label: ${{ needs.provide-runner.outputs.label }}
- secrets:
- rc_auth: ${{ secrets.REMOTE_CACHE_AUTH }}
-
- main:
- name: Build and test
- needs:
- - provide-runner
- - prepare-vm
- runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
- steps:
- - name: Checkout PR
- uses: actions/checkout@v3
- if: github.event.pull_request.head.sha != ''
- with:
- ref: ${{ github.event.pull_request.head.sha }}
- - name: Checkout
- uses: actions/checkout@v3
- if: github.event.pull_request.head.sha == ''
- - name: Build
- id: build
- uses: ./.github/actions/build
- with:
- sanitizer: ${{ inputs.sanitizer }}
- - name: Run unit tests
- if: inputs.run_unit_tests
- uses: ./.github/actions/test
- with:
- log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}}
- test_label_regexp: ${{ inputs.test_label_regexp }}
- aws_key_id: ${{secrets.AWS_KEY_ID}}
- aws_key_value: ${{secrets.AWS_KEY_VALUE}}
- aws_bucket: ${{vars.AWS_BUCKET}}
- aws_endpoint: ${{vars.AWS_ENDPOINT}}
- testman_token: ${{secrets.TESTMO_TOKEN}}
- testman_url: ${{vars.TESTMO_URL}}
- testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
- - name: Run functional tests
- if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure'
- uses: ./.github/actions/test_python
- with:
- log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}}
- test_label_regexp: ${{ inputs.test_label_regexp }}
- aws_key_id: ${{secrets.AWS_KEY_ID}}
- aws_key_value: ${{secrets.AWS_KEY_VALUE}}
- aws_bucket: ${{vars.AWS_BUCKET}}
- aws_endpoint: ${{vars.AWS_ENDPOINT}}
-
- release-runner:
- name: Release self-hosted YC runner if provided on-demand
- needs:
- - provide-runner # required to get output from the start-runner job
- - main # required to wait when the main job is done
- runs-on: ubuntu-latest
- if: always()
- steps:
- - name: Stop YC runner
- uses: yc-actions/yc-github-runner@v1
- with:
- mode: stop
- yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
- label: ${{ needs.provide-runner.outputs.label }}
- instance-id: ${{ needs.provide-runner.outputs.instance-id }}
-
diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml
deleted file mode 100644
index fa726a115d..0000000000
--- a/.github/workflows/build_and_test_provisioned.yml
+++ /dev/null
@@ -1,80 +0,0 @@
-name: Build-and-Test-Provisioned
-
-on:
- workflow_call:
- inputs:
- runner_label:
- required: true
- type: string
- run_build:
- type: boolean
- default: true
- sanitizer:
- required: false
- type: string
- run_unit_tests:
- type: boolean
- default: true
- run_functional_tests:
- type: boolean
- default: false
- test_label_regexp:
- required: false
- type: string
- workflow_dispatch:
- inputs:
- runner_label:
- required: true
- type: string
- run_build:
- type: boolean
- default: true
- sanitizer:
- required: false
- type: string
- run_unit_tests:
- type: boolean
- default: true
- run_functional_tests:
- type: boolean
- default: true
- test_label_regexp:
- required: false
- type: string
-
-jobs:
- main:
- name: Build and test
- runs-on: [ self-hosted, Provisioned, "${{ inputs.runner_label }}" ]
- steps:
- - name: Checkout
- uses: actions/checkout@v3
- - name: Build
- uses: ./.github/actions/build
- if: inputs.run_build
- with:
- sanitizer: ${{ inputs.sanitizer }}
- - name: Run unit tests
- uses: ./.github/actions/test
- if: inputs.run_unit_tests
- with:
- log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}}
- test_label_regexp: ${{ inputs.test_label_regexp }}
- aws_key_id: ${{secrets.AWS_KEY_ID}}
- aws_key_value: ${{secrets.AWS_KEY_VALUE}}
- aws_bucket: ${{vars.AWS_BUCKET}}
- aws_endpoint: ${{vars.AWS_ENDPOINT}}
- testman_token: ${{secrets.TESTMO_TOKEN}}
- testman_url: ${{vars.TESTMO_URL}}
- testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
- - name: Run functional tests
- if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure'
- uses: ./.github/actions/test_python
- with:
- log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}}
- test_label_regexp: ${{ inputs.test_label_regexp }}
- aws_key_id: ${{secrets.AWS_KEY_ID}}
- aws_key_value: ${{secrets.AWS_KEY_VALUE}}
- aws_bucket: ${{vars.AWS_BUCKET}}
- aws_endpoint: ${{vars.AWS_ENDPOINT}}
-
diff --git a/.github/workflows/nightly_run.yaml b/.github/workflows/nightly_run.yaml
deleted file mode 100644
index 015b68cd05..0000000000
--- a/.github/workflows/nightly_run.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: Nightly-run
-on:
- schedule:
- - cron: "0 1 * * *"
- workflow_dispatch:
- inputs:
- test_label_regexp:
- required: false
- type: string
-
-jobs:
- build_ondemand:
- name: Build/test X64
- strategy:
- matrix: ${{ fromJSON(vars.NIGHTLY_ONDEMAND_MATRIX) }}
- fail-fast: false
-
- uses: ./.github/workflows/build_and_test_ondemand.yml
- with:
- sanitizer: ${{matrix.sanitizer}}
- test_label_regexp: ${{inputs.test_label_regexp}}
- secrets: inherit
-
- build_provisioned:
- name: Build/test ARM64
- strategy:
- matrix: ${{ fromJSON(vars.NIGHTLY_PROVISIONED_MATRIX) }}
- fail-fast: false
- uses: ./.github/workflows/build_and_test_provisioned.yml
- with:
- runner_label: ARM64
- sanitizer: ${{matrix.sanitizer}}
- test_label_regexp: ${{inputs.test_label_regexp}}
- secrets: inherit
diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml
deleted file mode 100644
index c50690b1de..0000000000
--- a/.github/workflows/pr_check.yml
+++ /dev/null
@@ -1,90 +0,0 @@
-name: PR check
-on:
- pull_request_target:
- branches:
- - 'main'
- - 'stable-*'
- paths-ignore:
- - 'ydb/docs/**'
- types:
- - 'opened'
- - 'synchronize'
- - 'reopened'
- - 'labeled'
-jobs:
-
- check-running-allowed:
- runs-on: ubuntu-latest
- outputs:
- result: ${{ steps.check-ownership-membership.outputs.result }}
- steps:
- - name: Check if running tests is allowed
- id: check-ownership-membership
- uses: actions/github-script@v6
- with:
- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
- script: |
- // This is used primarily in forks. Repository owner
- // should be allowed to run anything.
- const userLogin = context.payload.pull_request.user.login;
- if (context.payload.repository.owner.login == userLogin) {
- return true;
- }
-
- const response = await github.rest.orgs.checkMembershipForUser({
- org: context.payload.organization.login,
- username: userLogin,
- });
-
- // How to interpret membership status code:
- // https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user
- if (response.status == '204') {
- return true;
- }
-
- const labels = context.payload.pull_request.labels;
- const okToTestLabel = labels.find(
- label => label.name == 'ok-to-test'
- );
- return okToTestLabel !== undefined;
- - name: comment-if-waiting-on-ok
- if: steps.check-ownership-membership.outputs.result == 'false' &&
- github.event.action == 'opened'
- uses: actions/github-script@v6
- with:
- script: |
- github.rest.issues.createComment({
- issue_number: context.issue.number,
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!'
- });
- - name: cleanup-test-label
- uses: actions/github-script@v6
- with:
- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
- script: |
- const { owner, repo } = context.repo;
- const prNumber = context.payload.pull_request.number;
- const labelToRemove = 'ok-to-test';
- try {
- const result = await github.rest.issues.removeLabel({
- owner,
- repo,
- issue_number: prNumber,
- name: labelToRemove
- });
- } catch(e) {
- // ignore the 404 error that arises
- // when the label did not exist for the
- // organization member
- console.log(e);
- }
- build_and_test:
- needs:
- - check-running-allowed
- if: needs.check-running-allowed.outputs.result == 'true'
- uses: ./.github/workflows/build_and_test_ondemand.yml
- with:
- test_label_regexp: '(SMALL|MEDIUM)'
- secrets: inherit
diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml
deleted file mode 100644
index dab4270299..0000000000
--- a/.github/workflows/prepare_vm_for_build.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: Prepare VM for YDB build
-
-on:
- workflow_call:
- inputs:
- runner_label:
- required: true
- type: string
- secrets:
- rc_auth:
- required: false
- workflow_dispatch:
- inputs:
- runner_label:
- required: true
- type: string
-
-jobs:
- prepare:
- runs-on: ${{ inputs.runner_label }} # run the job on a particular runner
- steps:
- - name: Install YDB Build dependencies
- shell: bash
- run: |
- wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
- wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add -
- echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
- echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
- sudo apt-get update
- sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev parallel
- sudo pip3 install conan==1.59 pytest==7.1.3 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow
- - name: Install AllureCtl
- if: false
- shell: bash
- run: |
- wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl
- chmod +x ~/allurectl
- - name: Checkout ccache
- uses: actions/checkout@v3
- with:
- repository: ccache/ccache
- path: ccache
- - name: Build ccache
- shell: bash
- run: |
- pwd
- cd ccache
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j32
- make install
- mkdir -p ~/.ccache
- echo 'remote_storage = http://${{secrets.rc_auth}}${{vars.REMOTE_CACHE_URL}}' > /root/.ccache/ccache.conf
- echo 'max_size = 50G' >> /root/.ccache/ccache.conf
- ccache -p