bring workflows from main

5 files changed, 396 insertions, 0 deletions

diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml
new file mode 100644
index 0000000000..bc3a01ff34
--- /dev/null
+++ b/.github/workflows/build_and_test_ondemand.yml
@@ -0,0 +1,137 @@
+name: Build-and-Test-On-demand
+
+on:
+  workflow_call:
+    inputs:
+      sanitizer:
+        required: false
+        type: string
+      test_label_regexp:
+        required: false
+        type: string
+      image:
+        type: string
+        required: false
+        default: fd8earpjmhevh8h6ug5o
+      run_unit_tests:
+        type: boolean
+        default: true
+      run_functional_tests:
+        type: boolean
+        default: false
+  workflow_dispatch:
+    inputs: 
+      sanitizer:
+        required: false
+        type: string
+      test_label_regexp:
+        required: false
+        type: string
+      image:
+        type: string
+        required: false
+        default: fd8earpjmhevh8h6ug5o
+      run_unit_tests:
+        type: boolean
+        default: true
+      run_functional_tests:
+        type: boolean
+        default: false
+    
+jobs:
+
+  provide-runner:
+    name: Start self-hosted YC runner
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{steps.start-yc-runner.outputs.label}}
+      instance-id: ${{steps.start-yc-runner.outputs.instance-id}}
+    steps:
+      - name: Start YC runner
+        id: start-yc-runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: start
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          folder-id: ${{secrets.YC_FOLDER}}
+          image-id: ${{inputs.image}}
+          disk-size: 930GB
+          disk-type: network-ssd-nonreplicated
+          cores: 32
+          memory: 64GB
+          core-fraction: 100
+          zone-id: ru-central1-b
+          subnet-id: ${{secrets.YC_SUBNET}}
+
+  prepare-vm:
+    name: Prepare runner
+    uses: ./.github/workflows/prepare_vm_for_build.yml
+    needs: provide-runner
+    with:
+      runner_label: ${{ needs.provide-runner.outputs.label }}
+    secrets:
+      rc_auth: ${{ secrets.REMOTE_CACHE_AUTH }}  
+
+  main:
+    name: Build and test
+    needs: 
+    - provide-runner
+    - prepare-vm
+    runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
+    steps:
+    - name: Checkout PR
+      uses: actions/checkout@v3
+      if: github.event.pull_request.head.sha != ''
+      with:
+          ref: ${{ github.event.pull_request.head.sha }}
+    - name: Checkout
+      uses: actions/checkout@v3
+      if: github.event.pull_request.head.sha == ''
+    - name: Build
+      id: build
+      uses: ./.github/actions/build
+      with: 
+        sanitizer: ${{ inputs.sanitizer }}
+    - name: Run unit tests
+      if: inputs.run_unit_tests
+      uses: ./.github/actions/test
+      with:
+        log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}}
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        aws_bucket: ${{vars.AWS_BUCKET}}
+        aws_endpoint: ${{vars.AWS_ENDPOINT}}
+        testman_token: ${{secrets.TESTMO_TOKEN}}
+        testman_url: ${{vars.TESTMO_URL}}
+        testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
+    - name: Run functional tests
+      if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure'
+      uses: ./.github/actions/test_python
+      with:
+        log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}}
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        aws_bucket: ${{vars.AWS_BUCKET}}
+        aws_endpoint: ${{vars.AWS_ENDPOINT}}
+
+  release-runner:
+    name: Release self-hosted YC runner if provided on-demand
+    needs:
+      - provide-runner # required to get output from the start-runner job
+      - main # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Stop YC runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: stop
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.provide-runner.outputs.label }}
+          instance-id: ${{ needs.provide-runner.outputs.instance-id }}
+          
diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml
new file mode 100644
index 0000000000..fa726a115d
--- /dev/null
+++ b/.github/workflows/build_and_test_provisioned.yml
@@ -0,0 +1,80 @@
+name: Build-and-Test-Provisioned
+
+on:
+  workflow_call:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+      run_build:
+        type: boolean
+        default: true
+      sanitizer:
+        required: false
+        type: string
+      run_unit_tests:
+        type: boolean
+        default: true
+      run_functional_tests:
+        type: boolean
+        default: false
+      test_label_regexp:
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+      run_build:
+        type: boolean
+        default: true
+      sanitizer:
+        required: false
+        type: string
+      run_unit_tests:
+        type: boolean
+        default: true
+      run_functional_tests:
+        type: boolean
+        default: true
+      test_label_regexp:
+        required: false
+        type: string
+    
+jobs:
+  main:
+    name: Build and test
+    runs-on: [ self-hosted, Provisioned, "${{ inputs.runner_label }}" ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Build
+      uses: ./.github/actions/build
+      if: inputs.run_build
+      with: 
+        sanitizer: ${{ inputs.sanitizer }}
+    - name: Run unit tests
+      uses: ./.github/actions/test
+      if: inputs.run_unit_tests
+      with: 
+        log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}}
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        aws_bucket: ${{vars.AWS_BUCKET}}
+        aws_endpoint: ${{vars.AWS_ENDPOINT}}
+        testman_token: ${{secrets.TESTMO_TOKEN}}
+        testman_url: ${{vars.TESTMO_URL}}
+        testman_project_id: ${{vars.TESTMO_PROJECT_ID}}
+    - name: Run functional tests
+      if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure'
+      uses: ./.github/actions/test_python
+      with:
+        log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}}
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+        aws_bucket: ${{vars.AWS_BUCKET}}
+        aws_endpoint: ${{vars.AWS_ENDPOINT}}
+      
diff --git a/.github/workflows/nightly_run.yaml b/.github/workflows/nightly_run.yaml
new file mode 100644
index 0000000000..015b68cd05
--- /dev/null
+++ b/.github/workflows/nightly_run.yaml
@@ -0,0 +1,34 @@
+name: Nightly-run
+on: 
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+    inputs:
+      test_label_regexp:
+        required: false
+        type: string
+  
+jobs:
+  build_ondemand:
+    name: Build/test X64
+    strategy:
+      matrix: ${{ fromJSON(vars.NIGHTLY_ONDEMAND_MATRIX) }}
+      fail-fast: false
+
+    uses: ./.github/workflows/build_and_test_ondemand.yml
+    with:
+      sanitizer: ${{matrix.sanitizer}}
+      test_label_regexp: ${{inputs.test_label_regexp}}
+    secrets: inherit
+
+  build_provisioned:
+    name: Build/test ARM64
+    strategy:
+      matrix: ${{ fromJSON(vars.NIGHTLY_PROVISIONED_MATRIX) }}
+      fail-fast: false
+    uses: ./.github/workflows/build_and_test_provisioned.yml
+    with:
+      runner_label: ARM64
+      sanitizer: ${{matrix.sanitizer}}
+      test_label_regexp: ${{inputs.test_label_regexp}}
+    secrets: inherit
diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml
new file mode 100644
index 0000000000..c50690b1de
--- /dev/null
+++ b/.github/workflows/pr_check.yml
@@ -0,0 +1,90 @@
+name: PR check
+on: 
+  pull_request_target:
+    branches:
+      - 'main'
+      - 'stable-*'
+    paths-ignore:
+      - 'ydb/docs/**'
+    types:
+      - 'opened'
+      - 'synchronize'
+      - 'reopened'
+      - 'labeled'
+jobs:
+
+  check-running-allowed:
+    runs-on: ubuntu-latest
+    outputs:
+      result: ${{ steps.check-ownership-membership.outputs.result }}
+    steps:
+      - name: Check if running tests is allowed
+        id: check-ownership-membership
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          script: |
+            // This is used primarily in forks. Repository owner 
+            // should be allowed to run anything. 
+            const userLogin = context.payload.pull_request.user.login;
+            if (context.payload.repository.owner.login == userLogin) {
+              return true;
+            }
+
+            const response = await github.rest.orgs.checkMembershipForUser({
+              org: context.payload.organization.login,
+              username: userLogin,
+            });
+
+            // How to interpret membership status code: 
+            // https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user
+            if (response.status == '204') {
+              return true;  
+            }
+
+            const labels = context.payload.pull_request.labels;
+            const okToTestLabel = labels.find(
+              label => label.name == 'ok-to-test'
+            );
+            return okToTestLabel !== undefined;
+      - name: comment-if-waiting-on-ok
+        if: steps.check-ownership-membership.outputs.result == 'false' &&
+            github.event.action == 'opened'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!'
+            });
+      - name: cleanup-test-label
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          script: |
+            const { owner, repo } = context.repo;
+            const prNumber = context.payload.pull_request.number;
+            const labelToRemove = 'ok-to-test';
+            try {
+              const result = await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: prNumber,
+                name: labelToRemove
+              });
+            } catch(e) {
+              // ignore the 404 error that arises
+              // when the label did not exist for the
+              // organization member
+              console.log(e);
+            }
+  build_and_test:
+    needs:
+      - check-running-allowed
+    if: needs.check-running-allowed.outputs.result == 'true'
+    uses: ./.github/workflows/build_and_test_ondemand.yml
+    with:
+      test_label_regexp: '(SMALL|MEDIUM)'
+    secrets: inherit
diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml
new file mode 100644
index 0000000000..dab4270299
--- /dev/null
+++ b/.github/workflows/prepare_vm_for_build.yml
@@ -0,0 +1,55 @@
+name: Prepare VM for YDB build
+
+on:
+  workflow_call:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+    secrets:
+      rc_auth:
+        required: false
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        required: true
+        type: string  
+        
+jobs:
+  prepare:
+    runs-on: ${{ inputs.runner_label }} # run the job on a particular runner
+    steps:
+    - name: Install YDB Build dependencies
+      shell: bash
+      run: |
+        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add -
+        echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
+        echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
+        sudo apt-get update
+        sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev parallel
+        sudo pip3 install conan==1.59 pytest==7.1.3 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow
+    - name: Install AllureCtl
+      if: false
+      shell: bash
+      run: |
+        wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl
+        chmod +x ~/allurectl
+    - name: Checkout ccache
+      uses: actions/checkout@v3
+      with:
+        repository: ccache/ccache
+        path: ccache
+    - name: Build ccache
+      shell: bash
+      run: |
+        pwd
+        cd ccache
+        mkdir build && cd build
+        cmake -DCMAKE_BUILD_TYPE=Release ..
+        make -j32
+        make install
+        mkdir -p ~/.ccache
+        echo 'remote_storage = http://${{secrets.rc_auth}}${{vars.REMOTE_CACHE_URL}}' > /root/.ccache/ccache.conf
+        echo 'max_size = 50G' >> /root/.ccache/ccache.conf
+        ccache -p