on-demand build

7 files changed, 276 insertions, 79 deletions

diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
new file mode 100644
index 00000000000..58d3353deb3
--- /dev/null
+++ b/.github/actions/build/action.yml
@@ -0,0 +1,38 @@
+name: Build
+description: Build YDB
+inputs:
+  sanitizer:
+    required: false
+    type: string
+    
+runs:
+  using: "composite"
+  steps:
+  - name: Configure for sanitizer
+    shell: bash
+    if: startsWith(inputs.sanitizer,'sanitize=')
+    run: |
+      mkdir -p ../build
+      patch -p1 < ydb/deploy/patches/0001-sanitizer-build.patch
+      cd ../build
+      rm -rf *
+      cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
+        -DCMAKE_CXX_FLAGS="-f${{ inputs.sanitizer }} -fno-omit-frame-pointer" \
+        -DCMAKE_C_FLAGS="-f${{ inputs.sanitizer }} -fno-omit-frame-pointer" \
+        ../ydb
+  - name: Configure
+    shell: bash
+    if: false == startsWith(inputs.sanitizer,'sanitize=')
+    run: |
+      mkdir -p ../build
+      cd ../build
+      rm -rf *
+      cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain ../ydb
+  - name: Build
+    shell: bash
+    run: |
+      ccache -z
+      cd ../build
+      ninja
+      ccache -s
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
new file mode 100644
index 00000000000..bc40b2a07c1
--- /dev/null
+++ b/.github/actions/test/action.yml
@@ -0,0 +1,49 @@
+name: build-and-test
+description: Build YDB and run Tests
+inputs:
+  log_suffix:
+    required: true
+    type: string
+  test_label_regexp:
+    required: false
+    type: string
+  aws_key_id:
+    required: true
+    type: string
+  aws_key_value: 
+    required: trye
+    type: string
+    
+runs:
+  using: "composite"
+  steps:
+  - name: Test
+    shell: bash
+    run: |
+      cd ../build/ydb
+      rm -rf $(pwd)/../../tmp/*
+      rm -rf $(pwd)/../../test_reports/*
+      mkdir -p ../../artifacts
+      rm -rf $(pwd)/../../artifacts/*
+
+      echo "Stdout log (gzip archive): https://storage.yandexcloud.net/ydb-tech-ci/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-stdout.gz" >> $GITHUB_STEP_SUMMARY
+
+      # Sed removes coloring from the output
+      TMPDIR=$(pwd)/../../tmp GTEST_OUTPUT="xml:$(pwd)/../../test_reports/" \
+        ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \
+              --output-junit $(pwd)/../../test_reports/ctest_report.xml \
+              -L '${{inputs.test_label_regexp}}' | \
+        sed -e 's/\x1b\[[0-9;]*m//g' | \
+        tee >(gzip --stdout > ../../artifacts/${{inputs.log_suffix}}-${{inputs.sanitizer}}-stdout.gz) | \
+        grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \
+        tee -a $GITHUB_STEP_SUMMARY
+  - name: Upload S3
+    uses: shallwefootball/s3-upload-action@master
+    if: always()
+    with:
+      aws_key_id: ${{inputs.AWS_KEY_ID }}
+      aws_secret_access_key: ${{inputs.AWS_KEY_VALUE}}
+      aws_bucket: ydb-tech-ci
+      source_dir: ../artifacts
+      destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}'
+      endpoint: https://storage.yandexcloud.net
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
deleted file mode 100644
index 94d94541e1d..00000000000
--- a/.github/workflows/build_and_test.yml
+++ /dev/null
@@ -1,75 +0,0 @@
-name: Build-and-Test
-
-on:
-  workflow_call:
-    inputs:
-      runner_label:
-        required: true
-        type: string
-      sanitizer:
-        required: false
-        type: string
-      test_label_regexp:
-        required: false
-        type: string
-    
-jobs:
-  main:
-
-    runs-on: [ self-hosted, "${{ inputs.runner_label }}" ]
-    
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Configure for sanitizer
-      shell: bash
-      if: startsWith(inputs.sanitizer,'sanitize=')
-      run: |
-        mkdir -p ../build
-        patch -p1 < ydb/deploy/patches/0001-sanitizer-build.patch
-        cd ../build
-        rm -rf *
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain -DCMAKE_CXX_FLAGS="${{ matrix.sanitizer }} -fno-omit-frame-pointer" -DCMAKE_C_FLAGS="${{ matrix.sanitizer }} -fno-omit-frame-pointer" ../ydb
-    - name: Configure
-      shell: bash
-      if: false == startsWith(inputs.sanitizer,'sanitize=')
-      run: |
-        mkdir -p ../build
-        cd ../build
-        rm -rf *
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain ../ydb
-    - name: Build
-      shell: bash
-      run: |
-        cd ../build
-        ninja
-    - name: Test
-      continue-on-error: true
-      shell: bash
-      run: |
-        cd ../build/ydb
-        rm -rf $(pwd)/../../tmp/*
-        rm -rf $(pwd)/../../test_reports/*
-        mkdir -p ../../artifacts
-        rm -rf $(pwd)/../../artifacts/*
-        
-        # Sed removes coloring from the output
-        TMPDIR=$(pwd)/../../tmp GTEST_OUTPUT="xml:$(pwd)/../../test_reports/" \
-          ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \
-                --output-junit $(pwd)/../../test_reports/ctest_report.xml \
-                -L ${{inputs.test_label_regexp}} | \
-          sed -e 's/\x1b\[[0-9;]*m//g' | \
-          tee >(gzip --stdout > ../../artifacts/${{inputs.runner_label}}-${{inputs.sanitizer}}-stdout.gz) | \
-          grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \
-          tee -a $GITHUB_STEP_SUMMARY
-        echo "Stdout log (gzip archive): https://storage.yandexcloud.net/ydb-tech-ci/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.runner_label}}-${{inputs.sanitizer}}-stdout.gz" >> $GITHUB_STEP_SUMMARY
-    - name: Upload S3
-      uses: shallwefootball/s3-upload-action@master
-      if: always()
-      with:
-        aws_key_id: ${{ secrets.AWS_KEY_ID }}
-        aws_secret_access_key: ${{ secrets.AWS_KEY_VALUE}}
-        aws_bucket: ydb-tech-ci
-        source_dir: ../artifacts
-        destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}'
-        endpoint: https://storage.yandexcloud.net
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml
new file mode 100644
index 00000000000..26ae7e8a35e
--- /dev/null
+++ b/.github/workflows/build_and_test_ondemand.yml
@@ -0,0 +1,86 @@
+name: Build-and-Test On-demand VM
+
+on:
+  workflow_call:
+    inputs:
+      sanitizer:
+        required: false
+        type: string
+      test_label_regexp:
+        required: false
+        type: string
+    
+jobs:
+
+  provide-runner:
+    name: Start self-hosted YC runner
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{steps.start-yc-runner.outputs.label}}
+      instance-id: ${{steps.start-yc-runner.outputs.instance-id}}
+    steps:
+      - name: Start YC runner
+        id: start-yc-runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: start
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          folder-id: ${{secrets.YC_FOLDER}}
+          image-id: fd8snjpoq85qqv0mk9gi
+          disk-size: 930GB
+          disk-type: network-ssd-nonreplicated
+          cores: 32
+          memory: 64GB
+          core-fraction: 100
+          zone-id: ru-central1-b
+          subnet-id: ${{secrets.YC_SUBNET}}
+
+  prepare-vm:
+    name: Prepare runner
+    uses: ./.github/workflows/prepare_vm_for_build.yml
+    needs: provide-runner
+    with:
+      runner_label: ${{ needs.provide-runner.outputs.label }}
+    secrets:
+      rc_auth: ${{ secrets.REMOTE_CACHE_AUTH }}  
+
+  main:
+    name: Build and test
+    needs: 
+    - provide-runner
+    - prepare-vm
+    runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Build
+      uses: ./.github/actions/build
+      with: 
+        sanitizer: ${{ inputs.sanitizer }}
+    - name: Test
+      uses: ./.github/actions/test
+      with: 
+        log_suffix: X64
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
+
+  release-runner:
+    name: Release self-hosted YC runner if provided on-demand
+    needs:
+      - provide-runner # required to get output from the start-runner job
+      - main # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Stop YC runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: stop
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.provide-runner.outputs.label }}
+          instance-id: ${{ needs.provide-runner.outputs.instance-id }}
+  
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml
new file mode 100644
index 00000000000..a955dbe7d9b
--- /dev/null
+++ b/.github/workflows/build_and_test_provisioned.yml
@@ -0,0 +1,33 @@
+name: Build-and-Test Provisioned VM
+
+on:
+  workflow_call:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+      sanitizer:
+        required: false
+        type: string
+      test_label_regexp:
+        required: false
+        type: string
+    
+jobs:
+  main:
+    name: Build and test
+    runs-on: [ self-hosted, Provisioned, "${{ inputs.runner_label }}" ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Build
+      uses: ./.github/actions/build
+      with: 
+        sanitizer: ${{ inputs.sanitizer }}
+    - name: Test
+      uses: ./.github/actions/test
+      with: 
+        log_suffix: ${{ inputs.runner_label }}
+        test_label_regexp: ${{ inputs.test_label_regexp }}
+        aws_key_id: ${{secrets.AWS_KEY_ID}}
+        aws_key_value: ${{secrets.AWS_KEY_VALUE}}
diff --git a/.github/workflows/nightly_run.yaml b/.github/workflows/nightly_run.yaml
index 1f8ceb41c8d..3524b3a9e2b 100644
--- a/.github/workflows/nightly_run.yaml
+++ b/.github/workflows/nightly_run.yaml
@@ -3,17 +3,34 @@ on:
   schedule:
     - cron: "0 1 * * *"
   workflow_dispatch:
+    inputs:
+      test_label_regexp:
+        required: false
+        type: string
   
 jobs:
-  build:
+  build_ondemand:
+    name: Build/test X64
     strategy:
       matrix:
-        arch: [ X64, ARM64 ]
         sanitizer: [ "", "sanitize=address" ]
       fail-fast: false
 
-    uses: ./.github/workflows/build_and_test.yml
+    uses: ./.github/workflows/build_and_test_ondemand.yml
     with:
-      runner_label: ${{matrix.arch}}
       sanitizer: ${{matrix.sanitizer}}
+      test_label_regexp: ${{inputs.test_label_regexp}}
     secrets: inherit
+
+  build_provisioned:
+    name: Build/test ARM64
+    strategy:
+      matrix:
+        sanitizer: [ "", "sanitize=address" ]
+      fail-fast: false
+    uses: ./.github/workflows/build_and_test_provisioned.yml
+    with:
+      runner_label: ARM64
+      sanitizer: ${{matrix.sanitizer}}
+      test_label_regexp: ${{inputs.test_label_regexp}}
+    secrets: inherit
\ No newline at end of file
diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml
new file mode 100644
index 00000000000..6804f87d7b4
--- /dev/null
+++ b/.github/workflows/prepare_vm_for_build.yml
@@ -0,0 +1,49 @@
+name: Prepare VM for YDB build
+
+on:
+  workflow_call:
+    inputs:
+      runner_label:
+        required: true
+        type: string
+    secrets:
+      rc_auth:
+        required: false
+        
+jobs:
+  prepare:
+    runs-on: ${{ inputs.runner_label }} # run the job on a particular runner
+    steps:
+    - name: Install YDB Build dependencies
+      shell: bash
+      run: |
+        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add -
+        echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
+        echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
+        sudo apt-get update
+        sudo apt-get -y install git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev
+        sudo pip3 install conan==1.59
+    - name: Install AllureCtl
+      shell: bash
+      run: |
+        wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl
+        chmod +x ~/allurectl
+    - name: Checkout ccache
+      uses: actions/checkout@v3
+      with:
+        repository: ccache/ccache
+        path: ccache
+    - name: Build ccache
+      shell: bash
+      run: |
+        pwd
+        cd ccache
+        mkdir build && cd build
+        cmake -DCMAKE_BUILD_TYPE=Release ..
+        make -j32
+        make install
+        mkdir -p ~/.ccache
+        echo 'remote_storage = http://${{secrets.rc_auth}}${{vars.REMOTE_CACHE_URL}}' > /root/.ccache/ccache.conf
+        echo 'max_size = 50G' >> /root/.ccache/ccache.conf
+        ccache -p