ci: prewarm ccache for ubuntu-18.04, 20.04, 22.04

ci: prewarm ccache for ubuntu-18.04, 20.04, 22.04

Pull Request resolved: #278

5 files changed, 162 insertions, 0 deletions

diff --git a/.github/prewarm/build.sh b/.github/prewarm/build.sh
new file mode 100644
index 0000000000..3d144264ad
--- /dev/null
+++ b/.github/prewarm/build.sh
@@ -0,0 +1,25 @@
+set -e
+
+export CONAN_USER_HOME=/ydbwork/build
+export CCACHE_SLOPPINESS=locale
+export CCACHE_BASEDIR=/ydbwork/
+
+export CONAN_USER_HOME=/ydbwork/build
+
+mkdir /ydbwork/build
+cd /ydbwork/build
+
+echo "::group::cmake"
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/ccache -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/ccache \
+-DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
+../ydb
+echo "::endgroup::"
+
+
+echo "::group::ninja"
+#ninja ydb/apps/ydb/all
+ninja
+echo "::endgroup::"
+
+ccache -s
diff --git a/.github/prewarm/ubuntu-1804.dockerfile b/.github/prewarm/ubuntu-1804.dockerfile
new file mode 100644
index 0000000000..ac5e7ba86a
--- /dev/null
+++ b/.github/prewarm/ubuntu-1804.dockerfile
@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1.0
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+## prepare system
+RUN apt-get update \
+    && apt-get install -y wget gnupg lsb-release curl xz-utils tzdata \
+    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - \
+    && echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
+    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
+    && echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" | tee /etc/apt/sources.list.d/llvm.list >/dev/null \
+    && apt-get update
+
+
+RUN apt-get install -y git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev llvm-12 \
+    && pip3 install conan==1.59 \
+    && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
+     tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
diff --git a/.github/prewarm/ubuntu-2004.dockerfile b/.github/prewarm/ubuntu-2004.dockerfile
new file mode 100644
index 0000000000..33850c3560
--- /dev/null
+++ b/.github/prewarm/ubuntu-2004.dockerfile
@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.0
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+## prepare system
+RUN apt-get update \
+    && apt-get install -y wget gnupg lsb-release curl xz-utils tzdata \
+    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - \
+    && echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
+    && apt-get update
+
+
+RUN apt-get install -y git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev llvm-12 \
+    && pip3 install conan==1.59 \
+    && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
+     tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
diff --git a/.github/prewarm/ubuntu-2204.dockerfile b/.github/prewarm/ubuntu-2204.dockerfile
new file mode 100644
index 0000000000..b50beb606d
--- /dev/null
+++ b/.github/prewarm/ubuntu-2204.dockerfile
@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:1.0
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+## prepare system
+RUN apt-get update && apt-get install -y wget gnupg lsb-release curl xz-utils tzdata
+
+
+RUN apt-get install -y git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev llvm-12 \
+    && pip3 install conan==1.59 \
+    && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
+     tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
diff --git a/.github/workflows/prewarm-ccache.yml b/.github/workflows/prewarm-ccache.yml
new file mode 100644
index 0000000000..8dc2228d98
--- /dev/null
+++ b/.github/workflows/prewarm-ccache.yml
@@ -0,0 +1,85 @@
+name: Prewarm-Ccache
+on:
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
+jobs:
+  provide-runner:
+    name: Start self-hosted YC runner
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-yc-runner.outputs.label }}
+      instance-id: ${{ steps.start-yc-runner.outputs.instance-id }}
+    steps:
+      - name: Start YC runner
+        id: start-yc-runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: start
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          folder-id: ${{ secrets.YC_FOLDER }}
+          image-id: fd8earpjmhevh8h6ug5o # TODO: create constant
+          disk-size: ${{ vars.DISK_SIZE && vars.DISK_SIZE || '279GB' }}
+          disk-type: network-ssd-nonreplicated
+          cores: 32
+          memory: 64GB
+          core-fraction: 100
+          zone-id: ru-central1-b
+          subnet-id: ${{ secrets.YC_SUBNET }}
+
+  prepare-vm:
+    name: Prepare runner
+    needs: provide-runner
+    runs-on: "${{ needs.provide-runner.outputs.label }}"
+    steps:
+      - name: install docker
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends docker.io
+  build:
+    needs:
+      - provide-runner
+      - prepare-vm
+    runs-on: "${{ needs.provide-runner.outputs.label }}"
+#    runs-on: linux
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        version: ["ubuntu-2204", "ubuntu-2004", "ubuntu-1804"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: build
+        shell: bash
+        run: |
+          echo "::group::docker-build"
+          docker build -f .github/prewarm/${{ matrix.version }}.dockerfile -t $IMG .
+          echo "::endgroup::"
+          
+          docker run --rm -v `pwd`:/ydbwork/ydb \
+            -e CCACHE_REMOTE_STORAGE="${CCACHE_REMOTE_STORAGE}" \
+            $IMG \
+            bash /ydbwork/ydb/.github/prewarm/build.sh
+        env:
+          IMG: runner-${{ matrix.version }}
+          CCACHE_REMOTE_STORAGE: ${{ vars.REMOTE_CACHE_URL && format('http://{0}{1}', secrets.REMOTE_CACHE_AUTH, vars.REMOTE_CACHE_URL) || ''}}
+  release-runner:
+    name: Release self-hosted YC runner if provided on-demand
+    needs:
+      - provide-runner
+      - build
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Stop YC runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: stop
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.provide-runner.outputs.label }}
+          instance-id: ${{ needs.provide-runner.outputs.instance-id }}