docker: build local_ydb using pyinstaller, enable s3 docker cache

docker: build local_ydb using pyinstaller, enable s3 docker cache

Pull Request resolved: 364

6 files changed, 77 insertions, 43 deletions

diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile
index 5651e27dd6..9389feea50 100644
--- a/.github/docker/Dockerfile
+++ b/.github/docker/Dockerfile
@@ -1,5 +1,4 @@
 # syntax=docker/dockerfile:1
-FROM cr.yandex/yc/yandex-docker-local-ydb@sha256:ce95513262a31fbe5b309521c362bde6d24ea68b7957bd874ef43d985a5d5e45 AS yandex_local_build
 FROM ubuntu:22.04 AS builder
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -7,26 +6,17 @@ ENV TZ=Etc/UTC
 
 
 RUN apt-get update \
-    && apt-get install -y sudo wget gnupg lsb-release curl xz-utils tzdata \
-    cmake python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
-    clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 \
-    && pip3 install conan==1.59 grpcio-tools \
+    && apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata \
+    cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
+    clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 file \
+    && pip3 install conan==1.59 grpcio-tools pyinstaller==5.13.2 six pyyaml packaging PyHamcrest cryptography \
     && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
          tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
 
 
-RUN apt-get install -y --no-install-recommends git
+COPY ydb/ /ydbwork/ydb/
 
-WORKDIR /ydbwork
-
-ARG GIT_REPO=https://github.com/ydb-platform/ydb.git
-ARG GIT_REF=main
-
-ENV GIT_REPO=$GIT_REPO
-ENV GIT_REF=$GIT_REF
-
-RUN mkdir ydb \
-    && git clone --depth=1 -b "${GIT_REF?}" ${GIT_REPO?}
+WORKDIR /ydbwork/
 
 RUN --mount=type=secret,id=ccache_remote_storage \
     mkdir build && cd build \
@@ -34,6 +24,8 @@ RUN --mount=type=secret,id=ccache_remote_storage \
     && export CCACHE_BASEDIR=/ydbwork/ \
     && export CCACHE_SLOPPINESS=locale \
     && export CCACHE_REMOTE_STORAGE="$(cat /run/secrets/ccache_remote_storage)" \
+    && export CC=/usr/bin/clang-14 \
+    && export CC_FOR_BUILD=$CC \
     && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
         -DCCACHE_PATH=/usr/local/bin/ccache \
         -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
@@ -41,7 +33,24 @@ RUN --mount=type=secret,id=ccache_remote_storage \
     && ninja ydb/apps/ydbd/ydbd ydb/apps/ydb/all \
     && ccache -s \
     && strip ydb/apps/ydbd/ydbd \
-    && strip ydb/apps/ydb/ydb
+    && strip ydb/apps/ydb/ydb \
+    && mv ./ydb/apps/ydbd/ydbd / \
+    && mv ./ydb/apps/ydb/ydb / \
+    && /ydbd -V  \
+    && /ydb version \
+    && cd .. && rm -rf build  # for reduce cache size
+
+
+
+# always use local_ydb.spec from main revision
+COPY main/ydb/public/tools/local_ydb/local_ydb.spec ydb/ydb/public/tools/local_ydb/
+
+RUN cd ydb && \
+    ./ydb/tests/oss/launch/compile_protos.sh . ydb library/cpp/actors && \
+    cd ydb/public/tools/local_ydb/ && \
+    pyinstaller local_ydb.spec && \
+    ./dist/local_ydb --help
+
 
 FROM ubuntu:22.04
 RUN apt-get update  \
@@ -53,18 +62,10 @@ RUN mkdir -p /root/ydb/bin/ \
     && mkdir -p ydb_certs \
     && echo '{"check_version":false}' > root/ydb/bin/config.json
 
-COPY --from=builder /ydbwork/build/ydb/apps/ydbd/ydbd /ydbwork/build/ydb/apps/ydb/ydb /
-
-COPY --from=yandex_local_build /local_ydb /
-
-COPY files/initialize_local_ydb.sh /initialize_local_ydb
-COPY files/health_check.sh /health_check
-COPY files/THIRD_PARTY_LICENSES /THIRD_PARTY_LICENSES
-COPY files/LICENSE /LICENSE
-
+COPY main/.github/docker/files/ /
+COPY --from=builder /ydbwork/ydb/ydb/public/tools/local_ydb/dist/local_ydb /
+COPY --from=builder /ydbd /ydb /
 
-RUN /ydbd -V
-RUN /ydb version
 
 # YDB grpc
 EXPOSE ${GRPC_TLS_PORT:-2135}
diff --git a/.github/docker/Dockerfile.dockerignore b/.github/docker/Dockerfile.dockerignore
new file mode 100644
index 0000000000..f3b64113ec
--- /dev/null
+++ b/.github/docker/Dockerfile.dockerignore
@@ -0,0 +1 @@
+**/.git
diff --git a/.github/docker/files/health_check.sh b/.github/docker/files/health_check
index c51e626f07..c51e626f07 100644
--- a/.github/docker/files/health_check.sh
+++ b/.github/docker/files/health_check
diff --git a/.github/docker/files/initialize_local_ydb.sh b/.github/docker/files/initialize_local_ydb
index c77725ce4f..c77725ce4f 100644
--- a/.github/docker/files/initialize_local_ydb.sh
+++ b/.github/docker/files/initialize_local_ydb
diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml
index a3eebe195c..e42441b2d4 100644
--- a/.github/workflows/docker_publish.yml
+++ b/.github/workflows/docker_publish.yml
@@ -15,6 +15,11 @@ on:
         required: true
         default: trunk
         description: "docker image tag"
+      local_ydb_ref:
+        type: string
+        required: true
+        default: main
+        description: "Git branch/tag revision to builld local_ydb"
 
 jobs:
   provide-runner:
@@ -47,14 +52,10 @@ jobs:
     needs: provide-runner
     runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
     steps:
-      - name: Checkout PR
-        uses: actions/checkout@v3
-        with:
-          sparse-checkout: .github
       - name: install docker
         shell: bash
         run: |
-          apt-get update 
+          apt-get update
           apt-get install -y --no-install-recommends docker.io
 
   build:
@@ -63,10 +64,27 @@ jobs:
       - prepare-vm
     runs-on: "${{ needs.provide-runner.outputs.label }}"
     steps:
+      - name: Checkout .github and local_ydb
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
+          sparse-checkout: |
+            .github
+            ydb/public/tools/local_ydb/
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          sparse-checkout: .github
+          ref: ${{ inputs.git_ref || 'main' }}
+          path: ydb
+
+      - name: get revision
+        shell: bash
+        id: get-sha
+        working-directory: ydb
+        run: |
+          echo "SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -78,18 +96,31 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/local-ydb
+          labels: |
+            ydb.revision=${{ steps.get-sha.outputs.SHA }}
+            org.opencontainers.image.revision=${{ steps.get-sha.outputs.SHA }}
+          tags: |
+            type=schedule,pattern=nightly
+            type=raw,value=${{ inputs.image_tag || 'trunk' }}
+
       - name: Build and push docker image
         uses: docker/build-push-action@v4
         with:
           push: true
-          context: .github/docker/
-          file: .github/docker/Dockerfile
-          tags: ghcr.io/${{ github.repository_owner }}/local-ydb:${{ inputs.image_tag || 'trunk' }}
+          context: .
+          file: main/.github/docker/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64
           provenance: false
-          build-args: |
-            GIT_REPO=${{ github.server_url }}/${{ github.repository }}
-            GIT_REF=${{ inputs.git_ref || 'main' }}
+          cache-from: type=s3,name=local_ydb,region=ru-central1,bucket=${{ vars.AWS_BUCKET }},endpoint_url=${{ vars.AWS_ENDPOINT }},access_key_id=${{ secrets.AWS_KEY_ID }},secret_access_key=${{ secrets.AWS_KEY_VALUE }}
+          cache-to: type=s3,name=local_ydb,region=ru-central1,bucket=${{ vars.AWS_BUCKET }},endpoint_url=${{ vars.AWS_ENDPOINT }},access_key_id=${{ secrets.AWS_KEY_ID }},secret_access_key=${{ secrets.AWS_KEY_VALUE }},mode=max
           secrets: |
             "ccache_remote_storage=${{ vars.REMOTE_CACHE_URL && format('http://{0}{1}', secrets.REMOTE_CACHE_AUTH, vars.REMOTE_CACHE_URL) || ''}}"
 
diff --git a/.mapping.json b/.mapping.json
index 5e2660d705..623a3d1bd2 100644
--- a/.mapping.json
+++ b/.mapping.json
@@ -7,10 +7,11 @@
   ".github/config/muted_shard.txt":"ydb/github_toplevel/.github/config/muted_shard.txt",
   ".github/config/muted_test.txt":"ydb/github_toplevel/.github/config/muted_test.txt",
   ".github/docker/Dockerfile":"ydb/github_toplevel/.github/docker/Dockerfile",
+  ".github/docker/Dockerfile.dockerignore":"ydb/github_toplevel/.github/docker/Dockerfile.dockerignore",
   ".github/docker/files/LICENSE":"ydb/github_toplevel/.github/docker/files/LICENSE",
   ".github/docker/files/THIRD_PARTY_LICENSES":"ydb/github_toplevel/.github/docker/files/THIRD_PARTY_LICENSES",
-  ".github/docker/files/health_check.sh":"ydb/github_toplevel/.github/docker/files/health_check.sh",
-  ".github/docker/files/initialize_local_ydb.sh":"ydb/github_toplevel/.github/docker/files/initialize_local_ydb.sh",
+  ".github/docker/files/health_check":"ydb/github_toplevel/.github/docker/files/health_check",
+  ".github/docker/files/initialize_local_ydb":"ydb/github_toplevel/.github/docker/files/initialize_local_ydb",
   ".github/prewarm/build.sh":"ydb/github_toplevel/.github/prewarm/build.sh",
   ".github/prewarm/ubuntu-1804.dockerfile":"ydb/github_toplevel/.github/prewarm/ubuntu-1804.dockerfile",
   ".github/prewarm/ubuntu-2004.dockerfile":"ydb/github_toplevel/.github/prewarm/ubuntu-2004.dockerfile",