docker: build local_ydb using pyinstaller, enable s3 docker cache

docker: build local_ydb using pyinstaller, enable s3 docker cache

Pull Request resolved: 364

1 files changed, 29 insertions, 28 deletions

diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile
index 5651e27dd6..9389feea50 100644
--- a/.github/docker/Dockerfile
+++ b/.github/docker/Dockerfile
@@ -1,5 +1,4 @@
 # syntax=docker/dockerfile:1
-FROM cr.yandex/yc/yandex-docker-local-ydb@sha256:ce95513262a31fbe5b309521c362bde6d24ea68b7957bd874ef43d985a5d5e45 AS yandex_local_build
 FROM ubuntu:22.04 AS builder
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -7,26 +6,17 @@ ENV TZ=Etc/UTC
 
 
 RUN apt-get update \
-    && apt-get install -y sudo wget gnupg lsb-release curl xz-utils tzdata \
-    cmake python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
-    clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 \
-    && pip3 install conan==1.59 grpcio-tools \
+    && apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata \
+    cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
+    clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 file \
+    && pip3 install conan==1.59 grpcio-tools pyinstaller==5.13.2 six pyyaml packaging PyHamcrest cryptography \
     && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
          tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
 
 
-RUN apt-get install -y --no-install-recommends git
+COPY ydb/ /ydbwork/ydb/
 
-WORKDIR /ydbwork
-
-ARG GIT_REPO=https://github.com/ydb-platform/ydb.git
-ARG GIT_REF=main
-
-ENV GIT_REPO=$GIT_REPO
-ENV GIT_REF=$GIT_REF
-
-RUN mkdir ydb \
-    && git clone --depth=1 -b "${GIT_REF?}" ${GIT_REPO?}
+WORKDIR /ydbwork/
 
 RUN --mount=type=secret,id=ccache_remote_storage \
     mkdir build && cd build \
@@ -34,6 +24,8 @@ RUN --mount=type=secret,id=ccache_remote_storage \
     && export CCACHE_BASEDIR=/ydbwork/ \
     && export CCACHE_SLOPPINESS=locale \
     && export CCACHE_REMOTE_STORAGE="$(cat /run/secrets/ccache_remote_storage)" \
+    && export CC=/usr/bin/clang-14 \
+    && export CC_FOR_BUILD=$CC \
     && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
         -DCCACHE_PATH=/usr/local/bin/ccache \
         -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
@@ -41,7 +33,24 @@ RUN --mount=type=secret,id=ccache_remote_storage \
     && ninja ydb/apps/ydbd/ydbd ydb/apps/ydb/all \
     && ccache -s \
     && strip ydb/apps/ydbd/ydbd \
-    && strip ydb/apps/ydb/ydb
+    && strip ydb/apps/ydb/ydb \
+    && mv ./ydb/apps/ydbd/ydbd / \
+    && mv ./ydb/apps/ydb/ydb / \
+    && /ydbd -V  \
+    && /ydb version \
+    && cd .. && rm -rf build  # for reduce cache size
+
+
+
+# always use local_ydb.spec from main revision
+COPY main/ydb/public/tools/local_ydb/local_ydb.spec ydb/ydb/public/tools/local_ydb/
+
+RUN cd ydb && \
+    ./ydb/tests/oss/launch/compile_protos.sh . ydb library/cpp/actors && \
+    cd ydb/public/tools/local_ydb/ && \
+    pyinstaller local_ydb.spec && \
+    ./dist/local_ydb --help
+
 
 FROM ubuntu:22.04
 RUN apt-get update  \
@@ -53,18 +62,10 @@ RUN mkdir -p /root/ydb/bin/ \
     && mkdir -p ydb_certs \
     && echo '{"check_version":false}' > root/ydb/bin/config.json
 
-COPY --from=builder /ydbwork/build/ydb/apps/ydbd/ydbd /ydbwork/build/ydb/apps/ydb/ydb /
-
-COPY --from=yandex_local_build /local_ydb /
-
-COPY files/initialize_local_ydb.sh /initialize_local_ydb
-COPY files/health_check.sh /health_check
-COPY files/THIRD_PARTY_LICENSES /THIRD_PARTY_LICENSES
-COPY files/LICENSE /LICENSE
-
+COPY main/.github/docker/files/ /
+COPY --from=builder /ydbwork/ydb/ydb/public/tools/local_ydb/dist/local_ydb /
+COPY --from=builder /ydbd /ydb /
 
-RUN /ydbd -V
-RUN /ydb version
 
 # YDB grpc
 EXPOSE ${GRPC_TLS_PORT:-2135}