feat(circuit-prover): Add circuit_prover Dockerfile and build rules (m…

…atter-labs#2939) This will enable us to build images for circuit proving (new version). Not tested. --------- Co-authored-by: Yury Akudovich <[email protected]>
lambdaclass · Sep 24, 2024 · 1095ae7 · 1095ae7
1 parent 7026637
commit 1095ae7
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 17 deletions.
diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml
@@ -108,10 +108,10 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-gar-prover-fri-gpu:
+  build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
     name: Build GAR prover FRI GPU
     needs: [ setup, build-push-prover-images ]
-    uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
+    uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
     if: contains(github.ref_name, 'prover')
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}

diff --git a/...ub/workflows/build-prover-fri-gpu-gar.yml → ...ri-gpu-gar-and-circuit-prover-gpu-gar.yml b/...ub/workflows/build-prover-fri-gpu-gar.yml → ...ri-gpu-gar-and-circuit-prover-gpu-gar.yml
@@ -1,4 +1,4 @@
-name: Build Prover FRI GPU with builtin setup data
+name: Build Prover FRI GPU & Circuit Prover GPU with builtin setup data
 
 on:
   workflow_call:
@@ -17,7 +17,7 @@ on:
         type: string
 
 jobs:
-  build-gar-prover-fri-gpu:
+  build:
     name: Build prover FRI GPU GAR
     runs-on: [matterlabs-ci-runner-high-performance]
     steps:
@@ -39,7 +39,15 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
-      - name: Build and push
+      - name: Login to Asia GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
+
+      - name: Login to Europe GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
+
+      - name: Build and push prover-gpu-fri-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:
           context: docker/prover-gpu-fri-gar
@@ -49,22 +57,36 @@ jobs:
           tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Login to Asia GAR
+      - name: Build and push prover-gpu-fri-gar to Asia GAR
         run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push to Asia GAR
+      - name: Build and push prover-gpu-fri-gar to Europe GAR
         run: |
           docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Login to Europe GAR
+      - name: Build and push circuit-prover-gpu-gar
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+        with:
+          context: docker/circuit-prover-gpu-gar
+          build-args: |
+            PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+          push: true
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+
+      - name: Build and push circuit-prover-gpu-gar to Asia GAR
         run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push to Europe GAR
+      - name: Build and push circuit-prover-gpu-gar to Europe GAR
         run: |
           docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -52,6 +52,7 @@ jobs:
           - witness-generator
           - prover-gpu-fri
           - witness-vector-generator
+          - circuit-prover-gpu
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor

diff --git a/.github/workflows/release-test-stage.yml b/.github/workflows/release-test-stage.yml
@@ -120,10 +120,10 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-gar-prover-fri-gpu:
+  build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
     name: Build GAR prover FRI GPU
     needs: [ setup, build-push-prover-images ]
-    uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
+    uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
     if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}

diff --git a/docker/circuit-prover-gpu-gar/Dockerfile b/docker/circuit-prover-gpu-gar/Dockerfile
@@ -0,0 +1,15 @@
+ARG PROVER_IMAGE
+FROM us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu:2.0-$PROVER_IMAGE as prover
+
+FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 as app
+
+# HACK copying to root is the only way to make Docker layer caching work for these files for some reason
+COPY *.bin /
+
+RUN apt-get update && apt-get install -y libpq5 ca-certificates openssl && rm -rf /var/lib/apt/lists/*
+
+# copy finalization hints required for assembly generation
+COPY --from=prover prover/data/keys/ /prover/data/keys/
+COPY --from=prover /usr/bin/zksync_circuit_prover /usr/bin/
+
+ENTRYPOINT ["zksync_circuit_prover"]
diff --git a/docker/circuit-prover-gpu/Dockerfile b/docker/circuit-prover-gpu/Dockerfile
@@ -0,0 +1,57 @@
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 AS builder
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG CUDA_ARCH=89
+ENV CUDAARCHS=${CUDA_ARCH}
+
+# set of args for use of sccache
+ARG SCCACHE_GCS_BUCKET=""
+ARG SCCACHE_GCS_SERVICE_ACCOUNT=""
+ARG SCCACHE_GCS_RW_MODE=""
+ARG RUSTC_WRAPPER=""
+
+ENV SCCACHE_GCS_BUCKET=${SCCACHE_GCS_BUCKET}
+ENV SCCACHE_GCS_SERVICE_ACCOUNT=${SCCACHE_GCS_SERVICE_ACCOUNT}
+ENV SCCACHE_GCS_RW_MODE=${SCCACHE_GCS_RW_MODE}
+ENV RUSTC_WRAPPER=${RUSTC_WRAPPER}
+
+RUN apt-get update && apt-get install -y curl clang openssl libssl-dev gcc g++ \
+    pkg-config build-essential libclang-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
+    rustup install nightly-2024-08-01 && \
+    rustup default nightly-2024-08-01
+
+RUN curl -Lo cmake-3.24.2-linux-x86_64.sh https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh && \
+    chmod +x cmake-3.24.2-linux-x86_64.sh && \
+    ./cmake-3.24.2-linux-x86_64.sh --skip-license --prefix=/usr/local
+
+# install sccache
+RUN curl -Lo sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz && \
+    tar -xzf sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz && \
+    cp sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/local/sbin/ && \
+    rm -rf sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz && \
+    rm -rf sccache-v0.8.1-x86_64-unknown-linux-musl && \
+    chmod +x /usr/local/sbin/sccache
+
+WORKDIR /usr/src/zksync
+COPY . .
+
+RUN cd prover && cargo build --release --bin zksync_circuit_prover
+
+FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
+
+RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
+
+# copy finalization hints required for assembly generation
+COPY prover/data/keys/ /prover/data/keys/
+
+COPY --from=builder /usr/src/zksync/prover/target/release/zksync_circuit_prover /usr/bin/
+
+ENTRYPOINT ["zksync_circuit_prover"]
diff --git a/infrastructure/zk/src/docker.ts b/infrastructure/zk/src/docker.ts
@@ -11,6 +11,7 @@ const IMAGES = [
     'witness-generator',
     'prover-gpu-fri',
     'witness-vector-generator',
+    'circuit-prover-gpu',
     'prover-fri-gateway',
     'prover-job-monitor',
     'proof-fri-gpu-compressor',
@@ -87,7 +88,8 @@ function defaultTagList(image: string, imageTagSha: string, imageTagShaTS: strin
             'prover-fri-gateway',
             'prover-gpu-fri',
             'witness-generator',
-            'witness-vector-generator'
+            'witness-vector-generator',
+            'circuit-prover-gpu'
         ].includes(image)
     ) {
         tagList.push(`2.0-${protocolVersionTag}-${imageTagShaTS}`, `${protocolVersionTag}-${imageTagShaTS}`);