feat: arm64 zk-environment rust Docker images and other (#296)

# What ❔ - Bump zksolc and zksvyper binaries, build solidity from source (to support arm64 in future) - Minor CI and Dockerfiles refactoring here and there - Bump ancient gcloud CLI in zk-env publish CI - Bump Ubuntu in zk-env to 22.04 - **Build zk-environment (non-CUDA) on arm64** ## Why ❔ To unblock building of the core services on arm64 arch ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`.
matter-labs · Oct 24, 2023 · 33174aa · 33174aa
1 parent 0f15919
commit 33174aa
Show file tree

Hide file tree

Showing 11 changed files with 145 additions and 364 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,6 +1,5 @@
 *
 !docker/prover/prover-entry.sh
-!docker/zk/entrypoint.sh
 !docker/local-node/entrypoint.sh
 !docker/external-node/entrypoint.sh
 !docker/contract-verifier/install-all-solc.sh

diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -33,7 +33,6 @@ jobs:
           ci_run zk db setup
           ci_run zk compiler all
           ci_run zk contract build
-          ci_run zk run yarn
 
       - name: Contracts unit tests
         run: ci_run zk test l1-contracts
@@ -119,20 +118,20 @@ jobs:
           mv solc-linux-amd64-v0.8.21+commit.d9974bed $(pwd)/etc/solc-bin/0.8.21/solc
           chmod +x $(pwd)/etc/solc-bin/0.8.21/solc
 
-          mkdir -p $(pwd)/etc/zksolc-bin/v1.3.14
-          wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.14
-          mv zksolc-linux-amd64-musl-v1.3.14 $(pwd)/etc/zksolc-bin/v1.3.14/zksolc
-          chmod +x $(pwd)/etc/zksolc-bin/v1.3.14/zksolc
+          mkdir -p $(pwd)/etc/zksolc-bin/v1.3.15
+          wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.15
+          mv zksolc-linux-amd64-musl-v1.3.15 $(pwd)/etc/zksolc-bin/v1.3.15/zksolc
+          chmod +x $(pwd)/etc/zksolc-bin/v1.3.15/zksolc
 
           mkdir -p $(pwd)/etc/vyper-bin/0.3.3
           wget -O vyper0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3%2Bcommit.48e326f0.linux
           mv vyper0.3.3 $(pwd)/etc/vyper-bin/0.3.3/vyper
           chmod +x $(pwd)/etc/vyper-bin/0.3.3/vyper
 
-          mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.10
-          wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.10
-          mv zkvyper-linux-amd64-musl-v1.3.10 $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper
-          chmod +x $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper
+          mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.11
+          wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.11
+          mv zkvyper-linux-amd64-musl-v1.3.11 $(pwd)/etc/zkvyper-bin/v1.3.11/zkvyper
+          chmod +x $(pwd)/etc/zkvyper-bin/v1.3.11/zkvyper
 
       - name: Start services
         run: |
@@ -217,20 +216,20 @@ jobs:
           mv solc-linux-amd64-v0.8.21+commit.d9974bed $(pwd)/etc/solc-bin/0.8.21/solc
           chmod +x $(pwd)/etc/solc-bin/0.8.21/solc
 
-          mkdir -p $(pwd)/etc/zksolc-bin/v1.3.14
-          wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.14
-          mv zksolc-linux-amd64-musl-v1.3.14 $(pwd)/etc/zksolc-bin/v1.3.14/zksolc
-          chmod +x $(pwd)/etc/zksolc-bin/v1.3.14/zksolc
+          mkdir -p $(pwd)/etc/zksolc-bin/v1.3.15
+          wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.15
+          mv zksolc-linux-amd64-musl-v1.3.15 $(pwd)/etc/zksolc-bin/v1.3.15/zksolc
+          chmod +x $(pwd)/etc/zksolc-bin/v1.3.15/zksolc
 
           mkdir -p $(pwd)/etc/vyper-bin/0.3.3
           wget -O vyper0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3%2Bcommit.48e326f0.linux
           mv vyper0.3.3 $(pwd)/etc/vyper-bin/0.3.3/vyper
           chmod +x $(pwd)/etc/vyper-bin/0.3.3/vyper
 
-          mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.10
-          wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.10
-          mv zkvyper-linux-amd64-musl-v1.3.10 $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper
-          chmod +x $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper
+          mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.11
+          wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.11
+          mv zkvyper-linux-amd64-musl-v1.3.11 $(pwd)/etc/zkvyper-bin/v1.3.11/zkvyper
+          chmod +x $(pwd)/etc/zkvyper-bin/v1.3.11/zkvyper
 
       - name: Start services
         run: |

diff --git a/.github/workflows/zk-environment.publish.yml b/.github/workflows/zk-environment.publish.yml
@@ -1,18 +1,21 @@
-name: publish zk-environment docker images
+name: Publish zk-environment Docker images
 
 on:
+  push:
+    branches:
+      - main
   pull_request:
     branches:
       - main
-    types: [closed]
     paths:
       - "docker/zk-environment/Dockerfile"
-      - ".github/workflows/zk-environment.publish.yml"
       - "docker/zk-environment-cuda-12-0/Dockerfile"
-      - ".github/workflows/zk-environment-cuda-12-0.publish.yml"
+      - ".github/workflows/zk-environment.publish.yml"
   workflow_dispatch:
-    branches:
-      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   changed_files:
@@ -36,111 +39,86 @@ jobs:
               - .github/workflows/zk-environment.publish.yml
             zk_env_cuda_12:
               - docker/zk-environment-cuda-12-0/Dockerfile
-              - .github/workflows/zk-environment-cuda-12-0.publish.yml
+              - .github/workflows/zk-environment.publish.yml
 
   zk_environment:
-    if: needs.changed_files.outputs.zk_environment == 'true' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
-    name: Push zk-environment docker image to Docker Hub
-    runs-on: ubuntu-latest
+    if: needs.changed_files.outputs.zk_environment == 'true'
+    name: Build and optionally push zk-environment Docker images to Docker Hub
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: matterlabs-ci-runner
+            arch: amd64
+          - runner: matterlabs-ci-runner-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
     needs: changed_files
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
         with:
           submodules: "recursive"
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2
-
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
+      - name: Build and optionally push zk-environment lightweight
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
-          context: .
+          file: docker/zk-environment/Dockerfile
           target: base
-          push: true
           tags: "matterlabs/zk-environment:latest2.0-lightweight"
-          file: docker/zk-environment/Dockerfile
-          no-cache: true
-
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
+          build-args: ARCH=${{ matrix.arch }}
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      - name: Build and optionally push zk-environment Rust nightly
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
-          context: .
-          target: nvidia-tools
-          push: true
-          tags: "matterlabs/zk-environment:latest2.0"
           file: docker/zk-environment/Dockerfile
-          no-cache: true
-
-  rust_nightly:
-    if: needs.changed_files.outputs.zk_environment == 'true' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
-    name: Push rust nightly docker image to Docker Hub
-    needs: changed_files
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
-        with:
-          submodules: "recursive"
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
-        with:
-          context: .
-          push: true
           target: rust-nightly
           tags: "matterlabs/zksync_rust:nightly"
+          build-args: ARCH=${{ matrix.arch }}
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      - name: Build and optionally push zk-environment CUDA 11.8
+        if: matrix.arch == 'amd64'
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
           file: docker/zk-environment/Dockerfile
-          no-cache: true
-
+          target: nvidia-tools
+          tags: "matterlabs/zk-environment:latest2.0"
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 
   zk_environment_cuda_12:
-    if: needs.changed_files.outputs.zk_environment_cuda_12 == 'true' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
-    name: Push zk-environment cuda 12 docker image to Docker Hub
+    if: needs.changed_files.outputs.zk_environment_cuda_12 == 'true'
+    name: Build and optionally push zk-environment CUDA 12 Docker image
     runs-on: [matterlabs-ci-runner]
     needs: changed_files
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:
           submodules: "recursive"
-
-      - name: Login to us-central1 GAR
+      - name: Log in to us-central1 GAR
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
-
       - name: Log in to Docker Hub
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
-
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2
+      - name: Build and optionally push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
-          context: .
           file: docker/zk-environment-cuda-12-0/Dockerfile
-          push: true
-          target: nvidia-tools
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
           tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-12-0:latest
             matterlabs/zk-environment:cuda-12-0-latest