chore(boojum): Merge main into boojum-integration (#164)

# What ❔

Merge main branch into boojum-integration

## Why ❔

To not get far behind

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.

---------

Signed-off-by: Danil <deniallugo@gmail.com>
Co-authored-by: Yury Akudovich <ya@matterlabs.dev>
Co-authored-by: Shahar Kaminsky <sk@matterlabs.dev>
Co-authored-by: Aleksandr Stepanov <alexandrst88@gmail.com>
Co-authored-by: Maksym <maksym.kryva@gmail.com>
Co-authored-by: Danil <deniallugo@gmail.com>
Co-authored-by: Alex Ostrovski <slowli@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: agolajko <57454127+agolajko@users.noreply.github.com>
Co-authored-by: AnastasiiaVashchuk <72273339+AnastasiiaVashchuk@users.noreply.github.com>

.github/release-please/config.json

@@ -1,7 +1,7 @@
 {
   "separate-pull-requests": true,
   "group-pull-request-title-pattern": "chore: release ${component} ${branch}",
-  "bootstrap-sha": "a980ac61f484a3b6bdbe11e606977d0d92d4e9ff",
+  "bootstrap-sha": "531757b5eb98da80b7e6d0ff7ad9fd0b970cc109",
   "bump-minor-pre-major": true,
   "bump-patch-for-minor-pre-major": true,
   "packages": {

.github/release-please/manifest.json

@@ -1,6 +1,6 @@
 {
   "sdk/zksync-web3.js": "0.15.4",
   "sdk/zksync-rs": "0.4.0",
-  "core": "15.0.1",
+  "core": "15.1.0",
   "prover": "7.1.1"
 }

.github/workflows/build-docker-from-tag.yml

@@ -0,0 +1,87 @@
+name: Build Image from tag
+on:
+  workflow_call:
+    inputs:
+      tag_name:
+        description: "Tag of an image to built"
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: "Tag of an image to built"
+        type: string
+        required: true
+  push:
+    tags:
+      - core-v**
+      - prover-v**
+
+concurrency: docker-build
+
+jobs:
+  setup:
+    name: Setup
+    runs-on: [ubuntu-latest]
+    outputs:
+      image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }}
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Generate output with git tag
+        id: set
+        run: |
+          git_tag=""
+          if [[ -z "${{ inputs.tag_name }}" ]]; then
+            git_tag="${GITHUB_REF#refs/*/}"
+          else
+            git_tag="${{ inputs.tag_name }}"
+          fi
+          version=$(cut -d "-" -f2 <<< ${git_tag})
+          echo "image_tag_suffix=${version}" >> $GITHUB_OUTPUT
+
+  build-push-core-images:
+    name: Build and push image
+    needs: [setup]
+    uses: ./.github/workflows/build-core-template.yml
+    if: contains(github.ref_name, 'core')
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+    with:
+      image_tag: ${{ needs.setup.outputs.image_tag }}
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+
+  build-push-prover-images:
+    name: Build and push image
+    needs: [setup]
+    uses: ./.github/workflows/build-prover-template.yml
+    if: contains(github.ref_name, 'prover')
+    with:
+      image_tag: ${{ needs.setup.outputs.image_tag }}
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+      ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+  build-gar-prover:
+    name: Build GAR prover
+    needs: [setup, build-push-prover-images]
+    uses: ./.github/workflows/build-gar-reusable.yml
+    if: contains(github.ref_name, 'prover')
+    with:
+      setup_keys_id: 4989f12
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+      push_asia: true
+
+  build-gar-prover-fri-gpu:
+    name: Build GAR prover FRI GPU
+    needs: [setup, build-push-prover-images]
+    uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
+    if: contains(github.ref_name, 'prover')
+    with:
+      setup_keys_id: 2d33a27-gpu
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}

.github/workflows/build-external-node-docker.yml

@@ -0,0 +1,51 @@
+name: External Node - Build & push docker image
+on:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Tag of a built image to deploy (latest2.0 by default)"
+        type: string
+        required: false
+        default: "latest2.0"
+
+jobs:
+  build-images:
+    name: External Node - Build and Push Docker Image
+    runs-on: [matterlabs-ci-runner]
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo CI=1 >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo CI=1 >> .env
+          echo IN_DOCKER=1 >> .env
+
+      - name: start-services
+        run: |
+          docker-compose -f docker-compose-runner.yml up -d zk geth postgres
+
+      - name: init
+        run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
+          ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts
+
+          ci_run zk
+          ci_run zk run yarn
+          ci_run cp etc/tokens/{test,localhost}.json
+          ci_run zk compiler all
+          ci_run zk contract build
+          ci_run zk f yarn run l2-contracts build
+
+      - name: update-image
+        run: |
+          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          ci_run zk docker build server-v2
+          ci_run gcloud auth configure-docker us-docker.pkg.dev -q
+          ci_run zk docker push external-node --custom-tag ${{ inputs.image_tag }}

.github/workflows/build-gar-reusable.yml

@@ -0,0 +1,81 @@
+name: Workflow template for Build Prover builtin Setup Keys
+
+on:
+  workflow_call:
+    inputs:
+      image_tag_suffix:
+        description: "Commit sha or git tag for Docker tag"
+        required: true
+        type: string
+      setup_keys_id:
+        description: "Commit sha for downloading keys from bucket dir"
+        required: true
+        type: string
+      push_asia:
+        description: "Push images to Asia GAR"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  build-gar-prover:
+    name: Build GAR prover
+    runs-on: [matterlabs-ci-runner]
+    strategy:
+      fail-fast: false
+      matrix:
+        setup_keys:
+          [
+            { prover_id: "0", keys_ids: "0,18" },
+            { prover_id: "1", keys_ids: "1,4" },
+            { prover_id: "2", keys_ids: "2,5" },
+            { prover_id: "3", keys_ids: "6,7" },
+            { prover_id: "4", keys_ids: "8,9" },
+            { prover_id: "5", keys_ids: "10,11" },
+            { prover_id: "6", keys_ids: "12,13" },
+            { prover_id: "7", keys_ids: "14,15" },
+            { prover_id: "8", keys_ids: "16,17" },
+            { prover_id: "9", keys_ids: "3" },
+          ]
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Download Setup Keys
+        run: |
+          gsutil cp gs://matterlabs-setup-keys-us/setup-keys/setup_2\^26.key docker/prover-gar/setup_2\^26.key
+          IFS=', ' read -r -a keys_ids <<< "${{ matrix.setup_keys.keys_ids }}"
+          printf "%s\n" "${keys_ids[@]}"| xargs -n 1 -P 8 -I {} gsutil cp -P gs://matterlabs-zksync-v2-infra-blob-store/prover_setup_keys/${{ inputs.setup_keys_id }}/setup_{}_key.bin docker/prover-gar/
+
+      - name: Login to us-central1 GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: docker/prover-gar
+          build-args: |
+            PROVER_IMAGE=${{ inputs.image_tag_suffix }}
+          push: true
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-v2-gar:2.0-${{ inputs.image_tag_suffix }}-prover-${{ matrix.setup_keys.prover_id }}-${{ inputs.setup_keys_id }}
+
+      - name: Login to asia-southeast1 GAR
+        if: "${{ inputs.push_asia }}"
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
+
+      - name: Push image to Asia
+        if: "${{ inputs.push_asia }}"
+        run: |
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-v2-gar:2.0-${{ inputs.image_tag_suffix }}-prover-${{ matrix.setup_keys.prover_id }}-${{ inputs.setup_keys_id }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-v2-gar:2.0-${{ inputs.image_tag_suffix }}-prover-${{ matrix.setup_keys.prover_id }}-${{ inputs.setup_keys_id }}

.github/workflows/build-local-node-docker.yml

@@ -0,0 +1,51 @@
+name: Local Node - Build docker image
+on:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Tag of a built image to deploy (latest2.0 by default)"
+        type: string
+        required: false
+        default: "latest2.0"
+
+jobs:
+  build-images:
+    name: Local Node - Build and Push Docker Image
+    runs-on: [matterlabs-ci-runner]
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo CI=1 >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo CI=1 >> .env
+          echo IN_DOCKER=1 >> .env
+
+      - name: start-services
+        run: |
+          docker-compose -f docker-compose-runner.yml up -d zk geth postgres
+
+      - name: init
+        run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
+          ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts
+
+          ci_run zk
+          ci_run zk run yarn
+          ci_run cp etc/tokens/{test,localhost}.json
+          ci_run zk compiler all
+          ci_run zk contract build
+          ci_run zk f yarn run l2-contracts build
+
+      - name: update-image
+        run: |
+          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          ci_run zk docker build server-v2
+          ci_run gcloud auth configure-docker us-docker.pkg.dev -q
+          ci_run zk docker push local-node --custom-tag ${{ inputs.image_tag }}

.github/workflows/build-prover-fri-gpu-gar.yml

@@ -0,0 +1,46 @@
+name: Build Prover FRI GPU with builtin setup data
+
+on:
+  workflow_call:
+    inputs:
+      image_tag_suffix:
+        description: "Commit sha or git tag for Docker tag"
+        required: true
+        type: string
+      setup_keys_id:
+        description: "Commit sha for downloading setup data from bucket dir"
+        required: true
+        type: string
+
+jobs:
+  build-gar-prover-fri-gpu:
+    name: Build prover FRI GPU GAR
+    runs-on: [matterlabs-ci-runner]
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Download Setup data
+        run: |
+          gsutil -m rsync -r gs://matterlabs-zksync-v2-infra-blob-store/prover_setup_data/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
+
+      - name: Login to us-central1 GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: docker/prover-gpu-fri-gar
+          build-args: |
+            PROVER_IMAGE=${{ inputs.image_tag_suffix }}
+          push: true
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}

.github/workflows/build-prover-template.yml

@@ -46,6 +46,7 @@ jobs:
     env:
       image_tag: ${{ inputs.image_tag }}
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
+      RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
     runs-on: [matterlabs-ci-runner]
     needs: [era-bellman-cuda]
     strategy:
@@ -126,7 +127,6 @@ jobs:
             if [ "$underscored_name" == "prover_gpu_fri" ]; then
               underscored_name="prover_fri"
             fi
-            ci_run rustup default nightly-2023-07-21
             ci_run echo [workspace] > Cargo.toml
             ci_run echo members = [\"prover/${underscored_name}\"] >> Cargo.toml
             ci_run cp prover/Cargo.lock Cargo.lock