Update from upstream

matter-labs · May 29, 2024 · 9878e9b · 9878e9b
2 parents 816523a + 2fc9a6c
commit 9878e9b
Show file tree

Hide file tree

Showing 273 changed files with 15,205 additions and 6,672 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -24,6 +24,7 @@ keys/setup
 !Cargo.toml
 !contracts/
 !setup_2\^26.key
+!setup_2\^24.key
 # It's required to remove .git from contracts,
 # otherwise yarn tries to use .git parent directory that
 # doesn't exist.

diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,4 +1,4 @@
 {
-  "core": "24.3.0",
-  "prover": "14.2.0"
+  "core": "24.4.0",
+  "prover": "14.3.0"
 }
diff --git a/.github/workflows/build-contract-verifier-template.yml b/.github/workflows/build-contract-verifier-template.yml
@@ -31,11 +31,11 @@ jobs:
     runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
     strategy:
       matrix:
-          components:
-            - contract-verifier
-            - verified-sources-fetcher
-          platforms:
-            - linux/amd64
+        components:
+          - contract-verifier
+          - verified-sources-fetcher
+        platforms:
+          - linux/amd64
 
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -143,8 +143,8 @@ jobs:
       - name: Show sccache stats
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   create_manifest:
     name: Create release manifest

diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml
@@ -36,15 +36,15 @@ jobs:
     runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
     strategy:
       matrix:
-          components:
-            - server-v2
-            - external-node
-            - snapshots-creator
-          platforms:
-            - linux/amd64
-          include:
-            - components: external-node
-              platforms: linux/arm64
+        components:
+          - server-v2
+          - external-node
+          - snapshots-creator
+        platforms:
+          - linux/amd64
+        include:
+          - components: external-node
+            platforms: linux/arm64
 
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -152,8 +152,8 @@ jobs:
       - name: Show sccache stats
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   create_manifest:
     name: Create release manifest

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -41,7 +41,7 @@ jobs:
       RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
       ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
-    runs-on: [matterlabs-ci-runner]
+    runs-on: [ matterlabs-ci-runner ]
     strategy:
       matrix:
         component:
@@ -51,6 +51,7 @@ jobs:
           - witness-vector-generator
           - prover-fri-gateway
           - proof-fri-compressor
+          - proof-fri-gpu-compressor
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
@@ -80,11 +81,17 @@ jobs:
           ci_run zk
 
       # We need the CRS only for the fri compressor.
-      - name: download CRS
+      - name: download CRS for CPU compressor
         if: matrix.component == 'proof-fri-compressor'
         run: |
           ci_run curl --retry 5 -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
+      - name: download CRS for GPU compressor
+        if: matrix.component == 'proof-fri-gpu-compressor'
+        run: |
+          ci_run curl --retry 5 -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
+
+
       - name: login to Docker registries
         if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
         run: |
@@ -138,15 +145,15 @@ jobs:
         env:
           DOCKER_ACTION: ${{ inputs.action }}
           COMPONENT: ${{ matrix.component }}
-        run: |          
+        run: |
           PASSED_ENV_VARS="ERA_BELLMAN_CUDA_RELEASE,CUDA_ARCH" \
           ci_run zk docker $DOCKER_ACTION $COMPONENT
 
       - name: Show sccache stats
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
   
   copy-images:
     name: Copy images between docker registries

diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -120,8 +120,8 @@ jobs:
       - name: Show sccache logs
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   integration:
     name: Integration (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, deployment_mode=${{ matrix.deployment_mode }})
@@ -202,7 +202,7 @@ jobs:
       # `sleep 5` because we need to wait until server started properly
       - name: Run server
         run: |
-          ci_run zk server --components=$SERVER_COMPONENTS &>server.log &
+          ci_run zk server --use-node-framework --components=$SERVER_COMPONENTS &>server.log &
           ci_run sleep 5
 
       - name: Run contract verifier
@@ -217,12 +217,28 @@ jobs:
         # We use `yarn` directly because the test launches `zk` commands in both server and EN envs.
         # An empty topmost environment helps avoid a mess when redefining env vars shared between both envs
         # (e.g., DATABASE_URL).
+        #
+        # Since `base_token` doesn't meaningfully influence the test, we use it as a flag for
+        # enabling / disabling tree during pruning.
         run: |
           if [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then
             ci_run zk config compile ext-node-validium
             ci_run zk config compile ext-node-validium-docker
           fi
-          ENABLE_CONSENSUS=${{ matrix.consensus }} DEPLOYMENT_MODE=${{ matrix.deployment_mode }} PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" ci_run yarn snapshot-recovery-test snapshot-recovery-test
+          ENABLE_CONSENSUS=${{ matrix.consensus }} \
+          DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
+          DISABLE_TREE_DURING_PRUNING=${{ matrix.base_token == 'Eth' }} \
+          ETH_CLIENT_WEB3_URL="http://reth:8545" \
+          PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,DISABLE_TREE_DURING_PRUNING,ETH_CLIENT_WEB3_URL" \
+            ci_run yarn recovery-test snapshot-recovery-test
+
+      - name: Genesis recovery test
+        run: |
+          ENABLE_CONSENSUS=${{ matrix.consensus }} \
+          DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
+          ETH_CLIENT_WEB3_URL="http://reth:8545" \
+          PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE,ETH_CLIENT_WEB3_URL" \
+            ci_run yarn recovery-test genesis-recovery-test
 
       - name: Fee projection tests
         run: ci_run zk test i fees
@@ -252,10 +268,13 @@ jobs:
 
       - name: Show snapshot-creator.log logs
         if: always()
-        run: ci_run cat core/tests/snapshot-recovery-test/snapshot-creator.log || true
+        run: ci_run cat core/tests/recovery-test/snapshot-creator.log || true
       - name: Show snapshot-recovery.log logs
         if: always()
-        run: ci_run cat core/tests/snapshot-recovery-test/snapshot-recovery.log || true
+        run: ci_run cat core/tests/recovery-test/snapshot-recovery.log || true
+      - name: Show genesis-recovery.log logs
+        if: always()
+        run: ci_run cat core/tests/recovery-test/genesis-recovery.log || true
 
       - name: Show revert.log logs
         if: always()
@@ -268,8 +287,8 @@ jobs:
       - name: Show sccache logs
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   external-node:
     name: External node (consensus=${{ matrix.consensus }}, base_token=${{ matrix.base_token }}, deployment_mode=${{ matrix.deployment_mode }})
@@ -389,5 +408,5 @@ jobs:
       - name: Show sccache logs
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
diff --git a/.gitignore b/.gitignore
@@ -10,7 +10,7 @@
 node_modules
 *.log
 target
-a.out 
+a.out
 .gitconfig
 cobertura.xml
 tags
@@ -30,6 +30,7 @@ Cargo.lock
 !/Cargo.lock
 !/infrastructure/zksync-crypto/Cargo.lock
 !/prover/Cargo.lock
+!/zk_toolbox/Cargo.lock
 
 /etc/env/target/*
 /etc/env/.current
@@ -96,3 +97,7 @@ hyperchain-*.yml
 
 # Prover keys that should not be commited
 prover/vk_setup_data_generator_server_fri/data/setup_*
+
+# Zk Toolbox
+chains/era/configs/*
+configs/*