Merge branch 'master' into sync-noir

* master: feat: devbox (#5772) docs: Yellow paper updates for the parity circuits (#6048) git subrepo push --branch=master noir-projects/aztec-nr git_subrepo.sh: Fix parent in .gitrepo file. [skip ci] chore: replace relative paths to noir-protocol-circuits git subrepo push --branch=master barretenberg feat: Pippenger benchmarks compatible with wasmtime (#6095) chore: bump bench-tx-size timeout (#6109) chore: ebs attach robustness (#6108) chore(ci): Run benchmarks on Earthly (#6089)
AztecProtocol · May 1, 2024 · 8834807 · 8834807
2 parents 0181082 + 72321f9
commit 8834807
Show file tree

Hide file tree

Showing 142 changed files with 1,635 additions and 5,871 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -482,37 +482,6 @@ jobs:
   # For old e2e tests see yarn-project/end-to-end/Earthfile
   # Semantics are similar to Dockerfile
 
-  # NOTE: Unlike other e2e, these will be re-enabled here as currently the logs functionality is not in the new earthfile setup
-  bench-publish-rollup:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
-  bench-process-history:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
-  bench-tx-size:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_tx_size_fees.test.ts ENABLE_GAS=1 DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
   e2e-join:
     docker:
       - image: cimg/base:2023.09
@@ -531,16 +500,6 @@ jobs:
           name: "Noop"
           command: echo Noop
 
-  bench-summary:
-    machine:
-      image: default
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Assemble benchmark summary from uploaded logs"
-          command: ./scripts/ci/assemble_e2e_benchmark.sh
-
   # Deploy & release jobs.
   deploy-and-release:
     machine:
@@ -775,16 +734,5 @@ workflows:
             - e2e-join
           <<: *defaults
 
-      # Benchmark jobs.
-      - bench-publish-rollup: *e2e_test
-      - bench-process-history: *e2e_test
-      - bench-tx-size: *e2e_test
-      - bench-summary:
-          requires:
-            - bench-publish-rollup
-            - bench-process-history
-            - bench-tx-size
-          <<: *defaults
-
       # Production releases.
       - deploy-and-release: *defaults_deploy
diff --git a/.devcontainer/dev/devcontainer.json b/.devcontainer/dev/devcontainer.json
@@ -1,17 +1,22 @@
 {
-  "image": "node:lts-bookworm-slim",
-  "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+  "name": "Development",
+  "build": {
+    "dockerfile": "../../build-images/Dockerfile",
+    "context": "../../build-images",
+    "target": "devbox"
   },
-  "postCreateCommand": "curl -s install.aztec.network | VERSION=master NON_INTERACTIVE=1 BIN_PATH=/usr/local/bin bash -s",
-  "customizations": {
-    "vscode": {
-      "settings": {},
-      "extensions": [
-        "noir-lang.vscode-noir"
-      ]
-    }
-  },
-  "workspaceMount": "source=${localWorkspaceFolder},target=/root/workspace,type=bind",
-  "workspaceFolder": "/root/workspace"
+  "containerUser": "aztec-dev",
+  // ubuntu:noble is currently not supported.
+  // Can possibly workaround cherry-picking from here:
+  //   https://github.com/devcontainers/features/blob/main/src/docker-in-docker/install.sh
+  //
+  // "image": "aztecprotocol/codespace",
+  // "features": {
+  //   "docker-in-docker": {
+  //     "version": "latest",
+  //     "moby": true,
+  //     "dockerDashComposeVersion": "v1"
+  //   }
+  // },
+  "mounts": ["source=devbox-home,target=/home/aztec-dev,type=volume"]
 }
diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml
@@ -1,14 +1,14 @@
 # Reusable setup workflow for CI tasks
 name: Setup Workflow
-description: 'Reusable setup steps'
+description: "Reusable setup steps"
 
 inputs:
   dockerhub_password:
     required: true
-    description: 'DockerHub Password'
+    description: "DockerHub Password"
   concurrency_key:
     required: false
-    description: 'Concurrency key for locking jobs'
+    description: "Concurrency key for locking jobs"
 runs:
   # define an action, runs in OS of caller
   using: composite
@@ -43,11 +43,14 @@ runs:
       with:
         # permission issue with spot runners, simpler to leave out
         use-cache: false
-        version: 'v0.8.5'
+        version: "v0.8.5"
 
     - name: Setup Env
       shell: bash
       run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }}
+      env:
+        PULL_REQUEST: "${{ github.event.pull_request.number }}"
+        BRANCH: "${{ github.ref_name }}"
 
     # As detailed in https://github.com/ben-z/gh-action-mutex
     # things do not become 'pending' in github actions, and instead just cancel one another

diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
@@ -32,7 +32,10 @@ jobs:
     needs: setup
     runs-on: master-arm
     steps:
-      - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
+      - {
+          uses: actions/checkout@v4,
+          with: { ref: "${{ github.event.pull_request.head.sha }}" },
+        }
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
@@ -48,15 +51,18 @@ jobs:
     needs: build
     runs-on: master-arm
     steps:
-      - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
+      - {
+          uses: actions/checkout@v4,
+          with: { ref: "${{ github.event.pull_request.head.sha }}" },
+        }
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
           concurrency_key: e2e-master-arm-e2e-tests
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 15
-        run: earthly -P --no-output +uniswap-trade-on-l1-from-l2 --e2e_mode=cache
+        run: earthly -P --no-output +uniswap-trade-on-l1-from-l2
 
   notify:
     needs: [e2e]

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,10 +11,12 @@ on:
       runner_action:
         description: "The action to take with the self-hosted runner (start, stop, restart)."
         required: false
+
 concurrency:
   # force parallelism in master
   group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
   cancel-in-progress: true
+
 jobs:
   setup:
     uses: ./.github/workflows/setup-runner.yml
@@ -64,10 +66,8 @@ jobs:
       matrix:
         test: ${{ fromJson( needs.build.outputs.e2e_list )}}
     steps:
-      - {
-          uses: actions/checkout@v4,
-          with: { ref: "${{ github.event.pull_request.head.sha }}" },
-        }
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
@@ -76,10 +76,35 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
-      # TODO
-      # - name: Upload logs
-      #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }}
+
+  bench-summary:
+    needs: e2e
+    runs-on: ${{ inputs.username || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 100 # Downloading base benchmark from master requires access to history
+          ref: "${{ github.event.pull_request.head.sha }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+          concurrency_key: build-${{ inputs.username || github.actor }}-x86
+      - name: "Build and upload bench aggregate file"
+        working-directory: ./yarn-project/scripts
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate
+      - name: "Download base benchmark"
+        if: ${{ github.event_name == 'pull_request' }}
+        run: scripts/logs/download_base_benchmark_from_s3.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          BENCH_FOLDER: ./yarn-project/scripts/bench
+          PULL_REQUEST: "${{ github.event.pull_request.number }}"
+      - name: "Generate summary comment if pull request"
+        if: ${{ github.event_name == 'pull_request' }}
+        working-directory: ./yarn-project/scripts
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment
 
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)

diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml
@@ -78,11 +78,11 @@ jobs:
         run: |
           # Compare the checked-out CI configuration files with the reference files
           if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
-            echo "Error: ci.yml changes in master (or PR base). Please merge these changes."
+            echo "Error: ci.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
             exit 1
           fi
           if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
-            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes."
+            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior."
             exit 1
           fi
 

diff --git a/.gitignore b/.gitignore
@@ -18,3 +18,7 @@ cmake-build-debug
 .DS_Store
 
 **/*.dockerignore
+
+# Earthly
+.arg
+.secret
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -37,7 +37,7 @@
     // Displays code coverage report information within vscode
     "ryanluker.vscode-coverage-gutters",
     // Spell checking
-    "streetsidesoftware.code-spell-checker",
+    "streetsidesoftware.code-spell-checker"
     // End C++/Circuits extensions
     ///////////////////////////////////////
   ],
@@ -58,7 +58,7 @@
     // Most features are disabled in `settings.json`
     // which confict with `clangd`
     // Since we ignore GDB, we no longer need this extension
-    "ms-vscode.cpptools",
+    "ms-vscode.cpptools"
     // End C++/Circuits unwanted extensions
     ///////////////////////////////////////
   ]

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -136,7 +136,7 @@
   // Clangd. Note that this setting may be overridden by user settings
   // to the default value "clangd".
   //
-  "clangd.path": "clangd-15",
+  "clangd.path": "clangd-16",
   //
   // C/C++ (should be disabled)
   //
@@ -165,6 +165,5 @@
     "**/target/**": true,
     "**/l1-contracts/lib/**": true,
     "**/barretenberg/cpp/build*/**": true
-  },
-  "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp"
+  }
 }
diff --git a/Earthfile b/Earthfile
@@ -1,5 +1,5 @@
 VERSION 0.8
-FROM ubuntu:lunar
+FROM ubuntu:noble
 
 build-ci:
     BUILD ./avm-transpiler/+build
@@ -16,19 +16,22 @@ build-ci:
     BUILD ./yarn-project/+end-to-end
     BUILD ./yarn-project/+aztec
 
-build-ci-small:
-    BUILD ./yarn-project/end-to-end/+e2e-escrow-contract
-
 build:
     # yarn-project has the entry point to Aztec
     BUILD ./yarn-project/+build
 
 test-end-to-end:
-    BUILD ./yarn-project/end-to-end/+test-all
+    BUILD ./yarn-project/end-to-end+e2e-tests
 
 bench:
   RUN echo hi
 
 release-meta:
     COPY .release-please-manifest.json /usr/src/.release-please-manifest.json
     SAVE ARTIFACT /usr/src /usr/src
+
+scripts:
+    FROM ubuntu:lunar
+    RUN apt-get update && apt-get install -y awscli
+    COPY scripts /usr/src/scripts
+    SAVE ARTIFACT /usr/src/scripts scripts
diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile
@@ -1,18 +1,17 @@
 VERSION 0.8
 IMPORT ../noir AS noir
-# we rely on noir source, which this image has
-FROM noir+nargo
 
-# move noir contents to /usr/src/noir
-RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src
-# work in avm-transpiler
-WORKDIR /usr/src/avm-transpiler
+source:
+  # we rely on noir source, which this image has
+  FROM noir+nargo
 
-RUN apt-get update && apt-get install -y git
+  # move noir contents to /usr/src/noir
+  RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src
+  # work in avm-transpiler
+  WORKDIR /usr/src/avm-transpiler
 
-COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .
+  COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .
 
-source:
   # for debugging rebuilds
   RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash
 
@@ -21,7 +20,7 @@ build:
     RUN ./scripts/bootstrap_native.sh
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
 
-run: 
+run:
     #TODO needed?
     FROM ubuntu:focal
     COPY +build/avm-transpiler /usr/src/avm-transpiler

diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 31ec6089135e12bf85240d50bc8bac066918dfa0
-	parent = 1449c338ca79f8d72b71484546aa46ddebb21779
+	commit = e43e6b0a1bd6e46298391ab921fc52d6423317db
+	parent = 5297b5bb2de63003fdb97b9ad75e06c485327b1c
 	method = merge
 	cmdver = 0.4.6
diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json
@@ -256,7 +256,7 @@
       "generator": "Ninja",
       "toolchainFile": "cmake/toolchains/wasm32-wasi.cmake",
       "environment": {
-        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk",
+        "WASI_SDK_PREFIX": "/opt/wasi-sdk",
         "CC": "$env{WASI_SDK_PREFIX}/bin/clang",
         "CXX": "$env{WASI_SDK_PREFIX}/bin/clang++",
         "AR": "$env{WASI_SDK_PREFIX}/bin/llvm-ar",