Merge remote-tracking branch 'origin/master' into lw/fixed_log_size

AztecProtocol · Oct 31, 2024 · 6da7a4f · 6da7a4f
2 parents fa04c56 + feace70
commit 6da7a4f
Show file tree

Hide file tree

Showing 161 changed files with 2,657 additions and 1,423 deletions.
diff --git a/.github/workflows/bb-msan.yml b/.github/workflows/bb-msan.yml
diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml
@@ -0,0 +1,98 @@
+# Checks bb (barretenberg prover library) prover with sanitizers
+# Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency).
+name: BB MSAN
+on:
+  push:
+    branches:
+      - master
+      - "*/bb-sanitizers*"
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - 'barretenberg/**'
+  workflow_dispatch:
+    inputs: {}
+
+concurrency:
+  # force parallelism in master
+  group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  # acts as prover performance baseline
+  bb-baseline:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check')
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Barretenberg Baseline Performance Check"
+        run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=clang16
+
+  # memory sanitzer for prover
+  bb-msan-check:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check')
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Barretenberg Prover MSAN Check"
+
+        run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check || true
+
+  # address sanitzer for prover
+  bb-asan-check:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-asan-check')
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Barretenberg Prover ASAN Check"
+        timeout-minutes: 720 # 12 hour timeout (proving)
+        run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan
+
+  # address sanitzer for prover
+  bb-tsan-check:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check')
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Barretenberg Prover TSAN Check"
+        timeout-minutes: 720 # 12 hour timeout (proving)
+        run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan || true
+
+  # undefined behavior sanitzer for prover
+  bb-ubsan-check:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-ubsan-check')
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Barretenberg Prover TSAN Check"
+        timeout-minutes: 720 # 12 hour timeout (proving)
+        run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan
diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml
@@ -5,7 +5,6 @@ on:
     branches:
       - master
       - "*/release-master*"
-      - ludamad-patch-2
   workflow_dispatch:
     inputs:
       tag:

diff --git a/Dockerfile.end-to-end.fast b/Dockerfile.end-to-end.fast
@@ -0,0 +1,25 @@
+# Use an ARG to define the architecture, defaulting to amd64
+ARG ARCH=amd64
+# aztec must be built from Dockerfile.fast
+FROM aztecprotocol/aztec AS aztec
+FROM aztecprotocol/build:1.0-${ARCH}
+
+# Install additional dependencies
+RUN apt-get update && apt-get install -y software-properties-common \
+    && add-apt-repository ppa:xtradeb/apps -y && apt-get update \
+    && apt-get install -y wget gnupg \
+    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
+    && apt update && apt install -y curl chromium netcat-openbsd \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CHROME_BIN="/usr/bin/chromium"
+ENV PATH=/opt/foundry/bin:$PATH
+ENV HARDWARE_CONCURRENCY=""
+ENV FAKE_PROOFS=""
+ENV PROVER_AGENT_CONCURRENCY=8
+
+COPY --from=aztec /usr/src/ /usr/src/
+WORKDIR /usr/src/yarn-project/end-to-end
+
+ENTRYPOINT ["yarn", "test"]
diff --git a/Dockerfile.fast b/Dockerfile.fast
@@ -20,99 +20,83 @@ RUN git init -b master \
     && git config user.email 'tech@aztecprotocol.com'
 
 # ---------- EXTRACT BUILD-SYSTEM ----------
-COPY build-system.tar.gz .
-RUN tar -xzf build-system.tar.gz \
-    && rm build-system.tar.gz && git add . \
+COPY build-system build-system
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # ---------- BUILD BARRETENBERG ----------
-COPY barretenberg.tar.gz .
-RUN tar -xzf barretenberg.tar.gz \
-    && rm barretenberg.tar.gz && git add . \
+COPY barretenberg barretenberg
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Bootstrap cache for barretenberg/cpp
 RUN --mount=type=secret,id=aws_access_key_id \
     --mount=type=secret,id=aws_secret_access_key \
-    cd barretenberg/cpp \
+    bash -c 'cd barretenberg \
     && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
     AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
-    && echo "barretenberg/cpp: Success"
-
-# Bootstrap cache for barretenberg/ts
-RUN --mount=type=secret,id=aws_access_key_id \
-    --mount=type=secret,id=aws_secret_access_key \
-    cd barretenberg/ts \
-    && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
-    AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
-    && echo "barretenberg/ts: Success"
+    ./bootstrap_cache.sh' \
+    && echo "barretenberg: Success"
 
 # ---------- BUILD NOIR ----------
-COPY noir.tar.gz .
-RUN tar -xzf noir.tar.gz \
-    && rm noir.tar.gz && git add . \
+ADD noir noir
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Bootstrap cache for Noir
 RUN --mount=type=secret,id=aws_access_key_id \
     --mount=type=secret,id=aws_secret_access_key \
-    cd noir \
+    bash -c 'cd noir \
     && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
     AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
+    ./bootstrap_cache.sh' \
     && echo "noir: Success"
 
 # ---------- BUILD L1 CONTRACTS ----------
-COPY l1-contracts.tar.gz .
-RUN tar -xzf l1-contracts.tar.gz \
-    && rm l1-contracts.tar.gz && git add . \
+ADD l1-contracts l1-contracts
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Bootstrap cache for L1 Contracts
 RUN --mount=type=secret,id=aws_access_key_id \
     --mount=type=secret,id=aws_secret_access_key \
-    cd l1-contracts \
+    bash -c 'cd l1-contracts \
     && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
     AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
+    ./bootstrap_cache.sh' \
     && echo "l1-contracts: Success"
 
 # ---------- BUILD AVM TRANSPILER ----------
-COPY avm-transpiler.tar.gz .
-RUN tar -xzf avm-transpiler.tar.gz \
-    && rm avm-transpiler.tar.gz && git add . \
+ADD avm-transpiler avm-transpiler
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Bootstrap cache for AVM Transpiler
 RUN --mount=type=secret,id=aws_access_key_id \
     --mount=type=secret,id=aws_secret_access_key \
-    cd avm-transpiler \
+    bash -c 'cd avm-transpiler \
     && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
     AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
+    ./bootstrap_cache.sh' \
     && echo "avm-transpiler: Success"
 
 # ---------- BUILD NOIR PROJECTS ----------
-COPY noir-projects.tar.gz .
-RUN tar -xzf noir-projects.tar.gz \
-    && rm noir-projects.tar.gz && git add . \
+ADD noir-projects noir-projects
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Bootstrap cache for Noir Projects
 RUN --mount=type=secret,id=aws_access_key_id \
     --mount=type=secret,id=aws_secret_access_key \
-    cd noir-projects \
+    bash -c 'cd noir-projects \
     && AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) \
     AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) \
-    ./bootstrap_cache.sh \
+    ./bootstrap_cache.sh' \
     && echo "noir-projects: Success"
 
 # ---------- BUILD YARN PROJECT ----------
-COPY yarn-project.tar.gz .
-RUN tar -xzf yarn-project.tar.gz \
-    && rm yarn-project.tar.gz && git add . \
+ADD yarn-project yarn-project
+RUN git add . \
     && git commit -m "Update git metadata" >/dev/null
 
 # Build yarn-project directly (no cache script)

diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 8b4fa8cbcb9e3bf94962569aa3f9857fe8452a7a
-	parent = 9d66c1abca1af9ddb0715627fad87c2efc612a1d
+	commit = b7f1c6ad51722d30e60f93035d675098d69da8eb
+	parent = 39cda86c3576c5cb94a7beb123b875a2ba37c26b
 	method = merge
 	cmdver = 0.4.6
diff --git a/barretenberg/bootstrap_cache.sh b/barretenberg/bootstrap_cache.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eu
+
+cd "$(dirname "$0")"
+
+# Run both tasks in the background
+(cd cpp && ./bootstrap_cache.sh "$@") &
+pid_cpp=$!
+(cd ts && ./bootstrap_cache.sh "$@") &
+pid_ts=$!
+
+# Wait for both processes and capture any non-zero exit codes
+wait $pid_cpp || exit_code=$?
+wait $pid_ts || exit_code=$?
+
+# Exit with the first non-zero exit code, if any
+exit ${exit_code:-0}
diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile
@@ -107,7 +107,15 @@ preset-msan-check:
   # install SRS needed for proving
   COPY --dir ./srs_db/+build/. srs_db
   RUN echo "Warning: If ./bin/client_ivc_tests is not found, there may be build failures above."
-  RUN cd build && ./bin/client_ivc_tests --gtest_also_run_disabled_tests
+  RUN cd build && ./bin/client_ivc_tests --gtest_filter="*BasicStructured"
+
+preset-check:
+  ARG preset
+  FROM +source
+  RUN cmake --preset $preset -Bbuild && cmake --build build --target client_ivc_tests
+  # install SRS needed for proving
+  COPY --dir ./srs_db/+build/. srs_db
+  RUN cd build && ./bin/client_ivc_tests --gtest_filter="*BasicStructured"
 
 preset-wasm:
   ARG TARGETARCH

diff --git a/barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_builder.test.cpp b/barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_builder.test.cpp
@@ -611,22 +611,20 @@ TEST(UltraCircuitConstructor, NonNativeFieldMultiplication)
 
     const auto split_into_limbs = [&](const uint512_t& input) {
         constexpr size_t NUM_BITS = 68;
-        std::array<fr, 5> limbs;
+        std::array<fr, 4> limbs;
         limbs[0] = input.slice(0, NUM_BITS).lo;
         limbs[1] = input.slice(NUM_BITS * 1, NUM_BITS * 2).lo;
         limbs[2] = input.slice(NUM_BITS * 2, NUM_BITS * 3).lo;
         limbs[3] = input.slice(NUM_BITS * 3, NUM_BITS * 4).lo;
-        limbs[4] = fr(input.lo);
         return limbs;
     };
 
-    const auto get_limb_witness_indices = [&](const std::array<fr, 5>& limbs) {
-        std::array<uint32_t, 5> limb_indices;
+    const auto get_limb_witness_indices = [&](const std::array<fr, 4>& limbs) {
+        std::array<uint32_t, 4> limb_indices;
         limb_indices[0] = circuit_constructor.add_variable(limbs[0]);
         limb_indices[1] = circuit_constructor.add_variable(limbs[1]);
         limb_indices[2] = circuit_constructor.add_variable(limbs[2]);
         limb_indices[3] = circuit_constructor.add_variable(limbs[3]);
-        limb_indices[4] = circuit_constructor.add_variable(limbs[4]);
         return limb_indices;
     };
     const uint512_t BINARY_BASIS_MODULUS = uint512_t(1) << (68 * 4);
@@ -671,22 +669,20 @@ TEST(UltraCircuitConstructor, NonNativeFieldMultiplicationSortCheck)
 
     const auto split_into_limbs = [&](const uint512_t& input) {
         constexpr size_t NUM_BITS = 68;
-        std::array<fr, 5> limbs;
+        std::array<fr, 4> limbs;
         limbs[0] = input.slice(0, NUM_BITS).lo;
         limbs[1] = input.slice(NUM_BITS * 1, NUM_BITS * 2).lo;
         limbs[2] = input.slice(NUM_BITS * 2, NUM_BITS * 3).lo;
         limbs[3] = input.slice(NUM_BITS * 3, NUM_BITS * 4).lo;
-        limbs[4] = fr(input.lo);
         return limbs;
     };
 
-    const auto get_limb_witness_indices = [&](const std::array<fr, 5>& limbs) {
-        std::array<uint32_t, 5> limb_indices;
+    const auto get_limb_witness_indices = [&](const std::array<fr, 4>& limbs) {
+        std::array<uint32_t, 4> limb_indices;
         limb_indices[0] = circuit_constructor.add_variable(limbs[0]);
         limb_indices[1] = circuit_constructor.add_variable(limbs[1]);
         limb_indices[2] = circuit_constructor.add_variable(limbs[2]);
         limb_indices[3] = circuit_constructor.add_variable(limbs[3]);
-        limb_indices[4] = circuit_constructor.add_variable(limbs[4]);
         return limb_indices;
     };
     const uint512_t BINARY_BASIS_MODULUS = uint512_t(1) << (68 * 4);

diff --git a/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt
@@ -13,7 +13,7 @@ set(DSL_DEPENDENCIES
     stdlib_schnorr
     stdlib_honk_verifier)
 
-if (NOT WASM)
+if (NOT WASM AND NOT DISABLE_AZTEC_VM)
     list(APPEND DSL_DEPENDENCIES libdeflate::libdeflate_static vm)
 endif()