feat: bench bb in pr's, docker shell utils (#3561)

- adds a CI task that runs bb benchmarks
- adds utilities for running alpine or ubuntu docker in an interactive
context (to be expanded as useful, for now just lets you rebuild in
ubuntu or alpine easily)

.circleci/config.yml

@@ -205,7 +205,7 @@ jobs:
       - run:
           name: "Deploy barretenberg docs"
           command: |
-            barretenberg/cpp/docs/deploy_doxygen.sh
+            barretenberg/cpp/scripts/ci/upload_doxygen_to_s3.sh
 
   barretenberg-stdlib-tests:
     docker:
@@ -240,6 +240,23 @@ jobs:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/bb-tests.sh
 
+  barretenberg-bench:
+    machine:
+      # NOTE: we usually use alpine build image when making spot images, but
+      # we are not able to upload to S3 with it
+      image: ubuntu-2204:2023.07.2
+    resource_class: medium 
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Benchmark"
+          command: cond_spot_run_build barretenberg-bench 32
+      - run:
+          name: "Upload"
+          command: |
+            barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh
+
   barretenberg-honk-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -1108,6 +1125,10 @@ workflows:
       - barretenberg-x86_64-linux-clang-fuzzing: *defaults
       - barretenberg-wasm-linux-clang: *defaults
       - barretenberg-x86_64-linux-clang-sol: *defaults
+      - barretenberg-bench:
+          requires:
+            - barretenberg-x86_64-linux-clang
+          <<: *defaults
       - barretenberg-proof-system-tests: *bb_test
       - barretenberg-honk-tests: *bb_test
       - barretenberg-dsl-tests: *bb_test

barretenberg/cpp/.dockerignore

@@ -19,6 +19,7 @@
 # Needed scripts.
 !scripts/install-wasi-sdk.sh
 !scripts/strip-wasm.sh
+!scripts/ci
 !./.clang-format
 !./format.sh
 

barretenberg/cpp/dockerfiles/Dockerfile.bench

@@ -0,0 +1,4 @@
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang
+WORKDIR /usr/src/barretenberg/cpp
+RUN apk update && apk add curl libstdc++ jq
+RUN ./scripts/ci/ultra_honk_bench.sh

barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang

@@ -11,13 +11,19 @@ RUN apk update \
         perl
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
-# Build bb binary. Everything else is built as part linux-clang-assert.
-RUN cmake --preset default && cmake --build --preset default --target bb
+# Build bb binary and targets needed for benchmarking. 
+# Everything else is built as part linux-clang-assert.
+# Benchmark targets want to run without asserts, so get built alongside bb.
+RUN cmake --preset default
+RUN cmake --build --preset default --target ultra_honk_rounds_bench --target bb --target grumpkin_srs_gen
 
 FROM alpine:3.18
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
+COPY --from=builder /usr/src/barretenberg/cpp/scripts/ci /usr/src/barretenberg/cpp/scripts/ci
 COPY --from=builder /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/ultra_honk_rounds_bench /usr/src/barretenberg/cpp/build/bin/ultra_honk_rounds_bench
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen
 # Copy libs for consuming projects.
 COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a
-COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a
+COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a

barretenberg/cpp/dockerfiles/interactive/Dockerfile.alpine

@@ -0,0 +1,20 @@
+# Use Alpine 3.18 as the base image
+FROM alpine:3.18
+
+# Install necessary packages
+RUN apk update && apk add \
+  bash \
+  build-base \
+  clang16 \
+  cmake \
+  ninja \
+  git \
+  curl \
+  perl \
+  libstdc++
+
+# Set the working directory in the container
+WORKDIR /usr/src/barretenberg/cpp
+
+# Set bash as the default command to keep the container running interactively
+CMD ["/bin/bash"]

barretenberg/cpp/dockerfiles/interactive/Dockerfile.ubuntu

@@ -0,0 +1,24 @@
+FROM ubuntu:lunar
+
+RUN apt update && apt install -y \
+  build-essential \
+  curl \
+  git \
+  cmake \
+  lsb-release \
+  wget \
+  software-properties-common \
+  gnupg \
+  ninja-build \
+  npm \
+\
+  libssl-dev \
+  jq \
+  bash \
+  libstdc++6
+
+WORKDIR /usr/src/barretenberg/cpp
+
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16
+
+CMD ["/bin/bash"]

barretenberg/cpp/scripts/ci/ultra_honk_bench.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+# enter script folder
+cd "$(dirname $0)"
+cd ../../srs_db
+./download_ignition.sh 1
+./download_grumpkin.sh
+cd ../build
+./bin/ultra_honk_rounds_bench --benchmark_format=json | tee ultra_honk_rounds_bench.json
+echo "Testing if we have created valid JSON."
+cat ultra_honk_rounds_bench.json | jq empty
+echo "JSON is valid. Continuing."

barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh

@@ -0,0 +1,19 @@
+# Uploads to S3 a recent barretenberg benchmark run. 
+#!/usr/bin/env bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+extract_repo barretenberg-bench /usr/src extracted-repo
+
+BUCKET_NAME="aztec-ci-artifacts"
+COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
+
+if [ "${BRANCH:-}" = "master" ]; then
+  TARGET_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH/"
+elif [ -n "${PULL_REQUEST:-}" ]; then
+  TARGET_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}"
+else
+  echo Skipping upload since no target folder was defined
+fi
+echo "Uploading to s3://$BUCKET_NAME/$TARGET_FOLDER"
+aws s3 cp extracted-repo/src/barretenberg/cpp/build/ultra_honk_rounds_bench.json "s3://$BUCKET_NAME/$TARGET_FOLDER/ultra_honk_rounds_bench.json"

barretenberg/cpp/scripts/docker_interactive.sh

@@ -0,0 +1,16 @@
+# Script to enter a docker shell.
+# This comes in two flavors, ubuntu and alpine.
+# This mounts the current folder into the image, allowing for in-docker development.
+# You then need to run cmake commands fresh to build. 
+# Ideally you can combine with a fresh clone of the repo to keep it persistently around, or move around build dirs as needed.
+# This is useful for debugging issues like bb being slow on alpine or running into errors on a specific version of gcc.
+# usage: ./docker_interactive.sh ubuntu or ./docker_interactive.sh alpine
+
+# Enter script directory.
+set -eu
+cd $(dirname $0)
+# allows for 'alpine' or 'ubuntu'
+ENVIRONMENT_KIND="${1:-alpine}"
+DOCKERFILE="../dockerfiles/interactive/Dockerfile.$ENVIRONMENT_KIND"
+docker build -t "env-$ENVIRONMENT_KIND" -f "$DOCKERFILE" .
+docker run -it --mount type=bind,source=$(pwd)/..,target=/usr/src/barretenberg/cpp "env-$ENVIRONMENT_KIND"

barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp

@@ -3,6 +3,10 @@
 #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp"
 #include "barretenberg/polynomials/polynomial_arithmetic.hpp"
 #include "barretenberg/srs/factories/file_crs_factory.hpp"
+
+#include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
+#include "barretenberg/ultra_honk/ultra_composer.hpp"
+
 #include <chrono>
 #include <cstdlib>
 
@@ -42,8 +46,6 @@ constexpr size_t NUM_POINTS = 1 << 16;
 std::vector<fr> scalars;
 static barretenberg::evaluation_domain small_domain;
 static barretenberg::evaluation_domain large_domain;
-auto reference_string =
-    std::make_shared<barretenberg::srs::factories::FileProverCrs<curve::BN254>>(NUM_POINTS, "../srs_db/ignition");
 
 const auto init = []() {
     small_domain = barretenberg::evaluation_domain(NUM_POINTS);
@@ -63,6 +65,9 @@ const auto init = []() {
 };
 // constexpr double add_to_mixed_add_complexity = 1.36;
 
+auto reference_string =
+    std::make_shared<barretenberg::srs::factories::FileProverCrs<curve::BN254>>(NUM_POINTS, "../srs_db/ignition");
+
 int pippenger()
 {
     scalar_multiplication::pippenger_runtime_state<curve::BN254> state(NUM_POINTS);
@@ -98,6 +103,7 @@ int coset_fft_regular()
 
 int main()
 {
+    barretenberg::srs::init_crs_factory("../srs_db/ignition");
     std::cout << "initializing" << std::endl;
     init();
     std::cout << "executing normal fft" << std::endl;

barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp

@@ -18,6 +18,6 @@ static void construct_proof_standard_power_of_2(State& state) noexcept
 }
 
 BENCHMARK(construct_proof_standard_power_of_2)
-    // 2**13 gates to 2**18 gates
-    ->DenseRange(13, 18)
+    // 2**15 gates to 2**20 gates
+    ->DenseRange(15, 20)
     ->Unit(::benchmark::kMillisecond);

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp

@@ -43,6 +43,6 @@ BENCHMARK_CAPTURE(construct_proof_ultrahonk,
     ->Unit(kMillisecond);
 
 BENCHMARK(construct_proof_ultrahonk_power_of_2)
-    // 2**13 gates to 2**18 gates
-    ->DenseRange(13, 18)
+    // 2**15 gates to 2**20 gates
+    ->DenseRange(15, 20)
     ->Unit(kMillisecond);

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp

@@ -41,6 +41,6 @@ BENCHMARK_CAPTURE(construct_proof_ultraplonk,
     ->Unit(kMillisecond);
 
 BENCHMARK(construct_proof_ultraplonk_power_of_2)
-    // 2**13 gates to 2**18 gates
-    ->DenseRange(13, 18)
+    // 2**15 gates to 2**20 gates
+    ->DenseRange(15, 20)
     ->Unit(kMillisecond);

barretenberg/cpp/src/barretenberg/proof_system/library/grand_product_library.hpp

@@ -69,9 +69,10 @@ void compute_grand_product(const size_t circuit_size,
     parallel_for(num_threads, [&](size_t thread_idx) {
         const size_t start = thread_idx * block_size;
         const size_t end = (thread_idx + 1) * block_size;
+        typename Flavor::AllValues evaluations;
+        auto evaluations_view = evaluations.get_all();
         for (size_t i = start; i < end; ++i) {
-            typename Flavor::AllValues evaluations;
-            for (auto [eval, full_poly] : zip_view(evaluations.get_all(), full_polynomials_view)) {
+            for (auto [eval, full_poly] : zip_view(evaluations_view, full_polynomials_view)) {
                 eval = full_poly.size() > i ? full_poly[i] : 0;
             }
             numerator[i] = GrandProdRelation::template compute_grand_product_numerator<Accumulator>(

barretenberg/cpp/srs_db/download_grumpkin.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # TODO(https://github.com/AztecProtocol/barretenberg/issues/813) We don't *actually* download grumpkin yet.
-# this just generates grumpkin points and puts in a place where run_acir_tests.sh expects it.
+# this just generates grumpkin points and links in a place where run_acir_tests.sh expects it.
 # The above issue tracks the final pieces here.
 set -eu
 

build-system/scripts/upload_logs_to_s3

@@ -17,7 +17,7 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
   exit 0
 fi
 
-# Duplicated in scripts/ci/assemble_e2e_benchmark.sh
+# Paths used in scripts/ci/assemble_e2e_benchmark.sh
 if [ "${BRANCH:-}" = "master" ]; then
   TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/"
 elif [ -n "${PULL_REQUEST:-}" ]; then

build_manifest.yml

@@ -104,6 +104,12 @@ barretenberg-acir-tests-bb.js:
     - bb.js
     - noir-compile-acir-tests
 
+barretenberg-bench:
+  buildDir: barretenberg/cpp
+  dockerfile: dockerfiles/Dockerfile.bench
+  dependencies:
+    - barretenberg-x86_64-linux-clang
+
 barretenberg-docs:
   buildDir: barretenberg/cpp
   dockerfile: docs/Dockerfile

scripts/ci/assemble_e2e_benchmark.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Grabs the log files uploaded in yarn-project/end-to-end/scripts/upload_logs_to_s3.sh
+# Grabs the log files uploaded in build-system/scripts/upload_logs_to_s3
 # that contain representative benchmarks, extracts whatever metrics are interesting,
 # and assembles a single file that shows the current state of the repository.
 
@@ -14,7 +14,7 @@ BASE_COMMIT_HASH=""
 BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json"
 BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json"
 
-# Adapted from yarn-project/end-to-end/scripts/upload_logs_to_s3.sh
+# Paths from build-system/scripts/upload_logs_to_s3
 if [ "${CIRCLE_BRANCH:-}" = "master" ]; then
   LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH"
   BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json"