From 54089190a4532988cec9f88d199263aeafa2c2f3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Jan 2024 15:05:50 -0500 Subject: [PATCH] feat: bench bb in pr's, docker shell utils (#3561) - adds a CI task that runs bb benchmarks - adds utilities for running alpine or ubuntu docker in an interactive context (to be expanded as useful, for now just lets you rebuild in ubuntu or alpine easily) --- .circleci/config.yml | 23 +++++++++++++++++- barretenberg/cpp/.dockerignore | 1 + barretenberg/cpp/dockerfiles/Dockerfile.bench | 4 ++++ .../dockerfiles/Dockerfile.x86_64-linux-clang | 12 +++++++--- .../dockerfiles/interactive/Dockerfile.alpine | 20 ++++++++++++++++ .../dockerfiles/interactive/Dockerfile.ubuntu | 24 +++++++++++++++++++ .../cpp/scripts/ci/ultra_honk_bench.sh | 14 +++++++++++ .../cpp/scripts/ci/upload_benchmarks_to_s3.sh | 19 +++++++++++++++ .../ci/upload_doxygen_to_s3.sh} | 0 .../cpp/scripts/docker_interactive.sh | 16 +++++++++++++ .../benchmark/pippenger_bench/main.cpp | 10 ++++++-- .../plonk_bench/standard_plonk.bench.cpp | 4 ++-- .../ultra_bench/ultra_honk.bench.cpp | 4 ++-- .../ultra_bench/ultra_plonk.bench.cpp | 4 ++-- .../library/grand_product_library.hpp | 5 ++-- barretenberg/cpp/srs_db/download_grumpkin.sh | 2 +- build-system/scripts/upload_logs_to_s3 | 2 +- build_manifest.yml | 6 +++++ scripts/ci/assemble_e2e_benchmark.sh | 4 ++-- 19 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 barretenberg/cpp/dockerfiles/Dockerfile.bench create mode 100644 barretenberg/cpp/dockerfiles/interactive/Dockerfile.alpine create mode 100644 barretenberg/cpp/dockerfiles/interactive/Dockerfile.ubuntu create mode 100755 barretenberg/cpp/scripts/ci/ultra_honk_bench.sh create mode 100755 barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh rename barretenberg/cpp/{docs/deploy_doxygen.sh => scripts/ci/upload_doxygen_to_s3.sh} (100%) create mode 100755 barretenberg/cpp/scripts/docker_interactive.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index b065a5ee4b2..f356ac7425d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,7 +205,7 @@ jobs: - run: name: "Deploy barretenberg docs" command: | - barretenberg/cpp/docs/deploy_doxygen.sh + barretenberg/cpp/scripts/ci/upload_doxygen_to_s3.sh barretenberg-stdlib-tests: docker: @@ -240,6 +240,23 @@ jobs: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/bb-tests.sh + barretenberg-bench: + machine: + # NOTE: we usually use alpine build image when making spot images, but + # we are not able to upload to S3 with it + image: ubuntu-2204:2023.07.2 + resource_class: medium + steps: + - *checkout + - *setup_env + - run: + name: "Benchmark" + command: cond_spot_run_build barretenberg-bench 32 + - run: + name: "Upload" + command: | + barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh + barretenberg-honk-tests: docker: - image: aztecprotocol/alpine-build-image @@ -1108,6 +1125,10 @@ workflows: - barretenberg-x86_64-linux-clang-fuzzing: *defaults - barretenberg-wasm-linux-clang: *defaults - barretenberg-x86_64-linux-clang-sol: *defaults + - barretenberg-bench: + requires: + - barretenberg-x86_64-linux-clang + <<: *defaults - barretenberg-proof-system-tests: *bb_test - barretenberg-honk-tests: *bb_test - barretenberg-dsl-tests: *bb_test diff --git a/barretenberg/cpp/.dockerignore b/barretenberg/cpp/.dockerignore index 27554eeebcc..e9b3e4636d3 100644 --- a/barretenberg/cpp/.dockerignore +++ b/barretenberg/cpp/.dockerignore @@ -19,6 +19,7 @@ # Needed scripts. !scripts/install-wasi-sdk.sh !scripts/strip-wasm.sh +!scripts/ci !./.clang-format !./format.sh diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.bench b/barretenberg/cpp/dockerfiles/Dockerfile.bench new file mode 100644 index 00000000000..0535d2fb3a9 --- /dev/null +++ b/barretenberg/cpp/dockerfiles/Dockerfile.bench @@ -0,0 +1,4 @@ +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang +WORKDIR /usr/src/barretenberg/cpp +RUN apk update && apk add curl libstdc++ jq +RUN ./scripts/ci/ultra_honk_bench.sh diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index 79ce703bf62..fe984e74e35 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -11,13 +11,19 @@ RUN apk update \ perl WORKDIR /usr/src/barretenberg/cpp COPY . . -# Build bb binary. Everything else is built as part linux-clang-assert. -RUN cmake --preset default && cmake --build --preset default --target bb +# Build bb binary and targets needed for benchmarking. +# Everything else is built as part linux-clang-assert. +# Benchmark targets want to run without asserts, so get built alongside bb. +RUN cmake --preset default +RUN cmake --build --preset default --target ultra_honk_rounds_bench --target bb --target grumpkin_srs_gen FROM alpine:3.18 WORKDIR /usr/src/barretenberg/cpp COPY . . +COPY --from=builder /usr/src/barretenberg/cpp/scripts/ci /usr/src/barretenberg/cpp/scripts/ci COPY --from=builder /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/ultra_honk_rounds_bench /usr/src/barretenberg/cpp/build/bin/ultra_honk_rounds_bench +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen # Copy libs for consuming projects. COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a -COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a diff --git a/barretenberg/cpp/dockerfiles/interactive/Dockerfile.alpine b/barretenberg/cpp/dockerfiles/interactive/Dockerfile.alpine new file mode 100644 index 00000000000..23ce19a9dc9 --- /dev/null +++ b/barretenberg/cpp/dockerfiles/interactive/Dockerfile.alpine @@ -0,0 +1,20 @@ +# Use Alpine 3.18 as the base image +FROM alpine:3.18 + +# Install necessary packages +RUN apk update && apk add \ + bash \ + build-base \ + clang16 \ + cmake \ + ninja \ + git \ + curl \ + perl \ + libstdc++ + +# Set the working directory in the container +WORKDIR /usr/src/barretenberg/cpp + +# Set bash as the default command to keep the container running interactively +CMD ["/bin/bash"] diff --git a/barretenberg/cpp/dockerfiles/interactive/Dockerfile.ubuntu b/barretenberg/cpp/dockerfiles/interactive/Dockerfile.ubuntu new file mode 100644 index 00000000000..ac58660df46 --- /dev/null +++ b/barretenberg/cpp/dockerfiles/interactive/Dockerfile.ubuntu @@ -0,0 +1,24 @@ +FROM ubuntu:lunar + +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ +\ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +WORKDIR /usr/src/barretenberg/cpp + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 + +CMD ["/bin/bash"] diff --git a/barretenberg/cpp/scripts/ci/ultra_honk_bench.sh b/barretenberg/cpp/scripts/ci/ultra_honk_bench.sh new file mode 100755 index 00000000000..39a4d03e1fc --- /dev/null +++ b/barretenberg/cpp/scripts/ci/ultra_honk_bench.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +# enter script folder +cd "$(dirname $0)" +cd ../../srs_db +./download_ignition.sh 1 +./download_grumpkin.sh +cd ../build +./bin/ultra_honk_rounds_bench --benchmark_format=json | tee ultra_honk_rounds_bench.json +echo "Testing if we have created valid JSON." +cat ultra_honk_rounds_bench.json | jq empty +echo "JSON is valid. Continuing." \ No newline at end of file diff --git a/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh b/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh new file mode 100755 index 00000000000..dc5d890704b --- /dev/null +++ b/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh @@ -0,0 +1,19 @@ +# Uploads to S3 a recent barretenberg benchmark run. +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +extract_repo barretenberg-bench /usr/src extracted-repo + +BUCKET_NAME="aztec-ci-artifacts" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +if [ "${BRANCH:-}" = "master" ]; then + TARGET_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH/" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" +else + echo Skipping upload since no target folder was defined +fi +echo "Uploading to s3://$BUCKET_NAME/$TARGET_FOLDER" +aws s3 cp extracted-repo/src/barretenberg/cpp/build/ultra_honk_rounds_bench.json "s3://$BUCKET_NAME/$TARGET_FOLDER/ultra_honk_rounds_bench.json" \ No newline at end of file diff --git a/barretenberg/cpp/docs/deploy_doxygen.sh b/barretenberg/cpp/scripts/ci/upload_doxygen_to_s3.sh similarity index 100% rename from barretenberg/cpp/docs/deploy_doxygen.sh rename to barretenberg/cpp/scripts/ci/upload_doxygen_to_s3.sh diff --git a/barretenberg/cpp/scripts/docker_interactive.sh b/barretenberg/cpp/scripts/docker_interactive.sh new file mode 100755 index 00000000000..2a38b1530cd --- /dev/null +++ b/barretenberg/cpp/scripts/docker_interactive.sh @@ -0,0 +1,16 @@ +# Script to enter a docker shell. +# This comes in two flavors, ubuntu and alpine. +# This mounts the current folder into the image, allowing for in-docker development. +# You then need to run cmake commands fresh to build. +# Ideally you can combine with a fresh clone of the repo to keep it persistently around, or move around build dirs as needed. +# This is useful for debugging issues like bb being slow on alpine or running into errors on a specific version of gcc. +# usage: ./docker_interactive.sh ubuntu or ./docker_interactive.sh alpine + +# Enter script directory. +set -eu +cd $(dirname $0) +# allows for 'alpine' or 'ubuntu' +ENVIRONMENT_KIND="${1:-alpine}" +DOCKERFILE="../dockerfiles/interactive/Dockerfile.$ENVIRONMENT_KIND" +docker build -t "env-$ENVIRONMENT_KIND" -f "$DOCKERFILE" . +docker run -it --mount type=bind,source=$(pwd)/..,target=/usr/src/barretenberg/cpp "env-$ENVIRONMENT_KIND" \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp index 3bcd6b920a1..85024ba6605 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp @@ -3,6 +3,10 @@ #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" #include "barretenberg/polynomials/polynomial_arithmetic.hpp" #include "barretenberg/srs/factories/file_crs_factory.hpp" + +#include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp" +#include "barretenberg/ultra_honk/ultra_composer.hpp" + #include #include @@ -42,8 +46,6 @@ constexpr size_t NUM_POINTS = 1 << 16; std::vector scalars; static barretenberg::evaluation_domain small_domain; static barretenberg::evaluation_domain large_domain; -auto reference_string = - std::make_shared>(NUM_POINTS, "../srs_db/ignition"); const auto init = []() { small_domain = barretenberg::evaluation_domain(NUM_POINTS); @@ -63,6 +65,9 @@ const auto init = []() { }; // constexpr double add_to_mixed_add_complexity = 1.36; +auto reference_string = + std::make_shared>(NUM_POINTS, "../srs_db/ignition"); + int pippenger() { scalar_multiplication::pippenger_runtime_state state(NUM_POINTS); @@ -98,6 +103,7 @@ int coset_fft_regular() int main() { + barretenberg::srs::init_crs_factory("../srs_db/ignition"); std::cout << "initializing" << std::endl; init(); std::cout << "executing normal fft" << std::endl; diff --git a/barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp index 09bd5ccd2ea..403367d9159 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/plonk_bench/standard_plonk.bench.cpp @@ -18,6 +18,6 @@ static void construct_proof_standard_power_of_2(State& state) noexcept } BENCHMARK(construct_proof_standard_power_of_2) - // 2**13 gates to 2**18 gates - ->DenseRange(13, 18) + // 2**15 gates to 2**20 gates + ->DenseRange(15, 20) ->Unit(::benchmark::kMillisecond); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp index e01487440b1..610f968157f 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp @@ -43,6 +43,6 @@ BENCHMARK_CAPTURE(construct_proof_ultrahonk, ->Unit(kMillisecond); BENCHMARK(construct_proof_ultrahonk_power_of_2) - // 2**13 gates to 2**18 gates - ->DenseRange(13, 18) + // 2**15 gates to 2**20 gates + ->DenseRange(15, 20) ->Unit(kMillisecond); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp index 7c44c18296b..c9fc158c532 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_plonk.bench.cpp @@ -41,6 +41,6 @@ BENCHMARK_CAPTURE(construct_proof_ultraplonk, ->Unit(kMillisecond); BENCHMARK(construct_proof_ultraplonk_power_of_2) - // 2**13 gates to 2**18 gates - ->DenseRange(13, 18) + // 2**15 gates to 2**20 gates + ->DenseRange(15, 20) ->Unit(kMillisecond); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/proof_system/library/grand_product_library.hpp b/barretenberg/cpp/src/barretenberg/proof_system/library/grand_product_library.hpp index e4afe8fe7ab..0f8b46dbcf3 100644 --- a/barretenberg/cpp/src/barretenberg/proof_system/library/grand_product_library.hpp +++ b/barretenberg/cpp/src/barretenberg/proof_system/library/grand_product_library.hpp @@ -69,9 +69,10 @@ void compute_grand_product(const size_t circuit_size, parallel_for(num_threads, [&](size_t thread_idx) { const size_t start = thread_idx * block_size; const size_t end = (thread_idx + 1) * block_size; + typename Flavor::AllValues evaluations; + auto evaluations_view = evaluations.get_all(); for (size_t i = start; i < end; ++i) { - typename Flavor::AllValues evaluations; - for (auto [eval, full_poly] : zip_view(evaluations.get_all(), full_polynomials_view)) { + for (auto [eval, full_poly] : zip_view(evaluations_view, full_polynomials_view)) { eval = full_poly.size() > i ? full_poly[i] : 0; } numerator[i] = GrandProdRelation::template compute_grand_product_numerator( diff --git a/barretenberg/cpp/srs_db/download_grumpkin.sh b/barretenberg/cpp/srs_db/download_grumpkin.sh index c2eefc6c56c..fb59a1ec806 100755 --- a/barretenberg/cpp/srs_db/download_grumpkin.sh +++ b/barretenberg/cpp/srs_db/download_grumpkin.sh @@ -1,6 +1,6 @@ #!/bin/sh # TODO(https://github.com/AztecProtocol/barretenberg/issues/813) We don't *actually* download grumpkin yet. -# this just generates grumpkin points and puts in a place where run_acir_tests.sh expects it. +# this just generates grumpkin points and links in a place where run_acir_tests.sh expects it. # The above issue tracks the final pieces here. set -eu diff --git a/build-system/scripts/upload_logs_to_s3 b/build-system/scripts/upload_logs_to_s3 index 25c7fe2e50a..3188a80ec0f 100755 --- a/build-system/scripts/upload_logs_to_s3 +++ b/build-system/scripts/upload_logs_to_s3 @@ -17,7 +17,7 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then exit 0 fi -# Duplicated in scripts/ci/assemble_e2e_benchmark.sh +# Paths used in scripts/ci/assemble_e2e_benchmark.sh if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" elif [ -n "${PULL_REQUEST:-}" ]; then diff --git a/build_manifest.yml b/build_manifest.yml index be3cf243273..8924d81aa80 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -104,6 +104,12 @@ barretenberg-acir-tests-bb.js: - bb.js - noir-compile-acir-tests +barretenberg-bench: + buildDir: barretenberg/cpp + dockerfile: dockerfiles/Dockerfile.bench + dependencies: + - barretenberg-x86_64-linux-clang + barretenberg-docs: buildDir: barretenberg/cpp dockerfile: docs/Dockerfile diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh index 4a7b1bbe015..811bef01e6f 100755 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ b/scripts/ci/assemble_e2e_benchmark.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Grabs the log files uploaded in yarn-project/end-to-end/scripts/upload_logs_to_s3.sh +# Grabs the log files uploaded in build-system/scripts/upload_logs_to_s3 # that contain representative benchmarks, extracts whatever metrics are interesting, # and assembles a single file that shows the current state of the repository. @@ -14,7 +14,7 @@ BASE_COMMIT_HASH="" BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json" BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" -# Adapted from yarn-project/end-to-end/scripts/upload_logs_to_s3.sh +# Paths from build-system/scripts/upload_logs_to_s3 if [ "${CIRCLE_BRANCH:-}" = "master" ]; then LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json"