feat: structured trace in client ivc (#6132)

Adds a version of the Full ClientIvc benchmark that utilizes a
structured trace, `FullStructured`.  See the PR description for 
more details.
---------

Co-authored-by: Rumata888 <isennovskiy@gmail.com>
Co-authored-by: codygunton <codygunton@gmail.com>

barretenberg/cpp/scripts/analyze_client_ivc_bench.py

@@ -71,3 +71,39 @@
     print(f"{key:<{max_label_length}}{time_ms:>8.0f}  {time_ms/total_time_ms:>8.2%}")
 
 
+# Relations breakdown
+# Note: The timings here are off likely because the tracking is occuring in a hot loop but 
+# they should be meaningful relative to one another
+print('\nRelation contributions (times to be interpreted relatively):')
+relations = [
+    "Arithmetic::accumulate(t)",
+    "Permutation::accumulate(t)",
+    "Lookup::accumulate(t)",
+    "DeltaRange::accumulate(t)",
+    "Elliptic::accumulate(t)",
+    "Auxiliary::accumulate(t)",
+    "EccOp::accumulate(t)",
+    "DatabusRead::accumulate(t)",
+    "PoseidonExt::accumulate(t)",
+    "PoseidonInt::accumulate(t)",
+]
+with open(PREFIX/IVC_BENCH_JSON, "r") as read_file:
+    read_result = json.load(read_file)
+    for _bench in read_result["benchmarks"]:
+        if _bench["name"] == BENCHMARK:
+            bench = _bench
+bench_components = dict(filter(lambda x: x[0] in relations, bench.items()))
+
+# For each kept time, get the proportion over all kept times.
+sum_of_kept_times_ms = sum(float(time)
+                           for _, time in bench_components.items())/1e6
+max_label_length = max(len(label) for label in relations)
+column = {"function": "function", "ms": "ms", "%": "% sum"}
+print(
+    f"{column['function']:<{max_label_length}}{column['ms']:>8}  {column['%']:>8}")
+for key in relations:
+    if key not in bench:
+        time_ms = 0
+    else:
+        time_ms = bench[key]/1e6
+    print(f"{key:<{max_label_length}}{time_ms:>8.0f}  {time_ms/sum_of_kept_times_ms:>8.2%}")

barretenberg/cpp/scripts/benchmark_client_ivc.sh

@@ -2,6 +2,7 @@
 set -eu
 
 TARGET="client_ivc_bench"
+# Note: to run structured trace version, change "Full" to "FullStructured" here and in analyze script
 FILTER="ClientIVCBench/Full/6$"
 BUILD_DIR=build-op-count-time
 

barretenberg/cpp/scripts/compare_branch_vs_baseline_remote.sh

@@ -11,7 +11,7 @@
 
 # Specify the benchmark suite and the "baseline" branch against which to compare
 BENCHMARK=${1:-goblin_bench}
-FILTER=${2:-""}
+FILTER=${2:-"*."}
 PRESET=${3:-clang16}
 BUILD_DIR=${4:-build}
 HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16}
@@ -24,12 +24,7 @@ echo -e "\nComparing $BENCHMARK between $BASELINE_BRANCH and current branch:"
 # Move above script dir.
 cd $(dirname $0)/..
 
-# Configure and build benchmark in feature branch
-echo -e "\nConfiguring and building $BENCHMARK in current feature branch...\n"
-cmake --preset $PRESET
-cmake --build --preset $PRESET --target $BENCHMARK
-
-# Run bench in current branch
+# Run benchmark in current branch
 echo -e "\nRunning benchmark in feature branch.."
 ./scripts/benchmark_remote.sh $BENCHMARK\
                               "./$BENCHMARK --benchmark_filter=$FILTER\
@@ -40,13 +35,8 @@ echo -e "\nRunning benchmark in feature branch.."
 
 scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/results_after.json $BUILD_DIR/
 
-# Configure and build benchmark in $BASELINE branch
-echo -e "\nConfiguring and building $BENCHMARK in $BASELINE_BRANCH...\n"
+# Run benchmark in baseline branch
 git checkout $BASELINE_BRANCH
-cmake --preset $PRESET
-cmake --build --preset $PRESET --target $BENCHMARK
-
-# Run bench in current branch
 echo -e "\nRunning benchmark in feature branch.."
 ./scripts/benchmark_remote.sh $BENCHMARK\
                               "./$BENCHMARK --benchmark_filter=$FILTER\

barretenberg/cpp/scripts/compare_branch_vs_baseline_remote_wasm.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Install requirements (numpy + scipy) for comparison script if necessary.
+# Note: By default, installation will occur in $HOME/.local/bin.
+# pip3 install --user -r $BUILD_DIR/_deps/benchmark-src/requirements.txt
+
+
+# This script is used to compare a suite of benchmarks between baseline (default: master) and
+# the branch from which the script is run. Simply check out the branch of interest, ensure
+# it is up to date with local master, and run the script.
+
+# Specify the benchmark suite and the "baseline" branch against which to compare
+BENCHMARK=${1:-goblin_bench}
+FILTER=${2:-"*."}
+PRESET=${3:-wasm-threads}
+BUILD_DIR=${4:-build-wasm-threads}
+HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16}
+
+BASELINE_BRANCH="master"
+BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools"
+
+echo -e "\nComparing $BENCHMARK between $BASELINE_BRANCH and current branch:"
+
+# Move above script dir.
+cd $(dirname $0)/..
+
+# Run benchmark in feature branch
+echo -e "\nRunning benchmark in feature branch.."
+./scripts/benchmark_wasm_remote.sh $BENCHMARK\
+                                   "./$BENCHMARK --benchmark_filter=$FILTER\
+                                                 --benchmark_out=../results_after.json\
+                                                 --benchmark_out_format=json"
+
+scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/results_after.json $BUILD_DIR/
+
+# Run benchmark in $BASELINE branch
+
+echo -e "\nRunning benchmark in baseline branch.."
+git checkout $BASELINE_BRANCH
+./scripts/benchmark_wasm_remote.sh $BENCHMARK\
+                                   "./$BENCHMARK --benchmark_filter=$FILTER\
+                                                 --benchmark_out=../results_before.json\
+                                                 --benchmark_out_format=json"
+
+scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/results_before.json $BUILD_DIR/
+
+# Call compare.py on the results (json) to get high level statistics.
+# See docs at https://github.com/google/benchmark/blob/main/docs/tools.md for more details.
+$BENCH_TOOLS_DIR/compare.py benchmarks $BUILD_DIR/results_before.json $BUILD_DIR/results_after.json
+
+# Return to branch from which the script was called
+git checkout -

barretenberg/cpp/scripts/compare_client_ivc_bench.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -eu
+
+./scripts/compare_branch_vs_baseline_remote_wasm.sh client_ivc_bench 'Full/6$'

barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp

@@ -173,6 +173,25 @@ BENCHMARK_DEFINE_F(ClientIVCBench, Full)(benchmark::State& state)
     }
 }
 
+/**
+ * @brief Benchmark the prover work for the full PG-Goblin IVC protocol
+ *
+ */
+BENCHMARK_DEFINE_F(ClientIVCBench, FullStructured)(benchmark::State& state)
+{
+    ClientIVC ivc;
+    ivc.structured_flag = true;
+    ivc.precompute_folding_verification_keys();
+    for (auto _ : state) {
+        BB_REPORT_OP_COUNT_IN_BENCH(state);
+        // Perform a specified number of iterations of function/kernel accumulation
+        perform_ivc_accumulation_rounds(state, ivc);
+
+        // Construct IVC scheme proof (fold, decider, merge, eccvm, translator)
+        ivc.prove();
+    }
+}
+
 /**
  * @brief Benchmark only the accumulation rounds
  *
@@ -252,6 +271,7 @@ BENCHMARK_DEFINE_F(ClientIVCBench, Translator)(benchmark::State& state)
         ->Arg(1 << 6)
 
 BENCHMARK_REGISTER_F(ClientIVCBench, Full)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, FullStructured)->Unit(benchmark::kMillisecond)->ARGS;
 BENCHMARK_REGISTER_F(ClientIVCBench, Accumulate)->Unit(benchmark::kMillisecond)->ARGS;
 BENCHMARK_REGISTER_F(ClientIVCBench, Decide)->Unit(benchmark::kMillisecond)->ARGS;
 BENCHMARK_REGISTER_F(ClientIVCBench, ECCVM)->Unit(benchmark::kMillisecond)->ARGS;

barretenberg/cpp/src/barretenberg/benchmark/relations_bench/relations.bench.cpp

@@ -1,4 +1,5 @@
 #include "barretenberg/eccvm/eccvm_flavor.hpp"
+#include "barretenberg/protogalaxy/protogalaxy_prover.hpp"
 #include "barretenberg/stdlib_circuit_builders/goblin_ultra_flavor.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_flavor.hpp"
 #include "barretenberg/translator_vm/goblin_translator_flavor.hpp"
@@ -13,46 +14,109 @@ namespace bb::benchmark::relations {
 using Fr = bb::fr;
 using Fq = grumpkin::fr;
 
-template <typename Flavor, typename Relation> void execute_relation(::benchmark::State& state)
+// Generic helper for executing Relation::accumulate for the template specified input type
+template <typename Flavor, typename Relation, typename Input, typename Accumulator>
+void execute_relation(::benchmark::State& state)
 {
     using FF = typename Flavor::FF;
-    using AllValues = typename Flavor::AllValues;
-    using SumcheckArrayOfValuesOverSubrelations = typename Relation::SumcheckArrayOfValuesOverSubrelations;
 
     auto params = bb::RelationParameters<FF>::get_random();
 
-    // Extract an array containing all the polynomial evaluations at a given row i
-    AllValues new_value{};
-    // Define the appropriate SumcheckArrayOfValuesOverSubrelations type for this relation and initialize to zero
-    SumcheckArrayOfValuesOverSubrelations accumulator;
-    // Evaluate each constraint in the relation and check that each is satisfied
+    // Instantiate zero-initialized inputs and accumulator
+    Input input{};
+    Accumulator accumulator;
 
     for (auto _ : state) {
-        Relation::accumulate(accumulator, new_value, params, 1);
+        Relation::accumulate(accumulator, input, params, 1);
     }
 }
-BENCHMARK(execute_relation<UltraFlavor, UltraArithmeticRelation<Fr>>);
-BENCHMARK(execute_relation<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
-BENCHMARK(execute_relation<UltraFlavor, EllipticRelation<Fr>>);
-BENCHMARK(execute_relation<UltraFlavor, AuxiliaryRelation<Fr>>);
-BENCHMARK(execute_relation<UltraFlavor, LookupRelation<Fr>>);
-BENCHMARK(execute_relation<UltraFlavor, UltraPermutationRelation<Fr>>);
-
-BENCHMARK(execute_relation<GoblinUltraFlavor, EccOpQueueRelation<Fr>>);
-
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorDecompositionRelation<Fr>>);
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorOpcodeConstraintRelation<Fr>>);
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorAccumulatorTransferRelation<Fr>>);
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorDeltaRangeConstraintRelation<Fr>>);
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorNonNativeFieldRelation<Fr>>);
-BENCHMARK(execute_relation<GoblinTranslatorFlavor, GoblinTranslatorPermutationRelation<Fr>>);
-
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMLookupRelation<Fq>>);
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMMSMRelation<Fq>>);
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMPointTableRelation<Fq>>);
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMSetRelation<Fq>>);
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMTranscriptRelation<Fq>>);
-BENCHMARK(execute_relation<ECCVMFlavor, ECCVMWnafRelation<Fq>>);
+
+// Single execution of relation on values (FF), e.g. Sumcheck verifier / PG perturbator work
+template <typename Flavor, typename Relation> void execute_relation_for_values(::benchmark::State& state)
+{
+    using Input = typename Flavor::AllValues;
+    using Accumulator = typename Relation::SumcheckArrayOfValuesOverSubrelations;
+
+    execute_relation<Flavor, Relation, Input, Accumulator>(state);
+}
+
+// Single execution of relation on Sumcheck univariates, i.e. Sumcheck/Decider prover work
+template <typename Flavor, typename Relation> void execute_relation_for_univariates(::benchmark::State& state)
+{
+    using Input = typename Flavor::ExtendedEdges;
+    using Accumulator = typename Relation::SumcheckTupleOfUnivariatesOverSubrelations;
+
+    execute_relation<Flavor, Relation, Input, Accumulator>(state);
+}
+
+// Single execution of relation on PG univariates, i.e. PG combiner work
+template <typename Flavor, typename Relation> void execute_relation_for_pg_univariates(::benchmark::State& state)
+{
+    using ProverInstances = ProverInstances_<Flavor>;
+    using ProtoGalaxyProver = ProtoGalaxyProver_<ProverInstances>;
+    using Input = ProtoGalaxyProver::ExtendedUnivariates;
+    using Accumulator = typename Relation::template ProtogalaxyTupleOfUnivariatesOverSubrelations<ProverInstances::NUM>;
+
+    execute_relation<Flavor, Relation, Input, Accumulator>(state);
+}
+
+// Ultra relations (PG prover combiner work)
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, UltraArithmeticRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, EllipticRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, AuxiliaryRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<UltraFlavor, UltraPermutationRelation<Fr>>);
+
+// Goblin-Ultra only relations (PG prover combiner work)
+BENCHMARK(execute_relation_for_pg_univariates<GoblinUltraFlavor, EccOpQueueRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<GoblinUltraFlavor, DatabusLookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<GoblinUltraFlavor, Poseidon2ExternalRelation<Fr>>);
+BENCHMARK(execute_relation_for_pg_univariates<GoblinUltraFlavor, Poseidon2InternalRelation<Fr>>);
+
+// Ultra relations (Sumcheck prover work)
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, UltraArithmeticRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, EllipticRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, AuxiliaryRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<UltraFlavor, UltraPermutationRelation<Fr>>);
+
+// Goblin-Ultra only relations (Sumcheck prover work)
+BENCHMARK(execute_relation_for_univariates<GoblinUltraFlavor, EccOpQueueRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<GoblinUltraFlavor, DatabusLookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<GoblinUltraFlavor, Poseidon2ExternalRelation<Fr>>);
+BENCHMARK(execute_relation_for_univariates<GoblinUltraFlavor, Poseidon2InternalRelation<Fr>>);
+
+// Ultra relations (verifier work)
+BENCHMARK(execute_relation_for_values<UltraFlavor, UltraArithmeticRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, DeltaRangeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, EllipticRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, AuxiliaryRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, LookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<UltraFlavor, UltraPermutationRelation<Fr>>);
+
+// Goblin-Ultra only relations (verifier work)
+BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, EccOpQueueRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, DatabusLookupRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, Poseidon2ExternalRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, Poseidon2InternalRelation<Fr>>);
+
+// Translator VM
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorDecompositionRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorOpcodeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorAccumulatorTransferRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorDeltaRangeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorNonNativeFieldRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorPermutationRelation<Fr>>);
+
+// ECCVM
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMLookupRelation<Fq>>);
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMMSMRelation<Fq>>);
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMPointTableRelation<Fq>>);
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMSetRelation<Fq>>);
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMTranscriptRelation<Fq>>);
+BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMWnafRelation<Fq>>);
 
 } // namespace bb::benchmark::relations
 

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -11,7 +11,7 @@ namespace bb {
 void ClientIVC::initialize(ClientCircuit& circuit)
 {
     goblin.merge(circuit); // Construct new merge proof
-    prover_fold_output.accumulator = std::make_shared<ProverInstance>(circuit);
+    prover_fold_output.accumulator = std::make_shared<ProverInstance>(circuit, structured_flag);
 }
 
 /**
@@ -24,7 +24,7 @@ void ClientIVC::initialize(ClientCircuit& circuit)
 ClientIVC::FoldProof ClientIVC::accumulate(ClientCircuit& circuit)
 {
     goblin.merge(circuit); // Add recursive merge verifier and construct new merge proof
-    prover_instance = std::make_shared<ProverInstance>(circuit);
+    prover_instance = std::make_shared<ProverInstance>(circuit, structured_flag);
     FoldingProver folding_prover({ prover_fold_output.accumulator, prover_instance });
     prover_fold_output = folding_prover.fold_instances();
     return prover_fold_output.folding_data;

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -78,6 +78,9 @@ class ClientIVC {
     // be needed in the real IVC as they are provided as inputs
     std::shared_ptr<ProverInstance> prover_instance;
 
+    // A flag indicating whether or not to construct a structured trace in the ProverInstance
+    bool structured_flag = false;
+
     void initialize(ClientCircuit& circuit);
 
     FoldProof accumulate(ClientCircuit& circuit);

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp

@@ -118,7 +118,7 @@ typename ExecutionTrace_<Flavor>::TraceData ExecutionTrace_<Flavor>::construct_t
 
         // If the trace is structured, we populate the data from the next block at a fixed block size offset
         if (is_structured) {
-            offset += builder.FIXED_BLOCK_SIZE;
+            offset += block.get_fixed_size();
         } else { // otherwise, the next block starts immediately following the previous one
             offset += block_size;
         }