feat: IVC integration tests using new accumulate model (#7946)

Introduces more in depth testing of the AztecIvc scheme, including tests
that mimic the benchmark setup as well as more in depth tests for the
inter-circuit databus consistency checks.

Other noteworthy components of this PR:
- Introduces utility class `PrivateFunctionExecutionMockCircuitProducer`
for consistently generating mock circuits (shared between the benchmark
and this new test suite). Subclass `MockDatabusProducer` incorporates
nontrivial databus interactions into these circuits for the first time.
- Removes the old `accumulate()` method on AztecIvc (which
simultaneously added recursive verifiers and performed prover work) in
favor of two methods that separate the logic but combine to the same
effect: `complete_kernel_circuit_logic()` and a new `accumulate()`
(which now only performs the prover work).

Confusing jump in benchmark explained in
[this](AztecProtocol/barretenberg#1072) issue:

```
-----------------------------------------------------------------------------------------
Benchmark                               Time             CPU   Iterations UserCounters...
-----------------------------------------------------------------------------------------
AztecIVCBench/FullStructured/6      38855 ms        36066 ms            1 Arithmetic::accumulate=4.03742M DEBUG:ProverPolynomials()=12 DEBUG:ProverPolynomials()(t)=3.68002G 

function                                  ms     % sum
construct_circuits(t)                   4474    11.61%
ProverInstance(Circuit&)(t)             6787    17.62%
ProtogalaxyProver::fold_instances(t)   21788    56.55%
Decider::construct_proof(t)             1655     4.30%
ECCVMProver(CircuitBuilder&)(t)          235     0.61%
ECCVMProver::construct_proof(t)         2585     6.71%
TranslatorProver::construct_proof(t)     824     2.14%
Goblin::merge(t)                         177     0.46%
```

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.cpp

@@ -2,33 +2,6 @@
 
 namespace bb {
 
-/**
- * @brief Accumulate a circuit into the IVC scheme
- * @details If this is the first circuit being accumulated, initialize the prover and verifier accumulators. Otherwise,
- * fold the instance for the provided circuit into the accumulator. When two fold proofs have been enqueued, two
- * recursive folding verifications are appended to the next circuit that is accumulated, which must be a kernel.
- * Similarly, merge proofs are stored in a queue and recursively verified in kernels.
- *
- * @param circuit Circuit to be accumulated/folded
- * @param precomputed_vk Optional precomputed VK (otherwise will be computed herein)
- */
-void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<VerificationKey>& precomputed_vk)
-{
-    circuit_count++; // increment the count of circuits processed into the IVC
-
-    // The aztec architecture dictates that every second circuit is a kernel. This check can be triggered/replaced by
-    // the presence of the recursive folding verify opcode once it is introduced into noir.
-    is_kernel = (circuit_count % 2 == 0);
-
-    // If present circuit is a kernel, perform required recursive PG and/or merge verifications and databus checks
-    if (is_kernel) {
-        complete_kernel_circuit_logic(circuit);
-    }
-
-    // Perform PG and/or merge proving
-    execute_accumulation_prover(circuit, precomputed_vk);
-}
-
 /**
  * @brief Append logic to complete a kernel circuit
  * @details A kernel circuit may contain some combination of PG recursive verification, merge recursive verification,
@@ -38,7 +11,7 @@ void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verifica
  */
 void AztecIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
 {
-    BB_OP_COUNT_TIME_NAME("construct_circuits");
+    circuit.databus_propagation_data.is_kernel = true;
 
     // The folding verification queue should be either empty or contain two fold proofs
     ASSERT(verification_queue.empty() || verification_queue.size() == 2);
@@ -70,8 +43,7 @@ void AztecIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
  * @param circuit
  * @param precomputed_vk
  */
-void AztecIVC::execute_accumulation_prover(ClientCircuit& circuit,
-                                           const std::shared_ptr<VerificationKey>& precomputed_vk)
+void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<VerificationKey>& precomputed_vk)
 {
     // Construct merge proof for the present circuit and add to merge verification queue
     MergeProof merge_proof = goblin.prove_merge(circuit);
@@ -86,12 +58,12 @@ void AztecIVC::execute_accumulation_prover(ClientCircuit& circuit,
 
     // Set the instance verification key from precomputed if available, else compute it
     instance_vk = precomputed_vk ? precomputed_vk : std::make_shared<VerificationKey>(prover_instance->proving_key);
-    instance_vk->databus_propagation_data.is_kernel = is_kernel; // Store whether the present circuit is a kernel
 
     // If this is the first circuit simply initialize the prover and verifier accumulator instances
-    if (circuit_count == 1) {
+    if (!initialized) {
         fold_output.accumulator = prover_instance;
         verifier_accumulator = std::make_shared<VerifierInstance>(instance_vk);
+        initialized = true;
     } else { // Otherwise, fold the new instance into the accumulator
         FoldingProver folding_prover({ fold_output.accumulator, prover_instance });
         fold_output = folding_prover.fold_instances();
@@ -162,36 +134,6 @@ HonkProof AztecIVC::decider_prove() const
     return decider_prover.construct_proof();
 }
 
-/**
- * @brief Given a set of circuits, compute the verification keys that will be required by the IVC scheme
- * @details The verification keys computed here are in general not the same as the verification keys for the
- * raw input circuits because recursive verifier circuits (merge and/or folding) may be appended to the incoming
- * circuits as part accumulation.
- * @note This method exists for convenience and is not not meant to be used in practice for IVC. Given a set of
- * circuits, it could be run once and for all to compute then save the required VKs. It also provides a convenient
- * (albeit innefficient) way of separating out the cost of computing VKs from a benchmark.
- *
- * @param circuits A copy of the circuits to be accumulated (passing by reference would alter the original circuits)
- * @return std::vector<std::shared_ptr<AztecIVC::VerificationKey>>
- */
-std::vector<std::shared_ptr<AztecIVC::VerificationKey>> AztecIVC::precompute_folding_verification_keys(
-    std::vector<ClientCircuit> circuits)
-{
-    std::vector<std::shared_ptr<VerificationKey>> vkeys;
-
-    for (auto& circuit : circuits) {
-        accumulate(circuit);
-        vkeys.emplace_back(instance_vk);
-    }
-
-    // Reset the scheme so it can be reused for actual accumulation, maintaining the trace structure setting as is
-    TraceStructure structure = trace_structure;
-    *this = AztecIVC();
-    this->trace_structure = structure;
-
-    return vkeys;
-}
-
 /**
  * @brief Construct and verify a proof for the IVC
  * @note Use of this method only makes sense when the prover and verifier are the same entity, e.g. in

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.hpp

@@ -90,16 +90,12 @@ class AztecIVC {
     // A flag indicating whether or not to construct a structured trace in the ProverInstance
     TraceStructure trace_structure = TraceStructure::NONE;
 
-    size_t circuit_count = 0; // the number of circuits processed into the IVC
-    bool is_kernel = false;   // is the present circuit a kernel
+    bool initialized = false; // Is the IVC accumulator initialized
 
     // Complete the logic of a kernel circuit (e.g. PG/merge recursive verification, databus consistency checks)
     void complete_kernel_circuit_logic(ClientCircuit& circuit);
 
     // Perform prover work for accumulation (e.g. PG folding, merge proving)
-    void execute_accumulation_prover(ClientCircuit& circuit,
-                                     const std::shared_ptr<VerificationKey>& precomputed_vk = nullptr);
-
     void accumulate(ClientCircuit& circuit, const std::shared_ptr<VerificationKey>& precomputed_vk = nullptr);
 
     Proof prove();
@@ -115,7 +111,5 @@ class AztecIVC {
     bool prove_and_verify();
 
     HonkProof decider_prove() const;
-
-    std::vector<std::shared_ptr<VerificationKey>> precompute_folding_verification_keys(std::vector<ClientCircuit>);
 };
 } // namespace bb

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.test.cpp

@@ -52,6 +52,48 @@ class AztecIVCTests : public ::testing::Test {
         return circuit;
     }
 
+    /**
+     * @brief A test utility for generating alternating mock app and kernel circuits and precomputing verification keys
+     *
+     */
+    class MockCircuitProducer {
+        using ClientCircuit = AztecIVC::ClientCircuit;
+
+        bool is_kernel = false;
+
+      public:
+        ClientCircuit create_next_circuit(AztecIVC& ivc, size_t log2_num_gates = 16)
+        {
+            ClientCircuit circuit{ ivc.goblin.op_queue };
+            circuit = create_mock_circuit(ivc, log2_num_gates); // construct mock base logic
+            if (is_kernel) {
+                ivc.complete_kernel_circuit_logic(circuit); // complete with recursive verifiers etc
+            }
+            is_kernel = !is_kernel; // toggle is_kernel on/off alternatingly
+
+            return circuit;
+        }
+
+        auto precompute_verification_keys(const size_t num_circuits,
+                                          TraceStructure trace_structure,
+                                          size_t log2_num_gates = 16)
+        {
+            AztecIVC ivc; // temporary IVC instance needed to produce the complete kernel circuits
+            ivc.trace_structure = trace_structure;
+
+            std::vector<std::shared_ptr<VerificationKey>> vkeys;
+
+            for (size_t idx = 0; idx < num_circuits; ++idx) {
+                ClientCircuit circuit = create_next_circuit(ivc, log2_num_gates); // create the next circuit
+                ivc.accumulate(circuit);                                          // accumulate the circuit
+                vkeys.emplace_back(ivc.instance_vk);                              // save the VK for the circuit
+            }
+            is_kernel = false;
+
+            return vkeys;
+        }
+    };
+
     /**
      * @brief Tamper with a proof by finding the first non-zero value and incrementing it by 1
      *
@@ -77,12 +119,14 @@ TEST_F(AztecIVCTests, Basic)
 {
     AztecIVC ivc;
 
+    MockCircuitProducer circuit_producer;
+
     // Initialize the IVC with an arbitrary circuit
-    Builder circuit_0 = create_mock_circuit(ivc);
+    Builder circuit_0 = circuit_producer.create_next_circuit(ivc);
     ivc.accumulate(circuit_0);
 
     // Create another circuit and accumulate
-    Builder circuit_1 = create_mock_circuit(ivc);
+    Builder circuit_1 = circuit_producer.create_next_circuit(ivc);
     ivc.accumulate(circuit_1);
 
     EXPECT_TRUE(ivc.prove_and_verify());
@@ -102,10 +146,12 @@ TEST_F(AztecIVCTests, BadProofFailure)
         AztecIVC ivc;
         ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-        // Construct a set of arbitrary circuits
+        MockCircuitProducer circuit_producer;
+
+        // Construct and accumulate a set of mocked private function execution circuits
         size_t NUM_CIRCUITS = 4;
         for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-            auto circuit = create_mock_circuit(ivc, /*log2_num_gates=*/5);
+            auto circuit = circuit_producer.create_next_circuit(ivc, /*log2_num_gates=*/5);
             ivc.accumulate(circuit);
         }
         EXPECT_TRUE(ivc.prove_and_verify());
@@ -116,10 +162,12 @@ TEST_F(AztecIVCTests, BadProofFailure)
         AztecIVC ivc;
         ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-        // Construct a set of arbitrary circuits
+        MockCircuitProducer circuit_producer;
+
+        // Construct and accumulate a set of mocked private function execution circuits
         size_t NUM_CIRCUITS = 4;
         for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-            auto circuit = create_mock_circuit(ivc, /*log2_num_gates=*/5);
+            auto circuit = circuit_producer.create_next_circuit(ivc, /*log2_num_gates=*/5);
             ivc.accumulate(circuit);
 
             if (idx == 2) {
@@ -136,10 +184,12 @@ TEST_F(AztecIVCTests, BadProofFailure)
         AztecIVC ivc;
         ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-        // Construct a set of arbitrary circuits
+        MockCircuitProducer circuit_producer;
+
+        // Construct and accumulate a set of mocked private function execution circuits
         size_t NUM_CIRCUITS = 4;
         for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-            auto circuit = create_mock_circuit(ivc, /*log2_num_gates=*/5);
+            auto circuit = circuit_producer.create_next_circuit(ivc, /*log2_num_gates=*/5);
             ivc.accumulate(circuit);
 
             if (idx == 2) {
@@ -156,10 +206,12 @@ TEST_F(AztecIVCTests, BadProofFailure)
         AztecIVC ivc;
         ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-        // Construct a set of arbitrary circuits
+        MockCircuitProducer circuit_producer;
+
+        // Construct and accumulate a set of mocked private function execution circuits
         size_t NUM_CIRCUITS = 4;
         for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-            auto circuit = create_mock_circuit(ivc, /*log2_num_gates=*/5);
+            auto circuit = circuit_producer.create_next_circuit(ivc, /*log2_num_gates=*/5);
             ivc.accumulate(circuit);
         }
 
@@ -181,15 +233,13 @@ TEST_F(AztecIVCTests, BasicLarge)
 {
     AztecIVC ivc;
 
-    // Construct a set of arbitrary circuits
+    MockCircuitProducer circuit_producer;
+
+    // Construct and accumulate a set of mocked private function execution circuits
     size_t NUM_CIRCUITS = 6;
     std::vector<Builder> circuits;
     for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-        circuits.emplace_back(create_mock_circuit(ivc));
-    }
-
-    // Accumulate each circuit
-    for (auto& circuit : circuits) {
+        auto circuit = circuit_producer.create_next_circuit(ivc);
         ivc.accumulate(circuit);
     }
 
@@ -205,17 +255,17 @@ TEST_F(AztecIVCTests, BasicStructured)
     AztecIVC ivc;
     ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-    // Construct some circuits of varying size
-    Builder circuit_0 = create_mock_circuit(ivc, /*log2_num_gates=*/5);
-    Builder circuit_1 = create_mock_circuit(ivc, /*log2_num_gates=*/6);
-    Builder circuit_2 = create_mock_circuit(ivc, /*log2_num_gates=*/7);
-    Builder circuit_3 = create_mock_circuit(ivc, /*log2_num_gates=*/8);
+    MockCircuitProducer circuit_producer;
 
-    // The circuits can be accumulated as normal due to the structured trace
-    ivc.accumulate(circuit_0);
-    ivc.accumulate(circuit_1);
-    ivc.accumulate(circuit_2);
-    ivc.accumulate(circuit_3);
+    size_t NUM_CIRCUITS = 4;
+
+    // Construct and accumulate some circuits of varying size
+    size_t log2_num_gates = 5;
+    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+        auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates);
+        ivc.accumulate(circuit);
+        log2_num_gates += 2;
+    }
 
     EXPECT_TRUE(ivc.prove_and_verify());
 };
@@ -228,19 +278,16 @@ TEST_F(AztecIVCTests, PrecomputedVerificationKeys)
 {
     AztecIVC ivc;
 
-    // Construct a set of arbitrary circuits
     size_t NUM_CIRCUITS = 4;
-    std::vector<Builder> circuits;
-    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-        circuits.emplace_back(create_mock_circuit(ivc));
-    }
 
-    // Precompute the verification keys that will be needed for the IVC
-    auto precomputed_vkeys = ivc.precompute_folding_verification_keys(circuits);
+    MockCircuitProducer circuit_producer;
+
+    auto precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, TraceStructure::NONE);
 
-    // Accumulate each circuit using the precomputed VKs
-    for (auto [circuit, precomputed_vk] : zip_view(circuits, precomputed_vkeys)) {
-        ivc.accumulate(circuit, precomputed_vk);
+    // Construct and accumulate set of circuits using the precomputed vkeys
+    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+        auto circuit = circuit_producer.create_next_circuit(ivc);
+        ivc.accumulate(circuit, precomputed_vks[idx]);
     }
 
     EXPECT_TRUE(ivc.prove_and_verify());
@@ -255,19 +302,18 @@ TEST_F(AztecIVCTests, StructuredPrecomputedVKs)
     AztecIVC ivc;
     ivc.trace_structure = TraceStructure::SMALL_TEST;
 
-    // Construct a set of arbitrary circuits
     size_t NUM_CIRCUITS = 4;
-    std::vector<Builder> circuits;
-    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
-        circuits.emplace_back(create_mock_circuit(ivc, /*log2_num_gates=*/5));
-    }
+    size_t log2_num_gates = 5; // number of gates in baseline mocked circuit
 
-    // Precompute the (structured) verification keys that will be needed for the IVC
-    auto precomputed_vkeys = ivc.precompute_folding_verification_keys(circuits);
+    MockCircuitProducer circuit_producer;
 
-    // Accumulate each circuit
-    for (auto [circuit, precomputed_vk] : zip_view(circuits, precomputed_vkeys)) {
-        ivc.accumulate(circuit, precomputed_vk);
+    auto precomputed_vks =
+        circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_structure, log2_num_gates);
+
+    // Construct and accumulate set of circuits using the precomputed vkeys
+    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+        auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates);
+        ivc.accumulate(circuit, precomputed_vks[idx]);
     }
 
     EXPECT_TRUE(ivc.prove_and_verify());