fix: Fixed the memory issue (AztecProtocol/barretenberg#509)

* Fixed the memory issue
* Switched recursion to use check_circuit (but still use composers)
* Low-impact fix of circular dependency

---------

Co-authored-by: codygunton <codygunton@gmail.com>

barretenberg/cpp/src/barretenberg/common/log.hpp

@@ -9,15 +9,23 @@
 #define BENCHMARK_INFO_SEPARATOR "#"
 #define BENCHMARK_INFO_SUFFIX "##BENCHMARK_INFO_SUFFIX##"
 #define GET_COMPOSER_NAME_STRING(composer)                                                                             \
-    (typeid(composer) == typeid(plonk::StandardPlonkComposer)               ? "StandardPlonk"                          \
-     : typeid(composer) == typeid(plonk::TurboPlonkComposer)                ? "TurboPlonk"                             \
-     : typeid(composer) == typeid(plonk::UltraPlonkComposer)                ? "UltraPlonk"                             \
-     : typeid(composer) == typeid(honk::StandardHonkComposer)               ? "StandardHonk"                           \
-     : typeid(composer) == typeid(honk::UltraHonkComposer)                  ? "UltraHonk"                              \
-     : typeid(composer) == typeid(proof_system::StandardCircuitConstructor) ? "StandardArithemtization"                \
-     : typeid(composer) == typeid(proof_system::TurboCircuitConstructor)    ? "TurboArithemtization"                   \
-     : typeid(composer) == typeid(proof_system::UltraCircuitConstructor)    ? "UltraArithmetization"                   \
-                                                                            : "NullPlonk")
+    (typeid(composer) == typeid(plonk::StandardPlonkComposer)                                                          \
+         ? "StandardPlonk"                                                                                             \
+         : typeid(composer) == typeid(plonk::TurboPlonkComposer)                                                       \
+               ? "TurboPlonk"                                                                                          \
+               : typeid(composer) == typeid(plonk::UltraPlonkComposer)                                                 \
+                     ? "UltraPlonk"                                                                                    \
+                     : typeid(composer) == typeid(honk::StandardHonkComposer)                                          \
+                           ? "StandardHonk"                                                                            \
+                           : typeid(composer) == typeid(honk::UltraHonkComposer)                                       \
+                                 ? "UltraHonk"                                                                         \
+                                 : typeid(composer) == typeid(proof_system::StandardCircuitConstructor)                \
+                                       ? "StandardArithemtization"                                                     \
+                                       : typeid(composer) == typeid(proof_system::TurboCircuitConstructor)             \
+                                             ? "TurboArithemtization"                                                  \
+                                             : typeid(composer) == typeid(proof_system::UltraCircuitConstructor)       \
+                                                   ? "UltraArithmetization"                                            \
+                                                   : "NullPlonk")
 
 namespace {
 

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -103,7 +103,7 @@ void create_circuit(Composer& composer, acir_format const& constraint_system)
         if (i == constraint_system.recursion_constraints.size() - 1) {
             std::vector<uint32_t> proof_output_witness_indices(constraint.output_aggregation_object.begin(),
                                                                constraint.output_aggregation_object.end());
-            composer.set_recursive_proof(proof_output_witness_indices);
+            composer.circuit_constructor.set_recursive_proof(proof_output_witness_indices);
         }
     }
 }
@@ -230,7 +230,7 @@ void create_circuit_with_witness(Composer& composer, acir_format const& constrai
         if (i == constraint_system.recursion_constraints.size() - 1) {
             std::vector<uint32_t> proof_output_witness_indices(constraint.output_aggregation_object.begin(),
                                                                constraint.output_aggregation_object.end());
-            composer.set_recursive_proof(proof_output_witness_indices);
+            composer.circuit_constructor.set_recursive_proof(proof_output_witness_indices);
         }
     }
 }

barretenberg/cpp/src/barretenberg/ecc/CMakeLists.txt

@@ -1,4 +1,4 @@
-barretenberg_module(ecc numeric crypto_keccak srs)
+barretenberg_module(ecc numeric crypto_keccak)
 
 if(DISABLE_ADX)
     message(STATUS "Disabling ADX assembly variant.")

barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/point_table.hpp

@@ -17,6 +17,7 @@ inline size_t point_table_size(size_t num_points)
 
 inline size_t point_table_buf_size(size_t num_points)
 {
+    // TODO(Cody): This could be trouble if we change curves.
     return sizeof(g1::affine_element) * point_table_size(num_points);
 }
 

barretenberg/cpp/src/barretenberg/honk/composer/composer_helper/ultra_honk_composer_helper.cpp

@@ -134,17 +134,23 @@ void UltraHonkComposerHelper::compute_witness(CircuitConstructor& circuit_constr
     // Copy memory read/write record data into proving key. Prover needs to know which gates contain a read/write
     // 'record' witness on the 4th wire. This wire value can only be fully computed once the first 3 wire polynomials
     // have been committed to. The 4th wire on these gates will be a random linear combination of the first 3 wires,
-    // using the plookup challenge `eta`
+    // using the plookup challenge `eta`. Because we shift the gates by the number of public inputs, we need to update
+    // the records with the public_inputs offset
+    const uint32_t public_inputs_count = static_cast<uint32_t>(circuit_constructor.public_inputs.size());
+    auto add_public_inputs_offset = [public_inputs_count](uint32_t gate_index) {
+        return gate_index + public_inputs_count;
+    };
     proving_key->memory_read_records = std::vector<uint32_t>();
     proving_key->memory_write_records = std::vector<uint32_t>();
-    proving_key->memory_read_records.reserve(circuit_constructor.memory_read_records.size());
-    proving_key->memory_write_records.reserve(circuit_constructor.memory_write_records.size());
-    std::copy(circuit_constructor.memory_read_records.begin(),
-              circuit_constructor.memory_read_records.end(),
-              std::back_inserter(proving_key->memory_read_records));
-    std::copy(circuit_constructor.memory_write_records.begin(),
-              circuit_constructor.memory_write_records.end(),
-              std::back_inserter(proving_key->memory_write_records));
+
+    std::transform(circuit_constructor.memory_read_records.begin(),
+                   circuit_constructor.memory_read_records.end(),
+                   std::back_inserter(proving_key->memory_read_records),
+                   add_public_inputs_offset);
+    std::transform(circuit_constructor.memory_write_records.begin(),
+                   circuit_constructor.memory_write_records.end(),
+                   std::back_inserter(proving_key->memory_write_records),
+                   add_public_inputs_offset);
 
     computed_witness = true;
 }

barretenberg/cpp/src/barretenberg/honk/composer/ultra_honk_composer.hpp

@@ -52,8 +52,8 @@ class UltraHonkComposer {
         , num_gates(circuit_constructor.num_gates)
         , variables(circuit_constructor.variables)
         , zero_idx(circuit_constructor.zero_idx)
-        , contains_recursive_proof(composer_helper.contains_recursive_proof)
-        , recursive_proof_public_input_indices(composer_helper.recursive_proof_public_input_indices)
+        , contains_recursive_proof(circuit_constructor.contains_recursive_proof)
+        , recursive_proof_public_input_indices(circuit_constructor.recursive_proof_public_input_indices)
     {
         // TODO(#217/#423): Related to issue of ensuring no identically 0 polynomials
         add_gates_to_ensure_all_polys_are_non_zero();

barretenberg/cpp/src/barretenberg/plonk/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(plonk proof_system transcript crypto_pedersen_commitment polynomials crypto_sha256 ecc crypto_blake3s)
+barretenberg_module(plonk proof_system transcript crypto_pedersen_commitment polynomials crypto_sha256 ecc crypto_blake3s srs)

barretenberg/cpp/src/barretenberg/plonk/composer/composer_helper/standard_plonk_composer_helper.cpp

@@ -76,9 +76,10 @@ std::shared_ptr<plonk::proving_key> StandardPlonkComposerHelper::compute_proving
     compute_standard_plonk_sigma_permutations<Flavor>(circuit_constructor, circuit_proving_key.get());
 
     circuit_proving_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
     // What does this line do exactly?
-    circuit_proving_key->contains_recursive_proof = contains_recursive_proof;
+    circuit_proving_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
     return circuit_proving_key;
 }
 
@@ -101,8 +102,9 @@ std::shared_ptr<plonk::verification_key> StandardPlonkComposerHelper::compute_ve
         plonk::compute_verification_key_common(circuit_proving_key, crs_factory_->get_verifier_crs());
     circuit_verification_key->composer_type = circuit_proving_key->composer_type;
     circuit_verification_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
-    circuit_verification_key->contains_recursive_proof = contains_recursive_proof;
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
+    circuit_verification_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
 
     return circuit_verification_key;
 }

barretenberg/cpp/src/barretenberg/plonk/composer/composer_helper/standard_plonk_composer_helper.hpp

@@ -25,8 +25,6 @@ class StandardPlonkComposerHelper {
     // The crs_factory holds the path to the srs and exposes methods to extract the srs elements
     std::shared_ptr<barretenberg::srs::factories::CrsFactory> crs_factory_;
 
-    std::vector<uint32_t> recursive_proof_public_input_indices;
-    bool contains_recursive_proof = false;
     bool computed_witness = false;
 
     StandardPlonkComposerHelper()
@@ -59,20 +57,6 @@ class StandardPlonkComposerHelper {
         };
         return result;
     }
-    void add_recursive_proof(CircuitConstructor& circuit_constructor,
-                             const std::vector<uint32_t>& proof_output_witness_indices)
-    {
-
-        if (contains_recursive_proof) {
-            circuit_constructor.failure("added recursive proof when one already exists");
-        }
-        contains_recursive_proof = true;
-
-        for (const auto& idx : proof_output_witness_indices) {
-            circuit_constructor.set_public_input(idx);
-            recursive_proof_public_input_indices.push_back((uint32_t)(circuit_constructor.public_inputs.size() - 1));
-        }
-    }
     std::shared_ptr<plonk::proving_key> compute_proving_key(const CircuitConstructor& circuit_constructor);
     std::shared_ptr<plonk::verification_key> compute_verification_key(const CircuitConstructor& circuit_constructor);
 

barretenberg/cpp/src/barretenberg/plonk/composer/composer_helper/turbo_plonk_composer_helper.cpp

@@ -50,8 +50,9 @@ std::shared_ptr<plonk::proving_key> TurboPlonkComposerHelper::compute_proving_ke
     // Compute sigma polynomials (TODO(kesha): we should update that late)
     compute_standard_plonk_sigma_permutations<Flavor>(circuit_constructor, circuit_proving_key.get());
     circuit_proving_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
-    circuit_proving_key->contains_recursive_proof = contains_recursive_proof;
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
+    circuit_proving_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
     return circuit_proving_key;
 }
 
@@ -74,8 +75,9 @@ std::shared_ptr<plonk::verification_key> TurboPlonkComposerHelper::compute_verif
         plonk::compute_verification_key_common(circuit_proving_key, crs_factory_->get_verifier_crs());
     circuit_verification_key->composer_type = circuit_proving_key->composer_type;
     circuit_verification_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
-    circuit_verification_key->contains_recursive_proof = contains_recursive_proof;
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
+    circuit_verification_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
 
     return circuit_verification_key;
 }

barretenberg/cpp/src/barretenberg/plonk/composer/composer_helper/turbo_plonk_composer_helper.hpp

@@ -23,9 +23,6 @@ class TurboPlonkComposerHelper {
     // The crs_factory holds the path to the srs and exposes methods to extract the srs elements
     std::shared_ptr<barretenberg::srs::factories::CrsFactory> crs_factory_;
 
-    std::vector<uint32_t> recursive_proof_public_input_indices;
-    bool contains_recursive_proof = false;
-
     bool computed_witness = false;
     TurboPlonkComposerHelper()
         : TurboPlonkComposerHelper(std::shared_ptr<barretenberg::srs::factories::CrsFactory>(
@@ -61,20 +58,6 @@ class TurboPlonkComposerHelper {
         };
         return result;
     }
-    void add_recursive_proof(CircuitConstructor& circuit_constructor,
-                             const std::vector<uint32_t>& proof_output_witness_indices)
-    {
-
-        if (contains_recursive_proof) {
-            circuit_constructor.failure("added recursive proof when one already exists");
-        }
-        contains_recursive_proof = true;
-
-        for (const auto& idx : proof_output_witness_indices) {
-            circuit_constructor.set_public_input(idx);
-            recursive_proof_public_input_indices.push_back((uint32_t)(circuit_constructor.public_inputs.size() - 1));
-        }
-    }
     static transcript::Manifest create_manifest(const size_t num_public_inputs)
     {
         // add public inputs....

barretenberg/cpp/src/barretenberg/plonk/composer/composer_helper/ultra_plonk_composer_helper.cpp

@@ -142,17 +142,23 @@ void UltraPlonkComposerHelper::compute_witness(CircuitConstructor& circuit_const
     // Copy memory read/write record data into proving key. Prover needs to know which gates contain a read/write
     // 'record' witness on the 4th wire. This wire value can only be fully computed once the first 3 wire polynomials
     // have been committed to. The 4th wire on these gates will be a random linear combination of the first 3 wires,
-    // using the plookup challenge `eta`
+    // using the plookup challenge `eta`. Because we shift the gates by the number of public inputs, we need to update
+    // the records with the public_inputs offset
+    const uint32_t public_inputs_count = static_cast<uint32_t>(circuit_constructor.public_inputs.size());
+    auto add_public_inputs_offset = [public_inputs_count](uint32_t gate_index) {
+        return gate_index + public_inputs_count;
+    };
     circuit_proving_key->memory_read_records = std::vector<uint32_t>();
     circuit_proving_key->memory_write_records = std::vector<uint32_t>();
-    circuit_proving_key->memory_read_records.reserve(circuit_constructor.memory_read_records.size());
-    circuit_proving_key->memory_write_records.reserve(circuit_constructor.memory_write_records.size());
-    std::copy(circuit_constructor.memory_read_records.begin(),
-              circuit_constructor.memory_read_records.end(),
-              std::back_inserter(circuit_proving_key->memory_read_records));
-    std::copy(circuit_constructor.memory_write_records.begin(),
-              circuit_constructor.memory_write_records.end(),
-              std::back_inserter(circuit_proving_key->memory_write_records));
+
+    std::transform(circuit_constructor.memory_read_records.begin(),
+                   circuit_constructor.memory_read_records.end(),
+                   std::back_inserter(circuit_proving_key->memory_read_records),
+                   add_public_inputs_offset);
+    std::transform(circuit_constructor.memory_write_records.begin(),
+                   circuit_constructor.memory_write_records.end(),
+                   std::back_inserter(circuit_proving_key->memory_write_records),
+                   add_public_inputs_offset);
 
     computed_witness = true;
 }
@@ -469,9 +475,10 @@ std::shared_ptr<proving_key> UltraPlonkComposerHelper::compute_proving_key(Circu
     circuit_proving_key->polynomial_store.put("s_fft", std::move(s_fft));
 
     circuit_proving_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
 
-    circuit_proving_key->contains_recursive_proof = contains_recursive_proof;
+    circuit_proving_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
 
     return circuit_proving_key;
 }
@@ -498,9 +505,10 @@ std::shared_ptr<plonk::verification_key> UltraPlonkComposerHelper::compute_verif
 
     // See `add_recusrive_proof()` for how this recursive data is assigned.
     circuit_verification_key->recursive_proof_public_input_indices =
-        std::vector<uint32_t>(recursive_proof_public_input_indices.begin(), recursive_proof_public_input_indices.end());
+        std::vector<uint32_t>(circuit_constructor.recursive_proof_public_input_indices.begin(),
+                              circuit_constructor.recursive_proof_public_input_indices.end());
 
-    circuit_verification_key->contains_recursive_proof = contains_recursive_proof;
+    circuit_verification_key->contains_recursive_proof = circuit_constructor.contains_recursive_proof;
 
     return circuit_verification_key;
 }