feat: Poseidon2 gates for Ultra arithmetisation (#7494)

Add Poseidon2 gates to the `UltraCircuitBuilder` which now ensures that
recursive verifier instantiated with the Ultra arithmetisation produce
the correct number of constraints.

Updates required:
* change verification key length and constant proof length constants
across the codebase (two selectors from the new gate whose commitments
need to be in the vk and the poseidon relation becomes the one with the
highest degree); changes to Prover.toml accordingly
* ensure the ultra recursive verifier still stays constant size now that
hashing produces gates
* small modification to solidity verifer to reflect the ones in cpp with
the caveat that the UltraKeccak flavor still doesnt support Poseidon
gate (changes coming in a followup PR)

Tube circuit changes in # of gates (post finalisation):
- number of gates prior this change, in master: 13947018
- number of gates post this change: 14038982

Closes AztecProtocol/barretenberg#1041

barretenberg/acir_tests/flows/honk_sol.sh

@@ -1,14 +1,18 @@
 #!/bin/sh
-set -eu
+set -eux
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/program.json"
+FLAGS="-c $CRS_PATH $VFLAG"
 
 export PROOF="$(pwd)/proof"
 export PROOF_AS_FIELDS="$(pwd)/proof_fields.json"
 
 # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs
-$BIN prove_keccak_ultra_honk -o proof
-$BIN write_vk_ultra_honk -o vk
+$BIN prove_keccak_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_ultra_keccak_honk -o vk $FLAGS $BFLAG
 $BIN proof_as_fields_honk -k vk -c $CRS_PATH -p $PROOF
-$BIN contract_ultra_honk -k vk -c $CRS_PATH -b ./target/program.json -o Verifier.sol
+$BIN contract_ultra_honk -k vk -c $CRS_PATH -o Verifier.sol
 
 # Export the paths to the environment variables for the js test runner
 export VERIFIER_PATH="$(pwd)/Verifier.sol"

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -197,6 +197,7 @@ bool proveAndVerifyHonkAcirFormat(acir_format::AcirFormat constraint_system, aci
 
     // Verify Honk proof
     auto verification_key = std::make_shared<VerificationKey>(prover.instance->proving_key);
+
     Verifier verifier{ verification_key };
 
     return verifier.verify_proof(proof);
@@ -589,7 +590,6 @@ void prove_tube(const std::string& output_path)
 
     builder->add_recursive_proof(current_aggregation_object);
 
-    info("num gates in tube circuit: ", builder->get_num_gates());
     using Prover = UltraProver_<UltraFlavor>;
     using Verifier = UltraVerifier_<UltraFlavor>;
     Prover tube_prover{ *builder };
@@ -819,7 +819,7 @@ void contract(const std::string& output_path, const std::string& vk_path)
  */
 void contract_honk(const std::string& output_path, const std::string& vk_path)
 {
-    using VerificationKey = UltraFlavor::VerificationKey;
+    using VerificationKey = UltraKeccakFlavor::VerificationKey;
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
 
     auto g2_data = get_bn254_g2_data(CRS_PATH);
@@ -1457,6 +1457,9 @@ int main(int argc, char* argv[])
         } else if (command == "write_vk_ultra_honk") {
             std::string output_path = get_option(args, "-o", "./target/vk");
             write_vk_honk<UltraFlavor>(bytecode_path, output_path);
+        } else if (command == "write_vk_ultra_keccak_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<UltraKeccakFlavor>(bytecode_path, output_path);
         } else if (command == "prove_mega_honk") {
             std::string output_path = get_option(args, "-o", "./proofs/proof");
             prove_honk<MegaFlavor>(bytecode_path, witness_path, output_path);

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_circuits.hpp

@@ -23,7 +23,8 @@ template <typename Builder> void generate_basic_arithmetic_circuit(Builder& buil
     stdlib::field_t a(stdlib::witness_t(&builder, fr::random_element()));
     stdlib::field_t b(stdlib::witness_t(&builder, fr::random_element()));
     stdlib::field_t c(&builder);
-    size_t passes = (1UL << log2_num_gates) / 4 - 4;
+    // Ensure the circuit is filled but finalisation doesn't make the circuit size go to the next power of two
+    size_t passes = (1UL << log2_num_gates) / 4 - 8;
     if (static_cast<int>(passes) <= 0) {
         throw_or_abort("too few gates");
     }

barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_checker.cpp

@@ -105,15 +105,16 @@ bool UltraCircuitChecker::check_block(Builder& builder,
         if (!result) {
             return report_fail("Failed Lookup check relation at row idx = ", idx);
         }
+        result = result && check_relation<PoseidonInternal>(values, params);
+        if (!result) {
+            return report_fail("Failed PoseidonInternal relation at row idx = ", idx);
+        }
+        result = result && check_relation<PoseidonExternal>(values, params);
+        if (!result) {
+            return report_fail("Failed PoseidonExternal relation at row idx = ", idx);
+        }
+
         if constexpr (IsMegaBuilder<Builder>) {
-            result = result && check_relation<PoseidonInternal>(values, params);
-            if (!result) {
-                return report_fail("Failed PoseidonInternal relation at row idx = ", idx);
-            }
-            result = result && check_relation<PoseidonExternal>(values, params);
-            if (!result) {
-                return report_fail("Failed PoseidonExternal relation at row idx = ", idx);
-            }
             result = result && check_databus_read(values, builder);
             if (!result) {
                 return report_fail("Failed databus read at row idx = ", idx);
@@ -289,10 +290,10 @@ void UltraCircuitChecker::populate_values(
     values.q_elliptic = block.q_elliptic()[idx];
     values.q_aux = block.q_aux()[idx];
     values.q_lookup = block.q_lookup_type()[idx];
+    values.q_poseidon2_internal = block.q_poseidon2_internal()[idx];
+    values.q_poseidon2_external = block.q_poseidon2_external()[idx];
     if constexpr (IsMegaBuilder<Builder>) {
         values.q_busread = block.q_busread()[idx];
-        values.q_poseidon2_internal = block.q_poseidon2_internal()[idx];
-        values.q_poseidon2_external = block.q_poseidon2_external()[idx];
     }
 }
 

barretenberg/cpp/src/barretenberg/commitment_schemes_recursion/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(commitment_schemes_recursion commitment_schemes stdlib_primitives)
+barretenberg_module(commitment_schemes_recursion commitment_schemes stdlib_poseidon2)

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/honk_contract.hpp

@@ -270,7 +270,7 @@ struct Transcript {
     Fr beta;
     Fr gamma;
     Fr[NUMBER_OF_ALPHAS] alphas;
-    Fr[LOG_N] gateChallenges;
+    Fr[CONST_PROOF_SIZE_LOG_N] gateChallenges;
     Fr[CONST_PROOF_SIZE_LOG_N] sumCheckUChallenges;
     Fr rho;
     // Zero morph
@@ -297,7 +297,7 @@ library TranscriptLib
 
         t.gateChallenges = generateGateChallenges(t.alphas[NUMBER_OF_ALPHAS - 1]);
 
-        t.sumCheckUChallenges = generateSumcheckChallenges(proof, t.gateChallenges[LOG_N - 1]);
+        t.sumCheckUChallenges = generateSumcheckChallenges(proof, t.gateChallenges[CONST_PROOF_SIZE_LOG_N - 1]);
         t.rho = generateRhoChallenge(proof, t.sumCheckUChallenges[CONST_PROOF_SIZE_LOG_N - 1]);
 
         t.zmY = generateZMYChallenge(t.rho, proof);
@@ -385,9 +385,9 @@ library TranscriptLib
         }
     }
 
-    function generateGateChallenges(Fr previousChallenge) internal view returns(Fr[LOG_N] memory gateChallenges)
+    function generateGateChallenges(Fr previousChallenge) internal view returns(Fr[CONST_PROOF_SIZE_LOG_N] memory gateChallenges)
     {
-        for (uint256 i = 0; i < LOG_N; i++) {
+        for (uint256 i = 0; i < CONST_PROOF_SIZE_LOG_N; i++) {
             previousChallenge = FrLib.fromBytes32(keccak256(abi.encodePacked(Fr.unwrap(previousChallenge))));
             gateChallenges[i] = previousChallenge;
         }

barretenberg/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp

@@ -109,7 +109,6 @@ template <class Fq, class Fr, class T> constexpr void affine_element<Fq, Fr, T>:
         x.data[1] = Fq::modulus.data[1];
         x.data[2] = Fq::modulus.data[2];
         x.data[3] = Fq::modulus.data[3];
-
     } else {
         (*this).x = Fq::zero();
         (*this).y = Fq::zero();

barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp

@@ -701,7 +701,7 @@ TEST_F(join_split_tests, test_0_input_notes_and_detect_circuit_change)
     // The below part detects any changes in the join-split circuit
     constexpr size_t DYADIC_CIRCUIT_SIZE = 1 << 16;
 
-    constexpr uint256_t CIRCUIT_HASH("0x470358e4d91c4c5296ef788b1165b2c439cd498f49c3f99386b002753ca3d0ee");
+    constexpr uint256_t CIRCUIT_HASH("0x9170317e02f4131b84f6b4efdd3ac23e5f392d815df37750c8f05a94c64797b2");
 
     const uint256_t circuit_hash = circuit.hash_circuit();
     // circuit is finalized now

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp

@@ -57,7 +57,18 @@ typename ExecutionTrace_<Flavor>::TraceData ExecutionTrace_<Flavor>::construct_t
 
     uint32_t offset = Flavor::has_zero_row ? 1 : 0; // Offset at which to place each block in the trace polynomials
     // For each block in the trace, populate wire polys, copy cycles and selector polys
-    for (auto& block : builder.blocks.get()) {
+
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1078): remove when Keccak flavor works with Poseidon
+    // gate
+    auto get_blocks = [&]() {
+        if constexpr (!HasKeccak<Flavor>) {
+            return builder.blocks.get();
+        } else {
+            return builder.blocks.get_for_ultra_keccak();
+        }
+    };
+
+    for (auto& block : get_blocks()) {
         auto block_size = static_cast<uint32_t>(block.size());
 
         // Update wire polynomials and copy cycles
@@ -76,10 +87,12 @@ typename ExecutionTrace_<Flavor>::TraceData ExecutionTrace_<Flavor>::construct_t
 
         // Insert the selector values for this block into the selector polynomials at the correct offset
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/398): implicit arithmetization/flavor consistency
-        for (auto [selector_poly, selector] : zip_view(trace_data.selectors, block.selectors)) {
+        for (size_t selector_idx = 0; selector_idx < NUM_USED_SELECTORS; selector_idx++) {
+            auto selector_poly = trace_data.selectors[selector_idx];
+            auto selector = block.selectors[selector_idx];
             for (size_t row_idx = 0; row_idx < block_size; ++row_idx) {
                 size_t trace_row_idx = row_idx + offset;
-                selector_poly[trace_row_idx] = selector[row_idx];
+                trace_data.selectors[selector_idx][trace_row_idx] = selector[row_idx];
             }
         }
 

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp

@@ -9,16 +9,21 @@ template <class Flavor> class ExecutionTrace_ {
     using Builder = typename Flavor::CircuitBuilder;
     using Polynomial = typename Flavor::Polynomial;
     using FF = typename Flavor::FF;
-    using TrackBlocks = typename Builder::Arithmetization::TraceBlocks;
+    using TraceBlocks = typename Builder::Arithmetization::TraceBlocks;
     using Wires = std::array<SlabVector<uint32_t>, Builder::NUM_WIRES>;
     using ProvingKey = typename Flavor::ProvingKey;
 
   public:
     static constexpr size_t NUM_WIRES = Builder::NUM_WIRES;
 
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1078): Since Keccak doesn't have knowledge of Poseidon2
+    // gate yet, we ignore the two related selectors
+    static constexpr size_t NUM_USED_SELECTORS =
+        !HasKeccak<Flavor> ? Builder::Arithmetization::NUM_SELECTORS : Builder::Arithmetization::NUM_SELECTORS - 2;
+
     struct TraceData {
         std::array<Polynomial, NUM_WIRES> wires;
-        std::array<Polynomial, Builder::Arithmetization::NUM_SELECTORS> selectors;
+        std::array<Polynomial, NUM_USED_SELECTORS> selectors;
         // A vector of sets (vectors) of addresses into the wire polynomials whose values are copy constrained
         std::vector<CyclicPermutation> copy_cycles;
         uint32_t ram_rom_offset = 0;    // offset of the RAM/ROM block in the execution trace
@@ -55,7 +60,7 @@ template <class Flavor> class ExecutionTrace_ {
                 }
                 {
                     ZoneScopedN("selector initialization");
-                    for (size_t idx = 0; idx < Builder::Arithmetization::NUM_SELECTORS; ++idx) {
+                    for (size_t idx = 0; idx < NUM_USED_SELECTORS; ++idx) {
                         selectors[idx] = Polynomial(proving_key.circuit_size);
                         std::string selector_tag = builder.selector_names[idx] + "_lagrange";
                         proving_key.polynomial_store.put(selector_tag, selectors[idx].share());

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -411,6 +411,9 @@ concept IsHonkFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWit
 template <typename T> 
 concept IsUltraFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor>;
 
+template <typename T>
+concept HasKeccak = IsAnyOf<T, UltraKeccakFlavor>;
+
 template <typename T>
 concept IsGoblinFlavor = IsAnyOf<T, MegaFlavor,
                                     MegaRecursiveFlavor_<UltraCircuitBuilder>,

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp

@@ -78,10 +78,4 @@ template <typename FF, size_t NUM_WIRES, size_t NUM_SELECTORS> class ExecutionTr
     void set_fixed_size(uint32_t size_in) { fixed_size = size_in; }
 };
 
-class TranslatorArith {
-  public:
-    static constexpr size_t NUM_WIRES = 81;
-    static constexpr size_t NUM_SELECTORS = 0;
-};
-
 } // namespace bb

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp

@@ -156,25 +156,15 @@ template <typename FF_> class MegaArith {
          * conventional Ultra arithmetization
          *
          */
-        void pad_additional()
-        {
-            q_busread().emplace_back(0);
-            q_poseidon2_external().emplace_back(0);
-            q_poseidon2_internal().emplace_back(0);
-        };
+        void pad_additional() { q_busread().emplace_back(0); };
 
         /**
          * @brief Resizes all selectors which are not part of the conventional Ultra arithmetization
          * @details Facilitates reuse of Ultra gate construction functions in arithmetizations which extend the
          * conventional Ultra arithmetization
          * @param new_size
          */
-        void resize_additional(size_t new_size)
-        {
-            q_busread().resize(new_size);
-            q_poseidon2_external().resize(new_size);
-            q_poseidon2_internal().resize(new_size);
-        };
+        void resize_additional(size_t new_size) { q_busread().resize(new_size); };
     };
 
     struct TraceBlocks : public MegaTraceBlocks<MegaTraceBlock> {

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp

@@ -21,8 +21,16 @@ template <typename FF_> class UltraArith {
         T elliptic;
         T aux;
         T lookup;
+        T poseidon_external;
+        T poseidon_internal;
 
-        auto get() { return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, aux, lookup }; }
+        auto get()
+        {
+            return RefArray{ pub_inputs, arithmetic, delta_range,       elliptic,
+                             aux,        lookup,     poseidon_external, poseidon_internal };
+        }
+
+        auto get_for_ultra_keccak() { return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, aux, lookup }; }
 
         bool operator==(const UltraTraceBlocks& other) const = default;
     };
@@ -38,12 +46,14 @@ template <typename FF_> class UltraArith {
             this->elliptic = FIXED_SIZE;
             this->aux = FIXED_SIZE;
             this->lookup = FIXED_SIZE;
+            this->poseidon_external = FIXED_SIZE;
+            this->poseidon_internal = FIXED_SIZE;
         }
     };
 
   public:
     static constexpr size_t NUM_WIRES = 4;
-    static constexpr size_t NUM_SELECTORS = 11;
+    static constexpr size_t NUM_SELECTORS = 13;
     using FF = FF_;
 
     class UltraTraceBlock : public ExecutionTraceBlock<FF, NUM_WIRES, NUM_SELECTORS> {
@@ -75,6 +85,8 @@ template <typename FF_> class UltraArith {
         auto& q_elliptic() { return this->selectors[8]; };
         auto& q_aux() { return this->selectors[9]; };
         auto& q_lookup_type() { return this->selectors[10]; };
+        auto& q_poseidon2_external() { return this->selectors[11]; };
+        auto& q_poseidon2_internal() { return this->selectors[12]; };
     };
 
     struct TraceBlocks : public UltraTraceBlocks<UltraTraceBlock> {
@@ -107,6 +119,12 @@ template <typename FF_> class UltraArith {
         }
 
         auto get()
+        {
+            return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range,       this->elliptic,
+                             this->aux,        this->lookup,     this->poseidon_external, this->poseidon_internal };
+        }
+
+        auto get_for_ultra_keccak()
         {
             return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range,
                              this->elliptic,   this->aux,        this->lookup };
@@ -121,6 +139,8 @@ template <typename FF_> class UltraArith {
             info("elliptic   :\t", this->elliptic.size());
             info("auxiliary  :\t", this->aux.size());
             info("lookups    :\t", this->lookup.size());
+            info("poseidon ext  :\t", this->poseidon_external.size());
+            info("poseidon int  :\t", this->poseidon_internal.size());
         }
 
         size_t get_total_structured_size()
@@ -152,9 +172,19 @@ template <typename FF_> class UltraArith {
     };
 
     // Note: These are needed for Plonk only (for poly storage in a std::map). Must be in same order as above struct.
-    inline static const std::vector<std::string> selector_names = { "q_m",        "q_c",   "q_1",       "q_2",
-                                                                    "q_3",        "q_4",   "q_arith",   "q_sort",
-                                                                    "q_elliptic", "q_aux", "table_type" };
+    inline static const std::vector<std::string> selector_names = { "q_m",
+                                                                    "q_c",
+                                                                    "q_1",
+                                                                    "q_2",
+                                                                    "q_3",
+                                                                    "q_4",
+                                                                    "q_arith",
+                                                                    "q_sort",
+                                                                    "q_elliptic",
+                                                                    "q_aux",
+                                                                    "table_type",
+                                                                    "q_poseidon2_external",
+                                                                    "q_poseidon2_internal" };
 };
 
 } // namespace bb

barretenberg/cpp/src/barretenberg/polynomials/pow.bench.cpp

@@ -17,7 +17,7 @@ void compute_pow_poly(benchmark::State& state)
         int64_t num_betas = state.range(0);
         std::vector<bb::fr> cur_betas(betas.begin(), betas.begin() + num_betas);
         PowPolynomial pow{ cur_betas };
-        pow.compute_values();
+        pow.compute_values(static_cast<size_t>(num_betas));
     }
 }