feat: prove then verify flow for honk (#5957)

Closes AztecProtocol/barretenberg#932.

Adds the prove then verify flow for Ultra and GoblinUltra Honk.

barretenberg/acir_tests/Dockerfile.bb

@@ -10,8 +10,12 @@ COPY . .
 # Run every acir test through native bb build prove_then_verify flow for UltraPlonk.
 # This ensures we test independent pk construction through real/garbage witness data paths.
 RUN FLOW=prove_then_verify ./run_acir_tests.sh
-# Construct and verify a UltraHonk proof for all acir programs
-RUN FLOW=prove_and_verify_ultra_honk ./run_acir_tests.sh
+# Construct and separately verify a UltraHonk proof for a single program
+RUN FLOW=prove_then_verify_ultra_honk ./run_acir_tests.sh double_verify_nested_proof
+# Construct and separately verify a GoblinUltraHonk proof for all acir programs
+RUN FLOW=prove_then_verify_goblin_ultra_honk ./run_acir_tests.sh
+# Construct and verify a UltraHonk proof for a single program
+RUN FLOW=prove_and_verify_ultra_honk ./run_acir_tests.sh double_verify_nested_proof
 # Construct and verify a Goblin UltraHonk (GUH) proof for a single arbitrary program
 RUN FLOW=prove_and_verify_goblin_ultra_honk ./run_acir_tests.sh 6_array
 # Construct and verify a UltraHonk proof for all ACIR programs using the new witness stack workflow

barretenberg/acir_tests/flows/prove_then_verify_goblin_ultra_honk.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/acir.gz"
+FLAGS="-c $CRS_PATH $VFLAG"
+
+# Test we can perform the proof/verify flow.
+# This ensures we test independent pk construction through real/garbage witness data paths.
+$BIN prove_goblin_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_goblin_ultra_honk -o vk $FLAGS $BFLAG
+$BIN verify_goblin_ultra_honk -k vk -p proof $FLAGS

barretenberg/acir_tests/flows/prove_then_verify_ultra_honk.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/acir.gz"
+FLAGS="-c $CRS_PATH $VFLAG"
+
+# Test we can perform the proof/verify flow.
+# This ensures we test independent pk construction through real/garbage witness data paths.
+$BIN prove_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_ultra_honk -o vk $FLAGS $BFLAG
+$BIN verify_ultra_honk -k vk -p proof $FLAGS

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -579,6 +579,116 @@ bool avm_verify(const std::filesystem::path& proof_path)
     return true;
 }
 
+/**
+ * @brief Creates a proof for an ACIR circuit
+ *
+ * Communication:
+ * - stdout: The proof is written to stdout as a byte array
+ * - Filesystem: The proof is written to the path specified by outputPath
+ *
+ * @param bytecodePath Path to the file containing the serialized circuit
+ * @param witnessPath Path to the file containing the serialized witness
+ * @param outputPath Path to write the proof to
+ */
+template <IsUltraFlavor Flavor>
+void prove_honk(const std::string& bytecodePath, const std::string& witnessPath, const std::string& outputPath)
+{
+    using Builder = Flavor::CircuitBuilder;
+    using Prover = UltraProver_<Flavor>;
+
+    auto constraint_system = get_constraint_system(bytecodePath);
+    auto witness = get_witness(witnessPath);
+
+    auto builder = acir_format::create_circuit<Builder>(constraint_system, 0, witness);
+
+    const size_t additional_gates_buffer = 15; // conservatively large to be safe
+    size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + additional_gates_buffer);
+    init_bn254_crs(srs_size);
+
+    // Construct Honk proof
+    Prover prover{ builder };
+    auto proof = prover.construct_proof();
+
+    if (outputPath == "-") {
+        writeRawBytesToStdout(to_buffer</*include_size=*/true>(proof));
+        vinfo("proof written to stdout");
+    } else {
+        write_file(outputPath, to_buffer</*include_size=*/true>(proof));
+        vinfo("proof written to: ", outputPath);
+    }
+}
+
+/**
+ * @brief Verifies a proof for an ACIR circuit
+ *
+ * Note: The fact that the proof was computed originally by parsing an ACIR circuit is not of importance
+ * because this method uses the verification key to verify the proof.
+ *
+ * Communication:
+ * - proc_exit: A boolean value is returned indicating whether the proof is valid.
+ *   an exit code of 0 will be returned for success and 1 for failure.
+ *
+ * @param proof_path Path to the file containing the serialized proof
+ * @param vk_path Path to the file containing the serialized verification key
+ * @return true If the proof is valid
+ * @return false If the proof is invalid
+ */
+template <IsUltraFlavor Flavor> bool verify_honk(const std::string& proof_path, const std::string& vk_path)
+{
+    using VerificationKey = Flavor::VerificationKey;
+    using Verifier = UltraVerifier_<Flavor>;
+    using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
+
+    auto g2_data = get_bn254_g2_data(CRS_PATH);
+    srs::init_crs_factory({}, g2_data);
+    auto proof = from_buffer<std::vector<bb::fr>>(read_file(proof_path));
+    auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(read_file(vk_path)));
+    verification_key->pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
+
+    Verifier verifier{ verification_key };
+
+    bool verified = verifier.verify_proof(proof);
+
+    vinfo("verified: ", verified);
+    return verified;
+}
+
+/**
+ * @brief Writes a verification key for an ACIR circuit to a file
+ *
+ * Communication:
+ * - stdout: The verification key is written to stdout as a byte array
+ * - Filesystem: The verification key is written to the path specified by outputPath
+ *
+ * @param bytecodePath Path to the file containing the serialized circuit
+ * @param outputPath Path to write the verification key to
+ */
+template <IsUltraFlavor Flavor> void write_vk_honk(const std::string& bytecodePath, const std::string& outputPath)
+{
+    using Builder = Flavor::CircuitBuilder;
+    using ProverInstance = ProverInstance_<Flavor>;
+    using VerificationKey = Flavor::VerificationKey;
+
+    auto constraint_system = get_constraint_system(bytecodePath);
+    auto builder = acir_format::create_circuit<Builder>(constraint_system, 0, {});
+
+    const size_t additional_gates_buffer = 15; // conservatively large to be safe
+    size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + additional_gates_buffer);
+    init_bn254_crs(srs_size);
+
+    ProverInstance prover_inst(builder);
+    VerificationKey vk(
+        prover_inst.proving_key); // uses a partial form of the proving key which only has precomputed entities
+
+    auto serialized_vk = to_buffer(vk);
+    if (outputPath == "-") {
+        writeRawBytesToStdout(serialized_vk);
+        vinfo("vk written to stdout");
+    } else {
+        write_file(outputPath, serialized_vk);
+        vinfo("vk written to: ", outputPath);
+    }
+}
 /**
  * @brief Creates a proof for an ACIR circuit, outputs the proof and verification key in binary and 'field' format
  *
@@ -721,6 +831,22 @@ int main(int argc, char* argv[])
         } else if (command == "avm_verify") {
             std::filesystem::path proof_path = get_option(args, "-p", "./proofs/avm_proof");
             return avm_verify(proof_path) ? 0 : 1;
+        } else if (command == "prove_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./proofs/proof");
+            prove_honk<UltraFlavor>(bytecode_path, witness_path, output_path);
+        } else if (command == "verify_ultra_honk") {
+            return verify_honk<UltraFlavor>(proof_path, vk_path) ? 0 : 1;
+        } else if (command == "write_vk_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<UltraFlavor>(bytecode_path, output_path);
+        } else if (command == "prove_goblin_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./proofs/proof");
+            prove_honk<GoblinUltraFlavor>(bytecode_path, witness_path, output_path);
+        } else if (command == "verify_goblin_ultra_honk") {
+            return verify_honk<GoblinUltraFlavor>(proof_path, vk_path) ? 0 : 1;
+        } else if (command == "write_vk_goblin_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<GoblinUltraFlavor>(bytecode_path, output_path);
         } else {
             std::cerr << "Unknown command: " << command << "\n";
             return 1;

barretenberg/cpp/src/barretenberg/common/serialize.hpp

@@ -371,7 +371,8 @@ template <typename B, typename T> inline void read(B& it, std::optional<T>& opt_
 }
 
 template <typename T>
-concept HasGetAll = requires(T t) { t.get_all(); };
+concept HasGetAll = requires(T t) { t.get_all(); } && !
+msgpack_concepts::HasMsgPack<T>;
 
 // Write out a struct that defines get_all()
 template <typename B, HasGetAll T> inline void write(B& buf, T const& value)

barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp

@@ -35,12 +35,22 @@
                       _28,                                                                                             \
                       _29,                                                                                             \
                       _30,                                                                                             \
+                      _31,                                                                                             \
+                      _32,                                                                                             \
+                      _33,                                                                                             \
+                      _34,                                                                                             \
+                      _35,                                                                                             \
                       N,                                                                                               \
                       ...)                                                                                             \
     N
-// AD: support for 30 fields!? one may ask. Well, after 20 not being enough...
+// AD: support for 40 fields!? one may ask. Well, after 30 not being enough...
 #define VA_NARGS(...)                                                                                                  \
     VA_NARGS_IMPL(__VA_ARGS__,                                                                                         \
+                  35,                                                                                                  \
+                  34,                                                                                                  \
+                  33,                                                                                                  \
+                  32,                                                                                                  \
+                  31,                                                                                                  \
                   30,                                                                                                  \
                   29,                                                                                                  \
                   28,                                                                                                  \
@@ -106,6 +116,11 @@
 #define _NVP28(x, ...) _NVP1(x), _NVP27(__VA_ARGS__)
 #define _NVP29(x, ...) _NVP1(x), _NVP28(__VA_ARGS__)
 #define _NVP30(x, ...) _NVP1(x), _NVP29(__VA_ARGS__)
+#define _NVP31(x, ...) _NVP1(x), _NVP30(__VA_ARGS__)
+#define _NVP32(x, ...) _NVP1(x), _NVP31(__VA_ARGS__)
+#define _NVP33(x, ...) _NVP1(x), _NVP32(__VA_ARGS__)
+#define _NVP34(x, ...) _NVP1(x), _NVP33(__VA_ARGS__)
+#define _NVP35(x, ...) _NVP1(x), _NVP34(__VA_ARGS__)
 
 #define CONCAT(a, b) a##b
 #define _NVP_N(n) CONCAT(_NVP, n)

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/goblin_ultra_flavor.hpp

@@ -427,6 +427,109 @@ class GoblinUltraFlavor {
                 commitment = proving_key.commitment_key->commit(polynomial);
             }
         }
+        // TODO(https://github.com/AztecProtocol/barretenberg/issues/964): Clean the boilerplate up.
+        VerificationKey(const size_t circuit_size,
+                        const size_t num_public_inputs,
+                        const size_t pub_inputs_offset,
+                        const Commitment& q_m,
+                        const Commitment& q_c,
+                        const Commitment& q_l,
+                        const Commitment& q_r,
+                        const Commitment& q_o,
+                        const Commitment& q_4,
+                        const Commitment& q_arith,
+                        const Commitment& q_delta_range,
+                        const Commitment& q_elliptic,
+                        const Commitment& q_aux,
+                        const Commitment& q_lookup,
+                        const Commitment& q_busread,
+                        const Commitment& q_poseidon2_external,
+                        const Commitment& q_poseidon2_internal,
+                        const Commitment& sigma_1,
+                        const Commitment& sigma_2,
+                        const Commitment& sigma_3,
+                        const Commitment& sigma_4,
+                        const Commitment& id_1,
+                        const Commitment& id_2,
+                        const Commitment& id_3,
+                        const Commitment& id_4,
+                        const Commitment& table_1,
+                        const Commitment& table_2,
+                        const Commitment& table_3,
+                        const Commitment& table_4,
+                        const Commitment& lagrange_first,
+                        const Commitment& lagrange_last,
+                        const Commitment& lagrange_ecc_op,
+                        const Commitment& databus_id)
+        {
+            this->circuit_size = circuit_size;
+            this->log_circuit_size = numeric::get_msb(this->circuit_size);
+            this->num_public_inputs = num_public_inputs;
+            this->pub_inputs_offset = pub_inputs_offset;
+            this->q_m = q_m;
+            this->q_c = q_c;
+            this->q_l = q_l;
+            this->q_r = q_r;
+            this->q_o = q_o;
+            this->q_4 = q_4;
+            this->q_arith = q_arith;
+            this->q_delta_range = q_delta_range;
+            this->q_elliptic = q_elliptic;
+            this->q_aux = q_aux;
+            this->q_lookup = q_lookup;
+            this->q_busread = q_busread;
+            this->q_poseidon2_external = q_poseidon2_external;
+            this->q_poseidon2_internal = q_poseidon2_internal;
+            this->sigma_1 = sigma_1;
+            this->sigma_2 = sigma_2;
+            this->sigma_3 = sigma_3;
+            this->sigma_4 = sigma_4;
+            this->id_1 = id_1;
+            this->id_2 = id_2;
+            this->id_3 = id_3;
+            this->id_4 = id_4;
+            this->table_1 = table_1;
+            this->table_2 = table_2;
+            this->table_3 = table_3;
+            this->table_4 = table_4;
+            this->lagrange_first = lagrange_first;
+            this->lagrange_last = lagrange_last;
+            this->lagrange_ecc_op = lagrange_ecc_op;
+            this->databus_id = databus_id;
+        }
+        MSGPACK_FIELDS(circuit_size,
+                       num_public_inputs,
+                       pub_inputs_offset,
+                       q_m,
+                       q_c,
+                       q_l,
+                       q_r,
+                       q_o,
+                       q_4,
+                       q_arith,
+                       q_delta_range,
+                       q_elliptic,
+                       q_aux,
+                       q_lookup,
+                       q_busread,
+                       q_poseidon2_external,
+                       q_poseidon2_internal,
+                       sigma_1,
+                       sigma_2,
+                       sigma_3,
+                       sigma_4,
+                       id_1,
+                       id_2,
+                       id_3,
+                       id_4,
+                       table_1,
+                       table_2,
+                       table_3,
+                       table_4,
+                       lagrange_first,
+                       lagrange_last,
+                       lagrange_ecc_op,
+                       databus_id);
     };
     /**
      * @brief A container for storing the partially evaluated multivariates produced by sumcheck.