refactor: remove 'extern template's, expand macros (AztecProtocol#3953)

- "extern template" was
[cargo-culted](https://en.wikipedia.org/wiki/Cargo_cult_programming).
Deleting it all changes no semantics at all. The feature seems at a
surface level to be used like other uses of C++ 'extern' that indicate
to look in a different compilation unit, such as is used by global
variables in a different translation unit. But, C++ actually already
looks in different translation units by default for all the symbols we
applied it to. The only use of extern template is actually if the source
is provided in a header, the exact opposite of how it was used in the
codebase.
- Simplifies usage of instantiation macros that were not doing much

barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

@@ -22,6 +22,7 @@ std::vector<uint8_t> download_bn254_g1_data(size_t num_points)
 std::vector<uint8_t> download_bn254_g2_data()
 {
     std::string url = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat";
+    // IMPORTANT: this currently uses a shell, DO NOT let user-controlled strings here.
     std::string command = "curl -s '" + url + "'";
     return exec_pipe(command);
 }
@@ -69,4 +70,4 @@ barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path&
     auto data = download_bn254_g2_data();
     write_file(g2_path, data);
     return from_buffer<barretenberg::g2::affine_element>(data.data());
-}
+}

barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/pedersen.hpp

@@ -30,6 +30,5 @@ template <typename Curve> class pedersen_commitment_base {
     static AffineElement commit_native(const std::vector<Fq>& inputs, GeneratorContext context = {});
 };
 
-extern template class pedersen_commitment_base<curve::Grumpkin>;
 using pedersen_commitment = pedersen_commitment_base<curve::Grumpkin>;
 } // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/pedersen.hpp

@@ -36,6 +36,5 @@ template <typename Curve> class pedersen_hash_base {
     static std::vector<Fq> convert_buffer(const std::vector<uint8_t>& input);
 };
 
-extern template class pedersen_hash_base<curve::Grumpkin>;
 using pedersen_hash = pedersen_hash_base<curve::Grumpkin>;
 } // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/sha256/sha256.hpp

@@ -15,10 +15,6 @@ hash sha256_block(const std::vector<uint8_t>& input);
 
 template <typename T> hash sha256(const T& input);
 
-extern template hash sha256<std::vector<uint8_t>>(const std::vector<uint8_t>& input);
-extern template hash sha256<std::array<uint8_t, 32>>(const std::array<uint8_t, 32>& input);
-extern template hash sha256<std::string>(const std::string& input);
-
 inline barretenberg::fr sha256_to_field(std::vector<uint8_t> const& input)
 {
     auto result = sha256::sha256(input);

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/acir_composer.cpp

@@ -10,6 +10,7 @@
 #include "barretenberg/plonk/proof_system/verification_key/sol_gen.hpp"
 #include "barretenberg/plonk/proof_system/verification_key/verification_key.hpp"
 #include "barretenberg/srs/factories/crs_factory.hpp"
+#include "barretenberg/stdlib/primitives/circuit_builders/circuit_builders_fwd.hpp"
 #include "contract.hpp"
 
 namespace acir_proofs {
@@ -35,6 +36,9 @@ template <typename Builder> void AcirComposer::create_circuit(acir_format::acir_
     vinfo("gates: ", builder_.get_total_circuit_size());
 }
 
+template void AcirComposer::create_circuit<proof_system::UltraCircuitBuilder>(
+    acir_format::acir_format& constraint_system);
+
 std::shared_ptr<proof_system::plonk::proving_key> AcirComposer::init_proving_key(
     acir_format::acir_format& constraint_system)
 {

barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp

@@ -114,8 +114,4 @@ template <typename Curve> struct pippenger_runtime_state {
     affine_product_runtime_state<Curve> get_affine_product_runtime_state(size_t num_threads, size_t thread_index);
 };
 
-extern template struct affine_product_runtime_state<curve::BN254>;
-extern template struct affine_product_runtime_state<curve::Grumpkin>;
-extern template struct pippenger_runtime_state<curve::BN254>;
-extern template struct pippenger_runtime_state<curve::Grumpkin>;
 } // namespace barretenberg::scalar_multiplication

barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp

@@ -173,108 +173,4 @@ typename Curve::Element pippenger_without_endomorphism_basis_points(typename Cur
 // Explicit instantiation
 // BN254
 
-extern template void generate_pippenger_point_table<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                  curve::BN254::AffineElement* table,
-                                                                  size_t num_points);
-
-extern template uint32_t construct_addition_chains<curve::BN254>(affine_product_runtime_state<curve::BN254>& state,
-                                                                 bool empty_bucket_counts = true);
-
-extern template void add_affine_points<curve::BN254>(curve::BN254::AffineElement* points,
-                                                     const size_t num_points,
-                                                     curve::BN254::BaseField* scratch_space);
-
-extern template void add_affine_points_with_edge_cases<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                     const size_t num_points,
-                                                                     curve::BN254::BaseField* scratch_space);
-
-extern template void evaluate_addition_chains<curve::BN254>(affine_product_runtime_state<curve::BN254>& state,
-                                                            const size_t max_bucket_bits,
-                                                            bool handle_edge_cases);
-extern template curve::BN254::Element pippenger_internal<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                       curve::BN254::ScalarField* scalars,
-                                                                       const size_t num_initial_points,
-                                                                       pippenger_runtime_state<curve::BN254>& state,
-                                                                       bool handle_edge_cases);
-
-extern template curve::BN254::Element evaluate_pippenger_rounds<curve::BN254>(
-    pippenger_runtime_state<curve::BN254>& state,
-    curve::BN254::AffineElement* points,
-    const size_t num_points,
-    bool handle_edge_cases = false);
-
-extern template curve::BN254::AffineElement* reduce_buckets<curve::BN254>(
-    affine_product_runtime_state<curve::BN254>& state, bool first_round = true, bool handle_edge_cases = false);
-
-extern template curve::BN254::Element pippenger<curve::BN254>(curve::BN254::ScalarField* scalars,
-                                                              curve::BN254::AffineElement* points,
-                                                              const size_t num_points,
-                                                              pippenger_runtime_state<curve::BN254>& state,
-                                                              bool handle_edge_cases = true);
-
-extern template curve::BN254::Element pippenger_unsafe<curve::BN254>(curve::BN254::ScalarField* scalars,
-                                                                     curve::BN254::AffineElement* points,
-                                                                     const size_t num_initial_points,
-                                                                     pippenger_runtime_state<curve::BN254>& state);
-
-extern template curve::BN254::Element pippenger_without_endomorphism_basis_points<curve::BN254>(
-    curve::BN254::ScalarField* scalars,
-    curve::BN254::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::BN254>& state);
-
-// Grumpkin
-
-extern template void generate_pippenger_point_table<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                                     curve::Grumpkin::AffineElement* table,
-                                                                     size_t num_points);
-
-extern template uint32_t construct_addition_chains<curve::Grumpkin>(
-    affine_product_runtime_state<curve::Grumpkin>& state, bool empty_bucket_counts = true);
-
-extern template void add_affine_points<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                        const size_t num_points,
-                                                        curve::Grumpkin::BaseField* scratch_space);
-
-extern template void add_affine_points_with_edge_cases<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                                        const size_t num_points,
-                                                                        curve::Grumpkin::BaseField* scratch_space);
-
-extern template void evaluate_addition_chains<curve::Grumpkin>(affine_product_runtime_state<curve::Grumpkin>& state,
-                                                               const size_t max_bucket_bits,
-                                                               bool handle_edge_cases);
-extern template curve::Grumpkin::Element pippenger_internal<curve::Grumpkin>(
-    curve::Grumpkin::AffineElement* points,
-    curve::Grumpkin::ScalarField* scalars,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state,
-    bool handle_edge_cases);
-
-extern template curve::Grumpkin::Element evaluate_pippenger_rounds<curve::Grumpkin>(
-    pippenger_runtime_state<curve::Grumpkin>& state,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_points,
-    bool handle_edge_cases = false);
-
-extern template curve::Grumpkin::AffineElement* reduce_buckets<curve::Grumpkin>(
-    affine_product_runtime_state<curve::Grumpkin>& state, bool first_round = true, bool handle_edge_cases = false);
-
-extern template curve::Grumpkin::Element pippenger<curve::Grumpkin>(curve::Grumpkin::ScalarField* scalars,
-                                                                    curve::Grumpkin::AffineElement* points,
-                                                                    const size_t num_points,
-                                                                    pippenger_runtime_state<curve::Grumpkin>& state,
-                                                                    bool handle_edge_cases = true);
-
-extern template curve::Grumpkin::Element pippenger_unsafe<curve::Grumpkin>(
-    curve::Grumpkin::ScalarField* scalars,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state);
-
-extern template curve::Grumpkin::Element pippenger_without_endomorphism_basis_points<curve::Grumpkin>(
-    curve::Grumpkin::ScalarField* scalars,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state);
-
 } // namespace barretenberg::scalar_multiplication

barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.hpp

@@ -74,7 +74,6 @@ template <ECCVMFlavor Flavor> class ECCVMComposer_ {
         commitment_key = std::make_shared<CommitmentKey>(circuit_size, crs_factory_);
     };
 };
-extern template class ECCVMComposer_<honk::flavor::ECCVM>;
 
 // TODO(#532): this pattern is weird; is this not instantiating the templates?
 using ECCVMComposer = ECCVMComposer_<honk::flavor::ECCVM>;

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.hpp

@@ -83,6 +83,4 @@ template <ECCVMFlavor Flavor> class ECCVMProver_ {
     plonk::proof proof;
 };
 
-extern template class ECCVMProver_<honk::flavor::ECCVM>;
-
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.hpp

@@ -39,7 +39,6 @@ template <typename Flavor> class ECCVMVerifier_ {
     std::shared_ptr<Transcript> transcript;
 };
 
-extern template class ECCVMVerifier_<honk::flavor::ECCVM>;
 using ECCVMVerifierGrumpkin = ECCVMVerifier_<honk::flavor::ECCVM>;
 
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/flavor/ecc_vm.hpp

@@ -16,7 +16,7 @@
 #include "barretenberg/relations/ecc_vm/ecc_wnaf_relation.hpp"
 #include "barretenberg/relations/relation_parameters.hpp"
 #include "barretenberg/relations/relation_types.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 #include <array>
 #include <concepts>
 #include <span>
@@ -930,22 +930,4 @@ class ECCVM : public ECCVMBase<barretenberg::g1, curve::Grumpkin, pcs::ipa::IPA<
 // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)
 
 } // namespace flavor
-namespace sumcheck {
-
-extern template class ECCVMTranscriptRelationImpl<grumpkin::fr>;
-extern template class ECCVMWnafRelationImpl<grumpkin::fr>;
-extern template class ECCVMPointTableRelationImpl<grumpkin::fr>;
-extern template class ECCVMMSMRelationImpl<grumpkin::fr>;
-extern template class ECCVMSetRelationImpl<grumpkin::fr>;
-extern template class ECCVMLookupRelationImpl<grumpkin::fr>;
-
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMTranscriptRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMWnafRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMPointTableRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMMSMRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMSetRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMLookupRelationImpl, flavor::ECCVM);
-
-DECLARE_SUMCHECK_PERMUTATION_CLASS(ECCVMSetRelationImpl, flavor::ECCVM);
-} // namespace sumcheck
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/flavor/generated/AvmMini_flavor.hpp

@@ -1,7 +1,7 @@
 
 
 #pragma once
-#include "../relation_definitions_fwd.hpp"
+#include "../relation_definitions.hpp"
 #include "barretenberg/commitment_schemes/kzg/kzg.hpp"
 #include "barretenberg/ecc/curves/bn254/g1.hpp"
 #include "barretenberg/polynomials/barycentric.hpp"

barretenberg/cpp/src/barretenberg/flavor/generated/Toy_flavor.hpp

@@ -1,7 +1,7 @@
 
 
 #pragma once
-#include "../relation_definitions_fwd.hpp"
+#include "../relation_definitions.hpp"
 #include "barretenberg/commitment_schemes/kzg/kzg.hpp"
 #include "barretenberg/ecc/curves/bn254/g1.hpp"
 #include "barretenberg/polynomials/barycentric.hpp"

barretenberg/cpp/src/barretenberg/flavor/goblin_translator.hpp

@@ -14,7 +14,7 @@
 #include "barretenberg/relations/translator_vm/translator_gen_perm_sort_relation.hpp"
 #include "barretenberg/relations/translator_vm/translator_non_native_field_relation.hpp"
 #include "barretenberg/relations/translator_vm/translator_permutation_relation.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 
 namespace proof_system::honk::flavor {
 
@@ -1138,22 +1138,4 @@ class GoblinTranslator {
 
     using Transcript = BaseTranscript;
 };
-} // namespace proof_system::honk::flavor
-
-namespace proof_system {
-
-extern template class GoblinTranslatorPermutationRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorGenPermSortRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorOpcodeConstraintRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorAccumulatorTransferRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorDecompositionRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorNonNativeFieldRelationImpl<barretenberg::fr>;
-
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorPermutationRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorGenPermSortRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorOpcodeConstraintRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorAccumulatorTransferRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorDecompositionRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorNonNativeFieldRelationImpl, honk::flavor::GoblinTranslator);
-
-} // namespace proof_system
+} // namespace proof_system::honk::flavor

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra.hpp

@@ -17,7 +17,7 @@
 #include "barretenberg/relations/relation_parameters.hpp"
 #include "barretenberg/relations/ultra_arithmetic_relation.hpp"
 #include "barretenberg/transcript/transcript.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 
 namespace proof_system::honk::flavor {
 

barretenberg/cpp/src/barretenberg/flavor/relation_definitions.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "barretenberg/relations/relation_types.hpp"
+
+// Utilities for explicit instantiation of relation and permutation classes.
+
+#define ExtendedEdge(Flavor) Flavor::ExtendedEdges
+#define EvaluationEdge(Flavor) Flavor::AllValues
+#define EntityEdge(Flavor) Flavor::AllEntities<Flavor::FF>
+
+#define ACCUMULATE(...) _ACCUMULATE(__VA_ARGS__)
+#define _ACCUMULATE(RelationImpl, Flavor, AccumulatorType, EdgeType)                                                   \
+    template void                                                                                                      \
+    RelationImpl<Flavor::FF>::accumulate<proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType,            \
+                                         EdgeType(Flavor)>(                                                            \
+        proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType&,                                            \
+        EdgeType(Flavor) const&,                                                                                       \
+        RelationParameters<Flavor::FF> const&,                                                                         \
+        Flavor::FF const&);
+
+#define PERMUTATION_METHOD(...) _PERMUTATION_METHOD(__VA_ARGS__)
+#define _PERMUTATION_METHOD(MethodName, RelationImpl, Flavor, AccumulatorType, EdgeType)                               \
+    template typename proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType                                \
+    RelationImpl<Flavor::FF>::MethodName<proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType,            \
+                                         EdgeType(Flavor)>(EdgeType(Flavor) const&,                                    \
+                                                           RelationParameters<Flavor::FF> const&);
+
+#define SUMCHECK_RELATION_CLASS(...) _SUMCHECK_RELATION_CLASS(__VA_ARGS__)
+#define DEFINE_SUMCHECK_RELATION_CLASS(RelationImpl, Flavor)                                                           \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckTupleOfUnivariatesOverSubrelations, ExtendedEdge)                         \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckArrayOfValuesOverSubrelations, EvaluationEdge)                            \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckArrayOfValuesOverSubrelations, EntityEdge)
+
+#define SUMCHECK_PERMUTATION_CLASS(...) _SUMCHECK_PERMUTATION_CLASS(__VA_ARGS__)
+#define DEFINE_SUMCHECK_PERMUTATION_CLASS(RelationImpl, Flavor)                                                        \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, UnivariateAccumulator0, ExtendedEdge)      \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, ValueAccumulator0, EvaluationEdge)         \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, ValueAccumulator0, EntityEdge)             \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, UnivariateAccumulator0, ExtendedEdge)    \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, ValueAccumulator0, EvaluationEdge)       \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, ValueAccumulator0, EntityEdge)