refactor: remove 'extern template's, expand macros

barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

@@ -22,6 +22,7 @@ std::vector<uint8_t> download_bn254_g1_data(size_t num_points)
 std::vector<uint8_t> download_bn254_g2_data()
 {
     std::string url = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat";
+    // IMPORTANT: this currently uses a shell, DO NOT let user-controlled strings here.
     std::string command = "curl -s '" + url + "'";
     return exec_pipe(command);
 }
@@ -69,4 +70,4 @@ barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path&
     auto data = download_bn254_g2_data();
     write_file(g2_path, data);
     return from_buffer<barretenberg::g2::affine_element>(data.data());
-}
+}

barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/pedersen.hpp

@@ -30,6 +30,5 @@ template <typename Curve> class pedersen_commitment_base {
     static AffineElement commit_native(const std::vector<Fq>& inputs, GeneratorContext context = {});
 };
 
-extern template class pedersen_commitment_base<curve::Grumpkin>;
 using pedersen_commitment = pedersen_commitment_base<curve::Grumpkin>;
 } // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/pedersen.hpp

@@ -36,6 +36,5 @@ template <typename Curve> class pedersen_hash_base {
     static std::vector<Fq> convert_buffer(const std::vector<uint8_t>& input);
 };
 
-extern template class pedersen_hash_base<curve::Grumpkin>;
 using pedersen_hash = pedersen_hash_base<curve::Grumpkin>;
 } // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/sha256/sha256.hpp

@@ -15,10 +15,6 @@ hash sha256_block(const std::vector<uint8_t>& input);
 
 template <typename T> hash sha256(const T& input);
 
-extern template hash sha256<std::vector<uint8_t>>(const std::vector<uint8_t>& input);
-extern template hash sha256<std::array<uint8_t, 32>>(const std::array<uint8_t, 32>& input);
-extern template hash sha256<std::string>(const std::string& input);
-
 inline barretenberg::fr sha256_to_field(std::vector<uint8_t> const& input)
 {
     auto result = sha256::sha256(input);

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/acir_composer.cpp

@@ -10,6 +10,7 @@
 #include "barretenberg/plonk/proof_system/verification_key/sol_gen.hpp"
 #include "barretenberg/plonk/proof_system/verification_key/verification_key.hpp"
 #include "barretenberg/srs/factories/crs_factory.hpp"
+#include "barretenberg/stdlib/primitives/circuit_builders/circuit_builders_fwd.hpp"
 #include "contract.hpp"
 
 namespace acir_proofs {
@@ -35,6 +36,9 @@ template <typename Builder> void AcirComposer::create_circuit(acir_format::acir_
     vinfo("gates: ", builder_.get_total_circuit_size());
 }
 
+template void AcirComposer::create_circuit<proof_system::UltraCircuitBuilder>(
+    acir_format::acir_format& constraint_system);
+
 std::shared_ptr<proof_system::plonk::proving_key> AcirComposer::init_proving_key(
     acir_format::acir_format& constraint_system)
 {

barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp

@@ -114,8 +114,4 @@ template <typename Curve> struct pippenger_runtime_state {
     affine_product_runtime_state<Curve> get_affine_product_runtime_state(size_t num_threads, size_t thread_index);
 };
 
-extern template struct affine_product_runtime_state<curve::BN254>;
-extern template struct affine_product_runtime_state<curve::Grumpkin>;
-extern template struct pippenger_runtime_state<curve::BN254>;
-extern template struct pippenger_runtime_state<curve::Grumpkin>;
 } // namespace barretenberg::scalar_multiplication

barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp

@@ -173,108 +173,4 @@ typename Curve::Element pippenger_without_endomorphism_basis_points(typename Cur
 // Explicit instantiation
 // BN254
 
-extern template void generate_pippenger_point_table<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                  curve::BN254::AffineElement* table,
-                                                                  size_t num_points);
-
-extern template uint32_t construct_addition_chains<curve::BN254>(affine_product_runtime_state<curve::BN254>& state,
-                                                                 bool empty_bucket_counts = true);
-
-extern template void add_affine_points<curve::BN254>(curve::BN254::AffineElement* points,
-                                                     const size_t num_points,
-                                                     curve::BN254::BaseField* scratch_space);
-
-extern template void add_affine_points_with_edge_cases<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                     const size_t num_points,
-                                                                     curve::BN254::BaseField* scratch_space);
-
-extern template void evaluate_addition_chains<curve::BN254>(affine_product_runtime_state<curve::BN254>& state,
-                                                            const size_t max_bucket_bits,
-                                                            bool handle_edge_cases);
-extern template curve::BN254::Element pippenger_internal<curve::BN254>(curve::BN254::AffineElement* points,
-                                                                       curve::BN254::ScalarField* scalars,
-                                                                       const size_t num_initial_points,
-                                                                       pippenger_runtime_state<curve::BN254>& state,
-                                                                       bool handle_edge_cases);
-
-extern template curve::BN254::Element evaluate_pippenger_rounds<curve::BN254>(
-    pippenger_runtime_state<curve::BN254>& state,
-    curve::BN254::AffineElement* points,
-    const size_t num_points,
-    bool handle_edge_cases = false);
-
-extern template curve::BN254::AffineElement* reduce_buckets<curve::BN254>(
-    affine_product_runtime_state<curve::BN254>& state, bool first_round = true, bool handle_edge_cases = false);
-
-extern template curve::BN254::Element pippenger<curve::BN254>(curve::BN254::ScalarField* scalars,
-                                                              curve::BN254::AffineElement* points,
-                                                              const size_t num_points,
-                                                              pippenger_runtime_state<curve::BN254>& state,
-                                                              bool handle_edge_cases = true);
-
-extern template curve::BN254::Element pippenger_unsafe<curve::BN254>(curve::BN254::ScalarField* scalars,
-                                                                     curve::BN254::AffineElement* points,
-                                                                     const size_t num_initial_points,
-                                                                     pippenger_runtime_state<curve::BN254>& state);
-
-extern template curve::BN254::Element pippenger_without_endomorphism_basis_points<curve::BN254>(
-    curve::BN254::ScalarField* scalars,
-    curve::BN254::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::BN254>& state);
-
-// Grumpkin
-
-extern template void generate_pippenger_point_table<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                                     curve::Grumpkin::AffineElement* table,
-                                                                     size_t num_points);
-
-extern template uint32_t construct_addition_chains<curve::Grumpkin>(
-    affine_product_runtime_state<curve::Grumpkin>& state, bool empty_bucket_counts = true);
-
-extern template void add_affine_points<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                        const size_t num_points,
-                                                        curve::Grumpkin::BaseField* scratch_space);
-
-extern template void add_affine_points_with_edge_cases<curve::Grumpkin>(curve::Grumpkin::AffineElement* points,
-                                                                        const size_t num_points,
-                                                                        curve::Grumpkin::BaseField* scratch_space);
-
-extern template void evaluate_addition_chains<curve::Grumpkin>(affine_product_runtime_state<curve::Grumpkin>& state,
-                                                               const size_t max_bucket_bits,
-                                                               bool handle_edge_cases);
-extern template curve::Grumpkin::Element pippenger_internal<curve::Grumpkin>(
-    curve::Grumpkin::AffineElement* points,
-    curve::Grumpkin::ScalarField* scalars,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state,
-    bool handle_edge_cases);
-
-extern template curve::Grumpkin::Element evaluate_pippenger_rounds<curve::Grumpkin>(
-    pippenger_runtime_state<curve::Grumpkin>& state,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_points,
-    bool handle_edge_cases = false);
-
-extern template curve::Grumpkin::AffineElement* reduce_buckets<curve::Grumpkin>(
-    affine_product_runtime_state<curve::Grumpkin>& state, bool first_round = true, bool handle_edge_cases = false);
-
-extern template curve::Grumpkin::Element pippenger<curve::Grumpkin>(curve::Grumpkin::ScalarField* scalars,
-                                                                    curve::Grumpkin::AffineElement* points,
-                                                                    const size_t num_points,
-                                                                    pippenger_runtime_state<curve::Grumpkin>& state,
-                                                                    bool handle_edge_cases = true);
-
-extern template curve::Grumpkin::Element pippenger_unsafe<curve::Grumpkin>(
-    curve::Grumpkin::ScalarField* scalars,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state);
-
-extern template curve::Grumpkin::Element pippenger_without_endomorphism_basis_points<curve::Grumpkin>(
-    curve::Grumpkin::ScalarField* scalars,
-    curve::Grumpkin::AffineElement* points,
-    const size_t num_initial_points,
-    pippenger_runtime_state<curve::Grumpkin>& state);
-
 } // namespace barretenberg::scalar_multiplication

barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.hpp

@@ -74,7 +74,6 @@ template <ECCVMFlavor Flavor> class ECCVMComposer_ {
         commitment_key = std::make_shared<CommitmentKey>(circuit_size, crs_factory_);
     };
 };
-extern template class ECCVMComposer_<honk::flavor::ECCVM>;
 
 // TODO(#532): this pattern is weird; is this not instantiating the templates?
 using ECCVMComposer = ECCVMComposer_<honk::flavor::ECCVM>;

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.hpp

@@ -83,6 +83,4 @@ template <ECCVMFlavor Flavor> class ECCVMProver_ {
     plonk::proof proof;
 };
 
-extern template class ECCVMProver_<honk::flavor::ECCVM>;
-
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.hpp

@@ -39,7 +39,6 @@ template <typename Flavor> class ECCVMVerifier_ {
     std::shared_ptr<Transcript> transcript;
 };
 
-extern template class ECCVMVerifier_<honk::flavor::ECCVM>;
 using ECCVMVerifierGrumpkin = ECCVMVerifier_<honk::flavor::ECCVM>;
 
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/flavor/ecc_vm.hpp

@@ -16,7 +16,7 @@
 #include "barretenberg/relations/ecc_vm/ecc_wnaf_relation.hpp"
 #include "barretenberg/relations/relation_parameters.hpp"
 #include "barretenberg/relations/relation_types.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 #include <array>
 #include <concepts>
 #include <span>
@@ -930,22 +930,4 @@ class ECCVM : public ECCVMBase<barretenberg::g1, curve::Grumpkin, pcs::ipa::IPA<
 // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)
 
 } // namespace flavor
-namespace sumcheck {
-
-extern template class ECCVMTranscriptRelationImpl<grumpkin::fr>;
-extern template class ECCVMWnafRelationImpl<grumpkin::fr>;
-extern template class ECCVMPointTableRelationImpl<grumpkin::fr>;
-extern template class ECCVMMSMRelationImpl<grumpkin::fr>;
-extern template class ECCVMSetRelationImpl<grumpkin::fr>;
-extern template class ECCVMLookupRelationImpl<grumpkin::fr>;
-
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMTranscriptRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMWnafRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMPointTableRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMMSMRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMSetRelationImpl, flavor::ECCVM);
-DECLARE_SUMCHECK_RELATION_CLASS(ECCVMLookupRelationImpl, flavor::ECCVM);
-
-DECLARE_SUMCHECK_PERMUTATION_CLASS(ECCVMSetRelationImpl, flavor::ECCVM);
-} // namespace sumcheck
 } // namespace proof_system::honk

barretenberg/cpp/src/barretenberg/flavor/generated/AvmMini_flavor.hpp

@@ -1,7 +1,7 @@
 
 
 #pragma once
-#include "../relation_definitions_fwd.hpp"
+#include "../relation_definitions.hpp"
 #include "barretenberg/commitment_schemes/kzg/kzg.hpp"
 #include "barretenberg/ecc/curves/bn254/g1.hpp"
 #include "barretenberg/polynomials/barycentric.hpp"

barretenberg/cpp/src/barretenberg/flavor/generated/Toy_flavor.hpp

@@ -1,7 +1,7 @@
 
 
 #pragma once
-#include "../relation_definitions_fwd.hpp"
+#include "../relation_definitions.hpp"
 #include "barretenberg/commitment_schemes/kzg/kzg.hpp"
 #include "barretenberg/ecc/curves/bn254/g1.hpp"
 #include "barretenberg/polynomials/barycentric.hpp"

barretenberg/cpp/src/barretenberg/flavor/goblin_translator.hpp

@@ -14,7 +14,7 @@
 #include "barretenberg/relations/translator_vm/translator_gen_perm_sort_relation.hpp"
 #include "barretenberg/relations/translator_vm/translator_non_native_field_relation.hpp"
 #include "barretenberg/relations/translator_vm/translator_permutation_relation.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 
 namespace proof_system::honk::flavor {
 
@@ -1138,22 +1138,4 @@ class GoblinTranslator {
 
     using Transcript = BaseTranscript;
 };
-} // namespace proof_system::honk::flavor
-
-namespace proof_system {
-
-extern template class GoblinTranslatorPermutationRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorGenPermSortRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorOpcodeConstraintRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorAccumulatorTransferRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorDecompositionRelationImpl<barretenberg::fr>;
-extern template class GoblinTranslatorNonNativeFieldRelationImpl<barretenberg::fr>;
-
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorPermutationRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorGenPermSortRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorOpcodeConstraintRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorAccumulatorTransferRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorDecompositionRelationImpl, honk::flavor::GoblinTranslator);
-DECLARE_SUMCHECK_RELATION_CLASS(GoblinTranslatorNonNativeFieldRelationImpl, honk::flavor::GoblinTranslator);
-
-} // namespace proof_system
+} // namespace proof_system::honk::flavor

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra.hpp

@@ -17,7 +17,7 @@
 #include "barretenberg/relations/relation_parameters.hpp"
 #include "barretenberg/relations/ultra_arithmetic_relation.hpp"
 #include "barretenberg/transcript/transcript.hpp"
-#include "relation_definitions_fwd.hpp"
+#include "relation_definitions.hpp"
 
 namespace proof_system::honk::flavor {
 

barretenberg/cpp/src/barretenberg/flavor/relation_definitions.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "barretenberg/relations/relation_types.hpp"
+
+// Utilities for explicit instantiation of relation and permutation classes.
+
+#define ExtendedEdge(Flavor) Flavor::ExtendedEdges
+#define EvaluationEdge(Flavor) Flavor::AllValues
+#define EntityEdge(Flavor) Flavor::AllEntities<Flavor::FF>
+
+#define ACCUMULATE(...) _ACCUMULATE(__VA_ARGS__)
+#define _ACCUMULATE(RelationImpl, Flavor, AccumulatorType, EdgeType)                                                   \
+    template void                                                                                                      \
+    RelationImpl<Flavor::FF>::accumulate<proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType,            \
+                                         EdgeType(Flavor)>(                                                            \
+        proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType&,                                            \
+        EdgeType(Flavor) const&,                                                                                       \
+        RelationParameters<Flavor::FF> const&,                                                                         \
+        Flavor::FF const&);
+
+#define PERMUTATION_METHOD(...) _PERMUTATION_METHOD(__VA_ARGS__)
+#define _PERMUTATION_METHOD(MethodName, RelationImpl, Flavor, AccumulatorType, EdgeType)                               \
+    template typename proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType                                \
+    RelationImpl<Flavor::FF>::MethodName<proof_system::Relation<RelationImpl<Flavor::FF>>::AccumulatorType,            \
+                                         EdgeType(Flavor)>(EdgeType(Flavor) const&,                                    \
+                                                           RelationParameters<Flavor::FF> const&);
+
+#define SUMCHECK_RELATION_CLASS(...) _SUMCHECK_RELATION_CLASS(__VA_ARGS__)
+#define DEFINE_SUMCHECK_RELATION_CLASS(RelationImpl, Flavor)                                                           \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckTupleOfUnivariatesOverSubrelations, ExtendedEdge)                         \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckArrayOfValuesOverSubrelations, EvaluationEdge)                            \
+    ACCUMULATE(RelationImpl, Flavor, SumcheckArrayOfValuesOverSubrelations, EntityEdge)
+
+#define SUMCHECK_PERMUTATION_CLASS(...) _SUMCHECK_PERMUTATION_CLASS(__VA_ARGS__)
+#define DEFINE_SUMCHECK_PERMUTATION_CLASS(RelationImpl, Flavor)                                                        \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, UnivariateAccumulator0, ExtendedEdge)      \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, ValueAccumulator0, EvaluationEdge)         \
+    PERMUTATION_METHOD(compute_permutation_numerator, RelationImpl, Flavor, ValueAccumulator0, EntityEdge)             \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, UnivariateAccumulator0, ExtendedEdge)    \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, ValueAccumulator0, EvaluationEdge)       \
+    PERMUTATION_METHOD(compute_permutation_denominator, RelationImpl, Flavor, ValueAccumulator0, EntityEdge)