From 133cb68dbd6a771bdda622ebf47f0d153eec7252 Mon Sep 17 00:00:00 2001 From: Maddiaa <47148561+Maddiaa0@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:27:06 +0100 Subject: [PATCH] feat: bb logup proving support (#54) * feat: circuit builder does not need to compute inverses, prover does so * fix: integration cleanup * chore: sweep * chore: clippy and fmt --- bberg/src/circuit_builder.rs | 9 ++- bberg/src/composer_builder.rs | 4 +- bberg/src/flavor_builder.rs | 106 ++++++++++++++++++++++++---------- bberg/src/lookup_builder.rs | 2 +- bberg/src/prover_builder.rs | 101 ++++++++++++++++++++++++++------ bberg/src/verifier_builder.rs | 12 +++- bberg/src/vm_builder.rs | 35 +++++++---- 7 files changed, 204 insertions(+), 65 deletions(-) diff --git a/bberg/src/circuit_builder.rs b/bberg/src/circuit_builder.rs index 77e15d2e51..35eec1508b 100644 --- a/bberg/src/circuit_builder.rs +++ b/bberg/src/circuit_builder.rs @@ -10,8 +10,9 @@ pub trait CircuitBuilder { name: &str, relations: &[String], permutations: &[String], - fixed: &[String], - shifted: &[String], + all_cols_without_inverses: &[String], + all_cols: &[String], + to_be_shifted: &[String], all_cols_with_shifts: &[String], ); } @@ -62,6 +63,7 @@ impl CircuitBuilder for BBFiles { name: &str, relations: &[String], permutations: &[String], + all_cols_without_inverses: &[String], all_cols: &[String], to_be_shifted: &[String], all_cols_with_shifts: &[String], @@ -97,7 +99,8 @@ impl CircuitBuilder for BBFiles { }; // Apply transformations - let compute_polys_assignemnt = map_with_newline(all_cols, compute_polys_transformation); + let compute_polys_assignemnt = + map_with_newline(all_cols_without_inverses, compute_polys_transformation); let all_poly_shifts = map_with_newline(to_be_shifted, all_polys_transformation); let check_circuit_for_each_relation = map_with_newline(relations, check_circuit_transformation); diff --git a/bberg/src/composer_builder.rs b/bberg/src/composer_builder.rs index 26e0cdd57e..2cbd8576d1 100644 --- a/bberg/src/composer_builder.rs +++ b/bberg/src/composer_builder.rs @@ -40,7 +40,7 @@ void {name}Composer::compute_witness(CircuitConstructor& circuit) compute_witness(circuit_constructor); compute_commitment_key(circuit_constructor.get_circuit_subgroup_size()); - {name}Prover output_state(proving_key, commitment_key); + {name}Prover output_state(proving_key, proving_key->commitment_key); return output_state; }} @@ -167,7 +167,7 @@ class {name}Composer {{ void compute_commitment_key(size_t circuit_size) {{ - commitment_key = std::make_shared(circuit_size); + proving_key->commitment_key = std::make_shared(circuit_size); }}; }}; diff --git a/bberg/src/flavor_builder.rs b/bberg/src/flavor_builder.rs index c29178f38a..61c80938a4 100644 --- a/bberg/src/flavor_builder.rs +++ b/bberg/src/flavor_builder.rs @@ -1,6 +1,5 @@ use crate::{ file_writer::BBFiles, - permutation_builder::{get_inverses_from_permutations, Permutation}, utils::{get_relations_imports, map_with_newline, snake_case}, }; @@ -10,7 +9,7 @@ pub trait FlavorBuilder { &mut self, name: &str, relation_file_names: &[String], - permutations: &[Permutation], + lookups: &[String], fixed: &[String], witness: &[String], all_cols: &[String], @@ -26,7 +25,7 @@ impl FlavorBuilder for BBFiles { &mut self, name: &str, relation_file_names: &[String], - permutations: &[Permutation], + lookups: &[String], fixed: &[String], witness: &[String], all_cols: &[String], @@ -34,19 +33,15 @@ impl FlavorBuilder for BBFiles { shifted: &[String], all_cols_and_shifts: &[String], ) { - // TODO: move elsewhere and rename - let inverses = get_inverses_from_permutations(permutations); - let first_poly = &witness[0]; - let includes = flavor_includes(&snake_case(name), relation_file_names, &inverses); + let includes = flavor_includes(&snake_case(name), relation_file_names, lookups); let num_precomputed = fixed.len(); let num_witness = witness.len(); let num_all = all_cols_and_shifts.len(); // Top of file boilerplate let class_aliases = create_class_aliases(); - let relation_definitions = - create_relation_definitions(name, relation_file_names, permutations); + let relation_definitions = create_relation_definitions(name, relation_file_names, lookups); let container_size_definitions = container_size_definitions(num_precomputed, num_witness, num_all); @@ -56,7 +51,8 @@ impl FlavorBuilder for BBFiles { let all_entities = create_all_entities(all_cols, to_be_shifted, shifted, all_cols_and_shifts); - let proving_and_verification_key = create_proving_and_verification_key(to_be_shifted); + let proving_and_verification_key = + create_proving_and_verification_key(name, lookups, to_be_shifted); let polynomial_views = create_polynomial_views(first_poly); let commitment_labels_class = create_commitment_labels(all_cols); @@ -116,8 +112,8 @@ class {name}Flavor {{ } /// Imports located at the top of the flavor files -fn flavor_includes(name: &str, relation_file_names: &[String], permutations: &[String]) -> String { - let relation_imports = get_relations_imports(name, relation_file_names, permutations); +fn flavor_includes(name: &str, relation_file_names: &[String], lookups: &[String]) -> String { + let relation_imports = get_relations_imports(name, relation_file_names, lookups); format!( " @@ -150,13 +146,17 @@ fn create_relations_tuple(master_name: &str, relation_file_names: &[String]) -> } /// Creates comma separated relations tuple file -/// TODO(md): maybe need the filename in here too if we scope these -fn create_permutations_tuple(permutations: &[Permutation]) -> String { - permutations - .iter() - .map(|perm| format!("{}_relation", perm.attribute.clone().unwrap())) - .collect::>() - .join(", ") +fn create_lookups_tuple(lookups: &[String]) -> Option { + if lookups.is_empty() { + return None; + } + Some( + lookups + .iter() + .map(|lookup| format!("{}_relation", lookup.clone())) + .collect::>() + .join(", "), + ) } /// Create Class Aliases @@ -190,17 +190,23 @@ fn create_class_aliases() -> &'static str { fn create_relation_definitions( name: &str, relation_file_names: &[String], - permutations: &[Permutation], + lookups: &[String], ) -> String { // Relations tuple = ns::relation_name_0, ns::relation_name_1, ... ns::relation_name_n (comma speratated) let comma_sep_relations = create_relations_tuple(name, relation_file_names); - let comma_sep_perms: String = create_permutations_tuple(permutations); + let comma_sep_lookups: Option = create_lookups_tuple(lookups); + + // We only include the grand product relations if we are given lookups + let mut grand_product_relations = String::new(); let mut all_relations = comma_sep_relations.to_string(); - if !permutations.is_empty() { - all_relations = all_relations + &format!(", {comma_sep_perms}"); + if let Some(lookups) = comma_sep_lookups { + all_relations = all_relations + &format!(", {lookups}"); + grand_product_relations = format!("using GrandProductRelations = std::tuple<{lookups}>;"); } format!(" + {grand_product_relations} + using Relations = std::tuple<{all_relations}>; static constexpr size_t MAX_PARTIAL_RELATION_LENGTH = compute_max_partial_relation_length(); @@ -209,7 +215,7 @@ fn create_relation_definitions( // random polynomial e.g. For \\sum(x) [A(x) * B(x) + C(x)] * PowZeta(X), relation length = 2 and random relation // length = 3 static constexpr size_t BATCHED_RELATION_PARTIAL_LENGTH = MAX_PARTIAL_RELATION_LENGTH + 1; - static constexpr size_t NUM_RELATIONS = std::tuple_size::value; + static constexpr size_t NUM_RELATIONS = std::tuple_size_v; template using ProtogalaxyTupleOfTuplesOfUnivariates = @@ -284,7 +290,6 @@ fn create_witness_entities(witness: &[String]) -> String { let pointer_view = create_flavor_members(witness); let wires = return_ref_vector("get_wires", witness); - let sorted_polys = return_ref_vector("get_sorted_polynomials", &[]); format!( " @@ -295,7 +300,6 @@ fn create_witness_entities(witness: &[String]) -> String { {pointer_view} {wires} - {sorted_polys} }}; " ) @@ -333,8 +337,14 @@ fn create_all_entities( ) } -fn create_proving_and_verification_key(to_be_shifted: &[String]) -> String { +fn create_proving_and_verification_key( + flavor_name: &str, + lookups: &[String], + to_be_shifted: &[String], +) -> String { let get_to_be_shifted = return_ref_vector("get_to_be_shifted", to_be_shifted); + let compute_logderivative_inverses = + create_compute_logderivative_inverses(flavor_name, lookups); format!(" public: @@ -346,8 +356,7 @@ fn create_proving_and_verification_key(to_be_shifted: &[String]) -> String { {get_to_be_shifted} - // The plookup wires that store plookup read data. - std::array get_table_column_wires() {{ return {{}}; }}; + {compute_logderivative_inverses} }}; using VerificationKey = VerificationKey_, VerifierCommitmentKey>; @@ -377,6 +386,19 @@ fn create_polynomial_views(first_poly: &String) -> String { ProverPolynomials(ProverPolynomials&& o) noexcept = default; ProverPolynomials& operator=(ProverPolynomials&& o) noexcept = default; ~ProverPolynomials() = default; + + ProverPolynomials(ProvingKey& proving_key) + {{ + for (auto [prover_poly, key_poly] : zip_view(this->get_unshifted(), proving_key.get_all())) {{ + ASSERT(flavor_get_label(*this, prover_poly) == flavor_get_label(proving_key, key_poly)); + prover_poly = key_poly.share(); + }} + for (auto [prover_poly, key_poly] : zip_view(this->get_shifted(), proving_key.get_to_be_shifted())) {{ + ASSERT(flavor_get_label(*this, prover_poly) == (flavor_get_label(proving_key, key_poly) + \"_shift\")); + prover_poly = key_poly.shifted(); + }} + }} + [[nodiscard]] size_t get_polynomial_size() const {{ return {first_poly}.size(); }} /** * @brief Returns the evaluations of all prover polynomials at one point on the boolean hypercube, which @@ -419,6 +441,12 @@ fn create_polynomial_views(first_poly: &String) -> String { */ using ExtendedEdges = ProverUnivariates; + /** + * @brief A container for the witness commitments. + * + */ + using WitnessCommitments = WitnessEntities; + ") } @@ -452,7 +480,6 @@ fn create_commitment_labels(all_ents: &[String]) -> String { private: using Base = AllEntities; - public: CommitmentLabels() : AllEntities() {{ @@ -463,6 +490,25 @@ fn create_commitment_labels(all_ents: &[String]) -> String { ) } +fn create_compute_logderivative_inverses(flavor_name: &str, lookups: &[String]) -> String { + let compute_inverse_transformation = |lookup_name: &String| { + format!("bb::compute_logderivative_inverse<{flavor_name}Flavor, {lookup_name}_relation>(prover_polynomials, relation_parameters, this->circuit_size);") + }; + + let compute_inverses = map_with_newline(lookups, compute_inverse_transformation); + + format!( + " + void compute_logderivative_inverses(const RelationParameters& relation_parameters) + {{ + ProverPolynomials prover_polynomials = ProverPolynomials(*this); + + {compute_inverses} + }} + " + ) +} + fn create_key_dereference(fixed: &[String]) -> String { let deref_transformation = |name: &String| format!("{name} = verification_key->{name};"); diff --git a/bberg/src/lookup_builder.rs b/bberg/src/lookup_builder.rs index e70b864efd..dc3a23d0b7 100644 --- a/bberg/src/lookup_builder.rs +++ b/bberg/src/lookup_builder.rs @@ -188,7 +188,7 @@ fn create_lookup_settings_file(lookup: &Lookup) -> String { let lookup_tuple_size = columns_per_set; // NOTE: hardcoded until optimizations required - let inverse_degree = 2; + let inverse_degree = 4; let read_term_degree = 0; let write_term_degree = 0; let read_term_types = "{0}"; diff --git a/bberg/src/prover_builder.rs b/bberg/src/prover_builder.rs index 71dbc35b8b..b7a313e2d3 100644 --- a/bberg/src/prover_builder.rs +++ b/bberg/src/prover_builder.rs @@ -1,10 +1,15 @@ use crate::file_writer::BBFiles; -use crate::utils::snake_case; +use crate::utils::{map_with_newline, snake_case}; pub trait ProverBuilder { - fn create_prover_cpp(&mut self, name: &str); - fn create_prover_hpp(&mut self, name: &str); + + fn create_prover_cpp( + &mut self, + name: &str, + commitment_polys: &[String], + lookup_names: &[String], + ); } impl ProverBuilder for BBFiles { @@ -31,6 +36,7 @@ impl ProverBuilder for BBFiles { void execute_preamble_round(); void execute_wire_commitments_round(); + void execute_log_derivative_inverse_round(); void execute_relation_check_rounds(); void execute_zeromorph_rounds(); @@ -49,6 +55,7 @@ impl ProverBuilder for BBFiles { ProverPolynomials prover_polynomials; CommitmentLabels commitment_labels; + typename Flavor::WitnessCommitments witness_commitments; Polynomial quotient_W; @@ -72,9 +79,20 @@ impl ProverBuilder for BBFiles { ); } - fn create_prover_cpp(&mut self, name: &str) { + /// Create the prover cpp file + /// + /// Committed polys are included as we manually unroll all commitments, as we do not commit to everything + fn create_prover_cpp( + &mut self, + name: &str, + commitment_polys: &[String], + lookup_names: &[String], + ) { let include_str = includes_cpp(&snake_case(name)); + let polynomial_commitment_phase = create_commitments_phase(commitment_polys); + let log_derivative_inverse_phase = create_log_derivative_inverse_round(lookup_names); + let prover_cpp = format!(" {include_str} @@ -121,19 +139,21 @@ impl ProverBuilder for BBFiles { }} /** - * @brief Compute commitments to the first three wires + * @brief Compute commitments to all of the witness wires (apart from the logderivative inverse wires) * */ void {name}Prover::execute_wire_commitments_round() {{ - auto wire_polys = key->get_wires(); - auto labels = commitment_labels.get_wires(); - for (size_t idx = 0; idx < wire_polys.size(); ++idx) {{ - transcript->send_to_verifier(labels[idx], commitment_key->commit(wire_polys[idx])); - }} + + {polynomial_commitment_phase} + }} - + void {name}Prover::execute_log_derivative_inverse_round() + {{ + + {log_derivative_inverse_phase} + }} /** * @brief Run Sumcheck resulting in u = (u_1,...,u_d) challenges and all evaluations at u being calculated. @@ -187,13 +207,8 @@ impl ProverBuilder for BBFiles { // Compute wire commitments execute_wire_commitments_round(); - // TODO: not implemented for codegen just yet // Compute sorted list accumulator and commitment - // execute_log_derivative_commitments_round(); - - // Fiat-Shamir: bbeta & gamma - // Compute grand product(s) and commitments. - // execute_grand_product_computation_round(); + execute_log_derivative_inverse_round(); // Fiat-Shamir: alpha // Run sumcheck subprotocol. @@ -252,3 +267,55 @@ fn includes_cpp(name: &str) -> String { " ) } + +/// Commitment Transform +/// +/// Produces code to perform kzg commitment, then stores in the witness_commitments struct +fn commitment_transform(name: &String) -> String { + format!("witness_commitments.{name} = commitment_key->commit(key->{name});") +} + +/// Send to Verifier Transform +/// +/// Sends commitment produces in commitment_transform to the verifier +fn send_to_verifier_transform(name: &String) -> String { + format!("transcript->send_to_verifier(commitment_labels.{name}, witness_commitments.{name});") +} + +fn create_commitments_phase(polys_to_commit_to: &[String]) -> String { + let all_commit_operations = map_with_newline(polys_to_commit_to, commitment_transform); + let send_to_verifier_operations = + map_with_newline(polys_to_commit_to, send_to_verifier_transform); + + format!( + " + // Commit to all polynomials (apart from logderivative inverse polynomials, which are committed to in the later logderivative phase) + {all_commit_operations} + + // Send all commitments to the verifier + {send_to_verifier_operations} + " + ) +} + +fn create_log_derivative_inverse_round(lookup_operations: &[String]) -> String { + let all_commit_operations = map_with_newline(lookup_operations, commitment_transform); + let send_to_verifier_operations = + map_with_newline(lookup_operations, send_to_verifier_transform); + + format!( + " + auto [beta, gamm] = transcript->template get_challenges(\"beta\", \"gamma\"); + relation_parameters.beta = beta; + relation_parameters.gamma = gamm; + + key->compute_logderivative_inverses(relation_parameters); + + // Commit to all logderivative inverse polynomials + {all_commit_operations} + + // Send all commitments to the verifier + {send_to_verifier_operations} + " + ) +} diff --git a/bberg/src/verifier_builder.rs b/bberg/src/verifier_builder.rs index b23cfe6010..db789e18cc 100644 --- a/bberg/src/verifier_builder.rs +++ b/bberg/src/verifier_builder.rs @@ -4,13 +4,13 @@ use crate::{ }; pub trait VerifierBuilder { - fn create_verifier_cpp(&mut self, name: &str, witness: &[String]); + fn create_verifier_cpp(&mut self, name: &str, witness: &[String], inverses: &[String]); fn create_verifier_hpp(&mut self, name: &str); } impl VerifierBuilder for BBFiles { - fn create_verifier_cpp(&mut self, name: &str, witness: &[String]) { + fn create_verifier_cpp(&mut self, name: &str, witness: &[String], inverses: &[String]) { let include_str = includes_cpp(&snake_case(name)); let wire_transformation = |n: &String| { @@ -19,6 +19,7 @@ impl VerifierBuilder for BBFiles { ) }; let wire_commitments = map_with_newline(witness, wire_transformation); + let inverse_commitments = map_with_newline(inverses, wire_transformation); let ver_cpp = format!(" {include_str} @@ -70,6 +71,13 @@ impl VerifierBuilder for BBFiles { // Get commitments to VM wires {wire_commitments} + + auto [beta, gamm] = transcript->template get_challenges(\"beta\", \"gamma\"); + relation_parameters.beta = beta; + relation_parameters.gamma = gamm; + + // Get commitments to inverses + {inverse_commitments} // Execute Sumcheck Verifier const size_t log_circuit_size = numeric::get_msb(circuit_size); diff --git a/bberg/src/vm_builder.rs b/bberg/src/vm_builder.rs index ced22d7b4a..ee877a2342 100644 --- a/bberg/src/vm_builder.rs +++ b/bberg/src/vm_builder.rs @@ -29,7 +29,11 @@ struct ColumnGroups { fixed: Vec, /// witness or commit columns in pil -> will be found in proof witness: Vec, - /// fixed + witness columns + /// witness or commit columns in pil, with out the inverse columns + witnesses_without_inverses: Vec, + /// fixed + witness columns without lookup inverses + all_cols_without_inverses: Vec, + /// fixed + witness columns with lookup inverses all_cols: Vec, /// Columns that will not be shifted unshifted: Vec, @@ -85,7 +89,9 @@ pub(crate) fn analyzed_to_cpp( let ColumnGroups { fixed, witness, + witnesses_without_inverses, all_cols, + all_cols_without_inverses, unshifted: _unshifted, to_be_shifted, shifted, @@ -100,6 +106,7 @@ pub(crate) fn analyzed_to_cpp( file_name, &relations, &inverses, + &all_cols_without_inverses, &all_cols, &to_be_shifted, &all_cols_with_shifts, @@ -109,7 +116,7 @@ pub(crate) fn analyzed_to_cpp( bb_files.create_flavor_hpp( file_name, &relations, - &permutations, + &inverses, &fixed, &witness, &all_cols, @@ -123,11 +130,11 @@ pub(crate) fn analyzed_to_cpp( bb_files.create_composer_hpp(file_name); // ----------------------- Create the Verifier files ----------------------- - bb_files.create_verifier_cpp(file_name, &witness); + bb_files.create_verifier_cpp(file_name, &witnesses_without_inverses, &inverses); bb_files.create_verifier_hpp(file_name); // ----------------------- Create the Prover files ----------------------- - bb_files.create_prover_cpp(file_name); + bb_files.create_prover_cpp(file_name, &witnesses_without_inverses, &inverses); bb_files.create_prover_hpp(file_name); } @@ -161,22 +168,30 @@ fn get_all_col_names( let witness_names = collect_col(witness, sanitize); let inverses = flatten(&[perm_inverses, lookup_inverses]); - let witness_names = flatten(&[witness_names, inverses.clone(), lookup_counts]); + let witnesses_without_inverses = flatten(&[witness_names.clone(), lookup_counts.clone()]); + let witnesses_with_inverses = flatten(&[witness_names, inverses.clone(), lookup_counts]); // Group columns by properties let shifted = transform_map(to_be_shifted, append_shift); - let all_cols: Vec = flatten(&[fixed_names.clone(), witness_names.clone()]); - let unshifted: Vec = flatten(&[fixed_names.clone(), witness_names.clone()]) + let all_cols_without_inverses: Vec = + flatten(&[fixed_names.clone(), witnesses_without_inverses.clone()]); + let all_cols: Vec = flatten(&[fixed_names.clone(), witnesses_with_inverses.clone()]); + let unshifted: Vec = flatten(&[fixed_names.clone(), witnesses_with_inverses.clone()]) .into_iter() .filter(|name| !shifted.contains(name)) .collect(); - let all_cols_with_shifts: Vec = - flatten(&[fixed_names.clone(), witness_names.clone(), shifted.clone()]); + let all_cols_with_shifts: Vec = flatten(&[ + fixed_names.clone(), + witnesses_with_inverses.clone(), + shifted.clone(), + ]); ColumnGroups { fixed: fixed_names, - witness: witness_names, + witness: witnesses_with_inverses, + all_cols_without_inverses, + witnesses_without_inverses, all_cols, unshifted, to_be_shifted: to_be_shifted.to_vec(),