From dfd423531d01b8a87706a1f8d9048c3eb7ae1380 Mon Sep 17 00:00:00 2001 From: nemo Date: Thu, 10 Sep 2020 11:48:24 -0400 Subject: [PATCH 1/4] feat: split up window post API into separate calls --- filecoin-proofs/src/api/post.rs | 311 +++++++++++++++++++- filecoin-proofs/src/types/mod.rs | 16 + filecoin-proofs/tests/api.rs | 77 ++++- storage-proofs/core/src/compound_proof.rs | 24 ++ storage-proofs/post/src/fallback/vanilla.rs | 54 +++- 5 files changed, 473 insertions(+), 9 deletions(-) diff --git a/filecoin-proofs/src/api/post.rs b/filecoin-proofs/src/api/post.rs index fe5257c60..424b319a9 100644 --- a/filecoin-proofs/src/api/post.rs +++ b/filecoin-proofs/src/api/post.rs @@ -3,7 +3,7 @@ use std::hash::{Hash, Hasher as StdHasher}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; -use anyhow::{ensure, Context, Result}; +use anyhow::{anyhow, ensure, Context, Result}; use bincode::deserialize; use generic_array::typenum::Unsigned; use log::{info, trace}; @@ -25,7 +25,8 @@ use crate::caches::{get_post_params, get_post_verifying_key}; use crate::constants::*; use crate::parameters::{window_post_setup_params, winning_post_setup_params}; use crate::types::{ - ChallengeSeed, Commitment, PersistentAux, PoStConfig, ProverId, SectorSize, TemporaryAux, + ChallengeSeed, Commitment, FallbackPoStVanillaProof, PersistentAux, PoStConfig, ProverId, + SectorSize, SnarkProof, TemporaryAux, VanillaProof, }; use crate::PoStType; @@ -241,7 +242,75 @@ pub fn clear_caches( Ok(()) } -pub type SnarkProof = Vec; +/// Generates a Window proof-of-spacetime with provided vanilla proofs. +pub fn generate_winning_post_with_vanilla( + post_config: &PoStConfig, + randomness: &ChallengeSeed, + prover_id: ProverId, + vanilla_proofs: Vec>, +) -> Result { + info!("generate_winning_post_with_vanilla:start"); + ensure!( + post_config.typ == PoStType::Winning, + "invalid post config type" + ); + + ensure!( + vanilla_proofs.len() == post_config.sector_count, + "invalid amount of vanilla proofs" + ); + + let randomness_safe: ::Domain = + as_safe_commitment(randomness, "randomness")?; + let prover_id_safe: ::Domain = + as_safe_commitment(&prover_id, "prover_id")?; + + let vanilla_params = winning_post_setup_params(&post_config)?; + + let setup_params = compound_proof::SetupParams { + vanilla_params, + partitions: None, + priority: post_config.priority, + }; + let pub_params: compound_proof::PublicParams> = + fallback::FallbackPoStCompound::setup(&setup_params)?; + let groth_params = get_post_params::(&post_config)?; + + let mut pub_sectors = Vec::with_capacity(vanilla_proofs.len()); + for vanilla_proof in &vanilla_proofs { + pub_sectors.push(fallback::PublicSector { + id: vanilla_proof.sector_id, + comm_r: vanilla_proof.comm_r, + }); + } + + let pub_inputs = fallback::PublicInputs { + randomness: randomness_safe, + prover_id: prover_id_safe, + sectors: &pub_sectors, + k: None, + }; + + let partitions = 1; + let partitioned_proofs = partition_vanilla_proofs( + &pub_params.vanilla_params, + &pub_inputs, + partitions, + &vanilla_proofs, + )?; + + let proof = fallback::FallbackPoStCompound::prove_with_vanilla( + &pub_params, + &pub_inputs, + partitioned_proofs, + &groth_params, + )?; + let proof = proof.to_vec()?; + + info!("generate_winning_post_with_vanilla:finish"); + + Ok(proof) +} /// Generates a Winning proof-of-spacetime. pub fn generate_winning_post( @@ -328,9 +397,9 @@ pub fn generate_winning_post( Ok(proof) } -/// Given some randomness and a the length of available sectors, generates the challenged sector. +/// Given some randomness and the length of available sectors, generates the challenged sector. /// -/// The returned values are indicies in the range of `0..sector_set_size`, requiring the caller +/// The returned values are indices in the range of `0..sector_set_size`, requiring the caller /// to match the index to the correct sector. pub fn generate_winning_post_sector_challenge( post_config: &PoStConfig, @@ -362,6 +431,117 @@ pub fn generate_winning_post_sector_challenge( result } +/// Generates the challenges per SectorId required for a Window proof-of-spacetime. +// FIXME: rename to avoid confusion with fallback::generate_sector_challenges +pub fn generate_sector_challenges( + post_config: &PoStConfig, + randomness: &ChallengeSeed, + pub_sectors: &[SectorId], + _prover_id: ProverId, +) -> Result>> { + info!("generate_sector_challenges:start"); + ensure!( + post_config.typ == PoStType::Window || post_config.typ == PoStType::Winning, + "invalid post config type" + ); + + let randomness_safe: ::Domain = + as_safe_commitment(randomness, "randomness")?; + + let public_params = fallback::PublicParams { + sector_size: u64::from(post_config.sector_size), + challenge_count: post_config.challenge_count, + sector_count: post_config.sector_count, + }; + + let mut sector_challenges: BTreeMap> = BTreeMap::new(); + + let num_sectors_per_chunk = post_config.sector_count; + let partitions = if post_config.typ == PoStType::Window { + match get_partitions_for_window_post(pub_sectors.len(), &post_config) { + Some(x) => x, + None => 1, + } + } else { + assert_eq!(post_config.typ, PoStType::Winning); + + 1 + }; + + for partition_index in 0..partitions { + let sectors = pub_sectors + .chunks(num_sectors_per_chunk) + .nth(partition_index) + .ok_or_else(|| anyhow!("invalid number of sectors/partition index"))?; + + for (i, sector) in sectors.iter().enumerate() { + let mut challenges = Vec::new(); + + for n in 0..post_config.challenge_count { + let challenge_index = ((partition_index * post_config.sector_count + i) + * post_config.challenge_count + + n) as u64; + let challenged_leaf_start = fallback::generate_leaf_challenge( + &public_params, + randomness_safe, + u64::from(*sector), + challenge_index, + )?; + challenges.push(challenged_leaf_start); + } + + sector_challenges.insert(*sector, challenges); + } + } + + info!("generate_sector_challenges:finish"); + + Ok(sector_challenges) +} + +/// Generates a single vanilla proof required for Window proof-of-spacetime. +pub fn generate_single_vanilla_proof( + post_config: &PoStConfig, + sector_id: SectorId, + replica: &PrivateReplicaInfo, + challenges: &[u64], +) -> Result> { + let sector_count = 1; + + info!("generate_single_vanilla_proof:start"); + ensure!( + post_config.sector_count == sector_count, + "invalid post config sector size ({} required)", + sector_count + ); + + let tree = &replica.merkle_tree(post_config.sector_size)?; + let comm_r = replica.safe_comm_r()?; + let comm_c = replica.safe_comm_c()?; + let comm_r_last = replica.safe_comm_r_last()?; + + let mut priv_sectors = Vec::with_capacity(sector_count); + priv_sectors.push(fallback::PrivateSector { + tree, + comm_c, + comm_r_last, + }); + + let priv_inputs = fallback::PrivateInputs:: { + sectors: &priv_sectors, + }; + + let vanilla_proof = fallback::vanilla_proofs(sector_id, &priv_inputs, challenges)?; + + info!("generate_single_vanilla_proof:finish"); + + Ok(FallbackPoStVanillaProof { + sector_id, + comm_r, + vanilla_proof, + }) +} + /// Verifies a winning proof-of-spacetime. /// /// The provided `replicas` must be the same ones as passed to `generate_winning_post`, and be based on @@ -461,6 +641,127 @@ pub fn verify_winning_post( Ok(true) } +// Partition a flat vector of vanilla proofs. +pub fn partition_vanilla_proofs( + pub_params: &fallback::PublicParams, + pub_inputs: &fallback::PublicInputs<::Domain>, + partition_count: usize, + vanilla_proofs: &[FallbackPoStVanillaProof], +) -> Result>> { + info!("partition_vanilla_proofs:start"); + + let num_sectors_per_chunk = pub_params.sector_count; + let num_sectors = pub_inputs.sectors.len(); + + ensure!( + num_sectors <= partition_count * num_sectors_per_chunk, + "cannot prove the provided number of sectors: {} > {} * {}", + num_sectors, + partition_count, + num_sectors_per_chunk, + ); + + let mut partition_proofs = Vec::new(); + + for (j, pub_sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + let mut proofs = Vec::with_capacity(num_sectors_per_chunk); + + for pub_sector in pub_sectors_chunk.iter() { + let cur_proof = vanilla_proofs + .iter() + .find(|&proof| proof.sector_id == pub_sector.id) + .expect("failed to locate sector proof"); + + proofs.extend(cur_proof.vanilla_proof.sectors.clone()); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while proofs.len() < num_sectors_per_chunk { + proofs.push(proofs[proofs.len() - 1].clone()); + } + + partition_proofs + .push(fallback::Proof::<::Proof> { sectors: proofs }); + } + + info!("partition_vanilla_proofs:finish"); + + Ok(partition_proofs) +} + +/// Generates a Window proof-of-spacetime with provided vanilla proofs. +pub fn generate_window_post_with_vanilla( + post_config: &PoStConfig, + randomness: &ChallengeSeed, + prover_id: ProverId, + vanilla_proofs: Vec>, +) -> Result { + info!("generate_window_post_with_vanilla:start"); + ensure!( + post_config.typ == PoStType::Window, + "invalid post config type" + ); + + let randomness_safe: ::Domain = + as_safe_commitment(randomness, "randomness")?; + let prover_id_safe: ::Domain = + as_safe_commitment(&prover_id, "prover_id")?; + + let vanilla_params = window_post_setup_params(&post_config); + let partitions = get_partitions_for_window_post(vanilla_proofs.len(), &post_config); + + let setup_params = compound_proof::SetupParams { + vanilla_params, + partitions, + priority: post_config.priority, + }; + + let partitions = match partitions { + Some(x) => x, + None => 1, + }; + + let pub_params: compound_proof::PublicParams> = + fallback::FallbackPoStCompound::setup(&setup_params)?; + let groth_params = get_post_params::(&post_config)?; + + let mut pub_sectors = Vec::with_capacity(vanilla_proofs.len()); + for vanilla_proof in &vanilla_proofs { + pub_sectors.push(fallback::PublicSector { + id: vanilla_proof.sector_id, + comm_r: vanilla_proof.comm_r, + }); + } + + let pub_inputs = fallback::PublicInputs { + randomness: randomness_safe, + prover_id: prover_id_safe, + sectors: &pub_sectors, + k: None, + }; + + let partitioned_proofs = partition_vanilla_proofs( + &pub_params.vanilla_params, + &pub_inputs, + partitions, + &vanilla_proofs, + )?; + + let proof = fallback::FallbackPoStCompound::prove_with_vanilla( + &pub_params, + &pub_inputs, + partitioned_proofs, + &groth_params, + )?; + + info!("generate_window_post_with_vanilla:finish"); + + Ok(proof.to_vec()?) +} + /// Generates a Window proof-of-spacetime. pub fn generate_window_post( post_config: &PoStConfig, diff --git a/filecoin-proofs/src/types/mod.rs b/filecoin-proofs/src/types/mod.rs index fcef9f623..f6da62cd4 100644 --- a/filecoin-proofs/src/types/mod.rs +++ b/filecoin-proofs/src/types/mod.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use storage_proofs::hasher::Hasher; use storage_proofs::porep::stacked; +use storage_proofs::post::fallback::*; +use storage_proofs::sector::*; use crate::constants::*; @@ -83,3 +85,17 @@ pub struct SealPreCommitPhase1Output { pub config: StoreConfig, pub comm_d: Commitment, } + +pub type SnarkProof = Vec; +pub type VanillaProof = Proof<::Proof>; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FallbackPoStVanillaProof { + pub sector_id: SectorId, + pub comm_r: ::Domain, + #[serde(bound( + serialize = "VanillaProof: Serialize", + deserialize = "VanillaProof: Deserialize<'de>" + ))] + pub vanilla_proof: VanillaProof, // Has comm_c, comm_r_last, inclusion_proofs +} diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index 18a5a2d59..ed3a76d7e 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -178,16 +178,47 @@ fn winning_post(sector_size: u64, fake: bool) - assert_eq!(challenged_sectors[0], 0); // with a sector_count of 1, the only valid index is 0 let pub_replicas = vec![(sector_id, PublicReplicaInfo::new(comm_r)?)]; + let private_replica_info = + PrivateReplicaInfo::new(replica.path().into(), comm_r, cache_dir.path().into())?; + + ///////////////////////////////////////////// + // The following methods of proof generation are functionally equivalent: + // 1) + // + /* let priv_replicas = vec![( sector_id, - PrivateReplicaInfo::new(replica.path().into(), comm_r, cache_dir.path().into())?, + private_replica_info, )]; - let proof = generate_winning_post::(&config, &randomness, &priv_replicas[..], prover_id)?; + */ + // + // 2) + let mut vanilla_proofs = Vec::with_capacity(sector_count); + let challenges = + generate_sector_challenges::(&config, &randomness, &vec![sector_id], prover_id)?; + + let single_proof = generate_single_vanilla_proof::( + &config, + sector_id, + &private_replica_info, + &challenges[§or_id], + )?; + + vanilla_proofs.push(single_proof); + + let proof = generate_winning_post_with_vanilla::( + &config, + &randomness, + prover_id, + vanilla_proofs, + )?; + ///////////////////////////////////////////// let valid = verify_winning_post::(&config, &randomness, &pub_replicas[..], prover_id, &proof)?; assert!(valid, "proof did not verify"); + Ok(()) } @@ -334,10 +365,50 @@ fn window_post( priority: false, }; - let proof = generate_window_post::(&config, &randomness, &priv_replicas, prover_id)?; + ///////////////////////////////////////////// + // The following methods of proof generation are functionally equivalent: + // 1) + //let proof = generate_window_post::(&config, &randomness, &priv_replicas, prover_id)?; + // + // 2) + let replica_sectors = priv_replicas + .iter() + .map(|(sector, _replica)| *sector) + .collect::>(); + + let challenges = + generate_sector_challenges::(&config, &randomness, &replica_sectors, prover_id)?; + + // This 'single config' is used when generating a single vanilla proof. + let single_config = PoStConfig { + sector_size: sector_size.into(), + sector_count: 1, + challenge_count: WINDOW_POST_CHALLENGE_COUNT, + typ: PoStType::Window, + priority: false, + }; + + let mut vanilla_proofs = Vec::with_capacity(replica_sectors.len()); + + for (sector_id, replica) in priv_replicas.iter() { + let sector_challenges = &challenges[sector_id]; + let single_proof = generate_single_vanilla_proof::( + &single_config, + *sector_id, + replica, + sector_challenges, + )?; + + vanilla_proofs.push(single_proof); + } + + let proof = + generate_window_post_with_vanilla::(&config, &randomness, prover_id, vanilla_proofs)?; + ///////////////////////////////////////////// let valid = verify_window_post::(&config, &randomness, &pub_replicas, prover_id, &proof)?; assert!(valid, "proof did not verify"); + Ok(()) } diff --git a/storage-proofs/core/src/compound_proof.rs b/storage-proofs/core/src/compound_proof.rs index 667753134..0b39d731f 100644 --- a/storage-proofs/core/src/compound_proof.rs +++ b/storage-proofs/core/src/compound_proof.rs @@ -105,6 +105,30 @@ where Ok(MultiProof::new(groth_proofs, &groth_params.vk)) } + fn prove_with_vanilla<'b>( + pub_params: &PublicParams<'a, S>, + pub_in: &S::PublicInputs, + vanilla_proofs: Vec, + groth_params: &'b groth16::MappedParameters, + ) -> Result> { + let partition_count = Self::partition_count(pub_params); + + // This will always run at least once, since there cannot be zero partitions. + ensure!(partition_count > 0, "There must be partitions"); + + info!("snark_proof:start"); + let groth_proofs = Self::circuit_proofs( + pub_in, + vanilla_proofs, + &pub_params.vanilla_params, + groth_params, + pub_params.priority, + )?; + info!("snark_proof:finish"); + + Ok(MultiProof::new(groth_proofs, &groth_params.vk)) + } + // verify is equivalent to ProofScheme::verify. fn verify<'b>( public_params: &PublicParams<'a, S>, diff --git a/storage-proofs/post/src/fallback/vanilla.rs b/storage-proofs/post/src/fallback/vanilla.rs index a7f0c7995..d4d170ac7 100644 --- a/storage-proofs/post/src/fallback/vanilla.rs +++ b/storage-proofs/post/src/fallback/vanilla.rs @@ -236,6 +236,58 @@ enum ProofOrFault { Fault(SectorId), } +// Generates a single vanilla proof, given the private inputs and sector challenges. +pub fn vanilla_proofs( + sector_id: SectorId, + priv_inputs: &PrivateInputs, + challenges: &[u64], +) -> Result> { + ensure!( + priv_inputs.sectors.len() == 1, + "vanilla_proofs called with multiple sector proofs" + ); + + let priv_sector = &priv_inputs.sectors[0]; + let comm_c = priv_sector.comm_c; + let comm_r_last = priv_sector.comm_r_last; + let tree = priv_sector.tree; + + let tree_leafs = tree.leafs(); + let rows_to_discard = default_rows_to_discard(tree_leafs, Tree::Arity::to_usize()); + + trace!( + "Generating proof for tree leafs {} and arity {}", + tree_leafs, + Tree::Arity::to_usize(), + ); + + let inclusion_proofs = (0..challenges.len()) + .into_par_iter() + .map(|challenged_leaf_index| { + let challenged_leaf_start = challenges[challenged_leaf_index]; + let proof = + tree.gen_cached_proof(challenged_leaf_start as usize, Some(rows_to_discard))?; + + ensure!( + proof.validate(challenged_leaf_start as usize) + && proof.root() == priv_sector.comm_r_last, + "Generated vanilla proof for sector {} is invalid", + sector_id + ); + + Ok(proof) + }) + .collect::>>()?; + + Ok(Proof { + sectors: vec![SectorProof { + inclusion_proofs, + comm_c, + comm_r_last, + }], + }) +} + impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> { type PublicParams = PublicParams; type SetupParams = SetupParams; @@ -455,7 +507,7 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> ensure!( challenge_count == inclusion_proofs.len(), - "unexpected umber of inclusion proofs: {} != {}", + "unexpected number of inclusion proofs: {} != {}", challenge_count, inclusion_proofs.len() ); From a5612133a345f4a41599ddc5d9e6999631946fb2 Mon Sep 17 00:00:00 2001 From: nemo Date: Thu, 10 Sep 2020 12:28:55 -0400 Subject: [PATCH 2/4] feat: verify partition proofs before returning them --- filecoin-proofs/src/api/post.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/filecoin-proofs/src/api/post.rs b/filecoin-proofs/src/api/post.rs index 424b319a9..5f41ef8ad 100644 --- a/filecoin-proofs/src/api/post.rs +++ b/filecoin-proofs/src/api/post.rs @@ -16,6 +16,7 @@ use storage_proofs::merkle::{ }; use storage_proofs::multi_proof::MultiProof; use storage_proofs::post::fallback; +use storage_proofs::proof::ProofScheme; use storage_proofs::sector::*; use storage_proofs::settings; use storage_proofs::util::default_rows_to_discard; @@ -689,6 +690,15 @@ pub fn partition_vanilla_proofs( info!("partition_vanilla_proofs:finish"); + ensure!( + fallback::FallbackPoSt::::verify_all_partitions( + pub_params, + pub_inputs, + &partition_proofs + )? == true, + "partitioned vanilla proofs failed to verify" + ); + Ok(partition_proofs) } From 328c04f249dd3d915456f264a15f70c5dbc853ae Mon Sep 17 00:00:00 2001 From: nemo Date: Thu, 10 Sep 2020 17:18:54 -0400 Subject: [PATCH 3/4] feat: fix winning post api split --- filecoin-proofs/src/api/post.rs | 107 ++++++++++++++++---- filecoin-proofs/tests/api.rs | 16 ++- storage-proofs/post/src/fallback/vanilla.rs | 9 +- 3 files changed, 106 insertions(+), 26 deletions(-) diff --git a/filecoin-proofs/src/api/post.rs b/filecoin-proofs/src/api/post.rs index 5f41ef8ad..a3876ad22 100644 --- a/filecoin-proofs/src/api/post.rs +++ b/filecoin-proofs/src/api/post.rs @@ -16,6 +16,7 @@ use storage_proofs::merkle::{ }; use storage_proofs::multi_proof::MultiProof; use storage_proofs::post::fallback; +use storage_proofs::post::fallback::SectorProof; use storage_proofs::proof::ProofScheme; use storage_proofs::sector::*; use storage_proofs::settings; @@ -294,6 +295,7 @@ pub fn generate_winning_post_with_vanilla( let partitions = 1; let partitioned_proofs = partition_vanilla_proofs( + &post_config, &pub_params.vanilla_params, &pub_inputs, partitions, @@ -432,9 +434,9 @@ pub fn generate_winning_post_sector_challenge( result } -/// Generates the challenges per SectorId required for a Window proof-of-spacetime. -// FIXME: rename to avoid confusion with fallback::generate_sector_challenges -pub fn generate_sector_challenges( +/// Generates the challenges per SectorId required for either a Window +/// proof-of-spacetime or a Winning proof-of-spacetime. +pub fn generate_fallback_sector_challenges( post_config: &PoStConfig, randomness: &ChallengeSeed, pub_sectors: &[SectorId], @@ -642,14 +644,21 @@ pub fn verify_winning_post( Ok(true) } -// Partition a flat vector of vanilla proofs. +// Partition a flat vector of vanilla proofs -- the post_config type +// is required in order to determine the proper shape of the returned +// partitioned proofs. pub fn partition_vanilla_proofs( + post_config: &PoStConfig, pub_params: &fallback::PublicParams, pub_inputs: &fallback::PublicInputs<::Domain>, partition_count: usize, vanilla_proofs: &[FallbackPoStVanillaProof], ) -> Result>> { info!("partition_vanilla_proofs:start"); + ensure!( + post_config.typ == PoStType::Window || post_config.typ == PoStType::Winning, + "invalid post config type" + ); let num_sectors_per_chunk = pub_params.sector_count; let num_sectors = pub_inputs.sectors.len(); @@ -664,28 +673,83 @@ pub fn partition_vanilla_proofs( let mut partition_proofs = Vec::new(); - for (j, pub_sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { - trace!("processing partition {}", j); + // Note that the partition proofs returned are shaped differently + // based on which type of PoSt is being considered. + if post_config.typ == PoStType::Window { + for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); - let mut proofs = Vec::with_capacity(num_sectors_per_chunk); + let mut proofs = Vec::with_capacity(num_sectors_per_chunk); - for pub_sector in pub_sectors_chunk.iter() { - let cur_proof = vanilla_proofs - .iter() - .find(|&proof| proof.sector_id == pub_sector.id) - .expect("failed to locate sector proof"); + for pub_sector in sectors_chunk.iter() { + let cur_proof = vanilla_proofs + .iter() + .find(|&proof| proof.sector_id == pub_sector.id) + .expect("failed to locate sector proof"); - proofs.extend(cur_proof.vanilla_proof.sectors.clone()); - } + // Note: Window post requires all inclusion proofs (based on the challenge + // count per sector) per sector proof. + proofs.extend(cur_proof.vanilla_proof.sectors.clone()); + } - // If there were less than the required number of sectors provided, we duplicate the last one - // to pad the proof out, such that it works in the circuit part. - while proofs.len() < num_sectors_per_chunk { - proofs.push(proofs[proofs.len() - 1].clone()); + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while proofs.len() < num_sectors_per_chunk { + proofs.push(proofs[proofs.len() - 1].clone()); + } + + partition_proofs + .push(fallback::Proof::<::Proof> { sectors: proofs }); } + } else { + for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + // Winning post sector_count is winning post challenges per sector + ensure!( + post_config.sector_count == vanilla_proofs[j].vanilla_proof.sectors.len(), + "invalid number of sector proofs for Winning PoSt" + ); + + let mut proofs = Vec::with_capacity(post_config.challenge_count); + + // Sanity check incoming structure + ensure!( + sectors_chunk.len() == 1, + "Invalid sector chunk for Winning PoSt" + ); + ensure!( + sectors_chunk[0].vanilla_proof.sectors.len() == 1, + "Invalid sector count for Winning PoSt chunk" + ); + + let cur_sector_proof = §ors_chunk[0].vanilla_proof.sectors[0]; + + // Unroll inclusions proofs from the single provided sector_proof (per partition) + // into individual sector proofs, required for winning post. + for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() { + proofs.push(SectorProof { + inclusion_proofs: vec![cur_inclusion_proof.clone()], + comm_c: cur_sector_proof.comm_c, + comm_r_last: cur_sector_proof.comm_r_last, + }); + } - partition_proofs - .push(fallback::Proof::<::Proof> { sectors: proofs }); + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while proofs.len() < num_sectors_per_chunk { + proofs.push(proofs[proofs.len() - 1].clone()); + } + + // Winning post Challenge count is the total winning post challenges + ensure!( + proofs.len() == post_config.challenge_count, + "invalid number of partition proofs based on Winning PoSt challenges" + ); + + partition_proofs + .push(fallback::Proof::<::Proof> { sectors: proofs }); + } } info!("partition_vanilla_proofs:finish"); @@ -695,7 +759,7 @@ pub fn partition_vanilla_proofs( pub_params, pub_inputs, &partition_proofs - )? == true, + )?, "partitioned vanilla proofs failed to verify" ); @@ -754,6 +818,7 @@ pub fn generate_window_post_with_vanilla( }; let partitioned_proofs = partition_vanilla_proofs( + &post_config, &pub_params.vanilla_params, &pub_inputs, partitions, diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index ed3a76d7e..a28603c73 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -195,8 +195,12 @@ fn winning_post(sector_size: u64, fake: bool) - // // 2) let mut vanilla_proofs = Vec::with_capacity(sector_count); - let challenges = - generate_sector_challenges::(&config, &randomness, &vec![sector_id], prover_id)?; + let challenges = generate_fallback_sector_challenges::( + &config, + &randomness, + &vec![sector_id], + prover_id, + )?; let single_proof = generate_single_vanilla_proof::( &config, @@ -376,8 +380,12 @@ fn window_post( .map(|(sector, _replica)| *sector) .collect::>(); - let challenges = - generate_sector_challenges::(&config, &randomness, &replica_sectors, prover_id)?; + let challenges = generate_fallback_sector_challenges::( + &config, + &randomness, + &replica_sectors, + prover_id, + )?; // This 'single config' is used when generating a single vanilla proof. let single_config = PoStConfig { diff --git a/storage-proofs/post/src/fallback/vanilla.rs b/storage-proofs/post/src/fallback/vanilla.rs index d4d170ac7..e120efc13 100644 --- a/storage-proofs/post/src/fallback/vanilla.rs +++ b/storage-proofs/post/src/fallback/vanilla.rs @@ -109,7 +109,7 @@ pub struct SectorProof { serialize = "MerkleProof: Serialize", deserialize = "MerkleProof: serde::de::DeserializeOwned" ))] - inclusion_proofs: + pub inclusion_proofs: Vec>, pub comm_c: ::Domain, pub comm_r_last: ::Domain, @@ -148,6 +148,13 @@ impl SectorProof

{ .map(MerkleProofTrait::as_options) .collect() } + + // Returns a read-only reference. + pub fn inclusion_proofs( + &self, + ) -> &Vec> { + &self.inclusion_proofs + } } #[derive(Debug, Clone)] From 41396974567583a530b8670c99426beff5ab10f8 Mon Sep 17 00:00:00 2001 From: nemo Date: Fri, 11 Sep 2020 08:39:21 -0400 Subject: [PATCH 4/4] fix: apply review feedback --- filecoin-proofs/src/api/post.rs | 208 +++++++++++--------- filecoin-proofs/src/types/mod.rs | 5 +- storage-proofs/core/src/compound_proof.rs | 4 +- storage-proofs/post/src/fallback/vanilla.rs | 12 +- 4 files changed, 121 insertions(+), 108 deletions(-) diff --git a/filecoin-proofs/src/api/post.rs b/filecoin-proofs/src/api/post.rs index a3876ad22..861d453b9 100644 --- a/filecoin-proofs/src/api/post.rs +++ b/filecoin-proofs/src/api/post.rs @@ -27,7 +27,7 @@ use crate::caches::{get_post_params, get_post_verifying_key}; use crate::constants::*; use crate::parameters::{window_post_setup_params, winning_post_setup_params}; use crate::types::{ - ChallengeSeed, Commitment, FallbackPoStVanillaProof, PersistentAux, PoStConfig, ProverId, + ChallengeSeed, Commitment, FallbackPoStSectorProof, PersistentAux, PoStConfig, ProverId, SectorSize, SnarkProof, TemporaryAux, VanillaProof, }; use crate::PoStType; @@ -244,12 +244,12 @@ pub fn clear_caches( Ok(()) } -/// Generates a Window proof-of-spacetime with provided vanilla proofs. +/// Generates a Winning proof-of-spacetime with provided vanilla proofs. pub fn generate_winning_post_with_vanilla( post_config: &PoStConfig, randomness: &ChallengeSeed, prover_id: ProverId, - vanilla_proofs: Vec>, + vanilla_proofs: Vec>, ) -> Result { info!("generate_winning_post_with_vanilla:start"); ensure!( @@ -293,7 +293,10 @@ pub fn generate_winning_post_with_vanilla( k: None, }; - let partitions = 1; + let partitions = match pub_params.partitions { + Some(x) => x, + None => 1, + }; let partitioned_proofs = partition_vanilla_proofs( &post_config, &pub_params.vanilla_params, @@ -460,15 +463,16 @@ pub fn generate_fallback_sector_challenges( let mut sector_challenges: BTreeMap> = BTreeMap::new(); let num_sectors_per_chunk = post_config.sector_count; - let partitions = if post_config.typ == PoStType::Window { - match get_partitions_for_window_post(pub_sectors.len(), &post_config) { + let partitions = match post_config.typ { + PoStType::Window => match get_partitions_for_window_post(pub_sectors.len(), &post_config) { Some(x) => x, None => 1, - } - } else { - assert_eq!(post_config.typ, PoStType::Winning); + }, + PoStType::Winning => { + assert_eq!(post_config.typ, PoStType::Winning); - 1 + 1 + } }; for partition_index in 0..partitions { @@ -484,13 +488,13 @@ pub fn generate_fallback_sector_challenges( let challenge_index = ((partition_index * post_config.sector_count + i) * post_config.challenge_count + n) as u64; - let challenged_leaf_start = fallback::generate_leaf_challenge( + let challenged_leaf = fallback::generate_leaf_challenge( &public_params, randomness_safe, u64::from(*sector), challenge_index, )?; - challenges.push(challenged_leaf_start); + challenges.push(challenged_leaf); } sector_challenges.insert(*sector, challenges); @@ -502,27 +506,31 @@ pub fn generate_fallback_sector_challenges( Ok(sector_challenges) } -/// Generates a single vanilla proof required for Window proof-of-spacetime. +/// Generates a single vanilla proof required for either Window proof-of-spacetime +/// or Winning proof-of-spacetime. pub fn generate_single_vanilla_proof( post_config: &PoStConfig, sector_id: SectorId, replica: &PrivateReplicaInfo, challenges: &[u64], -) -> Result> { - let sector_count = 1; - +) -> Result> { info!("generate_single_vanilla_proof:start"); - ensure!( - post_config.sector_count == sector_count, - "invalid post config sector size ({} required)", - sector_count - ); let tree = &replica.merkle_tree(post_config.sector_size)?; let comm_r = replica.safe_comm_r()?; let comm_c = replica.safe_comm_c()?; let comm_r_last = replica.safe_comm_r_last()?; + // There is only enough information in the arguments to generate a + // single vanilla proof, so the behaviour is unexpected if the + // sector_count provided is > 1. + let sector_count = post_config.sector_count; + ensure!( + post_config.sector_count == 1, + "invalid post config sector size ({} required)", + sector_count + ); + let mut priv_sectors = Vec::with_capacity(sector_count); priv_sectors.push(fallback::PrivateSector { tree, @@ -534,11 +542,11 @@ pub fn generate_single_vanilla_proof( sectors: &priv_sectors, }; - let vanilla_proof = fallback::vanilla_proofs(sector_id, &priv_inputs, challenges)?; + let vanilla_proof = fallback::vanilla_proof(sector_id, &priv_inputs, challenges)?; info!("generate_single_vanilla_proof:finish"); - Ok(FallbackPoStVanillaProof { + Ok(FallbackPoStSectorProof { sector_id, comm_r, vanilla_proof, @@ -644,15 +652,15 @@ pub fn verify_winning_post( Ok(true) } -// Partition a flat vector of vanilla proofs -- the post_config type -// is required in order to determine the proper shape of the returned -// partitioned proofs. +// Partition a flat vector of vanilla sector proofs. The post_config +// (PoSt) type is required in order to determine the proper shape of +// the returned partitioned proofs. pub fn partition_vanilla_proofs( post_config: &PoStConfig, pub_params: &fallback::PublicParams, pub_inputs: &fallback::PublicInputs<::Domain>, partition_count: usize, - vanilla_proofs: &[FallbackPoStVanillaProof], + vanilla_proofs: &[FallbackPoStSectorProof], ) -> Result>> { info!("partition_vanilla_proofs:start"); ensure!( @@ -675,80 +683,84 @@ pub fn partition_vanilla_proofs( // Note that the partition proofs returned are shaped differently // based on which type of PoSt is being considered. - if post_config.typ == PoStType::Window { - for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { - trace!("processing partition {}", j); - - let mut proofs = Vec::with_capacity(num_sectors_per_chunk); - - for pub_sector in sectors_chunk.iter() { - let cur_proof = vanilla_proofs - .iter() - .find(|&proof| proof.sector_id == pub_sector.id) - .expect("failed to locate sector proof"); - - // Note: Window post requires all inclusion proofs (based on the challenge - // count per sector) per sector proof. - proofs.extend(cur_proof.vanilla_proof.sectors.clone()); - } - - // If there were less than the required number of sectors provided, we duplicate the last one - // to pad the proof out, such that it works in the circuit part. - while proofs.len() < num_sectors_per_chunk { - proofs.push(proofs[proofs.len() - 1].clone()); + match post_config.typ { + PoStType::Window => { + for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + let mut sector_proofs = Vec::with_capacity(num_sectors_per_chunk); + + for pub_sector in sectors_chunk.iter() { + let cur_proof = vanilla_proofs + .iter() + .find(|&proof| proof.sector_id == pub_sector.id) + .expect("failed to locate sector proof"); + + // Note: Window post requires all inclusion proofs (based on the challenge + // count per sector) per sector proof. + sector_proofs.extend(cur_proof.vanilla_proof.sectors.clone()); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, + }); } - - partition_proofs - .push(fallback::Proof::<::Proof> { sectors: proofs }); } - } else { - for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() { - trace!("processing partition {}", j); - - // Winning post sector_count is winning post challenges per sector - ensure!( - post_config.sector_count == vanilla_proofs[j].vanilla_proof.sectors.len(), - "invalid number of sector proofs for Winning PoSt" - ); - - let mut proofs = Vec::with_capacity(post_config.challenge_count); - - // Sanity check incoming structure - ensure!( - sectors_chunk.len() == 1, - "Invalid sector chunk for Winning PoSt" - ); - ensure!( - sectors_chunk[0].vanilla_proof.sectors.len() == 1, - "Invalid sector count for Winning PoSt chunk" - ); - - let cur_sector_proof = §ors_chunk[0].vanilla_proof.sectors[0]; - - // Unroll inclusions proofs from the single provided sector_proof (per partition) - // into individual sector proofs, required for winning post. - for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() { - proofs.push(SectorProof { - inclusion_proofs: vec![cur_inclusion_proof.clone()], - comm_c: cur_sector_proof.comm_c, - comm_r_last: cur_sector_proof.comm_r_last, + PoStType::Winning => { + for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + // Sanity check incoming structure + ensure!( + sectors_chunk.len() == 1, + "Invalid sector chunk for Winning PoSt" + ); + ensure!( + sectors_chunk[0].vanilla_proof.sectors.len() == 1, + "Invalid sector count for Winning PoSt chunk" + ); + + // Winning post sector_count is winning post challenges per sector + ensure!( + post_config.sector_count == sectors_chunk[j].vanilla_proof.sectors.len(), + "invalid number of sector proofs for Winning PoSt" + ); + + let mut sector_proofs = Vec::with_capacity(post_config.challenge_count); + let cur_sector_proof = §ors_chunk[0].vanilla_proof.sectors[0]; + + // Unroll inclusions proofs from the single provided sector_proof (per partition) + // into individual sector proofs, required for winning post. + for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() { + sector_proofs.push(SectorProof { + inclusion_proofs: vec![cur_inclusion_proof.clone()], + comm_c: cur_sector_proof.comm_c, + comm_r_last: cur_sector_proof.comm_r_last, + }); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + // Winning post Challenge count is the total winning post challenges + ensure!( + sector_proofs.len() == post_config.challenge_count, + "invalid number of partition proofs based on Winning PoSt challenges" + ); + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, }); } - - // If there were less than the required number of sectors provided, we duplicate the last one - // to pad the proof out, such that it works in the circuit part. - while proofs.len() < num_sectors_per_chunk { - proofs.push(proofs[proofs.len() - 1].clone()); - } - - // Winning post Challenge count is the total winning post challenges - ensure!( - proofs.len() == post_config.challenge_count, - "invalid number of partition proofs based on Winning PoSt challenges" - ); - - partition_proofs - .push(fallback::Proof::<::Proof> { sectors: proofs }); } } @@ -771,7 +783,7 @@ pub fn generate_window_post_with_vanilla( post_config: &PoStConfig, randomness: &ChallengeSeed, prover_id: ProverId, - vanilla_proofs: Vec>, + vanilla_proofs: Vec>, ) -> Result { info!("generate_window_post_with_vanilla:start"); ensure!( diff --git a/filecoin-proofs/src/types/mod.rs b/filecoin-proofs/src/types/mod.rs index f6da62cd4..9de822861 100644 --- a/filecoin-proofs/src/types/mod.rs +++ b/filecoin-proofs/src/types/mod.rs @@ -89,8 +89,11 @@ pub struct SealPreCommitPhase1Output { pub type SnarkProof = Vec; pub type VanillaProof = Proof<::Proof>; +// This FallbackPoStSectorProof is used during Fallback PoSt, but +// contains only Vanilla proof information and is not a full Fallback +// PoSt proof. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct FallbackPoStVanillaProof { +pub struct FallbackPoStSectorProof { pub sector_id: SectorId, pub comm_r: ::Domain, #[serde(bound( diff --git a/storage-proofs/core/src/compound_proof.rs b/storage-proofs/core/src/compound_proof.rs index 0b39d731f..518531520 100644 --- a/storage-proofs/core/src/compound_proof.rs +++ b/storage-proofs/core/src/compound_proof.rs @@ -78,7 +78,7 @@ where // This will always run at least once, since there cannot be zero partitions. ensure!(partition_count > 0, "There must be partitions"); - info!("vanilla_proof:start"); + info!("vanilla_proofs:start"); let vanilla_proofs = S::prove_all_partitions( &pub_params.vanilla_params, &pub_in, @@ -86,7 +86,7 @@ where partition_count, )?; - info!("vanilla_proof:finish"); + info!("vanilla_proofs:finish"); let sanity_check = S::verify_all_partitions(&pub_params.vanilla_params, &pub_in, &vanilla_proofs)?; diff --git a/storage-proofs/post/src/fallback/vanilla.rs b/storage-proofs/post/src/fallback/vanilla.rs index e120efc13..2988e9299 100644 --- a/storage-proofs/post/src/fallback/vanilla.rs +++ b/storage-proofs/post/src/fallback/vanilla.rs @@ -244,14 +244,14 @@ enum ProofOrFault { } // Generates a single vanilla proof, given the private inputs and sector challenges. -pub fn vanilla_proofs( +pub fn vanilla_proof( sector_id: SectorId, priv_inputs: &PrivateInputs, challenges: &[u64], ) -> Result> { ensure!( priv_inputs.sectors.len() == 1, - "vanilla_proofs called with multiple sector proofs" + "vanilla_proof called with multiple sector proofs" ); let priv_sector = &priv_inputs.sectors[0]; @@ -271,13 +271,11 @@ pub fn vanilla_proofs( let inclusion_proofs = (0..challenges.len()) .into_par_iter() .map(|challenged_leaf_index| { - let challenged_leaf_start = challenges[challenged_leaf_index]; - let proof = - tree.gen_cached_proof(challenged_leaf_start as usize, Some(rows_to_discard))?; + let challenged_leaf = challenges[challenged_leaf_index]; + let proof = tree.gen_cached_proof(challenged_leaf as usize, Some(rows_to_discard))?; ensure!( - proof.validate(challenged_leaf_start as usize) - && proof.root() == priv_sector.comm_r_last, + proof.validate(challenged_leaf as usize) && proof.root() == priv_sector.comm_r_last, "Generated vanilla proof for sector {} is invalid", sector_id );