diff --git a/Cargo.lock b/Cargo.lock index 92a44ed17de..767ed43748b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,6 +42,17 @@ dependencies = [ "synstructure", ] +[[package]] +name = "addchain" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1177222c93a7bb492002e9a3cd947c7fd869e085d6e81a9e415ff1be65b3489c" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "addr2line" version = "0.13.0" @@ -172,6 +183,25 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdcf67bb7ba7797a081cd19009948ab533af7c355d5caf1d08c777582d351e9c" +[[package]] +name = "bellman" +version = "0.6.0" +source = "git+https://github.com/zcashfoundation/librustzcash.git?branch=batch-bellman#c33a8cbefd66bbfaf46794ced6377a62d6189723" +dependencies = [ + "bit-vec 0.4.4", + "blake2s_simd", + "byteorder", + "crossbeam", + "ff", + "futures 0.1.29", + "futures-cpupool", + "group", + "num_cpus", + "pairing", + "rand_core 0.5.1", + "subtle", +] + [[package]] name = "bincode" version = "1.3.1" @@ -188,9 +218,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" dependencies = [ - "bit-vec", + "bit-vec 0.6.2", ] +[[package]] +name = "bit-vec" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" + [[package]] name = "bit-vec" version = "0.6.2" @@ -700,6 +736,31 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "ff" +version = "0.6.0" +source = "git+https://github.com/zcashfoundation/librustzcash.git?branch=batch-bellman#c33a8cbefd66bbfaf46794ced6377a62d6189723" +dependencies = [ + "byteorder", + "ff_derive", + "rand_core 0.5.1", + "subtle", +] + +[[package]] +name = "ff_derive" +version = "0.6.0" +source = "git+https://github.com/zcashfoundation/librustzcash.git?branch=batch-bellman#c33a8cbefd66bbfaf46794ced6377a62d6189723" +dependencies = [ + "addchain", + "num-bigint", + "num-integer", + "num-traits", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.38", +] + [[package]] name = "fixed-hash" version = "0.6.1" @@ -750,6 +811,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futures" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" + [[package]] name = "futures" version = "0.3.5" @@ -781,6 +848,16 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures 0.1.29", + "num_cpus", +] + [[package]] name = "futures-executor" version = "0.3.5" @@ -899,6 +976,18 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +[[package]] +name = "group" +version = "0.6.0" +source = "git+https://github.com/zcashfoundation/librustzcash.git?branch=batch-bellman#c33a8cbefd66bbfaf46794ced6377a62d6189723" +dependencies = [ + "byteorder", + "ff", + "rand 0.7.3", + "rand_xorshift", + "subtle", +] + [[package]] name = "gumdrop" version = "0.7.0" @@ -1403,6 +1492,17 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-format" version = "0.4.0" @@ -1469,6 +1569,18 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "pairing" +version = "0.16.0" +source = "git+https://github.com/zcashfoundation/librustzcash.git?branch=batch-bellman#c33a8cbefd66bbfaf46794ced6377a62d6189723" +dependencies = [ + "byteorder", + "ff", + "group", + "rand_core 0.5.1", + "subtle", +] + [[package]] name = "parity-scale-codec" version = "1.3.4" @@ -2441,7 +2553,7 @@ version = "0.1.0" dependencies = [ "color-eyre", "ed25519-zebra 2.1.1", - "futures", + "futures 0.3.5", "futures-core", "pin-project", "rand 0.7.3", @@ -2874,7 +2986,7 @@ dependencies = [ "displaydoc", "ed25519-zebra 1.0.0", "equihash", - "futures", + "futures 0.3.5", "hex", "jubjub", "lazy_static", @@ -2903,12 +3015,14 @@ version = "3.0.0-alpha.0" name = "zebra-consensus" version = "3.0.0-alpha.0" dependencies = [ + "bellman", "chrono", "color-eyre", - "futures", + "futures 0.3.5", "futures-util", "metrics", "once_cell", + "pairing", "rand 0.7.3", "redjubjub", "spandoc", @@ -2934,7 +3048,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "futures", + "futures 0.3.5", "hex", "indexmap", "metrics", @@ -2970,7 +3084,7 @@ version = "3.0.0-alpha.0" dependencies = [ "color-eyre", "dirs", - "futures", + "futures 0.3.5", "hex", "lazy_static", "once_cell", @@ -2993,7 +3107,7 @@ name = "zebra-test" version = "3.0.0-alpha.0" dependencies = [ "color-eyre", - "futures", + "futures 0.3.5", "hex", "lazy_static", "regex", @@ -3030,7 +3144,7 @@ dependencies = [ "chrono", "color-eyre", "dirs", - "futures", + "futures 0.3.5", "gumdrop", "hyper", "inferno", diff --git a/Cargo.toml b/Cargo.toml index 50678668060..c1a2fb1e042 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,7 @@ panic = "abort" [profile.release] panic = "abort" + +[patch.crates-io] +bellman = { git = "https://github.com/zcashfoundation/librustzcash.git", branch = "batch-bellman" } +pairing = { git = "https://github.com/zcashfoundation/librustzcash.git", branch = "batch-bellman" } diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index bb748d2bd59..f78582045c1 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -10,6 +10,8 @@ chrono = "0.4.13" color-eyre = "0.5" rand = "0.7" redjubjub = "0.2" +bellman = "0.6" +pairing = "0.16" metrics = "0.12" futures = "0.3.5" diff --git a/zebra-consensus/src/primitives.rs b/zebra-consensus/src/primitives.rs index 4887c12748a..005cb7486c6 100644 --- a/zebra-consensus/src/primitives.rs +++ b/zebra-consensus/src/primitives.rs @@ -1,8 +1,13 @@ //! Asynchronous verification of cryptographic primitives. +pub mod groth16; pub mod redjubjub; /// The maximum batch size for any of the batch verifiers. const MAX_BATCH_SIZE: usize = 64; + /// The maximum latency bound for any of the batch verifiers. const MAX_BATCH_LATENCY: std::time::Duration = std::time::Duration::from_millis(100); + +/// The size of the buffer in the broadcast channels used by batch verifiers. +const BROADCAST_BUFFER_SIZE: usize = 10; diff --git a/zebra-consensus/src/primitives/groth16.rs b/zebra-consensus/src/primitives/groth16.rs new file mode 100644 index 00000000000..501cfe106b2 --- /dev/null +++ b/zebra-consensus/src/primitives/groth16.rs @@ -0,0 +1,93 @@ +//! Async Groth16 batch verifier service + +use std::{ + future::Future, + mem, + pin::Pin, + task::{Context, Poll}, +}; + +use futures::future::{ready, Ready}; +use once_cell::sync::Lazy; +use rand::thread_rng; +use tokio::sync::broadcast::{channel, RecvError, Sender}; +use tower::Service; +use tower_batch::{Batch, BatchControl}; +use tower_fallback::Fallback; +use tower_util::ServiceFn; + +use bellman::{ + groth16::{batch, PreparedVerifyingKey}, + VerificationError, +}; +use pairing::bls12_381::Bls12; + +/// A Groth16 verification item, used as the request type of the service. +pub type Item = batch::Item; +pub type Error = VerificationError; + +// XXX we'd like to have a similar Fallback as for Redjubjub, +// but it's unclear how to do that without capturing a &PreparedVerifyingKey, +// which prevents us from casting to a function. + +pub struct Verifier { + batch: batch::Verifier, + // Making this 'static makes managing lifetimes much easier. + pvk: &'static PreparedVerifyingKey, + tx: Sender>, +} + +impl Verifier { + fn new(pvk: &'static PreparedVerifyingKey) -> Self { + let batch = batch::Verifier::default(); + let (tx, _) = channel(super::BROADCAST_BUFFER_SIZE); + Self { batch, tx, pvk } + } +} + +impl Service> for Verifier { + type Response = (); + type Error = Error; + type Future = Pin> + Send + 'static>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: BatchControl) -> Self::Future { + match req { + BatchControl::Item(item) => { + tracing::trace!("got item"); + self.batch.queue(item); + let mut rx = self.tx.subscribe(); + Box::pin(async move { + match rx.recv().await { + Ok(result) => result, + Err(RecvError::Lagged(_)) => { + tracing::error!( + "missed channel updates, BROADCAST_BUFFER_SIZE is too low!!" + ); + Err(Error::InvalidProof) + } + Err(RecvError::Closed) => panic!("verifier was dropped without flushing"), + } + }) + } + + BatchControl::Flush => { + tracing::trace!("got flush command"); + let batch = mem::take(&mut self.batch); + let _ = self.tx.send(batch.verify(thread_rng(), self.pvk)); + Box::pin(async { Ok(()) }) + } + } + } +} + +impl Drop for Verifier { + fn drop(&mut self) { + // We need to flush the current batch in case there are still any pending futures. + let batch = mem::take(&mut self.batch); + let _ = self.tx.send(batch.verify(thread_rng(), self.pvk)); + } +} diff --git a/zebra-consensus/src/primitives/redjubjub.rs b/zebra-consensus/src/primitives/redjubjub.rs index 780fe11d362..38b9f8dc59a 100644 --- a/zebra-consensus/src/primitives/redjubjub.rs +++ b/zebra-consensus/src/primitives/redjubjub.rs @@ -62,8 +62,7 @@ pub struct Verifier { impl Default for Verifier { fn default() -> Self { let batch = batch::Verifier::default(); - // XXX(hdevalence) what's a reasonable choice here? - let (tx, _) = channel(10); + let (tx, _) = channel(super::BROADCAST_BUFFER_SIZE); Self { tx, batch } } } @@ -90,8 +89,8 @@ impl Service> for Verifier { match rx.recv().await { Ok(result) => result, Err(RecvError::Lagged(_)) => { - tracing::warn!( - "missed channel updates for the correct signature batch!" + tracing::error!( + "missed channel updates, BROADCAST_BUFFER_SIZE is too low!!" ); Err(Error::InvalidSignature) }