From 809e67094d23be6d2be8c41a087b152db95b38bf Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Tue, 16 May 2023 15:45:27 -0700 Subject: [PATCH 1/2] chore: Remove sled & tempfile dev-dependencies --- Cargo.lock | 72 --------- Cargo.toml | 4 - cspell.json | 2 - src/acvm_interop/pwg/merkle.rs | 9 +- src/composer.rs | 4 +- src/crs.rs | 10 -- src/merkle.rs | 263 ++++++++++++++------------------- 7 files changed, 117 insertions(+), 247 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4912b70..25400755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,8 +61,6 @@ dependencies = [ "pkg-config", "reqwest", "rust-embed", - "sled", - "tempfile", "thiserror", "tokio", "wasmer", @@ -900,16 +898,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "funty" version = "1.1.0" @@ -977,15 +965,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.6" @@ -1344,16 +1323,6 @@ dependencies = [ "cc", ] -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.17" @@ -1560,31 +1529,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] - [[package]] name = "paste" version = "1.0.12" @@ -2203,22 +2147,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot", -] - [[package]] name = "smallvec" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index e9d49804..6e1c5302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,6 @@ getrandom = { version = "0.2", optional = true } [build-dependencies] pkg-config = "0.3" -[dev-dependencies] -sled = "0.34.6" -tempfile = "3.3" - [features] default = ["native"] native = [ diff --git a/cspell.json b/cspell.json index bdc8f643..1904bf5e 100644 --- a/cspell.json +++ b/cspell.json @@ -73,8 +73,6 @@ "acvm", "barretenberg", "indicatif", - "tempdir", - "tempfile", "wasmer", "getrandom" ] diff --git a/src/acvm_interop/pwg/merkle.rs b/src/acvm_interop/pwg/merkle.rs index da85ab59..765dc85c 100644 --- a/src/acvm_interop/pwg/merkle.rs +++ b/src/acvm_interop/pwg/merkle.rs @@ -92,11 +92,9 @@ mod tests { }, ]; - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); let mut msg_hasher: blake2::Blake2s = MessageHasher::new(); - let mut tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let mut tree: MerkleTree = MerkleTree::new(3); for test_vector in tests { let index = FieldElement::try_from_str(test_vector.index).unwrap(); @@ -142,10 +140,7 @@ mod tests { // This test uses `update_leaf` directly rather than `update_message` #[test] fn simple_shield() -> Result<(), Error> { - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); - - let mut tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let mut tree: MerkleTree = MerkleTree::new(3); let barretenberg = Barretenberg::new(); let pubkey_x = FieldElement::from_hex( diff --git a/src/composer.rs b/src/composer.rs index 2fa724ad..a0b5ea28 100644 --- a/src/composer.rs +++ b/src/composer.rs @@ -878,11 +878,9 @@ mod test { #[test] fn test_compute_merkle_root_constraint() -> Result<(), Error> { - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); let mut msg_hasher: blake2::Blake2s = MessageHasher::new(); - let tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let tree: MerkleTree = MerkleTree::new(3); let empty_leaf = vec![0; 64]; diff --git a/src/crs.rs b/src/crs.rs index 9862a713..e1f293a7 100644 --- a/src/crs.rs +++ b/src/crs.rs @@ -192,13 +192,3 @@ fn does_not_panic() { } //TODO check that p_points memory is properly free } -#[test] -#[ignore] -fn downloading() { - use tempfile::tempdir; - let dir = tempdir().unwrap(); - - let file_path = dir.path().to_path_buf().join("transcript00.dat"); - let res = download_crs(file_path); - assert_eq!(res, Ok(())); -} diff --git a/src/merkle.rs b/src/merkle.rs index 395610c4..6278be62 100644 --- a/src/merkle.rs +++ b/src/merkle.rs @@ -1,6 +1,6 @@ // TODO(#166): Rework this module to return results use acvm::FieldElement; -use std::{convert::TryInto, path::Path}; +use std::{collections::BTreeMap, convert::TryInto}; use crate::{pedersen::Pedersen, Barretenberg, Error}; @@ -61,130 +61,23 @@ fn flatten_path(path: Vec<(FieldElement, FieldElement)>) -> Vec { pub(crate) struct MerkleTree { depth: u32, total_size: u32, - db: sled::Db, + db: BTreeMap<&'static [u8], Vec>, + preimages_tree: BTreeMap<[u8; 16], Vec>, + hashes_tree: BTreeMap<[u8; 16], Vec>, barretenberg: PH, msg_hasher: MH, } -fn insert_root(db: &mut sled::Db, value: FieldElement) { - db.insert("ROOT".as_bytes(), value.to_be_bytes()).unwrap(); -} -fn fetch_root(db: &sled::Db) -> FieldElement { - let value = db - .get("ROOT".as_bytes()) - .unwrap() - .expect("merkle root should always be present"); - FieldElement::from_be_bytes_reduce(&value) -} -fn insert_depth(db: &mut sled::Db, value: u32) { - db.insert("DEPTH".as_bytes(), &value.to_be_bytes()).unwrap(); -} -fn fetch_depth(db: &sled::Db) -> u32 { - let value = db - .get("DEPTH".as_bytes()) - .unwrap() - .expect("depth should always be present"); - u32::from_be_bytes(value.to_vec().try_into().unwrap()) -} -fn insert_empty_index(db: &mut sled::Db, index: u32) { - // First fetch the depth to see that this is less than - let depth = fetch_depth(db); - let total_size = 1 << depth; - if index > total_size { - panic!("trying to insert at index {index}, but total width is {total_size}") - } - db.insert("EMPTY".as_bytes(), &index.to_be_bytes()).unwrap(); -} -fn fetch_empty_index(db: &sled::Db) -> u32 { - let value = db - .get("EMPTY".as_bytes()) - .unwrap() - .expect("empty index should always be present"); - u32::from_be_bytes(value.to_vec().try_into().unwrap()) -} -fn insert_preimage(db: &mut sled::Db, index: u32, value: Vec) { - let tree = db.open_tree("preimages").unwrap(); - - let index = index as u128; - tree.insert(index.to_be_bytes(), value).unwrap(); -} - -#[allow(dead_code)] -fn fetch_preimage(db: &sled::Db, index: usize) -> Vec { - let tree = db.open_tree("preimages").unwrap(); - - let index = index as u128; - tree.get(index.to_be_bytes()) - .unwrap() - .map(|i_vec| i_vec.to_vec()) - .unwrap() -} -fn fetch_hash(db: &sled::Db, index: usize) -> FieldElement { - let tree = db.open_tree("hashes").unwrap(); - let index = index as u128; - - tree.get(index.to_be_bytes()) - .unwrap() - .map(|i_vec| FieldElement::from_be_bytes_reduce(&i_vec)) - .unwrap() -} - -fn insert_hash(db: &mut sled::Db, index: u32, hash: FieldElement) { - let tree = db.open_tree("hashes").unwrap(); - let index = index as u128; - - tree.insert(index.to_be_bytes(), hash.to_be_bytes()) - .unwrap(); -} - -#[allow(dead_code)] -fn find_hash_from_value(db: &sled::Db, leaf_value: &FieldElement) -> Option { - let tree = db.open_tree("hashes").unwrap(); - - for index_db_lef_hash in tree.iter() { - let (key, db_leaf_hash) = index_db_lef_hash.unwrap(); - let index = u128::from_be_bytes(key.to_vec().try_into().unwrap()); - - if db_leaf_hash.to_vec() == leaf_value.to_be_bytes() { - return Some(index); - } - } - None -} - impl MerkleTree { - #[allow(dead_code)] - pub(crate) fn from_path>( - path: P, - barretenberg: PH, - msg_hasher: MH, - ) -> MerkleTree { - assert!(path.as_ref().exists(), "path does not exist"); - let config = sled::Config::new().path(path); - - let db = config.open().unwrap(); - - let depth = fetch_depth(&db); - - let total_size = 1u32 << depth; - - MerkleTree { - depth, - total_size, - barretenberg, - db, - msg_hasher, - } - } - - pub(crate) fn new>(depth: u32, path: P) -> MerkleTree { + pub(crate) fn new(depth: u32) -> Self { let barretenberg = PH::new(); let mut msg_hasher = MH::new(); assert!((1..=20).contains(&depth)); // Why can depth != 0 and depth not more than 20? - let config = sled::Config::new().path(path); - let mut db = config.open().unwrap(); + let db = BTreeMap::new(); + let preimages_tree = BTreeMap::new(); + let hashes_tree = BTreeMap::new(); let total_size = 1u32 << depth; @@ -208,27 +101,106 @@ impl MerkleTree { offset += layer_size; layer_size /= 2; } + + let mut merkle_tree = MerkleTree { + depth, + total_size, + barretenberg, + db, + preimages_tree, + hashes_tree, + msg_hasher, + }; + let root = current; - insert_root(&mut db, root); + merkle_tree.insert_root(root); for (index, hash) in hashes.into_iter().enumerate() { - insert_hash(&mut db, index as u32, hash) + merkle_tree.insert_hash(index as u32, hash) } for (index, image) in pre_images.into_iter().enumerate() { - insert_preimage(&mut db, index as u32, image) + merkle_tree.insert_preimage(index as u32, image) } - insert_depth(&mut db, depth); - insert_empty_index(&mut db, 0); + merkle_tree.insert_depth(depth); + merkle_tree.insert_empty_index(0); - MerkleTree { - depth, - total_size, - barretenberg, - db, - msg_hasher, + merkle_tree + } + + fn insert_root(&mut self, value: FieldElement) { + self.db.insert("ROOT".as_bytes(), value.to_be_bytes()); + } + fn fetch_root(&self) -> FieldElement { + let value = self + .db + .get("ROOT".as_bytes()) + .expect("merkle root should always be present"); + FieldElement::from_be_bytes_reduce(value) + } + fn insert_depth(&mut self, value: u32) { + self.db + .insert("DEPTH".as_bytes(), value.to_be_bytes().into()); + } + fn fetch_depth(&self) -> u32 { + let value = self + .db + .get("DEPTH".as_bytes()) + .expect("depth should always be present"); + u32::from_be_bytes(value.to_vec().try_into().unwrap()) + } + fn insert_empty_index(&mut self, index: u32) { + // First fetch the depth to see that this is less than + let depth = self.fetch_depth(); + let total_size = 1 << depth; + if index > total_size { + panic!("trying to insert at index {index}, but total width is {total_size}") } + self.db + .insert("EMPTY".as_bytes(), index.to_be_bytes().into()); + } + fn fetch_empty_index(&self) -> u32 { + let value = self + .db + .get("EMPTY".as_bytes()) + .expect("empty index should always be present"); + u32::from_be_bytes(value.to_vec().try_into().unwrap()) + } + fn insert_preimage(&mut self, index: u32, value: Vec) { + let index = index as u128; + self.preimages_tree.insert(index.to_be_bytes(), value); + } + #[allow(dead_code)] + fn fetch_preimage(&self, index: usize) -> Vec { + let index = index as u128; + self.preimages_tree + .get(&index.to_be_bytes()) + .unwrap() + .to_vec() + } + fn fetch_hash(&self, index: usize) -> FieldElement { + let index = index as u128; + + let i_vec = self.hashes_tree.get(&index.to_be_bytes()).unwrap(); + FieldElement::from_be_bytes_reduce(i_vec) + } + fn insert_hash(&mut self, index: u32, hash: FieldElement) { + let index = index as u128; + + self.hashes_tree + .insert(index.to_be_bytes(), hash.to_be_bytes()); + } + fn find_hash_from_value(&self, leaf_value: &FieldElement) -> Option { + for index_db_lef_hash in self.hashes_tree.iter() { + let (key, db_leaf_hash) = index_db_lef_hash; + let index = u128::from_be_bytes(key.to_vec().try_into().unwrap()); + + if db_leaf_hash.to_vec() == leaf_value.to_be_bytes() { + return Some(index); + } + } + None } pub(crate) fn get_hash_path(&self, mut index: usize) -> HashPath { @@ -239,8 +211,8 @@ impl MerkleTree { for _ in 0..self.depth { index &= (!0) - 1; path.push(( - fetch_hash(&self.db, offset + index), - fetch_hash(&self.db, offset + index + 1), + self.fetch_hash(offset + index), + self.fetch_hash(offset + index + 1), )); offset += layer_size as usize; layer_size /= 2; @@ -255,18 +227,17 @@ impl MerkleTree { new_message: &[u8], ) -> Result { let current = self.msg_hasher.hash(new_message); - - insert_preimage(&mut self.db, index as u32, new_message.to_vec()); + self.insert_preimage(index as u32, new_message.to_vec()); self.update_leaf(index, current) } fn check_if_index_valid_and_increment(&mut self, mut index: usize) { // Fetch the empty index - let empty_index = fetch_empty_index(&self.db) as usize; + let empty_index = self.fetch_empty_index() as usize; if empty_index == index { // increment the empty index index += 1; - insert_empty_index(&mut self.db, index as u32); + self.insert_empty_index(index as u32); } else { panic!("this is an regular append-only merkle tree. Tried to insert at {index}, but next empty is at {empty_index}"); } @@ -274,14 +245,14 @@ impl MerkleTree { #[allow(dead_code)] pub(crate) fn find_index_from_leaf(&self, leaf_value: &FieldElement) -> Option { - let index = find_hash_from_value(&self.db, leaf_value); + let index = self.find_hash_from_value(leaf_value); index.map(|val| val as usize) } #[allow(dead_code)] // TODO: this gets updated to be -1 on the latest barretenberg branch pub(crate) fn find_index_for_empty_leaf(&self) -> usize { - let index = fetch_empty_index(&self.db); + let index = self.fetch_empty_index(); index as usize } @@ -298,12 +269,12 @@ impl MerkleTree { let mut offset = 0usize; let mut layer_size = self.total_size; for _ in 0..self.depth { - insert_hash(&mut self.db, (offset + index) as u32, current); + self.insert_hash((offset + index) as u32, current); index &= (!0) - 1; current = self.barretenberg.hash( - &fetch_hash(&self.db, offset + index), - &fetch_hash(&self.db, offset + index + 1), + &self.fetch_hash(offset + index), + &self.fetch_hash(offset + index + 1), )?; offset += layer_size as usize; @@ -311,18 +282,18 @@ impl MerkleTree { index /= 2; } - insert_root(&mut self.db, current); + self.insert_root(current); Ok(current) } #[allow(dead_code)] /// Gets a message at `index`. This is not the leaf pub(crate) fn get_message_at_index(&self, index: usize) -> Vec { - fetch_preimage(&self.db, index) + self.fetch_preimage(index) } pub(crate) fn root(&self) -> FieldElement { - fetch_root(&self.db) + self.fetch_root() } #[allow(dead_code)] @@ -333,10 +304,8 @@ impl MerkleTree { #[test] fn basic_interop_initial_root() { - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); // Test that the initial root is computed correctly - let tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let tree: MerkleTree = MerkleTree::new(3); // Copied from barretenberg by copying the stdout from MemoryTree let expected_hex = "04ccfbbb859b8605546e03dcaf41393476642859ff7f99446c054b841f0e05c8"; assert_eq!(tree.root().to_hex(), expected_hex) @@ -344,10 +313,8 @@ fn basic_interop_initial_root() { #[test] fn basic_interop_hashpath() { - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); // Test that the hashpath is correct - let tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let tree: MerkleTree = MerkleTree::new(3); let path = tree.get_hash_path(0); @@ -375,9 +342,7 @@ fn basic_interop_hashpath() { #[test] fn basic_interop_update() -> Result<(), Error> { // Test that computing the HashPath is correct - use tempfile::tempdir; - let temp_dir = tempdir().unwrap(); - let mut tree: MerkleTree = MerkleTree::new(3, &temp_dir); + let mut tree: MerkleTree = MerkleTree::new(3); tree.update_message(0, &[0; 64])?; tree.update_message(1, &[1; 64])?; From 1ab6ba9297b0c5c9e9ddea3c11547df1f68272b2 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 17 May 2023 06:21:04 +0100 Subject: [PATCH 2/2] chore: space out functions --- src/merkle.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/merkle.rs b/src/merkle.rs index 6278be62..4446f47b 100644 --- a/src/merkle.rs +++ b/src/merkle.rs @@ -132,6 +132,7 @@ impl MerkleTree { fn insert_root(&mut self, value: FieldElement) { self.db.insert("ROOT".as_bytes(), value.to_be_bytes()); } + fn fetch_root(&self) -> FieldElement { let value = self .db @@ -139,10 +140,12 @@ impl MerkleTree { .expect("merkle root should always be present"); FieldElement::from_be_bytes_reduce(value) } + fn insert_depth(&mut self, value: u32) { self.db .insert("DEPTH".as_bytes(), value.to_be_bytes().into()); } + fn fetch_depth(&self) -> u32 { let value = self .db @@ -150,6 +153,7 @@ impl MerkleTree { .expect("depth should always be present"); u32::from_be_bytes(value.to_vec().try_into().unwrap()) } + fn insert_empty_index(&mut self, index: u32) { // First fetch the depth to see that this is less than let depth = self.fetch_depth(); @@ -160,6 +164,7 @@ impl MerkleTree { self.db .insert("EMPTY".as_bytes(), index.to_be_bytes().into()); } + fn fetch_empty_index(&self) -> u32 { let value = self .db @@ -167,10 +172,12 @@ impl MerkleTree { .expect("empty index should always be present"); u32::from_be_bytes(value.to_vec().try_into().unwrap()) } + fn insert_preimage(&mut self, index: u32, value: Vec) { let index = index as u128; self.preimages_tree.insert(index.to_be_bytes(), value); } + #[allow(dead_code)] fn fetch_preimage(&self, index: usize) -> Vec { let index = index as u128; @@ -179,18 +186,21 @@ impl MerkleTree { .unwrap() .to_vec() } + fn fetch_hash(&self, index: usize) -> FieldElement { let index = index as u128; let i_vec = self.hashes_tree.get(&index.to_be_bytes()).unwrap(); FieldElement::from_be_bytes_reduce(i_vec) } + fn insert_hash(&mut self, index: u32, hash: FieldElement) { let index = index as u128; self.hashes_tree .insert(index.to_be_bytes(), hash.to_be_bytes()); } + fn find_hash_from_value(&self, leaf_value: &FieldElement) -> Option { for index_db_lef_hash in self.hashes_tree.iter() { let (key, db_leaf_hash) = index_db_lef_hash;