Update default hamt hash function to sha256 and make algo generic

blockchain/state_manager/src/utils.rs

@@ -12,7 +12,7 @@ use address::{Address, Protocol};
 use bitfield::BitField;
 use blockstore::BlockStore;
 use cid::Cid;
-use fil_types::{RegisteredSealProof, SectorInfo, SectorNumber, SectorSize};
+use fil_types::{RegisteredSealProof, SectorInfo, SectorNumber, SectorSize, HAMT_BIT_WIDTH};
 use filecoin_proofs_api::{post::generate_winning_post_sector_challenge, ProverId};
 use forest_blocks::Tipset;
 use ipld_amt::Amt;
@@ -318,9 +318,10 @@ where
         })?;
     let mut miners: Vec<Address> = Vec::new();
     let block_store = &*state_manager.get_block_store();
-    let map = Hamt::load(&power_actor_state.claims, block_store)
-        .map_err(|err| Error::Other(err.to_string()))?;
-    map.for_each(|_: &String, k: String| -> Result<(), String> {
+    let map =
+        Hamt::<_>::load_with_bit_width(&power_actor_state.claims, block_store, HAMT_BIT_WIDTH)
+            .map_err(|err| Error::Other(err.to_string()))?;
+    map.for_each(|_, k: String| -> Result<(), String> {
         let address = Address::from_bytes(k.as_bytes()).map_err(|e| e.to_string())?;
         miners.push(address);
         Ok(())

ipld/hamt/Cargo.toml

@@ -14,14 +14,18 @@ ipld_blockstore = { path = "../blockstore" }
 forest_ipld = { path = "../" }
 serde_bytes = "0.11.3"
 thiserror = "1.0"
+sha2 = "0.9.1"
 
-# TODO replace fork with updated fork or make PR into https://github.com/stusmall/murmur3
+# TODO remove murmur3 support when finalized in Filecoin
 [dependencies.murmur3]
 git = "https://github.com/dignifiedquire/murmur3"
 branch = "nicer-hashing"
+optional = true
 
 [features]
-identity-hash = []
+default = ["identity", "murmur3"]
+identity = []
+murmur = ["murmur3"]
 
 [dev-dependencies]
 hex = "0.4.2"

ipld/hamt/src/bitfield.rs

@@ -63,20 +63,20 @@ impl Default for Bitfield {
 }
 
 impl Bitfield {
-    pub fn clear_bit(&mut self, idx: u8) {
+    pub fn clear_bit(&mut self, idx: u32) {
         let ai = idx / 64;
         let bi = idx % 64;
         self.0[ai as usize] &= u64::MAX - (1 << bi as u32);
     }
 
-    pub fn test_bit(&self, idx: u8) -> bool {
+    pub fn test_bit(&self, idx: u32) -> bool {
         let ai = idx / 64;
         let bi = idx % 64;
 
         self.0[ai as usize] & (1 << bi as u32) != 0
     }
 
-    pub fn set_bit(&mut self, idx: u8) {
+    pub fn set_bit(&mut self, idx: u32) {
         let ai = idx / 64;
         let bi = idx % 64;
 
@@ -100,28 +100,28 @@ impl Bitfield {
         Bitfield([0, 0, 0, 0])
     }
 
-    pub fn set_bits_le(self, bit: u8) -> Self {
+    pub fn set_bits_le(self, bit: u32) -> Self {
         if bit == 0 {
             return self;
         }
         self.set_bits_leq(bit - 1)
     }
 
-    pub fn set_bits_leq(mut self, bit: u8) -> Self {
+    pub fn set_bits_leq(mut self, bit: u32) -> Self {
         if bit < 64 {
-            self.0[0] = set_bits_leq(self.0[0], bit as u32);
+            self.0[0] = set_bits_leq(self.0[0], bit);
         } else if bit < 128 {
             self.0[0] = std::u64::MAX;
-            self.0[1] = set_bits_leq(self.0[1], bit as u32 - 64);
+            self.0[1] = set_bits_leq(self.0[1], bit - 64);
         } else if bit < 192 {
             self.0[0] = std::u64::MAX;
             self.0[1] = std::u64::MAX;
-            self.0[2] = set_bits_leq(self.0[2], bit as u32 - 128);
+            self.0[2] = set_bits_leq(self.0[2], bit - 128);
         } else {
             self.0[0] = std::u64::MAX;
             self.0[1] = std::u64::MAX;
             self.0[2] = std::u64::MAX;
-            self.0[3] = set_bits_leq(self.0[3], bit as u32 - 192);
+            self.0[3] = set_bits_leq(self.0[3], bit - 192);
         }
 
         self

ipld/hamt/src/hamt.rs

@@ -1,13 +1,15 @@
 // Copyright 2020 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use super::BytesKey;
 use crate::node::Node;
-use crate::{Error, Hash, DEFAULT_BIT_WIDTH};
+use crate::{Error, Hash, HashAlgorithm, Sha256, DEFAULT_BIT_WIDTH};
 use cid::{multihash::Blake2b256, Cid};
 use forest_ipld::{from_ipld, to_ipld, Ipld};
 use ipld_blockstore::BlockStore;
 use serde::{de::DeserializeOwned, Serialize, Serializer};
 use std::borrow::Borrow;
+use std::marker::PhantomData;
 
 /// Implementation of the HAMT data structure for IPLD.
 ///
@@ -18,24 +20,26 @@ use std::borrow::Borrow;
 ///
 /// let store = db::MemoryDB::default();
 ///
-/// let mut map: Hamt<usize, _> = Hamt::new(&store);
+/// let mut map: Hamt<_, usize> = Hamt::new(&store);
 /// map.set(1, "a".to_string()).unwrap();
 /// assert_eq!(map.get(&1).unwrap(), Some("a".to_string()));
 /// assert_eq!(map.delete(&1).unwrap(), true);
 /// assert_eq!(map.get::<_, String>(&1).unwrap(), None);
 /// let cid = map.flush().unwrap();
 /// ```
 #[derive(Debug)]
-pub struct Hamt<'a, K, BS> {
-    root: Node<K>,
+pub struct Hamt<'a, BS, K = BytesKey, H = Sha256> {
+    root: Node<K, H>,
     store: &'a BS,
 
-    bit_width: u8,
+    bit_width: u32,
+    hash: PhantomData<H>,
 }
 
-impl<K, BS> Serialize for Hamt<'_, K, BS>
+impl<BS, K, H> Serialize for Hamt<'_, BS, K, H>
 where
     K: Serialize,
+    H: HashAlgorithm,
 {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -45,27 +49,29 @@ where
     }
 }
 
-impl<'a, K: PartialEq, S: BlockStore> PartialEq for Hamt<'a, K, S> {
+impl<'a, K: PartialEq, S: BlockStore, H: HashAlgorithm> PartialEq for Hamt<'a, S, K, H> {
     fn eq(&self, other: &Self) -> bool {
         self.root == other.root
     }
 }
 
-impl<'a, K, BS> Hamt<'a, K, BS>
+impl<'a, BS, K, H> Hamt<'a, BS, K, H>
 where
     K: Hash + Eq + PartialOrd + Serialize + DeserializeOwned + Clone,
     BS: BlockStore,
+    H: HashAlgorithm,
 {
     pub fn new(store: &'a BS) -> Self {
         Self::new_with_bit_width(store, DEFAULT_BIT_WIDTH)
     }
 
     /// Construct hamt with a bit width
-    pub fn new_with_bit_width(store: &'a BS, bit_width: u8) -> Self {
+    pub fn new_with_bit_width(store: &'a BS, bit_width: u32) -> Self {
         Self {
             root: Node::default(),
             store,
             bit_width,
+            hash: Default::default(),
         }
     }
 
@@ -75,12 +81,13 @@ where
     }
 
     /// Lazily instantiate a hamt from this root Cid with a specified bit width.
-    pub fn load_with_bit_width(cid: &Cid, store: &'a BS, bit_width: u8) -> Result<Self, Error> {
+    pub fn load_with_bit_width(cid: &Cid, store: &'a BS, bit_width: u32) -> Result<Self, Error> {
         match store.get(cid)? {
             Some(root) => Ok(Self {
                 root,
                 store,
                 bit_width,
+                hash: Default::default(),
             }),
             None => Err(Error::CidNotFound(cid.to_string())),
         }
@@ -115,7 +122,7 @@ where
     ///
     /// let store = db::MemoryDB::default();
     ///
-    /// let mut map: Hamt<usize, _> = Hamt::new(&store);
+    /// let mut map: Hamt<_, usize> = Hamt::new(&store);
     /// map.set(37, "a".to_string()).unwrap();
     /// assert_eq!(map.is_empty(), false);
     ///
@@ -143,7 +150,7 @@ where
     ///
     /// let store = db::MemoryDB::default();
     ///
-    /// let mut map: Hamt<usize, _> = Hamt::new(&store);
+    /// let mut map: Hamt<_, usize> = Hamt::new(&store);
     /// map.set(1, "a".to_string()).unwrap();
     /// assert_eq!(map.get(&1).unwrap(), Some("a".to_string()));
     /// assert_eq!(map.get::<usize, String>(&2).unwrap(), None);
@@ -174,7 +181,7 @@ where
     ///
     /// let store = db::MemoryDB::default();
     ///
-    /// let mut map: Hamt<usize, _> = Hamt::new(&store);
+    /// let mut map: Hamt<_, usize> = Hamt::new(&store);
     /// map.set(1, "a".to_string()).unwrap();
     /// assert_eq!(map.contains_key(&1).unwrap(), true);
     /// assert_eq!(map.contains_key(&2).unwrap(), false);
@@ -202,7 +209,7 @@ where
     ///
     /// let store = db::MemoryDB::default();
     ///
-    /// let mut map: Hamt<usize, _> = Hamt::new(&store);
+    /// let mut map: Hamt<_, usize> = Hamt::new(&store);
     /// map.set(1, "a".to_string()).unwrap();
     /// assert_eq!(map.delete(&1).unwrap(), true);
     /// assert_eq!(map.delete(&1).unwrap(), false);
@@ -240,7 +247,7 @@ where
     ///
     /// let store = db::MemoryDB::default();
     ///
-    /// let mut map: Hamt<usize, _> = Hamt::new(&store);
+    /// let mut map: Hamt<_, usize> = Hamt::new(&store);
     /// map.set(1, 1).unwrap();
     /// map.set(4, 2).unwrap();
     ///

ipld/hamt/src/hash_algorithm.rs

@@ -0,0 +1,101 @@
+// Copyright 2020 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use crate::{Hash, HashedKey};
+use sha2::{Digest, Sha256 as Sha256Hasher};
+
+#[cfg(feature = "murmur")]
+use murmur3::murmur3_x64_128::MurmurHasher;
+
+/// Algorithm used as the hasher for the Hamt.
+pub trait HashAlgorithm {
+    fn hash<X: ?Sized>(key: &X) -> HashedKey
+    where
+        X: Hash;
+}
+
+/// Type is needed because the Sha256 hasher does not implement `std::hash::Hasher`
+#[derive(Default)]
+struct Sha2HasherWrapper(Sha256Hasher);
+
+impl Hasher for Sha2HasherWrapper {
+    fn finish(&self) -> u64 {
+        // u64 hash not used in hamt
+        0
+    }
+
+    fn write(&mut self, bytes: &[u8]) {
+        self.0.update(bytes);
+    }
+}
+
+#[derive(Debug)]
+pub enum Sha256 {}
+
+impl HashAlgorithm for Sha256 {
+    fn hash<X: ?Sized>(key: &X) -> HashedKey
+    where
+        X: Hash,
+    {
+        let mut hasher = Sha2HasherWrapper::default();
+        key.hash(&mut hasher);
+        hasher.0.finalize().into()
+    }
+}
+
+#[cfg(feature = "murmur")]
+#[derive(Debug)]
+pub enum Murmur3 {}
+
+#[cfg(feature = "murmur")]
+impl HashAlgorithm for Murmur3 {
+    fn hash<X: ?Sized>(key: &X) -> HashedKey
+    where
+        X: Hash,
+    {
+        let mut hasher = MurmurHasher::default();
+        key.hash(&mut hasher);
+        let mut bz: [u8; 32] = Default::default();
+        let hash: [u8; 16] = hasher.finalize().into();
+        bz[0..16].copy_from_slice(&hash);
+        bz
+    }
+}
+
+#[cfg(feature = "identity")]
+use std::hash::Hasher;
+
+#[cfg(feature = "identity")]
+#[derive(Default)]
+struct IdentityHasher {
+    bz: HashedKey,
+}
+#[cfg(feature = "identity")]
+impl Hasher for IdentityHasher {
+    fn finish(&self) -> u64 {
+        // u64 hash not used in hamt
+        0
+    }
+
+    fn write(&mut self, bytes: &[u8]) {
+        for (i, byte) in bytes.iter().take(self.bz.len()).enumerate() {
+            self.bz[i] = *byte;
+        }
+    }
+}
+
+#[cfg(feature = "identity")]
+#[derive(Debug)]
+pub enum Identity {}
+
+#[cfg(feature = "identity")]
+impl HashAlgorithm for Identity {
+    fn hash<X: ?Sized>(key: &X) -> HashedKey
+    where
+        X: Hash,
+    {
+        let mut ident_hasher = IdentityHasher::default();
+        key.hash(&mut ident_hasher);
+        ident_hasher.bz
+    }
+}