ZIP-221: Validate chain history commitments in the non-finalized state (

#2301) * sketch of implementation * refined implementation; still incomplete * update librustzcash, change zcash_history to work with it * simplified code per review; renamed MMR to HistoryTree * expand HistoryTree implementation * handle and propagate errors * simplify check.rs tracing * add suggested TODO * add HistoryTree::prune * fix bug in pruning * fix compilation of tests; still need to make them pass * Apply suggestions from code review Co-authored-by: teor <teor@riseup.net> * Apply suggestions from code review Co-authored-by: teor <teor@riseup.net> * improvements from code review * improve check.rs comments and variable names * fix HistoryTree which should use BTreeMap and not HashMap; fix non_finalized_state prop tests * fix finalized_state proptest * fix non_finalized_state tests by setting the correct commitments * renamed mmr.rs to history_tree.rs * Add HistoryTree struct * expand non_finalized_state protest * fix typo * Add HistoryTree struct * Update zebra-chain/src/primitives/zcash_history.rs Co-authored-by: Deirdre Connolly <deirdre@zfnd.org> * fix formatting * Apply suggestions from code review Co-authored-by: Deirdre Connolly <deirdre@zfnd.org> * history_tree.rs: fixes from code review * fixes to work with updated HistoryTree * Improvements from code review * Add Debug implementations to allow comparing Chains with proptest_assert_eq Co-authored-by: teor <teor@riseup.net> Co-authored-by: Deirdre Connolly <deirdre@zfnd.org>
ZcashFoundation · Jul 7, 2021 · 91b1fcb · 91b1fcb
1 parent fdfa3cb
commit 91b1fcb
Show file tree

Hide file tree

Showing 13 changed files with 487 additions and 114 deletions.
diff --git a/zebra-chain/src/block/commitment.rs b/zebra-chain/src/block/commitment.rs
@@ -149,6 +149,12 @@ impl From<[u8; 32]> for ChainHistoryMmrRootHash {
     }
 }
 
+impl From<ChainHistoryMmrRootHash> for [u8; 32] {
+    fn from(hash: ChainHistoryMmrRootHash) -> Self {
+        hash.0
+    }
+}
+
 /// A block commitment to chain history and transaction auth.
 /// - the chain history tree for all ancestors in the current network upgrade,
 ///   and

diff --git a/zebra-chain/src/history_tree.rs b/zebra-chain/src/history_tree.rs
@@ -32,6 +32,7 @@ pub enum HistoryTreeError {
 
 /// History tree (Merkle mountain range) structure that contains information about
 // the block history, as specified in [ZIP-221][https://zips.z.cash/zip-0221].
+#[derive(Debug)]
 pub struct HistoryTree {
     network: Network,
     network_upgrade: NetworkUpgrade,
@@ -244,3 +245,11 @@ impl Clone for HistoryTree {
         }
     }
 }
+
+impl PartialEq for HistoryTree {
+    fn eq(&self, other: &Self) -> bool {
+        self.hash() == other.hash()
+    }
+}
+
+impl Eq for HistoryTree {}
diff --git a/zebra-chain/src/primitives/zcash_history.rs b/zebra-chain/src/primitives/zcash_history.rs
@@ -45,7 +45,7 @@ impl From<&zcash_history::NodeData> for NodeData {
 /// An encoded entry in the tree.
 ///
 /// Contains the node data and information about its position in the tree.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Entry {
     inner: [u8; zcash_history::MAX_ENTRY_SIZE],
 }
@@ -231,6 +231,15 @@ impl Tree {
     }
 }
 
+impl std::fmt::Debug for Tree {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Tree")
+            .field("network", &self.network)
+            .field("network_upgrade", &self.network_upgrade)
+            .finish()
+    }
+}
+
 /// Convert a Block into a zcash_history::NodeData used in the MMR tree.
 ///
 /// `sapling_root` is the root of the Sapling note commitment tree of the block.

diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs
@@ -3,7 +3,11 @@ use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use thiserror::Error;
 
-use zebra_chain::{block, work::difficulty::CompactDifficulty};
+use zebra_chain::{
+    block::{self, ChainHistoryMmrRootHash},
+    history_tree::HistoryTreeError,
+    work::difficulty::CompactDifficulty,
+};
 
 /// A wrapper for type erased errors that is itself clonable and implements the
 /// Error trait
@@ -74,4 +78,17 @@ pub enum ValidateContextError {
         difficulty_threshold: CompactDifficulty,
         expected_difficulty: CompactDifficulty,
     },
+
+    #[error("block contains an invalid commitment")]
+    InvalidBlockCommitment(#[from] block::CommitmentError),
+
+    #[error("block history commitment {candidate_commitment:?} is different to the expected commitment {expected_commitment:?}")]
+    #[non_exhaustive]
+    InvalidHistoryCommitment {
+        candidate_commitment: ChainHistoryMmrRootHash,
+        expected_commitment: ChainHistoryMmrRootHash,
+    },
+
+    #[error("error building the history tree")]
+    HistoryTreeError(#[from] HistoryTreeError),
 }
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
@@ -178,9 +178,10 @@ impl StateService {
         let parent_hash = prepared.block.header.previous_block_hash;
 
         if self.disk.finalized_tip_hash() == parent_hash {
-            self.mem.commit_new_chain(prepared)?;
+            self.mem
+                .commit_new_chain(prepared, self.disk.history_tree().clone())?;
         } else {
-            self.mem.commit_block(prepared)?;
+            self.mem.commit_block(prepared, self.disk.history_tree())?;
         }
 
         Ok(())
@@ -222,7 +223,7 @@ impl StateService {
         assert!(relevant_chain.len() >= POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN,
                 "contextual validation requires at least 28 (POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN) blocks");
 
-        check::block_is_contextually_valid(
+        check::block_is_valid_for_recent_chain(
             prepared,
             self.network,
             self.disk.finalized_tip_height(),

diff --git a/zebra-state/src/service/arbitrary.rs b/zebra-state/src/service/arbitrary.rs
@@ -50,6 +50,29 @@ impl ValueTree for PreparedChainTree {
 pub struct PreparedChain {
     // the proptests are threaded (not async), so we want to use a threaded mutex here
     chain: std::sync::Mutex<Option<(Network, Arc<SummaryDebug<Vec<PreparedBlock>>>)>>,
+    // the height from which to start the chain. If None, starts at the genesis block
+    start_height: Option<Height>,
+}
+
+impl PreparedChain {
+    /// Create a PreparedChain strategy with Heartwood-onward blocks.
+    pub(super) fn new_heartwood() -> Self {
+        // The history tree only works with Heartwood onward.
+        // Since the network will be chosen later, we pick the larger
+        // between the mainnet and testnet Heartwood activation heights.
+        let main_height = NetworkUpgrade::Heartwood
+            .activation_height(Network::Mainnet)
+            .expect("must have height");
+        let test_height = NetworkUpgrade::Heartwood
+            .activation_height(Network::Testnet)
+            .expect("must have height");
+        let height = (std::cmp::max(main_height, test_height) + 1).expect("must be valid");
+
+        PreparedChain {
+            start_height: Some(height),
+            ..Default::default()
+        }
+    }
 }
 
 impl Strategy for PreparedChain {
@@ -60,7 +83,12 @@ impl Strategy for PreparedChain {
         let mut chain = self.chain.lock().unwrap();
         if chain.is_none() {
             // TODO: use the latest network upgrade (#1974)
-            let ledger_strategy = LedgerState::genesis_strategy(NetworkUpgrade::Nu5, None, false);
+            let ledger_strategy = match self.start_height {
+                Some(start_height) => {
+                    LedgerState::height_strategy(start_height, NetworkUpgrade::Nu5, None, false)
+                }
+                None => LedgerState::genesis_strategy(NetworkUpgrade::Nu5, None, false),
+            };
 
             let (network, blocks) = ledger_strategy
                 .prop_flat_map(|ledger| {
@@ -83,7 +111,9 @@ impl Strategy for PreparedChain {
         }
 
         let chain = chain.clone().expect("should be generated");
-        let count = (1..chain.1.len()).new_tree(runner)?;
+        // `count` must be 1 less since the first block is used to build the
+        // history tree.
+        let count = (1..chain.1.len() - 1).new_tree(runner)?;
         Ok(PreparedChainTree {
             chain: chain.1,
             count,

diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs
@@ -4,7 +4,7 @@ use std::borrow::Borrow;
 
 use chrono::Duration;
 use zebra_chain::{
-    block::{self, Block},
+    block::{self, Block, ChainHistoryMmrRootHash},
     parameters::POW_AVERAGING_WINDOW,
     parameters::{Network, NetworkUpgrade},
     work::difficulty::CompactDifficulty,
@@ -18,8 +18,11 @@ use difficulty::{AdjustedDifficulty, POW_MEDIAN_BLOCK_SPAN};
 
 pub(crate) mod difficulty;
 
-/// Check that `block` is contextually valid for `network`, based on the
-/// `finalized_tip_height` and `relevant_chain`.
+/// Check that the `prepared` block is contextually valid for `network`, based
+/// on the `finalized_tip_height` and `relevant_chain`.
+///
+/// This function performs checks that require a small number of recent blocks,
+/// including previous hash, previous height, and block difficulty.
 ///
 /// The relevant chain is an iterator over the ancestors of `block`, starting
 /// with its parent block.
@@ -28,12 +31,8 @@ pub(crate) mod difficulty;
 ///
 /// If the state contains less than 28
 /// (`POW_AVERAGING_WINDOW + POW_MEDIAN_BLOCK_SPAN`) blocks.
-#[tracing::instrument(
-    name = "contextual_validation",
-    fields(?network),
-    skip(prepared, network, finalized_tip_height, relevant_chain)
-)]
-pub(crate) fn block_is_contextually_valid<C>(
+#[tracing::instrument(skip(prepared, finalized_tip_height, relevant_chain))]
+pub(crate) fn block_is_valid_for_recent_chain<C>(
     prepared: &PreparedBlock,
     network: Network,
     finalized_tip_height: Option<block::Height>,
@@ -86,6 +85,40 @@ where
     Ok(())
 }
 
+/// Check that the `prepared` block is contextually valid for `network`, based
+/// on the `history_root_hash` of the history tree up to and including the
+/// previous block.
+#[tracing::instrument(skip(prepared))]
+pub(crate) fn block_commitment_is_valid_for_chain_history(
+    prepared: &PreparedBlock,
+    network: Network,
+    history_root_hash: &ChainHistoryMmrRootHash,
+) -> Result<(), ValidateContextError> {
+    match prepared.block.commitment(network)? {
+        block::Commitment::PreSaplingReserved(_)
+        | block::Commitment::FinalSaplingRoot(_)
+        | block::Commitment::ChainHistoryActivationReserved => {
+            // No contextual checks needed for those.
+            Ok(())
+        }
+        block::Commitment::ChainHistoryRoot(block_history_root_hash) => {
+            if block_history_root_hash == *history_root_hash {
+                Ok(())
+            } else {
+                Err(ValidateContextError::InvalidHistoryCommitment {
+                    candidate_commitment: block_history_root_hash,
+                    expected_commitment: *history_root_hash,
+                })
+            }
+        }
+        block::Commitment::ChainHistoryBlockTxAuthCommitment(_) => {
+            // TODO: Get auth_hash from block (ZIP-244), e.g.
+            // let auth_hash = prepared.block.auth_hash();
+            todo!("hash mmr_hash and auth_hash per ZIP-244 and compare")
+        }
+    }
+}
+
 /// Returns `ValidateContextError::OrphanedBlock` if the height of the given
 /// block is less than or equal to the finalized tip height.
 fn block_is_not_orphaned(

diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
@@ -7,6 +7,7 @@ mod tests;
 
 use std::{collections::HashMap, convert::TryInto, path::Path, sync::Arc};
 
+use zebra_chain::history_tree::HistoryTree;
 use zebra_chain::transparent;
 use zebra_chain::{
     block::{self, Block},
@@ -378,6 +379,11 @@ impl FinalizedState {
             })
     }
 
+    /// Returns the history tree for the finalized state.
+    pub fn history_tree(&self) -> &HistoryTree {
+        todo!("add history tree to finalized state");
+    }
+
     /// If the database is `ephemeral`, delete it.
     fn delete_ephemeral(&self) {
         if self.ephemeral {

diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs
@@ -14,6 +14,7 @@ use std::{collections::BTreeSet, mem, ops::Deref, sync::Arc};
 
 use zebra_chain::{
     block::{self, Block},
+    history_tree::HistoryTree,
     parameters::Network,
     transaction::{self, Transaction},
     transparent,
@@ -23,6 +24,8 @@ use crate::{FinalizedBlock, HashOrHeight, PreparedBlock, Utxo, ValidateContextEr
 
 use self::chain::Chain;
 
+use super::check;
+
 /// The state of the chains in memory, incuding queued blocks.
 #[derive(Default)]
 pub struct NonFinalizedState {
@@ -74,7 +77,14 @@ impl NonFinalizedState {
     }
 
     /// Commit block to the non-finalized state.
-    pub fn commit_block(&mut self, prepared: PreparedBlock) -> Result<(), ValidateContextError> {
+    ///
+    /// `finalized_tip_history_tree`: the history tree of the finalized tip used to recompute
+    /// the history tree, if needed.
+    pub fn commit_block(
+        &mut self,
+        prepared: PreparedBlock,
+        finalized_tip_history_tree: &HistoryTree,
+    ) -> Result<(), ValidateContextError> {
         let parent_hash = prepared.block.header.previous_block_hash;
         let (height, hash) = (prepared.height, prepared.hash);
 
@@ -85,8 +95,12 @@ impl NonFinalizedState {
             );
         }
 
-        let mut parent_chain = self.parent_chain(parent_hash)?;
-
+        let mut parent_chain = self.parent_chain(parent_hash, finalized_tip_history_tree)?;
+        check::block_commitment_is_valid_for_chain_history(
+            &prepared,
+            self.network,
+            &parent_chain.history_root_hash(),
+        )?;
         parent_chain.push(prepared)?;
         self.chain_set.insert(parent_chain);
         self.update_metrics_for_committed_block(height, hash);
@@ -95,12 +109,20 @@ impl NonFinalizedState {
 
     /// Commit block to the non-finalized state as a new chain where its parent
     /// is the finalized tip.
+    ///
+    /// `history_tree` must contain the history of the finalized tip.
     pub fn commit_new_chain(
         &mut self,
         prepared: PreparedBlock,
+        finalized_tip_history_tree: HistoryTree,
     ) -> Result<(), ValidateContextError> {
-        let mut chain = Chain::default();
+        let mut chain = Chain::new(finalized_tip_history_tree);
         let (height, hash) = (prepared.height, prepared.hash);
+        check::block_commitment_is_valid_for_chain_history(
+            &prepared,
+            self.network,
+            &chain.history_root_hash(),
+        )?;
         chain.push(prepared)?;
         self.chain_set.insert(Box::new(chain));
         self.update_metrics_for_committed_block(height, hash);
@@ -246,9 +268,13 @@ impl NonFinalizedState {
     ///
     /// The chain can be an existing chain in the non-finalized state or a freshly
     /// created fork, if needed.
+    ///
+    /// `finalized_tip_history_tree`: the history tree of the finalized tip used to recompute
+    /// the history tree, if needed.
     fn parent_chain(
         &mut self,
         parent_hash: block::Hash,
+        finalized_tip_history_tree: &HistoryTree,
     ) -> Result<Box<Chain>, ValidateContextError> {
         match self.take_chain_if(|chain| chain.non_finalized_tip_hash() == parent_hash) {
             // An existing chain in the non-finalized state
@@ -257,7 +283,11 @@ impl NonFinalizedState {
             None => Ok(Box::new(
                 self.chain_set
                     .iter()
-                    .find_map(|chain| chain.fork(parent_hash).transpose())
+                    .find_map(|chain| {
+                        chain
+                            .fork(parent_hash, finalized_tip_history_tree)
+                            .transpose()
+                    })
                     .expect(
                         "commit_block is only called with blocks that are ready to be commited",
                     )?,