From 1e997939837e9c1f0c087d6d28ac12e373c8c05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 25 Mar 2024 13:44:01 +0800 Subject: [PATCH 01/13] feat(testenv): add `make_checkpoint_tip` This creates a checkpoint linked list which contains all blocks. --- crates/testenv/src/lib.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/testenv/src/lib.rs b/crates/testenv/src/lib.rs index 8cba86261..4ae6ea6e3 100644 --- a/crates/testenv/src/lib.rs +++ b/crates/testenv/src/lib.rs @@ -1,7 +1,9 @@ -use bdk_chain::bitcoin::{ - address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash, - secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget, - ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid, +use bdk_chain::{ + bitcoin::{ + address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash, secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget, ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid + }, + local_chain::CheckPoint, + BlockId, }; use bitcoincore_rpc::{ bitcoincore_rpc_json::{GetBlockTemplateModes, GetBlockTemplateRules}, @@ -234,6 +236,18 @@ impl TestEnv { .send_to_address(address, amount, None, None, None, None, None, None)?; Ok(txid) } + + /// Create a checkpoint linked list of all the blocks in the chain. + pub fn make_checkpoint_tip(&self) -> CheckPoint { + CheckPoint::from_block_ids((0_u32..).map_while(|height| { + self.bitcoind + .client + .get_block_hash(height as u64) + .ok() + .map(|hash| BlockId { height, hash }) + })) + .expect("must craft tip") + } } #[cfg(test)] From bd62aa0fe199d676710c9909617198d62f4897c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 25 Mar 2024 13:39:21 +0800 Subject: [PATCH 02/13] feat(esplora)!: remove `EsploraExt::update_local_chain` Previously, we would update the `TxGraph` and `KeychainTxOutIndex` first, then create a second update for `LocalChain`. This required locking the receiving structures 3 times (instead of twice, which is optimal). This PR eliminates this requirement by making use of the new `query` method of `CheckPoint`. Examples are also updated to use the new API. --- crates/esplora/src/async_ext.rs | 532 ++++++++++------- crates/esplora/src/blocking_ext.rs | 552 +++++++++++------- crates/esplora/src/lib.rs | 22 +- crates/esplora/tests/async_ext.rs | 70 ++- crates/esplora/tests/blocking_ext.rs | 107 +++- crates/testenv/src/lib.rs | 4 +- example-crates/example_esplora/src/main.rs | 86 ++- .../wallet_esplora_async/src/main.rs | 17 +- .../wallet_esplora_blocking/src/main.rs | 27 +- 9 files changed, 861 insertions(+), 556 deletions(-) diff --git a/crates/esplora/src/async_ext.rs b/crates/esplora/src/async_ext.rs index bd7a2c850..d0ae80d5b 100644 --- a/crates/esplora/src/async_ext.rs +++ b/crates/esplora/src/async_ext.rs @@ -1,15 +1,18 @@ +use std::collections::BTreeSet; + use async_trait::async_trait; use bdk_chain::collections::btree_map; +use bdk_chain::Anchor; use bdk_chain::{ - bitcoin::{Amount, BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, + bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, collections::BTreeMap, local_chain::{self, CheckPoint}, BlockId, ConfirmationTimeHeightAnchor, TxGraph, }; -use esplora_client::TxStatus; +use esplora_client::{Amount, TxStatus}; use futures::{stream::FuturesOrdered, TryStreamExt}; -use crate::anchor_from_status; +use crate::{anchor_from_status, FullScanUpdate, SyncUpdate}; /// [`esplora_client::Error`] type Error = Box; @@ -22,36 +25,15 @@ type Error = Box; #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] #[cfg_attr(not(target_arch = "wasm32"), async_trait)] pub trait EsploraAsyncExt { - /// Prepare a [`LocalChain`] update with blocks fetched from Esplora. - /// - /// * `local_tip` is the previous tip of [`LocalChain::tip`]. - /// * `request_heights` is the block heights that we are interested in fetching from Esplora. - /// - /// The result of this method can be applied to [`LocalChain::apply_update`]. - /// - /// ## Consistency - /// - /// The chain update returned is guaranteed to be consistent as long as there is not a *large* re-org - /// during the call. The size of re-org we can tollerate is server dependent but will be at - /// least 10. - /// - /// [`LocalChain`]: bdk_chain::local_chain::LocalChain - /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip - /// [`LocalChain::apply_update`]: bdk_chain::local_chain::LocalChain::apply_update - async fn update_local_chain( - &self, - local_tip: CheckPoint, - request_heights: impl IntoIterator + Send> + Send, - ) -> Result; - - /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and - /// returns a [`TxGraph`] and a map of last active indices. + /// Scan keychain scripts for transactions against Esplora, returning an update that can be + /// applied to the receiving structures. /// + /// * `local_tip`: the previously seen tip from [`LocalChain::tip`]. /// * `keychain_spks`: keychains that we want to scan transactions for /// - /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated - /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in - /// parallel. + /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no + /// associated transactions. `parallel_requests` specifies the max number of HTTP requests to + /// make in parallel. /// /// ## Note /// @@ -65,19 +47,23 @@ pub trait EsploraAsyncExt { /// and [Sparrow](https://www.sparrowwallet.com/docs/faq.html#ive-restored-my-wallet-but-some-of-my-funds-are-missing). /// /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1. + /// + /// [`LocalChain::tip`]: local_chain::LocalChain::tip async fn full_scan( &self, + local_tip: CheckPoint, keychain_spks: BTreeMap< K, impl IntoIterator + Send> + Send, >, stop_gap: usize, parallel_requests: usize, - ) -> Result<(TxGraph, BTreeMap), Error>; + ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Esplora client) for the data /// specified and return a [`TxGraph`]. /// + /// * `local_tip`: the previously seen tip from [`LocalChain::tip`]. /// * `misc_spks`: scripts that we want to sync transactions for /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we @@ -86,210 +72,216 @@ pub trait EsploraAsyncExt { /// If the scripts to sync are unknown, such as when restoring or importing a keychain that /// may include scripts that have been used, use [`full_scan`] with the keychain. /// + /// [`LocalChain::tip`]: local_chain::LocalChain::tip /// [`full_scan`]: EsploraAsyncExt::full_scan async fn sync( &self, + local_tip: CheckPoint, misc_spks: impl IntoIterator + Send> + Send, txids: impl IntoIterator + Send> + Send, outpoints: impl IntoIterator + Send> + Send, parallel_requests: usize, - ) -> Result, Error>; + ) -> Result; } #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] #[cfg_attr(not(target_arch = "wasm32"), async_trait)] impl EsploraAsyncExt for esplora_client::AsyncClient { - async fn update_local_chain( - &self, - local_tip: CheckPoint, - request_heights: impl IntoIterator + Send> + Send, - ) -> Result { - // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are - // consistent. - let mut fetched_blocks = self - .get_blocks(None) - .await? - .into_iter() - .map(|b| (b.time.height, b.id)) - .collect::>(); - let new_tip_height = fetched_blocks - .keys() - .last() - .copied() - .expect("must have atleast one block"); - - // Fetch blocks of heights that the caller is interested in, skipping blocks that are - // already fetched when constructing `fetched_blocks`. - for height in request_heights { - // do not fetch blocks higher than remote tip - if height > new_tip_height { - continue; - } - // only fetch what is missing - if let btree_map::Entry::Vacant(entry) = fetched_blocks.entry(height) { - // ❗The return value of `get_block_hash` is not strictly guaranteed to be consistent - // with the chain at the time of `get_blocks` above (there could have been a deep - // re-org). Since `get_blocks` returns 10 (or so) blocks we are assuming that it's - // not possible to have a re-org deeper than that. - entry.insert(self.get_block_hash(height).await?); - } - } - - // Ensure `fetched_blocks` can create an update that connects with the original chain by - // finding a "Point of Agreement". - for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { - if height > new_tip_height { - continue; - } - - let fetched_hash = match fetched_blocks.entry(height) { - btree_map::Entry::Occupied(entry) => *entry.get(), - btree_map::Entry::Vacant(entry) => { - *entry.insert(self.get_block_hash(height).await?) - } - }; - - // We have found point of agreement so the update will connect! - if fetched_hash == local_hash { - break; - } - } - - Ok(local_chain::Update { - tip: CheckPoint::from_block_ids(fetched_blocks.into_iter().map(BlockId::from)) - .expect("must be in height order"), - introduce_older_blocks: true, - }) - } - async fn full_scan( &self, + local_tip: CheckPoint, keychain_spks: BTreeMap< K, impl IntoIterator + Send> + Send, >, stop_gap: usize, parallel_requests: usize, - ) -> Result<(TxGraph, BTreeMap), Error> { - type TxsOfSpkIndex = (u32, Vec); - let parallel_requests = Ord::max(parallel_requests, 1); - let mut graph = TxGraph::::default(); - let mut last_active_indexes = BTreeMap::::new(); - let stop_gap = Ord::max(stop_gap, 1); - - for (keychain, spks) in keychain_spks { - let mut spks = spks.into_iter(); - let mut last_index = Option::::None; - let mut last_active_index = Option::::None; - - loop { - let handles = spks - .by_ref() - .take(parallel_requests) - .map(|(spk_index, spk)| { - let client = self.clone(); - async move { - let mut last_seen = None; - let mut spk_txs = Vec::new(); - loop { - let txs = client.scripthash_txs(&spk, last_seen).await?; - let tx_count = txs.len(); - last_seen = txs.last().map(|tx| tx.txid); - spk_txs.extend(txs); - if tx_count < 25 { - break Result::<_, Error>::Ok((spk_index, spk_txs)); - } - } - } - }) - .collect::>(); + ) -> Result, Error> { + let update_blocks = init_chain_update(self, &local_tip).await?; + let (tx_graph, last_active_indices) = + full_scan_for_index_and_graph(self, keychain_spks, stop_gap, parallel_requests).await?; + let local_chain = + finalize_chain_update(self, &local_tip, tx_graph.all_anchors(), update_blocks).await?; + Ok(FullScanUpdate { + local_chain, + tx_graph, + last_active_indices, + }) + } - if handles.is_empty() { - break; - } + async fn sync( + &self, + local_tip: CheckPoint, + misc_spks: impl IntoIterator + Send> + Send, + txids: impl IntoIterator + Send> + Send, + outpoints: impl IntoIterator + Send> + Send, + parallel_requests: usize, + ) -> Result { + let update_blocks = init_chain_update(self, &local_tip).await?; + let tx_graph = + sync_for_index_and_graph(self, misc_spks, txids, outpoints, parallel_requests).await?; + let local_chain = + finalize_chain_update(self, &local_tip, tx_graph.all_anchors(), update_blocks).await?; + Ok(SyncUpdate { + tx_graph, + local_chain, + }) + } +} - for (index, txs) in handles.try_collect::>().await? { - last_index = Some(index); - if !txs.is_empty() { - last_active_index = Some(index); - } - for tx in txs { - let _ = graph.insert_tx(tx.to_tx()); - if let Some(anchor) = anchor_from_status(&tx.status) { - let _ = graph.insert_anchor(tx.txid, anchor); - } +/// Create the initial chain update. +/// +/// This atomically fetches the latest blocks from Esplora and additional blocks to ensure the +/// update can connect to the `start_tip`. +/// +/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks and +/// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for +/// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use +/// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when +/// alternating between chain-sources. +#[doc(hidden)] +pub async fn init_chain_update( + client: &esplora_client::AsyncClient, + local_tip: &CheckPoint, +) -> Result, Error> { + // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are + // consistent. + let mut fetched_blocks = client + .get_blocks(None) + .await? + .into_iter() + .map(|b| (b.time.height, b.id)) + .collect::>(); + let new_tip_height = fetched_blocks + .keys() + .last() + .copied() + .expect("must atleast have one block"); - let previous_outputs = tx.vin.iter().filter_map(|vin| { - let prevout = vin.prevout.as_ref()?; - Some(( - OutPoint { - txid: vin.txid, - vout: vin.vout, - }, - TxOut { - script_pubkey: prevout.scriptpubkey.clone(), - value: Amount::from_sat(prevout.value), - }, - )) - }); - - for (outpoint, txout) in previous_outputs { - let _ = graph.insert_txout(outpoint, txout); - } - } - } + // Ensure `fetched_blocks` can create an update that connects with the original chain by + // finding a "Point of Agreement". + for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { + if height > new_tip_height { + continue; + } - let last_index = last_index.expect("Must be set since handles wasn't empty."); - let gap_limit_reached = if let Some(i) = last_active_index { - last_index >= i.saturating_add(stop_gap as u32) - } else { - last_index + 1 >= stop_gap as u32 - }; - if gap_limit_reached { - break; - } + let fetched_hash = match fetched_blocks.entry(height) { + btree_map::Entry::Occupied(entry) => *entry.get(), + btree_map::Entry::Vacant(entry) => *entry.insert(client.get_block_hash(height).await?), + }; + + // We have found point of agreement so the update will connect! + if fetched_hash == local_hash { + break; + } + } + + Ok(fetched_blocks) +} + +/// Fetches missing checkpoints and finalizes the [`local_chain::Update`]. +/// +/// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an +/// existing checkpoint/block under `local_tip` or `update_blocks`. +#[doc(hidden)] +pub async fn finalize_chain_update( + client: &esplora_client::AsyncClient, + local_tip: &CheckPoint, + anchors: &BTreeSet<(A, Txid)>, + mut update_blocks: BTreeMap, +) -> Result { + let update_tip_height = update_blocks + .keys() + .last() + .copied() + .expect("must atleast have one block"); + + // We want to have a corresponding checkpoint per height. We iterate the heights of anchors + // backwards, comparing it against our `local_tip`'s chain and our current set of + // `update_blocks` to see if a corresponding checkpoint already exists. + let anchor_heights = anchors + .iter() + .rev() + .map(|(a, _)| a.anchor_block().height) + // filter out heights that surpass the update tip + .filter(|h| *h <= update_tip_height) + // filter out duplicate heights + .filter({ + let mut prev_height = Option::::None; + move |h| match prev_height.replace(*h) { + None => true, + Some(prev_h) => prev_h != *h, } + }); - if let Some(last_active_index) = last_active_index { - last_active_indexes.insert(keychain, last_active_index); + // We keep track of a checkpoint node of `local_tip` to make traversing the linked-list of + // checkpoints more efficient. + let mut curr_cp = local_tip.clone(); + + for h in anchor_heights { + if let Some(cp) = curr_cp.range(h..).last() { + curr_cp = cp.clone(); + if cp.height() == h { + continue; } } - - Ok((graph, last_active_indexes)) + if let btree_map::Entry::Vacant(entry) = update_blocks.entry(h) { + entry.insert(client.get_block_hash(h).await?); + } } - async fn sync( - &self, - misc_spks: impl IntoIterator + Send> + Send, - txids: impl IntoIterator + Send> + Send, - outpoints: impl IntoIterator + Send> + Send, - parallel_requests: usize, - ) -> Result, Error> { - let mut graph = self - .full_scan( - [( - (), - misc_spks - .into_iter() - .enumerate() - .map(|(i, spk)| (i as u32, spk)), - )] - .into(), - usize::MAX, - parallel_requests, - ) - .await - .map(|(g, _)| g)?; - - let mut txids = txids.into_iter(); + Ok(local_chain::Update { + tip: CheckPoint::from_block_ids( + update_blocks + .into_iter() + .map(|(height, hash)| BlockId { height, hash }), + ) + .expect("must be in order"), + introduce_older_blocks: true, + }) +} + +/// This performs a full scan to get an update for the [`TxGraph`] and +/// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex). +#[doc(hidden)] +pub async fn full_scan_for_index_and_graph( + client: &esplora_client::AsyncClient, + keychain_spks: BTreeMap< + K, + impl IntoIterator + Send> + Send, + >, + stop_gap: usize, + parallel_requests: usize, +) -> Result<(TxGraph, BTreeMap), Error> { + type TxsOfSpkIndex = (u32, Vec); + let parallel_requests = Ord::max(parallel_requests, 1); + let mut graph = TxGraph::::default(); + let mut last_active_indexes = BTreeMap::::new(); + + for (keychain, spks) in keychain_spks { + let mut spks = spks.into_iter(); + let mut last_index = Option::::None; + let mut last_active_index = Option::::None; + loop { - let handles = txids + let handles = spks .by_ref() .take(parallel_requests) - .filter(|&txid| graph.get_tx(txid).is_none()) - .map(|txid| { - let client = self.clone(); - async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) } + .map(|(spk_index, spk)| { + let client = client.clone(); + async move { + let mut last_seen = None; + let mut spk_txs = Vec::new(); + loop { + let txs = client.scripthash_txs(&spk, last_seen).await?; + let tx_count = txs.len(); + last_seen = txs.last().map(|tx| tx.txid); + spk_txs.extend(txs); + if tx_count < 25 { + break Result::<_, Error>::Ok((spk_index, spk_txs)); + } + } + } }) .collect::>(); @@ -297,38 +289,128 @@ impl EsploraAsyncExt for esplora_client::AsyncClient { break; } - for (txid, status) in handles.try_collect::>().await? { - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(txid, anchor); + for (index, txs) in handles.try_collect::>().await? { + last_index = Some(index); + if !txs.is_empty() { + last_active_index = Some(index); + } + for tx in txs { + let _ = graph.insert_tx(tx.to_tx()); + if let Some(anchor) = anchor_from_status(&tx.status) { + let _ = graph.insert_anchor(tx.txid, anchor); + } + + let previous_outputs = tx.vin.iter().filter_map(|vin| { + let prevout = vin.prevout.as_ref()?; + Some(( + OutPoint { + txid: vin.txid, + vout: vin.vout, + }, + TxOut { + script_pubkey: prevout.scriptpubkey.clone(), + value: Amount::from_sat(prevout.value), + }, + )) + }); + + for (outpoint, txout) in previous_outputs { + let _ = graph.insert_txout(outpoint, txout); + } } } + + let last_index = last_index.expect("Must be set since handles wasn't empty."); + let gap_limit_reached = if let Some(i) = last_active_index { + last_index >= i.saturating_add(stop_gap as u32) + } else { + last_index + 1 >= stop_gap as u32 + }; + if gap_limit_reached { + break; + } } - for op in outpoints.into_iter() { - if graph.get_tx(op.txid).is_none() { - if let Some(tx) = self.get_tx(&op.txid).await? { - let _ = graph.insert_tx(tx); - } - let status = self.get_tx_status(&op.txid).await?; - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(op.txid, anchor); - } + if let Some(last_active_index) = last_active_index { + last_active_indexes.insert(keychain, last_active_index); + } + } + + Ok((graph, last_active_indexes)) +} + +#[doc(hidden)] +pub async fn sync_for_index_and_graph( + client: &esplora_client::AsyncClient, + misc_spks: impl IntoIterator + Send> + Send, + txids: impl IntoIterator + Send> + Send, + outpoints: impl IntoIterator + Send> + Send, + parallel_requests: usize, +) -> Result, Error> { + let mut graph = full_scan_for_index_and_graph( + client, + [( + (), + misc_spks + .into_iter() + .enumerate() + .map(|(i, spk)| (i as u32, spk)), + )] + .into(), + usize::MAX, + parallel_requests, + ) + .await + .map(|(g, _)| g)?; + + let mut txids = txids.into_iter(); + loop { + let handles = txids + .by_ref() + .take(parallel_requests) + .filter(|&txid| graph.get_tx(txid).is_none()) + .map(|txid| { + let client = client.clone(); + async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) } + }) + .collect::>(); + + if handles.is_empty() { + break; + } + + for (txid, status) in handles.try_collect::>().await? { + if let Some(anchor) = anchor_from_status(&status) { + let _ = graph.insert_anchor(txid, anchor); } + } + } - if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _).await? { - if let Some(txid) = op_status.txid { - if graph.get_tx(txid).is_none() { - if let Some(tx) = self.get_tx(&txid).await? { - let _ = graph.insert_tx(tx); - } - let status = self.get_tx_status(&txid).await?; - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(txid, anchor); - } + for op in outpoints.into_iter() { + if graph.get_tx(op.txid).is_none() { + if let Some(tx) = client.get_tx(&op.txid).await? { + let _ = graph.insert_tx(tx); + } + let status = client.get_tx_status(&op.txid).await?; + if let Some(anchor) = anchor_from_status(&status) { + let _ = graph.insert_anchor(op.txid, anchor); + } + } + + if let Some(op_status) = client.get_output_status(&op.txid, op.vout as _).await? { + if let Some(txid) = op_status.txid { + if graph.get_tx(txid).is_none() { + if let Some(tx) = client.get_tx(&txid).await? { + let _ = graph.insert_tx(tx); + } + let status = client.get_tx_status(&txid).await?; + if let Some(anchor) = anchor_from_status(&status) { + let _ = graph.insert_anchor(txid, anchor); } } } } - Ok(graph) } + + Ok(graph) } diff --git a/crates/esplora/src/blocking_ext.rs b/crates/esplora/src/blocking_ext.rs index 52aefedcb..adfd33c09 100644 --- a/crates/esplora/src/blocking_ext.rs +++ b/crates/esplora/src/blocking_ext.rs @@ -1,7 +1,10 @@ +use std::collections::BTreeSet; use std::thread::JoinHandle; +use std::usize; use bdk_chain::collections::btree_map; use bdk_chain::collections::BTreeMap; +use bdk_chain::Anchor; use bdk_chain::{ bitcoin::{Amount, BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, local_chain::{self, CheckPoint}, @@ -10,9 +13,11 @@ use bdk_chain::{ use esplora_client::TxStatus; use crate::anchor_from_status; +use crate::FullScanUpdate; +use crate::SyncUpdate; /// [`esplora_client::Error`] -type Error = Box; +pub type Error = Box; /// Trait to extend the functionality of [`esplora_client::BlockingClient`]. /// @@ -20,36 +25,15 @@ type Error = Box; /// /// [crate-level documentation]: crate pub trait EsploraExt { - /// Prepare a [`LocalChain`] update with blocks fetched from Esplora. - /// - /// * `local_tip` is the previous tip of [`LocalChain::tip`]. - /// * `request_heights` is the block heights that we are interested in fetching from Esplora. - /// - /// The result of this method can be applied to [`LocalChain::apply_update`]. - /// - /// ## Consistency - /// - /// The chain update returned is guaranteed to be consistent as long as there is not a *large* re-org - /// during the call. The size of re-org we can tollerate is server dependent but will be at - /// least 10. - /// - /// [`LocalChain`]: bdk_chain::local_chain::LocalChain - /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip - /// [`LocalChain::apply_update`]: bdk_chain::local_chain::LocalChain::apply_update - fn update_local_chain( - &self, - local_tip: CheckPoint, - request_heights: impl IntoIterator, - ) -> Result; - - /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and - /// returns a [`TxGraph`] and a map of last active indices. + /// Scan keychain scripts for transactions against Esplora, returning an update that can be + /// applied to the receiving structures. /// + /// * `local_tip`: the previously seen tip from [`LocalChain::tip`]. /// * `keychain_spks`: keychains that we want to scan transactions for /// - /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated - /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in - /// parallel. + /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no + /// associated transactions. `parallel_requests` specifies the max number of HTTP requests to + /// make in parallel. /// /// ## Note /// @@ -63,16 +47,20 @@ pub trait EsploraExt { /// and [Sparrow](https://www.sparrowwallet.com/docs/faq.html#ive-restored-my-wallet-but-some-of-my-funds-are-missing). /// /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1. + /// + /// [`LocalChain::tip`]: local_chain::LocalChain::tip fn full_scan( &self, + local_tip: CheckPoint, keychain_spks: BTreeMap>, stop_gap: usize, parallel_requests: usize, - ) -> Result<(TxGraph, BTreeMap), Error>; + ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Esplora client) for the data /// specified and return a [`TxGraph`]. /// + /// * `local_tip`: the previously seen tip from [`LocalChain::tip`]. /// * `misc_spks`: scripts that we want to sync transactions for /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we @@ -81,251 +69,365 @@ pub trait EsploraExt { /// If the scripts to sync are unknown, such as when restoring or importing a keychain that /// may include scripts that have been used, use [`full_scan`] with the keychain. /// + /// [`LocalChain::tip`]: local_chain::LocalChain::tip /// [`full_scan`]: EsploraExt::full_scan fn sync( &self, + local_tip: CheckPoint, misc_spks: impl IntoIterator, txids: impl IntoIterator, outpoints: impl IntoIterator, parallel_requests: usize, - ) -> Result, Error>; + ) -> Result; } impl EsploraExt for esplora_client::BlockingClient { - fn update_local_chain( + fn full_scan( &self, local_tip: CheckPoint, - request_heights: impl IntoIterator, - ) -> Result { - // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are - // consistent. - let mut fetched_blocks = self - .get_blocks(None)? - .into_iter() - .map(|b| (b.time.height, b.id)) - .collect::>(); - let new_tip_height = fetched_blocks - .keys() - .last() - .copied() - .expect("must atleast have one block"); - - // Fetch blocks of heights that the caller is interested in, skipping blocks that are - // already fetched when constructing `fetched_blocks`. - for height in request_heights { - // do not fetch blocks higher than remote tip - if height > new_tip_height { - continue; - } - // only fetch what is missing - if let btree_map::Entry::Vacant(entry) = fetched_blocks.entry(height) { - // ❗The return value of `get_block_hash` is not strictly guaranteed to be consistent - // with the chain at the time of `get_blocks` above (there could have been a deep - // re-org). Since `get_blocks` returns 10 (or so) blocks we are assuming that it's - // not possible to have a re-org deeper than that. - entry.insert(self.get_block_hash(height)?); - } - } - - // Ensure `fetched_blocks` can create an update that connects with the original chain by - // finding a "Point of Agreement". - for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { - if height > new_tip_height { - continue; - } - - let fetched_hash = match fetched_blocks.entry(height) { - btree_map::Entry::Occupied(entry) => *entry.get(), - btree_map::Entry::Vacant(entry) => *entry.insert(self.get_block_hash(height)?), - }; - - // We have found point of agreement so the update will connect! - if fetched_hash == local_hash { - break; - } - } - - Ok(local_chain::Update { - tip: CheckPoint::from_block_ids(fetched_blocks.into_iter().map(BlockId::from)) - .expect("must be in height order"), - introduce_older_blocks: true, + keychain_spks: BTreeMap>, + stop_gap: usize, + parallel_requests: usize, + ) -> Result, Error> { + let update_blocks = init_chain_update_blocking(self, &local_tip)?; + let (tx_graph, last_active_indices) = full_scan_for_index_and_graph_blocking( + self, + keychain_spks, + stop_gap, + parallel_requests, + )?; + let local_chain = finalize_chain_update_blocking( + self, + &local_tip, + tx_graph.all_anchors(), + update_blocks, + )?; + Ok(FullScanUpdate { + local_chain, + tx_graph, + last_active_indices, }) } - fn full_scan( + fn sync( &self, - keychain_spks: BTreeMap>, - stop_gap: usize, + local_tip: CheckPoint, + misc_spks: impl IntoIterator, + txids: impl IntoIterator, + outpoints: impl IntoIterator, parallel_requests: usize, - ) -> Result<(TxGraph, BTreeMap), Error> { - type TxsOfSpkIndex = (u32, Vec); - let parallel_requests = Ord::max(parallel_requests, 1); - let mut graph = TxGraph::::default(); - let mut last_active_indexes = BTreeMap::::new(); - let stop_gap = Ord::max(stop_gap, 1); - - for (keychain, spks) in keychain_spks { - let mut spks = spks.into_iter(); - let mut last_index = Option::::None; - let mut last_active_index = Option::::None; - - loop { - let handles = spks - .by_ref() - .take(parallel_requests) - .map(|(spk_index, spk)| { - std::thread::spawn({ - let client = self.clone(); - move || -> Result { - let mut last_seen = None; - let mut spk_txs = Vec::new(); - loop { - let txs = client.scripthash_txs(&spk, last_seen)?; - let tx_count = txs.len(); - last_seen = txs.last().map(|tx| tx.txid); - spk_txs.extend(txs); - if tx_count < 25 { - break Ok((spk_index, spk_txs)); - } - } - } - }) - }) - .collect::>>>(); + ) -> Result { + let update_blocks = init_chain_update_blocking(self, &local_tip)?; + let tx_graph = sync_for_index_and_graph_blocking( + self, + misc_spks, + txids, + outpoints, + parallel_requests, + )?; + let local_chain = finalize_chain_update_blocking( + self, + &local_tip, + tx_graph.all_anchors(), + update_blocks, + )?; + Ok(SyncUpdate { + local_chain, + tx_graph, + }) + } +} - if handles.is_empty() { - break; - } +/// Create the initial chain update. +/// +/// This atomically fetches the latest blocks from Esplora and additional blocks to ensure the +/// update can connect to the `start_tip`. +/// +/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks and +/// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for +/// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use +/// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when +/// alternating between chain-sources. +#[doc(hidden)] +pub fn init_chain_update_blocking( + client: &esplora_client::BlockingClient, + local_tip: &CheckPoint, +) -> Result, Error> { + // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are + // consistent. + let mut fetched_blocks = client + .get_blocks(None)? + .into_iter() + .map(|b| (b.time.height, b.id)) + .collect::>(); + let new_tip_height = fetched_blocks + .keys() + .last() + .copied() + .expect("must atleast have one block"); - for handle in handles { - let (index, txs) = handle.join().expect("thread must not panic")?; - last_index = Some(index); - if !txs.is_empty() { - last_active_index = Some(index); - } - for tx in txs { - let _ = graph.insert_tx(tx.to_tx()); - if let Some(anchor) = anchor_from_status(&tx.status) { - let _ = graph.insert_anchor(tx.txid, anchor); - } + // Ensure `fetched_blocks` can create an update that connects with the original chain by + // finding a "Point of Agreement". + for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { + if height > new_tip_height { + continue; + } - let previous_outputs = tx.vin.iter().filter_map(|vin| { - let prevout = vin.prevout.as_ref()?; - Some(( - OutPoint { - txid: vin.txid, - vout: vin.vout, - }, - TxOut { - script_pubkey: prevout.scriptpubkey.clone(), - value: Amount::from_sat(prevout.value), - }, - )) - }); - - for (outpoint, txout) in previous_outputs { - let _ = graph.insert_txout(outpoint, txout); - } - } - } + let fetched_hash = match fetched_blocks.entry(height) { + btree_map::Entry::Occupied(entry) => *entry.get(), + btree_map::Entry::Vacant(entry) => *entry.insert(client.get_block_hash(height)?), + }; - let last_index = last_index.expect("Must be set since handles wasn't empty."); - let gap_limit_reached = if let Some(i) = last_active_index { - last_index >= i.saturating_add(stop_gap as u32) - } else { - last_index + 1 >= stop_gap as u32 - }; - if gap_limit_reached { - break; - } + // We have found point of agreement so the update will connect! + if fetched_hash == local_hash { + break; + } + } + + Ok(fetched_blocks) +} + +/// Fetches missing checkpoints and finalizes the [`local_chain::Update`]. +/// +/// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an +/// existing checkpoint/block under `local_tip` or `update_blocks`. +#[doc(hidden)] +pub fn finalize_chain_update_blocking( + client: &esplora_client::BlockingClient, + local_tip: &CheckPoint, + anchors: &BTreeSet<(A, Txid)>, + mut update_blocks: BTreeMap, +) -> Result { + let update_tip_height = update_blocks + .keys() + .last() + .copied() + .expect("must atleast have one block"); + + // We want to have a corresponding checkpoint per height. We iterate the heights of anchors + // backwards, comparing it against our `local_tip`'s chain and our current set of + // `update_blocks` to see if a corresponding checkpoint already exists. + let anchor_heights = anchors + .iter() + .rev() + .map(|(a, _)| a.anchor_block().height) + // filter out heights that surpass the update tip + .filter(|h| *h <= update_tip_height) + // filter out duplicate heights + .filter({ + let mut prev_height = Option::::None; + move |h| match prev_height.replace(*h) { + None => true, + Some(prev_h) => prev_h != *h, } + }); + + // We keep track of a checkpoint node of `local_tip` to make traversing the linked-list of + // checkpoints more efficient. + let mut curr_cp = local_tip.clone(); - if let Some(last_active_index) = last_active_index { - last_active_indexes.insert(keychain, last_active_index); + for h in anchor_heights { + if let Some(cp) = curr_cp.range(h..).last() { + curr_cp = cp.clone(); + if cp.height() == h { + continue; } } - - Ok((graph, last_active_indexes)) + if let btree_map::Entry::Vacant(entry) = update_blocks.entry(h) { + entry.insert(client.get_block_hash(h)?); + } } - fn sync( - &self, - misc_spks: impl IntoIterator, - txids: impl IntoIterator, - outpoints: impl IntoIterator, - parallel_requests: usize, - ) -> Result, Error> { - let mut graph = self - .full_scan( - [( - (), - misc_spks - .into_iter() - .enumerate() - .map(|(i, spk)| (i as u32, spk)), - )] - .into(), - usize::MAX, - parallel_requests, - ) - .map(|(g, _)| g)?; - - let mut txids = txids.into_iter(); + Ok(local_chain::Update { + tip: CheckPoint::from_block_ids( + update_blocks + .into_iter() + .map(|(height, hash)| BlockId { height, hash }), + ) + .expect("must be in order"), + introduce_older_blocks: true, + }) +} + +/// This performs a full scan to get an update for the [`TxGraph`] and +/// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex). +#[doc(hidden)] +pub fn full_scan_for_index_and_graph_blocking( + client: &esplora_client::BlockingClient, + keychain_spks: BTreeMap>, + stop_gap: usize, + parallel_requests: usize, +) -> Result<(TxGraph, BTreeMap), Error> { + type TxsOfSpkIndex = (u32, Vec); + let parallel_requests = Ord::max(parallel_requests, 1); + let mut tx_graph = TxGraph::::default(); + let mut last_active_indices = BTreeMap::::new(); + + for (keychain, spks) in keychain_spks { + let mut spks = spks.into_iter(); + let mut last_index = Option::::None; + let mut last_active_index = Option::::None; + loop { - let handles = txids + let handles = spks .by_ref() .take(parallel_requests) - .filter(|&txid| graph.get_tx(txid).is_none()) - .map(|txid| { + .map(|(spk_index, spk)| { std::thread::spawn({ - let client = self.clone(); - move || { - client - .get_tx_status(&txid) - .map_err(Box::new) - .map(|s| (txid, s)) + let client = client.clone(); + move || -> Result { + let mut last_seen = None; + let mut spk_txs = Vec::new(); + loop { + let txs = client.scripthash_txs(&spk, last_seen)?; + let tx_count = txs.len(); + last_seen = txs.last().map(|tx| tx.txid); + spk_txs.extend(txs); + if tx_count < 25 { + break Ok((spk_index, spk_txs)); + } + } } }) }) - .collect::>>>(); + .collect::>>>(); if handles.is_empty() { break; } for handle in handles { - let (txid, status) = handle.join().expect("thread must not panic")?; - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(txid, anchor); + let (index, txs) = handle.join().expect("thread must not panic")?; + last_index = Some(index); + if !txs.is_empty() { + last_active_index = Some(index); + } + for tx in txs { + let _ = tx_graph.insert_tx(tx.to_tx()); + if let Some(anchor) = anchor_from_status(&tx.status) { + let _ = tx_graph.insert_anchor(tx.txid, anchor); + } + + let previous_outputs = tx.vin.iter().filter_map(|vin| { + let prevout = vin.prevout.as_ref()?; + Some(( + OutPoint { + txid: vin.txid, + vout: vin.vout, + }, + TxOut { + script_pubkey: prevout.scriptpubkey.clone(), + value: Amount::from_sat(prevout.value), + }, + )) + }); + + for (outpoint, txout) in previous_outputs { + let _ = tx_graph.insert_txout(outpoint, txout); + } } } + + let last_index = last_index.expect("Must be set since handles wasn't empty."); + let gap_limit_reached = if let Some(i) = last_active_index { + last_index >= i.saturating_add(stop_gap as u32) + } else { + last_index + 1 >= stop_gap as u32 + }; + if gap_limit_reached { + break; + } } - for op in outpoints { - if graph.get_tx(op.txid).is_none() { - if let Some(tx) = self.get_tx(&op.txid)? { - let _ = graph.insert_tx(tx); - } - let status = self.get_tx_status(&op.txid)?; - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(op.txid, anchor); - } + if let Some(last_active_index) = last_active_index { + last_active_indices.insert(keychain, last_active_index); + } + } + + Ok((tx_graph, last_active_indices)) +} + +#[doc(hidden)] +pub fn sync_for_index_and_graph_blocking( + client: &esplora_client::BlockingClient, + misc_spks: impl IntoIterator, + txids: impl IntoIterator, + outpoints: impl IntoIterator, + parallel_requests: usize, +) -> Result, Error> { + let (mut tx_graph, _) = full_scan_for_index_and_graph_blocking( + client, + { + let mut keychains = BTreeMap::new(); + keychains.insert( + (), + misc_spks + .into_iter() + .enumerate() + .map(|(i, spk)| (i as u32, spk)), + ); + keychains + }, + usize::MAX, + parallel_requests, + )?; + + let mut txids = txids.into_iter(); + loop { + let handles = txids + .by_ref() + .take(parallel_requests) + .filter(|&txid| tx_graph.get_tx(txid).is_none()) + .map(|txid| { + std::thread::spawn({ + let client = client.clone(); + move || { + client + .get_tx_status(&txid) + .map_err(Box::new) + .map(|s| (txid, s)) + } + }) + }) + .collect::>>>(); + + if handles.is_empty() { + break; + } + + for handle in handles { + let (txid, status) = handle.join().expect("thread must not panic")?; + if let Some(anchor) = anchor_from_status(&status) { + let _ = tx_graph.insert_anchor(txid, anchor); } + } + } - if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _)? { - if let Some(txid) = op_status.txid { - if graph.get_tx(txid).is_none() { - if let Some(tx) = self.get_tx(&txid)? { - let _ = graph.insert_tx(tx); - } - let status = self.get_tx_status(&txid)?; - if let Some(anchor) = anchor_from_status(&status) { - let _ = graph.insert_anchor(txid, anchor); - } + for op in outpoints { + if tx_graph.get_tx(op.txid).is_none() { + if let Some(tx) = client.get_tx(&op.txid)? { + let _ = tx_graph.insert_tx(tx); + } + let status = client.get_tx_status(&op.txid)?; + if let Some(anchor) = anchor_from_status(&status) { + let _ = tx_graph.insert_anchor(op.txid, anchor); + } + } + + if let Some(op_status) = client.get_output_status(&op.txid, op.vout as _)? { + if let Some(txid) = op_status.txid { + if tx_graph.get_tx(txid).is_none() { + if let Some(tx) = client.get_tx(&txid)? { + let _ = tx_graph.insert_tx(tx); + } + let status = client.get_tx_status(&txid)?; + if let Some(anchor) = anchor_from_status(&status) { + let _ = tx_graph.insert_anchor(txid, anchor); } } } } - Ok(graph) } + + Ok(tx_graph) } diff --git a/crates/esplora/src/lib.rs b/crates/esplora/src/lib.rs index 535167ff2..c422a0833 100644 --- a/crates/esplora/src/lib.rs +++ b/crates/esplora/src/lib.rs @@ -16,7 +16,9 @@ //! [`TxGraph`]: bdk_chain::tx_graph::TxGraph //! [`example_esplora`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_esplora -use bdk_chain::{BlockId, ConfirmationTimeHeightAnchor}; +use std::collections::BTreeMap; + +use bdk_chain::{local_chain, BlockId, ConfirmationTimeHeightAnchor, TxGraph}; use esplora_client::TxStatus; pub use esplora_client; @@ -48,3 +50,21 @@ fn anchor_from_status(status: &TxStatus) -> Option None } } + +/// Update returns from a full scan. +pub struct FullScanUpdate { + /// The update to apply to the receiving [`LocalChain`](local_chain::LocalChain). + pub local_chain: local_chain::Update, + /// The update to apply to the receiving [`TxGraph`]. + pub tx_graph: TxGraph, + /// Last active indices for the corresponding keychains (`K`). + pub last_active_indices: BTreeMap, +} + +/// Update returned from a sync. +pub struct SyncUpdate { + /// The update to apply to the receiving [`LocalChain`](local_chain::LocalChain). + pub local_chain: local_chain::Update, + /// The update to apply to the receiving [`TxGraph`]. + pub tx_graph: TxGraph, +} diff --git a/crates/esplora/tests/async_ext.rs b/crates/esplora/tests/async_ext.rs index c71c214e9..5946bb4d8 100644 --- a/crates/esplora/tests/async_ext.rs +++ b/crates/esplora/tests/async_ext.rs @@ -2,7 +2,7 @@ use bdk_esplora::EsploraAsyncExt; use electrsd::bitcoind::anyhow; use electrsd::bitcoind::bitcoincore_rpc::RpcApi; use esplora_client::{self, Builder}; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::str::FromStr; use std::thread::sleep; use std::time::Duration; @@ -52,8 +52,12 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { sleep(Duration::from_millis(10)) } - let graph_update = client + // use a full checkpoint linked list (since this is not what we are testing) + let cp_tip = env.make_checkpoint_tip(); + + let sync_update = client .sync( + cp_tip.clone(), misc_spks.into_iter(), vec![].into_iter(), vec![].into_iter(), @@ -61,6 +65,24 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { ) .await?; + assert!( + { + let update_cps = sync_update + .local_chain + .tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + let superset_cps = cp_tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + superset_cps.is_superset(&update_cps) + }, + "update should not alter original checkpoint tip since we already started with all checkpoints", + ); + + let graph_update = sync_update.tx_graph; // Check to see if we have the floating txouts available from our two created transactions' // previous outputs in order to calculate transaction fees. for tx in graph_update.full_txs() { @@ -140,14 +162,24 @@ pub async fn test_async_update_tx_graph_stop_gap() -> anyhow::Result<()> { sleep(Duration::from_millis(10)) } + // use a full checkpoint linked list (since this is not what we are testing) + let cp_tip = env.make_checkpoint_tip(); + // A scan with a gap limit of 3 won't find the transaction, but a scan with a gap limit of 4 // will. - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 3, 1).await?; - assert!(graph_update.full_txs().next().is_none()); - assert!(active_indices.is_empty()); - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 4, 1).await?; - assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr); - assert_eq!(active_indices[&0], 3); + let full_scan_update = client + .full_scan(cp_tip.clone(), keychains.clone(), 3, 1) + .await?; + assert!(full_scan_update.tx_graph.full_txs().next().is_none()); + assert!(full_scan_update.last_active_indices.is_empty()); + let full_scan_update = client + .full_scan(cp_tip.clone(), keychains.clone(), 4, 1) + .await?; + assert_eq!( + full_scan_update.tx_graph.full_txs().next().unwrap().txid, + txid_4th_addr + ); + assert_eq!(full_scan_update.last_active_indices[&0], 3); // Now receive a coin on the last address. let txid_last_addr = env.bitcoind.client.send_to_address( @@ -167,16 +199,26 @@ pub async fn test_async_update_tx_graph_stop_gap() -> anyhow::Result<()> { // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will. // The last active indice won't be updated in the first case but will in the second one. - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 5, 1).await?; - let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect(); + let full_scan_update = client + .full_scan(cp_tip.clone(), keychains.clone(), 5, 1) + .await?; + let txs: HashSet<_> = full_scan_update + .tx_graph + .full_txs() + .map(|tx| tx.txid) + .collect(); assert_eq!(txs.len(), 1); assert!(txs.contains(&txid_4th_addr)); - assert_eq!(active_indices[&0], 3); - let (graph_update, active_indices) = client.full_scan(keychains, 6, 1).await?; - let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect(); + assert_eq!(full_scan_update.last_active_indices[&0], 3); + let full_scan_update = client.full_scan(cp_tip, keychains, 6, 1).await?; + let txs: HashSet<_> = full_scan_update + .tx_graph + .full_txs() + .map(|tx| tx.txid) + .collect(); assert_eq!(txs.len(), 2); assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr)); - assert_eq!(active_indices[&0], 9); + assert_eq!(full_scan_update.last_active_indices[&0], 9); Ok(()) } diff --git a/crates/esplora/tests/blocking_ext.rs b/crates/esplora/tests/blocking_ext.rs index 35e38a778..d35fab658 100644 --- a/crates/esplora/tests/blocking_ext.rs +++ b/crates/esplora/tests/blocking_ext.rs @@ -3,7 +3,7 @@ use bdk_chain::BlockId; use bdk_esplora::EsploraExt; use electrsd::bitcoind::anyhow; use electrsd::bitcoind::bitcoincore_rpc::RpcApi; -use esplora_client::{self, Builder}; +use esplora_client::{self, BlockHash, Builder}; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::str::FromStr; use std::thread::sleep; @@ -68,13 +68,35 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { sleep(Duration::from_millis(10)) } - let graph_update = client.sync( + // use a full checkpoint linked list (since this is not what we are testing) + let cp_tip = env.make_checkpoint_tip(); + + let sync_update = client.sync( + cp_tip.clone(), misc_spks.into_iter(), vec![].into_iter(), vec![].into_iter(), 1, )?; + assert!( + { + let update_cps = sync_update + .local_chain + .tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + let superset_cps = cp_tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + superset_cps.is_superset(&update_cps) + }, + "update should not alter original checkpoint tip since we already started with all checkpoints", + ); + + let graph_update = sync_update.tx_graph; // Check to see if we have the floating txouts available from our two created transactions' // previous outputs in order to calculate transaction fees. for tx in graph_update.full_txs() { @@ -155,14 +177,20 @@ pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> { sleep(Duration::from_millis(10)) } + // use a full checkpoint linked list (since this is not what we are testing) + let cp_tip = env.make_checkpoint_tip(); + // A scan with a stop_gap of 3 won't find the transaction, but a scan with a gap limit of 4 // will. - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 3, 1)?; - assert!(graph_update.full_txs().next().is_none()); - assert!(active_indices.is_empty()); - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 4, 1)?; - assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr); - assert_eq!(active_indices[&0], 3); + let full_scan_update = client.full_scan(cp_tip.clone(), keychains.clone(), 3, 1)?; + assert!(full_scan_update.tx_graph.full_txs().next().is_none()); + assert!(full_scan_update.last_active_indices.is_empty()); + let full_scan_update = client.full_scan(cp_tip.clone(), keychains.clone(), 4, 1)?; + assert_eq!( + full_scan_update.tx_graph.full_txs().next().unwrap().txid, + txid_4th_addr + ); + assert_eq!(full_scan_update.last_active_indices[&0], 3); // Now receive a coin on the last address. let txid_last_addr = env.bitcoind.client.send_to_address( @@ -182,16 +210,24 @@ pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> { // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will. // The last active indice won't be updated in the first case but will in the second one. - let (graph_update, active_indices) = client.full_scan(keychains.clone(), 5, 1)?; - let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect(); + let full_scan_update = client.full_scan(cp_tip.clone(), keychains.clone(), 5, 1)?; + let txs: HashSet<_> = full_scan_update + .tx_graph + .full_txs() + .map(|tx| tx.txid) + .collect(); assert_eq!(txs.len(), 1); assert!(txs.contains(&txid_4th_addr)); - assert_eq!(active_indices[&0], 3); - let (graph_update, active_indices) = client.full_scan(keychains, 6, 1)?; - let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect(); + assert_eq!(full_scan_update.last_active_indices[&0], 3); + let full_scan_update = client.full_scan(cp_tip.clone(), keychains, 6, 1)?; + let txs: HashSet<_> = full_scan_update + .tx_graph + .full_txs() + .map(|tx| tx.txid) + .collect(); assert_eq!(txs.len(), 2); assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr)); - assert_eq!(active_indices[&0], 9); + assert_eq!(full_scan_update.last_active_indices[&0], 9); Ok(()) } @@ -317,14 +353,38 @@ fn update_local_chain() -> anyhow::Result<()> { for (i, t) in test_cases.into_iter().enumerate() { println!("Case {}: {}", i, t.name); let mut chain = t.chain; + let cp_tip = chain.tip(); - let update = client - .update_local_chain(chain.tip(), t.request_heights.iter().copied()) - .map_err(|err| { - anyhow::format_err!("[{}:{}] `update_local_chain` failed: {}", i, t.name, err) + let new_blocks = + bdk_esplora::init_chain_update_blocking(&client, &cp_tip).map_err(|err| { + anyhow::format_err!("[{}:{}] `init_chain_update` failed: {}", i, t.name, err) })?; - let update_blocks = update + let mock_anchors = t + .request_heights + .iter() + .map(|&h| { + let anchor_blockhash: BlockHash = bdk_chain::bitcoin::hashes::Hash::hash( + &format!("hash_at_height_{}", h).into_bytes(), + ); + let txid: Txid = bdk_chain::bitcoin::hashes::Hash::hash( + &format!("txid_at_height_{}", h).into_bytes(), + ); + let anchor = BlockId { + height: h, + hash: anchor_blockhash, + }; + (anchor, txid) + }) + .collect::>(); + + let chain_update = bdk_esplora::finalize_chain_update_blocking( + &client, + &cp_tip, + &mock_anchors, + new_blocks, + )?; + let update_blocks = chain_update .tip .iter() .map(|cp| cp.block_id()) @@ -346,14 +406,15 @@ fn update_local_chain() -> anyhow::Result<()> { ) .collect::>(); - assert_eq!( - update_blocks, exp_update_blocks, + assert!( + update_blocks.is_superset(&exp_update_blocks), "[{}:{}] unexpected update", - i, t.name + i, + t.name ); let _ = chain - .apply_update(update) + .apply_update(chain_update) .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err)); // all requested heights must exist in the final chain diff --git a/crates/testenv/src/lib.rs b/crates/testenv/src/lib.rs index 4ae6ea6e3..b0147d0fc 100644 --- a/crates/testenv/src/lib.rs +++ b/crates/testenv/src/lib.rs @@ -1,6 +1,8 @@ use bdk_chain::{ bitcoin::{ - address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash, secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget, ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid + address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash, + secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget, + ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid, }, local_chain::CheckPoint, BlockId, diff --git a/example-crates/example_esplora/src/main.rs b/example-crates/example_esplora/src/main.rs index 9232081b6..3aa5b6d80 100644 --- a/example-crates/example_esplora/src/main.rs +++ b/example-crates/example_esplora/src/main.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, io::{self, Write}, sync::Mutex, }; @@ -60,6 +60,7 @@ enum EsploraCommands { esplora_args: EsploraArgs, }, } + impl EsploraCommands { fn esplora_args(&self) -> EsploraArgs { match self { @@ -149,20 +150,24 @@ fn main() -> anyhow::Result<()> { }; let client = esplora_cmd.esplora_args().client(args.network)?; - // Prepare the `IndexedTxGraph` update based on whether we are scanning or syncing. + // Prepare the `IndexedTxGraph` and `LocalChain` updates based on whether we are scanning or + // syncing. + // // Scanning: We are iterating through spks of all keychains and scanning for transactions for // each spk. We start with the lowest derivation index spk and stop scanning after `stop_gap` // number of consecutive spks have no transaction history. A Scan is done in situations of // wallet restoration. It is a special case. Applications should use "sync" style updates // after an initial scan. + // // Syncing: We only check for specified spks, utxos and txids to update their confirmation // status or fetch missing transactions. - let indexed_tx_graph_changeset = match &esplora_cmd { + let (local_chain_changeset, indexed_tx_graph_changeset) = match &esplora_cmd { EsploraCommands::Scan { stop_gap, scan_options, .. } => { + let local_tip = chain.lock().expect("mutex must not be poisoned").tip(); let keychain_spks = graph .lock() .expect("mutex must not be poisoned") @@ -189,23 +194,33 @@ fn main() -> anyhow::Result<()> { // is reached. It returns a `TxGraph` update (`graph_update`) and a structure that // represents the last active spk derivation indices of keychains // (`keychain_indices_update`). - let (mut graph_update, last_active_indices) = client - .full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests) + let mut update = client + .full_scan( + local_tip, + keychain_spks, + *stop_gap, + scan_options.parallel_requests, + ) .context("scanning for transactions")?; // We want to keep track of the latest time a transaction was seen unconfirmed. let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = graph_update.update_last_seen_unconfirmed(now); + let _ = update.tx_graph.update_last_seen_unconfirmed(now); let mut graph = graph.lock().expect("mutex must not be poisoned"); + let mut chain = chain.lock().expect("mutex must not be poisoned"); // Because we did a stop gap based scan we are likely to have some updates to our // deriviation indices. Usually before a scan you are on a fresh wallet with no // addresses derived so we need to derive up to last active addresses the scan found // before adding the transactions. - let (_, index_changeset) = graph.index.reveal_to_target_multi(&last_active_indices); - let mut indexed_tx_graph_changeset = graph.apply_update(graph_update); - indexed_tx_graph_changeset.append(index_changeset.into()); - indexed_tx_graph_changeset + (chain.apply_update(update.local_chain)?, { + let (_, index_changeset) = graph + .index + .reveal_to_target_multi(&update.last_active_indices); + let mut indexed_tx_graph_changeset = graph.apply_update(update.tx_graph); + indexed_tx_graph_changeset.append(index_changeset.into()); + indexed_tx_graph_changeset + }) } EsploraCommands::Sync { mut unused_spks, @@ -231,12 +246,13 @@ fn main() -> anyhow::Result<()> { let mut outpoints: Box> = Box::new(core::iter::empty()); let mut txids: Box> = Box::new(core::iter::empty()); + let local_tip = chain.lock().expect("mutex must not be poisoned").tip(); + // Get a short lock on the structures to get spks, utxos, and txs that we are interested // in. { let graph = graph.lock().unwrap(); let chain = chain.lock().unwrap(); - let chain_tip = chain.tip().block_id(); if *all_spks { let all_spks = graph @@ -276,7 +292,7 @@ fn main() -> anyhow::Result<()> { let init_outpoints = graph.index.outpoints().iter().cloned(); let utxos = graph .graph() - .filter_chain_unspents(&*chain, chain_tip, init_outpoints) + .filter_chain_unspents(&*chain, local_tip.block_id(), init_outpoints) .map(|(_, utxo)| utxo) .collect::>(); outpoints = Box::new( @@ -299,7 +315,7 @@ fn main() -> anyhow::Result<()> { // `EsploraExt::update_tx_graph_without_keychain`. let unconfirmed_txids = graph .graph() - .list_chain_txs(&*chain, chain_tip) + .list_chain_txs(&*chain, local_tip.block_id()) .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed()) .map(|canonical_tx| canonical_tx.tx_node.txid) .collect::>(); @@ -311,48 +327,30 @@ fn main() -> anyhow::Result<()> { } } - let mut graph_update = - client.sync(spks, txids, outpoints, scan_options.parallel_requests)?; + let mut update = client.sync( + local_tip, + spks, + txids, + outpoints, + scan_options.parallel_requests, + )?; // Update last seen unconfirmed let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = graph_update.update_last_seen_unconfirmed(now); + let _ = update.tx_graph.update_last_seen_unconfirmed(now); - graph.lock().unwrap().apply_update(graph_update) + ( + chain.lock().unwrap().apply_update(update.local_chain)?, + graph.lock().unwrap().apply_update(update.tx_graph), + ) } }; println!(); - // Now that we're done updating the `IndexedTxGraph`, it's time to update the `LocalChain`! We - // want the `LocalChain` to have data about all the anchors in the `TxGraph` - for this reason, - // we want retrieve the blocks at the heights of the newly added anchors that are missing from - // our view of the chain. - let (missing_block_heights, tip) = { - let chain = &*chain.lock().unwrap(); - let missing_block_heights = indexed_tx_graph_changeset - .graph - .missing_heights_from(chain) - .collect::>(); - let tip = chain.tip(); - (missing_block_heights, tip) - }; - - println!("prev tip: {}", tip.height()); - println!("missing block heights: {:?}", missing_block_heights); - - // Here, we actually fetch the missing blocks and create a `local_chain::Update`. - let chain_changeset = { - let chain_update = client - .update_local_chain(tip, missing_block_heights) - .context("scanning for blocks")?; - println!("new tip: {}", chain_update.tip.height()); - chain.lock().unwrap().apply_update(chain_update)? - }; - // We persist the changes let mut db = db.lock().unwrap(); - db.stage((chain_changeset, indexed_tx_graph_changeset)); + db.stage((local_chain_changeset, indexed_tx_graph_changeset)); db.commit()?; Ok(()) } diff --git a/example-crates/wallet_esplora_async/src/main.rs b/example-crates/wallet_esplora_async/src/main.rs index 14e9e38dd..5aa10fbca 100644 --- a/example-crates/wallet_esplora_async/src/main.rs +++ b/example-crates/wallet_esplora_async/src/main.rs @@ -53,18 +53,17 @@ async fn main() -> Result<(), anyhow::Error> { (k, k_spks) }) .collect(); - let (mut update_graph, last_active_indices) = client - .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS) - .await?; + let mut update = client + .full_scan(prev_tip, keychain_spks, STOP_GAP, PARALLEL_REQUESTS) + .await?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = update_graph.update_last_seen_unconfirmed(now); - let missing_heights = update_graph.missing_heights(wallet.local_chain()); - let chain_update = client.update_local_chain(prev_tip, missing_heights).await?; + let _ = update.tx_graph.update_last_seen_unconfirmed(now); + let update = Update { - last_active_indices, - graph: update_graph, - chain: Some(chain_update), + last_active_indices: update.last_active_indices, + graph: update.tx_graph, + chain: Some(update.local_chain), }; wallet.apply_update(update)?; wallet.commit()?; diff --git a/example-crates/wallet_esplora_blocking/src/main.rs b/example-crates/wallet_esplora_blocking/src/main.rs index 1815650d4..248c3c6bb 100644 --- a/example-crates/wallet_esplora_blocking/src/main.rs +++ b/example-crates/wallet_esplora_blocking/src/main.rs @@ -36,7 +36,6 @@ fn main() -> Result<(), anyhow::Error> { let client = esplora_client::Builder::new("https://blockstream.info/testnet/api").build_blocking(); - let prev_tip = wallet.latest_checkpoint(); let keychain_spks = wallet .all_unbounded_spk_iters() .into_iter() @@ -53,20 +52,20 @@ fn main() -> Result<(), anyhow::Error> { }) .collect(); - let (mut update_graph, last_active_indices) = - client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?; - + let mut update = client.full_scan( + wallet.latest_checkpoint(), + keychain_spks, + STOP_GAP, + PARALLEL_REQUESTS, + )?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = update_graph.update_last_seen_unconfirmed(now); - let missing_heights = update_graph.missing_heights(wallet.local_chain()); - let chain_update = client.update_local_chain(prev_tip, missing_heights)?; - let update = Update { - last_active_indices, - graph: update_graph, - chain: Some(chain_update), - }; - - wallet.apply_update(update)?; + let _ = update.tx_graph.update_last_seen_unconfirmed(now); + + wallet.apply_update(Update { + last_active_indices: update.last_active_indices, + graph: update.tx_graph, + chain: Some(update.local_chain), + })?; wallet.commit()?; println!(); From 886d72e3d541d088320bbdad6804057f32aca684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 26 Mar 2024 15:11:43 +0800 Subject: [PATCH 03/13] chore(chain)!: rm `missing_heights` and `missing_heights_from` methods These methods are no longer needed as we can determine missing heights directly from the `CheckPoint` tip. --- crates/chain/src/tx_graph.rs | 87 +----------------- crates/chain/tests/test_tx_graph.rs | 133 ---------------------------- 2 files changed, 2 insertions(+), 218 deletions(-) diff --git a/crates/chain/src/tx_graph.rs b/crates/chain/src/tx_graph.rs index b74ebeda7..f6144e7a2 100644 --- a/crates/chain/src/tx_graph.rs +++ b/crates/chain/src/tx_graph.rs @@ -89,8 +89,8 @@ //! [`insert_txout`]: TxGraph::insert_txout use crate::{ - collections::*, keychain::Balance, local_chain::LocalChain, Anchor, Append, BlockId, - ChainOracle, ChainPosition, FullTxOut, + collections::*, keychain::Balance, Anchor, Append, BlockId, ChainOracle, ChainPosition, + FullTxOut, }; use alloc::collections::vec_deque::VecDeque; use alloc::sync::Arc; @@ -759,69 +759,6 @@ impl TxGraph { } impl TxGraph { - /// Find missing block heights of `chain`. - /// - /// This works by scanning through anchors, and seeing whether the anchor block of the anchor - /// exists in the [`LocalChain`]. The returned iterator does not output duplicate heights. - pub fn missing_heights<'a>(&'a self, chain: &'a LocalChain) -> impl Iterator + 'a { - // Map of txids to skip. - // - // Usually, if a height of a tx anchor is missing from the chain, we would want to return - // this height in the iterator. The exception is when the tx is confirmed in chain. All the - // other missing-height anchors of this tx can be skipped. - // - // * Some(true) => skip all anchors of this txid - // * Some(false) => do not skip anchors of this txid - // * None => we do not know whether we can skip this txid - let mut txids_to_skip = HashMap::::new(); - - // Keeps track of the last height emitted so we don't double up. - let mut last_height_emitted = Option::::None; - - self.anchors - .iter() - .filter(move |(_, txid)| { - let skip = *txids_to_skip.entry(*txid).or_insert_with(|| { - let tx_anchors = match self.txs.get(txid) { - Some((_, anchors, _)) => anchors, - None => return true, - }; - let mut has_missing_height = false; - for anchor_block in tx_anchors.iter().map(Anchor::anchor_block) { - match chain.get(anchor_block.height) { - None => { - has_missing_height = true; - continue; - } - Some(chain_cp) => { - if chain_cp.hash() == anchor_block.hash { - return true; - } - } - } - } - !has_missing_height - }); - #[cfg(feature = "std")] - debug_assert!({ - println!("txid={} skip={}", txid, skip); - true - }); - !skip - }) - .filter_map(move |(a, _)| { - let anchor_block = a.anchor_block(); - if Some(anchor_block.height) != last_height_emitted - && chain.get(anchor_block.height).is_none() - { - last_height_emitted = Some(anchor_block.height); - Some(anchor_block.height) - } else { - None - } - }) - } - /// Get the position of the transaction in `chain` with tip `chain_tip`. /// /// Chain data is fetched from `chain`, a [`ChainOracle`] implementation. @@ -1330,8 +1267,6 @@ impl ChangeSet { /// /// This is useful if you want to find which heights you need to fetch data about in order to /// confirm or exclude these anchors. - /// - /// See also: [`TxGraph::missing_heights`] pub fn anchor_heights(&self) -> impl Iterator + '_ where A: Anchor, @@ -1346,24 +1281,6 @@ impl ChangeSet { !duplicate }) } - - /// Returns an iterator for the [`anchor_heights`] in this changeset that are not included in - /// `local_chain`. This tells you which heights you need to include in `local_chain` in order - /// for it to conclusively act as a [`ChainOracle`] for the transaction anchors this changeset - /// will add. - /// - /// [`ChainOracle`]: crate::ChainOracle - /// [`anchor_heights`]: Self::anchor_heights - pub fn missing_heights_from<'a>( - &'a self, - local_chain: &'a LocalChain, - ) -> impl Iterator + 'a - where - A: Anchor, - { - self.anchor_heights() - .filter(move |&height| local_chain.get(height).is_none()) - } } impl Append for ChangeSet { diff --git a/crates/chain/tests/test_tx_graph.rs b/crates/chain/tests/test_tx_graph.rs index 7940913ef..1c7a90f72 100644 --- a/crates/chain/tests/test_tx_graph.rs +++ b/crates/chain/tests/test_tx_graph.rs @@ -1087,139 +1087,6 @@ fn update_last_seen_unconfirmed() { assert_eq!(graph.full_txs().next().unwrap().last_seen_unconfirmed, 2); } -#[test] -fn test_missing_blocks() { - /// An anchor implementation for testing, made up of `(the_anchor_block, random_data)`. - #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord, core::hash::Hash)] - struct TestAnchor(BlockId); - - impl Anchor for TestAnchor { - fn anchor_block(&self) -> BlockId { - self.0 - } - } - - struct Scenario<'a> { - name: &'a str, - graph: TxGraph, - chain: LocalChain, - exp_heights: &'a [u32], - } - - const fn new_anchor(height: u32, hash: BlockHash) -> TestAnchor { - TestAnchor(BlockId { height, hash }) - } - - fn new_scenario<'a>( - name: &'a str, - graph_anchors: &'a [(Txid, TestAnchor)], - chain: &'a [(u32, BlockHash)], - exp_heights: &'a [u32], - ) -> Scenario<'a> { - Scenario { - name, - graph: { - let mut g = TxGraph::default(); - for (txid, anchor) in graph_anchors { - let _ = g.insert_anchor(*txid, anchor.clone()); - } - g - }, - chain: { - let (mut c, _) = LocalChain::from_genesis_hash(h!("genesis")); - for (height, hash) in chain { - let _ = c.insert_block(BlockId { - height: *height, - hash: *hash, - }); - } - c - }, - exp_heights, - } - } - - fn run(scenarios: &[Scenario]) { - for scenario in scenarios { - let Scenario { - name, - graph, - chain, - exp_heights, - } = scenario; - - let heights = graph.missing_heights(chain).collect::>(); - assert_eq!(&heights, exp_heights, "scenario: {}", name); - } - } - - run(&[ - new_scenario( - "2 txs with the same anchor (2:B) which is missing from chain", - &[ - (h!("tx_1"), new_anchor(2, h!("B"))), - (h!("tx_2"), new_anchor(2, h!("B"))), - ], - &[(1, h!("A")), (3, h!("C"))], - &[2], - ), - new_scenario( - "2 txs with different anchors at the same height, one of the anchors is missing", - &[ - (h!("tx_1"), new_anchor(2, h!("B1"))), - (h!("tx_2"), new_anchor(2, h!("B2"))), - ], - &[(1, h!("A")), (2, h!("B1"))], - &[], - ), - new_scenario( - "tx with 2 anchors of same height which are missing from the chain", - &[ - (h!("tx"), new_anchor(3, h!("C1"))), - (h!("tx"), new_anchor(3, h!("C2"))), - ], - &[(1, h!("A")), (4, h!("D"))], - &[3], - ), - new_scenario( - "tx with 2 anchors at the same height, chain has this height but does not match either anchor", - &[ - (h!("tx"), new_anchor(4, h!("D1"))), - (h!("tx"), new_anchor(4, h!("D2"))), - ], - &[(4, h!("D3")), (5, h!("E"))], - &[], - ), - new_scenario( - "tx with 2 anchors at different heights, one anchor exists in chain, should return nothing", - &[ - (h!("tx"), new_anchor(3, h!("C"))), - (h!("tx"), new_anchor(4, h!("D"))), - ], - &[(4, h!("D")), (5, h!("E"))], - &[], - ), - new_scenario( - "tx with 2 anchors at different heights, first height is already in chain with different hash, iterator should only return 2nd height", - &[ - (h!("tx"), new_anchor(5, h!("E1"))), - (h!("tx"), new_anchor(6, h!("F1"))), - ], - &[(4, h!("D")), (5, h!("E")), (7, h!("G"))], - &[6], - ), - new_scenario( - "tx with 2 anchors at different heights, neither height is in chain, both heights should be returned", - &[ - (h!("tx"), new_anchor(3, h!("C"))), - (h!("tx"), new_anchor(4, h!("D"))), - ], - &[(1, h!("A")), (2, h!("B"))], - &[3, 4], - ), - ]); -} - #[test] /// The `map_anchors` allow a caller to pass a function to reconstruct the [`TxGraph`] with any [`Anchor`], /// even though the function is non-deterministic. From 494d253493f1bc914adba16a28ccf1bc0a0f4ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 26 Mar 2024 20:12:51 +0800 Subject: [PATCH 04/13] feat(testenv): add `genesis_hash` method This gets the genesis hash of the env blockchain. --- crates/testenv/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/testenv/src/lib.rs b/crates/testenv/src/lib.rs index b0147d0fc..2edd06eb2 100644 --- a/crates/testenv/src/lib.rs +++ b/crates/testenv/src/lib.rs @@ -250,6 +250,12 @@ impl TestEnv { })) .expect("must craft tip") } + + /// Get the genesis hash of the blockchain. + pub fn genesis_hash(&self) -> anyhow::Result { + let hash = self.bitcoind.client.get_block_hash(0)?; + Ok(hash) + } } #[cfg(test)] From a6e613e6b978b995abf6c92a16df0300b113aa2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 26 Mar 2024 20:29:20 +0800 Subject: [PATCH 05/13] test(esplora): add `test_finalize_chain_update` We ensure that calling `finalize_chain_update` does not result in a chain which removed previous heights and all anchor heights are included. --- crates/esplora/tests/async_ext.rs | 172 +++++++++++++++++++++++++++ crates/esplora/tests/blocking_ext.rs | 168 ++++++++++++++++++++++++++ 2 files changed, 340 insertions(+) diff --git a/crates/esplora/tests/async_ext.rs b/crates/esplora/tests/async_ext.rs index 5946bb4d8..e053ba72b 100644 --- a/crates/esplora/tests/async_ext.rs +++ b/crates/esplora/tests/async_ext.rs @@ -1,3 +1,6 @@ +use bdk_chain::bitcoin::hashes::Hash; +use bdk_chain::local_chain::LocalChain; +use bdk_chain::BlockId; use bdk_esplora::EsploraAsyncExt; use electrsd::bitcoind::anyhow; use electrsd::bitcoind::bitcoincore_rpc::RpcApi; @@ -10,6 +13,175 @@ use std::time::Duration; use bdk_chain::bitcoin::{Address, Amount, Txid}; use bdk_testenv::TestEnv; +macro_rules! h { + ($index:literal) => {{ + bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes()) + }}; +} + +/// Ensure that update does not remove heights (from original), and all anchor heights are included. +#[tokio::test] +pub async fn test_finalize_chain_update() -> anyhow::Result<()> { + struct TestCase<'a> { + name: &'a str, + /// Initial blockchain height to start the env with. + initial_env_height: u32, + /// Initial checkpoint heights to start with. + initial_cps: &'a [u32], + /// The final blockchain height of the env. + final_env_height: u32, + /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch + /// the blockhash from the env. + anchors: &'a [(u32, Txid)], + } + + let test_cases = [ + TestCase { + name: "chain_extends", + initial_env_height: 60, + initial_cps: &[59, 60], + final_env_height: 90, + anchors: &[], + }, + TestCase { + name: "introduce_older_heights", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 50, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + TestCase { + name: "introduce_older_heights_after_chain_extends", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 100, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + ]; + + for (i, t) in test_cases.into_iter().enumerate() { + println!("[{}] running test case: {}", i, t.name); + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + + // set env to `initial_env_height` + if let Some(to_mine) = t + .initial_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height().await? < t.initial_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft initial `local_chain` + let local_chain = { + let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let chain_tip = chain.tip(); + let update_blocks = bdk_esplora::init_chain_update(&client, &chain_tip).await?; + let update_anchors = t + .initial_cps + .iter() + .map(|&height| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + Txid::all_zeros(), + )) + }) + .collect::>>()?; + let chain_update = bdk_esplora::finalize_chain_update( + &client, + &chain_tip, + &update_anchors, + update_blocks, + ) + .await?; + chain.apply_update(chain_update)?; + chain + }; + println!("local chain height: {}", local_chain.tip().height()); + + // extend env chain + if let Some(to_mine) = t + .final_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height().await? < t.final_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft update + let update = { + let local_tip = local_chain.tip(); + let update_blocks = bdk_esplora::init_chain_update(&client, &local_tip).await?; + let update_anchors = t + .anchors + .iter() + .map(|&(height, txid)| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + txid, + )) + }) + .collect::>()?; + bdk_esplora::finalize_chain_update(&client, &local_tip, &update_anchors, update_blocks) + .await? + }; + + // apply update + let mut updated_local_chain = local_chain.clone(); + updated_local_chain.apply_update(update)?; + println!( + "updated local chain height: {}", + updated_local_chain.tip().height() + ); + + assert!( + { + let initial_heights = local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + let updated_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + updated_heights.is_superset(&initial_heights) + }, + "heights from the initial chain must all be in the updated chain", + ); + + assert!( + { + let exp_anchor_heights = t + .anchors + .iter() + .map(|(h, _)| *h) + .chain(t.initial_cps.iter().copied()) + .collect::>(); + let anchor_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + anchor_heights.is_superset(&exp_anchor_heights) + }, + "anchor heights must all be in updated chain", + ); + } + + Ok(()) +} #[tokio::test] pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { let env = TestEnv::new()?; diff --git a/crates/esplora/tests/blocking_ext.rs b/crates/esplora/tests/blocking_ext.rs index d35fab658..a9078d031 100644 --- a/crates/esplora/tests/blocking_ext.rs +++ b/crates/esplora/tests/blocking_ext.rs @@ -1,3 +1,4 @@ +use bdk_chain::bitcoin::hashes::Hash; use bdk_chain::local_chain::LocalChain; use bdk_chain::BlockId; use bdk_esplora::EsploraExt; @@ -26,6 +27,173 @@ macro_rules! local_chain { }}; } +/// Ensure that update does not remove heights (from original), and all anchor heights are included. +#[test] +pub fn test_finalize_chain_update() -> anyhow::Result<()> { + struct TestCase<'a> { + name: &'a str, + /// Initial blockchain height to start the env with. + initial_env_height: u32, + /// Initial checkpoint heights to start with. + initial_cps: &'a [u32], + /// The final blockchain height of the env. + final_env_height: u32, + /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch + /// the blockhash from the env. + anchors: &'a [(u32, Txid)], + } + + let test_cases = [ + TestCase { + name: "chain_extends", + initial_env_height: 60, + initial_cps: &[59, 60], + final_env_height: 90, + anchors: &[], + }, + TestCase { + name: "introduce_older_heights", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 50, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + TestCase { + name: "introduce_older_heights_after_chain_extends", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 100, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + ]; + + for (i, t) in test_cases.into_iter().enumerate() { + println!("[{}] running test case: {}", i, t.name); + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking()?; + + // set env to `initial_env_height` + if let Some(to_mine) = t + .initial_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height()? < t.initial_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft initial `local_chain` + let local_chain = { + let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let chain_tip = chain.tip(); + let update_blocks = bdk_esplora::init_chain_update_blocking(&client, &chain_tip)?; + let update_anchors = t + .initial_cps + .iter() + .map(|&height| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + Txid::all_zeros(), + )) + }) + .collect::>>()?; + let chain_update = bdk_esplora::finalize_chain_update_blocking( + &client, + &chain_tip, + &update_anchors, + update_blocks, + )?; + chain.apply_update(chain_update)?; + chain + }; + println!("local chain height: {}", local_chain.tip().height()); + + // extend env chain + if let Some(to_mine) = t + .final_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height()? < t.final_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft update + let update = { + let local_tip = local_chain.tip(); + let update_blocks = bdk_esplora::init_chain_update_blocking(&client, &local_tip)?; + let update_anchors = t + .anchors + .iter() + .map(|&(height, txid)| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + txid, + )) + }) + .collect::>()?; + bdk_esplora::finalize_chain_update_blocking( + &client, + &local_tip, + &update_anchors, + update_blocks, + )? + }; + + // apply update + let mut updated_local_chain = local_chain.clone(); + updated_local_chain.apply_update(update)?; + println!( + "updated local chain height: {}", + updated_local_chain.tip().height() + ); + + assert!( + { + let initial_heights = local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + let updated_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + updated_heights.is_superset(&initial_heights) + }, + "heights from the initial chain must all be in the updated chain", + ); + + assert!( + { + let exp_anchor_heights = t + .anchors + .iter() + .map(|(h, _)| *h) + .chain(t.initial_cps.iter().copied()) + .collect::>(); + let anchor_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + anchor_heights.is_superset(&exp_anchor_heights) + }, + "anchor heights must all be in updated chain", + ); + } + + Ok(()) +} + #[test] pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { let env = TestEnv::new()?; From 519cd75d23fbb72321b0b189dca12afbfd78c0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 3 Apr 2024 14:29:02 +0800 Subject: [PATCH 06/13] test(esplora): move esplora tests into src files Since we want to keep these methods private. --- crates/esplora/Cargo.toml | 1 + crates/esplora/src/async_ext.rs | 193 ++++++++++++- crates/esplora/src/blocking_ext.rs | 400 ++++++++++++++++++++++++++- crates/esplora/tests/async_ext.rs | 172 ------------ crates/esplora/tests/blocking_ext.rs | 389 +------------------------- 5 files changed, 579 insertions(+), 576 deletions(-) diff --git a/crates/esplora/Cargo.toml b/crates/esplora/Cargo.toml index 822090ad9..ac00bf716 100644 --- a/crates/esplora/Cargo.toml +++ b/crates/esplora/Cargo.toml @@ -25,6 +25,7 @@ miniscript = { version = "11.0.0", optional = true, default-features = false } bdk_testenv = { path = "../testenv", default_features = false } electrsd = { version= "0.27.1", features = ["bitcoind_25_0", "esplora_a33e97e1", "legacy"] } tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } +anyhow = "1" [features] default = ["std", "async-https", "blocking"] diff --git a/crates/esplora/src/async_ext.rs b/crates/esplora/src/async_ext.rs index d0ae80d5b..b387cb771 100644 --- a/crates/esplora/src/async_ext.rs +++ b/crates/esplora/src/async_ext.rs @@ -139,8 +139,7 @@ impl EsploraAsyncExt for esplora_client::AsyncClient { /// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use /// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when /// alternating between chain-sources. -#[doc(hidden)] -pub async fn init_chain_update( +async fn init_chain_update( client: &esplora_client::AsyncClient, local_tip: &CheckPoint, ) -> Result, Error> { @@ -183,8 +182,7 @@ pub async fn init_chain_update( /// /// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an /// existing checkpoint/block under `local_tip` or `update_blocks`. -#[doc(hidden)] -pub async fn finalize_chain_update( +async fn finalize_chain_update( client: &esplora_client::AsyncClient, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, @@ -243,8 +241,7 @@ pub async fn finalize_chain_update( /// This performs a full scan to get an update for the [`TxGraph`] and /// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex). -#[doc(hidden)] -pub async fn full_scan_for_index_and_graph( +async fn full_scan_for_index_and_graph( client: &esplora_client::AsyncClient, keychain_spks: BTreeMap< K, @@ -339,8 +336,7 @@ pub async fn full_scan_for_index_and_graph( Ok((graph, last_active_indexes)) } -#[doc(hidden)] -pub async fn sync_for_index_and_graph( +async fn sync_for_index_and_graph( client: &esplora_client::AsyncClient, misc_spks: impl IntoIterator + Send> + Send, txids: impl IntoIterator + Send> + Send, @@ -414,3 +410,184 @@ pub async fn sync_for_index_and_graph( Ok(graph) } + +#[cfg(test)] +mod test { + use std::{collections::BTreeSet, time::Duration}; + + use bdk_chain::{ + bitcoin::{hashes::Hash, Txid}, + local_chain::LocalChain, + BlockId, + }; + use bdk_testenv::TestEnv; + use electrsd::bitcoind::bitcoincore_rpc::RpcApi; + use esplora_client::Builder; + + use crate::async_ext::{finalize_chain_update, init_chain_update}; + + macro_rules! h { + ($index:literal) => {{ + bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes()) + }}; + } + + /// Ensure that update does not remove heights (from original), and all anchor heights are included. + #[tokio::test] + pub async fn test_finalize_chain_update() -> anyhow::Result<()> { + struct TestCase<'a> { + name: &'a str, + /// Initial blockchain height to start the env with. + initial_env_height: u32, + /// Initial checkpoint heights to start with. + initial_cps: &'a [u32], + /// The final blockchain height of the env. + final_env_height: u32, + /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch + /// the blockhash from the env. + anchors: &'a [(u32, Txid)], + } + + let test_cases = [ + TestCase { + name: "chain_extends", + initial_env_height: 60, + initial_cps: &[59, 60], + final_env_height: 90, + anchors: &[], + }, + TestCase { + name: "introduce_older_heights", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 50, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + TestCase { + name: "introduce_older_heights_after_chain_extends", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 100, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + ]; + + for (i, t) in test_cases.into_iter().enumerate() { + println!("[{}] running test case: {}", i, t.name); + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + + // set env to `initial_env_height` + if let Some(to_mine) = t + .initial_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height().await? < t.initial_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft initial `local_chain` + let local_chain = { + let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let chain_tip = chain.tip(); + let update_blocks = init_chain_update(&client, &chain_tip).await?; + let update_anchors = t + .initial_cps + .iter() + .map(|&height| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + Txid::all_zeros(), + )) + }) + .collect::>>()?; + let chain_update = + finalize_chain_update(&client, &chain_tip, &update_anchors, update_blocks) + .await?; + chain.apply_update(chain_update)?; + chain + }; + println!("local chain height: {}", local_chain.tip().height()); + + // extend env chain + if let Some(to_mine) = t + .final_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height().await? < t.final_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft update + let update = { + let local_tip = local_chain.tip(); + let update_blocks = init_chain_update(&client, &local_tip).await?; + let update_anchors = t + .anchors + .iter() + .map(|&(height, txid)| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + txid, + )) + }) + .collect::>()?; + finalize_chain_update(&client, &local_tip, &update_anchors, update_blocks).await? + }; + + // apply update + let mut updated_local_chain = local_chain.clone(); + updated_local_chain.apply_update(update)?; + println!( + "updated local chain height: {}", + updated_local_chain.tip().height() + ); + + assert!( + { + let initial_heights = local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + let updated_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + updated_heights.is_superset(&initial_heights) + }, + "heights from the initial chain must all be in the updated chain", + ); + + assert!( + { + let exp_anchor_heights = t + .anchors + .iter() + .map(|(h, _)| *h) + .chain(t.initial_cps.iter().copied()) + .collect::>(); + let anchor_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + anchor_heights.is_superset(&exp_anchor_heights) + }, + "anchor heights must all be in updated chain", + ); + } + + Ok(()) + } +} diff --git a/crates/esplora/src/blocking_ext.rs b/crates/esplora/src/blocking_ext.rs index adfd33c09..419a2ae66 100644 --- a/crates/esplora/src/blocking_ext.rs +++ b/crates/esplora/src/blocking_ext.rs @@ -148,8 +148,7 @@ impl EsploraExt for esplora_client::BlockingClient { /// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use /// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when /// alternating between chain-sources. -#[doc(hidden)] -pub fn init_chain_update_blocking( +fn init_chain_update_blocking( client: &esplora_client::BlockingClient, local_tip: &CheckPoint, ) -> Result, Error> { @@ -191,8 +190,7 @@ pub fn init_chain_update_blocking( /// /// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an /// existing checkpoint/block under `local_tip` or `update_blocks`. -#[doc(hidden)] -pub fn finalize_chain_update_blocking( +fn finalize_chain_update_blocking( client: &esplora_client::BlockingClient, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, @@ -251,8 +249,7 @@ pub fn finalize_chain_update_blocking( /// This performs a full scan to get an update for the [`TxGraph`] and /// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex). -#[doc(hidden)] -pub fn full_scan_for_index_and_graph_blocking( +fn full_scan_for_index_and_graph_blocking( client: &esplora_client::BlockingClient, keychain_spks: BTreeMap>, stop_gap: usize, @@ -347,8 +344,7 @@ pub fn full_scan_for_index_and_graph_blocking( Ok((tx_graph, last_active_indices)) } -#[doc(hidden)] -pub fn sync_for_index_and_graph_blocking( +fn sync_for_index_and_graph_blocking( client: &esplora_client::BlockingClient, misc_spks: impl IntoIterator, txids: impl IntoIterator, @@ -431,3 +427,391 @@ pub fn sync_for_index_and_graph_blocking( Ok(tx_graph) } + +#[cfg(test)] +mod test { + use crate::blocking_ext::{finalize_chain_update_blocking, init_chain_update_blocking}; + use bdk_chain::bitcoin::hashes::Hash; + use bdk_chain::bitcoin::Txid; + use bdk_chain::local_chain::LocalChain; + use bdk_chain::BlockId; + use bdk_testenv::TestEnv; + use electrsd::bitcoind::bitcoincore_rpc::RpcApi; + use esplora_client::{BlockHash, Builder}; + use std::collections::{BTreeMap, BTreeSet}; + use std::time::Duration; + + macro_rules! h { + ($index:literal) => {{ + bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes()) + }}; + } + + macro_rules! local_chain { + [ $(($height:expr, $block_hash:expr)), * ] => {{ + #[allow(unused_mut)] + bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect()) + .expect("chain must have genesis block") + }}; + } + + /// Ensure that update does not remove heights (from original), and all anchor heights are included. + #[test] + pub fn test_finalize_chain_update() -> anyhow::Result<()> { + struct TestCase<'a> { + name: &'a str, + /// Initial blockchain height to start the env with. + initial_env_height: u32, + /// Initial checkpoint heights to start with. + initial_cps: &'a [u32], + /// The final blockchain height of the env. + final_env_height: u32, + /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch + /// the blockhash from the env. + anchors: &'a [(u32, Txid)], + } + + let test_cases = [ + TestCase { + name: "chain_extends", + initial_env_height: 60, + initial_cps: &[59, 60], + final_env_height: 90, + anchors: &[], + }, + TestCase { + name: "introduce_older_heights", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 50, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + TestCase { + name: "introduce_older_heights_after_chain_extends", + initial_env_height: 50, + initial_cps: &[10, 15], + final_env_height: 100, + anchors: &[(11, h!("A")), (14, h!("B"))], + }, + ]; + + for (i, t) in test_cases.into_iter().enumerate() { + println!("[{}] running test case: {}", i, t.name); + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + + // set env to `initial_env_height` + if let Some(to_mine) = t + .initial_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height()? < t.initial_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft initial `local_chain` + let local_chain = { + let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let chain_tip = chain.tip(); + let update_blocks = init_chain_update_blocking(&client, &chain_tip)?; + let update_anchors = t + .initial_cps + .iter() + .map(|&height| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + Txid::all_zeros(), + )) + }) + .collect::>>()?; + let chain_update = finalize_chain_update_blocking( + &client, + &chain_tip, + &update_anchors, + update_blocks, + )?; + chain.apply_update(chain_update)?; + chain + }; + println!("local chain height: {}", local_chain.tip().height()); + + // extend env chain + if let Some(to_mine) = t + .final_env_height + .checked_sub(env.make_checkpoint_tip().height()) + { + env.mine_blocks(to_mine as _, None)?; + } + while client.get_height()? < t.final_env_height { + std::thread::sleep(Duration::from_millis(10)); + } + + // craft update + let update = { + let local_tip = local_chain.tip(); + let update_blocks = init_chain_update_blocking(&client, &local_tip)?; + let update_anchors = t + .anchors + .iter() + .map(|&(height, txid)| -> anyhow::Result<_> { + Ok(( + BlockId { + height, + hash: env.bitcoind.client.get_block_hash(height as _)?, + }, + txid, + )) + }) + .collect::>()?; + finalize_chain_update_blocking(&client, &local_tip, &update_anchors, update_blocks)? + }; + + // apply update + let mut updated_local_chain = local_chain.clone(); + updated_local_chain.apply_update(update)?; + println!( + "updated local chain height: {}", + updated_local_chain.tip().height() + ); + + assert!( + { + let initial_heights = local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + let updated_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + updated_heights.is_superset(&initial_heights) + }, + "heights from the initial chain must all be in the updated chain", + ); + + assert!( + { + let exp_anchor_heights = t + .anchors + .iter() + .map(|(h, _)| *h) + .chain(t.initial_cps.iter().copied()) + .collect::>(); + let anchor_heights = updated_local_chain + .iter_checkpoints() + .map(|cp| cp.height()) + .collect::>(); + anchor_heights.is_superset(&exp_anchor_heights) + }, + "anchor heights must all be in updated chain", + ); + } + + Ok(()) + } + + #[test] + fn update_local_chain() -> anyhow::Result<()> { + const TIP_HEIGHT: u32 = 50; + + let env = TestEnv::new()?; + let blocks = { + let bitcoind_client = &env.bitcoind.client; + assert_eq!(bitcoind_client.get_block_count()?, 1); + [ + (0, bitcoind_client.get_block_hash(0)?), + (1, bitcoind_client.get_block_hash(1)?), + ] + .into_iter() + .chain((2..).zip(env.mine_blocks((TIP_HEIGHT - 1) as usize, None)?)) + .collect::>() + }; + // so new blocks can be seen by Electrs + let env = env.reset_electrsd()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + + struct TestCase { + name: &'static str, + chain: LocalChain, + request_heights: &'static [u32], + exp_update_heights: &'static [u32], + } + + let test_cases = [ + TestCase { + name: "request_later_blocks", + chain: local_chain![(0, blocks[&0]), (21, blocks[&21])], + request_heights: &[22, 25, 28], + exp_update_heights: &[21, 22, 25, 28], + }, + TestCase { + name: "request_prev_blocks", + chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (5, blocks[&5])], + request_heights: &[4], + exp_update_heights: &[4, 5], + }, + TestCase { + name: "request_prev_blocks_2", + chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (10, blocks[&10])], + request_heights: &[4, 6], + exp_update_heights: &[4, 6, 10], + }, + TestCase { + name: "request_later_and_prev_blocks", + chain: local_chain![(0, blocks[&0]), (7, blocks[&7]), (11, blocks[&11])], + request_heights: &[8, 9, 15], + exp_update_heights: &[8, 9, 11, 15], + }, + TestCase { + name: "request_tip_only", + chain: local_chain![(0, blocks[&0]), (5, blocks[&5]), (49, blocks[&49])], + request_heights: &[TIP_HEIGHT], + exp_update_heights: &[49], + }, + TestCase { + name: "request_nothing", + chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, blocks[&23])], + request_heights: &[], + exp_update_heights: &[23], + }, + TestCase { + name: "request_nothing_during_reorg", + chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, h!("23"))], + request_heights: &[], + exp_update_heights: &[13, 23], + }, + TestCase { + name: "request_nothing_during_reorg_2", + chain: local_chain![ + (0, blocks[&0]), + (21, blocks[&21]), + (22, h!("22")), + (23, h!("23")) + ], + request_heights: &[], + exp_update_heights: &[21, 22, 23], + }, + TestCase { + name: "request_prev_blocks_during_reorg", + chain: local_chain![ + (0, blocks[&0]), + (21, blocks[&21]), + (22, h!("22")), + (23, h!("23")) + ], + request_heights: &[17, 20], + exp_update_heights: &[17, 20, 21, 22, 23], + }, + TestCase { + name: "request_later_blocks_during_reorg", + chain: local_chain![ + (0, blocks[&0]), + (9, blocks[&9]), + (22, h!("22")), + (23, h!("23")) + ], + request_heights: &[25, 27], + exp_update_heights: &[9, 22, 23, 25, 27], + }, + TestCase { + name: "request_later_blocks_during_reorg_2", + chain: local_chain![(0, blocks[&0]), (9, h!("9"))], + request_heights: &[10], + exp_update_heights: &[0, 9, 10], + }, + TestCase { + name: "request_later_and_prev_blocks_during_reorg", + chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (9, h!("9"))], + request_heights: &[8, 11], + exp_update_heights: &[1, 8, 9, 11], + }, + ]; + + for (i, t) in test_cases.into_iter().enumerate() { + println!("Case {}: {}", i, t.name); + let mut chain = t.chain; + let cp_tip = chain.tip(); + + let new_blocks = init_chain_update_blocking(&client, &cp_tip).map_err(|err| { + anyhow::format_err!("[{}:{}] `init_chain_update` failed: {}", i, t.name, err) + })?; + + let mock_anchors = t + .request_heights + .iter() + .map(|&h| { + let anchor_blockhash: BlockHash = bdk_chain::bitcoin::hashes::Hash::hash( + &format!("hash_at_height_{}", h).into_bytes(), + ); + let txid: Txid = bdk_chain::bitcoin::hashes::Hash::hash( + &format!("txid_at_height_{}", h).into_bytes(), + ); + let anchor = BlockId { + height: h, + hash: anchor_blockhash, + }; + (anchor, txid) + }) + .collect::>(); + + let chain_update = + finalize_chain_update_blocking(&client, &cp_tip, &mock_anchors, new_blocks)?; + let update_blocks = chain_update + .tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + + let exp_update_blocks = t + .exp_update_heights + .iter() + .map(|&height| { + let hash = blocks[&height]; + BlockId { height, hash } + }) + .chain( + // Electrs Esplora `get_block` call fetches 10 blocks which is included in the + // update + blocks + .range(TIP_HEIGHT - 9..) + .map(|(&height, &hash)| BlockId { height, hash }), + ) + .collect::>(); + + assert!( + update_blocks.is_superset(&exp_update_blocks), + "[{}:{}] unexpected update", + i, + t.name + ); + + let _ = chain + .apply_update(chain_update) + .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err)); + + // all requested heights must exist in the final chain + for height in t.request_heights { + let exp_blockhash = blocks.get(height).expect("block must exist in bitcoind"); + assert_eq!( + chain.get(*height).map(|cp| cp.hash()), + Some(*exp_blockhash), + "[{}:{}] block {}:{} must exist in final chain", + i, + t.name, + height, + exp_blockhash + ); + } + } + + Ok(()) + } +} diff --git a/crates/esplora/tests/async_ext.rs b/crates/esplora/tests/async_ext.rs index e053ba72b..5946bb4d8 100644 --- a/crates/esplora/tests/async_ext.rs +++ b/crates/esplora/tests/async_ext.rs @@ -1,6 +1,3 @@ -use bdk_chain::bitcoin::hashes::Hash; -use bdk_chain::local_chain::LocalChain; -use bdk_chain::BlockId; use bdk_esplora::EsploraAsyncExt; use electrsd::bitcoind::anyhow; use electrsd::bitcoind::bitcoincore_rpc::RpcApi; @@ -13,175 +10,6 @@ use std::time::Duration; use bdk_chain::bitcoin::{Address, Amount, Txid}; use bdk_testenv::TestEnv; -macro_rules! h { - ($index:literal) => {{ - bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes()) - }}; -} - -/// Ensure that update does not remove heights (from original), and all anchor heights are included. -#[tokio::test] -pub async fn test_finalize_chain_update() -> anyhow::Result<()> { - struct TestCase<'a> { - name: &'a str, - /// Initial blockchain height to start the env with. - initial_env_height: u32, - /// Initial checkpoint heights to start with. - initial_cps: &'a [u32], - /// The final blockchain height of the env. - final_env_height: u32, - /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch - /// the blockhash from the env. - anchors: &'a [(u32, Txid)], - } - - let test_cases = [ - TestCase { - name: "chain_extends", - initial_env_height: 60, - initial_cps: &[59, 60], - final_env_height: 90, - anchors: &[], - }, - TestCase { - name: "introduce_older_heights", - initial_env_height: 50, - initial_cps: &[10, 15], - final_env_height: 50, - anchors: &[(11, h!("A")), (14, h!("B"))], - }, - TestCase { - name: "introduce_older_heights_after_chain_extends", - initial_env_height: 50, - initial_cps: &[10, 15], - final_env_height: 100, - anchors: &[(11, h!("A")), (14, h!("B"))], - }, - ]; - - for (i, t) in test_cases.into_iter().enumerate() { - println!("[{}] running test case: {}", i, t.name); - - let env = TestEnv::new()?; - let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); - let client = Builder::new(base_url.as_str()).build_async()?; - - // set env to `initial_env_height` - if let Some(to_mine) = t - .initial_env_height - .checked_sub(env.make_checkpoint_tip().height()) - { - env.mine_blocks(to_mine as _, None)?; - } - while client.get_height().await? < t.initial_env_height { - std::thread::sleep(Duration::from_millis(10)); - } - - // craft initial `local_chain` - let local_chain = { - let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); - let chain_tip = chain.tip(); - let update_blocks = bdk_esplora::init_chain_update(&client, &chain_tip).await?; - let update_anchors = t - .initial_cps - .iter() - .map(|&height| -> anyhow::Result<_> { - Ok(( - BlockId { - height, - hash: env.bitcoind.client.get_block_hash(height as _)?, - }, - Txid::all_zeros(), - )) - }) - .collect::>>()?; - let chain_update = bdk_esplora::finalize_chain_update( - &client, - &chain_tip, - &update_anchors, - update_blocks, - ) - .await?; - chain.apply_update(chain_update)?; - chain - }; - println!("local chain height: {}", local_chain.tip().height()); - - // extend env chain - if let Some(to_mine) = t - .final_env_height - .checked_sub(env.make_checkpoint_tip().height()) - { - env.mine_blocks(to_mine as _, None)?; - } - while client.get_height().await? < t.final_env_height { - std::thread::sleep(Duration::from_millis(10)); - } - - // craft update - let update = { - let local_tip = local_chain.tip(); - let update_blocks = bdk_esplora::init_chain_update(&client, &local_tip).await?; - let update_anchors = t - .anchors - .iter() - .map(|&(height, txid)| -> anyhow::Result<_> { - Ok(( - BlockId { - height, - hash: env.bitcoind.client.get_block_hash(height as _)?, - }, - txid, - )) - }) - .collect::>()?; - bdk_esplora::finalize_chain_update(&client, &local_tip, &update_anchors, update_blocks) - .await? - }; - - // apply update - let mut updated_local_chain = local_chain.clone(); - updated_local_chain.apply_update(update)?; - println!( - "updated local chain height: {}", - updated_local_chain.tip().height() - ); - - assert!( - { - let initial_heights = local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - let updated_heights = updated_local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - updated_heights.is_superset(&initial_heights) - }, - "heights from the initial chain must all be in the updated chain", - ); - - assert!( - { - let exp_anchor_heights = t - .anchors - .iter() - .map(|(h, _)| *h) - .chain(t.initial_cps.iter().copied()) - .collect::>(); - let anchor_heights = updated_local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - anchor_heights.is_superset(&exp_anchor_heights) - }, - "anchor heights must all be in updated chain", - ); - } - - Ok(()) -} #[tokio::test] pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { let env = TestEnv::new()?; diff --git a/crates/esplora/tests/blocking_ext.rs b/crates/esplora/tests/blocking_ext.rs index a9078d031..3f8ff6932 100644 --- a/crates/esplora/tests/blocking_ext.rs +++ b/crates/esplora/tests/blocking_ext.rs @@ -1,10 +1,7 @@ -use bdk_chain::bitcoin::hashes::Hash; -use bdk_chain::local_chain::LocalChain; -use bdk_chain::BlockId; use bdk_esplora::EsploraExt; use electrsd::bitcoind::anyhow; use electrsd::bitcoind::bitcoincore_rpc::RpcApi; -use esplora_client::{self, BlockHash, Builder}; +use esplora_client::{self, Builder}; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::str::FromStr; use std::thread::sleep; @@ -13,187 +10,6 @@ use std::time::Duration; use bdk_chain::bitcoin::{Address, Amount, Txid}; use bdk_testenv::TestEnv; -macro_rules! h { - ($index:literal) => {{ - bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes()) - }}; -} - -macro_rules! local_chain { - [ $(($height:expr, $block_hash:expr)), * ] => {{ - #[allow(unused_mut)] - bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect()) - .expect("chain must have genesis block") - }}; -} - -/// Ensure that update does not remove heights (from original), and all anchor heights are included. -#[test] -pub fn test_finalize_chain_update() -> anyhow::Result<()> { - struct TestCase<'a> { - name: &'a str, - /// Initial blockchain height to start the env with. - initial_env_height: u32, - /// Initial checkpoint heights to start with. - initial_cps: &'a [u32], - /// The final blockchain height of the env. - final_env_height: u32, - /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch - /// the blockhash from the env. - anchors: &'a [(u32, Txid)], - } - - let test_cases = [ - TestCase { - name: "chain_extends", - initial_env_height: 60, - initial_cps: &[59, 60], - final_env_height: 90, - anchors: &[], - }, - TestCase { - name: "introduce_older_heights", - initial_env_height: 50, - initial_cps: &[10, 15], - final_env_height: 50, - anchors: &[(11, h!("A")), (14, h!("B"))], - }, - TestCase { - name: "introduce_older_heights_after_chain_extends", - initial_env_height: 50, - initial_cps: &[10, 15], - final_env_height: 100, - anchors: &[(11, h!("A")), (14, h!("B"))], - }, - ]; - - for (i, t) in test_cases.into_iter().enumerate() { - println!("[{}] running test case: {}", i, t.name); - - let env = TestEnv::new()?; - let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); - let client = Builder::new(base_url.as_str()).build_blocking()?; - - // set env to `initial_env_height` - if let Some(to_mine) = t - .initial_env_height - .checked_sub(env.make_checkpoint_tip().height()) - { - env.mine_blocks(to_mine as _, None)?; - } - while client.get_height()? < t.initial_env_height { - std::thread::sleep(Duration::from_millis(10)); - } - - // craft initial `local_chain` - let local_chain = { - let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); - let chain_tip = chain.tip(); - let update_blocks = bdk_esplora::init_chain_update_blocking(&client, &chain_tip)?; - let update_anchors = t - .initial_cps - .iter() - .map(|&height| -> anyhow::Result<_> { - Ok(( - BlockId { - height, - hash: env.bitcoind.client.get_block_hash(height as _)?, - }, - Txid::all_zeros(), - )) - }) - .collect::>>()?; - let chain_update = bdk_esplora::finalize_chain_update_blocking( - &client, - &chain_tip, - &update_anchors, - update_blocks, - )?; - chain.apply_update(chain_update)?; - chain - }; - println!("local chain height: {}", local_chain.tip().height()); - - // extend env chain - if let Some(to_mine) = t - .final_env_height - .checked_sub(env.make_checkpoint_tip().height()) - { - env.mine_blocks(to_mine as _, None)?; - } - while client.get_height()? < t.final_env_height { - std::thread::sleep(Duration::from_millis(10)); - } - - // craft update - let update = { - let local_tip = local_chain.tip(); - let update_blocks = bdk_esplora::init_chain_update_blocking(&client, &local_tip)?; - let update_anchors = t - .anchors - .iter() - .map(|&(height, txid)| -> anyhow::Result<_> { - Ok(( - BlockId { - height, - hash: env.bitcoind.client.get_block_hash(height as _)?, - }, - txid, - )) - }) - .collect::>()?; - bdk_esplora::finalize_chain_update_blocking( - &client, - &local_tip, - &update_anchors, - update_blocks, - )? - }; - - // apply update - let mut updated_local_chain = local_chain.clone(); - updated_local_chain.apply_update(update)?; - println!( - "updated local chain height: {}", - updated_local_chain.tip().height() - ); - - assert!( - { - let initial_heights = local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - let updated_heights = updated_local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - updated_heights.is_superset(&initial_heights) - }, - "heights from the initial chain must all be in the updated chain", - ); - - assert!( - { - let exp_anchor_heights = t - .anchors - .iter() - .map(|(h, _)| *h) - .chain(t.initial_cps.iter().copied()) - .collect::>(); - let anchor_heights = updated_local_chain - .iter_checkpoints() - .map(|cp| cp.height()) - .collect::>(); - anchor_heights.is_superset(&exp_anchor_heights) - }, - "anchor heights must all be in updated chain", - ); - } - - Ok(()) -} - #[test] pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { let env = TestEnv::new()?; @@ -399,206 +215,3 @@ pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> { Ok(()) } - -#[test] -fn update_local_chain() -> anyhow::Result<()> { - const TIP_HEIGHT: u32 = 50; - - let env = TestEnv::new()?; - let blocks = { - let bitcoind_client = &env.bitcoind.client; - assert_eq!(bitcoind_client.get_block_count()?, 1); - [ - (0, bitcoind_client.get_block_hash(0)?), - (1, bitcoind_client.get_block_hash(1)?), - ] - .into_iter() - .chain((2..).zip(env.mine_blocks((TIP_HEIGHT - 1) as usize, None)?)) - .collect::>() - }; - // so new blocks can be seen by Electrs - let env = env.reset_electrsd()?; - let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); - let client = Builder::new(base_url.as_str()).build_blocking(); - - struct TestCase { - name: &'static str, - chain: LocalChain, - request_heights: &'static [u32], - exp_update_heights: &'static [u32], - } - - let test_cases = [ - TestCase { - name: "request_later_blocks", - chain: local_chain![(0, blocks[&0]), (21, blocks[&21])], - request_heights: &[22, 25, 28], - exp_update_heights: &[21, 22, 25, 28], - }, - TestCase { - name: "request_prev_blocks", - chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (5, blocks[&5])], - request_heights: &[4], - exp_update_heights: &[4, 5], - }, - TestCase { - name: "request_prev_blocks_2", - chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (10, blocks[&10])], - request_heights: &[4, 6], - exp_update_heights: &[4, 6, 10], - }, - TestCase { - name: "request_later_and_prev_blocks", - chain: local_chain![(0, blocks[&0]), (7, blocks[&7]), (11, blocks[&11])], - request_heights: &[8, 9, 15], - exp_update_heights: &[8, 9, 11, 15], - }, - TestCase { - name: "request_tip_only", - chain: local_chain![(0, blocks[&0]), (5, blocks[&5]), (49, blocks[&49])], - request_heights: &[TIP_HEIGHT], - exp_update_heights: &[49], - }, - TestCase { - name: "request_nothing", - chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, blocks[&23])], - request_heights: &[], - exp_update_heights: &[23], - }, - TestCase { - name: "request_nothing_during_reorg", - chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, h!("23"))], - request_heights: &[], - exp_update_heights: &[13, 23], - }, - TestCase { - name: "request_nothing_during_reorg_2", - chain: local_chain![ - (0, blocks[&0]), - (21, blocks[&21]), - (22, h!("22")), - (23, h!("23")) - ], - request_heights: &[], - exp_update_heights: &[21, 22, 23], - }, - TestCase { - name: "request_prev_blocks_during_reorg", - chain: local_chain![ - (0, blocks[&0]), - (21, blocks[&21]), - (22, h!("22")), - (23, h!("23")) - ], - request_heights: &[17, 20], - exp_update_heights: &[17, 20, 21, 22, 23], - }, - TestCase { - name: "request_later_blocks_during_reorg", - chain: local_chain![ - (0, blocks[&0]), - (9, blocks[&9]), - (22, h!("22")), - (23, h!("23")) - ], - request_heights: &[25, 27], - exp_update_heights: &[9, 22, 23, 25, 27], - }, - TestCase { - name: "request_later_blocks_during_reorg_2", - chain: local_chain![(0, blocks[&0]), (9, h!("9"))], - request_heights: &[10], - exp_update_heights: &[0, 9, 10], - }, - TestCase { - name: "request_later_and_prev_blocks_during_reorg", - chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (9, h!("9"))], - request_heights: &[8, 11], - exp_update_heights: &[1, 8, 9, 11], - }, - ]; - - for (i, t) in test_cases.into_iter().enumerate() { - println!("Case {}: {}", i, t.name); - let mut chain = t.chain; - let cp_tip = chain.tip(); - - let new_blocks = - bdk_esplora::init_chain_update_blocking(&client, &cp_tip).map_err(|err| { - anyhow::format_err!("[{}:{}] `init_chain_update` failed: {}", i, t.name, err) - })?; - - let mock_anchors = t - .request_heights - .iter() - .map(|&h| { - let anchor_blockhash: BlockHash = bdk_chain::bitcoin::hashes::Hash::hash( - &format!("hash_at_height_{}", h).into_bytes(), - ); - let txid: Txid = bdk_chain::bitcoin::hashes::Hash::hash( - &format!("txid_at_height_{}", h).into_bytes(), - ); - let anchor = BlockId { - height: h, - hash: anchor_blockhash, - }; - (anchor, txid) - }) - .collect::>(); - - let chain_update = bdk_esplora::finalize_chain_update_blocking( - &client, - &cp_tip, - &mock_anchors, - new_blocks, - )?; - let update_blocks = chain_update - .tip - .iter() - .map(|cp| cp.block_id()) - .collect::>(); - - let exp_update_blocks = t - .exp_update_heights - .iter() - .map(|&height| { - let hash = blocks[&height]; - BlockId { height, hash } - }) - .chain( - // Electrs Esplora `get_block` call fetches 10 blocks which is included in the - // update - blocks - .range(TIP_HEIGHT - 9..) - .map(|(&height, &hash)| BlockId { height, hash }), - ) - .collect::>(); - - assert!( - update_blocks.is_superset(&exp_update_blocks), - "[{}:{}] unexpected update", - i, - t.name - ); - - let _ = chain - .apply_update(chain_update) - .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err)); - - // all requested heights must exist in the final chain - for height in t.request_heights { - let exp_blockhash = blocks.get(height).expect("block must exist in bitcoind"); - assert_eq!( - chain.get(*height).map(|cp| cp.hash()), - Some(*exp_blockhash), - "[{}:{}] block {}:{} must exist in final chain", - i, - t.name, - height, - exp_blockhash - ); - } - } - - Ok(()) -} From eded1a7ea0c6a4b9664826df4f77b714cbad0bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 15 Apr 2024 14:37:11 +0800 Subject: [PATCH 07/13] feat(chain): introduce `CheckPoint::insert` Co-authored-by: LLFourn --- crates/chain/src/local_chain.rs | 37 ++++++++++++++ crates/chain/tests/test_local_chain.rs | 71 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index 100a9662c..a86f1a77d 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -187,6 +187,43 @@ impl CheckPoint { core::ops::Bound::Unbounded => true, }) } + + /// Inserts `block_id` at its height within the chain. + /// + /// The effect of `insert` depends on whether a height already exists. If it doesn't the + /// `block_id` we inserted and all pre-existing blocks higher than it will be re-inserted after + /// it. If the height already existed and has a conflicting block hash then it will be purged + /// along with all block followin it. The returned chain will have a tip of the `block_id` + /// passed in. Of course, if the `block_id` was already present then this just returns `self`. + #[must_use] + pub fn insert(self, block_id: BlockId) -> Self { + assert_ne!(block_id.height, 0, "cannot insert the genesis block"); + + let mut cp = self.clone(); + let mut tail = vec![]; + let base = loop { + if cp.height() == block_id.height { + if cp.hash() == block_id.hash { + return self; + } + // if we have a conflict we just return the inserted block because the tail is by + // implication invalid. + tail = vec![]; + break cp.prev().expect("can't be called on genesis block"); + } + + if cp.height() < block_id.height { + break cp; + } + + tail.push(cp.block_id()); + cp = cp.prev().expect("will break before genesis block"); + }; + + base + .extend(core::iter::once(block_id).chain(tail.into_iter().rev())) + .expect("tail is in order") + } } /// Iterates over checkpoints backwards. diff --git a/crates/chain/tests/test_local_chain.rs b/crates/chain/tests/test_local_chain.rs index 70a089016..e5ec74816 100644 --- a/crates/chain/tests/test_local_chain.rs +++ b/crates/chain/tests/test_local_chain.rs @@ -577,6 +577,77 @@ fn checkpoint_query() { } } +#[test] +fn checkpoint_insert() { + struct TestCase<'a> { + /// The name of the test. + name: &'a str, + /// The original checkpoint chain to call [`CheckPoint::insert`] on. + chain: &'a [(u32, BlockHash)], + /// The `block_id` to insert. + to_insert: (u32, BlockHash), + /// The expected final checkpoint chain after calling [`CheckPoint::insert`]. + exp_final_chain: &'a [(u32, BlockHash)], + } + + let test_cases = [ + TestCase { + name: "insert_above_tip", + chain: &[(1, h!("a")), (2, h!("b"))], + to_insert: (4, h!("d")), + exp_final_chain: &[(1, h!("a")), (2, h!("b")), (4, h!("d"))], + }, + TestCase { + name: "insert_already_exists_expect_no_change", + chain: &[(1, h!("a")), (2, h!("b")), (3, h!("c"))], + to_insert: (2, h!("b")), + exp_final_chain: &[(1, h!("a")), (2, h!("b")), (3, h!("c"))], + }, + TestCase { + name: "insert_in_middle", + chain: &[(2, h!("b")), (4, h!("d")), (5, h!("e"))], + to_insert: (3, h!("c")), + exp_final_chain: &[(2, h!("b")), (3, h!("c")), (4, h!("d")), (5, h!("e"))], + }, + TestCase { + name: "replace_one", + chain: &[(3, h!("c")), (4, h!("d")), (5, h!("e"))], + to_insert: (5, h!("E")), + exp_final_chain: &[(3, h!("c")), (4, h!("d")), (5, h!("E"))], + }, + TestCase { + name: "insert_conflict_should_evict", + chain: &[(3, h!("c")), (4, h!("d")), (5, h!("e")), (6, h!("f"))], + to_insert: (4, h!("D")), + exp_final_chain: &[(3, h!("c")), (4, h!("D"))], + }, + ]; + + fn genesis_block() -> impl Iterator { + core::iter::once((0, h!("_"))).map(BlockId::from) + } + + for (i, t) in test_cases.into_iter().enumerate() { + println!("Running [{}] '{}'", i, t.name); + + let chain = CheckPoint::from_block_ids( + genesis_block().chain(t.chain.iter().copied().map(BlockId::from)), + ) + .expect("test formed incorrectly, must construct checkpoint chain"); + + let exp_final_chain = CheckPoint::from_block_ids( + genesis_block().chain(t.exp_final_chain.iter().copied().map(BlockId::from)), + ) + .expect("test formed incorrectly, must construct checkpoint chain"); + + assert_eq!( + chain.insert(t.to_insert.into()), + exp_final_chain, + "unexpected final chain" + ); + } +} + #[test] fn local_chain_apply_header_connected_to() { fn header_from_prev_blockhash(prev_blockhash: BlockHash) -> Header { From 72fe65b65f297ebb7160eee6859c46e29c2d9528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 16 Apr 2024 17:30:58 +0800 Subject: [PATCH 08/13] feat(esplora)!: simplify chain update logic Co-authored-by: LLFourn --- crates/chain/src/local_chain.rs | 3 +- crates/esplora/src/async_ext.rs | 189 +++++++++++++------------ crates/esplora/src/blocking_ext.rs | 215 ++++++++++++++--------------- 3 files changed, 196 insertions(+), 211 deletions(-) diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index a86f1a77d..6fa85d422 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -220,8 +220,7 @@ impl CheckPoint { cp = cp.prev().expect("will break before genesis block"); }; - base - .extend(core::iter::once(block_id).chain(tail.into_iter().rev())) + base.extend(core::iter::once(block_id).chain(tail.into_iter().rev())) .expect("tail is in order") } } diff --git a/crates/esplora/src/async_ext.rs b/crates/esplora/src/async_ext.rs index b387cb771..1abc28c81 100644 --- a/crates/esplora/src/async_ext.rs +++ b/crates/esplora/src/async_ext.rs @@ -1,7 +1,6 @@ use std::collections::BTreeSet; use async_trait::async_trait; -use bdk_chain::collections::btree_map; use bdk_chain::Anchor; use bdk_chain::{ bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, @@ -97,11 +96,11 @@ impl EsploraAsyncExt for esplora_client::AsyncClient { stop_gap: usize, parallel_requests: usize, ) -> Result, Error> { - let update_blocks = init_chain_update(self, &local_tip).await?; + let latest_blocks = fetch_latest_blocks(self).await?; let (tx_graph, last_active_indices) = full_scan_for_index_and_graph(self, keychain_spks, stop_gap, parallel_requests).await?; let local_chain = - finalize_chain_update(self, &local_tip, tx_graph.all_anchors(), update_blocks).await?; + chain_update(self, &latest_blocks, &local_tip, tx_graph.all_anchors()).await?; Ok(FullScanUpdate { local_chain, tx_graph, @@ -117,11 +116,11 @@ impl EsploraAsyncExt for esplora_client::AsyncClient { outpoints: impl IntoIterator + Send> + Send, parallel_requests: usize, ) -> Result { - let update_blocks = init_chain_update(self, &local_tip).await?; + let latest_blocks = fetch_latest_blocks(self).await?; let tx_graph = sync_for_index_and_graph(self, misc_spks, txids, outpoints, parallel_requests).await?; let local_chain = - finalize_chain_update(self, &local_tip, tx_graph.all_anchors(), update_blocks).await?; + chain_update(self, &latest_blocks, &local_tip, tx_graph.all_anchors()).await?; Ok(SyncUpdate { tx_graph, local_chain, @@ -129,112 +128,105 @@ impl EsploraAsyncExt for esplora_client::AsyncClient { } } -/// Create the initial chain update. +/// Fetch latest blocks from Esplora in an atomic call. /// -/// This atomically fetches the latest blocks from Esplora and additional blocks to ensure the -/// update can connect to the `start_tip`. -/// -/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks and +/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks AND /// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for /// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use /// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when /// alternating between chain-sources. -async fn init_chain_update( +async fn fetch_latest_blocks( client: &esplora_client::AsyncClient, - local_tip: &CheckPoint, ) -> Result, Error> { - // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are - // consistent. - let mut fetched_blocks = client + Ok(client .get_blocks(None) .await? .into_iter() .map(|b| (b.time.height, b.id)) - .collect::>(); - let new_tip_height = fetched_blocks - .keys() - .last() - .copied() - .expect("must atleast have one block"); - - // Ensure `fetched_blocks` can create an update that connects with the original chain by - // finding a "Point of Agreement". - for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { - if height > new_tip_height { - continue; - } + .collect()) +} - let fetched_hash = match fetched_blocks.entry(height) { - btree_map::Entry::Occupied(entry) => *entry.get(), - btree_map::Entry::Vacant(entry) => *entry.insert(client.get_block_hash(height).await?), - }; +/// Used instead of [`esplora_client::BlockingClient::get_block_hash`]. +/// +/// This first checks the previously fetched `latest_blocks` before fetching from Esplora again. +async fn fetch_block( + client: &esplora_client::AsyncClient, + latest_blocks: &BTreeMap, + height: u32, +) -> Result, Error> { + if let Some(&hash) = latest_blocks.get(&height) { + return Ok(Some(hash)); + } - // We have found point of agreement so the update will connect! - if fetched_hash == local_hash { - break; - } + // We avoid fetching blocks higher than previously fetched `latest_blocks` as the local chain + // tip is used to signal for the last-synced-up-to-height. + let &tip_height = latest_blocks + .keys() + .last() + .expect("must have atleast one entry"); + if height > tip_height { + return Ok(None); } - Ok(fetched_blocks) + Ok(Some(client.get_block_hash(height).await?)) } -/// Fetches missing checkpoints and finalizes the [`local_chain::Update`]. +/// Create the [`local_chain::Update`]. /// -/// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an -/// existing checkpoint/block under `local_tip` or `update_blocks`. -async fn finalize_chain_update( +/// We want to have a corresponding checkpoint per anchor height. However, checkpoints fetched +/// should not surpass `latest_blocks`. +async fn chain_update( client: &esplora_client::AsyncClient, + latest_blocks: &BTreeMap, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, - mut update_blocks: BTreeMap, ) -> Result { - let update_tip_height = update_blocks - .keys() - .last() - .copied() - .expect("must atleast have one block"); - - // We want to have a corresponding checkpoint per height. We iterate the heights of anchors - // backwards, comparing it against our `local_tip`'s chain and our current set of - // `update_blocks` to see if a corresponding checkpoint already exists. - let anchor_heights = anchors - .iter() - .rev() - .map(|(a, _)| a.anchor_block().height) - // filter out heights that surpass the update tip - .filter(|h| *h <= update_tip_height) - // filter out duplicate heights - .filter({ - let mut prev_height = Option::::None; - move |h| match prev_height.replace(*h) { - None => true, - Some(prev_h) => prev_h != *h, - } - }); + let mut point_of_agreement = None; + let mut conflicts = vec![]; + for local_cp in local_tip.iter() { + let remote_hash = match fetch_block(client, latest_blocks, local_cp.height()).await? { + Some(hash) => hash, + None => continue, + }; + if remote_hash == local_cp.hash() { + point_of_agreement = Some(local_cp.clone()); + break; + } else { + // it is not strictly necessary to include all the conflicted heights (we do need the + // first one) but it seems prudent to make sure the updated chain's heights are a + // superset of the existing chain after update. + conflicts.push(BlockId { + height: local_cp.height(), + hash: remote_hash, + }); + } + } - // We keep track of a checkpoint node of `local_tip` to make traversing the linked-list of - // checkpoints more efficient. - let mut curr_cp = local_tip.clone(); + let mut tip = point_of_agreement.expect("remote esplora should have same genesis block"); - for h in anchor_heights { - if let Some(cp) = curr_cp.range(h..).last() { - curr_cp = cp.clone(); - if cp.height() == h { - continue; - } - } - if let btree_map::Entry::Vacant(entry) = update_blocks.entry(h) { - entry.insert(client.get_block_hash(h).await?); + tip = tip + .extend(conflicts.into_iter().rev()) + .expect("evicted are in order"); + + for anchor in anchors { + let height = anchor.0.anchor_block().height; + if tip.get(height).is_none() { + let hash = match fetch_block(client, latest_blocks, height).await? { + Some(hash) => hash, + None => continue, + }; + tip = tip.insert(BlockId { height, hash }); } } + // insert the most recent blocks at the tip to make sure we update the tip and make the update + // robust. + for (&height, &hash) in latest_blocks.iter() { + tip = tip.insert(BlockId { height, hash }); + } + Ok(local_chain::Update { - tip: CheckPoint::from_block_ids( - update_blocks - .into_iter() - .map(|(height, hash)| BlockId { height, hash }), - ) - .expect("must be in order"), + tip, introduce_older_blocks: true, }) } @@ -424,7 +416,7 @@ mod test { use electrsd::bitcoind::bitcoincore_rpc::RpcApi; use esplora_client::Builder; - use crate::async_ext::{finalize_chain_update, init_chain_update}; + use crate::async_ext::{chain_update, fetch_latest_blocks}; macro_rules! h { ($index:literal) => {{ @@ -493,9 +485,8 @@ mod test { // craft initial `local_chain` let local_chain = { let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); - let chain_tip = chain.tip(); - let update_blocks = init_chain_update(&client, &chain_tip).await?; - let update_anchors = t + // force `chain_update_blocking` to add all checkpoints in `t.initial_cps` + let anchors = t .initial_cps .iter() .map(|&height| -> anyhow::Result<_> { @@ -508,10 +499,14 @@ mod test { )) }) .collect::>>()?; - let chain_update = - finalize_chain_update(&client, &chain_tip, &update_anchors, update_blocks) - .await?; - chain.apply_update(chain_update)?; + let update = chain_update( + &client, + &fetch_latest_blocks(&client).await?, + &chain.tip(), + &anchors, + ) + .await?; + chain.apply_update(update)?; chain }; println!("local chain height: {}", local_chain.tip().height()); @@ -529,9 +524,7 @@ mod test { // craft update let update = { - let local_tip = local_chain.tip(); - let update_blocks = init_chain_update(&client, &local_tip).await?; - let update_anchors = t + let anchors = t .anchors .iter() .map(|&(height, txid)| -> anyhow::Result<_> { @@ -544,7 +537,13 @@ mod test { )) }) .collect::>()?; - finalize_chain_update(&client, &local_tip, &update_anchors, update_blocks).await? + chain_update( + &client, + &fetch_latest_blocks(&client).await?, + &local_chain.tip(), + &anchors, + ) + .await? }; // apply update diff --git a/crates/esplora/src/blocking_ext.rs b/crates/esplora/src/blocking_ext.rs index 419a2ae66..5b7cd6288 100644 --- a/crates/esplora/src/blocking_ext.rs +++ b/crates/esplora/src/blocking_ext.rs @@ -2,7 +2,6 @@ use std::collections::BTreeSet; use std::thread::JoinHandle; use std::usize; -use bdk_chain::collections::btree_map; use bdk_chain::collections::BTreeMap; use bdk_chain::Anchor; use bdk_chain::{ @@ -89,19 +88,14 @@ impl EsploraExt for esplora_client::BlockingClient { stop_gap: usize, parallel_requests: usize, ) -> Result, Error> { - let update_blocks = init_chain_update_blocking(self, &local_tip)?; + let latest_blocks = fetch_latest_blocks(self)?; let (tx_graph, last_active_indices) = full_scan_for_index_and_graph_blocking( self, keychain_spks, stop_gap, parallel_requests, )?; - let local_chain = finalize_chain_update_blocking( - self, - &local_tip, - tx_graph.all_anchors(), - update_blocks, - )?; + let local_chain = chain_update(self, &latest_blocks, &local_tip, tx_graph.all_anchors())?; Ok(FullScanUpdate { local_chain, tx_graph, @@ -117,7 +111,7 @@ impl EsploraExt for esplora_client::BlockingClient { outpoints: impl IntoIterator, parallel_requests: usize, ) -> Result { - let update_blocks = init_chain_update_blocking(self, &local_tip)?; + let latest_blocks = fetch_latest_blocks(self)?; let tx_graph = sync_for_index_and_graph_blocking( self, misc_spks, @@ -125,12 +119,7 @@ impl EsploraExt for esplora_client::BlockingClient { outpoints, parallel_requests, )?; - let local_chain = finalize_chain_update_blocking( - self, - &local_tip, - tx_graph.all_anchors(), - update_blocks, - )?; + let local_chain = chain_update(self, &latest_blocks, &local_tip, tx_graph.all_anchors())?; Ok(SyncUpdate { local_chain, tx_graph, @@ -138,111 +127,104 @@ impl EsploraExt for esplora_client::BlockingClient { } } -/// Create the initial chain update. -/// -/// This atomically fetches the latest blocks from Esplora and additional blocks to ensure the -/// update can connect to the `start_tip`. +/// Fetch latest blocks from Esplora in an atomic call. /// -/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks and +/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks AND /// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for /// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use /// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when /// alternating between chain-sources. -fn init_chain_update_blocking( +fn fetch_latest_blocks( client: &esplora_client::BlockingClient, - local_tip: &CheckPoint, ) -> Result, Error> { - // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are - // consistent. - let mut fetched_blocks = client + Ok(client .get_blocks(None)? .into_iter() .map(|b| (b.time.height, b.id)) - .collect::>(); - let new_tip_height = fetched_blocks - .keys() - .last() - .copied() - .expect("must atleast have one block"); - - // Ensure `fetched_blocks` can create an update that connects with the original chain by - // finding a "Point of Agreement". - for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) { - if height > new_tip_height { - continue; - } + .collect()) +} - let fetched_hash = match fetched_blocks.entry(height) { - btree_map::Entry::Occupied(entry) => *entry.get(), - btree_map::Entry::Vacant(entry) => *entry.insert(client.get_block_hash(height)?), - }; +/// Used instead of [`esplora_client::BlockingClient::get_block_hash`]. +/// +/// This first checks the previously fetched `latest_blocks` before fetching from Esplora again. +fn fetch_block( + client: &esplora_client::BlockingClient, + latest_blocks: &BTreeMap, + height: u32, +) -> Result, Error> { + if let Some(&hash) = latest_blocks.get(&height) { + return Ok(Some(hash)); + } - // We have found point of agreement so the update will connect! - if fetched_hash == local_hash { - break; - } + // We avoid fetching blocks higher than previously fetched `latest_blocks` as the local chain + // tip is used to signal for the last-synced-up-to-height. + let &tip_height = latest_blocks + .keys() + .last() + .expect("must have atleast one entry"); + if height > tip_height { + return Ok(None); } - Ok(fetched_blocks) + Ok(Some(client.get_block_hash(height)?)) } -/// Fetches missing checkpoints and finalizes the [`local_chain::Update`]. +/// Create the [`local_chain::Update`]. /// -/// A checkpoint is considered "missing" if an anchor (of `anchors`) points to a height without an -/// existing checkpoint/block under `local_tip` or `update_blocks`. -fn finalize_chain_update_blocking( +/// We want to have a corresponding checkpoint per anchor height. However, checkpoints fetched +/// should not surpass `latest_blocks`. +fn chain_update( client: &esplora_client::BlockingClient, + latest_blocks: &BTreeMap, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, - mut update_blocks: BTreeMap, ) -> Result { - let update_tip_height = update_blocks - .keys() - .last() - .copied() - .expect("must atleast have one block"); - - // We want to have a corresponding checkpoint per height. We iterate the heights of anchors - // backwards, comparing it against our `local_tip`'s chain and our current set of - // `update_blocks` to see if a corresponding checkpoint already exists. - let anchor_heights = anchors - .iter() - .rev() - .map(|(a, _)| a.anchor_block().height) - // filter out heights that surpass the update tip - .filter(|h| *h <= update_tip_height) - // filter out duplicate heights - .filter({ - let mut prev_height = Option::::None; - move |h| match prev_height.replace(*h) { - None => true, - Some(prev_h) => prev_h != *h, - } - }); + let mut point_of_agreement = None; + let mut conflicts = vec![]; + for local_cp in local_tip.iter() { + let remote_hash = match fetch_block(client, latest_blocks, local_cp.height())? { + Some(hash) => hash, + None => continue, + }; + if remote_hash == local_cp.hash() { + point_of_agreement = Some(local_cp.clone()); + break; + } else { + // it is not strictly necessary to include all the conflicted heights (we do need the + // first one) but it seems prudent to make sure the updated chain's heights are a + // superset of the existing chain after update. + conflicts.push(BlockId { + height: local_cp.height(), + hash: remote_hash, + }); + } + } - // We keep track of a checkpoint node of `local_tip` to make traversing the linked-list of - // checkpoints more efficient. - let mut curr_cp = local_tip.clone(); + let mut tip = point_of_agreement.expect("remote esplora should have same genesis block"); - for h in anchor_heights { - if let Some(cp) = curr_cp.range(h..).last() { - curr_cp = cp.clone(); - if cp.height() == h { - continue; - } - } - if let btree_map::Entry::Vacant(entry) = update_blocks.entry(h) { - entry.insert(client.get_block_hash(h)?); + tip = tip + .extend(conflicts.into_iter().rev()) + .expect("evicted are in order"); + + for anchor in anchors { + let height = anchor.0.anchor_block().height; + if tip.get(height).is_none() { + let hash = match fetch_block(client, latest_blocks, height)? { + Some(hash) => hash, + None => continue, + }; + tip = tip.insert(BlockId { height, hash }); } } + // insert the most recent blocks at the tip to make sure we update the tip and make the update + // robust. + for (&height, &hash) in latest_blocks.iter() { + tip = tip.insert(BlockId { height, hash }); + } + Ok(local_chain::Update { - tip: CheckPoint::from_block_ids( - update_blocks - .into_iter() - .map(|(height, hash)| BlockId { height, hash }), - ) - .expect("must be in order"), + tip, introduce_older_blocks: true, }) } @@ -430,7 +412,7 @@ fn sync_for_index_and_graph_blocking( #[cfg(test)] mod test { - use crate::blocking_ext::{finalize_chain_update_blocking, init_chain_update_blocking}; + use crate::blocking_ext::{chain_update, fetch_latest_blocks}; use bdk_chain::bitcoin::hashes::Hash; use bdk_chain::bitcoin::Txid; use bdk_chain::local_chain::LocalChain; @@ -462,7 +444,7 @@ mod test { name: &'a str, /// Initial blockchain height to start the env with. initial_env_height: u32, - /// Initial checkpoint heights to start with. + /// Initial checkpoint heights to start with in the local chain. initial_cps: &'a [u32], /// The final blockchain height of the env. final_env_height: u32, @@ -516,9 +498,8 @@ mod test { // craft initial `local_chain` let local_chain = { let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?); - let chain_tip = chain.tip(); - let update_blocks = init_chain_update_blocking(&client, &chain_tip)?; - let update_anchors = t + // force `chain_update_blocking` to add all checkpoints in `t.initial_cps` + let anchors = t .initial_cps .iter() .map(|&height| -> anyhow::Result<_> { @@ -531,13 +512,13 @@ mod test { )) }) .collect::>>()?; - let chain_update = finalize_chain_update_blocking( + let update = chain_update( &client, - &chain_tip, - &update_anchors, - update_blocks, + &fetch_latest_blocks(&client)?, + &chain.tip(), + &anchors, )?; - chain.apply_update(chain_update)?; + chain.apply_update(update)?; chain }; println!("local chain height: {}", local_chain.tip().height()); @@ -555,9 +536,7 @@ mod test { // craft update let update = { - let local_tip = local_chain.tip(); - let update_blocks = init_chain_update_blocking(&client, &local_tip)?; - let update_anchors = t + let anchors = t .anchors .iter() .map(|&(height, txid)| -> anyhow::Result<_> { @@ -570,7 +549,12 @@ mod test { )) }) .collect::>()?; - finalize_chain_update_blocking(&client, &local_tip, &update_anchors, update_blocks)? + chain_update( + &client, + &fetch_latest_blocks(&client)?, + &local_chain.tip(), + &anchors, + )? }; // apply update @@ -640,8 +624,12 @@ mod test { struct TestCase { name: &'static str, + /// Original local chain to start off with. chain: LocalChain, + /// Heights of floating anchors. [`chain_update_blocking`] will request for checkpoints + /// of these heights. request_heights: &'static [u32], + /// The expected local chain result (heights only). exp_update_heights: &'static [u32], } @@ -738,11 +726,6 @@ mod test { for (i, t) in test_cases.into_iter().enumerate() { println!("Case {}: {}", i, t.name); let mut chain = t.chain; - let cp_tip = chain.tip(); - - let new_blocks = init_chain_update_blocking(&client, &cp_tip).map_err(|err| { - anyhow::format_err!("[{}:{}] `init_chain_update` failed: {}", i, t.name, err) - })?; let mock_anchors = t .request_heights @@ -761,9 +744,13 @@ mod test { (anchor, txid) }) .collect::>(); + let chain_update = chain_update( + &client, + &fetch_latest_blocks(&client)?, + &chain.tip(), + &mock_anchors, + )?; - let chain_update = - finalize_chain_update_blocking(&client, &cp_tip, &mock_anchors, new_blocks)?; let update_blocks = chain_update .tip .iter() From 1269b0610efb7bd86d92a909800f9330568c797a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 17 Apr 2024 10:44:26 +0800 Subject: [PATCH 09/13] test(chain): fix incorrect test case --- crates/chain/tests/test_local_chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/chain/tests/test_local_chain.rs b/crates/chain/tests/test_local_chain.rs index e5ec74816..c331008db 100644 --- a/crates/chain/tests/test_local_chain.rs +++ b/crates/chain/tests/test_local_chain.rs @@ -672,9 +672,9 @@ fn local_chain_apply_header_connected_to() { let test_cases = [ { - let header = header_from_prev_blockhash(h!("A")); + let header = header_from_prev_blockhash(h!("_")); let hash = header.block_hash(); - let height = 2; + let height = 1; let connected_to = BlockId { height, hash }; TestCase { name: "connected_to_self_header_applied_to_self", From 77d35954c1f3a18f767267e9097f63ca11c709ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 17 Apr 2024 10:02:12 +0800 Subject: [PATCH 10/13] feat(chain)!: rm `local_chain::Update` The intention is to remove the `Update::introduce_older_blocks` parameter and update the local chain directly with `CheckPoint`. This simplifies the API and there is a way to do this efficiently. --- crates/bdk/src/wallet/mod.rs | 2 +- crates/bitcoind_rpc/tests/test_emitter.rs | 25 ++------ crates/chain/src/local_chain.rs | 62 +++---------------- crates/chain/tests/common/mod.rs | 9 +-- crates/chain/tests/test_local_chain.rs | 4 +- crates/electrum/src/electrum_ext.rs | 9 +-- crates/esplora/src/async_ext.rs | 13 ++-- crates/esplora/src/blocking_ext.rs | 14 ++--- crates/esplora/src/lib.rs | 10 +-- crates/esplora/tests/async_ext.rs | 1 - crates/esplora/tests/blocking_ext.rs | 1 - .../example_bitcoind_rpc_polling/src/main.rs | 11 +--- 12 files changed, 39 insertions(+), 122 deletions(-) diff --git a/crates/bdk/src/wallet/mod.rs b/crates/bdk/src/wallet/mod.rs index 846878823..f11abb30b 100644 --- a/crates/bdk/src/wallet/mod.rs +++ b/crates/bdk/src/wallet/mod.rs @@ -107,7 +107,7 @@ pub struct Update { /// Update for the wallet's internal [`LocalChain`]. /// /// [`LocalChain`]: local_chain::LocalChain - pub chain: Option, + pub chain: Option, } /// The changes made to a wallet by applying an [`Update`]. diff --git a/crates/bitcoind_rpc/tests/test_emitter.rs b/crates/bitcoind_rpc/tests/test_emitter.rs index f2e2a5d59..c517740f5 100644 --- a/crates/bitcoind_rpc/tests/test_emitter.rs +++ b/crates/bitcoind_rpc/tests/test_emitter.rs @@ -4,7 +4,7 @@ use bdk_bitcoind_rpc::Emitter; use bdk_chain::{ bitcoin::{Address, Amount, Txid}, keychain::Balance, - local_chain::{self, CheckPoint, LocalChain}, + local_chain::{CheckPoint, LocalChain}, Append, BlockId, IndexedTxGraph, SpkTxOutIndex, }; use bdk_testenv::TestEnv; @@ -47,10 +47,7 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> { ); assert_eq!( - local_chain.apply_update(local_chain::Update { - tip: emission.checkpoint, - introduce_older_blocks: false, - })?, + local_chain.apply_update(emission.checkpoint,)?, BTreeMap::from([(height, Some(hash))]), "chain update changeset is unexpected", ); @@ -95,10 +92,7 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> { ); assert_eq!( - local_chain.apply_update(local_chain::Update { - tip: emission.checkpoint, - introduce_older_blocks: false, - })?, + local_chain.apply_update(emission.checkpoint,)?, if exp_height == exp_hashes.len() - reorged_blocks.len() { core::iter::once((height, Some(hash))) .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None))) @@ -168,10 +162,7 @@ fn test_into_tx_graph() -> anyhow::Result<()> { while let Some(emission) = emitter.next_block()? { let height = emission.block_height(); - let _ = chain.apply_update(local_chain::Update { - tip: emission.checkpoint, - introduce_older_blocks: false, - })?; + let _ = chain.apply_update(emission.checkpoint)?; let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height); assert!(indexed_additions.is_empty()); } @@ -232,10 +223,7 @@ fn test_into_tx_graph() -> anyhow::Result<()> { { let emission = emitter.next_block()?.expect("must get mined block"); let height = emission.block_height(); - let _ = chain.apply_update(local_chain::Update { - tip: emission.checkpoint, - introduce_older_blocks: false, - })?; + let _ = chain.apply_update(emission.checkpoint)?; let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height); assert!(indexed_additions.graph.txs.is_empty()); assert!(indexed_additions.graph.txouts.is_empty()); @@ -294,8 +282,7 @@ fn process_block( block: Block, block_height: u32, ) -> anyhow::Result<()> { - recv_chain - .apply_update(CheckPoint::from_header(&block.header, block_height).into_update(false))?; + recv_chain.apply_update(CheckPoint::from_header(&block.header, block_height))?; let _ = recv_graph.apply_block(block, block_height); Ok(()) } diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index 6fa85d422..792b987f5 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -96,16 +96,6 @@ impl CheckPoint { .expect("must construct checkpoint") } - /// Convenience method to convert the [`CheckPoint`] into an [`Update`]. - /// - /// For more information, refer to [`Update`]. - pub fn into_update(self, introduce_older_blocks: bool) -> Update { - Update { - tip: self, - introduce_older_blocks, - } - } - /// Puts another checkpoint onto the linked list representing the blockchain. /// /// Returns an `Err(self)` if the block you are pushing on is not at a greater height that the one you @@ -251,31 +241,6 @@ impl IntoIterator for CheckPoint { } } -/// Used to update [`LocalChain`]. -/// -/// This is used as input for [`LocalChain::apply_update`]. It contains the update's chain `tip` and -/// a flag `introduce_older_blocks` which signals whether this update intends to introduce missing -/// blocks to the original chain. -/// -/// Block-by-block syncing mechanisms would typically create updates that builds upon the previous -/// tip. In this case, `introduce_older_blocks` would be `false`. -/// -/// Script-pubkey based syncing mechanisms may not introduce transactions in a chronological order -/// so some updates require introducing older blocks (to anchor older transactions). For -/// script-pubkey based syncing, `introduce_older_blocks` would typically be `true`. -#[derive(Debug, Clone, PartialEq)] -pub struct Update { - /// The update chain's new tip. - pub tip: CheckPoint, - - /// Whether the update allows for introducing older blocks. - /// - /// Refer to [struct-level documentation] for more. - /// - /// [struct-level documentation]: Update - pub introduce_older_blocks: bool, -} - /// This is a local implementation of [`ChainOracle`]. #[derive(Debug, Clone, PartialEq)] pub struct LocalChain { @@ -390,23 +355,16 @@ impl LocalChain { /// the existing chain and invalidate the block after it (if it exists) by including a block at /// the same height but with a different hash to explicitly exclude it as a connection point. /// - /// Additionally, an empty chain can be updated with any chain, and a chain with a single block - /// can have it's block invalidated by an update chain with a block at the same height but - /// different hash. + /// Additionally, a chain with a single block can have it's block invalidated by an update + /// chain with a block at the same height but different hash. /// /// # Errors /// /// An error will occur if the update does not correctly connect with `self`. /// - /// Refer to [`Update`] for more about the update struct. - /// /// [module-level documentation]: crate::local_chain - pub fn apply_update(&mut self, update: Update) -> Result { - let changeset = merge_chains( - self.tip.clone(), - update.tip.clone(), - update.introduce_older_blocks, - )?; + pub fn apply_update(&mut self, update: CheckPoint) -> Result { + let changeset = merge_chains(self.tip.clone(), update.clone())?; // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in // `.apply_changeset` self.apply_changeset(&changeset) @@ -464,11 +422,8 @@ impl LocalChain { conn => Some(conn), }; - let update = Update { - tip: CheckPoint::from_block_ids([conn, prev, Some(this)].into_iter().flatten()) - .expect("block ids must be in order"), - introduce_older_blocks: false, - }; + let update = CheckPoint::from_block_ids([conn, prev, Some(this)].into_iter().flatten()) + .expect("block ids must be in order"); self.apply_update(update) .map_err(ApplyHeaderError::CannotConnect) @@ -769,7 +724,6 @@ impl std::error::Error for ApplyHeaderError {} fn merge_chains( original_tip: CheckPoint, update_tip: CheckPoint, - introduce_older_blocks: bool, ) -> Result { let mut changeset = ChangeSet::default(); let mut orig = original_tip.into_iter(); @@ -829,11 +783,9 @@ fn merge_chains( } point_of_agreement_found = true; prev_orig_was_invalidated = false; - // OPTIMIZATION 1 -- If we know that older blocks cannot be introduced without - // invalidation, we can break after finding the point of agreement. // OPTIMIZATION 2 -- if we have the same underlying pointer at this point, we // can guarantee that no older blocks are introduced. - if !introduce_older_blocks || Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) { + if Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) { return Ok(changeset); } } else { diff --git a/crates/chain/tests/common/mod.rs b/crates/chain/tests/common/mod.rs index 6116484ce..586e64043 100644 --- a/crates/chain/tests/common/mod.rs +++ b/crates/chain/tests/common/mod.rs @@ -32,12 +32,9 @@ macro_rules! local_chain { macro_rules! chain_update { [ $(($height:expr, $hash:expr)), * ] => {{ #[allow(unused_mut)] - bdk_chain::local_chain::Update { - tip: bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect()) - .expect("chain must have genesis block") - .tip(), - introduce_older_blocks: true, - } + bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect()) + .expect("chain must have genesis block") + .tip() }}; } diff --git a/crates/chain/tests/test_local_chain.rs b/crates/chain/tests/test_local_chain.rs index c331008db..cff4a8e73 100644 --- a/crates/chain/tests/test_local_chain.rs +++ b/crates/chain/tests/test_local_chain.rs @@ -3,7 +3,7 @@ use std::ops::{Bound, RangeBounds}; use bdk_chain::{ local_chain::{ AlterCheckPointError, ApplyHeaderError, CannotConnectError, ChangeSet, CheckPoint, - LocalChain, MissingGenesisError, Update, + LocalChain, MissingGenesisError, }, BlockId, }; @@ -17,7 +17,7 @@ mod common; struct TestLocalChain<'a> { name: &'static str, chain: LocalChain, - update: Update, + update: CheckPoint, exp: ExpectedResult<'a>, } diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 3ff467fce..5a6c5d116 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,6 +1,6 @@ use bdk_chain::{ bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, - local_chain::{self, CheckPoint}, + local_chain::CheckPoint, tx_graph::{self, TxGraph}, Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; @@ -124,7 +124,7 @@ impl RelevantTxids { #[derive(Debug)] pub struct ElectrumUpdate { /// Chain update - pub chain_update: local_chain::Update, + pub chain_update: CheckPoint, /// Transaction updates from electrum pub relevant_txids: RelevantTxids, } @@ -232,10 +232,7 @@ impl ElectrumExt for A { continue; // reorg } - let chain_update = local_chain::Update { - tip, - introduce_older_blocks: true, - }; + let chain_update = tip; let keychain_update = request_spks .into_keys() diff --git a/crates/esplora/src/async_ext.rs b/crates/esplora/src/async_ext.rs index 1abc28c81..9d02646c6 100644 --- a/crates/esplora/src/async_ext.rs +++ b/crates/esplora/src/async_ext.rs @@ -5,7 +5,7 @@ use bdk_chain::Anchor; use bdk_chain::{ bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, collections::BTreeMap, - local_chain::{self, CheckPoint}, + local_chain::CheckPoint, BlockId, ConfirmationTimeHeightAnchor, TxGraph, }; use esplora_client::{Amount, TxStatus}; @@ -47,7 +47,7 @@ pub trait EsploraAsyncExt { /// /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1. /// - /// [`LocalChain::tip`]: local_chain::LocalChain::tip + /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip async fn full_scan( &self, local_tip: CheckPoint, @@ -71,7 +71,7 @@ pub trait EsploraAsyncExt { /// If the scripts to sync are unknown, such as when restoring or importing a keychain that /// may include scripts that have been used, use [`full_scan`] with the keychain. /// - /// [`LocalChain::tip`]: local_chain::LocalChain::tip + /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip /// [`full_scan`]: EsploraAsyncExt::full_scan async fn sync( &self, @@ -180,7 +180,7 @@ async fn chain_update( latest_blocks: &BTreeMap, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, -) -> Result { +) -> Result { let mut point_of_agreement = None; let mut conflicts = vec![]; for local_cp in local_tip.iter() { @@ -225,10 +225,7 @@ async fn chain_update( tip = tip.insert(BlockId { height, hash }); } - Ok(local_chain::Update { - tip, - introduce_older_blocks: true, - }) + Ok(tip) } /// This performs a full scan to get an update for the [`TxGraph`] and diff --git a/crates/esplora/src/blocking_ext.rs b/crates/esplora/src/blocking_ext.rs index 5b7cd6288..703738569 100644 --- a/crates/esplora/src/blocking_ext.rs +++ b/crates/esplora/src/blocking_ext.rs @@ -6,7 +6,7 @@ use bdk_chain::collections::BTreeMap; use bdk_chain::Anchor; use bdk_chain::{ bitcoin::{Amount, BlockHash, OutPoint, ScriptBuf, TxOut, Txid}, - local_chain::{self, CheckPoint}, + local_chain::CheckPoint, BlockId, ConfirmationTimeHeightAnchor, TxGraph, }; use esplora_client::TxStatus; @@ -47,7 +47,7 @@ pub trait EsploraExt { /// /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1. /// - /// [`LocalChain::tip`]: local_chain::LocalChain::tip + /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip fn full_scan( &self, local_tip: CheckPoint, @@ -68,7 +68,7 @@ pub trait EsploraExt { /// If the scripts to sync are unknown, such as when restoring or importing a keychain that /// may include scripts that have been used, use [`full_scan`] with the keychain. /// - /// [`LocalChain::tip`]: local_chain::LocalChain::tip + /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip /// [`full_scan`]: EsploraExt::full_scan fn sync( &self, @@ -178,7 +178,7 @@ fn chain_update( latest_blocks: &BTreeMap, local_tip: &CheckPoint, anchors: &BTreeSet<(A, Txid)>, -) -> Result { +) -> Result { let mut point_of_agreement = None; let mut conflicts = vec![]; for local_cp in local_tip.iter() { @@ -223,10 +223,7 @@ fn chain_update( tip = tip.insert(BlockId { height, hash }); } - Ok(local_chain::Update { - tip, - introduce_older_blocks: true, - }) + Ok(tip) } /// This performs a full scan to get an update for the [`TxGraph`] and @@ -752,7 +749,6 @@ mod test { )?; let update_blocks = chain_update - .tip .iter() .map(|cp| cp.block_id()) .collect::>(); diff --git a/crates/esplora/src/lib.rs b/crates/esplora/src/lib.rs index c422a0833..37d7dd26e 100644 --- a/crates/esplora/src/lib.rs +++ b/crates/esplora/src/lib.rs @@ -18,7 +18,7 @@ use std::collections::BTreeMap; -use bdk_chain::{local_chain, BlockId, ConfirmationTimeHeightAnchor, TxGraph}; +use bdk_chain::{local_chain::CheckPoint, BlockId, ConfirmationTimeHeightAnchor, TxGraph}; use esplora_client::TxStatus; pub use esplora_client; @@ -53,8 +53,8 @@ fn anchor_from_status(status: &TxStatus) -> Option /// Update returns from a full scan. pub struct FullScanUpdate { - /// The update to apply to the receiving [`LocalChain`](local_chain::LocalChain). - pub local_chain: local_chain::Update, + /// The update to apply to the receiving [`LocalChain`](bdk_chain::local_chain::LocalChain). + pub local_chain: CheckPoint, /// The update to apply to the receiving [`TxGraph`]. pub tx_graph: TxGraph, /// Last active indices for the corresponding keychains (`K`). @@ -63,8 +63,8 @@ pub struct FullScanUpdate { /// Update returned from a sync. pub struct SyncUpdate { - /// The update to apply to the receiving [`LocalChain`](local_chain::LocalChain). - pub local_chain: local_chain::Update, + /// The update to apply to the receiving [`LocalChain`](bdk_chain::local_chain::LocalChain). + pub local_chain: CheckPoint, /// The update to apply to the receiving [`TxGraph`]. pub tx_graph: TxGraph, } diff --git a/crates/esplora/tests/async_ext.rs b/crates/esplora/tests/async_ext.rs index 5946bb4d8..f6954fe11 100644 --- a/crates/esplora/tests/async_ext.rs +++ b/crates/esplora/tests/async_ext.rs @@ -69,7 +69,6 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { { let update_cps = sync_update .local_chain - .tip .iter() .map(|cp| cp.block_id()) .collect::>(); diff --git a/crates/esplora/tests/blocking_ext.rs b/crates/esplora/tests/blocking_ext.rs index 3f8ff6932..40e446a4e 100644 --- a/crates/esplora/tests/blocking_ext.rs +++ b/crates/esplora/tests/blocking_ext.rs @@ -67,7 +67,6 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> { { let update_cps = sync_update .local_chain - .tip .iter() .map(|cp| cp.block_id()) .collect::>(); diff --git a/example-crates/example_bitcoind_rpc_polling/src/main.rs b/example-crates/example_bitcoind_rpc_polling/src/main.rs index 88b83067b..be9e1839f 100644 --- a/example-crates/example_bitcoind_rpc_polling/src/main.rs +++ b/example-crates/example_bitcoind_rpc_polling/src/main.rs @@ -188,10 +188,7 @@ fn main() -> anyhow::Result<()> { let mut db = db.lock().unwrap(); let chain_changeset = chain - .apply_update(local_chain::Update { - tip: emission.checkpoint, - introduce_older_blocks: false, - }) + .apply_update(emission.checkpoint) .expect("must always apply as we receive blocks in order from emitter"); let graph_changeset = graph.apply_block_relevant(&emission.block, height); db.stage((chain_changeset, graph_changeset)); @@ -301,12 +298,8 @@ fn main() -> anyhow::Result<()> { let changeset = match emission { Emission::Block(block_emission) => { let height = block_emission.block_height(); - let chain_update = local_chain::Update { - tip: block_emission.checkpoint, - introduce_older_blocks: false, - }; let chain_changeset = chain - .apply_update(chain_update) + .apply_update(block_emission.checkpoint) .expect("must always apply as we receive blocks in order from emitter"); let graph_changeset = graph.apply_block_relevant(&block_emission.block, height); From daf588f016ec3118c875db8ed6b55fa03683f0f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 17 Apr 2024 14:06:44 +0800 Subject: [PATCH 11/13] feat(chain): optimize `merge_chains` --- crates/chain/src/local_chain.rs | 35 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index 792b987f5..957d1ad0a 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -364,13 +364,17 @@ impl LocalChain { /// /// [module-level documentation]: crate::local_chain pub fn apply_update(&mut self, update: CheckPoint) -> Result { - let changeset = merge_chains(self.tip.clone(), update.clone())?; - // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in - // `.apply_changeset` - self.apply_changeset(&changeset) - .map_err(|_| CannotConnectError { - try_include_height: 0, - })?; + let (changeset, can_replace) = merge_chains(self.tip.clone(), update.clone())?; + if can_replace { + self.tip = update; + } else { + // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in + // `.apply_changeset` + self.apply_changeset(&changeset) + .map_err(|_| CannotConnectError { + try_include_height: 0, + })?; + } Ok(changeset) } @@ -721,10 +725,14 @@ impl core::fmt::Display for ApplyHeaderError { #[cfg(feature = "std")] impl std::error::Error for ApplyHeaderError {} +/// Applies `update_tip` onto `original_tip`. +/// +/// On success, a tuple is returned `(changeset, can_replace)`. If `can_replace` is true, then the +/// `update_tip` can replace the `original_tip`. fn merge_chains( original_tip: CheckPoint, update_tip: CheckPoint, -) -> Result { +) -> Result<(ChangeSet, bool), CannotConnectError> { let mut changeset = ChangeSet::default(); let mut orig = original_tip.into_iter(); let mut update = update_tip.into_iter(); @@ -736,6 +744,11 @@ fn merge_chains( let mut prev_orig_was_invalidated = false; let mut potentially_invalidated_heights = vec![]; + // Flag to set if heights are removed from original chain. If no heights are removed, and we + // have a matching node pointer between the two chains, we can conclude that the update tip can + // just replace the original tip. + let mut has_removed_heights = false; + // To find the difference between the new chain and the original we iterate over both of them // from the tip backwards in tandem. We always dealing with the highest one from either chain // first and move to the next highest. The crucial logic is applied when they have blocks at the @@ -761,6 +774,8 @@ fn merge_chains( prev_orig_was_invalidated = false; prev_orig = curr_orig.take(); + has_removed_heights = true; + // OPTIMIZATION: we have run out of update blocks so we don't need to continue // iterating because there's no possibility of adding anything to changeset. if u.is_none() { @@ -786,7 +801,7 @@ fn merge_chains( // OPTIMIZATION 2 -- if we have the same underlying pointer at this point, we // can guarantee that no older blocks are introduced. if Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) { - return Ok(changeset); + return Ok((changeset, !has_removed_heights)); } } else { // We have an invalidation height so we set the height to the updated hash and @@ -820,5 +835,5 @@ fn merge_chains( } } - Ok(changeset) + Ok((changeset, false)) } From 2f22987c9e924800f8682b2dcbdde60fd26b069a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 22 Apr 2024 10:39:37 +0800 Subject: [PATCH 12/13] chore(chain): fix comment --- crates/chain/src/local_chain.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index 957d1ad0a..20f7e0e03 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -368,8 +368,7 @@ impl LocalChain { if can_replace { self.tip = update; } else { - // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in - // `.apply_changeset` + // `._check_changeset_is_applied` is called in `.apply_changeset` self.apply_changeset(&changeset) .map_err(|_| CannotConnectError { try_include_height: 0, From 96a9aa6e63474dbd93a2ef969eef5b07c79e6491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 22 Apr 2024 11:59:18 +0800 Subject: [PATCH 13/13] feat(chain): refactor `merge_chains` `merge_chains` now returns a tuple of the resultant checkpoint AND changeset. This is arguably a more readable/understandable setup. To do this, we had to create `CheckPoint::apply_changeset` which is kept as a private method. Thank you @ValuedMammal for the suggestion. Co-authored-by: valuedvalued mammal --- crates/chain/src/local_chain.rs | 135 +++++++++++++++++--------------- 1 file changed, 74 insertions(+), 61 deletions(-) diff --git a/crates/chain/src/local_chain.rs b/crates/chain/src/local_chain.rs index 20f7e0e03..157b9db27 100644 --- a/crates/chain/src/local_chain.rs +++ b/crates/chain/src/local_chain.rs @@ -213,6 +213,46 @@ impl CheckPoint { base.extend(core::iter::once(block_id).chain(tail.into_iter().rev())) .expect("tail is in order") } + + /// Apply `changeset` to the checkpoint. + fn apply_changeset(mut self, changeset: &ChangeSet) -> Result { + if let Some(start_height) = changeset.keys().next().cloned() { + // changes after point of agreement + let mut extension = BTreeMap::default(); + // point of agreement + let mut base: Option = None; + + for cp in self.iter() { + if cp.height() >= start_height { + extension.insert(cp.height(), cp.hash()); + } else { + base = Some(cp); + break; + } + } + + for (&height, &hash) in changeset { + match hash { + Some(hash) => { + extension.insert(height, hash); + } + None => { + extension.remove(&height); + } + }; + } + + let new_tip = match base { + Some(base) => base + .extend(extension.into_iter().map(BlockId::from)) + .expect("extension is strictly greater than base"), + None => LocalChain::from_blocks(extension)?.tip(), + }; + self = new_tip; + } + + Ok(self) + } } /// Iterates over checkpoints backwards. @@ -348,32 +388,22 @@ impl LocalChain { /// Applies the given `update` to the chain. /// - /// The method returns [`ChangeSet`] on success. This represents the applied changes to `self`. + /// The method returns [`ChangeSet`] on success. This represents the changes applied to `self`. /// /// There must be no ambiguity about which of the existing chain's blocks are still valid and /// which are now invalid. That is, the new chain must implicitly connect to a definite block in /// the existing chain and invalidate the block after it (if it exists) by including a block at /// the same height but with a different hash to explicitly exclude it as a connection point. /// - /// Additionally, a chain with a single block can have it's block invalidated by an update - /// chain with a block at the same height but different hash. - /// /// # Errors /// /// An error will occur if the update does not correctly connect with `self`. /// /// [module-level documentation]: crate::local_chain pub fn apply_update(&mut self, update: CheckPoint) -> Result { - let (changeset, can_replace) = merge_chains(self.tip.clone(), update.clone())?; - if can_replace { - self.tip = update; - } else { - // `._check_changeset_is_applied` is called in `.apply_changeset` - self.apply_changeset(&changeset) - .map_err(|_| CannotConnectError { - try_include_height: 0, - })?; - } + let (new_tip, changeset) = merge_chains(self.tip.clone(), update)?; + self.tip = new_tip; + self._check_changeset_is_applied(&changeset); Ok(changeset) } @@ -465,43 +495,10 @@ impl LocalChain { /// Apply the given `changeset`. pub fn apply_changeset(&mut self, changeset: &ChangeSet) -> Result<(), MissingGenesisError> { - if let Some(start_height) = changeset.keys().next().cloned() { - // changes after point of agreement - let mut extension = BTreeMap::default(); - // point of agreement - let mut base: Option = None; - - for cp in self.iter_checkpoints() { - if cp.height() >= start_height { - extension.insert(cp.height(), cp.hash()); - } else { - base = Some(cp); - break; - } - } - - for (&height, &hash) in changeset { - match hash { - Some(hash) => { - extension.insert(height, hash); - } - None => { - extension.remove(&height); - } - }; - } - - let new_tip = match base { - Some(base) => base - .extend(extension.into_iter().map(BlockId::from)) - .expect("extension is strictly greater than base"), - None => LocalChain::from_blocks(extension)?.tip(), - }; - self.tip = new_tip; - - debug_assert!(self._check_changeset_is_applied(changeset)); - } - + let old_tip = self.tip.clone(); + let new_tip = old_tip.apply_changeset(changeset)?; + self.tip = new_tip; + debug_assert!(self._check_changeset_is_applied(changeset)); Ok(()) } @@ -731,10 +728,10 @@ impl std::error::Error for ApplyHeaderError {} fn merge_chains( original_tip: CheckPoint, update_tip: CheckPoint, -) -> Result<(ChangeSet, bool), CannotConnectError> { +) -> Result<(CheckPoint, ChangeSet), CannotConnectError> { let mut changeset = ChangeSet::default(); - let mut orig = original_tip.into_iter(); - let mut update = update_tip.into_iter(); + let mut orig = original_tip.iter(); + let mut update = update_tip.iter(); let mut curr_orig = None; let mut curr_update = None; let mut prev_orig: Option = None; @@ -743,10 +740,11 @@ fn merge_chains( let mut prev_orig_was_invalidated = false; let mut potentially_invalidated_heights = vec![]; - // Flag to set if heights are removed from original chain. If no heights are removed, and we - // have a matching node pointer between the two chains, we can conclude that the update tip can - // just replace the original tip. - let mut has_removed_heights = false; + // If we can, we want to return the update tip as the new tip because this allows checkpoints + // in multiple locations to keep the same `Arc` pointers when they are being updated from each + // other using this function. We can do this as long as long as the update contains every + // block's height of the original chain. + let mut is_update_height_superset_of_original = true; // To find the difference between the new chain and the original we iterate over both of them // from the tip backwards in tandem. We always dealing with the highest one from either chain @@ -773,7 +771,7 @@ fn merge_chains( prev_orig_was_invalidated = false; prev_orig = curr_orig.take(); - has_removed_heights = true; + is_update_height_superset_of_original = false; // OPTIMIZATION: we have run out of update blocks so we don't need to continue // iterating because there's no possibility of adding anything to changeset. @@ -800,7 +798,17 @@ fn merge_chains( // OPTIMIZATION 2 -- if we have the same underlying pointer at this point, we // can guarantee that no older blocks are introduced. if Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) { - return Ok((changeset, !has_removed_heights)); + if is_update_height_superset_of_original { + return Ok((update_tip, changeset)); + } else { + let new_tip = + original_tip.apply_changeset(&changeset).map_err(|_| { + CannotConnectError { + try_include_height: 0, + } + })?; + return Ok((new_tip, changeset)); + } } } else { // We have an invalidation height so we set the height to the updated hash and @@ -834,5 +842,10 @@ fn merge_chains( } } - Ok((changeset, false)) + let new_tip = original_tip + .apply_changeset(&changeset) + .map_err(|_| CannotConnectError { + try_include_height: 0, + })?; + Ok((new_tip, changeset)) }