Introduce blockchain and tools subcommand (paritytech#14)

* Improve the error handling in BitcoindBackend

* Add tools subcommand

* Add blockchain subcommand

* Update getting_started.md

* Make gettxoutsetinfo interruputible

* Split into installation.md and usage.md

README.md

@@ -25,6 +25,12 @@ planning stages.
 by downloading all headers and the state at a specific block, in decentralized manner.
 - 🔗 **Substrate Integration**: Utilizes Substrate framework to provide production-level blockchain infrastructures.
 
+## Run Tests
+
+```bash
+cargo test --workspace --all
+```
+
 ## Disclaimer
 
 **Do not use Subcoin in production.** It is a heavy work in progress, not feature-complete and the code

crates/pallet-bitcoin/src/lib.rs

@@ -46,6 +46,11 @@ impl Txid {
 
         Self(H256::from(d))
     }
+
+    pub fn into_bitcoin_txid(self) -> bitcoin::Txid {
+        bitcoin::consensus::Decodable::consensus_decode(&mut self.encode().as_slice())
+            .expect("txid must be encoded correctly; qed")
+    }
 }
 
 impl core::fmt::Debug for Txid {
@@ -61,11 +66,11 @@ impl core::fmt::Debug for Txid {
 #[derive(Debug, TypeInfo, Encode, Decode)]
 pub struct Coin {
     /// Whether the coin is from a coinbase transaction.
-    is_coinbase: bool,
+    pub is_coinbase: bool,
     /// Transfer value in satoshis.
-    amount: u64,
+    pub amount: u64,
     /// Spending condition of the output.
-    script_pubkey: Vec<u8>,
+    pub script_pubkey: Vec<u8>,
 }
 
 impl MaxEncodedLen for Coin {
@@ -192,6 +197,12 @@ pub fn coin_storage_key<T: Config>(bitcoin_txid: bitcoin::Txid, index: Vout) ->
     Coins::<T>::storage_double_map_final_key(txid, index)
 }
 
+pub fn coin_storage_prefix<T: Config>() -> [u8; 32] {
+    use frame_support::StoragePrefixedMap;
+
+    Coins::<T>::final_prefix()
+}
+
 impl<T: Config> Pallet<T> {
     fn decode_transaction(btc_tx: Vec<u8>) -> BitcoinTransaction {
         BitcoinTransaction::consensus_decode(&mut btc_tx.as_slice()).unwrap_or_else(|_| {

crates/subcoin-node/src/cli.rs

@@ -1,7 +1,9 @@
 pub mod params;
 
+use crate::commands::blockchain::{Blockchain, BlockchainCmd};
 use crate::commands::import_blocks::{ImportBlocks, ImportBlocksCmd};
 use crate::commands::run::{Run, RunCmd};
+use crate::commands::tools::Tools;
 use crate::substrate_cli::SubstrateCli;
 use clap::Parser;
 use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE};
@@ -22,6 +24,14 @@ pub enum Command {
     /// Import blocks.
     ImportBlocks(ImportBlocks),
 
+    /// Utility tools.
+    #[command(subcommand)]
+    Tools(Tools),
+
+    /// Blockchain.
+    #[command(subcommand)]
+    Blockchain(Blockchain),
+
     /// Validate blocks.
     CheckBlock(Box<sc_cli::CheckBlockCmd>),
 
@@ -220,6 +230,26 @@ pub fn run() -> sc_cli::Result<()> {
                 ))
             })
         }
+        Command::Tools(tools) => tools.run(),
+        Command::Blockchain(blockchain) => {
+            let block_execution_strategy = blockchain.block_execution_strategy();
+            let cmd = BlockchainCmd::new(blockchain);
+            let runner = SubstrateCli.create_runner(&cmd)?;
+            runner.async_run(|config| {
+                let subcoin_service::NodeComponents {
+                    client,
+                    task_manager,
+                    ..
+                } = subcoin_service::new_node(subcoin_service::SubcoinConfiguration {
+                    network: bitcoin::Network::Bitcoin,
+                    config: &config,
+                    block_execution_strategy,
+                    no_hardware_benchmarks: true,
+                    storage_monitor,
+                })?;
+                Ok((cmd.run(client), task_manager))
+            })
+        }
         Command::CheckBlock(cmd) => {
             let runner = SubstrateCli.create_runner(&*cmd)?;
             runner.async_run(|config| {

crates/subcoin-node/src/commands.rs

@@ -1,2 +1,4 @@
+pub mod blockchain;
 pub mod import_blocks;
 pub mod run;
+pub mod tools;

crates/subcoin-node/src/commands/blockchain.rs

@@ -0,0 +1,135 @@
+use crate::cli::params::CommonParams;
+use crate::utils::Yield;
+use pallet_bitcoin::Coin;
+use sc_cli::{ImportParams, NodeKeyParams, SharedParams};
+use sc_client_api::{HeaderBackend, StorageProvider};
+use sc_consensus_nakamoto::BlockExecutionStrategy;
+use sp_core::storage::StorageKey;
+use sp_core::Decode;
+use std::sync::Arc;
+use subcoin_primitives::{BackendExt, CoinStorageKey};
+use subcoin_service::FullClient;
+
+/// Blockchain.
+#[derive(Debug, clap::Subcommand)]
+pub enum Blockchain {
+    /// Statistics about the unspent transaction output set.
+    #[command(name = "gettxoutsetinfo")]
+    GetTxOutSetInfo {
+        #[clap(long)]
+        height: Option<u32>,
+
+        #[allow(missing_docs)]
+        #[clap(flatten)]
+        common_params: CommonParams,
+    },
+}
+
+impl Blockchain {
+    pub fn block_execution_strategy(&self) -> BlockExecutionStrategy {
+        match self {
+            Self::GetTxOutSetInfo { common_params, .. } => common_params.block_execution_strategy(),
+        }
+    }
+}
+
+pub enum BlockchainCmd {
+    GetTxOutSetInfo {
+        height: Option<u32>,
+        shared_params: SharedParams,
+    },
+}
+
+impl BlockchainCmd {
+    /// Constructs a new instance of [`BlockchainCmd`].
+    pub fn new(blockchain: Blockchain) -> Self {
+        match blockchain {
+            Blockchain::GetTxOutSetInfo {
+                height,
+                common_params,
+            } => Self::GetTxOutSetInfo {
+                height,
+                shared_params: common_params.as_shared_params(),
+            },
+        }
+    }
+
+    fn shared_params(&self) -> &SharedParams {
+        match self {
+            Self::GetTxOutSetInfo { shared_params, .. } => shared_params,
+        }
+    }
+
+    pub async fn run(self, client: Arc<FullClient>) -> sc_cli::Result<()> {
+        match self {
+            Self::GetTxOutSetInfo { height, .. } => gettxoutsetinfo(&client, height).await,
+        }
+    }
+}
+
+impl sc_cli::CliConfiguration for BlockchainCmd {
+    fn shared_params(&self) -> &SharedParams {
+        BlockchainCmd::shared_params(self)
+    }
+
+    fn import_params(&self) -> Option<&ImportParams> {
+        None
+    }
+
+    fn node_key_params(&self) -> Option<&NodeKeyParams> {
+        None
+    }
+}
+
+async fn gettxoutsetinfo(client: &Arc<FullClient>, height: Option<u32>) -> sc_cli::Result<()> {
+    const FINAL_PREFIX_LEN: usize = 32;
+
+    let storage_prefix = subcoin_service::CoinStorageKey.storage_prefix();
+    let storage_key = StorageKey(storage_prefix.to_vec());
+    let block_number = height.unwrap_or_else(|| client.info().best_number);
+    let block_hash = client.hash(block_number)?.unwrap();
+    let pairs_iter = client
+        .storage_pairs(block_hash, Some(&storage_key), None)?
+        .map(|(key, data)| (key.0, data.0));
+
+    let mut txouts = 0;
+    let mut bogosize = 0;
+    let mut total_amount = 0;
+
+    let genesis_txid: bitcoin::Txid =
+        "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"
+            .parse()
+            .expect("Genesis txid must be correct; qed");
+
+    for (key, value) in pairs_iter {
+        let (txid, _vout) =
+            <(pallet_bitcoin::Txid, u32)>::decode(&mut &key.as_slice()[FINAL_PREFIX_LEN..])
+                .expect("Key type must be correct; qed");
+        let txid = txid.into_bitcoin_txid();
+        // output in genesis tx is excluded in gettxoutsetinfo.
+        if txid == genesis_txid {
+            continue;
+        }
+        let coin = Coin::decode(&mut value.as_slice())
+            .expect("Coin read from DB must be decoded successfully; qed");
+        txouts += 1;
+        total_amount += coin.amount;
+        // https://github.com/bitcoin/bitcoin/blob/33af14e31b9fa436029a2bb8c2b11de8feb32f86/src/kernel/coinstats.cpp#L40
+        bogosize += 50 + coin.script_pubkey.len();
+
+        // Yield here allows to make the process interruptible by ctrl_c.
+        Yield::new().await;
+    }
+
+    let bitcoin_block_hash = client
+        .block_hash(block_number)
+        .expect("Bitcoin block hash missing");
+
+    println!("block_number: {block_number}");
+    println!("block_hash: {bitcoin_block_hash}");
+    println!("txouts: {txouts}");
+    println!("bogosize: {bogosize}");
+    println!("total_amount: {:.8}", total_amount as f64 / 100_000_000.0);
+
+    Ok(())
+}

crates/subcoin-node/src/commands/import_blocks.rs

@@ -1,4 +1,5 @@
 use crate::cli::params::CommonParams;
+use crate::utils::Yield;
 use bitcoin_explorer::BitcoinDB;
 use sc_cli::{ImportParams, NodeKeyParams, SharedParams};
 use sc_client_api::HeaderBackend;
@@ -80,7 +81,7 @@ impl ImportBlocksCmd {
     ) -> sc_cli::Result<()> {
         let from = (client.info().best_number + 1) as usize;
 
-        let bitcoind_backend = BitcoinBackend::new(&data_dir);
+        let bitcoind_backend = BitcoinBackend::new(&data_dir)?;
         let max = bitcoind_backend.block_count();
         let to = self.to.unwrap_or(max).min(max);
 
@@ -114,7 +115,7 @@ impl ImportBlocksCmd {
             );
 
         for index in from..=to {
-            let block = bitcoind_backend.block_at(index);
+            let block = bitcoind_backend.block_at(index)?;
             bitcoin_block_import
                 .import_block(block)
                 .await
@@ -234,53 +235,27 @@ struct BitcoinBackend {
 }
 
 impl BitcoinBackend {
-    fn new(path: impl AsRef<Path>) -> Self {
-        let db = BitcoinDB::new(path.as_ref(), true).expect("Failed to open Bitcoin DB");
-        Self { db }
+    fn new(path: impl AsRef<Path>) -> sc_cli::Result<Self> {
+        let db = BitcoinDB::new(path.as_ref(), true)
+            .map_err(|err| sc_cli::Error::Application(Box::new(err)))?;
+        Ok(Self { db })
     }
 
-    fn block_at(&self, height: usize) -> bitcoin::Block {
+    fn block_at(&self, height: usize) -> sc_cli::Result<bitcoin::Block> {
         use bitcoin::consensus::Decodable;
 
-        let raw_block = self.db.get_raw_block(height).expect("Failed to get block");
+        let raw_block = self.db.get_raw_block(height).map_err(|err| {
+            std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("Failed to get bitcoin block at #{height}: {err}"),
+            )
+        })?;
 
-        bitcoin::Block::consensus_decode(&mut raw_block.as_slice())
-            .expect("Bad block in the database")
+        Ok(bitcoin::Block::consensus_decode(&mut raw_block.as_slice())
+            .expect("Bad block in the database"))
     }
 
     fn block_count(&self) -> usize {
         self.db.get_block_count()
     }
 }
-
-/// A future that will always `yield` on the first call of `poll` but schedules the
-/// current task for re-execution.
-///
-/// This is done by getting the waker and calling `wake_by_ref` followed by returning
-/// `Pending`. The next time the `poll` is called, it will return `Ready`.
-struct Yield(bool);
-
-impl Yield {
-    fn new() -> Self {
-        Self(false)
-    }
-}
-
-impl futures::Future for Yield {
-    type Output = ();
-
-    fn poll(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut futures::task::Context<'_>,
-    ) -> futures::task::Poll<()> {
-        use futures::task::Poll;
-
-        if !self.0 {
-            self.0 = true;
-            cx.waker().wake_by_ref();
-            Poll::Pending
-        } else {
-            Poll::Ready(())
-        }
-    }
-}