Make analyzing index easier (#850)

justfile

@@ -95,6 +95,9 @@ download-log unit='ord' host='ordinals.com':
   ssh root@{{host}} 'mkdir -p tmp && journalctl -u {{unit}} > tmp/{{unit}}.log'
   rsync --progress --compress root@{{host}}:tmp/{{unit}}.log tmp/{{unit}}.log
 
+download-index unit='ord' host='ordinals.com':
+  rsync --progress --compress root@{{host}}:/var/lib/{{unit}}/index.redb tmp/{{unit}}.index.redb
+
 graph log:
   ./bin/graph $1
 

src/index.rs

@@ -156,7 +156,11 @@ impl Index {
       bail!("failed to create data dir `{}`: {err}", data_dir.display());
     }
 
-    let database_path = data_dir.join("index.redb");
+    let database_path = if let Some(database_path) = &options.index {
+      database_path.clone()
+    } else {
+      data_dir.join("index.redb")
+    };
 
     let database = match unsafe { redb::Database::open(&database_path) } {
       Ok(database) => database,

src/index/updater.rs

@@ -73,8 +73,8 @@ impl Updater {
 
     let rx = Self::fetch_blocks_from(index, self.height)?;
 
-    let mut uncomitted = 0;
-    for i in 0.. {
+    let mut uncommitted = 0;
+    loop {
       let block = match rx.recv() {
         Ok(block) => block,
         Err(mpsc::RecvError) => break,
@@ -90,11 +90,11 @@ impl Updater {
         }
       }
 
-      uncomitted += 1;
+      uncommitted += 1;
 
-      if i % 5000 == 0 {
+      if uncommitted == 5000 {
         self.commit(wtx)?;
-        uncomitted = 0;
+        uncommitted = 0;
         wtx = index.begin_write()?;
         let height = wtx
           .open_table(HEIGHT_TO_BLOCK_HASH)?
@@ -124,7 +124,7 @@ impl Updater {
       }
     }
 
-    if uncomitted > 0 {
+    if uncommitted > 0 {
       self.commit(wtx)?;
     }
 

src/options.rs

@@ -15,6 +15,8 @@ pub(crate) struct Options {
   data_dir: Option<PathBuf>,
   #[clap(long, help = "Limit index to <HEIGHT_LIMIT> blocks.")]
   pub(crate) height_limit: Option<u64>,
+  #[clap(long, help = "Use index at <INDEX>.")]
+  pub(crate) index: Option<PathBuf>,
   #[clap(long, help = "Index ordinal ranges.")]
   pub(crate) index_ordinals: bool,
   #[clap(long, help = "Connect to Bitcoin Core RPC at <RPC_URL>.")]

src/subcommand.rs

@@ -17,7 +17,7 @@ pub(crate) enum Subcommand {
   Epochs,
   Find(find::Find),
   Index,
-  Info,
+  Info(info::Info),
   ListRanges(list::List),
   Parse(parse::Parse),
   Range(range::Range),
@@ -34,7 +34,7 @@ impl Subcommand {
       Self::Epochs => epochs::run(),
       Self::Find(find) => find.run(options),
       Self::Index => index::run(options),
-      Self::Info => info::run(options),
+      Self::Info(info) => info.run(options),
       Self::ListRanges(list) => list.run(options),
       Self::Parse(parse) => parse.run(),
       Self::Range(range) => range.run(),

src/subcommand/info.rs

@@ -1,8 +1,35 @@
 use super::*;
 
-pub(crate) fn run(options: Options) -> Result {
-  let index = Index::open(&options)?;
-  index.update()?;
-  serde_json::to_writer(io::stdout(), &index.info()?)?;
-  Ok(())
+#[derive(Debug, Parser)]
+pub(crate) struct Info {
+  #[clap(long)]
+  transactions: bool,
+}
+
+impl Info {
+  pub(crate) fn run(self, options: Options) -> Result {
+    let index = Index::open(&options)?;
+    index.update()?;
+    let info = index.info()?;
+
+    if self.transactions {
+      println!("start\tend\tcount\telapsed");
+
+      for window in info.transactions.windows(2) {
+        let start = &window[0];
+        let end = &window[1];
+        println!(
+          "{}\t{}\t{}\t{:.2}",
+          start.starting_block_count,
+          end.starting_block_count,
+          end.starting_block_count - start.starting_block_count,
+          (end.starting_timestamp - start.starting_timestamp) as f64 / 1000.0 / 60.0
+        );
+      }
+    } else {
+      serde_json::to_writer(io::stdout(), &info)?;
+    }
+
+    Ok(())
+  }
 }

tests/index.rs

@@ -1,29 +1,17 @@
 use super::*;
 
 #[test]
-fn custom_index_size() {
+fn custom_index_path() {
   let rpc_server = test_bitcoincore_rpc::spawn();
   rpc_server.mine_blocks(1);
 
-  let output = CommandBuilder::new("--max-index-size 1mib --index-ordinals find 0")
+  let tempdir = TempDir::new().unwrap();
+
+  let index_path = tempdir.path().join("foo.redb");
+
+  CommandBuilder::new(format!("--index {} index", index_path.display()))
     .rpc_server(&rpc_server)
-    .expected_stdout("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b:0:0\n")
     .run();
 
-  assert_eq!(
-    output
-      .tempdir
-      .path()
-      .join(if cfg!(target_os = "macos") {
-        "Library/Application Support/"
-      } else {
-        ".local/share"
-      })
-      .join("ord")
-      .join("index.redb")
-      .metadata()
-      .unwrap()
-      .len(),
-    1 << 20
-  );
+  assert!(index_path.is_file())
 }

tests/info.rs

@@ -1,7 +1,7 @@
 use super::*;
 
 #[test]
-fn initial() {
+fn json() {
   let rpc_server = test_bitcoincore_rpc::spawn();
   CommandBuilder::new("--index-ordinals info")
     .rpc_server(&rpc_server)
@@ -10,3 +10,42 @@ fn initial() {
     )
     .run();
 }
+
+#[test]
+fn transactions() {
+  let rpc_server = test_bitcoincore_rpc::spawn();
+
+  let tempdir = TempDir::new().unwrap();
+
+  let index_path = tempdir.path().join("index.redb");
+
+  CommandBuilder::new(format!(
+    "--index-ordinals --index {} info --transactions",
+    index_path.display()
+  ))
+  .rpc_server(&rpc_server)
+  .expected_stdout("start\tend\tcount\telapsed\n")
+  .run();
+
+  rpc_server.mine_blocks(10);
+
+  CommandBuilder::new(format!(
+    "--index-ordinals --index {} info --transactions",
+    index_path.display()
+  ))
+  .rpc_server(&rpc_server)
+  .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n")
+  .run();
+
+  rpc_server.mine_blocks(10);
+
+  CommandBuilder::new(format!(
+    "--index-ordinals --index {} info --transactions",
+    index_path.display()
+  ))
+  .rpc_server(&rpc_server)
+  .expected_stdout("start\tend\tcount\telapsed\n")
+  .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n")
+  .stdout_regex("start\tend\tcount\telapsed\n0\t1\t1\t\\d+\\.\\d+\n1\t11\t10\t\\d+\\.\\d+\n")
+  .run();
+}

tests/lib.rs

@@ -37,6 +37,7 @@ mod command_builder;
 mod epochs;
 mod expected;
 mod find;
+mod index;
 mod info;
 mod list;
 mod parse;