diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 85ea05852f..7cc6bd0692 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,13 +44,18 @@ jobs: cargo update --locked --package ord - name: Test - run: cargo test --all + run: | + cargo test --all + cargo test --all --features redb - name: Clippy - run: cargo clippy --all --all-targets + run: | + cargo clippy --all --all-targets + cargo clippy --all --all-targets --features redb - name: Format - run: cargo fmt --all -- --check + run: | + cargo fmt --all -- --check - name: Check for Forbidden Words if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/.gitignore b/.gitignore index f20da860b3..749a2f9f75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -.idea/ +/.idea/ +/index.lmdb /index.redb /target diff --git a/Cargo.lock b/Cargo.lock index 799a266196..1db8ffaf07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "bitflags" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" + [[package]] name = "bitflags" version = "1.3.2" @@ -151,13 +157,26 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + [[package]] name = "clap" version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "textwrap 0.11.0", "unicode-width", ] @@ -169,7 +188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5f1fea81f183005ced9e59cdb01737ef2423956dac5a6d731b06b2ecfaa3467" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "indexmap", "lazy_static", @@ -300,6 +319,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctrlc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19c6cedffdc8c03a3346d723eb20bd85a13362bb96dc2ac000842c6381ec7bf" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -470,6 +499,12 @@ dependencies = [ "slab", ] +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "getrandom" version = "0.1.16" @@ -489,7 +524,7 @@ checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -886,6 +921,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + [[package]] name = "ntapi" version = "0.3.7" @@ -895,6 +943,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -926,8 +984,10 @@ version = "0.0.0" dependencies = [ "bitcoin", "bitcoincore-rpc", + "chrono", "clap 3.1.0", "criterion", + "ctrlc", "derive_more", "dirs", "env_logger", @@ -941,12 +1001,36 @@ dependencies = [ "jsonrpc-derive", "jsonrpc-http-server", "log", + "ord-lmdb-zero", + "rayon", "redb", "regex", "tempfile", "unindent", ] +[[package]] +name = "ord-liblmdb-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0708b9f740bd632d4fe7468b314c4c45d994811a811e82acc193ced0f6b358ba" +dependencies = [ + "gcc", + "libc", +] + +[[package]] +name = "ord-lmdb-zero" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6186bbfce46615119f72e26acfe341b79a88c18c2d994d3e4b2c9029a24bf09" +dependencies = [ + "bitflags 0.9.1", + "libc", + "ord-liblmdb-sys", + "supercow", +] + [[package]] name = "os_str_bytes" version = "6.0.0" @@ -1166,7 +1250,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1335,6 +1419,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "supercow" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63" + [[package]] name = "syn" version = "1.0.86" @@ -1384,6 +1474,17 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -1579,9 +1680,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" diff --git a/Cargo.toml b/Cargo.toml index 28aa43f480..6f9bffbd0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,9 @@ autotests = false [dependencies] bitcoin = "0.27.1" bitcoincore-rpc = "0.14.0" +chrono = "0.4.19" clap = { version = "3.1.0", features = ["derive"] } +ctrlc = "3.2.1" derive_more = "0.99.17" dirs = "4.0.0" env_logger = "0.9.0" @@ -18,7 +20,9 @@ integer-cbrt = "0.1.2" integer-sqrt = "0.1.5" jsonrpc = "0.12.1" log = "0.4.14" -redb = "0.0.5" +ord-lmdb-zero = "0.4.5" +rayon = "1.5.1" +redb = { version = "0.0.5", optional = true } [dev-dependencies] criterion = "0.3.5" diff --git a/src/index.rs b/src/index.rs index 377d315de6..83a8d927d5 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,6 +1,7 @@ use { super::*, bitcoincore_rpc::{Auth, Client, RpcApi}, + rayon::iter::{IntoParallelRefIterator, ParallelIterator}, }; pub(crate) struct Index { @@ -10,41 +11,28 @@ pub(crate) struct Index { } impl Index { - const HEIGHT_TO_HASH: TableDefinition<'static, u64, [u8]> = - TableDefinition::new("HEIGHT_TO_HASH"); - const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<'static, [u8], [u8]> = - TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); - const KEY_TO_SATPOINT: TableDefinition<'static, [u8], [u8]> = - TableDefinition::new("KEY_TO_SATPOINT"); - - pub(crate) fn open(options: Options) -> Result { + pub(crate) fn open(options: &Options) -> Result { let client = Client::new( - &options.rpc_url.ok_or("This command requires `--rpc-url`")?, + options + .rpc_url + .as_ref() + .ok_or("This command requires `--rpc-url`")?, options .cookie_file - .map(Auth::CookieFile) + .as_ref() + .map(|path| Auth::CookieFile(path.clone())) .unwrap_or(Auth::None), )?; - let result = unsafe { Database::open("index.redb") }; - - let database = match result { - Ok(database) => database, - Err(redb::Error::Io(error)) if error.kind() == io::ErrorKind::NotFound => unsafe { - Database::create("index.redb", options.index_size.0)? - }, - Err(error) => return Err(error.into()), - }; - Ok(Self { client, - database, + database: Database::open(options)?, sleep_until: Cell::new(Instant::now()), }) } #[allow(clippy::self_named_constructors)] - pub(crate) fn index(options: Options) -> Result { + pub(crate) fn index(options: &Options) -> Result { let index = Self::open(options)?; index.index_ranges()?; @@ -53,36 +41,7 @@ impl Index { } pub(crate) fn print_info(&self) -> Result { - let tx = self.database.begin_write()?; - - let height_to_hash = tx.open_table(&Self::HEIGHT_TO_HASH)?; - - let blocks_indexed = height_to_hash - .range(0..)? - .rev() - .next() - .map(|(height, _hash)| height + 1) - .unwrap_or(0); - - let outputs_indexed = tx.open_table(&Self::OUTPOINT_TO_ORDINAL_RANGES)?.len()?; - - tx.abort()?; - - let stats = self.database.stats()?; - - println!("blocks indexed: {}", blocks_indexed); - println!("outputs indexed: {}", outputs_indexed); - println!("tree height: {}", stats.tree_height()); - println!("free pages: {}", stats.free_pages()); - println!("stored: {}", Bytes(stats.stored_bytes())); - println!("overhead: {}", Bytes(stats.overhead_bytes())); - println!("fragmented: {}", Bytes(stats.fragmented_bytes())); - println!( - "index size: {}", - Bytes(std::fs::metadata("index.redb")?.len().try_into()?) - ); - - Ok(()) + self.database.print_info() } fn client(&self) -> &Client { @@ -101,44 +60,58 @@ impl Index { &self.client } + fn decode_ordinal_range(bytes: [u8; 11]) -> (u64, u64) { + let n = u128::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], + bytes[9], bytes[10], 0, 0, 0, 0, 0, + ]); + + // 51 bit base + let base = (n & ((1 << 51) - 1)) as u64; + // 33 bit delta + let delta = (n >> 51) as u64; + + (base, base + delta) + } + fn index_ranges(&self) -> Result { log::info!("Indexing ranges…"); + let mut wtx = self.database.begin_write()?; + loop { - let wtx = self.database.begin_write()?; + let start = Instant::now(); + let mut ordinal_ranges_written = 0; - let mut height_to_hash = wtx.open_table(&Self::HEIGHT_TO_HASH)?; - let height = height_to_hash - .range(0..)? - .rev() - .next() - .map(|(height, _hash)| height + 1) - .unwrap_or(0); + let height = wtx.height()?; let block = match self.block(height)? { Some(block) => block, None => { - wtx.abort()?; - break; + wtx.commit()?; + return Ok(()); } }; + let time: DateTime = DateTime::from_utc( + NaiveDateTime::from_timestamp(block.header.time as i64, 0), + Utc, + ); + log::info!( - "Indexing block {height} with {} transactions…", + "Block {height} at {} with {} transactions…", + time, block.txdata.len() ); if let Some(prev_height) = height.checked_sub(1) { - let prev_hash = height_to_hash.get(&prev_height)?.unwrap(); + let prev_hash = wtx.blockhash_at_height(prev_height)?.unwrap(); if prev_hash != block.header.prev_blockhash.as_ref() { return Err("Reorg detected at or before {prev_height}".into()); } } - let mut outpoint_to_ordinal_ranges = wtx.open_table(&Self::OUTPOINT_TO_ORDINAL_RANGES)?; - let mut key_to_satpoint = wtx.open_table(&Self::KEY_TO_SATPOINT)?; - let mut coinbase_inputs = VecDeque::new(); let h = Height(height); @@ -147,7 +120,14 @@ impl Index { coinbase_inputs.push_front((start.n(), (start + h.subsidy()).n())); } - for (tx_offset, tx) in block.txdata.iter().enumerate().skip(1) { + let txdata = block + .txdata + .as_slice() + .par_iter() + .map(|tx| (tx.txid(), tx)) + .collect::>(); + + for (tx_offset, (txid, tx)) in txdata.iter().enumerate().skip(1) { log::trace!("Indexing transaction {tx_offset}…"); let mut input_ordinal_ranges = VecDeque::new(); @@ -156,64 +136,88 @@ impl Index { let mut key = Vec::new(); input.previous_output.consensus_encode(&mut key)?; - let ordinal_ranges = outpoint_to_ordinal_ranges - .get(key.as_slice())? + let ordinal_ranges = wtx + .get_ordinal_ranges(key.as_slice())? .ok_or("Could not find outpoint in index")?; - for chunk in ordinal_ranges.chunks_exact(16) { - let start = u64::from_le_bytes(chunk[0..8].try_into().unwrap()); - let end = u64::from_le_bytes(chunk[8..16].try_into().unwrap()); - input_ordinal_ranges.push_back((start, end)); + let new = input_ordinal_ranges.len(); + + for chunk in ordinal_ranges.chunks_exact(11) { + input_ordinal_ranges.push_back(Self::decode_ordinal_range(chunk.try_into().unwrap())); + } + + for (start, _end) in input_ordinal_ranges.range(new..) { + wtx.remove_satpoint( + &Key { + ordinal: *start, + block: 0, + transaction: 0, + } + .encode(), + )?; } + + wtx.remove_outpoint(&key)?; } self.index_transaction( height, tx_offset as u64, + *txid, tx, + &mut wtx, &mut input_ordinal_ranges, - &mut outpoint_to_ordinal_ranges, - &mut key_to_satpoint, + &mut ordinal_ranges_written, )?; coinbase_inputs.extend(input_ordinal_ranges); } - if let Some(tx) = block.txdata.first() { + if let Some((txid, tx)) = txdata.first() { self.index_transaction( height, 0, + *txid, tx, + &mut wtx, &mut coinbase_inputs, - &mut outpoint_to_ordinal_ranges, - &mut key_to_satpoint, + &mut ordinal_ranges_written, )?; } - height_to_hash.insert(&height, &block.block_hash())?; - wtx.commit()?; - } + wtx.set_blockhash_at_height(height, block.block_hash())?; + if height % 1000 == 0 { + wtx.commit()?; + wtx = self.database.begin_write()?; + } - Ok(()) + log::info!( + "Wrote {ordinal_ranges_written} ordinal ranges in {}ms", + (Instant::now() - start).as_millis(), + ); + + if INTERRUPTS.load(atomic::Ordering::Relaxed) > 0 { + wtx.commit()?; + return Ok(()); + } + } } fn index_transaction( &self, block: u64, tx_offset: u64, + txid: Txid, tx: &Transaction, + wtx: &mut WriteTransaction, input_ordinal_ranges: &mut VecDeque<(u64, u64)>, - outpoint_to_ordinal_ranges: &mut Table<[u8], [u8]>, - key_to_satpoint: &mut Table<[u8], [u8]>, + ordinal_ranges_written: &mut u64, ) -> Result { for (vout, output) in tx.output.iter().enumerate() { let outpoint = OutPoint { - txid: tx.txid(), vout: vout as u32, + txid, }; - let mut outpoint_encoded = Vec::new(); - outpoint.consensus_encode(&mut outpoint_encoded)?; - let mut ordinals = Vec::new(); let mut remaining = output.value; @@ -238,7 +242,7 @@ impl Index { outpoint, } .consensus_encode(&mut satpoint)?; - key_to_satpoint.insert( + wtx.insert_satpoint( &Key { ordinal: assigned.0, block, @@ -248,13 +252,21 @@ impl Index { &satpoint, )?; - ordinals.extend_from_slice(&assigned.0.to_le_bytes()); - ordinals.extend_from_slice(&assigned.1.to_le_bytes()); + let base = assigned.0; + let delta = assigned.1 - assigned.0; + + let n = base as u128 | (delta as u128) << 51; + + ordinals.extend_from_slice(&n.to_le_bytes()[0..11]); remaining -= assigned.1 - assigned.0; + + *ordinal_ranges_written += 1; } - outpoint_to_ordinal_ranges.insert(&outpoint_encoded, &ordinals)?; + let mut outpoint_encoded = Vec::new(); + outpoint.consensus_encode(&mut outpoint_encoded)?; + wtx.insert_outpoint(&outpoint_encoded, &ordinals)?; } Ok(()) @@ -271,33 +283,14 @@ impl Index { } pub(crate) fn find(&self, ordinal: Ordinal) -> Result> { - let rtx = self.database.begin_read()?; - - let height_to_hash = match rtx.open_table(&Self::HEIGHT_TO_HASH) { - Ok(height_to_hash) => height_to_hash, - Err(redb::Error::TableDoesNotExist(_)) => return Ok(None), - Err(err) => return Err(err.into()), - }; - - match height_to_hash.range(0..)?.rev().next() { - Some((height, _hash)) if height >= ordinal.height().0 => {} - _ => return Ok(None), + if self.database.height()? <= ordinal.height().0 { + return Ok(None); } - let key_to_satpoint = match rtx.open_table(&Self::KEY_TO_SATPOINT) { - Ok(key_to_satpoint) => key_to_satpoint, - Err(redb::Error::TableDoesNotExist(_)) => return Ok(None), - Err(err) => return Err(err.into()), - }; - - match key_to_satpoint - .range([].as_slice()..=Key::new(ordinal).encode().as_slice())? - .rev() - .next() - { + match self.database.find(ordinal)? { Some((start_key, start_satpoint)) => { - let start_key = Key::decode(start_key)?; - let start_satpoint = SatPoint::consensus_decode(start_satpoint)?; + let start_key = Key::decode(&start_key)?; + let start_satpoint = SatPoint::consensus_decode(start_satpoint.as_slice())?; Ok(Some(( start_key.block, start_key.transaction, @@ -311,24 +304,19 @@ impl Index { } } - pub(crate) fn list(&self, outpoint: OutPoint) -> Result> { - let rtx = self.database.begin_read()?; - let outpoint_to_ordinal_ranges = rtx.open_table(&Self::OUTPOINT_TO_ORDINAL_RANGES)?; - - let mut key = Vec::new(); - outpoint.consensus_encode(&mut key)?; - - let ordinal_ranges = outpoint_to_ordinal_ranges - .get(key.as_slice())? - .ok_or("Could not find outpoint in index")?; - - let mut output = Vec::new(); - for chunk in ordinal_ranges.chunks_exact(16) { - let start = u64::from_le_bytes(chunk[0..8].try_into().unwrap()); - let end = u64::from_le_bytes(chunk[8..16].try_into().unwrap()); - output.push((start, end)); + pub(crate) fn list(&self, outpoint: OutPoint) -> Result>> { + let mut outpoint_encoded = Vec::new(); + outpoint.consensus_encode(&mut outpoint_encoded)?; + let ordinal_ranges = self.database.list(&outpoint_encoded)?; + match ordinal_ranges { + Some(ordinal_ranges) => { + let mut output = Vec::new(); + for chunk in ordinal_ranges.chunks_exact(11) { + output.push(Self::decode_ordinal_range(chunk.try_into().unwrap())); + } + Ok(Some(output)) + } + None => Ok(None), } - - Ok(output) } } diff --git a/src/lmdb_database.rs b/src/lmdb_database.rs new file mode 100644 index 0000000000..bd5748920e --- /dev/null +++ b/src/lmdb_database.rs @@ -0,0 +1,247 @@ +use { + super::*, + ord_lmdb_zero::{self as lmdb, EnvBuilder, Environment}, + std::fs, +}; + +const HEIGHT_TO_HASH: &str = "HEIGHT_TO_HASH"; +const KEY_TO_SATPOINT: &str = "KEY_TO_SATPOINT"; +const OUTPOINT_TO_ORDINAL_RANGES: &str = "OUTPOINT_TO_ORDINAL_RANGES"; + +trait LmdbResultExt { + fn into_option(self) -> Result>; +} + +impl LmdbResultExt for lmdb::Result { + fn into_option(self) -> Result> { + match self { + Ok(value) => Ok(Some(value)), + Err(lmdb::Error::Code(-30798)) => Ok(None), + Err(error) => Err(error.into()), + } + } +} + +pub(crate) struct Database(Environment); + +impl Database { + pub(crate) fn open(options: &Options) -> Result { + let path = "index.lmdb"; + + fs::create_dir_all(path)?; + + let env = unsafe { + let mut builder = EnvBuilder::new()?; + + builder.set_maxdbs(3)?; + builder.set_mapsize(options.index_size.0)?; + + builder + .open(path, lmdb::open::Flags::empty(), 0o600) + .unwrap() + }; + + Ok(Self(env)) + } + + pub(crate) fn begin_write(&self) -> Result { + WriteTransaction::new(&self.0) + } + + pub(crate) fn print_info(&self) -> Result { + let stat = self.0.stat()?; + + let blocks_indexed = self.height()?; + + println!("blocks indexed: {}", blocks_indexed); + println!( + "data and metadata: {}", + ((stat.branch_pages + stat.leaf_pages + stat.overflow_pages) as u64) * stat.psize as u64 + ); + + Ok(()) + } + + pub(crate) fn find(&self, ordinal: Ordinal) -> Result, Vec)>> { + let key_to_satpoint = lmdb::Database::open( + &self.0, + Some(KEY_TO_SATPOINT), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + let tx = lmdb::ReadTransaction::new(&self.0)?; + + let mut cursor = tx.cursor(key_to_satpoint)?; + + let key = Key::new(ordinal).encode(); + + let access = tx.access(); + cursor + .seek_range_k::<[u8], [u8]>(&access, key.as_slice()) + .into_option()?; + + Ok( + cursor + .prev::<[u8], [u8]>(&access) + .into_option()? + .map(|(start_key, start_satpoint)| (start_key.to_vec(), start_satpoint.to_vec())), + ) + } + + pub(crate) fn height(&self) -> Result { + let height_to_hash = lmdb::Database::open( + &self.0, + Some(HEIGHT_TO_HASH), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + let tx = lmdb::ReadTransaction::new(&self.0)?; + + let height = tx + .cursor(&height_to_hash)? + .last::<[u8], [u8]>(&tx.access()) + .into_option()? + .map(|(key, _value)| u64::from_be_bytes(key.try_into().unwrap()) + 1) + .unwrap_or_default(); + + Ok(height) + } + + pub(crate) fn list(&self, outpoint: &[u8]) -> Result>> { + let outpoint_to_ordinal_ranges = &lmdb::Database::open( + &self.0, + Some(OUTPOINT_TO_ORDINAL_RANGES), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + Ok( + lmdb::ReadTransaction::new(&self.0)? + .access() + .get::<[u8], [u8]>(outpoint_to_ordinal_ranges, outpoint) + .into_option()? + .map(|ranges| ranges.to_vec()), + ) + } +} + +pub(crate) struct WriteTransaction<'a> { + height_to_hash: lmdb::Database<'a>, + lmdb_write_transaction: lmdb::WriteTransaction<'a>, + key_to_satpoint: lmdb::Database<'a>, + outpoint_to_ordinal_ranges: lmdb::Database<'a>, +} + +impl<'a> WriteTransaction<'a> { + pub(crate) fn new(environment: &'a Environment) -> Result { + let height_to_hash = lmdb::Database::open( + environment, + Some(HEIGHT_TO_HASH), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + let outpoint_to_ordinal_ranges = lmdb::Database::open( + environment, + Some(OUTPOINT_TO_ORDINAL_RANGES), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + let key_to_satpoint = lmdb::Database::open( + environment, + Some(KEY_TO_SATPOINT), + &lmdb::DatabaseOptions::new(lmdb::db::CREATE), + )?; + + let lmdb_write_transaction = lmdb::WriteTransaction::new(environment)?; + + Ok(Self { + lmdb_write_transaction, + height_to_hash, + outpoint_to_ordinal_ranges, + key_to_satpoint, + }) + } + + pub(crate) fn commit(self) -> Result { + Ok(self.lmdb_write_transaction.commit()?) + } + + pub(crate) fn height(&self) -> Result { + Ok( + self + .lmdb_write_transaction + .cursor(&self.height_to_hash)? + .last::<[u8], [u8]>(&self.lmdb_write_transaction.access()) + .into_option()? + .map(|(key, _value)| u64::from_be_bytes(key.try_into().unwrap()) + 1) + .unwrap_or_default(), + ) + } + + pub(crate) fn blockhash_at_height(&self, height: u64) -> Result>> { + Ok( + self + .lmdb_write_transaction + .access() + .get::<[u8], [u8]>(&self.height_to_hash, &height.to_be_bytes()) + .into_option()? + .map(|value| value.to_vec()), + ) + } + + pub(crate) fn set_blockhash_at_height(&mut self, height: u64, blockhash: BlockHash) -> Result { + self.lmdb_write_transaction.access().put( + &self.height_to_hash, + &height.to_be_bytes(), + blockhash.as_ref(), + lmdb::put::Flags::empty(), + )?; + Ok(()) + } + + pub(crate) fn insert_outpoint(&mut self, outpoint: &[u8], ordinal_ranges: &[u8]) -> Result { + self.lmdb_write_transaction.access().put( + &self.outpoint_to_ordinal_ranges, + outpoint, + ordinal_ranges, + lmdb::put::Flags::empty(), + )?; + Ok(()) + } + + pub(crate) fn remove_outpoint(&mut self, outpoint: &[u8]) -> Result { + self + .lmdb_write_transaction + .access() + .del_key(&self.outpoint_to_ordinal_ranges, outpoint)?; + Ok(()) + } + + pub(crate) fn get_ordinal_ranges(&self, outpoint: &[u8]) -> Result>> { + Ok( + self + .lmdb_write_transaction + .access() + .get::<[u8], [u8]>(&self.outpoint_to_ordinal_ranges, outpoint) + .into_option()? + .map(|value| value.to_vec()), + ) + } + + pub(crate) fn insert_satpoint(&mut self, key: &[u8], satpoint: &[u8]) -> Result { + self.lmdb_write_transaction.access().put( + &self.key_to_satpoint, + key, + satpoint, + lmdb::put::Flags::empty(), + )?; + Ok(()) + } + + pub(crate) fn remove_satpoint(&mut self, key: &[u8]) -> Result { + let mut cursor = self.lmdb_write_transaction.cursor(&self.key_to_satpoint)?; + let mut access = self.lmdb_write_transaction.access(); + cursor.seek_range_k::<[u8], [u8]>(&access, key)?; + cursor.del(&mut access, lmdb::del::Flags::empty())?; + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index 3fbaebdacf..75e3d51bcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,19 @@ +#![allow(clippy::too_many_arguments)] + use { crate::{ arguments::Arguments, bytes::Bytes, epoch::Epoch, height::Height, index::Index, key::Key, options::Options, ordinal::Ordinal, sat_point::SatPoint, subcommand::Subcommand, }, bitcoin::{ - blockdata::constants::COIN_VALUE, consensus::Decodable, consensus::Encodable, Block, OutPoint, - Transaction, + blockdata::constants::COIN_VALUE, consensus::Decodable, consensus::Encodable, Block, BlockHash, + OutPoint, Transaction, Txid, }, + chrono::{DateTime, NaiveDateTime, Utc}, clap::Parser, derive_more::{Display, FromStr}, integer_cbrt::IntegerCubeRoot, integer_sqrt::IntegerSquareRoot, - redb::{Database, ReadableTable, Table, TableDefinition}, std::{ cell::Cell, cmp::Ordering, @@ -22,6 +24,7 @@ use { path::PathBuf, process, str::FromStr, + sync::atomic::{self, AtomicU64}, time::{Duration, Instant}, }, }; @@ -37,12 +40,33 @@ mod ordinal; mod sat_point; mod subcommand; +#[cfg(feature = "redb")] +mod redb_database; +#[cfg(feature = "redb")] +use redb_database::{Database, WriteTransaction}; + +#[cfg(not(feature = "redb"))] +mod lmdb_database; +#[cfg(not(feature = "redb"))] +use lmdb_database::{Database, WriteTransaction}; + type Error = Box; type Result = std::result::Result; +static INTERRUPTS: AtomicU64 = AtomicU64::new(0); + fn main() { env_logger::init(); + ctrlc::set_handler(move || { + let interrupts = INTERRUPTS.fetch_add(1, atomic::Ordering::Relaxed); + + if interrupts > 5 { + process::exit(1); + } + }) + .expect("Error setting ctrl-c handler"); + if let Err(error) = Arguments::parse().run() { eprintln!("error: {}", error); process::exit(1); diff --git a/src/redb_database.rs b/src/redb_database.rs new file mode 100644 index 0000000000..a6331c48a6 --- /dev/null +++ b/src/redb_database.rs @@ -0,0 +1,198 @@ +use { + super::*, + redb::{ReadableTable, TableDefinition}, +}; + +const HEIGHT_TO_HASH: TableDefinition = TableDefinition::new("HEIGHT_TO_HASH"); +const OUTPOINT_TO_ORDINAL_RANGES: TableDefinition<[u8], [u8]> = + TableDefinition::new("OUTPOINT_TO_ORDINAL_RANGES"); +const KEY_TO_SATPOINT: TableDefinition<[u8], [u8]> = TableDefinition::new("KEY_TO_SATPOINT"); + +pub(crate) struct Database(redb::Database); + +impl Database { + pub(crate) fn open(options: &Options) -> Result { + let database = match unsafe { redb::Database::open("index.redb") } { + Ok(database) => database, + Err(redb::Error::Io(error)) if error.kind() == io::ErrorKind::NotFound => unsafe { + redb::Database::create("index.redb", options.index_size.0)? + }, + Err(error) => return Err(error.into()), + }; + + let tx = database.begin_write()?; + + tx.open_table(&HEIGHT_TO_HASH)?; + tx.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?; + tx.open_table(&KEY_TO_SATPOINT)?; + + tx.commit()?; + + Ok(Self(database)) + } + + pub(crate) fn begin_write(&self) -> Result { + WriteTransaction::new(&self.0) + } + + pub(crate) fn print_info(&self) -> Result { + let tx = self.0.begin_read()?; + + let height_to_hash = tx.open_table(&HEIGHT_TO_HASH)?; + + let blocks_indexed = height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0); + + let outputs_indexed = tx.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?.len()?; + + let stats = self.0.stats()?; + + println!("blocks indexed: {}", blocks_indexed); + println!("outputs indexed: {}", outputs_indexed); + println!("tree height: {}", stats.tree_height()); + println!("free pages: {}", stats.free_pages()); + println!("stored: {}", Bytes(stats.stored_bytes())); + println!("overhead: {}", Bytes(stats.overhead_bytes())); + println!("fragmented: {}", Bytes(stats.fragmented_bytes())); + println!( + "index size: {}", + Bytes(std::fs::metadata("index.redb")?.len().try_into()?) + ); + + Ok(()) + } + + pub(crate) fn height(&self) -> Result { + let tx = self.0.begin_read()?; + + let height_to_hash = tx.open_table(&HEIGHT_TO_HASH)?; + + Ok( + height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0), + ) + } + + pub(crate) fn find(&self, ordinal: Ordinal) -> Result, Vec)>> { + let rtx = self.0.begin_read()?; + + let height_to_hash = rtx.open_table(&HEIGHT_TO_HASH)?; + + match height_to_hash.range(0..)?.rev().next() { + Some((height, _hash)) if height >= ordinal.height().0 => {} + _ => return Ok(None), + } + + let key_to_satpoint = rtx.open_table(&KEY_TO_SATPOINT)?; + + match key_to_satpoint + .range([].as_slice()..=Key::new(ordinal).encode().as_slice())? + .rev() + .next() + { + Some((start_key, start_satpoint)) => Ok(Some((start_key.to_vec(), start_satpoint.to_vec()))), + None => Ok(None), + } + } + + pub(crate) fn list(&self, outpoint: &[u8]) -> Result>> { + Ok( + self + .0 + .begin_read()? + .open_table(&OUTPOINT_TO_ORDINAL_RANGES)? + .get(outpoint)? + .map(|outpoint| outpoint.to_vec()), + ) + } +} + +pub(crate) struct WriteTransaction<'a> { + inner: redb::DatabaseTransaction<'a>, + height_to_hash: redb::Table<'a, u64, [u8]>, + outpoint_to_ordinal_ranges: redb::Table<'a, [u8], [u8]>, + key_to_satpoint: redb::Table<'a, [u8], [u8]>, +} + +impl<'a> WriteTransaction<'a> { + pub(crate) fn new(database: &'a redb::Database) -> Result { + let inner = database.begin_write()?; + let height_to_hash = inner.open_table(&HEIGHT_TO_HASH)?; + let outpoint_to_ordinal_ranges = inner.open_table(&OUTPOINT_TO_ORDINAL_RANGES)?; + let key_to_satpoint = inner.open_table(&KEY_TO_SATPOINT)?; + + Ok(Self { + inner, + height_to_hash, + outpoint_to_ordinal_ranges, + key_to_satpoint, + }) + } + + pub(crate) fn commit(self) -> Result { + self.inner.commit()?; + Ok(()) + } + + pub(crate) fn height(&self) -> Result { + Ok( + self + .height_to_hash + .range(0..)? + .rev() + .next() + .map(|(height, _hash)| height + 1) + .unwrap_or(0), + ) + } + + pub(crate) fn blockhash_at_height(&self, height: u64) -> Result> { + Ok(self.height_to_hash.get(&height)?) + } + + pub(crate) fn set_blockhash_at_height(&mut self, height: u64, blockhash: BlockHash) -> Result { + self.height_to_hash.insert(&height, &blockhash)?; + Ok(()) + } + + pub(crate) fn insert_outpoint(&mut self, outpoint: &[u8], ordinal_ranges: &[u8]) -> Result { + self + .outpoint_to_ordinal_ranges + .insert(outpoint, ordinal_ranges)?; + Ok(()) + } + + pub(crate) fn remove_outpoint(&mut self, outpoint: &[u8]) -> Result { + self.outpoint_to_ordinal_ranges.remove(outpoint)?; + Ok(()) + } + + pub(crate) fn get_ordinal_ranges(&self, outpoint: &[u8]) -> Result> { + Ok(self.outpoint_to_ordinal_ranges.get(outpoint)?) + } + + pub(crate) fn insert_satpoint(&mut self, key: &[u8], satpoint: &[u8]) -> Result { + self.key_to_satpoint.insert(key, satpoint)?; + Ok(()) + } + + pub(crate) fn remove_satpoint(&mut self, key: &[u8]) -> Result { + let key = self + .key_to_satpoint + .range(key..)? + .next() + .unwrap() + .0 + .to_vec(); + self.key_to_satpoint.remove(&key)?; + Ok(()) + } +} diff --git a/src/subcommand/find.rs b/src/subcommand/find.rs index 18f85c7ea6..def66e7b13 100644 --- a/src/subcommand/find.rs +++ b/src/subcommand/find.rs @@ -9,7 +9,7 @@ pub(crate) struct Find { impl Find { pub(crate) fn run(self, options: Options) -> Result<()> { - let index = Index::index(options)?; + let index = Index::index(&options)?; match index.find(self.ordinal)? { Some((block, tx, satpoint)) => { diff --git a/src/subcommand/index.rs b/src/subcommand/index.rs index 56f4139958..4f270b4e8e 100644 --- a/src/subcommand/index.rs +++ b/src/subcommand/index.rs @@ -1,6 +1,6 @@ use super::*; pub(crate) fn run(options: Options) -> Result<()> { - Index::index(options)?; + Index::index(&options)?; Ok(()) } diff --git a/src/subcommand/info.rs b/src/subcommand/info.rs index e62ce5a676..10cc010e5e 100644 --- a/src/subcommand/info.rs +++ b/src/subcommand/info.rs @@ -1,6 +1,6 @@ use super::*; pub(crate) fn run(options: Options) -> Result { - Index::open(options)?.print_info()?; + Index::open(&options)?.print_info()?; Ok(()) } diff --git a/src/subcommand/list.rs b/src/subcommand/list.rs index cf34cb74ee..a2ed797334 100644 --- a/src/subcommand/list.rs +++ b/src/subcommand/list.rs @@ -7,13 +7,16 @@ pub(crate) struct List { impl List { pub(crate) fn run(self, options: Options) -> Result<()> { - let index = Index::index(options)?; - let ranges = index.list(self.outpoint)?; + let index = Index::index(&options)?; - for (start, end) in ranges { - println!("[{start},{end})"); + match index.list(self.outpoint)? { + Some(ranges) => { + for (start, end) in ranges { + println!("[{start},{end})"); + } + Ok(()) + } + None => Err("Output not found".into()), } - - Ok(()) } } diff --git a/tests/find.rs b/tests/find.rs index 279c2f24c3..2aee8ab4b1 100644 --- a/tests/find.rs +++ b/tests/find.rs @@ -129,7 +129,7 @@ fn empty_index() -> Result { } #[test] -fn unmined_satoshi() -> Result { +fn unmined_satoshi_in_second_block() -> Result { Test::new()? .block() .expected_stderr("error: Ordinal has not been mined as of index height\n") @@ -137,3 +137,12 @@ fn unmined_satoshi() -> Result { .command("find 5000000000") .run() } + +#[test] +fn unmined_satoshi_in_first_block() -> Result { + Test::new()? + .expected_stderr("error: Ordinal has not been mined as of index height\n") + .expected_status(1) + .command("find 0") + .run() +} diff --git a/tests/index.rs b/tests/index.rs index 607b13e2bb..0a1d2adcdc 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -1,19 +1,5 @@ use super::*; -#[test] -fn default_index_size() -> Result { - let tempdir = Test::new()? - .command("find 0") - .expected_stdout("0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0:0\n") - .block() - .output()? - .tempdir; - - assert_eq!(tempdir.path().join("index.redb").metadata()?.len(), 1 << 20); - - Ok(()) -} - #[test] fn incremental_indexing() -> Result { let output = Test::new()? @@ -37,6 +23,7 @@ fn incremental_indexing() -> Result { } #[test] +#[cfg(feature = "redb")] fn custom_index_size() -> Result { let tempdir = Test::new()? .command("--index-size 2097152 find 0") @@ -51,6 +38,7 @@ fn custom_index_size() -> Result { } #[test] +#[cfg(feature = "redb")] fn human_readable_index_size() -> Result { let tempdir = Test::new()? .command("--index-size 2mib find 0") @@ -63,3 +51,18 @@ fn human_readable_index_size() -> Result { Ok(()) } + +#[test] +#[cfg(feature = "redb")] +fn default_index_size() -> Result { + let tempdir = Test::new()? + .command("find 0") + .expected_stdout("0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0:0\n") + .block() + .output()? + .tempdir; + + assert_eq!(tempdir.path().join("index.redb").metadata()?.len(), 1 << 20); + + Ok(()) +} diff --git a/tests/info.rs b/tests/info.rs index 44c108f4df..ccb3ec0cb4 100644 --- a/tests/info.rs +++ b/tests/info.rs @@ -1,6 +1,7 @@ use super::*; #[test] +#[cfg(feature = "redb")] fn basic() -> Result { let output = Test::new()?.command("index").block().output()?; @@ -21,3 +22,20 @@ fn basic() -> Result { ) .run() } + +#[test] +#[cfg(not(feature = "redb"))] +fn basic() -> Result { + let output = Test::new()?.command("index").block().output()?; + + Test::with_tempdir(output.tempdir) + .command("info") + .stdout_regex( + r" + blocks indexed: 1 + data and metadata: \d+ + " + .unindent(), + ) + .run() +} diff --git a/tests/integration.rs b/tests/integration.rs index 4afb004d4b..7f6f6244d6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -166,10 +166,14 @@ impl Test { close_handle.close(); + let stdout = str::from_utf8(&output.stdout)?; let stderr = str::from_utf8(&output.stderr)?; if output.status.code() != Some(self.expected_status) { - panic!("Test failed: {}\n{}", output.status, stderr); + panic!( + "Test failed: {}\nstdout:\n{}\nstderr:\n{}", + output.status, stdout, stderr + ); } let re = Regex::new(r"(?m)^\[.*\n")?; @@ -180,8 +184,6 @@ impl Test { assert_eq!(re.replace_all(stderr, ""), self.expected_stderr); - let stdout = str::from_utf8(&output.stdout)?; - match self.expected_stdout { Expected::String(expected_stdout) => assert_eq!(stdout, expected_stdout), Expected::Regex(expected_stdout) => assert!( diff --git a/tests/list.rs b/tests/list.rs index 119f79cd08..ba12f453e2 100644 --- a/tests/list.rs +++ b/tests/list.rs @@ -209,3 +209,19 @@ fn null_input() -> Result { .expected_stdout("") .run() } + +#[test] +fn old_transactions_are_pruned() -> Result { + Test::new()? + .command("list 0396bc915f141f7de025f72ae9b6bb8dcdb5f444fc245d8fac486ba67a38eef9:0") + .block() + .block() + .transaction(TransactionOptions { + slots: &[(0, 0, 0)], + output_count: 1, + fee: 50 * 100_000_000, + }) + .expected_stderr("error: Output not found\n") + .expected_status(1) + .run() +}