diff --git a/Cargo.toml b/Cargo.toml index 8c9268bab..e7072cdea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ eyre = { version = "0.6", default-features = false, features = [ "track-caller", futures = { version = "0.3", default-features = false } humantime = { version = "2.1.0", default-features = false } humantime-serde = { version = "1.1", default-features = false } +iota-crypto = { version = "0.15", default-features = false, features = [ "blake2b" ] } iota-types = { version = "1.0.0-rc.4", default-features = false, features = [ "api", "block", "std" ] } mongodb = { version = "2.3", default-features = false, features = [ "tokio-runtime" ] } packable = { version = "0.7", default-features = false } @@ -74,9 +75,6 @@ zeroize = { version = "1.5", default-features = false, features = [ "std" ], opt inx = { version = "1.0.0-beta.8", default-features = false, optional = true } tonic = { version = "0.8", default-features = false, optional = true } -# PoI -iota-crypto = { version = "0.15", default-features = false, features = [ "blake2b" ], optional = true } - [dev-dependencies] iota-types = { version = "1.0.0-rc.4", default-features = false, features = [ "api", "block", "std", "rand" ] } rand = { version = "0.8", default-features = false, features = [ "std" ] } @@ -118,7 +116,6 @@ metrics = [ ] poi = [ "api", - "dep:iota-crypto", ] rand = [ "iota-types/rand", diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index df2e160ba..cbadd5d93 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -360,6 +360,13 @@ impl ClArgs { tracing::info!("Indexes built successfully."); return Ok(PostCommand::Exit); } + Subcommands::MigrateTo { version } => { + tracing::info!("Connecting to database using hosts: `{}`.", config.mongodb.hosts_str()?); + let db = chronicle::db::MongoDb::connect(&config.mongodb).await?; + crate::migrations::migrate(version, &db).await?; + tracing::info!("Migration completed successfully."); + return Ok(PostCommand::Exit); + } _ => (), } } @@ -396,6 +403,8 @@ pub enum Subcommands { }, /// Manually build indexes. BuildIndexes, + /// Migrate to a new version. + MigrateTo { version: String }, } #[derive(Copy, Clone, PartialEq, Eq)] diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 3b4fd7526..37761837d 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -8,6 +8,7 @@ mod api; mod cli; mod config; +mod migrations; mod process; #[cfg(feature = "inx")] mod stardust_inx; diff --git a/src/bin/inx-chronicle/migrations/migrate_1_0_0_beta_31.rs b/src/bin/inx-chronicle/migrations/migrate_1_0_0_beta_31.rs new file mode 100644 index 000000000..ccfb5033a --- /dev/null +++ b/src/bin/inx-chronicle/migrations/migrate_1_0_0_beta_31.rs @@ -0,0 +1,120 @@ +// Copyright 2023 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use chronicle::{ + db::{collections::OutputCollection, MongoDb, MongoDbCollection, MongoDbCollectionExt}, + types::stardust::block::output::{AliasId, NftId, OutputId}, +}; +use futures::TryStreamExt; +use mongodb::{bson::doc, options::IndexOptions, IndexModel}; +use serde::Deserialize; + +pub const PREV_VERSION: &str = "1.0.0-beta.30"; + +pub async fn migrate(db: &MongoDb) -> eyre::Result<()> { + let collection = db.collection::(); + + #[derive(Deserialize)] + struct Res { + output_id: OutputId, + } + + // Convert the outputs with implicit IDs + let outputs = collection + .aggregate::( + [ + doc! { "$match": { "$or": [ + { "output.alias_id": AliasId::implicit() }, + { "output.nft_id": NftId::implicit() } + ] } }, + doc! { "$project": { + "output_id": "$_id" + } }, + ], + None, + ) + .await? + .map_ok(|res| res.output_id) + .try_collect::>() + .await?; + + for output_id in outputs { + // Alias and nft are the same length so both can be done this way since they are just serialized as bytes + let id = AliasId::from(output_id); + collection + .update_one( + doc! { "_id": output_id }, + doc! { "$set": { "details.indexed_id": id } }, + None, + ) + .await?; + } + + // Get the outputs that don't have implicit IDs + collection + .update_many( + doc! { + "output.kind": "alias", + "output.alias_id": { "$ne": AliasId::implicit() }, + }, + vec![doc! { "$set": { + "details.indexed_id": "$output.alias_id", + } }], + None, + ) + .await?; + + collection + .update_many( + doc! { + "output.kind": "nft", + "output.nft_id": { "$ne": NftId::implicit() }, + }, + vec![doc! { "$set": { + "details.indexed_id": "$output.nft_id", + } }], + None, + ) + .await?; + + collection + .update_many( + doc! { "output.kind": "foundry" }, + vec![doc! { "$set": { + "details.indexed_id": "$output.foundry_id", + } }], + None, + ) + .await?; + + collection + .collection() + .drop_index("output_alias_id_index", None) + .await?; + + collection + .collection() + .drop_index("output_foundry_id_index", None) + .await?; + + collection.collection().drop_index("output_nft_id_index", None).await?; + + collection + .create_index( + IndexModel::builder() + .keys(doc! { "details.indexed_id": 1 }) + .options( + IndexOptions::builder() + .name("output_indexed_id_index".to_string()) + .partial_filter_expression(doc! { + "details.indexed_id": { "$exists": true }, + }) + .build(), + ) + .build(), + None, + ) + .await?; + + Ok(()) +} diff --git a/src/bin/inx-chronicle/migrations/mod.rs b/src/bin/inx-chronicle/migrations/mod.rs new file mode 100644 index 000000000..92186f797 --- /dev/null +++ b/src/bin/inx-chronicle/migrations/mod.rs @@ -0,0 +1,22 @@ +// Copyright 2023 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use chronicle::db::MongoDb; +use eyre::bail; + +pub mod migrate_1_0_0_beta_31; + +pub async fn migrate(version: &str, db: &MongoDb) -> eyre::Result<()> { + let curr_version = std::env!("CARGO_PKG_VERSION"); + match version { + "1.0.0-beta.31" => { + if migrate_1_0_0_beta_31::PREV_VERSION == curr_version { + migrate_1_0_0_beta_31::migrate(db).await?; + } else { + bail!("cannot migrate to {} from {}", version, curr_version); + } + } + _ => bail!("cannot migrate version {}", version), + } + Ok(()) +} diff --git a/src/db/collections/outputs/indexer/mod.rs b/src/db/collections/outputs/indexer/mod.rs index c6b17d079..316bd04a4 100644 --- a/src/db/collections/outputs/indexer/mod.rs +++ b/src/db/collections/outputs/indexer/mod.rs @@ -15,7 +15,7 @@ use mongodb::{ options::IndexOptions, IndexModel, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; pub use self::{ alias::AliasOutputsQuery, basic::BasicOutputsQuery, foundry::FoundryOutputsQuery, nft::NftOutputsQuery, @@ -25,7 +25,7 @@ use crate::{ db::{collections::SortOrder, mongodb::MongoDbCollectionExt}, types::{ ledger::OutputMetadata, - stardust::block::output::{AliasId, FoundryId, NftId, OutputId}, + stardust::block::output::{AliasId, AliasOutput, FoundryId, FoundryOutput, NftId, NftOutput, OutputId}, tangle::MilestoneIndex, }, }; @@ -43,7 +43,8 @@ pub struct OutputsResult { pub outputs: Vec, } -#[derive(From)] +#[derive(Copy, Clone, Debug, Serialize, Deserialize, From)] +#[serde(untagged)] #[allow(missing_docs)] pub enum IndexedId { Alias(AliasId), @@ -51,6 +52,17 @@ pub enum IndexedId { Nft(NftId), } +impl IndexedId { + /// Get the indexed ID kind. + pub fn kind(&self) -> &'static str { + match self { + IndexedId::Alias(_) => AliasOutput::KIND, + IndexedId::Foundry(_) => FoundryOutput::KIND, + IndexedId::Nft(_) => NftOutput::KIND, + } + } +} + impl From for Bson { fn from(id: IndexedId) -> Self { match id { @@ -75,16 +87,12 @@ impl OutputCollection { ledger_index: MilestoneIndex, ) -> Result, Error> { let id = id.into(); - let id_string = match id { - IndexedId::Alias(_) => "output.alias_id", - IndexedId::Foundry(_) => "output.foundry_id", - IndexedId::Nft(_) => "output.nft_id", - }; let mut res = self .aggregate( [ doc! { "$match": { - id_string: id, + "output.kind": id.kind(), + "details.indexed_id": id, "metadata.booked.milestone_index": { "$lte": ledger_index }, "metadata.spent_metadata.spent.milestone_index": { "$not": { "$lte": ledger_index } } } }, @@ -183,44 +191,12 @@ impl OutputCollection { self.create_index( IndexModel::builder() - .keys(doc! { "output.alias_id": 1 }) - .options( - IndexOptions::builder() - .name("output_alias_id_index".to_string()) - .partial_filter_expression(doc! { - "output.alias_id": { "$exists": true }, - }) - .build(), - ) - .build(), - None, - ) - .await?; - - self.create_index( - IndexModel::builder() - .keys(doc! { "output.foundry_id": 1 }) - .options( - IndexOptions::builder() - .name("output_foundry_id_index".to_string()) - .partial_filter_expression(doc! { - "output.foundry_id": { "$exists": true }, - }) - .build(), - ) - .build(), - None, - ) - .await?; - - self.create_index( - IndexModel::builder() - .keys(doc! { "output.nft_id": 1 }) + .keys(doc! { "details.indexed_id": 1 }) .options( IndexOptions::builder() - .name("output_nft_id_index".to_string()) + .name("output_indexed_id_index".to_string()) .partial_filter_expression(doc! { - "output.nft_id": { "$exists": true }, + "details.indexed_id": { "$exists": true }, }) .build(), ) diff --git a/src/db/collections/outputs/mod.rs b/src/db/collections/outputs/mod.rs index 7a1e93b56..934e771ba 100644 --- a/src/db/collections/outputs/mod.rs +++ b/src/db/collections/outputs/mod.rs @@ -28,7 +28,7 @@ use crate::{ LedgerOutput, LedgerSpent, MilestoneIndexTimestamp, OutputMetadata, RentStructureBytes, SpentMetadata, }, stardust::block::{ - output::{Output, OutputId}, + output::{AliasId, NftId, Output, OutputId}, Address, BlockId, }, tangle::MilestoneIndex, @@ -95,6 +95,8 @@ struct OutputDetails { address: Option
, is_trivial_unlock: bool, rent_structure: RentStructureBytes, + #[serde(skip_serializing_if = "Option::is_none")] + indexed_id: Option, } impl From<&LedgerOutput> for OutputDocument { @@ -114,6 +116,26 @@ impl From<&LedgerOutput> for OutputDocument { address, is_trivial_unlock, rent_structure: rec.rent_structure, + indexed_id: match &rec.output { + Output::Alias(output) => Some( + if output.alias_id == AliasId::implicit() { + AliasId::from(rec.output_id) + } else { + output.alias_id + } + .into(), + ), + Output::Nft(output) => Some( + if output.nft_id == NftId::implicit() { + NftId::from(rec.output_id) + } else { + output.nft_id + } + .into(), + ), + Output::Foundry(output) => Some(output.foundry_id.into()), + _ => None, + }, }, } } diff --git a/src/db/mongodb/collection.rs b/src/db/mongodb/collection.rs index 73e09a3f4..8f0a779ea 100644 --- a/src/db/mongodb/collection.rs +++ b/src/db/mongodb/collection.rs @@ -125,6 +125,16 @@ pub trait MongoDbCollectionExt: MongoDbCollection { self.collection().update_one(doc, update, options).await } + /// Calls [`mongodb::Collection::update_many()`]. + async fn update_many( + &self, + doc: Document, + update: impl Into + Send + Sync, + options: impl Into> + Send + Sync, + ) -> Result { + self.collection().update_many(doc, update, options).await + } + /// Calls [`mongodb::Collection::replace_one()`] and coerces the document type. async fn replace_one( &self, diff --git a/src/types/stardust/block/output/alias.rs b/src/types/stardust/block/output/alias.rs index 36500e35d..5fd31976e 100644 --- a/src/types/stardust/block/output/alias.rs +++ b/src/types/stardust/block/output/alias.rs @@ -13,7 +13,7 @@ use super::{ feature::Feature, native_token::NativeToken, unlock_condition::{GovernorAddressUnlockCondition, StateControllerAddressUnlockCondition}, - OutputAmount, + OutputAmount, OutputId, }; use crate::types::{context::TryFromWithContext, util::bytify}; @@ -54,6 +54,12 @@ impl From for iota::dto::AliasIdDto { } } +impl From for AliasId { + fn from(value: OutputId) -> Self { + Self(value.hash()) + } +} + impl FromStr for AliasId { type Err = iota_types::block::Error; diff --git a/src/types/stardust/block/output/mod.rs b/src/types/stardust/block/output/mod.rs index c832f9c52..d27a479c2 100644 --- a/src/types/stardust/block/output/mod.rs +++ b/src/types/stardust/block/output/mod.rs @@ -15,6 +15,7 @@ pub mod treasury; use std::{borrow::Borrow, str::FromStr}; +use crypto::hashes::{blake2b::Blake2b256, Digest}; use iota_types::block::output as iota; use mongodb::bson::{doc, Bson}; use packable::PackableExt; @@ -42,7 +43,19 @@ use crate::types::{ }; /// The amount of tokens associated with an output. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize, derive_more::From)] +#[derive( + Copy, + Clone, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + derive_more::From, + derive_more::AddAssign, + derive_more::SubAssign, +)] pub struct OutputAmount(#[serde(with = "crate::types::util::stringify")] pub u64); /// The index of an output within a transaction. @@ -63,6 +76,16 @@ impl OutputId { pub fn to_hex(&self) -> String { prefix_hex::encode([self.transaction_id.0.as_ref(), &self.index.to_le_bytes()].concat()) } + + /// Hash the [`OutputId`] with BLAKE2b-256. + #[inline(always)] + pub fn hash(&self) -> [u8; 32] { + Blake2b256::digest(self.as_bytes()).into() + } + + fn as_bytes(&self) -> Vec { + [self.transaction_id.0.as_ref(), &self.index.to_le_bytes()].concat() + } } impl From for OutputId { diff --git a/src/types/stardust/block/output/nft.rs b/src/types/stardust/block/output/nft.rs index 6d9443acf..1fbcaa0c4 100644 --- a/src/types/stardust/block/output/nft.rs +++ b/src/types/stardust/block/output/nft.rs @@ -13,7 +13,7 @@ use super::{ unlock_condition::{ AddressUnlockCondition, ExpirationUnlockCondition, StorageDepositReturnUnlockCondition, TimelockUnlockCondition, }, - Feature, NativeToken, OutputAmount, + Feature, NativeToken, OutputAmount, OutputId, }; use crate::types::{context::TryFromWithContext, util::bytify}; @@ -54,6 +54,12 @@ impl From for iota::dto::NftIdDto { } } +impl From for NftId { + fn from(value: OutputId) -> Self { + Self(value.hash()) + } +} + impl FromStr for NftId { type Err = iota_types::block::Error;