fix(indexer)!: add indexed id to fix implicit aliases and nfts (#1075)

* Add indexed id to fix implicit aliases and nfts. Add migration binary.

* Move migrations to a CLI command

* Check for indexed id kind

* Make crypto dep mandatory

* Small fix

* Mongo doesn't want to hear that it has to match

* Use pipeline for update many

* Fix details level

---------

Co-authored-by: /alex/ <alexander.schmidt@iota.org>

Cargo.toml

@@ -32,6 +32,7 @@ eyre = { version = "0.6", default-features = false, features = [ "track-caller",
 futures = { version = "0.3", default-features = false }
 humantime = { version = "2.1.0", default-features = false }
 humantime-serde = { version = "1.1", default-features = false }
+iota-crypto = { version = "0.15", default-features = false, features = [ "blake2b" ] }
 iota-types = { version = "1.0.0-rc.4", default-features = false, features = [ "api", "block", "std" ] }
 mongodb = { version = "2.3", default-features = false, features = [ "tokio-runtime" ] }
 packable = { version = "0.7", default-features = false }
@@ -74,9 +75,6 @@ zeroize = { version = "1.5", default-features = false, features = [ "std" ], opt
 inx = { version = "1.0.0-beta.8", default-features = false, optional = true }
 tonic = { version = "0.8", default-features = false, optional = true }
 
-# PoI
-iota-crypto = { version = "0.15", default-features = false, features = [ "blake2b" ], optional = true }
-
 [dev-dependencies]
 iota-types = { version = "1.0.0-rc.4", default-features = false, features = [ "api", "block", "std", "rand" ] }
 rand = { version = "0.8", default-features = false, features = [ "std" ] }
@@ -118,7 +116,6 @@ metrics = [
 ]
 poi = [
     "api",
-    "dep:iota-crypto",
 ]
 rand = [
     "iota-types/rand",

src/bin/inx-chronicle/cli.rs

@@ -360,6 +360,13 @@ impl ClArgs {
                     tracing::info!("Indexes built successfully.");
                     return Ok(PostCommand::Exit);
                 }
+                Subcommands::MigrateTo { version } => {
+                    tracing::info!("Connecting to database using hosts: `{}`.", config.mongodb.hosts_str()?);
+                    let db = chronicle::db::MongoDb::connect(&config.mongodb).await?;
+                    crate::migrations::migrate(version, &db).await?;
+                    tracing::info!("Migration completed successfully.");
+                    return Ok(PostCommand::Exit);
+                }
                 _ => (),
             }
         }
@@ -396,6 +403,8 @@ pub enum Subcommands {
     },
     /// Manually build indexes.
     BuildIndexes,
+    /// Migrate to a new version.
+    MigrateTo { version: String },
 }
 
 #[derive(Copy, Clone, PartialEq, Eq)]

src/bin/inx-chronicle/main.rs

@@ -8,6 +8,7 @@
 mod api;
 mod cli;
 mod config;
+mod migrations;
 mod process;
 #[cfg(feature = "inx")]
 mod stardust_inx;

src/bin/inx-chronicle/migrations/migrate_1_0_0_beta_31.rs

@@ -0,0 +1,120 @@
+// Copyright 2023 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+use chronicle::{
+    db::{collections::OutputCollection, MongoDb, MongoDbCollection, MongoDbCollectionExt},
+    types::stardust::block::output::{AliasId, NftId, OutputId},
+};
+use futures::TryStreamExt;
+use mongodb::{bson::doc, options::IndexOptions, IndexModel};
+use serde::Deserialize;
+
+pub const PREV_VERSION: &str = "1.0.0-beta.30";
+
+pub async fn migrate(db: &MongoDb) -> eyre::Result<()> {
+    let collection = db.collection::<OutputCollection>();
+
+    #[derive(Deserialize)]
+    struct Res {
+        output_id: OutputId,
+    }
+
+    // Convert the outputs with implicit IDs
+    let outputs = collection
+        .aggregate::<Res>(
+            [
+                doc! { "$match": { "$or": [
+                    { "output.alias_id": AliasId::implicit() },
+                    { "output.nft_id": NftId::implicit() }
+                ] } },
+                doc! { "$project": {
+                    "output_id": "$_id"
+                } },
+            ],
+            None,
+        )
+        .await?
+        .map_ok(|res| res.output_id)
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    for output_id in outputs {
+        // Alias and nft are the same length so both can be done this way since they are just serialized as bytes
+        let id = AliasId::from(output_id);
+        collection
+            .update_one(
+                doc! { "_id": output_id },
+                doc! { "$set": { "details.indexed_id": id } },
+                None,
+            )
+            .await?;
+    }
+
+    // Get the outputs that don't have implicit IDs
+    collection
+        .update_many(
+            doc! {
+                "output.kind": "alias",
+                "output.alias_id": { "$ne": AliasId::implicit() },
+            },
+            vec![doc! { "$set": {
+                "details.indexed_id": "$output.alias_id",
+            } }],
+            None,
+        )
+        .await?;
+
+    collection
+        .update_many(
+            doc! {
+                "output.kind": "nft",
+                "output.nft_id": { "$ne": NftId::implicit() },
+            },
+            vec![doc! { "$set": {
+                "details.indexed_id": "$output.nft_id",
+            } }],
+            None,
+        )
+        .await?;
+
+    collection
+        .update_many(
+            doc! { "output.kind": "foundry" },
+            vec![doc! { "$set": {
+                "details.indexed_id": "$output.foundry_id",
+            } }],
+            None,
+        )
+        .await?;
+
+    collection
+        .collection()
+        .drop_index("output_alias_id_index", None)
+        .await?;
+
+    collection
+        .collection()
+        .drop_index("output_foundry_id_index", None)
+        .await?;
+
+    collection.collection().drop_index("output_nft_id_index", None).await?;
+
+    collection
+        .create_index(
+            IndexModel::builder()
+                .keys(doc! { "details.indexed_id": 1 })
+                .options(
+                    IndexOptions::builder()
+                        .name("output_indexed_id_index".to_string())
+                        .partial_filter_expression(doc! {
+                            "details.indexed_id": { "$exists": true },
+                        })
+                        .build(),
+                )
+                .build(),
+            None,
+        )
+        .await?;
+
+    Ok(())
+}

src/bin/inx-chronicle/migrations/mod.rs

@@ -0,0 +1,22 @@
+// Copyright 2023 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+use chronicle::db::MongoDb;
+use eyre::bail;
+
+pub mod migrate_1_0_0_beta_31;
+
+pub async fn migrate(version: &str, db: &MongoDb) -> eyre::Result<()> {
+    let curr_version = std::env!("CARGO_PKG_VERSION");
+    match version {
+        "1.0.0-beta.31" => {
+            if migrate_1_0_0_beta_31::PREV_VERSION == curr_version {
+                migrate_1_0_0_beta_31::migrate(db).await?;
+            } else {
+                bail!("cannot migrate to {} from {}", version, curr_version);
+            }
+        }
+        _ => bail!("cannot migrate version {}", version),
+    }
+    Ok(())
+}

src/db/collections/outputs/indexer/mod.rs

@@ -15,7 +15,7 @@ use mongodb::{
     options::IndexOptions,
     IndexModel,
 };
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 pub use self::{
     alias::AliasOutputsQuery, basic::BasicOutputsQuery, foundry::FoundryOutputsQuery, nft::NftOutputsQuery,
@@ -25,7 +25,7 @@ use crate::{
     db::{collections::SortOrder, mongodb::MongoDbCollectionExt},
     types::{
         ledger::OutputMetadata,
-        stardust::block::output::{AliasId, FoundryId, NftId, OutputId},
+        stardust::block::output::{AliasId, AliasOutput, FoundryId, FoundryOutput, NftId, NftOutput, OutputId},
         tangle::MilestoneIndex,
     },
 };
@@ -43,14 +43,26 @@ pub struct OutputsResult {
     pub outputs: Vec<OutputResult>,
 }
 
-#[derive(From)]
+#[derive(Copy, Clone, Debug, Serialize, Deserialize, From)]
+#[serde(untagged)]
 #[allow(missing_docs)]
 pub enum IndexedId {
     Alias(AliasId),
     Foundry(FoundryId),
     Nft(NftId),
 }
 
+impl IndexedId {
+    /// Get the indexed ID kind.
+    pub fn kind(&self) -> &'static str {
+        match self {
+            IndexedId::Alias(_) => AliasOutput::KIND,
+            IndexedId::Foundry(_) => FoundryOutput::KIND,
+            IndexedId::Nft(_) => NftOutput::KIND,
+        }
+    }
+}
+
 impl From<IndexedId> for Bson {
     fn from(id: IndexedId) -> Self {
         match id {
@@ -75,16 +87,12 @@ impl OutputCollection {
         ledger_index: MilestoneIndex,
     ) -> Result<Option<IndexedOutputResult>, Error> {
         let id = id.into();
-        let id_string = match id {
-            IndexedId::Alias(_) => "output.alias_id",
-            IndexedId::Foundry(_) => "output.foundry_id",
-            IndexedId::Nft(_) => "output.nft_id",
-        };
         let mut res = self
             .aggregate(
                 [
                     doc! { "$match": {
-                        id_string: id,
+                        "output.kind": id.kind(),
+                        "details.indexed_id": id,
                         "metadata.booked.milestone_index": { "$lte": ledger_index },
                         "metadata.spent_metadata.spent.milestone_index": { "$not": { "$lte": ledger_index } }
                     } },
@@ -183,44 +191,12 @@ impl OutputCollection {
 
         self.create_index(
             IndexModel::builder()
-                .keys(doc! { "output.alias_id": 1 })
-                .options(
-                    IndexOptions::builder()
-                        .name("output_alias_id_index".to_string())
-                        .partial_filter_expression(doc! {
-                            "output.alias_id": { "$exists": true },
-                        })
-                        .build(),
-                )
-                .build(),
-            None,
-        )
-        .await?;
-
-        self.create_index(
-            IndexModel::builder()
-                .keys(doc! { "output.foundry_id": 1 })
-                .options(
-                    IndexOptions::builder()
-                        .name("output_foundry_id_index".to_string())
-                        .partial_filter_expression(doc! {
-                            "output.foundry_id": { "$exists": true },
-                        })
-                        .build(),
-                )
-                .build(),
-            None,
-        )
-        .await?;
-
-        self.create_index(
-            IndexModel::builder()
-                .keys(doc! { "output.nft_id": 1 })
+                .keys(doc! { "details.indexed_id": 1 })
                 .options(
                     IndexOptions::builder()
-                        .name("output_nft_id_index".to_string())
+                        .name("output_indexed_id_index".to_string())
                         .partial_filter_expression(doc! {
-                            "output.nft_id": { "$exists": true },
+                            "details.indexed_id": { "$exists": true },
                         })
                         .build(),
                 )

src/db/collections/outputs/mod.rs

@@ -28,7 +28,7 @@ use crate::{
             LedgerOutput, LedgerSpent, MilestoneIndexTimestamp, OutputMetadata, RentStructureBytes, SpentMetadata,
         },
         stardust::block::{
-            output::{Output, OutputId},
+            output::{AliasId, NftId, Output, OutputId},
             Address, BlockId,
         },
         tangle::MilestoneIndex,
@@ -95,6 +95,8 @@ struct OutputDetails {
     address: Option<Address>,
     is_trivial_unlock: bool,
     rent_structure: RentStructureBytes,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    indexed_id: Option<IndexedId>,
 }
 
 impl From<&LedgerOutput> for OutputDocument {
@@ -114,6 +116,26 @@ impl From<&LedgerOutput> for OutputDocument {
                 address,
                 is_trivial_unlock,
                 rent_structure: rec.rent_structure,
+                indexed_id: match &rec.output {
+                    Output::Alias(output) => Some(
+                        if output.alias_id == AliasId::implicit() {
+                            AliasId::from(rec.output_id)
+                        } else {
+                            output.alias_id
+                        }
+                        .into(),
+                    ),
+                    Output::Nft(output) => Some(
+                        if output.nft_id == NftId::implicit() {
+                            NftId::from(rec.output_id)
+                        } else {
+                            output.nft_id
+                        }
+                        .into(),
+                    ),
+                    Output::Foundry(output) => Some(output.foundry_id.into()),
+                    _ => None,
+                },
             },
         }
     }

src/db/mongodb/collection.rs

@@ -125,6 +125,16 @@ pub trait MongoDbCollectionExt: MongoDbCollection {
         self.collection().update_one(doc, update, options).await
     }
 
+    /// Calls [`mongodb::Collection::update_many()`].
+    async fn update_many(
+        &self,
+        doc: Document,
+        update: impl Into<UpdateModifications> + Send + Sync,
+        options: impl Into<Option<UpdateOptions>> + Send + Sync,
+    ) -> Result<UpdateResult, Error> {
+        self.collection().update_many(doc, update, options).await
+    }
+
     /// Calls [`mongodb::Collection::replace_one()`] and coerces the document type.
     async fn replace_one<T: Serialize + Send + Sync>(
         &self,

src/types/stardust/block/output/alias.rs

@@ -13,7 +13,7 @@ use super::{
     feature::Feature,
     native_token::NativeToken,
     unlock_condition::{GovernorAddressUnlockCondition, StateControllerAddressUnlockCondition},
-    OutputAmount,
+    OutputAmount, OutputId,
 };
 use crate::types::{context::TryFromWithContext, util::bytify};
 
@@ -54,6 +54,12 @@ impl From<AliasId> for iota::dto::AliasIdDto {
     }
 }
 
+impl From<OutputId> for AliasId {
+    fn from(value: OutputId) -> Self {
+        Self(value.hash())
+    }
+}
+
 impl FromStr for AliasId {
     type Err = iota_types::block::Error;