From 1b3b38c3f6fb2dcc46484bcce105d9ba4f807bb7 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 13 Jul 2022 13:35:31 +0200 Subject: [PATCH 01/18] Fix stardust feature deps; Add 'check-all-features' to canary --- .cargo/config.toml | 6 +- .github/workflows/_build.yml | 12 ++-- .github/workflows/canary.yml | 19 +++++++ Cargo.toml | 12 ++-- bin/inx-chronicle/src/api/extractors.rs | 69 +++++++++++++---------- bin/inx-chronicle/src/api/responses.rs | 66 ++++++++++++---------- bin/inx-chronicle/src/api/routes.rs | 39 +++++++------ bin/inx-chronicle/src/api/stardust/mod.rs | 4 +- bin/inx-chronicle/src/launcher.rs | 11 ++-- src/db/collections/block.rs | 2 +- src/db/collections/ledger_update.rs | 2 +- src/db/mod.rs | 1 + src/types/mod.rs | 2 + 13 files changed, 147 insertions(+), 98 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 7c6dfe8e9..57b9dc71a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,7 +1,7 @@ [alias] -ci-build-all = "build --all-targets --all-features" -ci-build-inx = "build --all-targets --no-default-features --features inx,stardust" -ci-build-api = "build --all-targets --no-default-features --features api-core,api-history,analytics,stardust" +ci-check-all = "check --all-targets --all-features" +ci-check-inx = "check --all-targets --no-default-features --features inx,stardust" +ci-check-api = "check --all-targets --no-default-features --features api-core,api-history,analytics,stardust" ci-clippy-all = "clippy --all-targets --all-features -- -D warnings" ci-clippy-inx = "clippy --all-targets --no-default-features --features inx,stardust -- -D warnings" diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index a258fb6ac..470177b53 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -42,22 +42,22 @@ jobs: - uses: Swatinem/rust-cache@v1 - - name: Build with all features + - name: Check (all features) uses: actions-rs/cargo@v1 with: - command: ci-build-all + command: ci-check-all - - name: Build with INX only + - name: Check (INX only) if: contains(inputs.os, 'ubuntu') uses: actions-rs/cargo@v1 with: - command: ci-build-inx + command: ci-check-inx - - name: Build with API only + - name: Check (API only) if: contains(inputs.os, 'ubuntu') uses: actions-rs/cargo@v1 with: - command: ci-build-api + command: ci-check-api - name: Test uses: actions-rs/cargo@v1 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index deee912fa..18cefa043 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -58,4 +58,23 @@ jobs: with: command: ci-udeps + check-all-features: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - uses: actions-rs/cargo@v1 + with: + command: install + args: --force cargo-all-features + + - uses: actions-rs/cargo@v1 + with: + command: check-all-features diff --git a/Cargo.toml b/Cargo.toml index 6578cfbb0..2acd054be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ uuid = { version = "1.1", default-features = false, features = ["v4"] } auth-helper = { version = "0.3", default-features = false, optional = true } axum = { version = "0.5", default-features = false, features = ["http1", "json", "query", "original-uri", "headers"], optional = true } ed25519 = { version = "1.5", default-features = false, features = ["alloc", "pkcs8", "pem"], optional = true } -ed25519-dalek = { version = "1.0", default-features = false, optional = true } +ed25519-dalek = { version = "1.0", default-features = false, features = ["u64_backend"], optional = true } hex = { version = "0.4", default-features = false, optional = true } hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "stream"], optional = true } lazy_static = { version = "1.4", default-features = false, optional = true } @@ -82,14 +82,13 @@ packable = { version = "0.4", default-features = false } [features] default = [ - "analytics", + "api-analytics", "api-history", "api-core", "inx", "stardust", "metrics", ] -analytics = [] api = [ "dep:auth-helper", "dep:axum", @@ -107,11 +106,14 @@ api = [ "dep:tower-http", "dep:zeroize", ] -api-history = [ +api-analytics = [] +api-core = [ "api", + "stardust", ] -api-core = [ +api-history = [ "api", + "stardust", ] console = [ "dep:console-subscriber", diff --git a/bin/inx-chronicle/src/api/extractors.rs b/bin/inx-chronicle/src/api/extractors.rs index b9f4e2c02..898eb13ad 100644 --- a/bin/inx-chronicle/src/api/extractors.rs +++ b/bin/inx-chronicle/src/api/extractors.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use axum::extract::{FromRequest, Query}; -use chronicle::types::stardust::milestone::MilestoneTimestamp; + use serde::Deserialize; use time::{Duration, OffsetDateTime}; @@ -44,43 +44,52 @@ pub struct TimeRangeQuery { end_timestamp: Option, } -#[derive(Copy, Clone)] -pub struct TimeRange { - pub start_timestamp: MilestoneTimestamp, - pub end_timestamp: MilestoneTimestamp, -} +#[cfg(feature = "stardust")] +mod stardust { + use super::*; + use chronicle::types::stardust::milestone::MilestoneTimestamp; -fn days_ago_utc(days: i64) -> u32 { - let then = OffsetDateTime::now_utc() - Duration::days(days); - then.unix_timestamp() as u32 -} + #[derive(Copy, Clone)] + pub struct TimeRange { + pub start_timestamp: MilestoneTimestamp, + pub end_timestamp: MilestoneTimestamp, + } -fn now_utc() -> u32 { - OffsetDateTime::now_utc().unix_timestamp() as u32 -} + fn days_ago_utc(days: i64) -> u32 { + let then = OffsetDateTime::now_utc() - Duration::days(days); + then.unix_timestamp() as u32 + } -#[async_trait] -impl FromRequest for TimeRange { - type Rejection = ApiError; + fn now_utc() -> u32 { + OffsetDateTime::now_utc().unix_timestamp() as u32 + } - async fn from_request(req: &mut axum::extract::RequestParts) -> Result { - let Query(TimeRangeQuery { - start_timestamp, - end_timestamp, - }) = Query::::from_request(req) - .await - .map_err(ApiError::QueryError)?; - let time_range = TimeRange { - start_timestamp: start_timestamp.unwrap_or_else(|| days_ago_utc(30)).into(), - end_timestamp: end_timestamp.unwrap_or_else(now_utc).into(), - }; - if time_range.end_timestamp < time_range.start_timestamp { - return Err(ApiError::BadTimeRange); + #[async_trait] + impl FromRequest for TimeRange { + type Rejection = ApiError; + + async fn from_request(req: &mut axum::extract::RequestParts) -> Result { + let Query(TimeRangeQuery { + start_timestamp, + end_timestamp, + }) = Query::::from_request(req) + .await + .map_err(ApiError::QueryError)?; + let time_range = TimeRange { + start_timestamp: start_timestamp.unwrap_or_else(|| days_ago_utc(30)).into(), + end_timestamp: end_timestamp.unwrap_or_else(now_utc).into(), + }; + if time_range.end_timestamp < time_range.start_timestamp { + return Err(ApiError::BadTimeRange); + } + Ok(time_range) } - Ok(time_range) } } +#[cfg(feature = "stardust")] +pub use stardust::*; + #[derive(Copy, Clone, Deserialize)] #[serde(default)] pub struct Included { diff --git a/bin/inx-chronicle/src/api/responses.rs b/bin/inx-chronicle/src/api/responses.rs index e530df836..46662313f 100644 --- a/bin/inx-chronicle/src/api/responses.rs +++ b/bin/inx-chronicle/src/api/responses.rs @@ -1,10 +1,6 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use chronicle::{ - db::collections::SyncData, - types::{ledger::LedgerInclusionState, tangle::MilestoneIndex}, -}; use serde::{Deserialize, Serialize}; macro_rules! impl_success_response { @@ -33,31 +29,6 @@ pub struct InfoResponse { impl_success_response!(InfoResponse); -/// An aggregation type that represents the ranges of completed milestones and gaps. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct SyncDataDto(pub SyncData); - -impl_success_response!(SyncDataDto); - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Record { - pub id: String, - pub inclusion_state: Option, - pub milestone_index: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Transfer { - pub transaction_id: String, - pub output_index: u16, - pub is_spending: bool, - pub inclusion_state: Option, - pub block_id: String, - pub amount: u64, -} - #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MaybeSpentOutput { @@ -71,3 +42,40 @@ pub struct Unlock { pub block_id: String, pub block: Value, } + +/// An aggregation type that represents the ranges of completed milestones and gaps. +#[cfg(feature = "stardust")] +mod stardust { + use serde::{Deserialize, Serialize}; + use chronicle::{ + db::collections::SyncData, + types::{ledger::LedgerInclusionState, tangle::MilestoneIndex}, + }; + + #[derive(Debug, Clone, Default, Serialize, Deserialize)] + pub struct SyncDataDto(pub SyncData); + + impl_success_response!(SyncDataDto); + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Record { + pub id: String, + pub inclusion_state: Option, + pub milestone_index: Option, + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Transfer { + pub transaction_id: String, + pub output_index: u16, + pub is_spending: bool, + pub inclusion_state: Option, + pub block_id: String, + pub amount: u64, + } +} + +#[cfg(feature = "stardust")] +pub use stardust::*; diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index b78b2355b..8a8a99a55 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -62,23 +62,28 @@ async fn login( } async fn is_healthy(database: Extension) -> bool { - let end = match database.get_latest_milestone().await { - Ok(Some(last)) => last, - _ => return false, - }; - - // Panic: The milestone_timestamp is guaranteeed to be valid. - let latest_ms_time = OffsetDateTime::from_unix_timestamp(end.milestone_timestamp.0 as i64).unwrap(); - - if OffsetDateTime::now_utc() > latest_ms_time + STALE_MILESTONE_DURATION { - return false; - } - - // Check if there are no gaps in the sync status. - match database.get_gaps().await { - Ok(gaps) => gaps.is_empty(), - _ => false, + #[cfg(feature = "stardust")] + { + let end = match database.get_latest_milestone().await { + Ok(Some(last)) => last, + _ => return false, + }; + + // Panic: The milestone_timestamp is guaranteeed to be valid. + let latest_ms_time = OffsetDateTime::from_unix_timestamp(end.milestone_timestamp.0 as i64).unwrap(); + + if OffsetDateTime::now_utc() > latest_ms_time + STALE_MILESTONE_DURATION { + return false; + } + + // Check if there are no gaps in the sync status. + match database.get_gaps().await { + Ok(gaps) => gaps.is_empty(), + _ => false, + } } + #[cfg(not(feature = "stardust"))] + true } pub async fn info(database: Extension) -> InfoResponse { @@ -97,7 +102,7 @@ pub async fn health(database: Extension) -> StatusCode { } } -#[cfg(feature = "api-history")] +#[cfg(all(feature = "stardust", feature = "api-history"))] pub async fn sync(database: Extension) -> ApiResult { Ok(SyncDataDto(database.get_sync_data(0.into()..=u32::MAX.into()).await?)) } diff --git a/bin/inx-chronicle/src/api/stardust/mod.rs b/bin/inx-chronicle/src/api/stardust/mod.rs index 6e4852593..698b9ce57 100644 --- a/bin/inx-chronicle/src/api/stardust/mod.rs +++ b/bin/inx-chronicle/src/api/stardust/mod.rs @@ -1,7 +1,7 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -#[cfg(feature = "analytics")] +#[cfg(feature = "api-analytics")] pub mod analytics; #[cfg(feature = "api-core")] pub mod core; @@ -14,7 +14,7 @@ pub fn routes() -> Router { #[allow(unused_mut)] let mut router = Router::new(); - #[cfg(feature = "analytics")] + #[cfg(feature = "api-analytics")] { router = router.nest("/analytics/v2", analytics::routes()); } diff --git a/bin/inx-chronicle/src/launcher.rs b/bin/inx-chronicle/src/launcher.rs index e5fa37862..02e0fb211 100644 --- a/bin/inx-chronicle/src/launcher.rs +++ b/bin/inx-chronicle/src/launcher.rs @@ -62,10 +62,13 @@ impl Actor for Launcher { let db = MongoDb::connect(&config.mongodb).await?; - db.create_output_indexes().await?; - db.create_block_indexes().await?; - db.create_ledger_update_indexes().await?; - db.create_milestone_indexes().await?; + #[cfg(feature = "stardust")] + { + db.create_output_indexes().await?; + db.create_block_indexes().await?; + db.create_ledger_update_indexes().await?; + db.create_milestone_indexes().await?; + } #[cfg(all(feature = "inx", feature = "stardust"))] if config.inx.enabled { diff --git a/src/db/collections/block.rs b/src/db/collections/block.rs index 0fcedb904..516f0cb80 100644 --- a/src/db/collections/block.rs +++ b/src/db/collections/block.rs @@ -255,7 +255,7 @@ impl MongoDb { } } -#[cfg(feature = "analytics")] +#[cfg(feature = "api-analytics")] mod analytics { use super::*; use crate::types::tangle::MilestoneIndex; diff --git a/src/db/collections/ledger_update.rs b/src/db/collections/ledger_update.rs index 3e156d722..2cfb1a513 100644 --- a/src/db/collections/ledger_update.rs +++ b/src/db/collections/ledger_update.rs @@ -264,7 +264,7 @@ impl MongoDb { } } -#[cfg(feature = "analytics")] +#[cfg(feature = "api-analytics")] mod analytics { use futures::TryStreamExt; use mongodb::bson; diff --git a/src/db/mod.rs b/src/db/mod.rs index cabd19cde..8455649e8 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 /// Module containing the collections in the database. +#[cfg(feature = "stardust")] pub mod collections; mod mongodb; diff --git a/src/types/mod.rs b/src/types/mod.rs index d1ed7357b..b8cb70255 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -4,11 +4,13 @@ #![allow(missing_docs)] // TODO Remove this once everything has settled. /// Module containing the ledger data models. +#[cfg(feature = "stardust")] pub mod ledger; /// Module containing Stardust data models. #[cfg(feature = "stardust")] pub mod stardust; /// Module containing the tangle models. +#[cfg(feature = "stardust")] pub mod tangle; /// Module contain utility functions. pub mod util; From cbcbbb1798bded18ef1255eaa5cd36c1fabb3a3b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 13 Jul 2022 13:39:33 +0200 Subject: [PATCH 02/18] Fix Cargo.toml --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2acd054be..bbdcbf80a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,9 +85,10 @@ default = [ "api-analytics", "api-history", "api-core", + "api-history", "inx", - "stardust", "metrics", + "stardust", ] api = [ "dep:auth-helper", From 83c5172def4c71aba136b50a02fe476a25027d5f Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 13 Jul 2022 13:41:07 +0200 Subject: [PATCH 03/18] Fix format --- bin/inx-chronicle/src/api/extractors.rs | 4 ++-- bin/inx-chronicle/src/api/responses.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/inx-chronicle/src/api/extractors.rs b/bin/inx-chronicle/src/api/extractors.rs index 898eb13ad..521e4642e 100644 --- a/bin/inx-chronicle/src/api/extractors.rs +++ b/bin/inx-chronicle/src/api/extractors.rs @@ -3,7 +3,6 @@ use async_trait::async_trait; use axum::extract::{FromRequest, Query}; - use serde::Deserialize; use time::{Duration, OffsetDateTime}; @@ -46,9 +45,10 @@ pub struct TimeRangeQuery { #[cfg(feature = "stardust")] mod stardust { - use super::*; use chronicle::types::stardust::milestone::MilestoneTimestamp; + use super::*; + #[derive(Copy, Clone)] pub struct TimeRange { pub start_timestamp: MilestoneTimestamp, diff --git a/bin/inx-chronicle/src/api/responses.rs b/bin/inx-chronicle/src/api/responses.rs index 46662313f..0e70b3518 100644 --- a/bin/inx-chronicle/src/api/responses.rs +++ b/bin/inx-chronicle/src/api/responses.rs @@ -46,11 +46,11 @@ pub struct Unlock { /// An aggregation type that represents the ranges of completed milestones and gaps. #[cfg(feature = "stardust")] mod stardust { - use serde::{Deserialize, Serialize}; use chronicle::{ db::collections::SyncData, types::{ledger::LedgerInclusionState, tangle::MilestoneIndex}, }; + use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct SyncDataDto(pub SyncData); From 746718f43386961d74e1c7abf7c9e1c504e2bcfb Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 13:06:09 +0200 Subject: [PATCH 04/18] Fix 'ci-check-api' alias --- .cargo/config.toml | 4 ++-- Cargo.toml | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 57b9dc71a..14d948a31 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,11 +1,11 @@ [alias] ci-check-all = "check --all-targets --all-features" ci-check-inx = "check --all-targets --no-default-features --features inx,stardust" -ci-check-api = "check --all-targets --no-default-features --features api-core,api-history,analytics,stardust" +ci-check-api = "check --all-targets --no-default-features --features api-core,api-history,api-analytics,stardust" ci-clippy-all = "clippy --all-targets --all-features -- -D warnings" ci-clippy-inx = "clippy --all-targets --no-default-features --features inx,stardust -- -D warnings" -ci-clippy-api = "clippy --all-targets --no-default-features --features api-core,api-history,analytics,stardust -- -D warnings" +ci-clippy-api = "clippy --all-targets --no-default-features --features api-core,api-history,api-analytics,stardust -- -D warnings" ci-doctest = "test --doc --all-features --release" ci-doc = "doc --all-features --no-deps --document-private-items" diff --git a/Cargo.toml b/Cargo.toml index bbdcbf80a..cd8a15a35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,7 +107,10 @@ api = [ "dep:tower-http", "dep:zeroize", ] -api-analytics = [] +api-analytics = [ + "api", + "stardust", +] api-core = [ "api", "stardust", From 0dfae5b72331131d42470fb898106902f6b9d5a9 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 13:31:13 +0200 Subject: [PATCH 05/18] Consolidate api features --- .cargo/config.toml | 4 ++-- Cargo.toml | 16 +--------------- bin/inx-chronicle/src/api/routes.rs | 2 +- bin/inx-chronicle/src/api/stardust/mod.rs | 20 +++++--------------- src/db/collections/block.rs | 2 +- src/db/collections/ledger_update.rs | 2 +- 6 files changed, 11 insertions(+), 35 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 14d948a31..9fbae6b65 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,11 +1,11 @@ [alias] ci-check-all = "check --all-targets --all-features" ci-check-inx = "check --all-targets --no-default-features --features inx,stardust" -ci-check-api = "check --all-targets --no-default-features --features api-core,api-history,api-analytics,stardust" +ci-check-api = "check --all-targets --no-default-features --features api,stardust" ci-clippy-all = "clippy --all-targets --all-features -- -D warnings" ci-clippy-inx = "clippy --all-targets --no-default-features --features inx,stardust -- -D warnings" -ci-clippy-api = "clippy --all-targets --no-default-features --features api-core,api-history,api-analytics,stardust -- -D warnings" +ci-clippy-api = "clippy --all-targets --no-default-features --features api,stardust -- -D warnings" ci-doctest = "test --doc --all-features --release" ci-doc = "doc --all-features --no-deps --document-private-items" diff --git a/Cargo.toml b/Cargo.toml index cd8a15a35..11cc31f4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,10 +82,7 @@ packable = { version = "0.4", default-features = false } [features] default = [ - "api-analytics", - "api-history", - "api-core", - "api-history", + "api", "inx", "metrics", "stardust", @@ -106,17 +103,6 @@ api = [ "dep:tower", "dep:tower-http", "dep:zeroize", -] -api-analytics = [ - "api", - "stardust", -] -api-core = [ - "api", - "stardust", -] -api-history = [ - "api", "stardust", ] console = [ diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 8a8a99a55..7883c17d4 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -102,7 +102,7 @@ pub async fn health(database: Extension) -> StatusCode { } } -#[cfg(all(feature = "stardust", feature = "api-history"))] +#[cfg(all(feature = "stardust", feature = "api"))] pub async fn sync(database: Extension) -> ApiResult { Ok(SyncDataDto(database.get_sync_data(0.into()..=u32::MAX.into()).await?)) } diff --git a/bin/inx-chronicle/src/api/stardust/mod.rs b/bin/inx-chronicle/src/api/stardust/mod.rs index 698b9ce57..dc531a8ad 100644 --- a/bin/inx-chronicle/src/api/stardust/mod.rs +++ b/bin/inx-chronicle/src/api/stardust/mod.rs @@ -1,11 +1,8 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -#[cfg(feature = "api-analytics")] pub mod analytics; -#[cfg(feature = "api-core")] pub mod core; -#[cfg(feature = "api-history")] pub mod history; use axum::Router; @@ -14,19 +11,12 @@ pub fn routes() -> Router { #[allow(unused_mut)] let mut router = Router::new(); - #[cfg(feature = "api-analytics")] + #[cfg(feature = "api")] { - router = router.nest("/analytics/v2", analytics::routes()); - } - - #[cfg(feature = "api-history")] - { - router = router.nest("/history/v2", history::routes()); - } - - #[cfg(feature = "api-core")] - { - router = router.nest("/core/v2", core::routes()); + router = router + .nest("/analytics/v2", analytics::routes()) + .nest("/history/v2", history::routes()) + .nest("/core/v2", core::routes()); } router diff --git a/src/db/collections/block.rs b/src/db/collections/block.rs index 516f0cb80..db28a343c 100644 --- a/src/db/collections/block.rs +++ b/src/db/collections/block.rs @@ -255,7 +255,7 @@ impl MongoDb { } } -#[cfg(feature = "api-analytics")] +#[cfg(feature = "api")] mod analytics { use super::*; use crate::types::tangle::MilestoneIndex; diff --git a/src/db/collections/ledger_update.rs b/src/db/collections/ledger_update.rs index 2cfb1a513..60c072d15 100644 --- a/src/db/collections/ledger_update.rs +++ b/src/db/collections/ledger_update.rs @@ -264,7 +264,7 @@ impl MongoDb { } } -#[cfg(feature = "api-analytics")] +#[cfg(feature = "api")] mod analytics { use futures::TryStreamExt; use mongodb::bson; From f8003aaf420c5b8ab020a164dc1245ca456a83b8 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 13:39:28 +0200 Subject: [PATCH 06/18] Remove '--release' option for 'ci-test' and 'ci-doctest' --- .cargo/config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 9fbae6b65..31642d856 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -7,10 +7,10 @@ ci-clippy-all = "clippy --all-targets --all-features -- -D warnings" ci-clippy-inx = "clippy --all-targets --no-default-features --features inx,stardust -- -D warnings" ci-clippy-api = "clippy --all-targets --no-default-features --features api,stardust -- -D warnings" -ci-doctest = "test --doc --all-features --release" +ci-doctest = "test --doc --all-features" ci-doc = "doc --all-features --no-deps --document-private-items" ci-fmt = "fmt --all -- --check" -ci-test = "test --all-targets --all-features --release" +ci-test = "test --all-targets --all-features" ci-toml = "sort --grouped --check" ci-udeps = "udeps --all-targets --all-features --backend=depinfo" From 260cd7c6a7e82a8ab5fdab21d67fc2715bd8d2aa Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 13:46:33 +0200 Subject: [PATCH 07/18] Rename 'build' to 'check' in CI files and workflows --- .github/workflows/{_build.yml => _check.yml} | 4 +-- .github/workflows/canary.yml | 30 ++++++++++---------- .github/workflows/ci.yml | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) rename .github/workflows/{_build.yml => _check.yml} (97%) diff --git a/.github/workflows/_build.yml b/.github/workflows/_check.yml similarity index 97% rename from .github/workflows/_build.yml rename to .github/workflows/_check.yml index 470177b53..d9da71c49 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_check.yml @@ -1,4 +1,4 @@ -name: Build and Test +name: Check and Test on: workflow_call: @@ -11,7 +11,7 @@ on: type: string jobs: - build-and-test: + check-and-test: name: '${{ inputs.os }}, ${{ inputs.rust }}' runs-on: ${{ inputs.os }} # Unfortunately, we can't do this right now because `indexmap` does not seem to follow semver. diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 18cefa043..b3cd86076 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -6,29 +6,29 @@ on: jobs: - build-and-test-1: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test-1: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: windows-latest, rust: stable } - build-and-test-2: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test-2: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: macos-latest, rust: stable } - build-and-test-3: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test-3: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: ubuntu-latest, rust: beta } - build-and-test-4: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test-4: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: windows-latest, rust: beta } - build-and-test-5: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test-5: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: macos-latest, rust: beta } docker: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 382e2cdec..1da2e2204 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,9 @@ concurrency: cancel-in-progress: true jobs: - build-and-test: - name: "build and test" - uses: ./.github/workflows/_build.yml + check-and-test: + name: "check and test" + uses: ./.github/workflows/_check.yml with: { os: ubuntu-latest, rust: stable } format: From 924c43ea54487f17774a60358de4316a8855c001 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 14:52:00 +0200 Subject: [PATCH 08/18] Remove one cfg --- bin/inx-chronicle/src/api/routes.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 7883c17d4..16a656aef 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -62,6 +62,9 @@ async fn login( } async fn is_healthy(database: Extension) -> bool { + #[allow(unused_mut)] + let mut result = true; + #[cfg(feature = "stardust")] { let end = match database.get_latest_milestone().await { @@ -77,13 +80,13 @@ async fn is_healthy(database: Extension) -> bool { } // Check if there are no gaps in the sync status. - match database.get_gaps().await { + result = match database.get_gaps().await { Ok(gaps) => gaps.is_empty(), _ => false, - } - } - #[cfg(not(feature = "stardust"))] - true + }; + }; + + result } pub async fn info(database: Extension) -> InfoResponse { From eec7c689b7ebfb36dbfce156950978a8d457f8d3 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 14:55:39 +0200 Subject: [PATCH 09/18] Fix --- bin/inx-chronicle/src/api/routes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 16a656aef..97093d3cc 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -80,11 +80,11 @@ async fn is_healthy(database: Extension) -> bool { } // Check if there are no gaps in the sync status. - result = match database.get_gaps().await { + result &= match database.get_gaps().await { Ok(gaps) => gaps.is_empty(), _ => false, }; - }; + } result } From a67a90edc54a2f68a29bf23dab2eff691445afa7 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 17:22:49 +0200 Subject: [PATCH 10/18] Move 'is_healthy' check back to api --- bin/inx-chronicle/src/api/routes.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 97093d3cc..c59458b47 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -61,10 +61,7 @@ async fn login( } } -async fn is_healthy(database: Extension) -> bool { - #[allow(unused_mut)] - let mut result = true; - +async fn is_healthy(database: &MongoDb) -> bool { #[cfg(feature = "stardust")] { let end = match database.get_latest_milestone().await { @@ -80,13 +77,13 @@ async fn is_healthy(database: Extension) -> bool { } // Check if there are no gaps in the sync status. - result &= match database.get_gaps().await { + match database.get_gaps().await { Ok(gaps) => gaps.is_empty(), - _ => false, + _ => return false, }; } - result + true } pub async fn info(database: Extension) -> InfoResponse { From 453ba31d12108c4eb45c70bc6c5b74c743bb9bab Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 17:26:46 +0200 Subject: [PATCH 11/18] Log any error during health check --- bin/inx-chronicle/src/api/routes.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index c59458b47..7a98d5080 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -66,7 +66,11 @@ async fn is_healthy(database: &MongoDb) -> bool { { let end = match database.get_latest_milestone().await { Ok(Some(last)) => last, - _ => return false, + Ok(None) => return false, + Err(e) => { + log::error!("An error occured during health check: {e}"); + return false; + } }; // Panic: The milestone_timestamp is guaranteeed to be valid. From 8b08f70e6be5a355b13df5a5673fe92bb2a20338 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 17:46:07 +0200 Subject: [PATCH 12/18] PR comments --- bin/inx-chronicle/src/api/routes.rs | 7 +------ .../src/api/stardust/history/routes.rs | 6 +++++- bin/inx-chronicle/src/api/stardust/mod.rs | 16 ++++------------ src/db/collections/ledger_update.rs | 1 - 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 7a98d5080..ef92b48a6 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -13,7 +13,7 @@ use hyper::StatusCode; use serde::Deserialize; use time::{Duration, OffsetDateTime}; -use super::{auth::Auth, config::ApiData, error::ApiError, responses::*, ApiResult}; +use super::{auth::Auth, config::ApiData, error::ApiError, responses::*}; // Similar to Hornet, we enforce that the latest known milestone is newer than 5 minutes. This should give Chronicle // sufficient time to catch up with the node that it is connected too. The current milestone interval is 5 seconds. @@ -106,11 +106,6 @@ pub async fn health(database: Extension) -> StatusCode { } } -#[cfg(all(feature = "stardust", feature = "api"))] -pub async fn sync(database: Extension) -> ApiResult { - Ok(SyncDataDto(database.get_sync_data(0.into()..=u32::MAX.into()).await?)) -} - pub async fn not_found() -> ApiError { ApiError::NotFound } diff --git a/bin/inx-chronicle/src/api/stardust/history/routes.rs b/bin/inx-chronicle/src/api/stardust/history/routes.rs index da8fbae9a..984d1fd61 100644 --- a/bin/inx-chronicle/src/api/stardust/history/routes.rs +++ b/bin/inx-chronicle/src/api/stardust/history/routes.rs @@ -16,7 +16,7 @@ use super::{ TransactionsPerAddressResponse, TransactionsPerMilestoneResponse, TransferByAddress, TransferByMilestone, }, }; -use crate::api::{routes::sync, ApiError, ApiResult}; +use crate::api::{responses::SyncDataDto, ApiError, ApiResult}; pub fn routes() -> Router { Router::new().route("/gaps", get(sync)).nest( @@ -27,6 +27,10 @@ pub fn routes() -> Router { ) } +async fn sync(database: Extension) -> ApiResult { + Ok(SyncDataDto(database.get_sync_data(0.into()..=u32::MAX.into()).await?)) +} + async fn transactions_by_address_history( database: Extension, Path(address): Path, diff --git a/bin/inx-chronicle/src/api/stardust/mod.rs b/bin/inx-chronicle/src/api/stardust/mod.rs index dc531a8ad..31b737de5 100644 --- a/bin/inx-chronicle/src/api/stardust/mod.rs +++ b/bin/inx-chronicle/src/api/stardust/mod.rs @@ -8,16 +8,8 @@ pub mod history; use axum::Router; pub fn routes() -> Router { - #[allow(unused_mut)] - let mut router = Router::new(); - - #[cfg(feature = "api")] - { - router = router - .nest("/analytics/v2", analytics::routes()) - .nest("/history/v2", history::routes()) - .nest("/core/v2", core::routes()); - } - - router + Router::new() + .nest("/analytics/v2", analytics::routes()) + .nest("/history/v2", history::routes()) + .nest("/core/v2", core::routes()) } diff --git a/src/db/collections/ledger_update.rs b/src/db/collections/ledger_update.rs index 60c072d15..e34654ea4 100644 --- a/src/db/collections/ledger_update.rs +++ b/src/db/collections/ledger_update.rs @@ -264,7 +264,6 @@ impl MongoDb { } } -#[cfg(feature = "api")] mod analytics { use futures::TryStreamExt; use mongodb::bson; From 0801bd9d0b655a461ed51b0a5e0011fec2e9dec2 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 17:47:41 +0200 Subject: [PATCH 13/18] Remove one more feature gate --- src/db/collections/block.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/db/collections/block.rs b/src/db/collections/block.rs index db28a343c..8302ae6ab 100644 --- a/src/db/collections/block.rs +++ b/src/db/collections/block.rs @@ -255,7 +255,6 @@ impl MongoDb { } } -#[cfg(feature = "api")] mod analytics { use super::*; use crate::types::tangle::MilestoneIndex; From 1015ac9a9a46cb09f199c00e1eebb11d29165b3c Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 18:10:49 +0200 Subject: [PATCH 14/18] Bubble up errors --- bin/inx-chronicle/src/api/routes.rs | 43 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index ef92b48a6..d9c6d021c 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -14,6 +14,7 @@ use serde::Deserialize; use time::{Duration, OffsetDateTime}; use super::{auth::Auth, config::ApiData, error::ApiError, responses::*}; +use crate::api::error::InternalApiError; // Similar to Hornet, we enforce that the latest known milestone is newer than 5 minutes. This should give Chronicle // sufficient time to catch up with the node that it is connected too. The current milestone interval is 5 seconds. @@ -61,15 +62,14 @@ async fn login( } } -async fn is_healthy(database: &MongoDb) -> bool { +async fn is_healthy(database: &MongoDb) -> Result { #[cfg(feature = "stardust")] { let end = match database.get_latest_milestone().await { Ok(Some(last)) => last, - Ok(None) => return false, + Ok(None) => return Ok(false), Err(e) => { - log::error!("An error occured during health check: {e}"); - return false; + return Err(ApiError::Internal(InternalApiError::MongoDb(e))); } }; @@ -77,32 +77,49 @@ async fn is_healthy(database: &MongoDb) -> bool { let latest_ms_time = OffsetDateTime::from_unix_timestamp(end.milestone_timestamp.0 as i64).unwrap(); if OffsetDateTime::now_utc() > latest_ms_time + STALE_MILESTONE_DURATION { - return false; + return Ok(false); } // Check if there are no gaps in the sync status. match database.get_gaps().await { - Ok(gaps) => gaps.is_empty(), - _ => return false, + Ok(gaps) => { + if !gaps.is_empty() { + return Ok(false); + } + } + Err(e) => { + return Err(ApiError::Internal(InternalApiError::MongoDb(e))); + } }; } - true + Ok(true) } pub async fn info(database: Extension) -> InfoResponse { + let is_healthy = match is_healthy(&database).await { + Ok(res) => res, + Err(e) => { + log::error!("An error occured during health check: {e}"); + false + } + }; + InfoResponse { name: "Chronicle".into(), version: std::env!("CARGO_PKG_VERSION").to_string(), - is_healthy: is_healthy(database).await, + is_healthy, } } pub async fn health(database: Extension) -> StatusCode { - if is_healthy(database).await { - StatusCode::OK - } else { - StatusCode::SERVICE_UNAVAILABLE + match is_healthy(&database).await { + Ok(true) => StatusCode::OK, + Ok(false) => StatusCode::SERVICE_UNAVAILABLE, + Err(e) => { + log::error!("An error occured during health check: {e}"); + StatusCode::SERVICE_UNAVAILABLE + } } } From 7ad3b9954bcfe13336b1d4f4fa5e6a24177de9d9 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 22:56:04 +0200 Subject: [PATCH 15/18] I dont want to hear any complaints now anymore --- bin/inx-chronicle/src/api/routes.rs | 53 ++++++++++++----------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index d9c6d021c..f4a11a502 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -14,7 +14,6 @@ use serde::Deserialize; use time::{Duration, OffsetDateTime}; use super::{auth::Auth, config::ApiData, error::ApiError, responses::*}; -use crate::api::error::InternalApiError; // Similar to Hornet, we enforce that the latest known milestone is newer than 5 minutes. This should give Chronicle // sufficient time to catch up with the node that it is connected too. The current milestone interval is 5 seconds. @@ -65,12 +64,9 @@ async fn login( async fn is_healthy(database: &MongoDb) -> Result { #[cfg(feature = "stardust")] { - let end = match database.get_latest_milestone().await { - Ok(Some(last)) => last, - Ok(None) => return Ok(false), - Err(e) => { - return Err(ApiError::Internal(InternalApiError::MongoDb(e))); - } + let end = match database.get_latest_milestone().await? { + Some(last) => last, + None => return Ok(false), }; // Panic: The milestone_timestamp is guaranteeed to be valid. @@ -81,15 +77,13 @@ async fn is_healthy(database: &MongoDb) -> Result { } // Check if there are no gaps in the sync status. - match database.get_gaps().await { - Ok(gaps) => { - if !gaps.is_empty() { - return Ok(false); - } - } - Err(e) => { - return Err(ApiError::Internal(InternalApiError::MongoDb(e))); - } + if !database + .get_gaps() + .await? + .gaps + .is_empty() + { + return Ok(false); }; } @@ -97,29 +91,24 @@ async fn is_healthy(database: &MongoDb) -> Result { } pub async fn info(database: Extension) -> InfoResponse { - let is_healthy = match is_healthy(&database).await { - Ok(res) => res, - Err(e) => { - log::error!("An error occured during health check: {e}"); - false - } - }; - InfoResponse { name: "Chronicle".into(), version: std::env!("CARGO_PKG_VERSION").to_string(), - is_healthy, + is_healthy: is_healthy(&database).await.unwrap_or_else(|e| { + log::error!("An error occured during health check: {e}"); + false + }), } } pub async fn health(database: Extension) -> StatusCode { - match is_healthy(&database).await { - Ok(true) => StatusCode::OK, - Ok(false) => StatusCode::SERVICE_UNAVAILABLE, - Err(e) => { - log::error!("An error occured during health check: {e}"); - StatusCode::SERVICE_UNAVAILABLE - } + if is_healthy(&database).await.unwrap_or_else(|e| { + log::error!("An error occured during health check: {e}"); + false + }) { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE } } From da9f80f594fdc087fcec3eb2e990fc0853f1af9e Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 23:18:03 +0200 Subject: [PATCH 16/18] Fix warnings --- bin/inx-chronicle/src/launcher.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/inx-chronicle/src/launcher.rs b/bin/inx-chronicle/src/launcher.rs index 02e0fb211..0bf0bc083 100644 --- a/bin/inx-chronicle/src/launcher.rs +++ b/bin/inx-chronicle/src/launcher.rs @@ -2,9 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; +#[cfg(any(feature = "api", feature = "inx"))] +use chronicle::runtime::{ActorError, HandleEvent, Report}; use chronicle::{ db::MongoDb, - runtime::{Actor, ActorContext, ActorError, ErrorLevel, HandleEvent, Report, RuntimeError}, + runtime::{Actor, ActorContext, ErrorLevel, RuntimeError}, }; use clap::Parser; use thiserror::Error; @@ -44,6 +46,7 @@ impl Actor for Launcher { type State = ChronicleConfig; type Error = LauncherError; + #[allow(unused_variables)] async fn init(&mut self, cx: &mut ActorContext) -> Result { // TODO: move parsing command-line args to `main.rs` (only async closures make this low effort though) let cl_args = ClArgs::parse(); From 4733e7bf340bd4bdfbe1884871d339b0d8e0f8d3 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 23:41:40 +0200 Subject: [PATCH 17/18] Format+Fix --- bin/inx-chronicle/src/api/routes.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index f4a11a502..97d608f23 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -77,12 +77,7 @@ async fn is_healthy(database: &MongoDb) -> Result { } // Check if there are no gaps in the sync status. - if !database - .get_gaps() - .await? - .gaps - .is_empty() - { + if !database.get_gaps().await?.is_empty() { return Ok(false); }; } From 42167cdfcf2e7d5f60c1d06862dd5faf23c1b3b4 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 14 Jul 2022 23:55:47 +0200 Subject: [PATCH 18/18] Fix clippy --- bin/inx-chronicle/src/api/routes.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/inx-chronicle/src/api/routes.rs b/bin/inx-chronicle/src/api/routes.rs index 97d608f23..a7ec2c8ee 100644 --- a/bin/inx-chronicle/src/api/routes.rs +++ b/bin/inx-chronicle/src/api/routes.rs @@ -97,10 +97,12 @@ pub async fn info(database: Extension) -> InfoResponse { } pub async fn health(database: Extension) -> StatusCode { - if is_healthy(&database).await.unwrap_or_else(|e| { + let handle_error = |e| { log::error!("An error occured during health check: {e}"); false - }) { + }; + + if is_healthy(&database).await.unwrap_or_else(handle_error) { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE