From 920c7da80dcdf2c7a58345ef291ac492bd46517a Mon Sep 17 00:00:00 2001 From: -f Date: Sun, 12 Jan 2025 03:40:36 +0530 Subject: [PATCH 1/8] add highest_seen_tree_index --- rust/main/agents/relayer/src/relayer.rs | 48 +++++++++++++++++-- rust/main/hyperlane-base/src/db/mod.rs | 5 ++ .../src/db/rocks/hyperlane_db.rs | 15 ++++++ rust/main/hyperlane-base/src/metrics/core.rs | 19 ++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/rust/main/agents/relayer/src/relayer.rs b/rust/main/agents/relayer/src/relayer.rs index b1f013b6ae..4b037f7868 100644 --- a/rust/main/agents/relayer/src/relayer.rs +++ b/rust/main/agents/relayer/src/relayer.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::{Debug, Formatter}, sync::Arc, + time::Duration, }; use async_trait::async_trait; @@ -10,7 +11,7 @@ use eyre::Result; use futures_util::future::try_join_all; use hyperlane_base::{ broadcast::BroadcastMpscSender, - db::{HyperlaneRocksDB, DB}, + db::{HyperlaneDb, HyperlaneRocksDB, DB}, metrics::{AgentMetrics, MetricsUpdater}, settings::{ChainConf, IndexSettings}, AgentMetadata, BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, @@ -526,16 +527,38 @@ impl Relayer { return tokio::spawn(async {}).instrument(info_span!("MerkleTreeHookSync")); } }; - tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { + + let origin_clone = origin.clone(); + let origin_db = Arc::new(self.dbs.get(origin).unwrap().clone()); + let core_metrics = self.core_metrics.clone(); + + let hook_sync_handle = tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { contract_sync .clone() .sync( "merkle_tree_hook", SyncOptions::new(Some(cursor), tx_id_receiver), ) - .await + .await; })) - .instrument(info_span!("MerkleTreeHookSync")) + .instrument(info_span!("MerkleTreeHookSync")); + + let index_updater_handle = + tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { + loop { + Self::update_highest_seen_tree_index(&core_metrics, &origin_db, &origin_clone) + .await; + + // sleep for 5 seconds, ie, 5 seconds between updates + tokio::time::sleep(Duration::from_secs(5)).await; + } + })) + .instrument(info_span!("MerkleTreeHighestSeenIndexUpdater")); + + tokio::spawn(async move { + let _ = tokio::join!(hook_sync_handle, index_updater_handle); + }) + .instrument(info_span!("MerkleTreeHookSync_with_IndexUpdater")) } fn run_message_processor( @@ -620,6 +643,23 @@ impl Relayer { })) .instrument(span) } + + async fn update_highest_seen_tree_index( + core_metrics: &CoreMetrics, + origin_db: &Arc, + origin: &HyperlaneDomain, + ) { + let highest_seen_tree_index = origin_db + .retrieve_highest_seen_tree_index() + .unwrap_or_default(); + + if let Some(highest_seen_tree_index) = highest_seen_tree_index { + core_metrics + .highest_seen_tree_index() + .with_label_values(&[origin.name()]) + .set(highest_seen_tree_index as i64); + } + } } #[cfg(test)] diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index 04a3e59cc6..26df5cd0ca 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -135,6 +135,11 @@ pub trait HyperlaneDb: Send + Sync { leaf_index: &u32, ) -> DbResult>; + fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; + + /// Retrieve the highest seen tree index by the syncer + fn retrieve_highest_seen_tree_index(&self) -> DbResult>; + fn store_merkle_leaf_index_by_message_id( &self, message_id: &H256, diff --git a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs index 76a51bfe3c..b7f2ab3dc0 100644 --- a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs +++ b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs @@ -25,6 +25,7 @@ const NONCE_PROCESSED: &str = "nonce_processed_"; const GAS_PAYMENT_BY_SEQUENCE: &str = "gas_payment_by_sequence_"; const GAS_PAYMENT_BLOCK_BY_SEQUENCE: &str = "gas_payment_block_by_sequence_"; const HIGHEST_SEEN_MESSAGE_NONCE: &str = "highest_seen_message_nonce_"; +const HIGHEST_SEEN_TREE_INDEX: &str = "highest_seen_tree_index_"; const GAS_PAYMENT_FOR_MESSAGE_ID: &str = "gas_payment_sequence_for_message_id_v2_"; const GAS_PAYMENT_META_PROCESSED: &str = "gas_payment_meta_processed_v3_"; const GAS_EXPENDITURE_FOR_MESSAGE_ID: &str = "gas_expenditure_for_message_id_v2_"; @@ -211,6 +212,12 @@ impl HyperlaneRocksDB { &insertion.index(), &insertion_block_number, )?; + + let current_highest = self.retrieve_highest_seen_tree_index()?.unwrap_or_default(); + if insertion.index() > current_highest { + self.store_highest_seen_tree_index(&insertion.index())?; + } + // Return true to indicate the tree insertion was processed Ok(true) } @@ -614,6 +621,14 @@ impl HyperlaneDb for HyperlaneRocksDB { self.retrieve_value_by_key(MERKLE_TREE_INSERTION, leaf_index) } + fn retrieve_highest_seen_tree_index(&self) -> DbResult> { + self.retrieve_value_by_key(HIGHEST_SEEN_TREE_INDEX, &bool::default()) + } + + fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()> { + self.store_value_by_key(HIGHEST_SEEN_TREE_INDEX, &bool::default(), index) + } + fn store_merkle_leaf_index_by_message_id( &self, message_id: &H256, diff --git a/rust/main/hyperlane-base/src/metrics/core.rs b/rust/main/hyperlane-base/src/metrics/core.rs index d3cbaf3db4..5bed85b60d 100644 --- a/rust/main/hyperlane-base/src/metrics/core.rs +++ b/rust/main/hyperlane-base/src/metrics/core.rs @@ -39,6 +39,7 @@ pub struct CoreMetrics { span_events: IntCounterVec, last_known_message_nonce: IntGaugeVec, latest_leaf_index: IntGaugeVec, + highest_seen_tree_index: IntGaugeVec, submitter_queue_length: IntGaugeVec, operations_processed_count: IntCounterVec, @@ -122,6 +123,15 @@ impl CoreMetrics { &["origin"], registry )?; + let highest_seen_tree_index = register_int_gauge_vec_with_registry!( + opts!( + namespaced!("highest_seen_tree_index"), + "Highest tree index seen by the relayer", + const_labels_ref + ), + &["origin"], + registry + )?; let observed_validator_latest_index = register_int_gauge_vec_with_registry!( opts!( @@ -189,6 +199,7 @@ impl CoreMetrics { span_events, last_known_message_nonce, latest_leaf_index, + highest_seen_tree_index, submitter_queue_length, @@ -329,6 +340,14 @@ impl CoreMetrics { self.latest_leaf_index.clone() } + /// Reports the current highest tree index seen by the relayer. + /// + /// Labels: + /// - `origin`: Origin chain the tree index is being tracked at. + pub fn highest_seen_tree_index(&self) -> IntGaugeVec { + self.highest_seen_tree_index.clone() + } + /// Latest message nonce in the validator. /// /// Phase: From 49e08a0460d0ab66776922cb8ce6a6bb038cf6e6 Mon Sep 17 00:00:00 2001 From: -f Date: Sun, 12 Jan 2025 04:23:14 +0530 Subject: [PATCH 2/8] rename to latest_tree_insertion --- .../agents/relayer/src/merkle_tree/processor.rs | 8 ++++---- rust/main/hyperlane-base/src/metrics/core.rs | 14 +++++++------- .../src/invariants/termination_invariants.rs | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/rust/main/agents/relayer/src/merkle_tree/processor.rs b/rust/main/agents/relayer/src/merkle_tree/processor.rs index e12a7fc343..0977c2cbe6 100644 --- a/rust/main/agents/relayer/src/merkle_tree/processor.rs +++ b/rust/main/agents/relayer/src/merkle_tree/processor.rs @@ -76,7 +76,7 @@ impl MerkleTreeProcessor { // Update the metrics // we assume that leaves are inserted in order so this will be monotonically increasing self.metrics - .latest_leaf_index_gauge + .latest_tree_insertion_gauge .set(insertion.index() as i64); Some(insertion) } else { @@ -89,14 +89,14 @@ impl MerkleTreeProcessor { #[derive(Debug)] pub struct MerkleTreeProcessorMetrics { - latest_leaf_index_gauge: IntGauge, + latest_tree_insertion_gauge: IntGauge, } impl MerkleTreeProcessorMetrics { pub fn new(metrics: &CoreMetrics, origin: &HyperlaneDomain) -> Self { Self { - latest_leaf_index_gauge: metrics - .latest_leaf_index() + latest_tree_insertion_gauge: metrics + .latest_tree_insertion() .with_label_values(&[origin.name()]), } } diff --git a/rust/main/hyperlane-base/src/metrics/core.rs b/rust/main/hyperlane-base/src/metrics/core.rs index 5bed85b60d..24cca269f2 100644 --- a/rust/main/hyperlane-base/src/metrics/core.rs +++ b/rust/main/hyperlane-base/src/metrics/core.rs @@ -38,7 +38,7 @@ pub struct CoreMetrics { span_counts: IntCounterVec, span_events: IntCounterVec, last_known_message_nonce: IntGaugeVec, - latest_leaf_index: IntGaugeVec, + latest_tree_insertion: IntGaugeVec, highest_seen_tree_index: IntGaugeVec, submitter_queue_length: IntGaugeVec, @@ -114,10 +114,10 @@ impl CoreMetrics { registry )?; - let latest_leaf_index = register_int_gauge_vec_with_registry!( + let latest_tree_insertion = register_int_gauge_vec_with_registry!( opts!( - namespaced!("latest_leaf_index"), - "Latest leaf index inserted into the merkle tree", + namespaced!("latest_tree_insertion"), + "Latest leaf inserted into the merkle tree", const_labels_ref ), &["origin"], @@ -198,7 +198,7 @@ impl CoreMetrics { span_counts, span_events, last_known_message_nonce, - latest_leaf_index, + latest_tree_insertion, highest_seen_tree_index, submitter_queue_length, @@ -336,8 +336,8 @@ impl CoreMetrics { /// /// Labels: /// - `origin`: Origin chain the leaf index is being tracked at. - pub fn latest_leaf_index(&self) -> IntGaugeVec { - self.latest_leaf_index.clone() + pub fn latest_tree_insertion(&self) -> IntGaugeVec { + self.latest_tree_insertion.clone() } /// Reports the current highest tree index seen by the relayer. diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index cb851af8d2..246adad513 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -137,6 +137,21 @@ pub fn termination_invariants_met( return Ok(false); } + // check for each origin that the highest tree index seen by the syncer == the latest tree index inserted into the local tree + let latest_tree_insertion: Vec = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_latest_tree_insertion", + &hashmap! {}, + )?; + let highest_seen_tree_index: Vec = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_highest_seen_tree_index", + &hashmap! {}, + )?; + for (i, item) in latest_tree_insertion.iter().enumerate() { + assert_eq!(*item, highest_seen_tree_index[i]); + } + if let Some((solana_cli_tools_path, solana_config_path)) = solana_cli_tools_path.zip(solana_config_path) { From dde2523422c17e5ef95fcdc0ce5b2f1504e8cbce Mon Sep 17 00:00:00 2001 From: -f Date: Sun, 12 Jan 2025 04:37:42 +0530 Subject: [PATCH 3/8] fix check --- rust/main/agents/relayer/src/msg/processor.rs | 4 ++++ rust/main/agents/validator/src/submit.rs | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/main/agents/relayer/src/msg/processor.rs b/rust/main/agents/relayer/src/msg/processor.rs index 21c6b1def1..cdb90db707 100644 --- a/rust/main/agents/relayer/src/msg/processor.rs +++ b/rust/main/agents/relayer/src/msg/processor.rs @@ -742,6 +742,10 @@ mod test { /// Retrieve the nonce of the highest processed message we're aware of fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; + fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; + + fn retrieve_highest_seen_tree_index(&self) -> DbResult>; + } } diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index f4779c6656..dd915033da 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -464,7 +464,8 @@ mod test { ) -> DbResult>; fn store_highest_seen_message_nonce_number(&self, nonce: &u32) -> DbResult<()>; fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; - + fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; + fn retrieve_highest_seen_tree_index(&self) -> DbResult>; } } From 9d82cdf7b600531024fee198189eb268a0b92d32 Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 13 Jan 2025 19:15:13 +0530 Subject: [PATCH 4/8] update invariant --- .../src/invariants/termination_invariants.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 246adad513..a6f1500890 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -137,20 +137,19 @@ pub fn termination_invariants_met( return Ok(false); } - // check for each origin that the highest tree index seen by the syncer == the latest tree index inserted into the local tree - let latest_tree_insertion: Vec = fetch_metric( - RELAYER_METRICS_PORT, - "hyperlane_latest_tree_insertion", - &hashmap! {}, - )?; + let highest_seen_tree_index: Vec = fetch_metric( RELAYER_METRICS_PORT, "hyperlane_highest_seen_tree_index", &hashmap! {}, )?; - for (i, item) in latest_tree_insertion.iter().enumerate() { - assert_eq!(*item, highest_seen_tree_index[i]); - } + // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions + // LHS: sum(highest_seen_tree_index) + len(highest_seen_tree_index) (each is index so we add 1 to each) + // RHS: eth_messages_expected + (config.kathy_messages as u32 / 4) (double insertions) + assert_eq!( + highest_seen_tree_index.iter().sum::() + highest_seen_tree_index.len() as u32, + eth_messages_expected + (config.kathy_messages as u32 / 4) + ); if let Some((solana_cli_tools_path, solana_config_path)) = solana_cli_tools_path.zip(solana_config_path) From a82b89bb126b0dcd691f32dea59fc4cad1a06c76 Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 13 Jan 2025 19:40:21 +0530 Subject: [PATCH 5/8] double --- .../run-locally/src/invariants/termination_invariants.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index a6f1500890..a0967c82da 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -137,7 +137,6 @@ pub fn termination_invariants_met( return Ok(false); } - let highest_seen_tree_index: Vec = fetch_metric( RELAYER_METRICS_PORT, "hyperlane_highest_seen_tree_index", @@ -145,10 +144,10 @@ pub fn termination_invariants_met( )?; // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions // LHS: sum(highest_seen_tree_index) + len(highest_seen_tree_index) (each is index so we add 1 to each) - // RHS: eth_messages_expected + (config.kathy_messages as u32 / 4) (double insertions) + // RHS: eth_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) assert_eq!( - highest_seen_tree_index.iter().sum::() + highest_seen_tree_index.len() as u32, - eth_messages_expected + (config.kathy_messages as u32 / 4) + highest_seen_tree_index.iter().sum::() + highest_seen_tree_index.len() as u32, + eth_messages_expected + (config.kathy_messages as u32 / 4) * 2 ); if let Some((solana_cli_tools_path, solana_config_path)) = From 09c6b20957a40bb2f14d65378f76d9498ad042b7 Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 13 Jan 2025 20:23:23 +0530 Subject: [PATCH 6/8] adjust to total messages --- .../run-locally/src/invariants/termination_invariants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index a0967c82da..1a3111d7f2 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -144,10 +144,10 @@ pub fn termination_invariants_met( )?; // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions // LHS: sum(highest_seen_tree_index) + len(highest_seen_tree_index) (each is index so we add 1 to each) - // RHS: eth_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) + // RHS: total_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) assert_eq!( highest_seen_tree_index.iter().sum::() + highest_seen_tree_index.len() as u32, - eth_messages_expected + (config.kathy_messages as u32 / 4) * 2 + total_messages_expected + (config.kathy_messages as u32 / 4) * 2 ); if let Some((solana_cli_tools_path, solana_config_path)) = From 17cfce4997563708074655feaa869fa6aa617115 Mon Sep 17 00:00:00 2001 From: -f Date: Wed, 15 Jan 2025 13:39:41 +0530 Subject: [PATCH 7/8] revert highest_seen_tree_index --- rust/main/agents/relayer/src/msg/processor.rs | 4 -- rust/main/agents/relayer/src/relayer.rs | 48 ++----------------- rust/main/agents/validator/src/submit.rs | 3 +- rust/main/hyperlane-base/src/db/mod.rs | 5 -- .../src/db/rocks/hyperlane_db.rs | 15 ------ rust/main/hyperlane-base/src/metrics/core.rs | 19 -------- .../src/invariants/termination_invariants.rs | 43 +++++++++-------- 7 files changed, 27 insertions(+), 110 deletions(-) diff --git a/rust/main/agents/relayer/src/msg/processor.rs b/rust/main/agents/relayer/src/msg/processor.rs index cdb90db707..21c6b1def1 100644 --- a/rust/main/agents/relayer/src/msg/processor.rs +++ b/rust/main/agents/relayer/src/msg/processor.rs @@ -742,10 +742,6 @@ mod test { /// Retrieve the nonce of the highest processed message we're aware of fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; - fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; - - fn retrieve_highest_seen_tree_index(&self) -> DbResult>; - } } diff --git a/rust/main/agents/relayer/src/relayer.rs b/rust/main/agents/relayer/src/relayer.rs index 4b037f7868..b1f013b6ae 100644 --- a/rust/main/agents/relayer/src/relayer.rs +++ b/rust/main/agents/relayer/src/relayer.rs @@ -2,7 +2,6 @@ use std::{ collections::{HashMap, HashSet}, fmt::{Debug, Formatter}, sync::Arc, - time::Duration, }; use async_trait::async_trait; @@ -11,7 +10,7 @@ use eyre::Result; use futures_util::future::try_join_all; use hyperlane_base::{ broadcast::BroadcastMpscSender, - db::{HyperlaneDb, HyperlaneRocksDB, DB}, + db::{HyperlaneRocksDB, DB}, metrics::{AgentMetrics, MetricsUpdater}, settings::{ChainConf, IndexSettings}, AgentMetadata, BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, @@ -527,38 +526,16 @@ impl Relayer { return tokio::spawn(async {}).instrument(info_span!("MerkleTreeHookSync")); } }; - - let origin_clone = origin.clone(); - let origin_db = Arc::new(self.dbs.get(origin).unwrap().clone()); - let core_metrics = self.core_metrics.clone(); - - let hook_sync_handle = tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { + tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { contract_sync .clone() .sync( "merkle_tree_hook", SyncOptions::new(Some(cursor), tx_id_receiver), ) - .await; + .await })) - .instrument(info_span!("MerkleTreeHookSync")); - - let index_updater_handle = - tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { - loop { - Self::update_highest_seen_tree_index(&core_metrics, &origin_db, &origin_clone) - .await; - - // sleep for 5 seconds, ie, 5 seconds between updates - tokio::time::sleep(Duration::from_secs(5)).await; - } - })) - .instrument(info_span!("MerkleTreeHighestSeenIndexUpdater")); - - tokio::spawn(async move { - let _ = tokio::join!(hook_sync_handle, index_updater_handle); - }) - .instrument(info_span!("MerkleTreeHookSync_with_IndexUpdater")) + .instrument(info_span!("MerkleTreeHookSync")) } fn run_message_processor( @@ -643,23 +620,6 @@ impl Relayer { })) .instrument(span) } - - async fn update_highest_seen_tree_index( - core_metrics: &CoreMetrics, - origin_db: &Arc, - origin: &HyperlaneDomain, - ) { - let highest_seen_tree_index = origin_db - .retrieve_highest_seen_tree_index() - .unwrap_or_default(); - - if let Some(highest_seen_tree_index) = highest_seen_tree_index { - core_metrics - .highest_seen_tree_index() - .with_label_values(&[origin.name()]) - .set(highest_seen_tree_index as i64); - } - } } #[cfg(test)] diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index dd915033da..f4779c6656 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -464,8 +464,7 @@ mod test { ) -> DbResult>; fn store_highest_seen_message_nonce_number(&self, nonce: &u32) -> DbResult<()>; fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; - fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; - fn retrieve_highest_seen_tree_index(&self) -> DbResult>; + } } diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index 26df5cd0ca..04a3e59cc6 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -135,11 +135,6 @@ pub trait HyperlaneDb: Send + Sync { leaf_index: &u32, ) -> DbResult>; - fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()>; - - /// Retrieve the highest seen tree index by the syncer - fn retrieve_highest_seen_tree_index(&self) -> DbResult>; - fn store_merkle_leaf_index_by_message_id( &self, message_id: &H256, diff --git a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs index b7f2ab3dc0..76a51bfe3c 100644 --- a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs +++ b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs @@ -25,7 +25,6 @@ const NONCE_PROCESSED: &str = "nonce_processed_"; const GAS_PAYMENT_BY_SEQUENCE: &str = "gas_payment_by_sequence_"; const GAS_PAYMENT_BLOCK_BY_SEQUENCE: &str = "gas_payment_block_by_sequence_"; const HIGHEST_SEEN_MESSAGE_NONCE: &str = "highest_seen_message_nonce_"; -const HIGHEST_SEEN_TREE_INDEX: &str = "highest_seen_tree_index_"; const GAS_PAYMENT_FOR_MESSAGE_ID: &str = "gas_payment_sequence_for_message_id_v2_"; const GAS_PAYMENT_META_PROCESSED: &str = "gas_payment_meta_processed_v3_"; const GAS_EXPENDITURE_FOR_MESSAGE_ID: &str = "gas_expenditure_for_message_id_v2_"; @@ -212,12 +211,6 @@ impl HyperlaneRocksDB { &insertion.index(), &insertion_block_number, )?; - - let current_highest = self.retrieve_highest_seen_tree_index()?.unwrap_or_default(); - if insertion.index() > current_highest { - self.store_highest_seen_tree_index(&insertion.index())?; - } - // Return true to indicate the tree insertion was processed Ok(true) } @@ -621,14 +614,6 @@ impl HyperlaneDb for HyperlaneRocksDB { self.retrieve_value_by_key(MERKLE_TREE_INSERTION, leaf_index) } - fn retrieve_highest_seen_tree_index(&self) -> DbResult> { - self.retrieve_value_by_key(HIGHEST_SEEN_TREE_INDEX, &bool::default()) - } - - fn store_highest_seen_tree_index(&self, index: &u32) -> DbResult<()> { - self.store_value_by_key(HIGHEST_SEEN_TREE_INDEX, &bool::default(), index) - } - fn store_merkle_leaf_index_by_message_id( &self, message_id: &H256, diff --git a/rust/main/hyperlane-base/src/metrics/core.rs b/rust/main/hyperlane-base/src/metrics/core.rs index 24cca269f2..38f63fb311 100644 --- a/rust/main/hyperlane-base/src/metrics/core.rs +++ b/rust/main/hyperlane-base/src/metrics/core.rs @@ -39,7 +39,6 @@ pub struct CoreMetrics { span_events: IntCounterVec, last_known_message_nonce: IntGaugeVec, latest_tree_insertion: IntGaugeVec, - highest_seen_tree_index: IntGaugeVec, submitter_queue_length: IntGaugeVec, operations_processed_count: IntCounterVec, @@ -123,15 +122,6 @@ impl CoreMetrics { &["origin"], registry )?; - let highest_seen_tree_index = register_int_gauge_vec_with_registry!( - opts!( - namespaced!("highest_seen_tree_index"), - "Highest tree index seen by the relayer", - const_labels_ref - ), - &["origin"], - registry - )?; let observed_validator_latest_index = register_int_gauge_vec_with_registry!( opts!( @@ -199,7 +189,6 @@ impl CoreMetrics { span_events, last_known_message_nonce, latest_tree_insertion, - highest_seen_tree_index, submitter_queue_length, @@ -340,14 +329,6 @@ impl CoreMetrics { self.latest_tree_insertion.clone() } - /// Reports the current highest tree index seen by the relayer. - /// - /// Labels: - /// - `origin`: Origin chain the tree index is being tracked at. - pub fn highest_seen_tree_index(&self) -> IntGaugeVec { - self.highest_seen_tree_index.clone() - } - /// Latest message nonce in the validator. /// /// Phase: diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 1a3111d7f2..6e69fd5e63 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -7,7 +7,7 @@ use crate::utils::get_matching_lines; use maplit::hashmap; use relayer::GAS_EXPENDITURE_LOG_MESSAGE; -use crate::invariants::SOL_MESSAGES_EXPECTED; +use crate::invariants::common::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP}; use crate::logging::log; use crate::solana::solana_termination_invariants_met; use crate::{ @@ -30,7 +30,15 @@ pub fn termination_invariants_met( } else { 0 }; + let sol_messages_with_non_matching_igp = if config.sealevel_enabled { + SOL_MESSAGES_WITH_NON_MATCHING_IGP + } else { + 0 + }; + + // this is total messages expected to be delivered let total_messages_expected = eth_messages_expected + sol_messages_expected; + let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp; let lengths = fetch_metric( RELAYER_METRICS_PORT, @@ -38,8 +46,13 @@ pub fn termination_invariants_met( &hashmap! {}, )?; assert!(!lengths.is_empty(), "Could not find queue length metric"); - if lengths.iter().sum::() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES { - log!("Relayer queues not empty. Lengths: {:?}", lengths); + if lengths.iter().sum::() + != ZERO_MERKLE_INSERTION_KATHY_MESSAGES + sol_messages_with_non_matching_igp + { + log!( + "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}", + lengths + ); return Ok(false); }; @@ -128,28 +141,15 @@ pub fn termination_invariants_met( // TestSendReceiver randomly breaks gas payments up into // two. So we expect at least as many gas payments as messages. - if gas_payment_events_count < total_messages_expected { + if gas_payment_events_count < total_messages_dispatched { log!( "Relayer has {} gas payment events, expected at least {}", gas_payment_events_count, - total_messages_expected + total_messages_dispatched ); return Ok(false); } - let highest_seen_tree_index: Vec = fetch_metric( - RELAYER_METRICS_PORT, - "hyperlane_highest_seen_tree_index", - &hashmap! {}, - )?; - // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions - // LHS: sum(highest_seen_tree_index) + len(highest_seen_tree_index) (each is index so we add 1 to each) - // RHS: total_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) - assert_eq!( - highest_seen_tree_index.iter().sum::() + highest_seen_tree_index.len() as u32, - total_messages_expected + (config.kathy_messages as u32 / 4) * 2 - ); - if let Some((solana_cli_tools_path, solana_config_path)) = solana_cli_tools_path.zip(solana_config_path) { @@ -166,12 +166,13 @@ pub fn termination_invariants_met( )? .iter() .sum::(); - if dispatched_messages_scraped != total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES + if dispatched_messages_scraped + != total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES { log!( "Scraper has scraped {} dispatched messages, expected {}", dispatched_messages_scraped, - total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES, + total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES, ); return Ok(false); } @@ -203,7 +204,7 @@ pub fn termination_invariants_met( log!( "Scraper has scraped {} delivered messages, expected {}", delivered_messages_scraped, - total_messages_expected + total_messages_expected + sol_messages_with_non_matching_igp ); return Ok(false); } From b6a5a925376e80a35a94ae6cc961b8d7b0fe1327 Mon Sep 17 00:00:00 2001 From: -f Date: Wed, 15 Jan 2025 13:46:31 +0530 Subject: [PATCH 8/8] err --- .../run-locally/src/invariants/common.rs | 1 + rust/main/utils/run-locally/src/main.rs | 7 +- rust/main/utils/run-locally/src/solana.rs | 91 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/rust/main/utils/run-locally/src/invariants/common.rs b/rust/main/utils/run-locally/src/invariants/common.rs index 35a0c5eae4..1f603db53b 100644 --- a/rust/main/utils/run-locally/src/invariants/common.rs +++ b/rust/main/utils/run-locally/src/invariants/common.rs @@ -1,3 +1,4 @@ // This number should be even, so the messages can be split into two equal halves // sent before and after the relayer spins up, to avoid rounding errors. pub const SOL_MESSAGES_EXPECTED: u32 = 20; +pub const SOL_MESSAGES_WITH_NON_MATCHING_IGP: u32 = 1; diff --git a/rust/main/utils/run-locally/src/main.rs b/rust/main/utils/run-locally/src/main.rs index 4686c15446..7aeb3ae101 100644 --- a/rust/main/utils/run-locally/src/main.rs +++ b/rust/main/utils/run-locally/src/main.rs @@ -227,7 +227,7 @@ fn main() -> ExitCode { "GASPAYMENTENFORCEMENT", r#"[{ "type": "minimum", - "payment": "1", + "payment": "1" }]"#, ) .arg( @@ -465,6 +465,11 @@ fn main() -> ExitCode { initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone()) .join(); } + initiate_solana_non_matching_igp_paying_transfer( + solana_path.clone(), + solana_config_path.clone(), + ) + .join(); } log!("Setup complete! Agents running in background..."); diff --git a/rust/main/utils/run-locally/src/solana.rs b/rust/main/utils/run-locally/src/solana.rs index 6dd137857f..3e32524cdd 100644 --- a/rust/main/utils/run-locally/src/solana.rs +++ b/rust/main/utils/run-locally/src/solana.rs @@ -73,6 +73,8 @@ const SOLANA_REMOTE_CHAIN_ID: &str = "13376"; pub const SOLANA_CHECKPOINT_LOCATION: &str = "/tmp/test_sealevel_checkpoints_0x70997970c51812dc3a010c7d01b50e0d17dc79c8"; +const SOLANA_GAS_ORACLE_CONFIG_FILE: &str = + "../sealevel/environments/local-e2e/gas-oracle-configs.json"; const SOLANA_OVERHEAD_CONFIG_FILE: &str = "../sealevel/environments/local-e2e/overheads.json"; // Install the CLI tools and return the path to the bin dir. @@ -280,6 +282,7 @@ pub fn start_solana_test_validator( .join(); sealevel_client + .clone() .cmd("validator-announce") .cmd("announce") .arg("validator", "0x70997970c51812dc3a010c7d01b50e0d17dc79c8") @@ -291,6 +294,43 @@ pub fn start_solana_test_validator( .run() .join(); + sealevel_client + .clone() + .cmd("igp") + .cmd("init-igp-account") + .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U") + .arg("environment", SOLANA_ENV_NAME) + .arg("environments-dir", SOLANA_ENVS_DIR) + .arg("chain", "sealeveltest1") + .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE) + .arg("gas-oracle-config-file", SOLANA_GAS_ORACLE_CONFIG_FILE) + .arg( + "account-salt", + "0x0000000000000000000000000000000000000000000000000000000000000001", + ) + .run() + .join(); + + sealevel_client + .cmd("igp") + .cmd("init-overhead-igp-account") + .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U") + .arg("environment", SOLANA_ENV_NAME) + .arg("environments-dir", SOLANA_ENVS_DIR) + .arg("chain", "sealeveltest1") + .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE) + .arg("overhead-config-file", SOLANA_OVERHEAD_CONFIG_FILE) + .arg( + "inner-igp-account", + "8EniU8dQaGQ3HWWtT77V7hrksheygvEu6TtzJ3pX1nKM", + ) + .arg( + "account-salt", + "0x0000000000000000000000000000000000000000000000000000000000000001", + ) + .run() + .join(); + log!("Local Solana chain started and hyperlane programs deployed and initialized successfully"); (solana_config_path, validator) @@ -341,6 +381,57 @@ pub fn initiate_solana_hyperlane_transfer( message_id } +#[apply(as_task)] +#[allow(clippy::get_first)] +pub fn initiate_solana_non_matching_igp_paying_transfer( + solana_cli_tools_path: PathBuf, + solana_config_path: PathBuf, +) -> String { + let sender = Program::new(concat_path(&solana_cli_tools_path, "solana")) + .arg("config", solana_config_path.to_str().unwrap()) + .arg("keypair", SOLANA_KEYPAIR) + .cmd("address") + .run_with_output() + .join() + .get(0) + .expect("failed to get sender address") + .trim() + .to_owned(); + + let output = sealevel_client(&solana_cli_tools_path, &solana_config_path) + .cmd("token") + .cmd("transfer-remote") + .cmd(SOLANA_KEYPAIR) + .cmd("10000000000") + .cmd(SOLANA_REMOTE_CHAIN_ID) + .cmd(sender) // send to self + .cmd("native") + .arg("program-id", "CGn8yNtSD3aTTqJfYhUb6s1aVTN75NzwtsFKo1e83aga") + .run_with_output() + .join(); + let non_matching_igp_message_id = get_message_id_from_logs(output.clone()) + .unwrap_or_else(|| panic!("failed to get message id from logs: {:?}", output)); + + log!( + "paying gas to a different IGP account for message id: {}", + non_matching_igp_message_id + ); + sealevel_client(&solana_cli_tools_path, &solana_config_path) + .cmd("igp") + .cmd("pay-for-gas") + .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U") + .arg("message-id", non_matching_igp_message_id.clone()) + .arg("destination-domain", SOLANA_REMOTE_CHAIN_ID) + .arg("gas", "100000") + .arg( + "account-salt", + "0x0000000000000000000000000000000000000000000000000000000000000001", + ) + .run() + .join(); + non_matching_igp_message_id +} + fn get_message_id_from_logs(logs: Vec) -> Option { let message_id_regex = Regex::new(r"Dispatched message to \d+, ID 0x([0-9a-fA-F]+)").unwrap(); for log in logs {