feat(relayer): add core metric for the highest seen index in tree

rust/main/agents/relayer/src/merkle_tree/processor.rs

@@ -76,7 +76,7 @@ impl MerkleTreeProcessor {
             // Update the metrics
             // we assume that leaves are inserted in order so this will be monotonically increasing
             self.metrics
-                .latest_leaf_index_gauge
+                .latest_tree_insertion_gauge
                 .set(insertion.index() as i64);
             Some(insertion)
         } else {
@@ -89,14 +89,14 @@ impl MerkleTreeProcessor {
 
 #[derive(Debug)]
 pub struct MerkleTreeProcessorMetrics {
-    latest_leaf_index_gauge: IntGauge,
+    latest_tree_insertion_gauge: IntGauge,
 }
 
 impl MerkleTreeProcessorMetrics {
     pub fn new(metrics: &CoreMetrics, origin: &HyperlaneDomain) -> Self {
         Self {
-            latest_leaf_index_gauge: metrics
-                .latest_leaf_index()
+            latest_tree_insertion_gauge: metrics
+                .latest_tree_insertion()
                 .with_label_values(&[origin.name()]),
         }
     }

rust/main/hyperlane-base/src/metrics/core.rs

@@ -38,7 +38,7 @@ pub struct CoreMetrics {
     span_counts: IntCounterVec,
     span_events: IntCounterVec,
     last_known_message_nonce: IntGaugeVec,
-    latest_leaf_index: IntGaugeVec,
+    latest_tree_insertion: IntGaugeVec,
     submitter_queue_length: IntGaugeVec,
 
     operations_processed_count: IntCounterVec,
@@ -113,10 +113,10 @@ impl CoreMetrics {
             registry
         )?;
 
-        let latest_leaf_index = register_int_gauge_vec_with_registry!(
+        let latest_tree_insertion = register_int_gauge_vec_with_registry!(
             opts!(
-                namespaced!("latest_leaf_index"),
-                "Latest leaf index inserted into the merkle tree",
+                namespaced!("latest_tree_insertion"),
+                "Latest leaf inserted into the merkle tree",
                 const_labels_ref
             ),
             &["origin"],
@@ -188,7 +188,7 @@ impl CoreMetrics {
             span_counts,
             span_events,
             last_known_message_nonce,
-            latest_leaf_index,
+            latest_tree_insertion,
 
             submitter_queue_length,
 
@@ -325,8 +325,8 @@ impl CoreMetrics {
     ///
     /// Labels:
     /// - `origin`: Origin chain the leaf index is being tracked at.
-    pub fn latest_leaf_index(&self) -> IntGaugeVec {
-        self.latest_leaf_index.clone()
+    pub fn latest_tree_insertion(&self) -> IntGaugeVec {
+        self.latest_tree_insertion.clone()
     }
 
     /// Latest message nonce in the validator.

rust/main/utils/run-locally/src/invariants/common.rs

@@ -1,3 +1,4 @@
 // This number should be even, so the messages can be split into two equal halves
 // sent before and after the relayer spins up, to avoid rounding errors.
 pub const SOL_MESSAGES_EXPECTED: u32 = 20;
+pub const SOL_MESSAGES_WITH_NON_MATCHING_IGP: u32 = 1;

rust/main/utils/run-locally/src/invariants/termination_invariants.rs

@@ -7,7 +7,7 @@ use crate::utils::get_matching_lines;
 use maplit::hashmap;
 use relayer::GAS_EXPENDITURE_LOG_MESSAGE;
 
-use crate::invariants::SOL_MESSAGES_EXPECTED;
+use crate::invariants::common::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP};
 use crate::logging::log;
 use crate::solana::solana_termination_invariants_met;
 use crate::{
@@ -30,16 +30,29 @@ pub fn termination_invariants_met(
     } else {
         0
     };
+    let sol_messages_with_non_matching_igp = if config.sealevel_enabled {
+        SOL_MESSAGES_WITH_NON_MATCHING_IGP
+    } else {
+        0
+    };
+
+    // this is total messages expected to be delivered
     let total_messages_expected = eth_messages_expected + sol_messages_expected;
+    let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp;
 
     let lengths = fetch_metric(
         RELAYER_METRICS_PORT,
         "hyperlane_submitter_queue_length",
         &hashmap! {},
     )?;
     assert!(!lengths.is_empty(), "Could not find queue length metric");
-    if lengths.iter().sum::<u32>() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
-        log!("Relayer queues not empty. Lengths: {:?}", lengths);
+    if lengths.iter().sum::<u32>()
+        != ZERO_MERKLE_INSERTION_KATHY_MESSAGES + sol_messages_with_non_matching_igp
+    {
+        log!(
+            "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}",
+            lengths
+        );
         return Ok(false);
     };
 
@@ -128,11 +141,11 @@ pub fn termination_invariants_met(
 
     // TestSendReceiver randomly breaks gas payments up into
     // two. So we expect at least as many gas payments as messages.
-    if gas_payment_events_count < total_messages_expected {
+    if gas_payment_events_count < total_messages_dispatched {
         log!(
             "Relayer has {} gas payment events, expected at least {}",
             gas_payment_events_count,
-            total_messages_expected
+            total_messages_dispatched
         );
         return Ok(false);
     }
@@ -153,12 +166,13 @@ pub fn termination_invariants_met(
     )?
     .iter()
     .sum::<u32>();
-    if dispatched_messages_scraped != total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES
+    if dispatched_messages_scraped
+        != total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES
     {
         log!(
             "Scraper has scraped {} dispatched messages, expected {}",
             dispatched_messages_scraped,
-            total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES,
+            total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES,
         );
         return Ok(false);
     }
@@ -190,7 +204,7 @@ pub fn termination_invariants_met(
         log!(
             "Scraper has scraped {} delivered messages, expected {}",
             delivered_messages_scraped,
-            total_messages_expected
+            total_messages_expected + sol_messages_with_non_matching_igp
         );
         return Ok(false);
     }

rust/main/utils/run-locally/src/main.rs

@@ -227,7 +227,7 @@ fn main() -> ExitCode {
             "GASPAYMENTENFORCEMENT",
             r#"[{
                 "type": "minimum",
-                "payment": "1",
+                "payment": "1"
             }]"#,
         )
         .arg(
@@ -465,6 +465,11 @@ fn main() -> ExitCode {
             initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone())
                 .join();
         }
+        initiate_solana_non_matching_igp_paying_transfer(
+            solana_path.clone(),
+            solana_config_path.clone(),
+        )
+        .join();
     }
 
     log!("Setup complete! Agents running in background...");

rust/main/utils/run-locally/src/solana.rs

@@ -73,6 +73,8 @@ const SOLANA_REMOTE_CHAIN_ID: &str = "13376";
 pub const SOLANA_CHECKPOINT_LOCATION: &str =
     "/tmp/test_sealevel_checkpoints_0x70997970c51812dc3a010c7d01b50e0d17dc79c8";
 
+const SOLANA_GAS_ORACLE_CONFIG_FILE: &str =
+    "../sealevel/environments/local-e2e/gas-oracle-configs.json";
 const SOLANA_OVERHEAD_CONFIG_FILE: &str = "../sealevel/environments/local-e2e/overheads.json";
 
 // Install the CLI tools and return the path to the bin dir.
@@ -280,6 +282,7 @@ pub fn start_solana_test_validator(
         .join();
 
     sealevel_client
+        .clone()
         .cmd("validator-announce")
         .cmd("announce")
         .arg("validator", "0x70997970c51812dc3a010c7d01b50e0d17dc79c8")
@@ -291,6 +294,43 @@ pub fn start_solana_test_validator(
         .run()
         .join();
 
+    sealevel_client
+        .clone()
+        .cmd("igp")
+        .cmd("init-igp-account")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("environment", SOLANA_ENV_NAME)
+        .arg("environments-dir", SOLANA_ENVS_DIR)
+        .arg("chain", "sealeveltest1")
+        .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE)
+        .arg("gas-oracle-config-file", SOLANA_GAS_ORACLE_CONFIG_FILE)
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+
+    sealevel_client
+        .cmd("igp")
+        .cmd("init-overhead-igp-account")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("environment", SOLANA_ENV_NAME)
+        .arg("environments-dir", SOLANA_ENVS_DIR)
+        .arg("chain", "sealeveltest1")
+        .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE)
+        .arg("overhead-config-file", SOLANA_OVERHEAD_CONFIG_FILE)
+        .arg(
+            "inner-igp-account",
+            "8EniU8dQaGQ3HWWtT77V7hrksheygvEu6TtzJ3pX1nKM",
+        )
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+
     log!("Local Solana chain started and hyperlane programs deployed and initialized successfully");
 
     (solana_config_path, validator)
@@ -341,6 +381,57 @@ pub fn initiate_solana_hyperlane_transfer(
     message_id
 }
 
+#[apply(as_task)]
+#[allow(clippy::get_first)]
+pub fn initiate_solana_non_matching_igp_paying_transfer(
+    solana_cli_tools_path: PathBuf,
+    solana_config_path: PathBuf,
+) -> String {
+    let sender = Program::new(concat_path(&solana_cli_tools_path, "solana"))
+        .arg("config", solana_config_path.to_str().unwrap())
+        .arg("keypair", SOLANA_KEYPAIR)
+        .cmd("address")
+        .run_with_output()
+        .join()
+        .get(0)
+        .expect("failed to get sender address")
+        .trim()
+        .to_owned();
+
+    let output = sealevel_client(&solana_cli_tools_path, &solana_config_path)
+        .cmd("token")
+        .cmd("transfer-remote")
+        .cmd(SOLANA_KEYPAIR)
+        .cmd("10000000000")
+        .cmd(SOLANA_REMOTE_CHAIN_ID)
+        .cmd(sender) // send to self
+        .cmd("native")
+        .arg("program-id", "CGn8yNtSD3aTTqJfYhUb6s1aVTN75NzwtsFKo1e83aga")
+        .run_with_output()
+        .join();
+    let non_matching_igp_message_id = get_message_id_from_logs(output.clone())
+        .unwrap_or_else(|| panic!("failed to get message id from logs: {:?}", output));
+
+    log!(
+        "paying gas to a different IGP account for message id: {}",
+        non_matching_igp_message_id
+    );
+    sealevel_client(&solana_cli_tools_path, &solana_config_path)
+        .cmd("igp")
+        .cmd("pay-for-gas")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("message-id", non_matching_igp_message_id.clone())
+        .arg("destination-domain", SOLANA_REMOTE_CHAIN_ID)
+        .arg("gas", "100000")
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+    non_matching_igp_message_id
+}
+
 fn get_message_id_from_logs(logs: Vec<String>) -> Option<String> {
     let message_id_regex = Regex::new(r"Dispatched message to \d+, ID 0x([0-9a-fA-F]+)").unwrap();
     for log in logs {