narrowed it down to empty proof

merkle-tree/indexed/src/lib.rs

@@ -254,25 +254,48 @@ where
         low_element_next_value: &mut BigUint,
         low_leaf_proof: &mut BoundedVec<[u8; 32]>,
     ) -> Result<(), IndexedMerkleTreeError> {
+        #[cfg(target_os = "solana")]
+        {
+            solana_program::msg!("patch_elements_and_proof");
+            solana_program::msg!("new_element: {:?}", new_element);
+            solana_program::msg!("low_element: {:?}", low_element);
+            solana_program::msg!("low_element_next_value: {:?}", low_element_next_value);
+            solana_program::msg!("indexed_changelog_index: {:?}", indexed_changelog_index);
+            // for (i, entry) in self.indexed_changelog.iter().enumerate() {
+            //     solana_program::msg!("entry {}: {:?}", i, entry.element);
+
+            //     if i == 7 {
+            //         break;
+            //     }
+            // }
+        }
         let next_indexed_changelog_indices: Vec<usize> = self
             .indexed_changelog
             .iter_from(indexed_changelog_index)?
             .skip(1)
             .enumerate()
             .filter_map(|(index, changelog_entry)| {
+                // #[cfg(target_os = "solana")]
+                // solana_program::msg!("changelog_entry: {:?}", changelog_entry);
                 if changelog_entry.element.index == low_element.index {
                     Some((indexed_changelog_index + 1 + index) % self.indexed_changelog.len())
                 } else {
                     None
                 }
             })
             .collect();
+        #[cfg(target_os = "solana")]
+        solana_program::msg!(
+            "next_indexed_changelog_indices: {:?}",
+            next_indexed_changelog_indices
+        );
 
         let mut new_low_element = None;
 
         for next_indexed_changelog_index in next_indexed_changelog_indices {
             let changelog_entry = &mut self.indexed_changelog[next_indexed_changelog_index];
-
+            // #[cfg(target_os = "solana")]
+            // solana_program::msg!("patching with: changelog_entry: {:?}", changelog_entry);
             let next_element_value = BigUint::from_bytes_be(&changelog_entry.element.next_value);
             if next_element_value < new_element.value {
                 // If the next element is lower than the current element, it means
@@ -304,14 +327,53 @@ where
         if let Some((new_low_element_changelog_index, new_low_element)) = new_low_element {
             let new_low_element_changelog_entry =
                 &self.indexed_changelog[new_low_element_changelog_index];
-
+            // #[cfg(target_os = "solana")]
+            // solana_program::msg!(
+            //     "new_low_element_changelog_entry: {:?}",
+            //     new_low_element_changelog_entry
+            // );
             *changelog_index = new_low_element_changelog_entry.changelog_index;
             *low_element = IndexedElement {
                 index: new_low_element_changelog_entry.element.index,
                 value: new_low_element.clone(),
                 next_index: new_low_element_changelog_entry.element.next_index,
             };
+            #[cfg(target_os = "solana")]
+            solana_program::msg!("low_leaf_proof len: {:?}", low_leaf_proof.len());
+            #[cfg(target_os = "solana")]
+            {
+                solana_program::msg!(
+                    "new_low_element_changelog_entry len: {:?}",
+                    new_low_element_changelog_entry.proof.len()
+                );
+                solana_program::msg!(
+                    "new_low_element_changelog_entry proof: {:?}",
+                    new_low_element_changelog_entry.proof
+                );
+                solana_program::msg!(
+                    "new_low_element_changelog_entry element: {:?}",
+                    new_low_element_changelog_entry.element
+                );
+            }
+            #[cfg(not(target_os = "solana"))]
+            {
+                println!(
+                    "new_low_element_changelog_entry len: {:?}",
+                    new_low_element_changelog_entry.proof.len()
+                );
+                println!(
+                    "new_low_element_changelog_entry proof: {:?}",
+                    new_low_element_changelog_entry.proof
+                );
+                println!(
+                    "new_low_element_changelog_entry element: {:?}",
+                    new_low_element_changelog_entry.element
+                );
+            }
+
             low_leaf_proof.clone_from(&new_low_element_changelog_entry.proof);
+            #[cfg(target_os = "solana")]
+            solana_program::msg!("low_leaf_proof len: {:?}", low_leaf_proof.len());
             new_element.next_index = low_element.next_index;
 
             // Start the patching process from scratch for the new low element.
@@ -416,7 +478,16 @@ where
         let mut proof = BoundedVec::with_capacity(self.height);
         let new_leaf = new_element.hash::<H>(&low_element_next_value)?;
         let (new_changelog_index, _) = self.merkle_tree.append_with_proof(&new_leaf, &mut proof)?;
-
+        #[cfg(target_os = "solana")]
+        {
+            solana_program::msg!(" appeneded with proof");
+            solana_program::msg!("proof: {:?}", proof);
+        }
+        #[cfg(not(target_os = "solana"))]
+        {
+            println!(" appeneded with proof");
+            println!("proof: {:?}", proof);
+        }
         // Prepare raw new element to save in changelog.
         let raw_new_element = RawIndexedElement {
             value: bigint_to_be_bytes_array::<32>(&new_element.value).unwrap(),

merkle-tree/indexed/tests/tests.rs

@@ -855,8 +855,9 @@ pub fn functional_non_inclusion_test() {
 fn functional_changelog_test_1() {
     let address_1 = 30_u32.to_biguint().unwrap();
     let address_2 = 10_u32.to_biguint().unwrap();
+    let address_3 = 11_u32.to_biguint().unwrap();
 
-    perform_change_log_test::<false, false, 10, 16, 16, 0, 16>(&[address_1, address_2]);
+    perform_change_log_test::<false, false, 10, 16, 16, 0, 16>(&[address_1, address_2, address_3]);
 }
 
 /// Performs conflicting Merkle tree updates where:
@@ -1102,6 +1103,131 @@ fn perform_change_log_test<
                     &mut low_element_proof,
                 )
                 .unwrap();
+            for i in onchain_indexed_merkle_tree.changelog.iter() {
+                println!("indexed array state element {:?} ", i);
+            }
         }
     }
 }
+
+fn perform_change_log_test_2<
+    const DOUBLE_SPEND: bool,
+    const WRAP_AROUND: bool,
+    const HEIGHT: usize,
+    const CHANGELOG: usize,
+    const ROOTS: usize,
+    const CANOPY: usize,
+    const INDEXED_CHANGELOG: usize,
+>(
+    addresses: &[BigUint],
+) {
+    // Initialize the trees and indexed array.
+    let mut relayer_indexed_array =
+        IndexedArray::<Poseidon, usize, INDEXING_ARRAY_ELEMENTS>::default();
+    relayer_indexed_array.init().unwrap();
+    let mut relayer_merkle_tree =
+        reference::IndexedMerkleTree::<Poseidon, usize>::new(HEIGHT, CANOPY).unwrap();
+    let mut onchain_indexed_merkle_tree = IndexedMerkleTree::<Poseidon, usize, HEIGHT>::new(
+        HEIGHT,
+        CHANGELOG,
+        ROOTS,
+        CANOPY,
+        INDEXED_CHANGELOG,
+    )
+    .unwrap();
+    onchain_indexed_merkle_tree.init().unwrap();
+    onchain_indexed_merkle_tree.add_highest_element().unwrap();
+    relayer_merkle_tree.init().unwrap();
+    assert_eq!(
+        relayer_merkle_tree.root(),
+        onchain_indexed_merkle_tree.root(),
+        "environment setup failed relayer and onchain indexed Merkle tree roots are inconsistent"
+    );
+
+    // Perform updates for each actor, where every of them is using the same
+    // changelog indices, generating a conflict which needs to be solved by
+    // patching from changelog.
+    let mut indexed_arrays = vec![relayer_indexed_array.clone(); addresses.len()];
+    let changelog_index = onchain_indexed_merkle_tree.changelog_index();
+    let indexed_changelog_index = onchain_indexed_merkle_tree.indexed_changelog_index();
+    for (i, (address, indexed_array)) in addresses.iter().zip(indexed_arrays.iter_mut()).enumerate()
+    {
+        let (old_low_address, old_low_address_next_value) = indexed_array
+            .find_low_element_for_nonexistent(&address)
+            .unwrap();
+        let address_bundle = indexed_array
+            .new_element_with_low_element_index(old_low_address.index, address)
+            .unwrap();
+
+        let mut low_element_proof = relayer_merkle_tree
+            .get_proof_of_leaf(old_low_address.index, false)
+            .unwrap();
+
+        if DOUBLE_SPEND && i > 0 {
+            let res = onchain_indexed_merkle_tree.update(
+                changelog_index,
+                indexed_changelog_index,
+                address_bundle.new_element,
+                old_low_address,
+                old_low_address_next_value,
+                &mut low_element_proof,
+            );
+            assert!(matches!(
+                res,
+                Err(IndexedMerkleTreeError::NewElementGreaterOrEqualToNextElement)
+            ));
+        } else if WRAP_AROUND && (i + 1) * 2 > INDEXED_CHANGELOG {
+            // After a wrap-around of the indexed changelog, we expect leaf
+            // updates to break immediately.
+            let res = onchain_indexed_merkle_tree.update(
+                changelog_index,
+                indexed_changelog_index,
+                address_bundle.new_element.clone(),
+                old_low_address.clone(),
+                old_low_address_next_value,
+                &mut low_element_proof,
+            );
+            println!("changelog_index {:?}", changelog_index);
+            println!("indexed_changelog_index {:?}", indexed_changelog_index);
+            println!(
+                "address_bundle new_element_next_value{:?}",
+                address_bundle.new_element_next_value
+            );
+            println!(
+                "address_bundle new_element {:?}",
+                address_bundle.new_element
+            );
+
+            println!("old_low_address {:?}", old_low_address);
+            println!("res {:?}", res);
+            assert!(matches!(
+                res,
+                Err(IndexedMerkleTreeError::ConcurrentMerkleTree(
+                    ConcurrentMerkleTreeError::CannotUpdateLeaf
+                ))
+            ));
+        } else {
+            onchain_indexed_merkle_tree
+                .update(
+                    changelog_index,
+                    indexed_changelog_index,
+                    address_bundle.new_element,
+                    old_low_address,
+                    old_low_address_next_value,
+                    &mut low_element_proof,
+                )
+                .unwrap();
+        }
+    }
+}
+
+///
+#[test]
+fn debug_changelog() {
+    let address = 10_u32.to_biguint().unwrap();
+
+    perform_change_log_test_2::<true, false, 10, 16, 16, 0, 16>(&[
+        address.clone(),
+        address.clone(),
+    ]);
+}

test-programs/e2e-test/tests/test.rs

@@ -32,7 +32,7 @@ async fn test_10000_all() {
         KeypairActionConfig::all_default(),
         GeneralActionConfig::default(),
         10000,
-        None,
+        Some(17695267035445945231),
     )
     .await;
     env.execute_rounds().await;

test-utils/src/assert_address_merkle_tree.rs

@@ -15,6 +15,7 @@ pub async fn assert_address_merkle_tree_initialized<R: RpcConnection>(
     expected_next_index: usize,
     expected_rightmost_leaf: &[u8; 32],
     owner_pubkey: &Pubkey,
+    expected_indexed_changelog_length: usize,
 ) {
     let merkle_tree = AccountZeroCopy::<account_compression::AddressMerkleTreeAccount>::new(
         rpc,
@@ -115,4 +116,17 @@ pub async fn assert_address_merkle_tree_initialized<R: RpcConnection>(
         expected_roots_length.saturating_sub(1)
     );
     assert_eq!(&merkle_tree.rightmost_leaf(), expected_rightmost_leaf);
+
+    assert_eq!(
+        merkle_tree.changelog.len(),
+        expected_indexed_changelog_length
+    );
+    assert_eq!(
+        merkle_tree.changelog.len(),
+        expected_indexed_changelog_length
+    );
+    // for i in merkle_tree.changelog.iter() {
+    //     println!("changelog: {:?}", i);
+    // }
+    // panic!()
 }

test-utils/src/test_env.rs

@@ -521,6 +521,7 @@ pub async fn create_address_merkle_tree_and_queue_account<R: RpcConnection>(
         expected_next_index,
         &expected_right_most_leaf,
         owner,
+        merkle_tree_config.address_changelog_size as usize,
     )
     .await;
 

test-utils/src/test_forester.rs

@@ -261,6 +261,14 @@ pub async fn empty_address_queue_test<const INDEXED_ARRAY_SIZE: usize, R: RpcCon
     let relayer_merkle_tree = &mut address_tree_bundle.merkle_tree;
     let relayer_indexing_array = &mut address_tree_bundle.indexed_array;
     let mut update_errors: Vec<RpcError> = Vec::new();
+    let address_merkle_tree =
+        get_indexed_merkle_tree::<AddressMerkleTreeAccount, R, Poseidon, usize, 26>(
+            rpc,
+            address_merkle_tree_pubkey,
+        )
+        .await;
+    let indexed_changelog_index = address_merkle_tree.indexed_changelog_index() as u16;
+    let changelog_index = address_merkle_tree.changelog_index() as u16;
     loop {
         let pre_forester_counter = if !signer_is_owner {
             rpc.get_anchor_account::<ForesterEpoch>(
@@ -282,6 +290,7 @@ pub async fn empty_address_queue_test<const INDEXED_ARRAY_SIZE: usize, R: RpcCon
             unsafe { get_hash_set::<QueueAccount, R>(rpc, address_queue_pubkey).await };
 
         let address = address_queue.first_no_seq().unwrap();
+        println!("address: {:?}", address);
         if address.is_none() {
             break;
         }
@@ -298,6 +307,9 @@ pub async fn empty_address_queue_test<const INDEXED_ARRAY_SIZE: usize, R: RpcCon
         let low_address_proof = initial_merkle_tree_state
             .get_proof_of_leaf(old_low_address.index, false)
             .unwrap();
+        // print:
+        // - state of onchain indexed array
+        // - local element
 
         let old_sequence_number = address_merkle_tree.sequence_number();
         let old_root = address_merkle_tree.root();
@@ -313,8 +325,8 @@ pub async fn empty_address_queue_test<const INDEXED_ARRAY_SIZE: usize, R: RpcCon
             old_low_address.next_index as u64,
             bigint_to_be_bytes_array(&old_low_address_next_value).unwrap(),
             low_address_proof.to_array().unwrap(),
-            None,
-            None,
+            Some(changelog_index),
+            Some(indexed_changelog_index),
             signer_is_owner,
         )
         .await