soroban-rpc: get expiration ledger sequence at source (#916)

* Bump Rust and Go dependencies

* Start migrating soroban-rpc

* Fix get_ledger_changes() invocation

* wip migrating the soroban-cli (incomplete!)

* Appease rust

* Bump core dependencies

* Bump Go dependencies again

* Fix test comparisons

* Fix BumpAndRestoreFootprint test

* Improve restore test a bit further

* Adjust tests further

* Fix typo

* Fix GetPreflight test

* Get CLI to compile and fix most TODOs

* run cargo md-gen

* Appease clippy

* Add comment about expiration entry efficiency

* Address review feedback

* Take a stab at extracting the spec from the state

* update

* linter

* limit get_ledger_entries to expiration ledger entries.

* fix cli serialization

* update test

* further fix..

* further fix.

* further fix

* update test

* fix typo

* fix: update CLI and tests for new getLedgerEntries (#1021)

* fix: add latest_modified_ledger to FullLedgerEntry

Also simplifies parsing FullLedgerEntry

* fix: CLI test for contract read

* fix: update tests to use getLedgerEntries

* fix: nit

* step

* update

* fix tests

* updating

* update per code review

---------

Co-authored-by: Alfonso Acosta <alfonso@stellar.org>
Co-authored-by: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com>
Co-authored-by: Willem Wyndham <willem@ahalabs.dev>

cmd/crates/soroban-test/tests/it/hello_world.rs

@@ -310,7 +310,7 @@ fn contract_data_read() {
         .arg("--durability=persistent")
         .assert()
         .success()
-        .stdout("COUNTER,1,4096\n");
+        .stdout("COUNTER,1,4,4096\n");
 
     sandbox
         .new_assert_cmd("contract")
@@ -331,7 +331,7 @@ fn contract_data_read() {
         .arg("--durability=persistent")
         .assert()
         .success()
-        .stdout("COUNTER,2,4096\n");
+        .stdout("COUNTER,2,4,4096\n");
 }
 
 #[test]

cmd/soroban-cli/src/commands/contract/bump.rs

@@ -175,7 +175,7 @@ impl Cmd {
 
         if operations[0].changes.is_empty() {
             let entry = client.get_full_ledger_entries(&keys).await?;
-            let expire = entry.entries[0].expiration.expiration_ledger_seq;
+            let expire = entry.entries[0].expiration_ledger_seq;
             if entry.latest_ledger + i64::from(ledgers_to_expire) < i64::from(expire) {
                 return Ok(expire);
             }

cmd/soroban-cli/src/commands/contract/read.rs

@@ -1,15 +1,13 @@
 use std::{
-    convert::Into,
     fmt::Debug,
     io::{self, stdout},
 };
 
 use clap::{command, Parser, ValueEnum};
-use sha2::{Digest, Sha256};
 use soroban_env_host::{
     xdr::{
-        ContractDataEntry, Error as XdrError, ExpirationEntry, Hash, LedgerEntryData, LedgerKey,
-        LedgerKeyContractData, ScVal, WriteXdr,
+        ContractDataEntry, Error as XdrError, LedgerEntryData, LedgerKey, LedgerKeyContractData,
+        ScVal, WriteXdr,
     },
     HostError,
 };
@@ -113,18 +111,16 @@ impl Cmd {
     fn run_in_sandbox(&self) -> Result<FullLedgerEntries, Error> {
         let state = self.config.get_state()?;
         let ledger_entries = &state.ledger_entries;
-
+        let latest_ledger = u32::try_from(state.ledger_entries.len()).unwrap();
         let keys = self.key.parse_keys()?;
         let entries = ledger_entries
             .iter()
             .map(|(k, v)| (k.as_ref().clone(), (v.0.as_ref().clone(), v.1)))
             .filter(|(k, _v)| keys.contains(k))
             .map(|(key, (v, expiration))| {
                 Ok(FullLedgerEntry {
-                    expiration: ExpirationEntry {
-                        key_hash: Hash(Sha256::digest(key.to_xdr()?).into()),
-                        expiration_ledger_seq: expiration.unwrap_or_default(),
-                    },
+                    expiration_ledger_seq: expiration.unwrap_or_default(),
+                    last_modified_ledger: latest_ledger,
                     key,
                     val: v.data,
                 })
@@ -145,7 +141,8 @@ impl Cmd {
         for FullLedgerEntry {
             key,
             val,
-            expiration,
+            expiration_ledger_seq,
+            last_modified_ledger,
         } in &entries.entries
         {
             let (
@@ -155,8 +152,6 @@ impl Cmd {
             else {
                 return Err(Error::OnlyDataAllowed);
             };
-            let expiration = expiration.expiration_ledger_seq;
-
             let output = match self.output {
                 Output::String => [
                     soroban_spec_tools::to_string(key).map_err(|e| Error::CannotPrintResult {
@@ -167,7 +162,8 @@ impl Cmd {
                         result: val.clone(),
                         error: e,
                     })?,
-                    expiration.to_string(),
+                    last_modified_ledger.to_string(),
+                    expiration_ledger_seq.to_string(),
                 ],
                 Output::Json => [
                     serde_json::to_string_pretty(&key).map_err(|error| {
@@ -182,7 +178,13 @@ impl Cmd {
                             error,
                         }
                     })?,
-                    serde_json::to_string_pretty(&expiration).map_err(|error| {
+                    serde_json::to_string_pretty(&last_modified_ledger).map_err(|error| {
+                        Error::CannotPrintJsonResult {
+                            result: val.clone(),
+                            error,
+                        }
+                    })?,
+                    serde_json::to_string_pretty(&expiration_ledger_seq).map_err(|error| {
                         Error::CannotPrintJsonResult {
                             result: val.clone(),
                             error,
@@ -192,7 +194,8 @@ impl Cmd {
                 Output::Xdr => [
                     key.to_xdr_base64()?,
                     val.to_xdr_base64()?,
-                    expiration.to_xdr_base64()?,
+                    last_modified_ledger.to_xdr_base64()?,
+                    expiration_ledger_seq.to_xdr_base64()?,
                 ],
             };
             out.write_record(output)

cmd/soroban-cli/src/rpc/mod.rs

@@ -3,14 +3,16 @@ use itertools::Itertools;
 use jsonrpsee_core::params::ObjectParams;
 use jsonrpsee_core::{self, client::ClientT, rpc_params};
 use jsonrpsee_http_client::{HeaderMap, HttpClient, HttpClientBuilder};
-use serde_aux::prelude::{deserialize_default_from_null, deserialize_number_from_string};
-use sha2::{Digest, Sha256};
+use serde_aux::prelude::{
+    deserialize_default_from_null, deserialize_number_from_string,
+    deserialize_option_number_from_string,
+};
 use soroban_env_host::xdr::{
     self, AccountEntry, AccountId, ContractDataEntry, DiagnosticEvent, Error as XdrError,
-    ExpirationEntry, LedgerEntryData, LedgerFootprint, LedgerKey, LedgerKeyAccount,
-    LedgerKeyExpiration, PublicKey, ReadXdr, SequenceNumber, SorobanAuthorizationEntry,
-    SorobanResources, Transaction, TransactionEnvelope, TransactionMeta, TransactionMetaV3,
-    TransactionResult, TransactionV1Envelope, Uint256, VecM, WriteXdr,
+    LedgerEntryData, LedgerFootprint, LedgerKey, LedgerKeyAccount, PublicKey, ReadXdr,
+    SequenceNumber, SorobanAuthorizationEntry, SorobanResources, Transaction, TransactionEnvelope,
+    TransactionMeta, TransactionMetaV3, TransactionResult, TransactionV1Envelope, Uint256, VecM,
+    WriteXdr,
 };
 use soroban_env_host::xdr::{DepthLimitedRead, SorobanAuthorizedFunction};
 use soroban_sdk::token;
@@ -96,8 +98,8 @@ pub enum Error {
     SpecBase64(#[from] soroban_spec::read::ParseSpecBase64Error),
     #[error("Fee was too large {0}")]
     LargeFee(u64),
-    #[error("Failed to parse  LedgerEntryData")]
-    FailedParseLedgerEntryData,
+    #[error("Failed to parse LedgerEntryData\nkey:{0:?}\nvalue:{1:?}\nexpiration:{2:?}")]
+    FailedParseLedgerEntryData(LedgerKey, LedgerEntryData, LedgerEntryData),
 }
 
 #[derive(serde::Deserialize, serde::Serialize, Debug)]
@@ -146,8 +148,18 @@ pub struct GetTransactionResponse {
 pub struct LedgerEntryResult {
     pub key: String,
     pub xdr: String,
-    #[serde(rename = "lastModifiedLedgerSeq")]
-    pub last_modified_ledger: String,
+    #[serde(
+        rename = "lastModifiedLedgerSeq",
+        deserialize_with = "deserialize_number_from_string"
+    )]
+    pub last_modified_ledger: u32,
+    #[serde(
+        rename = "expirationLedgerSeq",
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "deserialize_option_number_from_string",
+        default
+    )]
+    pub expiration_ledger_seq: Option<u32>,
 }
 
 #[derive(serde::Deserialize, serde::Serialize, Debug)]
@@ -427,7 +439,8 @@ pub enum EventStart {
 pub struct FullLedgerEntry {
     pub key: LedgerKey,
     pub val: LedgerEntryData,
-    pub expiration: ExpirationEntry,
+    pub last_modified_ledger: u32,
+    pub expiration_ledger_seq: u32,
 }
 
 #[derive(Debug)]
@@ -495,10 +508,10 @@ impl Client {
 
     pub async fn verify_network_passphrase(&self, expected: Option<&str>) -> Result<String, Error> {
         let server = self.get_network().await?.passphrase;
-        if expected != Some(&server) {
-            if let Some(expected_val) = expected {
+        if let Some(expected) = expected {
+            if expected != server {
                 return Err(Error::InvalidNetworkPassphrase {
-                    expected: expected_val.to_string(),
+                    expected: expected.to_string(),
                     server,
                 });
             }
@@ -769,35 +782,35 @@ soroban config identity fund {address} --helper-url <url>"#
     ) -> Result<FullLedgerEntries, Error> {
         let keys = ledger_keys
             .iter()
-            .map(|key| Ok(into_keys(key.clone())?.into_iter()))
-            .flatten_ok()
-            .collect::<Result<Vec<_>, Error>>()?;
-        tracing::trace!("{keys:#?}");
+            .filter(|key| !matches!(key, LedgerKey::Expiration(_)))
+            .map(Clone::clone)
+            .collect::<Vec<_>>();
+        tracing::trace!("keys: {keys:#?}");
         let GetLedgerEntriesResponse {
             entries,
             latest_ledger,
         } = self.get_ledger_entries(&keys).await?;
-        tracing::trace!(?entries);
+        tracing::trace!("raw: {entries:#?}");
         let entries = entries
-            .as_deref()
             .unwrap_or_default()
             .iter()
-            .tuple_windows()
-            .map(|(key_res, entry_res)| {
-                let expiration = LedgerEntryData::from_xdr_base64(&entry_res.xdr)?;
-                if let LedgerEntryData::Expiration(expiration) = expiration {
-                    let key = LedgerKey::from_xdr_base64(&key_res.key)?;
-                    let val = LedgerEntryData::from_xdr_base64(&key_res.xdr)?;
+            .map(
+                |LedgerEntryResult {
+                     key,
+                     xdr,
+                     last_modified_ledger,
+                     expiration_ledger_seq,
+                 }| {
                     Ok(FullLedgerEntry {
-                        key,
-                        val,
-                        expiration,
+                        key: LedgerKey::from_xdr_base64(key)?,
+                        val: LedgerEntryData::from_xdr_base64(xdr)?,
+                        expiration_ledger_seq: expiration_ledger_seq.unwrap_or_default(),
+                        last_modified_ledger: *last_modified_ledger,
                     })
-                } else {
-                    Err(Error::FailedParseLedgerEntryData)
-                }
-            })
+                },
+            )
             .collect::<Result<Vec<_>, Error>>()?;
+        tracing::trace!("parsed: {entries:#?}");
         Ok(FullLedgerEntries {
             entries,
             latest_ledger,
@@ -954,13 +967,6 @@ pub fn parse_cursor(c: &str) -> Result<(u64, i32), Error> {
     Ok((toid_part, start_index))
 }
 
-fn into_keys(key: LedgerKey) -> Result<[LedgerKey; 2], Error> {
-    let expiration = LedgerKey::Expiration(LedgerKeyExpiration {
-        key_hash: xdr::Hash(Sha256::digest(key.to_xdr()?).into()),
-    });
-    Ok([key, expiration])
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

cmd/soroban-rpc/internal/db/ledgerentry.go

@@ -2,6 +2,7 @@ package db
 
 import (
 	"context"
+	"crypto/sha256"
 	"database/sql"
 	"fmt"
 
@@ -23,8 +24,9 @@ type LedgerEntryReader interface {
 }
 
 type LedgerKeyAndEntry struct {
-	Key   xdr.LedgerKey
-	Entry xdr.LedgerEntry
+	Key                 xdr.LedgerKey
+	Entry               xdr.LedgerEntry
+	ExpirationLedgerSeq *uint32 // optional expiration ledger seq, when applicable.
 }
 
 type LedgerEntryReadTx interface {
@@ -216,32 +218,71 @@ func (l *ledgerEntryReadTx) getRawLedgerEntries(keys ...string) (map[string]stri
 	return result, nil
 }
 
-func GetLedgerEntry(tx LedgerEntryReadTx, key xdr.LedgerKey) (bool, xdr.LedgerEntry, error) {
+func GetLedgerEntry(tx LedgerEntryReadTx, key xdr.LedgerKey) (bool, xdr.LedgerEntry, *uint32, error) {
 	keyEntries, err := tx.GetLedgerEntries(key)
 	if err != nil {
-		return false, xdr.LedgerEntry{}, err
+		return false, xdr.LedgerEntry{}, nil, err
 	}
 	switch len(keyEntries) {
 	case 0:
-		return false, xdr.LedgerEntry{}, nil
+		return false, xdr.LedgerEntry{}, nil, nil
 	case 1:
 		// expected length
-		return true, keyEntries[0].Entry, nil
+		return true, keyEntries[0].Entry, keyEntries[0].ExpirationLedgerSeq, nil
 	default:
-		return false, xdr.LedgerEntry{}, fmt.Errorf("multiple entries (%d) for key %v", len(keyEntries), key)
+		return false, xdr.LedgerEntry{}, nil, fmt.Errorf("multiple entries (%d) for key %v", len(keyEntries), key)
 	}
 }
 
+// isExpirableKey check to see if the key type is expected to be accompanied by a LedgerExpirationEntry
+func isExpirableKey(key xdr.LedgerKey) bool {
+	switch key.Type {
+	case xdr.LedgerEntryTypeContractData:
+		return true
+	case xdr.LedgerEntryTypeContractCode:
+		return true
+	default:
+	}
+	return false
+}
+
+func entryKeyToExpirationEntryKey(key xdr.LedgerKey) (xdr.LedgerKey, error) {
+	buf, err := key.MarshalBinary()
+	if err != nil {
+		return xdr.LedgerKey{}, err
+	}
+	var expirationEntry xdr.LedgerKey
+	err = expirationEntry.SetExpiration(xdr.Hash(sha256.Sum256(buf)))
+	if err != nil {
+		return xdr.LedgerKey{}, err
+	}
+	return expirationEntry, nil
+}
+
 func (l *ledgerEntryReadTx) GetLedgerEntries(keys ...xdr.LedgerKey) ([]LedgerKeyAndEntry, error) {
-	encodedKeys := make([]string, len(keys))
+	encodedKeys := make([]string, len(keys), 2*len(keys))
 	encodedKeyToKey := make(map[string]xdr.LedgerKey, len(keys))
-	for i, k := range keys {
+	encodedKeyToEncodedExpirationLedgerKey := make(map[string]string, len(keys))
+	for _, k := range keys {
 		encodedKey, err := encodeLedgerKey(l.buffer, k)
 		if err != nil {
 			return nil, err
 		}
-		encodedKeys[i] = encodedKey
+		encodedKeys = append(encodedKeys, encodedKey)
 		encodedKeyToKey[encodedKey] = k
+		if !isExpirableKey(k) {
+			continue
+		}
+		expirationEntryKey, err := entryKeyToExpirationEntryKey(k)
+		if err != nil {
+			return nil, err
+		}
+		encodedExpirationKey, err := encodeLedgerKey(l.buffer, expirationEntryKey)
+		if err != nil {
+			return nil, err
+		}
+		encodedKeyToEncodedExpirationLedgerKey[encodedKey] = encodedExpirationKey
+		encodedKeys = append(encodedKeys, encodedExpirationKey)
 	}
 
 	rawResult, err := l.getRawLedgerEntries(encodedKeys...)
@@ -259,7 +300,22 @@ func (l *ledgerEntryReadTx) GetLedgerEntries(keys ...xdr.LedgerKey) ([]LedgerKey
 		if err := xdr.SafeUnmarshal([]byte(encodedEntry), &entry); err != nil {
 			return nil, errors.Wrap(err, "cannot decode ledger entry from DB")
 		}
-		result = append(result, LedgerKeyAndEntry{key, entry})
+		encodedExpKey, has := encodedKeyToEncodedExpirationLedgerKey[encodedKey]
+		if !has {
+			result = append(result, LedgerKeyAndEntry{key, entry, nil})
+			continue
+		}
+		encodedExpEntry, ok := rawResult[encodedExpKey]
+		if !ok {
+			// missing expiration key. this should no happen.
+			return nil, errors.New("missing expiration key entry")
+		}
+		var expEntry xdr.LedgerEntry
+		if err := xdr.SafeUnmarshal([]byte(encodedExpEntry), &expEntry); err != nil {
+			return nil, errors.Wrap(err, "cannot decode expiration ledger entry from DB")
+		}
+		expSeq := uint32(expEntry.Data.Expiration.ExpirationLedgerSeq)
+		result = append(result, LedgerKeyAndEntry{key, entry, &expSeq})
 	}
 
 	return result, nil