diff --git a/Cargo.lock b/Cargo.lock index 3133d68aad4..8a5851de88e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4417,7 +4417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pki-types", ] @@ -5031,7 +5031,7 @@ dependencies = [ "clap 4.5.23", "hyper 1.5.2", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pemfile 2.2.0", "serde_json", "tokio", @@ -5160,7 +5160,7 @@ dependencies = [ "hyper 1.5.2", "hyper-util", "log", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -5624,7 +5624,7 @@ dependencies = [ "rand 0.8.5", "rcgen", "reqwest 0.12.9", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-acme", "rustls-pemfile 2.2.0", "rustls-platform-verifier", @@ -5717,7 +5717,7 @@ dependencies = [ "rcgen", "regex", "reqwest 0.12.9", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pemfile 2.2.0", "serde", "serde_bytes", @@ -6049,7 +6049,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.20", + "rustls 0.23.18", "serde", "serde_cbor", "tokio", @@ -6970,7 +6970,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rsa", - "rustls 0.23.20", + "rustls 0.23.18", "serde", "sha2 0.10.8", "simple_asn1", @@ -7731,7 +7731,7 @@ dependencies = [ "ic-types-test-utils", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.20", + "rustls 0.23.18", "tempfile", "tokio", ] @@ -7931,7 +7931,7 @@ dependencies = [ "ic-types", "pkcs8", "rand 0.8.5", - "rustls 0.23.20", + "rustls 0.23.18", "signature", "time", "tokio", @@ -7967,7 +7967,7 @@ dependencies = [ "ic-types", "json5", "maplit", - "rustls 0.23.20", + "rustls 0.23.18", "serde", "thiserror 2.0.8", "x509-parser", @@ -7980,7 +7980,7 @@ dependencies = [ "ic-base-types", "ic-crypto-tls-interfaces", "mockall", - "rustls 0.23.20", + "rustls 0.23.18", ] [[package]] @@ -8516,7 +8516,7 @@ dependencies = [ "rand 0.8.5", "reqwest 0.12.9", "rstest", - "rustls 0.23.20", + "rustls 0.23.18", "serde", "serde_bytes", "serde_cbor", @@ -8637,7 +8637,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "rstest", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pemfile 2.2.0", "serde", "serde_json", @@ -10472,7 +10472,6 @@ dependencies = [ "csv", "hex", "ic-base-types", - "ic-ledger-canister-core", "ic-nns-constants", "ic-nns-governance-api", "ic-nns-gtc", @@ -10703,7 +10702,7 @@ dependencies = [ "quinn", "quinn-udp", "rcgen", - "rustls 0.23.20", + "rustls 0.23.18", "serde", "slog", "tempfile", @@ -10867,7 +10866,7 @@ dependencies = [ "prost 0.13.4", "quinn", "rstest", - "rustls 0.23.20", + "rustls 0.23.18", "slog", "socket2 0.5.8", "static_assertions", @@ -14716,7 +14715,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem 3.0.4", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pemfile 2.2.0", "secrecy", "serde", @@ -17839,7 +17838,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.18", "socket2 0.5.8", "thiserror 2.0.8", "tokio", @@ -17857,7 +17856,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-pki-types", "slab", "thiserror 2.0.8", @@ -18508,7 +18507,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -18994,9 +18993,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "brotli 7.0.0", "brotli-decompressor", @@ -19098,7 +19097,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.20", + "rustls 0.23.18", "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki 0.102.8", @@ -21031,7 +21030,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.18", "tokio", ] diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 09c62d8bd34..36611ea481b 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -17,6 +17,8 @@ canisters( "registry": "registry-canister.wasm.gz", "governance": "governance-canister.wasm.gz", "ledger": "ledger-canister_notify-method.wasm.gz", + "ledger_v1": "ledger-canister_notify-method.wasm.gz", + "ledger_v2": "ledger-canister_notify-method.wasm.gz", "archive": "ledger-archive-node-canister.wasm.gz", "index": "ic-icp-index-canister.wasm.gz", "root": "root-canister.wasm.gz", @@ -47,6 +49,8 @@ canisters( "registry": "mainnet_nns_registry_canister", "governance": "mainnet_nns_governance_canister", "ledger": "mainnet_icp_ledger_canister", + "ledger_v1": "mainnet_icp_ledger_canister-v1", + "ledger_v2": "mainnet_icp_ledger_canister-v2", "archive": "mainnet_icp_ledger-archive-node-canister", "index": "mainnet_icp_index_canister", "root": "mainnet_nns_root-canister", diff --git a/mainnet-canister-revisions.json b/mainnet-canister-revisions.json index f940ecf64b1..bd87ae10e6d 100644 --- a/mainnet-canister-revisions.json +++ b/mainnet-canister-revisions.json @@ -63,6 +63,14 @@ "rev": "7c6309cb5bec7ab28ed657ac7672af08a59fc1ba", "sha256": "a9ed1cb9dda555e0fc1038825eb7b3a6b366f17aa4b88575184c7537e864e551" }, + "ledger_v1": { + "rev": "6dcfafb491092704d374317d9a72a7ad2475d7c9", + "sha256": "4fe38a91a3130e9d8b39e3413ae3b3f46c40d3fbd507df1b6092f962d970a7ea" + }, + "ledger_v2": { + "rev": "dac2f36f96d7549d82fa8e3c714979255ce57afd", + "sha256": "50c05fd687883fe788c0bb91996de358d8f856ba56088c6ff47767ea853001d7" + }, "lifeline": { "rev": "a0207146be211cdff83321c99e9e70baa62733c7", "sha256": "76978515223287ece643bc7ca087eb310412b737e2382a73b8ae55fcb458da5b" diff --git a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs index 30e9d972a98..a97b5af638e 100644 --- a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs +++ b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs @@ -4,22 +4,18 @@ use ic_canister_log::{log, Sink}; use ic_ledger_core::approvals::{ AllowanceTable, AllowancesData, ApproveError, InsufficientAllowance, }; -use ic_ledger_core::tokens::Zero; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, VecDeque}; use std::ops::Range; use std::time::Duration; use crate::archive::{ArchivingGuardError, FailedToArchiveBlocks, LedgerArchivingGuard}; -use ic_ledger_core::balances::{BalanceError, Balances, BalancesStore, InspectableBalancesStore}; +use ic_ledger_core::balances::{BalanceError, Balances, BalancesStore}; use ic_ledger_core::block::{BlockIndex, BlockType, EncodedBlock, FeeCollector}; use ic_ledger_core::timestamp::TimeStamp; use ic_ledger_core::tokens::TokensType; use ic_ledger_hash_of::HashOf; -/// The memo to use for balances burned and approvals reset to 0 during trimming -const TRIMMED_MEMO: u64 = u64::MAX; - #[derive(Debug, Deserialize, Serialize)] pub struct TransactionInfo { pub block_timestamp: TimeStamp, @@ -158,13 +154,6 @@ pub trait LedgerData: LedgerContext { /// The maximum number of transactions that we attempt to purge in one go. fn max_transactions_to_purge(&self) -> usize; - /// The maximum size of the balances map. - fn max_number_of_accounts(&self) -> usize; - - /// How many accounts with lowest balances to purge when the number of accounts exceeds - /// [LedgerData::max_number_of_accounts]. - fn accounts_overflow_trim_quantity(&self) -> usize; - // Token configuration /// Token name (e.g., Bitcoin). @@ -208,30 +197,12 @@ pub enum TransferError { const APPROVE_PRUNE_LIMIT: usize = 100; /// Adds a new block with the specified transaction to the ledger. -/// Trim balances if necessary. pub fn apply_transaction( ledger: &mut L, transaction: L::Transaction, now: TimeStamp, effective_fee: L::Tokens, ) -> Result<(BlockIndex, HashOf), TransferError> -where - L: LedgerData, - L::BalancesStore: InspectableBalancesStore, -{ - let result = apply_transaction_no_trimming(ledger, transaction, now, effective_fee); - trim_balances(ledger, now); - result -} - -/// Adds a new block with the specified transaction to the ledger. -/// Do not perform any balance trimming. -pub fn apply_transaction_no_trimming( - ledger: &mut L, - transaction: L::Transaction, - now: TimeStamp, - effective_fee: L::Tokens, -) -> Result<(BlockIndex, HashOf), TransferError> where L: LedgerData, { @@ -322,44 +293,6 @@ where Ok((height, ledger.blockchain().last_hash.unwrap())) } -/// Trim balances. Can be used e.g. if the ledger is low on heap memory. -fn trim_balances(ledger: &mut L, now: TimeStamp) -where - L: LedgerData, - L::BalancesStore: InspectableBalancesStore, -{ - let effective_max_number_of_accounts = - ledger.max_number_of_accounts() + ledger.accounts_overflow_trim_quantity() - 1; - - let to_trim = if ledger.balances().store.len() > effective_max_number_of_accounts { - select_accounts_to_trim(ledger) - } else { - vec![] - }; - - for (balance, account) in to_trim { - let burn_tx = L::Transaction::burn(account, None, balance, Some(now), Some(TRIMMED_MEMO)); - - burn_tx - .apply(ledger, now, L::Tokens::zero()) - .expect("failed to burn funds that must have existed"); - - let parent_hash = ledger.blockchain().last_hash; - let fee_collector = ledger.fee_collector().cloned(); - - ledger - .blockchain_mut() - .add_block(L::Block::from_transaction( - parent_hash, - burn_tx, - now, - L::Tokens::zero(), - fee_collector, - )) - .unwrap(); - } -} - /// Finds the archive canister that contains the block with the specified height. pub fn find_block_in_archive(ledger: &L, block_height: u64) -> Option { let index = ledger @@ -453,39 +386,6 @@ pub fn purge_old_transactions(ledger: &mut L, now: TimeStamp) -> num_tx_purged } -// Find the specified number of accounts with lowest balances so that their -// balances can be reclaimed. -fn select_accounts_to_trim(ledger: &L) -> Vec<(L::Tokens, L::AccountId)> -where - L: LedgerData, - L::BalancesStore: InspectableBalancesStore, - L::Tokens: TokensType, -{ - let mut to_trim: std::collections::BinaryHeap<(L::Tokens, L::AccountId)> = - std::collections::BinaryHeap::new(); - - let num_accounts = ledger.accounts_overflow_trim_quantity(); - let mut iter = ledger.balances().store.iter(); - - // Accumulate up to `trim_quantity` accounts - for (account, balance) in iter.by_ref().take(num_accounts) { - to_trim.push((balance.clone(), account.clone())); - } - - for (account, balance) in iter { - // If any account's balance is lower than the maximum in our set, - // include that account, and remove the current maximum - if let Some((greatest_balance, _)) = to_trim.peek() { - if balance < greatest_balance { - to_trim.push((balance.clone(), account.clone())); - to_trim.pop(); - } - } - } - - to_trim.into_vec() -} - /// Asynchronously archives a suffix of the locally available blockchain. /// /// NOTE: only one archiving task can run at each point in time. diff --git a/rs/ledger_suite/common/ledger_core/src/balances.rs b/rs/ledger_suite/common/ledger_core/src/balances.rs index 68d9791b879..cd229dfcabe 100644 --- a/rs/ledger_suite/common/ledger_core/src/balances.rs +++ b/rs/ledger_suite/common/ledger_core/src/balances.rs @@ -19,13 +19,6 @@ pub trait BalancesStore { F: FnMut(Option<&Self::Tokens>) -> Result; } -#[allow(clippy::len_without_is_empty)] -pub trait InspectableBalancesStore: BalancesStore { - fn iter(&self) -> Box + '_>; - - fn len(&self) -> usize; -} - impl BalancesStore for BTreeMap where AccountId: Eq + Clone + std::cmp::Ord, @@ -63,20 +56,6 @@ where } } -impl InspectableBalancesStore for BTreeMap -where - AccountId: Eq + Clone + std::cmp::Ord, - Tokens: TokensType, -{ - fn len(&self) -> usize { - self.len() - } - - fn iter(&self) -> Box + '_> { - Box::new(self.iter()) - } -} - /// An error returned by `Balances` if the debit operation fails. #[derive(Debug)] pub enum BalanceError { diff --git a/rs/ledger_suite/common/ledger_core/src/tokens.rs b/rs/ledger_suite/common/ledger_core/src/tokens.rs index 3707aa1dae4..1693324ce1b 100644 --- a/rs/ledger_suite/common/ledger_core/src/tokens.rs +++ b/rs/ledger_suite/common/ledger_core/src/tokens.rs @@ -1,10 +1,15 @@ use candid::{CandidType, Nat}; +use ic_stable_structures::{storable::Bound, Storable}; use minicbor::{Decode, Encode}; use num_traits::{Bounded, ToPrimitive}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::borrow::Cow; use std::fmt; use std::fmt::Debug; +#[cfg(test)] +mod tests; + /// Performs addition that returns `None` instead of wrapping around on /// overflow. /// @@ -302,3 +307,20 @@ impl From for Nat { Nat::from(value.e8s) } } + +impl Storable for Tokens { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(self.e8s.to_le_bytes().to_vec()) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + Self { + e8s: u64::from_le_bytes(bytes.into_owned().as_slice().try_into().unwrap()), + } + } + + const BOUND: Bound = Bound::Bounded { + max_size: 8, + is_fixed_size: true, + }; +} diff --git a/rs/ledger_suite/common/ledger_core/src/tokens/tests.rs b/rs/ledger_suite/common/ledger_core/src/tokens/tests.rs new file mode 100644 index 00000000000..5d0324be9c8 --- /dev/null +++ b/rs/ledger_suite/common/ledger_core/src/tokens/tests.rs @@ -0,0 +1,12 @@ +use crate::tokens::Tokens; +use ic_stable_structures::Storable; +use proptest::prelude::{any, prop_assert_eq, proptest}; + +#[test] +fn tokens_serialization() { + proptest!(|(e8s in any::())| { + let tokens = Tokens { e8s }; + let new_tokens = Tokens::from_bytes(tokens.to_bytes()); + prop_assert_eq!(new_tokens, tokens); + }) +} diff --git a/rs/ledger_suite/icp/ledger.did b/rs/ledger_suite/icp/ledger.did index a4778eb2446..7466333461c 100644 --- a/rs/ledger_suite/icp/ledger.did +++ b/rs/ledger_suite/icp/ledger.did @@ -302,8 +302,6 @@ type InitArgs = record { token_symbol: opt text; token_name: opt text; feature_flags : opt FeatureFlags; - maximum_number_of_accounts : opt nat64; - accounts_overflow_trim_quantity: opt nat64; }; type Icrc1BlockIndex = nat; diff --git a/rs/ledger_suite/icp/ledger/BUILD.bazel b/rs/ledger_suite/icp/ledger/BUILD.bazel index 0d9bea50cb9..fb6d2de14ae 100644 --- a/rs/ledger_suite/icp/ledger/BUILD.bazel +++ b/rs/ledger_suite/icp/ledger/BUILD.bazel @@ -118,12 +118,16 @@ rust_ic_test( ":ledger-canister-wasm-allowance-getter", ":ledger-canister-wasm-low-limits", ":ledger-canister-wasm-next-version", + "@mainnet_icp_ledger_canister-v1//file", + "@mainnet_icp_ledger_canister-v2//file", "@mainnet_icp_ledger_canister//file", ], env = { "RUST_TEST_THREADS": "4", "CARGO_MANIFEST_DIR": "rs/ledger_suite/icp/ledger", "ICP_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_icp_ledger_canister//file)", + "ICP_LEDGER_DEPLOYED_VERSION_V1_WASM_PATH": "$(rootpath @mainnet_icp_ledger_canister-v1//file)", + "ICP_LEDGER_DEPLOYED_VERSION_V2_WASM_PATH": "$(rootpath @mainnet_icp_ledger_canister-v2//file)", "LEDGER_CANISTER_WASM_PATH": "$(rootpath :ledger-canister-wasm)", "LEDGER_CANISTER_ALLOWANCE_GETTER_WASM_PATH": "$(rootpath :ledger-canister-wasm-allowance-getter)", "LEDGER_CANISTER_NEXT_VERSION_WASM_PATH": "$(rootpath :ledger-canister-wasm-next-version)", diff --git a/rs/ledger_suite/icp/ledger/src/lib.rs b/rs/ledger_suite/icp/ledger/src/lib.rs index d401537dd74..98a27c87ea5 100644 --- a/rs/ledger_suite/icp/ledger/src/lib.rs +++ b/rs/ledger_suite/icp/ledger/src/lib.rs @@ -5,6 +5,7 @@ use ic_ledger_canister_core::blockchain::Blockchain; use ic_ledger_canister_core::ledger::{ self as core_ledger, LedgerContext, LedgerData, TransactionInfo, }; +use ic_ledger_core::balances::BalancesStore; use ic_ledger_core::{ approvals::{Allowance, AllowanceTable, AllowancesData}, balances::Balances, @@ -75,6 +76,7 @@ fn default_ledger_version() -> u64 { const UPGRADES_MEMORY_ID: MemoryId = MemoryId::new(0); const ALLOWANCES_MEMORY_ID: MemoryId = MemoryId::new(1); const ALLOWANCES_EXPIRATIONS_MEMORY_ID: MemoryId = MemoryId::new(2); +const BALANCES_MEMORY_ID: MemoryId = MemoryId::new(3); #[derive(Clone, Debug, Encode, Decode)] struct StorableAllowance { @@ -142,12 +144,17 @@ thread_local! { #[allow(clippy::type_complexity)] pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); + + // account -> tokens - map storing ledger balances. + pub static BALANCES_MEMORY: RefCell>> = + MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(BALANCES_MEMORY_ID)))); } #[derive(Copy, Clone, Serialize, Deserialize, Debug)] pub enum LedgerField { Allowances, AllowancesExpirations, + Balances, } #[derive(Copy, Clone, Serialize, Deserialize, Debug)] @@ -168,24 +175,28 @@ impl Default for LedgerState { /// We have the following ledger versions: /// * 0 - the whole ledger state is stored on the heap. /// * 1 - the allowances are stored in stable structures. +/// * 2 - the balances are stored in stable structures. #[cfg(not(feature = "next-ledger-version"))] -pub const LEDGER_VERSION: u64 = 1; +pub const LEDGER_VERSION: u64 = 2; #[cfg(feature = "next-ledger-version")] -pub const LEDGER_VERSION: u64 = 2; +pub const LEDGER_VERSION: u64 = 3; + +type StableLedgerBalances = Balances; #[derive(Debug, Deserialize, Serialize)] pub struct Ledger { balances: LedgerBalances, #[serde(default)] + stable_balances: StableLedgerBalances, + #[serde(default)] approvals: LedgerAllowances, #[serde(default)] stable_approvals: AllowanceTable, pub blockchain: Blockchain, - // A cap on the maximum number of accounts. + // DEPRECATED pub maximum_number_of_accounts: usize, - // When maximum number of accounts is exceeded, a specified number of - // accounts with lowest balances are removed. + // DEPRECATED accounts_overflow_trim_quantity: usize, pub minting_account_id: Option, pub icrc1_minting_account: Option, @@ -229,17 +240,17 @@ pub struct Ledger { impl LedgerContext for Ledger { type AccountId = AccountIdentifier; type AllowancesData = StableAllowancesData; - type BalancesStore = BTreeMap; + type BalancesStore = StableBalances; type Tokens = Tokens; fn balances(&self) -> &Balances { panic_if_not_ready(); - &self.balances + &self.stable_balances } fn balances_mut(&mut self) -> &mut Balances { panic_if_not_ready(); - &mut self.balances + &mut self.stable_balances } fn approvals(&self) -> &AllowanceTable { @@ -275,14 +286,6 @@ impl LedgerData for Ledger { Self::MAX_TRANSACTIONS_TO_PURGE } - fn max_number_of_accounts(&self) -> usize { - self.maximum_number_of_accounts - } - - fn accounts_overflow_trim_quantity(&self) -> usize { - self.accounts_overflow_trim_quantity - } - fn token_name(&self) -> &str { &self.token_name } @@ -330,11 +333,12 @@ impl Default for Ledger { fn default() -> Self { Self { approvals: Default::default(), + stable_balances: StableLedgerBalances::default(), stable_approvals: Default::default(), balances: LedgerBalances::default(), blockchain: Blockchain::default(), - maximum_number_of_accounts: 28_000_000, - accounts_overflow_trim_quantity: 100_000, + maximum_number_of_accounts: 0, + accounts_overflow_trim_quantity: 0, minting_account_id: None, icrc1_minting_account: None, blocks_notified: IntMap::new(), @@ -450,8 +454,6 @@ impl Ledger { token_symbol: Option, token_name: Option, feature_flags: Option, - maximum_number_of_accounts: Option, - accounts_overflow_trim_quantity: Option, ) { self.token_symbol = token_symbol.unwrap_or_else(|| "ICP".to_string()); self.token_name = token_name.unwrap_or_else(|| "Internet Computer".to_string()); @@ -479,12 +481,6 @@ impl Ledger { if let Some(feature_flags) = feature_flags { self.feature_flags = feature_flags; } - if let Some(maximum_number_of_accounts) = maximum_number_of_accounts { - self.maximum_number_of_accounts = maximum_number_of_accounts; - } - if let Some(accounts_overflow_trim_quantity) = accounts_overflow_trim_quantity { - self.accounts_overflow_trim_quantity = accounts_overflow_trim_quantity; - } } pub fn change_notification_state( @@ -593,9 +589,23 @@ impl Ledger { } } + pub fn migrate_one_balance(&mut self) -> bool { + match self.balances.store.pop_first() { + Some((account, tokens)) => { + self.stable_balances.credit(&account, tokens); + true + } + None => false, + } + } + pub fn clear_arrivals(&mut self) { self.approvals.allowances_data.clear_arrivals(); } + + pub fn copy_token_pool(&mut self) { + self.stable_balances.token_pool = self.balances.token_pool; + } } pub fn add_payment( @@ -650,6 +660,16 @@ pub fn clear_stable_allowance_data() { }); } +pub fn clear_stable_balances_data() { + BALANCES_MEMORY.with_borrow_mut(|balances| { + balances.clear_new(); + }); +} + +pub fn balances_len() -> u64 { + BALANCES_MEMORY.with_borrow(|balances| balances.len()) +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct StableAllowancesData {} @@ -733,3 +753,40 @@ impl AllowancesData for StableAllowancesData { panic!("The method `clear_arrivals` should not be called for StableAllowancesData") } } + +#[derive(Serialize, Deserialize, Debug, Default, PartialEq)] +pub struct StableBalances {} + +impl BalancesStore for StableBalances { + type AccountId = AccountIdentifier; + type Tokens = Tokens; + + fn get_balance(&self, k: &AccountIdentifier) -> Option { + BALANCES_MEMORY.with_borrow(|balances| balances.get(k)) + } + + fn update(&mut self, k: AccountIdentifier, mut f: F) -> Result + where + F: FnMut(Option<&Tokens>) -> Result, + { + let entry = BALANCES_MEMORY.with_borrow(|balances| balances.get(&k)); + match entry { + Some(v) => { + let new_v = f(Some(&v))?; + if new_v != Tokens::ZERO { + BALANCES_MEMORY.with_borrow_mut(|balances| balances.insert(k, new_v)); + } else { + BALANCES_MEMORY.with_borrow_mut(|balances| balances.remove(&k)); + } + Ok(new_v) + } + None => { + let new_v = f(None)?; + if new_v != Tokens::ZERO { + BALANCES_MEMORY.with_borrow_mut(|balances| balances.insert(k, new_v)); + } + Ok(new_v) + } + } + } +} diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs index 433c750b487..ec70136b3cc 100644 --- a/rs/ledger_suite/icp/ledger/src/main.rs +++ b/rs/ledger_suite/icp/ledger/src/main.rs @@ -57,9 +57,9 @@ use icrc_ledger_types::{ icrc21::{errors::Icrc21Error, requests::ConsentMessageRequest, responses::ConsentInfo}, }; use ledger_canister::{ - clear_stable_allowance_data, is_ready, ledger_state, panic_if_not_ready, set_ledger_state, - Ledger, LedgerField, LedgerState, LEDGER, LEDGER_VERSION, MAX_MESSAGE_SIZE_BYTES, - UPGRADES_MEMORY, + balances_len, clear_stable_allowance_data, clear_stable_balances_data, is_ready, ledger_state, + panic_if_not_ready, set_ledger_state, Ledger, LedgerField, LedgerState, LEDGER, LEDGER_VERSION, + MAX_MESSAGE_SIZE_BYTES, UPGRADES_MEMORY, }; use num_traits::cast::ToPrimitive; #[allow(unused_imports)] @@ -113,8 +113,6 @@ fn init( token_symbol: Option, token_name: Option, feature_flags: Option, - maximum_number_of_accounts: Option, - accounts_overflow_trim_quantity: Option, ) { print(format!( "[ledger] init(): minting account is {}", @@ -131,8 +129,6 @@ fn init( token_symbol, token_name, feature_flags, - maximum_number_of_accounts, - accounts_overflow_trim_quantity, ); match max_message_size_bytes { None => { @@ -712,8 +708,6 @@ fn canister_init(arg: LedgerCanisterPayload) { arg.token_symbol, arg.token_name, arg.feature_flags, - arg.maximum_number_of_accounts, - arg.accounts_overflow_trim_quantity, ), LedgerCanisterPayload::Upgrade(_) => { trap_with("Cannot initialize the canister with an Upgrade argument. Please provide an Init argument."); @@ -745,8 +739,6 @@ fn main() { arg.token_symbol, arg.token_name, arg.feature_flags, - arg.maximum_number_of_accounts, - arg.accounts_overflow_trim_quantity, ), Err(old_err) => trap_with(&format!("Unable to decode init argument.\nDecode as new init returned the error {}\nDecode as old init returned the error {}", new_err, old_err)) @@ -760,7 +752,7 @@ fn main() { const BUFFER_SIZE: usize = 8388608; #[cfg(not(feature = "low-upgrade-instruction-limits"))] -const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 250_000_000_000; #[cfg(not(feature = "low-upgrade-instruction-limits"))] const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; @@ -839,6 +831,12 @@ fn post_upgrade(args: Option) { PRE_UPGRADE_INSTRUCTIONS_CONSUMED .with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); + if upgrade_from_version < 2 { + set_ledger_state(LedgerState::Migrating(LedgerField::Balances)); + print(format!("Upgrading from version {upgrade_from_version} which does not store balances in stable structures, clearing stable balances data.").as_str()); + clear_stable_balances_data(); + ledger.copy_token_pool(); + } if upgrade_from_version == 0 { set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); print("Upgrading from version 0 which does not use stable structures, clearing stable allowance data."); @@ -864,6 +862,7 @@ fn migrate_next_part(instruction_limit: u64) { STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow_mut() += 1); let mut migrated_allowances = 0; let mut migrated_expirations = 0; + let mut migrated_balances = 0; print("Migrating part of the ledger state."); @@ -884,6 +883,13 @@ fn migrate_next_part(instruction_limit: u64) { LedgerField::AllowancesExpirations => { if ledger.migrate_one_expiration() { migrated_expirations += 1; + } else { + set_ledger_state(LedgerState::Migrating(LedgerField::Balances)); + } + } + LedgerField::Balances => { + if ledger.migrate_one_balance() { + migrated_balances += 1; } else { set_ledger_state(LedgerState::Ready); } @@ -891,8 +897,8 @@ fn migrate_next_part(instruction_limit: u64) { } } let instructions_migration = instruction_counter() - instructions_migration_start; - let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_migration}, total instructions used in message: {}." , - instruction_counter()); + let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations} balances: {migrated_balances}. Migration step instructions: {instructions_migration}, total instructions used in message: {}." , + instruction_counter()); if !is_ready() { print(format!( "Migration partially done. Scheduling the next part. {msg}" @@ -1500,7 +1506,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i )?; w.encode_gauge( "ledger_balance_store_entries", - ledger.balances().store.len() as f64, + balances_len() as f64, "Total number of accounts in the balance store.", )?; } diff --git a/rs/ledger_suite/icp/ledger/src/tests.rs b/rs/ledger_suite/icp/ledger/src/tests.rs index c5fe4fd4a70..a3bf66c0566 100644 --- a/rs/ledger_suite/icp/ledger/src/tests.rs +++ b/rs/ledger_suite/icp/ledger/src/tests.rs @@ -1,4 +1,4 @@ -use crate::{AccountIdentifier, Ledger, StorableAllowance}; +use crate::{balances_len, AccountIdentifier, Ledger, StorableAllowance}; use ic_base_types::{CanisterId, PrincipalId}; use ic_ledger_canister_core::{ archive::Archive, @@ -7,14 +7,15 @@ use ic_ledger_canister_core::{ }; use ic_ledger_core::{ approvals::Allowance, + balances::BalancesStore, block::{BlockIndex, BlockType}, timestamp::TimeStamp, - tokens::{CheckedAdd, CheckedSub, Tokens}, + tokens::Tokens, }; use ic_stable_structures::Storable; use icp_ledger::{ - apply_operation, ArchiveOptions, Block, LedgerBalances, Memo, Operation, PaymentError, - Transaction, TransferError, DEFAULT_TRANSFER_FEE, + apply_operation, ArchiveOptions, Block, Memo, Operation, PaymentError, Transaction, + TransferError, DEFAULT_TRANSFER_FEE, }; use proptest::prelude::{any, prop_assert_eq, proptest}; use proptest::strategy::Strategy; @@ -34,71 +35,6 @@ fn ts(n: u64) -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(n) } -#[test] -fn balances_overflow() { - let balances = LedgerBalances::new(); - let mut state = Ledger { - balances, - maximum_number_of_accounts: 8, - accounts_overflow_trim_quantity: 2, - minting_account_id: Some(PrincipalId::new_user_test_id(137).into()), - ..Default::default() - }; - assert_eq!(state.balances.token_pool, Tokens::MAX); - println!( - "minting canister initial balance: {}", - state.balances.token_pool - ); - let mut credited = Tokens::ZERO; - - // 11 accounts. The one with 0 will not be added - // The rest will be added and trigger a trim of 2 once - // the total number reaches 8 + 2 - // the number of active accounts won't go below 8 after trimming - for i in 0..11 { - let amount = Tokens::new(i, 0).unwrap(); - state - .add_payment( - Memo::default(), - Operation::Mint { - to: PrincipalId::new_user_test_id(i).into(), - amount, - }, - None, - ) - .unwrap(); - credited = credited.checked_add(&amount).unwrap(); - } - println!("amount credited to accounts: {}", credited); - - println!("balances: {:?}", state.balances); - - // The two accounts with lowest balances, 0 and 1 respectively, have been - // removed - assert_eq!(state.balances.store.len(), 8); - assert_eq!( - state - .balances - .account_balance(&PrincipalId::new_user_test_id(0).into()), - Tokens::ZERO - ); - assert_eq!( - state - .balances - .account_balance(&PrincipalId::new_user_test_id(1).into()), - Tokens::ZERO - ); - // We have credited 55 Tokens to various accounts but the three accounts - // with lowest balances, 0, 1 and 2, should have been removed and their - // balance returned to the minting canister - let expected_minting_canister_balance = Tokens::MAX - .checked_sub(&credited) - .unwrap() - .checked_add(&Tokens::new(1 + 2, 0).unwrap()) - .unwrap(); - assert_eq!(state.balances.token_pool, expected_minting_canister_balance); -} - #[test] fn balances_remove_accounts_with_zero_balance() { let mut ctx = Ledger::default(); @@ -116,8 +52,8 @@ fn balances_remove_accounts_with_zero_balance() { .unwrap(); // verify that an account entry exists for the `canister` assert_eq!( - ctx.balances().store.get(&canister), - Some(&Tokens::from_e8s(1000)) + ctx.balances().store.get_balance(&canister), + Some(&Tokens::from_e8s(1000)).copied() ); // make 2 transfers that empty the account for _ in 0..2 { @@ -136,15 +72,15 @@ fn balances_remove_accounts_with_zero_balance() { } // target canister's balance adds up assert_eq!( - ctx.balances().store.get(&target_canister), - Some(&Tokens::from_e8s(800)) + ctx.balances().store.get_balance(&target_canister), + Some(&Tokens::from_e8s(800)).copied() ); // source canister has been removed - assert_eq!(ctx.balances().store.get(&canister), None); + assert_eq!(ctx.balances().store.get_balance(&canister), None); assert_eq!(ctx.balances().account_balance(&canister), Tokens::ZERO); // one account left in the store - assert_eq!(ctx.balances().store.len(), 1); + assert_eq!(balances_len(), 1); apply_operation( &mut ctx, @@ -159,11 +95,11 @@ fn balances_remove_accounts_with_zero_balance() { ) .unwrap(); // No new account should have been created - assert_eq!(ctx.balances().store.len(), 1); + assert_eq!(balances_len(), 1); // and the fee should have been taken from sender assert_eq!( - ctx.balances().store.get(&target_canister), - Some(&Tokens::from_e8s(700)) + ctx.balances().store.get_balance(&target_canister), + Some(&Tokens::from_e8s(700)).copied() ); apply_operation( @@ -177,7 +113,7 @@ fn balances_remove_accounts_with_zero_balance() { .unwrap(); // No new account should have been created - assert_eq!(ctx.balances().store.len(), 1); + assert_eq!(balances_len(), 1); apply_operation( &mut ctx, @@ -191,7 +127,7 @@ fn balances_remove_accounts_with_zero_balance() { .unwrap(); // And burn should have exhausted the target_canister - assert_eq!(ctx.balances().store.len(), 0); + assert_eq!(balances_len(), 0); } #[test] @@ -264,8 +200,6 @@ fn serialize() { Some("ICP".into()), Some("icp".into()), None, - None, - None, ); let txn = Transaction::new( @@ -604,8 +538,6 @@ fn get_blocks_returns_correct_blocks() { Some("ICP".into()), Some("icp".into()), None, - None, - None, ); for i in 0..10 { @@ -669,8 +601,6 @@ fn test_purge() { Some("ICP".into()), Some("icp".into()), None, - None, - None, ); let little_later = genesis + Duration::from_millis(1); diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index 4b7573de082..e088014260a 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -59,6 +59,14 @@ fn ledger_wasm_mainnet() -> Vec { std::fs::read(std::env::var("ICP_LEDGER_DEPLOYED_VERSION_WASM_PATH").unwrap()).unwrap() } +fn ledger_wasm_mainnet_v1() -> Vec { + std::fs::read(std::env::var("ICP_LEDGER_DEPLOYED_VERSION_V1_WASM_PATH").unwrap()).unwrap() +} + +fn ledger_wasm_mainnet_v2() -> Vec { + std::fs::read(std::env::var("ICP_LEDGER_DEPLOYED_VERSION_V2_WASM_PATH").unwrap()).unwrap() +} + fn ledger_wasm_allowance_getter() -> Vec { ic_test_utilities_load_wasm::load_wasm( std::env::var("CARGO_MANIFEST_DIR").unwrap(), @@ -91,8 +99,6 @@ fn encode_init_args( .transfer_fee(Tokens::try_from(args.transfer_fee).unwrap()) .token_symbol_and_name(&args.token_symbol, &args.token_name) .feature_flags(FeatureFlags { icrc2: true }) - .maximum_number_of_accounts(args.maximum_number_of_accounts) - .accounts_overflow_trim_quantity(args.accounts_overflow_trim_quantity) .build() .unwrap() } @@ -509,8 +515,6 @@ fn check_old_init() { token_symbol: Some("ICP".into()), token_name: Some("Internet Computer".into()), feature_flags: None, - maximum_number_of_accounts: None, - accounts_overflow_trim_quantity: None, }) .unwrap(); env.install_canister(ledger_wasm(), old_init, None) @@ -1227,8 +1231,16 @@ fn test_block_transformation() { } #[test] -fn test_upgrade_serialization() { - let ledger_wasm_mainnet = ledger_wasm_mainnet(); +fn test_upgrade_serialization_from_master() { + test_upgrade_serialization(ledger_wasm_mainnet()); +} + +#[test] +fn test_upgrade_serialization_from_v2() { + test_upgrade_serialization(ledger_wasm_mainnet_v2()); +} + +fn test_upgrade_serialization(ledger_wasm_mainnet: Vec) { let ledger_wasm_current = ledger_wasm(); let minter = Arc::new(minter_identity()); @@ -1250,13 +1262,12 @@ fn test_upgrade_serialization() { upgrade_args, minter, false, - false, + true, ); } -#[ignore] // TODO: Re-enable as part of FI-1440 #[test] -fn test_multi_step_migration() { +fn test_multi_step_migration_from_mainnet() { ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration( ledger_wasm_mainnet(), ledger_wasm_low_instruction_limits(), @@ -1264,6 +1275,15 @@ fn test_multi_step_migration() { ); } +#[test] +fn test_multi_step_migration_from_v2() { + ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration( + ledger_wasm_mainnet_v2(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} + #[test] fn test_downgrade_from_incompatible_version() { ic_ledger_suite_state_machine_tests::test_downgrade_from_incompatible_version( @@ -1271,13 +1291,21 @@ fn test_downgrade_from_incompatible_version() { ledger_wasm_next_version(), ledger_wasm(), encode_init_args, - true, + false, ); } -#[ignore] // TODO: Re-enable as part of FI-1440 #[test] -fn test_stable_migration_endpoints_disabled() { +fn test_stable_migration_endpoints_disabled_from_mainnet() { + test_stable_migration_endpoints_disabled(ledger_wasm_mainnet()); +} + +#[test] +fn test_stable_migration_endpoints_disabled_from_v2() { + test_stable_migration_endpoints_disabled(ledger_wasm_mainnet_v2()); +} + +fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec) { let send_args = SendArgs { memo: icp_ledger::Memo::default(), amount: Tokens::from_e8s(1), @@ -1302,7 +1330,7 @@ fn test_stable_migration_endpoints_disabled() { .unwrap(); ic_ledger_suite_state_machine_tests::icrc1_test_stable_migration_endpoints_disabled( - ledger_wasm_mainnet(), + ledger_wasm_mainnet, ledger_wasm_low_instruction_limits(), encode_init_args, vec![ @@ -1313,9 +1341,8 @@ fn test_stable_migration_endpoints_disabled() { ); } -#[ignore] // TODO: Re-enable as part of FI-1440 #[test] -fn test_incomplete_migration() { +fn test_incomplete_migration_from_mainnet() { ic_ledger_suite_state_machine_tests::test_incomplete_migration( ledger_wasm_mainnet(), ledger_wasm_low_instruction_limits(), @@ -1323,9 +1350,17 @@ fn test_incomplete_migration() { ); } -#[ignore] // TODO: Re-enable as part of FI-1440 #[test] -fn test_incomplete_migration_to_current() { +fn test_incomplete_migration_from_v2() { + ic_ledger_suite_state_machine_tests::test_incomplete_migration( + ledger_wasm_mainnet_v2(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} + +#[test] +fn test_incomplete_migration_to_current_from_mainnet() { ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current( ledger_wasm_mainnet(), ledger_wasm_low_instruction_limits(), @@ -1333,9 +1368,17 @@ fn test_incomplete_migration_to_current() { ); } -#[ignore] // TODO: Re-enable as part of FI-1440 #[test] -fn test_metrics_while_migrating() { +fn test_incomplete_migration_to_current_from_v2() { + ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current( + ledger_wasm_mainnet_v2(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} + +#[test] +fn test_metrics_while_migrating_from_mainnet() { ic_ledger_suite_state_machine_tests::test_metrics_while_migrating( ledger_wasm_mainnet(), ledger_wasm_low_instruction_limits(), @@ -1343,6 +1386,24 @@ fn test_metrics_while_migrating() { ); } +#[test] +fn test_metrics_while_migrating_from_v2() { + ic_ledger_suite_state_machine_tests::test_metrics_while_migrating( + ledger_wasm_mainnet_v2(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} + +#[test] +fn test_upgrade_from_v1_not_possible() { + ic_ledger_suite_state_machine_tests::test_upgrade_from_v1_not_possible( + ledger_wasm_mainnet_v1(), + ledger_wasm(), + encode_init_args, + ); +} + #[test] fn test_approve_smoke() { ic_ledger_suite_state_machine_tests::test_approve_smoke(ledger_wasm(), encode_init_args); @@ -1506,11 +1567,6 @@ fn test_transfer_from_burn() { ic_ledger_suite_state_machine_tests::test_transfer_from_burn(ledger_wasm(), encode_init_args); } -#[test] -fn test_balances_overflow() { - ic_ledger_suite_state_machine_tests::test_balances_overflow(ledger_wasm(), encode_init_args); -} - #[test] fn account_identifier_test() { let env = StateMachine::new(); diff --git a/rs/ledger_suite/icp/src/lib.rs b/rs/ledger_suite/icp/src/lib.rs index b83917ef97c..90c381de74f 100644 --- a/rs/ledger_suite/icp/src/lib.rs +++ b/rs/ledger_suite/icp/src/lib.rs @@ -474,8 +474,6 @@ pub struct InitArgs { pub token_symbol: Option, pub token_name: Option, pub feature_flags: Option, - pub maximum_number_of_accounts: Option, - pub accounts_overflow_trim_quantity: Option, } impl LedgerCanisterInitPayload { @@ -508,8 +506,6 @@ pub struct LedgerCanisterInitPayloadBuilder { token_symbol: Option, token_name: Option, feature_flags: Option, - maximum_number_of_accounts: Option, - accounts_overflow_trim_quantity: Option, } impl LedgerCanisterInitPayloadBuilder { @@ -526,8 +522,6 @@ impl LedgerCanisterInitPayloadBuilder { token_symbol: None, token_name: None, feature_flags: None, - maximum_number_of_accounts: None, - accounts_overflow_trim_quantity: None, } } @@ -582,23 +576,6 @@ impl LedgerCanisterInitPayloadBuilder { self } - pub fn maximum_number_of_accounts(mut self, maximum_number_of_accounts: Option) -> Self { - if let Some(maximum_number_of_accounts) = maximum_number_of_accounts { - self.maximum_number_of_accounts = Some(maximum_number_of_accounts as usize); - } - self - } - - pub fn accounts_overflow_trim_quantity( - mut self, - accounts_overflow_trim_quantity: Option, - ) -> Self { - if let Some(accounts_overflow_trim_quantity) = accounts_overflow_trim_quantity { - self.accounts_overflow_trim_quantity = Some(accounts_overflow_trim_quantity as usize); - } - self - } - pub fn build(self) -> Result { let minting_account = self .minting_account @@ -632,8 +609,6 @@ impl LedgerCanisterInitPayloadBuilder { token_symbol: self.token_symbol, token_name: self.token_name, feature_flags: self.feature_flags, - maximum_number_of_accounts: self.maximum_number_of_accounts, - accounts_overflow_trim_quantity: self.accounts_overflow_trim_quantity, }, ))) } diff --git a/rs/ledger_suite/icp/tests/golden_nns_state.rs b/rs/ledger_suite/icp/tests/golden_nns_state.rs index 74030fe52c3..8766ed034e9 100644 --- a/rs/ledger_suite/icp/tests/golden_nns_state.rs +++ b/rs/ledger_suite/icp/tests/golden_nns_state.rs @@ -22,7 +22,7 @@ use ic_nns_constants::{ LEDGER_CANISTER_INDEX_IN_NNS_SUBNET, LEDGER_INDEX_CANISTER_INDEX_IN_NNS_SUBNET, }; use ic_nns_test_utils_golden_nns_state::new_state_machine_with_golden_nns_state_or_panic; -use ic_state_machine_tests::{StateMachine, UserError}; +use ic_state_machine_tests::{ErrorCode, StateMachine, UserError}; use icp_ledger::{ AccountIdentifier, Archives, Block, FeatureFlags, LedgerCanisterPayload, UpgradeArgs, }; @@ -30,9 +30,8 @@ use std::time::Instant; /// The number of instructions that can be executed in a single canister upgrade. /// The limit () -/// is actually 300B, but in the ledger implementation we use a value slightly lower than the old -/// limit 200B. -const CANISTER_UPGRADE_INSTRUCTION_LIMIT: u64 = 190_000_000_000; +/// is actually 300B, but in the ledger implementation we use a slightly lower limit of 250B. +const CANISTER_UPGRADE_INSTRUCTION_LIMIT: u64 = 250_000_000_000; const INDEX_CANISTER_ID: CanisterId = CanisterId::from_u64(LEDGER_INDEX_CANISTER_INDEX_IN_NNS_SUBNET); const LEDGER_CANISTER_ID: CanisterId = CanisterId::from_u64(LEDGER_CANISTER_INDEX_IN_NNS_SUBNET); @@ -320,8 +319,21 @@ impl Setup { pub fn downgrade_to_mainnet(&self) { println!("Downgrading to mainnet version"); self.upgrade_index(&self.mainnet_wasms.index); - self.upgrade_ledger(&self.mainnet_wasms.ledger) - .expect("should successfully downgrade to the mainnet version"); + match self.upgrade_ledger(&self.mainnet_wasms.ledger) { + Ok(_) => { + panic!("should fail to downgrade ledger to mainnet version"); + } + Err(err) => { + // The ledger will still be running the master version, while the other canisters + // will be downgraded to the mainnet version. This is not an ideal situation, nor + // is it expected to happen in practice, but for the moment let's run the test to + // completion. + err.assert_contains( + ErrorCode::CanisterCalledTrap, + "Trying to downgrade from incompatible version", + ); + } + } self.check_ledger_metrics(ExpectMigration::No); self.upgrade_archive_canisters(&self.mainnet_wasms.archive); } diff --git a/rs/ledger_suite/icp/tests/upgrade_downgrade.rs b/rs/ledger_suite/icp/tests/upgrade_downgrade.rs index 9ea817b32d2..46ec2e24d3e 100644 --- a/rs/ledger_suite/icp/tests/upgrade_downgrade.rs +++ b/rs/ledger_suite/icp/tests/upgrade_downgrade.rs @@ -451,7 +451,7 @@ fn should_upgrade_and_downgrade_canister_suite() { setup.assert_index_ledger_parity(true); setup.upgrade_index_canister(UpgradeToVersion::MainNet); - setup.upgrade_ledger_canister(UpgradeToVersion::MainNet, true); + setup.upgrade_ledger_canister(UpgradeToVersion::MainNet, false); setup.upgrade_archive_canisters(UpgradeToVersion::MainNet); setup.assert_index_ledger_parity(true); diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index ee7d2fc00bf..77dd0c654aa 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -22,9 +22,7 @@ use ic_ledger_canister_core::runtime::Runtime; use ic_ledger_canister_core::{ archive::ArchiveCanisterWasm, blockchain::Blockchain, - ledger::{ - apply_transaction_no_trimming, block_locations, LedgerContext, LedgerData, TransactionInfo, - }, + ledger::{apply_transaction, block_locations, LedgerContext, LedgerData, TransactionInfo}, range_utils, }; use ic_ledger_core::balances::BalancesStore; @@ -677,14 +675,12 @@ impl Ledger { ) }); let mint = Transaction::mint(account, amount, Some(now), None); - apply_transaction_no_trimming(&mut ledger, mint, now, Tokens::ZERO).unwrap_or_else( - |err| { - panic!( - "failed to mint {} tokens to {}: {:?}", - balance, account, err - ) - }, - ); + apply_transaction(&mut ledger, mint, now, Tokens::ZERO).unwrap_or_else(|err| { + panic!( + "failed to mint {} tokens to {}: {:?}", + balance, account, err + ) + }); } ledger @@ -782,14 +778,6 @@ impl LedgerData for Ledger { MAX_TRANSACTIONS_TO_PURGE } - fn max_number_of_accounts(&self) -> usize { - self.maximum_number_of_accounts - } - - fn accounts_overflow_trim_quantity(&self) -> usize { - self.accounts_overflow_trim_quantity - } - fn token_name(&self) -> &str { &self.token_name } diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 18643abf59e..2014271b4a3 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -21,7 +21,7 @@ use ic_icrc1_ledger::{ }; use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; use ic_ledger_canister_core::ledger::{ - apply_transaction_no_trimming, archive_blocks, LedgerAccess, LedgerContext, LedgerData, + apply_transaction, archive_blocks, LedgerAccess, LedgerContext, LedgerData, TransferError as CoreTransferError, }; use ic_ledger_canister_core::runtime::heap_memory_size_bytes; @@ -670,7 +670,7 @@ fn execute_transfer_not_async( ) }; - let (block_idx, _) = apply_transaction_no_trimming(ledger, tx, now, effective_fee)?; + let (block_idx, _) = apply_transaction(ledger, tx, now, effective_fee)?; Ok(block_idx) }) } @@ -869,7 +869,7 @@ async fn icrc2_approve(arg: ApproveArgs) -> Result { memo: arg.memo, }; - let (block_idx, _) = apply_transaction_no_trimming(ledger, tx, now, expected_fee_tokens) + let (block_idx, _) = apply_transaction(ledger, tx, now, expected_fee_tokens) .map_err(convert_transfer_error) .map_err(|err| { let err: ApproveError = match err.try_into() { diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index c38b394251a..5a84a1166d1 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -101,8 +101,6 @@ pub struct InitArgs { pub metadata: Vec<(String, Value)>, pub archive_options: ArchiveOptions, pub feature_flags: Option, - pub maximum_number_of_accounts: Option, - pub accounts_overflow_trim_quantity: Option, } #[derive(Clone, Eq, PartialEq, Debug, CandidType)] @@ -119,7 +117,6 @@ pub struct UpgradeArgs { pub transfer_fee: Option, pub change_fee_collector: Option, pub feature_flags: Option, - pub accounts_overflow_trim_quantity: Option, pub change_archive_options: Option, } @@ -894,8 +891,6 @@ fn init_args(initial_balances: Vec<(Account, u64)>) -> InitArgs { max_transactions_per_response: None, }, feature_flags: Some(FeatureFlags { icrc2: true }), - maximum_number_of_accounts: None, - accounts_overflow_trim_quantity: None, } } @@ -1666,8 +1661,6 @@ pub fn test_archive_controllers(ledger_wasm: Vec) { max_transactions_per_response: None, }, feature_flags: args.feature_flags, - maximum_number_of_accounts: args.maximum_number_of_accounts, - accounts_overflow_trim_quantity: args.accounts_overflow_trim_quantity, }) } @@ -1696,8 +1689,6 @@ pub fn test_archive_no_additional_controllers(ledger_wasm: Vec) { max_transactions_per_response: None, }, feature_flags: args.feature_flags, - maximum_number_of_accounts: args.maximum_number_of_accounts, - accounts_overflow_trim_quantity: args.accounts_overflow_trim_quantity, }) } @@ -1731,8 +1722,6 @@ pub fn test_archive_duplicate_controllers(ledger_wasm: Vec) { max_transactions_per_response: None, }, feature_flags: args.feature_flags, - maximum_number_of_accounts: args.maximum_number_of_accounts, - accounts_overflow_trim_quantity: args.accounts_overflow_trim_quantity, }) } let p100 = PrincipalId::new_user_test_id(100); @@ -2767,7 +2756,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); } - for i in 2..30 { + for i in 2..40 { let to = Account::from(PrincipalId::new_user_test_id(i).0); transfer(&env, canister_id, account, to, 100).expect("failed to transfer funds"); } @@ -3974,51 +3963,6 @@ where assert_eq!(total_supply(&env, canister_id), 60_000); } -pub fn test_balances_overflow(ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T) -where - T: CandidType, -{ - let env = StateMachine::new(); - - let args = encode_init_args(InitArgs { - maximum_number_of_accounts: Some(8), - accounts_overflow_trim_quantity: Some(2), - ..init_args(vec![]) - }); - let args = Encode!(&args).unwrap(); - let canister_id = env.install_canister(ledger_wasm, args, None).unwrap(); - - let minter = minting_account(&env, canister_id).unwrap(); - - let mut credited = 0; - for i in 0..11 { - transfer( - &env, - canister_id, - minter, - PrincipalId::new_user_test_id(i).0, - i, - ) - .expect("failed to mint tokens"); - credited += i; - } - assert_eq!( - balance_of(&env, canister_id, PrincipalId::new_user_test_id(1).0), - 0 - ); - assert_eq!( - balance_of(&env, canister_id, PrincipalId::new_user_test_id(2).0), - 0 - ); - for i in 3..11 { - assert_eq!( - balance_of(&env, canister_id, PrincipalId::new_user_test_id(i).0), - i - ); - } - assert_eq!(total_supply(&env, canister_id), credited - 1 - 2); -} - pub fn test_icrc1_test_suite( ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T, diff --git a/rs/nns/inspector/BUILD.bazel b/rs/nns/inspector/BUILD.bazel index 531724a2940..0ad7574ed85 100644 --- a/rs/nns/inspector/BUILD.bazel +++ b/rs/nns/inspector/BUILD.bazel @@ -4,7 +4,6 @@ package(default_visibility = ["//visibility:public"]) DEPENDENCIES = [ # Keep sorted. - "//rs/ledger_suite/common/ledger_canister_core", "//rs/ledger_suite/icp:icp_ledger", "//rs/ledger_suite/icp/ledger", "//rs/nns/constants", diff --git a/rs/nns/inspector/Cargo.toml b/rs/nns/inspector/Cargo.toml index 83ceb74fc7a..8d38ba9a6d9 100644 --- a/rs/nns/inspector/Cargo.toml +++ b/rs/nns/inspector/Cargo.toml @@ -9,7 +9,6 @@ clap = { workspace = true } csv = "1.1" hex = { workspace = true } ic-base-types = { path = "../../types/base_types" } -ic-ledger-canister-core = { path = "../../ledger_suite/common/ledger_canister_core" } ic-nns-constants = { path = "../constants" } ic-nns-governance-api = { path = "../governance/api" } ic-nns-gtc = { path = "../gtc" } diff --git a/rs/nns/inspector/src/main.rs b/rs/nns/inspector/src/main.rs index 8c77da9169a..ee12e062c8d 100644 --- a/rs/nns/inspector/src/main.rs +++ b/rs/nns/inspector/src/main.rs @@ -2,10 +2,9 @@ use clap::Parser; use ic_base_types::CanisterId; -use ic_ledger_canister_core::ledger::LedgerContext; use ic_nns_constants::{ CYCLES_MINTING_CANISTER_ID, GENESIS_TOKEN_CANISTER_ID, GOVERNANCE_CANISTER_ID, - LEDGER_CANISTER_ID, REGISTRY_CANISTER_ID, + REGISTRY_CANISTER_ID, }; use ic_nns_governance_api::pb::v1::{Governance as GovernanceProto, Neuron}; use ic_nns_gtc::pb::v1::Gtc as GtcProto; @@ -14,7 +13,7 @@ use prost::Message; use std::{ convert::TryInto, fs::File, - io::{Read, Write}, + io::Read, path::{Path, PathBuf}, string::ToString, }; @@ -71,13 +70,6 @@ fn main() { Err(e) => eprintln!("Could not extract the governance stable memory: {}", e), } - // Ledger. - // Stable memory = binary CBOR, type Ledger (rust struct). - match extract_stable_memory(&args, LEDGER_CANISTER_ID, "ledger_stable_memory.cbor") { - Ok(cbor) => decode_ledger_stable_memory(cbor, &args.output), - Err(e) => eprintln!("Could not extract the ledger stable memory: {}", e), - } - // Gtc. // Stable memory = binary proto, type Gtc. match extract_stable_memory(&args, GENESIS_TOKEN_CANISTER_ID, "gtc_stable_memory.pb") { @@ -279,69 +271,3 @@ fn decode_gtc_stable_memory(gtc_pb: PathBuf, output: &Path, rs: &Path) { } eprintln!("Wrote gtc_accounts_EXTRACT.csv"); } - -/// The type of record expected to be found in the ledger canister's stable -/// memory. -#[derive(serde::Serialize)] -struct LedgerBalanceRecord { - account_identitifier: String, - balance_e8s: u64, -} - -/// Decode stable memory for the ledger canister. -fn decode_ledger_stable_memory(cbor: PathBuf, output: &Path) { - // For the same argument as above, do NOT deserialize to a specific struct. - // This is an audit tool, we don't want any risk of dropping data. - // So go the schema-less way. - use serde_cbor::value::Value; - let value_res: serde_cbor::Result = - serde_cbor::from_reader(File::open(cbor.clone()).unwrap()); - let val = match value_res { - Err(e) => { - eprintln!( - "Could not parse the cbor for the ledger stable memory: {}", - e - ); - return; - } - Ok(v) => v, - }; - // The following is very slow and the output is very verbose. Serialization to - // json is not a viable alternative as some keys are not strings, which is - // allowed in cbor but not in json. - match write!( - File::create(output.join("ledger_stable_memory.txt")).unwrap(), - "{:#?}", - val - ) { - Ok(()) => eprintln!("Wrote ledger_stable_memory.txt"), - Err(e) => eprintln!("Could not write ledger_stable_memory.txt: {}", e), - }; - - let ledger: ledger_canister::Ledger = match serde_cbor::from_reader(File::open(cbor).unwrap()) { - Err(e) => { - eprintln!( - "Could parse the ledger stable memory as a Ledger struct: {}", - e - ); - return; - } - Ok(l) => l, - }; - let mut records: Vec = ledger - .balances() - .store - .iter() - .map(|(key, icpts)| LedgerBalanceRecord { - account_identitifier: key.to_string(), - balance_e8s: icpts.get_e8s(), - }) - .collect(); - records.sort_by_key(|record| record.account_identitifier.clone()); - let mut csv_writer = - csv::Writer::from_path(output.join("ledger_balances_EXTRACT.csv")).unwrap(); - for r in records.into_iter() { - csv_writer.serialize(r).unwrap(); - } - eprintln!("Wrote ledger_balances_EXTRACT.csv"); -}