From ecb1f8a9d739213cca027660d74b3b090c0d5d3e Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:54:02 -0700 Subject: [PATCH] [TieredStorage] Include Hot Account in StoredAccountMeta and ReadableAccount (#33544) #### Problem All account storage formats are required to implement both StoredAccountMeta and ReadableAccount, but the implementation for the hot account format is missing. #### Summary of Changes This PR includes hot account format into StoredAccountMeta and ReadableAccount enum. This will allow the TieredStorageReader in the future PRs to return hot account format in its `get_account` implementation. --- accounts-db/src/account_storage/meta.rs | 31 ++++++++++++++++++++++++- accounts-db/src/append_vec.rs | 18 ++++++++++---- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs index dba672292310f5..21b117c0a15d3a 100644 --- a/accounts-db/src/account_storage/meta.rs +++ b/accounts-db/src/account_storage/meta.rs @@ -1,5 +1,9 @@ use { - crate::{append_vec::AppendVecStoredAccountMeta, storable_accounts::StorableAccounts}, + crate::{ + append_vec::AppendVecStoredAccountMeta, + storable_accounts::StorableAccounts, + tiered_storage::{hot::HotAccountMeta, readable::TieredReadableAccount}, + }, solana_sdk::{account::ReadableAccount, hash::Hash, pubkey::Pubkey, stake_history::Epoch}, std::{borrow::Borrow, marker::PhantomData}, }; @@ -12,6 +16,10 @@ pub struct StoredAccountInfo { pub size: usize, } +lazy_static! { + static ref DEFAULT_ACCOUNT_HASH: Hash = Hash::default(); +} + /// Goal is to eliminate copies and data reshaping given various code paths that store accounts. /// This struct contains what is needed to store accounts to a storage /// 1. account & pubkey (StorableAccounts) @@ -100,66 +108,82 @@ impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: #[derive(PartialEq, Eq, Debug)] pub enum StoredAccountMeta<'storage> { AppendVec(AppendVecStoredAccountMeta<'storage>), + Hot(TieredReadableAccount<'storage, HotAccountMeta>), } impl<'storage> StoredAccountMeta<'storage> { pub fn pubkey(&self) -> &'storage Pubkey { match self { Self::AppendVec(av) => av.pubkey(), + Self::Hot(hot) => hot.address(), } } pub fn hash(&self) -> &'storage Hash { match self { Self::AppendVec(av) => av.hash(), + Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH), } } pub fn stored_size(&self) -> usize { match self { Self::AppendVec(av) => av.stored_size(), + Self::Hot(_) => unimplemented!(), } } pub fn offset(&self) -> usize { match self { Self::AppendVec(av) => av.offset(), + Self::Hot(hot) => hot.index(), } } pub fn data(&self) -> &'storage [u8] { match self { Self::AppendVec(av) => av.data(), + Self::Hot(hot) => hot.data(), } } pub fn data_len(&self) -> u64 { match self { Self::AppendVec(av) => av.data_len(), + Self::Hot(hot) => hot.data().len() as u64, } } pub fn write_version(&self) -> StoredMetaWriteVersion { match self { Self::AppendVec(av) => av.write_version(), + Self::Hot(hot) => hot.write_version().unwrap_or_default(), } } pub fn meta(&self) -> &StoredMeta { match self { Self::AppendVec(av) => av.meta(), + // Hot account does not support this API as it does not + // use the same in-memory layout as StoredMeta. + Self::Hot(_) => unreachable!(), } } pub fn set_meta(&mut self, meta: &'storage StoredMeta) { match self { Self::AppendVec(av) => av.set_meta(meta), + // Hot account does not support this API as it does not + // use the same in-memory layout as StoredMeta. + Self::Hot(_) => unreachable!(), } } pub(crate) fn sanitize(&self) -> bool { match self { Self::AppendVec(av) => av.sanitize(), + // Hot account currently doesn't have the concept of sanitization. + Self::Hot(_) => unimplemented!(), } } } @@ -168,26 +192,31 @@ impl<'storage> ReadableAccount for StoredAccountMeta<'storage> { fn lamports(&self) -> u64 { match self { Self::AppendVec(av) => av.lamports(), + Self::Hot(hot) => hot.lamports(), } } fn data(&self) -> &[u8] { match self { Self::AppendVec(av) => av.data(), + Self::Hot(hot) => hot.data(), } } fn owner(&self) -> &Pubkey { match self { Self::AppendVec(av) => av.owner(), + Self::Hot(hot) => hot.owner(), } } fn executable(&self) -> bool { match self { Self::AppendVec(av) => av.executable(), + Self::Hot(hot) => hot.executable(), } } fn rent_epoch(&self) -> Epoch { match self { Self::AppendVec(av) => av.rent_epoch(), + Self::Hot(hot) => hot.rent_epoch(), } } } diff --git a/accounts-db/src/append_vec.rs b/accounts-db/src/append_vec.rs index 941c0a9afe298b..fce45672f2a9bd 100644 --- a/accounts-db/src/append_vec.rs +++ b/accounts-db/src/append_vec.rs @@ -686,6 +686,8 @@ pub mod tests { pub(crate) fn ref_executable_byte(&self) -> &u8 { match self { Self::AppendVec(av) => av.ref_executable_byte(), + // Tests currently only cover AppendVec. + Self::Hot(_) => unreachable!(), } } } @@ -1181,7 +1183,9 @@ pub mod tests { av.append_account_test(&create_test_account(10)).unwrap(); let accounts = av.accounts(0); - let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap(); + let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap() else { + panic!("StoredAccountMeta can only be AppendVec in this test."); + }; account.set_data_len_unsafe(crafted_data_len); assert_eq!(account.data_len(), crafted_data_len); @@ -1209,7 +1213,9 @@ pub mod tests { av.append_account_test(&create_test_account(10)).unwrap(); let accounts = av.accounts(0); - let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap(); + let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap() else { + panic!("StoredAccountMeta can only be AppendVec in this test."); + }; account.set_data_len_unsafe(too_large_data_len); assert_eq!(account.data_len(), too_large_data_len); @@ -1245,14 +1251,18 @@ pub mod tests { assert_eq!(*accounts[0].ref_executable_byte(), 0); assert_eq!(*accounts[1].ref_executable_byte(), 1); - let StoredAccountMeta::AppendVec(account) = &accounts[0]; + let StoredAccountMeta::AppendVec(account) = &accounts[0] else { + panic!("StoredAccountMeta can only be AppendVec in this test."); + }; let crafted_executable = u8::max_value() - 1; account.set_executable_as_byte(crafted_executable); // reload crafted accounts let accounts = av.accounts(0); - let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap(); + let StoredAccountMeta::AppendVec(account) = accounts.first().unwrap() else { + panic!("StoredAccountMeta can only be AppendVec in this test."); + }; // upper 7-bits are not 0, so sanitization should fail assert!(!account.sanitize_executable());