footer.rs

use {
    crate::tiered_storage::{
        error::TieredStorageError, file::TieredStorageFile, index::IndexBlockFormat,
        mmap_utils::get_type, TieredStorageResult as TsResult,
    },
    memmap2::Mmap,
    solana_sdk::{hash::Hash, pubkey::Pubkey},
    std::{mem, path::Path},
};

pub const FOOTER_FORMAT_VERSION: u64 = 1;

/// The size of the footer struct + the magic number at the end.
pub const FOOTER_SIZE: usize =
    mem::size_of::<TieredStorageFooter>() + mem::size_of::<TieredStorageMagicNumber>();
static_assertions::const_assert_eq!(mem::size_of::<TieredStorageFooter>(), 160);

/// The size of the ending part of the footer.  This size should remain unchanged
/// even when the footer's format changes.
pub const FOOTER_TAIL_SIZE: usize = 24;

/// The ending 8 bytes of a valid tiered account storage file.
pub const FOOTER_MAGIC_NUMBER: u64 = 0x502A2AB5; // SOLALABS -> SOLANA LABS

#[derive(Debug, PartialEq, Eq)]
#[repr(C)]
pub struct TieredStorageMagicNumber(pub u64);

impl Default for TieredStorageMagicNumber {
    fn default() -> Self {
        Self(FOOTER_MAGIC_NUMBER)
    }
}

#[repr(u16)]
#[derive(
    Clone,
    Copy,
    Debug,
    Default,
    Eq,
    Hash,
    PartialEq,
    num_enum::IntoPrimitive,
    num_enum::TryFromPrimitive,
)]
pub enum AccountMetaFormat {
    #[default]
    Hot = 0,
    // Temporarily comment out to avoid unimplemented!() block
    // Cold = 1,
}

#[repr(u16)]
#[derive(
    Clone,
    Copy,
    Debug,
    Default,
    Eq,
    Hash,
    PartialEq,
    num_enum::IntoPrimitive,
    num_enum::TryFromPrimitive,
)]
pub enum AccountBlockFormat {
    #[default]
    AlignedRaw = 0,
    Lz4 = 1,
}

#[repr(u16)]
#[derive(
    Clone,
    Copy,
    Debug,
    Default,
    Eq,
    Hash,
    PartialEq,
    num_enum::IntoPrimitive,
    num_enum::TryFromPrimitive,
)]
pub enum OwnersBlockFormat {
    #[default]
    LocalIndex = 0,
}

#[derive(Debug, PartialEq, Eq, Clone)]
#[repr(C)]
pub struct TieredStorageFooter {
    // formats
    /// The format of the account meta entry.
    pub account_meta_format: AccountMetaFormat,
    /// The format of the owners block.
    pub owners_block_format: OwnersBlockFormat,
    /// The format of the account index block.
    pub index_block_format: IndexBlockFormat,
    /// The format of the account block.
    pub account_block_format: AccountBlockFormat,

    // Account-block related
    /// The number of account entries.
    pub account_entry_count: u32,
    /// The size of each account meta entry in bytes.
    pub account_meta_entry_size: u32,
    /// The default size of an account block before compression.
    ///
    /// If the size of one account (meta + data + optional fields) before
    /// compression is bigger than this number, than it is considered a
    /// blob account and it will have its own account block.
    pub account_block_size: u64,

    // Owner-related
    /// The number of owners.
    pub owner_count: u32,
    /// The size of each owner entry.
    pub owner_entry_size: u32,

    // Offsets
    // Note that offset to the account blocks is omitted as it's always 0.
    /// The offset pointing to the first byte of the account index block.
    pub index_block_offset: u64,
    /// The offset pointing to the first byte of the owners block.
    pub owners_block_offset: u64,

    // account range
    /// The smallest account address in this file.
    pub min_account_address: Pubkey,
    /// The largest account address in this file.
    pub max_account_address: Pubkey,

    /// A hash that represents a tiered accounts file for consistency check.
    pub hash: Hash,

    // The below fields belong to footer tail.
    // The sum of their sizes should match FOOTER_TAIL_SIZE.
    /// The size of the footer including the magic number.
    pub footer_size: u64,
    /// The format version of the tiered accounts file.
    pub format_version: u64,
    // This field is persisted in the storage but not in this struct.
    // The number should match FOOTER_MAGIC_NUMBER.
    // pub magic_number: u64,
}

impl Default for TieredStorageFooter {
    fn default() -> Self {
        Self {
            account_meta_format: AccountMetaFormat::default(),
            owners_block_format: OwnersBlockFormat::default(),
            index_block_format: IndexBlockFormat::default(),
            account_block_format: AccountBlockFormat::default(),
            account_entry_count: 0,
            account_meta_entry_size: 0,
            account_block_size: 0,
            owner_count: 0,
            owner_entry_size: 0,
            index_block_offset: 0,
            owners_block_offset: 0,
            hash: Hash::new_unique(),
            min_account_address: Pubkey::default(),
            max_account_address: Pubkey::default(),
            footer_size: FOOTER_SIZE as u64,
            format_version: FOOTER_FORMAT_VERSION,
        }
    }
}

impl TieredStorageFooter {
    pub fn new_from_path(path: impl AsRef<Path>) -> TsResult<Self> {
        let file = TieredStorageFile::new_readonly(path);
        Self::new_from_footer_block(&file)
    }

    pub fn write_footer_block(&self, file: &TieredStorageFile) -> TsResult<()> {
        file.write_type(self)?;
        file.write_type(&TieredStorageMagicNumber::default())?;

        Ok(())
    }

    pub fn new_from_footer_block(file: &TieredStorageFile) -> TsResult<Self> {
        let mut footer_size: u64 = 0;
        let mut footer_version: u64 = 0;
        let mut magic_number = TieredStorageMagicNumber(0);

        file.seek_from_end(-(FOOTER_TAIL_SIZE as i64))?;
        file.read_type(&mut footer_size)?;
        file.read_type(&mut footer_version)?;
        file.read_type(&mut magic_number)?;

        if magic_number != TieredStorageMagicNumber::default() {
            return Err(TieredStorageError::MagicNumberMismatch(
                TieredStorageMagicNumber::default().0,
                magic_number.0,
            ));
        }

        let mut footer = Self::default();
        file.seek_from_end(-(footer_size as i64))?;
        file.read_type(&mut footer)?;

        Ok(footer)
    }

    pub fn new_from_mmap(map: &Mmap) -> TsResult<&TieredStorageFooter> {
        let offset = map.len().saturating_sub(FOOTER_TAIL_SIZE);
        let (footer_size, offset) = get_type::<u64>(map, offset)?;
        let (_footer_version, offset) = get_type::<u64>(map, offset)?;
        let (magic_number, _offset) = get_type::<TieredStorageMagicNumber>(map, offset)?;

        if *magic_number != TieredStorageMagicNumber::default() {
            return Err(TieredStorageError::MagicNumberMismatch(
                TieredStorageMagicNumber::default().0,
                magic_number.0,
            ));
        }

        let (footer, _offset) =
            get_type::<TieredStorageFooter>(map, map.len().saturating_sub(*footer_size as usize))?;

        Ok(footer)
    }
}

#[cfg(test)]
mod tests {
    use {
        super::*,
        crate::{
            append_vec::test_utils::get_append_vec_path, tiered_storage::file::TieredStorageFile,
        },
        memoffset::offset_of,
        solana_sdk::hash::Hash,
    };

    #[test]
    fn test_footer() {
        let path = get_append_vec_path("test_file_footer");
        let expected_footer = TieredStorageFooter {
            account_meta_format: AccountMetaFormat::Hot,
            owners_block_format: OwnersBlockFormat::LocalIndex,
            index_block_format: IndexBlockFormat::AddressAndOffset,
            account_block_format: AccountBlockFormat::AlignedRaw,
            account_entry_count: 300,
            account_meta_entry_size: 24,
            account_block_size: 4096,
            owner_count: 250,
            owner_entry_size: 32,
            index_block_offset: 1069600,
            owners_block_offset: 1081200,
            hash: Hash::new_unique(),
            min_account_address: Pubkey::default(),
            max_account_address: Pubkey::new_unique(),
            footer_size: FOOTER_SIZE as u64,
            format_version: FOOTER_FORMAT_VERSION,
        };

        // Persist the expected footer.
        {
            let file = TieredStorageFile::new_writable(&path.path).unwrap();
            expected_footer.write_footer_block(&file).unwrap();
        }

        // Reopen the same storage, and expect the persisted footer is
        // the same as what we have written.
        {
            let footer = TieredStorageFooter::new_from_path(&path.path).unwrap();
            assert_eq!(expected_footer, footer);
        }
    }

    #[test]
    fn test_footer_layout() {
        assert_eq!(offset_of!(TieredStorageFooter, account_meta_format), 0x00);
        assert_eq!(offset_of!(TieredStorageFooter, owners_block_format), 0x02);
        assert_eq!(offset_of!(TieredStorageFooter, index_block_format), 0x04);
        assert_eq!(offset_of!(TieredStorageFooter, account_block_format), 0x06);
        assert_eq!(offset_of!(TieredStorageFooter, account_entry_count), 0x08);
        assert_eq!(
            offset_of!(TieredStorageFooter, account_meta_entry_size),
            0x0C
        );
        assert_eq!(offset_of!(TieredStorageFooter, account_block_size), 0x10);
        assert_eq!(offset_of!(TieredStorageFooter, owner_count), 0x18);
        assert_eq!(offset_of!(TieredStorageFooter, owner_entry_size), 0x1C);
        assert_eq!(offset_of!(TieredStorageFooter, index_block_offset), 0x20);
        assert_eq!(offset_of!(TieredStorageFooter, owners_block_offset), 0x28);
        assert_eq!(offset_of!(TieredStorageFooter, min_account_address), 0x30);
        assert_eq!(offset_of!(TieredStorageFooter, max_account_address), 0x50);
        assert_eq!(offset_of!(TieredStorageFooter, hash), 0x70);
        assert_eq!(offset_of!(TieredStorageFooter, footer_size), 0x90);
        assert_eq!(offset_of!(TieredStorageFooter, format_version), 0x98);
    }
}