Moves create_all_accounts_run_and_snapshot_dirs() into accounts-db utils

accounts-db/src/accounts_db.rs

@@ -68,7 +68,7 @@ use {
         rent_collector::RentCollector,
         sorted_storages::SortedStorages,
         storable_accounts::StorableAccounts,
-        u64_align,
+        u64_align, utils,
         verify_accounts_hash_in_background::VerifyAccountsHashInBackground,
     },
     blake3::traits::digest::Digest,
@@ -1197,90 +1197,15 @@ impl AccountStorageEntry {
     }
 }
 
-/// To allow generating a bank snapshot directory with full state information, we need to
-/// hardlink account appendvec files from the runtime operation directory to a snapshot
-/// hardlink directory.  This is to create the run/ and snapshot sub directories for an
-/// account_path provided by the user.  These two sub directories are on the same file
-/// system partition to allow hard-linking.
-pub fn create_accounts_run_and_snapshot_dirs(
-    account_dir: impl AsRef<Path>,
-) -> std::io::Result<(PathBuf, PathBuf)> {
-    let run_path = account_dir.as_ref().join("run");
-    let snapshot_path = account_dir.as_ref().join("snapshot");
-    if (!run_path.is_dir()) || (!snapshot_path.is_dir()) {
-        // If the "run/" or "snapshot" sub directories do not exist, the directory may be from
-        // an older version for which the appendvec files are at this directory.  Clean up
-        // them first.
-        // This will be done only once when transitioning from an old image without run directory
-        // to this new version using run and snapshot directories.
-        // The run/ content cleanup will be done at a later point.  The snapshot/ content persists
-        // across the process boot, and will be purged by the account_background_service.
-        if fs::remove_dir_all(&account_dir).is_err() {
-            delete_contents_of_path(&account_dir);
-        }
-        fs::create_dir_all(&run_path)?;
-        fs::create_dir_all(&snapshot_path)?;
-    }
-
-    Ok((run_path, snapshot_path))
-}
-
-/// For all account_paths, create the run/ and snapshot/ sub directories.
-/// If an account_path directory does not exist, create it.
-/// It returns (account_run_paths, account_snapshot_paths) or error
-pub fn create_all_accounts_run_and_snapshot_dirs(
-    account_paths: &[PathBuf],
-) -> std::io::Result<(Vec<PathBuf>, Vec<PathBuf>)> {
-    let mut run_dirs = Vec::with_capacity(account_paths.len());
-    let mut snapshot_dirs = Vec::with_capacity(account_paths.len());
-    for account_path in account_paths {
-        // create the run/ and snapshot/ sub directories for each account_path
-        let (run_dir, snapshot_dir) = create_accounts_run_and_snapshot_dirs(account_path)?;
-        run_dirs.push(run_dir);
-        snapshot_dirs.push(snapshot_dir);
-    }
-    Ok((run_dirs, snapshot_dirs))
-}
-
-/// Delete the files and subdirectories in a directory.
-/// This is useful if the process does not have permission
-/// to delete the top level directory it might be able to
-/// delete the contents of that directory.
-pub fn delete_contents_of_path(path: impl AsRef<Path>) {
-    match fs::read_dir(&path) {
-        Err(err) => {
-            warn!(
-                "Failed to delete contents of '{}': could not read dir: {err}",
-                path.as_ref().display(),
-            )
-        }
-        Ok(dir_entries) => {
-            for entry in dir_entries.flatten() {
-                let sub_path = entry.path();
-                let result = if sub_path.is_dir() {
-                    fs::remove_dir_all(&sub_path)
-                } else {
-                    fs::remove_file(&sub_path)
-                };
-                if let Err(err) = result {
-                    warn!(
-                        "Failed to delete contents of '{}': {err}",
-                        sub_path.display(),
-                    );
-                }
-            }
-        }
-    }
-}
-
 pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec<TempDir>, Vec<PathBuf>)> {
     let temp_dirs: IoResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
     let temp_dirs = temp_dirs?;
 
     let paths: IoResult<Vec<_>> = temp_dirs
         .iter()
         .map(|temp_dir| {
-            create_accounts_run_and_snapshot_dirs(temp_dir).map(|(run_dir, _snapshot_dir)| run_dir)
+            utils::create_accounts_run_and_snapshot_dirs(temp_dir)
+                .map(|(run_dir, _snapshot_dir)| run_dir)
         })
         .collect();
     let paths = paths?;

accounts-db/src/lib.rs

@@ -46,6 +46,7 @@ pub mod storable_accounts;
 pub mod tiered_storage;
 pub mod transaction_error_metrics;
 pub mod transaction_results;
+pub mod utils;
 mod verify_accounts_hash_in_background;
 pub mod waitable_condvar;
 

accounts-db/src/utils.rs

@@ -0,0 +1,119 @@
+use {
+    log::*,
+    std::{
+        fs,
+        path::{Path, PathBuf},
+    },
+};
+
+pub const ACCOUNTS_RUN_DIR: &str = "run";
+pub const ACCOUNTS_SNAPSHOT_DIR: &str = "snapshot";
+
+/// For all account_paths, create the run/ and snapshot/ sub directories.
+/// If an account_path directory does not exist, create it.
+/// It returns (account_run_paths, account_snapshot_paths) or error
+pub fn create_all_accounts_run_and_snapshot_dirs(
+    account_paths: &[PathBuf],
+) -> std::io::Result<(Vec<PathBuf>, Vec<PathBuf>)> {
+    let mut run_dirs = Vec::with_capacity(account_paths.len());
+    let mut snapshot_dirs = Vec::with_capacity(account_paths.len());
+    for account_path in account_paths {
+        // create the run/ and snapshot/ sub directories for each account_path
+        let (run_dir, snapshot_dir) = create_accounts_run_and_snapshot_dirs(account_path)?;
+        run_dirs.push(run_dir);
+        snapshot_dirs.push(snapshot_dir);
+    }
+    Ok((run_dirs, snapshot_dirs))
+}
+
+/// To allow generating a bank snapshot directory with full state information, we need to
+/// hardlink account appendvec files from the runtime operation directory to a snapshot
+/// hardlink directory.  This is to create the run/ and snapshot sub directories for an
+/// account_path provided by the user.  These two sub directories are on the same file
+/// system partition to allow hard-linking.
+pub fn create_accounts_run_and_snapshot_dirs(
+    account_dir: impl AsRef<Path>,
+) -> std::io::Result<(PathBuf, PathBuf)> {
+    let run_path = account_dir.as_ref().join(ACCOUNTS_RUN_DIR);
+    let snapshot_path = account_dir.as_ref().join(ACCOUNTS_SNAPSHOT_DIR);
+    if (!run_path.is_dir()) || (!snapshot_path.is_dir()) {
+        // If the "run/" or "snapshot" sub directories do not exist, the directory may be from
+        // an older version for which the appendvec files are at this directory.  Clean up
+        // them first.
+        // This will be done only once when transitioning from an old image without run directory
+        // to this new version using run and snapshot directories.
+        // The run/ content cleanup will be done at a later point.  The snapshot/ content persists
+        // across the process boot, and will be purged by the account_background_service.
+        if fs::remove_dir_all(&account_dir).is_err() {
+            delete_contents_of_path(&account_dir);
+        }
+        fs::create_dir_all(&run_path)?;
+        fs::create_dir_all(&snapshot_path)?;
+    }
+
+    Ok((run_path, snapshot_path))
+}
+
+/// Delete the files and subdirectories in a directory.
+/// This is useful if the process does not have permission
+/// to delete the top level directory it might be able to
+/// delete the contents of that directory.
+pub fn delete_contents_of_path(path: impl AsRef<Path>) {
+    match fs::read_dir(&path) {
+        Err(err) => {
+            warn!(
+                "Failed to delete contents of '{}': could not read dir: {err}",
+                path.as_ref().display(),
+            )
+        }
+        Ok(dir_entries) => {
+            for entry in dir_entries.flatten() {
+                let sub_path = entry.path();
+                let result = if sub_path.is_dir() {
+                    fs::remove_dir_all(&sub_path)
+                } else {
+                    fs::remove_file(&sub_path)
+                };
+                if let Err(err) = result {
+                    warn!(
+                        "Failed to delete contents of '{}': {err}",
+                        sub_path.display(),
+                    );
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use {super::*, tempfile::TempDir};
+
+    #[test]
+    pub fn test_create_all_accounts_run_and_snapshot_dirs() {
+        let (_tmp_dirs, account_paths): (Vec<TempDir>, Vec<PathBuf>) = (0..4)
+            .map(|_| {
+                let tmp_dir = tempfile::TempDir::new().unwrap();
+                let account_path = tmp_dir.path().join("accounts");
+                (tmp_dir, account_path)
+            })
+            .unzip();
+
+        // create the `run/` and `snapshot/` dirs, and ensure they're there
+        let (account_run_paths, account_snapshot_paths) =
+            create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap();
+        account_run_paths.iter().all(|path| path.is_dir());
+        account_snapshot_paths.iter().all(|path| path.is_dir());
+
+        // delete a `run/` and `snapshot/` dir, then re-create it
+        let account_path_first = account_paths.first().unwrap();
+        delete_contents_of_path(account_path_first);
+        assert!(account_path_first.exists());
+        assert!(!account_path_first.join(ACCOUNTS_RUN_DIR).exists());
+        assert!(!account_path_first.join(ACCOUNTS_SNAPSHOT_DIR).exists());
+
+        _ = create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap();
+        account_run_paths.iter().all(|path| path.is_dir());
+        account_snapshot_paths.iter().all(|path| path.is_dir());
+    }
+}

ledger-tool/src/ledger_utils.rs

@@ -3,7 +3,9 @@ use {
     clap::{value_t, value_t_or_exit, values_t_or_exit, ArgMatches},
     crossbeam_channel::unbounded,
     log::*,
-    solana_accounts_db::hardened_unpack::open_genesis_config,
+    solana_accounts_db::{
+        hardened_unpack::open_genesis_config, utils::create_all_accounts_run_and_snapshot_dirs,
+    },
     solana_core::{
         accounts_hash_verifier::AccountsHashVerifier, validator::BlockVerificationMethod,
     },
@@ -35,8 +37,8 @@ use {
         snapshot_config::SnapshotConfig,
         snapshot_hash::StartingSnapshotHashes,
         snapshot_utils::{
-            self, clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs,
-            move_and_async_delete_path_contents, SnapshotError,
+            self, clean_orphaned_account_snapshot_dirs, move_and_async_delete_path_contents,
+            SnapshotError,
         },
     },
     solana_sdk::{
@@ -68,7 +70,7 @@ pub(crate) enum LoadAndProcessLedgerError {
     CleanOrphanedAccountSnapshotDirectories(#[source] SnapshotError),
 
     #[error("failed to create all run and snapshot directories: {0}")]
-    CreateAllAccountsRunAndSnapshotDirectories(#[source] SnapshotError),
+    CreateAllAccountsRunAndSnapshotDirectories(#[source] std::io::Error),
 
     #[error("custom accounts path is not supported with seconday blockstore access")]
     CustomAccountsPathUnsupported(#[source] BlockstoreError),

local-cluster/src/integration_tests.rs

@@ -17,7 +17,7 @@ use {
         validator_configs::*,
     },
     log::*,
-    solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs,
+    solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs,
     solana_core::{
         consensus::{tower_storage::FileTowerStorage, Tower, SWITCH_FORK_THRESHOLD},
         validator::{is_snapshot_config_valid, ValidatorConfig},

local-cluster/src/local_cluster.rs

@@ -6,7 +6,7 @@ use {
     },
     itertools::izip,
     log::*,
-    solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs,
+    solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs,
     solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient},
     solana_core::{
         consensus::tower_storage::FileTowerStorage,

local-cluster/tests/local_cluster.rs

@@ -7,7 +7,7 @@ use {
     rand::seq::IteratorRandom,
     serial_test::serial,
     solana_accounts_db::{
-        accounts_db::create_accounts_run_and_snapshot_dirs, hardened_unpack::open_genesis_config,
+        hardened_unpack::open_genesis_config, utils::create_accounts_run_and_snapshot_dirs,
     },
     solana_client::thin_client::ThinClient,
     solana_core::{

runtime/src/snapshot_bank_utils.rs

@@ -13,11 +13,10 @@ use {
         snapshot_hash::SnapshotHash,
         snapshot_package::{AccountsPackage, AccountsPackageKind, SnapshotKind, SnapshotPackage},
         snapshot_utils::{
-            self, archive_snapshot_package, delete_contents_of_path,
-            deserialize_snapshot_data_file, deserialize_snapshot_data_files, get_bank_snapshot_dir,
-            get_highest_bank_snapshot_post, get_highest_full_snapshot_archive_info,
-            get_highest_incremental_snapshot_archive_info, get_snapshot_file_name,
-            get_storages_to_serialize, hard_link_storages_to_snapshot,
+            self, archive_snapshot_package, deserialize_snapshot_data_file,
+            deserialize_snapshot_data_files, get_bank_snapshot_dir, get_highest_bank_snapshot_post,
+            get_highest_full_snapshot_archive_info, get_highest_incremental_snapshot_archive_info,
+            get_snapshot_file_name, get_storages_to_serialize, hard_link_storages_to_snapshot,
             rebuild_storages_from_snapshot_dir, serialize_snapshot_data_file,
             verify_and_unarchive_snapshots, verify_unpacked_snapshots_dir_and_version,
             AddBankSnapshotError, ArchiveFormat, BankSnapshotInfo, BankSnapshotType, SnapshotError,
@@ -36,6 +35,7 @@ use {
         accounts_hash::AccountsHash,
         accounts_index::AccountSecondaryIndexes,
         accounts_update_notifier_interface::AccountsUpdateNotifier,
+        utils::delete_contents_of_path,
     },
     solana_measure::{measure, measure::Measure},
     solana_sdk::{
@@ -1259,9 +1259,9 @@ mod tests {
             bank_forks::BankForks,
             genesis_utils,
             snapshot_utils::{
-                clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs,
-                create_tmp_accounts_dir_for_tests, get_bank_snapshots, get_bank_snapshots_post,
-                get_bank_snapshots_pre, get_highest_bank_snapshot, purge_bank_snapshot,
+                clean_orphaned_account_snapshot_dirs, create_tmp_accounts_dir_for_tests,
+                get_bank_snapshots, get_bank_snapshots_post, get_bank_snapshots_pre,
+                get_highest_bank_snapshot, purge_bank_snapshot,
                 purge_bank_snapshots_older_than_slot, purge_incomplete_bank_snapshots,
                 purge_old_bank_snapshots, purge_old_bank_snapshots_at_startup,
                 snapshot_storage_rebuilder::get_slot_and_append_vec_id, ArchiveFormat,
@@ -2091,34 +2091,6 @@ mod tests {
         assert_eq!(snapshot.slot, 1);
     }
 
-    #[test]
-    pub fn test_create_all_accounts_run_and_snapshot_dirs() {
-        let (_tmp_dirs, account_paths): (Vec<TempDir>, Vec<PathBuf>) = (0..4)
-            .map(|_| {
-                let tmp_dir = tempfile::TempDir::new().unwrap();
-                let account_path = tmp_dir.path().join("accounts");
-                (tmp_dir, account_path)
-            })
-            .unzip();
-
-        // create the `run/` and `snapshot/` dirs, and ensure they're there
-        let (account_run_paths, account_snapshot_paths) =
-            create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap();
-        account_run_paths.iter().all(|path| path.is_dir());
-        account_snapshot_paths.iter().all(|path| path.is_dir());
-
-        // delete a `run/` and `snapshot/` dir, then re-create it
-        let account_path_first = account_paths.first().unwrap();
-        delete_contents_of_path(account_path_first);
-        assert!(account_path_first.exists());
-        assert!(!account_path_first.join("run").exists());
-        assert!(!account_path_first.join("snapshot").exists());
-
-        _ = create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap();
-        account_run_paths.iter().all(|path| path.is_dir());
-        account_snapshot_paths.iter().all(|path| path.is_dir());
-    }
-
     #[test]
     fn test_clean_orphaned_account_snapshot_dirs() {
         let genesis_config = GenesisConfig::default();