From 24fb6fca43629b34c74cc3b1d56c0f30d2a503ae Mon Sep 17 00:00:00 2001 From: Ryan Quinn Ford Date: Thu, 16 Jan 2025 09:26:44 +0100 Subject: [PATCH 1/4] setting rocksdb as default --- Cargo.lock | 1 + crates/cli/src/cfg.rs | 22 ++++++++++-------- crates/cli/src/main.rs | 42 ++++++++++++++++------------------- crates/storage/Cargo.toml | 1 + crates/storage/src/rocksdb.rs | 32 ++++++++++++++++++++++++-- crates/tests/src/lib.rs | 8 +++++-- 6 files changed, 70 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a220b44..d1d75065 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4761,6 +4761,7 @@ version = "0.1.0" dependencies = [ "anyhow", "auto_impl", + "dirs 5.0.1", "jmt", "log", "prism-common", diff --git a/crates/cli/src/cfg.rs b/crates/cli/src/cfg.rs index afe153d7..9bfe7b20 100644 --- a/crates/cli/src/cfg.rs +++ b/crates/cli/src/cfg.rs @@ -6,7 +6,7 @@ use dotenvy::dotenv; use log::{error, warn}; use prism_errors::{DataAvailabilityError, GeneralError}; use prism_prover::webserver::WebServerConfig; -use prism_storage::redis::RedisConfig; +use prism_storage::rocksdb::RocksDBConfig; use serde::{Deserialize, Serialize}; use std::{fs, path::Path, sync::Arc}; @@ -31,7 +31,7 @@ pub struct CommandArgs { log_level: String, #[arg(short = 'r', long)] - redis_client: Option, + rocksdb_path: Option, #[arg(long)] verifying_key: Option, @@ -101,7 +101,7 @@ pub struct Config { #[serde(skip_serializing_if = "Option::is_none")] pub celestia_config: Option, pub da_layer: DALayerOption, - pub redis_config: Option, + pub rocksdb_config: Option, pub verifying_key: Option, pub verifying_key_algorithm: String, } @@ -112,7 +112,7 @@ impl Default for Config { webserver: Some(WebServerConfig::default()), celestia_config: Some(CelestiaConfig::default()), da_layer: DALayerOption::default(), - redis_config: Some(RedisConfig::default()), + rocksdb_config: Some(RocksDBConfig::default()), verifying_key: None, verifying_key_algorithm: "ed25519".to_string(), } @@ -151,15 +151,19 @@ pub fn load_config(args: CommandArgs) -> Result { Ok(final_config) } -fn get_config_path(args: &CommandArgs) -> Result { +fn get_prism_home(args: &CommandArgs) -> Result { args.config_path .clone() - .or_else(|| home_dir().map(|path| format!("{}/.prism/config.toml", path.to_string_lossy()))) + .or_else(|| home_dir().map(|path| format!("{}/.prism/", path.to_string_lossy()))) .ok_or_else(|| { GeneralError::MissingArgumentError("could not determine config path".to_string()).into() }) } +fn get_config_path(args: &CommandArgs) -> Result { + get_prism_home(args).map(|path| format!("{}/config.toml", path)) +} + fn ensure_config_file_exists(config_path: &str) -> Result<()> { if !Path::new(config_path).exists() { if let Some(parent) = Path::new(config_path).parent() { @@ -177,7 +181,7 @@ fn ensure_config_file_exists(config_path: &str) -> Result<()> { fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { let webserver_config = &config.webserver.unwrap_or_default(); - let redis_config = &config.redis_config.unwrap_or_default(); + let rocksdb_config = &config.rocksdb_config.unwrap_or_default(); let celestia_config = &config.celestia_config.unwrap_or_default(); Config { @@ -186,8 +190,8 @@ fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { host: args.webserver.host.unwrap_or(webserver_config.host.clone()), port: args.webserver.port.unwrap_or(webserver_config.port), }), - redis_config: Some(RedisConfig { - connection_string: args.redis_client.unwrap_or(redis_config.connection_string.clone()), + rocksdb_config: Some(RocksDBConfig { + path: args.rocksdb_path.unwrap_or(rocksdb_config.path.clone()), }), celestia_config: Some(CelestiaConfig { connection_string: args diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d4f6af70..f6210d88 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -11,7 +11,7 @@ use std::io::{Error, ErrorKind}; use node_types::NodeType; use prism_lightclient::LightClient; use prism_prover::Prover; -use prism_storage::RedisConnection; +use prism_storage::rocksdb::RocksDBConnection; use std::{str::FromStr, sync::Arc}; #[macro_use] @@ -67,11 +67,11 @@ async fn main() -> std::io::Result<()> { .await .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; - let redis_config = config - .clone() - .redis_config - .ok_or_else(|| Error::new(ErrorKind::NotFound, "redis configuration not found"))?; - let redis_connections = RedisConnection::new(&redis_config) + let rocksdb_config = config.clone().rocksdb_config.ok_or_else(|| { + Error::new(ErrorKind::NotFound, "rocksdb configuration not found") + })?; + + let rocksdb = RocksDBConnection::new(&rocksdb_config) .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; let signing_key_chain = KeyStoreType::KeyChain(KeyChain) @@ -107,12 +107,10 @@ async fn main() -> std::io::Result<()> { }; Arc::new( - Prover::new(Arc::new(Box::new(redis_connections)), da, &prover_cfg).map_err( - |e| { - error!("error initializing prover: {}", e); - Error::new(ErrorKind::Other, e.to_string()) - }, - )?, + Prover::new(Arc::new(Box::new(rocksdb)), da, &prover_cfg).map_err(|e| { + error!("error initializing prover: {}", e); + Error::new(ErrorKind::Other, e.to_string()) + })?, ) } Commands::FullNode(args) => { @@ -123,11 +121,11 @@ async fn main() -> std::io::Result<()> { .await .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; - let redis_config = config - .clone() - .redis_config - .ok_or_else(|| Error::new(ErrorKind::NotFound, "redis configuration not found"))?; - let redis_connections = RedisConnection::new(&redis_config) + let rocksdb_config = config.clone().rocksdb_config.ok_or_else(|| { + Error::new(ErrorKind::NotFound, "rocksdb configuration not found") + })?; + + let rocksdb = RocksDBConnection::new(&rocksdb_config) .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; let signing_key_chain = KeyStoreType::KeyChain(KeyChain) @@ -170,12 +168,10 @@ async fn main() -> std::io::Result<()> { }; Arc::new( - Prover::new(Arc::new(Box::new(redis_connections)), da, &prover_cfg).map_err( - |e| { - error!("error initializing prover: {}", e); - Error::new(ErrorKind::Other, e.to_string()) - }, - )?, + Prover::new(Arc::new(Box::new(rocksdb)), da, &prover_cfg).map_err(|e| { + error!("error initializing prover: {}", e); + Error::new(ErrorKind::Other, e.to_string()) + })?, ) } }; diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index 392a1887..b733ac65 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -12,6 +12,7 @@ readme.workspace = true [dependencies] serde = { workspace = true } +dirs = { workspace = true } redis = { workspace = true } log = { workspace = true } anyhow = { workspace = true } diff --git a/crates/storage/src/rocksdb.rs b/crates/storage/src/rocksdb.rs index 63a2835c..8e88668c 100644 --- a/crates/storage/src/rocksdb.rs +++ b/crates/storage/src/rocksdb.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use crate::Database; use anyhow::{anyhow, Result}; +use dirs::home_dir; use jmt::{ storage::{LeafNode, Node, NodeBatch, NodeKey, TreeReader, TreeWriter}, KeyHash, OwnedValue, Version, @@ -13,6 +14,7 @@ use prism_serde::{ hex::{FromHex, ToHex}, }; use rocksdb::{DBWithThreadMode, MultiThreaded, Options, DB}; +use serde::{Deserialize, Serialize}; const KEY_PREFIX_COMMITMENTS: &str = "commitments:epoch_"; const KEY_PREFIX_NODE: &str = "node:"; @@ -20,6 +22,30 @@ const KEY_PREFIX_VALUE_HISTORY: &str = "value_history:"; type RocksDB = DBWithThreadMode; +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct RocksDBConfig { + pub path: String, +} + +impl RocksDBConfig { + pub fn new(path: &str) -> Self { + Self { + path: path.to_string(), + } + } +} + +impl Default for RocksDBConfig { + fn default() -> Self { + Self { + path: home_dir() + .map(|path| format!("{}/.prism/db/", path.to_string_lossy())) + .unwrap() + .to_string(), + } + } +} + #[derive(Clone)] pub struct RocksDBConnection { connection: Arc, @@ -27,7 +53,8 @@ pub struct RocksDBConnection { } impl RocksDBConnection { - pub fn new(path: &str) -> Result { + pub fn new(cfg: &RocksDBConfig) -> Result { + let path = &cfg.path; let db = DB::open_default(path)?; Ok(Self { @@ -180,7 +207,8 @@ mod tests { fn setup_db() -> (TempDir, RocksDBConnection) { let temp_dir = TempDir::new().unwrap(); - let db = RocksDBConnection::new(temp_dir.path().to_str().unwrap()).unwrap(); + let cfg = RocksDBConfig::new(temp_dir.path().to_str().unwrap()); + let db = RocksDBConnection::new(&cfg).unwrap(); (temp_dir, db) } diff --git a/crates/tests/src/lib.rs b/crates/tests/src/lib.rs index e7bc01b4..0c4063f5 100644 --- a/crates/tests/src/lib.rs +++ b/crates/tests/src/lib.rs @@ -12,7 +12,10 @@ use prism_da::{ use prism_keys::{CryptoAlgorithm, SigningKey}; use prism_lightclient::LightClient; use prism_prover::Prover; -use prism_storage::{rocksdb::RocksDBConnection, Database}; +use prism_storage::{ + rocksdb::{RocksDBConfig, RocksDBConnection}, + Database, +}; use rand::{rngs::StdRng, Rng, SeedableRng}; use sp1_sdk::{HashableKey, Prover as _, ProverClient}; use std::sync::Arc; @@ -24,7 +27,8 @@ pub const PRISM_ELF: &[u8] = include_bytes!("../../../elf/riscv32im-succinct-zkv fn setup_db() -> Arc> { let temp_dir = TempDir::new().unwrap(); - let db = RocksDBConnection::new(temp_dir.path().to_str().unwrap()).unwrap(); + let cfg = RocksDBConfig::new(temp_dir.path().to_str().unwrap()); + let db = RocksDBConnection::new(&cfg).unwrap(); Arc::new(Box::new(db) as Box) } From eebdc7e34da5f3838b3c3a933c807b11967ca4bd Mon Sep 17 00:00:00 2001 From: Ryan Quinn Ford Date: Thu, 16 Jan 2025 11:20:49 +0100 Subject: [PATCH 2/4] refactoring config to allow for choosing db --- Cargo.lock | 1 - crates/cli/src/cfg.rs | 108 +++++++++++++++++++++++++-------- crates/cli/src/main.rs | 39 ++++-------- crates/storage/Cargo.toml | 1 - crates/storage/src/database.rs | 8 +++ crates/storage/src/redis.rs | 2 +- crates/storage/src/rocksdb.rs | 14 +---- justfile | 23 ------- 8 files changed, 106 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1d75065..3a220b44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4761,7 +4761,6 @@ version = "0.1.0" dependencies = [ "anyhow", "auto_impl", - "dirs 5.0.1", "jmt", "log", "prism-common", diff --git a/crates/cli/src/cfg.rs b/crates/cli/src/cfg.rs index 9bfe7b20..034d55f4 100644 --- a/crates/cli/src/cfg.rs +++ b/crates/cli/src/cfg.rs @@ -1,12 +1,18 @@ use anyhow::{Context, Result}; -use clap::{Args, Parser, Subcommand}; +use clap::{Args, Parser, Subcommand, ValueEnum}; use config::{builder::DefaultState, ConfigBuilder, File}; use dirs::home_dir; use dotenvy::dotenv; use log::{error, warn}; use prism_errors::{DataAvailabilityError, GeneralError}; use prism_prover::webserver::WebServerConfig; -use prism_storage::rocksdb::RocksDBConfig; +use prism_storage::{ + database::StorageBackend, + inmemory::InMemoryDatabase, + redis::RedisConfig, + rocksdb::{RocksDBConfig, RocksDBConnection}, + Database, RedisConnection, +}; use serde::{Deserialize, Serialize}; use std::{fs, path::Path, sync::Arc}; @@ -30,8 +36,8 @@ pub struct CommandArgs { #[arg(short, long, default_value = "INFO")] log_level: String, - #[arg(short = 'r', long)] - rocksdb_path: Option, + #[arg(short = 'n', long, default_value = "local")] + network_name: Option, #[arg(long)] verifying_key: Option, @@ -43,7 +49,10 @@ pub struct CommandArgs { verifying_key_algorithm: Option, #[arg(long)] - config_path: Option, + home_path: Option, + + #[command(flatten)] + database: DatabaseArgs, #[command(flatten)] celestia: CelestiaArgs, @@ -74,7 +83,7 @@ struct CelestiaArgs { #[arg(long)] operation_namespace_id: Option, - // Height to start searching the DA layer for SNARKs on + /// Height to start searching the DA layer for SNARKs on #[arg(short = 's', long)] celestia_start_height: Option, } @@ -101,18 +110,18 @@ pub struct Config { #[serde(skip_serializing_if = "Option::is_none")] pub celestia_config: Option, pub da_layer: DALayerOption, - pub rocksdb_config: Option, + pub db: StorageBackend, pub verifying_key: Option, pub verifying_key_algorithm: String, } -impl Default for Config { - fn default() -> Self { +impl Config { + fn with_home(path: &str) -> Self { Config { webserver: Some(WebServerConfig::default()), celestia_config: Some(CelestiaConfig::default()), da_layer: DALayerOption::default(), - rocksdb_config: Some(RocksDBConfig::default()), + db: StorageBackend::RocksDB(RocksDBConfig::new(path)), verifying_key: None, verifying_key_algorithm: "ed25519".to_string(), } @@ -126,16 +135,38 @@ pub enum DALayerOption { InMemory, } +#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize, Deserialize, ValueEnum)] +pub enum DBValues { + #[default] + RocksDB, + InMemory, + Redis, +} + +#[derive(Args, Deserialize, Clone, Debug)] +pub struct DatabaseArgs { + #[arg(long, value_enum)] + db_type: Option, + + /// Path to the RocksDB database, used when `db_type` is `rocks-db` + #[arg(long)] + rocksdb_path: Option, + + /// Connection string to Redis, used when `db_type` is `redis` + #[arg(long)] + redis_url: Option, +} + pub fn load_config(args: CommandArgs) -> Result { dotenv().ok(); std::env::set_var("RUST_LOG", args.clone().log_level); pretty_env_logger::init(); - let config_path = get_config_path(&args).context("Failed to determine config path")?; - ensure_config_file_exists(&config_path).context("Failed to ensure config file exists")?; + let home_path = get_prism_home(&args).context("Failed to determine prism home path")?; + ensure_config_file_exists(&home_path).context("Failed to ensure config file exists")?; let config_source = ConfigBuilder::::default() - .add_source(File::with_name(&config_path)) + .add_source(File::with_name(&format!("{}/config.toml", home_path))) .build() .context("Failed to build config")?; @@ -152,25 +183,25 @@ pub fn load_config(args: CommandArgs) -> Result { } fn get_prism_home(args: &CommandArgs) -> Result { - args.config_path + let network_name = args.network_name.clone().unwrap_or_else(|| "custom".to_string()); + args.home_path .clone() - .or_else(|| home_dir().map(|path| format!("{}/.prism/", path.to_string_lossy()))) + .or_else(|| { + home_dir().map(|path| format!("{}/.prism/{}/", path.to_string_lossy(), network_name)) + }) .ok_or_else(|| { GeneralError::MissingArgumentError("could not determine config path".to_string()).into() }) } -fn get_config_path(args: &CommandArgs) -> Result { - get_prism_home(args).map(|path| format!("{}/config.toml", path)) -} - -fn ensure_config_file_exists(config_path: &str) -> Result<()> { +fn ensure_config_file_exists(home_path: &str) -> Result<()> { + let config_path = &format!("{}/config.toml", home_path); if !Path::new(config_path).exists() { if let Some(parent) = Path::new(config_path).parent() { fs::create_dir_all(parent).context("Failed to create config directory")?; } - let default_config = Config::default(); + let default_config = Config::with_home(home_path); let config_toml = toml::to_string(&default_config).context("Failed to serialize default config")?; @@ -181,8 +212,8 @@ fn ensure_config_file_exists(config_path: &str) -> Result<()> { fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { let webserver_config = &config.webserver.unwrap_or_default(); - let rocksdb_config = &config.rocksdb_config.unwrap_or_default(); let celestia_config = &config.celestia_config.unwrap_or_default(); + let prism_home = get_prism_home(&args.clone()).unwrap(); Config { webserver: Some(WebServerConfig { @@ -190,9 +221,15 @@ fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { host: args.webserver.host.unwrap_or(webserver_config.host.clone()), port: args.webserver.port.unwrap_or(webserver_config.port), }), - rocksdb_config: Some(RocksDBConfig { - path: args.rocksdb_path.unwrap_or(rocksdb_config.path.clone()), - }), + db: match args.database.db_type { + None | Some(DBValues::RocksDB) => StorageBackend::RocksDB(RocksDBConfig { + path: args.database.rocksdb_path.unwrap_or(prism_home), + }), + Some(DBValues::Redis) => StorageBackend::Redis(RedisConfig { + connection_string: args.database.redis_url.unwrap_or_default(), + }), + Some(DBValues::InMemory) => StorageBackend::InMemory, + }, celestia_config: Some(CelestiaConfig { connection_string: args .celestia @@ -219,6 +256,27 @@ fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { } } +pub fn initialize_db(cfg: &Config) -> Result>> { + match &cfg.db { + StorageBackend::RocksDB(cfg) => { + let db = RocksDBConnection::new(cfg) + .map_err(|e| GeneralError::InitializationError(e.to_string())) + .context("Failed to initialize RocksDB")?; + + Ok(Arc::new(Box::new(db) as Box)) + } + StorageBackend::InMemory => Ok(Arc::new( + Box::new(InMemoryDatabase::new()) as Box + )), + StorageBackend::Redis(cfg) => { + let db = RedisConnection::new(cfg) + .map_err(|e| GeneralError::InitializationError(e.to_string())) + .context("Failed to initialize Redis")?; + Ok(Arc::new(Box::new(db) as Box)) + } + } +} + pub async fn initialize_da_layer( config: &Config, ) -> Result> { diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index f6210d88..c3665541 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,7 +1,7 @@ mod cfg; mod node_types; -use cfg::{initialize_da_layer, load_config, Cli, Commands}; +use cfg::{initialize_da_layer, initialize_db, load_config, Cli, Commands}; use clap::Parser; use keystore_rs::{KeyChain, KeyStore, KeyStoreType}; use prism_keys::{CryptoAlgorithm, SigningKey, VerifyingKey}; @@ -11,7 +11,6 @@ use std::io::{Error, ErrorKind}; use node_types::NodeType; use prism_lightclient::LightClient; use prism_prover::Prover; -use prism_storage::rocksdb::RocksDBConnection; use std::{str::FromStr, sync::Arc}; #[macro_use] @@ -67,12 +66,8 @@ async fn main() -> std::io::Result<()> { .await .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; - let rocksdb_config = config.clone().rocksdb_config.ok_or_else(|| { - Error::new(ErrorKind::NotFound, "rocksdb configuration not found") - })?; - - let rocksdb = RocksDBConnection::new(&rocksdb_config) - .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; + let db = + initialize_db(&config).map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; let signing_key_chain = KeyStoreType::KeyChain(KeyChain) .get_signing_key() @@ -106,12 +101,10 @@ async fn main() -> std::io::Result<()> { start_height: config.celestia_config.unwrap_or_default().start_height, }; - Arc::new( - Prover::new(Arc::new(Box::new(rocksdb)), da, &prover_cfg).map_err(|e| { - error!("error initializing prover: {}", e); - Error::new(ErrorKind::Other, e.to_string()) - })?, - ) + Arc::new(Prover::new(db, da, &prover_cfg).map_err(|e| { + error!("error initializing prover: {}", e); + Error::new(ErrorKind::Other, e.to_string()) + })?) } Commands::FullNode(args) => { let config = load_config(args.clone()) @@ -121,12 +114,8 @@ async fn main() -> std::io::Result<()> { .await .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; - let rocksdb_config = config.clone().rocksdb_config.ok_or_else(|| { - Error::new(ErrorKind::NotFound, "rocksdb configuration not found") - })?; - - let rocksdb = RocksDBConnection::new(&rocksdb_config) - .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; + let db = + initialize_db(&config).map_err(|e| Error::new(ErrorKind::Other, e.to_string()))?; let signing_key_chain = KeyStoreType::KeyChain(KeyChain) .get_signing_key() @@ -167,12 +156,10 @@ async fn main() -> std::io::Result<()> { start_height: config.celestia_config.unwrap_or_default().start_height, }; - Arc::new( - Prover::new(Arc::new(Box::new(rocksdb)), da, &prover_cfg).map_err(|e| { - error!("error initializing prover: {}", e); - Error::new(ErrorKind::Other, e.to_string()) - })?, - ) + Arc::new(Prover::new(db, da, &prover_cfg).map_err(|e| { + error!("error initializing prover: {}", e); + Error::new(ErrorKind::Other, e.to_string()) + })?) } }; diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index b733ac65..392a1887 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -12,7 +12,6 @@ readme.workspace = true [dependencies] serde = { workspace = true } -dirs = { workspace = true } redis = { workspace = true } log = { workspace = true } anyhow = { workspace = true } diff --git a/crates/storage/src/database.rs b/crates/storage/src/database.rs index 85f2e6b1..21f2498e 100644 --- a/crates/storage/src/database.rs +++ b/crates/storage/src/database.rs @@ -3,6 +3,14 @@ use auto_impl::auto_impl; use jmt::storage::{TreeReader, TreeWriter}; use prism_common::digest::Digest; use prism_errors::{DatabaseError, PrismError}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub enum StorageBackend { + RocksDB(crate::rocksdb::RocksDBConfig), + InMemory, + Redis(crate::redis::RedisConfig), +} #[auto_impl(&, Box, Arc)] pub trait Database: Send + Sync + TreeReader + TreeWriter { diff --git a/crates/storage/src/redis.rs b/crates/storage/src/redis.rs index 8fa6830f..410d4904 100644 --- a/crates/storage/src/redis.rs +++ b/crates/storage/src/redis.rs @@ -23,7 +23,7 @@ use prism_errors::DatabaseError; use crate::database::{convert_to_connection_error, Database}; use log::debug; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct RedisConfig { pub connection_string: String, } diff --git a/crates/storage/src/rocksdb.rs b/crates/storage/src/rocksdb.rs index 8e88668c..f5b3adfc 100644 --- a/crates/storage/src/rocksdb.rs +++ b/crates/storage/src/rocksdb.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use crate::Database; use anyhow::{anyhow, Result}; -use dirs::home_dir; use jmt::{ storage::{LeafNode, Node, NodeBatch, NodeKey, TreeReader, TreeWriter}, KeyHash, OwnedValue, Version, @@ -22,7 +21,7 @@ const KEY_PREFIX_VALUE_HISTORY: &str = "value_history:"; type RocksDB = DBWithThreadMode; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct RocksDBConfig { pub path: String, } @@ -35,17 +34,6 @@ impl RocksDBConfig { } } -impl Default for RocksDBConfig { - fn default() -> Self { - Self { - path: home_dir() - .map(|path| format!("{}/.prism/db/", path.to_string_lossy())) - .unwrap() - .to_string(), - } - } -} - #[derive(Clone)] pub struct RocksDBConnection { connection: Arc, diff --git a/justfile b/justfile index 66c05556..04d5a4a0 100644 --- a/justfile +++ b/justfile @@ -145,29 +145,6 @@ install-deps: done; \ fi - # Install Redis if not present - if ! command -v redis-server > /dev/null; then \ - echo "Installing Redis..."; \ - if [ "$OS" = "Mac" ]; then \ - if ! command -v brew > /dev/null; then \ - echo "Homebrew is not installed. Please install Homebrew first."; \ - exit 1; \ - fi; \ - brew install redis; \ - elif [ "$OS" = "Linux" ]; then \ - if {{is_root}}; then \ - apt update; \ - apt install redis-server -y; \ - else \ - sudo apt update; \ - sudo apt install redis-server -y; \ - fi; \ - fi; \ - echo "Redis installation complete!"; \ - else \ - echo "Redis is already installed."; \ - fi - if ! cargo prove --version > /dev/null 2>&1; then \ echo "Installing SP1..." curl -L https://sp1.succinct.xyz | bash; \ From e5aabfec2e8c5418fade49fdb4088cb2bb3ca5a9 Mon Sep 17 00:00:00 2001 From: Ryan Quinn Ford Date: Sat, 18 Jan 2025 11:52:31 +0100 Subject: [PATCH 3/4] defaults --- crates/cli/src/cfg.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/cfg.rs b/crates/cli/src/cfg.rs index 034d55f4..eb8ac632 100644 --- a/crates/cli/src/cfg.rs +++ b/crates/cli/src/cfg.rs @@ -145,15 +145,16 @@ pub enum DBValues { #[derive(Args, Deserialize, Clone, Debug)] pub struct DatabaseArgs { - #[arg(long, value_enum)] - db_type: Option, + #[arg(long, value_enum, default_value_t = DBValues::RocksDB)] + /// Storage backend to use. Default: `rocks-db` + db_type: DBValues, /// Path to the RocksDB database, used when `db_type` is `rocks-db` #[arg(long)] rocksdb_path: Option, /// Connection string to Redis, used when `db_type` is `redis` - #[arg(long)] + #[arg(long, required_if_eq("db_type", "redis"))] redis_url: Option, } @@ -222,13 +223,13 @@ fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { port: args.webserver.port.unwrap_or(webserver_config.port), }), db: match args.database.db_type { - None | Some(DBValues::RocksDB) => StorageBackend::RocksDB(RocksDBConfig { + DBValues::RocksDB => StorageBackend::RocksDB(RocksDBConfig { path: args.database.rocksdb_path.unwrap_or(prism_home), }), - Some(DBValues::Redis) => StorageBackend::Redis(RedisConfig { + DBValues::Redis => StorageBackend::Redis(RedisConfig { connection_string: args.database.redis_url.unwrap_or_default(), }), - Some(DBValues::InMemory) => StorageBackend::InMemory, + DBValues::InMemory => StorageBackend::InMemory, }, celestia_config: Some(CelestiaConfig { connection_string: args From 21064e463c3760e41f6ddf7c964c85bfc65dfb95 Mon Sep 17 00:00:00 2001 From: Ryan Quinn Ford Date: Sat, 18 Jan 2025 12:02:17 +0100 Subject: [PATCH 4/4] review fixes --- crates/cli/src/cfg.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/cfg.rs b/crates/cli/src/cfg.rs index eb8ac632..aebc6913 100644 --- a/crates/cli/src/cfg.rs +++ b/crates/cli/src/cfg.rs @@ -121,7 +121,7 @@ impl Config { webserver: Some(WebServerConfig::default()), celestia_config: Some(CelestiaConfig::default()), da_layer: DALayerOption::default(), - db: StorageBackend::RocksDB(RocksDBConfig::new(path)), + db: StorageBackend::RocksDB(RocksDBConfig::new(&format!("{}/data", path))), verifying_key: None, verifying_key_algorithm: "ed25519".to_string(), } @@ -164,8 +164,13 @@ pub fn load_config(args: CommandArgs) -> Result { pretty_env_logger::init(); let home_path = get_prism_home(&args).context("Failed to determine prism home path")?; + ensure_config_file_exists(&home_path).context("Failed to ensure config file exists")?; + if let Some(rocksdb_path) = &args.database.rocksdb_path { + fs::create_dir_all(rocksdb_path).context("Failed to create RocksDB directory")?; + } + let config_source = ConfigBuilder::::default() .add_source(File::with_name(&format!("{}/config.toml", home_path))) .build() @@ -224,7 +229,7 @@ fn apply_command_line_args(config: Config, args: CommandArgs) -> Config { }), db: match args.database.db_type { DBValues::RocksDB => StorageBackend::RocksDB(RocksDBConfig { - path: args.database.rocksdb_path.unwrap_or(prism_home), + path: args.database.rocksdb_path.unwrap_or_else(|| format!("{}/data", prism_home)), }), DBValues::Redis => StorageBackend::Redis(RedisConfig { connection_string: args.database.redis_url.unwrap_or_default(),