storage: refactor and rocksdb implementation

Cargo.toml

@@ -21,13 +21,15 @@ members = [
     "crates/groth16",
     "crates/errors",
     "crates/sp1",
+    "crates/storage",
 ]
 default-members = [
     "crates/prism",
     "crates/common",
     "crates/nova",
     "crates/groth16",
     "crates/errors",
+    "crates/storage",
 ]
 resolver = "2"
 
@@ -78,10 +80,12 @@ bincode = "1.3.3"
 sp1-zkvm = { version = "1.2.0" }
 sp1-sdk = { version = "1.2.0" }
 prism-common = { path = "crates/common" }
+prism-storage = { path = "crates/storage" }
 prism-nova = { path = "crates/nova" }
 prism-errors = { path = "crates/errors" }
 prism-main = { path = "crates/prism" }
 prism-groth16 = { path = "crates/groth16" }
+rocksdb = "0.21.0"
 
 [patch.crates-io]
 sha2-v0-10-8 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", package = "sha2", branch = "patch-sha2-v0.10.8" }

crates/prism/Cargo.toml

@@ -49,6 +49,7 @@ jmt = { workspace = true }
 sha2 = { workspace = true }
 auto_impl = { workspace = true }
 prism-common = { workspace = true }
+prism-storage = { workspace = true }
 prism-errors = { workspace = true }
 prism-groth16 = { workspace = true }
 sp1-sdk = { workspace = true }

crates/prism/src/cfg.rs

@@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize};
 use std::{fs, path::Path, sync::Arc};
 
 use crate::da::{celestia::CelestiaConnection, DataAvailabilityLayer};
+use prism_storage::RedisConfig;
 
 #[derive(Clone, Debug, Subcommand, Deserialize)]
 pub enum Commands {
@@ -97,18 +98,6 @@ impl Default for WebServerConfig {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct RedisConfig {
-    pub connection_string: String,
-}
-
-impl Default for RedisConfig {
-    fn default() -> Self {
-        RedisConfig {
-            connection_string: "redis://127.0.0.1/".to_string(),
-        }
-    }
-}
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct CelestiaConfig {
@@ -300,7 +289,7 @@ pub async fn initialize_da_layer(
             unreachable!() // This line should never be reached due to the return in the last iteration
         }
         DALayerOption::InMemory => {
-            let (da_layer, _height_rx, _block_rx) = InMemoryDataAvailabilityLayer::new(1);
+            let (da_layer, _height_rx, _block_rx) = InMemoryDataAvailabilityLayer::new(30);
             Ok(Arc::new(da_layer) as Arc<dyn DataAvailabilityLayer + 'static>)
         }
         DALayerOption::None => Err(anyhow!(PrismError::ConfigError(

crates/prism/src/lib.rs

@@ -2,7 +2,6 @@ pub mod cfg;
 pub mod consts;
 pub mod da;
 pub mod node_types;
-pub mod storage;
 pub mod utils;
 pub mod webserver;
 #[macro_use]

crates/prism/src/main.rs

@@ -2,7 +2,6 @@ mod cfg;
 pub mod consts;
 pub mod da;
 mod node_types;
-pub mod storage;
 mod utils;
 mod webserver;
 
@@ -13,7 +12,7 @@ use keystore_rs::{KeyChain, KeyStore, KeyStoreType};
 use crate::cfg::{CommandLineArgs, Commands};
 use node_types::{lightclient::LightClient, sequencer::Sequencer, NodeType};
 use std::sync::Arc;
-use storage::RedisConnection;
+use prism_storage::RedisConnection;
 
 #[macro_use]
 extern crate log;

crates/prism/src/node_types/sequencer.rs

@@ -24,9 +24,10 @@ use crate::{
     consts::{CHANNEL_BUFFER_SIZE, DA_RETRY_COUNT, DA_RETRY_INTERVAL},
     da::{DataAvailabilityLayer, FinalizedEpoch},
     node_types::NodeType,
-    storage::Database,
     webserver::{OperationInput, WebServer},
 };
+
+use prism_storage::Database;
 use prism_common::{
     hashchain::{Hashchain, HashchainEntry},
     operation::{AccountSource, Operation},
@@ -470,8 +471,8 @@ mod tests {
     use crate::{
         cfg::{Config, RedisConfig},
         da::memory::InMemoryDataAvailabilityLayer,
-        storage::RedisConnection,
     };
+    use prism_storage::RedisConnection;
     use base64::{engine::general_purpose::STANDARD as engine, Engine as _};
     use keystore_rs::create_signing_key;
     use serial_test::serial;

crates/storage/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "prism-storage"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+readme.workspace = true
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+redis = { workspace = true }
+mockall = { workspace = true }
+tokio = { workspace = true }
+log = { workspace = true }
+anyhow = { workspace = true }
+bincode = { workspace = true }
+hex = { workspace = true }
+jmt = { workspace = true }
+pretty_env_logger = { workspace = true }
+config = { workspace = true }
+thiserror = { workspace = true }
+prism-errors = { workspace = true }
+prism-common = { workspace = true }
+auto_impl = {workspace = true}
+rocksdb = {workspace = true}

crates/storage/src/database.rs

@@ -0,0 +1,29 @@
+use anyhow::Result;
+use auto_impl::auto_impl;
+use jmt::storage::{TreeReader, TreeWriter};
+use prism_common::{
+    hashchain::{Hashchain, HashchainEntry},
+    operation::Operation,
+    tree::Digest,
+};
+use prism_errors::{DatabaseError, PrismError};
+
+#[auto_impl(&, Box, Arc)]
+pub trait Database: Send + Sync + TreeReader + TreeWriter {
+    fn get_hashchain(&self, key: &str) -> Result<Hashchain>;
+    fn set_hashchain(&self, incoming_operation: &Operation, value: &[HashchainEntry])
+        -> Result<()>;
+
+    fn get_commitment(&self, epoch: &u64) -> Result<String>;
+    fn set_commitment(&self, epoch: &u64, commitment: &Digest) -> Result<()>;
+
+    fn get_epoch(&self) -> Result<u64>;
+    fn set_epoch(&self, epoch: &u64) -> Result<()>;
+
+    #[cfg(test)]
+    fn flush_database(&self) -> Result<()>;
+}
+
+pub fn convert_to_connection_error(e: redis::RedisError) -> PrismError {
+    PrismError::Database(DatabaseError::ConnectionError(e.to_string()))
+}

crates/storage/src/lib.rs

@@ -0,0 +1,6 @@
+pub mod database;
+pub mod redis;
+pub mod rocksdb;
+
+pub use crate::database::Database;
+pub use crate::redis::{RedisConfig, RedisConnection};