Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(exex): use rmp-serde for WAL storage #11353

Merged
merged 21 commits into from
Oct 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions crates/exex/exex/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ reth-chain-state.workspace = true
reth-chainspec.workspace = true
reth-config.workspace = true
reth-evm.workspace = true
reth-exex-types = { workspace = true, features = ["serde"] }
reth-exex-types = { workspace = true, features = ["serde", "serde-bincode-compat"] }
reth-fs-util.workspace = true
reth-metrics.workspace = true
reth-node-api.workspace = true
Expand Down Expand Up @@ -46,7 +46,7 @@ eyre.workspace = true
itertools.workspace = true
metrics.workspace = true
parking_lot.workspace = true
serde_json.workspace = true
rmp-serde = "1.3"
tracing.workspace = true

[dev-dependencies]
Expand Down
14 changes: 10 additions & 4 deletions crates/exex/exex/src/wal/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,11 @@ impl Storage {
Err(err) => return Err(err.into()),
};

// TODO(alexey): use rmp-serde when Alloy and Reth serde issues are resolved
Ok(serde_json::from_reader(&mut file)?)
// Deserialize using the bincode- and msgpack-compatible serde wrapper
let notification: reth_exex_types::serde_bincode_compat::ExExNotification<'_> =
rmp_serde::decode::from_read(&mut file)?;

Ok(Some(notification.into()))
}

/// Writes the notification to the file with the given ID.
Expand All @@ -130,9 +133,12 @@ impl Storage {
let file_path = self.file_path(file_id);
debug!(?file_path, "Writing notification to WAL");

// Serialize using the bincode- and msgpack-compatible serde wrapper
let notification =
reth_exex_types::serde_bincode_compat::ExExNotification::from(notification);

Ok(reth_fs_util::atomic_write_file(&file_path, |file| {
// TODO(alexey): use rmp-serde when Alloy and Reth serde issues are resolved
serde_json::to_writer(file, notification)
rmp_serde::encode::write(file, &notification)
})?)
}
}
Expand Down
3 changes: 2 additions & 1 deletion crates/primitives-traits/src/header/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ pub use alloy_consensus::Header;

use alloy_primitives::{Address, BlockNumber, B256, U256};

/// Bincode-compatible header type serde implementations.
#[cfg(feature = "serde-bincode-compat")]
pub(super) mod serde_bincode_compat {
pub mod serde_bincode_compat {
pub use super::sealed::serde_bincode_compat::SealedHeader;
}

Expand Down
4 changes: 2 additions & 2 deletions crates/primitives-traits/src/header/sealed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,14 @@ pub(super) mod serde_bincode_compat {
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_primitives_traits::{header::SealedHeader, serde_bincode_compat};
/// use reth_primitives_traits::{serde_bincode_compat, SealedHeader};
/// use serde::{Deserialize, Serialize};
/// use serde_with::serde_as;
///
/// #[serde_as]
/// #[derive(Serialize, Deserialize)]
/// struct Data {
/// #[serde_as(as = "serde_bincode_compat::header::SealedHeader")]
/// #[serde_as(as = "serde_bincode_compat::SealedHeader")]
/// header: SealedHeader,
/// }
/// ```
Expand Down
2 changes: 1 addition & 1 deletion crates/primitives-traits/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,5 +53,5 @@ pub use header::{BlockHeader, Header, HeaderError, SealedHeader};
/// Read more: <https://github.com/bincode-org/bincode/issues/326>
#[cfg(feature = "serde-bincode-compat")]
pub mod serde_bincode_compat {
pub use super::header::serde_bincode_compat::*;
pub use super::header::{serde_bincode_compat as header, serde_bincode_compat::*};
}
4 changes: 2 additions & 2 deletions crates/primitives/src/transaction/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1995,7 +1995,7 @@ pub mod serde_bincode_compat {
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_primitives_traits::{serde_bincode_compat, Transaction};
/// use reth_primitives::{serde_bincode_compat, Transaction};
/// use serde::{Deserialize, Serialize};
/// use serde_with::serde_as;
///
Expand Down Expand Up @@ -2069,7 +2069,7 @@ pub mod serde_bincode_compat {
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_primitives_traits::{serde_bincode_compat, TransactionSigned};
/// use reth_primitives::{serde_bincode_compat, TransactionSigned};
/// use serde::{Deserialize, Serialize};
/// use serde_with::serde_as;
///
Expand Down
Loading