Cherry-picked Move to mimalloc

Kash-Protocol · Jan 5, 2024 · b61f84b · b61f84b
1 parent a9772bf
commit b61f84b
Show file tree

Hide file tree

Showing 12 changed files with 156 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -53,6 +53,7 @@ members = [
     "rothschild",
     "metrics/perf_monitor",
     "metrics/core",
+    "utils/alloc",
 ]
 
 [workspace.package]
@@ -73,6 +74,9 @@ include = [
 ]
 
 [workspace.dependencies]
+mimalloc = { version = "0.1.39", default-features = false, features = [
+    'override',
+] }
 # kash-testing-integration = { version = "0.13.1", path = "testing/integration" }
 kash-addresses = { version = "0.13.1", path = "crypto/addresses" }
 kash-addressmanager = { version = "0.13.1", path = "components/addressmanager" }
@@ -124,6 +128,7 @@ kash-wrpc-proxy = { version = "0.13.1", path = "rpc/wrpc/proxy" }
 kash-wrpc-server = { version = "0.13.1", path = "rpc/wrpc/server" }
 kash-wrpc-wasm = { version = "0.13.1", path = "rpc/wrpc/wasm" }
 kashd = { version = "0.13.1", path = "kashd" }
+kash-alloc = { version = "0.13.1", path = "utils/alloc" }
 
 # external
 aes = "0.8.3"
@@ -162,7 +167,9 @@ faster-hex = "0.6.1" # TODO "0.8.1" - fails unit tests
 fixedstr = { version = "0.5.4", features = ["serde"] }
 flate2 = "1.0.28"
 futures = { version = "0.3.29" }
-futures-util = { version = "0.3.29", default-features = false, features = [ "alloc", ] }
+futures-util = { version = "0.3.29", default-features = false, features = [
+    "alloc",
+] }
 getrandom = { version = "0.2.10", features = ["js"] }
 h2 = "0.3.21"
 heapless = "0.7.16"
@@ -237,7 +244,10 @@ web-sys = "=0.3.64"
 xxhash-rust = { version = "0.8.7", features = ["xxh3"] }
 zeroize = { version = "1.6.0", default-features = false, features = ["alloc"] }
 pin-project-lite = "0.2.13"
-tower-http = { version = "0.4.4", features = ["map-response-body", "map-request-body"] }
+tower-http = { version = "0.4.4", features = [
+    "map-response-body",
+    "map-request-body",
+] }
 tower = "0.4.7"
 hyper = "0.14.27"
 # workflow dependencies that are not a part of core libraries

diff --git a/kashd/Cargo.toml b/kashd/Cargo.toml
@@ -13,6 +13,8 @@ name = "kashd_lib"
 crate-type = ["cdylib", "lib"]
 
 [dependencies]
+kash-alloc.workspace = true # This changes the global allocator for all of the next dependencies so should be kept first
+
 kash-addresses.workspace = true
 kash-addressmanager.workspace = true
 kash-consensus-core.workspace = true
@@ -50,5 +52,5 @@ tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] }
 workflow-log.workspace = true
 
 [features]
-heap = ["dhat"]
+heap = ["dhat", "kash-alloc/heap"]
 devnet-prealloc = ["kash-consensus/devnet-prealloc"]
diff --git a/kashd/src/main.rs b/kashd/src/main.rs
@@ -4,6 +4,7 @@ extern crate kash_hashes;
 
 use std::sync::Arc;
 
+use kash_alloc::init_allocator_with_default_settings;
 use kash_core::{info, signals::Signals};
 use kash_utils::fd_budget;
 use kashd_lib::{
@@ -19,6 +20,8 @@ pub fn main() {
     #[cfg(feature = "heap")]
     let _profiler = dhat::Profiler::builder().file_name("kashd-heap.json").build();
 
+    init_allocator_with_default_settings();
+
     let args = parse_args();
 
     match fd_budget::try_set_fd_limit(DESIRED_DAEMON_SOFT_FD_LIMIT) {

diff --git a/simpa/Cargo.toml b/simpa/Cargo.toml
@@ -9,6 +9,7 @@ include.workspace = true
 license.workspace = true
 
 [dependencies]
+kash-alloc.workspace = true            # This changes the global allocator for all of the next dependencies so should be kept first
 kash-consensus-core.workspace = true
 kash-consensus-notify.workspace = true
 kash-consensus.workspace = true
@@ -20,7 +21,7 @@ kash-utils.workspace = true
 
 async-channel.workspace = true
 clap.workspace = true
-dhat = {workspace = true, optional = true}
+dhat = { workspace = true, optional = true }
 futures-util.workspace = true
 futures.workspace = true
 indexmap.workspace = true
@@ -34,4 +35,4 @@ secp256k1.workspace = true
 tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] }
 
 [features]
-heap = ["dhat"]
+heap = ["dhat", "kash-alloc/heap"]
diff --git a/simpa/src/main.rs b/simpa/src/main.rs
@@ -2,6 +2,7 @@ use async_channel::unbounded;
 use clap::Parser;
 use futures::{future::try_join_all, Future};
 use itertools::Itertools;
+use kash_alloc::init_allocator_with_default_settings;
 use kash_consensus::{
     config::ConfigBuilder,
     consensus::Consensus,
@@ -118,6 +119,8 @@ fn main() {
     #[cfg(feature = "heap")]
     let _profiler = dhat::Profiler::builder().file_name("simpa-heap.json").build();
 
+    init_allocator_with_default_settings();
+
     // Get CLI arguments
     let args = Args::parse();
 

diff --git a/testing/integration/Cargo.toml b/testing/integration/Cargo.toml
@@ -8,6 +8,7 @@ include.workspace = true
 license.workspace = true
 
 [dependencies]
+kash-alloc.workspace = true            # This changes the global allocator for all of the next dependencies so should be kept first
 kash-addresses.workspace = true
 kash-consensus-core.workspace = true
 kash-consensus-notify.workspace = true

diff --git a/testing/integration/src/consensus_integration_tests.rs b/testing/integration/src/consensus_integration_tests.rs
@@ -3,6 +3,7 @@
 //!
 
 use async_channel::unbounded;
+use kash_alloc::init_allocator_with_default_settings;
 use kash_consensus::config::genesis::GENESIS;
 use kash_consensus::config::{Config, ConfigBuilder};
 use kash_consensus::consensus::factory::Factory as ConsensusFactory;
@@ -176,16 +177,19 @@ fn reachability_stretch_test(use_attack_json: bool) {
 
 #[test]
 fn test_attack_json() {
+    init_allocator_with_default_settings();
     reachability_stretch_test(true);
 }
 
 #[test]
 fn test_noattack_json() {
+    init_allocator_with_default_settings();
     reachability_stretch_test(false);
 }
 
 #[tokio::test]
 async fn consensus_sanity_test() {
+    init_allocator_with_default_settings();
     let genesis_child: Hash = 2.into();
     let config = ConfigBuilder::new(MAINNET_PARAMS).skip_proof_of_work().build();
     let consensus = TestConsensus::new(&config);
@@ -235,6 +239,7 @@ struct GhostdagTestBlock {
 
 #[tokio::test]
 async fn ghostdag_test() {
+    init_allocator_with_default_settings();
     let mut path_strings: Vec<String> =
         common::read_dir("testdata/dags").map(|f| f.unwrap().path().to_str().unwrap().to_owned()).collect();
     path_strings.sort();
@@ -319,6 +324,7 @@ fn strings_to_hashes(strings: &Vec<String>) -> Vec<Hash> {
 
 #[tokio::test]
 async fn block_window_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS)
         .skip_proof_of_work()
         .edit_consensus_params(|p| {
@@ -388,6 +394,7 @@ async fn block_window_test() {
 
 #[tokio::test]
 async fn header_in_isolation_validation_test() {
+    init_allocator_with_default_settings();
     let config = Config::new(MAINNET_PARAMS);
     let consensus = TestConsensus::new(&config);
     let wait_handles = consensus.init();
@@ -456,6 +463,7 @@ async fn header_in_isolation_validation_test() {
 
 #[tokio::test]
 async fn incest_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS).skip_proof_of_work().build();
     let consensus = TestConsensus::new(&config);
     let wait_handles = consensus.init();
@@ -484,6 +492,7 @@ async fn incest_test() {
 
 #[tokio::test]
 async fn missing_parents_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS).skip_proof_of_work().build();
     let consensus = TestConsensus::new(&config);
     let wait_handles = consensus.init();
@@ -508,6 +517,7 @@ async fn missing_parents_test() {
 // as a known invalid.
 #[tokio::test]
 async fn known_invalid_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS).skip_proof_of_work().build();
     let consensus = TestConsensus::new(&config);
     let wait_handles = consensus.init();
@@ -533,6 +543,7 @@ async fn known_invalid_test() {
 
 #[tokio::test]
 async fn median_time_test() {
+    init_allocator_with_default_settings();
     struct Test {
         name: &'static str,
         config: Config,
@@ -606,6 +617,7 @@ async fn median_time_test() {
 
 #[tokio::test]
 async fn mergeset_size_limit_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS).skip_proof_of_work().build();
     let consensus = TestConsensus::new(&config);
     let wait_handles = consensus.init();
@@ -832,39 +844,46 @@ impl KashdGoParams {
 
 #[tokio::test]
 async fn goref_custom_pruning_depth_test() {
+    init_allocator_with_default_settings();
     json_test("testdata/dags_for_json_tests/goref_custom_pruning_depth", false).await
 }
 
 #[tokio::test]
 async fn goref_notx_test() {
+    init_allocator_with_default_settings();
     json_test("testdata/dags_for_json_tests/goref-notx-5000-blocks", false).await
 }
 
 #[tokio::test]
 async fn goref_notx_concurrent_test() {
+    init_allocator_with_default_settings();
     json_test("testdata/dags_for_json_tests/goref-notx-5000-blocks", true).await
 }
 
 #[tokio::test]
 async fn goref_tx_small_test() {
+    init_allocator_with_default_settings();
     json_test("testdata/dags_for_json_tests/goref-905-tx-265-blocks", false).await
 }
 
 #[tokio::test]
 async fn goref_tx_small_concurrent_test() {
+    init_allocator_with_default_settings();
     json_test("testdata/dags_for_json_tests/goref-905-tx-265-blocks", true).await
 }
 
 #[ignore]
 #[tokio::test]
 async fn goref_tx_big_test() {
+    init_allocator_with_default_settings();
     // TODO: add this directory to a data repo and fetch dynamically
     json_test("testdata/dags_for_json_tests/goref-1.6M-tx-10K-blocks", false).await
 }
 
 #[ignore]
 #[tokio::test]
 async fn goref_tx_big_concurrent_test() {
+    init_allocator_with_default_settings();
     // TODO: add this file to a data repo and fetch dynamically
     json_test("testdata/dags_for_json_tests/goref-1.6M-tx-10K-blocks", true).await
 }
@@ -1230,6 +1249,7 @@ fn hex_decode(src: &str) -> Vec<u8> {
 
 #[tokio::test]
 async fn bounded_merge_depth_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(DEVNET_PARAMS)
         .skip_proof_of_work()
         .edit_consensus_params(|p| {
@@ -1309,6 +1329,7 @@ async fn bounded_merge_depth_test() {
 
 #[tokio::test]
 async fn difficulty_test() {
+    init_allocator_with_default_settings();
     async fn add_block(consensus: &TestConsensus, block_time: Option<u64>, parents: Vec<Hash>) -> Header {
         let selected_parent = consensus.ghostdag_manager().find_selected_parent(parents.iter().copied());
         let block_time = block_time.unwrap_or_else(|| {
@@ -1627,6 +1648,7 @@ async fn difficulty_test() {
 
 #[tokio::test]
 async fn selected_chain_test() {
+    init_allocator_with_default_settings();
     kash_core::log::try_init_logger("info");
 
     let config = ConfigBuilder::new(MAINNET_PARAMS)
@@ -1695,6 +1717,7 @@ fn selected_chain_store_iterator(consensus: &TestConsensus, pruning_point: Hash)
 
 #[tokio::test]
 async fn staging_consensus_test() {
+    init_allocator_with_default_settings();
     let config = ConfigBuilder::new(MAINNET_PARAMS).build();
 
     let db_tempdir = get_kash_tempdir();