Cherry-picked patches from upstream and updated README

Cargo.toml

@@ -53,10 +53,11 @@ members = [
     "rothschild",
     "metrics/perf_monitor",
     "metrics/core",
+    "utils/alloc",
 ]
 
 [workspace.package]
-version = "0.13.1"
+version = "0.13.3"
 authors = ["Kash developers"]
 license = "MIT/Apache-2.0"
 repository = "https://github.com/Kash-Protocol/rusty-kash"
@@ -73,57 +74,61 @@ include = [
 ]
 
 [workspace.dependencies]
-# kash-testing-integration = { version = "0.13.1", path = "testing/integration" }
-kash-addresses = { version = "0.13.1", path = "crypto/addresses" }
-kash-addressmanager = { version = "0.13.1", path = "components/addressmanager" }
-kash-bip32 = { version = "0.13.1", path = "wallet/bip32" }
-kash-cli = { version = "0.13.1", path = "cli" }
-kash-connectionmanager = { version = "0.13.1", path = "components/connectionmanager" }
-kash-consensus = { version = "0.13.1", path = "consensus" }
-kash-consensus-core = { version = "0.13.1", path = "consensus/core" }
-kash-consensus-notify = { version = "0.13.1", path = "consensus/notify" }
-kash-consensus-wasm = { version = "0.13.1", path = "consensus/wasm" }
-kash-consensusmanager = { version = "0.13.1", path = "components/consensusmanager" }
-kash-core = { version = "0.13.1", path = "core" }
-kash-daemon = { version = "0.13.1", path = "daemon" }
-kash-database = { version = "0.13.1", path = "database" }
-kash-grpc-client = { version = "0.13.1", path = "rpc/grpc/client" }
-kash-grpc-core = { version = "0.13.1", path = "rpc/grpc/core" }
-kash-grpc-server = { version = "0.13.1", path = "rpc/grpc/server" }
-kash-hashes = { version = "0.13.1", path = "crypto/hashes" }
-kash-index-core = { version = "0.13.1", path = "indexes/core" }
-kash-index-processor = { version = "0.13.1", path = "indexes/processor" }
-kash-math = { version = "0.13.1", path = "math" }
-kash-merkle = { version = "0.13.1", path = "crypto/merkle" }
-kash-metrics-core = { version = "0.13.0", path = "metrics/core" }
-kash-mining = { version = "0.13.1", path = "mining" }
-kash-mining-errors = { version = "0.13.1", path = "mining/errors" }
-kash-muhash = { version = "0.13.1", path = "crypto/muhash" }
-kash-notify = { version = "0.13.1", path = "notify" }
-kash-os = { version = "0.13.1", path = "kash-os" }
-kash-p2p-flows = { version = "0.13.1", path = "protocol/flows" }
-kash-p2p-lib = { version = "0.13.1", path = "protocol/p2p" }
-kash-perf-monitor = { version = "0.13.1", path = "metrics/perf_monitor" }
-kash-pow = { version = "0.13.1", path = "consensus/pow" }
-kash-rpc-core = { version = "0.13.1", path = "rpc/core" }
-kash-rpc-macros = { version = "0.13.1", path = "rpc/macros" }
-kash-rpc-service = { version = "0.13.1", path = "rpc/service" }
-kash-txscript = { version = "0.13.1", path = "crypto/txscript" }
-kash-txscript-errors = { version = "0.13.1", path = "crypto/txscript/errors" }
-kash-utils = { version = "0.13.1", path = "utils" }
-kash-utils-tower = { version = "0.13.1", path = "utils/tower" }
-kash-utxoindex = { version = "0.13.1", path = "indexes/utxoindex" }
-kash-wallet = { version = "0.13.1", path = "wallet/native" }
-kash-wallet-cli-wasm = { version = "0.13.1", path = "wallet/wasm" }
-kash-wallet-core = { version = "0.13.1", path = "wallet/core" }
-kash-wallet-macros = { version = "0.13.0", path = "wallet/macros" }
-kash-wasm = { version = "0.13.1", path = "wasm" }
-kash-wrpc-client = { version = "0.13.1", path = "rpc/wrpc/client" }
-kash-wrpc-core = { version = "0.13.1", path = "rpc/wrpc/core" }
-kash-wrpc-proxy = { version = "0.13.1", path = "rpc/wrpc/proxy" }
-kash-wrpc-server = { version = "0.13.1", path = "rpc/wrpc/server" }
-kash-wrpc-wasm = { version = "0.13.1", path = "rpc/wrpc/wasm" }
-kashd = { version = "0.13.1", path = "kashd" }
+mimalloc = { version = "0.1.39", default-features = false, features = [
+    'override',
+] }
+# kash-testing-integration = { version = "0.13.3", path = "testing/integration" }
+kash-addresses = { version = "0.13.3", path = "crypto/addresses" }
+kash-addressmanager = { version = "0.13.3", path = "components/addressmanager" }
+kash-bip32 = { version = "0.13.3", path = "wallet/bip32" }
+kash-cli = { version = "0.13.3", path = "cli" }
+kash-connectionmanager = { version = "0.13.3", path = "components/connectionmanager" }
+kash-consensus = { version = "0.13.3", path = "consensus" }
+kash-consensus-core = { version = "0.13.3", path = "consensus/core" }
+kash-consensus-notify = { version = "0.13.3", path = "consensus/notify" }
+kash-consensus-wasm = { version = "0.13.3", path = "consensus/wasm" }
+kash-consensusmanager = { version = "0.13.3", path = "components/consensusmanager" }
+kash-core = { version = "0.13.3", path = "core" }
+kash-daemon = { version = "0.13.3", path = "daemon" }
+kash-database = { version = "0.13.3", path = "database" }
+kash-grpc-client = { version = "0.13.3", path = "rpc/grpc/client" }
+kash-grpc-core = { version = "0.13.3", path = "rpc/grpc/core" }
+kash-grpc-server = { version = "0.13.3", path = "rpc/grpc/server" }
+kash-hashes = { version = "0.13.3", path = "crypto/hashes" }
+kash-index-core = { version = "0.13.3", path = "indexes/core" }
+kash-index-processor = { version = "0.13.3", path = "indexes/processor" }
+kash-math = { version = "0.13.3", path = "math" }
+kash-merkle = { version = "0.13.3", path = "crypto/merkle" }
+kash-metrics-core = { version = "0.13.3", path = "metrics/core" }
+kash-mining = { version = "0.13.3", path = "mining" }
+kash-mining-errors = { version = "0.13.3", path = "mining/errors" }
+kash-muhash = { version = "0.13.3", path = "crypto/muhash" }
+kash-notify = { version = "0.13.3", path = "notify" }
+kash-os = { version = "0.13.3", path = "kash-os" }
+kash-p2p-flows = { version = "0.13.3", path = "protocol/flows" }
+kash-p2p-lib = { version = "0.13.3", path = "protocol/p2p" }
+kash-perf-monitor = { version = "0.13.3", path = "metrics/perf_monitor" }
+kash-pow = { version = "0.13.3", path = "consensus/pow" }
+kash-rpc-core = { version = "0.13.3", path = "rpc/core" }
+kash-rpc-macros = { version = "0.13.3", path = "rpc/macros" }
+kash-rpc-service = { version = "0.13.3", path = "rpc/service" }
+kash-txscript = { version = "0.13.3", path = "crypto/txscript" }
+kash-txscript-errors = { version = "0.13.3", path = "crypto/txscript/errors" }
+kash-utils = { version = "0.13.3", path = "utils" }
+kash-utils-tower = { version = "0.13.3", path = "utils/tower" }
+kash-utxoindex = { version = "0.13.3", path = "indexes/utxoindex" }
+kash-wallet = { version = "0.13.3", path = "wallet/native" }
+kash-wallet-cli-wasm = { version = "0.13.3", path = "wallet/wasm" }
+kash-wallet-core = { version = "0.13.3", path = "wallet/core" }
+kash-wallet-macros = { version = "0.13.3", path = "wallet/macros" }
+kash-wasm = { version = "0.13.3", path = "wasm" }
+kash-wrpc-client = { version = "0.13.3", path = "rpc/wrpc/client" }
+kash-wrpc-core = { version = "0.13.3", path = "rpc/wrpc/core" }
+kash-wrpc-proxy = { version = "0.13.3", path = "rpc/wrpc/proxy" }
+kash-wrpc-server = { version = "0.13.3", path = "rpc/wrpc/server" }
+kash-wrpc-wasm = { version = "0.13.3", path = "rpc/wrpc/wasm" }
+kashd = { version = "0.13.3", path = "kashd" }
+kash-alloc = { version = "0.13.3", path = "utils/alloc" }
 
 # external
 aes = "0.8.3"
@@ -162,7 +167,9 @@ faster-hex = "0.6.1" # TODO "0.8.1" - fails unit tests
 fixedstr = { version = "0.5.4", features = ["serde"] }
 flate2 = "1.0.28"
 futures = { version = "0.3.29" }
-futures-util = { version = "0.3.29", default-features = false, features = [ "alloc", ] }
+futures-util = { version = "0.3.29", default-features = false, features = [
+    "alloc",
+] }
 getrandom = { version = "0.2.10", features = ["js"] }
 h2 = "0.3.21"
 heapless = "0.7.16"
@@ -237,9 +244,13 @@ web-sys = "=0.3.64"
 xxhash-rust = { version = "0.8.7", features = ["xxh3"] }
 zeroize = { version = "1.6.0", default-features = false, features = ["alloc"] }
 pin-project-lite = "0.2.13"
-tower-http = { version = "0.4.4", features = ["map-response-body", "map-request-body"] }
+tower-http = { version = "0.4.4", features = [
+    "map-response-body",
+    "map-request-body",
+] }
 tower = "0.4.7"
 hyper = "0.14.27"
+chrono = "0.4.31"
 # workflow dependencies that are not a part of core libraries
 
 # workflow-perf-monitor = { path = "../../../workflow-perf-monitor-rs" }

README.md

@@ -1,20 +1,19 @@
+# Kash On Rust: The Future of Decentralized Stablecoin Payments
 
-<h1>Kash On Rust</h1>
+⚠️ IMPORTANT: Please note that Rusty Kash is currently in the development stage and not yet in Alpha. It may exhibit instability, and significant mining activities are advised against at this stage. We recommend joining our [Kash Discord Server](https://discord.gg/qKT4dAtxEc) for updates.
 
-Welcome to the Rust-based implementation of the Kash full-node and its ancillary libraries. This Alpha release serves as a drop-in replacement to the established <a href="https://github.com/Kash-Protocol/kashd">Golang node</a> (once the rust rewrite is completed), introducing developers to the possibilities of Rust in the Kash network's context.
+Welcome to the Rust-based implementation of the Kash full-node and its ancillary libraries. Rusty Kash, a groundbreaking cryptocurrency, is developed with a focus on establishing the purest form of decentralized stablecoin payments. We are proud that Rusty Kash is developed based on Rusty Kaspa, leveraging its advanced blockchain technology. Our project is in an active development phase, ensuring continuous alignment with Rusty Kaspa's updates to guarantee state-of-the-art features and security.
 
 We invite developers and blockchain enthusiasts to collaborate, test, and optimize our Rust implementation. Each line of code here is an opportunity to contribute to the open-source blockchain movement, shaping a platform designed for scalability and speed without compromising on decentralization.
 
-Your feedback, contributions, and issue reports will be integral to evolving this codebase from its Alpha phase into a mature and reliable node in the Kash network.
-
 ## Installation
   <details>
   <summary>Building on Linux</summary>
 
-  1. Install  general prerequisites
+  1. Install general prerequisites
 
       ```bash
-      sudo apt install build-essential libssl-dev pkg-config 
+      sudo apt install curl git build-essential libssl-dev pkg-config 
       ```
 
   2. Install Protobuf (required for gRPC)

cli/src/imports.rs

@@ -6,12 +6,10 @@ pub use crate::notifier::Notification;
 pub use crate::result::Result;
 pub use crate::utils::*;
 pub use async_trait::async_trait;
-pub use borsh::{BorshDeserialize, BorshSerialize};
 pub use cfg_if::cfg_if;
 pub use futures::stream::{Stream, StreamExt, TryStreamExt};
-pub use futures::{future::FutureExt, select, Future};
+pub use futures::{future::FutureExt, select};
 pub use kash_consensus_core::network::{NetworkId, NetworkType};
-pub use kash_daemon::DaemonEvent;
 pub use kash_utils::hex::*;
 pub use kash_wallet_core::derivation::gen0::import::*;
 pub use kash_wallet_core::prelude::*;
@@ -22,15 +20,12 @@ pub use regex::Regex;
 pub use separator::Separatable;
 pub use serde::{Deserialize, Serialize};
 pub use serde_json::{to_value, Value};
-pub use std::cmp;
 pub use std::collections::HashMap;
 pub use std::collections::VecDeque;
 pub use std::ops::Deref;
-pub use std::path::{Path, PathBuf};
-pub use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-pub use std::sync::{Arc, Mutex, MutexGuard};
+pub use std::sync::atomic::{AtomicBool, Ordering};
+pub use std::sync::{Arc, Mutex};
 pub use workflow_core::prelude::*;
 pub use workflow_core::runtime as application_runtime;
 pub use workflow_log::*;
-pub use workflow_nw::ipc::result::Result as IpcResult;
 pub use workflow_terminal::prelude::*;

components/addressmanager/src/lib.rs

@@ -15,7 +15,7 @@ use itertools::{
 };
 use kash_consensus_core::config::Config;
 use kash_core::{debug, info, task::tick::TickService, time::unix_now, warn};
-use kash_database::prelude::{StoreResultExtensions, DB};
+use kash_database::prelude::{CachePolicy, StoreResultExtensions, DB};
 use kash_utils::networking::IpAddress;
 use local_ip_address::list_afinet_netifas;
 use parking_lot::Mutex;
@@ -61,7 +61,7 @@ pub struct AddressManager {
 impl AddressManager {
     pub fn new(config: Arc<Config>, db: Arc<DB>, tick_service: Arc<TickService>) -> (Arc<Mutex<Self>>, Option<Extender>) {
         let mut instance = Self {
-            banned_address_store: DbBannedAddressesStore::new(db.clone(), MAX_ADDRESSES as u64),
+            banned_address_store: DbBannedAddressesStore::new(db.clone(), CachePolicy::Count(MAX_ADDRESSES)),
             address_store: address_store_with_cache::new(db),
             local_net_addresses: Vec::new(),
             config,
@@ -337,7 +337,7 @@ mod address_store_with_cache {
     };
 
     use itertools::Itertools;
-    use kash_database::prelude::DB;
+    use kash_database::prelude::{CachePolicy, DB};
     use kash_utils::networking::PrefixBucket;
     use rand::{
         distributions::{WeightedError, WeightedIndex},
@@ -359,7 +359,8 @@ mod address_store_with_cache {
 
     impl Store {
         fn new(db: Arc<DB>) -> Self {
-            let db_store = DbAddressesStore::new(db, 0);
+            // We manage the cache ourselves on this level, so we disable the inner builtin cache
+            let db_store = DbAddressesStore::new(db, CachePolicy::Empty);
             let mut addresses = HashMap::new();
             for (key, entry) in db_store.iterator().map(|res| res.unwrap()) {
                 addresses.insert(key, entry);

components/addressmanager/src/stores/address_store.rs

@@ -1,9 +1,10 @@
 use kash_database::{
     prelude::DB,
+    prelude::{CachePolicy, StoreError, StoreResult},
     prelude::{CachedDbAccess, DirectDbWriter},
-    prelude::{StoreError, StoreResult},
     registry::DatabaseStorePrefixes,
 };
+use kash_utils::mem_size::MemSizeEstimator;
 use serde::{Deserialize, Serialize};
 use std::net::Ipv6Addr;
 use std::{error::Error, fmt::Display, sync::Arc};
@@ -17,6 +18,8 @@ pub struct Entry {
     pub address: NetAddress,
 }
 
+impl MemSizeEstimator for Entry {}
+
 pub trait AddressesStoreReader {
     fn get(&self, key: AddressKey) -> Result<Entry, StoreError>;
 }
@@ -74,8 +77,8 @@ pub struct DbAddressesStore {
 }
 
 impl DbAddressesStore {
-    pub fn new(db: Arc<DB>, cache_size: u64) -> Self {
-        Self { db: Arc::clone(&db), access: CachedDbAccess::new(db, cache_size, DatabaseStorePrefixes::Addresses.into()) }
+    pub fn new(db: Arc<DB>, cache_policy: CachePolicy) -> Self {
+        Self { db: Arc::clone(&db), access: CachedDbAccess::new(db, cache_policy, DatabaseStorePrefixes::Addresses.into()) }
     }
 
     pub fn iterator(&self) -> impl Iterator<Item = Result<(AddressKey, Entry), Box<dyn Error>>> + '_ {

components/addressmanager/src/stores/banned_address_store.rs

@@ -1,15 +1,18 @@
 use kash_database::{
+    prelude::{CachePolicy, StoreError, StoreResult},
     prelude::{CachedDbAccess, DirectDbWriter, DB},
-    prelude::{StoreError, StoreResult},
     registry::DatabaseStorePrefixes,
 };
+use kash_utils::mem_size::MemSizeEstimator;
 use serde::{Deserialize, Serialize};
 use std::net::{IpAddr, Ipv6Addr};
 use std::{error::Error, fmt::Display, sync::Arc};
 
 #[derive(Clone, Copy, Serialize, Deserialize)]
 pub struct ConnectionBanTimestamp(pub u64);
 
+impl MemSizeEstimator for ConnectionBanTimestamp {}
+
 pub trait BannedAddressesStoreReader {
     fn get(&self, address: IpAddr) -> Result<ConnectionBanTimestamp, StoreError>;
 }
@@ -70,8 +73,8 @@ pub struct DbBannedAddressesStore {
 }
 
 impl DbBannedAddressesStore {
-    pub fn new(db: Arc<DB>, cache_size: u64) -> Self {
-        Self { db: Arc::clone(&db), access: CachedDbAccess::new(db, cache_size, DatabaseStorePrefixes::BannedAddresses.into()) }
+    pub fn new(db: Arc<DB>, cache_policy: CachePolicy) -> Self {
+        Self { db: Arc::clone(&db), access: CachedDbAccess::new(db, cache_policy, DatabaseStorePrefixes::BannedAddresses.into()) }
     }
 
     pub fn iterator(&self) -> impl Iterator<Item = Result<(IpAddr, ConnectionBanTimestamp), Box<dyn Error>>> + '_ {

components/connectionmanager/src/lib.rs

@@ -127,24 +127,29 @@ impl ConnectionManager {
 
             if !is_connected && request.next_attempt <= SystemTime::now() {
                 debug!("Connecting to peer request {}", address);
-                if self.p2p_adaptor.connect_peer(address.to_string()).await.is_err() {
-                    debug!("Failed connecting to peer request {}", address);
-                    if request.is_permanent {
-                        const MAX_ACCOUNTABLE_ATTEMPTS: u32 = 4;
-                        let retry_duration = Duration::from_secs(30u64 * 2u64.pow(min(request.attempts, MAX_ACCOUNTABLE_ATTEMPTS)));
-                        debug!("Will retry peer request {} in {}", address, DurationString::from(retry_duration));
-                        new_requests.insert(
-                            address,
-                            ConnectionRequest {
-                                next_attempt: SystemTime::now() + retry_duration,
-                                attempts: request.attempts + 1,
-                                is_permanent: true,
-                            },
-                        );
+                match self.p2p_adaptor.connect_peer(address.to_string()).await {
+                    Err(err) => {
+                        debug!("Failed connecting to peer request: {}, {}", address, err);
+                        if request.is_permanent {
+                            const MAX_ACCOUNTABLE_ATTEMPTS: u32 = 4;
+                            let retry_duration =
+                                Duration::from_secs(30u64 * 2u64.pow(min(request.attempts, MAX_ACCOUNTABLE_ATTEMPTS)));
+                            debug!("Will retry peer request {} in {}", address, DurationString::from(retry_duration));
+                            new_requests.insert(
+                                address,
+                                ConnectionRequest {
+                                    next_attempt: SystemTime::now() + retry_duration,
+                                    attempts: request.attempts + 1,
+                                    is_permanent: true,
+                                },
+                            );
+                        }
+                    }
+                    Ok(_) if request.is_permanent => {
+                        // Permanent requests are kept forever
+                        new_requests.insert(address, ConnectionRequest::new(true));
                     }
-                } else if request.is_permanent {
-                    // Permanent requests are kept forever
-                    new_requests.insert(address, ConnectionRequest::new(true));
+                    Ok(_) => {}
                 }
             } else {
                 new_requests.insert(address, request);
@@ -186,7 +191,7 @@ impl ConnectionManager {
             if progressing && !jobs.is_empty() {
                 // Log only if progress was made
                 info!(
-                    "Connection manager: has {}/{} outgoing P2P connections, trying to obtain {} additional connections...",
+                    "Connection manager: has {}/{} outgoing P2P connections, trying to obtain {} additional connection(s)...",
                     self.outbound_target - missing_connections,
                     self.outbound_target,
                     jobs.len(),

components/consensusmanager/src/batch.rs

@@ -0,0 +1,30 @@
+use kash_consensus_core::{api::BlockValidationFuture, block::Block};
+use std::fmt::Debug;
+
+pub struct BlockProcessingBatch {
+    pub blocks: Vec<Block>,
+    pub block_tasks: Option<Vec<BlockValidationFuture>>,
+    pub virtual_state_tasks: Option<Vec<BlockValidationFuture>>,
+}
+
+impl BlockProcessingBatch {
+    pub fn new(blocks: Vec<Block>, block_tasks: Vec<BlockValidationFuture>, virtual_state_tasks: Vec<BlockValidationFuture>) -> Self {
+        Self { blocks, block_tasks: Some(block_tasks), virtual_state_tasks: Some(virtual_state_tasks) }
+    }
+
+    pub fn zip(self) -> impl Iterator<Item = (Block, BlockValidationFuture)> {
+        self.blocks.into_iter().zip(self.virtual_state_tasks.unwrap())
+    }
+}
+
+impl Default for BlockProcessingBatch {
+    fn default() -> Self {
+        Self { blocks: Default::default(), block_tasks: Some(Default::default()), virtual_state_tasks: Some(Default::default()) }
+    }
+}
+
+impl Debug for BlockProcessingBatch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BlockProcessingBatch").field("blocks", &self.blocks).finish()
+    }
+}