diff --git a/.github/pr-custom-review.yml b/.github/pr-custom-review.yml index 3620f7e770674..059f4a283af07 100644 --- a/.github/pr-custom-review.yml +++ b/.github/pr-custom-review.yml @@ -8,12 +8,24 @@ rules: check_type: changed_files condition: include: .* - # excluding files from 'CI team' rules - exclude: ^\.gitlab-ci\.yml|^scripts/ci/.*|^\.github/.*|^\.config/nextest.toml + # excluding files from 'CI team' and 'FRAME coders' rules + exclude: ^\.gitlab-ci\.yml|^scripts/ci/.*|^\.github/.*|^\.config/nextest.toml|^frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*)) min_approvals: 2 teams: - core-devs + - name: FRAME coders + check_type: changed_files + condition: + include: ^frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*)) + all: + - min_approvals: 2 + teams: + - core-devs + - min_approvals: 1 + teams: + - frame-coders + - name: CI team check_type: changed_files condition: diff --git a/Cargo.lock b/Cargo.lock index d9cb87af47edd..62e99d437b21a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2700,6 +2700,7 @@ dependencies = [ "sp-arithmetic", "sp-core", "sp-core-hashing-proc-macro", + "sp-debug-derive", "sp-inherents", "sp-io", "sp-runtime", @@ -5484,9 +5485,10 @@ dependencies = [ "fs_extra", "git2", "glob", + "itertools", "tar", "tempfile", - "toml 0.7.3", + "toml_edit", ] [[package]] @@ -7431,9 +7433,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd4572a52711e2ccff02b4973ec7e4a5b5c23387ebbfbd6cd42b34755714cefc" +checksum = "4890dcb9556136a4ec2b0c51fa4a08c8b733b829506af8fff2e853f3a065985b" dependencies = [ "blake2", "crc32fast", @@ -10900,7 +10902,6 @@ dependencies = [ "sp-debug-derive", "sp-externalities", "sp-runtime-interface", - "sp-serializer", "sp-std", "sp-storage", "ss58-registry", @@ -11218,14 +11219,6 @@ dependencies = [ "substrate-wasm-builder", ] -[[package]] -name = "sp-serializer" -version = "4.0.0-dev" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "sp-session" version = "4.0.0-dev" @@ -11318,6 +11311,7 @@ dependencies = [ "sp-application-crypto", "sp-core", "sp-runtime", + "sp-std", ] [[package]] @@ -12300,9 +12294,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" dependencies = [ "bitflags", "bytes", @@ -12623,7 +12617,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.6", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index 7563a4a643286..2b1659d4fb37b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,7 +219,6 @@ members = [ "primitives/runtime-interface/test", "primitives/runtime-interface/test-wasm", "primitives/runtime-interface/test-wasm-deprecated", - "primitives/serializer", "primitives/session", "primitives/staking", "primitives/state-machine", diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml index ea15fce4cbf7e..b3f1434a9c27b 100644 --- a/bin/node/bench/Cargo.toml +++ b/bin/node/bench/Cargo.toml @@ -38,7 +38,7 @@ tempfile = "3.1.0" fs_extra = "1" rand = { version = "0.8.5", features = ["small_rng"] } lazy_static = "1.4.0" -parity-db = "0.4.7" +parity-db = "0.4.8" sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } futures = { version = "0.3.21", features = ["thread-pool"] } diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 7d522b404b9c4..aeb31f9444faf 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -22,7 +22,7 @@ kvdb-memorydb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } linked-hash-map = "0.5.4" log = "0.4.17" -parity-db = "0.4.7" +parity-db = "0.4.8" parking_lot = "0.12.1" sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-state-db = { version = "0.10.0-dev", path = "../state-db" } diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 79023923e8b05..a66c187cacf7b 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -243,17 +243,17 @@ //! More precise usage details are still being worked on and will likely change in the future. mod behaviour; -mod discovery; -mod peer_info; mod protocol; mod service; -mod transport; pub mod config; +pub mod discovery; pub mod error; pub mod event; pub mod network_state; +pub mod peer_info; pub mod request_responses; +pub mod transport; pub mod types; pub mod utils; diff --git a/client/network/src/peer_info.rs b/client/network/src/peer_info.rs index e4a5c5753a00c..aab3fc9487e8c 100644 --- a/client/network/src/peer_info.rs +++ b/client/network/src/peer_info.rs @@ -16,6 +16,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! [`PeerInfoBehaviour`] is implementation of `NetworkBehaviour` that holds information about peers +//! in cache. + use crate::utils::interval; use either::Either; diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index 9e63ce98878a6..4136b34fc0e8e 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -16,6 +16,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! Transport that serves as a common ground for all connections. + use either::Either; use libp2p::{ core::{ diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index c6228d739048d..7047991126184 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -18,6 +18,6 @@ log = "0.4.17" serde_json = "1.0.85" tokio = { version = "1.22.0", features = ["parking_lot"] } prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } -tower-http = { version = "0.3.4", features = ["cors"] } +tower-http = { version = "0.4.0", features = ["cors"] } tower = "0.4.13" http = "0.2.8" diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index a33bce50aa40f..e71e211767f3d 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -173,7 +173,7 @@ fn construct_genesis_should_work_with_native() { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000 * DOLLARS, ) - .build_storage(); + .build(); let genesis_hash = insert_genesis_block(&mut storage); let backend = InMemoryBackend::from((storage, StateVersion::default())); @@ -204,7 +204,7 @@ fn construct_genesis_should_work_with_wasm() { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000 * DOLLARS, ) - .build_storage(); + .build(); let genesis_hash = insert_genesis_block(&mut storage); let backend = InMemoryBackend::from((storage, StateVersion::default())); diff --git a/client/statement-store/Cargo.toml b/client/statement-store/Cargo.toml index 5a4948cee0a6c..936aeb6e3cd82 100644 --- a/client/statement-store/Cargo.toml +++ b/client/statement-store/Cargo.toml @@ -19,7 +19,7 @@ futures = "0.3.21" futures-timer = "3.0.2" log = "0.4.17" parking_lot = "0.12.1" -parity-db = "0.4.7" +parity-db = "0.4.8" tokio = { version = "1.22.0", features = ["time"] } sp-statement-store = { version = "4.0.0-dev", path = "../../primitives/statement-store" } prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } diff --git a/client/storage-monitor/src/lib.rs b/client/storage-monitor/src/lib.rs index 277a74de08baf..655b940e8bedc 100644 --- a/client/storage-monitor/src/lib.rs +++ b/client/storage-monitor/src/lib.rs @@ -27,12 +27,15 @@ use std::{ const LOG_TARGET: &str = "storage-monitor"; +/// Result type used in this crate. +pub type Result = std::result::Result; + /// Error type used in this crate. #[derive(Debug, thiserror::Error)] pub enum Error { #[error("IO Error")] IOError(#[from] io::Error), - #[error("Out of storage space: available {0}MB, required {1}MB")] + #[error("Out of storage space: available {0}MiB, required {1}MiB")] StorageOutOfSpace(u64, u64), } @@ -42,7 +45,7 @@ pub struct StorageMonitorParams { /// Required available space on database storage. If available space for DB storage drops below /// the given threshold, node will be gracefully terminated. If `0` is given monitoring will be /// disabled. - #[arg(long = "db-storage-threshold", value_name = "MB", default_value_t = 1000)] + #[arg(long = "db-storage-threshold", value_name = "MiB", default_value_t = 1024)] pub threshold: u64, /// How often available space is polled. @@ -50,14 +53,14 @@ pub struct StorageMonitorParams { pub polling_period: u32, } -/// Storage monitor service: checks the available space for the filesystem for fiven path. +/// Storage monitor service: checks the available space for the filesystem for given path. pub struct StorageMonitorService { /// watched path path: PathBuf, /// number of megabytes that shall be free on the filesystem for watched path threshold: u64, - /// storage space polling period (seconds) - polling_period: u32, + /// storage space polling period + polling_period: Duration, } impl StorageMonitorService { @@ -66,7 +69,7 @@ impl StorageMonitorService { parameters: StorageMonitorParams, database: DatabaseSource, spawner: &impl SpawnEssentialNamed, - ) -> Result<(), Error> { + ) -> Result<()> { Ok(match (parameters.threshold, database.path()) { (0, _) => { log::info!( @@ -83,8 +86,7 @@ impl StorageMonitorService { (threshold, Some(path)) => { log::debug!( target: LOG_TARGET, - "Initializing StorageMonitorService for db path: {:?}", - path, + "Initializing StorageMonitorService for db path: {path:?}", ); Self::check_free_space(&path, threshold)?; @@ -92,7 +94,7 @@ impl StorageMonitorService { let storage_monitor_service = StorageMonitorService { path: path.to_path_buf(), threshold, - polling_period: parameters.polling_period, + polling_period: Duration::from_secs(parameters.polling_period.into()), }; spawner.spawn_essential( @@ -108,22 +110,22 @@ impl StorageMonitorService { /// below threshold. async fn run(self) { loop { - tokio::time::sleep(Duration::from_secs(self.polling_period.into())).await; + tokio::time::sleep(self.polling_period).await; if Self::check_free_space(&self.path, self.threshold).is_err() { break }; } } - /// Returns free space in MB, or error if statvfs failed. - fn free_space(path: &Path) -> Result { - Ok(fs4::available_space(path).map(|s| s / 1_000_000)?) + /// Returns free space in MiB, or error if statvfs failed. + fn free_space(path: &Path) -> Result { + Ok(fs4::available_space(path).map(|s| s / 1024 / 1024)?) } - /// Checks if the amount of free space for given `path` is above given `threshold`. + /// Checks if the amount of free space for given `path` is above given `threshold` in MiB. /// If it dropped below, error is returned. /// System errors are silently ignored. - fn check_free_space(path: &Path, threshold: u64) -> Result<(), Error> { + fn check_free_space(path: &Path, threshold: u64) -> Result<()> { match StorageMonitorService::free_space(path) { Ok(available_space) => { log::trace!( @@ -132,14 +134,14 @@ impl StorageMonitorService { ); if available_space < threshold { - log::error!(target: LOG_TARGET, "Available space {available_space}MB for path `{}` dropped below threshold: {threshold}MB , terminating...", path.display()); + log::error!(target: LOG_TARGET, "Available space {available_space}MiB for path `{}` dropped below threshold: {threshold}MiB , terminating...", path.display()); Err(Error::StorageOutOfSpace(available_space, threshold)) } else { Ok(()) } }, Err(e) => { - log::error!(target: LOG_TARGET, "Could not read available space: {:?}.", e); + log::error!(target: LOG_TARGET, "Could not read available space: {e:?}."); Err(e) }, } diff --git a/docs/node-template-release.md b/docs/node-template-release.md index 4f4977a9df03c..911e6a2bbe71a 100644 --- a/docs/node-template-release.md +++ b/docs/node-template-release.md @@ -27,33 +27,26 @@ by running the following command. delete files/directories that are removed from the source. So you need to manually check and remove them in the destination. -3. There are actually three packages in the Node Template, `node-template` (the node), -`node-template-runtime` (the runtime), and `pallet-template`, and each has its own `Cargo.toml`. -Inside these three files, dependencies are listed in expanded form and linked to a certain git -commit in Substrate remote repository, such as: +3. There is a `Cargo.toml` file in the root directory. Inside, dependencies are listed form and +linked to a certain git commit in Substrate remote repository, such as: ```toml - [dev-dependencies.sp-core] - default-features = false - git = 'https://github.com/paritytech/substrate.git' - rev = 'c1fe59d060600a10eebb4ace277af1fee20bad17' - version = '3.0.0' + sp-core = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", rev = "de80d0107336a9c7a2efdc0199015e4d67fcbdb5", default-features = false } ``` - We will update each of them to the shortened form and link them to the Rust - [crate registry](https://crates.io/). After confirming the versioned package is published in - the crate, the above will become: + We will update each of them to link to the Rust [crate registry](https://crates.io/). +After confirming the versioned package is published in the crate, the above will become: ```toml - [dev-dependencies] - sp-core = { version = '3.0.0', default-features = false } + [workspace.dependencies] + sp-core = { version = "7.0.0", default-features = false } ``` P.S: This step can be automated if we update `node-template-release` package in `scripts/ci/node-template-release`. -4. Once the three `Cargo.toml`s are updated, compile and confirm that the Node Template builds. Then -commit the changes to a new branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR. +4. Once the `Cargo.toml` is updated, compile and confirm that the Node Template builds. Then commit +the changes to a new branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR. > Note that there is a chance the code in Substrate Node Template works with the linked Substrate git commit but not with published packages due to the latest (as yet) unpublished features. In this case, diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index c25f33ae3eaf9..33aacd9323bf2 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -1654,7 +1654,7 @@ pub mod pallet { impl, I: 'static> AccountTouch for Pallet { type Balance = DepositBalanceOf; - fn deposit_required() -> Self::Balance { + fn deposit_required(_: T::AssetId) -> Self::Balance { T::AssetAccountDeposit::get() } diff --git a/frame/election-provider-multi-phase/test-staking-e2e/src/lib.rs b/frame/election-provider-multi-phase/test-staking-e2e/src/lib.rs index c396b008fa4c5..2b4a255dcf606 100644 --- a/frame/election-provider-multi-phase/test-staking-e2e/src/lib.rs +++ b/frame/election-provider-multi-phase/test-staking-e2e/src/lib.rs @@ -20,6 +20,7 @@ mod mock; pub(crate) const LOG_TARGET: &str = "tests::e2e-epm"; +use frame_support::assert_ok; use mock::*; use sp_core::Get; use sp_npos_elections::{to_supports, StakedAssignment}; @@ -204,3 +205,72 @@ fn continous_slashes_below_offending_threshold() { } }); } + +#[test] +/// When validators are slashed, they are chilled and removed from the current `VoterList`. Thus, +/// the slashed validator should not be considered in the next validator set. However, if the +/// slashed validator sets its intention to validate again in the same era when it was slashed and +/// chilled, the validator may not be removed from the active validator set across eras, provided +/// it would selected in the subsequent era if there was no slash. Nominators of the slashed +/// validator will also be slashed and chilled, as expected, but the nomination intentions will +/// remain after the validator re-set the intention to be validating again. +/// +/// This behaviour is due to removing implicit chill upon slash +/// . +/// +/// Related to . +fn set_validation_intention_after_chilled() { + use frame_election_provider_support::SortedListProvider; + use pallet_staking::{Event, Forcing, Nominators}; + + let staking_builder = StakingExtBuilder::default(); + let epm_builder = EpmExtBuilder::default(); + + ExtBuilder::default() + .staking(staking_builder) + .epm(epm_builder) + .build_and_execute(|| { + assert_eq!(active_era(), 0); + // validator is part of the validator set. + assert!(Session::validators().contains(&81)); + assert!(::VoterList::contains(&81)); + + // nominate validator 81. + assert_ok!(Staking::nominate(RuntimeOrigin::signed(21), vec![81])); + assert_eq!(Nominators::::get(21).unwrap().targets, vec![81]); + + // validator is slashed. it is removed from the `VoterList` through chilling but in the + // current era, the validator is still part of the active validator set. + add_slash(&81); + assert!(Session::validators().contains(&81)); + assert!(!::VoterList::contains(&81)); + assert_eq!( + staking_events(), + [ + Event::Chilled { stash: 81 }, + Event::ForceEra { mode: Forcing::ForceNew }, + Event::SlashReported { + validator: 81, + slash_era: 0, + fraction: Perbill::from_percent(10) + } + ], + ); + + // after the nominator is slashed and chilled, the nominations remain. + assert_eq!(Nominators::::get(21).unwrap().targets, vec![81]); + + // validator sets intention to stake again in the same era it was chilled. + assert_ok!(Staking::validate(RuntimeOrigin::signed(81), Default::default())); + + // progress era and check that the slashed validator is still part of the validator + // set. + assert!(start_next_active_era().is_ok()); + assert_eq!(active_era(), 1); + assert!(Session::validators().contains(&81)); + assert!(::VoterList::contains(&81)); + + // nominations are still active as before the slash. + assert_eq!(Nominators::::get(21).unwrap().targets, vec![81]); + }) +} diff --git a/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs b/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs index e1abbb909cbb1..490179e91ddda 100644 --- a/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs +++ b/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs @@ -124,11 +124,11 @@ impl pallet_balances::Config for Runtime { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; - type WeightInfo = (); + type MaxHolds = ConstU32<1>; + type MaxFreezes = traits::ConstU32<1>; type HoldIdentifier = (); type FreezeIdentifier = (); - type MaxHolds = traits::ConstU32<1>; - type MaxFreezes = traits::ConstU32<1>; + type WeightInfo = (); } impl pallet_timestamp::Config for Runtime { @@ -781,3 +781,11 @@ pub(crate) fn set_minimum_election_score( .map(|_| ()) .map_err(|_| ()) } + +pub(crate) fn staking_events() -> Vec> { + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| if let RuntimeEvent::Staking(inner) = e { Some(inner) } else { None }) + .collect::>() +} diff --git a/frame/ranked-collective/src/lib.rs b/frame/ranked-collective/src/lib.rs index 288fd78d6e718..6296403d2a1ce 100644 --- a/frame/ranked-collective/src/lib.rs +++ b/frame/ranked-collective/src/lib.rs @@ -112,36 +112,39 @@ impl, I: 'static, M: GetMaxVoters> Tally { pub type TallyOf = Tally>; pub type PollIndexOf = <>::Polls as Polling>>::Index; +pub type ClassOf = <>::Polls as Polling>>::Class; type AccountIdLookupOf = <::Lookup as StaticLookup>::Source; -impl, I: 'static, M: GetMaxVoters> VoteTally for Tally { - fn new(_: Rank) -> Self { +impl, I: 'static, M: GetMaxVoters>> + VoteTally> for Tally +{ + fn new(_: ClassOf) -> Self { Self { bare_ayes: 0, ayes: 0, nays: 0, dummy: PhantomData } } - fn ayes(&self, _: Rank) -> Votes { + fn ayes(&self, _: ClassOf) -> Votes { self.bare_ayes } - fn support(&self, class: Rank) -> Perbill { + fn support(&self, class: ClassOf) -> Perbill { Perbill::from_rational(self.bare_ayes, M::get_max_voters(class)) } - fn approval(&self, _: Rank) -> Perbill { + fn approval(&self, _: ClassOf) -> Perbill { Perbill::from_rational(self.ayes, 1.max(self.ayes + self.nays)) } #[cfg(feature = "runtime-benchmarks")] - fn unanimity(class: Rank) -> Self { + fn unanimity(class: ClassOf) -> Self { Self { - bare_ayes: M::get_max_voters(class), + bare_ayes: M::get_max_voters(class.clone()), ayes: M::get_max_voters(class), nays: 0, dummy: PhantomData, } } #[cfg(feature = "runtime-benchmarks")] - fn rejection(class: Rank) -> Self { + fn rejection(class: ClassOf) -> Self { Self { bare_ayes: 0, ayes: 0, nays: M::get_max_voters(class), dummy: PhantomData } } #[cfg(feature = "runtime-benchmarks")] - fn from_requirements(support: Perbill, approval: Perbill, class: Rank) -> Self { + fn from_requirements(support: Perbill, approval: Perbill, class: ClassOf) -> Self { let c = M::get_max_voters(class); let ayes = support * c; let nays = ((ayes as u64) * 1_000_000_000u64 / approval.deconstruct() as u64) as u32 - ayes; @@ -149,14 +152,17 @@ impl, I: 'static, M: GetMaxVoters> VoteTally for Tally } #[cfg(feature = "runtime-benchmarks")] - fn setup(class: Rank, granularity: Perbill) { - if M::get_max_voters(class) == 0 { + fn setup(class: ClassOf, granularity: Perbill) { + if M::get_max_voters(class.clone()) == 0 { let max_voters = granularity.saturating_reciprocal_mul(1u32); for i in 0..max_voters { let who: T::AccountId = frame_benchmarking::account("ranked_collective_benchmarking", i, 0); - crate::Pallet::::do_add_member_to_rank(who, class) - .expect("could not add members for benchmarks"); + crate::Pallet::::do_add_member_to_rank( + who, + T::MinRankOfClass::convert(class.clone()), + ) + .expect("could not add members for benchmarks"); } assert_eq!(M::get_max_voters(class), max_voters); } @@ -234,14 +240,17 @@ impl Convert for Geometric { } } -/// Trait for getting the maximum number of voters for a given rank. +/// Trait for getting the maximum number of voters for a given poll class. pub trait GetMaxVoters { - /// Return the maximum number of voters for the rank `r`. - fn get_max_voters(r: Rank) -> MemberIndex; + /// Poll class type. + type Class; + /// Return the maximum number of voters for the poll class `c`. + fn get_max_voters(c: Self::Class) -> MemberIndex; } impl, I: 'static> GetMaxVoters for Pallet { - fn get_max_voters(r: Rank) -> MemberIndex { - MemberCount::::get(r) + type Class = ClassOf; + fn get_max_voters(c: Self::Class) -> MemberIndex { + MemberCount::::get(T::MinRankOfClass::convert(c)) } } @@ -346,7 +355,7 @@ pub mod pallet { /// Convert the tally class into the minimum rank required to vote on the poll. If /// `Polls::Class` is the same type as `Rank`, then `Identity` can be used here to mean /// "a rank of at least the poll class". - type MinRankOfClass: Convert<>>::Class, Rank>; + type MinRankOfClass: Convert, Rank>; /// Convert a rank_delta into a number of votes the rank gets. /// diff --git a/frame/ranked-collective/src/tests.rs b/frame/ranked-collective/src/tests.rs index 91244a90b0b6a..04519bc0f8e22 100644 --- a/frame/ranked-collective/src/tests.rs +++ b/frame/ranked-collective/src/tests.rs @@ -25,10 +25,10 @@ use frame_support::{ parameter_types, traits::{ConstU16, ConstU32, ConstU64, EitherOf, Everything, MapSuccess, Polling}, }; -use sp_core::H256; +use sp_core::{Get, H256}; use sp_runtime::{ testing::Header, - traits::{BlakeTwo256, Identity, IdentityLookup, ReduceBy}, + traits::{BlakeTwo256, IdentityLookup, ReduceBy}, }; use super::*; @@ -36,6 +36,7 @@ use crate as pallet_ranked_collective; type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; +type Class = Rank; frame_support::construct_runtime!( pub enum Test where @@ -95,7 +96,7 @@ impl Polling> for TestPolls { type Index = u8; type Votes = Votes; type Moment = u64; - type Class = Rank; + type Class = Class; fn classes() -> Vec { vec![0, 1, 2] } @@ -164,6 +165,19 @@ impl Polling> for TestPolls { } } +/// Convert the tally class into the minimum rank required to vote on the poll. +/// MinRank(Class) = Class - Delta +pub struct MinRankOfClass(PhantomData); +impl> Convert for MinRankOfClass { + fn convert(a: Class) -> Rank { + a.saturating_sub(Delta::get()) + } +} + +parameter_types! { + pub static MinRankOfClassDelta: Rank = 0; +} + impl Config for Test { type WeightInfo = (); type RuntimeEvent = RuntimeEvent; @@ -180,7 +194,7 @@ impl Config for Test { MapSuccess, ReduceBy>>, >; type Polls = TestPolls; - type MinRankOfClass = Identity; + type MinRankOfClass = MinRankOfClass; type VoteWeight = Geometric; } @@ -499,3 +513,43 @@ fn do_add_member_to_rank_works() { assert_eq!(member_count(max_rank + 1), 0); }) } + +#[test] +fn tally_support_correct() { + new_test_ext().execute_with(|| { + // add members, + // rank 1: accounts 1, 2, 3 + // rank 2: accounts 2, 3 + // rank 3: accounts 3. + assert_ok!(Club::add_member(RuntimeOrigin::root(), 1)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 1)); + assert_ok!(Club::add_member(RuntimeOrigin::root(), 2)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 2)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 2)); + assert_ok!(Club::add_member(RuntimeOrigin::root(), 3)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 3)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 3)); + assert_ok!(Club::promote_member(RuntimeOrigin::root(), 3)); + + // init tally with 1 aye vote. + let tally: TallyOf = Tally::from_parts(1, 1, 0); + + // with minRank(Class) = Class + // for class 3, 100% support. + MinRankOfClassDelta::set(0); + assert_eq!(tally.support(3), Perbill::from_rational(1u32, 1)); + + // with minRank(Class) = (Class - 1) + // for class 3, ~50% support. + MinRankOfClassDelta::set(1); + assert_eq!(tally.support(3), Perbill::from_rational(1u32, 2)); + + // with minRank(Class) = (Class - 2) + // for class 3, ~33% support. + MinRankOfClassDelta::set(2); + assert_eq!(tally.support(3), Perbill::from_rational(1u32, 3)); + + // reset back. + MinRankOfClassDelta::set(0); + }); +} diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 1d9838572857d..82a0956da7b61 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -669,12 +669,12 @@ impl Pallet { /// Clear all era information for given era. pub(crate) fn clear_era_information(era_index: EraIndex) { - #[allow(deprecated)] - >::remove_prefix(era_index, None); - #[allow(deprecated)] - >::remove_prefix(era_index, None); - #[allow(deprecated)] - >::remove_prefix(era_index, None); + let mut cursor = >::clear_prefix(era_index, u32::MAX, None); + debug_assert!(cursor.maybe_cursor.is_none()); + cursor = >::clear_prefix(era_index, u32::MAX, None); + debug_assert!(cursor.maybe_cursor.is_none()); + cursor = >::clear_prefix(era_index, u32::MAX, None); + debug_assert!(cursor.maybe_cursor.is_none()); >::remove(era_index); >::remove(era_index); >::remove(era_index); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index a725d43515b4c..c5c9c669d2b48 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -631,7 +631,7 @@ pub mod pallet { MaxNominatorsCount::::put(x); } - for &(ref stash, ref controller, balance, ref status) in &self.stakers { + for &(ref stash, _, balance, ref status) in &self.stakers { crate::log!( trace, "inserting genesis staker: {:?} => {:?} => {:?}", @@ -650,11 +650,11 @@ pub mod pallet { )); frame_support::assert_ok!(match status { crate::StakerStatus::Validator => >::validate( - T::RuntimeOrigin::from(Some(controller.clone()).into()), + T::RuntimeOrigin::from(Some(stash.clone()).into()), Default::default(), ), crate::StakerStatus::Nominator(votes) => >::nominate( - T::RuntimeOrigin::from(Some(controller.clone()).into()), + T::RuntimeOrigin::from(Some(stash.clone()).into()), votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), ), _ => Ok(()), diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index d8b1b9c248720..6ed67c8081b5b 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -27,6 +27,7 @@ sp-arithmetic = { version = "6.0.0", default-features = false, path = "../../pri sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../primitives/inherents" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-weights = { version = "4.0.0", default-features = false, path = "../../primitives/weights" } +sp-debug-derive = { default-features = false, path = "../../primitives/debug-derive" } tt-call = "1.0.8" frame-support-procedural = { version = "4.0.0-dev", default-features = false, path = "./procedural" } paste = "1.0" @@ -75,7 +76,9 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "sp-staking/runtime-benchmarks" ] -try-runtime = [] +try-runtime = [ + "sp-debug-derive/force-debug" +] # By default some types have documentation, `no-metadata-docs` allows to reduce the documentation # in the metadata. no-metadata-docs = ["frame-support-procedural/no-metadata-docs", "sp-api/no-metadata-docs"] diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 895b09a17e083..b8aa5674ddde5 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -648,12 +648,13 @@ pub fn derive_debug_no_bound(input: TokenStream) -> TokenStream { } /// Derive [`Debug`], if `std` is enabled it uses `frame_support::DebugNoBound`, if `std` is not -/// enabled it just returns `""`. +/// enabled it just returns `""`. /// This behaviour is useful to prevent bloating the runtime WASM blob from unneeded code. #[proc_macro_derive(RuntimeDebugNoBound)] pub fn derive_runtime_debug_no_bound(input: TokenStream) -> TokenStream { - #[cfg(not(feature = "std"))] - { + if cfg!(any(feature = "std", feature = "try-runtime")) { + debug_no_bound::derive_debug_no_bound(input) + } else { let input: syn::DeriveInput = match syn::parse(input) { Ok(input) => input, Err(e) => return e.to_compile_error().into(), @@ -666,18 +667,13 @@ pub fn derive_runtime_debug_no_bound(input: TokenStream) -> TokenStream { const _: () = { impl #impl_generics core::fmt::Debug for #name #ty_generics #where_clause { fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { - fmt.write_str("") + fmt.write_str("") } } }; ) .into() } - - #[cfg(feature = "std")] - { - debug_no_bound::derive_debug_no_bound(input) - } } /// Derive [`PartialEq`] but do not bound any generic. Docs are at diff --git a/frame/support/procedural/src/pallet/expand/genesis_config.rs b/frame/support/procedural/src/pallet/expand/genesis_config.rs index 85ad20dd45c9e..7c66ef7d355d4 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_config.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_config.rs @@ -89,7 +89,7 @@ pub fn expand_genesis_config(def: &mut Def) -> proc_macro2::TokenStream { attrs.push(syn::parse_quote!( #[doc = r" Can be used to configure the - [genesis state](https://docs.substrate.io/v3/runtime/chain-specs#the-genesis-state) + [genesis state](https://docs.substrate.io/build/genesis-configuration/) of this pallet. "] )); diff --git a/frame/support/src/traits/misc.rs b/frame/support/src/traits/misc.rs index e21fcfbb24179..a6f8c46d63951 100644 --- a/frame/support/src/traits/misc.rs +++ b/frame/support/src/traits/misc.rs @@ -1160,8 +1160,8 @@ pub trait AccountTouch { /// The type for currency units of the deposit. type Balance; - /// The deposit amount of a native currency required for creating an asset account. - fn deposit_required() -> Self::Balance; + /// The deposit amount of a native currency required for creating an account of the `asset`. + fn deposit_required(asset: AssetId) -> Self::Balance; /// Create an account for `who` of the `asset` with a deposit taken from the `depositor`. fn touch(asset: AssetId, who: AccountId, depositor: AccountId) -> DispatchResult; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 1b830105bb154..7a97ace736bb4 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -66,6 +66,7 @@ #[cfg(feature = "std")] use serde::Serialize; +use sp_io::hashing::blake2_256; #[cfg(feature = "runtime-benchmarks")] use sp_runtime::traits::TrailingZeroInput; use sp_runtime::{ @@ -1316,6 +1317,8 @@ impl Pallet { // populate environment ExecutionPhase::::put(Phase::Initialization); storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &0u32); + let entropy = (b"frame_system::initialize", parent_hash).using_encoded(blake2_256); + storage::unhashed::put_raw(well_known_keys::INTRABLOCK_ENTROPY, &entropy[..]); >::put(number); >::put(digest); >::put(parent_hash); @@ -1365,6 +1368,7 @@ impl Pallet { ); ExecutionPhase::::kill(); AllExtrinsicsLen::::kill(); + storage::unhashed::kill(well_known_keys::INTRABLOCK_ENTROPY); // The following fields // @@ -1614,25 +1618,35 @@ impl Pallet { .ok_or(Error::::FailedToExtractRuntimeVersion)?; cfg_if::cfg_if! { - if #[cfg(all(feature = "runtime-benchmarks", not(test)))] { + if #[cfg(all(feature = "runtime-benchmarks", not(test)))] { // Let's ensure the compiler doesn't optimize our fetching of the runtime version away. core::hint::black_box((new_version, current_version)); Ok(()) - } else { - if new_version.spec_name != current_version.spec_name { - return Err(Error::::InvalidSpecName.into()) - } + } else { + if new_version.spec_name != current_version.spec_name { + return Err(Error::::InvalidSpecName.into()) + } - if new_version.spec_version <= current_version.spec_version { - return Err(Error::::SpecVersionNeedsToIncrease.into()) - } + if new_version.spec_version <= current_version.spec_version { + return Err(Error::::SpecVersionNeedsToIncrease.into()) + } - Ok(()) - } + Ok(()) + } } } } +/// Returns a 32 byte datum which is guaranteed to be universally unique. `entropy` is provided +/// as a facility to reduce the potential for precalculating results. +pub fn unique(entropy: impl Encode) -> [u8; 32] { + let mut last = [0u8; 32]; + sp_io::storage::read(well_known_keys::INTRABLOCK_ENTROPY, &mut last[..], 0); + let next = (b"frame_system::unique", entropy, last).using_encoded(blake2_256); + sp_io::storage::set(well_known_keys::INTRABLOCK_ENTROPY, &next); + next +} + /// Event handler which registers a provider when created. pub struct Provider(PhantomData); impl HandleLifetime for Provider { diff --git a/frame/system/src/tests.rs b/frame/system/src/tests.rs index dc245f0642e2f..05a7e96fdecb1 100644 --- a/frame/system/src/tests.rs +++ b/frame/system/src/tests.rs @@ -56,6 +56,43 @@ fn origin_works() { assert_eq!(x.unwrap(), RawOrigin::::Signed(1u64)); } +#[test] +fn unique_datum_works() { + new_test_ext().execute_with(|| { + System::initialize(&1, &[0u8; 32].into(), &Default::default()); + assert!(sp_io::storage::exists(well_known_keys::INTRABLOCK_ENTROPY)); + + let h1 = unique(b""); + assert_eq!( + 32, + sp_io::storage::read(well_known_keys::INTRABLOCK_ENTROPY, &mut [], 0).unwrap() + ); + let h2 = unique(b""); + assert_eq!( + 32, + sp_io::storage::read(well_known_keys::INTRABLOCK_ENTROPY, &mut [], 0).unwrap() + ); + assert_ne!(h1, h2); + + let h3 = unique(b"Hello"); + assert_eq!( + 32, + sp_io::storage::read(well_known_keys::INTRABLOCK_ENTROPY, &mut [], 0).unwrap() + ); + assert_ne!(h2, h3); + + let h4 = unique(b"Hello"); + assert_eq!( + 32, + sp_io::storage::read(well_known_keys::INTRABLOCK_ENTROPY, &mut [], 0).unwrap() + ); + assert_ne!(h3, h4); + + System::finalize(); + assert!(!sp_io::storage::exists(well_known_keys::INTRABLOCK_ENTROPY)); + }); +} + #[test] fn stored_map_works() { new_test_ext().execute_with(|| { diff --git a/primitives/application-crypto/Cargo.toml b/primitives/application-crypto/Cargo.toml index 37b58bc2bf649..dc97948fdf5c8 100644 --- a/primitives/application-crypto/Cargo.toml +++ b/primitives/application-crypto/Cargo.toml @@ -18,7 +18,7 @@ targets = ["x86_64-unknown-linux-gnu"] sp-core = { version = "7.0.0", default-features = false, path = "../core" } codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", optional = true, features = ["derive"] } +serde = { version = "1.0.136", default-features = false, optional = true, features = ["derive", "alloc"] } sp-std = { version = "5.0.0", default-features = false, path = "../std" } sp-io = { version = "7.0.0", default-features = false, path = "../io" } @@ -29,11 +29,18 @@ std = [ "sp-core/std", "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-std/std", "sp-io/std", ] +# Serde support without relying on std features. +serde = [ + "dep:serde", + "sp-core/serde", + "scale-info/serde", +] + # This feature enables all crypto primitives for `no_std` builds like microcontrollers # or Intel SGX. # For the regular wasm runtime builds this should not be used. diff --git a/primitives/application-crypto/src/lib.rs b/primitives/application-crypto/src/lib.rs index 3e8f2f5a77b3a..11be54e29c702 100644 --- a/primitives/application-crypto/src/lib.rs +++ b/primitives/application-crypto/src/lib.rs @@ -23,20 +23,25 @@ pub use sp_core::crypto::{key_types, CryptoTypeId, KeyTypeId}; #[doc(hidden)] #[cfg(feature = "full_crypto")] -pub use sp_core::crypto::{DeriveError, DeriveJunction, Pair, SecretStringError, Ss58Codec}; +pub use sp_core::crypto::{DeriveError, Pair, SecretStringError}; +#[cfg(any(feature = "full_crypto", feature = "serde"))] +pub use sp_core::crypto::{DeriveJunction, Ss58Codec}; #[doc(hidden)] pub use sp_core::{ self, crypto::{ByteArray, CryptoType, Derive, IsWrappedBy, Public, UncheckedFrom, Wraps}, RuntimeDebug, }; +#[doc(hidden)] +#[cfg(all(not(feature = "std"), feature = "serde"))] +pub use sp_std::alloc::{format, string::String}; #[doc(hidden)] pub use codec; #[doc(hidden)] pub use scale_info; #[doc(hidden)] -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub use serde; #[doc(hidden)] pub use sp_std::{ops::Deref, vec::Vec}; @@ -282,7 +287,7 @@ macro_rules! app_crypto_public_not_full_crypto { #[macro_export] macro_rules! app_crypto_public_common { ($public:ty, $sig:ty, $key_type:expr, $crypto_type:expr) => { - $crate::app_crypto_public_common_if_std!(); + $crate::app_crypto_public_common_if_serde!(); impl AsRef<[u8]> for Public { fn as_ref(&self) -> &[u8] { @@ -323,11 +328,11 @@ macro_rules! app_crypto_public_common { }; } -/// Implements traits for the public key type if `feature = "std"` is enabled. -#[cfg(feature = "std")] +/// Implements traits for the public key type if `feature = "serde"` is enabled. +#[cfg(feature = "serde")] #[doc(hidden)] #[macro_export] -macro_rules! app_crypto_public_common_if_std { +macro_rules! app_crypto_public_common_if_serde { () => { impl $crate::Derive for Public { fn derive>( @@ -338,15 +343,15 @@ macro_rules! app_crypto_public_common_if_std { } } - impl std::fmt::Display for Public { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + impl core::fmt::Display for Public { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { use $crate::Ss58Codec; write!(f, "{}", self.0.to_ss58check()) } } impl $crate::serde::Serialize for Public { - fn serialize(&self, serializer: S) -> std::result::Result + fn serialize(&self, serializer: S) -> core::result::Result where S: $crate::serde::Serializer, { @@ -356,11 +361,14 @@ macro_rules! app_crypto_public_common_if_std { } impl<'de> $crate::serde::Deserialize<'de> for Public { - fn deserialize(deserializer: D) -> std::result::Result + fn deserialize(deserializer: D) -> core::result::Result where D: $crate::serde::Deserializer<'de>, { use $crate::Ss58Codec; + #[cfg(all(not(feature = "std"), feature = "serde"))] + use $crate::{format, String}; + Public::from_ss58check(&String::deserialize(deserializer)?) .map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e))) } @@ -368,10 +376,10 @@ macro_rules! app_crypto_public_common_if_std { }; } -#[cfg(not(feature = "std"))] +#[cfg(not(feature = "serde"))] #[doc(hidden)] #[macro_export] -macro_rules! app_crypto_public_common_if_std { +macro_rules! app_crypto_public_common_if_serde { () => { impl $crate::Derive for Public {} }; diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index 92eebd4a6d8d6..122691623b8fd 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -21,7 +21,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", default-features = integer-sqrt = "0.1.2" num-traits = { version = "0.2.8", default-features = false } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } static_assertions = "1.1.0" sp-std = { version = "5.0.0", default-features = false, path = "../std" } @@ -37,9 +37,14 @@ std = [ "codec/std", "num-traits/std", "scale-info/std", - "serde", + "serde/std", "sp-std/std", ] +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", +] [[bench]] name = "bench" diff --git a/primitives/arithmetic/src/fixed_point.rs b/primitives/arithmetic/src/fixed_point.rs index 67fbd5bc79c88..08b788e9abd76 100644 --- a/primitives/arithmetic/src/fixed_point.rs +++ b/primitives/arithmetic/src/fixed_point.rs @@ -32,9 +32,12 @@ use sp_std::{ prelude::*, }; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::string::{String, ToString}; + /// Integer types that can be used to interact with `FixedPointNumber` implementations. pub trait FixedPointOperand: Copy @@ -928,14 +931,12 @@ macro_rules! implement_fixed { } } - #[cfg(feature = "std")] impl sp_std::fmt::Display for $name { fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { write!(f, "{}", self.0) } } - #[cfg(feature = "std")] impl sp_std::str::FromStr for $name { type Err = &'static str; @@ -948,7 +949,7 @@ macro_rules! implement_fixed { // Manual impl `Serialize` as serde_json does not support i128. // TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] impl Serialize for $name { fn serialize(&self, serializer: S) -> Result where @@ -960,7 +961,7 @@ macro_rules! implement_fixed { // Manual impl `Deserialize` as serde_json does not support i128. // TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] impl<'de> Deserialize<'de> for $name { fn deserialize(deserializer: D) -> Result where diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index e7ba08a532909..d2eceafab5ed0 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -53,12 +53,12 @@ use traits::{BaseArithmetic, One, SaturatedConversion, Unsigned, Zero}; use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; /// Arithmetic errors. #[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, MaxEncodedLen)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum ArithmeticError { /// Underflow. Underflow, diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index 611a4e05e1027..fe88b72e24c22 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use crate::traits::{ @@ -556,7 +556,7 @@ macro_rules! implement_per_thing { /// A fixed point representation of a number in the range [0, 1]. /// #[doc = $title] - #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Encode, Copy, Clone, PartialEq, Eq, codec::MaxEncodedLen, PartialOrd, Ord, scale_info::TypeInfo)] pub struct $name($type); diff --git a/primitives/consensus/babe/Cargo.toml b/primitives/consensus/babe/Cargo.toml index 7e171811a7fe2..5e57f276aac8b 100644 --- a/primitives/consensus/babe/Cargo.toml +++ b/primitives/consensus/babe/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] async-trait = { version = "0.1.57", optional = true } codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } sp-application-crypto = { version = "7.0.0", default-features = false, path = "../../application-crypto" } sp-consensus = { version = "0.10.0-dev", optional = true, path = "../common" } @@ -34,7 +34,7 @@ std = [ "async-trait", "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-api/std", "sp-application-crypto/std", "sp-consensus", @@ -46,3 +46,13 @@ std = [ "sp-std/std", "sp-timestamp", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-application-crypto/serde", + "sp-consensus-slots/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/consensus/babe/src/lib.rs b/primitives/consensus/babe/src/lib.rs index dc161525a8511..c083bfd9a313e 100644 --- a/primitives/consensus/babe/src/lib.rs +++ b/primitives/consensus/babe/src/lib.rs @@ -25,7 +25,7 @@ pub mod inherents; use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sp_runtime::{traits::Header, ConsensusEngineId, RuntimeDebug}; use sp_std::vec::Vec; @@ -217,7 +217,7 @@ impl BabeConfiguration { /// Types of allowed slots. #[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, MaxEncodedLen, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum AllowedSlots { /// Only allow primary slots. PrimarySlots, @@ -241,7 +241,7 @@ impl AllowedSlots { /// Configuration data used by the BABE consensus engine that may change with epochs. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug, MaxEncodedLen, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct BabeEpochConfiguration { /// A constant value that is used in the threshold calculation formula. /// Expressed as a rational where the first member of the tuple is the diff --git a/primitives/consensus/beefy/Cargo.toml b/primitives/consensus/beefy/Cargo.toml index 144c8452d71b0..cf5f660b22007 100644 --- a/primitives/consensus/beefy/Cargo.toml +++ b/primitives/consensus/beefy/Cargo.toml @@ -13,8 +13,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", optional = true, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", default-features = false, optional = true, features = ["derive", "alloc"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } sp-application-crypto = { version = "7.0.0", default-features = false, path = "../../application-crypto" } sp-core = { version = "7.0.0", default-features = false, path = "../../core" } @@ -34,7 +34,7 @@ default = ["std"] std = [ "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-api/std", "sp-application-crypto/std", "sp-core/std", @@ -43,3 +43,12 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-application-crypto/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/consensus/beefy/src/mmr.rs b/primitives/consensus/beefy/src/mmr.rs index 465008dc22499..c303cae2fdcc7 100644 --- a/primitives/consensus/beefy/src/mmr.rs +++ b/primitives/consensus/beefy/src/mmr.rs @@ -101,7 +101,7 @@ impl MmrLeafVersion { /// Details of a BEEFY authority set. #[derive(Debug, Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo, MaxEncodedLen)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct BeefyAuthoritySet { /// Id of the set. /// diff --git a/primitives/consensus/grandpa/Cargo.toml b/primitives/consensus/grandpa/Cargo.toml index 446471da8361a..a1846aa76ac1f 100644 --- a/primitives/consensus/grandpa/Cargo.toml +++ b/primitives/consensus/grandpa/Cargo.toml @@ -18,7 +18,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", default-features = grandpa = { package = "finality-grandpa", version = "0.16.2", default-features = false, features = ["derive-codec"] } log = { version = "0.4.17", default-features = false } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", features = ["derive", "alloc"], default-features = false, optional = true } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } sp-application-crypto = { version = "7.0.0", default-features = false, path = "../../application-crypto" } sp-core = { version = "7.0.0", default-features = false, path = "../../core" } @@ -33,7 +33,7 @@ std = [ "grandpa/std", "log/std", "scale-info/std", - "serde", + "serde/std", "sp-api/std", "sp-application-crypto/std", "sp-core/std", @@ -41,3 +41,12 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-application-crypto/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/consensus/grandpa/src/lib.rs b/primitives/consensus/grandpa/src/lib.rs index 728ce3092888b..baeaee4738e48 100644 --- a/primitives/consensus/grandpa/src/lib.rs +++ b/primitives/consensus/grandpa/src/lib.rs @@ -22,7 +22,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::Serialize; use codec::{Codec, Decode, Encode, Input}; @@ -140,8 +140,8 @@ pub struct GrandpaJustification { } /// A scheduled change of authority set. -#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] +#[cfg_attr(feature = "serde", derive(Serialize))] pub struct ScheduledChange { /// The new authorities after the change, along with their respective weights. pub next_authorities: AuthorityList, @@ -150,8 +150,8 @@ pub struct ScheduledChange { } /// An consensus log item for GRANDPA. -#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)] +#[cfg_attr(feature = "serde", derive(Serialize))] pub enum ConsensusLog { /// Schedule an authority set change. /// diff --git a/primitives/consensus/slots/Cargo.toml b/primitives/consensus/slots/Cargo.toml index a4d85dff6ea85..29aa3c10c2f55 100644 --- a/primitives/consensus/slots/Cargo.toml +++ b/primitives/consensus/slots/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive", "max-encoded-len"] } scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } -serde = { version = "1.0", features = ["derive"], optional = true } +serde = { version = "1.0", default-features = false, features = ["derive", "alloc"], optional = true } sp-std = { version = "5.0.0", default-features = false, path = "../../std" } sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../timestamp" } @@ -24,7 +24,13 @@ default = ["std"] std = [ "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-std/std", "sp-timestamp/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", +] diff --git a/primitives/consensus/slots/src/lib.rs b/primitives/consensus/slots/src/lib.rs index 1f2fa585df7f2..30bb42e2c7589 100644 --- a/primitives/consensus/slots/src/lib.rs +++ b/primitives/consensus/slots/src/lib.rs @@ -25,7 +25,7 @@ use sp_timestamp::Timestamp; /// Unit type wrapper that represents a slot. #[derive(Debug, Encode, MaxEncodedLen, Decode, Eq, Clone, Copy, Default, Ord, TypeInfo)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Slot(u64); impl core::ops::Deref for Slot { diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 0c203a03d904d..d0ec98b1c3080 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -16,10 +16,10 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive","max-encoded-len"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } log = { version = "0.4.17", default-features = false } -serde = { version = "1.0.136", optional = true, features = ["derive"] } +serde = { version = "1.0.136", optional = true, default-features = false, features = ["derive", "alloc"] } bounded-collections = { version = "0.1.4", default-features = false } primitive-types = { version = "0.12.0", default-features = false, features = ["codec", "scale-info"] } -impl-serde = { version = "0.4.0", optional = true } +impl-serde = { version = "0.4.0", default-features = false, optional = true } hash-db = { version = "0.16.0", default-features = false } hash256-std-hasher = { version = "0.15.2", default-features = false } bs58 = { version = "0.4.0", default-features = false, optional = true } @@ -31,6 +31,7 @@ zeroize = { version = "1.4.3", default-features = false } secrecy = { version = "0.8.0", default-features = false } lazy_static = { version = "1.4.0", default-features = false, optional = true } parking_lot = { version = "0.12.1", optional = true } +ss58-registry = { version = "1.34.0", default-features = false } sp-std = { version = "5.0.0", default-features = false, path = "../std" } sp-debug-derive = { version = "5.0.0", default-features = false, path = "../debug-derive" } sp-storage = { version = "7.0.0", default-features = false, path = "../storage" } @@ -49,14 +50,12 @@ libsecp256k1 = { version = "0.7", default-features = false, features = ["static- schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false } merlin = { version = "2.0", default-features = false } secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"], optional = true } -ss58-registry = { version = "1.34.0", default-features = false } sp-core-hashing = { version = "5.0.0", path = "./hashing", default-features = false, optional = true } sp-runtime-interface = { version = "7.0.0", default-features = false, path = "../runtime-interface" } # bls crypto w3f-bls = { version = "0.1.3", default-features = false, optional = true} [dev-dependencies] -sp-serializer = { version = "4.0.0-dev", path = "../serializer" } rand = "0.8.5" criterion = "0.4.0" serde_json = "1.0" @@ -84,13 +83,13 @@ std = [ "primitive-types/serde", "primitive-types/byteorder", "primitive-types/rustc-hex", - "impl-serde", + "impl-serde/std", "codec/std", "scale-info/std", "hash256-std-hasher/std", "hash-db/std", "sp-std/std", - "serde", + "serde/std", "blake2/std", "array-bytes", "ed25519-zebra/std", @@ -116,6 +115,20 @@ std = [ "dyn-clonable", ] +# Serde support without relying on std features. +serde = [ + "dep:serde", + "array-bytes", + "blake2", + "bs58/alloc", + "scale-info/serde", + "secrecy/alloc", + "impl-serde", + "primitive-types/serde_no_std", + "sp-storage/serde", + "sp-core-hashing", +] + # This feature enables all crypto primitives for `no_std` builds like microcontrollers # or Intel SGX. # For the regular wasm runtime builds this should not be used. diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index 27b24057b0e80..5947603cd0942 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -33,13 +33,16 @@ pub use secrecy::{ExposeSecret, SecretString}; use sp_runtime_interface::pass_by::PassByInner; #[doc(hidden)] pub use sp_std::ops::Deref; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::{ + alloc::{format, string::String}, + vec, +}; use sp_std::{hash::Hash, str, vec::Vec}; +pub use ss58_registry::{from_known_address_format, Ss58AddressFormat, Ss58AddressFormatRegistry}; /// Trait to zeroize a memory buffer. pub use zeroize::Zeroize; -#[cfg(feature = "full_crypto")] -pub use ss58_registry::{from_known_address_format, Ss58AddressFormat, Ss58AddressFormatRegistry}; - /// The root phrase for our publicly known keys. pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk"; @@ -49,7 +52,6 @@ pub const DEV_ADDRESS: &str = "5DfhGyQdFobKM8NsWvEeAKk5EQQgYe9AydgJ7rMB6E1EqRzV" /// The length of the junction identifier. Note that this is also referred to as the /// `CHAIN_CODE_LENGTH` in the context of Schnorrkel. -#[cfg(feature = "full_crypto")] pub const JUNCTION_ID_LEN: usize = 32; /// Similar to `From`, except that the onus is on the part of the caller to ensure @@ -113,7 +115,7 @@ pub enum DeriveError { /// a new secret key from an existing secret key and, in the case of `SoftRaw` and `SoftIndex` /// a new public key from an existing public key. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Encode, Decode)] -#[cfg(feature = "full_crypto")] +#[cfg(any(feature = "full_crypto", feature = "serde"))] pub enum DeriveJunction { /// Soft (vanilla) derivation. Public keys have a correspondent derivation. Soft([u8; JUNCTION_ID_LEN]), @@ -121,7 +123,7 @@ pub enum DeriveJunction { Hard([u8; JUNCTION_ID_LEN]), } -#[cfg(feature = "full_crypto")] +#[cfg(any(feature = "full_crypto", feature = "serde"))] impl DeriveJunction { /// Consume self to return a soft derive junction with the same chain code. pub fn soften(self) -> Self { @@ -180,7 +182,7 @@ impl DeriveJunction { } } -#[cfg(feature = "full_crypto")] +#[cfg(any(feature = "full_crypto", feature = "serde"))] impl> From for DeriveJunction { fn from(j: T) -> DeriveJunction { let j = j.as_ref(); @@ -208,7 +210,7 @@ impl> From for DeriveJunction { #[cfg_attr(not(feature = "std"), derive(Debug))] #[derive(Clone, Copy, Eq, PartialEq)] #[allow(missing_docs)] -#[cfg(feature = "full_crypto")] +#[cfg(any(feature = "full_crypto", feature = "serde"))] pub enum PublicError { #[cfg_attr(feature = "std", error("Base 58 requirement is violated"))] BadBase58, @@ -247,7 +249,6 @@ impl sp_std::fmt::Debug for PublicError { /// /// See /// for information on the codec. -#[cfg(feature = "full_crypto")] pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { /// A format filterer, can be used to ensure that `from_ss58check` family only decode for /// allowed identifiers. By default just refuses the two reserved identifiers. @@ -256,7 +257,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { } /// Some if the string is a properly encoded SS58Check address. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn from_ss58check(s: &str) -> Result { Self::from_ss58check_with_version(s).and_then(|(r, v)| match v { v if !v.is_custom() => Ok(r), @@ -266,7 +267,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { } /// Some if the string is a properly encoded SS58Check address. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn from_ss58check_with_version(s: &str) -> Result<(Self, Ss58AddressFormat), PublicError> { const CHECKSUM_LEN: usize = 2; let body_len = Self::LEN; @@ -321,7 +322,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { } /// Return the ss58-check string for this key. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn to_ss58check_with_version(&self, version: Ss58AddressFormat) -> String { // We mask out the upper two bits of the ident - SS58 Prefix currently only supports 14-bits let ident: u16 = u16::from(version) & 0b0011_1111_1111_1111; @@ -344,7 +345,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { } /// Return the ss58-check string for this key. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn to_ss58check(&self) -> String { self.to_ss58check_with_version(default_ss58_version()) } @@ -362,16 +363,16 @@ pub trait Derive: Sized { /// Derive a child key from a series of given junctions. /// /// Will be `None` for public keys if there are any hard junctions in there. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn derive>(&self, _path: Iter) -> Option { None } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] const PREFIX: &[u8] = b"SS58PRE"; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] fn ss58hash(data: &[u8]) -> Vec { use blake2::{Blake2b512, Digest}; @@ -382,19 +383,19 @@ fn ss58hash(data: &[u8]) -> Vec { } /// Default prefix number -#[cfg(feature = "std")] -static DEFAULT_VERSION: core::sync::atomic::AtomicU16 = std::sync::atomic::AtomicU16::new( +#[cfg(feature = "serde")] +static DEFAULT_VERSION: core::sync::atomic::AtomicU16 = core::sync::atomic::AtomicU16::new( from_known_address_format(Ss58AddressFormatRegistry::SubstrateAccount), ); /// Returns default SS58 format used by the current active process. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub fn default_ss58_version() -> Ss58AddressFormat { - DEFAULT_VERSION.load(std::sync::atomic::Ordering::Relaxed).into() + DEFAULT_VERSION.load(core::sync::atomic::Ordering::Relaxed).into() } /// Returns either the input address format or the default. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub fn unwrap_or_default_ss58_version(network: Option) -> Ss58AddressFormat { network.unwrap_or_else(default_ss58_version) } @@ -408,9 +409,9 @@ pub fn unwrap_or_default_ss58_version(network: Option) -> Ss5 /// This will enable the node to decode ss58 addresses with this prefix. /// /// This SS58 version/format is also only used by the node and not by the runtime. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub fn set_default_ss58_version(new_default: Ss58AddressFormat) { - DEFAULT_VERSION.store(new_default.into(), std::sync::atomic::Ordering::Relaxed); + DEFAULT_VERSION.store(new_default.into(), core::sync::atomic::Ordering::Relaxed); } #[cfg(feature = "std")] @@ -458,6 +459,11 @@ impl + AsRef<[u8]> + Public + Derive> Ss58Codec for T { } } +// Use the default implementations of the trait in serde feature. +// The std implementation is not available because of std only crate Regex. +#[cfg(all(not(feature = "std"), feature = "serde"))] +impl + AsRef<[u8]> + Public + Derive> Ss58Codec for T {} + /// Trait used for types that are really just a fixed-length array. pub trait ByteArray: AsRef<[u8]> + AsMut<[u8]> + for<'a> TryFrom<&'a [u8], Error = ()> { /// The "length" of the values of this type, which is always the same. @@ -507,7 +513,7 @@ impl ByteArray for AccountId32 { const LEN: usize = 32; } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Ss58Codec for AccountId32 {} impl AsRef<[u8]> for AccountId32 { @@ -591,7 +597,7 @@ impl sp_std::fmt::Debug for AccountId32 { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl serde::Serialize for AccountId32 { fn serialize(&self, serializer: S) -> Result where @@ -601,7 +607,7 @@ impl serde::Serialize for AccountId32 { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> serde::Deserialize<'de> for AccountId32 { fn deserialize(deserializer: D) -> Result where @@ -1063,7 +1069,7 @@ pub trait CryptoType { crate::RuntimeDebug, TypeInfo, )] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct KeyTypeId(pub [u8; 4]); impl From for KeyTypeId { @@ -1121,7 +1127,7 @@ pub trait VrfPublic: VrfCrypto { /// An identifier for a specific cryptographic algorithm used by a key pair #[derive(Debug, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct CryptoTypeId(pub [u8; 4]); /// Known key types; this also functions as a global registry of key types for projects wishing to diff --git a/primitives/core/src/ecdsa.rs b/primitives/core/src/ecdsa.rs index 2474fa7736b78..0c997bc290008 100644 --- a/primitives/core/src/ecdsa.rs +++ b/primitives/core/src/ecdsa.rs @@ -21,7 +21,7 @@ use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; use sp_runtime_interface::pass_by::PassByInner; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use crate::crypto::Ss58Codec; use crate::crypto::{ ByteArray, CryptoType, CryptoTypeId, Derive, Public as TraitPublic, UncheckedFrom, @@ -40,8 +40,10 @@ use secp256k1::{ ecdsa::{RecoverableSignature, RecoveryId}, Message, PublicKey, SecretKey, }; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::{format, string::String}; #[cfg(feature = "full_crypto")] use sp_std::vec::Vec; @@ -164,7 +166,7 @@ impl sp_std::fmt::Debug for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Public { fn serialize(&self, serializer: S) -> Result where @@ -174,7 +176,7 @@ impl Serialize for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Public { fn deserialize(deserializer: D) -> Result where @@ -204,7 +206,7 @@ impl TryFrom<&[u8]> for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Signature { fn serialize(&self, serializer: S) -> Result where @@ -214,7 +216,7 @@ impl Serialize for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Signature { fn deserialize(deserializer: D) -> Result where diff --git a/primitives/core/src/ed25519.rs b/primitives/core/src/ed25519.rs index d0e6bef97a7d9..29da78626f850 100644 --- a/primitives/core/src/ed25519.rs +++ b/primitives/core/src/ed25519.rs @@ -29,7 +29,7 @@ use crate::{ use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use crate::crypto::Ss58Codec; use crate::crypto::{CryptoType, CryptoTypeId, Derive, Public as TraitPublic, UncheckedFrom}; #[cfg(feature = "full_crypto")] @@ -38,9 +38,11 @@ use crate::crypto::{DeriveError, DeriveJunction, Pair as TraitPair, SecretString use core::convert::TryFrom; #[cfg(feature = "full_crypto")] use ed25519_zebra::{SigningKey, VerificationKey}; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use sp_runtime_interface::pass_by::PassByInner; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::{format, string::String}; use sp_std::ops::Deref; /// An identifier used to match public keys against ed25519 keys @@ -176,7 +178,7 @@ impl sp_std::fmt::Debug for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Public { fn serialize(&self, serializer: S) -> Result where @@ -186,7 +188,7 @@ impl Serialize for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Public { fn deserialize(deserializer: D) -> Result where @@ -216,7 +218,7 @@ impl TryFrom<&[u8]> for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Signature { fn serialize(&self, serializer: S) -> Result where @@ -226,7 +228,7 @@ impl Serialize for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Signature { fn deserialize(deserializer: D) -> Result where diff --git a/primitives/core/src/hash.rs b/primitives/core/src/hash.rs index ec7da98296bf3..ece9b1af794aa 100644 --- a/primitives/core/src/hash.rs +++ b/primitives/core/src/hash.rs @@ -32,7 +32,6 @@ pub fn convert_hash, H2: AsRef<[u8]>>(src: &H2) -> H1 #[cfg(test)] mod tests { use super::*; - use sp_serializer as ser; #[test] fn test_h160() { @@ -47,8 +46,11 @@ mod tests { ]; for (number, expected) in tests { - assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); - assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + assert_eq!( + format!("{:?}", expected), + serde_json::to_string_pretty(&number).expect("Json pretty print failed") + ); + assert_eq!(number, serde_json::from_str(&format!("{:?}", expected)).unwrap()); } } @@ -86,30 +88,33 @@ mod tests { ]; for (number, expected) in tests { - assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); - assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + assert_eq!( + format!("{:?}", expected), + serde_json::to_string_pretty(&number).expect("Json pretty print failed") + ); + assert_eq!(number, serde_json::from_str(&format!("{:?}", expected)).unwrap()); } } #[test] fn test_invalid() { - assert!(ser::from_str::( + assert!(serde_json::from_str::( "\"0x000000000000000000000000000000000000000000000000000000000000000\"" ) .unwrap_err() .is_data()); - assert!(ser::from_str::( + assert!(serde_json::from_str::( "\"0x000000000000000000000000000000000000000000000000000000000000000g\"" ) .unwrap_err() .is_data()); - assert!(ser::from_str::( + assert!(serde_json::from_str::( "\"0x00000000000000000000000000000000000000000000000000000000000000000\"" ) .unwrap_err() .is_data()); - assert!(ser::from_str::("\"\"").unwrap_err().is_data()); - assert!(ser::from_str::("\"0\"").unwrap_err().is_data()); - assert!(ser::from_str::("\"10\"").unwrap_err().is_data()); + assert!(serde_json::from_str::("\"\"").unwrap_err().is_data()); + assert!(serde_json::from_str::("\"0\"").unwrap_err().is_data()); + assert!(serde_json::from_str::("\"10\"").unwrap_err().is_data()); } } diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index f9541b02e2903..b61009bc640ee 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -34,16 +34,16 @@ macro_rules! map { #[doc(hidden)] pub use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub use serde; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sp_runtime_interface::pass_by::{PassByEnum, PassByInner}; use sp_std::{ops::Deref, prelude::*}; pub use sp_debug_derive::RuntimeDebug; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub use impl_serde::serialize as bytes; #[cfg(feature = "full_crypto")] @@ -139,8 +139,8 @@ impl ExecutionContext { /// Hex-serialized shim for `Vec`. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord))] -pub struct Bytes(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec); +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize, Hash, PartialOrd, Ord))] +pub struct Bytes(#[cfg_attr(feature = "serde", serde(with = "bytes"))] pub Vec); impl From> for Bytes { fn from(s: Vec) -> Self { @@ -209,7 +209,7 @@ impl sp_std::ops::Deref for OpaqueMetadata { PassByInner, TypeInfo, )] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct OpaquePeerId(pub Vec); impl OpaquePeerId { @@ -394,6 +394,45 @@ macro_rules! impl_maybe_marker { } } +/// Macro for creating `Maybe*` marker traits. +/// +/// Such a maybe-marker trait requires the given bound when either `feature = std` or `feature = +/// serde` is activated. +/// +/// # Example +/// +/// ``` +/// sp_core::impl_maybe_marker_std_or_serde! { +/// /// A marker for a type that implements `Debug` when `feature = serde` or `feature = std`. +/// trait MaybeDebug: std::fmt::Debug; +/// /// A marker for a type that implements `Debug + Display` when `feature = serde` or `feature = std`. +/// trait MaybeDebugDisplay: std::fmt::Debug, std::fmt::Display; +/// } +/// ``` +#[macro_export] +macro_rules! impl_maybe_marker_std_or_serde { + ( + $( + $(#[$doc:meta] )+ + trait $trait_name:ident: $( $trait_bound:path ),+; + )+ + ) => { + $( + $(#[$doc])+ + #[cfg(any(feature = "serde", feature = "std"))] + pub trait $trait_name: $( $trait_bound + )+ {} + #[cfg(any(feature = "serde", feature = "std"))] + impl $trait_name for T {} + + $(#[$doc])+ + #[cfg(not(any(feature = "serde", feature = "std")))] + pub trait $trait_name {} + #[cfg(not(any(feature = "serde", feature = "std")))] + impl $trait_name for T {} + )+ + } +} + /// The maximum number of bytes that can be allocated at one time. // The maximum possible allocation size was chosen rather arbitrary, 32 MiB should be enough for // everybody. @@ -445,7 +484,7 @@ macro_rules! generate_feature_enabled_macro { } // Work around for: - #[doc(hidden)] + #[doc(hidden)] pub use [<_ $macro_name>] as $macro_name; } }; diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index a9e6639807023..a6cef85e6ac1b 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -58,7 +58,7 @@ pub trait OffchainStorage: Clone + Send + Sync { /// A type of supported crypto. #[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[repr(C)] pub enum StorageKind { /// Persistent storage is non-revertible and not fork-aware. It means that any value @@ -208,14 +208,14 @@ impl OpaqueMultiaddr { #[derive( Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode, )] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Timestamp(u64); /// Duration type #[derive( Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode, )] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Duration(u64); impl Duration { diff --git a/primitives/core/src/sr25519.rs b/primitives/core/src/sr25519.rs index fead9e2a09df3..cfcdd6a9fdd41 100644 --- a/primitives/core/src/sr25519.rs +++ b/primitives/core/src/sr25519.rs @@ -19,15 +19,20 @@ //! //! Note: `CHAIN_CODE_LENGTH` must be equal to `crate::crypto::JUNCTION_ID_LEN` //! for this to work. - -#[cfg(feature = "std")] +#[cfg(any(feature = "full_crypto", feature = "serde"))] +use crate::crypto::DeriveJunction; +#[cfg(feature = "serde")] use crate::crypto::Ss58Codec; #[cfg(feature = "full_crypto")] -use crate::crypto::{DeriveError, DeriveJunction, Pair as TraitPair, SecretStringError}; +use crate::crypto::{DeriveError, Pair as TraitPair, SecretStringError}; #[cfg(feature = "full_crypto")] use schnorrkel::{ - derive::{ChainCode, Derivation, CHAIN_CODE_LENGTH}, - signing_context, ExpansionMode, Keypair, MiniSecretKey, PublicKey, SecretKey, + derive::CHAIN_CODE_LENGTH, signing_context, ExpansionMode, Keypair, MiniSecretKey, SecretKey, +}; +#[cfg(any(feature = "full_crypto", feature = "serde"))] +use schnorrkel::{ + derive::{ChainCode, Derivation}, + PublicKey, }; use sp_std::vec::Vec; @@ -41,9 +46,11 @@ use sp_std::ops::Deref; #[cfg(feature = "full_crypto")] use schnorrkel::keys::{MINI_SECRET_KEY_LENGTH, SECRET_KEY_LENGTH}; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use sp_runtime_interface::pass_by::PassByInner; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::{format, string::String}; // signing context #[cfg(feature = "full_crypto")] @@ -176,7 +183,7 @@ impl sp_std::fmt::Debug for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Public { fn serialize(&self, serializer: S) -> Result where @@ -186,7 +193,7 @@ impl Serialize for Public { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Public { fn deserialize(deserializer: D) -> Result where @@ -216,7 +223,7 @@ impl TryFrom<&[u8]> for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl Serialize for Signature { fn serialize(&self, serializer: S) -> Result where @@ -226,7 +233,7 @@ impl Serialize for Signature { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Signature { fn deserialize(deserializer: D) -> Result where @@ -339,7 +346,7 @@ impl Derive for Public { /// Derive a child key from a series of given junctions. /// /// `None` if there are any hard junctions in there. - #[cfg(feature = "std")] + #[cfg(feature = "serde")] fn derive>(&self, path: Iter) -> Option { let mut acc = PublicKey::from_bytes(self.as_ref()).ok()?; for j in path { diff --git a/primitives/core/src/uint.rs b/primitives/core/src/uint.rs index df49511e84943..b251671dbeea5 100644 --- a/primitives/core/src/uint.rs +++ b/primitives/core/src/uint.rs @@ -23,7 +23,6 @@ pub use primitive_types::{U256, U512}; mod tests { use super::*; use codec::{Decode, Encode}; - use sp_serializer as ser; macro_rules! test { ($name: ident, $test_name: ident) => { @@ -43,14 +42,17 @@ mod tests { ]; for (number, expected) in tests { - assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); - assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + assert_eq!( + format!("{:?}", expected), + serde_json::to_string_pretty(&number).expect("Json pretty print failed") + ); + assert_eq!(number, serde_json::from_str(&format!("{:?}", expected)).unwrap()); } // Invalid examples - assert!(ser::from_str::<$name>("\"0x\"").unwrap_err().is_data()); - assert!(ser::from_str::<$name>("\"0xg\"").unwrap_err().is_data()); - assert!(ser::from_str::<$name>("\"\"").unwrap_err().is_data()); + assert!(serde_json::from_str::<$name>("\"0x\"").unwrap_err().is_data()); + assert!(serde_json::from_str::<$name>("\"0xg\"").unwrap_err().is_data()); + assert!(serde_json::from_str::<$name>("\"\"").unwrap_err().is_data()); } }; } @@ -78,10 +80,10 @@ mod tests { #[test] fn test_large_values() { assert_eq!( - ser::to_string_pretty(&!U256::zero()), + serde_json::to_string_pretty(&!U256::zero()).expect("Json pretty print failed"), "\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"" ); - assert!(ser::from_str::( + assert!(serde_json::from_str::( "\"0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"" ) .unwrap_err() diff --git a/primitives/merkle-mountain-range/Cargo.toml b/primitives/merkle-mountain-range/Cargo.toml index cb3d6ddd51ad2..4d2c050899493 100644 --- a/primitives/merkle-mountain-range/Cargo.toml +++ b/primitives/merkle-mountain-range/Cargo.toml @@ -16,7 +16,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", default-features = scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } log = { version = "0.4.17", default-features = false } mmr-lib = { package = "ckb-merkle-mountain-range", version = "0.5.2", default-features = false } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", features = ["derive", "alloc"], default-features = false, optional = true } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } sp-debug-derive = { version = "5.0.0", default-features = false, path = "../debug-derive" } @@ -33,10 +33,18 @@ std = [ "codec/std", "log/std", "mmr-lib/std", - "serde", + "serde/std", "sp-api/std", "sp-core/std", "sp-debug-derive/std", "sp-runtime/std", "sp-std/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/merkle-mountain-range/src/lib.rs b/primitives/merkle-mountain-range/src/lib.rs index 436755c032908..6c0e75005ead8 100644 --- a/primitives/merkle-mountain-range/src/lib.rs +++ b/primitives/merkle-mountain-range/src/lib.rs @@ -101,13 +101,13 @@ impl FullLeaf /// This type does not implement SCALE encoding/decoding on purpose to avoid confusion, /// it would have to be SCALE-compatible with the concrete leaf type, but due to SCALE limitations /// it's not possible to know how many bytes the encoding of concrete leaf type uses. -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(RuntimeDebug, Clone, PartialEq)] pub struct OpaqueLeaf( /// Raw bytes of the leaf type encoded in its compact form. /// /// NOTE it DOES NOT include length prefix (like `Vec` encoding would). - #[cfg_attr(feature = "std", serde(with = "sp_core::bytes"))] + #[cfg_attr(feature = "serde", serde(with = "sp_core::bytes"))] pub Vec, ); diff --git a/primitives/npos-elections/Cargo.toml b/primitives/npos-elections/Cargo.toml index 50a7a5bcd6088..9006a4d199189 100644 --- a/primitives/npos-elections/Cargo.toml +++ b/primitives/npos-elections/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } sp-arithmetic = { version = "6.0.0", default-features = false, path = "../arithmetic" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } sp-runtime = { version = "7.0.0", default-features = false, path = "../runtime" } @@ -31,9 +31,18 @@ bench = [] std = [ "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-arithmetic/std", "sp-core/std", "sp-runtime/std", "sp-std/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-arithmetic/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/npos-elections/src/assignments.rs b/primitives/npos-elections/src/assignments.rs index 9390cd1f4f9fc..2ac2b9bebd771 100644 --- a/primitives/npos-elections/src/assignments.rs +++ b/primitives/npos-elections/src/assignments.rs @@ -18,7 +18,7 @@ //! Structs and helpers for distributing a voter's stake among various winners. use crate::{ExtendedBalance, IdentifierT, PerThing128}; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use codec::{Decode, Encode}; use sp_arithmetic::{ traits::{Bounded, Zero}, @@ -29,7 +29,7 @@ use sp_std::vec::Vec; /// A voter's stake assignment among a set of targets, represented as ratios. #[derive(RuntimeDebug, Clone, Default)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] +#[cfg_attr(feature = "serde", derive(PartialEq, Eq, Encode, Decode))] pub struct Assignment { /// Voter's identifier. pub who: AccountId, @@ -97,7 +97,7 @@ impl Assignment { /// A voter's stake assignment among a set of targets, represented as absolute values in the scale /// of [`ExtendedBalance`]. #[derive(RuntimeDebug, Clone, Default)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] +#[cfg_attr(feature = "serde", derive(PartialEq, Eq, Encode, Decode))] pub struct StakedAssignment { /// Voter's identifier pub who: AccountId, diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index 716c4b283c68e..253a231602f77 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -77,7 +77,7 @@ use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sp_arithmetic::{traits::Zero, Normalizable, PerThing, Rational128, ThresholdOrd}; use sp_core::{bounded::BoundedVec, RuntimeDebug}; @@ -144,7 +144,7 @@ pub type ExtendedBalance = u128; /// 2. `sum_stake`. /// 3. `sum_stake_squared`. #[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, Debug, Default)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct ElectionScore { /// The minimal winner, in terms of total backing stake. /// @@ -430,7 +430,7 @@ pub struct ElectionResult { /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet they /// do not necessarily have to be the same. #[derive(RuntimeDebug, Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Support { /// Total support. pub total: ExtendedBalance, diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index 4ec55162e9774..0d67a89c38c03 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -22,7 +22,7 @@ log = { version = "0.4.17", default-features = false } paste = "1.0" rand = { version = "0.8.5", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } sp-application-crypto = { version = "7.0.0", default-features = false, path = "../application-crypto" } sp-arithmetic = { version = "6.0.0", default-features = false, path = "../arithmetic" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } @@ -50,7 +50,7 @@ std = [ "log/std", "rand", "scale-info/std", - "serde", + "serde/std", "sp-application-crypto/std", "sp-arithmetic/std", "sp-core/std", @@ -58,3 +58,13 @@ std = [ "sp-std/std", "sp-weights/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-application-crypto/serde", + "sp-arithmetic/serde", + "sp-core/serde", + "sp-weights/serde", +] diff --git a/primitives/runtime/src/generic/block.rs b/primitives/runtime/src/generic/block.rs index 1df747a16c839..6261e412eb8ad 100644 --- a/primitives/runtime/src/generic/block.rs +++ b/primitives/runtime/src/generic/block.rs @@ -20,7 +20,7 @@ #[cfg(feature = "std")] use std::fmt; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use crate::{ @@ -76,9 +76,9 @@ impl fmt::Display for BlockId { /// Abstraction over a substrate block. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, scale_info::TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] -#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] pub struct Block { /// The block header. pub header: Header, @@ -114,9 +114,9 @@ where /// Abstraction over a substrate block and justification. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] -#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] pub struct SignedBlock { /// Full block. pub block: Block, diff --git a/primitives/runtime/src/generic/digest.rs b/primitives/runtime/src/generic/digest.rs index 143fc2ce58d90..d7db0f91a4821 100644 --- a/primitives/runtime/src/generic/digest.rs +++ b/primitives/runtime/src/generic/digest.rs @@ -17,8 +17,10 @@ //! Generic implementation of a digest. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::format; use sp_std::prelude::*; @@ -34,7 +36,7 @@ use sp_core::RuntimeDebug; /// Generic header digest. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, Default)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Digest { /// A list of logs in the digest. pub logs: Vec, @@ -106,7 +108,7 @@ pub enum DigestItem { RuntimeEnvironmentUpdated, } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl serde::Serialize for DigestItem { fn serialize(&self, seq: S) -> Result where @@ -116,7 +118,7 @@ impl serde::Serialize for DigestItem { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'a> serde::Deserialize<'a> for DigestItem { fn deserialize(de: D) -> Result where diff --git a/primitives/runtime/src/generic/era.rs b/primitives/runtime/src/generic/era.rs index 79dea7258c9e1..bdc7e9930fb1d 100644 --- a/primitives/runtime/src/generic/era.rs +++ b/primitives/runtime/src/generic/era.rs @@ -17,7 +17,7 @@ //! Generic implementation of an unchecked (pre-verification) extrinsic. -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use crate::codec::{Decode, Encode, Error, Input, Output}; @@ -30,7 +30,7 @@ pub type Phase = u64; /// An era to describe the longevity of a transaction. #[derive(PartialEq, Eq, Clone, Copy, sp_core::RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum Era { /// The transaction is valid forever. The genesis hash must be present in the signed content. Immortal, diff --git a/primitives/runtime/src/generic/header.rs b/primitives/runtime/src/generic/header.rs index e8b99efd42c75..7c1faaefb419a 100644 --- a/primitives/runtime/src/generic/header.rs +++ b/primitives/runtime/src/generic/header.rs @@ -26,22 +26,22 @@ use crate::{ MaybeSerializeDeserialize, Member, SimpleBitOps, }, }; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sp_core::U256; use sp_std::fmt::Debug; /// Abstraction over a block header for a substrate chain. #[derive(Encode, Decode, PartialEq, Eq, Clone, sp_core::RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] -#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] pub struct Header + TryFrom, Hash: HashT> { /// The parent hash. pub parent_hash: Hash::Output, /// The block number. #[cfg_attr( - feature = "std", + feature = "serde", serde(serialize_with = "serialize_number", deserialize_with = "deserialize_number") )] #[codec(compact)] @@ -54,7 +54,7 @@ pub struct Header + TryFrom, Hash: HashT> { pub digest: Digest, } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub fn serialize_number + TryFrom>( val: &T, s: S, @@ -66,7 +66,7 @@ where serde::Serialize::serialize(&u256, s) } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub fn deserialize_number<'a, D, T: Copy + Into + TryFrom>(d: D) -> Result where D: serde::Deserializer<'a>, diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index d147e7b6c1505..b9d7b9eb1b6c8 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -29,6 +29,8 @@ use crate::{ use codec::{Compact, Decode, Encode, EncodeLike, Error, Input}; use scale_info::{build::Fields, meta_type, Path, StaticTypeInfo, Type, TypeInfo, TypeParameter}; use sp_io::hashing::blake2_256; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::format; use sp_std::{fmt, prelude::*}; /// Current version of the [`UncheckedExtrinsic`] encoded format. @@ -317,7 +319,7 @@ where { } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl serde::Serialize for UncheckedExtrinsic { @@ -329,7 +331,7 @@ impl s } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'a, Address: Decode, Signature: Decode, Call: Decode, Extra: SignedExtension> serde::Deserialize<'a> for UncheckedExtrinsic { diff --git a/primitives/runtime/src/legacy/byte_sized_error.rs b/primitives/runtime/src/legacy/byte_sized_error.rs index b552d6af3066e..a592c751231dc 100644 --- a/primitives/runtime/src/legacy/byte_sized_error.rs +++ b/primitives/runtime/src/legacy/byte_sized_error.rs @@ -20,12 +20,12 @@ use crate::{ArithmeticError, TokenError}; use codec::{Decode, Encode}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; /// [`ModuleError`] type definition before BlockBuilder API version 6. #[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct ModuleError { /// Module index, matching the metadata module index. pub index: u8, @@ -33,7 +33,7 @@ pub struct ModuleError { pub error: u8, /// Optional error message. #[codec(skip)] - #[cfg_attr(feature = "std", serde(skip_deserializing))] + #[cfg_attr(feature = "serde", serde(skip_deserializing))] pub message: Option<&'static str>, } @@ -45,12 +45,12 @@ impl PartialEq for ModuleError { /// [`DispatchError`] type definition before BlockBuilder API version 6. #[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo, PartialEq)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum DispatchError { /// Some error occurred. Other( #[codec(skip)] - #[cfg_attr(feature = "std", serde(skip_deserializing))] + #[cfg_attr(feature = "serde", serde(skip_deserializing))] &'static str, ), /// Failed to lookup some data. diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 13ef157dff19e..7be5bebf5de80 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -24,7 +24,7 @@ pub use codec; #[doc(hidden)] pub use scale_info; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] #[doc(hidden)] pub use serde; #[doc(hidden)] @@ -52,6 +52,8 @@ use sp_std::prelude::*; use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +#[cfg(all(not(feature = "std"), feature = "serde"))] +use sp_std::alloc::format; pub mod curve; pub mod generic; @@ -122,7 +124,7 @@ pub type EncodedJustification = Vec; /// Collection of justifications for a given block, multiple justifications may /// be provided by different consensus engines for the same block. -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] pub struct Justifications(Vec); @@ -179,7 +181,7 @@ impl From for Justifications { use traits::{Lazy, Verify}; use crate::traits::IdentifyAccount; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] pub use serde::{de::DeserializeOwned, Deserialize, Serialize}; /// Complex storage builder stuff. @@ -235,7 +237,7 @@ impl BuildStorage for () { pub type ConsensusEngineId = [u8; 4]; /// Signature verify that can work with any known signature types. -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Eq, PartialEq, Clone, Encode, Decode, MaxEncodedLen, RuntimeDebug, TypeInfo)] pub enum MultiSignature { /// An Ed25519 signature. @@ -299,7 +301,7 @@ impl TryFrom for ecdsa::Signature { /// Public key for any known crypto algorithm. #[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum MultiSigner { /// An Ed25519 identity. Ed25519(ed25519::Public), @@ -427,7 +429,7 @@ impl Verify for MultiSignature { /// Signature verify that can work with any known signature types.. #[derive(Eq, PartialEq, Clone, Default, Encode, Decode, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct AnySignature(H512); impl Verify for AnySignature { @@ -475,7 +477,7 @@ pub type DispatchResultWithInfo = sp_std::result::Result, } @@ -495,7 +497,7 @@ impl PartialEq for ModuleError { /// Errors related to transactional storage layers. #[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, MaxEncodedLen)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum TransactionalError { /// Too many transactional layers have been spawned. LimitReached, @@ -520,12 +522,12 @@ impl From for DispatchError { /// Reason why a dispatch call failed. #[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo, PartialEq, MaxEncodedLen)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum DispatchError { /// Some error occurred. Other( #[codec(skip)] - #[cfg_attr(feature = "std", serde(skip_deserializing))] + #[cfg_attr(feature = "serde", serde(skip_deserializing))] &'static str, ), /// Failed to lookup some data. @@ -605,7 +607,7 @@ impl From for DispatchError { /// Description of what went wrong when trying to complete an operation on a token. #[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, MaxEncodedLen)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum TokenError { /// Funds are unavailable. FundsUnavailable, @@ -879,7 +881,7 @@ impl sp_std::fmt::Debug for OpaqueExtrinsic { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl ::serde::Serialize for OpaqueExtrinsic { fn serialize(&self, seq: S) -> Result where @@ -889,7 +891,7 @@ impl ::serde::Serialize for OpaqueExtrinsic { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'a> ::serde::Deserialize<'a> for OpaqueExtrinsic { fn deserialize(de: D) -> Result where diff --git a/primitives/runtime/src/runtime_string.rs b/primitives/runtime/src/runtime_string.rs index f8f183ec785c4..aa0bd52e56fe4 100644 --- a/primitives/runtime/src/runtime_string.rs +++ b/primitives/runtime/src/runtime_string.rs @@ -129,7 +129,7 @@ impl std::fmt::Display for RuntimeString { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl serde::Serialize for RuntimeString { fn serialize(&self, serializer: S) -> Result { match self { @@ -139,10 +139,10 @@ impl serde::Serialize for RuntimeString { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl<'de> serde::Deserialize<'de> for RuntimeString { fn deserialize>(de: D) -> Result { - String::deserialize(de).map(Self::Owned) + Ok(Self::Owned(serde::Deserialize::deserialize(de)?)) } } diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index 95f977077e704..d37db4802d78f 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -28,7 +28,7 @@ use crate::{ DispatchResult, }; use impl_trait_for_tuples::impl_for_tuples; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sp_application_crypto::AppCrypto; pub use sp_arithmetic::traits::{ @@ -716,7 +716,7 @@ pub trait Hash: /// Blake2-256 Hash implementation. #[derive(PartialEq, Eq, Clone, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct BlakeTwo256; impl Hasher for BlakeTwo256 { @@ -743,7 +743,7 @@ impl Hash for BlakeTwo256 { /// Keccak-256 Hash implementation. #[derive(PartialEq, Eq, Clone, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Keccak256; impl Hasher for Keccak256 { @@ -824,11 +824,13 @@ sp_core::impl_maybe_marker!( /// A type that implements Hash when in std environment. trait MaybeHash: sp_std::hash::Hash; +); - /// A type that implements Serialize when in std environment. +sp_core::impl_maybe_marker_std_or_serde!( + /// A type that implements Serialize when in std environment or serde feature is activated. trait MaybeSerialize: Serialize; - /// A type that implements Serialize, DeserializeOwned and Debug when in std environment. + /// A type that implements Serialize, DeserializeOwned and Debug when in std environment or serde feature is activated. trait MaybeSerializeDeserialize: DeserializeOwned, Serialize; ); diff --git a/primitives/runtime/src/transaction_validity.rs b/primitives/runtime/src/transaction_validity.rs index 072609c667b41..836948493823c 100644 --- a/primitives/runtime/src/transaction_validity.rs +++ b/primitives/runtime/src/transaction_validity.rs @@ -36,7 +36,7 @@ pub type TransactionTag = Vec; /// An invalid transaction validity. #[derive(Clone, PartialEq, Eq, Encode, Decode, Copy, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum InvalidTransaction { /// The call of the transaction is not expected. Call, @@ -119,7 +119,7 @@ impl From for &'static str { /// An unknown transaction validity. #[derive(Clone, PartialEq, Eq, Encode, Decode, Copy, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum UnknownTransaction { /// Could not lookup some information that is required to validate the transaction. CannotLookup, @@ -143,7 +143,7 @@ impl From for &'static str { /// Errors that can occur while checking the validity of a transaction. #[derive(Clone, PartialEq, Eq, Encode, Decode, Copy, RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum TransactionValidityError { /// The transaction is invalid. Invalid(InvalidTransaction), diff --git a/primitives/serializer/Cargo.toml b/primitives/serializer/Cargo.toml deleted file mode 100644 index 585e4b4e0dc20..0000000000000 --- a/primitives/serializer/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "sp-serializer" -version = "4.0.0-dev" -authors = ["Parity Technologies "] -edition = "2021" -license = "Apache-2.0" -homepage = "https://substrate.io" -repository = "https://github.com/paritytech/substrate/" -description = "Substrate customizable serde serializer." -documentation = "https://docs.rs/sp-serializer" -readme = "README.md" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -serde = "1.0.136" -serde_json = "1.0.85" diff --git a/primitives/serializer/README.md b/primitives/serializer/README.md deleted file mode 100644 index 083a0857cc0d2..0000000000000 --- a/primitives/serializer/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Substrate customizable serde serializer. - -The idea is that we can later change the implementation -to something more compact, but for now we're using JSON. - -License: Apache-2.0 \ No newline at end of file diff --git a/primitives/serializer/src/lib.rs b/primitives/serializer/src/lib.rs deleted file mode 100644 index 3d42707143d5e..0000000000000 --- a/primitives/serializer/src/lib.rs +++ /dev/null @@ -1,45 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Substrate customizable serde serializer. -//! -//! The idea is that we can later change the implementation -//! to something more compact, but for now we're using JSON. - -#![warn(missing_docs)] - -pub use serde_json::{from_reader, from_slice, from_str, Error, Result}; - -const PROOF: &str = "Serializers are infallible; qed"; - -/// Serialize the given data structure as a pretty-printed String of JSON. -pub fn to_string_pretty(value: &T) -> String { - serde_json::to_string_pretty(value).expect(PROOF) -} - -/// Serialize the given data structure as a JSON byte vector. -pub fn encode(value: &T) -> Vec { - serde_json::to_vec(value).expect(PROOF) -} - -/// Serialize the given data structure as JSON into the IO stream. -pub fn to_writer( - writer: W, - value: &T, -) -> Result<()> { - serde_json::to_writer(writer, value) -} diff --git a/primitives/storage/Cargo.toml b/primitives/storage/Cargo.toml index 72ebc40fc1135..c8271669f6721 100644 --- a/primitives/storage/Cargo.toml +++ b/primitives/storage/Cargo.toml @@ -15,12 +15,24 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } -impl-serde = { version = "0.4.0", optional = true } +impl-serde = { version = "0.4.0", optional = true, default-features = false } ref-cast = "1.0.0" -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } sp-debug-derive = { version = "5.0.0", default-features = false, path = "../debug-derive" } sp-std = { version = "5.0.0", default-features = false, path = "../std" } [features] default = [ "std" ] -std = [ "codec/std", "impl-serde", "serde", "sp-debug-derive/std", "sp-std/std" ] +std = [ + "codec/std", + "impl-serde/std", + "serde/std", + "sp-debug-derive/std", + "sp-std/std", +] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "impl-serde", +] diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 06995005d9f56..aa1bc8e305c9b 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -21,7 +21,7 @@ use core::fmt::Display; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sp_debug_derive::RuntimeDebug; @@ -35,11 +35,11 @@ use sp_std::{ /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr( - feature = "std", + feature = "serde", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode) )] pub struct StorageKey( - #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, + #[cfg_attr(feature = "serde", serde(with = "impl_serde::serialize"))] pub Vec, ); impl AsRef<[u8]> for StorageKey { @@ -100,11 +100,11 @@ impl From> for TrackedStorageKey { /// Storage key of a child trie, it contains the prefix to the key. #[derive(PartialEq, Eq, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] #[repr(transparent)] #[derive(RefCast)] pub struct PrefixedStorageKey( - #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] Vec, + #[cfg_attr(feature = "serde", serde(with = "impl_serde::serialize"))] Vec, ); impl Deref for PrefixedStorageKey { @@ -142,11 +142,11 @@ impl PrefixedStorageKey { /// Storage data associated to a [`StorageKey`]. #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr( - feature = "std", + feature = "serde", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode, Default) )] pub struct StorageData( - #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, + #[cfg_attr(feature = "serde", serde(with = "impl_serde::serialize"))] pub Vec, ); /// Map of data to use in a storage, it is a collection of @@ -178,8 +178,8 @@ pub struct Storage { /// Storage change set #[derive(RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, PartialEq, Eq))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize, PartialEq, Eq))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] pub struct StorageChangeSet { /// Block hash pub block: Hash, @@ -204,6 +204,9 @@ pub mod well_known_keys { /// Encodes to `0x3a65787472696e7369635f696e646578`. pub const EXTRINSIC_INDEX: &[u8] = b":extrinsic_index"; + /// Current intra-block entropy (a universally unique `[u8; 32]` value) is stored here. + pub const INTRABLOCK_ENTROPY: &[u8] = b":intrablock_entropy"; + /// Prefix of child storage keys. pub const CHILD_STORAGE_KEY_PREFIX: &[u8] = b":child_storage:"; @@ -242,7 +245,7 @@ pub const TRIE_VALUE_NODE_THRESHOLD: u32 = 33; /// Information related to a child state. #[derive(Debug, Clone)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] +#[cfg_attr(feature = "serde", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] pub enum ChildInfo { /// This is the one used by default. ParentKeyId(ChildTrieParentKeyId), @@ -389,7 +392,7 @@ impl ChildType { /// to be a unique id that will be use only once. Those unique id also required to be long enough to /// avoid any unique id to be prefixed by an other unique id. #[derive(Debug, Clone)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] +#[cfg_attr(feature = "serde", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] pub struct ChildTrieParentKeyId { /// Data is the storage key without prefix. data: Vec, diff --git a/primitives/test-primitives/Cargo.toml b/primitives/test-primitives/Cargo.toml index f6d57ce70fc05..77df69b9d1455 100644 --- a/primitives/test-primitives/Cargo.toml +++ b/primitives/test-primitives/Cargo.toml @@ -14,10 +14,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive"], optional = true } sp-application-crypto = { version = "7.0.0", default-features = false, path = "../application-crypto" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } sp-runtime = { version = "7.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "5.0.0", default-features = false, path = "../std" } [features] default = [ @@ -25,9 +26,18 @@ default = [ ] std = [ "codec/std", - "serde", + "serde/std", "sp-application-crypto/std", "sp-core/std", "sp-runtime/std", + "sp-std/std", "scale-info/std", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "sp-application-crypto/serde", + "sp-core/serde", + "sp-runtime/serde", +] diff --git a/primitives/test-primitives/src/lib.rs b/primitives/test-primitives/src/lib.rs index 035acc7a35ef6..913cb762d92a5 100644 --- a/primitives/test-primitives/src/lib.rs +++ b/primitives/test-primitives/src/lib.rs @@ -26,6 +26,7 @@ use sp_application_crypto::sr25519; pub use sp_core::{hash::H256, RuntimeDebug}; use sp_runtime::traits::{BlakeTwo256, Extrinsic as ExtrinsicT, Verify}; +use sp_std::vec::Vec; /// Extrinsic for test-runtime. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug, scale_info::TypeInfo)] @@ -34,7 +35,7 @@ pub enum Extrinsic { StorageChange(Vec, Option>), } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] impl serde::Serialize for Extrinsic { fn serialize(&self, seq: S) -> Result where diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index 6b425aa7b9d27..9cebb8f01d82e 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -15,10 +15,10 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } -impl-serde = { version = "0.4.0", optional = true } +impl-serde = { version = "0.4.0", default-features = false, optional = true } parity-wasm = { version = "0.45", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", features = ["derive"], optional = true } +serde = { version = "1.0.136", default-features = false, features = ["derive", "alloc"], optional = true } thiserror = { version = "1.0.30", optional = true } sp-core-hashing-proc-macro = { version = "5.0.0", path = "../core/hashing/proc-macro" } sp-runtime = { version = "7.0.0", default-features = false, path = "../runtime" } @@ -29,11 +29,18 @@ sp-version-proc-macro = { version = "4.0.0-dev", default-features = false, path default = ["std"] std = [ "codec/std", - "impl-serde", + "impl-serde/std", "parity-wasm", "scale-info/std", - "serde", + "serde/std", "sp-runtime/std", "sp-std/std", "thiserror", ] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "impl-serde", + "sp-runtime/serde", +] diff --git a/primitives/version/src/lib.rs b/primitives/version/src/lib.rs index 214606acc534a..bd8408bb4a48a 100644 --- a/primitives/version/src/lib.rs +++ b/primitives/version/src/lib.rs @@ -33,7 +33,7 @@ #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use std::collections::HashSet; @@ -156,8 +156,8 @@ macro_rules! create_apis_vec { /// `authoring_version`, absolutely not `impl_version` since they change the semantics of the /// runtime. #[derive(Clone, PartialEq, Eq, Encode, Default, sp_runtime::RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] pub struct RuntimeVersion { /// Identifies the different Substrate runtimes. There'll be at least polkadot and node. /// A different on-chain spec_name to that of the native runtime would normally result @@ -190,7 +190,7 @@ pub struct RuntimeVersion { /// List of supported API "features" along with their versions. #[cfg_attr( - feature = "std", + feature = "serde", serde( serialize_with = "apis_serialize::serialize", deserialize_with = "apis_serialize::deserialize", @@ -389,11 +389,12 @@ impl GetNativeVersion for std::sync::Arc { } } -#[cfg(feature = "std")] +#[cfg(feature = "serde")] mod apis_serialize { use super::*; use impl_serde::serialize as bytes; use serde::{de, ser::SerializeTuple, Serializer}; + use sp_std::vec::Vec; #[derive(Serialize)] struct ApiId<'a>(#[serde(serialize_with = "serialize_bytesref")] &'a super::ApiId, &'a u32); @@ -428,7 +429,7 @@ mod apis_serialize { impl<'de> de::Visitor<'de> for Visitor { type Value = ApisVec; - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fn expecting(&self, formatter: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { formatter.write_str("a sequence of api id and version tuples") } diff --git a/primitives/weights/Cargo.toml b/primitives/weights/Cargo.toml index af1b5647e24d7..534d790884794 100644 --- a/primitives/weights/Cargo.toml +++ b/primitives/weights/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.136", optional = true, features = ["derive"] } +serde = { version = "1.0.136", default-features = false, optional = true, features = ["derive", "alloc"] } smallvec = "1.8.0" sp-arithmetic = { version = "6.0.0", default-features = false, path = "../arithmetic" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } @@ -27,7 +27,7 @@ default = [ "std" ] std = [ "codec/std", "scale-info/std", - "serde", + "serde/std", "sp-arithmetic/std", "sp-core/std", "sp-debug-derive/std", @@ -36,3 +36,11 @@ std = [ # By default some types have documentation, `full-metadata-docs` allows to add documentation to # more types in the metadata. full-metadata-docs = ["scale-info/docs"] + +# Serde support without relying on std features. +serde = [ + "dep:serde", + "scale-info/serde", + "sp-arithmetic/serde", + "sp-core/serde", +] diff --git a/primitives/weights/src/lib.rs b/primitives/weights/src/lib.rs index 55d9104b00b77..36cf864dd5389 100644 --- a/primitives/weights/src/lib.rs +++ b/primitives/weights/src/lib.rs @@ -29,7 +29,7 @@ mod weight_v2; use codec::{CompactAs, Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; -#[cfg(feature = "std")] +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use sp_arithmetic::{ @@ -69,8 +69,8 @@ pub mod constants { MaxEncodedLen, TypeInfo, )] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(transparent))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(transparent))] #[deprecated(note = "Will be removed soon; use `Weight` instead.")] pub struct OldWeight(pub u64); diff --git a/primitives/weights/src/weight_v2.rs b/primitives/weights/src/weight_v2.rs index cb1cedf18ce87..2aede666dd3da 100644 --- a/primitives/weights/src/weight_v2.rs +++ b/primitives/weights/src/weight_v2.rs @@ -25,7 +25,7 @@ use super::*; #[derive( Encode, Decode, MaxEncodedLen, TypeInfo, Eq, PartialEq, Copy, Clone, RuntimeDebug, Default, )] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Weight { #[codec(compact)] /// The weight of computational time used based on some reference hardware. diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 6e86a01b4d54f..f49e36c6a1057 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -226,28 +226,20 @@ test-linux-stable: --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} # we need to update cache only from one job - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi - artifacts: - when: always - paths: - - target/nextest/default/junit.xml - reports: - junit: target/nextest/default/junit.xml - -test-linux-stable-upload-test-results: - stage: test - needs: - - job: test-linux-stable - artifacts: true - extends: - - .docker-env - - .test-refs - script: + # Upload tests results to Elasticsearch + - echo "Upload test results to Elasticsearch" - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json - "curl -v -XPOST --http1.1 -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} -H 'Content-Type: application/json' -d @target/nextest/default/junit.json" + artifacts: + when: always + paths: + - target/nextest/default/junit.xml + reports: + junit: target/nextest/default/junit.xml test-frame-support: stage: test diff --git a/scripts/ci/node-template-release/Cargo.toml b/scripts/ci/node-template-release/Cargo.toml index aca8d6e1f81da..e93810876c7d9 100644 --- a/scripts/ci/node-template-release/Cargo.toml +++ b/scripts/ci/node-template-release/Cargo.toml @@ -18,4 +18,5 @@ git2 = "0.16" glob = "0.3" tar = "0.4" tempfile = "3" -toml = "0.7" +toml_edit = "0.19" +itertools = "0.10" diff --git a/scripts/ci/node-template-release/src/main.rs b/scripts/ci/node-template-release/src/main.rs index 91a7e865458cf..c752522991978 100644 --- a/scripts/ci/node-template-release/src/main.rs +++ b/scripts/ci/node-template-release/src/main.rs @@ -1,30 +1,32 @@ -use clap::Parser; - use std::{ collections::HashMap, fs::{self, File, OpenOptions}, - io::{Read, Write}, path::{Path, PathBuf}, process::Command, }; -use glob; - +use clap::Parser; +use flate2::{write::GzEncoder, Compression}; use fs_extra::dir::{self, CopyOptions}; - -use tempfile; - use git2; - -use toml; - +use glob; +use itertools::Itertools; use tar; - -use flate2::{write::GzEncoder, Compression}; +use tempfile; +use toml_edit::{self, value, Array, Item, Table}; const SUBSTRATE_GIT_URL: &str = "https://github.com/paritytech/substrate.git"; -type CargoToml = HashMap; +type CargoToml = toml_edit::Document; + +#[derive(Debug, PartialEq)] +struct Dependency { + name: String, + version: Option, + default_features: Option, +} + +type Dependencies = HashMap>; #[derive(Parser)] struct Options { @@ -36,8 +38,14 @@ struct Options { output: PathBuf, } +/// Copy the `node-template` to the given path. +fn copy_node_template(node_template: &Path, dest_path: &Path) { + let options = CopyOptions::new(); + dir::copy(node_template, dest_path, &options).expect("Copies node-template to tmp dir"); +} + /// Find all `Cargo.toml` files in the given path. -fn find_cargo_tomls(path: PathBuf) -> Vec { +fn find_cargo_tomls(path: &PathBuf) -> Vec { let path = format!("{}/**/*.toml", path.display()); let glob = glob::glob(&path).expect("Generates globbing pattern"); @@ -55,10 +63,18 @@ fn find_cargo_tomls(path: PathBuf) -> Vec { result } -/// Copy the `node-template` to the given path. -fn copy_node_template(node_template: &Path, dest_path: &Path) { - let options = CopyOptions::new(); - dir::copy(node_template, dest_path, &options).expect("Copies node-template to tmp dir"); +/// Parse the given `Cargo.toml`. +fn parse_cargo_toml(file: &Path) -> CargoToml { + fs::read_to_string(file) + .unwrap_or_else(|e| panic!("Failed to read `{}`: {}", file.display(), e)) + .parse() + .unwrap_or_else(|e| panic!("Failed to parse `{}`: {}", file.display(), e)) +} + +/// Write the given `Cargo.toml` to the given path. +fn write_cargo_toml(path: &Path, cargo_toml: CargoToml) { + fs::write(path, cargo_toml.to_string()) + .unwrap_or_else(|e| panic!("Failed to write `{}`: {}", path.display(), e)); } /// Gets the latest commit id of the repository given by `path`. @@ -76,82 +92,125 @@ fn get_git_commit_id(path: &Path) -> String { format!("{}", commit_id) } -/// Parse the given `Cargo.toml` into a `HashMap` -fn parse_cargo_toml(file: &Path) -> CargoToml { - let mut content = String::new(); - File::open(file) - .expect("Cargo.toml exists") - .read_to_string(&mut content) - .expect("Reads file"); - toml::from_str(&content).expect("Cargo.toml is a valid toml file") -} - -/// Replaces all substrate path dependencies with a git dependency. -fn replace_path_dependencies_with_git( - cargo_toml_path: &Path, - commit_id: &str, +/// Rewrites git dependencies: +/// - inserts `workspace = true`; +/// - removes `path`; +/// - removes `version`; +/// - removes `default-features` +/// - and returns the dependencies that were rewritten. +fn update_git_dependencies bool>( cargo_toml: &mut CargoToml, -) { - let mut cargo_toml_path = cargo_toml_path.to_path_buf(); - // remove `Cargo.toml` - cargo_toml_path.pop(); - - for &table in &["dependencies", "build-dependencies", "dev-dependencies"] { - let mut dependencies: toml::value::Table = - match cargo_toml.remove(table).and_then(|v| v.try_into().ok()) { - Some(deps) => deps, - None => continue, - }; - - let deps_rewritten = dependencies - .iter() - .filter_map(|(k, v)| { - v.clone().try_into::().ok().map(move |v| (k, v)) + path_filter: F, +) -> Dependencies { + let process_dep = |dep: (toml_edit::KeyMut, &mut Item)| -> Option { + let (key, value) = dep; + value + .as_table_like_mut() + .filter(|dep| { + dep.get("path").and_then(|path| path.as_str()).map(path_filter).unwrap_or(false) }) - .filter(|t| { - t.1.contains_key("path") && { - // if the path does not exists, we need to add this as git dependency - t.1.get("path") - .unwrap() - .as_str() - .map(|path| !cargo_toml_path.join(path).exists()) - .unwrap_or(false) + .map(|dep| { + dep.insert("workspace", toml_edit::value(true)); + dep.remove("path"); + + Dependency { + name: key.get().to_string(), + version: dep + .remove("version") + .and_then(|version| version.as_str().map(|s| s.to_string())), + default_features: dep.remove("default-features").and_then(|b| b.as_bool()), } }) - .map(|(k, mut v)| { - // remove `path` and add `git` and `rev` - v.remove("path"); - v.insert("git".into(), SUBSTRATE_GIT_URL.into()); - v.insert("rev".into(), commit_id.into()); + }; + + ["dependencies", "build-dependencies", "dev-dependencies"] + .into_iter() + .map(|table| -> (String, HashMap) { + ( + table.to_string(), + cargo_toml[table] + .as_table_mut() + .into_iter() + .flat_map(|deps| deps.iter_mut().filter_map(process_dep)) + .map(|dep| (dep.name.clone(), dep)) + .collect(), + ) + }) + .collect() +} - (k.clone(), v.into()) - }) - .collect::>(); +/// Processes all `Cargo.toml` files, aggregates dependencies and saves the changes. +fn process_cargo_tomls(cargo_tomls: &Vec) -> Dependencies { + /// Merge dependencies from one collection in another. + fn merge_deps(into: &mut Dependencies, from: Dependencies) { + from.into_iter().for_each(|(table, deps)| { + into.entry(table).or_insert_with(HashMap::new).extend(deps); + }); + } - dependencies.extend(deps_rewritten.into_iter()); + cargo_tomls.iter().fold(Dependencies::new(), |mut acc, path| { + let mut cargo_toml = parse_cargo_toml(&path); - cargo_toml.insert(table.into(), dependencies.into()); - } + let mut cargo_toml_path = path.clone(); + cargo_toml_path.pop(); // remove `Cargo.toml` from the path + let deps = update_git_dependencies(&mut cargo_toml, |dep_path| { + !cargo_toml_path.join(dep_path).exists() + }); + + write_cargo_toml(&path, cargo_toml); + merge_deps(&mut acc, deps); + acc + }) } /// Update the top level (workspace) `Cargo.toml` file. /// -/// - Adds `profile.release` = `panic = unwind` /// - Adds `workspace` definition -fn update_top_level_cargo_toml( +/// - Adds dependencies +/// - Adds `profile.release` = `panic = unwind` +fn update_root_cargo_toml( cargo_toml: &mut CargoToml, - workspace_members: Vec<&PathBuf>, - node_template_path: &Path, + members: &[String], + deps: Dependencies, + commit_id: &str, ) { - let mut panic_unwind = toml::value::Table::new(); - panic_unwind.insert("panic".into(), "unwind".into()); - - let mut profile = toml::value::Table::new(); - profile.insert("release".into(), panic_unwind.into()); - - cargo_toml.insert("profile".into(), profile.into()); + let mut workspace = Table::new(); + workspace.insert("members", value(Array::from_iter(members.iter()))); + + let mut workspace_dependencies = Table::new(); + deps.values() + .flatten() + .sorted_by_key(|(name, _)| *name) + .for_each(|(name, dep)| { + if let Some(version) = &dep.version { + workspace_dependencies[name]["version"] = value(version); + } + if let Some(default_features) = dep.default_features { + workspace_dependencies[name]["default-features"] = value(default_features); + } + workspace_dependencies[name]["git"] = value(SUBSTRATE_GIT_URL); + workspace_dependencies[name]["rev"] = value(commit_id); + }); + + workspace.insert("dependencies", Item::Table(workspace_dependencies)); + cargo_toml.insert("workspace", Item::Table(workspace)); + + let mut panic_unwind = Table::new(); + panic_unwind.insert("panic", value("unwind")); + let mut profile = Table::new(); + profile.insert("release", Item::Table(panic_unwind)); + cargo_toml.insert("profile", Item::Table(profile.into())); +} - let members = workspace_members +fn process_root_cargo_toml( + root_cargo_toml_path: &Path, + root_deps: Dependencies, + cargo_tomls: &[PathBuf], + node_template_path: &PathBuf, + commit_id: &str, +) { + let mut root_cargo_toml = parse_cargo_toml(root_cargo_toml_path); + let workspace_members = cargo_tomls .iter() .map(|p| { p.strip_prefix(node_template_path) @@ -165,16 +224,8 @@ fn update_top_level_cargo_toml( }) .collect::>(); - let mut members_section = toml::value::Table::new(); - members_section.insert("members".into(), members.into()); - - cargo_toml.insert("workspace".into(), members_section.into()); -} - -fn write_cargo_toml(path: &Path, cargo_toml: CargoToml) { - let content = toml::to_string_pretty(&cargo_toml).expect("Creates `Cargo.toml`"); - let mut file = File::create(path).expect(&format!("Creates `{}`.", path.display())); - write!(file, "{}", content).expect("Writes `Cargo.toml`"); + update_root_cargo_toml(&mut root_cargo_toml, &workspace_members, root_deps, commit_id); + write_cargo_toml(&root_cargo_toml_path, root_cargo_toml); } /// Build and test the generated node-template @@ -211,8 +262,8 @@ fn build_and_test(path: &Path, cargo_tomls: &[PathBuf]) { fn main() { let options = Options::parse(); + // Copy node-template to a temp build dir. let build_dir = tempfile::tempdir().expect("Creates temp build dir"); - let node_template_folder = options .node_template .canonicalize() @@ -220,46 +271,38 @@ fn main() { .file_name() .expect("Node template folder is last element of path") .to_owned(); + copy_node_template(&options.node_template, build_dir.path()); // The path to the node-template in the build dir. let node_template_path = build_dir.path().join(node_template_folder); + let root_cargo_toml_path = node_template_path.join("Cargo.toml"); - copy_node_template(&options.node_template, build_dir.path()); - let mut cargo_tomls = find_cargo_tomls(build_dir.path().to_owned()); - - let commit_id = get_git_commit_id(&options.node_template); - let top_level_cargo_toml_path = node_template_path.join("Cargo.toml"); + // Get all `Cargo.toml` files in the node-template + let mut cargo_tomls = find_cargo_tomls(&node_template_path); - // Check if top level Cargo.toml exists. If not, create one in the destination - if !cargo_tomls.contains(&top_level_cargo_toml_path) { - // create the top_level_cargo_toml + // Check if top level Cargo.toml exists. If not, create one in the destination, + // else remove it from the list, as this requires some special treatments. + if let Some(index) = cargo_tomls.iter().position(|x| *x == root_cargo_toml_path) { + cargo_tomls.remove(index); + } else { OpenOptions::new() .create(true) .write(true) - .open(top_level_cargo_toml_path.clone()) + .open(root_cargo_toml_path.clone()) .expect("Create root level `Cargo.toml` failed."); - - // push into our data structure - cargo_tomls.push(PathBuf::from(top_level_cargo_toml_path.clone())); } - cargo_tomls.iter().for_each(|t| { - let mut cargo_toml = parse_cargo_toml(&t); - replace_path_dependencies_with_git(&t, &commit_id, &mut cargo_toml); - - // Check if this is the top level `Cargo.toml`, as this requires some special treatments. - if top_level_cargo_toml_path == *t { - // All workspace member `Cargo.toml` file paths. - let workspace_members = - cargo_tomls.iter().filter(|p| **p != top_level_cargo_toml_path).collect(); - - update_top_level_cargo_toml(&mut cargo_toml, workspace_members, &node_template_path); - } - - write_cargo_toml(&t, cargo_toml); - }); + // Process all `Cargo.toml` files. + let root_deps = process_cargo_tomls(&cargo_tomls); + process_root_cargo_toml( + &root_cargo_toml_path, + root_deps, + &cargo_tomls, + &node_template_path, + &get_git_commit_id(&options.node_template), + ); - // adding root rustfmt to node template build path + // Add root rustfmt to node template build path. let node_template_rustfmt_toml_path = node_template_path.join("rustfmt.toml"); let root_rustfmt_toml = &options.node_template.join("../../rustfmt.toml"); if root_rustfmt_toml.exists() { @@ -277,3 +320,103 @@ fn main() { tar.append_dir_all("substrate-node-template", node_template_path) .expect("Writes substrate-node-template archive"); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_update_git_dependencies() { + let toml = r#" +[dev-dependencies] +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } + +[dependencies] +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +sp-io = { version = "7.0.0", path = "../../../../primitives/io" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/system" } +"#; + let mut cargo_toml = toml.parse::().expect("invalid doc"); + let actual_deps = update_git_dependencies(&mut cargo_toml, |_| true); + + assert_eq!(actual_deps.len(), 3); + assert_eq!(actual_deps.get("dependencies").unwrap().len(), 2); + assert_eq!(actual_deps.get("dev-dependencies").unwrap().len(), 0); + assert_eq!( + actual_deps.get("dependencies").unwrap().get("sp-io").unwrap(), + &Dependency { + name: "sp-io".into(), + version: Some("7.0.0".into()), + default_features: None + } + ); + assert_eq!( + actual_deps.get("dependencies").unwrap().get("frame-system").unwrap(), + &Dependency { + name: "frame-system".into(), + version: Some("4.0.0-dev".into()), + default_features: Some(false), + } + ); + + let expected_toml = r#" +[dev-dependencies] +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } + +[dependencies] +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +sp-io = { workspace = true } +frame-system = { workspace = true } +"#; + assert_eq!(cargo_toml.to_string(), expected_toml); + } + + #[test] + fn test_update_root_cargo_toml() { + let mut cargo_toml = CargoToml::new(); + update_root_cargo_toml( + &mut cargo_toml, + &vec!["node".into(), "pallets/template".into(), "runtime".into()], + Dependencies::from([ + ( + "dependencies".into(), + HashMap::from([ + ( + "sp-io".into(), + Dependency { + name: "sp-io".into(), + version: Some("7.0.0".into()), + default_features: None, + }, + ), + ( + "frame-system".into(), + Dependency { + name: "frame-system".into(), + version: Some("4.0.0-dev".into()), + default_features: Some(true), + }, + ), + ]), + ), + ("dev-dependencies".into(), HashMap::new()), + ("build-dependencies".into(), HashMap::new()), + ]), + "commit_id", + ); + + let expected_toml = r#"[workspace] +members = ["node", "pallets/template", "runtime"] + +[workspace.dependencies] +frame-system = { version = "4.0.0-dev", default-features = true, git = "https://github.com/paritytech/substrate.git", rev = "commit_id" } +sp-io = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", rev = "commit_id" } + +[profile] + +[profile.release] +panic = "unwind" +"#; + assert_eq!(cargo_toml.to_string(), expected_toml); + } +} diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index b2c1e8f47b1af..39ee6667ff9f0 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -102,7 +102,7 @@ impl GenesisInit for GenesisParameters { .with_heap_pages(self.heap_pages_override) .with_wasm_code(&self.wasm_code) .with_extra_storage(self.extra_storage.clone()) - .build_storage() + .build() } } diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 62ce26c55e38b..aa57eb1928fc2 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -18,8 +18,7 @@ //! Tool for creating the genesis block. use super::{ - currency, substrate_test_pallet, wasm_binary_unwrap, AccountId, AuthorityId, Balance, - GenesisConfig, + currency, substrate_test_pallet, wasm_binary_unwrap, AccountId, Balance, GenesisConfig, }; use codec::Encode; use sc_service::construct_genesis_block; @@ -34,14 +33,19 @@ use sp_runtime::{ BuildStorage, }; -/// Builder for generating storage from substrate-test-runtime genesis config. Default storage can -/// be extended with additional key-value pairs. +/// Builder for generating storage from substrate-test-runtime genesis config. +/// +/// Default storage can be extended with additional key-value pairs. pub struct GenesisStorageBuilder { - authorities: Vec, + /// Authorities accounts used by any component requiring an authority set (e.g. babe). + authorities: Vec, + /// Accounts to be endowed with some funds. balances: Vec<(AccountId, u64)>, + /// Override default number of heap pages. heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. extra_storage: Storage, + /// Optional wasm code override. wasm_code: Option>, } @@ -50,9 +54,9 @@ impl Default for GenesisStorageBuilder { fn default() -> Self { Self::new( vec![ - sr25519::Public::from(Sr25519Keyring::Alice).into(), - sr25519::Public::from(Sr25519Keyring::Bob).into(), - sr25519::Public::from(Sr25519Keyring::Charlie).into(), + Sr25519Keyring::Alice.into(), + Sr25519Keyring::Bob.into(), + Sr25519Keyring::Charlie.into(), ], (0..16_usize) .into_iter() @@ -74,7 +78,7 @@ impl GenesisStorageBuilder { /// from `extra_storage` will be injected into built storage. `HEAP_PAGES` key and value will /// also be placed into storage. pub fn new( - authorities: Vec, + authorities: Vec, endowed_accounts: Vec, balance: Balance, ) -> Self { @@ -104,17 +108,28 @@ impl GenesisStorageBuilder { } /// Builds the `GenesisConfig` and returns its storage. - pub fn build_storage(&mut self) -> Storage { + pub fn build(self) -> Storage { + let authorities_sr25519: Vec<_> = self + .authorities + .clone() + .into_iter() + .map(|id| sr25519::Public::from(id)) + .collect(); + let genesis_config = GenesisConfig { system: frame_system::GenesisConfig { code: self.wasm_code.clone().unwrap_or(wasm_binary_unwrap().to_vec()), }, babe: pallet_babe::GenesisConfig { - authorities: self.authorities.clone().into_iter().map(|x| (x, 1)).collect(), + authorities: authorities_sr25519 + .clone() + .into_iter() + .map(|x| (x.into(), 1)) + .collect(), epoch_config: Some(crate::TEST_RUNTIME_BABE_EPOCH_CONFIGURATION), }, substrate_test: substrate_test_pallet::GenesisConfig { - authorities: self.authorities.clone(), + authorities: authorities_sr25519.clone(), }, balances: pallet_balances::GenesisConfig { balances: self.balances.clone() }, }; diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 00db6a745b3f5..e1d66e86002e7 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -64,13 +64,11 @@ use sp_runtime::{ use sp_version::NativeVersion; use sp_version::RuntimeVersion; -// Ensure Babe and Aura use the same crypto to simplify things a bit. -pub use sp_consensus_babe::{AllowedSlots, AuthorityId, BabeEpochConfiguration, Slot}; +pub use sp_consensus_babe::{AllowedSlots, BabeEpochConfiguration, Slot}; pub use pallet_balances::Call as BalancesCall; pub type AuraId = sp_consensus_aura::sr25519::AuthorityId; - #[cfg(feature = "std")] pub use extrinsic::{ExtrinsicBuilder, Transfer}; @@ -616,10 +614,7 @@ impl_runtime_apis! { } fn authorities() -> Vec { - SubstrateTest::authorities().into_iter().map(|a| { - let authority: sr25519::Public = a.into(); - AuraId::from(authority) - }).collect() + SubstrateTest::authorities().into_iter().map(|auth| AuraId::from(auth)).collect() } } @@ -630,10 +625,9 @@ impl_runtime_apis! { slot_duration: Babe::slot_duration(), epoch_length: EpochDuration::get(), c: epoch_config.c, - authorities: SubstrateTest::authorities() - .into_iter().map(|x|(x, 1)).collect(), - randomness: Babe::randomness(), - allowed_slots: epoch_config.allowed_slots, + authorities: Babe::authorities().to_vec(), + randomness: Babe::randomness(), + allowed_slots: epoch_config.allowed_slots, } } @@ -1087,7 +1081,7 @@ mod tests { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000 * currency::DOLLARS, ) - .build_storage() + .build() .into() } @@ -1095,7 +1089,7 @@ mod tests { fn validate_storage_keys() { assert_eq!( genesismap::GenesisStorageBuilder::default() - .build_storage() + .build() .top .keys() .cloned() diff --git a/test-utils/runtime/src/substrate_test_pallet.rs b/test-utils/runtime/src/substrate_test_pallet.rs index 98ebd550b9eef..40e7af0b43769 100644 --- a/test-utils/runtime/src/substrate_test_pallet.rs +++ b/test-utils/runtime/src/substrate_test_pallet.rs @@ -21,8 +21,8 @@ //! functioning runtime. Some calls are allowed to be submitted as unsigned extrinsics, however most //! of them requires signing. Refer to `pallet::Call` for further details. -use crate::AuthorityId; use frame_support::{pallet_prelude::*, storage}; +use sp_core::sr25519::Public; use sp_runtime::transaction_validity::{ InvalidTransaction, TransactionSource, TransactionValidity, ValidTransaction, }; @@ -49,12 +49,12 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn authorities)] - pub type Authorities = StorageValue<_, Vec, ValueQuery>; + pub type Authorities = StorageValue<_, Vec, ValueQuery>; #[pallet::genesis_config] #[cfg_attr(feature = "std", derive(Default))] pub struct GenesisConfig { - pub authorities: Vec, + pub authorities: Vec, } #[pallet::genesis_build]