diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90e0c2b007a4b..f27543026f42d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -150,7 +150,7 @@ test-linux-stable: &test-linux paths: - ${CI_COMMIT_SHORT_SHA}_warnings.log -test-dependency-rules: &test-linux +test-dependency-rules: stage: test <<: *docker-env except: @@ -159,7 +159,7 @@ test-dependency-rules: &test-linux script: - .maintain/ensure-deps.sh -test-frame-staking: &test-frame-staking +test-frame-staking: stage: test <<: *docker-env variables: @@ -175,7 +175,7 @@ test-frame-staking: &test-frame-staking - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features std - sccache -s -test-wasmtime: &test-wasmtime +test-wasmtime: stage: test <<: *docker-env variables: @@ -244,7 +244,7 @@ node-exits: - ./.maintain/check_for_exit.sh -test-full-crypto-feature: &test-full-crypto-feature +test-full-crypto-feature: stage: test <<: *docker-env variables: @@ -265,17 +265,18 @@ test-full-crypto-feature: &test-full-crypto-feature #### stage: build -build-linux-substrate: +build-linux-substrate: &build-binary stage: build <<: *collect-artifacts <<: *docker-env <<: *build-only + before_script: + - mkdir -p ./artifacts/substrate/ except: variables: - $DEPLOY_TAG script: - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mkdir -p ./artifacts/substrate/ - mv ./target/release/substrate ./artifacts/substrate/. - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then @@ -292,19 +293,13 @@ build-linux-substrate: - sccache -s build-linux-subkey: - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-only - except: - variables: - - $DEPLOY_TAG + <<: *build-binary + before_script: + - mkdir -p ./artifacts/subkey script: - cd ./bin/utils/subkey - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose - cd - - - sccache -s - - mkdir -p ./artifacts/subkey - mv ./target/release/subkey ./artifacts/subkey/. - echo -n "Subkey version = " - ./artifacts/subkey/subkey --version | diff --git a/Cargo.lock b/Cargo.lock index f951617658729..e918ca65c23d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,7 +490,7 @@ dependencies = [ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "sc-keystore 2.0.0", "sp-core 2.0.0", - "structopt 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3053,7 +3053,7 @@ dependencies = [ "sp-runtime 2.0.0", "sp-timestamp 2.0.0", "sp-transaction-pool 2.0.0", - "structopt 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-build-script-utils 2.0.0", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3801,7 +3801,6 @@ dependencies = [ "frame-support 2.0.0", "frame-system 2.0.0", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ref_thread_local 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", @@ -4367,14 +4366,28 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.2.6" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "proc-macro-error-attr 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "proc-macro-error-attr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "syn-mid 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "proc-macro-hack" version = "0.5.11" @@ -4738,11 +4751,6 @@ dependencies = [ "rust-argon2 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ref_thread_local" -version = "0.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "regex" version = "1.3.1" @@ -5029,7 +5037,7 @@ dependencies = [ "sp-panic-handler 2.0.0", "sp-runtime 2.0.0", "sp-state-machine 2.0.0", - "structopt 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -6671,20 +6679,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "structopt" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt-derive 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt-derive 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "structopt-derive" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-error 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-error 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -6714,6 +6722,7 @@ name = "subkey" version = "2.0.0" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", "frame-system 2.0.0", "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -6912,6 +6921,16 @@ dependencies = [ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "syn-mid" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synstructure" version = "0.12.3" @@ -8459,7 +8478,8 @@ dependencies = [ "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" "checksum primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0253db64c26d8b4e7896dd2063b516d2a1b9e0a5da26b5b78335f236d1e9522" "checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" -"checksum proc-macro-error 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "aeccfe4d5d8ea175d5f0e4a2ad0637e0f4121d63bd99d356fb1f39ab2e7c6097" +"checksum proc-macro-error 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "53c98547ceaea14eeb26fcadf51dc70d01a2479a7839170eae133721105e4428" +"checksum proc-macro-error-attr 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c2bf5d493cf5d3e296beccfd61794e445e830dfc8070a9c248ad3ee071392c6c" "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" @@ -8499,7 +8519,6 @@ dependencies = [ "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecedbca3bf205f8d8f5c2b44d83cd0690e39ee84b951ed649e9f1841132b66d" -"checksum ref_thread_local 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d813022b2e00774a48eaf43caaa3c20b45f040ba8cbf398e2e8911a06668dbe6" "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" "checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" @@ -8559,8 +8578,8 @@ dependencies = [ "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum string-interner 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd710eadff449a1531351b0e43eb81ea404336fa2f56c777427ab0e32a4cf183" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -"checksum structopt 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "30b3a3e93f5ad553c38b3301c8a0a0cec829a36783f6a0c467fc4bf553a5f5bf" -"checksum structopt-derive 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea692d40005b3ceba90a9fe7a78fa8d4b82b0ce627eebbffc329aab850f3410e" +"checksum structopt 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "884ae79d6aad1e738f4a70dff314203fd498490a63ebc4d03ea83323c40b7b72" +"checksum structopt-derive 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0a97f829a34a0a9d5b353a881025a23b8c9fd09d46be6045df6b22920dbd7a93" "checksum strum 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6138f8f88a16d90134763314e3fc76fa3ed6a7db4725d6acf9a3ef95a3188d22" "checksum strum_macros 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" "checksum substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" @@ -8568,6 +8587,7 @@ dependencies = [ "checksum subtle 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" "checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" +"checksum syn-mid 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9fd3937748a7eccff61ba5b90af1a20dbf610858923a9192ea0ecb0cb77db1d0" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum sysinfo 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4b2468c629cffba39c0a4425849ab3cdb03d9dfacba69684609aea04d08ff9" "checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" diff --git a/Cargo.toml b/Cargo.toml index 505cd299d9c71..e5940e5b7cdf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ members = [ "client/telemetry", "client/transaction-pool", "client/transaction-pool/graph", + "utils/prometheus", "utils/wasm-builder-runner", "utils/grafana-data-source", "utils/grafana-data-source/test", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 0ce5115831a28..47efe3c70235f 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -31,7 +31,7 @@ hex-literal = "0.2.1" jsonrpc-core = "14.0.3" log = "0.4.8" rand = "0.7.2" -structopt = "0.3.3" +structopt = "=0.3.7" # primitives sp-authority-discovery = { version = "2.0.0", path = "../../../primitives/authority-discovery" } @@ -107,7 +107,7 @@ tempfile = "3.1.0" [build-dependencies] sc-cli = { version = "2.0.0", package = "sc-cli", path = "../../../client/cli" } build-script-utils = { version = "2.0.0", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } -structopt = "0.3.3" +structopt = "=0.3.7" vergen = "3.0.4" [features] diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 8cd8cb9f33d8c..3b11ff31252d0 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -20,8 +20,8 @@ use tokio::runtime::{Builder as RuntimeBuilder, Runtime}; use sc_cli::{IntoExit, NoCustom, SharedParams, ImportParams, error}; use sc_service::{AbstractService, Roles as ServiceRoles, Configuration}; use log::info; -use structopt::{StructOpt, clap::App}; -use sc_cli::{display_role, parse_and_prepare, AugmentClap, GetSharedParams, ParseAndPrepare}; +use structopt::StructOpt; +use sc_cli::{display_role, parse_and_prepare, GetSharedParams, ParseAndPrepare}; use crate::{service, ChainSpec, load_spec}; use crate::factory_impl::FactoryState; use node_transaction_factory::RuntimeAdapter; @@ -88,12 +88,6 @@ pub struct FactoryCmd { pub import_params: ImportParams, } -impl AugmentClap for FactoryCmd { - fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { - FactoryCmd::augment_clap(app) - } -} - /// Parse command line arguments into service configuration. pub fn run(args: I, exit: E, version: sc_cli::VersionInfo) -> error::Result<()> where I: IntoIterator, diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index e31419d3b9f2e..c8d79afbced11 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -11,4 +11,4 @@ sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } node-cli = { version = "2.0.0", path = "../../node/cli" } sp-core = { version = "2.0.0", path = "../../../primitives/core" } rand = "0.7.2" -structopt = "0.3.3" +structopt = "=0.3.7" diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 6d687ce9109f2..c042b76f4a820 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -22,6 +22,7 @@ pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } rpassword = "4.0.1" itertools = "0.8.2" +derive_more = { version = "0.99.2" } [features] bench = [] diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index f10c949dbec25..d586fac6f906f 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -31,7 +31,7 @@ use sp_core::{ }; use sp_runtime::{traits::{IdentifyAccount, Verify}, generic::Era}; use std::{ - convert::{TryInto, TryFrom}, io::{stdin, Read}, str::FromStr, path::PathBuf, fs, + convert::{TryInto, TryFrom}, io::{stdin, Read}, str::FromStr, path::PathBuf, fs, fmt, }; mod vanity; @@ -234,12 +234,12 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { ]) } -fn main() { +fn main() -> Result<(), Error> { let usage = get_usage(); let matches = get_app(&usage).get_matches(); if matches.is_present("ed25519") { - return execute::(matches) + return execute::(matches); } if matches.is_present("secp256k1") { return execute::(matches) @@ -253,24 +253,41 @@ fn main() { /// /// If the `URI` given as CLI argument is a file, the file content is taken as `URI`. /// If no `URI` is given to the CLI, the user is prompted for it. -fn get_uri(match_name: &str, matches: &ArgMatches) -> String { - if let Some(uri) = matches.value_of(match_name) { +fn get_uri(match_name: &str, matches: &ArgMatches) -> Result { + let uri = if let Some(uri) = matches.value_of(match_name) { let file = PathBuf::from(uri); if file.is_file() { - fs::read_to_string(uri) - .expect(&format!("Failed to read `URI` file: {}", uri)) + fs::read_to_string(uri)? .trim_end() .into() } else { uri.into() } } else { - rpassword::read_password_from_tty(Some("URI: ")) - .expect("Failed to read `URI`") + rpassword::read_password_from_tty(Some("URI: "))? + }; + + Ok(uri) +} + +#[derive(derive_more::Display, derive_more::From)] +enum Error { + Static(&'static str), + Io(std::io::Error), + Formatted(String), +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) } } -fn execute(matches: ArgMatches) +fn static_err(msg: &'static str) -> Result<(), Error> { + Err(Error::Static(msg)) +} + +fn execute(matches: ArgMatches) -> Result<(), Error> where SignatureOf: SignatureT, PublicOf: PublicT, @@ -279,49 +296,55 @@ where let password = matches.value_of("password"); let password = if password.is_some() && password_interactive { - panic!("`--password` given and `--password-interactive` selected!"); + return static_err("`--password` given and `--password-interactive` selected!"); } else if password_interactive { Some( - rpassword::read_password_from_tty(Some("Key password: ")) - .expect("Reads password from tty") + rpassword::read_password_from_tty(Some("Key password: "))? ) } else { password.map(Into::into) }; let password = password.as_ref().map(String::as_str); - let maybe_network: Option = matches.value_of("network").map(|network| { + let maybe_network: Option = match matches.value_of("network").map(|network| { network .try_into() - .expect("Invalid network name. See --help for available networks.") - }); + .map_err(|_| Error::Static("Invalid network name. See --help for available networks.")) + }) { + Some(Err(e)) => return Err(e), + Some(Ok(v)) => Some(v), + None => None, + }; + if let Some(network) = maybe_network { set_default_ss58_version(network); } match matches.subcommand() { ("generate", Some(matches)) => { - let mnemonic = generate_mnemonic(matches); + let mnemonic = generate_mnemonic(matches)?; C::print_from_uri(mnemonic.phrase(), password, maybe_network); } ("inspect", Some(matches)) => { - C::print_from_uri(&get_uri("uri", &matches), password, maybe_network); + C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network); } ("sign", Some(matches)) => { - let suri = get_uri("suri", &matches); + let suri = get_uri("suri", &matches)?; let should_decode = matches.is_present("hex"); - let message = read_message_from_stdin(should_decode); - let signature = do_sign::(&suri, message, password); + + let message = read_message_from_stdin(should_decode)?; + let signature = do_sign::(&suri, message, password)?; println!("{}", signature); } ("verify", Some(matches)) => { - let uri = get_uri("uri", &matches); + let uri = get_uri("uri", &matches)?; let should_decode = matches.is_present("hex"); - let message = read_message_from_stdin(should_decode); - let is_valid_signature = do_verify::(matches, &uri, message); + + let message = read_message_from_stdin(should_decode)?; + let is_valid_signature = do_verify::(matches, &uri, message)?; if is_valid_signature { println!("Signature verifies correctly."); } else { - println!("Signature invalid."); + return static_err("Signature invalid."); } } ("vanity", Some(matches)) => { @@ -329,17 +352,17 @@ where .value_of("pattern") .map(str::to_string) .unwrap_or_default(); - let result = vanity::generate_key::(&desired).expect("Key generation failed"); + let result = vanity::generate_key::(&desired)?; let formated_seed = format_seed::(result.seed); C::print_from_uri(&formated_seed, None, maybe_network); } ("transfer", Some(matches)) => { - let signer = read_pair::(matches.value_of("from"), password); - let index = read_required_parameter::(matches, "index"); - let genesis_hash = read_genesis_hash(matches); + let signer = read_pair::(matches.value_of("from"), password)?; + let index = read_required_parameter::(matches, "index")?; + let genesis_hash = read_genesis_hash(matches)?; let to: AccountId = read_account_id(matches.value_of("to")); - let amount = read_required_parameter::(matches, "amount"); + let amount = read_required_parameter::(matches, "amount")?; let function = Call::Balances(BalancesCall::transfer(to.into(), amount)); let extrinsic = create_extrinsic::(function, index, signer, genesis_hash); @@ -347,9 +370,9 @@ where print_extrinsic(extrinsic); } ("sign-transaction", Some(matches)) => { - let signer = read_pair::(matches.value_of("suri"), password); - let index = read_required_parameter::(matches, "nonce"); - let genesis_hash = read_genesis_hash(matches); + let signer = read_pair::(matches.value_of("suri"), password)?; + let index = read_required_parameter::(matches, "nonce")?; + let genesis_hash = read_genesis_hash(matches)?; let call = matches.value_of("call").expect("call is required; qed"); let function: Call = hex::decode(&call) @@ -363,81 +386,86 @@ where } _ => print_usage(&matches), } + + Ok(()) } /// Creates a new randomly generated mnemonic phrase. -fn generate_mnemonic(matches: &ArgMatches) -> Mnemonic { - let words = matches - .value_of("words") - .map(|x| usize::from_str(x).expect("Invalid number given for --words")) - .map(|x| { - MnemonicType::for_word_count(x) - .expect("Invalid number of words given for phrase: must be 12/15/18/21/24") - }) - .unwrap_or(MnemonicType::Words12); - Mnemonic::new(words, Language::English) -} - -fn do_sign(suri: &str, message: Vec, password: Option<&str>) -> String +fn generate_mnemonic(matches: &ArgMatches) -> Result { + let words = match matches.value_of("words") { + Some(words) => { + let num = usize::from_str(words).map_err(|_| Error::Static("Invalid number given for --words"))?; + MnemonicType::for_word_count(num) + .map_err(|_| Error::Static("Invalid number of words given for phrase: must be 12/15/18/21/24"))? + }, + None => MnemonicType::Words12, + }; + Ok(Mnemonic::new(words, Language::English)) +} + +fn do_sign(suri: &str, message: Vec, password: Option<&str>) -> Result where SignatureOf: SignatureT, PublicOf: PublicT, { - let pair = read_pair::(Some(suri), password); + let pair = read_pair::(Some(suri), password)?; let signature = pair.sign(&message); - format_signature::(&signature) + Ok(format_signature::(&signature)) } -fn do_verify(matches: &ArgMatches, uri: &str, message: Vec) -> bool +fn do_verify(matches: &ArgMatches, uri: &str, message: Vec) -> Result where SignatureOf: SignatureT, PublicOf: PublicT, { - let signature = read_signature::(matches); + + let signature = read_signature::(matches)?; let pubkey = read_public_key::(Some(uri)); - <::Pair as Pair>::verify(&signature, &message, &pubkey) + Ok(<::Pair as Pair>::verify(&signature, &message, &pubkey)) +} + +fn decode_hex>(message: T) -> Result, Error> { + hex::decode(message).map_err(|e| Error::Formatted(format!("Invalid hex ({})", e))) } -fn read_message_from_stdin(should_decode: bool) -> Vec { +fn read_message_from_stdin(should_decode: bool) -> Result, Error> { let mut message = vec![]; stdin() .lock() - .read_to_end(&mut message) - .expect("Error reading from stdin"); + .read_to_end(&mut message)?; if should_decode { - message = hex::decode(&message).expect("Invalid hex in message"); + message = decode_hex(&message)?; } - message + Ok(message) } -fn read_required_parameter(matches: &ArgMatches, name: &str) -> T where +fn read_required_parameter(matches: &ArgMatches, name: &str) -> Result where ::Err: std::fmt::Debug, { let str_value = matches .value_of(name) .expect("parameter is required; thus it can't be None; qed"); - str::parse::(str_value).unwrap_or_else(|_| - panic!("Invalid `{}' parameter; expecting an integer.", name) + str::parse::(str_value).map_err(|_| + Error::Formatted(format!("Invalid `{}' parameter; expecting an integer.", name)) ) } -fn read_genesis_hash(matches: &ArgMatches) -> H256 { +fn read_genesis_hash(matches: &ArgMatches) -> Result { let genesis_hash: Hash = match matches.value_of("genesis").unwrap_or("alex") { "elm" => hex!["10c08714a10c7da78f40a60f6f732cf0dba97acfb5e2035445b032386157d5c3"].into(), "alex" => hex!["dcd1346701ca8396496e52aa2785b1748deb6db09551b72159dcb3e08991025b"].into(), - h => hex::decode(h) - .ok() - .and_then(|x| Decode::decode(&mut &x[..]).ok()) + h => Decode::decode(&mut &decode_hex(h)?[..]) .expect("Invalid genesis hash or unrecognised chain identifier"), }; println!( "Using a genesis hash of {}", HexDisplay::from(&genesis_hash.as_ref()) ); - genesis_hash + Ok(genesis_hash) } -fn read_signature(matches: &ArgMatches) -> SignatureOf where +fn read_signature(matches: &ArgMatches) -> Result, Error> +where SignatureOf: SignatureT, PublicOf: PublicT, { @@ -445,19 +473,20 @@ fn read_signature(matches: &ArgMatches) -> SignatureOf where .value_of("sig") .expect("signature parameter is required; thus it can't be None; qed"); let mut signature = <::Pair as Pair>::Signature::default(); - let sig_data = hex::decode(sig_data).expect("signature is invalid hex"); + let sig_data = decode_hex(sig_data)?; if sig_data.len() != signature.as_ref().len() { - panic!( + return Err(Error::Formatted(format!( "signature has an invalid length. read {} bytes, expected {} bytes", sig_data.len(), signature.as_ref().len(), - ); + ))); } signature.as_mut().copy_from_slice(&sig_data); - signature + Ok(signature) } -fn read_public_key(matched_uri: Option<&str>) -> PublicOf where +fn read_public_key(matched_uri: Option<&str>) -> PublicOf +where PublicOf: PublicT, { let uri = matched_uri.expect("parameter is required; thus it can't be None; qed"); @@ -494,12 +523,12 @@ fn read_account_id(matched_uri: Option<&str>) -> AccountId { fn read_pair( matched_suri: Option<&str>, password: Option<&str>, -) -> ::Pair where +) -> Result<::Pair, Error> where SignatureOf: SignatureT, PublicOf: PublicT, { - let suri = matched_suri.expect("parameter is required; thus it can't be None; qed"); - C::pair_from_suri(suri, password) + let suri = matched_suri.ok_or(Error::Static("parameter is required; thus it can't be None; qed"))?; + Ok(C::pair_from_suri(suri, password)) } fn format_signature(signature: &SignatureOf) -> String { @@ -591,7 +620,7 @@ mod tests { let matches = app.clone().get_matches_from(arg_vec); let matches = matches.subcommand().1.unwrap(); - let mnemonic = generate_mnemonic(matches); + let mnemonic = generate_mnemonic(matches).expect("generate failed"); let (pair, seed) = <::Pair as Pair>::from_phrase(mnemonic.phrase(), password) @@ -601,14 +630,15 @@ mod tests { let seed = format_seed::(seed); let message = "Blah Blah\n".as_bytes().to_vec(); - let signature = do_sign::(&seed, message.clone(), password); + let signature = do_sign::(&seed, message.clone(), password).expect("signing failed"); // Verify the previous signature. let arg_vec = vec!["subkey", "verify", &signature[..], &public_key[..]]; let matches = get_app(&usage).get_matches_from(arg_vec); let matches = matches.subcommand().1.unwrap(); - assert!(do_verify::(matches, &public_key, message)); + + assert!(do_verify::(matches, &public_key, message).expect("verify failed")); } #[test] diff --git a/bin/utils/subkey/src/vanity.rs b/bin/utils/subkey/src/vanity.rs index 33e8559b1fbc4..ee5a2f2cce516 100644 --- a/bin/utils/subkey/src/vanity.rs +++ b/bin/utils/subkey/src/vanity.rs @@ -62,7 +62,7 @@ fn calculate_score(_desired: &str, key: &str) -> usize { 0 } -pub(super) fn generate_key(desired: &str) -> Result, &str> where +pub(super) fn generate_key(desired: &str) -> Result, &'static str> where PublicOf: PublicT, { if desired.is_empty() { diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 49f327405bac3..5c7ac86904164 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -29,9 +29,10 @@ sp-core = { version = "2.0.0", path = "../../primitives/core" } sc-service = { version = "2.0.0", default-features = false, path = "../service" } sp-state-machine = { version = "2.0.0", path = "../../primitives/state-machine" } sc-telemetry = { version = "2.0.0", path = "../telemetry" } +sc-prometheus = { path = "../../utils/prometheus" } sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } names = "0.11.0" -structopt = "0.3.3" +structopt = "=0.3.7" sc-tracing = { version = "2.0.0", path = "../tracing" } [target.'cfg(not(target_os = "unknown"))'.dependencies] diff --git a/client/cli/src/informant/display.rs b/client/cli/src/informant/display.rs index 1742becb865c4..f0b6e61176de7 100644 --- a/client/cli/src/informant/display.rs +++ b/client/cli/src/informant/display.rs @@ -21,7 +21,7 @@ use sc_network::SyncState; use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating}; use sc_service::NetworkStatus; use std::{convert::{TryFrom, TryInto}, fmt, time}; - +use sc_prometheus::prometheus_gauge; /// State of the informant display system. /// /// This is the system that handles the line that gets regularly printed and that looks something @@ -63,7 +63,10 @@ impl InformantDisplay { let (status, target) = match (net_status.sync_state, net_status.best_seen_block) { (SyncState::Idle, _) => ("Idle".into(), "".into()), (SyncState::Downloading, None) => (format!("Syncing{}", speed), "".into()), - (SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed), format!(", target=#{}", n)), + (SyncState::Downloading, Some(n)) => { + prometheus_gauge!(TARGET_NUM => n.saturated_into().try_into().unwrap()); + (format!("Syncing{}", speed), format!(", target=#{}", n)) + } }; info!( diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index d1b3388432d10..e2323694f1b51 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -48,7 +48,7 @@ use std::{ use names::{Generator, Name}; use regex::Regex; -use structopt::{StructOpt, clap::AppSettings}; +use structopt::{StructOpt, StructOptInternal, clap::AppSettings}; #[doc(hidden)] pub use structopt::clap::App; use params::{ @@ -57,7 +57,7 @@ use params::{ NodeKeyParams, NodeKeyType, Cors, CheckBlockCmd, }; pub use params::{NoCustom, CoreParams, SharedParams, ImportParams, ExecutionStrategy}; -pub use traits::{GetSharedParams, AugmentClap}; +pub use traits::GetSharedParams; use app_dirs::{AppInfo, AppDataType}; use log::info; use lazy_static::lazy_static; @@ -196,7 +196,7 @@ pub fn parse_and_prepare<'a, CC, RP, I>( ) -> ParseAndPrepare<'a, CC, RP> where CC: StructOpt + Clone + GetSharedParams, - RP: StructOpt + Clone + AugmentClap, + RP: StructOpt + Clone + StructOptInternal, I: IntoIterator, ::Item: Into + Clone, { @@ -941,7 +941,14 @@ where config.tracing_targets = cli.tracing_targets.into(); config.tracing_receiver = cli.tracing_receiver.into(); - + + // Override prometheus + match cli.prometheus_endpoint { + None => {config.prometheus_endpoint = None;}, + Some(x) => { + config.prometheus_endpoint = Some(parse_address(&format!("{}:{}", x, 33333), cli.prometheus_port)?); + } + } // Imply forced authoring on --dev config.force_authoring = cli.shared_params.dev || cli.force_authoring; diff --git a/client/cli/src/params.rs b/client/cli/src/params.rs index e8d00978a8d79..79de3384e3168 100644 --- a/client/cli/src/params.rs +++ b/client/cli/src/params.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::traits::{AugmentClap, GetSharedParams}; +use crate::traits::GetSharedParams; use std::{str::FromStr, path::PathBuf}; -use structopt::{StructOpt, clap::{arg_enum, App, AppSettings, SubCommand, Arg}}; +use structopt::{StructOpt, StructOptInternal, clap::{arg_enum, App, AppSettings, SubCommand, Arg}}; pub use crate::execution_strategy::ExecutionStrategy; @@ -208,7 +208,7 @@ pub struct NetworkConfigurationParams { #[allow(missing_docs)] #[structopt(flatten)] - pub node_key_params: NodeKeyParams + pub node_key_params: NodeKeyParams, } arg_enum! { @@ -278,7 +278,7 @@ pub struct NodeKeyParams { /// If the file does not exist, it is created with a newly generated secret key of /// the chosen type. #[structopt(long = "node-key-file", value_name = "FILE")] - pub node_key_file: Option + pub node_key_file: Option, } /// Parameters used to create the pool configuration. @@ -440,6 +440,12 @@ pub struct RunCmd { /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. #[structopt(long = "ws-external")] pub ws_external: bool, + /// Prometheus exporter TCP port. + #[structopt(long = "prometheus-port", value_name = "PORT")] + pub prometheus_port: Option, + /// Prometheus exporter IP addr. + #[structopt(long = "prometheus-addr", value_name = "Local IP address")] + pub prometheus_endpoint: Option, /// Listen to all Websocket interfaces. /// @@ -623,14 +629,14 @@ impl StructOpt for Keyring { unimplemented!("Should not be called for `TestAccounts`.") } - fn from_clap(m: &::structopt::clap::ArgMatches) -> Self { + fn from_clap(m: &structopt::clap::ArgMatches) -> Self { Keyring { account: TEST_ACCOUNTS_CLI_VALUES.iter().find(|a| m.is_present(&a.name)).map(|a| a.variant), } } } -impl AugmentClap for Keyring { +impl StructOptInternal for Keyring { fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { TEST_ACCOUNTS_CLI_VALUES.iter().fold(app, |app, a| { let conflicts_with_strs = a.conflicts_with.iter().map(|s| s.as_str()).collect::>(); @@ -646,12 +652,6 @@ impl AugmentClap for Keyring { } } -impl Keyring { - fn is_subcommand() -> bool { - false - } -} - /// Default to verbosity level 0, if none is provided. fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { let pos = s.find(' '); @@ -705,8 +705,6 @@ fn parse_cors(s: &str) -> Result> { Ok(if is_all { Cors::All } else { Cors::List(origins) }) } -impl_augment_clap!(RunCmd); - /// The `build-spec` command used to build a specification. #[derive(Debug, StructOpt, Clone)] pub struct BuildSpecCmd { @@ -895,7 +893,7 @@ pub enum CoreParams { impl StructOpt for CoreParams where CC: StructOpt + GetSharedParams, - RP: StructOpt + AugmentClap + RP: StructOpt + StructOptInternal, { fn clap<'a, 'b>() -> App<'a, 'b> { RP::augment_clap( @@ -964,7 +962,7 @@ impl StructOpt for NoCustom { } } -impl AugmentClap for NoCustom { +impl StructOptInternal for NoCustom { fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { app } @@ -985,7 +983,7 @@ pub struct MergeParameters { pub right: R, } -impl StructOpt for MergeParameters where L: StructOpt + AugmentClap, R: StructOpt { +impl StructOpt for MergeParameters where L: StructOpt + StructOptInternal, R: StructOpt { fn clap<'a, 'b>() -> App<'a, 'b> { L::augment_clap(R::clap()) } diff --git a/client/cli/src/traits.rs b/client/cli/src/traits.rs index dddba0b25ab2c..2f4007c846f90 100644 --- a/client/cli/src/traits.rs +++ b/client/cli/src/traits.rs @@ -14,30 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::{StructOpt, clap::App}; use crate::params::SharedParams; -/// Something that can augment a clap app with further parameters. -/// `derive(StructOpt)` is implementing this function by default, so a macro `impl_augment_clap!` -/// is provided to simplify the implementation of this trait. -pub trait AugmentClap: StructOpt { - /// Augment the given clap `App` with further parameters. - fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b>; -} - -/// Macro for implementing the `AugmentClap` trait. -/// This requires that the given type uses `derive(StructOpt)`! -#[macro_export] -macro_rules! impl_augment_clap { - ( $type:ident ) => { - impl $crate::AugmentClap for $type { - fn augment_clap<'a, 'b>(app: $crate::App<'a, 'b>) -> $crate::App<'a, 'b> { - $type::augment_clap(app) - } - } - } -} - /// Supports getting common params. pub trait GetSharedParams { /// Returns shared params if any. diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index ce27b0995c5ef..887e8415f39b7 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -54,6 +54,7 @@ parity-multiaddr = { package = "parity-multiaddr", version = "0.5.0" } grafana-data-source = { version = "2.0.0", path = "../../utils/grafana-data-source" } sc-tracing = { version = "2.0.0", path = "../tracing" } tracing = "0.1.10" +sc-prometheus = { package = "sc-prometheus", path="../../utils/prometheus"} [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 9b5afe957a115..a5f10ee992245 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -53,6 +53,7 @@ use std::{ use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_transaction_pool::{TransactionPool, TransactionPoolMaintainer}; +use sc_prometheus::prometheus_gauge; use sp_blockchain; use grafana_data_source::{self, record_metrics}; @@ -954,6 +955,17 @@ ServiceBuilder< "bandwidth_upload" => bandwidth_upload, "used_state_cache_size" => used_state_cache_size, ); + prometheus_gauge!( + MEMPOOL_SIZE => used_state_cache_size as u64, + NODE_MEMORY => memory as u64, + NODE_CPU => cpu_usage as u64, + TX_COUNT => txpool_status.ready as u64, + FINALITY_HEIGHT => finalized_number as u64, + BEST_HEIGHT => best_number as u64, + P2P_PEERS_NUM => num_peers as u64, + P2P_NODE_DOWNLOAD => net_status.average_download_per_sec as u64, + P2P_NODE_UPLOAD => net_status.average_upload_per_sec as u64 + ); let _ = record_metrics!( "peers" => num_peers, "height" => best_number, @@ -964,8 +976,7 @@ ServiceBuilder< "bandwidth_download" => bandwidth_download, "bandwidth_upload" => bandwidth_upload, "used_state_cache_size" => used_state_cache_size, - ); - + ); Ok(()) }).select(exit.clone().map(Ok).compat()).then(|_| Ok(())); let _ = to_spawn_tx.unbounded_send(Box::new(tel_task)); @@ -1103,7 +1114,13 @@ ServiceBuilder< .then(|_| Ok(())))); telemetry }); - + // prometheus init + match config.prometheus_endpoint { + None => (), + Some(x) => { + let _prometheus = sc_prometheus::init_prometheus(x); + } + } // Grafana data source if let Some(port) = config.grafana_port { let future = select( diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 0b5152e24828d..d2bcca7a22b25 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -75,6 +75,8 @@ pub struct Configuration { pub rpc_cors: Option>, /// Grafana data source http port. `None` if disabled. pub grafana_port: Option, + /// Promteheus IP addr. `None` if disabled. and defult port 33333 + pub prometheus_endpoint: Option, /// Telemetry service URL. `None` if disabled. pub telemetry_endpoints: Option, /// External WASM transport for the telemetry. If `Some`, when connection to a telemetry @@ -153,6 +155,7 @@ impl Configuration where rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: Some(vec![]), + prometheus_endpoint: None, grafana_port: None, telemetry_endpoints: None, telemetry_external_transport: None, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index d9dc9bd004d52..2f3fd5d9849fa 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -17,8 +17,6 @@ //! Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. //! Manages communication between them. -#![warn(missing_docs)] - pub mod config; #[macro_use] pub mod chain_ops; diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index f66e4a65da200..38a81bbe74b0b 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -192,6 +192,7 @@ fn node_config ( rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: None, + prometheus_endpoint: None, grafana_port: None, telemetry_endpoints: None, telemetry_external_transport: None, diff --git a/frame/indices/Cargo.toml b/frame/indices/Cargo.toml index 574c2d3eb6f3d..d05a7654d1240 100644 --- a/frame/indices/Cargo.toml +++ b/frame/indices/Cargo.toml @@ -16,9 +16,6 @@ sp-core = { version = "2.0.0", default-features = false, path = "../../primitive frame-support = { version = "2.0.0", default-features = false, path = "../support" } frame-system = { version = "2.0.0", default-features = false, path = "../system" } -[dev-dependencies] -ref_thread_local = "0.0.0" - [features] default = ["std"] std = [ diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index 2a1cb0746f816..150664126d010 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -18,8 +18,7 @@ #![cfg(test)] -use std::collections::HashSet; -use ref_thread_local::{ref_thread_local, RefThreadLocal}; +use std::{cell::RefCell, collections::HashSet}; use sp_runtime::testing::Header; use sp_runtime::Perbill; use sp_core::H256; @@ -30,23 +29,23 @@ impl_outer_origin!{ pub enum Origin for Runtime where system = frame_system {} } -ref_thread_local! { - static managed ALIVE: HashSet = HashSet::new(); +thread_local! { + static ALIVE: RefCell> = Default::default(); } pub fn make_account(who: u64) { - ALIVE.borrow_mut().insert(who); + ALIVE.with(|a| a.borrow_mut().insert(who)); Indices::on_new_account(&who); } pub fn kill_account(who: u64) { - ALIVE.borrow_mut().remove(&who); + ALIVE.with(|a| a.borrow_mut().remove(&who)); } pub struct TestIsDeadAccount {} impl IsDeadAccount for TestIsDeadAccount { fn is_dead_account(who: &u64) -> bool { - !ALIVE.borrow_mut().contains(who) + !ALIVE.with(|a| a.borrow_mut().contains(who)) } } @@ -70,6 +69,7 @@ parameter_types! { pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); } + impl frame_system::Trait for Runtime { type Origin = Origin; type Index = u64; @@ -88,6 +88,7 @@ impl frame_system::Trait for Runtime { type Version = (); type ModuleToIndex = (); } + impl Trait for Runtime { type AccountIndex = u64; type IsDeadAccount = TestIsDeadAccount; @@ -97,9 +98,11 @@ impl Trait for Runtime { pub fn new_test_ext() -> sp_io::TestExternalities { { - let mut h = ALIVE.borrow_mut(); - h.clear(); - for i in 1..5 { h.insert(i); } + ALIVE.with(|a| { + let mut h = a.borrow_mut(); + h.clear(); + for i in 1..5 { h.insert(i); } + }); } let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml new file mode 100644 index 0000000000000..8ac6d7001833f --- /dev/null +++ b/utils/prometheus/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sc-prometheus" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "prometheus utils" +edition = "2018" + +[dependencies] +hyper = "0.12" +lazy_static = "1.0" +log = "0.4" +prometheus = { version = "0.7", features = ["nightly", "process"]} +tokio = "0.1" +sp-runtime = { package = "sp-runtime",path = "../../primitives/runtime" } + +[dev-dependencies] +reqwest = "0.9" \ No newline at end of file diff --git a/utils/prometheus/README.md b/utils/prometheus/README.md new file mode 100644 index 0000000000000..3efe80d57de94 --- /dev/null +++ b/utils/prometheus/README.md @@ -0,0 +1,501 @@ +# Substrate Prometheus Node Exporter +![grants](./photo_2019-12-13_16-32-53.jpg) +## Introduction + +Prometheus is one of the most widely used monitoring tool for managing high availability services supported by [Cloud Native Computing Foundation](https://www.cncf.io/). By providing Prometheus exporter in substrate, node operators can easily adopt widely used display/alert tool such as Grafana without seting-up/operating external Prometheus agent through RPC connections. Easy access to such monitoring tools will benefit parachain develepers/operators and validators to have much higher availability quality of their services. + +## List of Contents + +Hack Prometheus in Substrate + - Prometheus starter + - CLI Config + - Metrics Add + +Metrics + - List of available metrics + +Start Prometheus + - Install prometheus + - Edit Prometheus config file + - Start Prometheus + +Start Grafana + - Install Grafana + +## Substrate Dev hack +### Prometheus starter + +Here is the entry point of prometheus core module in Parity Substrate. + +utils/prometheus/src/lib.rs +```rust +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; +use hyper::http::StatusCode; +use hyper::rt::Future; +use hyper::service::service_fn_ok; +use hyper::{Body, Request, Response, Server}; +pub use prometheus::{Encoder, HistogramOpts, Opts, TextEncoder}; +pub use prometheus::{Histogram, IntCounter, IntGauge, Result}; +pub use sp_runtime::traits::SaturatedConversion; +use std::net::SocketAddr; + +pub mod metrics; + +/// Initializes the metrics context, and starts an HTTP server +/// to serve metrics. +pub fn init_prometheus(prometheus_addr: SocketAddr) { + let addr = prometheus_addr; + let server = Server::bind(&addr) + .serve(|| { + // This is the `Service` that will handle the connection. + // `service_fn_ok` is a helper to convert a function that + // returns a Response into a `Service`. + service_fn_ok(move |req: Request| { + if req.uri().path() == "/metrics" { + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + Response::builder() + .status(StatusCode::OK) + .header("Content-Type", encoder.format_type()) + .body(Body::from(buffer)) + .expect("Sends OK(200) response with one or more data metrics") + } else { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found.")) + .expect("Sends NOT_FOUND(404) message with no data metric") + } + }) + }) + .map_err(|e| error!("server error: {}", e)); + + info!("Exporting metrics at http://{}/metrics", addr); + + let mut rt = tokio::runtime::Builder::new() + .core_threads(1) // one thread is sufficient + .build() + .expect("Builds one thread of tokio runtime exporter for prometheus"); + + std::thread::spawn(move || { + rt.spawn(server); + rt.shutdown_on_idle().wait().unwrap(); + }); +} + +#[macro_export] +macro_rules! prometheus_gauge( + ($($metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set_gauge(&$metric, $value); + )* + } +); + +#[macro_export] +macro_rules! prometheus_histogram( + ($($metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set_histogram(&$metric, $value); + )* + } +); + +/* +TODO: Make abstract type for all metrics(e.g. Gauge, Histogram, Counter) with generic traits so that all metrics can be set up with one function `set` +#[macro_export] +macro_rules! prometheus( + ($($a: expr; $metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set(#$a, &$metric, $value); + )* + } +); +*/ +``` + +Here is the dependancies of the module. +client/prometheus/Cargo.toml +```toml +[package] +name = "sc-prometheus" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "prometheus utils" +edition = "2018" + +[dependencies] +hyper = "0.12" +lazy_static = "1.0" +log = "0.4" +prometheus = { version = "0.7", features = ["nightly", "process"]} +tokio = "0.1" +sp-runtime = { package = "sp-runtime",path = "../../primitives/runtime" } + +[dev-dependencies] +reqwest = "0.9" +``` + +**Abbreviation of the package in service manager of parity substrate** +client/service/Cargo.toml +```toml +[dependencies] +sc-prometheus = { package = "sc-prometheus", path="../../utils/prometheus"} +``` + +**Metrics builder as same as substrate-telemetry** +client/service/src/builder.rsL1271 , L1112 +```rust +use sc_prometheus::prometheus_gauge; +... + telemetry!( + SUBSTRATE_INFO; + "system.interval"; + "peers" => num_peers, + "height" => best_number, + "best" => ?best_hash, + "txcount" => txpool_status.ready, + "cpu" => cpu_usage, + "memory" => memory, + "finalized_height" => finalized_number, + "finalized_hash" => ?info.chain.finalized_hash, + "bandwidth_download" => bandwidth_download, + "bandwidth_upload" => bandwidth_upload, + "used_state_cache_size" => used_state_cache_size, + ); + prometheus_gauge!( + MEMPOOL_SIZE => used_state_cache_size as u64, + NODE_MEMORY => memory as u64, + NODE_CPU => cpu_usage as u64, + TX_COUNT => txpool_status.ready as u64, + FINALITY_HEIGHT => finalized_number as u64, + BEST_HEIGHT => best_number as u64, + P2P_PEERS_NUM => num_peers as u64, + P2P_NODE_DOWNLOAD => net_status.average_download_per_sec as u64, + P2P_NODE_UPLOAD => net_status.average_upload_per_sec as u64 + ); + let _ = record_metrics!( + "peers" => num_peers, + "height" => best_number, + "txcount" => txpool_status.ready, + "cpu" => cpu_usage, + "memory" => memory, + "finalized_height" => finalized_number, + "bandwidth_download" => bandwidth_download, + "bandwidth_upload" => bandwidth_upload, + "used_state_cache_size" => used_state_cache_size, + ); + Ok(()) + }).select(exit.clone().map(Ok).compat()).then(|_| Ok(())); + let _ = to_spawn_tx.unbounded_send(Box::new(tel_task)); + +... + // prometheus init + match config.prometheus_endpoint { + None => (), + Some(x) => { + let _prometheus = sc_prometheus::init_prometheus(x); + } + } + // Grafana data source + if let Some(port) = config.grafana_port { + let future = select( + grafana_data_source::run_server(port).boxed(), + exit.clone() + ).map(|either| match either { + Either::Left((result, _)) => result.map_err(|_| ()), + Either::Right(_) => Ok(()) + }).compat(); + + let _ = to_spawn_tx.unbounded_send(Box::new(future)); + } + + // Instrumentation + if let Some(tracing_targets) = config.tracing_targets.as_ref() { + let subscriber = sc_tracing::ProfilingSubscriber::new( + config.tracing_receiver, tracing_targets + ); + match tracing::subscriber::set_global_default(subscriber) { + Ok(_) => (), + Err(e) => error!(target: "tracing", "Unable to set global default subscriber {}", e), + } + } + + +``` +substrate/Cargo.toml +```toml +[workspace] +members = [ + "utils/prometheus", +``` +### CLI Config +client/cli/src/lib.rs +```rust +fn crate_run_node_config{ +... + // Override telemetry + if cli.no_telemetry { + config.telemetry_endpoints = None; + } else if !cli.telemetry_endpoints.is_empty() { + config.telemetry_endpoints = Some(TelemetryEndpoints::new(cli.telemetry_endpoints)); + } + + config.tracing_targets = cli.tracing_targets.into(); + config.tracing_receiver = cli.tracing_receiver.into(); + + // Override prometheus + match cli.prometheus_endpoint { + None => {config.prometheus_endpoint = None;}, + Some(x) => { + config.prometheus_endpoint = Some(parse_address(&format!("{}:{}", x, 33333), cli.prometheus_port)?); + } + } + // Imply forced authoring on --dev + config.force_authoring = cli.shared_params.dev || cli.force_authoring; + + Ok(config) +... +} +``` + +client/cli/src/params.rs +```rust +pub struct RunCmd{ +... +/// Prometheus exporter TCP port. + #[structopt(long = "prometheus-port", value_name = "PORT")] + pub prometheus_port: Option, + /// Prometheus exporter IP addr. + #[structopt(long = "prometheus-addr", value_name = "Local IP address")] + pub prometheus_endpoint: Option, +... +} +``` +client/service/src/config.rs +```rust +#[derive(Clone)] +pub struct Configuration { + ... + /// Promteheus IP addr. `None` if disabled. and defult port 33333 + pub prometheus_endpoint: Option, + ... +} +impl Configuration where + C: Default, + G: RuntimeGenesis, + E: Extension, +{ + /// Create default config for given chain spec. + pub fn default_with_spec(chain_spec: ChainSpec) -> Self { + let mut configuration = Configuration { + ... + prometheus_endpoints: None, + ... + }; + configuration.network.boot_nodes = configuration.chain_spec.boot_nodes().to_vec(); + + configuration.telemetry_endpoints = configuration.chain_spec.telemetry_endpoints().clone(); + + configuration + } +``` + + + +### Metrics Add +ex) consensus_FINALITY_HEIGHT + +utils/prometheus/src/metrics.rs + +```rust +pub use crate::*; + +/// Gauge type metrics generation function +pub fn try_create_int_gauge(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let gauge = IntGauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} + +///Gauge Metrics a value in injection. +pub fn set_gauge(gauge: &Result, value: u64) { + if let Ok(gauge) = gauge { + gauge.set(value as i64); + } +} + +///All of the metrics in the prometheus are managed by the lazy_static. +lazy_static! { + pub static ref FINALITY_HEIGHT: Result = try_create_int_gauge( + "consensus_finality_block_height_number", + "block is finality HEIGHT" + + ); +} +``` +client/service/Cargo.toml +```rust +... +sc-prometheus = { package = "sc-prometheus", path="../../utils/prometheus"} +... +``` +client/service/src/builder.rs +```rust +..... +use sc-prometheus::{prometheus_gauge}; +..... + let tel_task = state_rx.for_each(move |(net_status, _)| { + let info = client_.info(); + let best_number = info.chain.best_number.saturated_into::(); + let best_hash = info.chain.best_hash; + let num_peers = net_status.num_connected_peers; + let txpool_status = transaction_pool_.status(); + let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); + let bandwidth_download = net_status.average_download_per_sec; + let bandwidth_upload = net_status.average_upload_per_sec; + + let used_state_cache_size = match info.used_state_cache_size { + Some(size) => size, + None => 0, + }; + + // get cpu usage and memory usage of this process + let (cpu_usage, memory) = if let Some(self_pid) = self_pid { + if sys.refresh_process(self_pid) { + let proc = sys.get_process(self_pid) + .expect("Above refresh_process succeeds, this should be Some(), qed"); + (proc.cpu_usage(), proc.memory()) + } else { (0.0, 0) } + } else { (0.0, 0) }; + + telemetry!( + SUBSTRATE_INFO; + "system.interval"; + "peers" => num_peers, + "height" => best_number, + "best" => ?best_hash, + "txcount" => txpool_status.ready, + "cpu" => cpu_usage, + "memory" => memory, + "finalized_height" => finalized_number, + "finalized_hash" => ?info.chain.finalized_hash, + "bandwidth_download" => bandwidth_download, + "bandwidth_upload" => bandwidth_upload, + "used_state_cache_size" => used_state_cache_size, + ); + + prometheus_gauge!( + MEMPOOL_SIZE => used_state_cache_size as u64, + NODE_MEMORY => memory as u64, + NODE_CPU => cpu_usage as u64, + TX_COUNT => txpool_status.ready as u64, + FINALITY_HEIGHT => finalized_number as u64, + BEST_HEIGHT => best_number as u64, + P2P_PEERS_NUM => num_peers as u64, + P2P_NODE_DOWNLOAD => net_status.average_download_per_sec as u64, + P2P_NODE_UPLOAD => net_status.average_upload_per_sec as u64 + ); +..... +``` +## Metrics + +substrate can report and serve the Prometheus metrics, which in their turn can be consumed by Prometheus collector(s). + +This functionality is disabled by default. + +To enable the Prometheus metrics, set in your cli command (--prometheus-addr,--prometheus-port ). +Metrics will be served under /metrics on 33333 port by default. + +### List of available metrics + + +Consensus metrics, namespace: `substrate` + +| **Name** | **Type** | **Tags** | **Description** | +| -------------------------------------- | --------- | -------- | --------------------------------------------------------------- | +| consensus_finality_block_height_number | IntGauge | | finality Height of the chain | +| consensus_best_block_height_number | IntGauge | | best Height of the chain | +| consensus_target_syn_number | IntGauge | | syning Height target number | +| consensus_num_txs | Gauge | | Number of transactions | +| consensus_node_memory | IntGauge | | Node's primary memory | +| consensus_node_cpu | IntGauge | | Node's cpu load | +| p2p_peers_number | IntGauge | | Number of peers node's connected to | +| p2p_peer_receive_bytes_per_sec | IntGauge | | number of bytes received from a given peer | +| p2p_peer_send_bytes_per_sec | IntGauge | | number of bytes sent to a given peer | +| mempool_size | IntGauge | | Number of uncommitted transactions | +| Resource_receive_bytes_per_sec(Future) | IntGauge | | Operating System of bytes received | +| Resource_send_bytes_per_sec(Future) | IntGauge | | Operating System of bytes sent | +| Resource_cpu_use(Future) | IntGauge | | Operating System cpu load | +| Resource_disk_use(Future) | IntGauge | | Operating System disk use | +| validator_sign_prevote(Future) | IntGauge | validator addr | validator sign vote list | +| validator_sign_precommit(Future) | IntGauge | validator addr | validator sign commit list | + + +## Start Prometheus +### Install prometheus + +https://prometheus.io/download/ +```bash +wget +tar -zxvf +``` + +### Edit Prometheus config file + +You can visit [prometheus.yml](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus.yml) to download default `prometheus.yml`. + +Then edit `prometheus.yml` and add `jobs` : + +```yaml + - job_name: kusama + static_configs: + - targets: ['localhost:33333'] + labels: + instance: local-validator +``` + +> Note：value of targets is ip:port which used by substrate monitor + +### Start Prometheus + +```bash +cd +./prometheus +``` + +> The above example, you can save `prometheus.yml` at `~/volumes/prometheus` on your host machine + +You can visit `http://localhost:9090` to see prometheus data. + + + +## Start Grafana +### Install Grafana +https://grafana.com/docs/installation/debian/ + +```bash +apt-get install -y software-properties-common +sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main" +wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add - +sudo apt-get update +sudo apt-get install grafana +sudo service grafana-server start +./prometheus +``` + +You can visit `http://localhost:3000/` to open grafana and create your own dashboard. + +> Tips: The default username and password are both admin. We strongly recommend immediately changing your username & password after login + +### Seting Grafana + +Default ID:PW is admin. diff --git a/utils/prometheus/photo_2019-12-13_16-32-53.jpg b/utils/prometheus/photo_2019-12-13_16-32-53.jpg new file mode 100644 index 0000000000000..cdf44f88fb623 Binary files /dev/null and b/utils/prometheus/photo_2019-12-13_16-32-53.jpg differ diff --git a/utils/prometheus/src/lib.rs b/utils/prometheus/src/lib.rs new file mode 100644 index 0000000000000..4713f47291de0 --- /dev/null +++ b/utils/prometheus/src/lib.rs @@ -0,0 +1,90 @@ +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; +use hyper::http::StatusCode; +use hyper::rt::Future; +use hyper::service::service_fn_ok; +use hyper::{Body, Request, Response, Server}; +pub use prometheus::{Encoder, HistogramOpts, Opts, TextEncoder}; +pub use prometheus::{Histogram, IntCounter, IntGauge, Result}; +pub use sp_runtime::traits::SaturatedConversion; +use std::net::SocketAddr; + +pub mod metrics; + +/// Initializes the metrics context, and starts an HTTP server +/// to serve metrics. +pub fn init_prometheus(prometheus_addr: SocketAddr) { + let addr = prometheus_addr; + let server = Server::bind(&addr) + .serve(|| { + // This is the `Service` that will handle the connection. + // `service_fn_ok` is a helper to convert a function that + // returns a Response into a `Service`. + service_fn_ok(move |req: Request| { + if req.uri().path() == "/metrics" { + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + Response::builder() + .status(StatusCode::OK) + .header("Content-Type", encoder.format_type()) + .body(Body::from(buffer)) + .expect("Sends OK(200) response with one or more data metrics") + } else { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found.")) + .expect("Sends NOT_FOUND(404) message with no data metric") + } + }) + }) + .map_err(|e| error!("server error: {}", e)); + + info!("Exporting metrics at http://{}/metrics", addr); + + let mut rt = tokio::runtime::Builder::new() + .core_threads(1) // one thread is sufficient + .build() + .expect("Builds one thread of tokio runtime exporter for prometheus"); + + std::thread::spawn(move || { + rt.spawn(server); + rt.shutdown_on_idle().wait().unwrap(); + }); +} + +#[macro_export] +macro_rules! prometheus_gauge( + ($($metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set_gauge(&$metric, $value); + )* + } +); + +#[macro_export] +macro_rules! prometheus_histogram( + ($($metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set_histogram(&$metric, $value); + )* + } +); + +/* +TODO: Make abstract type for all metrics(e.g. Gauge, Histogram, Counter) with generic traits so that all metrics can be set up with one function `set` +#[macro_export] +macro_rules! prometheus( + ($($a: expr; $metric:expr => $value:expr),*) => { + use $crate::{metrics::*}; + $( + metrics::set(#$a, &$metric, $value); + )* + } +); +*/ \ No newline at end of file diff --git a/utils/prometheus/src/metrics.rs b/utils/prometheus/src/metrics.rs new file mode 100644 index 0000000000000..dfb9c4ec57720 --- /dev/null +++ b/utils/prometheus/src/metrics.rs @@ -0,0 +1,79 @@ +pub use crate::*; + +/// Gauge type metrics generation function +pub fn try_create_int_gauge(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let gauge = IntGauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} +/// histogram type metrics generation function +pub fn try_create_histogram(name: &str, help: &str) -> Result { + let opts = HistogramOpts::new(name, help); + let histogram = Histogram::with_opts(opts)?; + prometheus::register(Box::new(histogram.clone()))?; + Ok(histogram) +} + +///Gauge Metrics a value in injection. +pub fn set_gauge(gauge: &Result, value: u64) { + if let Ok(gauge) = gauge { + gauge.set(value as i64); + } +} +///histogram Metrics a value in injection. +pub fn set_histogram(histogram: &Result, value: f64) { + if let Ok(histogram) = histogram { + histogram.observe(value) + } +} +//All of the metrics in the prometheus are managed by the lazy_static. + +lazy_static! { + pub static ref FINALITY_HEIGHT: Result = try_create_int_gauge( + "consensus_finality_block_height_number", + "block is finality HEIGHT" + + ); + pub static ref BEST_HEIGHT: Result = try_create_int_gauge( + "consensus_best_block_height_number", + "block is best HEIGHT" + ); + + pub static ref BLOCK_INTERVAL_SECONDS: Result = try_create_histogram( + "consensus_block_interval_seconds", + "Time between this and last block(Block.Header.Time) in seconds" + ); + pub static ref P2P_PEERS_NUM: Result = try_create_int_gauge( + "p2p_peers_number", + "network gosip peers number" + ); + pub static ref TARGET_NUM: Result = try_create_int_gauge( + "consensus_target_syn_number", + "block syn target number" + ); + pub static ref TX_COUNT: Result = try_create_int_gauge( + "consensus_num_txs", + "Number of transactions" + ); + pub static ref NODE_MEMORY: Result = try_create_int_gauge( + "consensus_node_memory", + "node memory" + ); + pub static ref NODE_CPU: Result = try_create_int_gauge( + "consensus_node_cpu", + "node cpu" + ); + pub static ref MEMPOOL_SIZE: Result = try_create_int_gauge( + "mempool_size", + "Number of uncommitted transactions" + ); + pub static ref P2P_NODE_DOWNLOAD: Result = try_create_int_gauge( + "p2p_peers_receive_byte_per_sec", + "p2p_node_download_per_sec_byte" + ); + pub static ref P2P_NODE_UPLOAD: Result = try_create_int_gauge( + "p2p_peers_send_byte_per_sec", + "p2p_node_upload_per_sec_byte" + ); +} \ No newline at end of file diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs index bb32d62218742..66da930f3a16e 100644 --- a/utils/wasm-builder/src/wasm_project.rs +++ b/utils/wasm-builder/src/wasm_project.rs @@ -172,18 +172,69 @@ fn get_wasm_workspace_root() -> PathBuf { panic!("Could not find target dir in: {}", build_helper::out_dir().display()) } -fn create_wasm_workspace_project(wasm_workspace: &Path, cargo_manifest: &Path) { - let members = WalkDir::new(wasm_workspace) +/// Find all workspace members. +/// +/// Each folder in `wasm_workspace` is seen as a member of the workspace. Exceptions are +/// folders starting with "." and the "target" folder. +/// +/// Every workspace member that is not valid anymore is deleted (the folder of it). A +/// member is not valid anymore when the `wasm-project` dependency points to an non-existing +/// folder or the package name is not valid. +fn find_and_clear_workspace_members(wasm_workspace: &Path) -> Vec { + let mut members = WalkDir::new(wasm_workspace) .min_depth(1) .max_depth(1) .into_iter() .filter_map(|p| p.ok()) .map(|d| d.into_path()) - .filter(|p| p.is_dir() && !p.ends_with("target")) + .filter(|p| p.is_dir()) .filter_map(|p| p.file_name().map(|f| f.to_owned()).and_then(|s| s.into_string().ok())) - .filter(|f| !f.starts_with(".")) + .filter(|f| !f.starts_with(".") && f != "target") .collect::>(); + let mut i = 0; + while i != members.len() { + let path = wasm_workspace.join(&members[i]).join("Cargo.toml"); + + // Extract the `wasm-project` dependency. + // If the path can be extracted and is valid and the package name matches, + // the member is valid. + if let Some(mut wasm_project) = fs::read_to_string(path) + .ok() + .and_then(|s| toml::from_str::(&s).ok()) + .and_then(|mut t| t.remove("dependencies")) + .and_then(|p| p.try_into::
().ok()) + .and_then(|mut t| t.remove("wasm_project")) + .and_then(|p| p.try_into::
().ok()) + { + if let Some(path) = wasm_project.remove("path") + .and_then(|p| p.try_into::().ok()) + { + if let Some(name) = wasm_project.remove("package") + .and_then(|p| p.try_into::().ok()) + { + let path = PathBuf::from(path); + if path.exists() { + if name == get_crate_name(&path.join("Cargo.toml")) { + i += 1; + continue + } + } + } + } + } + + fs::remove_dir_all(wasm_workspace.join(&members[i])) + .expect("Removing invalid workspace member can not fail; qed"); + members.remove(i); + } + + members +} + +fn create_wasm_workspace_project(wasm_workspace: &Path, cargo_manifest: &Path) { + let members = find_and_clear_workspace_members(wasm_workspace); + let crate_metadata = MetadataCommand::new() .manifest_path(cargo_manifest) .exec()