Merge pull request #2 from paritytech/ashley-prometheus

Prometheus: Remove Grafana data source and some other changes

Cargo.toml

@@ -50,8 +50,6 @@ members = [
 	"client/transaction-pool/graph",
 	"utils/prometheus",
 	"utils/wasm-builder-runner",
-	"utils/grafana-data-source",
-	"utils/grafana-data-source/test",
 	"frame/assets",
 	"frame/aura",
 	"frame/authority-discovery",

client/cli/Cargo.toml

@@ -29,7 +29,7 @@ sp-core = { version = "2.0.0", path = "../../primitives/core" }
 sc-service = { version = "2.0.0", default-features = false, path = "../service" }
 sp-state-machine = { version = "2.0.0", path = "../../primitives/state-machine" }
 sc-telemetry = { version = "2.0.0", path = "../telemetry" }
-sc-prometheus = { path = "../../utils/prometheus" }
+prometheus-endpoint = { path = "../../utils/prometheus" }
 sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" }
 names = "0.11.0"
 structopt = "=0.3.7"

client/cli/src/informant/display.rs

@@ -18,10 +18,20 @@ use ansi_term::Colour;
 use sc_client_api::ClientInfo;
 use log::info;
 use sc_network::SyncState;
-use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating};
+use sp_runtime::traits::{
+	Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating, UniqueSaturatedInto
+};
 use sc_service::NetworkStatus;
 use std::{convert::{TryFrom, TryInto}, fmt, time};
-use sc_prometheus::prometheus_gauge;
+use prometheus_endpoint::{create_gauge, Gauge, U64};
+
+prometheus_endpoint::lazy_static! {
+	pub static ref SYNC_TARGET: Gauge<U64> = create_gauge(
+		"sync_target_number",
+		"block sync target number"
+	);
+}
+
 /// State of the informant display system.
 ///
 /// This is the system that handles the line that gets regularly printed and that looks something
@@ -64,7 +74,7 @@ impl<B: BlockT> InformantDisplay<B> {
 			(SyncState::Idle, _) => ("Idle".into(), "".into()),
 			(SyncState::Downloading, None) => (format!("Syncing{}", speed), "".into()),
 			(SyncState::Downloading, Some(n)) => {
-				prometheus_gauge!(TARGET_NUM => n.saturated_into().try_into().unwrap());
+				SYNC_TARGET.set(n.unique_saturated_into() as u64);
 				(format!("Syncing{}", speed), format!(", target=#{}", n))
 			}
 		};

client/cli/src/lib.rs

@@ -909,13 +909,12 @@ where
 
 	let rpc_interface: &str = interface_str(cli.rpc_external, cli.unsafe_rpc_external, cli.validator)?;
 	let ws_interface: &str = interface_str(cli.ws_external, cli.unsafe_ws_external, cli.validator)?;
-	let grafana_interface: &str = if cli.grafana_external { "0.0.0.0" } else { "127.0.0.1" };
 	let prometheus_interface: &str = if cli.prometheus_external { "0.0.0.0" } else { "127.0.0.1" };
 
 	config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), cli.rpc_port)?);
 	config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), cli.ws_port)?);
-	config.grafana_port = Some(
-		parse_address(&format!("{}:{}", grafana_interface, 9955), cli.grafana_port)?
+	config.prometheus_port = Some(
+		parse_address(&format!("{}:{}", prometheus_interface, 9955), cli.prometheus_port)?
 	);
 
 	config.rpc_ws_max_connections = cli.ws_max_connections;
@@ -942,12 +941,7 @@ where
 
 	config.tracing_targets = cli.tracing_targets.into();
 	config.tracing_receiver = cli.tracing_receiver.into();
-
-	// Override prometheus
-	if cli.prometheus_external {
-			config.prometheus_port = Some(
-		parse_address(&format!("{}:{}", prometheus_interface, 33333), cli.prometheus_port)?
-	)}
+
 	// Imply forced authoring on --dev
 	config.force_authoring = cli.shared_params.dev || cli.force_authoring;
 

client/cli/src/params.rs

@@ -298,7 +298,7 @@ arg_enum! {
 	pub enum TracingReceiver {
 		Log,
 		Telemetry,
-		Grafana,
+		Prometheus,
 	}
 }
 
@@ -307,7 +307,7 @@ impl Into<sc_tracing::TracingReceiver> for TracingReceiver {
 		match self {
 			TracingReceiver::Log => sc_tracing::TracingReceiver::Log,
 			TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry,
-			TracingReceiver::Grafana => sc_tracing::TracingReceiver::Grafana,
+			TracingReceiver::Prometheus => sc_tracing::TracingReceiver::Prometheus,
 		}
 	}
 }
@@ -440,24 +440,18 @@ pub struct RunCmd {
 	/// Use `--unsafe-ws-external` to suppress the warning if you understand the risks.
 	#[structopt(long = "ws-external")]
 	pub ws_external: bool,
-	/// Prometheus exporter TCP port.
-	#[structopt(long = "prometheus-port", value_name = "PORT")]
-	pub prometheus_port: Option<u16>,
-	/// Prometheus exporter on/off external".
-	#[structopt(long = "prometheus-external")]
-	pub prometheus_external: bool,
 
 	/// Listen to all Websocket interfaces.
 	///
 	/// Same as `--ws-external`.
 	#[structopt(long = "unsafe-ws-external")]
 	pub unsafe_ws_external: bool,
 
-	/// Listen to all Grafana data source interfaces.
+	/// Listen to all Prometheus endpoint interfaces.
 	///
 	/// Default is local.
-	#[structopt(long = "grafana-external")]
-	pub grafana_external: bool,
+	#[structopt(long = "prometheus-external")]
+	pub prometheus_external: bool,
 
 	/// Specify HTTP RPC server TCP port.
 	#[structopt(long = "rpc-port", value_name = "PORT")]
@@ -481,9 +475,9 @@ pub struct RunCmd {
 	#[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))]
 	pub rpc_cors: Option<Cors>,
 
-	/// Specify Grafana data source server TCP Port.
-	#[structopt(long = "grafana-port", value_name = "PORT")]
-	pub grafana_port: Option<u16>,
+	/// Specify Prometheus endpoint TCP Port.
+	#[structopt(long = "prometheus-port", value_name = "PORT")]
+	pub prometheus_port: Option<u16>,
 
 	/// The human-readable name for this node.
 	///

client/service/Cargo.toml

@@ -51,10 +51,9 @@ sc-rpc = { version = "2.0.0", path = "../rpc" }
 sc-telemetry = { version = "2.0.0", path = "../telemetry" }
 sc-offchain = { version = "2.0.0", path = "../offchain" }
 parity-multiaddr = { package = "parity-multiaddr", version = "0.5.0" }
-grafana-data-source = { version = "2.0.0", path = "../../utils/grafana-data-source" }
+prometheus-endpoint = { path = "../../utils/prometheus" }
 sc-tracing = { version = "2.0.0", path = "../tracing" }
 tracing = "0.1.10"
-sc-prometheus = { package = "sc-prometheus", path="../../utils/prometheus"}
 
 [dev-dependencies]
 substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }

client/service/src/builder.rs

@@ -53,11 +53,43 @@ use std::{
 use sysinfo::{get_current_pid, ProcessExt, System, SystemExt};
 use sc_telemetry::{telemetry, SUBSTRATE_INFO};
 use sp_transaction_pool::{TransactionPool, TransactionPoolMaintainer};
-use sc_prometheus::prometheus_gauge;
-
 use sp_blockchain;
-use grafana_data_source::{self, record_metrics};
-
+use prometheus_endpoint::{create_gauge, Gauge, U64, F64};
+
+prometheus_endpoint::lazy_static! {
+	pub static ref FINALITY_HEIGHT: Gauge<U64> = create_gauge(
+		"consensus_finality_block_height_number",
+		"block is finality HEIGHT"
+	);
+	pub static ref BEST_HEIGHT: Gauge<U64> = create_gauge(
+		"consensus_best_block_height_number",
+		"block is best HEIGHT"
+	);
+	pub static ref P2P_PEERS_NUM: Gauge<U64> = create_gauge(
+		"p2p_peers_number",
+		"network gosip peers number"
+	);
+	pub static ref TX_COUNT: Gauge<U64> = create_gauge(
+		"consensus_num_txs",
+		"Number of transactions"
+	);
+	pub static ref NODE_MEMORY: Gauge<U64> = create_gauge(
+		"consensus_node_memory",
+		"node memory"
+	);
+	pub static ref NODE_CPU: Gauge<F64> = create_gauge(
+		"consensus_node_cpu",
+		"node cpu"
+	);
+	pub static ref P2P_NODE_DOWNLOAD: Gauge<U64> = create_gauge(
+		"p2p_peers_receive_byte_per_sec",
+		"p2p_node_download_per_sec_byte"
+	);
+	pub static ref P2P_NODE_UPLOAD: Gauge<U64> = create_gauge(
+		"p2p_peers_send_byte_per_sec",
+		"p2p_node_upload_per_sec_byte"
+	);
+}
 /// Aggregator for the components required to build a service.
 ///
 /// # Usage
@@ -956,28 +988,14 @@ ServiceBuilder<
 				"bandwidth_upload" => bandwidth_upload,
 				"used_state_cache_size" => used_state_cache_size,
 			);
-			prometheus_gauge!(
-				STATE_CACHE_SIZE => used_state_cache_size as u64,
-				NODE_MEMORY => memory as u64,
-				NODE_CPU => cpu_usage as u64,
-				TX_COUNT => txpool_status.ready as u64,
-				FINALITY_HEIGHT => finalized_number as u64,
-				BEST_HEIGHT => best_number as u64,
-				P2P_PEERS_NUM => num_peers as u64,
-				P2P_NODE_DOWNLOAD => net_status.average_download_per_sec as u64,
-				P2P_NODE_UPLOAD => net_status.average_upload_per_sec as u64
-			  );
-			let _ = record_metrics!(
-				"peers" => num_peers,
-				"height" => best_number,
-				"txcount" => txpool_status.ready,
-				"cpu" => cpu_usage,
-				"memory" => memory,
-				"finalized_height" => finalized_number,
-				"bandwidth_download" => bandwidth_download,
-				"bandwidth_upload" => bandwidth_upload,
-				"used_state_cache_size" => used_state_cache_size,
-			  );
+			NODE_MEMORY.set(memory);
+			NODE_CPU.set(f64::from(cpu_usage));
+			TX_COUNT.set(txpool_status.ready as u64);
+			FINALITY_HEIGHT.set(finalized_number);
+			BEST_HEIGHT.set(best_number);
+			P2P_PEERS_NUM.set(num_peers as u64);
+			P2P_NODE_DOWNLOAD.set(net_status.average_download_per_sec);
+			P2P_NODE_UPLOAD.set(net_status.average_upload_per_sec);
 			Ok(())
 		}).select(exit.clone().map(Ok).compat()).then(|_| Ok(()));
 		let _ = to_spawn_tx.unbounded_send(Box::new(tel_task));
@@ -1115,29 +1133,18 @@ ServiceBuilder<
 				.then(|_| Ok(()))));
 			telemetry
 		});
-		// prometheus init
+		// Prometheus endpoint
 		if let Some(port) = config.prometheus_port {
 			let future = select(
-					sc_prometheus::init_prometheus(port).boxed()
-					,exit.clone()
-				).map(|either| match either {
-					Either::Left((result, _)) => result.map_err(|_| ()),
-					Either::Right(_) => Ok(())
-				}).compat();
-				let _ = to_spawn_tx.unbounded_send(Box::new(future));
-		}
-		// Grafana data source
-		if let Some(port) = config.grafana_port {
-			let future = select(
-				grafana_data_source::run_server(port).boxed(),
+				prometheus_endpoint::init_prometheus(port).boxed(),
 				exit.clone()
 			).map(|either| match either {
 				Either::Left((result, _)) => result.map_err(|_| ()),
 				Either::Right(_) => Ok(())
 			}).compat();
 
 			let _ = to_spawn_tx.unbounded_send(Box::new(future));
-    }
+		}
 
 		// Instrumentation
 		if let Some(tracing_targets) = config.tracing_targets.as_ref() {