diff --git a/CLI.md b/CLI.md index 0c1ddbd5..d06ac74c 100644 --- a/CLI.md +++ b/CLI.md @@ -66,6 +66,18 @@ This document contains the help content for the `unleash-edge` command-line prog * `--token-header ` — token header to use for edge authorization Default value: `Authorization` +* `--disable-metrics-batch-endpoint` — Disables /internal-backstage/metricsbatch endpoint + + This endpoint shows the current cached client metrics +* `--disable-metrics-endpoint` — Disables /internal-backstage/metrics endpoint + + Typically used for prometheus scraping metrics. +* `--disable-features-endpoint` — Disables /internal-backstage/features endpoint + + Used to show current cached features across environments +* `--disable-tokens-endpoint` — Disables /internal-backstage/tokens endpoint + + Used to show tokens used to refresh feature caches, but also tokens already validated/invalidated against upstream @@ -130,6 +142,7 @@ Run in edge mode * `--redis-write-connection-timeout-milliseconds ` — Timeout (in milliseconds) for waiting for a successful connection to redis when persisting Default value: `2000` +* `--s3-bucket-name ` — Bucket name to use for storing feature and token data * `--token-header ` — Token header to use for both edge authorization and communication with the upstream server Default value: `Authorization` @@ -139,6 +152,13 @@ Run in edge mode * `--dynamic` — If set to true, Edge starts with dynamic behavior. Dynamic behavior means that Edge will accept tokens outside the scope of the startup tokens Default value: `false` +* `--prometheus-remote-write-url ` — Sets a remote write url for prometheus metrics, if this is set, prometheus metrics will be written upstream +* `--prometheus-push-interval ` — Sets the interval for prometheus push metrics, only relevant if `prometheus_remote_write_url` is set. Defaults to 60 seconds + + Default value: `60` +* `--prometheus-username ` +* `--prometheus-password ` +* `--prometheus-user-id ` @@ -196,3 +216,4 @@ Perform a ready check against a running edge instance This document was generated automatically by clap-markdown. + diff --git a/Cargo.lock b/Cargo.lock index 9ffc675e..d1e24672 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2896,15 +2896,14 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "reqwest", "thiserror", ] [[package]] name = "prometheus-reqwest-remote-write" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048bf9b7b460e4ee902571219fee43afe4dc46a77f3beb095d53f36070fdf650" +checksum = "c6da310719525cefd34f55a5ec472047b0e2d49d1c71281e7a5a85c14ff50a74" dependencies = [ "prometheus", "prost", diff --git a/server/Cargo.toml b/server/Cargo.toml index ea3d2bac..d7dc5cef 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -55,8 +55,8 @@ opentelemetry_sdk = { version = "0.24.0", features = [ "serde_json", "logs", ] } -prometheus = { version = "0.13.4", features = ["process", "push"] } -prometheus-reqwest-remote-write = { version = "0.1.1" } +prometheus = { version = "0.13.4", features = ["process"] } +prometheus-reqwest-remote-write = { version = "0.2.1" } prometheus-static-metric = "0.5.1" rand = "0.8.5" redis = { version = "0.27.0", features = [ diff --git a/server/src/cli.rs b/server/src/cli.rs index 2f685adf..60ca4d65 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -278,6 +278,30 @@ pub struct HealthCheckArgs { pub ca_certificate_file: Option, } +#[derive(Args, Debug, Clone)] +pub struct InternalBackstageArgs { + /// Disables /internal-backstage/metricsbatch endpoint + /// + /// This endpoint shows the current cached client metrics + #[clap(long, env, global = true)] + pub disable_metrics_batch_endpoint: bool, + /// Disables /internal-backstage/metrics endpoint + /// + /// Typically used for prometheus scraping metrics. + #[clap(long, env, global = true)] + pub disable_metrics_endpoint: bool, + /// Disables /internal-backstage/features endpoint + /// + /// Used to show current cached features across environments + #[clap(long, env, global = true)] + pub disable_features_endpoint: bool, + /// Disables /internal-backstage/tokens endpoint + /// + /// Used to show tokens used to refresh feature caches, but also tokens already validated/invalidated against upstream + #[clap(long, env, global = true)] + pub disable_tokens_endpoint: bool, +} + #[derive(Args, Debug, Clone)] pub struct TokenHeader { /// Token header to use for edge authorization. @@ -350,6 +374,9 @@ pub struct CliArgs { /// token header to use for edge authorization. #[clap(long, env, global = true, default_value = "Authorization")] pub token_header: TokenHeader, + + #[clap(flatten)] + pub internal_backstage: InternalBackstageArgs, } #[derive(Args, Debug, Clone)] diff --git a/server/src/internal_backstage.rs b/server/src/internal_backstage.rs index 19b30f71..af21a6aa 100644 --- a/server/src/internal_backstage.rs +++ b/server/src/internal_backstage.rs @@ -10,13 +10,13 @@ use serde::{Deserialize, Serialize}; use unleash_types::client_features::ClientFeatures; use unleash_types::client_metrics::ClientApplication; -use crate::auth::token_validator::TokenValidator; use crate::error::EdgeError; use crate::http::feature_refresher::FeatureRefresher; use crate::metrics::actix_web_metrics::PrometheusMetricsHandler; use crate::metrics::client_metrics::MetricsCache; use crate::types::{BuildInfo, EdgeJsonResult, EdgeToken, TokenInfo, TokenRefresh}; use crate::types::{ClientMetric, MetricsInfo, Status}; +use crate::{auth::token_validator::TokenValidator, cli::InternalBackstageArgs}; #[derive(Debug, Serialize, Deserialize)] pub struct EdgeStatus { @@ -131,14 +131,21 @@ pub async fn features( pub fn configure_internal_backstage( cfg: &mut web::ServiceConfig, metrics_handler: PrometheusMetricsHandler, + internal_backtage_args: InternalBackstageArgs, ) { - cfg.service(health) - .service(info) - .service(tokens) - .service(ready) - .service(metrics_batch) - .service(web::resource("/metrics").route(web::get().to(metrics_handler))) - .service(features); + cfg.service(health).service(info).service(ready); + if !internal_backtage_args.disable_tokens_endpoint { + cfg.service(tokens); + } + if !internal_backtage_args.disable_metrics_endpoint { + cfg.service(web::resource("/metrics").route(web::get().to(metrics_handler))); + } + if !internal_backtage_args.disable_metrics_batch_endpoint { + cfg.service(metrics_batch); + } + if !internal_backtage_args.disable_features_endpoint { + cfg.service(features); + } } #[cfg(test)] diff --git a/server/src/main.rs b/server/src/main.rs index 5a8d7310..85307cc8 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -56,6 +56,8 @@ async fn main() -> Result<(), anyhow::Error> { app_name: args.clone().app_name, instance_id: args.clone().instance_id, }; + let app_name = args.app_name.clone(); + let internal_backstage_args = args.internal_backstage.clone(); let ( (token_cache, features_cache, engine_cache), token_validator, @@ -114,6 +116,7 @@ async fn main() -> Result<(), anyhow::Error> { internal_backstage::configure_internal_backstage( service_cfg, metrics_handler.clone(), + internal_backstage_args.clone(), ) })) .service( @@ -169,7 +172,7 @@ async fn main() -> Result<(), anyhow::Error> { _ = validator.schedule_revalidation_of_startup_tokens(edge.tokens, lazy_feature_refresher) => { tracing::info!("Token validator validation of startup tokens was unexpectedly shut down"); } - _ = metrics_pusher::prometheus_remote_write(prom_registry_for_write, edge.prometheus_remote_write_url, edge.prometheus_push_interval, edge.prometheus_username, edge.prometheus_password) => { + _ = metrics_pusher::prometheus_remote_write(prom_registry_for_write, edge.prometheus_remote_write_url, edge.prometheus_push_interval, edge.prometheus_username, edge.prometheus_password, app_name) => { tracing::info!("Prometheus push unexpectedly shut down"); } } diff --git a/server/src/metrics/metrics_pusher.rs b/server/src/metrics/metrics_pusher.rs index 59b5b51a..18c6d9b6 100644 --- a/server/src/metrics/metrics_pusher.rs +++ b/server/src/metrics/metrics_pusher.rs @@ -1,6 +1,7 @@ use base64::Engine; use prometheus_reqwest_remote_write::WriteRequest; use reqwest::{header, Client}; +use tracing::debug; fn get_http_client(username: Option, password: Option) -> Client { if let Some(uname) = username.clone() { @@ -31,6 +32,7 @@ pub async fn prometheus_remote_write( interval: u64, username: Option, password: Option, + app_name: String, ) { let sleep_duration = tokio::time::Duration::from_secs(interval); let client = get_http_client(username, password); @@ -38,7 +40,7 @@ pub async fn prometheus_remote_write( loop { tokio::select! { _ = tokio::time::sleep(sleep_duration) => { - remote_write_prom(registry.clone(), address.clone(), client.clone()).await; + remote_write_prom(registry.clone(), address.clone(), client.clone(), app_name.clone()).await; } } } @@ -52,9 +54,17 @@ pub async fn prometheus_remote_write( } } -async fn remote_write_prom(registry: prometheus::Registry, url: String, client: reqwest::Client) { - let write_request = WriteRequest::from_metric_families(registry.gather()) - .expect("Could not format write request"); +async fn remote_write_prom( + registry: prometheus::Registry, + url: String, + client: reqwest::Client, + app_name: String, +) { + let write_request = WriteRequest::from_metric_families( + registry.gather(), + Some(vec![("app_name".into(), app_name)]), + ) + .expect("Could not format write request"); let http_request = write_request .build_http_request(client.clone(), &url, "unleash_edge") .expect("Failed to build http request"); @@ -64,9 +74,10 @@ async fn remote_write_prom(registry: prometheus::Registry, url: String, client: if !r.status().is_success() { tracing::warn!("Prometheus push failed with status: {}", r.status()); } + debug!("Prometheus push successful"); } Err(e) => { - tracing::warn!("Prometheus push failed with error: {}", e); + tracing::warn!("Prometheus push failed with error: {:?}", e); } } }