From 83b2589a102da5f6c858570ce6aada83f9d668fd Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Sat, 14 Dec 2024 01:06:13 +0000 Subject: [PATCH] feat: redactions Fixes https://github.com/jdx/mise/discussions/3524 --- docs/tasks/task-configuration.md | 54 +++++++++++++++++++++++ e2e/tasks/test_task_redactions | 40 +++++++++++++++++ src/cli/run.rs | 31 +++++++------ src/cmd.rs | 30 ++++++++++--- src/config/config_file/mise_toml.rs | 8 ++++ src/config/config_file/mod.rs | 18 ++++++++ src/config/mod.rs | 45 +++++++++++++++++++ src/main.rs | 2 + src/redactions.rs | 34 +++++++++++++++ src/wildcard.rs | 67 +++++++++++++++++++++++++++++ 10 files changed, 311 insertions(+), 18 deletions(-) create mode 100644 e2e/tasks/test_task_redactions create mode 100644 src/redactions.rs create mode 100644 src/wildcard.rs diff --git a/docs/tasks/task-configuration.md b/docs/tasks/task-configuration.md index 29cb1a3a08..bc564d1d21 100644 --- a/docs/tasks/task-configuration.md +++ b/docs/tasks/task-configuration.md @@ -307,3 +307,57 @@ run = "echo task4" ::: If you want auto-completion/validation in included toml tasks files, you can use the following JSON schema: + +## `[redactions]` options + +Redactions are a way to hide sensitive information from the output of tasks. This is useful for things like +API keys, passwords, or other sensitive information that you don't want to accidentally leak in logs or +other output. + +### `redactions.env` + +- **Type**: `string[]` + +A list of environment variables to redact from the output. + +```toml +[redactions] +env = ["API_KEY", "PASSWORD"] +[tasks.test] +run = "echo $API_KEY" +``` + +Running the above task will output `echo [redacted]` instead. + +You can also specify these as a glob pattern, e.g.: `redactions.env = ["SECRETS_*"]`. + +### `redactions.vars` + +- **Type**: `string[]` + +A list of [vars](#vars) to redact from the output. + +```toml +[vars] +secret = "mysecret" +[tasks.test] +run = "echo {{vars.secret}}" +``` + +:::tip +This is generally useful when using `mise.local.toml` to put secret vars in which can be shared +with any other `mise.toml` file in the hierarchy. +::: + +## `[vars]` options + +Vars are variables that can be shared between tasks like environment variables but they are not +passed as environment variables to the scripts. They are defined in the `vars` section of the +`mise.toml` file. + +```toml +[vars] +e2e_args = '--headless' +[tasks.test] +run = './scripts/test-e2e.sh {{vars.e2e_args}}' +``` diff --git a/e2e/tasks/test_task_redactions b/e2e/tasks/test_task_redactions new file mode 100644 index 0000000000..4212256a50 --- /dev/null +++ b/e2e/tasks/test_task_redactions @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +cat <mise.toml +[env] +SECRET = "my_secret" + +[tasks.a] +run = 'echo secret: \$SECRET' + +[redactions] +env = ["SECRET"] +EOF + +assert "mise run a" "secret: [redacted]" + +cat <mise.toml +[tasks.a] +run = 'echo secret: {{ vars.secret }}' + +[redactions] +vars = ["secret"] + +[vars] +secret = "my_secret" +EOF + +assert "mise run a" "secret: [redacted]" + +cat <mise.toml +[env] +SECRET_FOO = "my_secret_wild" + +[tasks.a] +run = 'echo secret: \$SECRET_FOO' + +[redactions] +env = ["SECRET*"] +EOF + +assert "mise run a" "secret: [redacted]" diff --git a/src/cli/run.rs b/src/cli/run.rs index 49815f696a..7fed325c7b 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -361,12 +361,12 @@ impl Run { env: &BTreeMap, prefix: &str, ) -> Result<()> { + let config = Config::get(); let script = script.trim_start(); - let cmd = trunc( - &style::ebold(format!("$ {script} {args}", args = args.join(" "))) - .bright() - .to_string(), - ); + let cmd = style::ebold(format!("$ {script} {args}", args = args.join(" "))) + .bright() + .to_string(); + let cmd = trunc(&config.redact(cmd)?); if !self.quiet(Some(task)) { eprintln!("{prefix} {cmd}"); } @@ -452,6 +452,7 @@ impl Run { env: &BTreeMap, prefix: &str, ) -> Result<()> { + let config = Config::get(); let mut env = env.clone(); let command = file.to_string_lossy().to_string(); let args = task.args.iter().cloned().collect_vec(); @@ -465,7 +466,8 @@ impl Run { } let cmd = format!("{} {}", display_path(file), args.join(" ")); - let cmd = trunc(&style::ebold(format!("$ {cmd}")).bright().to_string()); + let cmd = style::ebold(format!("$ {cmd}")).bright().to_string(); + let cmd = trunc(&config.redact(cmd)?); if !self.quiet(Some(task)) { eprintln!("{prefix} {cmd}"); } @@ -493,23 +495,28 @@ impl Run { env: &BTreeMap, prefix: &str, ) -> Result<()> { + let config = Config::get(); let program = program.to_executable(); + let redactions = config.redactions()?; let mut cmd = CmdLineRunner::new(program.clone()) .args(args) .envs(env) + .redact(redactions.clone()) .raw(self.raw(Some(task))); cmd.with_pass_signals(); match self.output(Some(task)) { TaskOutput::Prefix => cmd = cmd.prefix(format!("{prefix} ")), - TaskOutput::Quiet | TaskOutput::Interleave => { - cmd = cmd - .stdin(Stdio::inherit()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - } TaskOutput::Silent => { cmd = cmd.stdout(Stdio::null()).stderr(Stdio::null()); } + TaskOutput::Quiet | TaskOutput::Interleave => { + if redactions.is_empty() { + cmd = cmd + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + } + } } let dir = self.cwd(task)?; if !dir.exists() { diff --git a/src/cmd.rs b/src/cmd.rs index ca2ad850b9..f734bf144d 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -11,6 +11,7 @@ use std::thread; use color_eyre::Result; use duct::{Expression, IntoExecutablePath}; use eyre::Context; +use indexmap::IndexSet; use once_cell::sync::Lazy; #[cfg(not(any(test, target_os = "windows")))] use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2}; @@ -100,6 +101,7 @@ pub struct CmdLineRunner<'a> { pr: Option<&'a dyn SingleReport>, stdin: Option, prefix: String, + redactions: IndexSet, raw: bool, pass_signals: bool, } @@ -126,6 +128,7 @@ impl<'a> CmdLineRunner<'a> { pr: None, stdin: None, prefix: String::new(), + redactions: Default::default(), raw: false, pass_signals: false, } @@ -174,6 +177,13 @@ impl<'a> CmdLineRunner<'a> { self } + pub fn redact(mut self, redactions: impl IntoIterator) -> Self { + for r in redactions { + self.redactions.insert(r); + } + self + } + pub fn prefix(mut self, prefix: impl Into) -> Self { self.prefix = prefix.into(); self @@ -337,11 +347,11 @@ impl<'a> CmdLineRunner<'a> { for line in rx { match line { ChildProcessOutput::Stdout(line) => { - self.on_stdout(&line); + self.on_stdout(line.clone()); combined_output.push(line); } ChildProcessOutput::Stderr(line) => { - self.on_stderr(&line); + self.on_stderr(line.clone()); combined_output.push(line); } ChildProcessOutput::ExitStatus(s) => { @@ -377,11 +387,15 @@ impl<'a> CmdLineRunner<'a> { } } - fn on_stdout(&self, line: &str) { + fn on_stdout(&self, mut line: String) { + line = self + .redactions + .iter() + .fold(line, |acc, r| acc.replace(r, "[redacted]")); let _lock = OUTPUT_LOCK.lock().unwrap(); if let Some(pr) = self.pr { if !line.trim().is_empty() { - pr.set_message(line.into()) + pr.set_message(line) } } else if console::colors_enabled() { println!("{}{line}\x1b[0m", self.prefix); @@ -390,12 +404,16 @@ impl<'a> CmdLineRunner<'a> { } } - fn on_stderr(&self, line: &str) { + fn on_stderr(&self, mut line: String) { + line = self + .redactions + .iter() + .fold(line, |acc, r| acc.replace(r, "[redacted]")); let _lock = OUTPUT_LOCK.lock().unwrap(); match self.pr { Some(pr) => { if !line.trim().is_empty() { - pr.println(line.into()) + pr.println(line) } } None => { diff --git a/src/config/config_file/mise_toml.rs b/src/config/config_file/mise_toml.rs index 2160c73178..1d4eb60adf 100644 --- a/src/config/config_file/mise_toml.rs +++ b/src/config/config_file/mise_toml.rs @@ -21,6 +21,7 @@ use crate::config::settings::SettingsPartial; use crate::config::{Alias, AliasMap}; use crate::file::{create_dir_all, display_path}; use crate::hooks::{Hook, Hooks}; +use crate::redactions::Redactions; use crate::registry::REGISTRY; use crate::task::Task; use crate::tera::{get_tera, BASE_CONTEXT}; @@ -55,6 +56,8 @@ pub struct MiseToml { #[serde(default)] plugins: HashMap, #[serde(default)] + redactions: Redactions, + #[serde(default)] task_config: TaskConfig, #[serde(default)] tasks: Tasks, @@ -438,6 +441,10 @@ impl ConfigFile for MiseToml { &self.task_config } + fn redactions(&self) -> &Redactions { + &self.redactions + } + fn watch_files(&self) -> eyre::Result> { self.watch_files .iter() @@ -534,6 +541,7 @@ impl Clone for MiseToml { doc: self.doc.clone(), hooks: self.hooks.clone(), tools: self.tools.clone(), + redactions: self.redactions.clone(), plugins: self.plugins.clone(), tasks: self.tasks.clone(), task_config: self.task_config.clone(), diff --git a/src/config/config_file/mod.rs b/src/config/config_file/mod.rs index 90005c2a4e..5cc179b485 100644 --- a/src/config/config_file/mod.rs +++ b/src/config/config_file/mod.rs @@ -23,7 +23,9 @@ use crate::errors::Error::UntrustedConfig; use crate::file::display_path; use crate::hash::hash_to_str; use crate::hooks::Hook; +use crate::redactions::Redactions; use crate::task::Task; +use crate::tera::{get_tera, BASE_CONTEXT}; use crate::toolset::{ToolRequest, ToolRequestSet, ToolSource, ToolVersionList, Toolset}; use crate::ui::{prompt, style}; use crate::watch_files::WatchFile; @@ -65,6 +67,9 @@ pub trait ConfigFile: Debug + Send + Sync { None => None, } } + fn config_root(&self) -> PathBuf { + config_root(self.get_path()) + } fn plugins(&self) -> eyre::Result> { Ok(Default::default()) } @@ -87,6 +92,14 @@ pub trait ConfigFile: Debug + Send + Sync { fn aliases(&self) -> eyre::Result { Ok(Default::default()) } + + fn tera(&self) -> (tera::Tera, tera::Context) { + let tera = get_tera(Some(&self.config_root())); + let mut ctx = BASE_CONTEXT.clone(); + ctx.insert("config_root", &self.config_root()); + (tera, ctx) + } + fn task_config(&self) -> &TaskConfig { static DEFAULT_TASK_CONFIG: Lazy = Lazy::new(TaskConfig::default); &DEFAULT_TASK_CONFIG @@ -96,6 +109,11 @@ pub trait ConfigFile: Debug + Send + Sync { Ok(&DEFAULT_VARS) } + fn redactions(&self) -> &Redactions { + static DEFAULT_REDACTIONS: Lazy = Lazy::new(Redactions::default); + &DEFAULT_REDACTIONS + } + fn watch_files(&self) -> Result> { Ok(Default::default()) } diff --git a/src/config/mod.rs b/src/config/mod.rs index 58b338c832..db4cc90218 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -37,7 +37,9 @@ use crate::cli::self_update::SelfUpdate; use crate::hook_env::WatchFilePattern; use crate::hooks::Hook; use crate::plugins::PluginType; +use crate::redactions::Redactions; use crate::watch_files::WatchFile; +use crate::wildcard::Wildcard; pub use settings::SETTINGS; type AliasMap = IndexMap; @@ -54,6 +56,7 @@ pub struct Config { aliases: AliasMap, env: OnceCell, env_with_sources: OnceCell, + redactions: OnceCell>, shorthands: OnceLock, tasks: OnceCell>, tool_request_set: OnceCell, @@ -622,6 +625,48 @@ impl Config { .chain(SETTINGS.env_files().iter().map(|p| p.as_path().into())) .collect()) } + + pub fn redactions(&self) -> Result<&IndexSet> { + self.redactions.get_or_try_init(|| { + let mut redactions = Redactions::default(); + for cf in self.config_files.values() { + let r = cf.redactions(); + if !r.is_empty() { + let mut r = r.clone(); + let (tera, ctx) = cf.tera(); + r.render(&mut tera.clone(), &ctx)?; + redactions.merge(r); + } + } + if redactions.is_empty() { + return Ok(Default::default()); + } + + let ts = self.get_toolset()?; + let env = ts.full_env()?; + + let env_matcher = Wildcard::new(redactions.env.clone()); + let var_matcher = Wildcard::new(redactions.vars.clone()); + + let env_vals = env + .into_iter() + .filter(|(k, _)| env_matcher.match_any(k)) + .map(|(_, v)| v); + let var_vals = self + .vars + .iter() + .filter(|(k, _)| var_matcher.match_any(k)) + .map(|(_, v)| v.to_string()); + Ok(env_vals.chain(var_vals).collect()) + }) + } + + pub fn redact(&self, mut input: String) -> Result { + for redaction in self.redactions()? { + input = input.replace(redaction, "[redacted]"); + } + Ok(input) + } } fn get_project_root(config_files: &ConfigMap) -> Option { diff --git a/src/main.rs b/src/main.rs index 09e367e01d..87c5546002 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,6 +53,7 @@ mod migrate; mod path_env; mod plugins; mod rand; +mod redactions; mod registry; pub(crate) mod result; mod runtime_symlinks; @@ -69,6 +70,7 @@ mod ui; mod uv; mod versions_host; mod watch_files; +mod wildcard; pub(crate) use crate::exit::exit; pub(crate) use crate::result::Result; diff --git a/src/redactions.rs b/src/redactions.rs new file mode 100644 index 0000000000..e77b1f37fb --- /dev/null +++ b/src/redactions.rs @@ -0,0 +1,34 @@ +use indexmap::IndexSet; + +#[derive(Default, Clone, Debug, serde::Deserialize)] +pub struct Redactions { + #[serde(default)] + pub env: IndexSet, + #[serde(default)] + pub vars: IndexSet, +} + +impl Redactions { + pub fn merge(&mut self, other: Self) { + for e in other.env { + self.env.insert(e); + } + for v in other.vars { + self.vars.insert(v); + } + } + + pub fn render(&mut self, tera: &mut tera::Tera, ctx: &tera::Context) -> eyre::Result<()> { + for r in self.env.clone().drain(..) { + self.env.insert(tera.render_str(&r, ctx)?); + } + for r in self.vars.clone().drain(..) { + self.vars.insert(tera.render_str(&r, ctx)?); + } + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.env.is_empty() && self.vars.is_empty() + } +} diff --git a/src/wildcard.rs b/src/wildcard.rs new file mode 100644 index 0000000000..5e7f289a09 --- /dev/null +++ b/src/wildcard.rs @@ -0,0 +1,67 @@ +use std::str::Chars; + +pub struct Wildcard { + patterns: Vec, +} + +impl Wildcard { + pub fn new(patterns: impl IntoIterator>) -> Self { + Self { + patterns: patterns.into_iter().map(Into::into).collect(), + } + } + + pub fn match_any(&self, input: &str) -> bool { + for pattern in &self.patterns { + if wildcard_match_single(input, pattern) { + return true; + } + } + false + } +} + +fn wildcard_match_single(input: &str, wildcard: &str) -> bool { + let mut input_chars = input.chars(); + let mut wildcard_chars = wildcard.chars(); + + loop { + match (input_chars.next(), wildcard_chars.next()) { + (Some(input_char), Some(wildcard_char)) => { + if wildcard_char == '*' { + return wildcard_match_single_star(input_chars, wildcard_chars); + } else if wildcard_char == '?' || input_char == wildcard_char { + continue; + } else { + return false; + } + } + (None, None) => return true, + (None, Some(wildcard_char)) => return wildcard_char == '*', + (Some(_), None) => return false, + } + } +} + +fn wildcard_match_single_star(mut input_chars: Chars, mut wildcard_chars: Chars) -> bool { + loop { + match wildcard_chars.next() { + Some(wildcard_char) => { + if wildcard_char == '*' { + continue; + } else { + while let Some(input_char) = input_chars.next() { + if wildcard_match_single( + &input_char.to_string(), + &wildcard_char.to_string(), + ) { + return wildcard_match_single_star(input_chars, wildcard_chars); + } + } + return false; + } + } + None => return true, + } + } +}