From 089d03e1dfca9c44ff883052396ecd83d36f29dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Wed, 6 Apr 2022 09:28:44 +0200 Subject: [PATCH 1/4] fix wrong impl of env.passthrough and env.volumes on targets --- docs/cross_toml.md | 12 +++++++-- src/cross_toml.rs | 66 ++++++++++++++++++++++------------------------ 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/docs/cross_toml.md b/docs/cross_toml.md index 32e0a39dc..662c541af 100644 --- a/docs/cross_toml.md +++ b/docs/cross_toml.md @@ -25,9 +25,17 @@ The `target` key allows you to specify parameters for specific compilation targe ```toml [target.aarch64-unknown-linux-gnu] -volumes = ["VOL1_ARG", "VOL2_ARG"] -passthrough = ["VAR1", "VAR2"] xargo = false image = "test-image" runner = "custom-runner" ``` + +# `target.TARGET.env` +The `target` key allows you to specify environment variables that should be used for a specific compilation target. +This is similar to `build.env`, but allows you to be more specific per target. + +```toml +[target.x86_64-unknown-linux-gnu.env] +volumes = ["VOL1_ARG", "VOL2_ARG"] +passthrough = ["IMPORTANT_ENV_VARIABLES"] +``` \ No newline at end of file diff --git a/src/cross_toml.rs b/src/cross_toml.rs index f080157e6..b2aa3bb65 100644 --- a/src/cross_toml.rs +++ b/src/cross_toml.rs @@ -5,9 +5,9 @@ use crate::{Target, TargetList}; use serde::Deserialize; use std::collections::HashMap; -/// Build environment configuration -#[derive(Debug, Deserialize, PartialEq)] -pub struct CrossBuildEnvConfig { +/// Environment configuration +#[derive(Debug, Deserialize, PartialEq, Default)] +pub struct CrossEnvConfig { #[serde(default)] volumes: Vec, #[serde(default)] @@ -18,7 +18,8 @@ pub struct CrossBuildEnvConfig { #[derive(Debug, Deserialize, PartialEq, Default)] #[serde(rename_all = "kebab-case")] pub struct CrossBuildConfig { - env: Option, + #[serde(default)] + env: CrossEnvConfig, xargo: Option, default_target: Option, } @@ -26,13 +27,11 @@ pub struct CrossBuildConfig { /// Target configuration #[derive(Debug, Deserialize, PartialEq)] pub struct CrossTargetConfig { - #[serde(default)] - passthrough: Vec, - #[serde(default)] - volumes: Vec, xargo: Option, image: Option, runner: Option, + #[serde(default)] + env: CrossEnvConfig, } /// Cross configuration @@ -40,7 +39,8 @@ pub struct CrossTargetConfig { pub struct CrossToml { #[serde(default, rename = "target")] pub targets: HashMap, - pub build: Option, + #[serde(default)] + pub build: CrossBuildConfig, } impl CrossToml { @@ -76,7 +76,7 @@ impl CrossToml { /// Returns the `build.xargo` or the `target.{}.xargo` part of `Cross.toml` pub fn xargo(&self, target: &Target) -> (Option, Option) { - let build_xargo = self.build.as_ref().and_then(|b| b.xargo); + let build_xargo = self.build.xargo; let target_xargo = self.get_target(target).and_then(|t| t.xargo); (build_xargo, target_xargo) @@ -84,33 +84,31 @@ impl CrossToml { /// Returns the list of environment variables to pass through for `build`, pub fn env_passthrough_build(&self) -> Vec { - self.get_build_env() - .map_or(Vec::new(), |e| e.passthrough.clone()) + self.build.env.passthrough.clone() } /// Returns the list of environment variables to pass through for `target`, pub fn env_passthrough_target(&self, target: &Target) -> Vec { self.get_target(target) - .map_or(Vec::new(), |t| t.passthrough.clone()) + .map_or(Vec::new(), |t| t.env.passthrough.clone()) } /// Returns the list of environment variables to pass through for `build`, pub fn env_volumes_build(&self) -> Vec { - self.get_build_env() - .map_or(Vec::new(), |e| e.volumes.clone()) + self.build.env.volumes.clone() } /// Returns the list of environment variables to pass through for `target`, pub fn env_volumes_target(&self, target: &Target) -> Vec { self.get_target(target) - .map_or(Vec::new(), |t| t.volumes.clone()) + .map_or(Vec::new(), |t| t.env.volumes.clone()) } /// Returns the default target to build, pub fn default_target(&self, target_list: &TargetList) -> Option { self.build + .default_target .as_ref() - .and_then(|b| b.default_target.as_ref()) .map(|t| Target::from(t, target_list)) } @@ -118,11 +116,6 @@ impl CrossToml { fn get_target(&self, target: &Target) -> Option<&CrossTargetConfig> { self.targets.get(target) } - - /// Returns a reference to the [`CrossBuildEnvConfig`] - fn get_build_env(&self) -> Option<&CrossBuildEnvConfig> { - self.build.as_ref().and_then(|b| b.env.as_ref()) - } } #[cfg(test)] @@ -133,7 +126,7 @@ mod tests { pub fn parse_empty_toml() -> Result<()> { let cfg = CrossToml { targets: HashMap::new(), - build: None, + build: CrossBuildConfig::default(), }; let parsed_cfg = CrossToml::from_str("")?; @@ -146,14 +139,14 @@ mod tests { pub fn parse_build_toml() -> Result<()> { let cfg = CrossToml { targets: HashMap::new(), - build: Some(CrossBuildConfig { - env: Some(CrossBuildEnvConfig { + build: CrossBuildConfig { + env: CrossEnvConfig { volumes: vec!["VOL1_ARG".to_string(), "VOL2_ARG".to_string()], passthrough: vec!["VAR1".to_string(), "VAR2".to_string()], - }), + }, xargo: Some(true), default_target: None, - }), + }, }; let test_str = r#" @@ -179,8 +172,10 @@ mod tests { triple: "aarch64-unknown-linux-gnu".to_string(), }, CrossTargetConfig { - passthrough: vec!["VAR1".to_string(), "VAR2".to_string()], - volumes: vec!["VOL1_ARG".to_string(), "VOL2_ARG".to_string()], + env: CrossEnvConfig { + passthrough: vec!["VAR1".to_string(), "VAR2".to_string()], + volumes: vec!["VOL1_ARG".to_string(), "VOL2_ARG".to_string()], + }, xargo: Some(false), image: Some("test-image".to_string()), runner: None, @@ -189,15 +184,16 @@ mod tests { let cfg = CrossToml { targets: target_map, - build: None, + build: CrossBuildConfig::default(), }; let test_str = r#" - [target.aarch64-unknown-linux-gnu] - volumes = ["VOL1_ARG", "VOL2_ARG"] - passthrough = ["VAR1", "VAR2"] - xargo = false - image = "test-image" + [target.aarch64-unknown-linux-gnu.env] + volumes = ["VOL1_ARG", "VOL2_ARG"] + passthrough = ["VAR1", "VAR2"] + [target.aarch64-unknown-linux-gnu] + xargo = false + image = "test-image" "#; let parsed_cfg = CrossToml::from_str(test_str)?; From 4a858e5e1776e5e2227b18824c9f34296c11079d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Wed, 6 Apr 2022 14:39:03 +0200 Subject: [PATCH 2/4] make gitignore more general --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1de565933..e333188d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -target \ No newline at end of file +**/target/ +**/.idea/ +**/.vscode/*.* +**/*.log +/cargo-timing*.html \ No newline at end of file From 05b4a3b22853fb438aad83a5712bbd41344317a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Wed, 6 Apr 2022 14:25:34 +0200 Subject: [PATCH 3/4] add ci testing for markdown toml fences --- .github/workflows/ci.yml | 2 +- Cargo.lock | 59 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 6 +++- README.md | 4 +-- src/config.rs | 2 +- src/cross_toml.rs | 16 +++++----- src/main.rs | 5 +++- src/tests.rs | 53 +++++++++++++++++++++++++++++++++ src/tests/toml.rs | 63 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 195 insertions(+), 15 deletions(-) create mode 100644 src/tests.rs create mode 100644 src/tests/toml.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 362000fdc..bbc967cef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --locked + args: --locked --all-targets --workspace timeout-minutes: 5 generate-matrix: diff --git a/Cargo.lock b/Cargo.lock index ffe034fdc..e3b216bad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "atty" version = "0.2.14" @@ -103,15 +112,17 @@ dependencies = [ "dunce", "eyre", "home", - "lazy_static", "libc", "nix", + "once_cell", + "regex", "rustc_version", "serde", "serde_ignored", "serde_json", "shell-escape", "toml", + "walkdir", "which", "winapi", ] @@ -269,6 +280,23 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -290,6 +318,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.6" @@ -434,6 +471,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "which" version = "4.2.4" @@ -461,6 +509,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 14e3ba4af..f765996ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ atty = "0.2" color-eyre = "0.6" eyre = "0.6" home = "0.5" -lazy_static = "1" rustc_version = "0.4" toml = "0.5" which = { version = "4", default_features = false } @@ -33,3 +32,8 @@ dunce = "1" [profile.release] lto = true + +[dev-dependencies] +regex = "1" +once_cell = "1" +walkdir = "2" diff --git a/README.md b/README.md index c1e09744e..9ae5418a5 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ non-standard targets (i.e. something not reported by rustc/rustup). However, you can use the `build.xargo` or `target.{{TARGET}}.xargo` field in `Cross.toml` to force the use of `xargo`: -``` toml +```toml # all the targets will use `xargo` [build] xargo = true @@ -234,7 +234,7 @@ xargo = true Or, -``` toml +```toml # only this target will use `xargo` [target.aarch64-unknown-linux-gnu] xargo = true diff --git a/src/config.rs b/src/config.rs index 3ec2ab8a0..17f810c5b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -319,7 +319,7 @@ mod tests { use std::matches; fn toml(content: &str) -> Result { - Ok(CrossToml::from_str(content).wrap_err("couldn't parse toml")?) + Ok(CrossToml::parse(content).wrap_err("couldn't parse toml")?.0) } #[test] diff --git a/src/cross_toml.rs b/src/cross_toml.rs index b2aa3bb65..2c33d384d 100644 --- a/src/cross_toml.rs +++ b/src/cross_toml.rs @@ -3,7 +3,7 @@ use crate::errors::*; use crate::{Target, TargetList}; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; /// Environment configuration #[derive(Debug, Deserialize, PartialEq, Default)] @@ -45,10 +45,10 @@ pub struct CrossToml { impl CrossToml { /// Parses the [`CrossToml`] from a string - pub fn from_str(toml_str: &str) -> Result { + pub fn parse(toml_str: &str) -> Result<(Self, BTreeSet)> { let tomld = &mut toml::Deserializer::new(toml_str); - let mut unused = std::collections::BTreeSet::new(); + let mut unused = BTreeSet::new(); let cfg = serde_ignored::deserialize(tomld, |path| { unused.insert(path.to_string()); @@ -57,11 +57,11 @@ impl CrossToml { if !unused.is_empty() { eprintln!( "Warning: found unused key(s) in Cross configuration:\n > {}", - unused.into_iter().collect::>().join(", ") + unused.clone().into_iter().collect::>().join(", ") ); } - Ok(cfg) + Ok((cfg, unused)) } /// Returns the `target.{}.image` part of `Cross.toml` @@ -128,7 +128,7 @@ mod tests { targets: HashMap::new(), build: CrossBuildConfig::default(), }; - let parsed_cfg = CrossToml::from_str("")?; + let (parsed_cfg, _) = CrossToml::parse("")?; assert_eq!(parsed_cfg, cfg); @@ -157,7 +157,7 @@ mod tests { volumes = ["VOL1_ARG", "VOL2_ARG"] passthrough = ["VAR1", "VAR2"] "#; - let parsed_cfg = CrossToml::from_str(test_str)?; + let (parsed_cfg, _) = CrossToml::parse(test_str)?; assert_eq!(parsed_cfg, cfg); @@ -195,7 +195,7 @@ mod tests { xargo = false image = "test-image" "#; - let parsed_cfg = CrossToml::from_str(test_str)?; + let (parsed_cfg, _) = CrossToml::parse(test_str)?; assert_eq!(parsed_cfg, cfg); diff --git a/src/main.rs b/src/main.rs index f05d1ba4c..284c07fcc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,8 @@ #![deny(missing_debug_implementations, rust_2018_idioms)] +#[cfg(test)] +mod tests; + mod cargo; mod cli; mod config; @@ -405,7 +408,7 @@ fn toml(root: &Root) -> Result> { let content = file::read(&path) .wrap_err_with(|| format!("could not read file `{}`", path.display()))?; - let config = CrossToml::from_str(&content) + let (config, _) = CrossToml::parse(&content) .wrap_err_with(|| format!("failed to parse file `{}` as TOML", path.display()))?; Ok(Some(config)) diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 000000000..5fa299105 --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,53 @@ +mod toml; + +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, +}; + +use once_cell::sync::OnceCell; +use serde::Deserialize; + +static WORKSPACE: OnceCell = OnceCell::new(); + +/// Returns the cargo workspace for the manifest +pub fn get_cargo_workspace() -> &'static Path { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + WORKSPACE.get_or_init(|| { + #[derive(Deserialize)] + struct Manifest { + workspace_root: PathBuf, + } + let output = std::process::Command::new( + std::env::var("CARGO") + .ok() + .unwrap_or_else(|| "cargo".to_string()), + ) + .arg("metadata") + .arg("--format-version=1") + .arg("--no-deps") + .current_dir(manifest_dir) + .output() + .unwrap(); + let manifest: Manifest = serde_json::from_slice(&output.stdout).unwrap(); + manifest.workspace_root + }) +} + +pub fn walk_dir<'a>( + root: &'_ Path, + skip: &'a [impl AsRef], +) -> impl Iterator> + 'a { + walkdir::WalkDir::new(root).into_iter().filter_entry(|e| { + if skip + .iter() + .map(|s| -> &std::ffi::OsStr { s.as_ref() }) + .any(|dir| e.file_name() == dir) + { + return false; + } else if e.file_type().is_dir() { + return true; + } + e.path().extension() == Some("md".as_ref()) + }) +} diff --git a/src/tests/toml.rs b/src/tests/toml.rs new file mode 100644 index 000000000..e39389e29 --- /dev/null +++ b/src/tests/toml.rs @@ -0,0 +1,63 @@ +use std::io::Read; + +use once_cell::sync::Lazy; +use regex::{Regex, RegexBuilder}; + +static TOML_REGEX: Lazy = Lazy::new(|| { + RegexBuilder::new(r#"```toml\n(.*?)```"#) + .multi_line(true) + .dot_matches_new_line(true) + .build() + .unwrap() +}); + +#[test] +fn toml_check() -> Result<(), Box> { + let workspace_root = super::get_cargo_workspace(); + let walk = super::walk_dir( + workspace_root, + &[ + "target", + ".git", + "src", + "CODE_OF_CONDUCT.md", + "CHANGELOG.md", + ], + ); + + for dir_entry in walk { + let dir_entry = dir_entry?; + if dir_entry.file_type().is_dir() { + continue; + } + eprintln!("File: {:?}", dir_entry.path().display()); + let mut file = std::fs::File::open(dir_entry.path()).unwrap(); + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + for matches in TOML_REGEX.captures_iter(&contents) { + let fence = matches.get(1).unwrap(); + eprintln!( + "testing snippet at: {}:{:?}", + dir_entry.path().display(), + text_line_no(&contents, fence.range().start), + ); + assert!(crate::cross_toml::CrossToml::parse(fence.as_str())? + .1 + .is_empty()); + } + } + Ok(()) +} + +pub fn text_line_no(text: &str, index: usize) -> usize { + let mut line_no = 0; + let mut count = 0; + for line in text.split('\n') { + line_no += 1; + count += line.as_bytes().len() + 1; + if count >= index { + break; + } + } + line_no +} From c2563d454434687337ef01d3792dcd2a9145424a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Thu, 7 Apr 2022 12:28:14 +0200 Subject: [PATCH 4/4] assert unused keys is empty --- src/cross_toml.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cross_toml.rs b/src/cross_toml.rs index 2c33d384d..fbddb9352 100644 --- a/src/cross_toml.rs +++ b/src/cross_toml.rs @@ -128,9 +128,10 @@ mod tests { targets: HashMap::new(), build: CrossBuildConfig::default(), }; - let (parsed_cfg, _) = CrossToml::parse("")?; + let (parsed_cfg, unused) = CrossToml::parse("")?; assert_eq!(parsed_cfg, cfg); + assert!(unused.is_empty()); Ok(()) } @@ -157,9 +158,10 @@ mod tests { volumes = ["VOL1_ARG", "VOL2_ARG"] passthrough = ["VAR1", "VAR2"] "#; - let (parsed_cfg, _) = CrossToml::parse(test_str)?; + let (parsed_cfg, unused) = CrossToml::parse(test_str)?; assert_eq!(parsed_cfg, cfg); + assert!(unused.is_empty()); Ok(()) } @@ -195,9 +197,10 @@ mod tests { xargo = false image = "test-image" "#; - let (parsed_cfg, _) = CrossToml::parse(test_str)?; + let (parsed_cfg, unused) = CrossToml::parse(test_str)?; assert_eq!(parsed_cfg, cfg); + assert!(unused.is_empty()); Ok(()) }