Skip to content

Commit

Permalink
Auto merge of #6686 - matthiaskrgr:lintcheck_git, r=flip1995
Browse files Browse the repository at this point in the history 
lintcheck: support git sources

This adds support for git sources in `cargo dev-lintcheck`

You can add a git source to `clippy_dev/lintcheck_crates.toml` by having a `git_url` and a `git_hash` key instead of the `versions` array.
The repo will the be cloned and checked out to the requested commit before checking it with clippy.

Fixes #6642

changelog: lintcheck: support git sources
  • Loading branch information
@bors
bors committed Feb 7, 2021
2 parents 83b7b16 + 2bffbfc commit c1ce78f
Show file tree
Hide file tree
Showing 3 changed files with 319 additions and 86 deletions.
35 changes: 18 additions & 17 deletions clippy_dev/lintcheck_crates.toml
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
[crates]
# some of these are from cargotest
cargo = ['0.49.0']
iron = ['0.6.1']
ripgrep = ['12.1.1']
xsv = ['0.13.0']
#tokei = ['12.0.4']
rayon = ['1.5.0']
serde = ['1.0.118']
cargo = {name = "cargo", versions = ['0.49.0']}
iron = {name = "iron", versions = ['0.6.1']}
ripgrep = {name = "ripgrep", versions = ['12.1.1']}
xsv = {name = "xsv", versions = ['0.13.0']}
#tokei = { name = "tokei", versions = ['12.0.4']}
rayon = {name = "rayon", versions = ['1.5.0']}
serde = {name = "serde", versions = ['1.0.118']}
# top 10 crates.io dls
bitflags = ['1.2.1']
libc = ['0.2.81']
log = ['0.4.11']
proc-macro2 = ['1.0.24']
quote = ['1.0.7']
rand = ['0.7.3']
rand_core = ['0.6.0']
regex = ['1.3.2']
syn = ['1.0.54']
unicode-xid = ['0.2.1']
bitflags = {name = "bitflags", versions = ['1.2.1']}
libc = {name = "libc", versions = ['0.2.81']}
log = {name = "log", versions = ['0.4.11']}
proc-macro2 = {name = "proc-macro2", versions = ['1.0.24']}
puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
quote = {name = "quote", versions = ['1.0.7']}
rand = {name = "rand", versions = ['0.7.3']}
rand_core = {name = "rand_core", versions = ['0.6.0']}
regex = {name = "regex", versions = ['1.3.2']}
syn = {name = "syn", versions = ['1.0.54']}
unicode-xid = {name = "unicode-xid", versions = ['0.2.1']}
167 changes: 118 additions & 49 deletions clippy_dev/src/lintcheck.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,29 @@ use serde_json::Value;
// use this to store the crates when interacting with the crates.toml file
#[derive(Debug, Serialize, Deserialize)]
struct CrateList {
crates: HashMap<String, Vec<String>>,
crates: HashMap<String, TomlCrate>,
}

// crate data we stored in the toml, can have multiple versions per crate
// A single TomlCrate is laster mapped to several CrateSources in that case
#[derive(Debug, Serialize, Deserialize)]
struct TomlCrate {
name: String,
versions: Vec<String>,
versions: Option<Vec<String>>,
git_url: Option<String>,
git_hash: Option<String>,
}

// represents an archive we download from crates.io
#[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq)]
struct CrateSource {
name: String,
version: String,
enum CrateSource {
CratesIo { name: String, version: String },
Git { name: String, url: String, commit: String },
}

// represents the extracted sourcecode of a crate
// we actually don't need to special-case git repos here because it does not matter for clippy, yay!
// (clippy only needs a simple path)
#[derive(Debug)]
struct Crate {
version: String,
Expand Down Expand Up @@ -69,40 +74,71 @@ impl std::fmt::Display for ClippyWarning {

impl CrateSource {
fn download_and_extract(&self) -> Crate {
let extract_dir = PathBuf::from("target/lintcheck/crates");
let krate_download_dir = PathBuf::from("target/lintcheck/downloads");

// url to download the crate from crates.io
let url = format!(
"https://crates.io/api/v1/crates/{}/{}/download",
self.name, self.version
);
println!("Downloading and extracting {} {} from {}", self.name, self.version, url);
let _ = std::fs::create_dir("target/lintcheck/");
let _ = std::fs::create_dir(&krate_download_dir);
let _ = std::fs::create_dir(&extract_dir);

let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", &self.name, &self.version));
// don't download/extract if we already have done so
if !krate_file_path.is_file() {
// create a file path to download and write the crate data into
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
// copy the crate into the file
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();

// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archive = tar::Archive::new(ungz_tar);
archive.unpack(&extract_dir).expect("Failed to extract!");
}
// crate is extracted, return a new Krate object which contains the path to the extracted
// sources that clippy can check
Crate {
version: self.version.clone(),
name: self.name.clone(),
path: extract_dir.join(format!("{}-{}/", self.name, self.version)),
match self {
CrateSource::CratesIo { name, version } => {
let extract_dir = PathBuf::from("target/lintcheck/crates");
let krate_download_dir = PathBuf::from("target/lintcheck/downloads");

// url to download the crate from crates.io
let url = format!("https://crates.io/api/v1/crates/{}/{}/download", name, version);
println!("Downloading and extracting {} {} from {}", name, version, url);
let _ = std::fs::create_dir("target/lintcheck/");
let _ = std::fs::create_dir(&krate_download_dir);
let _ = std::fs::create_dir(&extract_dir);

let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", name, version));
// don't download/extract if we already have done so
if !krate_file_path.is_file() {
// create a file path to download and write the crate data into
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
// copy the crate into the file
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();

// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archive = tar::Archive::new(ungz_tar);
archive.unpack(&extract_dir).expect("Failed to extract!");
}
// crate is extracted, return a new Krate object which contains the path to the extracted
// sources that clippy can check
Crate {
version: version.clone(),
name: name.clone(),
path: extract_dir.join(format!("{}-{}/", name, version)),
}
},
CrateSource::Git { name, url, commit } => {
let repo_path = {
let mut repo_path = PathBuf::from("target/lintcheck/downloads");
// add a -git suffix in case we have the same crate from crates.io and a git repo
repo_path.push(format!("{}-git", name));
repo_path
};
// clone the repo if we have not done so
if !repo_path.is_dir() {
println!("Cloning {} and checking out {}", url, commit);
Command::new("git")
.arg("clone")
.arg(url)
.arg(&repo_path)
.output()
.expect("Failed to clone git repo!");
}
// check out the commit/branch/whatever
Command::new("git")
.arg("checkout")
.arg(commit)
.output()
.expect("Failed to check out commit");

Crate {
version: commit.clone(),
name: name.clone(),
path: repo_path,
}
},
}
}
}
Expand All @@ -114,7 +150,7 @@ impl Crate {

let shared_target_dir = clippy_project_root().join("target/lintcheck/shared_target_dir/");

let all_output = std::process::Command::new(cargo_clippy_path)
let all_output = std::process::Command::new(&cargo_clippy_path)
.env("CARGO_TARGET_DIR", shared_target_dir)
// lint warnings will look like this:
// src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
Expand All @@ -128,10 +164,16 @@ impl Crate {
])
.current_dir(&self.path)
.output()
.unwrap();
.unwrap_or_else(|error| {
panic!(
"Encountered error:\n{:?}\ncargo_clippy_path: {}\ncrate path:{}\n",
error,
&cargo_clippy_path.display(),
&self.path.display()
);
});
let stdout = String::from_utf8_lossy(&all_output.stdout);
let output_lines = stdout.lines();
//dbg!(&output_lines);
let warnings: Vec<ClippyWarning> = output_lines
.into_iter()
// get all clippy warnings
Expand Down Expand Up @@ -160,19 +202,40 @@ fn read_crates() -> Vec<CrateSource> {
let tomlcrates: Vec<TomlCrate> = crate_list
.crates
.into_iter()
.map(|(name, versions)| TomlCrate { name, versions })
.map(|(_cratename, tomlcrate)| tomlcrate)
.collect();

// flatten TomlCrates into CrateSources (one TomlCrates may represent several versions of a crate =>
// multiple Cratesources)
let mut crate_sources = Vec::new();
tomlcrates.into_iter().for_each(|tk| {
tk.versions.iter().for_each(|ver| {
crate_sources.push(CrateSource {
// if we have multiple versions, save each one
if let Some(ref versions) = tk.versions {
versions.iter().for_each(|ver| {
crate_sources.push(CrateSource::CratesIo {
name: tk.name.clone(),
version: ver.to_string(),
});
})
}
// otherwise, we should have a git source
if tk.git_url.is_some() && tk.git_hash.is_some() {
crate_sources.push(CrateSource::Git {
name: tk.name.clone(),
version: ver.to_string(),
url: tk.git_url.clone().unwrap(),
commit: tk.git_hash.clone().unwrap(),
});
})
}
// if we have a version as well as a git data OR only one git data, something is funky
if tk.versions.is_some() && (tk.git_url.is_some() || tk.git_hash.is_some())
|| tk.git_hash.is_some() != tk.git_url.is_some()
{
eprintln!("tomlkrate: {:?}", tk);
if tk.git_hash.is_some() != tk.git_url.is_some() {
panic!("Encountered TomlCrate with only one of git_hash and git_url!")
}
unreachable!("Failed to translate TomlCrate into CrateSource!");
}
});
crate_sources
}
Expand Down Expand Up @@ -228,8 +291,14 @@ pub fn run(clap_config: &ArgMatches) {
let crates = read_crates();

let clippy_warnings: Vec<ClippyWarning> = if let Some(only_one_crate) = clap_config.value_of("only") {
// if we don't have the specified crated in the .toml, throw an error
if !crates.iter().any(|krate| krate.name == only_one_crate) {
// if we don't have the specified crate in the .toml, throw an error
if !crates.iter().any(|krate| {
let name = match krate {
CrateSource::CratesIo { name, .. } => name,
CrateSource::Git { name, .. } => name,
};
name == only_one_crate
}) {
eprintln!(
"ERROR: could not find crate '{}' in clippy_dev/lintcheck_crates.toml",
only_one_crate
Expand Down
Loading

0 comments on commit c1ce78f

Please sign in to comment.