Skip to content

Commit

Permalink
Async loading for outdir and proc-macro
Browse files Browse the repository at this point in the history
  • Loading branch information
@edwin0cheng
edwin0cheng committed Jan 23, 2021
1 parent 3cd994d commit 11dc6c7
Show file tree
Hide file tree
Showing 10 changed files with 334 additions and 139 deletions.
282 changes: 169 additions & 113 deletions crates/project_model/src/build_data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,13 @@ use std::{
use anyhow::Result;
use cargo_metadata::{BuildScript, Message, Package, PackageId};
use itertools::Itertools;
use la_arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;
use stdx::JodChild;

use crate::{cfg_flag::CfgFlag, CargoConfig};

#[derive(Debug, Clone, Default)]
pub(crate) struct BuildDataMap {
data: FxHashMap<PackageId, BuildData>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BuildData {
/// List of config flags defined by this package's build script
Expand All @@ -33,133 +30,190 @@ pub struct BuildData {
pub out_dir: Option<AbsPathBuf>,
/// Path to the proc-macro library file if this package exposes proc-macros
pub proc_macro_dylib_path: Option<AbsPathBuf>,

/// State for build data, used for updating
fetch: Option<(Idx<BuildDataConfig>, PackageId)>,
}

impl BuildDataMap {
pub(crate) fn new(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}
#[derive(Debug)]
pub(crate) struct BuildDataConfig {
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}
#[derive(Debug, Default)]
pub struct BuildDataLoader {
arena: Arena<BuildDataConfig>,
data: FxHashMap<Idx<BuildDataConfig>, BuildDataMap>,
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.data.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}
type BuildDataMap = FxHashMap<PackageId, BuildData>;

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) =
message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.data.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
pub(crate) fn packages_to_build_data(
packages: &Vec<Package>,
idx: Option<Idx<BuildDataConfig>>,
) -> BuildDataMap {
let mut res: BuildDataMap = FxHashMap::default();
for meta_pkg in packages {
let mut build_data = BuildData::default();
build_data.fetch = idx.map(|it| (it, meta_pkg.id.clone()));
inject_cargo_env(meta_pkg, &mut build_data);
res.insert(meta_pkg.id.clone(), build_data);
}
res
}

impl BuildData {
pub(crate) fn update(&mut self, loader: &BuildDataLoader) {
if let Some((ws, pid)) = &self.fetch {
let new_data = loader.get(ws, pid).cloned();
let BuildData { mut cfgs, mut envs, out_dir, proc_macro_dylib_path, .. } =
match new_data {
None => return,
Some(it) => it,
};

if let Some(out_dir) = &out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
self.envs.push(("OUT_DIR".to_string(), out_dir));
}
}
self.cfgs.append(&mut cfgs);
self.envs.append(&mut envs);
self.out_dir = out_dir;
self.proc_macro_dylib_path = proc_macro_dylib_path;
self.fetch = None;
}
res.inject_cargo_env(packages);
Ok(res)
}
}

pub(crate) fn with_cargo_env(packages: &Vec<Package>) -> Self {
let mut res = Self::default();
res.inject_cargo_env(packages);
res
impl BuildDataLoader {
pub(crate) fn new_config(
&mut self,
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
) -> Idx<BuildDataConfig> {
self.arena.alloc(BuildDataConfig {
cargo_toml: cargo_toml.to_path_buf().clone(),
cargo_features: cargo_features.clone(),
})
}

pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> {
self.data.get(id)
pub fn load(&mut self, progress: &dyn Fn(String)) -> Result<()> {
for (idx, config) in self.arena.iter() {
self.data
.insert(idx, load_workspace(&config.cargo_toml, &config.cargo_features, progress)?);
}
Ok(())
}

fn inject_cargo_env(&mut self, packages: &Vec<Package>) {
for meta_pkg in packages {
let resource = self.data.entry(meta_pkg.id.clone()).or_default();
inject_cargo_env(meta_pkg, &mut resource.envs);
pub(crate) fn get(
&self,
idx: &Idx<BuildDataConfig>,
package_id: &PackageId,
) -> Option<&BuildData> {
self.data.get(idx).and_then(|it| it.get(&package_id))
}
}

if let Some(out_dir) = &resource.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
resource.envs.push(("OUT_DIR".to_string(), out_dir));
fn load_workspace(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = FxHashMap::<PackageId, BuildData>::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
}

Ok(res)
}

// FIXME: File a better way to know if it is a dylib
Expand All @@ -173,7 +227,9 @@ fn is_dylib(path: &Path) -> bool {
/// Recreates the compile-time environment variables that Cargo sets.
///
/// Should be synced with <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates>
fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) {
fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) {
let env = &mut build_data.envs;

// FIXME: Missing variables:
// CARGO, CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_<name>

Expand Down
19 changes: 15 additions & 4 deletions crates/project_model/src/cargo_workspace.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use la_arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;

use crate::build_data::{BuildData, BuildDataMap};
use crate::build_data::{packages_to_build_data, BuildData, BuildDataLoader};
use crate::utf8_stdout;

/// `CargoWorkspace` represents the logical structure of, well, a Cargo
Expand Down Expand Up @@ -157,6 +157,7 @@ impl CargoWorkspace {
pub fn from_cargo_metadata(
cargo_toml: &AbsPath,
config: &CargoConfig,
build_data_loader: &mut BuildDataLoader,
progress: &dyn Fn(String),
) -> Result<CargoWorkspace> {
let mut meta = MetadataCommand::new();
Expand Down Expand Up @@ -228,12 +229,14 @@ impl CargoWorkspace {
)
})?;

let resources = if config.load_out_dirs_from_check {
BuildDataMap::new(cargo_toml, config, &meta.packages, progress)?
let build_data_config = if config.load_out_dirs_from_check {
Some(build_data_loader.new_config(cargo_toml, config))
} else {
BuildDataMap::with_cargo_env(&meta.packages)
None
};

let resources = packages_to_build_data(&meta.packages, build_data_config);

let mut pkg_by_id = FxHashMap::default();
let mut packages = Arena::default();
let mut targets = Arena::default();
Expand Down Expand Up @@ -337,4 +340,12 @@ impl CargoWorkspace {
fn is_unique(&self, name: &str) -> bool {
self.packages.iter().filter(|(_, v)| v.name == name).count() == 1
}

pub(crate) fn update_build_data(&mut self, loader: &BuildDataLoader) {
let keys: Vec<Idx<PackageData>> = self.packages.iter().map(|it| it.0).collect();

for id in keys {
self.packages[id].build_data.update(loader);
}
}
}
1 change: 1 addition & 0 deletions crates/project_model/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashSet;

pub use crate::{
build_data::BuildDataLoader,
cargo_workspace::{
CargoConfig, CargoWorkspace, Package, PackageData, PackageDependency, Target, TargetData,
TargetKind,
Expand Down
Loading

0 comments on commit 11dc6c7

Please sign in to comment.