diff --git a/src/build.rs b/src/build.rs index 34f255631..71ba42347 100644 --- a/src/build.rs +++ b/src/build.rs @@ -3,12 +3,14 @@ use std::collections::HashSet; use std::ffi::OsString; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Write}; use fs_err as fs; use fs_err::File; +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::{io::Read, path::PathBuf}; use itertools::Itertools; use miette::IntoDiagnostic; @@ -17,6 +19,7 @@ use rattler_shell::shell; use crate::env_vars::write_env_script; use crate::metadata::{Directories, Output}; use crate::packaging::{package_conda, record_files}; +use crate::recipe::parser::ScriptContent; use crate::render::resolved_dependencies::{install_environments, resolve_dependencies}; use crate::source::fetch_sources; use crate::test::TestConfiguration; @@ -39,44 +42,80 @@ pub fn get_conda_build_script( ) -> Result { let recipe = &output.recipe; - let default_script = if output.build_configuration.target_platform.is_windows() { - ["build.bat".to_owned()] + let script = recipe.build().script(); + let default_extension = if output.build_configuration.target_platform.is_windows() { + "bat" } else { - ["build.sh".to_owned()] + "sh" }; + let script_content = match script.contents() { + // No script was specified, so we try to read the default script. If the file cannot be + // found we return an empty string. + ScriptContent::Default => { + let recipe_file = directories + .recipe_dir + .join(Path::new("build").with_extension(default_extension)); + match std::fs::read_to_string(recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => String::new(), + Err(e) => { + return Err(e); + } + Ok(content) => content, + } + } - let script = if recipe.build().scripts().is_empty() { - &default_script - } else { - recipe.build().scripts() - }; - - let script = script.iter().join("\n"); - - let script = if script.ends_with(".sh") || script.ends_with(".bat") { - let recipe_file = directories.recipe_dir.join(script); - tracing::info!("Reading recipe file: {:?}", recipe_file); - - if !recipe_file.exists() { - if recipe.build().scripts().is_empty() { - tracing::info!("Empty build script"); - String::new() + // The scripts path was explicitly specified. If the file cannot be found we error out. + ScriptContent::Path(path) => { + let path_with_ext = if path.extension().is_none() { + Cow::Owned(path.with_extension(default_extension)) } else { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("Recipe file {:?} does not exist", recipe_file), - )); + Cow::Borrowed(path.as_path()) + }; + let recipe_file = directories.recipe_dir.join(path_with_ext); + match std::fs::read_to_string(&recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("recipe file {:?} does not exist", recipe_file.display()), + )); + } + Err(e) => { + return Err(e); + } + Ok(content) => content, } - } else { - let mut orig_build_file = File::open(recipe_file)?; - let mut orig_build_file_text = String::new(); - orig_build_file.read_to_string(&mut orig_build_file_text)?; - orig_build_file_text } - } else { - script + // The scripts content was specified but it is still ambiguous whether it is a path or the + // contents of the string. Try to read the file as a script but fall back to using the string + // as the contents itself if the file is missing. + ScriptContent::CommandOrPath(path) => { + let content = + if !path.contains('\n') && (path.ends_with(".bat") || path.ends_with(".sh")) { + let recipe_file = directories.recipe_dir.join(Path::new(path)); + match std::fs::read_to_string(recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(e) => { + return Err(e); + } + Ok(content) => Some(content), + } + } else { + None + }; + match content { + Some(content) => content, + None => path.to_owned(), + } + } + ScriptContent::Commands(commands) => commands.iter().join("\n"), + ScriptContent::Command(command) => command.to_owned(), }; + if script.interpreter().is_some() { + // We don't support an interpreter yet + tracing::error!("build.script.interpreter is not supported yet"); + } + if cfg!(unix) { let build_env_script_path = directories.work_dir.join("build_env.sh"); let preamble = @@ -89,7 +128,7 @@ pub fn get_conda_build_script( format!("Failed to write build env script: {}", e), ) })?; - let full_script = format!("{}\n{}", preamble, script); + let full_script = format!("{}\n{}", preamble, script_content); let build_script_path = directories.work_dir.join("conda_build.sh"); let mut build_script_file = File::create(&build_script_path)?; @@ -110,7 +149,7 @@ pub fn get_conda_build_script( ) })?; - let full_script = format!("{}\n{}", preamble, script); + let full_script = format!("{}\n{}", preamble, script_content); let build_script_path = directories.work_dir.join("conda_build.bat"); let mut build_script_file = File::create(&build_script_path)?; diff --git a/src/env_vars.rs b/src/env_vars.rs index ddb0ba74f..fabc31650 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -377,23 +377,11 @@ pub fn write_env_script( shell_type.set_env_var(&mut s, &k, &v)?; } - for env_key in output.recipe.build().script_env().passthrough() { - let var = std::env::var(env_key); - if let Ok(var) = var { - shell_type.set_env_var(&mut s, env_key, var.as_str())?; - } else { - tracing::warn!( - "Could not find passthrough environment variable: {}", - env_key - ); - } - } - - for (k, v) in output.recipe.build().script_env().env() { + for (k, v) in output.recipe.build().script().env() { shell_type.set_env_var(&mut s, k, v)?; } - if !output.recipe.build().script_env().secrets().is_empty() { + if !output.recipe.build().script().secrets().is_empty() { tracing::error!("Secrets are not supported yet"); } diff --git a/src/recipe/parser.rs b/src/recipe/parser.rs index 504d6d26f..3d4d65924 100644 --- a/src/recipe/parser.rs +++ b/src/recipe/parser.rs @@ -21,15 +21,17 @@ mod build; mod output; mod package; mod requirements; +mod script; mod source; mod test; pub use self::{ about::About, - build::{Build, RunExports, ScriptEnv}, + build::{Build, RunExports}, output::find_outputs_from_src, package::{OutputPackage, Package}, requirements::{Compiler, Dependency, PinSubpackage, Requirements}, + script::{Script, ScriptContent}, source::{Checksum, GitSource, GitUrl, PathSource, Source, UrlSource}, test::{PackageContent, Test}, }; @@ -133,7 +135,7 @@ impl Recipe { ErrorKind::InvalidField("recipe".to_string().into()), help = "The recipe field is only allowed in conjunction with multiple outputs" - )) + )); } "source" => source = value.try_convert(key_str)?, "build" => build = value.try_convert(key_str)?, @@ -146,7 +148,7 @@ impl Recipe { return Err(_partialerror!( *key.span(), ErrorKind::InvalidField(invalid_key.to_string().into()), - )) + )); } } } diff --git a/src/recipe/parser/build.rs b/src/recipe/parser/build.rs index a8c3a237c..1321ed2c1 100644 --- a/src/recipe/parser/build.rs +++ b/src/recipe/parser/build.rs @@ -1,8 +1,10 @@ -use std::{collections::BTreeMap, str::FromStr}; +use std::str::FromStr; use rattler_conda_types::{package::EntryPoint, NoArchKind, NoArchType, PackageName}; use serde::{Deserialize, Serialize}; +use super::Dependency; +use crate::recipe::parser::script::Script; use crate::{ _partialerror, recipe::{ @@ -14,8 +16,6 @@ use crate::{ }, }; -use super::Dependency; - /// The build options contain information about how to build the package and some additional /// metadata about the package. #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -29,9 +29,7 @@ pub struct Build { pub(super) skip: bool, /// The build script can be either a list of commands or a path to a script. By /// default, the build script is set to `build.sh` or `build.bat` on Unix and Windows respectively. - pub(super) script: Vec, - /// Environment variables to pass through or set in the script - pub(super) script_env: ScriptEnv, + pub(super) script: Script, /// A recipe can choose to ignore certain run exports of its dependencies pub(super) ignore_run_exports: Vec, /// A recipe can choose to ignore all run exports of coming from some packages @@ -63,13 +61,8 @@ impl Build { } /// Get the build script. - pub fn scripts(&self) -> &[String] { - self.script.as_slice() - } - - /// Get the build script environment. - pub const fn script_env(&self) -> &ScriptEnv { - &self.script_env + pub fn script(&self) -> &Script { + &self.script } /// Get run exports. @@ -134,7 +127,6 @@ impl TryConvertNode for RenderedMappingNode { build.skip = conds.iter().any(|&v| v); } "script" => build.script = value.try_convert(key_str)?, - "script_env" => build.script_env = value.try_convert(key_str)?, "ignore_run_exports" => { build.ignore_run_exports = value.try_convert(key_str)?; } @@ -162,7 +154,7 @@ impl TryConvertNode for RenderedMappingNode { return Err(_partialerror!( *key.span(), ErrorKind::InvalidField(invalid.to_string().into()), - )) + )); } } } @@ -171,102 +163,6 @@ impl TryConvertNode for RenderedMappingNode { } } -/// Extra environment variables to set during the build script execution -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ScriptEnv { - /// Environments variables to leak into the build environment from the host system. - /// During build time these variables are recorded and stored in the package output. - /// Use `secrets` for environment variables that should not be recorded. - pub(super) passthrough: Vec, - /// Environment variables to set in the build environment. - pub(super) env: BTreeMap, - /// Environment variables to leak into the build environment from the host system that - /// contain sensitve information. Use with care because this might make recipes no - /// longer reproducible on other machines. - pub(super) secrets: Vec, -} - -impl ScriptEnv { - /// Check if the script environment is empty is all its fields. - pub fn is_empty(&self) -> bool { - self.passthrough.is_empty() && self.env.is_empty() && self.secrets.is_empty() - } - - /// Get the passthrough environment variables. - /// - /// Those are the environments variables to leak into the build environment from the host system. - /// - /// During build time these variables are recorded and stored in the package output. - /// Use `secrets` for environment variables that should not be recorded. - pub fn passthrough(&self) -> &[String] { - self.passthrough.as_slice() - } - - /// Get the environment variables to set in the build environment. - pub fn env(&self) -> &BTreeMap { - &self.env - } - - /// Get the secrets environment variables. - /// - /// Environment variables to leak into the build environment from the host system that - /// contain sensitve information. - /// - /// # Warning - /// Use with care because this might make recipes no longer reproducible on other machines. - pub fn secrets(&self) -> &[String] { - self.secrets.as_slice() - } -} - -impl TryConvertNode for RenderedNode { - fn try_convert(&self, name: &str) -> Result { - self.as_mapping() - .ok_or_else(|| _partialerror!(*self.span(), ErrorKind::ExpectedMapping)) - .and_then(|m| m.try_convert(name)) - } -} - -impl TryConvertNode for RenderedMappingNode { - fn try_convert(&self, name: &str) -> Result { - let invalid = self - .keys() - .find(|k| matches!(k.as_str(), "env" | "passthrough" | "secrets")); - - if let Some(invalid) = invalid { - return Err(_partialerror!( - *invalid.span(), - ErrorKind::InvalidField(invalid.to_string().into()), - help = format!("valid keys for {name} are `env`, `passthrough` or `secrets`") - )); - } - - let env = self - .get("env") - .map(|node| node.try_convert("env")) - .transpose()? - .unwrap_or_default(); - - let passthrough = self - .get("passthrough") - .map(|node| node.try_convert("passthrough")) - .transpose()? - .unwrap_or_default(); - - let secrets = self - .get("secrets") - .map(|node| node.try_convert("secrets")) - .transpose()? - .unwrap_or_default(); - - Ok(ScriptEnv { - passthrough, - env, - secrets, - }) - } -} - /// Run exports are applied to downstream packages that depend on this package. #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct RunExports { @@ -394,7 +290,7 @@ impl TryConvertNode for RenderedMappingNode { *key.span(), ErrorKind::InvalidField(invalid.to_owned().into()), help = format!("fields for {name} should be one of: `weak`, `strong`, `noarch`, `strong_constrains`, or `weak_constrains`") - )) + )); } } } @@ -422,7 +318,7 @@ impl TryConvertNode for RenderedScalarNode { *self.span(), ErrorKind::InvalidField(invalid.to_owned().into()), help = format!("expected `python` or `generic` for {name}"), - )) + )); } }; Ok(noarch) diff --git a/src/recipe/parser/script.rs b/src/recipe/parser/script.rs new file mode 100644 index 000000000..92b749ebd --- /dev/null +++ b/src/recipe/parser/script.rs @@ -0,0 +1,306 @@ +use crate::{ + _partialerror, + recipe::custom_yaml::{ + HasSpan, RenderedMappingNode, RenderedNode, RenderedScalarNode, RenderedSequenceNode, + TryConvertNode, + }, + recipe::error::{ErrorKind, PartialParsingError}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::{borrow::Cow, collections::BTreeMap, path::PathBuf}; + +/// Defines the script to run to build the package. +#[derive(Debug, Default, Clone)] +pub struct Script { + /// The interpreter to use for the script. + pub(super) interpreter: Option, + /// Environment variables to set in the build environment. + pub(super) env: BTreeMap, + /// Environment variables to leak into the build environment from the host system that + /// contain sensitve information. Use with care because this might make recipes no + /// longer reproducible on other machines. + pub(super) secrets: Vec, + /// The contents of the script, either a path or a list of commands. + pub(super) content: ScriptContent, +} + +impl Serialize for Script { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + #[serde(untagged)] + enum RawScriptContent<'a> { + Command { content: &'a String }, + Commands { content: &'a Vec }, + Path { file: &'a PathBuf }, + } + + #[derive(Serialize)] + #[serde(untagged)] + enum RawScript<'a> { + CommandOrPath(&'a String), + Commands(&'a Vec), + Object { + #[serde(skip_serializing_if = "Option::is_none")] + interpreter: Option<&'a String>, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + env: &'a BTreeMap, + #[serde(skip_serializing_if = "Vec::is_empty")] + secrets: &'a Vec, + #[serde(skip_serializing_if = "Option::is_none", flatten)] + content: Option>, + }, + } + + let raw_script = match &self.content { + ScriptContent::CommandOrPath(content) => RawScript::CommandOrPath(content), + ScriptContent::Commands(content) + if self.interpreter.is_none() && self.env.is_empty() && self.secrets.is_empty() => + { + RawScript::Commands(content) + } + _ => RawScript::Object { + interpreter: self.interpreter.as_ref(), + env: &self.env, + secrets: &self.secrets, + content: match &self.content { + ScriptContent::Command(content) => Some(RawScriptContent::Command { content }), + ScriptContent::Commands(content) => { + Some(RawScriptContent::Commands { content }) + } + ScriptContent::Path(file) => Some(RawScriptContent::Path { file }), + ScriptContent::Default => None, + ScriptContent::CommandOrPath(_) => unreachable!(), + }, + }, + }; + + raw_script.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Script { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum RawScriptContent { + Command { content: String }, + Commands { content: Vec }, + Path { file: PathBuf }, + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum RawScript { + CommandOrPath(String), + Commands(Vec), + Object { + #[serde(default)] + interpreter: Option, + #[serde(default)] + env: BTreeMap, + #[serde(default)] + secrets: Vec, + content: Option, + }, + } + + let raw_script = RawScript::deserialize(deserializer)?; + Ok(match raw_script { + RawScript::CommandOrPath(str) => ScriptContent::CommandOrPath(str).into(), + RawScript::Commands(commands) => ScriptContent::Commands(commands).into(), + RawScript::Object { + interpreter, + env, + secrets, + content, + } => Self { + interpreter, + env, + secrets, + content: match content { + Some(RawScriptContent::Command { content }) => ScriptContent::Command(content), + Some(RawScriptContent::Commands { content }) => { + ScriptContent::Commands(content) + } + Some(RawScriptContent::Path { file }) => ScriptContent::Path(file), + None => ScriptContent::Default, + }, + }, + }) + } +} + +impl Script { + /// Returns the interpreter to use to execute the script + pub fn interpreter(&self) -> Option<&str> { + self.interpreter.as_deref() + } + + /// Returns the script contents + pub fn contents(&self) -> &ScriptContent { + &self.content + } + + /// Get the environment variables to set in the build environment. + pub fn env(&self) -> &BTreeMap { + &self.env + } + + /// Get the secrets environment variables. + /// + /// Environment variables to leak into the build environment from the host system that + /// contain sensitve information. + /// + /// # Warning + /// Use with care because this might make recipes no longer reproducible on other machines. + pub fn secrets(&self) -> &[String] { + self.secrets.as_slice() + } +} + +impl From for Script { + fn from(value: ScriptContent) -> Self { + Self { + interpreter: None, + env: Default::default(), + secrets: Default::default(), + content: value, + } + } +} + +impl TryConvertNode