From 4ad5c97458ee0d9ee3895be69d072b6f9d39c85c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 27 Jan 2023 00:22:04 -0800 Subject: [PATCH] Add `pants_from_sources` support. (#77) Closes #30 --- .github/workflows/ci.yml | 23 ++-- BUILD_ROOT | 0 CHANGES.md | 18 +++ Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 7 ++ package/src/main.rs | 247 ++++++++++++++++++++++++++++++++++++--- src/build_root.rs | 8 +- src/main.rs | 59 +++++++++- 9 files changed, 334 insertions(+), 32 deletions(-) create mode 100644 BUILD_ROOT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa666d7..da9f635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,9 @@ jobs: strategy: matrix: os: [ubuntu-22.04, macos-11, macOS-11-ARM64, windows-2022] + env: + PY: python3.9 + SCIE_PANTS_DEV_CACHE: .scie_pants_dev_cache steps: - uses: actions/checkout@v3 - name: Check Formatting @@ -38,6 +41,18 @@ jobs: run: cargo clippy --all - name: Unit Tests run: cargo test --all + - name: Setup Python 3.9 + if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-11' }} + uses: actions/setup-python@v4 + with: + # N.B.: We need Python 3.9 for running Pants goals against our tools.pex Python tools + # codebase as well as running Pants from sources in ITs. + python-version: "3.9" + - name: Cache Build and IT Artifacts + uses: actions/cache@v3 + with: + path: ${{ env.SCIE_PANTS_DEV_CACHE }} + key: ${{ runner.os }}-${{ runner.arch }}-scie-pants-v0 - name: Build, Package & Integration Tests if: ${{ matrix.os == 'macOS-11-ARM64' }} run: | @@ -58,13 +73,6 @@ jobs: run: | PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN=${{ secrets.GITHUB_TOKEN }} \ cargo run -p package -- test - - name: Setup Python 3.9 - if: ${{ matrix.os == 'ubuntu-22.04' }} - uses: actions/setup-python@v4 - with: - # N.B.: We need Python 3.9 for running Pants goals against our tools.pex Python tools - # codebase. - python-version: "3.9" - name: Build, Package & Integration Tests if: ${{ matrix.os == 'ubuntu-22.04' }} run: | @@ -85,4 +93,3 @@ jobs: cargo run -p package -- test \ --tools-pex dist/tools.pex --scie-pants dist/scie-pants-linux-x86_64 \ --tools-pex-mismatch-warn - diff --git a/BUILD_ROOT b/BUILD_ROOT new file mode 100644 index 0000000..e69de29 diff --git a/CHANGES.md b/CHANGES.md index 2260621..82c6e99 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,23 @@ # Release Notes +## 0.3.0 + +This release adds support for running Pants from a local Pants clone. This is useful for testing out +unreleased Pants changes. + +This feature used to be provided by a bespoke `pants_from_sources` script copied around to various +repositories; an example of which is [here]( +https://github.com/pantsbuild/example-python/blob/1b38d08821865e3756024950bc000bdbd0161b95/pants_from_sources). + +There are two ways to activate this mode: +1. Execute `pants` with the `PANTS_SOURCE` environment variable set as the path to the Pants repo + whose Pants code you'd like to run against your repo. +2. Copy, hardlink or symlink your `pants` binary to `pants_from_sources` and execute that. + +The first activation method is new. The second mode follows the bespoke `./pants_from_sources` +conventions and assumes `PANTS_SOURCE=../pants`. You can override that by setting the`PANTS_SOURCE` +env var as in the first activation method. + ## 0.2.2 This release fixes the scie-pants scie to not expose the interpreter used to run a Pants diff --git a/Cargo.lock b/Cargo.lock index f3b5196..37a2bab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -560,7 +560,7 @@ dependencies = [ [[package]] name = "scie-pants" -version = "0.2.2" +version = "0.3.0" dependencies = [ "anyhow", "dirs", diff --git a/Cargo.toml b/Cargo.toml index a7936bc..5c8f4a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "scie-pants" description = "Protects your Pants from the elements." -version = "0.2.2" +version = "0.3.0" edition = "2021" authors = [ "John Sirois ", diff --git a/README.md b/README.md index 856bc19..d8e93d2 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,13 @@ provides the following: If you run `scie-pants` in a directory where Pants is not already set up, it will prompt you, and you can let it set up the latest Pants stable version for your project. ++ Built-in [`pants_from_sources`]( + https://github.com/pantsbuild/example-python/blob/1b38d08821865e3756024950bc000bdbd0161b95/pants_from_sources) + support. You can either execute `scie-pants` with `PANTS_SOURCE` set to the path of a local clone + of the [Pants](https://github.com/pantsbuild/pants) repo or else copy, link or symlink your + `scie-pants` executable to `pants_from_sources` and execute that. In this case `PANTS_SOURCE` will + default to `../pants` just as was the case in the bespoke `./pants_from_sources` scripts. + + Partial support for firewalls: Currently, you can only re-direct the URLs scie-pants uses to fetch [Python Build Standalone]( diff --git a/package/src/main.rs b/package/src/main.rs index c1002eb..5adfade 100644 --- a/package/src/main.rs +++ b/package/src/main.rs @@ -24,7 +24,7 @@ use url::Url; const BINARY: &str = "scie-pants"; const PTEX_TAG: &str = "v0.6.0"; -const SCIE_JUMP_TAG: &str = "v0.8.0"; +const SCIE_JUMP_TAG: &str = "v0.10.0"; const CARGO: &str = env!("CARGO"); const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); @@ -195,10 +195,23 @@ fn _execute_with_input(command: &mut Command, stdin_data: Option<&[u8]>) -> Resu )) })?; if !output.status.success() { - return Err(Code::FAILURE.with_message(format!( + let mut message_lines = vec![format!( "Command {command:?} failed with exit code: {code:?}", code = output.status.code() - ))); + )]; + if output.stdout.is_empty() { + message_lines.push("STDOUT not captured.".to_string()) + } else { + message_lines.push("STDOUT:".to_string()); + message_lines.push(String::from_utf8_lossy(output.stdout.as_slice()).to_string()); + } + if output.stderr.is_empty() { + message_lines.push("STDERR not captured.".to_string()) + } else { + message_lines.push("STDERR:".to_string()); + message_lines.push(String::from_utf8_lossy(output.stderr.as_slice()).to_string()); + } + return Err(Code::FAILURE.with_message(message_lines.join(EOL))); } Ok(output) } @@ -224,6 +237,26 @@ fn hardlink(src: &Path, dst: &Path) -> ExitResult { }) } +fn softlink(src: &Path, dst: &Path) -> ExitResult { + #[cfg(unix)] + use std::os::unix::fs::symlink; + #[cfg(windows)] + use std::os::windows::fs::symlink_file as symlink; + + info!( + "Soft linking {src} -> {dst}", + src = src.display(), + dst = dst.display() + ); + symlink(src, dst).map_err(|e| { + Code::FAILURE.with_message(format!( + "Failed to soft link {src} -> {dst}: {e}", + src = src.display(), + dst = dst.display() + )) + }) +} + fn rename(src: &Path, dst: &Path) -> ExitResult { info!( "Renaming {src} -> {dst}", @@ -256,9 +289,22 @@ fn copy(src: &Path, dst: &Path) -> ExitResult { .map(|_| ()) } +fn remove_dir(path: &Path) -> ExitResult { + if path.exists() { + std::fs::remove_dir_all(path).map_err(|e| { + Code::FAILURE.with_message(format!( + "Failed to remove directory at {path}: {e}", + path = path.display() + )) + }) + } else { + Ok(()) + } +} + fn ensure_directory(path: &Path, clean: bool) -> ExitResult { - if clean && path.exists() { - if let Err(e) = std::fs::remove_dir_all(path) { + if clean { + if let Err(e) = remove_dir(path) { warn!( "Failed to clean directory at {path}: {e}", path = path.display() @@ -280,16 +326,17 @@ fn create_tempdir() -> Result { } fn touch(path: &Path) -> ExitResult { - write_file(path, []) + write_file(path, true, []) } -fn write_file>(path: &Path, content: C) -> ExitResult { +fn write_file>(path: &Path, append: bool, content: C) -> ExitResult { if let Some(parent) = path.parent() { ensure_directory(parent, false)?; } let mut fd = std::fs::OpenOptions::new() .create(true) - .append(true) + .write(true) + .append(append) .open(path) .map_err(|e| { Code::FAILURE.with_message(format!("Failed to open {path}: {e}", path = path.display())) @@ -476,6 +523,30 @@ fn build_a_scie_project(a_scie_project_repo: &Path, target: &str, dest_dir: &Pat .map(|_| ()) } +fn dev_cache_dir() -> Result { + if let Ok(cache_dir) = env::var("SCIE_PANTS_DEV_CACHE") { + let cache_path = PathBuf::from(cache_dir); + ensure_directory(&cache_path, false)?; + return cache_path.canonicalize().map_err(|e| { + Code::FAILURE.with_message(format!( + "Failed to resolve the absolute path of SCIE_PANTS_DEV_CACHE={cache_dir}: {e}", + cache_dir = cache_path.display() + )) + }); + } + + let cache_dir = dirs::cache_dir() + .ok_or_else(|| { + Code::FAILURE.with_message( + "Failed to look up user cache dir for caching scie project downloads".to_string(), + ) + })? + .join("scie-pants") + .join("dev"); + ensure_directory(&cache_dir, false)?; + Ok(cache_dir) +} + fn fetch_a_scie_project( build_context: &BuildContext, project_name: &str, @@ -486,16 +557,7 @@ fn fetch_a_scie_project( static BOOTSTRAP_PTEX: OnceCell = OnceCell::new(); let file_name = binary_full_name(binary_name); - let cache_dir = dirs::cache_dir() - .ok_or_else(|| { - Code::FAILURE.with_message( - "Failed to look up user cache dir for caching scie project downloads".to_string(), - ) - })? - .join("scie-pants") - .join("dev") - .join("downloads") - .join(project_name); + let cache_dir = dev_cache_dir()?.join("downloads").join(project_name); ensure_directory(&cache_dir, false)?; // We proceed with single-checked locking, contention is not a concern in this build process! @@ -888,12 +950,159 @@ fn test( integration_test!( "Verify `.env` loading works (example-django should down grade to Pants 2.12.1)" ); - write_file(&clone_root.path().join(".env"), "PANTS_VERSION=2.12.1")?; + write_file( + &clone_root.path().join(".env"), + false, + "PANTS_VERSION=2.12.1", + )?; execute( Command::new(scie_pants_scie) .arg("-V") .current_dir(clone_root.path().join("example-django")), )?; + + integration_test!("Verify PANTS_SOURCE mode."); + let dev_cache_dir = dev_cache_dir()?; + let clone_dir = dev_cache_dir.join("clones"); + let pants_2_14_1_clone_dir = clone_dir.join("pants-2.14.1"); + let venv_dir = dev_cache_dir.join("venvs"); + let pants_2_14_1_venv_dir = venv_dir.join("pants-2.14.1"); + if !pants_2_14_1_clone_dir.exists() || !pants_2_14_1_venv_dir.exists() { + let clone_root_tmp = create_tempdir()?; + let clone_root_path = clone_root_tmp.path().to_str().ok_or_else(|| { + Code::FAILURE.with_message(format!( + "Failed to convert clone root path to UTF-8 string: {clone_root:?}" + )) + })?; + execute(Command::new("git").args(["init", clone_root_path]))?; + // N.B.: The release_2.14.1 tag has sha cfcb23a97434405a22537e584a0f4f26b4f2993b and we + // must pass a full sha to use the shallow fetch trick. + const PANTS_2_14_1_SHA: &str = "cfcb23a97434405a22537e584a0f4f26b4f2993b"; + execute( + Command::new("git") + .args([ + "fetch", + "--depth", + "1", + "https://github.com/pantsbuild/pants", + PANTS_2_14_1_SHA, + ]) + .current_dir(clone_root_tmp.path()), + )?; + execute( + Command::new("git") + .args(["reset", "--hard", PANTS_2_14_1_SHA]) + .current_dir(clone_root_tmp.path()), + )?; + write_file( + clone_root_tmp.path().join("patch").as_path(), + false, + r#" +diff --git a/build-support/pants_venv b/build-support/pants_venv +index 81e3bd7..4236f4b 100755 +--- a/build-support/pants_venv ++++ b/build-support/pants_venv +@@ -14,11 +14,13 @@ REQUIREMENTS=( + # NB: We house these outside the working copy to avoid needing to gitignore them, but also to + # dodge https://github.com/hashicorp/vagrant/issues/12057. + platform=$(uname -mps | sed 's/ /./g') +-venv_dir_prefix="${HOME}/.cache/pants/pants_dev_deps/${platform}" ++venv_dir_prefix="${PANTS_VENV_DIR_PREFIX:-${HOME}/.cache/pants/pants_dev_deps/${platform}}" ++ ++echo >&2 "The ${SCIE_PANTS_TEST_MODE:-Pants 2.14.1 clone} is working." + + function venv_dir() { + py_venv_version=$(${PY} -c 'import sys; print("".join(map(str, sys.version_info[0:2])))') +- echo "${venv_dir_prefix}.py${py_venv_version}.venv" ++ echo "${venv_dir_prefix}/py${py_venv_version}.venv" + } + + function activate_venv() { +"#, + )?; + execute( + Command::new("git") + .args(["apply", "patch"]) + .current_dir(clone_root_tmp.path()), + )?; + write_file( + clone_root_tmp + .path() + .join("src") + .join("python") + .join("pants") + .join("VERSION") + .as_path(), + false, + "2.14.1+Custom-Local", + )?; + + let venv_root_tmp = create_tempdir()?; + execute( + Command::new("./pants") + .arg("-V") + .env("PANTS_VENV_DIR_PREFIX", venv_root_tmp.path()) + .current_dir(clone_root_tmp.path()), + )?; + + remove_dir( + clone_root_tmp + .path() + .join("src") + .join("rust") + .join("engine") + .join("target") + .as_path(), + )?; + ensure_directory(&clone_dir, true)?; + rename(&clone_root_tmp.into_path(), &pants_2_14_1_clone_dir)?; + ensure_directory(&venv_dir, true)?; + rename(&venv_root_tmp.into_path(), &pants_2_14_1_venv_dir)?; + } + + let test_pants_from_sources = |command: &mut Command, expected_message: &str| { + let result = execute(command.stderr(Stdio::piped()))?; + let stderr = String::from_utf8(result.stderr).map_err(|e| { + Code::FAILURE.with_message(format!("Failed to decode Pants stderr: {e}")) + })?; + assert!( + stderr.contains(expected_message), + "STDERR did not contain {}:\n{}", + expected_message, + stderr + ); + Ok(()) + }; + + test_pants_from_sources( + Command::new(scie_pants_scie) + .arg("-V") + .env("PANTS_SOURCE", &pants_2_14_1_clone_dir) + .env("PANTS_VENV_DIR_PREFIX", &pants_2_14_1_venv_dir) + .env("SCIE_PANTS_TEST_MODE", "PANTS_SOURCE mode"), + "The PANTS_SOURCE mode is working.", + )?; + + integration_test!("Verify pants_from_sources mode."); + let side_by_side_root = create_tempdir()?; + let pants_dir = side_by_side_root.path().join("pants"); + softlink(&pants_2_14_1_clone_dir, &pants_dir)?; + let user_repo_dir = side_by_side_root.path().join("user-repo"); + ensure_directory(&user_repo_dir, true)?; + touch(user_repo_dir.join("pants.toml").as_path())?; + touch(user_repo_dir.join("BUILD_ROOT").as_path())?; + + let pants_from_sources = side_by_side_root.path().join("pants_from_sources"); + softlink(scie_pants_scie, &pants_from_sources)?; + + test_pants_from_sources( + Command::new(pants_from_sources) + .arg("-V") + .env("SCIE_PANTS_TEST_MODE", "pants_from_sources mode") + .env("PANTS_VENV_DIR_PREFIX", &pants_2_14_1_venv_dir) + .current_dir(user_repo_dir), + "The pants_from_sources mode is working.", + )?; } // Max Python supported is 3.8 and only Linux and macOS x86_64 wheels were released. diff --git a/src/build_root.rs b/src/build_root.rs index 52871da..c286d12 100644 --- a/src/build_root.rs +++ b/src/build_root.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0 (see LICENSE). use std::ops::Deref; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use logging_timer::time; @@ -41,3 +41,9 @@ impl Deref for BuildRoot { &self.0 } } + +impl AsRef for BuildRoot { + fn as_ref(&self) -> &Path { + self.0.as_path() + } +} diff --git a/src/main.rs b/src/main.rs index 5459478..eff0036 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,8 @@ // Licensed under the Apache License, Version 2.0 (see LICENSE). use std::env; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; +use std::path::PathBuf; use anyhow::{anyhow, bail, Context, Result}; use build_root::BuildRoot; @@ -183,6 +184,39 @@ fn get_pants_process() -> Result { }) } +fn get_pants_from_sources_process(pants_repo_location: PathBuf) -> Result { + let exe = pants_repo_location.join("pants").into_os_string(); + + let mut args = vec!["--no-verify-config".into()]; + args.extend(env::args().skip(1).map(OsString::from)); + + let version = std::fs::read_to_string( + pants_repo_location + .join("src") + .join("python") + .join("pants") + .join("VERSION"), + )?; + + // The ENABLE_PANTSD env var is a custom env var defined by the legacy `./pants_from_sources` + // script. We maintain support here in perpetuity because it's cheap and we don't break folks' + // workflows. + let enable_pantsd = env::var_os("ENABLE_PANTSD") + .or_else(|| env::var_os("PANTS_PANTSD")) + .unwrap_or_else(|| "false".into()); + + let env = vec![ + ("PANTS_VERSION".into(), version.trim().into()), + ("PANTS_PANTSD".into(), enable_pantsd), + ("no_proxy".into(), "*".into()), + ]; + + let build_root = BuildRoot::find(None)?; + env::set_current_dir(build_root)?; + + Ok(Process { exe, args, env }) +} + trait OrExit { fn or_exit(self) -> T; } @@ -199,6 +233,19 @@ impl OrExit for Result { } } +fn invoked_as_basename() -> Option { + let scie = env::var("SCIE_ARGV0").ok()?; + let exe_path = PathBuf::from(scie); + + #[cfg(windows)] + let basename = exe_path.file_stem().and_then(OsStr::to_str); + + #[cfg(unix)] + let basename = exe_path.file_name().and_then(OsStr::to_str); + + basename.map(str::to_owned) +} + fn main() { env_logger::init(); let _timer = timer!(Level::Debug; "MAIN"); @@ -213,7 +260,15 @@ fn main() { } } - let pants_process = get_pants_process().or_exit(); + let pants_process = if let Ok(value) = env::var("PANTS_SOURCE") { + get_pants_from_sources_process(PathBuf::from(value)) + } else if let Some("pants_from_sources") = invoked_as_basename().as_deref() { + get_pants_from_sources_process(PathBuf::from("..").join("pants")) + } else { + get_pants_process() + } + .or_exit(); + trace!("Launching: {pants_process:#?}"); let exit_code = pants_process.exec().or_exit(); std::process::exit(exit_code)