From 3db44311da8b75d137f2904267e67aa608f64775 Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 3 May 2023 23:07:16 +0800 Subject: [PATCH] Add test cases for git sdist generator --- .cirrus.yml | 2 +- .github/workflows/test.yml | 6 +- src/pyproject_toml.rs | 38 ++++++----- src/source_distribution.rs | 3 +- .../{python => }/pyproject.toml | 3 + tests/common/other.rs | 19 +++++- tests/run.rs | 64 ++++++++++++++++++- 7 files changed, 113 insertions(+), 22 deletions(-) rename test-crates/workspace_with_path_dep/{python => }/pyproject.toml (59%) diff --git a/.cirrus.yml b/.cirrus.yml index aa828d251..d1e9a8c7b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -28,7 +28,7 @@ freebsd_task: - echo $CIRRUS_OS - cat Cargo.lock install_script: - - pkg install -y bash python + - pkg install -y bash git python - python3 -m ensurepip <<: *BUILD_AND_TEST diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f296048ff..eefef87c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -287,12 +287,12 @@ jobs: CARGO_TERM_COLOR: always container: alpine:latest steps: - - uses: actions/checkout@v3 - name: Install build requirements run: | set -ex - apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd + apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd git pip3 install cffi virtualenv + - uses: actions/checkout@v3 - name: Cache cargo build uses: Swatinem/rust-cache@v2 with: @@ -485,6 +485,8 @@ jobs: container: pyston/pyston:2.3.5 steps: - uses: actions/checkout@v3 + - name: Fix git permissions + run: git config --global --add safe.directory $GITHUB_WORKSPACE - uses: dtolnay/rust-toolchain@stable id: rustup with: diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs index 5b05f929f..a2afb1c63 100644 --- a/src/pyproject_toml.rs +++ b/src/pyproject_toml.rs @@ -9,10 +9,11 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; /// The `[tool]` section of a pyproject.toml -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(rename_all = "kebab-case")] pub struct Tool { - maturin: Option, + /// maturin options + pub maturin: Option, } #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -116,29 +117,36 @@ pub enum SdistGenerator { } /// The `[tool.maturin]` section of a pyproject.toml -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(rename_all = "kebab-case")] pub struct ToolMaturin { // maturin specific options - // extension module name, accepts setuptools style import name like `foo.bar` - module_name: Option, - include: Option>, - exclude: Option>, - bindings: Option, + /// Module name, accepts setuptools style import name like `foo.bar` + pub module_name: Option, + /// Include files matching the given glob pattern(s) + pub include: Option>, + /// Exclude files matching the given glob pattern(s) + pub exclude: Option>, + /// Bindings type + pub bindings: Option, + /// Platform compatibility #[serde(alias = "manylinux")] - compatibility: Option, + pub compatibility: Option, + /// Skip audit wheel #[serde(default)] - skip_auditwheel: bool, + pub skip_auditwheel: bool, + /// Strip the final binary #[serde(default)] - strip: bool, + pub strip: bool, + /// Source distribution generator #[serde(default)] - sdist_generator: SdistGenerator, + pub sdist_generator: SdistGenerator, /// The directory with python module, contains `/__init__.py` - python_source: Option, + pub python_source: Option, /// Python packages to include - python_packages: Option>, + pub python_packages: Option>, /// Path to the wheel directory, defaults to `.data` - data: Option, + pub data: Option, /// Cargo compile targets pub targets: Option>, /// Target configuration diff --git a/src/source_distribution.rs b/src/source_distribution.rs index a22693ad3..dd93cff79 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -581,8 +581,9 @@ fn add_git_tracked_files_to_sdist( .context("Failed to run `git ls-files -z`")?; if !output.status.success() { bail!( - "Failed to query file list from git: {}\n--- Stdout:\n{}\n--- Stderr:\n{}", + "Failed to query file list from git: {}\n--- Project Path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}", output.status, + pyproject_dir.display(), String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); diff --git a/test-crates/workspace_with_path_dep/python/pyproject.toml b/test-crates/workspace_with_path_dep/pyproject.toml similarity index 59% rename from test-crates/workspace_with_path_dep/python/pyproject.toml rename to test-crates/workspace_with_path_dep/pyproject.toml index 50dd4f373..b79c30ced 100644 --- a/test-crates/workspace_with_path_dep/python/pyproject.toml +++ b/test-crates/workspace_with_path_dep/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["maturin>=0.14,<0.15"] build-backend = "maturin" + +[tool.maturin] +manifest-path = "python/Cargo.toml" diff --git a/tests/common/other.rs b/tests/common/other.rs index 095ea4ec2..e0fca6060 100644 --- a/tests/common/other.rs +++ b/tests/common/other.rs @@ -1,6 +1,7 @@ use anyhow::{Context, Result}; use clap::Parser; use flate2::read::GzDecoder; +use maturin::pyproject_toml::{SdistGenerator, ToolMaturin}; use maturin::{BuildOptions, CargoOptions, PlatformTag}; use pretty_assertions::assert_eq; use std::collections::BTreeSet; @@ -115,6 +116,7 @@ pub fn test_workspace_cargo_lock() -> Result<()> { pub fn test_source_distribution( package: impl AsRef, + sdist_generator: SdistGenerator, expected_files: Vec<&str>, expected_cargo_toml: Option<(&Path, &str)>, unique_name: &str, @@ -135,7 +137,22 @@ pub fn test_source_distribution( ..Default::default() }; - let build_context = build_options.into_build_context(false, false, false)?; + let mut build_context = build_options.into_build_context(false, false, false)?; + + // Override the sdist generator for testing + let mut pyproject_toml = build_context.pyproject_toml.take().unwrap(); + let mut tool = pyproject_toml.tool.clone().unwrap_or_default(); + if let Some(ref mut tool_maturin) = tool.maturin { + tool_maturin.sdist_generator = sdist_generator; + } else { + tool.maturin = Some(ToolMaturin { + sdist_generator, + ..Default::default() + }); + } + pyproject_toml.tool = Some(tool); + build_context.pyproject_toml = Some(pyproject_toml); + let (path, _) = build_context .build_source_distribution()? .context("Failed to build source distribution")?; diff --git a/tests/run.rs b/tests/run.rs index e72948eb7..dea8dedad 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -4,6 +4,7 @@ use common::{ develop, errors, get_python_implementation, handle_result, integration, other, test_python_path, }; use indoc::indoc; +use maturin::pyproject_toml::SdistGenerator; use maturin::Target; use std::env; use std::path::{Path, PathBuf}; @@ -441,6 +442,7 @@ fn workspace_members_non_local_dep_sdist() { ); handle_result(other::test_source_distribution( "test-crates/pyo3-pure", + SdistGenerator::Cargo, vec![ "pyo3_pure-0.1.0+abc123de/Cargo.lock", "pyo3_pure-0.1.0+abc123de/Cargo.toml", @@ -463,6 +465,7 @@ fn workspace_members_non_local_dep_sdist() { fn lib_with_path_dep_sdist() { handle_result(other::test_source_distribution( "test-crates/sdist_with_path_dep", + SdistGenerator::Cargo, vec![ "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml", "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs", @@ -502,6 +505,7 @@ fn lib_with_target_path_dep() { ); handle_result(other::test_source_distribution( "test-crates/sdist_with_target_path_dep", + SdistGenerator::Cargo, vec![ "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml", "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs", @@ -525,6 +529,7 @@ fn lib_with_target_path_dep() { fn pyo3_mixed_src_layout_sdist() { handle_result(other::test_source_distribution( "test-crates/pyo3-mixed-src/rust", + SdistGenerator::Cargo, vec![ "pyo3_mixed_src-2.1.3/pyproject.toml", "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/__init__.py", @@ -545,6 +550,7 @@ fn pyo3_mixed_src_layout_sdist() { fn pyo3_mixed_include_exclude_sdist() { handle_result(other::test_source_distribution( "test-crates/pyo3-mixed-include-exclude", + SdistGenerator::Cargo, vec![ // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded "pyo3_mixed_include_exclude-2.1.3/Cargo.lock", @@ -567,6 +573,33 @@ fn pyo3_mixed_include_exclude_sdist() { )) } +#[test] +fn pyo3_mixed_include_exclude_git_sdist_generator() { + handle_result(other::test_source_distribution( + "test-crates/pyo3-mixed-include-exclude", + SdistGenerator::Git, + vec![ + // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded + "pyo3_mixed_include_exclude-2.1.3/Cargo.lock", + "pyo3_mixed_include_exclude-2.1.3/Cargo.toml", + "pyo3_mixed_include_exclude-2.1.3/PKG-INFO", + "pyo3_mixed_include_exclude-2.1.3/README.md", + "pyo3_mixed_include_exclude-2.1.3/check_installed/check_installed.py", + // "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/exclude_this_file, excluded + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/__init__.py", + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/include_this_file", // included + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/__init__.py", + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/double.py", + "pyo3_mixed_include_exclude-2.1.3/pyproject.toml", + "pyo3_mixed_include_exclude-2.1.3/src/lib.rs", + // "pyo3_mixed_include_exclude-2.1.3/tests/test_pyo3_mixed_include_exclude.py", excluded + "pyo3_mixed_include_exclude-2.1.3/tox.ini", + ], + None, + "sdist-pyo3-mixed-include-exclude-git", + )) +} + #[test] fn pyo3_mixed_include_exclude_wheel_files() { handle_result(other::check_wheel_files( @@ -590,6 +623,7 @@ fn pyo3_mixed_include_exclude_wheel_files() { fn workspace_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace/py", + SdistGenerator::Cargo, vec![ "py-0.1.0/Cargo.lock", "py-0.1.0/Cargo.toml", @@ -606,19 +640,44 @@ fn workspace_sdist() { fn workspace_with_path_dep_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace_with_path_dep/python", + SdistGenerator::Cargo, vec![ "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/Cargo.toml", "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/Cargo.toml", "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/src/lib.rs", + "workspace_with_path_dep-0.1.0/python/Cargo.lock", + "workspace_with_path_dep-0.1.0/python/Cargo.toml", + "workspace_with_path_dep-0.1.0/python/src/lib.rs", + "workspace_with_path_dep-0.1.0/pyproject.toml", + "workspace_with_path_dep-0.1.0/PKG-INFO", + ], + None, + "sdist-workspace-with-path-dep", + )) +} + +#[test] +fn workspace_with_path_dep_git_sdist_generator() { + handle_result(other::test_source_distribution( + "test-crates/workspace_with_path_dep/python", + SdistGenerator::Git, + vec![ "workspace_with_path_dep-0.1.0/Cargo.lock", "workspace_with_path_dep-0.1.0/Cargo.toml", + "workspace_with_path_dep-0.1.0/dont_include_in_sdist/Cargo.toml", + "workspace_with_path_dep-0.1.0/dont_include_in_sdist/src/main.rs", + "workspace_with_path_dep-0.1.0/generic_lib/Cargo.toml", + "workspace_with_path_dep-0.1.0/generic_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/pyproject.toml", - "workspace_with_path_dep-0.1.0/src/lib.rs", + "workspace_with_path_dep-0.1.0/python/Cargo.toml", + "workspace_with_path_dep-0.1.0/python/src/lib.rs", + "workspace_with_path_dep-0.1.0/transitive_lib/Cargo.toml", + "workspace_with_path_dep-0.1.0/transitive_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/PKG-INFO", ], None, - "sdist-workspace-with-path-dep", + "sdist-workspace-with-path-dep-git", )) } @@ -627,6 +686,7 @@ fn workspace_with_path_dep_sdist() { fn workspace_inheritance_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace-inheritance/python", + SdistGenerator::Cargo, vec![ "workspace_inheritance-0.1.0/local_dependencies/generic_lib/Cargo.toml", "workspace_inheritance-0.1.0/local_dependencies/generic_lib/src/lib.rs",