Add test cases for git sdist generator

.cirrus.yml

@@ -28,7 +28,7 @@ freebsd_task:
       - echo $CIRRUS_OS
       - cat Cargo.lock
   install_script:
-    - pkg install -y bash python
+    - pkg install -y bash git python
     - python3 -m ensurepip
   <<: *BUILD_AND_TEST
 

.github/workflows/test.yml

@@ -287,12 +287,12 @@ jobs:
       CARGO_TERM_COLOR: always
     container: alpine:latest
     steps:
-      - uses: actions/checkout@v3
       - name: Install build requirements
         run: |
           set -ex
-          apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd
+          apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd git
           pip3 install cffi virtualenv
+      - uses: actions/checkout@v3
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
         with:
@@ -485,6 +485,8 @@ jobs:
     container: pyston/pyston:2.3.5
     steps:
       - uses: actions/checkout@v3
+      - name: Fix git permissions
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
       - uses: dtolnay/rust-toolchain@stable
         id: rustup
         with:

src/pyproject_toml.rs

@@ -9,10 +9,11 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 /// The `[tool]` section of a pyproject.toml
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
 #[serde(rename_all = "kebab-case")]
 pub struct Tool {
-    maturin: Option<ToolMaturin>,
+    /// maturin options
+    pub maturin: Option<ToolMaturin>,
 }
 
 #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
@@ -116,29 +117,36 @@ pub enum SdistGenerator {
 }
 
 /// The `[tool.maturin]` section of a pyproject.toml
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
 #[serde(rename_all = "kebab-case")]
 pub struct ToolMaturin {
     // maturin specific options
-    // extension module name, accepts setuptools style import name like `foo.bar`
-    module_name: Option<String>,
-    include: Option<Vec<GlobPattern>>,
-    exclude: Option<Vec<GlobPattern>>,
-    bindings: Option<String>,
+    /// Module name, accepts setuptools style import name like `foo.bar`
+    pub module_name: Option<String>,
+    /// Include files matching the given glob pattern(s)
+    pub include: Option<Vec<GlobPattern>>,
+    /// Exclude files matching the given glob pattern(s)
+    pub exclude: Option<Vec<GlobPattern>>,
+    /// Bindings type
+    pub bindings: Option<String>,
+    /// Platform compatibility
     #[serde(alias = "manylinux")]
-    compatibility: Option<PlatformTag>,
+    pub compatibility: Option<PlatformTag>,
+    /// Skip audit wheel
     #[serde(default)]
-    skip_auditwheel: bool,
+    pub skip_auditwheel: bool,
+    /// Strip the final binary
     #[serde(default)]
-    strip: bool,
+    pub strip: bool,
+    /// Source distribution generator
     #[serde(default)]
-    sdist_generator: SdistGenerator,
+    pub sdist_generator: SdistGenerator,
     /// The directory with python module, contains `<module_name>/__init__.py`
-    python_source: Option<PathBuf>,
+    pub python_source: Option<PathBuf>,
     /// Python packages to include
-    python_packages: Option<Vec<String>>,
+    pub python_packages: Option<Vec<String>>,
     /// Path to the wheel directory, defaults to `<module_name>.data`
-    data: Option<PathBuf>,
+    pub data: Option<PathBuf>,
     /// Cargo compile targets
     pub targets: Option<Vec<CargoTarget>>,
     /// Target configuration

src/source_distribution.rs

@@ -581,8 +581,9 @@ fn add_git_tracked_files_to_sdist(
         .context("Failed to run `git ls-files -z`")?;
     if !output.status.success() {
         bail!(
-            "Failed to query file list from git: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
+            "Failed to query file list from git: {}\n--- Project Path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
             output.status,
+            pyproject_dir.display(),
             String::from_utf8_lossy(&output.stdout),
             String::from_utf8_lossy(&output.stderr),
         );

test-crates/workspace_with_path_dep/python/pyproject.toml → test-crates/workspace_with_path_dep/pyproject.toml

@@ -1,3 +1,6 @@
 [build-system]
 requires = ["maturin>=0.14,<0.15"]
 build-backend = "maturin"
+
+[tool.maturin]
+manifest-path = "python/Cargo.toml"

tests/common/other.rs

@@ -1,6 +1,7 @@
 use anyhow::{Context, Result};
 use clap::Parser;
 use flate2::read::GzDecoder;
+use maturin::pyproject_toml::{SdistGenerator, ToolMaturin};
 use maturin::{BuildOptions, CargoOptions, PlatformTag};
 use pretty_assertions::assert_eq;
 use std::collections::BTreeSet;
@@ -115,6 +116,7 @@ pub fn test_workspace_cargo_lock() -> Result<()> {
 
 pub fn test_source_distribution(
     package: impl AsRef<Path>,
+    sdist_generator: SdistGenerator,
     expected_files: Vec<&str>,
     expected_cargo_toml: Option<(&Path, &str)>,
     unique_name: &str,
@@ -135,7 +137,22 @@ pub fn test_source_distribution(
         ..Default::default()
     };
 
-    let build_context = build_options.into_build_context(false, false, false)?;
+    let mut build_context = build_options.into_build_context(false, false, false)?;
+
+    // Override the sdist generator for testing
+    let mut pyproject_toml = build_context.pyproject_toml.take().unwrap();
+    let mut tool = pyproject_toml.tool.clone().unwrap_or_default();
+    if let Some(ref mut tool_maturin) = tool.maturin {
+        tool_maturin.sdist_generator = sdist_generator;
+    } else {
+        tool.maturin = Some(ToolMaturin {
+            sdist_generator,
+            ..Default::default()
+        });
+    }
+    pyproject_toml.tool = Some(tool);
+    build_context.pyproject_toml = Some(pyproject_toml);
+
     let (path, _) = build_context
         .build_source_distribution()?
         .context("Failed to build source distribution")?;

tests/run.rs

@@ -4,6 +4,7 @@ use common::{
     develop, errors, get_python_implementation, handle_result, integration, other, test_python_path,
 };
 use indoc::indoc;
+use maturin::pyproject_toml::SdistGenerator;
 use maturin::Target;
 use std::env;
 use std::path::{Path, PathBuf};
@@ -441,6 +442,7 @@ fn workspace_members_non_local_dep_sdist() {
     );
     handle_result(other::test_source_distribution(
         "test-crates/pyo3-pure",
+        SdistGenerator::Cargo,
         vec![
             "pyo3_pure-0.1.0+abc123de/Cargo.lock",
             "pyo3_pure-0.1.0+abc123de/Cargo.toml",
@@ -463,6 +465,7 @@ fn workspace_members_non_local_dep_sdist() {
 fn lib_with_path_dep_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/sdist_with_path_dep",
+        SdistGenerator::Cargo,
         vec![
             "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml",
             "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs",
@@ -502,6 +505,7 @@ fn lib_with_target_path_dep() {
     );
     handle_result(other::test_source_distribution(
         "test-crates/sdist_with_target_path_dep",
+        SdistGenerator::Cargo,
         vec![
             "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml",
             "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs",
@@ -525,6 +529,7 @@ fn lib_with_target_path_dep() {
 fn pyo3_mixed_src_layout_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/pyo3-mixed-src/rust",
+        SdistGenerator::Cargo,
         vec![
             "pyo3_mixed_src-2.1.3/pyproject.toml",
             "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/__init__.py",
@@ -545,6 +550,7 @@ fn pyo3_mixed_src_layout_sdist() {
 fn pyo3_mixed_include_exclude_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/pyo3-mixed-include-exclude",
+        SdistGenerator::Cargo,
         vec![
             // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded
             "pyo3_mixed_include_exclude-2.1.3/Cargo.lock",
@@ -567,6 +573,33 @@ fn pyo3_mixed_include_exclude_sdist() {
     ))
 }
 
+#[test]
+fn pyo3_mixed_include_exclude_git_sdist_generator() {
+    handle_result(other::test_source_distribution(
+        "test-crates/pyo3-mixed-include-exclude",
+        SdistGenerator::Git,
+        vec![
+            // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded
+            "pyo3_mixed_include_exclude-2.1.3/Cargo.lock",
+            "pyo3_mixed_include_exclude-2.1.3/Cargo.toml",
+            "pyo3_mixed_include_exclude-2.1.3/PKG-INFO",
+            "pyo3_mixed_include_exclude-2.1.3/README.md",
+            "pyo3_mixed_include_exclude-2.1.3/check_installed/check_installed.py",
+            // "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/exclude_this_file, excluded
+            "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/__init__.py",
+            "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/include_this_file", // included
+            "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/__init__.py",
+            "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/double.py",
+            "pyo3_mixed_include_exclude-2.1.3/pyproject.toml",
+            "pyo3_mixed_include_exclude-2.1.3/src/lib.rs",
+            // "pyo3_mixed_include_exclude-2.1.3/tests/test_pyo3_mixed_include_exclude.py", excluded
+            "pyo3_mixed_include_exclude-2.1.3/tox.ini",
+        ],
+        None,
+        "sdist-pyo3-mixed-include-exclude-git",
+    ))
+}
+
 #[test]
 fn pyo3_mixed_include_exclude_wheel_files() {
     handle_result(other::check_wheel_files(
@@ -590,6 +623,7 @@ fn pyo3_mixed_include_exclude_wheel_files() {
 fn workspace_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/workspace/py",
+        SdistGenerator::Cargo,
         vec![
             "py-0.1.0/Cargo.lock",
             "py-0.1.0/Cargo.toml",
@@ -606,19 +640,44 @@ fn workspace_sdist() {
 fn workspace_with_path_dep_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/workspace_with_path_dep/python",
+        SdistGenerator::Cargo,
         vec![
             "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/Cargo.toml",
             "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/src/lib.rs",
             "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/Cargo.toml",
             "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/src/lib.rs",
+            "workspace_with_path_dep-0.1.0/python/Cargo.lock",
+            "workspace_with_path_dep-0.1.0/python/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/python/src/lib.rs",
+            "workspace_with_path_dep-0.1.0/pyproject.toml",
+            "workspace_with_path_dep-0.1.0/PKG-INFO",
+        ],
+        None,
+        "sdist-workspace-with-path-dep",
+    ))
+}
+
+#[test]
+fn workspace_with_path_dep_git_sdist_generator() {
+    handle_result(other::test_source_distribution(
+        "test-crates/workspace_with_path_dep/python",
+        SdistGenerator::Git,
+        vec![
             "workspace_with_path_dep-0.1.0/Cargo.lock",
             "workspace_with_path_dep-0.1.0/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/dont_include_in_sdist/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/dont_include_in_sdist/src/main.rs",
+            "workspace_with_path_dep-0.1.0/generic_lib/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/generic_lib/src/lib.rs",
             "workspace_with_path_dep-0.1.0/pyproject.toml",
-            "workspace_with_path_dep-0.1.0/src/lib.rs",
+            "workspace_with_path_dep-0.1.0/python/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/python/src/lib.rs",
+            "workspace_with_path_dep-0.1.0/transitive_lib/Cargo.toml",
+            "workspace_with_path_dep-0.1.0/transitive_lib/src/lib.rs",
             "workspace_with_path_dep-0.1.0/PKG-INFO",
         ],
         None,
-        "sdist-workspace-with-path-dep",
+        "sdist-workspace-with-path-dep-git",
     ))
 }
 
@@ -627,6 +686,7 @@ fn workspace_with_path_dep_sdist() {
 fn workspace_inheritance_sdist() {
     handle_result(other::test_source_distribution(
         "test-crates/workspace-inheritance/python",
+        SdistGenerator::Cargo,
         vec![
             "workspace_inheritance-0.1.0/local_dependencies/generic_lib/Cargo.toml",
             "workspace_inheritance-0.1.0/local_dependencies/generic_lib/src/lib.rs",