Merge #1378

1378: Add support for packaging multiple pure Python packages r=messense a=messense

Implements #1372 (comment)

Co-authored-by: messense <[email protected]>

Author: bors[bot]

Changelog.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * **Breaking Change**: Remove deprecated `python-source` option in `Cargo.toml` in [#1335](https://github.com/PyO3/maturin/pull/1335)
 * **Breaking Change**: Turn `patchelf` version warning into a hard error in [#1335](https://github.com/PyO3/maturin/pull/1335)
 * **Breaking Change**: [`uniffi_bindgen` CLI](https://mozilla.github.io/uniffi-rs/tutorial/Prerequisites.html#the-uniffi-bindgen-cli-tool) is required for building `uniffi` bindings wheels in [#1352](https://github.com/PyO3/maturin/pull/1352)
+* Add support for packaging multiple pure Python packages in [#1378](https://github.com/PyO3/maturin/pull/1378)
 
 ## [0.14.7] - 2022-12-20
 

guide/src/metadata.md

@@ -117,6 +117,10 @@ bindings = "pyo3"
 compatibility = "manylinux2014"
 # Don't check for manylinux compliance
 skip-auditwheel = false
+# Python source directory
+python-source = "src"
+# Python packages to include
+python-packages = ["foo", "bar"]
 # Strip the library for minimum file size
 strip = true
 # Build artifacts with the specified Cargo profile

src/module_writer.rs

@@ -303,7 +303,7 @@ impl WheelWriter {
                 debug!("Adding {} from {}", target, python_path);
                 self.add_bytes(target, python_path.as_bytes())?;
             } else {
-                println!("⚠️ source code path contains non-Unicode sequences, editable installs may not work.");
+                eprintln!("⚠️ source code path contains non-Unicode sequences, editable installs may not work.");
             }
         }
         Ok(())
@@ -1098,31 +1098,46 @@ pub fn write_python_part(
     python_module: impl AsRef<Path>,
     pyproject_toml: Option<&PyProjectToml>,
 ) -> Result<()> {
-    let python_module = python_module.as_ref();
-    for absolute in WalkBuilder::new(python_module).hidden(false).build() {
-        let absolute = absolute?.into_path();
-        let relative = absolute
-            .strip_prefix(python_module.parent().unwrap())
-            .unwrap();
-        if absolute.is_dir() {
-            writer.add_directory(relative)?;
-        } else {
-            // Ignore native libraries from develop, if any
-            if let Some(extension) = relative.extension() {
-                if extension.to_string_lossy() == "so" {
-                    debug!("Ignoring native library {}", relative.display());
+    let python_module = python_module.as_ref().to_path_buf();
+    let python_dir = python_module.parent().unwrap().to_path_buf();
+    let mut python_packages = vec![python_module];
+    if let Some(pyproject_toml) = pyproject_toml {
+        if let Some(packages) = pyproject_toml.python_packages() {
+            for package in packages {
+                let package_path = python_dir.join(package);
+                if python_packages.iter().any(|p| *p == package_path) {
                     continue;
                 }
+                python_packages.push(package_path);
+            }
+        }
+    }
+
+    for package in python_packages {
+        for absolute in WalkBuilder::new(package).hidden(false).build() {
+            let absolute = absolute?.into_path();
+            let relative = absolute.strip_prefix(&python_dir).unwrap();
+            if absolute.is_dir() {
+                writer.add_directory(relative)?;
+            } else {
+                // Ignore native libraries from develop, if any
+                if let Some(extension) = relative.extension() {
+                    if extension.to_string_lossy() == "so" {
+                        debug!("Ignoring native library {}", relative.display());
+                        continue;
+                    }
+                }
+                writer
+                    .add_file(relative, &absolute)
+                    .context(format!("File to add file from {}", absolute.display()))?;
             }
-            writer
-                .add_file(relative, &absolute)
-                .context(format!("File to add file from {}", absolute.display()))?;
         }
     }
 
     // Include additional files
     if let Some(pyproject) = pyproject_toml {
-        let pyproject_dir = python_module.parent().unwrap();
+        // FIXME: in src-layout pyproject.toml isn't located directly in python dir
+        let pyproject_dir = &python_dir;
         if let Some(glob_patterns) = pyproject.include() {
             for pattern in glob_patterns
                 .iter()

src/project_layout.rs

@@ -13,6 +13,8 @@ const PYPROJECT_TOML: &str = "pyproject.toml";
 /// Whether this project is pure rust or rust mixed with python and whether it has wheel data
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ProjectLayout {
+    /// Contains the absolute path to the python source directory
+    pub python_dir: PathBuf,
     /// Contains the canonicalized (i.e. absolute) path to the python part of the project
     /// If none, we have a rust crate compiled into a shared library with only some glue python for cffi
     /// If some, we have a python package that is extended by a native rust module.
@@ -342,6 +344,7 @@ impl ProjectLayout {
         };
         debug!(
             project_root = %project_root.display(),
+            python_dir = %python_root.display(),
             rust_module = %rust_module.display(),
             python_module = %python_module.display(),
             extension_name = %extension_name,
@@ -369,13 +372,15 @@ impl ProjectLayout {
             println!("🍹 Building a mixed python/rust project");
 
             Ok(ProjectLayout {
+                python_dir: python_root,
                 python_module: Some(python_module),
                 rust_module,
                 extension_name,
                 data,
             })
         } else {
             Ok(ProjectLayout {
+                python_dir: python_root,
                 python_module: None,
                 rust_module: project_root.to_path_buf(),
                 extension_name,

src/pyproject_toml.rs

@@ -97,6 +97,8 @@ pub struct ToolMaturin {
     strip: bool,
     /// The directory with python module, contains `<module_name>/__init__.py`
     python_source: Option<PathBuf>,
+    /// Python packages to include
+    python_packages: Option<Vec<String>>,
     /// Path to the wheel directory, defaults to `<module_name>.data`
     data: Option<PathBuf>,
     // Some customizable cargo options
@@ -208,6 +210,12 @@ impl PyProjectToml {
             .and_then(|maturin| maturin.python_source.as_deref())
     }
 
+    /// Returns the value of `[tool.maturin.python-packages]` in pyproject.toml
+    pub fn python_packages(&self) -> Option<&[String]> {
+        self.maturin()
+            .and_then(|maturin| maturin.python_packages.as_deref())
+    }
+
     /// Returns the value of `[tool.maturin.data]` in pyproject.toml
     pub fn data(&self) -> Option<&Path> {
         self.maturin().and_then(|maturin| maturin.data.as_deref())
@@ -291,6 +299,7 @@ mod tests {
 
             [tool.maturin]
             manylinux = "2010"
+            python-packages = ["foo", "bar"]
             manifest-path = "Cargo.toml"
             profile = "dev"
             features = ["foo", "bar"]
@@ -317,6 +326,10 @@ mod tests {
             maturin.rustc_args,
             Some(vec!["-Z".to_string(), "unstable-options".to_string()])
         );
+        assert_eq!(
+            maturin.python_packages,
+            Some(vec!["foo".to_string(), "bar".to_string()])
+        );
     }
 
     #[test]

src/source_distribution.rs

@@ -620,26 +620,42 @@ pub fn source_distribution(
 
     let pyproject_dir = pyproject_toml_path.parent().unwrap();
     // Add python source files
-    if let Some(python_source) = build_context.project_layout.python_module.as_ref() {
-        for entry in ignore::Walk::new(python_source) {
-            let source = entry?.into_path();
-            // Technically, `ignore` crate should handle this,
-            // but somehow it doesn't on Alpine Linux running in GitHub Actions,
-            // so we do it manually here.
-            // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
-            if source
-                .extension()
-                .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
-                .unwrap_or_default()
-            {
-                debug!("Ignoring {}", source.display());
+    if let Some(python_module) = build_context.project_layout.python_module.as_ref() {
+        let mut python_packages = vec![python_module.to_path_buf()];
+        for package in build_context
+            .pyproject_toml
+            .as_ref()
+            .and_then(|toml| toml.python_packages())
+            .unwrap_or_default()
+        {
+            let package_path = build_context.project_layout.python_dir.join(package);
+            if python_packages.iter().any(|p| *p == package_path) {
                 continue;
             }
-            let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
-            if source.is_dir() {
-                writer.add_directory(target)?;
-            } else {
-                writer.add_file(target, &source)?;
+            python_packages.push(package_path);
+        }
+
+        for package in python_packages {
+            for entry in ignore::Walk::new(package) {
+                let source = entry?.into_path();
+                // Technically, `ignore` crate should handle this,
+                // but somehow it doesn't on Alpine Linux running in GitHub Actions,
+                // so we do it manually here.
+                // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
+                if source
+                    .extension()
+                    .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
+                    .unwrap_or_default()
+                {
+                    debug!("Ignoring {}", source.display());
+                    continue;
+                }
+                let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
+                if source.is_dir() {
+                    writer.add_directory(target)?;
+                } else {
+                    writer.add_file(target, &source)?;
+                }
             }
         }
     }

test-crates/pyo3-mixed-src/pyproject.toml

@@ -12,3 +12,6 @@ requires-python = ">=3.7"
 
 [project.scripts]
 get_42 = "pyo3_mixed_src:get_42"
+
+[tool.maturin]
+python-packages = ["pyo3_mixed_src", "tests"]

tests/run.rs

@@ -491,6 +491,7 @@ fn pyo3_mixed_src_layout_sdist() {
             "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/__init__.py",
             "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/python_module/__init__.py",
             "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/python_module/double.py",
+            "pyo3_mixed_src-2.1.3/src/tests/test_pyo3_mixed.py",
             "pyo3_mixed_src-2.1.3/rust/Cargo.toml",
             "pyo3_mixed_src-2.1.3/rust/Cargo.lock",
             "pyo3_mixed_src-2.1.3/rust/src/lib.rs",