Merge #1380

1380: Detect src-layout for pure Rust projects with multiple Python packages r=messense a=messense

Follow-up of #1378, `python-packages` actually slightly changes the meaning of mixed Rust/Python project layout.

Co-authored-by: messense <[email protected]>

Author: bors[bot]

src/build_context.rs

@@ -926,16 +926,18 @@ impl BuildContext {
             self.excludes(Format::Wheel)?,
         )?;
 
-        if let Some(python_module) = &self.project_layout.python_module {
-            if self.target.is_wasi() {
-                // TODO: Can we have python code and the wasm launchers coexisting
-                // without clashes?
-                bail!("Sorry, adding python code to a wasm binary is currently not supported")
-            }
-            if !self.editable {
-                write_python_part(&mut writer, python_module, self.pyproject_toml.as_ref())
-                    .context("Failed to add the python module to the package")?;
-            }
+        if self.project_layout.python_module.is_some() && self.target.is_wasi() {
+            // TODO: Can we have python code and the wasm launchers coexisting
+            // without clashes?
+            bail!("Sorry, adding python code to a wasm binary is currently not supported")
+        }
+        if !self.editable {
+            write_python_part(
+                &mut writer,
+                &self.project_layout,
+                self.pyproject_toml.as_ref(),
+            )
+            .context("Failed to add the python module to the package")?;
         }
 
         let mut artifacts_ref = Vec::with_capacity(artifacts.len());

src/module_writer.rs

@@ -295,9 +295,9 @@ impl WheelWriter {
         project_layout: &ProjectLayout,
         metadata21: &Metadata21,
     ) -> Result<()> {
-        if let Some(python_module) = &project_layout.python_module {
-            let absolute_path = python_module.normalize()?.into_path_buf();
-            if let Some(python_path) = absolute_path.parent().and_then(|p| p.to_str()) {
+        if project_layout.python_module.is_some() || !project_layout.python_packages.is_empty() {
+            let absolute_path = project_layout.python_dir.normalize()?.into_path_buf();
+            if let Some(python_path) = absolute_path.to_str() {
                 let name = metadata21.get_distribution_escaped();
                 let target = format!("{}.pth", name);
                 debug!("Adding {} from {}", target, python_path);
@@ -720,6 +720,10 @@ pub fn write_bindings_module(
         }
     };
 
+    if !editable {
+        write_python_part(writer, project_layout, pyproject_toml)
+            .context("Failed to add the python module to the package")?;
+    }
     if let Some(python_module) = &project_layout.python_module {
         if editable {
             let target = project_layout.rust_module.join(&so_filename);
@@ -735,9 +739,6 @@ pub fn write_bindings_module(
                 target.display()
             ))?;
         } else {
-            write_python_part(writer, python_module, pyproject_toml)
-                .context("Failed to add the python module to the package")?;
-
             let relative = project_layout
                 .rust_module
                 .strip_prefix(python_module.parent().unwrap())
@@ -789,14 +790,13 @@ pub fn write_cffi_module(
 ) -> Result<()> {
     let cffi_declarations = generate_cffi_declarations(crate_dir, target_dir, python)?;
 
-    let module;
+    if !editable {
+        write_python_part(writer, project_layout, pyproject_toml)
+            .context("Failed to add the python module to the package")?;
+    }
 
+    let module;
     if let Some(python_module) = &project_layout.python_module {
-        if !editable {
-            write_python_part(writer, python_module, pyproject_toml)
-                .context("Failed to add the python module to the package")?;
-        }
-
         if editable {
             let base_path = python_module.join(module_name);
             fs::create_dir_all(&base_path)?;
@@ -956,14 +956,13 @@ pub fn write_uniffi_module(
     } = generate_uniffi_bindings(crate_dir, target_dir, target_os)?;
     let py_init = format!("from .{} import *  # NOQA\n", binding_name);
 
-    let module;
+    if !editable {
+        write_python_part(writer, project_layout, pyproject_toml)
+            .context("Failed to add the python module to the package")?;
+    }
 
+    let module;
     if let Some(python_module) = &project_layout.python_module {
-        if !editable {
-            write_python_part(writer, python_module, pyproject_toml)
-                .context("Failed to add the python module to the package")?;
-        }
-
         if editable {
             let base_path = python_module.join(module_name);
             fs::create_dir_all(&base_path)?;
@@ -1095,28 +1094,26 @@ if __name__ == '__main__':
 /// Adds the python part of a mixed project to the writer,
 pub fn write_python_part(
     writer: &mut impl ModuleWriter,
-    python_module: impl AsRef<Path>,
+    project_layout: &ProjectLayout,
     pyproject_toml: Option<&PyProjectToml>,
 ) -> Result<()> {
-    let python_module = python_module.as_ref().to_path_buf();
-    let python_dir = python_module.parent().unwrap().to_path_buf();
-    let mut python_packages = vec![python_module];
-    if let Some(pyproject_toml) = pyproject_toml {
-        if let Some(packages) = pyproject_toml.python_packages() {
-            for package in packages {
-                let package_path = python_dir.join(package);
-                if python_packages.iter().any(|p| *p == package_path) {
-                    continue;
-                }
-                python_packages.push(package_path);
-            }
+    let python_dir = &project_layout.python_dir;
+    let mut python_packages = Vec::new();
+    if let Some(python_module) = project_layout.python_module.as_ref() {
+        python_packages.push(python_module.to_path_buf());
+    }
+    for package in &project_layout.python_packages {
+        let package_path = python_dir.join(package);
+        if python_packages.iter().any(|p| *p == package_path) {
+            continue;
         }
+        python_packages.push(package_path);
     }
 
     for package in python_packages {
         for absolute in WalkBuilder::new(package).hidden(false).build() {
             let absolute = absolute?.into_path();
-            let relative = absolute.strip_prefix(&python_dir).unwrap();
+            let relative = absolute.strip_prefix(python_dir).unwrap();
             if absolute.is_dir() {
                 writer.add_directory(relative)?;
             } else {
@@ -1137,7 +1134,7 @@ pub fn write_python_part(
     // Include additional files
     if let Some(pyproject) = pyproject_toml {
         // FIXME: in src-layout pyproject.toml isn't located directly in python dir
-        let pyproject_dir = &python_dir;
+        let pyproject_dir = python_dir;
         if let Some(glob_patterns) = pyproject.include() {
             for pattern in glob_patterns
                 .iter()

src/project_layout.rs

@@ -3,6 +3,7 @@ use crate::{CargoToml, Metadata21, PyProjectToml};
 use anyhow::{bail, format_err, Context, Result};
 use cargo_metadata::{Metadata, MetadataCommand};
 use normpath::PathExt as _;
+use std::collections::HashSet;
 use std::env;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -19,6 +20,8 @@ pub struct ProjectLayout {
     /// If none, we have a rust crate compiled into a shared library with only some glue python for cffi
     /// If some, we have a python package that is extended by a native rust module.
     pub python_module: Option<PathBuf>,
+    /// Python packages to include
+    pub python_packages: Vec<String>,
     /// Contains the canonicalized (i.e. absolute) path to the rust part of the project
     pub rust_module: PathBuf,
     /// Rust extension name
@@ -137,19 +140,30 @@ impl ProjectResolver {
         } else {
             manifest_dir
         };
+        let python_packages = pyproject
+            .and_then(|x| x.python_packages())
+            .unwrap_or_default()
+            .to_vec();
         let py_root = match pyproject.and_then(|x| x.python_source()) {
             Some(py_src) => project_root.join(py_src),
             None => match pyproject.and_then(|x| x.project_name()) {
                 Some(project_name) => {
                     // Detect src layout
-                    let import_name = project_name.replace('-', "_");
                     let rust_cargo_toml_found =
                         project_root.join("rust").join("Cargo.toml").is_file();
-                    let python_src_found = project_root
-                        .join("src")
-                        .join(import_name)
-                        .join("__init__.py")
-                        .is_file();
+                    let import_name = project_name.replace('-', "_");
+                    let mut package_init = HashSet::new();
+                    package_init.insert(
+                        project_root
+                            .join("src")
+                            .join(import_name)
+                            .join("__init__.py"),
+                    );
+                    for package in &python_packages {
+                        package_init
+                            .insert(project_root.join("src").join(package).join("__init__.py"));
+                    }
+                    let python_src_found = package_init.iter().any(|x| x.is_file());
                     if rust_cargo_toml_found && python_src_found {
                         project_root.join("src")
                     } else {
@@ -176,7 +190,8 @@ impl ProjectResolver {
                 }
             }),
         };
-        let project_layout = ProjectLayout::determine(project_root, extension_name, py_root, data)?;
+        let project_layout =
+            ProjectLayout::determine(project_root, extension_name, py_root, python_packages, data)?;
         Ok(Self {
             project_layout,
             cargo_toml_path: manifest_file,
@@ -259,12 +274,18 @@ impl ProjectResolver {
                     } else if let Some(project_name) = pyproject.project_name() {
                         // Check if python source directory in `src/<project_name>`
                         let import_name = project_name.replace('-', "_");
-                        if current_dir
-                            .join("src")
-                            .join(import_name)
-                            .join("__init__.py")
-                            .is_file()
-                        {
+                        let mut package_init = HashSet::new();
+                        package_init.insert(
+                            current_dir
+                                .join("src")
+                                .join(import_name)
+                                .join("__init__.py"),
+                        );
+                        for package in pyproject.python_packages().unwrap_or_default() {
+                            package_init
+                                .insert(current_dir.join("src").join(package).join("__init__.py"));
+                        }
+                        if package_init.iter().any(|x| x.is_file()) {
                             return Ok((path, pyproject_file));
                         }
                     }
@@ -322,6 +343,7 @@ impl ProjectLayout {
         project_root: impl AsRef<Path>,
         module_name: &str,
         python_root: PathBuf,
+        python_packages: Vec<String>,
         data: Option<PathBuf>,
     ) -> Result<ProjectLayout> {
         // A dot in the module name means the extension module goes into the module folder specified by the path
@@ -373,6 +395,7 @@ impl ProjectLayout {
 
             Ok(ProjectLayout {
                 python_dir: python_root,
+                python_packages,
                 python_module: Some(python_module),
                 rust_module,
                 extension_name,
@@ -381,6 +404,7 @@ impl ProjectLayout {
         } else {
             Ok(ProjectLayout {
                 python_dir: python_root,
+                python_packages,
                 python_module: None,
                 rust_module: project_root.to_path_buf(),
                 extension_name,

src/source_distribution.rs

@@ -620,42 +620,38 @@ pub fn source_distribution(
 
     let pyproject_dir = pyproject_toml_path.parent().unwrap();
     // Add python source files
+    let mut python_packages = Vec::new();
     if let Some(python_module) = build_context.project_layout.python_module.as_ref() {
-        let mut python_packages = vec![python_module.to_path_buf()];
-        for package in build_context
-            .pyproject_toml
-            .as_ref()
-            .and_then(|toml| toml.python_packages())
-            .unwrap_or_default()
-        {
-            let package_path = build_context.project_layout.python_dir.join(package);
-            if python_packages.iter().any(|p| *p == package_path) {
-                continue;
-            }
-            python_packages.push(package_path);
+        python_packages.push(python_module.to_path_buf());
+    }
+    for package in &build_context.project_layout.python_packages {
+        let package_path = build_context.project_layout.python_dir.join(package);
+        if python_packages.iter().any(|p| *p == package_path) {
+            continue;
         }
+        python_packages.push(package_path);
+    }
 
-        for package in python_packages {
-            for entry in ignore::Walk::new(package) {
-                let source = entry?.into_path();
-                // Technically, `ignore` crate should handle this,
-                // but somehow it doesn't on Alpine Linux running in GitHub Actions,
-                // so we do it manually here.
-                // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
-                if source
-                    .extension()
-                    .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
-                    .unwrap_or_default()
-                {
-                    debug!("Ignoring {}", source.display());
-                    continue;
-                }
-                let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
-                if source.is_dir() {
-                    writer.add_directory(target)?;
-                } else {
-                    writer.add_file(target, &source)?;
-                }
+    for package in python_packages {
+        for entry in ignore::Walk::new(package) {
+            let source = entry?.into_path();
+            // Technically, `ignore` crate should handle this,
+            // but somehow it doesn't on Alpine Linux running in GitHub Actions,
+            // so we do it manually here.
+            // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
+            if source
+                .extension()
+                .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
+                .unwrap_or_default()
+            {
+                debug!("Ignoring {}", source.display());
+                continue;
+            }
+            let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
+            if source.is_dir() {
+                writer.add_directory(target)?;
+            } else {
+                writer.add_file(target, &source)?;
             }
         }
     }