From 1bd34837f017ff232d1d0d040aa7e52e035116bb Mon Sep 17 00:00:00 2001 From: Sebastian Puetz Date: Wed, 29 Jul 2020 12:03:17 +0200 Subject: [PATCH] Add path option for Python source. --- Changelog.md | 3 ++ Readme.md | 2 + maturin/__init__.py | 1 + src/build_context.rs | 15 +++++++- src/build_options.rs | 6 ++- src/develop.rs | 5 ++- src/main.rs | 5 +++ test-crates/pyo3-mixed/pyproject.toml | 3 ++ .../{ => python}/pyo3_mixed/__init__.py | 0 .../pyo3_mixed/python_module/__init__.py | 0 .../pyo3_mixed/python_module/double.py | 0 test-crates/pyo3-mixed/tox.ini | 2 +- tests/test_develop.rs | 21 +++++++--- tests/test_integration.rs | 38 +++++++++++++++---- 14 files changed, 82 insertions(+), 19 deletions(-) rename test-crates/pyo3-mixed/{ => python}/pyo3_mixed/__init__.py (100%) rename test-crates/pyo3-mixed/{ => python}/pyo3_mixed/python_module/__init__.py (100%) rename test-crates/pyo3-mixed/{ => python}/pyo3_mixed/python_module/double.py (100%) diff --git a/Changelog.md b/Changelog.md index e8ac64fc8..604a45079 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 include arbitrary files in source distributions ([#296](https://github.com/PyO3/maturin/pull/296)). * cffi is installed if it's missing and python is running inside a virtualenv. * Add support for PyO3 `0.12`'s `PYO3_PYTHON` environment variable. [#331](https://github.com/PyO3/maturin/pull/331) +* Enable mixed layouts with Python source in different sub-directories. Either through `tools.maturin.py-src` + in `pyproject.toml` or through the `--py-src` flag on `maturin`. + ([#335](https://github.com/PyO3/maturin/pull/335)) ## 0.8.0 - 2020-04-03 diff --git a/Readme.md b/Readme.md index 5e5c6e86b..70922460f 100644 --- a/Readme.md +++ b/Readme.md @@ -91,6 +91,8 @@ my-project    └── lib.rs ``` +You can also specify the directory housing the package through the `--py-src` argument. + maturin will add the native extension as a module in your python folder. When using develop, maturin will copy the native library and for cffi also the glue code to your python folder. You should add those files to your gitignore. With cffi you can do `from .my_project import lib` and then use `lib.my_native_function`, with pyo3/rust-cpython you can directly `from .my_project import my_native_function`. diff --git a/maturin/__init__.py b/maturin/__init__.py index a9ab46f28..b3cd85995 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -30,6 +30,7 @@ "rustc-extra-args", "skip-auditwheel", "strip", + "py-src" ] diff --git a/src/build_context.rs b/src/build_context.rs index 86388b05f..16fc1af36 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -59,8 +59,19 @@ pub enum ProjectLayout { impl ProjectLayout { /// Checks whether a python module exists besides Cargo.toml with the right name - pub fn determine(project_root: impl AsRef, module_name: &str) -> Result { - let python_package_dir = project_root.as_ref().join(module_name); + pub fn determine( + project_root: impl AsRef, + module_name: &str, + py_src: Option>, + ) -> Result { + let python_package_dir = if let Some(py_src) = py_src { + project_root + .as_ref() + .join(py_src.as_ref()) + .join(module_name) + } else { + project_root.as_ref().join(module_name) + }; if python_package_dir.is_dir() { if !python_package_dir.join("__init__.py").is_file() { bail!("Found a directory with the module name ({}) next to Cargo.toml, which indicates a mixed python/rust project, but the directory didn't contain an __init__.py file.", module_name) diff --git a/src/build_options.rs b/src/build_options.rs index 5009cb1ec..272f9b11d 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -55,6 +55,9 @@ pub struct BuildOptions { /// directory in the project's target directory #[structopt(short, long, parse(from_os_str))] pub out: Option, + /// Path to the Python source in mixed projects. + #[structopt(long = "py-src", parse(from_os_str))] + pub py_src: Option, /// [deprecated, use --manylinux instead] Don't check for manylinux compliance #[structopt(long = "skip-auditwheel")] pub skip_auditwheel: bool, @@ -80,6 +83,7 @@ impl Default for BuildOptions { interpreter: Some(vec![]), bindings: None, manifest_path: PathBuf::from("Cargo.toml"), + py_src: None, out: None, skip_auditwheel: false, target: None, @@ -120,7 +124,7 @@ impl BuildOptions { .unwrap_or_else(|| &cargo_toml.package.name) .to_owned(); - let project_layout = ProjectLayout::determine(manifest_dir, &module_name)?; + let project_layout = ProjectLayout::determine(manifest_dir, &module_name, self.py_src)?; let target = Target::from_target_triple(self.target.clone())?; diff --git a/src/develop.rs b/src/develop.rs index 75bbd008a..17b0de14d 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -7,11 +7,12 @@ use crate::PythonInterpreter; use crate::Target; use anyhow::{anyhow, format_err, Context, Result}; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; /// Installs a crate by compiling it and copying the shared library to the right directory /// /// Works only in a virtualenv. +#[allow(clippy::too_many_arguments)] pub fn develop( bindings: Option, manifest_file: &Path, @@ -20,6 +21,7 @@ pub fn develop( venv_dir: &Path, release: bool, strip: bool, + py_src: Option, ) -> Result<()> { let target = Target::from_target_triple(None)?; @@ -32,6 +34,7 @@ pub fn develop( manifest_path: manifest_file.to_path_buf(), out: None, skip_auditwheel: false, + py_src, target: None, cargo_extra_args, rustc_extra_args, diff --git a/src/main.rs b/src/main.rs index 72e38bf0a..0589a8db7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -175,6 +175,9 @@ enum Opt { /// Use as `--rustc-extra-args="--my-arg"` #[structopt(long = "rustc-extra-args")] rustc_extra_args: Vec, + /// Path to the Python source in mixed projects. + #[structopt(long = "py-src", parse(from_os_str))] + py_src: Option, }, /// Build only a source distribution (sdist) without compiling. /// @@ -438,6 +441,7 @@ fn run() -> Result<()> { rustc_extra_args, release, strip, + py_src, } => { let venv_dir = match (env::var_os("VIRTUAL_ENV"),env::var_os("CONDA_PREFIX")) { (Some(dir), None) => PathBuf::from(dir), @@ -456,6 +460,7 @@ fn run() -> Result<()> { &venv_dir, release, strip, + py_src, )?; } Opt::SDist { manifest_path, out } => { diff --git a/test-crates/pyo3-mixed/pyproject.toml b/test-crates/pyo3-mixed/pyproject.toml index 90cb176e8..04a319812 100644 --- a/test-crates/pyo3-mixed/pyproject.toml +++ b/test-crates/pyo3-mixed/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["maturin"] build-backend = "maturin" + +[tool.maturin] +py-src = "py-src" \ No newline at end of file diff --git a/test-crates/pyo3-mixed/pyo3_mixed/__init__.py b/test-crates/pyo3-mixed/python/pyo3_mixed/__init__.py similarity index 100% rename from test-crates/pyo3-mixed/pyo3_mixed/__init__.py rename to test-crates/pyo3-mixed/python/pyo3_mixed/__init__.py diff --git a/test-crates/pyo3-mixed/pyo3_mixed/python_module/__init__.py b/test-crates/pyo3-mixed/python/pyo3_mixed/python_module/__init__.py similarity index 100% rename from test-crates/pyo3-mixed/pyo3_mixed/python_module/__init__.py rename to test-crates/pyo3-mixed/python/pyo3_mixed/python_module/__init__.py diff --git a/test-crates/pyo3-mixed/pyo3_mixed/python_module/double.py b/test-crates/pyo3-mixed/python/pyo3_mixed/python_module/double.py similarity index 100% rename from test-crates/pyo3-mixed/pyo3_mixed/python_module/double.py rename to test-crates/pyo3-mixed/python/pyo3_mixed/python_module/double.py diff --git a/test-crates/pyo3-mixed/tox.ini b/test-crates/pyo3-mixed/tox.ini index 0d4da57ac..f5255d1bf 100644 --- a/test-crates/pyo3-mixed/tox.ini +++ b/test-crates/pyo3-mixed/tox.ini @@ -6,5 +6,5 @@ skipsdist = true whitelist_externals = cargo deps = pytest commands = - cargo run --manifest-path ../Cargo.toml -- develop # You'll want to use `maturin develop` here + cargo run --manifest-path ../../Cargo.toml -- develop --py-src=python # You'll want to use `maturin develop` here pytest diff --git a/tests/test_develop.rs b/tests/test_develop.rs index 8933fae4e..a6839bd15 100644 --- a/tests/test_develop.rs +++ b/tests/test_develop.rs @@ -10,32 +10,40 @@ mod common; #[test] fn test_develop_pyo3_pure() { - handle_result(test_develop("test-crates/pyo3-pure", None)); + handle_result(test_develop("test-crates/pyo3-pure", None, None)); } #[test] fn test_develop_pyo3_mixed() { - handle_result(test_develop("test-crates/pyo3-mixed", None)); + handle_result(test_develop( + "test-crates/pyo3-mixed", + None, + Some("python".into()), + )); } #[test] fn test_develop_cffi_pure() { - handle_result(test_develop("test-crates/cffi-pure", None)); + handle_result(test_develop("test-crates/cffi-pure", None, None)); } #[test] fn test_develop_cffi_mixed() { - handle_result(test_develop("test-crates/cffi-mixed", None)); + handle_result(test_develop("test-crates/cffi-mixed", None, None)); } #[test] fn test_develop_hello_world() { - handle_result(test_develop("test-crates/hello-world", None)); + handle_result(test_develop("test-crates/hello-world", None, None)); } /// Creates a virtualenv and activates it, checks that the package isn't installed, uses /// "maturin develop" to install it and checks it is working -fn test_develop(package: impl AsRef, bindings: Option) -> Result<()> { +fn test_develop( + package: impl AsRef, + bindings: Option, + py_src: Option, +) -> Result<()> { maybe_mock_cargo(); let test_name = package @@ -96,6 +104,7 @@ fn test_develop(package: impl AsRef, bindings: Option) -> Result<( &venv_dir, false, cfg!(feature = "faster-tests"), + py_src.map(|src| src.into()), )?; check_installed(&package.as_ref(), &python)?; diff --git a/tests/test_integration.rs b/tests/test_integration.rs index ccedb609e..b145e71ca 100644 --- a/tests/test_integration.rs +++ b/tests/test_integration.rs @@ -13,39 +13,47 @@ mod common; #[test] fn test_integration_pyo3_pure() { - handle_result(test_integration("test-crates/pyo3-pure", None)); + handle_result(test_integration("test-crates/pyo3-pure", None, None)); } #[test] fn test_integration_pyo3_mixed() { - handle_result(test_integration("test-crates/pyo3-mixed", None)); + handle_result(test_integration( + "test-crates/pyo3-mixed", + None, + Some("python".into()), + )); } #[cfg(target_os = "windows")] #[test] #[ignore] fn test_integration_pyo3_pure_conda() { - handle_result(test_integration_conda("text-crates/pyo3-pure", None)); + handle_result(test_integration_conda("text-crates/pyo3-pure", None, None)); } #[test] fn test_integration_cffi_pure() { - handle_result(test_integration("test-crates/cffi-pure", None)); + handle_result(test_integration("test-crates/cffi-pure", None, None)); } #[test] fn test_integration_cffi_mixed() { - handle_result(test_integration("test-crates/cffi-mixed", None)); + handle_result(test_integration("test-crates/cffi-mixed", None, None)); } #[test] fn test_integration_hello_world() { - handle_result(test_integration("test-crates/hello-world", None)); + handle_result(test_integration("test-crates/hello-world", None, None)); } /// For each installed python version, this builds a wheel, creates a virtualenv if it /// doesn't exist, installs the package and runs check_installed.py -fn test_integration(package: impl AsRef, bindings: Option) -> Result<()> { +fn test_integration( + package: impl AsRef, + bindings: Option, + py_src: Option, +) -> Result<()> { maybe_mock_cargo(); let target = Target::from_target_triple(None)?; @@ -65,6 +73,11 @@ fn test_integration(package: impl AsRef, bindings: Option) -> Resu cli.push(bindings); } + if let Some(py_src) = py_src.as_ref() { + cli.push("--py-src"); + cli.push(py_src); + } + let options = BuildOptions::from_iter_safe(cli)?; let wheels = options @@ -153,7 +166,11 @@ fn create_conda_env(name: &str, major: usize, minor: usize) { } #[cfg(target_os = "windows")] -fn test_integration_conda(package: impl AsRef, bindings: Option) -> Result<()> { +fn test_integration_conda( + package: impl AsRef, + bindings: Option, + py_src: Option, +) -> Result<()> { use std::env; let package_string = package.as_ref().join("Cargo.toml").display().to_string(); @@ -188,6 +205,11 @@ fn test_integration_conda(package: impl AsRef, bindings: Option) - cli.push(bindings); } + if let Some(py_src) = py_src.as_ref() { + cli.push("--py-src"); + cli.push(py_src); + } + let options = BuildOptions::from_iter_safe(cli)?; let wheels = options