diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45198d8a3..a0e2fdb7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,6 +124,12 @@ jobs: sudo apt-get install -y mingw-w64 rustup target add x86_64-pc-windows-gnu cargo run -- build --no-sdist -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-gnu + - name: test compiling with PYO3_CONFIG_FILE + shell: bash + run: | + rustup target add x86_64-unknown-linux-gnu + export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt + cargo run -- build --no-sdist -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-unknown-linux-gnu --zig test-alpine: name: Test Alpine Linux diff --git a/Changelog.md b/Changelog.md index 710106cbb..bf4d6fd86 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Re-export `__all__` for pure Rust projects in [#886](https://github.com/PyO3/maturin/pull/886) * Stop setting `RUSTFLAGS` environment variable to an empty string in [#887](https://github.com/PyO3/maturin/pull/887) * Add hardcoded well-known sysconfigs for effortless cross compiling in [#896](https://github.com/PyO3/maturin/pull/896) +* Add support for `PYO3_CONFIG_FILE` in [#899](https://github.com/PyO3/maturin/pull/899) ## [0.12.14] - 2022-04-25 diff --git a/src/build_options.rs b/src/build_options.rs index 4ca0e8dae..9b4cc51ad 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -656,7 +656,18 @@ pub fn find_interpreter( match bridge { BridgeModel::Bindings(binding_name, _) => { let mut interpreters = Vec::new(); - if binding_name.starts_with("pyo3") && target.is_unix() && target.cross_compiling() { + if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") { + if !binding_name.starts_with("pyo3") { + bail!("Only pyo3 bindings can be configured with PYO3_CONFIG_FILE"); + } + let interpreter_config = + InterpreterConfig::from_pyo3_config(config_file.as_ref(), target) + .context("Invalid PYO3_CONFIG_FILE")?; + interpreters.push(PythonInterpreter::from_config(interpreter_config)); + } else if binding_name.starts_with("pyo3") + && target.is_unix() + && target.cross_compiling() + { if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") { let host_interpreters = find_host_interpreter(bridge, interpreter, target, min_python_minor)?; diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs index 86226cfc2..ea7120993 100644 --- a/src/python_interpreter/config.rs +++ b/src/python_interpreter/config.rs @@ -1,8 +1,13 @@ use super::InterpreterKind; use crate::target::{Arch, Os}; +use crate::Target; +use anyhow::{format_err, Context, Result}; +use fs_err as fs; use once_cell::sync::Lazy; use serde::Deserialize; use std::collections::HashMap; +use std::io::{BufRead, BufReader}; +use std::path::Path; /// Wellknown Python interpreter sysconfig values static WELLKNOWN_SYSCONFIG: Lazy>>> = @@ -62,6 +67,123 @@ impl InterpreterConfig { None } + /// Construct a new InterpreterConfig from a pyo3 config file + pub fn from_pyo3_config(config_file: &Path, target: &Target) -> Result { + let config_file = fs::File::open(config_file)?; + let reader = BufReader::new(config_file); + let lines = reader.lines(); + + macro_rules! parse_value { + ($variable:ident, $value:ident) => { + $variable = Some($value.trim().parse().context(format!( + concat!( + "failed to parse ", + stringify!($variable), + " from config value '{}'" + ), + $value + ))?) + }; + } + + let mut implementation = None; + let mut version = None; + let mut abiflags = None; + let mut ext_suffix = None; + let mut abi_tag = None; + let mut pointer_width = None; + + for (i, line) in lines.enumerate() { + let line = line.context("failed to read line from config")?; + let (key, value) = line + .split_once('=') + .with_context(|| format!("expected key=value pair on line {}", i + 1))?; + match key { + "implementation" => parse_value!(implementation, value), + "version" => parse_value!(version, value), + "abiflags" => parse_value!(abiflags, value), + "ext_suffix" => parse_value!(ext_suffix, value), + "abi_tag" => parse_value!(abi_tag, value), + "pointer_width" => parse_value!(pointer_width, value), + _ => continue, + } + } + let version: String = version.context("missing value for version")?; + let (ver_major, ver_minor) = version + .split_once('.') + .context("Invalid python interpreter version")?; + let major = ver_major.parse::().with_context(|| { + format!( + "Invalid python interpreter major version '{}', expect a digit", + ver_major + ) + })?; + let minor = ver_minor.parse::().with_context(|| { + format!( + "Invalid python interpreter minor version '{}', expect a digit", + ver_minor + ) + })?; + let implementation = implementation.unwrap_or_else(|| "cpython".to_string()); + let interpreter_kind = implementation.parse().map_err(|e| format_err!("{}", e))?; + let abi_tag = match interpreter_kind { + InterpreterKind::CPython => { + if (major, minor) >= (3, 8) { + abi_tag.unwrap_or_else(|| format!("{}{}", major, minor)) + } else { + abi_tag.unwrap_or_else(|| format!("{}{}m", major, minor)) + } + } + InterpreterKind::PyPy => abi_tag.unwrap_or_else(|| "pp73".to_string()), + }; + let file_ext = if target.is_windows() { "pyd" } else { "so" }; + let ext_suffix = if target.is_linux() || target.is_macos() { + // See https://github.com/pypa/auditwheel/issues/349 + let target_env = if (major, minor) >= (3, 11) { + target.target_env().to_string() + } else { + "gnu".to_string() + }; + match interpreter_kind { + InterpreterKind::CPython => ext_suffix.unwrap_or_else(|| { + // Eg: .cpython-38-x86_64-linux-gnu.so + format!( + ".cpython-{}-{}-{}-{}.{}", + abi_tag, + target.get_python_arch(), + target.get_python_os(), + target_env, + file_ext, + ) + }), + InterpreterKind::PyPy => ext_suffix.unwrap_or_else(|| { + // Eg: .pypy38-pp73-x86_64-linux-gnu.so + format!( + ".pypy{}{}-{}-{}-{}-{}.{}", + major, + minor, + abi_tag, + target.get_python_arch(), + target.get_python_os(), + target_env, + file_ext, + ) + }), + } + } else { + ext_suffix.context("missing value for ext_suffix")? + }; + Ok(Self { + major, + minor, + interpreter_kind, + abiflags: abiflags.unwrap_or_default(), + ext_suffix, + abi_tag: Some(abi_tag), + pointer_width, + }) + } + /// Generate pyo3 config file content pub fn pyo3_config_file(&self) -> String { let mut content = format!( diff --git a/src/python_interpreter/mod.rs b/src/python_interpreter/mod.rs index 3d86ac867..4d736af37 100644 --- a/src/python_interpreter/mod.rs +++ b/src/python_interpreter/mod.rs @@ -10,7 +10,7 @@ use std::io; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::str; +use std::str::{self, FromStr}; mod config; @@ -248,6 +248,18 @@ impl fmt::Display for InterpreterKind { } } +impl FromStr for InterpreterKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "cpython" => Ok(InterpreterKind::CPython), + "pypy" => Ok(InterpreterKind::PyPy), + unknown => Err(format!("Unknown interpreter kind '{}'", unknown)), + } + } +} + /// The output format of [GET_INTERPRETER_METADATA] #[derive(Deserialize)] struct IntepreterMetadataMessage { diff --git a/src/target.rs b/src/target.rs index 29752ee1b..d03b2f7b8 100644 --- a/src/target.rs +++ b/src/target.rs @@ -296,6 +296,19 @@ impl Target { Ok(tag) } + /// Returns the name python uses in `sys.platform` for this architecture. + pub fn get_python_arch(&self) -> &str { + match self.arch { + Arch::Aarch64 => "aarch64", + Arch::Armv7L => "armv7l", + Arch::Powerpc64Le => "powerpc64le", + Arch::Powerpc64 => "powerpc64", + Arch::X86 => "i386", + Arch::X86_64 => "x86_64", + Arch::S390X => "s390x", + } + } + /// Returns the name python uses in `sys.platform` for this os pub fn get_python_os(&self) -> &str { match self.os { @@ -364,6 +377,11 @@ impl Target { self.arch } + /// Returns target environment + pub fn target_env(&self) -> Environment { + self.env + } + /// Returns true if the current platform is linux pub fn is_linux(&self) -> bool { self.os == Os::Linux diff --git a/test-crates/pyo3-mixed/pyo3-config.txt b/test-crates/pyo3-mixed/pyo3-config.txt new file mode 100644 index 000000000..da3ba7691 --- /dev/null +++ b/test-crates/pyo3-mixed/pyo3-config.txt @@ -0,0 +1,6 @@ +implementation=CPython +version=3.10 +shared=true +abi3=false +suppress_build_script_link_lines=false +pointer_width=64