From cd920c8b9adb323f98cca919339cfa7fca9b18fb Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 31 Dec 2022 17:50:17 +0800 Subject: [PATCH] Fallback to sysconfig interpreters for pyo3 bindings --- Changelog.md | 1 + src/build_options.rs | 79 ++++++++++++++++++++++++----------- src/python_interpreter/mod.rs | 19 +++++++-- 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/Changelog.md b/Changelog.md index 58d8d5bc5..a9894a5cc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **Breaking Change**: Turn `patchelf` version warning into a hard error in [#1335](https://github.com/PyO3/maturin/pull/1335) * **Breaking Change**: [`uniffi_bindgen` CLI](https://mozilla.github.io/uniffi-rs/tutorial/Prerequisites.html#the-uniffi-bindgen-cli-tool) is required for building `uniffi` bindings wheels in [#1352](https://github.com/PyO3/maturin/pull/1352) * Add support for packaging multiple pure Python packages in [#1378](https://github.com/PyO3/maturin/pull/1378) +* Fallback to sysconfig interpreters for pyo3 bindings in [#1381](https://github.com/PyO3/maturin/pull/1381) ## [0.14.7] - 2022-12-20 diff --git a/src/build_options.rs b/src/build_options.rs index 6c28a7cf4..cd0166063 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -13,6 +13,7 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::ops::{Deref, DerefMut}; use std::path::PathBuf; +use tracing::debug; // This is used for BridgeModel::Bindings("pyo3-ffi") and BridgeModel::Bindings("pyo3"). // These should be treated almost identically but must be correctly identified @@ -219,7 +220,6 @@ impl BuildOptions { ) -> Result> { match bridge { BridgeModel::Bindings(binding_name, _) | BridgeModel::Bin(Some((binding_name, _))) => { - let mut native_interpreters = false; let mut interpreters = Vec::new(); if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") { if !binding_name.starts_with("pyo3") { @@ -308,25 +308,12 @@ impl BuildOptions { interpreters = find_interpreter_in_sysconfig(interpreter, target, min_python_minor)?; } + } else if binding_name.starts_with("pyo3") { + // Only pyo3/pyo3-ffi bindings supports bundled sysconfig interpreters + interpreters = find_interpreter(bridge, interpreter, target, min_python_minor)?; } else { - match find_interpreter_in_host(bridge, interpreter, target, min_python_minor) { - Ok(host_interps) => { - interpreters = host_interps; - native_interpreters = true; - } - Err(err) => { - if binding_name.starts_with("pyo3") && target.is_unix() { - interpreters = find_interpreter_in_sysconfig( - interpreter, - target, - min_python_minor, - ) - .map_err(|_| err)?; - } else { - return Err(err); - } - } - } + interpreters = + find_interpreter_in_host(bridge, interpreter, target, min_python_minor)?; } let interpreters_str = interpreters @@ -334,11 +321,7 @@ impl BuildOptions { .map(ToString::to_string) .collect::>() .join(", "); - if native_interpreters { - println!("🐍 Found {}", interpreters_str); - } else { - println!("🐍 Found cross compiling target {}", interpreters_str); - } + println!("🐍 Found {}", interpreters_str); Ok(interpreters) } @@ -1000,6 +983,43 @@ fn find_single_python_interpreter( Ok(interpreter) } +/// Find python interpreters in host machine first, +/// fallback to bundled sysconfig if not found in host machine +fn find_interpreter( + bridge: &BridgeModel, + interpreter: &[PathBuf], + target: &Target, + min_python_minor: Option, +) -> Result> { + let mut interpreters = Vec::new(); + if !interpreter.is_empty() { + let mut missing = Vec::new(); + for interp in interpreter { + match PythonInterpreter::check_executable(interp.clone(), target, bridge) { + Ok(Some(interp)) => interpreters.push(interp), + _ => missing.push(interp.clone()), + } + } + if !missing.is_empty() { + let sysconfig_interps = + find_interpreter_in_sysconfig(&missing, target, min_python_minor)?; + interpreters.extend(sysconfig_interps); + } + } else { + interpreters = PythonInterpreter::find_all(target, bridge, min_python_minor) + .context("Finding python interpreters failed")?; + }; + + if interpreters.is_empty() { + if let Some(minor) = min_python_minor { + bail!("Couldn't find any python interpreters with version >= 3.{}. Please specify at least one with -i", minor); + } else { + bail!("Couldn't find any python interpreters. Please specify at least one with -i"); + } + } + Ok(interpreters) +} + /// Find python interpreters in the host machine fn find_interpreter_in_host( bridge: &BridgeModel, @@ -1076,7 +1096,16 @@ fn find_interpreter_in_sysconfig( python_impl, (ver_major, ver_minor), ) - .context("Failed to find a python interpreter")?; + .with_context(|| { + format!( + "Failed to find a {} {}.{} interpreter", + python_impl, ver_major, ver_minor + ) + })?; + debug!( + "Found {} {}.{} in bundled sysconfig", + sysconfig.interpreter_kind, sysconfig.major, sysconfig.minor, + ); interpreters.push(PythonInterpreter::from_config(sysconfig.clone())); } Ok(interpreters) diff --git a/src/python_interpreter/mod.rs b/src/python_interpreter/mod.rs index 9d01d2754..a0e29e4af 100644 --- a/src/python_interpreter/mod.rs +++ b/src/python_interpreter/mod.rs @@ -11,6 +11,7 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str::{self, FromStr}; +use tracing::debug; mod config; @@ -613,6 +614,10 @@ impl PythonInterpreter { .context(String::from_utf8_lossy(&output.stdout).trim().to_string())?; if (message.major == 2 && message.minor != 7) || (message.major == 3 && message.minor < 5) { + debug!( + "Skipping outdated python interpreter '{}'", + executable.as_ref().display() + ); return Ok(None); } @@ -636,6 +641,15 @@ impl PythonInterpreter { Some(message.platform.to_lowercase().replace(['-', '.'], "_")) }; + let executable = message + .executable + .map(PathBuf::from) + .unwrap_or_else(|| executable.as_ref().to_path_buf()); + debug!( + "Found {} interpreter at {}", + interpreter, + executable.display() + ); Ok(Some(PythonInterpreter { config: InterpreterConfig { major: message.major, @@ -648,10 +662,7 @@ impl PythonInterpreter { abi_tag: message.abi_tag, pointer_width: None, }, - executable: message - .executable - .map(PathBuf::from) - .unwrap_or_else(|| executable.as_ref().to_path_buf()), + executable, platform, runnable: true, implmentation_name: message.implementation_name,