Merge pull request #2272 from ravenexp/cross-python-implementation

pyo3-build-config: Add `PYO3_CROSS_PYTHON_IMPLEMENTATION` env var

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable for selecting the default cross Python implementation. [#2272](https://github.com/PyO3/pyo3/pull/2272)
 - Add new public `pyo3-build-config` API using the types from `target_lexicon` crate. Deprecate `cross_compiling()`. [#2253](https://github.com/PyO3/pyo3/pull/2253)
 - Allow dependent crates to access config values from `pyo3-build-config` via cargo link dep env vars. [#2092](https://github.com/PyO3/pyo3/pull/2092)
 - Added methods on `InterpreterConfig` to run Python scripts using the configured executable. [#2092](https://github.com/PyO3/pyo3/pull/2092)

guide/src/building_and_distribution.md

@@ -228,6 +228,7 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter
 * `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation.
 * `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target. This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Windows and Android or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`.
 * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
+* `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
 
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
 

pyo3-build-config/src/impl_.rs

@@ -690,6 +690,9 @@ pub struct CrossCompileConfig {
     /// The version of the Python library to link against.
     version: Option<PythonVersion>,
 
+    /// The target Python implementation hint (CPython or PyPy)
+    implementation: Option<PythonImplementation>,
+
     /// The compile target triple (e.g. aarch64-unknown-linux-gnu)
     target: Triple,
 }
@@ -707,12 +710,14 @@ impl CrossCompileConfig {
         if env_vars.any() || Self::is_cross_compiling_from_to(host, target) {
             let lib_dir = env_vars.lib_dir_path()?;
             let version = env_vars.parse_version()?;
+            let implementation = env_vars.parse_implementation()?;
             let target = target.clone();
 
             Ok(Some(CrossCompileConfig {
                 lib_dir,
-                target,
                 version,
+                implementation,
+                target,
             }))
         } else {
             Ok(None)
@@ -752,17 +757,37 @@ impl CrossCompileConfig {
     }
 }
 
-pub(crate) struct CrossCompileEnvVars {
+/// PyO3-specific cross compile environment variable values
+struct CrossCompileEnvVars {
+    /// `PYO3_CROSS`
     pyo3_cross: Option<OsString>,
+    /// `PYO3_CROSS_LIB_DIR`
     pyo3_cross_lib_dir: Option<OsString>,
+    /// `PYO3_CROSS_PYTHON_VERSION`
     pyo3_cross_python_version: Option<OsString>,
+    /// `PYO3_CROSS_PYTHON_IMPLEMENTATION`
+    pyo3_cross_python_implementation: Option<OsString>,
 }
 
 impl CrossCompileEnvVars {
+    /// Grabs the PyO3 cross-compile variables from the environment.
+    ///
+    /// Registers the build script to rerun if any of the variables changes.
+    fn from_env() -> Self {
+        CrossCompileEnvVars {
+            pyo3_cross: env_var("PYO3_CROSS"),
+            pyo3_cross_lib_dir: env_var("PYO3_CROSS_LIB_DIR"),
+            pyo3_cross_python_version: env_var("PYO3_CROSS_PYTHON_VERSION"),
+            pyo3_cross_python_implementation: env_var("PYO3_CROSS_PYTHON_IMPLEMENTATION"),
+        }
+    }
+
+    /// Checks if any of the variables is set.
     fn any(&self) -> bool {
         self.pyo3_cross.is_some()
             || self.pyo3_cross_lib_dir.is_some()
             || self.pyo3_cross_python_version.is_some()
+            || self.pyo3_cross_python_implementation.is_some()
     }
 
     /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value
@@ -784,6 +809,25 @@ impl CrossCompileEnvVars {
         Ok(version)
     }
 
+    /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value
+    /// into `PythonImplementation`.
+    fn parse_implementation(&self) -> Result<Option<PythonImplementation>> {
+        let implementation = self
+            .pyo3_cross_python_implementation
+            .as_ref()
+            .map(|os_string| {
+                let utf8_str = os_string
+                    .to_str()
+                    .ok_or("PYO3_CROSS_PYTHON_IMPLEMENTATION is not valid a UTF-8 string")?;
+                utf8_str
+                    .parse()
+                    .context("failed to parse PYO3_CROSS_PYTHON_IMPLEMENTATION")
+            })
+            .transpose()?;
+
+        Ok(implementation)
+    }
+
     /// Converts the stored `PYO3_CROSS_LIB_DIR` variable value (if any)
     /// into a `PathBuf` instance.
     ///
@@ -802,14 +846,6 @@ impl CrossCompileEnvVars {
     }
 }
 
-pub(crate) fn cross_compile_env_vars() -> CrossCompileEnvVars {
-    CrossCompileEnvVars {
-        pyo3_cross: env::var_os("PYO3_CROSS"),
-        pyo3_cross_lib_dir: env::var_os("PYO3_CROSS_LIB_DIR"),
-        pyo3_cross_python_version: env::var_os("PYO3_CROSS_PYTHON_VERSION"),
-    }
-}
-
 /// Detect whether we are cross compiling and return an assembled CrossCompileConfig if so.
 ///
 /// This function relies on PyO3 cross-compiling environment variables:
@@ -879,7 +915,7 @@ pub fn cross_compiling_from_to(
     host: &Triple,
     target: &Triple,
 ) -> Result<Option<CrossCompileConfig>> {
-    let env_vars = cross_compile_env_vars();
+    let env_vars = CrossCompileEnvVars::from_env();
     CrossCompileConfig::try_from_env_vars_host_target(env_vars, host, target)
 }
 
@@ -889,7 +925,7 @@ pub fn cross_compiling_from_to(
 /// This must be called from PyO3's build script, because it relies on environment
 /// variables such as `CARGO_CFG_TARGET_OS` which aren't available at any other time.
 pub fn cross_compiling_from_cargo_env() -> Result<Option<CrossCompileConfig>> {
-    let env_vars = cross_compile_env_vars();
+    let env_vars = CrossCompileEnvVars::from_env();
     let host = Triple::host();
     let target = target_triple_from_env();
 
@@ -1302,7 +1338,7 @@ fn cross_compile_from_sysconfigdata(
 /// Generates "default" cross compilation information for the target.
 ///
 /// This should work for most CPython extension modules when targeting
-/// Windows, MacOS and Linux.
+/// Windows, macOS and Linux.
 ///
 /// Must be called from a PyO3 crate build script.
 fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<InterpreterConfig> {
@@ -1315,7 +1351,9 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         )?;
 
     let abi3 = is_abi3();
-    let implementation = PythonImplementation::CPython;
+    let implementation = cross_compile_config
+        .implementation
+        .unwrap_or(PythonImplementation::CPython);
 
     let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
         let mingw = cross_compile_config.target.environment == Environment::Gnu;
@@ -1808,12 +1846,20 @@ mod tests {
 
     #[test]
     fn windows_hardcoded_cross_compile() {
-        let cross_config = CrossCompileConfig {
-            lib_dir: Some("C:\\some\\path".into()),
-            version: Some(PythonVersion { major: 3, minor: 7 }),
-            target: triple!("i686-pc-windows-msvc"),
+        let env_vars = CrossCompileEnvVars {
+            pyo3_cross: None,
+            pyo3_cross_lib_dir: Some("C:\\some\\path".into()),
+            pyo3_cross_python_implementation: None,
+            pyo3_cross_python_version: Some("3.7".into()),
         };
 
+        let host = triple!("x86_64-unknown-linux-gnu");
+        let target = triple!("i686-pc-windows-msvc");
+        let cross_config =
+            CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
+                .unwrap()
+                .unwrap();
+
         assert_eq!(
             default_cross_compile(&cross_config).unwrap(),
             InterpreterConfig {
@@ -1834,12 +1880,20 @@ mod tests {
 
     #[test]
     fn mingw_hardcoded_cross_compile() {
-        let cross_config = CrossCompileConfig {
-            lib_dir: Some("/usr/lib/mingw".into()),
-            version: Some(PythonVersion { major: 3, minor: 8 }),
-            target: triple!("i686-pc-windows-gnu"),
+        let env_vars = CrossCompileEnvVars {
+            pyo3_cross: None,
+            pyo3_cross_lib_dir: Some("/usr/lib/mingw".into()),
+            pyo3_cross_python_implementation: None,
+            pyo3_cross_python_version: Some("3.8".into()),
         };
 
+        let host = triple!("x86_64-unknown-linux-gnu");
+        let target = triple!("i686-pc-windows-gnu");
+        let cross_config =
+            CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
+                .unwrap()
+                .unwrap();
+
         assert_eq!(
             default_cross_compile(&cross_config).unwrap(),
             InterpreterConfig {
@@ -1860,12 +1914,20 @@ mod tests {
 
     #[test]
     fn unix_hardcoded_cross_compile() {
-        let cross_config = CrossCompileConfig {
-            lib_dir: Some("/usr/arm64/lib".into()),
-            version: Some(PythonVersion { major: 3, minor: 9 }),
-            target: triple!("aarch64-unknown-linux-gnu"),
+        let env_vars = CrossCompileEnvVars {
+            pyo3_cross: None,
+            pyo3_cross_lib_dir: Some("/usr/arm64/lib".into()),
+            pyo3_cross_python_implementation: None,
+            pyo3_cross_python_version: Some("3.9".into()),
         };
 
+        let host = triple!("x86_64-unknown-linux-gnu");
+        let target = triple!("aarch64-unknown-linux-gnu");
+        let cross_config =
+            CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
+                .unwrap()
+                .unwrap();
+
         assert_eq!(
             default_cross_compile(&cross_config).unwrap(),
             InterpreterConfig {
@@ -1884,6 +1946,42 @@ mod tests {
         );
     }
 
+    #[test]
+    fn pypy_hardcoded_cross_compile() {
+        let env_vars = CrossCompileEnvVars {
+            pyo3_cross: None,
+            pyo3_cross_lib_dir: None,
+            pyo3_cross_python_implementation: Some("PyPy".into()),
+            pyo3_cross_python_version: Some("3.10".into()),
+        };
+
+        let triple = triple!("x86_64-unknown-linux-gnu");
+        let cross_config =
+            CrossCompileConfig::try_from_env_vars_host_target(env_vars, &triple, &triple)
+                .unwrap()
+                .unwrap();
+
+        assert_eq!(
+            default_cross_compile(&cross_config).unwrap(),
+            InterpreterConfig {
+                implementation: PythonImplementation::PyPy,
+                version: PythonVersion {
+                    major: 3,
+                    minor: 10
+                },
+                shared: true,
+                abi3: false,
+                lib_name: Some("pypy3.10-c".into()),
+                lib_dir: None,
+                executable: None,
+                pointer_width: None,
+                build_flags: BuildFlags::default(),
+                suppress_build_script_link_lines: false,
+                extra_build_script_lines: vec![],
+            }
+        );
+    }
+
     #[test]
     fn default_lib_name_windows() {
         use PythonImplementation::*;
@@ -1975,6 +2073,7 @@ mod tests {
             pyo3_cross: None,
             pyo3_cross_lib_dir: None,
             pyo3_cross_python_version: Some("3.9".into()),
+            pyo3_cross_python_implementation: None,
         };
 
         assert_eq!(
@@ -1986,6 +2085,7 @@ mod tests {
             pyo3_cross: None,
             pyo3_cross_lib_dir: None,
             pyo3_cross_python_version: None,
+            pyo3_cross_python_implementation: None,
         };
 
         assert_eq!(env_vars.parse_version().unwrap(), None);
@@ -1994,6 +2094,7 @@ mod tests {
             pyo3_cross: None,
             pyo3_cross_lib_dir: None,
             pyo3_cross_python_version: Some("100".into()),
+            pyo3_cross_python_implementation: None,
         };
 
         assert!(env_vars.parse_version().is_err());
@@ -2068,6 +2169,7 @@ mod tests {
         let cross = CrossCompileConfig {
             lib_dir: Some(lib_dir.into()),
             version: Some(interpreter_config.version),
+            implementation: Some(interpreter_config.implementation),
             target: triple!("x86_64-unknown-linux-gnu"),
         };