From 3727ae0835ea12b03a7967815ecf01a03117e680 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 11 May 2022 18:37:52 -0700 Subject: [PATCH 01/16] ffi: add PyConfig::warn_default_encoding to 3.10+ This field was added in 3.10 but we missed it. This was causing offsets of subsequent fields to be wrong. This could lead to unexpected behavior or even crashes. --- CHANGELOG.md | 7 ++++++- pyo3-ffi/src/cpython/initconfig.rs | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a320cde5f0..47eb7714c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,12 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [Unreleased] + +### Fized + +- Added missing `warn_default_encoding` field to `PyConfig` on 3.10+. The previously missing field could result in incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370) +- Fixed order of `pathconfig_warnings` and `program_name` fields of `PyConfig` on 3.10+. Previously, the order of the fields was swapped and this could lead to incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370) ## [0.16.4] - 2022-04-14 diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 552a3664191..fca7ca39602 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -110,6 +110,8 @@ pub struct PyConfig { pub warnoptions: PyWideStringList, pub site_import: c_int, pub bytes_warning: c_int, + #[cfg(Py_3_10)] + pub warn_default_encoding: c_int, pub inspect: c_int, pub interactive: c_int, pub optimization_level: c_int, @@ -127,9 +129,9 @@ pub struct PyConfig { pub legacy_windows_stdio: c_int, pub check_hash_pycs_mode: *mut wchar_t, + pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, - pub pathconfig_warnings: c_int, pub pythonpath_env: *mut wchar_t, pub home: *mut wchar_t, #[cfg(Py_3_10)] From adbaa090bd70d008de35bd3d530a04648086b592 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 13 May 2022 17:52:31 -0700 Subject: [PATCH 02/16] ffi: add new 3.11 fields to PyConfig These are the new fields in 3.11 so far. 3.11 is supposedly feature frozen, so hopefully we don't need to revisit this struct before 3.11 final. --- pyo3-ffi/src/cpython/initconfig.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index fca7ca39602..5d70a7521a4 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -92,10 +92,14 @@ pub struct PyConfig { pub _use_peg_parser: c_int, pub tracemalloc: c_int, pub import_time: c_int, + #[cfg(Py_3_11)] + pub code_debug_ranges: c_int, pub show_ref_count: c_int, #[cfg(not(Py_3_9))] pub show_alloc_count: c_int, pub dump_refs: c_int, + #[cfg(Py_3_11)] + pub dump_refs_file: *mut wchar_t, pub malloc_stats: c_int, pub filesystem_encoding: *mut wchar_t, pub filesystem_errors: *mut wchar_t, @@ -129,6 +133,10 @@ pub struct PyConfig { pub legacy_windows_stdio: c_int, pub check_hash_pycs_mode: *mut wchar_t, + #[cfg(Py_3_11)] + pub use_frozen_modules: c_int, + #[cfg(Py_3_11)] + pub safe_path: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -139,6 +147,8 @@ pub struct PyConfig { pub module_search_paths_set: c_int, pub module_search_paths: PyWideStringList, + #[cfg(Py_3_11)] + pub stdlib_dir: *mut wchar_t, pub executable: *mut wchar_t, pub base_executable: *mut wchar_t, pub prefix: *mut wchar_t, @@ -155,6 +165,8 @@ pub struct PyConfig { pub _init_main: c_int, #[cfg(Py_3_9)] pub _isolated_interpreter: c_int, + #[cfg(Py_3_11)] + pub _is_python_build: c_int, #[cfg(all(Py_3_9, not(Py_3_10)))] pub orig_argv: PyWideStringList, } From 6369b1b2438f99c69c76afb7c2b839b3b8ac2b4f Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 14 May 2022 22:12:47 +0100 Subject: [PATCH 03/16] add regression test for PEP 587 --- CHANGELOG.md | 6 ++- Cargo.toml | 1 + pyo3-ffi/src/cpython/pylifecycle.rs | 2 +- tests/test_pep_587.rs | 60 +++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/test_pep_587.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 47eb7714c84..a91f6c7634f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Fized +### Added + +- Add FFI definition `Py_ExitStatusException`. [#2374](https://github.com/PyO3/pyo3/pull/2374) + +### Fixed - Added missing `warn_default_encoding` field to `PyConfig` on 3.10+. The previously missing field could result in incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370) - Fixed order of `pathconfig_warnings` and `program_name` fields of `PyConfig` on 3.10+. Previously, the order of the fields was swapped and this could lead to incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370) diff --git a/Cargo.toml b/Cargo.toml index 23df80c0d96..6ff88ba90fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ proptest = { version = "0.10.1", default-features = false, features = ["std"] } send_wrapper = "0.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.61" +widestring = "0.5.1" [build-dependencies] pyo3-build-config = { path = "pyo3-build-config", version = "0.16.4", features = ["resolve-config"] } diff --git a/pyo3-ffi/src/cpython/pylifecycle.rs b/pyo3-ffi/src/cpython/pylifecycle.rs index 3f1f16e26a4..08c47881e03 100644 --- a/pyo3-ffi/src/cpython/pylifecycle.rs +++ b/pyo3-ffi/src/cpython/pylifecycle.rs @@ -23,7 +23,7 @@ extern "C" { pub fn Py_RunMain() -> c_int; - // skipped Py_ExitStatusException + pub fn Py_ExitStatusException(status: PyStatus) -> !; // skipped _Py_RestoreSignals diff --git a/tests/test_pep_587.rs b/tests/test_pep_587.rs new file mode 100644 index 00000000000..ca5c3f65ea2 --- /dev/null +++ b/tests/test_pep_587.rs @@ -0,0 +1,60 @@ +#![cfg(all(Py_3_8, not(any(PyPy, Py_LIMITED_API))))] + +use pyo3::ffi; + +#[cfg(Py_3_10)] +use widestring::WideCString; + +#[test] +fn test_default_interpreter() { + macro_rules! ensure { + ($py_call:expr) => {{ + let status = $py_call; + unsafe { + if ffi::PyStatus_Exception(status) != 0 { + ffi::Py_ExitStatusException(status); + } + } + }}; + } + + let mut preconfig = unsafe { std::mem::zeroed() }; + + unsafe { ffi::PyPreConfig_InitPythonConfig(&mut preconfig) }; + preconfig.utf8_mode = 1; + + ensure!(unsafe { ffi::Py_PreInitialize(&preconfig) }); + + let mut config = unsafe { std::mem::zeroed() }; + unsafe { ffi::PyConfig_InitPythonConfig(&mut config) }; + + // Require manually calling _Py_InitializeMain to exercise more ffi code + config._init_main = 0; + + #[cfg(Py_3_10)] + unsafe { + ffi::PyConfig_SetBytesString( + &mut config, + &mut config.program_name, + "some_test\0".as_ptr().cast(), + ); + } + + ensure!(unsafe { ffi::Py_InitializeFromConfig(&config) }); + + // The GIL is held. + assert_eq!(unsafe { ffi::PyGILState_Check() }, 1); + + // Now proceed with the Python main initialization. + ensure!(unsafe { ffi::_Py_InitializeMain() }); + + // The GIL is held after finishing initialization. + assert_eq!(unsafe { ffi::PyGILState_Check() }, 1); + + // Confirm program name set above was picked up correctly + #[cfg(Py_3_10)] + { + let program_name = unsafe { WideCString::from_ptr_str(ffi::Py_GetProgramName().cast()) }; + assert_eq!(program_name.to_string().unwrap(), "some_test"); + } +} From efe22bd58ed3a5f63b947865ba7bf855a1c93369 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 16 Apr 2022 16:16:01 +0800 Subject: [PATCH 04/16] Add `generate-abi3-import-lib` feature to `pyo3-ffi` --- Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6ff88ba90fe..1d7ccdd0a62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,7 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] # Automatically generates `python3.dll` import libraries for Windows targets. -generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"] +generate-abi3-import-lib = ["pyo3-ffi/generate-abi3-import-lib"] # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the # Python interpreter if needed. diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 0e6703dd2d7..623b9277bf1 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -32,6 +32,8 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"] +# Automatically generates `python3.dll` import libraries for Windows targets. +generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] From fcecbacc30eb3a3444ccaead20ce547a0a52bde2 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 16 Apr 2022 23:13:06 +0800 Subject: [PATCH 05/16] pyo3-ffi: forward `extension-module` feature to `pyo3-build-config` --- Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1d7ccdd0a62..7ed156e8ac0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ pyproto = ["pyo3-macros/pyproto"] # Use this feature when building an extension module. # It tells the linker to keep the python symbols unresolved, # so that the module can also be used with statically linked python interpreters. -extension-module = ["pyo3-build-config/extension-module", "pyo3-ffi/extension-module"] +extension-module = ["pyo3-ffi/extension-module"] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 623b9277bf1..6691f29c1c8 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -21,7 +21,7 @@ default = [] # Use this feature when building an extension module. # It tells the linker to keep the python symbols unresolved, # so that the module can also be used with statically linked python interpreters. -extension-module = [] +extension-module = ["pyo3-build-config/extension-module"] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. abi3 = ["pyo3-build-config/abi3"] From a025e8ee6b48b65a55cf6150fc523a824bb3f6a0 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 10 May 2022 15:33:08 +0800 Subject: [PATCH 06/16] pyo3-build-config: switch to `python3_dll_a::ImportLibraryGenerator` --- pyo3-build-config/Cargo.toml | 4 ++-- pyo3-build-config/src/impl_.rs | 8 ++++---- .../src/{abi3_import_lib.rs => import_lib.rs} | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) rename pyo3-build-config/src/{abi3_import_lib.rs => import_lib.rs} (92%) diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index fb687b0fe71..77460cf9a7e 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -12,11 +12,11 @@ edition = "2018" [dependencies] once_cell = "1" -python3-dll-a = { version = "0.2", optional = true } +python3-dll-a = { version = "0.2.2", optional = true } target-lexicon = "0.12" [build-dependencies] -python3-dll-a = { version = "0.2", optional = true } +python3-dll-a = { version = "0.2.2", optional = true } target-lexicon = "0.12" [features] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 9cd659377c9..17cb020c993 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -3,8 +3,8 @@ // Optional python3.dll import library generator for Windows #[cfg(feature = "python3-dll-a")] -#[path = "abi3_import_lib.rs"] -mod abi3_import_lib; +#[path = "import_lib.rs"] +mod import_lib; use std::{ collections::{HashMap, HashSet}, @@ -1410,7 +1410,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result { // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?; + interpreter_config.lib_dir = self::import_lib::generate_abi3_import_lib(&host)?; } Ok(interpreter_config) diff --git a/pyo3-build-config/src/abi3_import_lib.rs b/pyo3-build-config/src/import_lib.rs similarity index 92% rename from pyo3-build-config/src/abi3_import_lib.rs rename to pyo3-build-config/src/import_lib.rs index 2a089fbadfe..ac88c5eefa5 100644 --- a/pyo3-build-config/src/abi3_import_lib.rs +++ b/pyo3-build-config/src/import_lib.rs @@ -3,7 +3,7 @@ use std::env; use std::path::PathBuf; -use python3_dll_a::generate_implib_for_target; +use python3_dll_a::ImportLibraryGenerator; use crate::errors::{Context, Result}; @@ -36,7 +36,8 @@ pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result let env = target.environment.to_string(); - generate_implib_for_target(&out_lib_dir, &arch, &env) + ImportLibraryGenerator::new(&arch, &env) + .generate(&out_lib_dir) .context("failed to generate python3.dll import library")?; let out_lib_dir_string = out_lib_dir From 29b8731f849533f47637709c98a0a10f81d272b4 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 10 May 2022 15:46:47 +0800 Subject: [PATCH 07/16] Add support for generating non-abi3 python import libraries for Windows targets --- Architecture.md | 4 ++-- Cargo.toml | 4 +++- examples/maturin-starter/Cargo.toml | 2 +- guide/src/building_and_distribution.md | 14 +++++++------- guide/src/features.md | 6 +++--- pyo3-build-config/src/impl_.rs | 12 +++++++++--- pyo3-build-config/src/import_lib.rs | 14 +++++++++----- pyo3-ffi/Cargo.toml | 4 +++- 8 files changed, 37 insertions(+), 23 deletions(-) diff --git a/Architecture.md b/Architecture.md index d0e873f80a8..b7614a74c67 100644 --- a/Architecture.md +++ b/Architecture.md @@ -193,9 +193,9 @@ Some of the functionality of `pyo3-build-config`: `PYO3_CROSS_PYTHON_IMPLEMENTATION`) or system files. When cross compiling extension modules it is often possible to make it work without any additional user input. - - When an experimental feature `generate-abi3-import-lib` is enabled, the `pyo3-ffi` build script can + - When an experimental feature `generate-import-lib` is enabled, the `pyo3-ffi` build script can generate `python3.dll` import libraries for Windows targets automatically via an external - [`python3-dll-a`] crate. This enables the users to cross compile abi3 extensions for Windows without + [`python3-dll-a`] crate. This enables the users to cross compile Python extensions for Windows without having to install any Windows Python libraries. diff --git a/Cargo.toml b/Cargo.toml index 7ed156e8ac0..51805f3f539 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,9 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] # Automatically generates `python3.dll` import libraries for Windows targets. -generate-abi3-import-lib = ["pyo3-ffi/generate-abi3-import-lib"] +generate-import-lib = ["pyo3-ffi/generate-import-lib"] +# Deprecated, replaced by `generate-import-lib` +generate-abi3-import-lib = ["generate-import-lib"] # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the # Python interpreter if needed. diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 21e8c90cc54..64318dcfe87 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -11,6 +11,6 @@ crate-type = ["cdylib"] pyo3 = { path = "../../", features = ["extension-module"] } [features] -abi3 = ["pyo3/abi3-py37", "pyo3/generate-abi3-import-lib"] +abi3 = ["pyo3/abi3-py37", "pyo3/generate-import-lib"] [workspace] diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index ce69030ca90..f1a0a587a39 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -162,7 +162,7 @@ PyO3 will still attempt to compile `abi3` extension modules after displaying a w On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`. -If the `python3.dll` import library is not available, an experimental `generate-abi3-import-lib` crate +If the `python3.dll` import library is not available, an experimental `generate-import-lib` crate feature may be enabled, and the required library will be created and used by PyO3 automatically. *Note*: MSVC targets require LLVM binutils (`llvm-dlltool`) to be available in `PATH` for @@ -242,12 +242,12 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`. * `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`. -An experimental `pyo3` crate feature `generate-abi3-import-lib` enables the user to cross-compile -"abi3" extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment +An experimental `pyo3` crate feature `generate-import-lib` enables the user to cross-compile +extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate -to generate import libraries for the Stable ABI Python DLL for MinGW-w64 and MSVC compile targets. -*Note*: MSVC targets require LLVM binutils to be available on the host system. -More specifically, `python3-dll-a` requires `llvm-dlltool` executable to be present in `PATH` when +to generate import libraries for the Python DLL for MinGW-w64 and MSVC compile targets. +*Note*: MSVC targets require LLVM binutils or MSVC build tools to be available on the host system. +More specifically, `python3-dll-a` requires `llvm-dlltool` or `lib.exe` executable to be present in `PATH` when targeting `*-pc-windows-msvc`. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): @@ -277,7 +277,7 @@ cargo build --target x86_64-pc-windows-gnu Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples. `PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets, -or when cross compiling "abi3" extension modules for Windows and the experimental `generate-abi3-import-lib` +or when cross compiling extension modules for Windows and the experimental `generate-import-lib` crate feature is enabled. The following resources may also be useful for cross-compiling: diff --git a/guide/src/features.md b/guide/src/features.md index d37059582da..3952425db96 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -30,12 +30,12 @@ These features are extensions of the `abi3` feature to specify the exact minimum See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail. -### `generate-abi3-import-lib` +### `generate-import-lib` -This experimental feature is used to generate import libraries for the Stable ABI Python DLL +This experimental feature is used to generate import libraries for Python DLL for MinGW-w64 and MSVC (cross-)compile targets. -Enabling it allows to (cross-)compile `abi3` extension modules to any Windows targets +Enabling it allows to (cross-)compile extension modules to any Windows targets without having to install the Windows Python distribution files for the target. See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 17cb020c993..f578d62c165 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1409,8 +1409,9 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result { // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - interpreter_config.lib_dir = self::import_lib::generate_abi3_import_lib(&host)?; + let py_version = if interpreter_config.abi3 { + None + } else { + Some(interpreter_config.version) + }; + interpreter_config.lib_dir = self::import_lib::generate_import_lib(&host, py_version)?; } Ok(interpreter_config) diff --git a/pyo3-build-config/src/import_lib.rs b/pyo3-build-config/src/import_lib.rs index ac88c5eefa5..ef941542a50 100644 --- a/pyo3-build-config/src/import_lib.rs +++ b/pyo3-build-config/src/import_lib.rs @@ -7,21 +7,24 @@ use python3_dll_a::ImportLibraryGenerator; use crate::errors::{Context, Result}; -use super::{Architecture, OperatingSystem, Triple}; +use super::{Architecture, OperatingSystem, PythonVersion, Triple}; -/// Generates the `python3.dll` import library for Windows targets. +/// Generates the `python3.dll` or `pythonXY.dll` import library for Windows targets. /// /// Places the generated import library into the build script output directory /// and returns the full library directory path. /// /// Does nothing if the target OS is not Windows. -pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result> { +pub(super) fn generate_import_lib( + target: &Triple, + py_version: Option, +) -> Result> { if target.operating_system != OperatingSystem::Windows { return Ok(None); } - let out_dir = env::var_os("OUT_DIR") - .expect("generate_abi3_import_lib() must be called from a build script"); + let out_dir = + env::var_os("OUT_DIR").expect("generate_import_lib() must be called from a build script"); // Put the newly created import library into the build script output directory. let mut out_lib_dir = PathBuf::from(out_dir); @@ -37,6 +40,7 @@ pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result let env = target.environment.to_string(); ImportLibraryGenerator::new(&arch, &env) + .version(py_version.map(|v| (v.major, v.minor))) .generate(&out_lib_dir) .context("failed to generate python3.dll import library")?; diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 6691f29c1c8..176b0c8df29 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -33,7 +33,9 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"] # Automatically generates `python3.dll` import libraries for Windows targets. -generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"] +generate-import-lib = ["pyo3-build-config/python3-dll-a"] +# Deprecated, replaced by `generate-import-lib` +generate-abi3-import-lib = ["generate-import-lib"] [build-dependencies] From 7104a0cc8e39abb976464a9e1106c35aafa9061a Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 10 May 2022 21:44:35 +0800 Subject: [PATCH 08/16] Add Windows non-abi3 cross compile test --- .github/workflows/ci.yml | 6 ++++++ examples/maturin-starter/Cargo.toml | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07013451403..222f998fe90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -276,11 +276,17 @@ jobs: env: XWIN_ARCH: x86_64 run: | + set -ex sudo apt-get install -y mingw-w64 llvm rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc python -m pip install cargo-xwin + # abi3 cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc + # non-abi3 + export PYO3_CROSS_PYTHON_VERSION=3.9 + cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu + cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc - name: Test cross compile to Windows with maturin if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} uses: messense/maturin-action@v1 diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 64318dcfe87..c7e3d23353f 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] pyo3 = { path = "../../", features = ["extension-module"] } [features] -abi3 = ["pyo3/abi3-py37", "pyo3/generate-import-lib"] +abi3 = ["pyo3/abi3-py37", "generate-import-lib"] +generate-import-lib = ["pyo3/generate-import-lib"] [workspace] From 02e4f197a36311138acd339b88c4c3f10e3c3fcc Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 11 May 2022 15:29:51 +0800 Subject: [PATCH 09/16] Change default python lib name for Windows when cross compiling Stop treating `gnu` abi as mingw by default. --- pyo3-build-config/src/impl_.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index f578d62c165..f0543dc859b 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1391,13 +1391,11 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Date: Wed, 11 May 2022 16:01:51 +0800 Subject: [PATCH 10/16] Update changelog for #2364 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a91f6c7634f..1c53cdd8dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add an experimental `generate-import-lib` feature to support auto-generating non-abi3 python import libraries for Windows targets. [#2364](https://github.com/PyO3/pyo3/pull/2364) - Add FFI definition `Py_ExitStatusException`. [#2374](https://github.com/PyO3/pyo3/pull/2374) +### Changed + +- Deprecate experimental `generate-abi3-import-lib` feature in favor of the new `generate-import-lib` feature. [#2364](https://github.com/PyO3/pyo3/pull/2364) + ### Fixed - Added missing `warn_default_encoding` field to `PyConfig` on 3.10+. The previously missing field could result in incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370) From f05cc91cab8fa786040c15e8aac2dc633d831650 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 14 May 2022 18:16:40 +0800 Subject: [PATCH 11/16] Auto generate Windows import libraries when using a pyo3 config file --- pyo3-build-config/src/impl_.rs | 21 +++++++++++++++++++++ pyo3-build-config/src/lib.rs | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index f0543dc859b..b83e1f13a46 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -456,6 +456,27 @@ print("mingw", get_platform().startswith("mingw")) }) } + pub fn fixup_import_libs(&mut self) -> Result<()> { + let target = target_triple_from_env(); + if self.lib_name.is_none() && target.operating_system == OperatingSystem::Windows { + self.lib_name = Some(default_lib_name_windows( + self.version, + self.implementation, + self.abi3, + false, + )); + } + // Auto generate python3.dll import libraries for Windows targets. + #[cfg(feature = "python3-dll-a")] + { + if self.lib_dir.is_none() { + let py_version = if self.abi3 { None } else { Some(self.version) }; + self.lib_dir = self::import_lib::generate_import_lib(&target, py_version)?; + } + } + Ok(()) + } + #[doc(hidden)] /// Serialize the `InterpreterConfig` and print it to the environment for Cargo to pass along /// to dependent packages during build time. diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index c227d6a321a..6d20fd8f362 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -177,7 +177,9 @@ pub mod pyo3_build_script_impl { #[cfg(feature = "resolve-config")] pub fn resolve_interpreter_config() -> Result { if !CONFIG_FILE.is_empty() { - InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + let mut interperter_config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))?; + interperter_config.fixup_import_libs()?; + Ok(interperter_config) } else if let Some(interpreter_config) = make_cross_compile_config()? { // This is a cross compile and need to write the config file. let path = resolve_cross_compile_config_path() From 1149dcf7d79606ee5456ee3f689b4c21f1ff936e Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 14 May 2022 19:33:16 +0800 Subject: [PATCH 12/16] Allow false positive `clippy::unnecessary-wraps` lint --- pyo3-build-config/src/impl_.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index b83e1f13a46..4fa2a49cb1e 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -456,6 +456,7 @@ print("mingw", get_platform().startswith("mingw")) }) } + #[allow(clippy::unnecessary_wraps)] pub fn fixup_import_libs(&mut self) -> Result<()> { let target = target_triple_from_env(); if self.lib_name.is_none() && target.operating_system == OperatingSystem::Windows { From 11b97d33c5dc41f6d04ebdd987045eea4ee2c7fe Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 3 May 2022 06:18:45 +0100 Subject: [PATCH 13/16] Fix CI for hashbrown 0.12.1 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 222f998fe90..b1a4e4cb6f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,7 @@ jobs: run: | set -x cargo update -p indexmap --precise 1.6.2 - cargo update -p hashbrown:0.12.0 --precise 0.9.1 + cargo update -p hashbrown:0.12.1 --precise 0.9.1 PROJECTS=("." "examples/decorator" "examples/maturin-starter" "examples/setuptools-rust-starter" "examples/word-count") for PROJ in ${PROJECTS[@]}; do cargo update --manifest-path "$PROJ/Cargo.toml" -p parking_lot --precise 0.11.0 From c4414f3cc18492f81b15dec9c1d82a9e4d2a4b2a Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 11 May 2022 02:26:53 +0800 Subject: [PATCH 14/16] Remove `#[doc(hidden)]` from trait impl items (#2365) See https://github.com/rust-lang/rust/pull/96008 --- src/internal_tricks.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 1f227bcb64e..9cb53336254 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -13,7 +13,6 @@ macro_rules! private_decl { macro_rules! private_impl { () => { - #[doc(hidden)] fn __private__(&self) -> crate::internal_tricks::PrivateMarker { crate::internal_tricks::PrivateMarker } From da7418754aa5c48e252fb7185126d05aa3905a1e Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Tue, 10 May 2022 02:00:56 -0400 Subject: [PATCH 15/16] Updating debugging docs with more info on rust-gdb (#2361) --- guide/src/debugging.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/guide/src/debugging.md b/guide/src/debugging.md index 3a624d2d392..c29930e8bf4 100644 --- a/guide/src/debugging.md +++ b/guide/src/debugging.md @@ -32,9 +32,16 @@ Run Valgrind with `valgrind --suppressions=valgrind-python.supp ./my-command --w ## Getting a stacktrace -The best start to investigate a crash such as an segmentation fault is a backtrace. +The best start to investigate a crash such as an segmentation fault is a backtrace. You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`. Alternatively you can use a debugger such as `gdb` to explore the issue. Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables. Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)). The mentioned issue contains a workaround for enabling pretty-printers in this case. * Link against a debug build of python as described in the previous chapter - * Run `gdb ` + * Run `rust-gdb ` + * Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!` * Enter `r` to run * After the crash occurred, enter `bt` or `bt full` to print the stacktrace + + Often it is helpful to run a small piece of Python code to exercise a section of Rust. + + ```console + rust-gdb --args python -c "import my_package; my_package.sum_to_string(1, 2)" + ``` From 456a96d6eb3371b0a78aa317c5f04c00ab41f903 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 15 May 2022 06:33:15 +0100 Subject: [PATCH 16/16] release: 0.16.5 --- CHANGELOG.md | 5 +++-- Cargo.toml | 8 ++++---- README.md | 4 ++-- examples/Cargo.toml | 2 +- examples/decorator/.template/pre-script.rhai | 2 +- examples/maturin-starter/.template/pre-script.rhai | 2 +- .../setuptools-rust-starter/.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 ++-- pyo3-macros-backend/Cargo.toml | 2 +- pyo3-macros/Cargo.toml | 4 ++-- 12 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c53cdd8dc8..39b65d41cd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.16.5] - 2022-05-15 ### Added @@ -1165,7 +1165,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.16.4...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.16.5...HEAD +[0.16.5]: https://github.com/pyo3/pyo3/compare/v0.16.4...v0.16.5 [0.16.3]: https://github.com/pyo3/pyo3/compare/v0.16.3...v0.16.4 [0.16.3]: https://github.com/pyo3/pyo3/compare/v0.16.2...v0.16.3 [0.16.2]: https://github.com/pyo3/pyo3/compare/v0.16.1...v0.16.2 diff --git a/Cargo.toml b/Cargo.toml index 51805f3f539..80aab8fc85a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.16.4" +version = "0.16.5" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -19,10 +19,10 @@ libc = "0.2.62" parking_lot = ">= 0.11, < 0.13" # ffi bindings to the python interpreter, split into a seperate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.16.4" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.16.5" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.16.4", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.16.5", optional = true } indoc = { version = "1.0.3", optional = true } unindent = { version = "0.1.4", optional = true } @@ -51,7 +51,7 @@ serde_json = "1.0.61" widestring = "0.5.1" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.16.4", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "0.16.5", features = ["resolve-config"] } [features] default = ["macros", "pyproto"] diff --git a/README.md b/README.md index 01637bbcc88..877608c2b22 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.16.4", features = ["extension-module"] } +pyo3 = { version = "0.16.5", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -132,7 +132,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.16.4" +version = "0.16.5" features = ["auto-initialize"] ``` diff --git a/examples/Cargo.toml b/examples/Cargo.toml index a46e14f5641..c31e0c19f90 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -5,7 +5,7 @@ publish = false edition = "2018" [dev-dependencies] -pyo3 = { version = "0.16.4", path = "..", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.16.5", path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 7ba10b00c8b..7d68d392bbd 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.16.4"); +variable::set("PYO3_VERSION", "0.16.5"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::rename(".template/tox.ini", "tox.ini"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 7ba10b00c8b..7d68d392bbd 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.16.4"); +variable::set("PYO3_VERSION", "0.16.5"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::rename(".template/tox.ini", "tox.ini"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 0dd10361442..566a8ec7fe7 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.16.4"); +variable::set("PYO3_VERSION", "0.16.5"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::rename(".template/tox.ini", "tox.ini"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 778da32d61c..dfec4963564 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.16.4"); +variable::set("PYO3_VERSION", "0.16.5"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/tox.ini", "tox.ini"); file::delete(".template"); diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 77460cf9a7e..4e05e4cea86 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.16.4" +version = "0.16.5" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 176b0c8df29..b14f33fb435 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.16.4" +version = "0.16.5" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -39,4 +39,4 @@ generate-abi3-import-lib = ["generate-import-lib"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.16.4", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.16.5", features = ["resolve-config"] } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index b8e543cd094..646598cd5af 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.16.4" +version = "0.16.5" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 7758fb3bc5a..8f7a3d2b897 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.16.4" +version = "0.16.5" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -23,4 +23,4 @@ abi3 = ["pyo3-macros-backend/abi3"] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "1.0.56", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.16.4" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.16.5" }