From a212be815f877140409f782eb11463767d1c6c90 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Mon, 9 Aug 2021 09:07:57 -0500 Subject: [PATCH 01/32] Fixed broken link in async-await.md --- guide/src/ecosystem/async-await.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 8daadf38e51..fc90020ec74 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -498,4 +498,4 @@ fn main() -> PyResult<()> { ## Additional Information - Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/testing) \ No newline at end of file +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) \ No newline at end of file From 3a3706e13b8515d3abf9f2a91fa43e9e22dfd83d Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 10 Aug 2021 07:50:52 +0100 Subject: [PATCH 02/32] guide: don't bother doctesting async guide --- guide/src/ecosystem/async-await.md | 74 +++++++++++++++--------------- src/lib.rs | 3 ++ 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index fc90020ec74..5e58cd3dcef 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -1,14 +1,14 @@ # Async / Await -If you are working with a Python library that makes use of async functions or wish to provide +If you are working with a Python library that makes use of async functions or wish to provide Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) -likely has the tools you need. It provides conversions between async functions in both Python and -Rust and was designed with first-class support for popular Rust runtimes such as -[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python -code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing +likely has the tools you need. It provides conversions between async functions in both Python and +Rust and was designed with first-class support for popular Rust runtimes such as +[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python +code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing Python libraries. -In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call +In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call async Python functions with PyO3, how to call async Rust functions from Python, and how to configure your codebase to manage the runtimes of both. @@ -163,13 +163,13 @@ maturin develop && python3 🔗 Found pyo3 bindings 🐍 Found CPython 3.8 at python3 Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.5 (default, Jan 27 2021, 15:41:15) +Python 3.8.5 (default, Jan 27 2021, 15:41:15) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import asyncio >>> >>> from my_async_module import rust_sleep ->>> +>>> >>> async def main(): >>> await rust_sleep() >>> @@ -188,19 +188,19 @@ async def py_sleep(): await asyncio.sleep(1) ``` -**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, +**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, we really don't need to know much about these `coroutine` objects. The key factor here is that calling an `async` function is _just like calling a regular function_, the only difference is that we have to do something special with the object that it returns. -Normally in Python, that something special is the `await` keyword, but in order to await this -coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. -That's where `pyo3-asyncio` comes in. -[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) +Normally in Python, that something special is the `await` keyword, but in order to await this +coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. +That's where `pyo3-asyncio` comes in. +[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) performs this conversion for us: -```rust no_run +```rust use pyo3::prelude::*; #[pyo3_asyncio::tokio::main] @@ -209,11 +209,11 @@ async fn main() -> PyResult<()> { // import the module containing the py_sleep function let example = py.import("example")?; - // calling the py_sleep method like a normal function + // calling the py_sleep method like a normal function // returns a coroutine let coroutine = example.call_method0("py_sleep")?; - // convert the coroutine into a Rust future using the + // convert the coroutine into a Rust future using the // tokio runtime pyo3_asyncio::tokio::into_future(coroutine) })?; @@ -225,12 +225,12 @@ async fn main() -> PyResult<()> { } ``` -> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the +> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the > [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. ## Awaiting a Rust Future in Python -Here we have the same async function as before written in Rust using the +Here we have the same async function as before written in Rust using the [`async-std`](https://async.rs/) runtime: ```rust @@ -243,12 +243,12 @@ async fn rust_sleep() { Similar to Python, Rust's async functions also return a special object called a `Future`: -```rust compile_fail +```rust let future = rust_sleep(); ``` -We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you -can use the `await` keyword with it. In order to do this, we'll call +We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you +can use the `await` keyword with it. In order to do this, we'll call [`pyo3_asyncio::async_std::future_into_py`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.future_into_py.html): ```rust @@ -284,38 +284,38 @@ doesn't always play well with Rust. Luckily, Rust's event loops are pretty flexible and don't _need_ control over the main thread, so in `pyo3-asyncio`, we decided the best way to handle Rust/Python interop was to just surrender the main -thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop +thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop implementations _prefer_ control over the main thread, this can still make some things awkward. ### PyO3 Asyncio Initialization Because Python needs to control the main thread, we can't use the convenient proc macros from Rust -runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main +runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main thread must block on [`pyo3_asyncio::run_forever`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.run_forever.html) or [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) since it's not a good idea to make long blocking calls during an async function. > Internally, these `#[main]` proc macros are expanded to something like this: -> ```rust compile_fail +> ```rust > fn main() { > // your async main fn > async fn _main_impl() { /* ... */ } -> Runtime::new().block_on(_main_impl()); +> Runtime::new().block_on(_main_impl()); > } > ``` > Making a long blocking call inside the `Future` that's being driven by `block_on` prevents that -> thread from doing anything else and can spell trouble for some runtimes (also this will actually -> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism -> that can avoid this problem, but again that's not something we can use here since we need it to +> thread from doing anything else and can spell trouble for some runtimes (also this will actually +> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism +> that can avoid this problem, but again that's not something we can use here since we need it to > block on the _main_ thread. -For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this -initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` +For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this +initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` while also satisfying the Python runtime's needs. Here's a full example of PyO3 initialization with the `async-std` runtime: -```rust no_run +```rust use pyo3::prelude::*; #[pyo3_asyncio::async_std::main] @@ -429,19 +429,19 @@ $ maturin develop && python3 🔗 Found pyo3 bindings 🐍 Found CPython 3.8 at python3 Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.8 (default, Apr 13 2021, 19:58:26) +Python 3.8.8 (default, Apr 13 2021, 19:58:26) [GCC 7.3.0] :: Anaconda, Inc. on linux Type "help", "copyright", "credits" or "license" for more information. >>> import asyncio >>> import uvloop ->>> +>>> >>> import my_async_module ->>> +>>> >>> uvloop.install() ->>> +>>> >>> async def main(): ... await my_async_module.rust_sleep() -... +... >>> asyncio.run(main()) >>> ``` @@ -498,4 +498,4 @@ fn main() -> PyResult<()> { ## Additional Information - Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) \ No newline at end of file +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) diff --git a/src/lib.rs b/src/lib.rs index 8089d503119..56b0154b241 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -545,4 +545,7 @@ pub mod doc_test { doctest!("guide/src/trait_bounds.md", guide_trait_bounds_md); doctest!("guide/src/types.md", guide_types_md); doctest!("guide/src/faq.md", faq); + + // deliberate choice not to test guide/ecosystem because those pages depend on external crates + // such as pyo3_asyncio. } From 5a5a3af638c8941145c126c261a4656f076b01ed Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 11 Aug 2021 23:49:01 +0100 Subject: [PATCH 03/32] changelog: fix missing PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08b11089d70..22dc80028eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add `indexmap` feature to add `ToPyObject`, `IntoPy` and `FromPyObject` implementations for `indexmap::IndexMap`. [#1728](https://github.com/PyO3/pyo3/pull/1728) - Add `pyo3_build_config::add_extension_module_link_args()` to use in build scripts to set linker arguments (for macOS). [#1755](https://github.com/PyO3/pyo3/pull/1755) -- Add `Python::with_gil_unchecked()` unsafe variation of `Python::with_gil()` to allow obtaining a `Python` in scenarios where `Python::with_gil()` would fail. +- Add `Python::with_gil_unchecked()` unsafe variation of `Python::with_gil()` to allow obtaining a `Python` in scenarios where `Python::with_gil()` would fail. [#1769](https://github.com/PyO3/pyo3/pull/1769) ### Changed From 74f4bfacacc57d7c54e535f87c0dc6eea09855e2 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 11 Aug 2021 23:50:58 +0100 Subject: [PATCH 04/32] guide: allow using newer gh-actions-pages version --- .github/workflows/guide.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/guide.yml b/.github/workflows/guide.yml index fce804bd4a4..9f67ea72d46 100644 --- a/.github/workflows/guide.yml +++ b/.github/workflows/guide.yml @@ -50,7 +50,7 @@ jobs: - name: Deploy if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - uses: peaceiris/actions-gh-pages@v3.7.0-8 + uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./gh-pages-build/ @@ -71,7 +71,7 @@ jobs: ln -sfT $TAG_NAME public/latest - name: Deploy - uses: peaceiris/actions-gh-pages@v3.7.0-8 + uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./public/ From aef1637b0a599d3d3f2023fc1a6e93271247ae6c Mon Sep 17 00:00:00 2001 From: Sanskar Jethi Date: Mon, 14 Jun 2021 01:35:56 +0530 Subject: [PATCH 05/32] Add documentation to call async python from rust --- guide/src/ecosystem/async-await.md | 59 +++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 5e58cd3dcef..7c92ea4641a 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -197,8 +197,10 @@ Normally in Python, that something special is the `await` keyword, but in order coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. That's where `pyo3-asyncio` comes in. [`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) -performs this conversion for us: +performs this conversion for us. +The following example uses `into_future` to call the `py_sleep` function shown above and then await the +coroutine object returned from the call: ```rust use pyo3::prelude::*; @@ -225,6 +227,61 @@ async fn main() -> PyResult<()> { } ``` +Alternatively, the below example shows how to write a `#[pyfunction]` which uses `into_future` to receive and await +a coroutine argument: + +```rust +#[pyfunction] +fn await_coro(coro: &PyAny) -> PyResult<()> { + // convert the coroutine into a Rust future using the + // async_std runtime + let f = pyo3_asyncio::async_std::into_future(coro)?; + + pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { + // await the future + f.await?; + Ok(()) + }) +} +``` + +This could be called from Python as: + +```python +import asyncio + +async def py_sleep(): + asyncio.sleep(1) + +await_coro(py_sleep()) +``` + +If for you wanted to pass a callable function to the `#[pyfunction]` instead, (i.e. the last line becomes `await_coro(py_sleep))`, then the above example needs to be tweaked to first call the callable to get the coroutine: + +```rust +#[pyfunction] +fn await_coro(callable: &PyAny) -> PyResult<()> { + // get the coroutine by calling the callable + let coro = callable.call0()?; + + // convert the coroutine into a Rust future using the + // async_std runtime + let f = pyo3_asyncio::async_std::into_future(coro)?; + + pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { + // await the future + f.await?; + Ok(()) + }) +} +``` + +This can be particularly helpful where you need to repeatedly create and await a coroutine. Trying to await the same coroutine multiple times will raise an error: + +```python +RuntimeError: cannot reuse already awaited coroutine +``` + > If you're interested in learning more about `coroutines` and `awaitables` in general, check out the > [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. From 4f4a7180762ae5011ce4aa70dea6391849b2838b Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 12 Aug 2021 00:27:49 +0100 Subject: [PATCH 06/32] ci: pin bitflags for MSRV job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8935329c6f..b671a5a65df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,7 @@ jobs: run: | cargo update -p indexmap --precise 1.6.2 cargo update -p hashbrown:0.11.2 --precise 0.9.1 + cargo update -p bitflags --precise 1.2.1 - name: Build docs run: cargo doc --no-deps --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}" From 62812f3fd93f97a3c58b6f2b1f5a978d24453cc0 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 12 Aug 2021 08:02:07 +0100 Subject: [PATCH 07/32] ffi: cleanup pystate --- CHANGELOG.md | 6 ++++ src/ffi/cpython/ceval.rs | 3 +- src/ffi/cpython/mod.rs | 6 ++-- src/ffi/cpython/pystate.rs | 57 ++++++++++++++++++++++++++++++++++++-- src/ffi/pystate.rs | 57 +++++++++++++++++++++++++------------- 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22dc80028eb..1329bea1106 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) + ## [0.14.2] - 2021-08-09 ### Added diff --git a/src/ffi/cpython/ceval.rs b/src/ffi/cpython/ceval.rs index 13173b1b146..29affbd72bb 100644 --- a/src/ffi/cpython/ceval.rs +++ b/src/ffi/cpython/ceval.rs @@ -1,5 +1,4 @@ -use crate::ffi::object::{freefunc, PyObject}; -use crate::ffi::pystate::Py_tracefunc; +use crate::ffi::{freefunc, PyObject, Py_tracefunc}; use std::os::raw::c_int; extern "C" { diff --git a/src/ffi/cpython/mod.rs b/src/ffi/cpython/mod.rs index 7e8c9c6aec0..19f3ccd12f1 100644 --- a/src/ffi/cpython/mod.rs +++ b/src/ffi/cpython/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod abstract_; // skipped bytearrayobject.h #[cfg(not(PyPy))] pub(crate) mod bytesobject; +#[cfg(not(PyPy))] pub(crate) mod ceval; pub(crate) mod code; pub(crate) mod compile; @@ -18,13 +19,12 @@ pub(crate) mod object; pub(crate) mod pydebug; #[cfg(all(Py_3_8, not(PyPy)))] pub(crate) mod pylifecycle; - -#[cfg(all(Py_3_8, not(PyPy)))] pub(crate) mod pystate; pub use self::abstract_::*; #[cfg(not(PyPy))] pub use self::bytesobject::*; +#[cfg(not(PyPy))] pub use self::ceval::*; pub use self::code::*; pub use self::compile::*; @@ -39,6 +39,4 @@ pub use self::object::*; pub use self::pydebug::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; - -#[cfg(all(Py_3_8, not(PyPy)))] pub use self::pystate::*; diff --git a/src/ffi/cpython/pystate.rs b/src/ffi/cpython/pystate.rs index 38491baeb31..1564815dd59 100644 --- a/src/ffi/cpython/pystate.rs +++ b/src/ffi/cpython/pystate.rs @@ -1,7 +1,19 @@ -use crate::ffi::pystate::{PyInterpreterState, PyThreadState}; +#[cfg(not(PyPy))] +use crate::ffi::PyThreadState; +use crate::ffi::{PyFrameObject, PyInterpreterState, PyObject}; use std::os::raw::c_int; -// Py_tracefunc is defined in ffi::pystate +// skipped _PyInterpreterState_RequiresIDRef +// skipped _PyInterpreterState_RequireIDRef + +// skipped _PyInterpreterState_GetMainModule + +pub type Py_tracefunc = extern "C" fn( + obj: *mut PyObject, + frame: *mut PyFrameObject, + what: c_int, + arg: *mut PyObject, +) -> c_int; pub const PyTrace_CALL: c_int = 0; pub const PyTrace_EXCEPTION: c_int = 1; @@ -12,13 +24,38 @@ pub const PyTrace_C_EXCEPTION: c_int = 5; pub const PyTrace_C_RETURN: c_int = 6; pub const PyTrace_OPCODE: c_int = 7; +// skipped PyTraceInfo +// skipped CFrame +// skipped _PyErr_StackItem +// skipped _PyStackChunk +// skipped _ts (aka PyThreadState) + extern "C" { - // PyGILState_Check is defined in ffi::pystate + // skipped _PyThreadState_Prealloc + // skipped _PyThreadState_UncheckedGet + // skipped _PyThreadState_GetDict + + #[cfg_attr(PyPy, link_name = "_PyPyGILState_Check")] + pub fn PyGILState_Check() -> c_int; + + // skipped _PyGILState_GetInterpreterStateUnsafe + // skipped _PyThread_CurrentFrames + // skipped _PyThread_CurrentExceptions + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Main() -> *mut PyInterpreterState; + #[cfg_attr(PyPy, link_name = "_PyPyInterpreterState_Head")] pub fn PyInterpreterState_Head() -> *mut PyInterpreterState; + #[cfg_attr(PyPy, link_name = "_PyPyInterpreterState_Next")] pub fn PyInterpreterState_Next(interp: *mut PyInterpreterState) -> *mut PyInterpreterState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_ThreadHead(interp: *mut PyInterpreterState) -> *mut PyThreadState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyThreadState_Next(tstate: *mut PyThreadState) -> *mut PyThreadState; + // skipped PyThreadState_DeleteCurrent } #[cfg(Py_3_9)] @@ -44,3 +81,17 @@ extern "C" { eval_frame: _PyFrameEvalFunction, ); } + +// skipped _PyInterpreterState_GetConfig +// skipped _PyInterpreterState_GetConfigCopy +// skipped _PyInterpreterState_SetConfig +// skipped _Py_GetConfig + +// skipped _PyCrossInterpreterData +// skipped _PyObject_GetCrossInterpreterData +// skipped _PyCrossInterpreterData_NewObject +// skipped _PyCrossInterpreterData_Release +// skipped _PyObject_CheckCrossInterpreterData +// skipped crossinterpdatafunc +// skipped _PyCrossInterpreterData_RegisterClass +// skipped _PyCrossInterpreterData_Lookup diff --git a/src/ffi/pystate.rs b/src/ffi/pystate.rs index 81631851e6b..e26619e7ec8 100644 --- a/src/ffi/pystate.rs +++ b/src/ffi/pystate.rs @@ -1,26 +1,49 @@ +#[cfg(not(PyPy))] use crate::ffi::moduleobject::PyModuleDef; use crate::ffi::object::PyObject; -use crate::ffi::PyFrameObject; -use std::os::raw::{c_int, c_long}; +use std::os::raw::c_int; +#[cfg(not(PyPy))] +use std::os::raw::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; +opaque_struct!(PyThreadState); opaque_struct!(PyInterpreterState); -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyThreadState { - pub ob_base: PyObject, - pub interp: *mut PyInterpreterState, -} - extern "C" { + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_New() -> *mut PyInterpreterState; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Clear(arg1: *mut PyInterpreterState); + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Delete(arg1: *mut PyInterpreterState); - //fn _PyState_AddModule(arg1: *mut PyObject, - // arg2: *mut PyModuleDef) -> c_int; + + #[cfg(all(Py_3_9, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_9, not(PyPy))))] + pub fn PyInterpreterState_Get() -> *mut PyInterpreterState; + + #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_8, not(PyPy))))] + pub fn PyInterpreterState_GetDict() -> *mut PyObject; + + #[cfg(all(Py_3_7, not(PyPy)))] + #[cfg_attr(docsrs, doc(all(Py_3_7, not(PyPy))))] + pub fn PyInterpreterState_GetID() -> i64; + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] + pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; + + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyThreadState_New")] pub fn PyThreadState_New(arg1: *mut PyInterpreterState) -> *mut PyThreadState; //fn _PyThreadState_Prealloc(arg1: *mut PyInterpreterState) @@ -39,6 +62,8 @@ extern "C" { pub fn PyThreadState_Swap(arg1: *mut PyThreadState) -> *mut PyThreadState; #[cfg_attr(PyPy, link_name = "PyPyThreadState_GetDict")] pub fn PyThreadState_GetDict() -> *mut PyObject; + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyThreadState_SetAsyncExc(arg1: c_long, arg2: *mut PyObject) -> c_int; } @@ -54,18 +79,12 @@ extern "C" { pub fn PyGILState_Ensure() -> PyGILState_STATE; #[cfg_attr(PyPy, link_name = "PyPyGILState_Release")] pub fn PyGILState_Release(arg1: PyGILState_STATE); + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyGILState_GetThisThreadState() -> *mut PyThreadState; - pub fn PyGILState_Check() -> c_int; } #[inline] pub unsafe fn PyThreadState_GET() -> *mut PyThreadState { PyThreadState_Get() } - -pub type Py_tracefunc = extern "C" fn( - obj: *mut PyObject, - frame: *mut PyFrameObject, - what: c_int, - arg: *mut PyObject, -) -> c_int; From a162563754f311e326a7389e1331c0716182ae55 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:07:15 +0100 Subject: [PATCH 08/32] ffi: fix PyPy symbols for `cpython/pystate.rs` --- src/ffi/cpython/pystate.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ffi/cpython/pystate.rs b/src/ffi/cpython/pystate.rs index 1564815dd59..7d2f9abf6a6 100644 --- a/src/ffi/cpython/pystate.rs +++ b/src/ffi/cpython/pystate.rs @@ -35,7 +35,7 @@ extern "C" { // skipped _PyThreadState_UncheckedGet // skipped _PyThreadState_GetDict - #[cfg_attr(PyPy, link_name = "_PyPyGILState_Check")] + #[cfg_attr(PyPy, link_name = "PyPyGILState_Check")] pub fn PyGILState_Check() -> c_int; // skipped _PyGILState_GetInterpreterStateUnsafe @@ -45,9 +45,9 @@ extern "C" { #[cfg(not(PyPy))] #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyInterpreterState_Main() -> *mut PyInterpreterState; - #[cfg_attr(PyPy, link_name = "_PyPyInterpreterState_Head")] + #[cfg_attr(PyPy, link_name = "PyPyInterpreterState_Head")] pub fn PyInterpreterState_Head() -> *mut PyInterpreterState; - #[cfg_attr(PyPy, link_name = "_PyPyInterpreterState_Next")] + #[cfg_attr(PyPy, link_name = "PyPyInterpreterState_Next")] pub fn PyInterpreterState_Next(interp: *mut PyInterpreterState) -> *mut PyInterpreterState; #[cfg(not(PyPy))] #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] From b87a08a1a7426f0bfdfcf198877ddbe0effd4496 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 13 Aug 2021 14:00:29 +0100 Subject: [PATCH 09/32] examples: make `word-count` example comparison fairer --- examples/word-count/src/lib.rs | 24 ++------------------ examples/word-count/tests/test_word_count.py | 1 - examples/word-count/word_count/__init__.py | 7 +++--- 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/examples/word-count/src/lib.rs b/examples/word-count/src/lib.rs index 1a078c6bdae..96f9f9e2fae 100644 --- a/examples/word-count/src/lib.rs +++ b/examples/word-count/src/lib.rs @@ -1,6 +1,3 @@ -// Source adopted from -// https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs - use pyo3::prelude::*; use rayon::prelude::*; @@ -24,28 +21,11 @@ fn search_sequential_allow_threads(py: Python, contents: &str, needle: &str) -> py.allow_threads(|| search_sequential(contents, needle)) } -fn matches(word: &str, needle: &str) -> bool { - let mut needle = needle.chars(); - for ch in word.chars().skip_while(|ch| !ch.is_alphabetic()) { - match needle.next() { - None => { - return !ch.is_alphabetic(); - } - Some(expect) => { - if ch.to_lowercase().next() != Some(expect) { - return false; - } - } - } - } - needle.next().is_none() -} - -/// Count the occurences of needle in line, case insensitive +/// Count the occurrences of needle in line, case insensitive fn count_line(line: &str, needle: &str) -> usize { let mut total = 0; for word in line.split(' ') { - if matches(word, needle) { + if word == needle { total += 1; } } diff --git a/examples/word-count/tests/test_word_count.py b/examples/word-count/tests/test_word_count.py index cb633622deb..5991e4ae1de 100644 --- a/examples/word-count/tests/test_word_count.py +++ b/examples/word-count/tests/test_word_count.py @@ -1,7 +1,6 @@ from concurrent.futures import ThreadPoolExecutor import pytest - import word_count diff --git a/examples/word-count/word_count/__init__.py b/examples/word-count/word_count/__init__.py index 8998f1e4d4b..8ce7a175471 100644 --- a/examples/word-count/word_count/__init__.py +++ b/examples/word-count/word_count/__init__.py @@ -8,11 +8,10 @@ ] -def search_py(contents, needle): +def search_py(contents: str, needle: str) -> int: total = 0 - for line in contents.split(): - words = line.split(" ") - for word in words: + for line in contents.splitlines(): + for word in line.split(" "): if word == needle: total += 1 return total From b0fed298135c8441cb8ab09e46f5222453c96266 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 13 Aug 2021 20:51:26 -0700 Subject: [PATCH 10/32] ffi: fix PyStatus._type The field wasn't defined previously. And the enum wasn't defined as `[repr(C)]`. This missing field could result in memory corruption if a Rust-allocated `PyStatus` was passed to a Python API, which could perform an out-of-bounds write. In my code, the out-of-bounds write corrupted a variable on the stack, leading to a segfault due to illegal memory access. However, this crash only occurred on Rust 1.54! So I initially mis-attribted it as a compiler bug / regression. It appears that a low-level Rust change in 1.54.0 changed the LLVM IR in such a way to cause LLVM optimization passes to produce sufficiently different assembly code, tickling the crash. See https://github.com/rust-lang/rust/issues/87947 if you want to see the wild goose chase I went on in Rust / LLVM land to potentially pin this on a compiler bug. Lessen learned: Rust crashes are almost certainly due to use of `unsafe`. --- CHANGELOG.md | 1 + src/ffi/cpython/initconfig.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1329bea1106..c4fb0cc8434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) +- Add missing `_type` field to `PyStatus` struct definition. ## [0.14.2] - 2021-08-09 diff --git a/src/ffi/cpython/initconfig.rs b/src/ffi/cpython/initconfig.rs index ee6fc75b53d..95d28b02bf3 100644 --- a/src/ffi/cpython/initconfig.rs +++ b/src/ffi/cpython/initconfig.rs @@ -4,6 +4,7 @@ use crate::ffi::Py_ssize_t; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_ulong}; +#[repr(C)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum _PyStatus_TYPE { _PyStatus_TYPE_OK = 0, @@ -14,6 +15,7 @@ pub enum _PyStatus_TYPE { #[repr(C)] #[derive(Copy, Clone)] pub struct PyStatus { + pub _type: _PyStatus_TYPE, pub func: *const c_char, pub err_msg: *const c_char, pub exitcode: c_int, From 08f1068a1b3653917afa0db57a621390b6dc2344 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 14 Aug 2021 07:40:52 +0100 Subject: [PATCH 11/32] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4fb0cc8434..47210574807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) -- Add missing `_type` field to `PyStatus` struct definition. +- Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) ## [0.14.2] - 2021-08-09 From a783a8da2c10bb261b747bf956bcc26d1430e2b3 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 10 Aug 2021 17:48:50 -0700 Subject: [PATCH 12/32] macros: raise AttributeError on property deletion requests The setter function will receive a NULL value on deletion requests. This wasn't properly handled before, leading to a panic. The new code raises AttributeError in this scenario instead. A test for the behavior has been added. Documentation has also been updated to reflect the behavior. --- CHANGELOG.md | 1 + guide/src/class.md | 4 ++++ pyo3-macros-backend/src/pymethod.rs | 6 +++++- tests/test_getter_setter.rs | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47210574807..1b3d06a3019 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) +- Raise `AttributeError` to avoid panic when calling `del` on a `[setter]` defined class property. [#1775](https://github.com/PyO3/pyo3/issues/1775) ## [0.14.2] - 2021-08-09 diff --git a/guide/src/class.md b/guide/src/class.md index 3b5d3509cfb..27e662a7a26 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -454,6 +454,10 @@ impl MyClass { In this case, the property `number` is defined and available from Python code as `self.number`. +Attributes defined by `#[setter]` or `#[pyo3(set)]` will always raise `AttributeError` on `del` +operations. Support for defining custom `del` behavior is tracked in +[#1778](https://github.com/PyO3/pyo3/issues/1778). + ## Instance methods To define a Python compatible method, an `impl` block for your struct has to be annotated with the diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 82b70fe665a..9af38f021ab 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -218,7 +218,11 @@ pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Resul ) -> std::os::raw::c_int { pyo3::callback::handle_panic(|_py| { #slf - let _value = _py.from_borrowed_ptr::(_value); + let _value = _py + .from_borrowed_ptr_or_opt(_value) + .ok_or_else(|| { + pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") + })?; let _val = pyo3::FromPyObject::extract(_value)?; pyo3::callback::convert(_py, #setter_impl) diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 53e99803bb7..38b9761ae8e 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -54,6 +54,8 @@ fn class_with_properties() { py_run!(py, inst, "inst.DATA = 20"); py_run!(py, inst, "assert inst.get_num() == 20 == inst.DATA"); + py_expect_exception!(py, inst, "del inst.DATA", PyAttributeError); + py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 20"); py_run!(py, inst, "inst.unwrapped = 42"); py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 42"); From 1741a03aefb185451e227f4b217c94f87d9b27ed Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 14 Aug 2021 07:43:32 +0100 Subject: [PATCH 13/32] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3d06a3019..7b846b4668d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) -- Raise `AttributeError` to avoid panic when calling `del` on a `[setter]` defined class property. [#1775](https://github.com/PyO3/pyo3/issues/1775) +- Raise `AttributeError` to avoid panic when calling `del` on a `#[setter]` defined class property. [#1779](https://github.com/PyO3/pyo3/issues/1779) ## [0.14.2] - 2021-08-09 From 90acff47c62433fbf77187f9dd69c94d0cea9b22 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 10 Aug 2021 13:17:23 -0700 Subject: [PATCH 14/32] ffi: define some cpython/unicodeobject bindings pyo3 doesn't currently define various Unicode bindings that allow the retrieval of raw data from Python strings. Said bindings are a prerequisite to possibly exposing this data in the Rust API (#1776). Even if those high-level APIs never materialize, the FFI bindings are necessary to enable consumers of the raw C API to utilize them. This commit partially defines the FFI bindings as defined in CPython's Include/cpython/unicodeobject.h file. I used the latest CPython 3.9 Git commit for defining the order of the symbols and the implementation of various inline preprocessor macros. I tried to be as faithful as possible to the original implementation, preserving intermediate `#define`s as inline functions. Missing symbols have been annotated with `skipped` and symbols currently defined in `src/ffi/unicodeobject.rs` have been annotated with `move`. The `state` field of `PyASCIIObject` is a bitfield, which Rust doesn't support. So we've provided accessor functions for retrieving these fields' values. No accessor functions are present because you shouldn't be touching these values from Rust code. Tests of the bitfield APIs and macro implementations have been added. --- src/ffi/cpython/mod.rs | 2 + src/ffi/cpython/unicodeobject.rs | 467 +++++++++++++++++++++++++++++++ 2 files changed, 469 insertions(+) create mode 100644 src/ffi/cpython/unicodeobject.rs diff --git a/src/ffi/cpython/mod.rs b/src/ffi/cpython/mod.rs index 19f3ccd12f1..b1701d1794e 100644 --- a/src/ffi/cpython/mod.rs +++ b/src/ffi/cpython/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod pydebug; #[cfg(all(Py_3_8, not(PyPy)))] pub(crate) mod pylifecycle; pub(crate) mod pystate; +pub(crate) mod unicodeobject; pub use self::abstract_::*; #[cfg(not(PyPy))] @@ -40,3 +41,4 @@ pub use self::pydebug::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; pub use self::pystate::*; +pub use self::unicodeobject::*; diff --git a/src/ffi/cpython/unicodeobject.rs b/src/ffi/cpython/unicodeobject.rs new file mode 100644 index 00000000000..160e7d0f697 --- /dev/null +++ b/src/ffi/cpython/unicodeobject.rs @@ -0,0 +1,467 @@ +use crate::ffi::{PyObject, PyUnicode_Check, Py_UCS1, Py_UCS2, Py_UCS4, Py_hash_t, Py_ssize_t}; +use libc::wchar_t; +use std::os::raw::{c_char, c_int, c_uint, c_void}; + +// skipped Py_UNICODE_ISSPACE() +// skipped Py_UNICODE_ISLOWER() +// skipped Py_UNICODE_ISUPPER() +// skipped Py_UNICODE_ISTITLE() +// skipped Py_UNICODE_ISLINEBREAK +// skipped Py_UNICODE_TOLOWER +// skipped Py_UNICODE_TOUPPER +// skipped Py_UNICODE_TOTITLE +// skipped Py_UNICODE_ISDECIMAL +// skipped Py_UNICODE_ISDIGIT +// skipped Py_UNICODE_ISNUMERIC +// skipped Py_UNICODE_ISPRINTABLE +// skipped Py_UNICODE_TODECIMAL +// skipped Py_UNICODE_TODIGIT +// skipped Py_UNICODE_TONUMERIC +// skipped Py_UNICODE_ISALPHA +// skipped Py_UNICODE_ISALNUM +// skipped Py_UNICODE_COPY +// skipped Py_UNICODE_FILL +// skipped Py_UNICODE_IS_SURROGATE +// skipped Py_UNICODE_IS_HIGH_SURROGATE +// skipped Py_UNICODE_IS_LOW_SURROGATE +// skipped Py_UNICODE_JOIN_SURROGATES +// skipped Py_UNICODE_HIGH_SURROGATE +// skipped Py_UNICODE_LOW_SURROGATE + +#[repr(C)] +pub struct PyASCIIObject { + pub ob_base: PyObject, + pub length: Py_ssize_t, + pub hash: Py_hash_t, + /// A bit field with various properties. + /// + /// Rust doesn't expose bitfields. So we have accessor functions for + /// retrieving values. + /// + /// unsigned int interned:2; // SSTATE_* constants. + /// unsigned int kind:3; // PyUnicode_*_KIND constants. + /// unsigned int compact:1; + /// unsigned int ascii:1; + /// unsigned int ready:1; + /// unsigned int :24; + pub state: u32, + pub wstr: *mut wchar_t, +} + +impl PyASCIIObject { + #[inline] + pub fn interned(&self) -> c_uint { + self.state & 3 + } + + #[inline] + pub fn kind(&self) -> c_uint { + (self.state >> 2) & 7 + } + + #[inline] + pub fn compact(&self) -> c_uint { + (self.state >> 5) & 1 + } + + #[inline] + pub fn ascii(&self) -> c_uint { + (self.state >> 6) & 1 + } + + #[inline] + pub fn ready(&self) -> c_uint { + (self.state >> 7) & 1 + } +} + +#[repr(C)] +pub struct PyCompactUnicodeObject { + pub _base: PyASCIIObject, + pub utf8_length: Py_ssize_t, + pub utf8: *mut c_char, + pub wstr_length: Py_ssize_t, +} + +#[repr(C)] +pub union PyUnicodeObjectData { + any: *mut c_void, + latin1: *mut Py_UCS1, + ucs2: *mut Py_UCS2, + ucs4: *mut Py_UCS4, +} + +#[repr(C)] +pub struct PyUnicodeObject { + pub _base: PyCompactUnicodeObject, + pub data: PyUnicodeObjectData, +} + +extern "C" { + #[cfg(not(PyPy))] + pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int; +} + +// skipped PyUnicode_GET_SIZE +// skipped PyUnicode_GET_DATA_SIZE +// skipped PyUnicode_AS_UNICODE +// skipped PyUnicode_AS_DATA + +pub const SSTATE_NOT_INTERNED: c_uint = 0; +pub const SSTATE_INTERNED_MORTAL: c_uint = 1; +pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; + +#[inline] +pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).ascii() +} + +#[inline] +pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { + (*(op as *mut PyASCIIObject)).compact() +} + +#[inline] +pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { + if (*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0 { + 1 + } else { + 0 + } +} + +#[cfg(not(Py_3_12))] +#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +pub const PyUnicode_WCHAR_KIND: c_uint = 0; + +pub const PyUnicode_1BYTE_KIND: c_uint = 1; +pub const PyUnicode_2BYTE_KIND: c_uint = 2; +pub const PyUnicode_4BYTE_KIND: c_uint = 4; + +#[inline] +pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { + PyUnicode_DATA(op) as *mut Py_UCS1 +} + +#[inline] +pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { + PyUnicode_DATA(op) as *mut Py_UCS2 +} + +#[inline] +pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { + PyUnicode_DATA(op) as *mut Py_UCS4 +} + +#[inline] +pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).kind() +} + +#[inline] +pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { + if PyUnicode_IS_ASCII(op) != 0 { + (op as *mut PyASCIIObject).offset(1) as *mut c_void + } else { + (op as *mut PyCompactUnicodeObject).offset(1) as *mut c_void + } +} + +#[inline] +pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { + debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); + + (*(op as *mut PyUnicodeObject)).data.any +} + +#[inline] +pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { + debug_assert!(PyUnicode_Check(op) != 0); + + if PyUnicode_IS_COMPACT(op) != 0 { + _PyUnicode_COMPACT_DATA(op) + } else { + _PyUnicode_NONCOMPACT_DATA(op) + } +} + +// skipped PyUnicode_WRITE +// skipped PyUnicode_READ +// skipped PyUnicode_READ_CHAR + +#[inline] +pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { + debug_assert!(PyUnicode_Check(op) != 0); + debug_assert!(PyUnicode_IS_READY(op) != 0); + + (*(op as *mut PyASCIIObject)).length +} + +#[inline] +pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { + (*(op as *mut PyASCIIObject)).ready() +} + +#[cfg(not(Py_3_12))] +#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +#[inline] +pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { + debug_assert!(PyUnicode_Check(op) != 0); + + if PyUnicode_IS_READY(op) != 0 { + 0 + } else { + _PyUnicode_Ready(op) + } +} + +// skipped PyUnicode_MAX_CHAR_VALUE +// skipped _PyUnicode_get_wstr_length +// skipped PyUnicode_WSTR_LENGTH + +extern "C" { + // move PyUnicode_New + + pub fn _PyUnicode_Ready(unicode: *mut PyObject) -> c_int; + + // skipped _PyUnicode_Copy + // move PyUnicode_CopyCharacters + // skipped _PyUnicode_FastCopyCharacters + // move PyUnicode_Fill + // skipped _PyUnicode_FastFill + // move PyUnicode_FromUnicode + // move PyUnicode_FromKindAndData + // skipped _PyUnicode_FromASCII + // skipped _PyUnicode_FindMaxChar + // move PyUnicode_AsUnicode + // skipped _PyUnicode_AsUnicode + // move PyUnicode_AsUnicodeAndSize + // skipped PyUnicode_GetMax +} + +// skipped _PyUnicodeWriter +// skipped _PyUnicodeWriter_Init +// skipped _PyUnicodeWriter_Prepare +// skipped _PyUnicodeWriter_PrepareInternal +// skipped _PyUnicodeWriter_PrepareKind +// skipped _PyUnicodeWriter_PrepareKindInternal +// skipped _PyUnicodeWriter_WriteChar +// skipped _PyUnicodeWriter_WriteStr +// skipped _PyUnicodeWriter_WriteSubstring +// skipped _PyUnicodeWriter_WriteASCIIString +// skipped _PyUnicodeWriter_WriteLatin1String +// skipped _PyUnicodeWriter_Finish +// skipped _PyUnicodeWriter_Dealloc +// skipped _PyUnicode_FormatAdvancedWriter + +// move PyUnicode_AsUTF8AndSize +// skipped _PyUnicode_AsStringAndSize +// move PyUnicode_AsUTF8 +// skipped _PyUnicode_AsString + +// move PyUnicode_Encode +// move PyUnicode_EncodeUTF7 +// skipped _PyUnicode_EncodeUTF7 + +// skipped _PyUnicode_AsUTF8String +// move PyUnicode_EncodeUTF8 + +// move PyUnicode_EncodeUTF32 +// skipped _PyUnicode_EncodeUTF32 + +// move PyUnicode_EncodeUTF16 +// skipped _PyUnicode_EncodeUTF16 + +// skipped _PyUnicode_DecodeUnicodeEscape +// move PyUnicode_EncodeUnicodeEscape +// move PyUnicode_EncodeRawUnicodeEscape + +// skipped _PyUnicode_AsLatin1String +// move PyUnicode_EncodeLatin1 + +// skipped _PyUnicode_AsASCIIString +// move PyUnicode_EncodeASCII + +// move PyUnicode_EncodeCharmap +// skipped _PyUnicode_EncodeCharmap +// move PyUnicode_TranslateCharmap + +// skipped PyUnicode_EncodeMBCS + +// move PyUnicode_EncodeDecimal +// move PyUnicode_TransformDecimalToASCII +// skipped _PyUnicode_TransformDecimalAndSpaceToASCII + +// skipped _PyUnicode_JoinArray +// skipped _PyUnicode_EqualToASCIIId +// skipped _PyUnicode_EqualToASCIIString +// skipped _PyUnicode_XStrip +// skipped _PyUnicode_InsertThousandsGrouping + +// skipped _Py_ascii_whitespace + +// skipped _PyUnicode_IsLowercase +// skipped _PyUnicode_IsUppercase +// skipped _PyUnicode_IsTitlecase +// skipped _PyUnicode_IsXidStart +// skipped _PyUnicode_IsXidContinue +// skipped _PyUnicode_IsWhitespace +// skipped _PyUnicode_IsLinebreak +// skipped _PyUnicode_ToLowercase +// skipped _PyUnicode_ToUppercase +// skipped _PyUnicode_ToTitlecase +// skipped _PyUnicode_ToLowerFull +// skipped _PyUnicode_ToTitleFull +// skipped _PyUnicode_ToUpperFull +// skipped _PyUnicode_ToFoldedFull +// skipped _PyUnicode_IsCaseIgnorable +// skipped _PyUnicode_IsCased +// skipped _PyUnicode_ToDecimalDigit +// skipped _PyUnicode_ToDigit +// skipped _PyUnicode_ToNumeric +// skipped _PyUnicode_IsDecimalDigit +// skipped _PyUnicode_IsDigit +// skipped _PyUnicode_IsNumeric +// skipped _PyUnicode_IsPrintable +// skipped _PyUnicode_IsAlpha +// skipped Py_UNICODE_strlen +// skipped Py_UNICODE_strcpy +// skipped Py_UNICODE_strcat +// skipped Py_UNICODE_strncpy +// skipped Py_UNICODE_strcmp +// skipped Py_UNICODE_strncmp +// skipped Py_UNICODE_strchr +// skipped Py_UNICODE_strrchr +// skipped _PyUnicode_FormatLong +// skipped PyUnicode_AsUnicodeCopy +// skipped _PyUnicode_FromId +// skipped _PyUnicode_EQ +// skipped _PyUnicode_ScanIdentifier + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::PyString; + use crate::{AsPyPointer, Python}; + + #[test] + fn ascii_object_bitfield() { + let ob_base: PyObject = unsafe { std::mem::zeroed() }; + + let mut o = PyASCIIObject { + ob_base, + length: 0, + hash: 0, + state: 0, + wstr: std::ptr::null_mut() as *mut wchar_t, + }; + + assert_eq!(o.interned(), 0); + assert_eq!(o.kind(), 0); + assert_eq!(o.compact(), 0); + assert_eq!(o.ascii(), 0); + assert_eq!(o.ready(), 0); + + for i in 0..4 { + o.state = i; + assert_eq!(o.interned(), i); + } + + for i in 0..8 { + o.state = i << 2; + assert_eq!(o.kind(), i); + } + + o.state = 1 << 5; + assert_eq!(o.compact(), 1); + + o.state = 1 << 6; + assert_eq!(o.ascii(), 1); + + o.state = 1 << 7; + assert_eq!(o.ready(), 1); + } + + #[test] + fn ascii() { + Python::with_gil(|py| { + // This test relies on implementation details of PyString. + let s = PyString::new(py, "hello, world"); + let ptr = s.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 1); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 1); + + assert!(!PyUnicode_1BYTE_DATA(ptr).is_null()); + // 2 and 4 byte macros return nonsense for this string instance. + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_1BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), s.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) + } + + #[test] + fn ucs4() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let py_string = PyString::new(py, s); + let ptr = py_string.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 0); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 0); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 0); + + assert!(!PyUnicode_4BYTE_DATA(ptr).is_null()); + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_4BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), py_string.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) + } +} From 1510855e5990a59b062456d73af2ab2b81b14bbc Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 10 Aug 2021 17:20:05 -0700 Subject: [PATCH 15/32] ffi: move limited API unicode symbols to cpython/unicodeobject.rs All symbols which are canonically defined in cpython/unicodeobject.h and had been defined in our unicodeobject.rs have been moved to our corresponding cpython/unicodeobject.rs file. This module is only present in non-limited build configurations, so we were able to drop the cfg annotations as part of moving the definitions. --- src/ffi/cpython/unicodeobject.rs | 221 ++++++++++++++++++++++++------- src/ffi/unicodeobject.rs | 143 -------------------- 2 files changed, 175 insertions(+), 189 deletions(-) diff --git a/src/ffi/cpython/unicodeobject.rs b/src/ffi/cpython/unicodeobject.rs index 160e7d0f697..243c0ede3cd 100644 --- a/src/ffi/cpython/unicodeobject.rs +++ b/src/ffi/cpython/unicodeobject.rs @@ -1,4 +1,6 @@ -use crate::ffi::{PyObject, PyUnicode_Check, Py_UCS1, Py_UCS2, Py_UCS4, Py_hash_t, Py_ssize_t}; +use crate::ffi::{ + PyObject, PyUnicode_Check, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_hash_t, Py_ssize_t, +}; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_uint, c_void}; @@ -226,22 +228,59 @@ pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { // skipped PyUnicode_WSTR_LENGTH extern "C" { - // move PyUnicode_New - + pub fn PyUnicode_New(size: Py_ssize_t, maxchar: Py_UCS4) -> *mut PyObject; pub fn _PyUnicode_Ready(unicode: *mut PyObject) -> c_int; // skipped _PyUnicode_Copy - // move PyUnicode_CopyCharacters + + pub fn PyUnicode_CopyCharacters( + to: *mut PyObject, + to_start: Py_ssize_t, + from: *mut PyObject, + from_start: Py_ssize_t, + how_many: Py_ssize_t, + ) -> Py_ssize_t; + // skipped _PyUnicode_FastCopyCharacters - // move PyUnicode_Fill + + pub fn PyUnicode_Fill( + unicode: *mut PyObject, + start: Py_ssize_t, + length: Py_ssize_t, + fill_char: Py_UCS4, + ) -> Py_ssize_t; + // skipped _PyUnicode_FastFill - // move PyUnicode_FromUnicode - // move PyUnicode_FromKindAndData + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] + pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; + + pub fn PyUnicode_FromKindAndData( + kind: c_int, + buffer: *const c_void, + size: Py_ssize_t, + ) -> *mut PyObject; + // skipped _PyUnicode_FromASCII // skipped _PyUnicode_FindMaxChar - // move PyUnicode_AsUnicode + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] + pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; + // skipped _PyUnicode_AsUnicode - // move PyUnicode_AsUnicodeAndSize + + #[cfg(not(Py_3_12))] + #[deprecated] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeAndSize")] + pub fn PyUnicode_AsUnicodeAndSize( + unicode: *mut PyObject, + size: *mut Py_ssize_t, + ) -> *mut Py_UNICODE; + // skipped PyUnicode_GetMax } @@ -260,43 +299,133 @@ extern "C" { // skipped _PyUnicodeWriter_Dealloc // skipped _PyUnicode_FormatAdvancedWriter -// move PyUnicode_AsUTF8AndSize -// skipped _PyUnicode_AsStringAndSize -// move PyUnicode_AsUTF8 -// skipped _PyUnicode_AsString - -// move PyUnicode_Encode -// move PyUnicode_EncodeUTF7 -// skipped _PyUnicode_EncodeUTF7 - -// skipped _PyUnicode_AsUTF8String -// move PyUnicode_EncodeUTF8 - -// move PyUnicode_EncodeUTF32 -// skipped _PyUnicode_EncodeUTF32 - -// move PyUnicode_EncodeUTF16 -// skipped _PyUnicode_EncodeUTF16 - -// skipped _PyUnicode_DecodeUnicodeEscape -// move PyUnicode_EncodeUnicodeEscape -// move PyUnicode_EncodeRawUnicodeEscape - -// skipped _PyUnicode_AsLatin1String -// move PyUnicode_EncodeLatin1 - -// skipped _PyUnicode_AsASCIIString -// move PyUnicode_EncodeASCII - -// move PyUnicode_EncodeCharmap -// skipped _PyUnicode_EncodeCharmap -// move PyUnicode_TranslateCharmap - -// skipped PyUnicode_EncodeMBCS - -// move PyUnicode_EncodeDecimal -// move PyUnicode_TransformDecimalToASCII -// skipped _PyUnicode_TransformDecimalAndSpaceToASCII +extern "C" { + #[cfg(Py_3_7)] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] + pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *const c_char; + + #[cfg(not(Py_3_7))] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] + pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *mut c_char; + + // skipped _PyUnicode_AsStringAndSize + + #[cfg(Py_3_7)] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] + pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *const c_char; + + #[cfg(not(Py_3_7))] + #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] + pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *mut c_char; + + // skipped _PyUnicode_AsString + + pub fn PyUnicode_Encode( + s: *const Py_UNICODE, + size: Py_ssize_t, + encoding: *const c_char, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeUTF7( + data: *const Py_UNICODE, + length: Py_ssize_t, + base64SetO: c_int, + base64WhiteSpace: c_int, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF7 + // skipped _PyUnicode_AsUTF8String + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] + pub fn PyUnicode_EncodeUTF8( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeUTF32( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + byteorder: c_int, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF32 + + pub fn PyUnicode_EncodeUTF16( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + byteorder: c_int, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeUTF16 + // skipped _PyUnicode_DecodeUnicodeEscape + + pub fn PyUnicode_EncodeUnicodeEscape( + data: *const Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeRawUnicodeEscape( + data: *const Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + // skipped _PyUnicode_AsLatin1String + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] + pub fn PyUnicode_EncodeLatin1( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_AsASCIIString + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] + pub fn PyUnicode_EncodeASCII( + data: *const Py_UNICODE, + length: Py_ssize_t, + errors: *const c_char, + ) -> *mut PyObject; + + pub fn PyUnicode_EncodeCharmap( + data: *const Py_UNICODE, + length: Py_ssize_t, + mapping: *mut PyObject, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped _PyUnicode_EncodeCharmap + + pub fn PyUnicode_TranslateCharmap( + data: *const Py_UNICODE, + length: Py_ssize_t, + table: *mut PyObject, + errors: *const c_char, + ) -> *mut PyObject; + + // skipped PyUnicode_EncodeMBCS + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] + pub fn PyUnicode_EncodeDecimal( + s: *mut Py_UNICODE, + length: Py_ssize_t, + output: *mut c_char, + errors: *const c_char, + ) -> c_int; + + #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] + pub fn PyUnicode_TransformDecimalToASCII( + s: *mut Py_UNICODE, + length: Py_ssize_t, + ) -> *mut PyObject; + + // skipped _PyUnicode_TransformDecimalAndSpaceToASCII +} // skipped _PyUnicode_JoinArray // skipped _PyUnicode_EqualToASCIIId diff --git a/src/ffi/unicodeobject.rs b/src/ffi/unicodeobject.rs index de51d816012..9f215cf8299 100644 --- a/src/ffi/unicodeobject.rs +++ b/src/ffi/unicodeobject.rs @@ -40,40 +40,11 @@ pub unsafe fn PyUnicode_CheckExact(op: *mut PyObject) -> c_int { pub const Py_UNICODE_REPLACEMENT_CHARACTER: Py_UCS4 = 0xFFFD; extern "C" { - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_New(size: Py_ssize_t, maxchar: Py_UCS4) -> *mut PyObject; - - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_CopyCharacters( - to: *mut PyObject, - to_start: Py_ssize_t, - from: *mut PyObject, - from_start: Py_ssize_t, - how_many: Py_ssize_t, - ) -> Py_ssize_t; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_Fill( - unicode: *mut PyObject, - start: Py_ssize_t, - length: Py_ssize_t, - fill_char: Py_UCS4, - ) -> Py_ssize_t; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] - pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromStringAndSize")] pub fn PyUnicode_FromStringAndSize(u: *const c_char, size: Py_ssize_t) -> *mut PyObject; pub fn PyUnicode_FromString(u: *const c_char) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_FromKindAndData( - kind: c_int, - buffer: *const c_void, - size: Py_ssize_t, - ) -> *mut PyObject; - pub fn PyUnicode_Substring( str: *mut PyObject, start: Py_ssize_t, @@ -86,17 +57,6 @@ extern "C" { copy_null: c_int, ) -> *mut Py_UCS4; pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] - pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; - #[cfg(all(not(Py_LIMITED_API), not(Py_3_12)))] - #[deprecated] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeAndSize")] - pub fn PyUnicode_AsUnicodeAndSize( - unicode: *mut PyObject, - size: *mut Py_ssize_t, - ) -> *mut Py_UNICODE; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")] pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")] @@ -143,20 +103,6 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromOrdinal")] pub fn PyUnicode_FromOrdinal(ordinal: c_int) -> *mut PyObject; pub fn PyUnicode_ClearFreeList() -> c_int; - #[cfg(any(not(Py_LIMITED_API), Py_3_10))] - #[cfg(Py_3_7)] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] - pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *const c_char; - #[cfg(not(Py_3_7))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8AndSize")] - pub fn PyUnicode_AsUTF8AndSize(unicode: *mut PyObject, size: *mut Py_ssize_t) -> *mut c_char; - #[cfg(not(Py_LIMITED_API))] - #[cfg(Py_3_7)] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] - pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *const c_char; - #[cfg(not(Py_3_7))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8")] - pub fn PyUnicode_AsUTF8(unicode: *mut PyObject) -> *mut c_char; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetDefaultEncoding")] pub fn PyUnicode_GetDefaultEncoding() -> *const c_char; #[cfg_attr(PyPy, link_name = "PyPyUnicode_Decode")] @@ -176,13 +122,6 @@ extern "C" { encoding: *const c_char, errors: *const c_char, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_Encode( - s: *const Py_UNICODE, - size: Py_ssize_t, - encoding: *const c_char, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsEncodedObject")] pub fn PyUnicode_AsEncodedObject( unicode: *mut PyObject, @@ -212,14 +151,6 @@ extern "C" { errors: *const c_char, consumed: *mut Py_ssize_t, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF7( - data: *const Py_UNICODE, - length: Py_ssize_t, - base64SetO: c_int, - base64WhiteSpace: c_int, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF8")] pub fn PyUnicode_DecodeUTF8( string: *const c_char, @@ -234,13 +165,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF8String")] pub fn PyUnicode_AsUTF8String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] - pub fn PyUnicode_EncodeUTF8( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF32")] pub fn PyUnicode_DecodeUTF32( string: *const c_char, @@ -257,13 +181,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF32String")] pub fn PyUnicode_AsUTF32String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF32( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - byteorder: c_int, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeUTF16")] pub fn PyUnicode_DecodeUTF16( string: *const c_char, @@ -280,13 +197,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUTF16String")] pub fn PyUnicode_AsUTF16String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUTF16( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - byteorder: c_int, - ) -> *mut PyObject; pub fn PyUnicode_DecodeUnicodeEscape( string: *const c_char, length: Py_ssize_t, @@ -294,22 +204,12 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicodeEscapeString")] pub fn PyUnicode_AsUnicodeEscapeString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; pub fn PyUnicode_DecodeRawUnicodeEscape( string: *const c_char, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_AsRawUnicodeEscapeString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeRawUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeLatin1")] pub fn PyUnicode_DecodeLatin1( string: *const c_char, @@ -318,13 +218,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsLatin1String")] pub fn PyUnicode_AsLatin1String(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] - pub fn PyUnicode_EncodeLatin1( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_DecodeASCII")] pub fn PyUnicode_DecodeASCII( string: *const c_char, @@ -333,13 +226,6 @@ extern "C" { ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsASCIIString")] pub fn PyUnicode_AsASCIIString(unicode: *mut PyObject) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] - pub fn PyUnicode_EncodeASCII( - data: *const Py_UNICODE, - length: Py_ssize_t, - errors: *const c_char, - ) -> *mut PyObject; pub fn PyUnicode_DecodeCharmap( string: *const c_char, length: Py_ssize_t, @@ -350,35 +236,6 @@ extern "C" { unicode: *mut PyObject, mapping: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_EncodeCharmap( - data: *const Py_UNICODE, - length: Py_ssize_t, - mapping: *mut PyObject, - errors: *const c_char, - ) -> *mut PyObject; - #[cfg(not(Py_LIMITED_API))] - pub fn PyUnicode_TranslateCharmap( - data: *const Py_UNICODE, - length: Py_ssize_t, - table: *mut PyObject, - errors: *const c_char, - ) -> *mut PyObject; - - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] - pub fn PyUnicode_EncodeDecimal( - s: *mut Py_UNICODE, - length: Py_ssize_t, - output: *mut c_char, - errors: *const c_char, - ) -> c_int; - #[cfg(not(Py_LIMITED_API))] - #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] - pub fn PyUnicode_TransformDecimalToASCII( - s: *mut Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; pub fn PyUnicode_DecodeLocaleAndSize( str: *const c_char, len: Py_ssize_t, From 62181cc3773c6fbd4d5117098c32441d3034c7d6 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 14 Aug 2021 07:56:15 +0100 Subject: [PATCH 16/32] ffi: PyPy and Python 3.10 attributes for unicodeobject --- src/ffi/cpython/unicodeobject.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ffi/cpython/unicodeobject.rs b/src/ffi/cpython/unicodeobject.rs index 243c0ede3cd..7c61243513b 100644 --- a/src/ffi/cpython/unicodeobject.rs +++ b/src/ffi/cpython/unicodeobject.rs @@ -228,11 +228,15 @@ pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { // skipped PyUnicode_WSTR_LENGTH extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_New")] pub fn PyUnicode_New(size: Py_ssize_t, maxchar: Py_UCS4) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "_PyPyUnicode_Ready")] pub fn _PyUnicode_Ready(unicode: *mut PyObject) -> c_int; // skipped _PyUnicode_Copy + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyUnicode_CopyCharacters( to: *mut PyObject, to_start: Py_ssize_t, @@ -243,6 +247,8 @@ extern "C" { // skipped _PyUnicode_FastCopyCharacters + #[cfg(not(PyPy))] + #[cfg_attr(docsrs, doc(cfg(not(PyPy))))] pub fn PyUnicode_Fill( unicode: *mut PyObject, start: Py_ssize_t, @@ -257,6 +263,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromKindAndData")] pub fn PyUnicode_FromKindAndData( kind: c_int, buffer: *const c_void, @@ -518,6 +525,7 @@ mod tests { } #[test] + #[cfg_attr(Py_3_10, allow(deprecated))] fn ascii() { Python::with_gil(|py| { // This test relies on implementation details of PyString. @@ -557,6 +565,7 @@ mod tests { } #[test] + #[cfg_attr(Py_3_10, allow(deprecated))] fn ucs4() { Python::with_gil(|py| { let s = "哈哈🐈"; From f2ebe82b4097d52a8ff8a5e4aa769d20f15f25b0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 14 Aug 2021 10:25:44 -0700 Subject: [PATCH 17/32] string: enable use of PyUnicode_AsUTF8AndSize on all Python versions This API is available in all supported Python versions and isn't deprecated. --- src/types/string.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/string.rs b/src/types/string.rs index 7da450bb279..fa6a2826010 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -49,8 +49,8 @@ impl PyString { pub fn to_str(&self) -> PyResult<&str> { let utf8_slice = { cfg_if::cfg_if! { - if #[cfg(any(not(Py_LIMITED_API), Py_3_10))] { - // PyUnicode_AsUTF8AndSize only available on limited API from Python 3.10 and up. + if #[cfg(not(Py_LIMITED_API))] { + // PyUnicode_AsUTF8AndSize only available on limited API. let mut size: ffi::Py_ssize_t = 0; let data = unsafe { ffi::PyUnicode_AsUTF8AndSize(self.as_ptr(), &mut size) }; if data.is_null() { From dd156cf8e0d7167192a43dcf799deae4a90494d6 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Mon, 16 Aug 2021 14:35:12 -0400 Subject: [PATCH 18/32] Expand supported num-complex versions When building an extension with rust-numpy and ndarray on the MSRV of 1.41 with complex numbers. The num-complex crate version needs to be 0.2 which was the pinned version as of ndarray 0.13.1 which was the last release of ndarray that supported building with rust 1.41. However, the pyo3 pinned version of 0.4 is incompatible with this and will cause an error when building because of the version mismatch. To fix this This commit expands the supported versions for num-complex to match what rust-numpy uses [1] so that we can build pyo3, numpy, ndarray, and num-complex in an extension with rust 1.41. Fixes #1798 [1] https://github.com/PyO3/rust-numpy/blob/v0.14.1/Cargo.toml#L19 --- CHANGELOG.md | 3 ++- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b846b4668d..b11b22bec8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed +- Raise `AttributeError` to avoid panic when calling `del` on a `#[setter]` defined class property. [#1779](https://github.com/PyO3/pyo3/issues/1779) - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) -- Raise `AttributeError` to avoid panic when calling `del` on a `#[setter]` defined class property. [#1779](https://github.com/PyO3/pyo3/issues/1779) +- Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) ## [0.14.2] - 2021-08-09 diff --git a/Cargo.toml b/Cargo.toml index 3667f11bf2e..54f4e2b8ba4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ inventory = { version = "0.1.4", optional = true } libc = "0.2.62" parking_lot = "0.11.0" num-bigint = { version = "0.4", optional = true } -num-complex = { version = "0.4", optional = true } +num-complex = { version = ">= 0.2, <= 0.4", optional = true } # must stay at 0.1.x for Rust 1.41 compatibility paste = { version = "0.1.18", optional = true } pyo3-macros = { path = "pyo3-macros", version = "=0.14.2", optional = true } From b48f4c95e758a6bd24af5257ac9d35eb628d897d Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Mon, 16 Aug 2021 18:22:46 -0400 Subject: [PATCH 19/32] Update Cargo.toml Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com> --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 54f4e2b8ba4..c9b0ac21ac4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ inventory = { version = "0.1.4", optional = true } libc = "0.2.62" parking_lot = "0.11.0" num-bigint = { version = "0.4", optional = true } -num-complex = { version = ">= 0.2, <= 0.4", optional = true } +num-complex = { version = ">= 0.2, < 0.5", optional = true } # must stay at 0.1.x for Rust 1.41 compatibility paste = { version = "0.1.18", optional = true } pyo3-macros = { path = "pyo3-macros", version = "=0.14.2", optional = true } From 3bdcbfa4b55c8810ed5bcf9f5e4f70cd35bba380 Mon Sep 17 00:00:00 2001 From: mejrs Date: Mon, 16 Aug 2021 16:32:11 +0200 Subject: [PATCH 20/32] test macro hygiene for pyclass --- pyo3-macros-backend/src/method.rs | 6 +- pyo3-macros-backend/src/proto_method.rs | 2 +- pyo3-macros-backend/src/pyclass.rs | 135 ++++++++++++------------ pyo3-macros-backend/src/pymethod.rs | 59 ++++++----- pyo3-macros-backend/src/pyproto.rs | 32 +++--- tests/test_proc_macro_hygiene.rs | 62 +++++++++++ 6 files changed, 181 insertions(+), 115 deletions(-) create mode 100644 tests/test_proc_macro_hygiene.rs diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 09fad20596b..603f8df6428 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -133,21 +133,21 @@ impl SelfType { match self { SelfType::Receiver { mutable: false } => { quote! { - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); let _ref = _cell.try_borrow()?; let _slf = &_ref; } } SelfType::Receiver { mutable: true } => { quote! { - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); let mut _ref = _cell.try_borrow_mut()?; let _slf = &mut _ref; } } SelfType::TryFromPyCell(span) => { quote_spanned! { *span => - let _cell = _py.from_borrowed_ptr::>(_slf); + let _cell = _py.from_borrowed_ptr::<::pyo3::PyCell<#cls>>(_slf); #[allow(clippy::useless_conversion)] // In case _slf is PyCell let _slf = std::convert::TryFrom::try_from(_cell)?; } diff --git a/pyo3-macros-backend/src/proto_method.rs b/pyo3-macros-backend/src/proto_method.rs index 3c14843fd78..8d8569b0834 100644 --- a/pyo3-macros-backend/src/proto_method.rs +++ b/pyo3-macros-backend/src/proto_method.rs @@ -95,7 +95,7 @@ pub(crate) fn impl_method_proto( }; Ok(quote! { - impl<'p> #module::#proto<'p> for #cls { + impl<'p> ::#module::#proto<'p> for #cls { #(#impl_types)* #res_type_def } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 14996df8a2d..deb017788fc 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -47,7 +47,7 @@ impl Default for PyClassArgs { freelist: None, name: None, module: None, - base: parse_quote! { pyo3::PyAny }, + base: parse_quote! { ::pyo3::PyAny }, has_dict: false, has_weaklist: false, is_gc: false, @@ -347,7 +347,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { pub struct #inventory_cls { methods: Vec, } - impl pyo3::class::impl_::PyMethodsInventory for #inventory_cls { + impl ::pyo3::class::impl_::PyMethodsInventory for #inventory_cls { fn new(methods: Vec) -> Self { Self { methods } } @@ -356,11 +356,11 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { } } - impl pyo3::class::impl_::HasMethodsInventory for #cls { + impl ::pyo3::class::impl_::HasMethodsInventory for #cls { type Methods = #inventory_cls; } - pyo3::inventory::collect!(#inventory_cls); + ::pyo3::inventory::collect!(#inventory_cls); } } @@ -380,31 +380,31 @@ fn impl_class( let alloc = attr.freelist.as_ref().map(|freelist| { quote! { - impl pyo3::class::impl_::PyClassWithFreeList for #cls { + impl ::pyo3::class::impl_::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(_py: pyo3::Python<'_>) -> &mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> { - static mut FREELIST: *mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _; + fn get_free_list(_py: ::pyo3::Python<'_>) -> &mut ::pyo3::impl_::freelist::FreeList<*mut ::pyo3::ffi::PyObject> { + static mut FREELIST: *mut ::pyo3::impl_::freelist::FreeList<*mut ::pyo3::ffi::PyObject> = 0 as *mut _; unsafe { if FREELIST.is_null() { - FREELIST = Box::into_raw(Box::new( - pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); + FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new( + ::pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); } &mut *FREELIST } } } - impl pyo3::class::impl_::PyClassAllocImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassAllocImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { #[inline] - fn alloc_impl(self) -> Option { - Some(pyo3::class::impl_::alloc_with_freelist::<#cls>) + fn alloc_impl(self) -> ::std::option::Option<::pyo3::ffi::allocfunc> { + ::std::option::Option::Some(::pyo3::class::impl_::alloc_with_freelist::<#cls>) } } - impl pyo3::class::impl_::PyClassFreeImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassFreeImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { #[inline] - fn free_impl(self) -> Option { - Some(pyo3::class::impl_::free_with_freelist::<#cls>) + fn free_impl(self) -> ::std::option::Option<::pyo3::ffi::freefunc> { + ::std::option::Option::Some(::pyo3::class::impl_::free_with_freelist::<#cls>) } } } @@ -414,23 +414,23 @@ fn impl_class( // insert space for weak ref let weakref = if attr.has_weaklist { - quote! { pyo3::pyclass_slots::PyClassWeakRefSlot } + quote! { ::pyo3::pyclass_slots::PyClassWeakRefSlot } } else if attr.has_extends { - quote! { ::WeakRef } + quote! { ::WeakRef } } else { - quote! { pyo3::pyclass_slots::PyClassDummySlot } + quote! { ::pyo3::pyclass_slots::PyClassDummySlot } }; let dict = if attr.has_dict { - quote! { pyo3::pyclass_slots::PyClassDictSlot } + quote! { ::pyo3::pyclass_slots::PyClassDictSlot } } else if attr.has_extends { - quote! { ::Dict } + quote! { ::Dict } } else { - quote! { pyo3::pyclass_slots::PyClassDummySlot } + quote! { ::pyo3::pyclass_slots::PyClassDummySlot } }; let module = if let Some(m) = &attr.module { - quote! { Some(#m) } + quote! { ::std::option::Option::Some(#m) } } else { - quote! { None } + quote! { ::std::option::Option::None } }; // Enforce at compile time that PyGCProtocol is implemented @@ -439,9 +439,9 @@ fn impl_class( let closure_token = syn::Ident::new(&closure_name, Span::call_site()); quote! { fn #closure_token() { - use pyo3::class; + use ::pyo3::class; - fn _assert_implements_protocol<'p, T: pyo3::class::PyGCProtocol<'p>>() {} + fn _assert_implements_protocol<'p, T: ::pyo3::class::PyGCProtocol<'p>>() {} _assert_implements_protocol::<#cls>(); } } @@ -450,12 +450,15 @@ fn impl_class( }; let (impl_inventory, for_each_py_method) = match methods_type { - PyClassMethodsType::Specialization => (None, quote! { visitor(collector.py_methods()); }), + PyClassMethodsType::Specialization => ( + ::std::option::Option::None, + quote! { visitor(collector.py_methods()); }, + ), PyClassMethodsType::Inventory => ( Some(impl_methods_inventory(cls)), quote! { - for inventory in pyo3::inventory::iter::<::Methods>() { - visitor(pyo3::class::impl_::PyMethodsInventory::get(inventory)); + for inventory in ::pyo3::inventory::iter::<::Methods>() { + visitor(::pyo3::class::impl_::PyMethodsInventory::get(inventory)); } }, ), @@ -463,17 +466,17 @@ fn impl_class( let base = &attr.base; let base_nativetype = if attr.has_extends { - quote! { ::BaseNativeType } + quote! { ::BaseNativeType } } else { - quote! { pyo3::PyAny } + quote! { ::pyo3::PyAny } }; // If #cls is not extended type, we allow Self->PyObject conversion let into_pyobject = if !attr.has_extends { quote! { - impl pyo3::IntoPy for #cls { - fn into_py(self, py: pyo3::Python) -> pyo3::PyObject { - pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) + impl ::pyo3::IntoPy<::pyo3::PyObject> for #cls { + fn into_py(self, py: ::pyo3::Python) -> ::pyo3::PyObject { + ::pyo3::IntoPy::into_py(::pyo3::Py::new(py, self).unwrap(), py) } } } @@ -482,13 +485,13 @@ fn impl_class( }; let thread_checker = if attr.has_unsendable { - quote! { pyo3::class::impl_::ThreadCheckerImpl<#cls> } + quote! { ::pyo3::class::impl_::ThreadCheckerImpl<#cls> } } else if attr.has_extends { quote! { - pyo3::class::impl_::ThreadCheckerInherited<#cls, <#cls as pyo3::class::impl_::PyClassImpl>::BaseType> + ::pyo3::class::impl_::ThreadCheckerInherited<#cls, <#cls as ::pyo3::class::impl_::PyClassImpl>::BaseType> } } else { - quote! { pyo3::class::impl_::ThreadCheckerStub<#cls> } + quote! { ::pyo3::class::impl_::ThreadCheckerStub<#cls> } }; let is_gc = attr.is_gc; @@ -496,54 +499,54 @@ fn impl_class( let is_subclass = attr.has_extends; Ok(quote! { - unsafe impl pyo3::type_object::PyTypeInfo for #cls { - type AsRefTarget = pyo3::PyCell; + unsafe impl ::pyo3::type_object::PyTypeInfo for #cls { + type AsRefTarget = ::pyo3::PyCell; const NAME: &'static str = #cls_name; - const MODULE: Option<&'static str> = #module; + const MODULE: ::std::option::Option<&'static str> = #module; #[inline] - fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { + fn type_object_raw(py: ::pyo3::Python<'_>) -> *mut ::pyo3::ffi::PyTypeObject { #deprecations - use pyo3::type_object::LazyStaticType; + use ::pyo3::type_object::LazyStaticType; static TYPE_OBJECT: LazyStaticType = LazyStaticType::new(); TYPE_OBJECT.get_or_init::(py) } } - impl pyo3::PyClass for #cls { + impl ::pyo3::PyClass for #cls { type Dict = #dict; type WeakRef = #weakref; type BaseNativeType = #base_nativetype; } - impl<'a> pyo3::derive_utils::ExtractExt<'a> for &'a #cls + impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a #cls { - type Target = pyo3::PyRef<'a, #cls>; + type Target = ::pyo3::PyRef<'a, #cls>; } - impl<'a> pyo3::derive_utils::ExtractExt<'a> for &'a mut #cls + impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a mut #cls { - type Target = pyo3::PyRefMut<'a, #cls>; + type Target = ::pyo3::PyRefMut<'a, #cls>; } #into_pyobject #impl_inventory - impl pyo3::class::impl_::PyClassImpl for #cls { + impl ::pyo3::class::impl_::PyClassImpl for #cls { const DOC: &'static str = #doc; const IS_GC: bool = #is_gc; const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; - type Layout = pyo3::PyCell; + type Layout = ::pyo3::PyCell; type BaseType = #base; type ThreadChecker = #thread_checker; - fn for_each_method_def(visitor: &mut dyn FnMut(&[pyo3::class::PyMethodDefType])) { - use pyo3::class::impl_::*; + fn for_each_method_def(visitor: &mut dyn ::std::ops::FnMut(&[::pyo3::class::PyMethodDefType])) { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); #for_each_py_method; visitor(collector.py_class_descriptors()); @@ -554,30 +557,30 @@ fn impl_class( visitor(collector.mapping_protocol_methods()); visitor(collector.number_protocol_methods()); } - fn get_new() -> Option { - use pyo3::class::impl_::*; + fn get_new() -> ::std::option::Option<::pyo3::ffi::newfunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.new_impl() } - fn get_alloc() -> Option { - use pyo3::class::impl_::*; + fn get_alloc() -> ::std::option::Option<::pyo3::ffi::allocfunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.alloc_impl() } - fn get_free() -> Option { - use pyo3::class::impl_::*; + fn get_free() -> ::std::option::Option<::pyo3::ffi::freefunc> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.free_impl() } - fn get_call() -> Option { - use pyo3::class::impl_::*; + fn get_call() -> ::std::option::Option<::pyo3::ffi::PyCFunctionWithKeywords> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.call_impl() } - fn for_each_proto_slot(visitor: &mut dyn FnMut(&[pyo3::ffi::PyType_Slot])) { + fn for_each_proto_slot(visitor: &mut dyn ::std::ops::FnMut(&[::pyo3::ffi::PyType_Slot])) { // Implementation which uses dtolnay specialization to load all slots. - use pyo3::class::impl_::*; + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); visitor(collector.object_protocol_slots()); visitor(collector.number_protocol_slots()); @@ -590,8 +593,8 @@ fn impl_class( visitor(collector.buffer_protocol_slots()); } - fn get_buffer() -> Option<&'static pyo3::class::impl_::PyBufferProcs> { - use pyo3::class::impl_::*; + fn get_buffer() -> ::std::option::Option<&'static ::pyo3::class::impl_::PyBufferProcs> { + use ::pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); collector.buffer_procs() } @@ -645,11 +648,11 @@ fn impl_descriptors( .collect::>()?; Ok(quote! { - impl pyo3::class::impl_::PyClassDescriptors<#cls> - for pyo3::class::impl_::PyClassImplCollector<#cls> + impl ::pyo3::class::impl_::PyClassDescriptors<#cls> + for ::pyo3::class::impl_::PyClassImplCollector<#cls> { - fn py_class_descriptors(self) -> &'static [pyo3::class::methods::PyMethodDefType] { - static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; + fn py_class_descriptors(self) -> &'static [::pyo3::class::methods::PyMethodDefType] { + static METHODS: &[::pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; METHODS } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 9af38f021ab..0991be14998 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -102,7 +102,7 @@ pub fn impl_py_method_def( }; let methoddef = spec.get_methoddef(quote! {{ #wrapper_def #wrapper_ident }}); Ok(quote! { - pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + ::pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) }) } @@ -110,7 +110,7 @@ fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result let wrapper_ident = syn::Ident::new("__wrap", Span::call_site()); let wrapper = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; Ok(quote! { - impl pyo3::class::impl_::PyClassNewImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassNewImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { fn new_impl(self) -> Option { Some({ #wrapper @@ -125,7 +125,7 @@ fn impl_py_method_def_call(cls: &syn::Type, spec: &FnSpec) -> Result for pyo3::class::impl_::PyClassImplCollector<#cls> { + impl ::pyo3::class::impl_::PyClassCallImpl<#cls> for ::pyo3::class::impl_::PyClassImplCollector<#cls> { fn call_impl(self) -> Option { Some({ #wrapper @@ -141,13 +141,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream { let deprecations = &spec.deprecations; let python_name = spec.null_terminated_python_name(); quote! { - pyo3::class::PyMethodDefType::ClassAttribute({ - pyo3::class::PyClassAttributeDef::new( + ::pyo3::class::PyMethodDefType::ClassAttribute({ + ::pyo3::class::PyClassAttributeDef::new( #python_name, - pyo3::class::methods::PyClassAttributeFactory({ - fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject { + ::pyo3::class::methods::PyClassAttributeFactory({ + fn __wrap(py: ::pyo3::Python<'_>) -> ::pyo3::PyObject { #deprecations - pyo3::IntoPy::into_py(#cls::#name(), py) + ::pyo3::IntoPy::into_py(#cls::#name(), py) } __wrap }) @@ -206,26 +206,26 @@ pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Resul PropertyType::Function { self_type, .. } => self_type.receiver(cls), }; Ok(quote! { - pyo3::class::PyMethodDefType::Setter({ + ::pyo3::class::PyMethodDefType::Setter({ #deprecations - pyo3::class::PySetterDef::new( + ::pyo3::class::PySetterDef::new( #python_name, - pyo3::class::methods::PySetter({ + ::pyo3::class::methods::PySetter({ unsafe extern "C" fn __wrap( - _slf: *mut pyo3::ffi::PyObject, - _value: *mut pyo3::ffi::PyObject, - _: *mut std::os::raw::c_void - ) -> std::os::raw::c_int { - pyo3::callback::handle_panic(|_py| { + _slf: *mut ::pyo3::ffi::PyObject, + _value: *mut ::pyo3::ffi::PyObject, + _: *mut ::std::os::raw::c_void + ) -> ::std::os::raw::c_int { + ::pyo3::callback::handle_panic(|_py| { #slf let _value = _py .from_borrowed_ptr_or_opt(_value) .ok_or_else(|| { - pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") + ::pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - let _val = pyo3::FromPyObject::extract(_value)?; + let _val = ::pyo3::FromPyObject::extract(_value)?; - pyo3::callback::convert(_py, #setter_impl) + ::pyo3::callback::convert(_py, #setter_impl) }) } __wrap @@ -266,12 +266,13 @@ pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Resul .. } => { // named struct field - quote!(_slf.#ident.clone()) + //quote!(_slf.#ident.clone()) + quote!(::std::clone::Clone::clone(&(_slf.#ident))) } PropertyType::Descriptor { field_index, .. } => { // tuple struct field let index = syn::Index::from(field_index); - quote!(_slf.#index.clone()) + quote!(::std::clone::Clone::clone(&(_slf.#index))) } PropertyType::Function { spec, .. } => impl_call_getter(cls, spec)?, }; @@ -281,18 +282,18 @@ pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Resul PropertyType::Function { self_type, .. } => self_type.receiver(cls), }; Ok(quote! { - pyo3::class::PyMethodDefType::Getter({ + ::pyo3::class::PyMethodDefType::Getter({ #deprecations - pyo3::class::PyGetterDef::new( + ::pyo3::class::PyGetterDef::new( #python_name, - pyo3::class::methods::PyGetter({ + ::pyo3::class::methods::PyGetter({ unsafe extern "C" fn __wrap( - _slf: *mut pyo3::ffi::PyObject, - _: *mut std::os::raw::c_void - ) -> *mut pyo3::ffi::PyObject { - pyo3::callback::handle_panic(|_py| { + _slf: *mut ::pyo3::ffi::PyObject, + _: *mut ::std::os::raw::c_void + ) -> *mut ::pyo3::ffi::PyObject { + ::pyo3::callback::handle_panic(|_py| { #slf - pyo3::callback::convert(_py, #getter_impl) + ::pyo3::callback::convert(_py, #getter_impl) }) } __wrap diff --git a/pyo3-macros-backend/src/pyproto.rs b/pyo3-macros-backend/src/pyproto.rs index 8080622916f..a3943d6ae76 100644 --- a/pyo3-macros-backend/src/pyproto.rs +++ b/pyo3-macros-backend/src/pyproto.rs @@ -68,7 +68,7 @@ fn impl_proto_impl( let flags = if m.can_coexist { // We need METH_COEXIST here to prevent __add__ from overriding __radd__ - Some(quote!(pyo3::ffi::METH_COEXIST)) + Some(quote!(::pyo3::ffi::METH_COEXIST)) } else { None }; @@ -106,11 +106,11 @@ fn impl_normal_methods( let methods_trait = proto.methods_trait(); let methods_trait_methods = proto.methods_trait_methods(); quote! { - impl pyo3::class::impl_::#methods_trait<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::#methods_trait<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { - fn #methods_trait_methods(self) -> &'static [pyo3::class::methods::PyMethodDefType] { - static METHODS: &[pyo3::class::methods::PyMethodDefType] = + fn #methods_trait_methods(self) -> &'static [::pyo3::class::methods::PyMethodDefType] { + static METHODS: &[::pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; METHODS } @@ -139,14 +139,14 @@ fn impl_proto_methods( if build_config.version <= PY39 && proto.name == "Buffer" { maybe_buffer_methods = Some(quote! { - impl pyo3::class::impl_::PyBufferProtocolProcs<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::PyBufferProtocolProcs<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { fn buffer_procs( self - ) -> Option<&'static pyo3::class::impl_::PyBufferProcs> { - static PROCS: pyo3::class::impl_::PyBufferProcs - = pyo3::class::impl_::PyBufferProcs { + ) -> Option<&'static ::pyo3::class::impl_::PyBufferProcs> { + static PROCS: ::pyo3::class::impl_::PyBufferProcs + = ::pyo3::class::impl_::PyBufferProcs { bf_getbuffer: Some(pyo3::class::buffer::getbuffer::<#ty>), bf_releasebuffer: Some(pyo3::class::buffer::releasebuffer::<#ty>), }; @@ -162,9 +162,9 @@ fn impl_proto_methods( let slot = syn::Ident::new(def.slot, Span::call_site()); let slot_impl = syn::Ident::new(def.slot_impl, Span::call_site()); quote! {{ - pyo3::ffi::PyType_Slot { - slot: pyo3::ffi::#slot, - pfunc: #module::#slot_impl::<#ty> as _ + ::pyo3::ffi::PyType_Slot { + slot: ::pyo3::ffi::#slot, + pfunc: ::#module::#slot_impl::<#ty> as _ } }} }) @@ -177,10 +177,10 @@ fn impl_proto_methods( quote! { #maybe_buffer_methods - impl pyo3::class::impl_::#slots_trait<#ty> - for pyo3::class::impl_::PyClassImplCollector<#ty> + impl ::pyo3::class::impl_::#slots_trait<#ty> + for ::pyo3::class::impl_::PyClassImplCollector<#ty> { - fn #slots_trait_slots(self) -> &'static [pyo3::ffi::PyType_Slot] { + fn #slots_trait_slots(self) -> &'static [::pyo3::ffi::PyType_Slot] { &[#(#tokens),*] } } diff --git a/tests/test_proc_macro_hygiene.rs b/tests/test_proc_macro_hygiene.rs new file mode 100644 index 00000000000..7bfcac0e16b --- /dev/null +++ b/tests/test_proc_macro_hygiene.rs @@ -0,0 +1,62 @@ +#![no_implicit_prelude] +#![allow(non_camel_case_types)] +#![allow(dead_code)] + +trait Use_unambiguous_imports { + type Error; +} + +struct Pyo3Shadowed; +type pyo3 = >::Error; + +struct CoreShadowed; +type core = >::Error; + +struct StdShadowed; +type std = >::Error; + +struct AllocShadowed; +type alloc = >::Error; + +#[::pyo3::proc_macro::pyclass] +#[derive(::std::clone::Clone)] +pub struct Foo; + +#[::pyo3::proc_macro::pyclass] +pub struct Foo2; + +#[::pyo3::proc_macro::pyclass( + name = "ActuallyBar", + freelist = 8, + weakref, + unsendable, + gc, + subclass, + extends = ::pyo3::types::PyDict, + module = "Spam" +)] +pub struct Bar { + #[pyo3(get, set)] + a: u8, + #[pyo3(get, set)] + b: Foo, + #[pyo3(get, set)] + c: ::std::option::Option<::pyo3::Py>, +} + +#[::pyo3::proc_macro::pyproto] +impl ::pyo3::class::gc::PyGCProtocol for Bar { + fn __traverse__( + &self, + visit: ::pyo3::class::gc::PyVisit, + ) -> ::std::result::Result<(), ::pyo3::class::gc::PyTraverseError> { + if let ::std::option::Option::Some(obj) = &self.c { + visit.call(obj)? + } + ::std::result::Result::Ok(()) + } + + fn __clear__(&mut self) { + self.c = ::std::option::Option::None; + } +} From f50a7a88589e21b922954821b86f9eb30f279040 Mon Sep 17 00:00:00 2001 From: mejrs Date: Tue, 17 Aug 2021 15:16:03 +0200 Subject: [PATCH 21/32] use macro to shadow --- tests/test_proc_macro_hygiene.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/test_proc_macro_hygiene.rs b/tests/test_proc_macro_hygiene.rs index 7bfcac0e16b..d12d8e1433b 100644 --- a/tests/test_proc_macro_hygiene.rs +++ b/tests/test_proc_macro_hygiene.rs @@ -1,22 +1,26 @@ #![no_implicit_prelude] -#![allow(non_camel_case_types)] -#![allow(dead_code)] -trait Use_unambiguous_imports { - type Error; -} - -struct Pyo3Shadowed; -type pyo3 = >::Error; +macro_rules! shadow { + ($name: ident) => { + ::paste::item! { + #[allow(non_camel_case_types, dead_code)] + unsafe trait [] {} -struct CoreShadowed; -type core = >::Error; + #[allow(non_camel_case_types, dead_code)] + struct []]> { + _ty: ::core::marker::PhantomData, + } -struct StdShadowed; -type std = >::Error; + #[allow(non_camel_case_types, dead_code)] + type $name = []<()>; + } + }; +} -struct AllocShadowed; -type alloc = >::Error; +shadow!(std); +shadow!(alloc); +shadow!(core); +shadow!(pyo3); #[::pyo3::proc_macro::pyclass] #[derive(::std::clone::Clone)] From b4f16339dcc529c152191204c1e7e775ab537e62 Mon Sep 17 00:00:00 2001 From: mejrs Date: Tue, 17 Aug 2021 15:19:19 +0200 Subject: [PATCH 22/32] fix extends param --- tests/test_proc_macro_hygiene.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_proc_macro_hygiene.rs b/tests/test_proc_macro_hygiene.rs index d12d8e1433b..747d6fbc7fa 100644 --- a/tests/test_proc_macro_hygiene.rs +++ b/tests/test_proc_macro_hygiene.rs @@ -36,7 +36,7 @@ pub struct Foo2; unsendable, gc, subclass, - extends = ::pyo3::types::PyDict, + extends = ::pyo3::types::PyAny, module = "Spam" )] pub struct Bar { From 99e7fe8da8436f027c0173e11630d05e4edfd81e Mon Sep 17 00:00:00 2001 From: mejrs Date: Tue, 17 Aug 2021 21:38:28 +0200 Subject: [PATCH 23/32] remove shadowing --- tests/test_proc_macro_hygiene.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/test_proc_macro_hygiene.rs b/tests/test_proc_macro_hygiene.rs index 747d6fbc7fa..5634128589c 100644 --- a/tests/test_proc_macro_hygiene.rs +++ b/tests/test_proc_macro_hygiene.rs @@ -1,27 +1,5 @@ #![no_implicit_prelude] -macro_rules! shadow { - ($name: ident) => { - ::paste::item! { - #[allow(non_camel_case_types, dead_code)] - unsafe trait [] {} - - #[allow(non_camel_case_types, dead_code)] - struct []]> { - _ty: ::core::marker::PhantomData, - } - - #[allow(non_camel_case_types, dead_code)] - type $name = []<()>; - } - }; -} - -shadow!(std); -shadow!(alloc); -shadow!(core); -shadow!(pyo3); - #[::pyo3::proc_macro::pyclass] #[derive(::std::clone::Clone)] pub struct Foo; From a85f3b498ad06828032814ae7a9070302d5af34a Mon Sep 17 00:00:00 2001 From: mejrs Date: Tue, 17 Aug 2021 22:32:41 +0200 Subject: [PATCH 24/32] fully disambiguate types --- pyo3-macros-backend/src/pyclass.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index deb017788fc..de80c2ccf41 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -345,13 +345,13 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { quote! { #[doc(hidden)] pub struct #inventory_cls { - methods: Vec, + methods: ::std::vec::Vec<::pyo3::class::PyMethodDefType>, } impl ::pyo3::class::impl_::PyMethodsInventory for #inventory_cls { - fn new(methods: Vec) -> Self { + fn new(methods: ::std::vec::Vec<::pyo3::class::PyMethodDefType>) -> Self { Self { methods } } - fn get(&'static self) -> &'static [pyo3::class::PyMethodDefType] { + fn get(&'static self) -> &'static [::pyo3::class::PyMethodDefType] { &self.methods } } From bbee0b29dbd676dc6ad380ea9ac55c8467c82aab Mon Sep 17 00:00:00 2001 From: Peter Schafhalter Date: Tue, 17 Aug 2021 14:21:56 -0700 Subject: [PATCH 25/32] Fix memory leak when calling Python from Rust --- src/python.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python.rs b/src/python.rs index b6757820693..5d5abd78c50 100644 --- a/src/python.rs +++ b/src/python.rs @@ -396,6 +396,7 @@ impl<'p> Python<'p> { return Err(PyErr::fetch(self)); } let res_ptr = ffi::PyEval_EvalCode(code_obj, globals, locals); + ffi::Py_DECREF(code_obj); self.from_owned_ptr_or_err(res_ptr) } From 4fdd4adcc629a1a960154e67a0faa5114ea222f8 Mon Sep 17 00:00:00 2001 From: Peter Schafhalter Date: Tue, 17 Aug 2021 14:37:39 -0700 Subject: [PATCH 26/32] Edit changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b11b22bec8a..4bd2f1c333b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) - Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) +- Add missing `Py_DECREF` to fix memory leak when calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806) ## [0.14.2] - 2021-08-09 From b8e932b2ec3bdb18444843cb0d7620f1c5f07e78 Mon Sep 17 00:00:00 2001 From: Peter Schafhalter Date: Tue, 17 Aug 2021 22:56:27 -0700 Subject: [PATCH 27/32] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bd2f1c333b..8affa1a64c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) - Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) -- Add missing `Py_DECREF` to fix memory leak when calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806) +- Add missing `Py_DECREF` to `Python::run_code` which fixes a memory leak when + calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806) ## [0.14.2] - 2021-08-09 From 8177535fc98413e2dcc1cb1737fa4c20a0623b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Le=C3=A9h?= <52788117+Ptrskay3@users.noreply.github.com> Date: Wed, 18 Aug 2021 12:50:57 +0200 Subject: [PATCH 28/32] Py_CompileString decref (#1810) * update changelog * fix memory leak in PyModule::from_code * add PR link to changelog * Add Py_DECREF also when PyImport_ExecCodeModuleEx fails * Remove duplicated calls, simplify logic Co-authored-by: messense Co-authored-by: messense --- CHANGELOG.md | 4 ++-- src/types/module.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8affa1a64c0..edac0046cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) - Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) -- Add missing `Py_DECREF` to `Python::run_code` which fixes a memory leak when - calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806) +- Add missing `Py_DECREF` to `Python::run_code` and `PyModule::from_code` which fixes a memory leak when + calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806), [#1810](https://github.com/PyO3/pyo3/pull/1810) ## [0.14.2] - 2021-08-09 diff --git a/src/types/module.rs b/src/types/module.rs index 483b8a1ac06..aa08fa2b010 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -127,6 +127,7 @@ impl PyModule { } let mptr = ffi::PyImport_ExecCodeModuleEx(module.as_ptr(), cptr, filename.as_ptr()); + ffi::Py_DECREF(cptr); if mptr.is_null() { return Err(PyErr::fetch(py)); } From 81fd23c65b2d3015f8c5491a8bc7775e88f3c2b7 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 18 Aug 2021 12:31:34 +0200 Subject: [PATCH 29/32] Use 'crate::' to refer to pyo3 Fixes: #1811 Signed-off-by: Christian Heimes --- CHANGELOG.md | 2 ++ src/types/datetime.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edac0046cea..850852b5763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) - Add missing `Py_DECREF` to `Python::run_code` and `PyModule::from_code` which fixes a memory leak when calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806), [#1810](https://github.com/PyO3/pyo3/pull/1810) +- Use crate:: syntax to refer to current crate in pyo3::types::datetime + [#1811](https://github.com/PyO3/pyo3/issues/1811) ## [0.14.2] - 2021-08-09 diff --git a/src/types/datetime.rs b/src/types/datetime.rs index cad4b79d2a9..990d82ee2e2 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -465,8 +465,8 @@ mod tests { #[cfg(not(PyPy))] #[test] fn test_new_with_fold() { - pyo3::Python::with_gil(|py| { - use pyo3::types::{PyDateTime, PyTimeAccess}; + crate::Python::with_gil(|py| { + use crate::types::{PyDateTime, PyTimeAccess}; let a = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, false); let b = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, true); From ee155ae8f85e9decd38a1e8c7de2b8f1fe2c8e83 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 10 Aug 2021 16:08:11 -0700 Subject: [PATCH 30/32] string: implement API to access raw string data With the recent implementation of non-limited unicode APIs, we're able to query Python's low-level state to access the raw bytes that Python is using to store string objects. This commit implements a safe Rust API for obtaining a view into Python's internals and representing the raw bytes Python is using to store strings. Not only do we allow accessing what Python has stored internally, but we also support coercing this data to a `Cow`. Closes #1776. --- CHANGELOG.md | 4 + src/types/mod.rs | 2 + src/types/string.rs | 286 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 850852b5763..f29714ed0fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- Add `PyString::data()` to access the raw bytes stored in a Python string. [#1794](https://github.com/PyO3/pyo3/issues/1794) + ### Fixed - Raise `AttributeError` to avoid panic when calling `del` on a `#[setter]` defined class property. [#1779](https://github.com/PyO3/pyo3/issues/1779) diff --git a/src/types/mod.rs b/src/types/mod.rs index f8bc24cf782..e635f440453 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -23,6 +23,8 @@ pub use self::num::PyLong as PyInt; pub use self::sequence::PySequence; pub use self::set::{PyFrozenSet, PySet}; pub use self::slice::{PySlice, PySliceIndices}; +#[cfg(not(Py_LIMITED_API))] +pub use self::string::PyStringData; pub use self::string::{PyString, PyString as PyUnicode}; pub use self::tuple::PyTuple; pub use self::typeobject::PyType; diff --git a/src/types/string.rs b/src/types/string.rs index fa6a2826010..9ac1a149839 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -1,14 +1,133 @@ // Copyright (c) 2017-present PyO3 Project and Contributors +#[cfg(not(Py_LIMITED_API))] +use crate::exceptions::PyUnicodeDecodeError; use crate::types::PyBytes; use crate::{ ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyNativeType, PyObject, PyResult, PyTryFrom, Python, ToPyObject, }; use std::borrow::Cow; +#[cfg(not(Py_LIMITED_API))] +use std::ffi::CStr; use std::os::raw::c_char; use std::str; +/// Represents raw data backing a Python `str`. +/// +/// Python internally stores strings in various representations. This enumeration +/// represents those variations. +#[cfg(not(Py_LIMITED_API))] +#[cfg_attr(docsrs, doc(cfg(not(Py_LIMITED_API))))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PyStringData<'a> { + /// UCS1 representation. + Ucs1(&'a [u8]), + + /// UCS2 representation. + Ucs2(&'a [u16]), + + /// UCS4 representation. + Ucs4(&'a [u32]), +} + +#[cfg(not(Py_LIMITED_API))] +impl<'a> PyStringData<'a> { + /// Obtain the raw bytes backing this instance as a [u8] slice. + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Ucs1(s) => s, + Self::Ucs2(s) => unsafe { + std::slice::from_raw_parts( + s.as_ptr() as *const u8, + s.len() * self.value_width_bytes(), + ) + }, + Self::Ucs4(s) => unsafe { + std::slice::from_raw_parts( + s.as_ptr() as *const u8, + s.len() * self.value_width_bytes(), + ) + }, + } + } + + /// Size in bytes of each value/item in the underlying slice. + #[inline] + pub fn value_width_bytes(&self) -> usize { + match self { + Self::Ucs1(_) => 1, + Self::Ucs2(_) => 2, + Self::Ucs4(_) => 4, + } + } + + /// Convert the raw data to a Rust string. + /// + /// For UCS-1 / UTF-8, returns a borrow into the original slice. For UCS-2 and UCS-4, + /// returns an owned string. + /// + /// Returns [PyUnicodeDecodeError] if the string data isn't valid in its purported + /// storage format. This should only occur for strings that were created via Python + /// C APIs that skip input validation (like `PyUnicode_FromKindAndData`) and should + /// never occur for strings that were created from Python code. + pub fn to_string(self, py: Python) -> PyResult> { + match self { + Self::Ucs1(data) => match str::from_utf8(data) { + Ok(s) => Ok(Cow::Borrowed(s)), + Err(e) => Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new_utf8( + py, data, e, + )?)), + }, + Self::Ucs2(data) => match String::from_utf16(data) { + Ok(s) => Ok(Cow::Owned(s)), + Err(e) => { + let mut message = e.to_string().as_bytes().to_vec(); + message.push(0); + + Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new( + py, + CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(&message).unwrap(), + )?)) + } + }, + Self::Ucs4(data) => match data.iter().map(|&c| std::char::from_u32(c)).collect() { + Some(s) => Ok(Cow::Owned(s)), + None => Err(crate::PyErr::from_instance(PyUnicodeDecodeError::new( + py, + CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), + )?)), + }, + } + } + + /// Convert the raw data to a Rust string, possibly with data loss. + /// + /// Invalid code points will be replaced with `U+FFFD REPLACEMENT CHARACTER`. + /// + /// Returns a borrow into original data, when possible, or owned data otherwise. + /// + /// The return value of this function should only disagree with [Self::to_string] + /// when that method would error. + pub fn to_string_lossy(self) -> Cow<'a, str> { + match self { + Self::Ucs1(data) => String::from_utf8_lossy(data), + Self::Ucs2(data) => Cow::Owned(String::from_utf16_lossy(data)), + Self::Ucs4(data) => Cow::Owned( + data.iter() + .map(|&c| std::char::from_u32(c).unwrap_or('\u{FFFD}')) + .collect(), + ), + } + } +} + /// Represents a Python `string` (a Unicode string object). /// /// This type is immutable. @@ -89,6 +208,45 @@ impl PyString { } } } + + /// Obtains the raw data backing the Python string. + /// + /// If the Python string object was created through legacy APIs, its internal + /// storage format will be canonicalized before data is returned. + #[cfg(not(Py_LIMITED_API))] + #[cfg_attr(docsrs, doc(cfg(not(Py_LIMITED_API))))] + pub fn data(&self) -> PyResult> { + let ptr = self.as_ptr(); + + if cfg!(not(Py_3_12)) { + #[allow(deprecated)] + let ready = unsafe { ffi::PyUnicode_READY(ptr) }; + if ready != 0 { + // Exception was created on failure. + return Err(crate::PyErr::fetch(self.py())); + } + } + + // The string should be in its canonical form after calling `PyUnicode_READY()`. + // And non-canonical form not possible after Python 3.12. So it should be safe + // to call these APIs. + let length = unsafe { ffi::PyUnicode_GET_LENGTH(ptr) } as usize; + let raw_data = unsafe { ffi::PyUnicode_DATA(ptr) }; + let kind = unsafe { ffi::PyUnicode_KIND(ptr) }; + + match kind { + ffi::PyUnicode_1BYTE_KIND => Ok(PyStringData::Ucs1(unsafe { + std::slice::from_raw_parts(raw_data as *const u8, length) + })), + ffi::PyUnicode_2BYTE_KIND => Ok(PyStringData::Ucs2(unsafe { + std::slice::from_raw_parts(raw_data as *const u16, length) + })), + ffi::PyUnicode_4BYTE_KIND => Ok(PyStringData::Ucs4(unsafe { + std::slice::from_raw_parts(raw_data as *const u32, length) + })), + _ => unreachable!(), + } + } } /// Converts a Rust `str` to a Python object. @@ -193,8 +351,16 @@ impl FromPyObject<'_> for char { #[cfg(test)] mod tests { use super::PyString; + #[cfg(not(Py_LIMITED_API))] + use super::PyStringData; + #[cfg(not(Py_LIMITED_API))] + use crate::exceptions::PyUnicodeDecodeError; + #[cfg(not(Py_LIMITED_API))] + use crate::type_object::PyTypeObject; use crate::Python; use crate::{FromPyObject, PyObject, PyTryFrom, ToPyObject}; + #[cfg(not(Py_LIMITED_API))] + use std::borrow::Cow; #[test] fn test_non_bmp() { @@ -297,4 +463,124 @@ mod tests { assert_eq!(format!("{}", s), "Hello\n"); }) } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs1() { + Python::with_gil(|py| { + let s = PyString::new(py, "hello, world"); + let data = s.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs1(b"hello, world")); + assert_eq!(data.to_string(py).unwrap(), Cow::Borrowed("hello, world")); + assert_eq!(data.to_string_lossy(), Cow::Borrowed("hello, world")); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs1_invalid() { + Python::with_gil(|py| { + // 0xfe is not allowed in UTF-8. + let buffer = b"f\xfe\0"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_1BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs1(b"f\xfe")); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-8' codec can't decode byte 0xfe in position 1")); + assert_eq!(data.to_string_lossy(), Cow::Borrowed("f�")); + }); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs2() { + Python::with_gil(|py| { + let s = py.eval("'foo\\ud800'", None, None).unwrap(); + let py_string = s.cast_as::().unwrap(); + let data = py_string.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs2(&[102, 111, 111, 0xd800])); + assert_eq!( + data.to_string_lossy(), + Cow::Owned::("foo�".to_string()) + ); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs2_invalid() { + Python::with_gil(|py| { + // U+FF22 (valid) & U+d800 (never valid) + let buffer = b"\x22\xff\x00\xd8\x00\x00"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_2BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs2(&[0xff22, 0xd800])); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-16' codec can't decode bytes in position 0-3")); + assert_eq!(data.to_string_lossy(), Cow::Owned::("Ｂ�".into())); + }); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs4() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let py_string = PyString::new(py, s); + let data = py_string.data().unwrap(); + + assert_eq!(data, PyStringData::Ucs4(&[21704, 21704, 128008])); + assert_eq!(data.to_string_lossy(), Cow::Owned::(s.to_string())); + }) + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_string_data_ucs4_invalid() { + Python::with_gil(|py| { + // U+20000 (valid) & U+d800 (never valid) + let buffer = b"\x00\x00\x02\x00\x00\xd8\x00\x00\x00\x00\x00\x00"; + let ptr = unsafe { + crate::ffi::PyUnicode_FromKindAndData( + crate::ffi::PyUnicode_4BYTE_KIND as _, + buffer.as_ptr() as *const _, + 2, + ) + }; + assert!(!ptr.is_null()); + let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let data = s.data().unwrap(); + assert_eq!(data, PyStringData::Ucs4(&[0x20000, 0xd800])); + let err = data.to_string(py).unwrap_err(); + assert_eq!(err.ptype(py), PyUnicodeDecodeError::type_object(py)); + assert!(err + .to_string() + .contains("'utf-32' codec can't decode bytes in position 0-7")); + assert_eq!(data.to_string_lossy(), Cow::Owned::("𠀀�".into())); + }); + } } From 19ca9e767d027a3364364fce9edf137fbda7db5e Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 14 Aug 2021 10:35:59 -0700 Subject: [PATCH 31/32] build: document InterpreterConfig fields I'm building functionality on top of this config and figured I'd take the time to document the fields to make things easier to understand. --- pyo3-build-config/src/impl_.rs | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index d9ee19ab98b..7b743b3a420 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -45,14 +45,60 @@ pub fn env_var(var: &str) -> Option { /// strategies are used to populate this type. #[cfg_attr(test, derive(Debug, PartialEq))] pub struct InterpreterConfig { + /// The Python implementation flavor. + /// + /// Serialized to `implementation`. pub implementation: PythonImplementation, + + /// Python `X.Y` version. e.g. `3.9`. + /// + /// Serialized to `version`. pub version: PythonVersion, + + /// Whether link library is shared. + /// + /// Serialized to `shared`. pub shared: bool, + + /// Whether linking against the stable/limited Python 3 API. + /// + /// Serialized to `abi3`. pub abi3: bool, + + /// The name of the link library defining Python. + /// + /// This effectively controls the `cargo:rustc-link-lib=` value to + /// control how libpython is linked. Values should not contain the `lib` + /// prefix. + /// + /// Serialized to `lib_name`. pub lib_name: Option, + + /// The directory containing the Python library to link against. + /// + /// The effectively controls the `cargo:rustc-link-search=native=` value + /// to add an additional library search path for the linker. + /// + /// Serialized to `lib_dir`. pub lib_dir: Option, + + /// Path of host `python` executable. + /// + /// This is a valid executable capable of running on the host/building machine. + /// For configurations derived by invoking a Python interpreter, it was the + /// executable invoked. + /// + /// Serialized to `executable`. pub executable: Option, + + /// Width in bits of pointers on the target machine. + /// + /// Serialized to `pointer_width`. pub pointer_width: Option, + + /// Additional relevant Python build flags / configuration settings. + /// + /// Serialized to `build_flags`. pub build_flags: BuildFlags, } From 5e53c4d0cf2b87337d72314bc6bde3132f3a5fbf Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 15 Aug 2021 23:37:08 +0100 Subject: [PATCH 32/32] release: 0.14.3 --- CHANGELOG.md | 12 ++++++------ Cargo.toml | 6 +++--- README.md | 4 ++-- pyo3-build-config/Cargo.toml | 2 +- pyo3-macros-backend/Cargo.toml | 4 ++-- pyo3-macros/Cargo.toml | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f29714ed0fe..ad4a980e6cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.14.3] - 2021-08-22 ### Added @@ -18,10 +18,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restrict FFI definitions `PyGILState_Check` and `Py_tracefunc` to the unlimited API. [#1787](https://github.com/PyO3/pyo3/pull/1787) - Add missing `_type` field to `PyStatus` struct definition. [#1791](https://github.com/PyO3/pyo3/pull/1791) - Reduce lower bound `num-complex` optional dependency to support interop with `rust-numpy` and `ndarray` when building with the MSRV of 1.41 [#1799](https://github.com/PyO3/pyo3/pull/1799) -- Add missing `Py_DECREF` to `Python::run_code` and `PyModule::from_code` which fixes a memory leak when - calling Python from Rust. [#1806](https://github.com/PyO3/pyo3/pull/1806), [#1810](https://github.com/PyO3/pyo3/pull/1810) -- Use crate:: syntax to refer to current crate in pyo3::types::datetime - [#1811](https://github.com/PyO3/pyo3/issues/1811) +- Fix memory leak in `Python::run_code`. [#1806](https://github.com/PyO3/pyo3/pull/1806) +- Fix memory leak in `PyModule::from_code`. [#1810](https://github.com/PyO3/pyo3/pull/1810) +- Remove use of `pyo3::` in `pyo3::types::datetime` which broke builds using `-Z avoid-dev-deps` [#1811](https://github.com/PyO3/pyo3/issues/1811) ## [0.14.2] - 2021-08-09 @@ -897,7 +896,8 @@ Yanked - Initial release -[unreleased]: https://github.com/pyo3/pyo3/compare/v0.14.1...HEAD +[unreleased]: https://github.com/pyo3/pyo3/compare/v0.14.3...HEAD +[0.14.3]: https://github.com/pyo3/pyo3/compare/v0.14.2...v0.14.3 [0.14.2]: https://github.com/pyo3/pyo3/compare/v0.14.1...v0.14.2 [0.14.1]: https://github.com/pyo3/pyo3/compare/v0.14.0...v0.14.1 [0.14.0]: https://github.com/pyo3/pyo3/compare/v0.13.2...v0.14.0 diff --git a/Cargo.toml b/Cargo.toml index c9b0ac21ac4..386de8df9eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.14.2" +version = "0.14.3" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -24,7 +24,7 @@ num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } # must stay at 0.1.x for Rust 1.41 compatibility paste = { version = "0.1.18", optional = true } -pyo3-macros = { path = "pyo3-macros", version = "=0.14.2", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.14.3", optional = true } unindent = { version = "0.1.4", optional = true } hashbrown = { version = ">= 0.9, < 0.12", optional = true } indexmap = { version = ">= 1.6, < 1.8", optional = true } @@ -42,7 +42,7 @@ pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initi serde_json = "1.0.61" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.14.2" } +pyo3-build-config = { path = "pyo3-build-config", version = "0.14.3" } [features] default = ["macros"] diff --git a/README.md b/README.md index faef74a32f7..ce14db4969a 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3] -version = "0.14.2" +version = "0.14.3" features = ["extension-module"] ``` @@ -108,7 +108,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.14.2" +version = "0.14.3" features = ["auto-initialize"] ``` diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index a94bbe70a94..dc1de0d482c 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.14.2" +version = "0.14.3" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 6f109c80dfa..ebe108984aa 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.14.2" +version = "0.14.3" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2018" [dependencies] quote = { version = "1", default-features = false } proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.2" } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.3" } [dependencies.syn] version = "1" diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index e6d2e0cfe33..a14589977ff 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.14.2" +version = "0.14.3" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,4 +16,4 @@ proc-macro = true [dependencies] quote = "1" syn = { version = "1", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.14.2" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.14.3" }