diff --git a/.cargo/config b/.cargo/config index c17c0f23439..e87eb0f23b1 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,6 +1,3 @@ -[alias] -xtask = "run --package xtask --" - [target.'cfg(feature = "cargo-clippy")'] rustflags = [ # TODO: remove these allows once msrv increased from 1.48 diff --git a/.github/bors.toml b/.github/bors.toml deleted file mode 100644 index dfe28aba8d9..00000000000 --- a/.github/bors.toml +++ /dev/null @@ -1,7 +0,0 @@ -delete_merged_branches = true -required_approvals = 0 -use_codeowners = false -status = [ - "conclusion", -] -timeout_sec = 21600 \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4473a007e8f..02c0d5d5c90 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,8 @@ -Thank you for contributing to pyo3! +Thank you for contributing to PyO3! + +By submitting these contributions you agree for them to be licensed under PyO3's [Apache-2.0 license](https://github.com/PyO3/pyo3#license). + +PyO3 is currently undergoing a relicensing process to match Rust's dual-license under `Apache-2.0` and `MIT` licenses. While that process is ongoing, if you are a first-time contributor please add your agreement as a comment in [#3108](https://github.com/PyO3/pyo3/pull/3108). Please consider adding the following to your pull request: - an entry for this PR in newsfragments - see [https://pyo3.rs/main/contributing.html#documenting-changes] @@ -6,5 +10,5 @@ Please consider adding the following to your pull request: - tests for all new or changed functions PyO3's CI pipeline will check your pull request. To run its tests -locally, you can run ```cargo xtask ci```. See its documentation - [here](https://github.com/PyO3/pyo3/tree/main/xtask#readme). +locally, you can run ```nox```. See ```nox --list-sessions``` +for a list of supported actions. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2155b82683c..f1e92149e7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ on: jobs: build: + continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 @@ -122,9 +123,11 @@ jobs: - uses: dorny/paths-filter@v2 # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} + if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} id: ffi-changes with: + base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' @@ -133,10 +136,10 @@ jobs: - '.github/workflows/build.yml' - name: Run pyo3-ffi-check + # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor + # is pypy 3.9 on windows + if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows')) }} run: nox -s ffi-check - # Allow failure on PyPy for now - continue-on-error: ${{ startsWith(inputs.python-version, 'pypy') }} - if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} - name: Test cross compilation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 122d2007a12..4bbd8aa46c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,11 +4,9 @@ on: push: branches: - main - # for bors r+ - - staging - # for bors try - - trying pull_request: + merge_group: + types: [checks_requested] workflow_dispatch: concurrency: @@ -34,6 +32,32 @@ jobs: - name: Check rust formatting (rustfmt) run: nox -s fmt-rust + check-msrv: + needs: [fmt] + runs-on: ubuntu-latest + if: github.ref != 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.48.0 + targets: x86_64-unknown-linux-gnu + components: clippy,rust-src + - uses: actions/setup-python@v4 + with: + architecture: "x64" + - uses: Swatinem/rust-cache@v2 + with: + key: check-msrv-1.48.0 + continue-on-error: true + - run: python -m pip install --upgrade pip && pip install nox + - name: Prepare minimal package versions + run: nox -s set-minimal-package-versions + - run: nox -s check-all + + env: + CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu + clippy: needs: [fmt] runs-on: ${{ matrix.platform.os }} @@ -80,16 +104,6 @@ jobs: rust-target: "i686-pc-windows-msvc", }, ] - include: - - rust: 1.48.0 - python-version: "3.11" - platform: - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - } - msrv: "MSRV" name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }} steps: - uses: actions/checkout@v3 @@ -106,15 +120,12 @@ jobs: key: clippy-${{ matrix.platform.rust-target }}-${{ matrix.platform.os }}-${{ matrix.rust }} continue-on-error: true - run: python -m pip install --upgrade pip && pip install nox - - if: matrix.msrv == 'MSRV' - name: Prepare minimal package versions (MSRV only) - run: nox -s set-minimal-package-versions - run: nox -s clippy-all env: CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} build-pr: - if: github.event_name == 'pull_request' + if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -158,7 +169,7 @@ jobs: } ] build-full: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -183,9 +194,11 @@ jobs: "3.9", "3.10", "3.11", - "pypy-3.7", - "pypy-3.8", - "pypy-3.9" + "3.12-dev", + "pypy3.7", + "pypy3.8", + "pypy3.9", + "pypy3.10", ] platform: [ @@ -353,6 +366,7 @@ jobs: conclusion: needs: - fmt + - check-msrv - clippy - build-pr - build-full diff --git a/.netlify/build.sh b/.netlify/build.sh index 74c8bca1927..6cab914b78c 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -17,21 +17,45 @@ mv pyo3-gh-pages netlify_build ## Configure netlify _redirects file # Add redirect for each documented version +set +x # these loops get very spammy and fill the deploy log + for d in netlify_build/v*; do version="${d/netlify_build\/v/}" echo "/v$version/doc/* https://docs.rs/pyo3/$version/:splat" >> netlify_build/_redirects + if [ $version != $PYO3_VERSION ]; then + # for old versions, mark the files in the latest version as the canonical URL + for file in $(find $d -type f); do + file_path="${file/$d\//}" + # remove index.html and/or .html suffix to match the page URL on the + # final netlfiy site + url_path="$file_path" + if [[ $file_path == index.html ]]; then + url_path="" + elif [[ $file_path == *.html ]]; then + url_path="${file_path%.html}" + fi + echo "/v$version/$url_path" >> netlify_build/_headers + if test -f "netlify_build/v$PYO3_VERSION/$file_path"; then + echo " Link: ; rel=\"canonical\"" >> netlify_build/_headers + else + # this file doesn't exist in the latest guide, don't index it + echo " X-Robots-Tag: noindex" >> netlify_build/_headers + fi + done + fi done # Add latest redirect -echo "/latest/* /v${PYO3_VERSION}/:splat" >> netlify_build/_redirects +echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects ## Add landing page redirect if [ "${CONTEXT}" == "deploy-preview" ]; then echo "/ /main/" >> netlify_build/_redirects else - echo "/ /v${PYO3_VERSION}/" >> netlify_build/_redirects + echo "/ /v${PYO3_VERSION}/ 302" >> netlify_build/_redirects fi +set -x ## Generate towncrier release notes pip install towncrier @@ -53,7 +77,7 @@ mv target/guide netlify_build/main/ ## Build public docs -cargo xtask doc +nox -s docs mv target/doc netlify_build/main/doc/ echo "" > netlify_build/main/doc/index.html @@ -61,7 +85,7 @@ echo "" > netlify_build/main/doc/in ## Build internal docs echo "
⚠️ Internal Docs ⚠️ Not Public API 👉 Official Docs Here
" > netlify_build/banner.html -RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" cargo xtask doc --internal +RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" nox -s docs -- nightly internal rm netlify_build/banner.html mkdir -p netlify_build/internal diff --git a/CHANGELOG.md b/CHANGELOG.md index 11c9b6bb772..fe87c5c3ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,31 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.19.1] - 2023-07-03 + +### Packaging + +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.14 [#3258](https://github.com/PyO3/pyo3/pull/3258) +- Extend range of supported versions of `indexmap` optional dependency to include version 2. [#3277](https://github.com/PyO3/pyo3/pull/3277) +- Support PyPy 3.10. [#3289](https://github.com/PyO3/pyo3/pull/3289) + +### Added + +- Add `pyo3::types::PyFrozenSetBuilder` to allow building a `PyFrozenSet` item by item. [#3156](https://github.com/PyO3/pyo3/pull/3156) +- Add support for converting to and from Python's `ipaddress.IPv4Address`/`ipaddress.IPv6Address` and `std::net::IpAddr`. [#3197](https://github.com/PyO3/pyo3/pull/3197) +- Add support for `num-bigint` feature in combination with `abi3`. [#3198](https://github.com/PyO3/pyo3/pull/3198) +- Add `PyErr_GetRaisedException()`, `PyErr_SetRaisedException()` to FFI definitions for Python 3.12 and later. [#3248](https://github.com/PyO3/pyo3/pull/3248) +- Add `Python::with_pool` which is a safer but more limited alternative to `Python::new_pool`. [#3263](https://github.com/PyO3/pyo3/pull/3263) +- Add `PyDict::get_item_with_error` on PyPy. [#3270](https://github.com/PyO3/pyo3/pull/3270) +- Allow `#[new]` methods may to return `Py` in order to return existing instances. [#3287](https://github.com/PyO3/pyo3/pull/3287) + +### Fixed + +- Fix conversion of classes implementing `__complex__` to `Complex` when using `abi3` or PyPy. [#3185](https://github.com/PyO3/pyo3/pull/3185) +- Stop suppressing unrelated exceptions in `PyAny::hasattr`. [#3271](https://github.com/PyO3/pyo3/pull/3271) +- Fix memory leak when creating `PySet` or `PyFrozenSet` or returning types converted into these internally, e.g. `HashSet` or `BTreeSet`. [#3286](https://github.com/PyO3/pyo3/pull/3286) + + ## [0.19.0] - 2023-05-31 ### Packaging @@ -1478,7 +1503,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.1...HEAD +[0.19.1]: https://github.com/pyo3/pyo3/compare/v0.19.0...v0.19.1 [0.19.0]: https://github.com/pyo3/pyo3/compare/v0.18.3...v0.19.0 [0.18.3]: https://github.com/pyo3/pyo3/compare/v0.18.2...v0.18.3 [0.18.2]: https://github.com/pyo3/pyo3/compare/v0.18.1...v0.18.2 diff --git a/Cargo.toml b/Cargo.toml index b0d84aae0d2..d6dd041bb90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.19.0" +version = "0.19.1" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -20,10 +20,10 @@ parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.1" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.19.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.19.1", optional = true } indoc = { version = "1.0.3", optional = true } unindent = { version = "0.1.4", optional = true } @@ -34,8 +34,8 @@ inventory = { version = "0.3.0", optional = true } anyhow = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false, optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } -hashbrown = { version = ">= 0.9, < 0.14", optional = true } -indexmap = { version = "1.6", optional = true } +hashbrown = { version = ">= 0.9, < 0.15", optional = true } +indexmap = { version = ">= 1.6, < 3", optional = true } num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } rust_decimal = { version = "1.0.0", default-features = false, optional = true } @@ -58,7 +58,7 @@ rust_decimal = { version = "1.8.0", features = ["std"] } widestring = "0.5.1" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } [features] default = ["macros"] @@ -176,14 +176,11 @@ harness = false [workspace] members = [ "pyo3-ffi", - "pyo3-ffi-check", - "pyo3-ffi-check/macro", "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", "pytests", "examples", - "xtask" ] [package.metadata.docs.rs] diff --git a/Contributing.md b/Contributing.md index a1fd31a34d9..c6d29e2d427 100644 --- a/Contributing.md +++ b/Contributing.md @@ -51,7 +51,7 @@ There are some specific areas of focus where help is currently needed for the do You can build the docs (including all features) with ```shell -cargo xtask doc --open +nox -s docs -- open ``` #### Doctests @@ -95,8 +95,10 @@ Tests run with all supported Python versions with the latest stable Rust compile If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. You can run these tests yourself with -```cargo xtask ci``` -See [its documentation](https://github.com/PyO3/pyo3/tree/main/xtask#readme) for more commands you can run. +```nox``` +and +```nox -l``` +lists further commands you can run. ### Documenting changes @@ -145,7 +147,7 @@ You can view what code is and isn't covered by PyO3's tests. We aim to have 100% - First, generate a `lcov.info` file with ```shell -cargo xtask coverage +nox -s coverage ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. diff --git a/README.md b/README.md index 594fbc7f519..cc45f93e29c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # PyO3 -[![actions status](https://github.com/PyO3/pyo3/workflows/CI/badge.svg)](https://github.com/PyO3/pyo3/actions) -[![benchmark](https://github.com/PyO3/pyo3/actions/workflows/bench.yml/badge.svg)](https://pyo3.rs/dev/bench/) -[![codecov](https://codecov.io/gh/PyO3/pyo3/branch/main/graph/badge.svg)](https://codecov.io/gh/PyO3/pyo3) -[![crates.io](https://img.shields.io/crates/v/pyo3)](https://crates.io/crates/pyo3) +[![actions status](https://img.shields.io/github/actions/workflow/status/PyO3/pyo3/ci.yml?branch=main&logo=github&style=)](https://github.com/PyO3/pyo3/actions) +[![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) +[![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) +[![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) [![minimum rustc 1.48](https://img.shields.io/badge/rustc-1.48+-blue.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![dev chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/PyO3/Lobby) -[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) +[![dev chat](https://img.shields.io/gitter/room/PyO3/Lobby?logo=gitter)](https://gitter.im/PyO3/Lobby) +[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.0", features = ["extension-module"] } +pyo3 = { version = "0.19.1", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.19.0" +version = "0.19.1" features = ["auto-initialize"] ``` @@ -216,7 +216,9 @@ about this topic. ## Articles and other media -- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - March 28, 2023 +- [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 +- [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 +- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - Mar 28, 2023 - [How Pydantic V2 leverages Rust's Superpowers](https://fosdem.org/2023/schedule/event/rust_how_pydantic_v2_leverages_rusts_superpowers/) - Feb 4, 2023 - [How we extended the River stats module with Rust using PyO3](https://boring-guy.sh/posts/river-rust/) - Dec 23, 2022 - [Nine Rules for Writing Python Extensions in Rust](https://towardsdatascience.com/nine-rules-for-writing-python-extensions-in-rust-d35ea3a4ec29?sk=f8d808d5f414154fdb811e4137011437) - Dec 31, 2021 @@ -244,7 +246,8 @@ If you don't have time to contribute yourself but still wish to support the proj ## License -PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). +PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). (The PyO3 project is in the process of collecting permission from past contributors to additionally license under the [MIT license](https://opensource.org/license/mit/), see [#3108](https://github.com/PyO3/pyo3/pull/3108). Once complete, PyO3 will additionally be licensed under the MIT license, the same as the Rust language itself is both Apache and MIT licensed.) + Python is licensed under the [Python License](https://docs.python.org/3/license.html). Deploys by Netlify diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e76132c4afe..bdc78d6a034 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "examples" +name = "pyo3-examples" version = "0.0.0" publish = false edition = "2018" [dev-dependencies] -pyo3 = { version = "0.19.0", path = "..", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.19.1", path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 0368bb1f432..088ea73bfbe 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/.template/pyproject.toml b/examples/decorator/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/decorator/.template/pyproject.toml +++ b/examples/decorator/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/decorator/pyproject.toml b/examples/decorator/pyproject.toml index c6196256d35..8575ca25fc2 100644 --- a/examples/decorator/pyproject.toml +++ b/examples/decorator/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 0368bb1f432..088ea73bfbe 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pyproject.toml b/examples/maturin-starter/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/maturin-starter/.template/pyproject.toml +++ b/examples/maturin-starter/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/pyproject.toml b/examples/maturin-starter/pyproject.toml index 9a18a20ea8b..fb9c808f283 100644 --- a/examples/maturin-starter/pyproject.toml +++ b/examples/maturin-starter/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 0915e77badd..158e1522040 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/plugin/plugin_api/pyproject.toml b/examples/plugin/plugin_api/pyproject.toml index 114687eddef..6350644ca14 100644 --- a/examples/plugin/plugin_api/pyproject.toml +++ b/examples/plugin/plugin_api/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 3ed1b01d30f..19ea7cc8520 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 68988ce3f48..a4bfa7ce13b 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,3 +1,3 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::delete(".template"); diff --git a/examples/word-count/pyproject.toml b/examples/word-count/pyproject.toml index d8ce3650277..6f88a5170f1 100644 --- a/examples/word-count/pyproject.toml +++ b/examples/word-count/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/guide/src/class.md b/guide/src/class.md index dae5cbc04ec..b5a27b517cc 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -60,7 +60,7 @@ To integrate Rust types with Python, PyO3 needs to place some restrictions on th Rust lifetimes are used by the Rust compiler to reason about a program's memory safety. They are a compile-time only concept; there is no way to access Rust lifetimes at runtime from a dynamic language like Python. -As soon as Rust data is exposed to Python, there is no guarantee which the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. +As soon as Rust data is exposed to Python, there is no guarantee that the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. When you need to share ownership of data between Python and Rust, instead of using borrowed references with lifetimes consider using reference-counted smart pointers such as [`Arc`] or [`Py`]. @@ -74,7 +74,7 @@ Because Python objects are freely shared between threads by the Python interpret ## Constructor -By default it is not possible to create an instance of a custom class from Python code. +By default, it is not possible to create an instance of a custom class from Python code. To declare a constructor, you need to define a method and annotate it with the `#[new]` attribute. Only Python's `__new__` method can be specified, `__init__` is not available. @@ -113,8 +113,11 @@ impl Nonzero { } ``` +If you want to return an existing object (for example, because your `new` +method caches the values it returns), `new` can return `pyo3::Py`. + As you can see, the Rust method name is not important here; this way you can -still use `new()` for a Rust-level constructor. +still, use `new()` for a Rust-level constructor. If no method marked with `#[new]` is declared, object instances can only be created from Rust, but not from Python. @@ -226,8 +229,10 @@ struct FrozenCounter { value: AtomicUsize, } -let py_counter: Py = Python::with_gil(|py| { - let counter = FrozenCounter { value: AtomicUsize::new(0) }; +let py_counter: Py = Python::with_gil(|py| { + let counter = FrozenCounter { + value: AtomicUsize::new(0), + }; Py::new(py, counter).unwrap() }); @@ -264,7 +269,7 @@ use the `extends` parameter for `pyclass` with the full path to the base class. For convenience, `(T, U)` implements `Into>` where `U` is the base class of `T`. -But for more deeply nested inheritance, you have to return `PyClassInitializer` +But for a more deeply nested inheritance, you have to return `PyClassInitializer` explicitly. To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, @@ -647,9 +652,9 @@ impl BaseClass { #[new] #[classmethod] fn py_new<'p>(cls: &'p PyType, py: Python<'p>) -> PyResult { - // Get an abstract attribute (presumably) declared on a subclass of this class. - let subclass_attr = cls.getattr("a_class_attr")?; - Ok(Self(subclass_attr.to_object(py))) + // Get an abstract attribute (presumably) declared on a subclass of this class. + let subclass_attr = cls.getattr("a_class_attr")?; + Ok(Self(subclass_attr.to_object(py))) } } ``` @@ -716,11 +721,72 @@ impl MyClass { } ``` +## Free functions + +Free functions defined using `#[pyfunction]` interact with classes through the same mechanisms as the self parameters of instance methods, i.e. they can take GIL-bound references, GIL-bound reference wrappers or GIL-indepedent references: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +struct MyClass { + my_field: i32, +} + +// Take a GIL-bound reference when the underlying `PyCell` is irrelevant. +#[pyfunction] +fn increment_field(my_class: &mut MyClass) { + my_class.my_field += 1; +} + +// Take a GIL-bound reference wrapper when borrowing should be automatic, +// but interaction with the underlying `PyCell` is desired. +#[pyfunction] +fn print_field(my_class: PyRef<'_, MyClass>) { + println!("{}", my_class.my_field); +} + +// Take a GIL-bound reference to the underlying cell +// when borrowing needs to be managed manually. +#[pyfunction] +fn increment_then_print_field(my_class: &PyCell) { + my_class.borrow_mut().my_field += 1; + + println!("{}", my_class.borrow().my_field); +} + +// Take a GIL-indepedent reference when you want to store the reference elsewhere. +#[pyfunction] +fn print_refcnt(my_class: Py, py: Python<'_>) { + println!("{}", my_class.get_refcnt(py)); +} +``` + +Classes can also be passed by value if they can be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +#[derive(Clone)] +struct MyClass { + my_field: Box, +} + +#[pyfunction] +fn dissamble_clone(my_class: MyClass) { + let MyClass { mut my_field } = my_class; + *my_field += 1; +} +``` + +Note that `#[derive(FromPyObject)]` on a class is usually not useful as it tries to construct a new Rust value by filling in the fields by looking up attributes of any given Python value. + ## Method arguments Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. -The following example defines a class `MyClass` with a method `method`. This method has a signature which sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: +The following example defines a class `MyClass` with a method `method`. This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: ```rust # use pyo3::prelude::*; @@ -756,7 +822,7 @@ impl MyClass { } ``` -In Python this might be used like: +In Python, this might be used like: ```python >>> import mymodule diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 38ed27c1f1e..911a9869973 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -13,7 +13,7 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| | `object` | - | `&PyAny` | -| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf` | `&PyUnicode` | +| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | | `bool` | `bool` | `&PyBool` | | `int` | Any integer type (`i32`, `u32`, `usize`, etc) | `&PyLong` | @@ -28,6 +28,7 @@ The table below contains the Python type and the corresponding function argument | `slice` | - | `&PySlice` | | `type` | - | `&PyType` | | `module` | - | `&PyModule` | +| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `datetime.datetime` | - | `&PyDateTime` | | `datetime.date` | - | `&PyDate` | | `datetime.time` | - | `&PyTime` | diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 2dd45a43ce7..2074590e325 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -123,7 +123,7 @@ You should also create a `pyproject.toml` with the following contents: ```toml [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/noxfile.py b/noxfile.py index d264cdc0467..23f04785fed 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,15 +4,14 @@ import subprocess import sys import tempfile -import time from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple import nox -nox.options.sessions = ["test", "clippy", "fmt"] +nox.options.sessions = ["test", "clippy", "fmt", "docs"] PYO3_DIR = Path(__file__).parent @@ -35,7 +34,7 @@ def test_rust(session: nox.Session): _run_cargo_test(session) _run_cargo_test(session, features="abi3") - if not "skip-full" in session.posargs: + if "skip-full" not in session.posargs: _run_cargo_test(session, features="full") _run_cargo_test(session, features="abi3 full") @@ -50,11 +49,10 @@ def test_py(session: nox.Session) -> None: @nox.session(venv_backend="none") def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) - _run(session, "cargo", "llvm-cov", "clean", "--workspace", external=True) + _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) - _run( + _run_cargo( session, - "cargo", "llvm-cov", "--package=pyo3", "--package=pyo3-build-config", @@ -65,7 +63,6 @@ def coverage(session: nox.Session) -> None: "--codecov", "--output-path", "coverage.json", - external=True, ) @@ -77,7 +74,8 @@ def fmt(session: nox.Session): @nox.session(name="fmt-rust", venv_backend="none") def fmt_rust(session: nox.Session): - _run(session, "cargo", "fmt", "--all", "--check", external=True) + _run_cargo(session, "fmt", "--all", "--check") + _run_cargo(session, "fmt", *_FFI_CHECK, "--all", "--check") @nox.session(name="fmt-py") @@ -96,26 +94,15 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: success = True env = env or os.environ for feature_set in _get_feature_sets(): - command = "clippy" - extra = ("--", "--deny=warnings") - if _get_rust_version()[:2] == (1, 48): - # 1.48 crashes during clippy because of lints requested - # in .cargo/config - command = "check" - extra = () try: - _run( + _run_cargo( session, - "cargo", - command, + "clippy", *feature_set, "--all-targets", "--workspace", - # linting pyo3-ffi-check requires docs to have been built or - # the macros will error; doesn't seem worth it on CI - "--exclude=pyo3-ffi-check", - *extra, - external=True, + "--", + "--deny=warnings", env=env, ) except Exception: @@ -126,31 +113,37 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: @nox.session(name="clippy-all", venv_backend="none") def clippy_all(session: nox.Session) -> None: success = True - with tempfile.NamedTemporaryFile("r+") as config: - env = os.environ.copy() - env["PYO3_CONFIG_FILE"] = config.name - env["PYO3_CI"] = "1" - def _clippy_with_config(implementation, version) -> bool: - config.seek(0) - config.truncate(0) - config.write( - f"""\ -implementation={implementation} -version={version} -suppress_build_script_link_lines=true -""" - ) - config.flush() + def _clippy_with_config(env: Dict[str, str]) -> None: + nonlocal success + success &= _clippy(session, env=env) - session.log(f"{implementation} {version}") - return _clippy(session, env=env) + _for_all_version_configs(session, _clippy_with_config) - for version in PY_VERSIONS: - success &= _clippy_with_config("CPython", version) + if not success: + session.error("one or more jobs failed") - for version in PYPY_VERSIONS: - success &= _clippy_with_config("PyPy", version) + +@nox.session(name="check-all", venv_backend="none") +def check_all(session: nox.Session) -> None: + success = True + + def _check(env: Dict[str, str]) -> None: + nonlocal success + for feature_set in _get_feature_sets(): + try: + _run_cargo( + session, + "check", + *feature_set, + "--all-targets", + "--workspace", + env=env, + ) + except Exception: + success = False + + _for_all_version_configs(session, _check) if not success: session.error("one or more jobs failed") @@ -159,13 +152,9 @@ def _clippy_with_config(implementation, version) -> bool: @nox.session(venv_backend="none") def publish(session: nox.Session) -> None: _run_cargo_publish(session, package="pyo3-build-config") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros-backend") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros") - time.sleep(10) _run_cargo_publish(session, package="pyo3-ffi") - time.sleep(10) _run_cargo_publish(session, package="pyo3") @@ -287,6 +276,43 @@ def test_emscripten(session: nox.Session): ) +@nox.session(venv_backend="none") +def docs(session: nox.Session) -> None: + rustdoc_flags = ["-Dwarnings"] + toolchain_flags = [] + cargo_flags = [] + + if "open" in session.posargs: + cargo_flags.append("--open") + + if "nightly" in session.posargs: + rustdoc_flags.append("--cfg docsrs") + toolchain_flags.append("+nightly") + cargo_flags.extend(["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"]) + + if "nightly" in session.posargs and "internal" in session.posargs: + rustdoc_flags.append("--Z unstable-options") + rustdoc_flags.append("--document-hidden-items") + cargo_flags.append("--document-private-items") + else: + cargo_flags.extend(["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"]) + + rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) + session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + + _run_cargo( + session, + *toolchain_flags, + "doc", + "--lib", + "--no-default-features", + "--features=full", + "--no-deps", + "--workspace", + *cargo_flags, + ) + + @nox.session(name="build-guide", venv_backend="none") def build_guide(session: nox.Session): _run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs) @@ -298,8 +324,6 @@ def format_guide(session: nox.Session): for path in Path("guide").glob("**/*.md"): session.log("Working on %s", path) - content = path.read_text() - lines = iter(path.read_text().splitlines(True)) new_lines = [] @@ -346,9 +370,8 @@ def format_guide(session: nox.Session): @nox.session(name="address-sanitizer", venv_backend="none") def address_sanitizer(session: nox.Session): - _run( + _run_cargo( session, - "cargo", "+nightly", "test", "--release", @@ -361,7 +384,6 @@ def address_sanitizer(session: nox.Session): "RUSTDOCFLAGS": "-Zsanitizer=address", "ASAN_OPTIONS": "detect_leaks=0", }, - external=True, ) @@ -459,20 +481,13 @@ def set_minimal_package_versions(session: nox.Session): "wasm-bindgen": "0.2.84", "syn": "1.0.109", } - # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. for project in projects: if project is None: - _run(session, "cargo", "update", external=True) + _run_cargo(session, "update") else: - _run( - session, - "cargo", - "update", - f"--manifest-path={project}/Cargo.toml", - external=True, - ) + _run_cargo(session, "update", f"--manifest-path={project}/Cargo.toml") for project in projects: lock_file = Path(project or "") / "Cargo.lock" @@ -506,22 +521,21 @@ def load_pkg_versions(): # supported on MSRV for project in projects: if project is None: - _run(session, "cargo", "metadata", silent=True, external=True) + _run_cargo(session, "metadata", silent=True) else: - _run( + _run_cargo( session, - "cargo", "metadata", f"--manifest-path={project}/Cargo.toml", silent=True, - external=True, ) @nox.session(name="ffi-check") def ffi_check(session: nox.Session): - session.run("cargo", "doc", "-p", "pyo3-ffi", "--no-deps", external=True) - _run(session, "cargo", "run", "-p", "pyo3-ffi-check", external=True) + _run_cargo(session, "doc", *_FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") + _run_cargo(session, "clippy", "--workspace", "--all-targets", *_FFI_CHECK) + _run_cargo(session, "run", *_FFI_CHECK) @lru_cache() @@ -603,6 +617,10 @@ def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: print("::endgroup::", file=sys.stderr) +def _run_cargo(session: nox.Session, *args: str, **kwargs: Any) -> None: + _run(session, "cargo", *args, **kwargs, external=True) + + def _run_cargo_test( session: nox.Session, *, @@ -624,7 +642,7 @@ def _run_cargo_test( def _run_cargo_publish(session: nox.Session, *, package: str) -> None: - _run(session, "cargo", "publish", f"--package={package}", external=True) + _run_cargo(session, "publish", f"--package={package}") def _run_cargo_set_package_version( @@ -642,3 +660,36 @@ def _run_cargo_set_package_version( def _get_output(*args: str) -> str: return subprocess.run(args, capture_output=True, text=True, check=True).stdout + + +def _for_all_version_configs( + session: nox.Session, job: Callable[[Dict[str, str]], None] +) -> None: + with tempfile.NamedTemporaryFile("r+") as config: + env = os.environ.copy() + env["PYO3_CONFIG_FILE"] = config.name + env["PYO3_CI"] = "1" + + def _job_with_config(implementation, version) -> bool: + config.seek(0) + config.truncate(0) + config.write( + f"""\ +implementation={implementation} +version={version} +suppress_build_script_link_lines=true +""" + ) + config.flush() + + session.log(f"{implementation} {version}") + return job(env) + + for version in PY_VERSIONS: + _job_with_config("CPython", version) + + for version in PYPY_VERSIONS: + _job_with_config("PyPy", version) + + +_FFI_CHECK = ("--manifest-path", "pyo3-ffi-check/Cargo.toml") diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index ac8d451fada..50445fc938a 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.19.0" +version = "0.19.1" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 6f57e0e6eed..8670b355bac 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -87,8 +87,8 @@ impl From<&'_ str> for Error { } impl From for Error { - fn from(_: std::convert::Infallible) -> Self { - unreachable!() + fn from(value: std::convert::Infallible) -> Self { + match value {} } } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index f870daa7ea0..07c5bcb6d37 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -161,6 +161,11 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 59 { println!("cargo:rustc-cfg=thread_local_const_init"); } + + // Enable use of `#[cfg(panic = "...")]` on Rust 1.60 and greater + if rustc_minor_version >= 60 { + println!("cargo:rustc-cfg=panic_unwind"); + } } /// Private exports used in PyO3's build.rs diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index e1663eb8cdc..06829298ea3 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,5 +13,10 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.63.0" +bindgen = "0.66.1" pyo3-build-config = { path = "../pyo3-build-config" } + +[workspace] +members = [ + "macro" +] diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index a9f51f654c6..999342fa9c9 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -11,4 +11,4 @@ proc-macro = true glob = "0.3" quote = "1" proc-macro2 = "1" -scraper = "0.12" +scraper = "0.17" diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index c537362530b..91a7dca6ee5 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -40,9 +40,9 @@ fn main() { pyo3_ffi_align, bindgen_align ); - } else { - pyo3_ffi_check_macro::for_all_fields!($name, check_field); } + + pyo3_ffi_check_macro::for_all_fields!($name, check_field); }}; } @@ -78,7 +78,8 @@ fn main() { non_camel_case_types, non_upper_case_globals, dead_code, - improper_ctypes + improper_ctypes, + clippy::all )] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index bda76ab6b55..4e910d3fa5a 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.19.0" +version = "0.19.1" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,4 +38,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index b77b2f988d0..fa5b1c20df9 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -2,7 +2,7 @@ use crate::object::*; use crate::pyport::Py_ssize_t; #[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_uchar, c_void}; +use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; // skipped _Py_CODEUNIT // skipped _Py_OPCODE @@ -11,6 +11,16 @@ use std::os::raw::{c_char, c_int, c_uchar, c_void}; #[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] opaque_struct!(_PyOpcache); +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoCached { + pub _co_code: *mut PyObject, + pub _co_varnames: *mut PyObject, + pub _co_cellvars: *mut PyObject, + pub _co_freevars: *mut PyObject, +} + #[cfg(all(not(PyPy), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -83,17 +93,24 @@ pub struct PyCodeObject { pub co_names: *mut PyObject, pub co_exceptiontable: *mut PyObject, pub co_flags: c_int, + #[cfg(not(Py_3_12))] pub co_warmup: c_int, + #[cfg(Py_3_12)] + pub _co_linearray_entry_size: c_short, pub co_argcount: c_int, pub co_posonlyargcount: c_int, pub co_kwonlyargcount: c_int, pub co_stacksize: c_int, pub co_firstlineno: c_int, + pub co_nlocalsplus: c_int, + #[cfg(Py_3_12)] + pub co_framesize: c_int, pub co_nlocals: c_int, pub co_nplaincellvars: c_int, pub co_ncellvars: c_int, pub co_nfreevars: c_int, + pub co_localsplusnames: *mut PyObject, pub co_localspluskinds: *mut PyObject, pub co_filename: *mut PyObject, @@ -101,9 +118,15 @@ pub struct PyCodeObject { pub co_qualname: *mut PyObject, pub co_linetable: *mut PyObject, pub co_weakreflist: *mut PyObject, + #[cfg(not(Py_3_12))] pub _co_code: *mut PyObject, + #[cfg(Py_3_12)] + pub _co_cached: *mut _PyCoCached, + #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, pub _co_firsttraceable: c_int, + #[cfg(Py_3_12)] + pub _co_linearray: *mut c_char, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 9a2afdb93e3..71af81e83e5 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,12 +30,27 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT +#[cfg(all(Py_3_12, not(PyPy)))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCompilerSrcLocation { + pub lineno: c_int, + pub end_lineno: c_int, + pub col_offset: c_int, + pub end_col_offset: c_int, +} + +// skipped SRC_LOCATION_FROM_AST + #[cfg(not(PyPy))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, + #[cfg(not(Py_3_12))] pub ff_lineno: c_int, + #[cfg(Py_3_12)] + pub ff_location: _PyCompilerSrcLocation, } pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 7d40ba08d38..17fe7559e1b 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -91,6 +91,8 @@ pub struct PyConfig { #[cfg(all(Py_3_9, not(Py_3_10)))] pub _use_peg_parser: c_int, pub tracemalloc: c_int, + #[cfg(Py_3_12)] + pub perf_profiling: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -137,6 +139,8 @@ pub struct PyConfig { pub use_frozen_modules: c_int, #[cfg(Py_3_11)] pub safe_path: c_int, + #[cfg(Py_3_12)] + pub int_max_str_digits: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -163,7 +167,7 @@ pub struct PyConfig { pub run_filename: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, - #[cfg(Py_3_9)] + #[cfg(all(Py_3_9, not(Py_3_12)))] pub _isolated_interpreter: c_int, #[cfg(Py_3_11)] pub _is_python_build: c_int, diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 870473b46da..76ab074f3d0 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -276,9 +276,11 @@ pub struct PyTypeObject { pub tp_finalize: Option, #[cfg(Py_3_8)] pub tp_vectorcall: Option, - #[cfg(any(all(PyPy, Py_3_8), all(not(PyPy), Py_3_8, not(Py_3_9))))] + #[cfg(Py_3_12)] + pub tp_watched: c_char, + #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] pub tp_print: Option, - #[cfg(PyPy)] + #[cfg(all(PyPy, not(Py_3_10)))] pub tp_pypy_flags: std::os::raw::c_long, #[cfg(py_sys_config = "COUNT_ALLOCS")] pub tp_allocs: Py_ssize_t, diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index abbffc0b8c2..fe7b4d4b045 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -65,6 +65,8 @@ pub struct PyImportErrorObject { pub msg: *mut PyObject, pub name: *mut PyObject, pub path: *mut PyObject, + #[cfg(Py_3_12)] + pub name_from: *mut PyObject, } #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 7cd3bfbc8fa..1bd93655c6a 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -118,10 +118,7 @@ where } const STATE_INTERNED_INDEX: usize = 0; -#[cfg(not(Py_3_12))] const STATE_INTERNED_WIDTH: u8 = 2; -#[cfg(Py_3_12)] -const STATE_INTERNED_WIDTH: u8 = 1; const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; const STATE_KIND_WIDTH: u8 = 3; @@ -268,7 +265,7 @@ impl PyASCIIObject { /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Returns one of: [`SSTATE_NOT_INTERNED`], [`SSTATE_INTERNED_MORTAL`], - /// or on CPython earlier than 3.12, [`SSTATE_INTERNED_IMMORTAL`] + /// or [`SSTATE_INTERNED_IMMORTAL`]. #[inline] pub unsafe fn interned(&self) -> c_uint { PyASCIIObjectState::from(self.state).interned() @@ -277,8 +274,7 @@ impl PyASCIIObject { /// Set the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Calling this function with an argument that is not [`SSTATE_NOT_INTERNED`], - /// [`SSTATE_INTERNED_MORTAL`], or on CPython earlier than 3.12, - /// [`SSTATE_INTERNED_IMMORTAL`] is invalid. + /// [`SSTATE_INTERNED_MORTAL`], or [`SSTATE_INTERNED_IMMORTAL`] is invalid. #[inline] pub unsafe fn set_interned(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); @@ -398,12 +394,14 @@ extern "C" { pub const SSTATE_NOT_INTERNED: c_uint = 0; pub const SSTATE_INTERNED_MORTAL: c_uint = 1; -#[cfg(not(Py_3_12))] pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; +#[cfg(Py_3_12)] +pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).ascii() @@ -420,7 +418,7 @@ pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { } #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +#[deprecated(note = "Removed in Python 3.12")] pub const PyUnicode_WCHAR_KIND: c_uint = 0; pub const PyUnicode_1BYTE_KIND: c_uint = 1; @@ -445,6 +443,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).kind() @@ -484,6 +483,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { #[inline] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).length @@ -502,8 +502,13 @@ pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ready() } +#[cfg(Py_3_12)] +#[inline] +pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int { + 0 +} + #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] #[inline] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { debug_assert!(crate::PyUnicode_Check(op) != 0); diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index b03fbb303a8..aa9435f6170 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -23,6 +23,7 @@ extern "C" { pub fn PyDict_New() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_GetItem")] pub fn PyDict_GetItem(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyDict_GetItemWithError")] pub fn PyDict_GetItemWithError(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_SetItem")] pub fn PyDict_SetItem(mp: *mut PyObject, key: *mut PyObject, item: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index 91d7e378799..b80f009b982 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -13,12 +13,14 @@ extern "C" { pub fn PyErr_Occurred() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyErr_Clear")] pub fn PyErr_Clear(); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_GetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Fetch")] pub fn PyErr_Fetch( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_SetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Restore")] pub fn PyErr_Restore(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyErr_GetExcInfo")] @@ -35,12 +37,22 @@ extern "C" { pub fn PyErr_GivenExceptionMatches(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyErr_ExceptionMatches")] pub fn PyErr_ExceptionMatches(arg1: *mut PyObject) -> c_int; + #[cfg_attr( + Py_3_12, + deprecated( + note = "Use PyErr_GetRaisedException() instead, to avoid any possible de-normalization." + ) + )] #[cfg_attr(PyPy, link_name = "PyPyErr_NormalizeException")] pub fn PyErr_NormalizeException( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg(Py_3_12)] + pub fn PyErr_GetRaisedException() -> *mut PyObject; + #[cfg(Py_3_12)] + pub fn PyErr_SetRaisedException(exc: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyException_SetTraceback")] pub fn PyException_SetTraceback(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyException_GetTraceback")] diff --git a/pyo3-ffi/src/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs index 86075475475..5ce6496834c 100644 --- a/pyo3-ffi/src/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -59,6 +59,8 @@ extern "C" { pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")] pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t; + #[cfg(not(Py_3_12))] + #[deprecated(note = "Removed in Python 3.12")] #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")] pub fn PyUnicode_GetSize(unicode: *mut PyObject) -> Py_ssize_t; pub fn PyUnicode_ReadChar(unicode: *mut PyObject, index: Py_ssize_t) -> Py_UCS4; diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index e08919cfe8d..4644fc6e764 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.19.0" +version = "0.19.1" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 1ba9c8abd70..61cdbb630c0 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! This crate contains the implementation of the proc macro attributes #![warn(elided_lifetimes_in_paths, unused_lifetimes)] diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 09b8b26d1a8..a867a301378 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::attributes::{TextSignatureAttribute, TextSignatureAttributeValue}; use crate::deprecations::{Deprecation, Deprecations}; use crate::params::impl_arg_params; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index cb2fda00c0f..e6db700718f 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Code generation for the function that initializes a python module and adds classes and function. use crate::{ diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 32a9be5d73a..b1e8538122c 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ method::{FnArg, FnSpec}, pyfunction::FunctionSignature, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 7e7ac6b615f..f723da0951f 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; use crate::attributes::{ diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 8f56b3e916a..53188c7c072 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ attributes::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index d0a1b6157cf..0ab0695eca5 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::collections::HashSet; use crate::{ diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 9689d863a44..38c325113f4 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; use crate::attributes::NameAttribute; @@ -206,7 +204,7 @@ pub fn gen_py_method( GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) } PyMethodProtoKind::Traverse => { - GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec.name)) + GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec)?) } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; @@ -400,7 +398,16 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>) -> Result MethodAndSlotDef { +fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { + if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { + return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ + Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ + Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ + i.e. `Python::with_gil` will panic.")); + } + + let rust_fn_ident = spec.name; + let associated_method = quote! { pub unsafe extern "C" fn __pymethod_traverse__( slf: *mut _pyo3::ffi::PyObject, @@ -416,10 +423,10 @@ fn impl_traverse_slot(cls: &syn::Type, rust_fn_ident: &syn::Ident) -> MethodAndS pfunc: #cls::__pymethod_traverse__ as _pyo3::ffi::traverseproc as _ } }; - MethodAndSlotDef { + Ok(MethodAndSlotDef { associated_method, slot_def, - } + }) } fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index c9e02d85fb7..5cffe120925 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use syn::{punctuated::Punctuated, spanned::Spanned, Token}; diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 6c531c0fecf..347b8d78ae5 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.19.0" +version = "0.19.1" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,4 +22,4 @@ abi3 = ["pyo3-macros-backend/abi3"] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "1.0.85", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.1" } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 387934310b9..37c7e6e9b99 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! This crate declares only the proc macro attributes, as a crate defining proc macro attributes //! must not contain any other public items. diff --git a/pyproject.toml b/pyproject.toml index f53001e2f93..ae99ebf75f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ exclude = ''' [tool.towncrier] filename = "CHANGELOG.md" -version = "0.19.0" +version = "0.19.1" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 9d2fb926c66..dfebfe31173 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [tool.pytest.ini_options] diff --git a/src/callback.rs b/src/callback.rs index 6d59253730e..611b1787478 100644 --- a/src/callback.rs +++ b/src/callback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Utilities for a Python callable object that invokes a Rust function. use crate::err::{PyErr, PyResult}; diff --git a/src/conversion.rs b/src/conversion.rs index c5bda16fd7c..85e9f04eb63 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Defines conversions between Rust and Python types. use crate::err::{self, PyDowncastError, PyResult}; #[cfg(feature = "experimental-inspect")] diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index a8aec454040..52ff2149813 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -1,8 +1,4 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - -#![cfg(all(feature = "num-bigint", not(any(Py_LIMITED_API))))] +#![cfg(feature = "num-bigint")] //! Conversions to and from [num-bigint](https://docs.rs/num-bigint)’s [`BigInt`] and [`BigUint`] types. //! //! This is useful for converting Python integers when they may not fit in Rust's built-in integer types. @@ -57,15 +53,16 @@ //! ``` use crate::{ - err, ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, - Python, ToPyObject, + ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, + ToPyObject, }; use num_bigint::{BigInt, BigUint}; use std::os::raw::{c_int, c_uchar}; +#[cfg(not(Py_LIMITED_API))] unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { - err::error_on_minusone( + crate::err::error_on_minusone( ob.py(), ffi::_PyLong_AsByteArray( ob.as_ptr() as *mut ffi::PyLongObject, @@ -77,13 +74,33 @@ unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyRe ) } +#[cfg(Py_LIMITED_API)] +unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { + use crate::intern; + let py = ob.py(); + let kwargs = if is_signed != 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(intern!(py, "signed"), true)?; + Some(kwargs) + } else { + None + }; + let bytes_obj = ob + .getattr(intern!(py, "to_bytes"))? + .call((buffer.len(), "little"), kwargs)?; + let bytes: &PyBytes = bytes_obj.downcast_unchecked(); + buffer.copy_from_slice(bytes.as_bytes()); + Ok(()) +} + macro_rules! bigint_conversion { ($rust_ty: ty, $is_signed: expr, $to_bytes: path, $from_bytes: path) => { #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl ToPyObject for $rust_ty { + #[cfg(not(Py_LIMITED_API))] fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); unsafe { - let bytes = $to_bytes(self); let obj = ffi::_PyLong_FromByteArray( bytes.as_ptr() as *const c_uchar, bytes.len(), @@ -93,6 +110,23 @@ macro_rules! bigint_conversion { PyObject::from_owned_ptr(py, obj) } } + + #[cfg(Py_LIMITED_API)] + fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); + let bytes_obj = PyBytes::new(py, &bytes); + let kwargs = if $is_signed > 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); + Some(kwargs) + } else { + None + }; + py.get_type::() + .call_method("from_bytes", (bytes_obj, "little"), kwargs) + .expect("int.from_bytes() failed during to_object()") // FIXME: #1813 or similar + .into() + } } #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] @@ -109,14 +143,33 @@ macro_rules! bigint_conversion { unsafe { let num: Py = Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))?; - let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); - let n_bytes = if n_bits == (-1isize as usize) { - return Err(PyErr::fetch(py)); - } else if n_bits == 0 { - 0 - } else { - (n_bits - 1 + $is_signed) / 8 + 1 + + let n_bytes = { + cfg_if::cfg_if! { + if #[cfg(not(Py_LIMITED_API))] { + // fast path + let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); + if n_bits == (-1isize as usize) { + return Err(crate::PyErr::fetch(py)); + } else if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } else { + // slow path + let n_bits_obj = num.getattr(py, crate::intern!(py, "bit_length"))?.call0(py)?; + let n_bits_int: &PyLong = n_bits_obj.downcast_unchecked(py); + let n_bits = n_bits_int.extract::()?; + if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } + } }; + if n_bytes <= 128 { let mut buffer = [0; 128]; extract(num.as_ref(py), &mut buffer[..n_bytes], $is_signed)?; diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index 217d862a542..df6b54b45bc 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -152,6 +152,18 @@ macro_rules! complex_conversion { #[cfg(any(Py_LIMITED_API, PyPy))] unsafe { + let obj = if obj.is_instance_of::() { + obj + } else if let Some(method) = + obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? + { + method.call0()? + } else { + // `obj` might still implement `__float__` or `__index__`, which will be + // handled by `PyComplex_{Real,Imag}AsDouble`, including propagating any + // errors if those methods don't exist / raise exceptions. + obj + }; let ptr = obj.as_ptr(); let real = ffi::PyComplex_RealAsDouble(ptr); if real == -1.0 { @@ -172,6 +184,7 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; + use crate::types::PyModule; #[test] fn from_complex() { @@ -197,4 +210,131 @@ mod tests { assert!(obj.extract::>(py).is_err()); }); } + #[test] + fn from_python_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + def __complex__(self): return 3.0+1.2j +class B: + def __float__(self): return 3.0 +class C: + def __index__(self): return 3 + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + // Before Python 3.8, `__index__` wasn't tried by `float`/`complex`. + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_inherited_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class First: pass +class ComplexMixin: + def __complex__(self): return 3.0+1.2j +class FloatMixin: + def __float__(self): return 3.0 +class IndexMixin: + def __index__(self): return 3 +class A(First, ComplexMixin): pass +class B(First, FloatMixin): pass +class C(First, IndexMixin): pass + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_noncallable_descriptor_magic() { + // Functions and lambdas implement the descriptor protocol in a way that makes + // `type(inst).attr(inst)` equivalent to `inst.attr()` for methods, but this isn't the only + // way the descriptor protocol might be implemented. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + @property + def __complex__(self): + return lambda: 3.0+1.2j + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } + #[test] + fn from_python_nondescriptor_magic() { + // Magic methods don't need to implement the descriptor protocol, if they're callable. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class MyComplex: + def __call__(self): return 3.0+1.2j +class A: + __complex__ = MyComplex() + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } } diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs new file mode 100755 index 00000000000..ca3c8728f9b --- /dev/null +++ b/src/conversions/std/ipaddr.rs @@ -0,0 +1,110 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use crate::exceptions::PyValueError; +use crate::sync::GILOnceCell; +use crate::types::PyType; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; + +impl FromPyObject<'_> for IpAddr { + fn extract(obj: &PyAny) -> PyResult { + match obj.getattr(intern!(obj.py(), "packed")) { + Ok(packed) => { + if let Ok(packed) = packed.extract::<[u8; 4]>() { + Ok(IpAddr::V4(Ipv4Addr::from(packed))) + } else if let Ok(packed) = packed.extract::<[u8; 16]>() { + Ok(IpAddr::V6(Ipv6Addr::from(packed))) + } else { + Err(PyValueError::new_err("invalid packed length")) + } + } + Err(_) => { + // We don't have a .packed attribute, so we try to construct an IP from str(). + obj.str()?.to_str()?.parse().map_err(PyValueError::new_err) + } + } + } +} + +impl ToPyObject for Ipv4Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV4_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV4_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv4Address") + .expect("failed to load ipaddress.IPv4Address") + .call1((u32::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv4Address") + .to_object(py) + } +} + +impl ToPyObject for Ipv6Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV6_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV6_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv6Address") + .expect("failed to load ipaddress.IPv6Address") + .call1((u128::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv6Address") + .to_object(py) + } +} + +impl ToPyObject for IpAddr { + fn to_object(&self, py: Python<'_>) -> PyObject { + match self { + IpAddr::V4(ip) => ip.to_object(py), + IpAddr::V6(ip) => ip.to_object(py), + } + } +} + +impl IntoPy for IpAddr { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +#[cfg(test)] +mod test_ipaddr { + use std::str::FromStr; + + use crate::types::PyString; + + use super::*; + + #[test] + fn test_roundtrip() { + Python::with_gil(|py| { + fn roundtrip(py: Python<'_>, ip: &str) { + let ip = IpAddr::from_str(ip).unwrap(); + let py_cls = if ip.is_ipv4() { + "IPv4Address" + } else { + "IPv6Address" + }; + + let pyobj = ip.into_py(py); + let repr = pyobj.as_ref(py).repr().unwrap().to_string_lossy(); + assert_eq!(repr, format!("{}('{}')", py_cls, ip)); + + let ip2: IpAddr = pyobj.extract(py).unwrap(); + assert_eq!(ip, ip2); + } + roundtrip(py, "127.0.0.1"); + roundtrip(py, "::1"); + roundtrip(py, "0.0.0.0"); + }); + } + + #[test] + fn test_from_pystring() { + Python::with_gil(|py| { + let py_str = PyString::new(py, "0:0:0:0:0:0:0:1"); + let ip: IpAddr = py_str.to_object(py).extract(py).unwrap(); + assert_eq!(ip, IpAddr::from_str("::1").unwrap()); + + let py_str = PyString::new(py, "invalid"); + assert!(py_str.to_object(py).extract::(py).is_err()); + }); + } +} diff --git a/src/conversions/std/mod.rs b/src/conversions/std/mod.rs index 6021c395288..f5e917d08ea 100644 --- a/src/conversions/std/mod.rs +++ b/src/conversions/std/mod.rs @@ -1,4 +1,5 @@ mod array; +mod ipaddr; mod map; mod num; mod osstr; diff --git a/src/derive_utils.rs b/src/derive_utils.rs index 25ea7f89fa7..4ccb38f901b 100644 --- a/src/derive_utils.rs +++ b/src/derive_utils.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Functionality for the code generated by the derive backend use crate::{types::PyModule, Python}; diff --git a/src/err/mod.rs b/src/err/mod.rs index 88b03986159..9c71b439352 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::{PyTraceback, PyType}; diff --git a/src/exceptions.rs b/src/exceptions.rs index d92e8b35dc3..9cbc8587fe2 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Exception and warning types defined by Python. //! //! The structs in this module represent Python's built-in exceptions and diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 97e838cf527..68ddab76305 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -2,6 +2,7 @@ use crate::ffi::*; use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; use crate::types::PyString; +#[cfg(not(Py_3_12))] use libc::wchar_t; #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons @@ -116,6 +117,7 @@ fn ascii_object_bitfield() { #[cfg(not(PyPy))] hash: 0, state: 0u32, + #[cfg(not(Py_3_12))] wstr: std::ptr::null_mut() as *mut wchar_t, }; @@ -124,9 +126,12 @@ fn ascii_object_bitfield() { assert_eq!(o.kind(), 0); assert_eq!(o.compact(), 0); assert_eq!(o.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 0); - for i in 0..4 { + let interned_count = if cfg!(Py_3_12) { 2 } else { 4 }; + + for i in 0..interned_count { o.set_interned(i); assert_eq!(o.interned(), i); } @@ -142,7 +147,9 @@ fn ascii_object_bitfield() { o.set_ascii(1); assert_eq!(o.ascii(), 1); + #[cfg(not(Py_3_12))] o.set_ready(1); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 1); } } @@ -163,6 +170,7 @@ fn ascii() { assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 1); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 1); @@ -203,6 +211,7 @@ fn ucs4() { assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 0); diff --git a/src/gil.rs b/src/gil.rs index fa96a910797..69b511c9ff3 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -1,12 +1,14 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Interaction with Python's global interpreter lock use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; use parking_lot::{const_mutex, Mutex, Once}; -use std::cell::{Cell, RefCell}; -use std::{mem, ptr::NonNull, sync::atomic}; +use std::cell::Cell; +#[cfg(debug_assertions)] +use std::cell::RefCell; +#[cfg(not(debug_assertions))] +use std::cell::UnsafeCell; +use std::{mem, ptr::NonNull}; static START: Once = Once::new(); @@ -35,7 +37,10 @@ thread_local_const_init! { static GIL_COUNT: Cell = const { Cell::new(0) }; /// Temporarily hold objects that will be released when the GILPool drops. - static OWNED_OBJECTS: RefCell>> = const { RefCell::new(Vec::new()) }; + #[cfg(debug_assertions)] + static OWNED_OBJECTS: RefCell = const { RefCell::new(Vec::new()) }; + #[cfg(not(debug_assertions))] + static OWNED_OBJECTS: UnsafeCell = const { UnsafeCell::new(Vec::new()) }; } const GIL_LOCKED_DURING_TRAVERSE: isize = -1; @@ -240,7 +245,6 @@ type PyObjVec = Vec>; /// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. struct ReferencePool { - dirty: atomic::AtomicBool, // .0 is INCREFs, .1 is DECREFs pointer_ops: Mutex<(PyObjVec, PyObjVec)>, } @@ -248,30 +252,27 @@ struct ReferencePool { impl ReferencePool { const fn new() -> Self { Self { - dirty: atomic::AtomicBool::new(false), pointer_ops: const_mutex((Vec::new(), Vec::new())), } } fn register_incref(&self, obj: NonNull) { self.pointer_ops.lock().0.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn register_decref(&self, obj: NonNull) { self.pointer_ops.lock().1.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn update_counts(&self, _py: Python<'_>) { - let prev = self.dirty.swap(false, atomic::Ordering::Acquire); - if !prev { + let mut ops = self.pointer_ops.lock(); + if ops.0.is_empty() && ops.1.is_empty() { return; } - let mut ops = self.pointer_ops.lock(); let (increfs, decrefs) = mem::take(&mut *ops); drop(ops); + // Always increase reference counts first - as otherwise objects which have a // nonzero total reference count might be incorrectly dropped by Python during // this update. @@ -379,7 +380,16 @@ impl GILPool { // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition POOL.update_counts(Python::assume_gil_acquired()); GILPool { - start: OWNED_OBJECTS.try_with(|o| o.borrow().len()).ok(), + start: OWNED_OBJECTS + .try_with(|owned_objects| { + #[cfg(debug_assertions)] + let len = owned_objects.borrow().len(); + #[cfg(not(debug_assertions))] + // SAFETY: This is not re-entrant. + let len = unsafe { (*owned_objects.get()).len() }; + len + }) + .ok(), _not_send: NOT_SEND, } } @@ -393,18 +403,21 @@ impl GILPool { impl Drop for GILPool { fn drop(&mut self) { - if let Some(obj_len_start) = self.start { - let dropping_obj = OWNED_OBJECTS.with(|holder| { - // `holder` must be dropped before calling Py_DECREF, or Py_DECREF may call - // `GILPool::drop` recursively, resulting in invalid borrowing. - let mut holder = holder.borrow_mut(); - if obj_len_start < holder.len() { - holder.split_off(obj_len_start) + if let Some(start) = self.start { + let owned_objects = OWNED_OBJECTS.with(|owned_objects| { + #[cfg(debug_assertions)] + let mut owned_objects = owned_objects.borrow_mut(); + #[cfg(not(debug_assertions))] + // SAFETY: `OWNED_OBJECTS` is released before calling Py_DECREF, + // or Py_DECREF may call `GILPool::drop` recursively, resulting in invalid borrowing. + let owned_objects = unsafe { &mut *owned_objects.get() }; + if start < owned_objects.len() { + owned_objects.split_off(start) } else { Vec::new() } }); - for obj in dropping_obj { + for obj in owned_objects { unsafe { ffi::Py_DECREF(obj.as_ptr()); } @@ -453,7 +466,15 @@ pub unsafe fn register_decref(obj: NonNull) { pub unsafe fn register_owned(_py: Python<'_>, obj: NonNull) { debug_assert!(gil_is_acquired()); // Ignores the error in case this function called from `atexit`. - let _ = OWNED_OBJECTS.try_with(|holder| holder.borrow_mut().push(obj)); + let _ = OWNED_OBJECTS.try_with(|owned_objects| { + #[cfg(debug_assertions)] + owned_objects.borrow_mut().push(obj); + #[cfg(not(debug_assertions))] + // SAFETY: This is not re-entrant. + unsafe { + (*owned_objects.get()).push(obj); + } + }); } /// Increments pyo3's internal GIL count - to be called whenever GILPool or GILGuard is created. @@ -489,7 +510,7 @@ mod tests { use crate::{ffi, gil, AsPyPointer, IntoPyPointer, PyObject, Python, ToPyObject}; #[cfg(not(target_arch = "wasm32"))] use parking_lot::{const_mutex, Condvar, Mutex}; - use std::{ptr::NonNull, sync::atomic::Ordering}; + use std::ptr::NonNull; fn get_object(py: Python<'_>) -> PyObject { // Convenience function for getting a single unique object, using `new_pool` so as to leave @@ -502,11 +523,27 @@ mod tests { } fn owned_object_count() -> usize { - OWNED_OBJECTS.with(|holder| holder.borrow().len()) + #[cfg(debug_assertions)] + let len = OWNED_OBJECTS.with(|owned_objects| owned_objects.borrow().len()); + #[cfg(not(debug_assertions))] + let len = OWNED_OBJECTS.with(|owned_objects| unsafe { (*owned_objects.get()).len() }); + len + } + + fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .0 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } - fn pool_not_dirty() -> bool { - !POOL.dirty.load(Ordering::SeqCst) + fn pool_dec_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .1 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } #[cfg(not(target_arch = "wasm32"))] @@ -584,13 +621,13 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // With the GIL held, reference cound will be decreased immediately. drop(reference); assert_eq!(obj.get_refcnt(py), 1); - assert!(pool_not_dirty()); + assert!(pool_dec_refs_does_not_contain(&obj)); }); } @@ -603,7 +640,7 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // Drop reference in a separate thread which doesn't have the GIL. std::thread::spawn(move || drop(reference)).join().unwrap(); diff --git a/src/impl_/freelist.rs b/src/impl_/freelist.rs index d5e3d1f8143..955a50a3549 100644 --- a/src/impl_/freelist.rs +++ b/src/impl_/freelist.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Support for [free allocation lists][1]. //! //! This can improve performance for types that are often created and deleted in quick succession. diff --git a/src/instance.rs b/src/instance.rs index d3927768ec9..0cbf8dbfc68 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::conversion::PyTryFrom; use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::gil; diff --git a/src/marker.rs b/src/marker.rs index c6d6fc40f96..e5b3ed81ca5 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Fundamental properties of objects tied to the Python interpreter. //! //! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded @@ -944,6 +940,72 @@ impl<'py> Python<'py> { } } +impl Python<'_> { + /// Creates a scope using a new pool for managing PyO3's owned references. + /// + /// This is a safe alterantive to [`new_pool`][Self::new_pool] as + /// it limits the closure to using the new GIL token at the cost of + /// being unable to capture existing GIL-bound references. + /// + /// Note that on stable Rust, this API suffers from the same the `SendWrapper` loophole + /// as [`allow_threads`][Self::allow_threads], c.f. the documentation of the [`Ungil`] trait, + /// + /// # Examples + /// + /// ```rust + /// # use pyo3::prelude::*; + /// Python::with_gil(|py| { + /// // Some long-running process like a webserver, which never releases the GIL. + /// loop { + /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. + /// py.with_pool(|py| { + /// // do stuff... + /// }); + /// # break; // Exit the loop so that doctest terminates! + /// } + /// }); + /// ``` + /// + /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// # use pyo3::types::PyString; + /// + /// Python::with_gil(|py| { + /// let old_str = PyString::new(py, "a message from the past"); + /// + /// py.with_pool(|_py| { + /// print!("{:?}", old_str); + /// }); + /// }); + /// ``` + /// + /// or continuing to use the old GIL token + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// + /// Python::with_gil(|old_py| { + /// old_py.with_pool(|_new_py| { + /// let _none = old_py.None(); + /// }); + /// }); + /// ``` + #[inline] + pub fn with_pool(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R + Ungil, + { + // SAFETY: The closure is `Ungil`, + // i.e. it does not capture any GIL-bound references + // and accesses only the newly created GIL token. + let pool = unsafe { GILPool::new() }; + + f(pool.python()) + } +} + impl<'unbound> Python<'unbound> { /// Unsafely creates a Python token with an unbounded lifetime. /// diff --git a/src/prelude.rs b/src/prelude.rs index a110ea5c1f5..ca0b0cf38db 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! PyO3's prelude. //! //! The purpose of this module is to alleviate imports of many commonly used items of the PyO3 crate diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 015f5791579..0f606853f74 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,7 +1,7 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; -use crate::{ffi, PyCell, PyClass, PyErr, PyResult, Python}; +use crate::{ffi, IntoPyPointer, Py, PyCell, PyClass, PyErr, PyResult, Python}; use crate::{ ffi::PyTypeObject, pycell::{ @@ -137,9 +137,14 @@ impl PyObjectInit for PyNativeTypeInitializer { /// ); /// }); /// ``` -pub struct PyClassInitializer { - init: T, - super_init: ::Initializer, +pub struct PyClassInitializer(PyClassInitializerImpl); + +enum PyClassInitializerImpl { + Existing(Py), + New { + init: T, + super_init: ::Initializer, + }, } impl PyClassInitializer { @@ -147,7 +152,7 @@ impl PyClassInitializer { /// /// It is recommended to use `add_subclass` instead of this method for most usage. pub fn new(init: T, super_init: ::Initializer) -> Self { - Self { init, super_init } + Self(PyClassInitializerImpl::New { init, super_init }) } /// Constructs a new initializer from an initializer for the base class. @@ -245,13 +250,18 @@ impl PyObjectInit for PyClassInitializer { contents: MaybeUninit>, } - let obj = self.super_init.into_new_object(py, subtype)?; + let (init, super_init) = match self.0 { + PyClassInitializerImpl::Existing(value) => return Ok(value.into_ptr()), + PyClassInitializerImpl::New { init, super_init } => (init, super_init), + }; + + let obj = super_init.into_new_object(py, subtype)?; let cell: *mut PartiallyInitializedPyCell = obj as _; std::ptr::write( (*cell).contents.as_mut_ptr(), PyCellContents { - value: ManuallyDrop::new(UnsafeCell::new(self.init)), + value: ManuallyDrop::new(UnsafeCell::new(init)), borrow_checker: ::Storage::new(), thread_checker: T::ThreadChecker::new(), dict: T::Dict::INIT, @@ -287,6 +297,13 @@ where } } +impl From> for PyClassInitializer { + #[inline] + fn from(value: Py) -> PyClassInitializer { + PyClassInitializer(PyClassInitializerImpl::Existing(value)) + } +} + // Implementation used by proc macros to allow anything convertible to PyClassInitializer to be // the return value of pyclass #[new] method (optionally wrapped in `Result`). impl IntoPyCallbackOutput> for U diff --git a/src/sync.rs b/src/sync.rs index 0f5a51631d3..3cb4206d239 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,5 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{types::PyString, Py, Python}; +use crate::{types::PyString, types::PyType, Py, PyErr, Python}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -169,6 +169,21 @@ impl GILOnceCell { } } +impl GILOnceCell> { + /// Get a reference to the contained Python type, initializing it if needed. + /// + /// This is a shorthand method for `get_or_init` which imports the type from Python on init. + pub(crate) fn get_or_try_init_type_ref<'py>( + &'py self, + py: Python<'py>, + module_name: &str, + attr_name: &str, + ) -> Result<&'py PyType, PyErr> { + self.get_or_try_init(py, || py.import(module_name)?.getattr(attr_name)?.extract()) + .map(|ty| ty.as_ref(py)) + } +} + /// Interns `text` as a Python string and stores a reference to it in static storage. /// /// A reference to the same Python string is returned on each invocation. diff --git a/src/type_object.rs b/src/type_object.rs index f36abd94788..3098b07217f 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Python type object information use crate::types::{PyAny, PyType}; diff --git a/src/types/any.rs b/src/types/any.rs index afdeb6ab573..f448e43bf05 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,7 +1,7 @@ use crate::class::basic::CompareOp; use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, ToPyObject}; use crate::err::{PyDowncastError, PyErr, PyResult}; -use crate::exceptions::PyTypeError; +use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::type_object::PyTypeInfo; #[cfg(not(PyPy))] use crate::types::PySuper; @@ -79,14 +79,37 @@ impl PyAny { /// /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. + /// + /// # Example: `intern!`ing the attribute name + /// + /// ``` + /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # + /// #[pyfunction] + /// fn has_version(sys: &PyModule) -> PyResult { + /// sys.hasattr(intern!(sys.py(), "version")) + /// } + /// # + /// # Python::with_gil(|py| { + /// # let sys = py.import("sys").unwrap(); + /// # has_version(sys).unwrap(); + /// # }); + /// ``` pub fn hasattr(&self, attr_name: N) -> PyResult where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult { + // PyObject_HasAttr suppresses all exceptions, which was the behaviour of `hasattr` in Python 2. + // Use an implementation which suppresses only AttributeError, which is consistent with `hasattr` in Python 3. + match any._getattr(attr_name) { + Ok(_) => Ok(true), + Err(err) if err.is_instance_of::(any.py()) => Ok(false), + Err(e) => Err(e), + } + } - unsafe { Ok(ffi::PyObject_HasAttr(self.as_ptr(), attr_name.as_ptr()) != 0) } + inner(self, attr_name.into_py(self.py())) } /// Retrieves an attribute value. @@ -115,12 +138,65 @@ impl PyAny { where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult<&PyAny> { + any._getattr(attr_name) + .map(|object| object.into_ref(any.py())) + } + inner(self, attr_name.into_py(self.py())) + } + + fn _getattr(&self, attr_name: Py) -> PyResult { unsafe { - let ret = ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()); - py.from_owned_ptr_or_err(ret) + Py::from_owned_ptr_or_err( + self.py(), + ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()), + ) + } + } + + /// Retrieve an attribute value, skipping the instance dictionary during the lookup but still + /// binding the object to the instance. + /// + /// This is useful when trying to resolve Python's "magic" methods like `__getitem__`, which + /// are looked up starting from the type object. This returns an `Option` as it is not + /// typically a direct error for the special lookup to fail, as magic methods are optional in + /// many situations in which they might be called. + /// + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// to intern `attr_name`. + #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. + pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult> + where + N: IntoPy>, + { + let py = self.py(); + let self_type = self.get_type(); + let attr = if let Ok(attr) = self_type.getattr(attr_name) { + attr + } else { + return Ok(None); + }; + + // Manually resolve descriptor protocol. + if cfg!(Py_3_10) + || unsafe { ffi::PyType_HasFeature(attr.get_type_ptr(), ffi::Py_TPFLAGS_HEAPTYPE) } != 0 + { + // This is the preferred faster path, but does not work on static types (generally, + // types defined in extension modules) before Python 3.10. + unsafe { + let descr_get_ptr = ffi::PyType_GetSlot(attr.get_type_ptr(), ffi::Py_tp_descr_get); + if descr_get_ptr.is_null() { + return Ok(Some(attr)); + } + let descr_get: ffi::descrgetfunc = std::mem::transmute(descr_get_ptr); + let ret = descr_get(attr.as_ptr(), self.as_ptr(), self_type.as_ptr()); + py.from_owned_ptr_or_err(ret).map(Some) + } + } else if let Ok(descr_get) = attr.get_type().getattr(crate::intern!(py, "__get__")) { + descr_get.call1((attr, self, self_type)).map(Some) + } else { + Ok(Some(attr)) } } @@ -974,9 +1050,82 @@ impl PyAny { #[cfg(test)] mod tests { use crate::{ - types::{IntoPyDict, PyBool, PyList, PyLong, PyModule}, + types::{IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, Python, ToPyObject, }; + + #[test] + fn test_lookup_special() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class CustomCallable: + def __call__(self): + return 1 + +class SimpleInt: + def __int__(self): + return 1 + +class InheritedInt(SimpleInt): pass + +class NoInt: pass + +class NoDescriptorInt: + __int__ = CustomCallable() + +class InstanceOverrideInt: + def __int__(self): + return 1 +instance_override = InstanceOverrideInt() +instance_override.__int__ = lambda self: 2 + +class ErrorInDescriptorInt: + @property + def __int__(self): + raise ValueError("uh-oh!") + +class NonHeapNonDescriptorInt: + # A static-typed callable that doesn't implement `__get__`. These are pretty hard to come by. + __int__ = int + "#, + "test.py", + "test", + ) + .unwrap(); + + let int = crate::intern!(py, "__int__"); + let eval_int = + |obj: &PyAny| obj.lookup_special(int)?.unwrap().call0()?.extract::(); + + let simple = module.getattr("SimpleInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(simple).unwrap(), 1); + let inherited = module.getattr("InheritedInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(inherited).unwrap(), 1); + let no_descriptor = module.getattr("NoDescriptorInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(no_descriptor).unwrap(), 1); + let missing = module.getattr("NoInt").unwrap().call0().unwrap(); + assert!(missing.lookup_special(int).unwrap().is_none()); + // Note the instance override should _not_ call the instance method that returns 2, + // because that's not how special lookups are meant to work. + let instance_override = module.getattr("instance_override").unwrap(); + assert_eq!(eval_int(instance_override).unwrap(), 1); + let descriptor_error = module + .getattr("ErrorInDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert!(descriptor_error.lookup_special(int).is_err()); + let nonheap_nondescriptor = module + .getattr("NonHeapNonDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert_eq!(eval_int(nonheap_nondescriptor).unwrap(), 0); + }) + } + #[test] fn test_call_for_non_existing_method() { Python::with_gil(|py| { @@ -1051,6 +1200,44 @@ class SimpleClass: }); } + #[test] + fn test_hasattr() { + Python::with_gil(|py| { + let x = 5.to_object(py).into_ref(py); + assert!(x.is_instance_of::()); + + assert!(x.hasattr("to_bytes").unwrap()); + assert!(!x.hasattr("bbbbbbytes").unwrap()); + }) + } + + #[cfg(feature = "macros")] + #[test] + fn test_hasattr_error() { + use crate::exceptions::PyValueError; + use crate::prelude::*; + + #[pyclass(crate = "crate")] + struct GetattrFail; + + #[pymethods(crate = "crate")] + impl GetattrFail { + fn __getattr__(&self, attr: PyObject) -> PyResult { + Err(PyValueError::new_err(attr)) + } + } + + Python::with_gil(|py| { + let obj = Py::new(py, GetattrFail).unwrap(); + let obj = obj.as_ref(py).as_ref(); + + assert!(obj + .hasattr("foo") + .unwrap_err() + .is_instance_of::(py)); + }) + } + #[test] fn test_nan_eq() { Python::with_gil(|py| { diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 80c453a6522..919810d2ddc 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 927c450085c..c11af6d71f8 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{PyErr, PyResult}; use crate::{ffi, AsPyPointer, Py, PyAny, Python}; use std::os::raw::c_char; diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 7da59e735e3..152ac0ffc61 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::Python; use crate::{ffi, AsPyPointer, PyAny}; use crate::{pyobject_native_type_core, PyErr, PyResult}; diff --git a/src/types/code.rs b/src/types/code.rs index fc7e3e9f83e..c0d0ce83926 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 2c995356e2c..8d5aae1f977 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -257,7 +257,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.DateTimeType, ); py.from_owned_ptr_or_err(ptr) @@ -294,7 +294,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), c_int::from(fold), api.DateTimeType, ); @@ -399,7 +399,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.TimeType, ); py.from_owned_ptr_or_err(ptr) @@ -423,7 +423,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), fold as c_int, api.TimeType, ); @@ -535,12 +535,12 @@ impl PyDeltaAccess for PyDelta { } } -// Utility function -fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { - // Convenience function for unpacking Options to either an Object or None +// Utility function which returns a borrowed reference to either +// the underlying tzinfo or None. +fn opt_to_pyobj(opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { match opt { Some(tzi) => tzi.as_ptr(), - None => py.None().as_ptr(), + None => unsafe { ffi::Py_None() }, } } diff --git a/src/types/dict.rs b/src/types/dict.rs index 0cf7e6ca7b6..b4ed14c3f89 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -1,13 +1,10 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use super::PyMapping; use crate::err::{self, PyErr, PyResult}; use crate::ffi::Py_ssize_t; use crate::types::{PyAny, PyList}; -use crate::{ffi, AsPyPointer, Python, ToPyObject}; #[cfg(not(PyPy))] -use crate::{IntoPyPointer, PyObject}; -use std::ptr::NonNull; +use crate::IntoPyPointer; +use crate::{ffi, AsPyPointer, PyObject, Python, ToPyObject}; /// Represents a Python `dict`. #[repr(transparent)] @@ -145,12 +142,16 @@ impl PyDict { where K: ToPyObject, { + self.get_item_impl(key.to_object(self.py())) + } + + fn get_item_impl(&self, key: PyObject) -> Option<&PyAny> { + let py = self.py(); unsafe { - let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.to_object(self.py()).as_ptr()); - NonNull::new(ptr).map(|p| { - // PyDict_GetItem return s borrowed ptr, must make it owned for safety (see #890). - self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())) - }) + let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.as_ptr()); + // PyDict_GetItem returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + PyObject::from_borrowed_ptr_or_opt(py, ptr).map(|pyobject| pyobject.into_ref(py)) } } @@ -159,19 +160,23 @@ impl PyDict { /// returns `Ok(None)` if item is not present, or `Err(PyErr)` if an error occurs. /// /// To get a `KeyError` for non-existing keys, use `PyAny::get_item_with_error`. - #[cfg(not(PyPy))] pub fn get_item_with_error(&self, key: K) -> PyResult> where K: ToPyObject, { - unsafe { - let ptr = - ffi::PyDict_GetItemWithError(self.as_ptr(), key.to_object(self.py()).as_ptr()); - if !ffi::PyErr_Occurred().is_null() { - return Err(PyErr::fetch(self.py())); - } + self.get_item_with_error_impl(key.to_object(self.py())) + } - Ok(NonNull::new(ptr).map(|p| self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())))) + fn get_item_with_error_impl(&self, key: PyObject) -> PyResult> { + let py = self.py(); + unsafe { + let ptr = ffi::PyDict_GetItemWithError(self.as_ptr(), key.as_ptr()); + // PyDict_GetItemWithError returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + PyObject::from_borrowed_ptr_or_opt(py, ptr) + .map(|pyobject| Ok(pyobject.into_ref(py))) + .or_else(|| PyErr::take(py).map(Err)) + .transpose() } } diff --git a/src/types/floatob.rs b/src/types/floatob.rs index ccf73dd8cde..82db228dfa1 100644 --- a/src/types/floatob.rs +++ b/src/types/floatob.rs @@ -1,6 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ diff --git a/src/types/frame.rs b/src/types/frame.rs index c16e143987d..160f3b3b6ed 100644 --- a/src/types/frame.rs +++ b/src/types/frame.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 5b728784a30..ff91be9251e 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,16 +1,45 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, PyObject, + Py, PyObject, }; use crate::{ffi, AsPyPointer, PyAny, Python, ToPyObject}; use std::ptr; +/// Allows building a Python `frozenset` one item at a time +pub struct PyFrozenSetBuilder<'py> { + py_frozen_set: &'py PyFrozenSet, +} + +impl<'py> PyFrozenSetBuilder<'py> { + /// Create a new `FrozenSetBuilder`. + /// Since this allocates a `PyFrozenSet` internally it may + /// panic when running out of memory. + pub fn new(py: Python<'py>) -> PyResult> { + Ok(PyFrozenSetBuilder { + py_frozen_set: PyFrozenSet::empty(py)?, + }) + } + + /// Adds an element to the set. + pub fn add(&mut self, key: K) -> PyResult<()> + where + K: ToPyObject, + { + let py = self.py_frozen_set.py(); + err::error_on_minusone(py, unsafe { + ffi::PySet_Add(self.py_frozen_set.as_ptr(), key.to_object(py).as_ptr()) + }) + } + + /// Finish building the set and take ownership of its current value + pub fn finalize(self) -> &'py PyFrozenSet { + self.py_frozen_set + } +} + /// Represents a Python `frozenset` #[repr(transparent)] pub struct PyFrozenSet(PyAny); @@ -180,7 +209,7 @@ pub(crate) fn new_from_iter( for obj in elements { unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; + err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; } } @@ -238,4 +267,25 @@ mod tests { } }); } + + #[test] + fn test_frozenset_builder() { + use super::PyFrozenSetBuilder; + + Python::with_gil(|py| { + let mut builder = PyFrozenSetBuilder::new(py).unwrap(); + + // add an item + builder.add(1).unwrap(); + builder.add(2).unwrap(); + builder.add(2).unwrap(); + + // finalize it + let set = builder.finalize(); + + assert!(set.contains(1).unwrap()); + assert!(set.contains(2).unwrap()); + assert!(!set.contains(3).unwrap()); + }); + } } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index cfbbd31b6dc..3b45c11378a 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, AsPyPointer, IntoPyPointer, Py, PyAny, PyErr, PyNativeType, PyResult, Python}; use crate::{PyDowncastError, PyTryFrom}; diff --git a/src/types/list.rs b/src/types/list.rs index 3e801276b95..5e951c29c8e 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use std::convert::TryInto; use crate::err::{self, PyResult}; diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 602302a953b..fa73176d142 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{PyDowncastError, PyErr, PyResult}; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; diff --git a/src/types/mod.rs b/src/types/mod.rs index 385d0af68dc..06a677ab62d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Various types defined by the Python interpreter such as `int`, `str` and `tuple`. pub use self::any::PyAny; @@ -21,7 +19,7 @@ pub use self::dict::{PyDictItems, PyDictKeys, PyDictValues}; pub use self::floatob::PyFloat; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::frame::PyFrame; -pub use self::frozenset::PyFrozenSet; +pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder}; pub use self::function::PyCFunction; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::function::PyFunction; diff --git a/src/types/module.rs b/src/types/module.rs index 970a22ff0ce..b6b25eacb95 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::callback::IntoPyCallbackOutput; use crate::err::{PyErr, PyResult}; use crate::exceptions; @@ -102,7 +98,7 @@ impl PyModule { /// let code = include_str!("../../assets/script.py"); /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, code, "example", "example")?; + /// PyModule::from_code(py, code, "example.py", "example")?; /// Ok(()) /// })?; /// # Ok(()) @@ -121,7 +117,7 @@ impl PyModule { /// let code = std::fs::read_to_string("assets/script.py")?; /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, &code, "example", "example")?; + /// PyModule::from_code(py, &code, "example.py", "example")?; /// Ok(()) /// })?; /// Ok(()) diff --git a/src/types/num.rs b/src/types/num.rs index 21f6d72d3cd..522517155f8 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, PyAny}; /// Represents a Python `int` object. diff --git a/src/types/sequence.rs b/src/types/sequence.rs index ccc2d895b72..73e110cd40e 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::exceptions::PyTypeError; #[cfg(feature = "experimental-inspect")] diff --git a/src/types/set.rs b/src/types/set.rs index 36774b9a014..af174e1b7c5 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -1,11 +1,8 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, + Py, }; use crate::{ffi, AsPyPointer, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -253,7 +250,7 @@ pub(crate) fn new_from_iter( for obj in elements { unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; + err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; } } diff --git a/src/types/slice.rs b/src/types/slice.rs index 61574f6490e..e82b535a45b 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{PyErr, PyResult}; use crate::ffi::{self, Py_ssize_t}; use crate::{AsPyPointer, PyAny, PyObject, Python, ToPyObject}; diff --git a/src/types/string.rs b/src/types/string.rs index 3998a6adf04..05b7109f280 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - #[cfg(not(Py_LIMITED_API))] use crate::exceptions::PyUnicodeDecodeError; use crate::types::PyBytes; diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 7e4393eb98d..82916558a55 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{error_on_minusone, PyResult}; use crate::ffi; use crate::types::PyString; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 3a124d3690e..429f2d0f848 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::convert::TryInto; use crate::ffi::{self, Py_ssize_t}; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 00ba62c3be1..ca4a5cdbef6 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::err::{self, PyResult}; use crate::{ffi, AsPyPointer, PyAny, PyTypeInfo, Python}; diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index 043971455b6..e07aa457de7 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -96,7 +96,7 @@ fn recursive_class_attributes() { } #[test] -#[cfg(panic = "unwind")] +#[cfg_attr(cfg_panic, cfg(panic = "unwind"))] fn test_fallible_class_attribute() { use pyo3::{exceptions::PyValueError, types::PyString}; diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index b9b0d152086..ff159c610f8 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::sync::GILOnceCell; use pyo3::types::IntoPyDict; #[pyclass] @@ -204,3 +205,62 @@ fn new_with_custom_error() { assert_eq!(err.to_string(), "ValueError: custom error"); }); } + +#[pyclass] +struct NewExisting { + #[pyo3(get)] + num: usize, +} + +#[pymethods] +impl NewExisting { + #[new] + fn new(py: pyo3::Python<'_>, val: usize) -> pyo3::Py { + static PRE_BUILT: GILOnceCell<[pyo3::Py; 2]> = GILOnceCell::new(); + let existing = PRE_BUILT.get_or_init(py, || { + [ + pyo3::PyCell::new(py, NewExisting { num: 0 }) + .unwrap() + .into(), + pyo3::PyCell::new(py, NewExisting { num: 1 }) + .unwrap() + .into(), + ] + }); + + if val < existing.len() { + return existing[val].clone_ref(py); + } + + pyo3::PyCell::new(py, NewExisting { num: val }) + .unwrap() + .into() + } +} + +#[test] +fn test_new_existing() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + + let obj1 = typeobj.call1((0,)).unwrap(); + let obj2 = typeobj.call1((0,)).unwrap(); + let obj3 = typeobj.call1((1,)).unwrap(); + let obj4 = typeobj.call1((1,)).unwrap(); + let obj5 = typeobj.call1((2,)).unwrap(); + let obj6 = typeobj.call1((2,)).unwrap(); + + assert!(obj1.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj2.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj3.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj4.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj5.getattr("num").unwrap().extract::().unwrap() == 2); + assert!(obj6.getattr("num").unwrap().extract::().unwrap() == 2); + + assert!(obj1.is(obj2)); + assert!(obj3.is(obj4)); + assert!(!obj1.is(obj3)); + assert!(!obj1.is(obj5)); + assert!(!obj5.is(obj6)); + }); +} diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 4625ea16431..5d933e13bc6 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -37,5 +37,5 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); - t.compile_fail("tests/ui/traverse_bare_self.rs"); + t.compile_fail("tests/ui/traverse.rs"); } diff --git a/tests/ui/traverse.rs b/tests/ui/traverse.rs new file mode 100644 index 00000000000..034224951c9 --- /dev/null +++ b/tests/ui/traverse.rs @@ -0,0 +1,27 @@ +use pyo3::prelude::*; +use pyo3::PyVisit; +use pyo3::PyTraverseError; + +#[pyclass] +struct TraverseTriesToTakePyRef {} + +#[pymethods] +impl TraverseTriesToTakePyRef { + fn __traverse__(slf: PyRef, visit: PyVisit) {} +} + +#[pyclass] +struct Class; + +#[pymethods] +impl Class { + fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + Ok(()) + } + + fn __clear__(&mut self) { + } +} + + +fn main() {} diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr new file mode 100644 index 00000000000..873ef864a23 --- /dev/null +++ b/tests/ui/traverse.stderr @@ -0,0 +1,23 @@ +error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:18:32 + | +18 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + | ^^^^^^^^^^ + +error[E0308]: mismatched types + --> tests/ui/traverse.rs:9:6 + | +8 | #[pymethods] + | ------------ arguments to this function are incorrect +9 | impl TraverseTriesToTakePyRef { + | ______^ +10 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} + | |___________________^ expected fn pointer, found fn item + | + = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` + found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` +note: function defined here + --> src/impl_/pymethods.rs + | + | pub unsafe fn call_traverse_impl( + | ^^^^^^^^^^^^^^^^^^ diff --git a/tests/ui/traverse_bare_self.rs b/tests/ui/traverse_bare_self.rs deleted file mode 100644 index 5adc316e43f..00000000000 --- a/tests/ui/traverse_bare_self.rs +++ /dev/null @@ -1,12 +0,0 @@ -use pyo3::prelude::*; -use pyo3::PyVisit; - -#[pyclass] -struct TraverseTriesToTakePyRef {} - -#[pymethods] -impl TraverseTriesToTakePyRef { - fn __traverse__(slf: PyRef, visit: PyVisit) {} -} - -fn main() {} diff --git a/tests/ui/traverse_bare_self.stderr b/tests/ui/traverse_bare_self.stderr deleted file mode 100644 index aba76145dc3..00000000000 --- a/tests/ui/traverse_bare_self.stderr +++ /dev/null @@ -1,17 +0,0 @@ -error[E0308]: mismatched types - --> tests/ui/traverse_bare_self.rs:8:6 - | -7 | #[pymethods] - | ------------ arguments to this function are incorrect -8 | impl TraverseTriesToTakePyRef { - | ______^ -9 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} - | |___________________^ expected fn pointer, found fn item - | - = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` - found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` -note: function defined here - --> src/impl_/pymethods.rs - | - | pub unsafe fn call_traverse_impl( - | ^^^^^^^^^^^^^^^^^^ diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml deleted file mode 100644 index 48a5a63fd3f..00000000000 --- a/xtask/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "xtask" -version = "0.1.0" -edition = "2018" -publish = false - -[[bin]] -name = "xtask" - -[dependencies] -anyhow = "1.0.51" - -# Clap 3 requires MSRV 1.54 -rustversion = "1.0" -structopt = { version = "0.3", default-features = false } -clap = { version = "2" } diff --git a/xtask/README.md b/xtask/README.md deleted file mode 100644 index 68d078e5595..00000000000 --- a/xtask/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Commands to test PyO3. - -To run these commands, you should be in PyO3's root directory, and run (for example) `cargo xtask ci`. - -``` -USAGE: - xtask.exe - -FLAGS: - -h, --help Prints help information - -V, --version Prints version information - -SUBCOMMANDS: - ci Runs everything - clippy Runs `clippy`, denying all warnings - coverage Runs `cargo llvm-cov` for the PyO3 codebase - default Only runs the fast things (this is used if no command is specified) - doc Attempts to render the documentation - fmt Checks Rust and Python code formatting with `rustfmt` and `black` - help Prints this message or the help of the given subcommand(s) - test Runs various variations on `cargo test` - test-py Runs the tests in examples/ and pytests/ -``` \ No newline at end of file diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs deleted file mode 100644 index f873816a017..00000000000 --- a/xtask/src/cli.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::utils::*; -use anyhow::{ensure, Result}; -use std::io; -use std::process::{Command, Output}; -use std::time::Instant; -use structopt::StructOpt; - -pub const MSRV: &str = "1.48"; - -#[derive(StructOpt)] -pub enum Subcommand { - /// Only runs the fast things (this is used if no command is specified) - Default, - /// Runs everything - Ci, - /// Checks Rust and Python code formatting with `rustfmt` and `black` - Fmt, - /// Runs `clippy`, denying all warnings. - Clippy, - /// Attempts to render the documentation. - Doc(DocOpts), - /// Runs various variations on `cargo test` - Test, - /// Runs the tests in examples/ and pytests/ - TestPy, -} - -impl Default for Subcommand { - fn default() -> Self { - Self::Default - } -} - -#[derive(StructOpt)] -pub struct DocOpts { - /// Whether to run the docs using nightly rustdoc - #[structopt(long)] - pub stable: bool, - /// Whether to open the docs after rendering. - #[structopt(long)] - pub open: bool, - /// Whether to show the private and hidden API. - #[structopt(long)] - pub internal: bool, -} - -impl Default for DocOpts { - fn default() -> Self { - Self { - stable: true, - open: false, - internal: false, - } - } -} - -impl Subcommand { - pub fn execute(self) -> Result<()> { - print_metadata()?; - - let start = Instant::now(); - - match self { - Subcommand::Default => { - crate::fmt::rust::run()?; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - } - Subcommand::Ci => { - let installed = Installed::new()?; - crate::fmt::rust::run()?; - if installed.black { - crate::fmt::python::run()?; - } else { - Installed::warn_black() - }; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - if installed.nox { - crate::pytests::run(None)?; - } else { - Installed::warn_nox() - }; - installed.assert()? - } - - Subcommand::Doc(opts) => crate::doc::run(opts)?, - Subcommand::Fmt => { - crate::fmt::rust::run()?; - crate::fmt::python::run()?; - } - Subcommand::Clippy => crate::clippy::run()?, - Subcommand::TestPy => crate::pytests::run(None)?, - Subcommand::Test => crate::test::run()?, - }; - - let dt = start.elapsed().as_secs(); - let minutes = dt / 60; - let seconds = dt % 60; - println!("\nxtask finished in {}m {}s.", minutes, seconds); - - Ok(()) - } -} - -/// Run a command as a child process, inheriting stdin, stdout and stderr. -pub fn run(command: &mut Command) -> Result<()> { - let command_str = format_command(command); - let github_actions = std::env::var_os("GITHUB_ACTIONS").is_some(); - if github_actions { - println!("::group::Running: {}", command_str); - } else { - println!("Running: {}", command_str); - } - - let status = command.spawn()?.wait()?; - - ensure! { - status.success(), - "process did not run successfully ({exit}): {command}", - exit = match status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - }; - - if github_actions { - println!("::endgroup::") - } - Ok(()) -} - -/// Like `run`, but does not inherit stdin, stdout and stderr. -pub fn run_with_output(command: &mut Command) -> Result { - let command_str = format_command(command); - - println!("Running: {}", command_str); - - let output = command.output()?; - - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}:\n{stderr}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - stderr = String::from_utf8_lossy(&output.stderr) - }; - - Ok(output) -} - -#[derive(Copy, Clone, Debug)] -pub struct Installed { - pub nox: bool, - pub black: bool, -} - -impl Installed { - pub fn new() -> anyhow::Result { - Ok(Self { - nox: Self::nox()?, - black: Self::black()?, - }) - } - - pub fn nox() -> anyhow::Result { - let output = std::process::Command::new("nox").arg("--version").output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_nox() { - eprintln!("Skipping: formatting Python code, because `nox` was not found"); - } - - pub fn black() -> anyhow::Result { - let output = std::process::Command::new("black") - .arg("--version") - .output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_black() { - eprintln!("Skipping: Python code formatting, because `black` was not found."); - } - - pub fn assert(&self) -> anyhow::Result<()> { - if self.nox && self.black { - Ok(()) - } else { - let mut err = - String::from("\n\nxtask was unable to run all tests due to some missing programs:"); - if !self.black { - err.push_str("\n`black` was not installed. (`pip install black`)"); - } - if !self.nox { - err.push_str("\n`nox` was not installed. (`pip install nox`)"); - } - - Err(anyhow::anyhow!(err)) - } - } -} diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs deleted file mode 100644 index eec5f5fdd0b..00000000000 --- a/xtask/src/clippy.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::cli; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--features=full") - .arg("--all-targets") - .arg("--workspace") - .arg("--") - .arg("-Dwarnings"), - )?; - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--all-targets") - .arg("--workspace") - .arg("--features=abi3,full") - .arg("--") - .arg("-Dwarnings"), - )?; - - Ok(()) -} diff --git a/xtask/src/doc.rs b/xtask/src/doc.rs deleted file mode 100644 index cde2c882914..00000000000 --- a/xtask/src/doc.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::cli; -use crate::cli::DocOpts; -use std::process::Command; -//--cfg docsrs --Z unstable-options --document-hidden-items - -pub fn run(opts: DocOpts) -> anyhow::Result<()> { - let mut flags = Vec::new(); - - if !opts.stable { - flags.push("--cfg docsrs"); - } - if opts.internal { - flags.push("--Z unstable-options"); - flags.push("--document-hidden-items"); - } - flags.push("-Dwarnings"); - - std::env::set_var("RUSTDOCFLAGS", flags.join(" ")); - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .args(if opts.stable { None } else { Some("+nightly") }) - .arg("doc") - .arg("--lib") - .arg("--no-default-features") - .arg("--features=full") - .arg("--no-deps") - .arg("--workspace") - .args(if opts.internal { - &["--document-private-items"][..] - } else { - &["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"][..] - }) - .args(if opts.stable { - &[][..] - } else { - &["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"][..] - }) - .args(if opts.open { Some("--open") } else { None }), - )?; - - Ok(()) -} diff --git a/xtask/src/fmt.rs b/xtask/src/fmt.rs deleted file mode 100644 index 8bc745246bc..00000000000 --- a/xtask/src/fmt.rs +++ /dev/null @@ -1,23 +0,0 @@ -pub mod rust { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("fmt") - .arg("--all") - .arg("--") - .arg("--check"), - )?; - Ok(()) - } -} - -pub mod python { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run(Command::new("black").arg(".").arg("--check"))?; - Ok(()) - } -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs deleted file mode 100644 index 421742685ca..00000000000 --- a/xtask/src/main.rs +++ /dev/null @@ -1,23 +0,0 @@ -use clap::ErrorKind::MissingArgumentOrSubcommand; -use structopt::StructOpt; - -pub mod cli; -pub mod clippy; -pub mod doc; -pub mod fmt; -pub mod pytests; -pub mod test; -pub mod utils; - -fn main() -> anyhow::Result<()> { - // Avoid spewing backtraces all over the command line - // For some reason this is automatically enabled on nightly compilers... - std::env::set_var("RUST_LIB_BACKTRACE", "0"); - - match cli::Subcommand::from_args_safe() { - Ok(c) => c.execute()?, - Err(e) if e.kind == MissingArgumentOrSubcommand => cli::Subcommand::default().execute()?, - Err(e) => return Err(e.into()), - } - Ok(()) -} diff --git a/xtask/src/pytests.rs b/xtask/src/pytests.rs deleted file mode 100644 index 78744c69bd1..00000000000 --- a/xtask/src/pytests.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::cli; -use anyhow::Result; -use std::{path::Path, process::Command}; - -pub fn run<'a>(env: impl IntoIterator + Copy) -> Result<()> { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(Path::new("pytests").join("noxfile.py")) - .envs(env), - )?; - - for entry in std::fs::read_dir("examples")? { - let path = entry?.path(); - if path.is_dir() && path.join("noxfile.py").exists() { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(path.join("noxfile.py")) - .envs(env), - )?; - } - } - Ok(()) -} diff --git a/xtask/src/test.rs b/xtask/src/test.rs deleted file mode 100644 index b85f3e4a504..00000000000 --- a/xtask/src/test.rs +++ /dev/null @@ -1,81 +0,0 @@ -use crate::cli::{self, MSRV}; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("test") - .arg("--lib") - .arg("--no-default-features") - .arg("--tests") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=full") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=abi3,full") - .arg("--quiet"), - )?; - - // If the MSRV toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg(MSRV), - )?; - - // Test MSRV - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg(format!("+{}", MSRV)) - .arg("test") - .arg("--no-default-features") - .arg("--features=full,auto-initialize") - .arg("--quiet"), - )?; - - // If the nightly toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg("nightly"), - )?; - - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg("+nightly") - .arg("test") - .arg("--no-default-features") - .arg("--features=full,nightly") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--manifest-path=pyo3-ffi/Cargo.toml") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--manifest-path=pyo3-build-config/Cargo.toml") - .arg("--quiet"), - )?; - - Ok(()) -} diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs deleted file mode 100644 index 045697e7357..00000000000 --- a/xtask/src/utils.rs +++ /dev/null @@ -1,65 +0,0 @@ -use anyhow::ensure; -use std::process::Command; - -// Replacement for str.split_once() on Rust older than 1.52 -#[rustversion::before(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - let mut iter = s.splitn(2, pat); - Some((iter.next()?, iter.next()?)) -} - -#[rustversion::since(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - s.split_once(pat) -} - -#[rustversion::since(1.57)] -pub fn format_command(command: &Command) -> String { - let mut buf = String::new(); - buf.push('`'); - buf.push_str(&command.get_program().to_string_lossy()); - for arg in command.get_args() { - buf.push(' '); - buf.push_str(&arg.to_string_lossy()); - } - buf.push('`'); - buf -} - -#[rustversion::before(1.57)] -pub fn format_command(command: &Command) -> String { - // Debug impl isn't as nice as the above, but will do on < 1.57 - format!("{:?}", command) -} - -pub fn get_output(command: &mut Command) -> anyhow::Result { - let output = command.output()?; - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = format_command(command), - }; - Ok(output) -} - -pub fn print_metadata() -> anyhow::Result<()> { - let rustc_output = std::process::Command::new("rustc") - .arg("--version") - .arg("--verbose") - .output()?; - let rustc_version = core::str::from_utf8(&rustc_output.stdout).unwrap(); - println!("Metadata: \n\n{}", rustc_version); - - let py_output = std::process::Command::new("python") - .arg("--version") - .arg("-V") - .output()?; - let py_version = core::str::from_utf8(&py_output.stdout).unwrap(); - println!("{}", py_version); - - Ok(()) -}