Merge branch 'master' into iterator-example

.github/workflows/test.yml

@@ -3,41 +3,91 @@ name: Test
 on: [push, pull_request]
 
 jobs:
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+      - run: pip install black==19.10b0
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Check python formatting (black)
+        run: black --check .
+      - name: Check rust formatting (rustfmt)
+        run: cargo fmt --all -- --check
+
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - run: make clippy
+
   build:
+    needs: [fmt] # don't wait for clippy as fails rarely and takes longer
     name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }}
     runs-on: ${{ matrix.platform.os }}
     strategy:
       max-parallel: 12
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: [3.5, 3.6, 3.7, 3.8, pypy3]
         platform: [
           { os: "macOS-latest",   python-architecture: "x64", rust-target: "x86_64-apple-darwin" },
           { os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" },
           { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" },
         ]
+        exclude:
+          # There is no 64-bit pypy on windows
+          - python-version: pypy3
+            platform: { os: "windows-latest", python-architecture: "x64" }
 
     steps:
       - uses: actions/checkout@v2
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install Rust
+          architecture: ${{ matrix.platform.python-architecture }}
+
+      - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
-          default: true
+          target: ${{ matrix.platform.rust-target }}
+
       - run: rustup set default-host ${{ matrix.platform.rust-target }}
+
       - name: Build without default features
         run: cargo build --no-default-features --verbose
+
       - name: Build with default features
-        run: cargo build --verbose
-      - name: Install test dependencies
+        run: cargo build --features "num-bigint num-complex" --verbose
+
+      # Run tests (except on PyPy, because no embedding API).
+      - if: matrix.python-version != 'pypy3'
+        name: Test
+        run: cargo test --features "num-bigint num-complex"
+
+      - name: Test proc-macro code
+        run: cargo test --manifest-path=pyo3-derive-backend/Cargo.toml
+
+      - name: Install python test dependencies
         run: |
           python -m pip install -U pip setuptools
           pip install setuptools-rust pytest pytest-benchmark tox tox-venv
-      - name: Test
-        run: ci/actions/test.sh
+
+      - name: Test example extension modules
         shell: bash
+        run: |
+          for example_dir in examples/*; do
+              cd $example_dir
+              tox -c "tox.ini" -e py
+              cd -
+          done
+
     env:
       RUST_BACKTRACE: 1

.travis.yml

@@ -16,8 +16,6 @@ jobs:
       python: "3.7"
     - name: Python 3.8
       python: "3.8"
-      # Run clippy and rustfmt
-      env: RUN_LINT=1
     - name: Python 3.9-dev
       python: "3.9-dev"
     - name: Nightly

CHANGELOG.md

@@ -21,21 +21,25 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Rename `PYTHON_SYS_EXECUTABLE` to `PYO3_PYTHON`. The old name will continue to work but will be undocumented, and will be removed in a future release. [#1039](https://github.com/PyO3/pyo3/pull/1039)
 - `PyType::as_type_ptr` is no longer `unsafe`. [#1047](https://github.com/PyO3/pyo3/pull/1047)
 - Change `PyIterator::from_object` to return `PyResult<PyIterator>` instead of `Result<PyIterator, PyDowncastError>`. [#1051](https://github.com/PyO3/pyo3/pull/1051)
+- `IntoPy` is no longer implied by `FromPy`. [#1063](https://github.com/PyO3/pyo3/pull/1063)
+- `PyObject` is now just a type alias for `Py<PyAny>`. [#1063](https://github.com/PyO3/pyo3/pull/1063)
 - Implement `Send + Sync` for `PyErr`. `PyErr::new`, `PyErr::from_type`, `PyException::py_err` and `PyException::into` have had these bounds added to their arguments. [#1067](https://github.com/PyO3/pyo3/pull/1067)
 - Change `#[pyproto]` to return NotImplemented for operators for which Python can try a reversed operation. [1072](https://github.com/PyO3/pyo3/pull/1072)
 
 ### Removed
 - Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
 - Remove `Python::register_any`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
 - Remove `GILGuard::acquire` from the public API. Use `Python::acquire_gil` or `Python::with_gil`. [#1036](https://github.com/PyO3/pyo3/pull/1036)
+- Remove `FromPy`. [#1063](https://github.com/PyO3/pyo3/pull/1063)
 
 ### Fixed
 - Conversion from types with an `__index__` method to Rust BigInts. [#1027](https://github.com/PyO3/pyo3/pull/1027)
 - Fix segfault with #[pyclass(dict, unsendable)] [#1058](https://github.com/PyO3/pyo3/pull/1058)
 - Don't rely on the order of structmembers to compute offsets in PyCell. Related to
   [#1058](https://github.com/PyO3/pyo3/pull/1058). [#1059](https://github.com/PyO3/pyo3/pull/1059)
 - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071)
-- Improve lifetime insertion in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093)
+- Fix best-effort build against PyPy 3.6. #[1092](https://github.com/PyO3/pyo3/pull/1092)
+- Improve lifetime elision in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093)
 
 ## [0.11.1] - 2020-06-30
 ### Added

Contributing.md

@@ -1,14 +1,62 @@
 # Contributing
 
-Thank you for contributing to pyo3!
+Thank you for your interest in contributing to PyO3! All are welcome - please consider reading our [Code of Conduct](Code-of-Conduct.md) to keep our community positive and inclusive.
 
-Here are some things you should check for submitting your pull request:
+If you are searching for ideas how to contribute, please read the "Getting started contributing" section. Once you've found an issue to contribute to, you may find the section "Writing pull requests" helpful.
 
- - Run `cargo fmt` (This is checked by travis ci)
- - Run `cargo clippy` and check there are no hard errors (There are a bunch of existing warnings; This is also checked by travis)
- - If applicable, add an entry in the changelog
- - If applicable, add documentation to all new items and extend the guide
- - If applicable, add tests for all new or fixed functions
- - Run `cargo test`
+## Getting started contributing
 
-You might want to run `tox` (`pip install tox`) locally to check compatibility with all supported python versions. If you're using linux or mac you might find the Makefile helpful for testing.
+Please join in with any part of PyO3 which interests you. We use Github issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it.
+
+The following sections also contain specific ideas on where to start contributing to PyO3.
+
+### Help users identify bugs
+
+The [PyO3 Gitter channel](https://gitter.im/PyO3/Lobby) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase.
+
+Helping others often reveals bugs, documentation weaknesses, and missing APIs. It's a good idea to open Github issues for these immediately so the resolution can be designed and implemented!
+
+### Implement issues ready for development
+
+Issues where the solution is clear and work is not in progress use the [needs-implementer](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-implemeter) label.
+
+Don't be afraid if the solution is not clear to you! The core PyO3 contributors will be happy to mentor you through any questions you have to help you write the solution.
+
+### Help write great docs
+
+PyO3 has a user guide (using mdbook) as well as the usual Rust API docs. The aim is for both of these to be detailed, easy to understand, and up-to-date. Pull requests are always welcome to fix typos, change wording, add examples, etc.
+
+There are some specific areas of focus where help is currently needed for the documentation:
+- Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label.
+- Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR!
+- Not all `unsafe` APIs had safety notes when they made. We'd like to ensure all `unsafe` APIs are carefully explained ([#698](https://github.com/PyO3/pyo3/issues/698)). If you see an `unsafe` function missing safety notes, please write some and open a PR!
+
+### Help design the next PyO3
+
+Issues which don't yet have a clear solution use the [needs-design](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-design) label.
+
+If any of these issues interest you, please join in with the conversation on the issue! All opinions are valued, and if you're interested in going further with e.g. draft PRs to experiment with API designs, even better!
+
+### Review pull requests
+
+Everybody is welcome to submit comments on open PRs. Please help ensure new PyO3 APIs are safe, performant, tidy, and easy to use!
+
+## Writing pull requests
+
+Here are a few things to note when you are writing PRs.
+
+### Continuous Integration
+
+The PyO3 repo uses a mixture of Github actions and Travis CI. PRs are blocked from merging if CI is not successful.
+
+Formatting, linting and tests are checked for all Rust and Python code. Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.8 with the minimum supported Rust version.
+
+### Minimum supported Rust version
+
+PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible.
+
+However, there will always be support for at least the last few Rust compiler versions, so that users have time to update.
+
+If your PR needs to bump the minimum supported Rust version, this is acceptable with the following conditions:
+- Any changes which require a more recent version than what is [currently available on stable Red Hat Enterprise Linux](https://access.redhat.com/documentation/en-us/red_hat_developer_tools/1/) will be postponed. (This is to allow package managers to update support for newer `rustc` versions; RHEL was arbitrarily picked because their update policy is clear.)
+- You might be asked to do extra work to tidy up other parts of the PyO3 codebase which can use the compiler version bump :)

Makefile

@@ -1,8 +1,5 @@
 .PHONY: test test_py publish clippy lint fmt
 
-# Constant used in clippy target
-CLIPPY_LINTS_TO_DENY := warnings
-
 test: lint test_py
 	cargo test
 
@@ -15,8 +12,7 @@ fmt:
 
 clippy:
 	@touch src/lib.rs  # Touching file to ensure that cargo clippy will re-check the project
-	cargo clippy --features="default num-bigint num-complex" --tests -- \
-		$(addprefix -D ,${CLIPPY_LINTS_TO_DENY})
+	cargo clippy --features="default num-bigint num-complex" --tests -- -Dwarnings
 	for example in examples/*; do (cd $$example/; cargo clippy) || exit 1; done
 
 lint: fmt clippy

README.md

@@ -13,13 +13,15 @@
 
 * API Documentation: [stable](https://docs.rs/pyo3/) |  [master](https://pyo3.rs/master/doc)
 
+* Contributing Notes: [github](https://github.com/PyO3/pyo3/blob/master/Contributing.md)
+
 A comparison with rust-cpython can be found [in the guide](https://pyo3.rs/master/rust_cpython.html).
 
 ## Usage
 
 PyO3 supports Python 3.5 and up. The minimum required Rust version is 1.39.0.
 
-PyPy is also supported (via cpyext) for Python 3.5 only, targeted PyPy version is 7.0.0.
+Building with PyPy is also possible (via cpyext) for Python 3.6, targeted PyPy version is 7.3+.
 Please refer to the [pypy section in the guide](https://pyo3.rs/master/pypy.html).
 
 You can either write a native Python module in Rust, or use Python from a Rust binary.

build.rs

@@ -306,12 +306,7 @@ fn get_library_link_name(version: &PythonVersion, ld_version: &str) -> String {
             Some(minor) => format!("{}", minor),
             None => String::new(),
         };
-        match version.implementation {
-            PythonInterpreterKind::CPython => {
-                format!("python{}{}", version.major, minor_or_empty_string)
-            }
-            PythonInterpreterKind::PyPy => format!("pypy{}-c", version.major),
-        }
+        format!("python{}{}", version.major, minor_or_empty_string)
     } else {
         match version.implementation {
             PythonInterpreterKind::CPython => format!("python{}", ld_version),
@@ -338,6 +333,11 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
 
 #[cfg(target_os = "macos")]
 fn get_macos_linkmodel(config: &InterpreterConfig) -> Result<String> {
+    // PyPy 3.6 ships with a shared library, but doesn't have Py_ENABLE_SHARED.
+    if config.version.implementation == PythonInterpreterKind::PyPy {
+        return Ok("shared".to_string());
+    }
+
     let script = r#"
 import sysconfig
 
@@ -356,19 +356,11 @@ else:
 fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
     // os x can be linked to a framework or static or dynamic, and
     // Py_ENABLE_SHARED is wrong; framework means shared library
+    let link_name = get_library_link_name(&config.version, &config.ld_version);
     match get_macos_linkmodel(config)?.as_ref() {
-        "static" => Ok(format!(
-            "cargo:rustc-link-lib=static={}",
-            get_library_link_name(&config.version, &config.ld_version)
-        )),
-        "shared" => Ok(format!(
-            "cargo:rustc-link-lib={}",
-            get_library_link_name(&config.version, &config.ld_version)
-        )),
-        "framework" => Ok(format!(
-            "cargo:rustc-link-lib={}",
-            get_library_link_name(&config.version, &config.ld_version)
-        )),
+        "static" => Ok(format!("cargo:rustc-link-lib=static={}", link_name,)),
+        "shared" => Ok(format!("cargo:rustc-link-lib={}", link_name)),
+        "framework" => Ok(format!("cargo:rustc-link-lib={}", link_name,)),
         other => bail!("unknown linkmodel {}", other),
     }
 }
@@ -382,6 +374,28 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
     ))
 }
 
+fn find_interpreter() -> Result<PathBuf> {
+    if let Some(exe) = env::var_os("PYO3_PYTHON") {
+        Ok(exe.into())
+    } else if let Some(exe) = env::var_os("PYTHON_SYS_EXECUTABLE") {
+        // Backwards-compatible name for PYO3_PYTHON; this may be removed at some point in the future.
+        Ok(exe.into())
+    } else {
+        ["python", "python3"]
+            .iter()
+            .find(|bin| {
+                if let Ok(out) = Command::new(bin).arg("--version").output() {
+                    // begin with `Python 3.X.X :: additional info`
+                    out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3")
+                } else {
+                    false
+                }
+            })
+            .map(PathBuf::from)
+            .ok_or_else(|| "Python 3.x interpreter not found".into())
+    }
+}
+
 /// Locate a suitable python interpreter and extract config from it.
 ///
 /// The following locations are checked in the order listed:
@@ -394,27 +408,7 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
 ///
 /// If none of the above works, an error is returned
 fn find_interpreter_and_get_config() -> Result<(InterpreterConfig, HashMap<String, String>)> {
-    let python_interpreter = if let Some(exe) = env::var_os("PYO3_PYTHON") {
-        exe.into()
-    } else if let Some(exe) = env::var_os("PYTHON_SYS_EXECUTABLE") {
-        // Backwards-compatible name for PYO3_PYTHON; this may be removed at some point in the future.
-        exe.into()
-    } else {
-        PathBuf::from(
-            ["python", "python3"]
-                .iter()
-                .find(|bin| {
-                    if let Ok(out) = Command::new(bin).arg("--version").output() {
-                        // begin with `Python 3.X.X :: additional info`
-                        out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3")
-                    } else {
-                        false
-                    }
-                })
-                .ok_or("Python 3.x interpreter not found")?,
-        )
-    };
-
+    let python_interpreter = find_interpreter()?;
     let interpreter_config = get_config_from_interpreter(&python_interpreter)?;
     if interpreter_config.version.major == 3 {
         return Ok((interpreter_config, get_config_vars(&python_interpreter)?));
@@ -426,7 +420,6 @@ fn find_interpreter_and_get_config() -> Result<(InterpreterConfig, HashMap<Strin
 /// Extract compilation vars from the specified interpreter.
 fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
     let script = r#"
-import json
 import platform
 import struct
 import sys

ci/travis/test.sh

@@ -10,11 +10,6 @@ else
   cargo build;
 fi
 
-if [[ $RUN_LINT == 1 ]]; then
-    pip install --pre black==19.3b0
-    make lint
-fi
-
 for example_dir in examples/*; do
     cd $example_dir
     if [[ $FEATURES == *"pypy"* ]]; then