Development towards 0.4.0

Cargo.toml

@@ -1,5 +1,7 @@
 [workspace]
 members = [
+    # Examples of usage
+    "examples",
     # Public API
     "pyo3_bindgen",
     # CLI tool
@@ -9,6 +11,12 @@ members = [
     # Procedural macros
     "pyo3_bindgen_macros",
 ]
+default-members = [
+    "pyo3_bindgen",
+    "pyo3_bindgen_cli",
+    "pyo3_bindgen_engine",
+    "pyo3_bindgen_macros",
+]
 resolver = "2"
 
 [workspace.package]
@@ -20,18 +28,19 @@ license = "MIT OR Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/AndrejOrsula/pyo3_bindgen"
 rust-version = "1.74"
-version = "0.3.1"
+version = "0.4.0"
 
 [workspace.dependencies]
-pyo3_bindgen = { path = "pyo3_bindgen", version = "0.3.1" }
-pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.3.1" }
-pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.3.1" }
+pyo3_bindgen = { path = "pyo3_bindgen", version = "0.4.0" }
+pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.4.0" }
+pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.4.0" }
 
 assert_cmd = { version = "2" }
 clap = { version = "4.5", features = ["derive"] }
 criterion = { version = "0.5" }
 indoc = { version = "2" }
 itertools = { version = "0.12" }
+libc = { version = "0.2" }
 predicates = { version = "3" }
 prettyplease = { version = "0.2" }
 proc-macro2 = { version = "1" }

README.md

@@ -20,7 +20,7 @@ An example of a generated Rust function signature and its intended usage is show
 ```py
 
 def answer_to(question: str) -> int:
-  """Returns answer to question."""
+  """Returns answer to a question."""
 
   return 42
 
@@ -44,9 +44,9 @@ if __name__ == "__main__":
 <td>
 
 ```rs
-/// Returns answer to question.
+/// Returns answer to a question.
 pub fn answer_to<'py>(
-  py: ::pyo3::marker::Python<'py>,
+  py: ::pyo3::Python<'py>,
   question: &str,
 ) -> ::pyo3::PyResult<i64> {
   ... // Calls function via `pyo3`
@@ -81,91 +81,115 @@ The workspace contains these packages:
 - **[pyo3_bindgen](pyo3_bindgen):** Public API for generation of bindings (in `build.rs` or via procedural macros)
 - **[pyo3_bindgen_cli](pyo3_bindgen_cli):** CLI tool for generation of bindings via `pyo3_bindgen` executable
 - **[pyo3_bindgen_engine](pyo3_bindgen_engine):** The underlying engine for generation of bindings
-- **[pyo3_bindgen_macros](pyo3_bindgen_macros):** \[Experimental\] Procedural macros for in-place generation
+- **[pyo3_bindgen_macros](pyo3_bindgen_macros):** Procedural macros for in-place generation
 
 ## Instructions
 
-Add `pyo3` as a dependency and `pyo3_bindgen` as a build dependency to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest (`auto-initialize` feature of `pyo3` is optional and shown here for your convenience).
+### <a href="#-option-1-build-script"><img src="https://rustacean.net/assets/rustacean-flat-noshadow.svg" width="16" height="16"></a> Option 1: Build script
+
+First, add `pyo3_bindgen` as a **build dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest. To actually use the generated bindings, you will also need to add `pyo3` as a regular dependency (or use the re-exported `pyo3_bindgen::pyo3` module).
 
 ```toml
+[build-dependencies]
+pyo3_bindgen = { version = "0.4" }
+
 [dependencies]
 pyo3 = { version = "0.20", features = ["auto-initialize"] }
-
-[build-dependencies]
-pyo3_bindgen = { version = "0.3" }
 ```
 
-### <a href="#-option-1-build-script"><img src="https://rustacean.net/assets/rustacean-flat-noshadow.svg" width="16" height="16"></a> Option 1: Build script
+Then, create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the selected Python modules. In this example, the bindings are simultaneously generated for the "os", "posixpath", and "sys" Python modules. At the end of the generation process, the Rust bindings are written to `${OUT_DIR}/bindings.rs`.
 
-Create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the `py_module` Python module.
+> With this approach, you can also customize the generation process via [`pyo3_bindgen::Config`](https://docs.rs/pyo3_bindgen/latest/pyo3_bindgen/struct.Config.html) that can be passed to the constructor, e.g. `Codegen::new(Config::builder().include_private(true).build())`.
 
 ```rs
-// build.rs
-use pyo3_bindgen::{Codegen, Config};
+//! build.rs
+use pyo3_bindgen::Codegen;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Generate Rust bindings to Python modules
-    Codegen::new(Config::default())?
-        .module_name("py_module")?
-        .build(std::path::Path::new(&std::env::var("OUT_DIR")?).join("bindings.rs"))?;
+    Codegen::default()
+        .module_names(["os", "posixpath", "sys"])?
+        .build(format!("{}/bindings.rs", std::env::var("OUT_DIR")?))?;
     Ok(())
 }
 ```
 
-Afterwards, include the generated bindings anywhere in your crate.
+Afterwards, you can include the generated Rust code via the `include!` macro anywhere in your crate and use the generated bindings as regular Rust modules. However, the bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held.
 
 ```rs
+//! src/main.rs
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
-pub use py_module::*;
+
+fn main() -> pyo3::PyResult<()> {
+    pyo3::Python::with_gil(|py| {
+        // Get the path to the Python executable via "sys" Python module
+        let python_exe_path = sys::executable(py)?;
+        // Get the current working directory via "os" Python module
+        let current_dir = os::getcwd(py)?;
+        // Get the relative path to the Python executable via "posixpath" Python module
+        let relpath_to_python_exe = posixpath::relpath(py, python_exe_path, current_dir)?;
+
+        println!("Relative path to Python executable: '{relpath_to_python_exe}'");
+        Ok(())
+    })
+}
 ```
 
-### <a href="#-option-2-cli-tool"><img src="https://www.svgrepo.com/show/353478/bash-icon.svg" width="16" height="16"></a> Option 2: CLI tool
+### <a href="#-option-2-procedural-macros-experimental"><img src="https://www.svgrepo.com/show/269868/lab.svg" width="16" height="16"></a> Option 2: Procedural macros (experimental)
 
-Install the `pyo3_bindgen` executable with `cargo`.
+As an alternative to build scripts, you can use procedural macros to generate the bindings in-place. First, add `pyo3_bindgen_macros` as a **regular dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest.
 
-```bash
-cargo install --locked pyo3_bindgen_cli
+```toml
+[dependencies]
+pyo3_bindgen = { version = "0.4" }
 ```
 
-Afterwards, run the `pyo3_bindgen` executable while passing the name of the target Python module.
+Subsequently, the `import_python!` macro can be used to generate Rust bindings for the selected Python modules anywhere in your crate. As demonstrated in the example below, Rust bindings are generated for the "math" Python module and can directly be used in the same scope. Similar to the previous approach, the generated bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held.
 
-```bash
-# Pass `--help` to show the usage and available options
-pyo3_bindgen -m py_module -o bindings.rs
-```
+> As opposed to using build scripts, this approach does not offer the same level of customization via `pyo3_bindgen::Config`. Furthermore, the procedural macro is quite experimental and might not work in all cases.
 
-### <a href="#-option-3-experimental-procedural-macros"><img src="https://www.svgrepo.com/show/269868/lab.svg" width="16" height="16"></a> Option 3 \[Experimental\]: Procedural macros
+```rs
+use pyo3_bindgen::import_python;
+import_python!("math");
+
+// Which Pi do you prefer?
+// a) 🐍 Pi from Python "math" module
+// b) 🦀 Pi from Rust standard library
+// c) 🥧 Pi from your favourite bakery
+pyo3::Python::with_gil(|py| {
+    let python_pi = math::pi(py).unwrap();
+    let rust_pi = std::f64::consts::PI;
+    assert_eq!(python_pi, rust_pi);
+})
+```
 
-> **Note:** This feature is experimental and will probably fail in many cases. It is recommended to use build scripts instead.
+### <a href="#-option-3-cli-tool"><img src="https://www.svgrepo.com/show/353478/bash-icon.svg" width="16" height="16"></a> Option 3: CLI tool
 
-Enable the `macros` feature of `pyo3_bindgen`.
+For a quick start and testing purposes, you can use the `pyo3_bindgen` executable to generate and inspect bindings for the selected Python modules. The executable is available as a standalone package and can be installed via `cargo`.
 
-```toml
-[build-dependencies]
-pyo3_bindgen = { version = "0.3", features = ["macros"] }
+```bash
+cargo install --locked pyo3_bindgen_cli
 ```
 
-Then, you can call the `import_python!` macro anywhere in your crate.
+Afterwards, run the `pyo3_bindgen` executable to generate Rust bindings for the selected Python modules. The generated bindings are printed to STDOUT by default, but they can also be written to a file via the `-o` option (see `pyo3_bindgen --help` for more options).
 
-```rs
-pyo3_bindgen::import_python!("py_module");
-pub use py_module::*;
+```bash
+pyo3_bindgen -m os sys numpy -o bindings.rs
 ```
 
 ## Status
 
 This project is in early development, and as such, the API of the generated bindings is not yet stable.
 
-- Not all Python types are mapped to their Rust equivalents yet. For this reason, some additional typecasting might be currently required when using the generated bindings (e.g. `let typed_value: py_module::MyClass = get_value()?.extract()?;`).
-- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Here are some preliminary results for version `0.3.0` with the default configuration (measured: parsing IO & codegen | not measured: compilation of the generated bindings, which takes much longer):
+- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Here are some preliminary results for version `0.3` with the default configuration (measured: parsing IO & codegen | not measured: compilation of the generated bindings, which takes much longer):
   - `sys`: 1.24 ms (0.66k total LoC)
   - `os`: 8.38 ms (3.88k total LoC)
   - `numpy`: 1.02 s (294k total LoC)
   - `torch`: 7.05 s (1.08M total LoC)
 - The generation of bindings should never panic as long as the target Python module can be successfully imported. If it does, please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as a bug.
 - The generated bindings should always be compilable and usable in Rust. If you encounter any issues, consider manually fixing the problematic parts of the bindings and please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as a bug.
-- However, the generated bindings are based on the introspection of the target Python module. Therefore, the correctness of the generated bindings is directly dependent on the quality of the type annotations and docstrings in the target Python module. Ideally, the generated bindings should be considered unsafe and serve as a starting point for safe and idiomatic Rust APIs.
-- Although implemented, the procedural macro does not work in many cases because PyO3 fails to import the target Python module when used from within a `proc_macro` crate. Therefore, it is recommended to use build scripts instead for now.
+- However, the generated bindings are based on the introspection of the target Python module. Therefore, the completeness and correctness of the generated bindings are directly dependent on the quality of the module structure, type annotations and docstrings in the target Python module. Ideally, the generated bindings should be considered unsafe and serve as a starting point for safe and idiomatic Rust APIs. If you find that something in the generated bindings is incorrect or missing, please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as well.
+- Not all Python types are mapped to their Rust equivalents yet. For this reason, some additional type-casting might be required when using the generated bindings (e.g. `let typed_value: MyType = any_value.extract()?;`).
+- Although implemented, the procedural macro might not work in many cases. Therefore, it is recommended that the build scripts be used wherever possible.
 
 ## License
 

examples/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "examples"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+version.workspace = true
+publish = false
+
+[dependencies]
+pyo3 = { workspace = true, features = ["auto-initialize"] }
+pyo3_bindgen = { workspace = true }
+
+[build-dependencies]
+pyo3_bindgen = { workspace = true }
+
+[[example]]
+name = "math"
+path = "math.rs"
+
+[[example]]
+name = "os_sys"
+path = "os_sys.rs"
+
+# [[example]]
+# name = "pygal"
+# path = "pygal.rs"
+
+[[example]]
+name = "random"
+path = "random.rs"

examples/build.rs

@@ -0,0 +1,8 @@
+use pyo3_bindgen::Codegen;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    Codegen::default()
+        .module_names(["os", "posixpath", "sys"])?
+        .build(format!("{}/bindings.rs", std::env::var("OUT_DIR")?))?;
+    Ok(())
+}

examples/math.rs

@@ -0,0 +1,26 @@
+//! Example demonstrating the use of the `import_python!` macro for the "math" module.
+//!
+//! Python equivalent:
+//!
+//! ```py
+//! import math
+//!
+//! python_pi = math.pi
+//! assert python_pi == 3.141592653589793
+//! print(f"Python Pi: {python_pi}")
+//! ```
+
+pyo3_bindgen::import_python!("math");
+
+fn main() {
+    // Which Pi do you prefer?
+    // a) 🐍 Pi from Python "math" module
+    // b) 🦀 Pi from Rust standard library
+    // c) 🥧 Pi from your favorite bakery
+    pyo3::Python::with_gil(|py| {
+        let python_pi = math::pi(py).unwrap();
+        let rust_pi = std::f64::consts::PI;
+        assert_eq!(python_pi, rust_pi);
+        println!("Python Pi: {}", python_pi);
+    })
+}

examples/os_sys.rs

@@ -0,0 +1,34 @@
+//! Example demonstrating the use of the `pyo3_bindgen` crate via build script for
+//! the "os", "posixpath", and "sys" Python modules.
+//!
+//! See `build.rs` for more details about the generation.
+//!
+//! Python equivalent:
+//!
+//! ```py
+//! import os
+//! import posixpath
+//! import sys
+//!
+//! python_exe_path = sys.executable
+//! current_dir = os.getcwd()
+//! relpath_to_python_exe = posixpath.relpath(python_exe_path, current_dir)
+//!
+//! print(f"Relative path to Python executable: '{relpath_to_python_exe}'")
+//! ```
+
+include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+
+fn main() -> pyo3::PyResult<()> {
+    pyo3::Python::with_gil(|py| {
+        // Get the path to the Python executable via "sys" Python module
+        let python_exe_path = sys::executable(py)?;
+        // Get the current working directory via "os" Python module
+        let current_dir = os::getcwd(py)?;
+        // Get the relative path to the Python executable via "posixpath" Python module
+        let relpath_to_python_exe = posixpath::relpath(py, python_exe_path, current_dir)?;
+
+        println!("Relative path to Python executable: '{relpath_to_python_exe}'");
+        Ok(())
+    })
+}