Skip to content

Commit

Permalink
Implement the wheel part of PEP 517 (#2)
Browse files Browse the repository at this point in the history
  • Loading branch information
konstin committed Jun 21, 2019
1 parent 3f9dd34 commit 671fcdb
Show file tree
Hide file tree
Showing 22 changed files with 629 additions and 144 deletions.
58 changes: 52 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ platforms = "0.2.0"
shlex = "0.1.1"
cbindgen = "0.8.7"
walkdir = "2.2.8"
flate2 = "1.0.9"
tar = "0.4.26"

[dev-dependencies]
indoc = "0.3.3"
Expand Down
28 changes: 27 additions & 1 deletion Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,32 @@ my-project
   └── lib.rs
```

## pyproject.toml

pyo3-pack has partial support for building through pyproject.toml. To use it, create a `pyproject.toml` next to your `Cargo.toml` with the following content:

```toml
[build-system]
requires = ["pyo3-pack", "toml"]
build-backend = "pyo3_pack"
```

You can then e.g. install your package with `pip install .`. With `pip install . -v` you can see the output of cargo and pyo3-pack.

You can use the options `manylinux`, `skip-auditwheel`, `bindings`, `cargo-extra-args` and `rustc-extra-args` under `[tool.pyo3-pack]` the same way you would when running pyo3-pack directly. The `bindings` key is required for cffi and bin projects as those can't be automatically detected.

```toml
[build-system]
requires = ["pyo3-pack", "toml"]
build-backend = "pyo3_pack"

[tool.pyo3-pack]
bindings = "cffi"
cargo-extra-args="-v"
```

Currently, only the wheel build part of [PEP 517](https://snarky.ca/clarifying-pep-518/) is supported. Building source distribution doesn't work yet, which means isolated builds will fail.

## Manylinux and auditwheel

For portability reasons, native python modules on linux must only dynamically link a set of very few libraries which are installed basically everywhere, hence the name manylinux. The pypa offers a special docker container and a tool called [auditwheel](https://github.com/pypa/auditwheel/) to ensure compliance with the [manylinux rules](https://www.python.org/dev/peps/pep-0513/#the-manylinux1-policy).
Expand All @@ -132,7 +158,7 @@ pyo3-pack itself is manylinux compliant when compiled for the musl target. The b

## PyPy

pyo3-pack can build wheels for pypy with pyo3. Note that pypy support in pyo3 is unreleased as of this writing. Also note that pypy [is not compatible with manylinux1](https://github.com/antocuni/pypy-wheels#why-not-manylinux1-wheels) and you can't publish pypy wheel to pypi pypy has been only tested manually and on linux. See [#115](https://github.com/PyO3/pyo3-pack/issues/115) for more details.
pyo3-pack can build wheels for pypy with pyo3. Note that pypy [is not compatible with manylinux1](https://github.com/antocuni/pypy-wheels#why-not-manylinux1-wheels) and you can't publish pypy wheel to pypi pypy has been only tested manually and on linux. See [#115](https://github.com/PyO3/pyo3-pack/issues/115) for more details.

### Build

Expand Down
105 changes: 105 additions & 0 deletions pyo3_pack/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""
pyo3-pack's implementation of the PEP 517 interface. Calls pyo3-pack through subprocess
Currently pyo3-pack doesn't have json output (nor is there a way to the user visible output off),
so this parse the user facing messages.
TODO: Don't require the user to specify toml as a requirement in the pyproject.toml
"""

import os
import shutil
import subprocess
from typing import List, Dict

import toml

available_options = [
"manylinux",
"skip-auditwheel",
"bindings",
"cargo-extra-args",
"rustc-extra-args",
]


def get_config() -> Dict[str, str]:
with open("pyproject.toml") as fp:
pyproject_toml = toml.load(fp)
return pyproject_toml.get("tool", {}).get("pyo3-pack", {})


def get_config_options() -> List[str]:
config = get_config()
options = []
for key, value in config.items():
if key not in available_options:
raise RuntimeError(
"{} is not a valid option for pyo3-pack. Valid are: {}".format(
key, ", ".join(available_options)
)
)
options.append("--{}={}".format(key, value))
return options


# noinspection PyUnusedLocal
def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
command = ["pyo3-pack", "build", "-i", "python"]
command.extend(get_config_options())

print("Running `{}`".format(" ".join(command)))
output = subprocess.check_output(command, universal_newlines=True)
print(output)
# Get the filename from `📦 Built wheel [for CPython 3.6m] to /home/user/project`
filename = os.path.split(output.strip().splitlines()[-1].split(" to ")[1])[1]
shutil.copy2("target/wheels/" + filename, os.path.join(wheel_directory, filename))
return filename


# noinspection PyUnusedLocal
def build_sdist(sdist_directory, config_settings=None):
command = [
"pyo3-pack",
"pep517",
"write-sdist",
"--sdist-directory",
sdist_directory,
]
command.extend(get_config_options())

print("Running `{}`".format(" ".join(command)))
output = subprocess.check_output(command, universal_newlines=True)
print(output)
return output.strip().splitlines()[-1]


# noinspection PyUnusedLocal
def get_requires_for_build_wheel(config_settings=None):
if get_config().get("bindings") == "cffi":
return ["cffi"]
else:
return []


# noinspection PyUnusedLocal
def get_requires_for_build_sdist(config_settings=None):
return []


# noinspection PyUnusedLocal
def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
command = [
"pyo3-pack",
"pep517",
"write-dist-info",
"--metadata-directory",
metadata_directory,
]
command.extend(get_config_options())

print("Running `{}`".format(" ".join(command)))
output = subprocess.check_output(command, universal_newlines=True)
print(output)
return output.strip().splitlines()[-1]
22 changes: 11 additions & 11 deletions src/build_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
use crate::auditwheel::auditwheel_rs;
use crate::compile;
use crate::compile::warn_missing_py_init;
use crate::module_writer::write_python_part;
use crate::module_writer::WheelWriter;
use crate::module_writer::{write_bin, write_bindings_module, write_cffi_module};
use crate::Manylinux;
Expand Down Expand Up @@ -215,20 +216,11 @@ impl BuildContext {
Ok(artifact)
}

fn get_unversal_tags(&self) -> (String, Vec<String>) {
let tag = format!(
"py2.py3-none-{platform}",
platform = self.target.get_platform_tag(&self.manylinux)
);
let tags = self.target.get_py2_and_py3_tags(&self.manylinux);
(tag, tags)
}

/// Builds a wheel with cffi bindings
pub fn build_cffi_wheel(&self) -> Result<PathBuf, Error> {
let artifact = self.compile_cdylib(None, None)?;

let (tag, tags) = self.get_unversal_tags();
let (tag, tags) = self.target.get_universal_tags(&self.manylinux);

let mut builder =
WheelWriter::new(&tag, &self.out, &self.metadata21, &self.scripts, &tags)?;
Expand Down Expand Up @@ -266,7 +258,7 @@ impl BuildContext {
auditwheel_rs(&artifact, &self.target, &self.manylinux)
.context("Failed to ensure manylinux compliance")?;

let (tag, tags) = self.get_unversal_tags();
let (tag, tags) = self.target.get_universal_tags(&self.manylinux);

if !self.scripts.is_empty() {
bail!("Defining entrypoints and working with a binary doesn't mix well");
Expand All @@ -275,6 +267,14 @@ impl BuildContext {
let mut builder =
WheelWriter::new(&tag, &self.out, &self.metadata21, &self.scripts, &tags)?;

match self.project_layout {
ProjectLayout::Mixed(ref python_module) => {
write_python_part(&mut builder, python_module, &self.module_name)
.context("Failed to add the python module to the package")?;
}
ProjectLayout::PureRust => {}
}

// I wouldn't know of any case where this would be the wrong (and neither do
// I know a better alternative)
let bin_name = artifact
Expand Down
Loading

0 comments on commit 671fcdb

Please sign in to comment.