From 566ffd2fb4af141278d1b951284599e9c6935fe9 Mon Sep 17 00:00:00 2001 From: Matthijs Brobbel Date: Tue, 8 Nov 2022 15:35:11 +0100 Subject: [PATCH] Add `[tool.maturin.include]` and `[tool.maturin.exclude]` --- guide/src/metadata.md | 5 ++ src/build_context.rs | 74 +++++++++++++++--- src/lib.rs | 2 +- src/module_writer.rs | 129 +++++++++++++++++++++++++++++-- src/pyproject_toml.rs | 151 ++++++++++++++++++++++++++++++++++++- src/source_distribution.rs | 47 ++++++++---- 6 files changed, 375 insertions(+), 33 deletions(-) diff --git a/guide/src/metadata.md b/guide/src/metadata.md index 0b10ac556..4b0088689 100644 --- a/guide/src/metadata.md +++ b/guide/src/metadata.md @@ -108,7 +108,12 @@ in the `tool.maturin` section of `pyproject.toml`. ```toml [tool.maturin] # Include arbitrary files in the sdist +# NOTE: deprecated, please use `include` with `format="sdist"` sdist-include = [] +# Include additional files +include = [] +# Exclude files +exclude = [] # Bindings type bindings = "pyo3" # Control the platform tag on linux diff --git a/src/build_context.rs b/src/build_context.rs index 37ed2369b..a95f56a31 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -10,12 +10,15 @@ use crate::project_layout::ProjectLayout; use crate::python_interpreter::InterpreterKind; use crate::source_distribution::source_distribution; use crate::{ - compile, BuildArtifact, Metadata21, ModuleWriter, PyProjectToml, PythonInterpreter, Target, + compile, BuildArtifact, Format, Metadata21, ModuleWriter, PyProjectToml, PythonInterpreter, + Target, }; use anyhow::{anyhow, bail, Context, Result}; use cargo_metadata::Metadata; use fs_err as fs; +use ignore::overrides::{Override, OverrideBuilder}; use lddtree::Library; +use normpath::PathExt; use regex::Regex; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; @@ -249,8 +252,9 @@ impl BuildContext { match self.pyproject_toml.as_ref() { Some(pyproject) => { - let sdist_path = source_distribution(self, pyproject) - .context("Failed to build source distribution")?; + let sdist_path = + source_distribution(self, pyproject, self.excludes(Format::Sdist)?) + .context("Failed to build source distribution")?; Ok(Some((sdist_path, "source".to_string()))) } None => Ok(None), @@ -450,6 +454,23 @@ impl BuildContext { Ok(()) } + fn excludes(&self, format: Format) -> Result> { + if let Some(pyproject) = self.pyproject_toml.as_ref() { + let pyproject_dir = self.pyproject_toml_path.normalize()?.into_path_buf(); + if let Some(glob_patterns) = &pyproject.exclude() { + let mut excludes = OverrideBuilder::new(pyproject_dir.parent().unwrap()); + for glob in glob_patterns + .iter() + .filter_map(|glob_pattern| glob_pattern.targets(format)) + { + excludes.add(glob)?; + } + return Ok(Some(excludes.build()?)); + } + } + Ok(None) + } + fn write_binding_wheel_abi3( &self, artifact: BuildArtifact, @@ -463,7 +484,13 @@ impl BuildContext { .get_platform_tag(platform_tags, self.universal2)?; let tag = format!("cp{}{}-abi3-{}", major, min_minor, platform); - let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?; + let mut writer = WheelWriter::new( + &tag, + &self.out, + &self.metadata21, + &[tag.clone()], + self.excludes(Format::Wheel)?, + )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; write_bindings_module( @@ -474,6 +501,8 @@ impl BuildContext { None, &self.target, self.editable, + &self.metadata21, + self.pyproject_toml.as_ref(), ) .context("Failed to add the files to the wheel")?; @@ -534,7 +563,13 @@ impl BuildContext { ) -> Result { let tag = python_interpreter.get_tag(&self.target, platform_tags, self.universal2)?; - let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?; + let mut writer = WheelWriter::new( + &tag, + &self.out, + &self.metadata21, + &[tag.clone()], + self.excludes(Format::Wheel)?, + )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; write_bindings_module( @@ -545,6 +580,8 @@ impl BuildContext { Some(python_interpreter), &self.target, self.editable, + &self.metadata21, + self.pyproject_toml.as_ref(), ) .context("Failed to add the files to the wheel")?; @@ -651,7 +688,13 @@ impl BuildContext { .target .get_universal_tags(platform_tags, self.universal2)?; - let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?; + let mut writer = WheelWriter::new( + &tag, + &self.out, + &self.metadata21, + &tags, + self.excludes(Format::Wheel)?, + )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; write_cffi_module( @@ -663,6 +706,8 @@ impl BuildContext { &artifact.path, &self.interpreter[0].executable, self.editable, + &self.metadata21, + self.pyproject_toml.as_ref(), )?; self.add_pth(&mut writer)?; @@ -754,7 +799,13 @@ impl BuildContext { self.metadata21.clone() }; - let mut writer = WheelWriter::new(&tag, &self.out, &metadata21, &tags)?; + let mut writer = WheelWriter::new( + &tag, + &self.out, + &metadata21, + &tags, + self.excludes(Format::Wheel)?, + )?; if let Some(python_module) = &self.project_layout.python_module { if self.target.is_wasi() { @@ -763,8 +814,13 @@ impl BuildContext { bail!("Sorry, adding python code to a wasm binary is currently not supported") } if !self.editable { - write_python_part(&mut writer, python_module) - .context("Failed to add the python module to the package")?; + write_python_part( + &mut writer, + python_module, + &metadata21, + self.pyproject_toml.as_ref(), + ) + .context("Failed to add the python module to the package")?; } } diff --git a/src/lib.rs b/src/lib.rs index 89fb65583..2ffb156bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,7 @@ pub use crate::module_writer::{ write_dist_info, ModuleWriter, PathWriter, SDistWriter, WheelWriter, }; pub use crate::new_project::{init_project, new_project, GenerateProjectOptions}; -pub use crate::pyproject_toml::PyProjectToml; +pub use crate::pyproject_toml::{Format, Formats, GlobPattern, PyProjectToml}; pub use crate::python_interpreter::PythonInterpreter; pub use crate::target::Target; #[cfg(feature = "upload")] diff --git a/src/module_writer.rs b/src/module_writer.rs index 00cf053ed..983d1c25b 100644 --- a/src/module_writer.rs +++ b/src/module_writer.rs @@ -1,11 +1,12 @@ //! The wheel format is (mostly) specified in PEP 427 use crate::project_layout::ProjectLayout; -use crate::{BridgeModel, Metadata21, PythonInterpreter, Target}; +use crate::{BridgeModel, Format, Metadata21, PyProjectToml, PythonInterpreter, Target}; use anyhow::{anyhow, bail, Context, Result}; use flate2::write::GzEncoder; use flate2::Compression; use fs_err as fs; use fs_err::File; +use ignore::overrides::Override; use ignore::WalkBuilder; use normpath::PathExt as _; use sha2::{Digest, Sha256}; @@ -202,6 +203,7 @@ pub struct WheelWriter { record: Vec<(String, String, usize)>, record_file: PathBuf, wheel_path: PathBuf, + excludes: Option, } impl ModuleWriter for WheelWriter { @@ -215,8 +217,12 @@ impl ModuleWriter for WheelWriter { bytes: &[u8], permissions: u32, ) -> Result<()> { + let target = target.as_ref(); + if self.exclude(target) { + return Ok(()); + } // The zip standard mandates using unix style paths - let target = target.as_ref().to_str().unwrap().replace('\\', "/"); + let target = target.to_str().unwrap().replace('\\', "/"); // Unlike users which can use the develop subcommand, the tests have to go through // packing a zip which pip than has to unpack. This makes this 2-3 times faster @@ -247,6 +253,7 @@ impl WheelWriter { wheel_dir: &Path, metadata21: &Metadata21, tags: &[String], + excludes: Option, ) -> Result { let wheel_path = wheel_dir.join(format!( "{}-{}-{}.whl", @@ -262,6 +269,7 @@ impl WheelWriter { record: Vec::new(), record_file: metadata21.get_dist_info_dir().join("RECORD"), wheel_path, + excludes, }; write_dist_info(&mut builder, metadata21, tags)?; @@ -289,6 +297,15 @@ impl WheelWriter { Ok(()) } + /// Returns `true` if the given path should be excluded + fn exclude(&self, path: impl AsRef) -> bool { + if let Some(excludes) = &self.excludes { + excludes.matched(path.as_ref(), false).is_whitelist() + } else { + false + } + } + /// Creates the record file and finishes the zip pub fn finish(mut self) -> Result { let compression_method = if cfg!(feature = "faster-tests") { @@ -318,6 +335,7 @@ pub struct SDistWriter { tar: tar::Builder>, path: PathBuf, files: HashSet, + excludes: Option, } impl ModuleWriter for SDistWriter { @@ -332,6 +350,10 @@ impl ModuleWriter for SDistWriter { permissions: u32, ) -> Result<()> { let target = target.as_ref(); + if self.exclude(target) { + return Ok(()); + } + if self.files.contains(target) { // Ignore duplicate files return Ok(()); @@ -354,6 +376,9 @@ impl ModuleWriter for SDistWriter { fn add_file(&mut self, target: impl AsRef, source: impl AsRef) -> Result<()> { let source = source.as_ref(); + if self.exclude(source) { + return Ok(()); + } let target = target.as_ref(); if source == self.path { bail!( @@ -381,7 +406,11 @@ impl ModuleWriter for SDistWriter { impl SDistWriter { /// Create a source distribution .tar.gz which can be subsequently expanded - pub fn new(wheel_dir: impl AsRef, metadata21: &Metadata21) -> Result { + pub fn new( + wheel_dir: impl AsRef, + metadata21: &Metadata21, + excludes: Option, + ) -> Result { let path = wheel_dir.as_ref().join(format!( "{}-{}.tar.gz", &metadata21.get_distribution_escaped(), @@ -396,9 +425,19 @@ impl SDistWriter { tar, path, files: HashSet::new(), + excludes, }) } + /// Returns `true` if the given path should be excluded + fn exclude(&self, path: impl AsRef) -> bool { + if let Some(excludes) = &self.excludes { + excludes.matched(path.as_ref(), false).is_whitelist() + } else { + false + } + } + /// Finished the .tar.gz archive pub fn finish(mut self) -> Result { self.tar.finish()?; @@ -634,6 +673,8 @@ pub fn write_bindings_module( python_interpreter: Option<&PythonInterpreter>, target: &Target, editable: bool, + metadata: &Metadata21, + pyproject_toml: Option<&PyProjectToml>, ) -> Result<()> { let ext_name = &project_layout.extension_name; let so_filename = match python_interpreter { @@ -664,7 +705,7 @@ pub fn write_bindings_module( target.display() ))?; } else { - write_python_part(writer, python_module) + write_python_part(writer, python_module, metadata, pyproject_toml) .context("Failed to add the python module to the package")?; let relative = project_layout @@ -714,6 +755,8 @@ pub fn write_cffi_module( artifact: &Path, python: &Path, editable: bool, + metadata21: &Metadata21, + pyproject_toml: Option<&PyProjectToml>, ) -> Result<()> { let cffi_declarations = generate_cffi_declarations(crate_dir, target_dir, python)?; @@ -721,7 +764,7 @@ pub fn write_cffi_module( if let Some(python_module) = &project_layout.python_module { if !editable { - write_python_part(writer, python_module) + write_python_part(writer, python_module, metadata21, pyproject_toml) .context("Failed to add the python module to the package")?; } @@ -848,11 +891,14 @@ if __name__ == '__main__': pub fn write_python_part( writer: &mut impl ModuleWriter, python_module: impl AsRef, + metadata21: &Metadata21, + pyproject_toml: Option<&PyProjectToml>, ) -> Result<()> { - for absolute in WalkBuilder::new(&python_module).hidden(false).build() { + let python_module = python_module.as_ref(); + for absolute in WalkBuilder::new(python_module).hidden(false).build() { let absolute = absolute?.into_path(); let relative = absolute - .strip_prefix(python_module.as_ref().parent().unwrap()) + .strip_prefix(python_module.parent().unwrap()) .unwrap(); if absolute.is_dir() { writer.add_directory(relative)?; @@ -870,6 +916,34 @@ pub fn write_python_part( } } + // Include additional files + if let Some(pyproject) = pyproject_toml { + let root_dir = PathBuf::from(format!( + "{}-{}", + &metadata21.get_distribution_escaped(), + &metadata21.get_version_escaped() + )); + if let Some(glob_patterns) = pyproject.include() { + for pattern in glob_patterns + .iter() + .filter_map(|glob_pattern| glob_pattern.targets(Format::Sdist)) + { + println!("📦 Including files matching \"{}\"", pattern); + for source in glob::glob(&python_module.join(pattern).to_string_lossy()) + .expect("No files found for pattern") + .filter_map(Result::ok) + { + let target = root_dir.join(source.strip_prefix(python_module).unwrap()); + if source.is_dir() { + writer.add_directory(target)?; + } else { + writer.add_file(target, source)?; + } + } + } + } + } + Ok(()) } @@ -972,3 +1046,44 @@ pub fn add_data(writer: &mut impl ModuleWriter, data: Option<&Path>) -> Result<( } Ok(()) } + +#[cfg(test)] +mod tests { + use ignore::overrides::OverrideBuilder; + + use super::*; + + #[test] + // The mechanism is the same for wheel_writer + fn sdist_writer_excludes() -> Result<(), Box> { + let metadata = Metadata21::default(); + let perm = 0o777; + + // No excludes + let tmp_dir = TempDir::new()?; + let mut writer = SDistWriter::new(&tmp_dir, &metadata, None)?; + assert!(writer.files.is_empty()); + writer.add_bytes_with_permissions("test", &[], perm)?; + assert_eq!(writer.files.len(), 1); + writer.finish()?; + tmp_dir.close()?; + + // A test filter + let tmp_dir = TempDir::new()?; + let mut excludes = OverrideBuilder::new(&tmp_dir); + excludes.add("test*")?; + excludes.add("!test2")?; + let mut writer = SDistWriter::new(&tmp_dir, &metadata, Some(excludes.build()?))?; + writer.add_bytes_with_permissions("test1", &[], perm)?; + writer.add_bytes_with_permissions("test3", &[], perm)?; + assert!(writer.files.is_empty()); + writer.add_bytes_with_permissions("test2", &[], perm)?; + assert!(!writer.files.is_empty()); + writer.add_bytes_with_permissions("yes", &[], perm)?; + assert_eq!(writer.files.len(), 2); + writer.finish()?; + tmp_dir.close()?; + + Ok(()) + } +} diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs index cd0629ccb..cb0ad0de6 100644 --- a/src/pyproject_toml.rs +++ b/src/pyproject_toml.rs @@ -12,12 +12,82 @@ pub struct Tool { maturin: Option, } +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +/// The target format for the include or exclude [GlobPattern]. +/// +/// See [Formats]. +pub enum Format { + /// Source distribution + Sdist, + /// Wheel + Wheel, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +/// A single [Format] or multiple [Format] values for a [GlobPattern]. +pub enum Formats { + /// A single [Format] value + Single(Format), + /// Multiple [Format] values + Multiple(Vec), +} + +impl Formats { + /// Returns `true` if the inner [Format] value(s) target the given [Format] + pub fn targets(&self, format: Format) -> bool { + match self { + Self::Single(val) if val == &format => true, + Self::Multiple(formats) if formats.contains(&format) => true, + _ => false, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +/// A glob pattern for the include and exclude configuration. +/// +/// See [PyProjectToml::include] and [PyProject::exclude]. +/// +/// Based on . +pub enum GlobPattern { + /// A glob + Path(String), + /// A glob `path` with a `format` key to specify one ore more [Format] values + WithFormat { + /// A glob + path: String, + /// One ore more [Format] values + format: Formats, + }, +} + +impl GlobPattern { + /// Returns the glob pattern for this patter if it targets the given [Format], else this returns `None`. + pub fn targets(&self, format: Format) -> Option<&str> { + match self { + // Not specified defaults to both + Self::Path(ref glob) => Some(glob), + Self::WithFormat { + path, + format: formats, + } if formats.targets(format) => Some(path), + _ => None, + } + } +} + /// The `[tool.maturin]` section of a pyproject.toml #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "kebab-case")] pub struct ToolMaturin { // maturin specific options + // TODO(0.15.0): remove deprecated sdist_include: Option>, + include: Option>, + exclude: Option>, bindings: Option, #[serde(alias = "manylinux")] compatibility: Option, @@ -98,10 +168,24 @@ impl PyProjectToml { } /// Returns the value of `[tool.maturin.sdist-include]` in pyproject.toml + #[deprecated( + since = "0.14.0", + note = "please use `PyProjectToml::include` ()" + )] pub fn sdist_include(&self) -> Option<&Vec> { self.maturin()?.sdist_include.as_ref() } + /// Returns the value of `[tool.maturin.include]` in pyproject.toml + pub fn include(&self) -> Option<&[GlobPattern]> { + self.maturin()?.include.as_ref().map(AsRef::as_ref) + } + + /// Returns the value of `[tool.maturin.exclude]` in pyproject.toml + pub fn exclude(&self) -> Option<&[GlobPattern]> { + self.maturin()?.exclude.as_ref().map(AsRef::as_ref) + } + /// Returns the value of `[tool.maturin.bindings]` in pyproject.toml pub fn bindings(&self) -> Option<&str> { self.maturin()?.bindings.as_deref() @@ -193,7 +277,10 @@ impl PyProjectToml { #[cfg(test)] mod tests { - use crate::PyProjectToml; + use crate::{ + pyproject_toml::{Format, Formats, GlobPattern, ToolMaturin}, + PyProjectToml, + }; use fs_err as fs; use pretty_assertions::assert_eq; use std::path::Path; @@ -261,4 +348,66 @@ mod tests { let without_constraint = PyProjectToml::new(pyproject_file).unwrap(); assert!(!without_constraint.warn_missing_maturin_version()); } + + #[test] + fn deserialize_include_exclude() { + let single = r#"include = ["single"]"#; + assert_eq!( + toml_edit::easy::from_str::(single) + .unwrap() + .include, + Some(vec![GlobPattern::Path("single".to_string())]) + ); + + let multiple = r#"include = ["one", "two"]"#; + assert_eq!( + toml_edit::easy::from_str::(multiple) + .unwrap() + .include, + Some(vec![ + GlobPattern::Path("one".to_string()), + GlobPattern::Path("two".to_string()) + ]) + ); + + let single_format = r#"include = [{path = "path", format="sdist"}]"#; + assert_eq!( + toml_edit::easy::from_str::(single_format) + .unwrap() + .include, + Some(vec![GlobPattern::WithFormat { + path: "path".to_string(), + format: Formats::Single(Format::Sdist) + },]) + ); + + let multiple_formats = r#"include = [{path = "path", format=["sdist", "wheel"]}]"#; + assert_eq!( + toml_edit::easy::from_str::(multiple_formats) + .unwrap() + .include, + Some(vec![GlobPattern::WithFormat { + path: "path".to_string(), + format: Formats::Multiple(vec![Format::Sdist, Format::Wheel]) + },]) + ); + + let mixed = r#"include = ["one", {path = "two", format="sdist"}, {path = "three", format=["sdist", "wheel"]}]"#; + assert_eq!( + toml_edit::easy::from_str::(mixed) + .unwrap() + .include, + Some(vec![ + GlobPattern::Path("one".to_string()), + GlobPattern::WithFormat { + path: "two".to_string(), + format: Formats::Single(Format::Sdist), + }, + GlobPattern::WithFormat { + path: "three".to_string(), + format: Formats::Multiple(vec![Format::Sdist, Format::Wheel]) + } + ]) + ); + } } diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 909d1d27e..7b8ee35e9 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,15 +1,16 @@ use crate::module_writer::{add_data, ModuleWriter}; use crate::polyfill::MetadataCommandExt; -use crate::{BuildContext, PyProjectToml, SDistWriter}; +use crate::{BuildContext, Format, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::{Metadata, MetadataCommand}; use fs_err as fs; +use ignore::overrides::Override; use normpath::PathExt as _; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; -use tracing::debug; +use tracing::{debug, warn}; const LOCAL_DEPENDENCIES_FOLDER: &str = "local_dependencies"; /// Inheritable workspace fields, see @@ -411,6 +412,7 @@ fn find_path_deps(cargo_metadata: &Metadata) -> Result> pub fn source_distribution( build_context: &BuildContext, pyproject: &PyProjectToml, + excludes: Option, ) -> Result { let metadata21 = &build_context.metadata21; let manifest_path = &build_context.manifest_path; @@ -427,7 +429,7 @@ pub fn source_distribution( let known_path_deps = find_path_deps(&build_context.cargo_metadata)?; - let mut writer = SDistWriter::new(&build_context.out, metadata21)?; + let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?; let root_dir = PathBuf::from(format!( "{}-{}", &metadata21.get_distribution_escaped(), @@ -551,20 +553,35 @@ pub fn source_distribution( } } + let mut include = |pattern| -> Result<()> { + println!("📦 Including files matching \"{}\"", pattern); + for source in glob::glob(&pyproject_dir.join(pattern).to_string_lossy()) + .expect("No files found for pattern") + .filter_map(Result::ok) + { + let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap()); + if source.is_dir() { + writer.add_directory(target)?; + } else { + writer.add_file(target, source)?; + } + } + Ok(()) + }; + if let Some(include_targets) = pyproject.sdist_include() { + warn!("`[tool.maturin.sdist-include]` is deprecated, please use `[tool.maturin.include]`"); for pattern in include_targets { - println!("📦 Including files matching \"{}\"", pattern); - for source in glob::glob(&pyproject_dir.join(pattern).to_string_lossy()) - .expect("No files found for pattern") - .filter_map(Result::ok) - { - let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap()); - if source.is_dir() { - writer.add_directory(target)?; - } else { - writer.add_file(target, source)?; - } - } + include(pattern.as_str())?; + } + } + + if let Some(glob_patterns) = pyproject.include() { + for pattern in glob_patterns + .iter() + .filter_map(|glob_pattern| glob_pattern.targets(Format::Sdist)) + { + include(pattern)?; } }