sdist-include option

Cargo.toml

@@ -34,6 +34,7 @@ name = "maturin"
 [dependencies]
 base64 = "0.12.0"
 bytesize = "1.0.0"
+glob = "0.3"
 cargo_metadata = "0.9.1"
 cbindgen = { version = "0.13.1", default-features = false }
 digest = { version = "0.8.1", features = ["std"] }

Changelog.md

@@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (for the cli, not for the crate).
 
-### 0.8.0 - 2020-04-03
+## Unreleased
+
+### Added
+
+* `[tool.maturin]` now supports `sdist-include = ["path/**/*"]` to
+include arbitrary files in source distributions ([#296](https://github.com/PyO3/maturin/pull/296)).
+
+## 0.8.0 - 2020-04-03
 
 ### Added
 
@@ -33,7 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
  * Remove python 2 from tags by ijl in [#254](https://github.com/PyO3/maturin/pull/254)
  * 32-bit wheels didn't work on linux. This has been fixed by dae in [#250](https://github.com/PyO3/maturin/pull/250)
- * The path of the RECORD file on windows used a backward slash instead of a forward slash 
+ * The path of the RECORD file on windows used a backward slash instead of a forward slash
 
 ## 0.7.7 - 2019-11-12
 

Readme.md

@@ -154,6 +154,13 @@ bindings = "cffi"
 manylinux = "off"
 ```
 
+To include arbitrary files in the sdist for use during compilation specify `sdist-include` as an array of globs:
+
+```toml
+[tool.maturin]
+sdist-include = ["path/**/*"]
+```
+
 Using tox with build isolation is currently blocked by a tox bug ([tox-dev/tox#1344](https://github.com/tox-dev/tox/issues/1344)). There's a `cargo sdist` command for only building a source distribution as workaround for [pypa/pip#6041](https://github.com/pypa/pip/issues/6041).
 
 ## Manylinux and auditwheel

maturin/__init__.py

@@ -18,13 +18,18 @@
 
 import toml
 
+# these are only used when creating the sdist, not when building it
+create_only_options = [
+    "sdist-include",
+]
+
 available_options = [
-    "manylinux",
-    "skip-auditwheel",
     "bindings",
-    "strip",
     "cargo-extra-args",
+    "manylinux",
     "rustc-extra-args",
+    "skip-auditwheel",
+    "strip",
 ]
 
 
@@ -38,12 +43,11 @@ def get_config_options() -> List[str]:
     config = get_config()
     options = []
     for key, value in config.items():
+        if key in create_only_options:
+            continue
         if key not in available_options:
-            raise RuntimeError(
-                "{} is not a valid option for maturin. Valid are: {}".format(
-                    key, ", ".join(available_options)
-                )
-            )
+            # attempt to install even if keys from newer or older versions are present
+            print("WARNING: {} is not a recognized option for maturin".format(key))
         options.append("--{}={}".format(key, value))
     return options
 

src/build_context.rs

@@ -132,13 +132,19 @@ impl BuildContext {
         fs::create_dir_all(&self.out)
             .context("Failed to create the target directory for the source distribution")?;
 
-        if get_pyproject_toml(self.manifest_path.parent().unwrap()).is_ok() {
-            warn_on_local_deps(&self.cargo_metadata);
-            let sdist_path = source_distribution(&self.out, &self.metadata21, &self.manifest_path)
+        match get_pyproject_toml(self.manifest_path.parent().unwrap()) {
+            Ok(pyproject) => {
+                warn_on_local_deps(&self.cargo_metadata);
+                let sdist_path = source_distribution(
+                    &self.out,
+                    &self.metadata21,
+                    &self.manifest_path,
+                    pyproject.sdist_include(),
+                )
                 .context("Failed to build source distribution")?;
-            Ok(Some((sdist_path, "source".to_string(), None)))
-        } else {
-            Ok(None)
+                Ok(Some((sdist_path, "source".to_string(), None)))
+            }
+            Err(_) => Ok(None),
         }
     }
 

src/main.rs

@@ -288,7 +288,7 @@ fn pep517(subcommand: PEP517Command) -> Result<(), Error> {
             let manifest_dir = manifest_path.parent().unwrap();
             let metadata21 = Metadata21::from_cargo_toml(&cargo_toml, &manifest_dir)
                 .context("Failed to parse Cargo.toml into python metadata")?;
-            let path = source_distribution(sdist_directory, &metadata21, &manifest_path)
+            let path = source_distribution(sdist_directory, &metadata21, &manifest_path, None)
                 .context("Failed to build source distribution")?;
             println!("{}", path.display());
         }
@@ -462,7 +462,7 @@ fn run() -> Result<(), Error> {
             let manifest_dir = manifest_path.parent().unwrap();
 
             // Ensure the project has a compliant pyproject.toml
-            get_pyproject_toml(&manifest_dir)
+            let pyproject = get_pyproject_toml(&manifest_dir)
                 .context("A pyproject.toml with a PEP 517 compliant `[build-system]` table is required to build a source distribution")?;
 
             let cargo_toml = CargoToml::from_path(&manifest_path)?;
@@ -482,8 +482,13 @@ fn run() -> Result<(), Error> {
             fs::create_dir_all(&wheel_dir)
                 .context("Failed to create the target directory for the source distribution")?;
 
-            source_distribution(&wheel_dir, &metadata21, &manifest_path)
-                .context("Failed to build source distribution")?;
+            source_distribution(
+                &wheel_dir,
+                &metadata21,
+                &manifest_path,
+                pyproject.sdist_include(),
+            )
+            .context("Failed to build source distribution")?;
         }
         Opt::PEP517(subcommand) => pep517(subcommand)?,
     }

src/source_distribution.rs

@@ -42,6 +42,7 @@ pub fn source_distribution(
     wheel_dir: impl AsRef<Path>,
     metadata21: &Metadata21,
     manifest_path: impl AsRef<Path>,
+    sdist_include: Option<&Vec<String>>,
 ) -> Result<PathBuf, Error> {
     let output = Command::new("cargo")
         .args(&["package", "--list", "--allow-dirty", "--manifest-path"])
@@ -89,6 +90,18 @@ pub fn source_distribution(
         writer.add_file(target, source)?;
     }
 
+    if let Some(include_targets) = sdist_include {
+        for pattern in include_targets {
+            println!("📦 Including files matching \"{}\"", pattern);
+            for source in glob::glob(pattern)
+                .expect("No files found for pattern")
+                .filter_map(Result::ok)
+            {
+                writer.add_file(manifest_dir.join(&source).to_path_buf(), source)?;
+            }
+        }
+    }
+
     writer.add_bytes("PKG-INFO", metadata21.to_file_contents().as_bytes())?;
 
     let source_distribution_path = writer.finish()?;
@@ -109,11 +122,32 @@ pub struct BuildSystem {
     build_backend: String,
 }
 
+/// The `[tool]` section of a pyproject.toml
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub struct Tool {
+    maturin: Option<ToolMaturin>,
+}
+
+/// The `[tool.maturin]` section of a pyproject.toml
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub struct ToolMaturin {
+    sdist_include: Option<Vec<String>>,
+}
+
 /// A pyproject.toml as specified in PEP 517
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "kebab-case")]
 pub struct PyProjectToml {
     build_system: BuildSystem,
+    tool: Option<Tool>,
+}
+
+impl PyProjectToml {
+    pub fn sdist_include(&self) -> Option<&Vec<String>> {
+        self.tool.as_ref()?.maturin.as_ref()?.sdist_include.as_ref()
+    }
 }
 
 /// Returns the contents of a pyproject.toml with a `[build-system]` entry or an error