diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 75a90fb13..7d21c57a3 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -16,6 +16,8 @@ jobs: with: python-version: 3.8 - name: Build Docker container - run: docker build --cache-from konstin2/maturin -t maturin . + run: | + docker pull konstin2/maturin + docker build --cache-from konstin2/maturin -t maturin . - name: Test the Docker container run: ./test-dockerfile.sh diff --git a/Cargo.toml b/Cargo.toml index cef3ed780..1b35071c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,8 +72,7 @@ scroll = "0.10.2" indoc = "1.0.2" [features] -default = ["auditwheel", "log", "upload", "rustls", "human-panic"] -auditwheel = [] +default = ["log", "upload", "rustls", "human-panic"] upload = ["reqwest", "rpassword", "configparser", "dirs"] password-storage = ["upload", "keyring"] log = ["pretty_env_logger"] diff --git a/src/auditwheel/audit.rs b/src/auditwheel/audit.rs new file mode 100644 index 000000000..4b3034e95 --- /dev/null +++ b/src/auditwheel/audit.rs @@ -0,0 +1,286 @@ +use crate::Manylinux; +use crate::Target; +use anyhow::Result; +use fs_err::File; +use goblin::elf::{sym::STT_FUNC, Elf}; +use goblin::strtab::Strtab; +use regex::Regex; +use scroll::Pread; +use std::collections::{HashMap, HashSet}; +use std::io; +use std::io::Read; +use std::path::Path; +use thiserror::Error; + +use super::policy::{Policy, POLICIES}; + +/// Error raised during auditing an elf file for manylinux compatibility +#[derive(Error, Debug)] +#[error("Ensuring manylinux compliance failed")] +pub enum AuditWheelError { + /// The wheel couldn't be read + #[error("Failed to read the wheel")] + IOError(#[source] io::Error), + /// Reexports elfkit parsing errors + #[error("Goblin failed to parse the elf file")] + GoblinError(#[source] goblin::error::Error), + /// The elf file isn't manylinux compatible. Contains the list of offending + /// libraries. + #[error( + "Your library links libpython ({0}), which libraries must not do. Have you forgotten to activate the extension-module feature?", + )] + LinksLibPythonError(String), + /// The elf file isn't manylinux compatible. Contains the list of offending + /// libraries. + #[error( + "Your library is not manylinux compliant because it links the following forbidden libraries: {0:?}", + )] + ManylinuxValidationError(Vec), + /// The elf file isn't manylinux compaible. Contains unsupported architecture + #[error( + "Your library is not manylinux compliant because it has unsupported architecture: {0}" + )] + UnsupportedArchitecture(String), +} + +/// Structure of "version needed" entries is documented in +/// https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-PDA/LSB-PDA.junk/symversion.html +#[derive(Clone, Copy, Debug, Pread)] +#[repr(C)] +struct GnuVersionNeed { + /// Version of structure. This value is currently set to 1, + /// and will be reset if the versioning implementation is incompatibly altered. + version: u16, + /// Number of associated verneed array entries. + cnt: u16, + /// Offset to the file name string in the section header, in bytes. + file: u32, + /// Offset to a corresponding entry in the vernaux array, in bytes. + aux: u32, + /// Offset to the next verneed entry, in bytes. + next: u32, +} + +/// Version Needed Auxiliary Entries +#[derive(Clone, Copy, Debug, Pread)] +#[repr(C)] +struct GnuVersionNeedAux { + /// Dependency name hash value (ELF hash function). + hash: u32, + /// Dependency information flag bitmask. + flags: u16, + /// Object file version identifier used in the .gnu.version symbol version array. + /// Bit number 15 controls whether or not the object is hidden; if this bit is set, + /// the object cannot be used and the static linker will ignore the symbol's presence in the object. + other: u16, + /// Offset to the dependency name string in the section header, in bytes. + name: u32, + /// Offset to the next vernaux entry, in bytes. + next: u32, +} + +#[derive(Clone, Debug)] +struct VersionedLibrary { + /// library name + name: String, + /// versions needed + versions: HashSet, +} + +/// Find required dynamic linked libraries with version information +fn find_versioned_libraries( + elf: &Elf, + buffer: &[u8], +) -> Result, AuditWheelError> { + let mut symbols = Vec::new(); + let section = elf + .section_headers + .iter() + .find(|h| &elf.shdr_strtab[h.sh_name] == ".gnu.version_r"); + if let Some(section) = section { + let linked_section = &elf.section_headers[section.sh_link as usize]; + linked_section + .check_size(buffer.len()) + .map_err(AuditWheelError::GoblinError)?; + let strtab = Strtab::parse( + buffer, + linked_section.sh_offset as usize, + linked_section.sh_size as usize, + 0x0, + ) + .map_err(AuditWheelError::GoblinError)?; + let num_versions = section.sh_info as usize; + let mut offset = section.sh_offset as usize; + for _ in 0..num_versions { + let ver = buffer + .gread::(&mut offset) + .map_err(goblin::error::Error::Scroll) + .map_err(AuditWheelError::GoblinError)?; + let mut versions = HashSet::new(); + for _ in 0..ver.cnt { + let ver_aux = buffer + .gread::(&mut offset) + .map_err(goblin::error::Error::Scroll) + .map_err(AuditWheelError::GoblinError)?; + let aux_name = &strtab[ver_aux.name as usize]; + versions.insert(aux_name.to_string()); + } + let name = &strtab[ver.file as usize]; + // Skip dynamic linker/loader + if name.starts_with("ld-linux") || name == "ld64.so.2" || name == "ld64.so.1" { + continue; + } + symbols.push(VersionedLibrary { + name: name.to_string(), + versions, + }); + } + } + Ok(symbols) +} + +/// Find incompliant symbols from symbol versions +fn find_incompliant_symbols( + elf: &Elf, + symbol_versions: &[String], +) -> Result, AuditWheelError> { + let mut symbols = Vec::new(); + let strtab = &elf.strtab; + for sym in &elf.syms { + if sym.st_type() == STT_FUNC { + let name = strtab + .get(sym.st_name) + .unwrap_or(Ok("BAD NAME")) + .map_err(AuditWheelError::GoblinError)?; + for symbol_version in symbol_versions { + if name.ends_with(&format!("@{}", symbol_version)) { + symbols.push(name.to_string()); + } + } + } + } + Ok(symbols) +} + +fn policy_is_satisfied( + policy: &Policy, + elf: &Elf, + arch: &str, + deps: &[String], + versioned_libraries: &[VersionedLibrary], +) -> Result { + let arch_versions = &policy + .symbol_versions + .get(arch) + .ok_or_else(|| AuditWheelError::UnsupportedArchitecture(arch.to_string()))?; + let mut offenders = HashSet::new(); + for dep in deps { + // Skip dynamic linker/loader + if dep.starts_with("ld-linux") || dep == "ld64.so.2" || dep == "ld64.so.1" { + continue; + } + if !policy.lib_whitelist.contains(dep) { + offenders.insert(dep.clone()); + } + } + for library in versioned_libraries { + if !policy.lib_whitelist.contains(&library.name) { + offenders.insert(library.name.clone()); + continue; + } + let mut versions: HashMap> = HashMap::new(); + for v in &library.versions { + let mut parts = v.splitn(2, '_'); + let name = parts.next().unwrap(); + let version = parts.next().unwrap(); + versions + .entry(name.to_string()) + .or_default() + .insert(version.to_string()); + } + for (name, versions_needed) in versions.iter() { + let versions_allowed = &arch_versions[name]; + if !versions_needed.is_subset(versions_allowed) { + let offending_versions: Vec<&str> = versions_needed + .difference(versions_allowed) + .map(|v| v.as_ref()) + .collect(); + let offending_symbol_versions: Vec = offending_versions + .iter() + .map(|v| format!("{}_{}", name, v)) + .collect(); + let offending_symbols = find_incompliant_symbols(&elf, &offending_symbol_versions)?; + let offender = if offending_symbols.is_empty() { + format!( + "{} offending versions: {}", + library.name, + offending_symbol_versions.join(", ") + ) + } else { + format!( + "{} offending symbols: {}", + library.name, + offending_symbols.join(", ") + ) + }; + offenders.insert(offender); + } + } + } + // Checks if we can give a more helpful error message + let is_libpython = Regex::new(r"^libpython3\.\d+\.so\.\d+\.\d+$").unwrap(); + let offenders: Vec = offenders.into_iter().collect(); + match offenders.as_slice() { + [] => Ok(policy.priority), + [lib] if is_libpython.is_match(lib) => { + Err(AuditWheelError::LinksLibPythonError(lib.clone())) + } + offenders => Err(AuditWheelError::ManylinuxValidationError( + offenders.to_vec(), + )), + } +} + +/// An reimplementation of auditwheel, which checks elf files for +/// manylinux compliance. Returns an error for non compliant elf files +/// +/// Checks for the libraries marked as NEEDED and symbol versions +pub fn auditwheel_rs( + path: &Path, + target: &Target, + manylinux: &Manylinux, +) -> Result, AuditWheelError> { + if !target.is_linux() || matches!(manylinux, Manylinux::Off) { + return Ok(None); + } + let arch = target.target_arch().to_string(); + let mut file = File::open(path).map_err(AuditWheelError::IOError)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer) + .map_err(AuditWheelError::IOError)?; + let elf = Elf::parse(&buffer).map_err(AuditWheelError::GoblinError)?; + // This returns essentially the same as ldd + let deps: Vec = elf.libraries.iter().map(ToString::to_string).collect(); + let versioned_libraries = find_versioned_libraries(&elf, &buffer)?; + + let policy = POLICIES + .iter() + .find(|p| p.name == manylinux.to_string()) + .unwrap(); + let mut match_policies = Vec::new(); + match_policies.push(policy_is_satisfied( + policy, + &elf, + &arch, + &deps, + &versioned_libraries, + )?); + for policy in policy.higher_priority_policies() { + if let Ok(priority) = policy_is_satisfied(policy, &elf, &arch, &deps, &versioned_libraries) + { + match_policies.push(priority); + } + } + let max_priority = match_policies.into_iter().max().unwrap_or_default(); + Ok(Policy::from_priority(max_priority)) +} diff --git a/src/auditwheel/mod.rs b/src/auditwheel/mod.rs index 2f7d908f3..6a54afdbc 100644 --- a/src/auditwheel/mod.rs +++ b/src/auditwheel/mod.rs @@ -1,265 +1,4 @@ -mod policy; +mod audit; +pub mod policy; -use crate::Manylinux; -use crate::Target; -use anyhow::Result; -use fs_err::File; -use goblin::elf::{sym::STT_FUNC, Elf}; -use goblin::strtab::Strtab; -use regex::Regex; -use scroll::Pread; -use std::collections::{HashMap, HashSet}; -use std::io; -use std::io::Read; -use std::path::Path; -use thiserror::Error; - -use policy::POLICIES; - -/// Error raised during auditing an elf file for manylinux compatibility -#[derive(Error, Debug)] -#[error("Ensuring manylinux compliance failed")] -pub enum AuditWheelError { - /// The wheel couldn't be read - #[error("Failed to read the wheel")] - IOError(#[source] io::Error), - /// Reexports elfkit parsing errors - #[error("Goblin failed to parse the elf file")] - GoblinError(#[source] goblin::error::Error), - /// The elf file isn't manylinux compatible. Contains the list of offending - /// libraries. - #[error( - "Your library links libpython ({0}), which libraries must not do. Have you forgotten to activate the extension-module feature?", - )] - LinksLibPythonError(String), - /// The elf file isn't manylinux compatible. Contains the list of offending - /// libraries. - #[error( - "Your library is not manylinux compliant because it links the following forbidden libraries: {0:?}", - )] - ManylinuxValidationError(Vec), - /// The elf file isn't manylinux compaible. Contains unsupported architecture - #[error( - "Your library is not manylinux compliant because it has unsupported architecture: {0}" - )] - UnsupportedArchitecture(String), -} - -/// Structure of "version needed" entries is documented in -/// https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-PDA/LSB-PDA.junk/symversion.html -#[derive(Clone, Copy, Debug, Pread)] -#[repr(C)] -struct GnuVersionNeed { - /// Version of structure. This value is currently set to 1, - /// and will be reset if the versioning implementation is incompatibly altered. - version: u16, - /// Number of associated verneed array entries. - cnt: u16, - /// Offset to the file name string in the section header, in bytes. - file: u32, - /// Offset to a corresponding entry in the vernaux array, in bytes. - aux: u32, - /// Offset to the next verneed entry, in bytes. - next: u32, -} - -/// Version Needed Auxiliary Entries -#[derive(Clone, Copy, Debug, Pread)] -#[repr(C)] -struct GnuVersionNeedAux { - /// Dependency name hash value (ELF hash function). - hash: u32, - /// Dependency information flag bitmask. - flags: u16, - /// Object file version identifier used in the .gnu.version symbol version array. - /// Bit number 15 controls whether or not the object is hidden; if this bit is set, - /// the object cannot be used and the static linker will ignore the symbol's presence in the object. - other: u16, - /// Offset to the dependency name string in the section header, in bytes. - name: u32, - /// Offset to the next vernaux entry, in bytes. - next: u32, -} - -#[derive(Clone, Debug)] -struct VersionedLibrary { - /// library name - name: String, - /// versions needed - versions: HashSet, -} - -/// Find required dynamic linked libraries with version information -fn find_versioned_libraries( - elf: &Elf, - buffer: &[u8], -) -> Result, AuditWheelError> { - let mut symbols = Vec::new(); - let section = elf - .section_headers - .iter() - .find(|h| &elf.shdr_strtab[h.sh_name] == ".gnu.version_r"); - if let Some(section) = section { - let linked_section = &elf.section_headers[section.sh_link as usize]; - linked_section - .check_size(buffer.len()) - .map_err(AuditWheelError::GoblinError)?; - let strtab = Strtab::parse( - buffer, - linked_section.sh_offset as usize, - linked_section.sh_size as usize, - 0x0, - ) - .map_err(AuditWheelError::GoblinError)?; - let num_versions = section.sh_info as usize; - let mut offset = section.sh_offset as usize; - for _ in 0..num_versions { - let ver = buffer - .gread::(&mut offset) - .map_err(goblin::error::Error::Scroll) - .map_err(AuditWheelError::GoblinError)?; - let mut versions = HashSet::new(); - for _ in 0..ver.cnt { - let ver_aux = buffer - .gread::(&mut offset) - .map_err(goblin::error::Error::Scroll) - .map_err(AuditWheelError::GoblinError)?; - let aux_name = &strtab[ver_aux.name as usize]; - versions.insert(aux_name.to_string()); - } - let name = &strtab[ver.file as usize]; - // Skip dynamic linker/loader - if name.starts_with("ld-linux") || name == "ld64.so.2" || name == "ld64.so.1" { - continue; - } - symbols.push(VersionedLibrary { - name: name.to_string(), - versions, - }); - } - } - Ok(symbols) -} - -/// Find incompliant symbols from symbol versions -fn find_incompliant_symbols( - elf: &Elf, - symbol_versions: &[String], -) -> Result, AuditWheelError> { - let mut symbols = Vec::new(); - let strtab = &elf.strtab; - for sym in &elf.syms { - if sym.st_type() == STT_FUNC { - let name = strtab - .get(sym.st_name) - .unwrap_or(Ok("BAD NAME")) - .map_err(AuditWheelError::GoblinError)?; - for symbol_version in symbol_versions { - if name.ends_with(&format!("@{}", symbol_version)) { - symbols.push(name.to_string()); - } - } - } - } - Ok(symbols) -} - -/// An (incomplete) reimplementation of auditwheel, which checks elf files for -/// manylinux compliance. Returns an error for non compliant elf files -/// -/// Only checks for the libraries marked as NEEDED, but not for symbol versions -/// (e.g. requiring a too recent glibc isn't caught). -pub fn auditwheel_rs( - path: &Path, - target: &Target, - manylinux: &Manylinux, -) -> Result<(), AuditWheelError> { - if !target.is_linux() || matches!(manylinux, Manylinux::Off) { - return Ok(()); - } - let policy = POLICIES - .iter() - .find(|p| p.name == manylinux.to_string()) - .unwrap(); - let arch = target.target_arch().to_string(); - let arch_versions = &policy - .symbol_versions - .get(&arch) - .ok_or(AuditWheelError::UnsupportedArchitecture(arch))?; - let mut file = File::open(path).map_err(AuditWheelError::IOError)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer) - .map_err(AuditWheelError::IOError)?; - let elf = Elf::parse(&buffer).map_err(AuditWheelError::GoblinError)?; - // This returns essentially the same as ldd - let deps: Vec = elf.libraries.iter().map(ToString::to_string).collect(); - let versioned_libraries = find_versioned_libraries(&elf, &buffer)?; - - let mut offenders = HashSet::new(); - for dep in deps { - // Skip dynamic linker/loader - if dep.starts_with("ld-linux") || dep == "ld64.so.2" || dep == "ld64.so.1" { - continue; - } - if !policy.lib_whitelist.contains(&dep) { - offenders.insert(dep); - } - } - for library in versioned_libraries { - if !policy.lib_whitelist.contains(&library.name) { - offenders.insert(library.name.clone()); - continue; - } - let mut versions: HashMap> = HashMap::new(); - for v in &library.versions { - let mut parts = v.splitn(2, '_'); - let name = parts.next().unwrap(); - let version = parts.next().unwrap(); - versions - .entry(name.to_string()) - .or_default() - .insert(version.to_string()); - } - for (name, versions_needed) in versions.iter() { - let versions_allowed = &arch_versions[name]; - if !versions_needed.is_subset(versions_allowed) { - let offending_versions: Vec<&str> = versions_needed - .difference(versions_allowed) - .map(|v| v.as_ref()) - .collect(); - let offending_symbol_versions: Vec = offending_versions - .iter() - .map(|v| format!("{}_{}", name, v)) - .collect(); - let offending_symbols = find_incompliant_symbols(&elf, &offending_symbol_versions)?; - let offender = if offending_symbols.is_empty() { - format!( - "{} offending versions: {}", - library.name, - offending_symbol_versions.join(", ") - ) - } else { - format!( - "{} offending symbols: {}", - library.name, - offending_symbols.join(", ") - ) - }; - offenders.insert(offender); - } - } - } - - // Checks if we can give a more helpful error message - let is_libpython = Regex::new(r"^libpython3\.\d+\.so\.\d+\.\d+$").unwrap(); - let offenders: Vec = offenders.into_iter().collect(); - match offenders.as_slice() { - [] => Ok(()), - [lib] if is_libpython.is_match(lib) => { - Err(AuditWheelError::LinksLibPythonError(lib.clone())) - } - offenders => Err(AuditWheelError::ManylinuxValidationError( - offenders.to_vec(), - )), - } -} +pub use self::audit::*; diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index 5cbda3918..b4af20a4f 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -1,5 +1,7 @@ +use crate::Manylinux; use once_cell::sync::Lazy; use serde::Deserialize; +use std::cmp::{Ordering, PartialOrd}; use std::collections::{HashMap, HashSet}; pub static POLICIES: Lazy> = Lazy::new(|| { @@ -9,7 +11,7 @@ pub static POLICIES: Lazy> = Lazy::new(|| { }); /// Manylinux policy -#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] pub struct Policy { /// manylinux platform tag name pub name: String, @@ -22,9 +24,44 @@ pub struct Policy { pub lib_whitelist: HashSet, } +impl Default for Policy { + fn default() -> Self { + // defaults to linux + Policy::from_priority(0).unwrap() + } +} + +impl PartialOrd for Policy { + fn partial_cmp(&self, other: &Self) -> Option { + self.priority.partial_cmp(&other.priority) + } +} + +impl Policy { + /// Get highest priority policy than self + pub fn higher_priority_policies(&self) -> impl Iterator { + POLICIES.iter().filter(move |p| p.priority > self.priority) + } + + /// Get manylinux platform tag from this policy + pub fn manylinux_tag(&self) -> Manylinux { + self.name.parse().expect("Manylinux variants is incomplete") + } + + /// Get policy by it's manylinux platform tag name + pub fn from_name(name: &str) -> Option { + POLICIES.iter().find(|p| p.name == name).cloned() + } + + /// Get policy by it's priority + pub fn from_priority(priority: i64) -> Option { + POLICIES.iter().find(|p| p.priority == priority).cloned() + } +} + #[cfg(test)] mod test { - use super::POLICIES; + use super::{Policy, POLICIES}; #[test] fn test_load_policy() { @@ -41,4 +78,28 @@ mod test { assert!(cxxabi.contains(*version)); } } + + #[test] + fn test_policy_manylinux_tag() { + for policy in POLICIES.iter() { + let _tag = policy.manylinux_tag(); + } + } + + #[test] + fn test_policy_from_name() { + use crate::Manylinux; + + let tags = &[ + Manylinux::Manylinux1, + Manylinux::Manylinux2010, + Manylinux::Manylinux2014, + Manylinux::Manylinux_2_24, + Manylinux::Off, + ]; + for manylinux in tags { + let policy = Policy::from_name(&manylinux.to_string()); + assert!(policy.is_some()); + } + } } diff --git a/src/build_context.rs b/src/build_context.rs index d723e5c6a..b0f3a59bd 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1,5 +1,5 @@ -#[cfg(feature = "auditwheel")] use crate::auditwheel::auditwheel_rs; +use crate::auditwheel::policy::Policy; use crate::compile; use crate::compile::warn_missing_py_init; use crate::module_writer::write_python_part; @@ -133,13 +133,12 @@ impl BuildContext { .context("Failed to create the target directory for the wheels")?; let wheels = match &self.bridge { - BridgeModel::Cffi => vec![(self.build_cffi_wheel()?, "py3".to_string())], - BridgeModel::Bin => vec![(self.build_bin_wheel()?, "py3".to_string())], + BridgeModel::Cffi => self.build_cffi_wheel()?, + BridgeModel::Bin => self.build_bin_wheel()?, BridgeModel::Bindings(_) => self.build_binding_wheels()?, - BridgeModel::BindingsAbi3(major, minor) => vec![( - self.build_binding_wheel_abi3(*major, *minor)?, - format!("cp{}{}", major, minor), - )], + BridgeModel::BindingsAbi3(major, minor) => { + self.build_binding_wheel_abi3(*major, *minor)? + } }; Ok(wheels) @@ -166,16 +165,33 @@ impl BuildContext { } } - /// For abi3 we only need to build a single wheel and we don't even need a python interpreter - /// for it - pub fn build_binding_wheel_abi3(&self, major: u8, min_minor: u8) -> Result { - // On windows, we have picked an interpreter to set the location of python.lib, - // otherwise it's none - let artifact = self.compile_cdylib(self.interpreter.get(0), Some(&self.module_name))?; + fn auditwheel( + &self, + python_interpreter: Option<&PythonInterpreter>, + artifact: &Path, + manylinux: &Manylinux, + ) -> Result> { + if !self.skip_auditwheel { + let target = python_interpreter + .map(|x| &x.target) + .unwrap_or(&self.target); + + let policy = auditwheel_rs(&artifact, target, manylinux) + .context(format!("Failed to ensure {} compliance", manylinux))?; + Ok(policy) + } else { + Ok(None) + } + } - let platform = self - .target - .get_platform_tag(&self.manylinux, self.universal2); + fn write_binding_wheel_abi3( + &self, + artifact: &Path, + manylinux: &Manylinux, + major: u8, + min_minor: u8, + ) -> Result<(PathBuf, String)> { + let platform = self.target.get_platform_tag(&manylinux, self.universal2); let tag = format!("cp{}{}-abi3-{}", major, min_minor, platform); let mut writer = WheelWriter::new( @@ -190,7 +206,7 @@ impl BuildContext { &mut writer, &self.project_layout, &self.module_name, - &artifact, + artifact, None, &self.target, false, @@ -198,6 +214,24 @@ impl BuildContext { .context("Failed to add the files to the wheel")?; let wheel_path = writer.finish()?; + Ok((wheel_path, format!("cp{}{}", major, min_minor))) + } + + /// For abi3 we only need to build a single wheel and we don't even need a python interpreter + /// for it + pub fn build_binding_wheel_abi3( + &self, + major: u8, + min_minor: u8, + ) -> Result> { + let mut wheels = Vec::new(); + // On windows, we have picked an interpreter to set the location of python.lib, + // otherwise it's none + let python_interpreter = self.interpreter.get(0); + let artifact = self.compile_cdylib(python_interpreter, Some(&self.module_name))?; + let policy = self.auditwheel(python_interpreter, &artifact, &self.manylinux)?; + let (wheel_path, tag) = + self.write_binding_wheel_abi3(&artifact, &self.manylinux, major, min_minor)?; println!( "📦 Built wheel for abi3 Python ≥ {}.{} to {}", @@ -205,8 +239,63 @@ impl BuildContext { min_minor, wheel_path.display() ); + wheels.push((wheel_path, tag)); + + if let Some(policy) = policy { + if policy > Policy::from_name(&self.manylinux.to_string()).unwrap() { + let manylinux = policy.manylinux_tag(); + if self + .auditwheel(python_interpreter, &artifact, &manylinux) + .is_ok() + { + println!( + "📦 Wheel is eligible for a higher priority tag. You requested {} but I have found this wheel is eligible for {}", + &self.manylinux, + manylinux, + ); + let (wheel_path, tag) = + self.write_binding_wheel_abi3(&artifact, &manylinux, major, min_minor)?; + println!("📦 Fixed-up wheel written to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + } + } + } + + Ok(wheels) + } + + fn write_binding_wheel( + &self, + python_interpreter: &PythonInterpreter, + artifact: &Path, + manylinux: &Manylinux, + ) -> Result<(PathBuf, String)> { + let tag = python_interpreter.get_tag(manylinux, self.universal2); + + let mut writer = WheelWriter::new( + &tag, + &self.out, + &self.metadata21, + &self.scripts, + &[tag.clone()], + )?; - Ok(wheel_path) + write_bindings_module( + &mut writer, + &self.project_layout, + &self.module_name, + &artifact, + Some(&python_interpreter), + &self.target, + false, + ) + .context("Failed to add the files to the wheel")?; + + let wheel_path = writer.finish()?; + Ok(( + wheel_path, + format!("cp{}{}", python_interpreter.major, python_interpreter.minor), + )) } /// Builds wheels for a Cargo project for all given python versions. @@ -221,30 +310,9 @@ impl BuildContext { for python_interpreter in &self.interpreter { let artifact = self.compile_cdylib(Some(&python_interpreter), Some(&self.module_name))?; - - let tag = python_interpreter.get_tag(&self.manylinux, self.universal2); - - let mut writer = WheelWriter::new( - &tag, - &self.out, - &self.metadata21, - &self.scripts, - &[tag.clone()], - )?; - - write_bindings_module( - &mut writer, - &self.project_layout, - &self.module_name, - &artifact, - Some(&python_interpreter), - &self.target, - false, - ) - .context("Failed to add the files to the wheel")?; - - let wheel_path = writer.finish()?; - + let policy = self.auditwheel(Some(&python_interpreter), &artifact, &self.manylinux)?; + let (wheel_path, tag) = + self.write_binding_wheel(python_interpreter, &artifact, &self.manylinux)?; println!( "📦 Built wheel for {} {}.{}{} to {}", python_interpreter.interpreter_kind, @@ -254,10 +322,27 @@ impl BuildContext { wheel_path.display() ); - wheels.push(( - wheel_path, - format!("cp{}{}", python_interpreter.major, python_interpreter.minor), - )); + wheels.push((wheel_path, tag)); + + if let Some(policy) = policy { + if policy > Policy::from_name(&self.manylinux.to_string()).unwrap() { + let manylinux = policy.manylinux_tag(); + if self + .auditwheel(Some(&python_interpreter), &artifact, &manylinux) + .is_ok() + { + println!( + "📦 Wheel is eligible for a higher priority tag. You requested {} but I have found this wheel is eligible for {}", + &self.manylinux, + manylinux, + ); + let (wheel_path, tag) = + self.write_binding_wheel(python_interpreter, &artifact, &manylinux)?; + println!("📦 Fixed-up wheel written to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + } + } + } } Ok(wheels) @@ -282,15 +367,6 @@ impl BuildContext { in the lib section of your Cargo.toml?", ) })?; - #[cfg(feature = "auditwheel")] - if !self.skip_auditwheel { - let target = python_interpreter - .map(|x| &x.target) - .unwrap_or(&self.target); - - auditwheel_rs(&artifact, target, &self.manylinux) - .context(format!("Failed to ensure {} compliance", self.manylinux))?; - } if let Some(module_name) = module_name { warn_missing_py_init(&artifact, module_name) @@ -300,13 +376,12 @@ impl BuildContext { Ok(artifact) } - /// Builds a wheel with cffi bindings - pub fn build_cffi_wheel(&self) -> Result { - let artifact = self.compile_cdylib(None, None)?; - - let (tag, tags) = self - .target - .get_universal_tags(&self.manylinux, self.universal2); + fn write_cffi_wheel( + &self, + artifact: &Path, + manylinux: &Manylinux, + ) -> Result<(PathBuf, String)> { + let (tag, tags) = self.target.get_universal_tags(manylinux, self.universal2); let mut builder = WheelWriter::new(&tag, &self.out, &self.metadata21, &self.scripts, &tags)?; @@ -322,31 +397,40 @@ impl BuildContext { )?; let wheel_path = builder.finish()?; - - println!("📦 Built wheel to {}", wheel_path.display()); - - Ok(wheel_path) + Ok((wheel_path, "py3".to_string())) } - /// Builds a wheel that contains a binary - /// - /// Runs [auditwheel_rs()] if not deactivated - pub fn build_bin_wheel(&self) -> Result { - let artifacts = compile(&self, None, &self.bridge) - .context("Failed to build a native library through cargo")?; + /// Builds a wheel with cffi bindings + pub fn build_cffi_wheel(&self) -> Result> { + let mut wheels = Vec::new(); + let artifact = self.compile_cdylib(None, None)?; + let (wheel_path, tag) = self.write_cffi_wheel(&artifact, &self.manylinux)?; + let policy = self.auditwheel(None, &artifact, &self.manylinux)?; - let artifact = artifacts - .get("bin") - .cloned() - .ok_or_else(|| anyhow!("Cargo didn't build a binary"))?; + println!("📦 Built wheel to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + + if let Some(policy) = policy { + if policy > Policy::from_name(&self.manylinux.to_string()).unwrap() { + let manylinux = policy.manylinux_tag(); + if self.auditwheel(None, &artifact, &manylinux).is_ok() { + println!( + "📦 Wheel is eligible for a higher priority tag. You requested {} but I have found this wheel is eligible for {}", + &self.manylinux, + manylinux, + ); + let (wheel_path, tag) = self.write_cffi_wheel(&artifact, &manylinux)?; + println!("📦 Fixed-up wheel written to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + } + } + } - #[cfg(feature = "auditwheel")] - auditwheel_rs(&artifact, &self.target, &self.manylinux) - .context(format!("Failed to ensure {} compliance", self.manylinux))?; + Ok(wheels) + } - let (tag, tags) = self - .target - .get_universal_tags(&self.manylinux, self.universal2); + fn write_bin_wheel(&self, artifact: &Path, manylinux: &Manylinux) -> Result<(PathBuf, String)> { + let (tag, tags) = self.target.get_universal_tags(manylinux, self.universal2); if !self.scripts.is_empty() { bail!("Defining entrypoints and working with a binary doesn't mix well"); @@ -371,9 +455,44 @@ impl BuildContext { write_bin(&mut builder, &artifact, &self.metadata21, bin_name)?; let wheel_path = builder.finish()?; + Ok((wheel_path, "py3".to_string())) + } + /// Builds a wheel that contains a binary + /// + /// Runs [auditwheel_rs()] if not deactivated + pub fn build_bin_wheel(&self) -> Result> { + let mut wheels = Vec::new(); + let artifacts = compile(&self, None, &self.bridge) + .context("Failed to build a native library through cargo")?; + + let artifact = artifacts + .get("bin") + .cloned() + .ok_or_else(|| anyhow!("Cargo didn't build a binary"))?; + + let policy = self.auditwheel(None, &artifact, &self.manylinux)?; + + let (wheel_path, tag) = self.write_bin_wheel(&artifact, &self.manylinux)?; println!("📦 Built wheel to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + + if let Some(policy) = policy { + if policy > Policy::from_name(&self.manylinux.to_string()).unwrap() { + let manylinux = policy.manylinux_tag(); + if self.auditwheel(None, &artifact, &manylinux).is_ok() { + println!( + "📦 Wheel is eligible for a higher priority tag. You requested {} but I have found this wheel is eligible for {}", + &self.manylinux, + manylinux, + ); + let (wheel_path, tag) = self.write_bin_wheel(&artifact, &manylinux)?; + println!("📦 Fixed-up wheel written to {}", wheel_path.display()); + wheels.push((wheel_path, tag)); + } + } + } - Ok(wheel_path) + Ok(wheels) } } diff --git a/src/lib.rs b/src/lib.rs index d777a718f..26a751a93 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,6 @@ #![deny(missing_docs)] -#[cfg(feature = "auditwheel")] pub use crate::auditwheel::{auditwheel_rs, AuditWheelError}; pub use crate::build_context::BridgeModel; pub use crate::build_context::BuildContext; @@ -48,7 +47,6 @@ pub use { crate::upload::{upload, UploadError}, }; -#[cfg(feature = "auditwheel")] mod auditwheel; mod build_context; mod build_options; diff --git a/src/target.rs b/src/target.rs index 2028ee0ce..d1bd596ee 100644 --- a/src/target.rs +++ b/src/target.rs @@ -21,6 +21,8 @@ enum OS { /// Decides how to handle manylinux compliance #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub enum Manylinux { + /// Use the manylinux1 tag + Manylinux1, /// Use the manylinux2010 tag Manylinux2010, /// Use the manylinux2014 tag @@ -35,6 +37,7 @@ pub enum Manylinux { impl fmt::Display for Manylinux { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { + Manylinux::Manylinux1 => write!(f, "manylinux1"), Manylinux::Manylinux2010 => write!(f, "manylinux2010"), Manylinux::Manylinux2014 => write!(f, "manylinux2014"), Manylinux::Manylinux_2_24 => write!(f, "manylinux_2_24"), @@ -48,10 +51,11 @@ impl FromStr for Manylinux { fn from_str(value: &str) -> Result { match value { - "2010" => Ok(Manylinux::Manylinux2010), - "2014" => Ok(Manylinux::Manylinux2014), - "2_24" => Ok(Manylinux::Manylinux_2_24), - "off" => Ok(Manylinux::Off), + "1" | "manylinux1" => Ok(Manylinux::Manylinux1), + "2010" | "manylinux2010" => Ok(Manylinux::Manylinux2010), + "2014" | "manylinux2014" => Ok(Manylinux::Manylinux2014), + "2_24" | "manylinux_2_24" => Ok(Manylinux::Manylinux_2_24), + "off" | "linux" => Ok(Manylinux::Off), _ => Err("Invalid value for the manylinux option"), } } diff --git a/test-dockerfile.sh b/test-dockerfile.sh index a92d0729b..77ec0558a 100755 --- a/test-dockerfile.sh +++ b/test-dockerfile.sh @@ -13,7 +13,7 @@ source venv-docker/bin/activate venv-docker/bin/pip install -U pip cffi -docker run --rm -v $(pwd)/test-crates/hello-world:/io maturin build --no-sdist -b bin +docker run -e RUST_BACKTRACE=1 --rm -v $(pwd)/test-crates/hello-world:/io maturin build --no-sdist -b bin venv-docker/bin/pip install hello-world --no-index --find-links test-crates/hello-world/target/wheels/ @@ -21,7 +21,7 @@ if [[ $(venv-docker/bin/python test-crates/hello-world/check_installed/check_ins exit 1 fi -docker run --rm -v $(pwd)/test-crates/cffi-pure:/io maturin build --no-sdist -b cffi +docker run -e RUST_BACKTRACE=1 --rm -v $(pwd)/test-crates/cffi-pure:/io maturin build --no-sdist -b cffi venv-docker/bin/pip install cffi-pure --no-index --find-links test-crates/cffi-pure/target/wheels/ @@ -29,7 +29,7 @@ if [[ $(venv-docker/bin/python test-crates/cffi-pure/check_installed/check_insta exit 1 fi -docker run --rm -v $(pwd)/test-crates/cffi-mixed:/io maturin build --no-sdist -b cffi +docker run -e RUST_BACKTRACE=1 --rm -v $(pwd)/test-crates/cffi-mixed:/io maturin build --no-sdist -b cffi venv-docker/bin/pip install cffi-mixed --no-index --find-links test-crates/cffi-mixed/target/wheels/ @@ -37,7 +37,7 @@ if [[ $(venv-docker/bin/python test-crates/cffi-mixed/check_installed/check_inst exit 1 fi -docker run --rm -v $(pwd)/test-crates/pyo3-pure:/io maturin build --no-sdist -i python3.8 +docker run -e RUST_BACKTRACE=1 --rm -v $(pwd)/test-crates/pyo3-pure:/io maturin build --no-sdist -i python3.8 venv-docker/bin/pip install pyo3-pure --no-index --find-links test-crates/pyo3-pure/target/wheels/ @@ -45,7 +45,7 @@ if [[ $(venv-docker/bin/python test-crates/pyo3-pure/check_installed/check_insta exit 1 fi -docker run --rm -v $(pwd)/test-crates/pyo3-mixed:/io maturin build --no-sdist -i python3.8 +docker run -e RUST_BACKTRACE=1 --rm -v $(pwd)/test-crates/pyo3-mixed:/io maturin build --no-sdist -i python3.8 venv-docker/bin/pip install pyo3-mixed --find-links test-crates/pyo3-mixed/target/wheels/