From d0afcc8da9f788081ad7a7785c51810201f9ca21 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 20 Jul 2021 11:58:53 +0800 Subject: [PATCH 1/2] Rename policy.json to manylinux-policy.json --- src/auditwheel/{policy.json => manylinux-policy.json} | 0 src/auditwheel/policy.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/auditwheel/{policy.json => manylinux-policy.json} (100%) diff --git a/src/auditwheel/policy.json b/src/auditwheel/manylinux-policy.json similarity index 100% rename from src/auditwheel/policy.json rename to src/auditwheel/manylinux-policy.json diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index 54188ae8d..26183243a 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -10,7 +10,7 @@ use std::fmt::{Display, Formatter}; /// priority to lowest pub static POLICIES: Lazy> = Lazy::new(|| { // https://github.com/pypa/auditwheel/blob/master/auditwheel/policy/policy.json - let mut policies: Vec = serde_json::from_slice(include_bytes!("policy.json")) + let mut policies: Vec = serde_json::from_slice(include_bytes!("manylinux-policy.json")) .expect("invalid manylinux policy.json file"); policies.sort_by_key(|policy| -policy.priority); policies From 8d5a82ea4345b040f664ee34384debbd3cbc9355 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 20 Jul 2021 12:03:31 +0800 Subject: [PATCH 2/2] Add auditwheel support for musllinux --- Changelog.md | 1 + src/auditwheel/audit.rs | 46 +++++++++++++++++--------- src/auditwheel/mod.rs | 2 +- src/auditwheel/musllinux-policy.json | 48 ++++++++++++++++++++++++++++ src/auditwheel/policy.rs | 46 +++++++++++++++++++------- 5 files changed, 115 insertions(+), 28 deletions(-) create mode 100644 src/auditwheel/musllinux-policy.json diff --git a/Changelog.md b/Changelog.md index 2f6a39589..f952eab76 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] * Add path option for Python source in [#584](https://github.com/PyO3/maturin/pull/584) +* Add auditwheel support for musllinux in [#597](https://github.com/PyO3/maturin/pull/597) * `[tool.maturin]` options from `pyproject.toml` will be used automatically in [#605](https://github.com/PyO3/maturin/pull/605) * Skip unavailable Python interpreters from pyenv in [#609](https://github.com/PyO3/maturin/pull/609) diff --git a/src/auditwheel/audit.rs b/src/auditwheel/audit.rs index e969bb566..fea6871d7 100644 --- a/src/auditwheel/audit.rs +++ b/src/auditwheel/audit.rs @@ -1,6 +1,6 @@ -use super::policy::{Policy, POLICIES}; +use super::policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; use crate::auditwheel::PlatformTag; -use crate::Target; +use crate::target::{Arch, Target}; use anyhow::Result; use fs_err::File; use goblin::elf::{sym::STT_FUNC, Elf}; @@ -253,18 +253,6 @@ pub fn auditwheel_rs( if !target.is_linux() || platform_tag == Some(PlatformTag::Linux) { return Ok(Policy::default()); } - if let Some(musl_tag @ PlatformTag::Musllinux { .. }) = platform_tag { - // TODO: add support for musllinux: https://github.com/pypa/auditwheel/issues/305 - eprintln!("⚠️ Warning: no auditwheel support for musllinux yet"); - // HACK: fake a musllinux policy - return Ok(Policy { - name: musl_tag.to_string(), - aliases: Vec::new(), - priority: 0, - symbol_versions: Default::default(), - lib_whitelist: Default::default(), - }); - } let arch = target.target_arch().to_string(); let mut file = File::open(path).map_err(AuditWheelError::IoError)?; let mut buffer = Vec::new(); @@ -276,8 +264,36 @@ pub fn auditwheel_rs( let versioned_libraries = find_versioned_libraries(&elf, &buffer)?; // Find the highest possible policy, if any + let platform_policies = match platform_tag { + Some(PlatformTag::Manylinux { .. }) | None => MANYLINUX_POLICIES.clone(), + Some(PlatformTag::Musllinux { .. }) => { + MUSLLINUX_POLICIES + .clone() + .into_iter() + .map(|mut policy| { + // Fixup musl libc lib_whitelist + if policy.lib_whitelist.remove("libc.so") { + let new_soname = match target.target_arch() { + Arch::Aarch64 => "libc.musl-aarch64.so.1", + Arch::Armv7L => "libc.musl-armv7.so.1", + Arch::Powerpc64Le => "libc.musl-ppc64le.so.1", + Arch::Powerpc64 => "", // musllinux doesn't support ppc64 + Arch::X86 => "libc.musl-x86.so.1", + Arch::X86_64 => "libc.musl-x86_64.so.1", + Arch::S390X => "libc.musl-s390x.so.1", + }; + if !new_soname.is_empty() { + policy.lib_whitelist.insert(new_soname.to_string()); + } + } + policy + }) + .collect() + } + Some(PlatformTag::Linux) => unreachable!(), + }; let mut highest_policy = None; - for policy in POLICIES.iter() { + for policy in platform_policies.iter() { let result = policy_is_satisfied(policy, &elf, &arch, &deps, &versioned_libraries); match result { Ok(_) => { diff --git a/src/auditwheel/mod.rs b/src/auditwheel/mod.rs index 78ad1d179..c704287a0 100644 --- a/src/auditwheel/mod.rs +++ b/src/auditwheel/mod.rs @@ -4,4 +4,4 @@ mod policy; pub use self::audit::*; pub use platform_tag::PlatformTag; -pub use policy::{Policy, POLICIES}; +pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; diff --git a/src/auditwheel/musllinux-policy.json b/src/auditwheel/musllinux-policy.json new file mode 100644 index 000000000..6cf8c3186 --- /dev/null +++ b/src/auditwheel/musllinux-policy.json @@ -0,0 +1,48 @@ +[ + {"name": "linux", + "aliases": [], + "priority": 0, + "symbol_versions": {}, + "lib_whitelist": [] + }, + {"name": "musllinux_1_1", + "aliases": [], + "priority": 100, + "symbol_versions": { + "i686": { + }, + "x86_64": { + }, + "aarch64": { + }, + "ppc64le": { + }, + "s390x": { + }, + "armv7l": { + } + }, + "lib_whitelist": [ + "libc.so" + ]}, + {"name": "musllinux_1_2", + "aliases": [], + "priority": 90, + "symbol_versions": { + "i686": { + }, + "x86_64": { + }, + "aarch64": { + }, + "ppc64le": { + }, + "s390x": { + }, + "armv7l": { + } + }, + "lib_whitelist": [ + "libc.so" + ]} +] diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index 26183243a..8bf04d1ad 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -8,14 +8,24 @@ use std::fmt::{Display, Formatter}; /// The policies (allowed symbols) for the different manylinux tags, sorted from highest /// priority to lowest -pub static POLICIES: Lazy> = Lazy::new(|| { - // https://github.com/pypa/auditwheel/blob/master/auditwheel/policy/policy.json +pub static MANYLINUX_POLICIES: Lazy> = Lazy::new(|| { + // https://github.com/pypa/auditwheel/blob/master/auditwheel/policy/manylinux-policy.json let mut policies: Vec = serde_json::from_slice(include_bytes!("manylinux-policy.json")) .expect("invalid manylinux policy.json file"); policies.sort_by_key(|policy| -policy.priority); policies }); +/// The policies (allowed symbols) for the different musllinux tags, sorted from highest +/// priority to lowest +pub static MUSLLINUX_POLICIES: Lazy> = Lazy::new(|| { + // https://github.com/pypa/auditwheel/blob/master/auditwheel/policy/musllinux-policy.json + let mut policies: Vec = serde_json::from_slice(include_bytes!("musllinux-policy.json")) + .expect("invalid musllinux policy.json file"); + policies.sort_by_key(|policy| -policy.priority); + policies +}); + /// Manylinux policy #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct Policy { @@ -35,7 +45,7 @@ pub struct Policy { impl Default for Policy { fn default() -> Self { // defaults to linux - Policy::from_priority(0).unwrap() + Policy::from_name("linux").unwrap() } } @@ -62,7 +72,12 @@ impl PartialOrd for Policy { impl Policy { /// Get highest priority policy than self pub fn higher_priority_policies(&self) -> impl Iterator { - POLICIES.iter().filter(move |p| p.priority > self.priority) + let policies = if self.name.starts_with("musllinux") { + &MUSLLINUX_POLICIES + } else { + &MANYLINUX_POLICIES + }; + policies.iter().filter(move |p| p.priority > self.priority) } /// Get platform tag from this policy @@ -72,21 +87,21 @@ impl Policy { /// Get policy by it's platform tag name pub fn from_name(name: &str) -> Option { - POLICIES + let policies = if name.starts_with("musllinux") { + &MUSLLINUX_POLICIES + } else { + &MANYLINUX_POLICIES + }; + policies .iter() .find(|p| p.name == name || p.aliases.iter().any(|alias| alias == name)) .cloned() } - - /// Get policy by it's priority - pub fn from_priority(priority: i64) -> Option { - POLICIES.iter().find(|p| p.priority == priority).cloned() - } } #[cfg(test)] mod test { - use super::{Policy, POLICIES}; + use super::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; #[test] fn test_load_policy() { @@ -106,7 +121,14 @@ mod test { #[test] fn test_policy_manylinux_tag() { - for policy in POLICIES.iter() { + for policy in MANYLINUX_POLICIES.iter() { + let _tag = policy.platform_tag(); + } + } + + #[test] + fn test_policy_musllinux_tag() { + for policy in MUSLLINUX_POLICIES.iter() { let _tag = policy.platform_tag(); } }