Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rustc_target: Add various aarch64 features #128192

Merged
merged 5 commits into from
Aug 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions compiler/rustc_codegen_llvm/src/attributes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -521,13 +521,20 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(

let function_features = function_features
.iter()
.flat_map(|feat| {
llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}"))
})
// Convert to LLVMFeatures and filter out unavailable ones
.flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat))
// Convert LLVMFeatures & dependencies to +<feats>s
.flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
.chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(),
InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(),
}))
// HACK: LLVM versions 19+ do not have the FPMR feature and treat it as always enabled
// It only exists as a feature in LLVM 18, cannot be passed down for any other version
.chain(match &*cx.tcx.sess.target.arch {
"aarch64" if llvm_util::get_version().0 == 18 => vec!["+fpmr".to_string()],
_ => vec![],
})
.collect::<Vec<String>>();

if cx.tcx.sess.target.is_like_wasm {
Expand Down
92 changes: 59 additions & 33 deletions compiler/rustc_codegen_llvm/src/llvm_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
// Though note that Rust can also be build with an external precompiled version of LLVM
// which might lead to failures if the oldest tested / supported LLVM version
// doesn't yet support the relevant intrinsics
pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
let arch = if sess.target.arch == "x86_64" {
"x86"
} else if sess.target.arch == "arm64ec" {
Expand All @@ -218,40 +218,59 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a
&*sess.target.arch
};
match (arch, s) {
("x86", "sse4.2") => {
LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32"))
}
("x86", "pclmulqdq") => LLVMFeature::new("pclmul"),
("x86", "rdrand") => LLVMFeature::new("rdrnd"),
("x86", "bmi1") => LLVMFeature::new("bmi"),
("x86", "cmpxchg16b") => LLVMFeature::new("cx16"),
("x86", "lahfsahf") => LLVMFeature::new("sahf"),
("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"),
("aarch64", "dpb") => LLVMFeature::new("ccpp"),
("aarch64", "dpb2") => LLVMFeature::new("ccdp"),
("aarch64", "frintts") => LLVMFeature::new("fptoint"),
("aarch64", "fcma") => LLVMFeature::new("complxnum"),
("aarch64", "pmuv3") => LLVMFeature::new("perfmon"),
("aarch64", "paca") => LLVMFeature::new("pauth"),
("aarch64", "pacg") => LLVMFeature::new("pauth"),
("x86", "sse4.2") => Some(LLVMFeature::with_dependency(
"sse4.2",
TargetFeatureFoldStrength::EnableOnly("crc32"),
)),
("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")),
("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")),
("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")),
("aarch64", "frintts") => Some(LLVMFeature::new("fptoint")),
("aarch64", "fcma") => Some(LLVMFeature::new("complxnum")),
("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")),
("aarch64", "paca") => Some(LLVMFeature::new("pauth")),
("aarch64", "pacg") => Some(LLVMFeature::new("pauth")),
("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")),
("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")),
// Rust ties fp and neon together.
("aarch64", "neon") => {
LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8"))
Some(LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")))
}
// In LLVM neon implicitly enables fp, but we manually enable
// neon when a feature only implicitly enables fp
("aarch64", "fhm") => LLVMFeature::new("fp16fml"),
("aarch64", "fp16") => LLVMFeature::new("fullfp16"),
("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")),
("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")),
// Filter out features that are not supported by the current LLVM version
("aarch64", "faminmax") if get_version().0 < 18 => None,
("aarch64", "fp8") if get_version().0 < 18 => None,
("aarch64", "fp8dot2") if get_version().0 < 18 => None,
("aarch64", "fp8dot4") if get_version().0 < 18 => None,
("aarch64", "fp8fma") if get_version().0 < 18 => None,
("aarch64", "fpmr") if get_version().0 != 18 => None,
("aarch64", "lut") if get_version().0 < 18 => None,
("aarch64", "sme-f8f16") if get_version().0 < 18 => None,
("aarch64", "sme-f8f32") if get_version().0 < 18 => None,
("aarch64", "sme-fa64") if get_version().0 < 18 => None,
("aarch64", "sme-lutv2") if get_version().0 < 18 => None,
("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None,
("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None,
("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None,
("aarch64", "v9.5a") if get_version().0 < 18 => None,
// In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called
// `fast-unaligned-access`. In LLVM 19, it was split back out.
("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => {
LLVMFeature::new("fast-unaligned-access")
Some(LLVMFeature::new("fast-unaligned-access"))
}
// For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled.
("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => {
LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))
Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")))
}
(_, s) => LLVMFeature::new(s),
(_, s) => Some(LLVMFeature::new(s)),
}
}

Expand Down Expand Up @@ -291,13 +310,17 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
return true;
}
// check that all features in a given smallvec are enabled
for llvm_feature in to_llvm_features(sess, feature) {
let cstr = SmallCStr::new(llvm_feature);
if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
return false;
if let Some(feat) = to_llvm_features(sess, feature) {
for llvm_feature in feat {
let cstr = SmallCStr::new(llvm_feature);
if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
return false;
}
}
true
} else {
false
}
true
})
.map(|(feature, _, _)| Symbol::intern(feature)),
);
Expand Down Expand Up @@ -386,9 +409,9 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
.target
.supported_target_features()
.iter()
.map(|(feature, _gate, _implied)| {
.filter_map(|(feature, _gate, _implied)| {
// LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name;
let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name;
let desc =
match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() {
Some(index) => {
Expand All @@ -398,7 +421,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
None => "",
};

(*feature, desc)
Some((*feature, desc))
})
.collect::<Vec<_>>();

Expand Down Expand Up @@ -595,7 +618,7 @@ pub(crate) fn global_llvm_features(
if feature_state.is_none() {
let rust_feature =
supported_features.iter().find_map(|&(rust_feature, _, _)| {
let llvm_features = to_llvm_features(sess, rust_feature);
let llvm_features = to_llvm_features(sess, rust_feature)?;
if llvm_features.contains(feature)
&& !llvm_features.contains(rust_feature)
{
Expand Down Expand Up @@ -641,7 +664,7 @@ pub(crate) fn global_llvm_features(
// passing requests down to LLVM. This means that all in-language
// features also work on the command line instead of having two
// different names when the LLVM name and the Rust name differ.
let llvm_feature = to_llvm_features(sess, feature);
let llvm_feature = to_llvm_features(sess, feature)?;

Some(
std::iter::once(format!(
Expand Down Expand Up @@ -691,6 +714,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> {
let feature = s
.strip_prefix(&['+', '-'][..])
.unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s }));
if s.is_empty() {
return None;
}
// Rustc-specific feature requests like `+crt-static` or `-crt-static`
// are not passed down to LLVM.
if RUSTC_SPECIFIC_FEATURES.contains(&feature) {
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_feature/src/unstable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ declare_features! (
// FIXME: Document these and merge with the list below.

// Unstable `#[target_feature]` directives.
(unstable, aarch64_unstable_target_feature, "CURRENT_RUSTC_VERSION", Some(44839)),
(unstable, aarch64_ver_target_feature, "1.27.0", Some(44839)),
(unstable, arm_target_feature, "1.27.0", Some(44839)),
(unstable, avx512_target_feature, "1.27.0", Some(44839)),
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_span/src/symbol.rs
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,7 @@ symbols! {
_task_context,
a32,
aarch64_target_feature,
aarch64_unstable_target_feature,
aarch64_ver_target_feature,
abi,
abi_amdgpu_kernel,
Expand Down
78 changes: 73 additions & 5 deletions compiler/rustc_target/src/target_features.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("bti", Stable, &[]),
// FEAT_CRC
("crc", Stable, &[]),
// FEAT_CSSC
("cssc", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_DIT
("dit", Stable, &[]),
// FEAT_DotProd
Expand All @@ -107,21 +109,37 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("dpb", Stable, &[]),
// FEAT_DPB2
("dpb2", Stable, &["dpb"]),
// FEAT_ECV
("ecv", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_F32MM
("f32mm", Stable, &["sve"]),
// FEAT_F64MM
("f64mm", Stable, &["sve"]),
// FEAT_FAMINMAX
("faminmax", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_FCMA
("fcma", Stable, &["neon"]),
// FEAT_FHM
("fhm", Stable, &["fp16"]),
// FEAT_FLAGM
("flagm", Stable, &[]),
// FEAT_FLAGM2
("flagm2", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_FP16
// Rust ties FP and Neon: https://github.com/rust-lang/rust/pull/91608
("fp16", Stable, &["neon"]),
// FEAT_FP8
("fp8", Unstable(sym::aarch64_unstable_target_feature), &["faminmax", "lut", "bf16"]),
// FEAT_FP8DOT2
("fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["fp8dot4"]),
// FEAT_FP8DOT4
("fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["fp8fma"]),
// FEAT_FP8FMA
("fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["fp8"]),
// FEAT_FRINTTS
("frintts", Stable, &[]),
// FEAT_HBC
("hbc", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_I8MM
("i8mm", Stable, &[]),
// FEAT_JSCVT
Expand All @@ -131,6 +149,14 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("lor", Stable, &[]),
// FEAT_LSE
("lse", Stable, &[]),
// FEAT_LSE128
("lse128", Unstable(sym::aarch64_unstable_target_feature), &["lse"]),
// FEAT_LSE2
("lse2", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_LUT
("lut", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_MOPS
("mops", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_MTE & FEAT_MTE2
("mte", Stable, &[]),
// FEAT_AdvSimd & FEAT_FP
Expand All @@ -143,14 +169,16 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("pan", Stable, &[]),
// FEAT_PMUv3
("pmuv3", Stable, &[]),
// FEAT_RAND
// FEAT_RNG
("rand", Stable, &[]),
// FEAT_RAS & FEAT_RASv1p1
("ras", Stable, &[]),
// FEAT_RCPC
// FEAT_LRCPC
("rcpc", Stable, &[]),
// FEAT_RCPC2
// FEAT_LRCPC2
("rcpc2", Stable, &["rcpc"]),
// FEAT_LRCPC3
("rcpc3", Unstable(sym::aarch64_unstable_target_feature), &["rcpc2"]),
// FEAT_RDM
("rdm", Stable, &["neon"]),
// FEAT_SB
Expand All @@ -161,10 +189,36 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("sha3", Stable, &["sha2"]),
// FEAT_SM3 & FEAT_SM4
("sm4", Stable, &["neon"]),
// FEAT_SME
("sme", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
// FEAT_SME_F16F16
("sme-f16f16", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
// FEAT_SME_F64F64
("sme-f64f64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
// FEAT_SME_F8F16
("sme-f8f16", Unstable(sym::aarch64_unstable_target_feature), &["sme-f8f32"]),
// FEAT_SME_F8F32
("sme-f8f32", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
// FEAT_SME_FA64
("sme-fa64", Unstable(sym::aarch64_unstable_target_feature), &["sme", "sve2"]),
// FEAT_SME_I16I64
("sme-i16i64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
// FEAT_SME_LUTv2
("sme-lutv2", Unstable(sym::aarch64_unstable_target_feature), &[]),
// FEAT_SME2
("sme2", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
// FEAT_SME2p1
("sme2p1", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
// FEAT_SPE
("spe", Stable, &[]),
// FEAT_SSBS & FEAT_SSBS2
("ssbs", Stable, &[]),
// FEAT_SSVE_FP8FDOT2
("ssve-fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8dot4"]),
// FEAT_SSVE_FP8FDOT4
("ssve-fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8fma"]),
// FEAT_SSVE_FP8FMA
("ssve-fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
// FEAT_SVE
// It was decided that SVE requires Neon: https://github.com/rust-lang/rust/pull/91608
//
Expand All @@ -173,16 +227,20 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
//
// "For backwards compatibility, Neon and VFP are required in the latest architectures."
("sve", Stable, &["neon"]),
// FEAT_SVE_B16B16 (SVE or SME Instructions)
("sve-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
// FEAT_SVE2
("sve2", Stable, &["sve"]),
// FEAT_SVE2_AES
// FEAT_SVE_AES & FEAT_SVE_PMULL128
("sve2-aes", Stable, &["sve2", "aes"]),
// FEAT_SVE2_BitPerm
("sve2-bitperm", Stable, &["sve2"]),
// FEAT_SVE2_SHA3
("sve2-sha3", Stable, &["sve2", "sha3"]),
// FEAT_SVE2_SM4
("sve2-sm4", Stable, &["sve2", "sm4"]),
// FEAT_SVE2p1
("sve2p1", Unstable(sym::aarch64_unstable_target_feature), &["sve2"]),
// FEAT_TME
("tme", Stable, &[]),
(
Expand All @@ -199,9 +257,19 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
("v8.4a", Unstable(sym::aarch64_ver_target_feature), &["v8.3a", "dotprod", "dit", "flagm"]),
("v8.5a", Unstable(sym::aarch64_ver_target_feature), &["v8.4a", "ssbs", "sb", "dpb2", "bti"]),
("v8.6a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "bf16", "i8mm"]),
("v8.7a", Unstable(sym::aarch64_ver_target_feature), &[]),
("v8.7a", Unstable(sym::aarch64_ver_target_feature), &["v8.6a", "wfxt"]),
("v8.8a", Unstable(sym::aarch64_ver_target_feature), &["v8.7a", "hbc", "mops"]),
("v8.9a", Unstable(sym::aarch64_ver_target_feature), &["v8.8a", "cssc"]),
("v9.1a", Unstable(sym::aarch64_ver_target_feature), &["v9a", "v8.6a"]),
("v9.2a", Unstable(sym::aarch64_ver_target_feature), &["v9.1a", "v8.7a"]),
("v9.3a", Unstable(sym::aarch64_ver_target_feature), &["v9.2a", "v8.8a"]),
("v9.4a", Unstable(sym::aarch64_ver_target_feature), &["v9.3a", "v8.9a"]),
("v9.5a", Unstable(sym::aarch64_ver_target_feature), &["v9.4a"]),
("v9a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "sve2"]),
// FEAT_VHE
("vh", Stable, &[]),
// FEAT_WFxT
("wfxt", Unstable(sym::aarch64_unstable_target_feature), &[]),
// tidy-alphabetical-end
];

Expand Down
Loading
Loading