Support s390x z13 vector ABI

taiki-e · taiki-e · commit 0a6b7ad6e48f · 2024-10-14T20:11:35.000+09:00
diff --git a/compiler/rustc_monomorphize/src/collector/abi_check.rs b/compiler/rustc_monomorphize/src/collector/abi_check.rs
@@ -2,14 +2,15 @@ use rustc_abi::Abi;
 use rustc_middle::ty::{self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt};
 use rustc_span::def_id::DefId;
 use rustc_span::{Span, Symbol};
-use rustc_target::abi::call::{FnAbi, PassMode};
+use rustc_target::abi::call::{FnAbi, PassMode, RegKind};
 
 use crate::errors::{AbiErrorDisabledVectorTypeCall, AbiErrorDisabledVectorTypeDef};
 
 // Represents the least-constraining feature that is required for vector types up to a certain size
 // to have their "proper" ABI.
 const X86_VECTOR_FEATURES: &'static [(u64, &'static str)] =
     &[(128, "sse"), (256, "avx"), (512, "avx512f")];
+const S390X_VECTOR_FEATURES: &'static [(u64, &'static str)] = &[(128, "vector")];
 
 fn do_check_abi<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -22,6 +23,8 @@ fn do_check_abi<'tcx>(
     } else if tcx.sess.target.arch == "aarch64" {
         // ABI on aarch64 does not depend on target features.
         return;
+    } else if tcx.sess.target.arch == "s390x" {
+        S390X_VECTOR_FEATURES
     } else {
         // FIXME: add support for non-tier1 architectures
         return;
@@ -31,6 +34,7 @@ fn do_check_abi<'tcx>(
         let size = arg_abi.layout.size;
         if matches!(arg_abi.layout.abi, Abi::Vector { .. })
             && !matches!(arg_abi.mode, PassMode::Indirect { .. })
+            || matches!(&arg_abi.mode, PassMode::Cast { cast, .. } if cast.rest.unit.kind == RegKind::Vector)
         {
             let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) {
                 Some((_, feature)) => feature,
@@ -48,7 +52,7 @@ fn do_check_abi<'tcx>(
 
 /// Checks that the ABI of a given instance of a function does not contain vector-passed arguments
 /// or return values for which the corresponding target feature is not enabled.
-pub fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
+pub(crate) fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
     let param_env = ParamEnv::reveal_all();
     let Ok(abi) = tcx.fn_abi_of_instance(param_env.and((instance, ty::List::empty()))) else {
         // An error will be reported during codegen if we cannot determine the ABI of this
@@ -65,7 +69,7 @@ pub fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
 
 /// Checks that a call expression does not try to pass a vector-passed argument which requires a
 /// target feature that the caller does not have, as doing so causes UB because of ABI mismatch.
-pub fn check_call_site_abi<'tcx>(
+pub(crate) fn check_call_site_abi<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,
     span: Span,
diff --git a/compiler/rustc_monomorphize/src/errors.rs b/compiler/rustc_monomorphize/src/errors.rs
@@ -96,7 +96,7 @@ pub(crate) struct UnknownCguCollectionMode<'a> {
 #[derive(Diagnostic)]
 #[diag(monomorphize_abi_error_disabled_vector_type_def)]
 #[help]
-pub struct AbiErrorDisabledVectorTypeDef<'a> {
+pub(crate) struct AbiErrorDisabledVectorTypeDef<'a> {
     #[primary_span]
     pub span: Span,
     pub required_feature: &'a str,
@@ -105,7 +105,7 @@ pub struct AbiErrorDisabledVectorTypeDef<'a> {
 #[derive(Diagnostic)]
 #[diag(monomorphize_abi_error_disabled_vector_type_call)]
 #[help]
-pub struct AbiErrorDisabledVectorTypeCall<'a> {
+pub(crate) struct AbiErrorDisabledVectorTypeCall<'a> {
     #[primary_span]
     pub span: Span,
     pub required_feature: &'a str,
diff --git a/compiler/rustc_target/src/abi/call/s390x.rs b/compiler/rustc_target/src/abi/call/s390x.rs
@@ -1,16 +1,35 @@
-// FIXME: The assumes we're using the non-vector ABI, i.e., compiling
-// for a pre-z13 machine or using -mno-vx.
-
-use crate::abi::call::{ArgAbi, FnAbi, Reg};
-use crate::abi::{HasDataLayout, TyAbiInterface};
+use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind};
+use crate::abi::{Abi, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
 use crate::spec::HasTargetSpec;
 
+fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>, expected_size: Size) -> bool
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
+    match layout.abi {
+        Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false,
+        Abi::Vector { .. } => layout.size == expected_size,
+        Abi::Aggregate { .. } => {
+            for i in 0..layout.fields.count() {
+                if contains_vector(cx, layout.field(cx, i), expected_size) {
+                    return true;
+                }
+            }
+            false
+        }
+    }
+}
+
 fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
-    if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 {
+    let size = ret.layout.size;
+    if size.bits() <= 128 && matches!(ret.layout.abi, Abi::Vector { .. }) {
+        return;
+    }
+    if !ret.layout.is_aggregate() && size.bits() <= 64 {
         ret.extend_integer_width_to(64);
-    } else {
-        ret.make_indirect();
+        return;
     }
+    ret.make_indirect();
 }
 
 fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
@@ -32,19 +51,25 @@ where
         }
         return;
     }
-    if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 {
+
+    let size = arg.layout.size;
+    if size.bits() <= 128 && contains_vector(cx, arg.layout, size) {
+        arg.cast_to(Reg { kind: RegKind::Vector, size });
+        return;
+    }
+    if !arg.layout.is_aggregate() && size.bits() <= 64 {
         arg.extend_integer_width_to(64);
         return;
     }
 
     if arg.layout.is_single_fp_element(cx) {
-        match arg.layout.size.bytes() {
+        match size.bytes() {
             4 => arg.cast_to(Reg::f32()),
             8 => arg.cast_to(Reg::f64()),
             _ => arg.make_indirect(),
         }
     } else {
-        match arg.layout.size.bytes() {
+        match size.bytes() {
             1 => arg.cast_to(Reg::i8()),
             2 => arg.cast_to(Reg::i16()),
             4 => arg.cast_to(Reg::i32()),
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs
@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
     base.endian = Endian::Big;
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
-    // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
-    // ABI. Pass the -vector feature string to LLVM to respect this assumption.
-    base.features = "-vector".into();
     base.max_atomic_width = Some(128);
     base.min_global_align = Some(16);
     base.stack_probes = StackProbeType::Inline;
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs
@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
     base.endian = Endian::Big;
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
-    // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
-    // ABI. Pass the -vector feature string to LLVM to respect this assumption.
-    base.features = "-vector".into();
     base.max_atomic_width = Some(128);
     base.min_global_align = Some(16);
     base.static_position_independent_executables = true;
diff --git a/tests/assembly/s390x-vector-abi.rs b/tests/assembly/s390x-vector-abi.rs
@@ -0,0 +1,228 @@
+//@ revisions: z10 z10_vector z13 z13_no_vector
+// ignore-tidy-linelength
+//@ assembly-output: emit-asm
+//@ compile-flags: -O -Z merge-functions=disabled
+//@[z10] compile-flags: --target s390x-unknown-linux-gnu --cfg no_vector
+//@[z10] needs-llvm-components: systemz
+//@[z10_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-feature=+vector
+//@[z10_vector] needs-llvm-components: systemz
+//@[z13] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13
+//@[z13] needs-llvm-components: systemz
+//@[z13_no_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 -C target-feature=-vector --cfg no_vector
+//@[z13_no_vector] needs-llvm-components: systemz
+
+#![feature(no_core, lang_items, repr_simd, s390x_target_feature)]
+#![no_core]
+#![crate_type = "lib"]
+#![allow(non_camel_case_types)]
+
+// Cases where vector feature is disabled are rejected.
+// See tests/ui/simd-abi-checks-s390x.rs for test for them.
+
+#[lang = "sized"]
+pub trait Sized {}
+#[lang = "copy"]
+pub trait Copy {}
+#[lang = "freeze"]
+pub trait Freeze {}
+
+impl<T: Copy, const N: usize> Copy for [T; N] {}
+
+#[repr(simd)]
+pub struct i8x8([i8; 8]);
+#[repr(simd)]
+pub struct i8x16([i8; 16]);
+#[repr(simd)]
+pub struct i8x32([i8; 32]);
+#[repr(C)]
+pub struct Wrapper<T>(T);
+#[repr(transparent)]
+pub struct TransparentWrapper<T>(T);
+
+impl Copy for i8 {}
+impl Copy for i64 {}
+impl Copy for i8x8 {}
+impl Copy for i8x16 {}
+impl Copy for i8x32 {}
+impl<T: Copy> Copy for Wrapper<T> {}
+impl<T: Copy> Copy for TransparentWrapper<T> {}
+
+// CHECK-LABEL: vector_ret_small:
+// CHECK: vlrepg %v24, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret_small(x: &i8x8) -> i8x8 {
+    *x
+}
+// CHECK-LABEL: vector_ret:
+// CHECK: vl %v24, 0(%r2), 3
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret(x: &i8x16) -> i8x16 {
+    *x
+}
+// CHECK-LABEL: vector_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 0(%r3), 4
+// z13-NEXT: vl %v1, 16(%r3), 4
+// z13-NEXT: vst %v1, 16(%r2), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret_large(x: &i8x32) -> i8x32 {
+    *x
+}
+
+// CHECK-LABEL: vector_wrapper_ret_small:
+// CHECK: mvc 0(8,%r2), 0(%r3)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret_small(x: &Wrapper<i8x8>) -> Wrapper<i8x8> {
+    *x
+}
+// CHECK-LABEL: vector_wrapper_ret:
+// CHECK: mvc 0(16,%r2), 0(%r3)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret(x: &Wrapper<i8x16>) -> Wrapper<i8x16> {
+    *x
+}
+// CHECK-LABEL: vector_wrapper_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 16(%r3), 4
+// z13-NEXT: vst %v0, 16(%r2), 4
+// z13-NEXT: vl %v0, 0(%r3), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret_large(x: &Wrapper<i8x32>) -> Wrapper<i8x32> {
+    *x
+}
+
+// CHECK-LABEL: vector_transparent_wrapper_ret_small:
+// CHECK: vlrepg %v24, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret_small(
+    x: &TransparentWrapper<i8x8>,
+) -> TransparentWrapper<i8x8> {
+    *x
+}
+// CHECK-LABEL: vector_transparent_wrapper_ret:
+// CHECK: vl %v24, 0(%r2), 3
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret(
+    x: &TransparentWrapper<i8x16>,
+) -> TransparentWrapper<i8x16> {
+    *x
+}
+// CHECK-LABEL: vector_transparent_wrapper_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 0(%r3), 4
+// z13-NEXT: vl %v1, 16(%r3), 4
+// z13-NEXT: vst %v1, 16(%r2), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret_large(
+    x: &TransparentWrapper<i8x32>,
+) -> TransparentWrapper<i8x32> {
+    *x
+}
+
+// CHECK-LABEL: vector_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg_small(x: i8x8) -> i64 {
+    unsafe { *(&x as *const i8x8 as *const i64) }
+}
+// CHECK-LABEL: vector_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg(x: i8x16) -> i64 {
+    unsafe { *(&x as *const i8x16 as *const i64) }
+}
+// CHECK-LABEL: vector_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg_large(x: i8x32) -> i64 {
+    unsafe { *(&x as *const i8x32 as *const i64) }
+}
+
+// CHECK-LABEL: vector_wrapper_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg_small(x: Wrapper<i8x8>) -> i64 {
+    unsafe { *(&x as *const Wrapper<i8x8> as *const i64) }
+}
+// CHECK-LABEL: vector_wrapper_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg(x: Wrapper<i8x16>) -> i64 {
+    unsafe { *(&x as *const Wrapper<i8x16> as *const i64) }
+}
+// CHECK-LABEL: vector_wrapper_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg_large(x: Wrapper<i8x32>) -> i64 {
+    unsafe { *(&x as *const Wrapper<i8x32> as *const i64) }
+}
+
+// CHECK-LABEL: vector_transparent_wrapper_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg_small(x: TransparentWrapper<i8x8>) -> i64 {
+    unsafe { *(&x as *const TransparentWrapper<i8x8> as *const i64) }
+}
+// CHECK-LABEL: vector_transparent_wrapper_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg(x: TransparentWrapper<i8x16>) -> i64 {
+    unsafe { *(&x as *const TransparentWrapper<i8x16> as *const i64) }
+}
+// CHECK-LABEL: vector_transparent_wrapper_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg_large(x: TransparentWrapper<i8x32>) -> i64 {
+    unsafe { *(&x as *const TransparentWrapper<i8x32> as *const i64) }
+}
diff --git a/tests/ui/simd-abi-checks-s390x.rs b/tests/ui/simd-abi-checks-s390x.rs
diff --git a/tests/ui/simd-abi-checks-s390x.z10.stderr b/tests/ui/simd-abi-checks-s390x.z10.stderr
diff --git a/tests/ui/simd-abi-checks-s390x.z13_no_vector.stderr b/tests/ui/simd-abi-checks-s390x.z13_no_vector.stderr
