Support s390x z13 vector ABI

compiler/rustc_monomorphize/src/collector/abi_check.rs

@@ -2,14 +2,15 @@ use rustc_abi::Abi;
 use rustc_middle::ty::{self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt};
 use rustc_span::def_id::DefId;
 use rustc_span::{Span, Symbol};
-use rustc_target::abi::call::{FnAbi, PassMode};
+use rustc_target::abi::call::{FnAbi, PassMode, RegKind};
 
 use crate::errors::{AbiErrorDisabledVectorTypeCall, AbiErrorDisabledVectorTypeDef};
 
 // Represents the least-constraining feature that is required for vector types up to a certain size
 // to have their "proper" ABI.
 const X86_VECTOR_FEATURES: &'static [(u64, &'static str)] =
  &[(128, "sse"), (256, "avx"), (512, "avx512f")];
+const S390X_VECTOR_FEATURES: &'static [(u64, &'static str)] = &[(128, "vector")];
 
 fn do_check_abi<'tcx>(
  tcx: TyCtxt<'tcx>,
@@ -22,6 +23,8 @@ fn do_check_abi<'tcx>(
  } else if tcx.sess.target.arch == "aarch64" {
  // ABI on aarch64 does not depend on target features.
  return;
+ } else if tcx.sess.target.arch == "s390x" {
+ S390X_VECTOR_FEATURES
  } else {
  // FIXME: add support for non-tier1 architectures
  return;
@@ -31,6 +34,7 @@ fn do_check_abi<'tcx>(
  let size = arg_abi.layout.size;
  if matches!(arg_abi.layout.abi, Abi::Vector { .. })
  && !matches!(arg_abi.mode, PassMode::Indirect { .. })
+ || matches!(&arg_abi.mode, PassMode::Cast { cast, .. } if cast.rest.unit.kind == RegKind::Vector)
  {
  let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) {
  Some((_, feature)) => feature,

compiler/rustc_target/src/abi/call/s390x.rs

@@ -1,16 +1,35 @@
-// FIXME: The assumes we're using the non-vector ABI, i.e., compiling
-// for a pre-z13 machine or using -mno-vx.
-
-use crate::abi::call::{ArgAbi, FnAbi, Reg};
-use crate::abi::{HasDataLayout, TyAbiInterface};
+use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind};
+use crate::abi::{Abi, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
 use crate::spec::HasTargetSpec;
 
+fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>, expected_size: Size) -> bool
+where
+ Ty: TyAbiInterface<'a, C> + Copy,
+{
+ match layout.abi {
+ Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false,
+ Abi::Vector { .. } => layout.size == expected_size,
+ Abi::Aggregate { .. } => {
+ for i in 0..layout.fields.count() {
+ if contains_vector(cx, layout.field(cx, i), expected_size) {
+ return true;
+ }
+ }
+ false
+ }
+ }
+}
+
 fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
- if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 {
+ let size = ret.layout.size;
+ if size.bits() <= 128 && matches!(ret.layout.abi, Abi::Vector { .. }) {
+ return;
+ }
+ if !ret.layout.is_aggregate() && size.bits() <= 64 {
  ret.extend_integer_width_to(64);
- } else {
- ret.make_indirect();
+ return;
  }
+ ret.make_indirect();
 }
 
 fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
@@ -32,19 +51,25 @@ where
  }
  return;
  }
- if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 {
+
+ let size = arg.layout.size;
+ if size.bits() <= 128 && contains_vector(cx, arg.layout, size) {
+ arg.cast_to(Reg { kind: RegKind::Vector, size });
+ return;
+ }
+ if !arg.layout.is_aggregate() && size.bits() <= 64 {
  arg.extend_integer_width_to(64);
  return;
  }
 
  if arg.layout.is_single_fp_element(cx) {
- match arg.layout.size.bytes() {
+ match size.bytes() {
  4 => arg.cast_to(Reg::f32()),
  8 => arg.cast_to(Reg::f64()),
  _ => arg.make_indirect(),
  }
  } else {
- match arg.layout.size.bytes() {
+ match size.bytes() {
  1 => arg.cast_to(Reg::i8()),
  2 => arg.cast_to(Reg::i16()),
  4 => arg.cast_to(Reg::i32()),

compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs

@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
  base.endian = Endian::Big;
  // z10 is the oldest CPU supported by LLVM
  base.cpu = "z10".into();
- // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
- // ABI. Pass the -vector feature string to LLVM to respect this assumption.
- base.features = "-vector".into();
  base.max_atomic_width = Some(128);
  base.min_global_align = Some(16);
  base.stack_probes = StackProbeType::Inline;

compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs

@@ -6,9 +6,6 @@ pub(crate) fn target() -> Target {
  base.endian = Endian::Big;
  // z10 is the oldest CPU supported by LLVM
  base.cpu = "z10".into();
- // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector
- // ABI. Pass the -vector feature string to LLVM to respect this assumption.
- base.features = "-vector".into();
  base.max_atomic_width = Some(128);
  base.min_global_align = Some(16);
  base.static_position_independent_executables = true;

tests/assembly/s390x-vector-abi.rs

@@ -0,0 +1,228 @@
+//@ revisions: z10 z10_vector z13 z13_no_vector
+// ignore-tidy-linelength
+//@ assembly-output: emit-asm
+//@ compile-flags: -O -Z merge-functions=disabled
+//@[z10] compile-flags: --target s390x-unknown-linux-gnu --cfg no_vector
+//@[z10] needs-llvm-components: systemz
+//@[z10_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-feature=+vector
+//@[z10_vector] needs-llvm-components: systemz
+//@[z13] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13
+//@[z13] needs-llvm-components: systemz
+//@[z13_no_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 -C target-feature=-vector --cfg no_vector
+//@[z13_no_vector] needs-llvm-components: systemz
+
+#![feature(no_core, lang_items, repr_simd, s390x_target_feature)]
+#![no_core]
+#![crate_type = "lib"]
+#![allow(non_camel_case_types)]
+
+// Cases where vector feature is disabled are rejected.
+// See tests/ui/simd-abi-checks-s390x.rs for test for them.
+
+#[lang = "sized"]
+pub trait Sized {}
+#[lang = "copy"]
+pub trait Copy {}
+#[lang = "freeze"]
+pub trait Freeze {}
+
+impl<T: Copy, const N: usize> Copy for [T; N] {}
+
+#[repr(simd)]
+pub struct i8x8([i8; 8]);
+#[repr(simd)]
+pub struct i8x16([i8; 16]);
+#[repr(simd)]
+pub struct i8x32([i8; 32]);
+#[repr(C)]
+pub struct Wrapper<T>(T);
+#[repr(transparent)]
+pub struct TransparentWrapper<T>(T);
+
+impl Copy for i8 {}
+impl Copy for i64 {}
+impl Copy for i8x8 {}
+impl Copy for i8x16 {}
+impl Copy for i8x32 {}
+impl<T: Copy> Copy for Wrapper<T> {}
+impl<T: Copy> Copy for TransparentWrapper<T> {}
+
+// CHECK-LABEL: vector_ret_small:
+// CHECK: vlrepg %v24, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret_small(x: &i8x8) -> i8x8 {
+ *x
+}
+// CHECK-LABEL: vector_ret:
+// CHECK: vl %v24, 0(%r2), 3
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret(x: &i8x16) -> i8x16 {
+ *x
+}
+// CHECK-LABEL: vector_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 0(%r3), 4
+// z13-NEXT: vl %v1, 16(%r3), 4
+// z13-NEXT: vst %v1, 16(%r2), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_ret_large(x: &i8x32) -> i8x32 {
+ *x
+}
+
+// CHECK-LABEL: vector_wrapper_ret_small:
+// CHECK: mvc 0(8,%r2), 0(%r3)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret_small(x: &Wrapper<i8x8>) -> Wrapper<i8x8> {
+ *x
+}
+// CHECK-LABEL: vector_wrapper_ret:
+// CHECK: mvc 0(16,%r2), 0(%r3)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret(x: &Wrapper<i8x16>) -> Wrapper<i8x16> {
+ *x
+}
+// CHECK-LABEL: vector_wrapper_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 16(%r3), 4
+// z13-NEXT: vst %v0, 16(%r2), 4
+// z13-NEXT: vl %v0, 0(%r3), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_ret_large(x: &Wrapper<i8x32>) -> Wrapper<i8x32> {
+ *x
+}
+
+// CHECK-LABEL: vector_transparent_wrapper_ret_small:
+// CHECK: vlrepg %v24, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret_small(
+ x: &TransparentWrapper<i8x8>,
+) -> TransparentWrapper<i8x8> {
+ *x
+}
+// CHECK-LABEL: vector_transparent_wrapper_ret:
+// CHECK: vl %v24, 0(%r2), 3
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret(
+ x: &TransparentWrapper<i8x16>,
+) -> TransparentWrapper<i8x16> {
+ *x
+}
+// CHECK-LABEL: vector_transparent_wrapper_ret_large:
+// z10: vl %v0, 16(%r3), 4
+// z10-NEXT: vl %v1, 0(%r3), 4
+// z10-NEXT: vst %v0, 16(%r2), 4
+// z10-NEXT: vst %v1, 0(%r2), 4
+// z10-NEXT: br %r14
+// z13: vl %v0, 0(%r3), 4
+// z13-NEXT: vl %v1, 16(%r3), 4
+// z13-NEXT: vst %v1, 16(%r2), 4
+// z13-NEXT: vst %v0, 0(%r2), 4
+// z13-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_ret_large(
+ x: &TransparentWrapper<i8x32>,
+) -> TransparentWrapper<i8x32> {
+ *x
+}
+
+// CHECK-LABEL: vector_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg_small(x: i8x8) -> i64 {
+ unsafe { *(&x as *const i8x8 as *const i64) }
+}
+// CHECK-LABEL: vector_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg(x: i8x16) -> i64 {
+ unsafe { *(&x as *const i8x16 as *const i64) }
+}
+// CHECK-LABEL: vector_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_arg_large(x: i8x32) -> i64 {
+ unsafe { *(&x as *const i8x32 as *const i64) }
+}
+
+// CHECK-LABEL: vector_wrapper_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg_small(x: Wrapper<i8x8>) -> i64 {
+ unsafe { *(&x as *const Wrapper<i8x8> as *const i64) }
+}
+// CHECK-LABEL: vector_wrapper_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg(x: Wrapper<i8x16>) -> i64 {
+ unsafe { *(&x as *const Wrapper<i8x16> as *const i64) }
+}
+// CHECK-LABEL: vector_wrapper_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_wrapper_arg_large(x: Wrapper<i8x32>) -> i64 {
+ unsafe { *(&x as *const Wrapper<i8x32> as *const i64) }
+}
+
+// CHECK-LABEL: vector_transparent_wrapper_arg_small:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg_small(x: TransparentWrapper<i8x8>) -> i64 {
+ unsafe { *(&x as *const TransparentWrapper<i8x8> as *const i64) }
+}
+// CHECK-LABEL: vector_transparent_wrapper_arg:
+// CHECK: vlgvg %r2, %v24, 0
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg(x: TransparentWrapper<i8x16>) -> i64 {
+ unsafe { *(&x as *const TransparentWrapper<i8x16> as *const i64) }
+}
+// CHECK-LABEL: vector_transparent_wrapper_arg_large:
+// CHECK: lg %r2, 0(%r2)
+// CHECK-NEXT: br %r14
+#[cfg_attr(no_vector, target_feature(enable = "vector"))]
+#[no_mangle]
+unsafe extern "C" fn vector_transparent_wrapper_arg_large(x: TransparentWrapper<i8x32>) -> i64 {
+ unsafe { *(&x as *const TransparentWrapper<i8x32> as *const i64) }
+}