From 31be70eac55b009ace329ce5e359725dea5b6776 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 4 Mar 2025 20:15:42 +0100 Subject: [PATCH 01/30] add `vec_gather_element` --- crates/core_arch/src/s390x/macros.rs | 16 ++++ crates/core_arch/src/s390x/vector.rs | 107 +++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/crates/core_arch/src/s390x/macros.rs b/crates/core_arch/src/s390x/macros.rs index 2c40fedae2..bf2ef35705 100644 --- a/crates/core_arch/src/s390x/macros.rs +++ b/crates/core_arch/src/s390x/macros.rs @@ -250,6 +250,19 @@ macro_rules! l_t_t { u8 }; + (vector_bool_long_long ) => { + u64 + }; + (vector_bool_int ) => { + u32 + }; + (vector_bool_short ) => { + u16 + }; + (vector_bool_char ) => { + u8 + }; + (vector_float) => { f32 }; @@ -338,6 +351,9 @@ macro_rules! t_u { (vector_bool_int) => { vector_unsigned_int }; + (vector_bool_long_long) => { + vector_unsigned_long_long + }; (vector_unsigned_char) => { vector_unsigned_char }; diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 3ec27b579c..761660ee32 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2593,6 +2593,78 @@ mod sealed { vec_vgfmaf(self, b, c) } } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vgef, D = 3))] + unsafe fn vgef( + a: vector_unsigned_int, + b: vector_unsigned_int, + c: *const u32, + ) -> vector_unsigned_int { + static_assert_uimm_bits!(D, 2); + let offset: u32 = simd_extract(b, D); + let ptr = c.byte_offset(offset as isize); + let value = ptr.read(); + simd_insert(a, D, value) + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vgeg, D = 1))] + unsafe fn vgeg( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: *const u64, + ) -> vector_unsigned_long_long { + static_assert_uimm_bits!(D, 1); + let offset: u64 = simd_extract(b, D); + let ptr = c.byte_offset(offset as isize); + let value = ptr.read(); + simd_insert(a, D, value) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorGatherElement { + type Element; + type Offset; + unsafe fn vec_gather_element( + self, + b: Self::Offset, + c: *const Self::Element, + ) -> Self; + } + + macro_rules! impl_vec_gather_element { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorGatherElement for $ty { + type Element = l_t_t!($ty); + type Offset = t_u!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_gather_element(self, b: Self::Offset, c: *const Self::Element) -> Self { + transmute($instr::(transmute(self), b, c.cast())) + } + } + )* + } + } + + impl_vec_gather_element! { + vgef vector_signed_int + vgef vector_bool_int + vgef vector_unsigned_int + + vgeg vector_signed_long_long + vgeg vector_bool_long_long + vgeg vector_unsigned_long_long + + vgef vector_float + vgeg vector_double + } } /// Load Count to Block Boundary @@ -3739,6 +3811,17 @@ pub unsafe fn vec_gfmsum_accum_128( transmute(vgfmag(a, b, transmute(c))) } +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_gather_element( + a: T, + b: T::Offset, + c: *const T::Element, +) -> T { + a.vec_gather_element::(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -4676,4 +4759,28 @@ mod tests { let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) }; assert_eq!(d, 0xE000E000E000E000E000E000E000E); } + + #[simd_test(enable = "vector")] + fn test_vec_gather_element() { + let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; + let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]; + + let v1 = vector_unsigned_int([1, 2, 3, 4]); + let v2 = vector_unsigned_int([1, 2, 3, 4]); + + let sizeof_int = core::mem::size_of::() as u32; + let v3 = vector_unsigned_int([ + 5 * sizeof_int, + 8 * sizeof_int, + 9 * sizeof_int, + 6 * sizeof_int, + ]); + + unsafe { + let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr()); + assert_eq!(d1.as_array(), &[15, 2, 3, 4]); + let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr()); + assert_eq!(d2.as_array(), &[25, 2, 3, 4]); + } + } } From 7777e5c509f50bccf80328a318b6acdf5d9f9c48 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 4 Mar 2025 20:25:26 +0100 Subject: [PATCH 02/30] add `vec_bperm_u128` --- crates/core_arch/src/s390x/vector.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 761660ee32..eb60812c54 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -206,6 +206,8 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128; + + #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -3811,6 +3813,19 @@ pub unsafe fn vec_gfmsum_accum_128( transmute(vgfmag(a, b, transmute(c))) } +/// Vector Bit Permute +#[inline] +#[target_feature(enable = "vector-enhancements-1")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vbperm))] +pub unsafe fn vec_bperm_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_long_long { + vbperm(a, b) +} + +/// Vector Gather Element #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -4760,6 +4775,16 @@ mod tests { assert_eq!(d, 0xE000E000E000E000E000E000E000E); } + #[simd_test(enable = "vector-enhancements-1")] + fn test_vec_bperm_u128() { + let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]); + let b = vector_unsigned_char([ + 0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255, + ]); + let d = unsafe { vec_bperm_u128(a, b) }; + assert_eq!(d.as_array(), &[0xF00, 0]); + } + #[simd_test(enable = "vector")] fn test_vec_gather_element() { let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; From f3253592e0f72e3ae76c0bac40cff08efb8056b6 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 4 Mar 2025 20:59:18 +0100 Subject: [PATCH 03/30] add `vec_sel` --- crates/core_arch/src/s390x/macros.rs | 3 ++ crates/core_arch/src/s390x/vector.rs | 74 ++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/crates/core_arch/src/s390x/macros.rs b/crates/core_arch/src/s390x/macros.rs index bf2ef35705..4f0f84ec91 100644 --- a/crates/core_arch/src/s390x/macros.rs +++ b/crates/core_arch/src/s390x/macros.rs @@ -396,6 +396,9 @@ macro_rules! t_b { (vector_bool_int) => { vector_bool_int }; + (vector_bool_long_long) => { + vector_bool_long_long + }; (vector_signed_char) => { vector_bool_char }; diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index eb60812c54..f55c28428d 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2667,6 +2667,58 @@ mod sealed { vgef vector_float vgeg vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSel: Sized { + unsafe fn vec_sel(self, b: Self, c: Mask) -> Self; + } + + macro_rules! impl_vec_sel { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSel for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self { + let b = simd_and(b, transmute(c)); + let a = simd_and(self, simd_xor(transmute(c), transmute(vector_signed_char([!0; 16])))); + simd_or(a, b) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSel for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self { + // defer to the implementation with an unsigned mask + self.vec_sel(b, transmute::<_, t_u!($ty)>(c)) + } + } + )* + } + } + + impl_vec_sel! { + vector_signed_char + vector_signed_short + vector_signed_int + vector_signed_long_long + + vector_unsigned_char + vector_unsigned_short + vector_unsigned_int + vector_unsigned_long_long + + vector_bool_char + vector_bool_short + vector_bool_int + vector_bool_long_long + + vector_float + vector_double + } } /// Load Count to Block Boundary @@ -3837,6 +3889,14 @@ pub unsafe fn vec_gather_element( a.vec_gather_element::(b, c) } +/// Vector Select +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sel, U>(a: T, b: T, c: U) -> T { + a.vec_sel(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -4785,6 +4845,20 @@ mod tests { assert_eq!(d.as_array(), &[0xF00, 0]); } + #[simd_test(enable = "vector")] + fn test_vec_sel() { + let a = vector_signed_int([1, 2, 3, 4]); + let b = vector_signed_int([5, 6, 7, 8]); + + let e = vector_unsigned_int([9, 10, 11, 12]); + let f = vector_unsigned_int([9, 9, 11, 11]); + + let c: vector_bool_int = unsafe { simd_eq(e, f) }; + assert_eq!(c.as_array(), &[!0, 0, !0, 0]); + let d: vector_signed_int = unsafe { vec_sel(a, b, c) }; + assert_eq!(d.as_array(), &[5, 2, 7, 4]); + } + #[simd_test(enable = "vector")] fn test_vec_gather_element() { let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; From a9a6405ffc568809caa5ae604de39bb9164ee79e Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 4 Mar 2025 21:27:27 +0100 Subject: [PATCH 04/30] add `vec_scatter` --- crates/core_arch/src/s390x/vector.rs | 68 +++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index f55c28428d..098e018fa2 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2606,7 +2606,7 @@ mod sealed { ) -> vector_unsigned_int { static_assert_uimm_bits!(D, 2); let offset: u32 = simd_extract(b, D); - let ptr = c.byte_offset(offset as isize); + let ptr = c.byte_add(offset as usize); let value = ptr.read(); simd_insert(a, D, value) } @@ -2621,7 +2621,7 @@ mod sealed { ) -> vector_unsigned_long_long { static_assert_uimm_bits!(D, 1); let offset: u64 = simd_extract(b, D); - let ptr = c.byte_offset(offset as isize); + let ptr = c.byte_add(offset as usize); let value = ptr.read(); simd_insert(a, D, value) } @@ -2668,6 +2668,70 @@ mod sealed { vgeg vector_double } + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vscef, D = 3))] + unsafe fn vscef(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) { + static_assert_uimm_bits!(D, 2); + let value = simd_extract(a, D); + let offset: u32 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + ptr.write(value); + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vsceg, D = 1))] + unsafe fn vsceg( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: *mut u64, + ) { + static_assert_uimm_bits!(D, 1); + let value = simd_extract(a, D); + let offset: u64 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + ptr.write(value); + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorScatterElement { + type Element; + type Offset; + unsafe fn vec_scatter_element(self, b: Self::Offset, c: *mut Self::Element); + } + + macro_rules! impl_vec_scatter_element { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorScatterElement for $ty { + type Element = l_t_t!($ty); + type Offset = t_u!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_scatter_element(self, b: Self::Offset, c: *mut Self::Element) { + $instr::(transmute(self), b, c.cast()) + } + } + )* + } + } + + impl_vec_scatter_element! { + vscef vector_signed_int + vscef vector_bool_int + vscef vector_unsigned_int + + vsceg vector_signed_long_long + vsceg vector_bool_long_long + vsceg vector_unsigned_long_long + + vscef vector_float + vsceg vector_double + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorSel: Sized { unsafe fn vec_sel(self, b: Self, c: Mask) -> Self; From 25746d14bf5d73f82ad5dec251df78a467462bb3 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 12:44:01 +0100 Subject: [PATCH 05/30] add `vec_fp_test_data_class` --- crates/core_arch/src/s390x/vector.rs | 125 +++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 098e018fa2..7db2015964 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -208,6 +208,9 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128; #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple; + #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -2783,6 +2786,38 @@ mod sealed { vector_float vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorFpTestDataClass { + type Result; + unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFpTestDataClass for vector_float { + type Result = vector_bool_int; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result { + let PackedTuple { x, y } = vftcisb(self, CLASS); + unsafe { ptr.write(y) }; + x + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFpTestDataClass for vector_double { + type Result = vector_bool_long_long; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result { + let PackedTuple { x, y } = vftcidb(self, CLASS); + unsafe { ptr.write(y) }; + x + } + } } /// Load Count to Block Boundary @@ -3961,6 +3996,61 @@ pub unsafe fn vec_sel, U>(a: T, b: T, c: U) -> T { a.vec_sel(b, c) } +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NOT_NORMAL: u32 = + __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY; + +/// Vector Floating-Point Test Data Class +/// +/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_fp_test_data_class( + a: T, + c: *mut i32, +) -> T::Result { + a.vec_fp_test_data_class::(c) +} + #[cfg(test)] mod tests { use super::*; @@ -4946,4 +5036,39 @@ mod tests { assert_eq!(d2.as_array(), &[25, 2, 3, 4]); } } + + #[simd_test(enable = "vector")] + fn test_vec_fp_test_data_class() { + let mut cc = 42; + + let v1 = vector_double([0.0, f64::NAN]); + let v2 = vector_double([f64::INFINITY, 1.0]); + let v3 = vector_double([1.0, 2.0]); + + unsafe { + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[!0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[0, !0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[!0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc); + assert_eq!(cc, 3); + assert_eq!(d.as_array(), &[0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[0, !0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc); + assert_eq!(cc, 0); + assert_eq!(d.as_array(), &[!0, !0]); + } + } } From 900615014960b315a2fe23ab3c5a28841cdbdeb6 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 14:23:15 +0100 Subject: [PATCH 06/30] add `vec_test_mask` --- crates/core_arch/src/s390x/vector.rs | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 7db2015964..75d4337df0 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -211,6 +211,8 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple; #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -2818,6 +2820,44 @@ mod sealed { x } } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorTestMask { + type Mask; + unsafe fn vec_test_mask(self, other: Self::Mask) -> i32; + } + + macro_rules! impl_vec_test_mask { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorTestMask for $ty { + type Mask = t_u!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 { + vtm(transmute(self), transmute(other)) + } + } + )* + } + } + + impl_vec_test_mask! { + vector_signed_char + vector_signed_short + vector_signed_int + vector_signed_long_long + + vector_unsigned_char + vector_unsigned_short + vector_unsigned_int + vector_unsigned_long_long + + vector_float + vector_double + } } /// Load Count to Block Boundary @@ -4051,6 +4091,16 @@ pub unsafe fn vec_fp_test_data_class(c) } +/// Vector Test under Mask +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_test_mask(a: T, b: T::Mask) -> i32 { + // I can't find much information about this, but this might just be a check for whether the + // bitwise and of a and b is non-zero? + a.vec_test_mask(b) +} + #[cfg(test)] mod tests { use super::*; @@ -5071,4 +5121,25 @@ mod tests { assert_eq!(d.as_array(), &[!0, !0]); } } + + #[simd_test(enable = "vector")] + fn test_vec_test_mask() { + unsafe { + let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]); + let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]); + assert_eq!(vec_test_mask(v, m), 3); + + let v = vector_unsigned_long_long([u64::MAX; 2]); + let m = vector_unsigned_long_long([0; 2]); + assert_eq!(vec_test_mask(v, m), 0); + + let v = vector_unsigned_long_long([0; 2]); + let m = vector_unsigned_long_long([u64::MAX; 2]); + assert_eq!(vec_test_mask(v, m), 0); + + let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]); + let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]); + assert_eq!(vec_test_mask(v, m), 3); + } + } } From 7914100b7c55d9861f12993404d77faff29e5657 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 15:07:59 +0100 Subject: [PATCH 07/30] add `vec_search_string_cc` and `vec_search_string_until_zero_cc` --- crates/core_arch/src/s390x/vector.rs | 159 +++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 75d4337df0..65f36bf4d0 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -213,6 +213,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple; #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32; + + #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple; + + #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -2858,6 +2866,63 @@ mod sealed { vector_float vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSearchString { + unsafe fn vec_search_string_cc( + self, + b: Self, + c: vector_unsigned_char, + d: &mut i32, + ) -> vector_unsigned_char; + + unsafe fn vec_search_string_until_zero_cc( + self, + b: Self, + c: vector_unsigned_char, + d: &mut i32, + ) -> vector_unsigned_char; + } + + macro_rules! impl_vec_search_string{ + ($($intr_s:ident $intr_sz:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSearchString for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char, d: &mut i32) -> vector_unsigned_char { + let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c); + *d = y; + x + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char, d: &mut i32) -> vector_unsigned_char { + let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c); + *d = y; + x + } + } + + )* + } + } + + impl_vec_search_string! { + vstrsb vstrszb vector_signed_char + vstrsb vstrszb vector_bool_char + vstrsb vstrszb vector_unsigned_char + + vstrsh vstrszh vector_signed_short + vstrsh vstrszh vector_bool_short + vstrsh vstrszh vector_unsigned_short + + vstrsf vstrszf vector_signed_int + vstrsf vstrszf vector_bool_int + vstrsf vstrszf vector_unsigned_int + } } /// Load Count to Block Boundary @@ -4101,6 +4166,32 @@ pub unsafe fn vec_test_mask(a: T, b: T::Mask) -> i32 a.vec_test_mask(b) } +/// Vector Search String +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_search_string_cc( + a: T, + b: T, + c: vector_unsigned_char, + d: &mut i32, +) -> vector_unsigned_char { + a.vec_search_string_cc(b, c, d) +} + +/// Vector Search String Until Zero +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_search_string_until_zero_cc( + a: T, + b: T, + c: vector_unsigned_char, + d: &mut i32, +) -> vector_unsigned_char { + a.vec_search_string_until_zero_cc(b, c, d) +} + #[cfg(test)] mod tests { use super::*; @@ -5142,4 +5233,72 @@ mod tests { assert_eq!(vec_test_mask(v, m), 3); } } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_search_string_cc() { + unsafe { + let b = vector_unsigned_char(*b"ABCD------------"); + let c = vector_unsigned_char([4; 16]); + let mut d = 0i32; + + let haystack = vector_unsigned_char(*b"__ABCD__________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 2); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"___ABCD_________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 3); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"________________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0_________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0__ABCD___"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 9); + assert_eq!(d, 2); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_search_string_until_zero_cc() { + unsafe { + let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0"); + let c = vector_unsigned_char([16; 16]); + let mut d = 0i32; + + let haystack = vector_unsigned_char(*b"__ABCD__________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 2); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"___ABCD_________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 3); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"________________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0_________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 1); + + let haystack = vector_unsigned_char(*b"______\0__ABCD___"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 1); + } + } } From a48b50419d7d0e8baa74763ce518d32283cb2d32 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 16:34:03 +0100 Subject: [PATCH 08/30] add `vec_double` and `vec_float` --- crates/core_arch/src/s390x/vector.rs | 158 +++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 65f36bf4d0..8a63b2ad52 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2923,6 +2923,86 @@ mod sealed { vstrsf vstrszf vector_bool_int vstrsf vstrszf vector_unsigned_int } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vcdgb))] + pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double { + simd_as(a) + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vcdlgb))] + pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double { + simd_as(a) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorDouble { + unsafe fn vec_double(self) -> vector_double; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorDouble for vector_signed_long_long { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_double(self) -> vector_double { + vcdgb(self) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorDouble for vector_unsigned_long_long { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_double(self) -> vector_double { + vcdlgb(self) + } + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr( + all(test, target_feature = "vector-enhancements-2"), + assert_instr(vcefb) + )] + pub unsafe fn vcefb(a: vector_signed_int) -> vector_float { + simd_as(a) + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr( + all(test, target_feature = "vector-enhancements-2"), + assert_instr(vcelfb) + )] + pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float { + simd_as(a) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorFloat { + unsafe fn vec_float(self) -> vector_float; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFloat for vector_signed_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_float(self) -> vector_float { + vcefb(self) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFloat for vector_unsigned_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_float(self) -> vector_float { + vcelfb(self) + } + } } /// Load Count to Block Boundary @@ -4192,6 +4272,48 @@ pub unsafe fn vec_search_string_until_zero_cc( a.vec_search_string_until_zero_cc(b, c, d) } +/// Vector Convert from float (even elements) to double +#[inline] +#[target_feature(enable = "vector-enhancements-1")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +// FIXME: this emits `vflls` where `vldeb` is expected +// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))] +pub unsafe fn vec_doublee(a: vector_float) -> vector_double { + let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) }); + simd_as(even) +} + +/// Vector Convert from double to float (even elements) +#[inline] +#[target_feature(enable = "vector-enhancements-1")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +// FIXME: the C version uses a shuffle mask with poison; we can't do that +// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))] +pub unsafe fn vec_floate(a: vector_double) -> vector_float { + let truncated: f32x2 = simd_as(a); + simd_shuffle( + truncated, + truncated, + const { u32x4::from_array([0, 0, 1, 1]) }, + ) +} + +/// Vector Convert from int to float +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float { + a.vec_float() +} + +/// Vector Convert from long long to double +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double { + a.vec_double() +} + #[cfg(test)] mod tests { use super::*; @@ -5301,4 +5423,40 @@ mod tests { assert_eq!(d, 1); } } + + #[simd_test(enable = "vector")] + fn test_vec_doublee() { + unsafe { + let v = vector_float([1.0, 2.0, 3.0, 4.0]); + assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]); + + let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]); + let d = vec_doublee(v); + assert!(d.as_array()[0].is_nan()); + assert_eq!(d.as_array()[1], f64::INFINITY); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_floate() { + // NOTE: indices 1 and 3 can have an arbitrary value. With the C version + // these are poison values, our version initializes the memory but its + // value still should not be relied upon by application code. + unsafe { + let v = vector_double([1.0, 2.0]); + let d = vec_floate(v); + assert_eq!(d.as_array()[0], 1.0); + assert_eq!(d.as_array()[2], 2.0); + + let v = vector_double([f64::NAN, f64::INFINITY]); + let d = vec_floate(v); + assert!(d.as_array()[0].is_nan()); + assert_eq!(d.as_array()[2], f32::INFINITY); + + let v = vector_double([f64::MIN, f64::MAX]); + let d = vec_floate(v); + assert_eq!(d.as_array()[0], f64::MIN as f32); + assert_eq!(d.as_array()[2], f64::MAX as f32); + } + } } From 814f140fb4cc70dbe85cc7c36cdc59fc7c080639 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 18:42:56 +0100 Subject: [PATCH 09/30] add `vec_extend_s64` --- crates/core_arch/src/s390x/vector.rs | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 8a63b2ad52..918cf0a8b1 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -3003,6 +3003,72 @@ mod sealed { vcelfb(self) } } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorExtendSigned64 { + unsafe fn vec_extend_s64(self) -> vector_signed_long_long; + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegb))] + pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i8x2>( + a, + a, + const { u32x2::from_array([7, 15]) }, + )) + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegh))] + pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i16x2>( + a, + a, + const { u32x2::from_array([3, 7]) }, + )) + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegf))] + pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i32x2>( + a, + a, + const { u32x2::from_array([1, 3]) }, + )) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_char { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegb(self) + } + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_short { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegh(self) + } + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegf(self) + } + } } /// Load Count to Block Boundary @@ -4314,6 +4380,14 @@ pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double { a.vec_double() } +/// Vector Sign Extend to Doubleword +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long { + a.vec_extend_s64() +} + #[cfg(test)] mod tests { use super::*; @@ -5459,4 +5533,18 @@ mod tests { assert_eq!(d.as_array()[2], f64::MAX as f32); } } + + #[simd_test(enable = "vector")] + fn test_vec_extend_s64() { + unsafe { + let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]); + + let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]); + assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]); + + let v = vector_signed_int([0, 1, 2, 3]); + assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]); + } + } } From cd185a8609511a90d34a2045f78352ba2614f1b8 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 19:13:48 +0100 Subject: [PATCH 10/30] add `vec_signed` and `vec_unsigned` --- crates/core_arch/src/s390x/vector.rs | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 918cf0a8b1..2b9f94017e 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -3069,6 +3069,35 @@ mod sealed { vsegf(self) } } + + // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats. + // This is what C provides, but even IBM does not clearly document these constraints. + // + // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSigned { + type Result; + unsafe fn vec_signed(self) -> Self::Result; + } + + test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] } + test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] } + + impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int } + impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorUnsigned { + type Result; + unsafe fn vec_unsigned(self) -> Self::Result; + } + + test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] } + test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] } + + impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int } + impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long } } /// Load Count to Block Boundary @@ -4388,6 +4417,22 @@ pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_sig a.vec_extend_s64() } +/// Vector Convert floating point to signed +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_signed(a: T) -> T::Result { + a.vec_signed() +} + +/// Vector Convert floating point to unsigned +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_unsigned(a: T) -> T::Result { + a.vec_unsigned() +} + #[cfg(test)] mod tests { use super::*; @@ -5547,4 +5592,27 @@ mod tests { assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]); } } + + #[simd_test(enable = "vector")] + fn test_vec_signed() { + unsafe { + let v = vector_float([1.0, 2.5, -2.5, -0.0]); + assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]); + + let v = vector_double([2.5, -2.5]); + assert_eq!(vec_signed(v).as_array(), &[2, -2]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_unsigned() { + // NOTE: converting a negative floating point value is UB! + unsafe { + let v = vector_float([1.0, 2.5, 3.5, 0.0]); + assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]); + + let v = vector_double([2.5, 3.5]); + assert_eq!(vec_unsigned(v).as_array(), &[2, 3]); + } + } } From 900d502e3be4aa327f450b5a9195595cbab80464 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 21:54:57 +0100 Subject: [PATCH 11/30] add `vec_cp_until_zero` and `vec_cp_until_zero_cc` --- crates/core_arch/src/s390x/vector.rs | 115 +++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 2b9f94017e..69fabbbe63 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -221,6 +221,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple; #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple; #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple; + + #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int; + + #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple; + #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -3098,6 +3106,68 @@ mod sealed { impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int } impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCopyUntilZero { + unsafe fn vec_cp_until_zero(self) -> Self; + } + + test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] } + test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] } + test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCopyUntilZeroCC { + unsafe fn vec_cp_until_zero_cc(self, cc: *mut i32) -> Self; + } + + test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple [vistrbs, vistrbs] } + test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple [vistrhs, vistrhs] } + test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple [vistrfs, vistrfs] } + + macro_rules! impl_vec_copy_until_zero_cc { + ($($intr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCopyUntilZeroCC for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cp_until_zero_cc(self, cc: *mut i32) -> Self { + let PackedTuple { x,y } = $intr(transmute(self)); + cc.write(y); + transmute(x) + } + } + + )* + } + } + + impl_vec_copy_until_zero_cc! { + vec_vistrbs vector_signed_char + vec_vistrbs vector_bool_char + vec_vistrbs vector_unsigned_char + + vec_vistrhs vector_signed_short + vec_vistrhs vector_bool_short + vec_vistrhs vector_unsigned_short + + vec_vistrfs vector_signed_int + vec_vistrfs vector_bool_int + vec_vistrfs vector_unsigned_int + } } /// Load Count to Block Boundary @@ -4433,6 +4503,22 @@ pub unsafe fn vec_unsigned(a: T) -> T::Result { a.vec_unsigned() } +/// Vector Copy Until Zero +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cp_until_zero(a: T) -> T { + a.vec_cp_until_zero() +} + +/// Vector Copy Until Zero +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cp_until_zero_cc(a: T, cc: *mut i32) -> T { + a.vec_cp_until_zero_cc(cc) +} + #[cfg(test)] mod tests { use super::*; @@ -5615,4 +5701,33 @@ mod tests { assert_eq!(vec_unsigned(v).as_array(), &[2, 3]); } } + + #[simd_test(enable = "vector")] + fn test_vec_cp_until_zero() { + unsafe { + let v = vector_signed_int([1, 2, 3, 4]); + let d = vec_cp_until_zero(v); + assert_eq!(d.as_array(), &[1, 2, 3, 4]); + + let v = vector_signed_int([1, 2, 0, 4]); + let d = vec_cp_until_zero(v); + assert_eq!(d.as_array(), &[1, 2, 0, 0]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_cp_until_zero_cc() { + let mut cc = 0; + unsafe { + let v = vector_signed_int([1, 2, 3, 4]); + let d = vec_cp_until_zero_cc(v, &mut cc); + assert_eq!(d.as_array(), &[1, 2, 3, 4]); + assert_eq!(cc, 3); + + let v = vector_signed_int([1, 2, 0, 4]); + let d = vec_cp_until_zero_cc(v, &mut cc); + assert_eq!(d.as_array(), &[1, 2, 0, 0]); + assert_eq!(cc, 0); + } + } } From 01c208f1f4ba44a821ab968fb038812081a43727 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 5 Mar 2025 22:14:45 +0100 Subject: [PATCH 12/30] add `vec_msum_u128` --- crates/core_arch/src/s390x/vector.rs | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 69fabbbe63..c30d35f16a 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -229,6 +229,8 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple; #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple; #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple; + + #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -4519,6 +4521,27 @@ pub unsafe fn vec_cp_until_zero_cc(a: T, cc: * a.vec_cp_until_zero_cc(cc) } +/// Vector Multiply Sum Logical +#[inline] +#[target_feature(enable = "vector-enhancements-1")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr( + all(test, target_feature = "vector-enhancements-1"), + assert_instr(vmslg, D = 4) +)] +pub unsafe fn vec_msum_u128( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: vector_unsigned_char, +) -> vector_unsigned_char { + const { + if !matches!(D, 0 | 4 | 8 | 12) { + panic!("D needs to be one of 0, 4, 8, 12"); + } + }; + transmute(vmslg(a, b, transmute(c), D)) +} + #[cfg(test)] mod tests { use super::*; @@ -5730,4 +5753,26 @@ mod tests { assert_eq!(cc, 0); } } + + #[simd_test(enable = "vector-enhancements-1")] + fn test_vec_msum_u128() { + let a = vector_unsigned_long_long([1, 2]); + let b = vector_unsigned_long_long([3, 4]); + + unsafe { + let c: vector_unsigned_char = transmute(100u128); + + let d: u128 = transmute(vec_msum_u128::<0>(a, b, c)); + assert_eq!(d, (1 * 3) + (2 * 4) + 100); + + let d: u128 = transmute(vec_msum_u128::<4>(a, b, c)); + assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100); + + let d: u128 = transmute(vec_msum_u128::<8>(a, b, c)); + assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100); + + let d: u128 = transmute(vec_msum_u128::<12>(a, b, c)); + assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100); + } + } } From bd478a9457026ab160e95cc0b0bd469d2b7ef1f1 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 00:40:06 +0100 Subject: [PATCH 13/30] add `vec_sld`, `vec_sldb`, `vec_sldw` and `vec_srdb` --- crates/core_arch/src/s390x/vector.rs | 165 +++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index c30d35f16a..04cb951316 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -97,6 +97,10 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.fshl.v16i8"] fn fshlb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; #[link_name = "llvm.fshl.v8i16"] fn fshlh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; #[link_name = "llvm.fshl.v4i32"] fn fshlf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; @@ -3170,6 +3174,86 @@ mod sealed { vec_vistrfs vector_bool_int vec_vistrfs vector_unsigned_int } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSrdb { + unsafe fn vec_srdb(self, b: Self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSld { + unsafe fn vec_sld(self, b: Self) -> Self; + + unsafe fn vec_sldw(self, b: Self) -> Self; + + unsafe fn vec_sldb(self, b: Self) -> Self; + } + + // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955 + // ideally we could implement this in terms of llvm.fshl.i128 + // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128; + // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } )) + + macro_rules! impl_vec_sld { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSld for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sld(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 4); + transmute(vsldb(transmute(self), transmute(b), C)) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sldw(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 2); + transmute(vsldb(transmute(self), transmute(b), const { 4 * C })) + } + + #[inline] + #[target_feature(enable = "vector-enhancements-2")] + unsafe fn vec_sldb(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 3); + transmute(vsld(transmute(self), transmute(b), C)) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSrdb for $ty { + #[inline] + #[target_feature(enable = "vector-enhancements-2")] + unsafe fn vec_srdb(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 3); + transmute(vsrd(transmute(self), transmute(b), C)) + } + } + )* + } + } + + impl_vec_sld! { + vector_signed_char + vector_bool_char + vector_unsigned_char + + vector_signed_short + vector_bool_short + vector_unsigned_short + + vector_signed_int + vector_bool_int + vector_unsigned_int + + vector_signed_long_long + vector_bool_long_long + vector_unsigned_long_long + + vector_float + vector_double + } } /// Load Count to Block Boundary @@ -4542,6 +4626,42 @@ pub unsafe fn vec_msum_u128( transmute(vmslg(a, b, transmute(c), D)) } +/// Vector Shift Left Double by Byte +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sld(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 4); + a.vec_sld::(b) +} + +/// Vector Shift Left Double by Word +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sldw(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 2); + a.vec_sldw::(b) +} + +/// Vector Shift Left Double by Bit +#[inline] +#[target_feature(enable = "vector-enhancements-2")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sldb(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 3); + a.vec_sldb::(b) +} + +/// Vector Shift Right Double by Bit +#[inline] +#[target_feature(enable = "vector-enhancements-2")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_srdb(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 3); + a.vec_srdb::(b) +} + #[cfg(test)] mod tests { use super::*; @@ -5775,4 +5895,49 @@ mod tests { assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100); } } + + #[simd_test(enable = "vector")] + fn test_vec_sld() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sld::<_, 4>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_sldw() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sldw::<_, 1>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_sldb() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sldb::<_, 4>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_srdb() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_srdb::<_, 4>(a, b); + println!("{:x?}", &d); + assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + } + } } From 747870fd45407eaac061dd3e6681b7f7cad05d52 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:27:00 +0100 Subject: [PATCH 14/30] add `vec_cmprg` --- crates/core_arch/src/s390x/vector.rs | 105 +++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 04cb951316..accae48b29 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -235,6 +235,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple; #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128; + + #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char; + #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short; + #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int; + + #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -365,6 +373,13 @@ const fn validate_block_boundary(block_boundary: u16) -> u32 { block_boundary as u32 >> 7 } +enum FindImm { + Eq = 4, + Ne = 12, + EqIdx = 0, + NeIdx = 8, +} + #[macro_use] mod sealed { use super::*; @@ -1922,13 +1937,6 @@ mod sealed { }; } - enum FindImm { - Eq = 4, - Ne = 12, - EqIdx = 0, - NeIdx = 8, - } - #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEq { type Result; @@ -3254,6 +3262,56 @@ mod sealed { vector_float vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCompareRange: Sized { + type Result; + + unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result; + unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32); + } + + macro_rules! impl_compare_range { + ($($ty:ident $vstrc:ident $vstrcs:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCompareRange for $ty { + type Result = t_b!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result { + const { + if !matches!(IMM, 0 | 4 | 8 | 12) { + panic!("IMM needs to be one of 0, 4, 8, 12"); + } + }; + + $vstrc(self, b, c, IMM) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32) { + const { + if !matches!(IMM, 0 | 4 | 8 | 12) { + panic!("IMM needs to be one of 0, 4, 8, 12"); + } + }; + + let PackedTuple { x, y } = $vstrcs(self, b, c, IMM); + (x,y) + } + } + )* + } + } + + impl_compare_range! { + vector_unsigned_char vstrcb vstrcbs + vector_unsigned_short vstrch vstrchs + vector_unsigned_int vstrcf vstrcfs + } } /// Load Count to Block Boundary @@ -4662,6 +4720,13 @@ pub unsafe fn vec_srdb(a: T, b: T) -> T { a.vec_srdb::(b) } +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::Eq as u32 }>(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -5936,8 +6001,32 @@ mod tests { unsafe { let d = vec_srdb::<_, 4>(a, b); - println!("{:x?}", &d); assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); } } + + #[simd_test(enable = "vector")] + fn test_vec_cmprg() { + const GT: u32 = 0x20000000; + const LT: u32 = 0x40000000; + const EQ: u32 = 0x80000000; + + let a = vector_unsigned_int([11, 22, 33, 44]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, !0, 0]); + + let c = vector_unsigned_int([GT, LT, 0, 0]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, 0, 0]); + + let a = vector_unsigned_int([11, 22, 33, 30]); + let b = vector_unsigned_int([10, 20, 30, 30]); + + let c = vector_unsigned_int([GT, LT, EQ, EQ]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, 0, !0]); + } } From 6a03b2fae4e71959405f50f80612f6eb47bd9660 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:32:16 +0100 Subject: [PATCH 15/30] add `vec_cmpnrg` --- crates/core_arch/src/s390x/vector.rs | 38 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index accae48b29..72514c34fb 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -4720,6 +4720,7 @@ pub unsafe fn vec_srdb(a: T, b: T) -> T { a.vec_srdb::(b) } +/// Vector Compare Ranges #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -4727,6 +4728,14 @@ pub unsafe fn vec_cmprg(a: T, b: T, c: T) -> T::R a.vstrc::<{ FindImm::Eq as u32 }>(b, c) } +/// Vector Compare Not in Ranges +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::Ne as u32 }>(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -6005,12 +6014,12 @@ mod tests { } } + const GT: u32 = 0x20000000; + const LT: u32 = 0x40000000; + const EQ: u32 = 0x80000000; + #[simd_test(enable = "vector")] fn test_vec_cmprg() { - const GT: u32 = 0x20000000; - const LT: u32 = 0x40000000; - const EQ: u32 = 0x80000000; - let a = vector_unsigned_int([11, 22, 33, 44]); let b = vector_unsigned_int([10, 20, 30, 40]); @@ -6029,4 +6038,25 @@ mod tests { let d = unsafe { vec_cmprg(a, b, c) }; assert_eq!(d.as_array(), &[!0, 0, 0, !0]); } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg() { + let a = vector_unsigned_int([11, 22, 33, 44]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, 0, !0]); + + let c = vector_unsigned_int([GT, LT, 0, 0]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, !0, !0]); + + let a = vector_unsigned_int([11, 22, 33, 30]); + let b = vector_unsigned_int([10, 20, 30, 30]); + + let c = vector_unsigned_int([GT, LT, EQ, EQ]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, !0, 0]); + } } From 434d285c3e5c3e08f238536bf940abbdac06ddcf Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:35:33 +0100 Subject: [PATCH 16/30] add `vec_cmprg_idx` and `vec_cmpnrg_idx` --- crates/core_arch/src/s390x/vector.rs | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 72514c34fb..e4de4d107d 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -4736,6 +4736,22 @@ pub unsafe fn vec_cmpnrg(a: T, b: T, c: T) -> T:: a.vstrc::<{ FindImm::Ne as u32 }>(b, c) } +/// Vector Compare Ranges Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg_idx(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c) +} + +/// Vector Compare Not in Ranges Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg_idx(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -6059,4 +6075,24 @@ mod tests { let d = unsafe { vec_cmpnrg(a, b, c) }; assert_eq!(d.as_array(), &[0, !0, !0, 0]); } + + #[simd_test(enable = "vector")] + fn test_vec_cmprg_idx() { + let a = vector_unsigned_int([1, 11, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 4, 0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg_idx() { + let a = vector_unsigned_int([1, 11, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 0, 0, 0]); + } } From 66b2a463e6bab3811e577b9e79e412973edd05e6 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:39:16 +0100 Subject: [PATCH 17/30] add `vec_cmprg_cc` and friends --- crates/core_arch/src/s390x/vector.rs | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index e4de4d107d..dcd1aa6947 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -4752,6 +4752,46 @@ pub unsafe fn vec_cmpnrg_idx(a: T, b: T, c: T) -> a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c) } +/// Vector Compare Ranges with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { + let (x,y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c); + d.write(y); + x +} + +/// Vector Compare Not in Ranges with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { + let (x,y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c); + d.write(y); + x +} + +/// Vector Compare Ranges Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg_idx_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { + let (x,y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c); + d.write(y); + x +} + +/// Vector Compare Not in Ranges Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg_idx_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { + let (x,y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c); + d.write(y); + x +} + #[cfg(test)] mod tests { use super::*; From 4dbc8afdfe7d3693960840098371a90e05048082 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:48:18 +0100 Subject: [PATCH 18/30] add `vec_cmprg_or_0_idx` and `vec_cmpnrg_or_0_idx` --- crates/core_arch/src/s390x/vector.rs | 124 +++++++++++++++++++++------ 1 file changed, 100 insertions(+), 24 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index dcd1aa6947..54bfc98af5 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -243,6 +243,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char; + #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short; + #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int; + + #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -3268,11 +3276,19 @@ mod sealed { type Result; unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result; + unsafe fn vstrcz(self, b: Self, c: Self) -> Self::Result; unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32); + unsafe fn vstrczs(self, b: Self, c: Self) -> (Self::Result, i32); + } + + const fn validate_compare_range_imm(imm: u32) { + if !matches!(imm, 0 | 4 | 8 | 12) { + panic!("IMM needs to be one of 0, 4, 8, 12"); + } } macro_rules! impl_compare_range { - ($($ty:ident $vstrc:ident $vstrcs:ident)*) => { + ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => { $( #[unstable(feature = "stdarch_s390x", issue = "135681")] impl VectorCompareRange for $ty { @@ -3281,36 +3297,41 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result { - const { - if !matches!(IMM, 0 | 4 | 8 | 12) { - panic!("IMM needs to be one of 0, 4, 8, 12"); - } - }; - + const { validate_compare_range_imm }; $vstrc(self, b, c, IMM) } #[inline] #[target_feature(enable = "vector")] - unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32) { - const { - if !matches!(IMM, 0 | 4 | 8 | 12) { - panic!("IMM needs to be one of 0, 4, 8, 12"); - } - }; + unsafe fn vstrcz(self, b: Self, c: Self) -> Self::Result { + const { validate_compare_range_imm }; + $vstrcz(self, b, c, IMM) + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32) { + const { validate_compare_range_imm }; let PackedTuple { x, y } = $vstrcs(self, b, c, IMM); (x,y) } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrczs(self, b: Self, c: Self) -> (Self::Result, i32) { + const { validate_compare_range_imm }; + let PackedTuple { x, y } = $vstrczs(self, b, c, IMM); + (x,y) + } } )* } } impl_compare_range! { - vector_unsigned_char vstrcb vstrcbs - vector_unsigned_short vstrch vstrchs - vector_unsigned_int vstrcf vstrcfs + vector_unsigned_char vstrcb vstrcbs vstrczb vstrczbs + vector_unsigned_short vstrch vstrchs vstrczh vstrczhs + vector_unsigned_int vstrcf vstrcfs vstrczf vstrczfs } } @@ -4756,8 +4777,13 @@ pub unsafe fn vec_cmpnrg_idx(a: T, b: T, c: T) -> #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cmprg_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { - let (x,y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c); +pub unsafe fn vec_cmprg_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c); d.write(y); x } @@ -4766,8 +4792,13 @@ pub unsafe fn vec_cmprg_cc(a: T, b: T, c: T, d: * #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cmpnrg_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { - let (x,y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c); +pub unsafe fn vec_cmpnrg_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c); d.write(y); x } @@ -4776,8 +4807,13 @@ pub unsafe fn vec_cmpnrg_cc(a: T, b: T, c: T, d: #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cmprg_idx_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { - let (x,y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c); +pub unsafe fn vec_cmprg_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c); d.write(y); x } @@ -4786,12 +4822,32 @@ pub unsafe fn vec_cmprg_idx_cc(a: T, b: T, c: T, #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cmpnrg_idx_cc(a: T, b: T, c: T, d: *mut i32) -> T::Result { - let (x,y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c); +pub unsafe fn vec_cmpnrg_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c); d.write(y); x } +/// Vector Compare Ranges or Zero Index#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg_or_0_idx(a: T, b: T, c: T) -> T::Result { + a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c) +} + +/// Vector Compare Not in Ranges or Zero Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg_or_0_idx(a: T, b: T, c: T) -> T::Result { + a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c) +} + #[cfg(test)] mod tests { use super::*; @@ -6135,4 +6191,24 @@ mod tests { let d = unsafe { vec_cmpnrg_idx(a, b, c) }; assert_eq!(d.as_array(), &[0, 0, 0, 0]); } + + #[simd_test(enable = "vector")] + fn test_vec_cmprg_or_0_idx() { + let a = vector_unsigned_int([1, 0, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg_or_0_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 4, 0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg_or_0_idx() { + let a = vector_unsigned_int([11, 33, 0, 22]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 8, 0, 0]); + } } From 83d5d2a7a2f6668ee38322913fc09d941cadce77 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:50:04 +0100 Subject: [PATCH 19/30] add `vec_cmprg_or_0_idx_cc` and `vec_cmpnrg_or_0_idx_cc` --- crates/core_arch/src/s390x/vector.rs | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 54bfc98af5..a0d60e55d1 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -4848,6 +4848,36 @@ pub unsafe fn vec_cmpnrg_or_0_idx(a: T, b: T, c: a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c) } +/// Vector Compare Ranges or Zero Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmprg_or_0_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c); + d.write(y); + x +} + +/// Vector Compare Not in Ranges or Zero Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpnrg_or_0_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c); + d.write(y); + x +} + #[cfg(test)] mod tests { use super::*; From e67cadf12e6c55c6fd8ae2d850c4fd097744c228 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 6 Mar 2025 16:51:22 +0100 Subject: [PATCH 20/30] let's not use `&mut` until we get confirmation it's OK --- crates/core_arch/src/s390x/vector.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index a0d60e55d1..c9d29c8d7d 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2903,14 +2903,14 @@ mod sealed { self, b: Self, c: vector_unsigned_char, - d: &mut i32, + d: *mut i32, ) -> vector_unsigned_char; unsafe fn vec_search_string_until_zero_cc( self, b: Self, c: vector_unsigned_char, - d: &mut i32, + d: *mut i32, ) -> vector_unsigned_char; } @@ -2921,17 +2921,17 @@ mod sealed { impl VectorSearchString for $ty { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char, d: &mut i32) -> vector_unsigned_char { + unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char, d: *mut i32) -> vector_unsigned_char { let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c); - *d = y; + d.write(y); x } #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char, d: &mut i32) -> vector_unsigned_char { + unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char, d: *mut i32) -> vector_unsigned_char { let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c); - *d = y; + d.write(y); x } } @@ -4584,7 +4584,7 @@ pub unsafe fn vec_search_string_cc( a: T, b: T, c: vector_unsigned_char, - d: &mut i32, + d: *mut i32, ) -> vector_unsigned_char { a.vec_search_string_cc(b, c, d) } @@ -4597,7 +4597,7 @@ pub unsafe fn vec_search_string_until_zero_cc( a: T, b: T, c: vector_unsigned_char, - d: &mut i32, + d: *mut i32, ) -> vector_unsigned_char { a.vec_search_string_until_zero_cc(b, c, d) } From c872f5b136a94a5b808da0be0e8eda788bcc1d6b Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Fri, 7 Mar 2025 23:47:58 +0100 Subject: [PATCH 21/30] add `vec_cmpgt`, `vec_cmplt`, `vec_cmpge`, `vec_cmple` --- crates/core_arch/src/s390x/vector.rs | 112 +++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index c9d29c8d7d..803ed12b7e 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -3333,6 +3333,62 @@ mod sealed { vector_unsigned_short vstrch vstrchs vstrczh vstrczhs vector_unsigned_int vstrcf vstrcfs vstrczf vstrczfs } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorComparePredicate: Sized { + type Result; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpgt(self, other: Self) -> Self::Result { + simd_gt(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpge(self, other: Self) -> Self::Result { + simd_ge(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmplt(self, other: Self) -> Self::Result { + simd_lt(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmple(self, other: Self) -> Self::Result { + simd_le(self, other) + } + } + + macro_rules! impl_compare_predicate { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorComparePredicate for $ty { + type Result = t_b!($ty); + } + )* + } + } + + impl_compare_predicate! { + vector_signed_char + vector_unsigned_char + + vector_signed_short + vector_unsigned_short + + vector_signed_int + vector_unsigned_int + vector_float + + vector_signed_long_long + vector_unsigned_long_long + vector_double + } } /// Load Count to Block Boundary @@ -4878,6 +4934,38 @@ pub unsafe fn vec_cmpnrg_or_0_idx_cc( x } +/// Vector Compare Greater Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpgt(a: T, b: T) -> T::Result { + a.vec_cmpgt(b) +} + +/// Vector Compare Greater Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpge(a: T, b: T) -> T::Result { + a.vec_cmpge(b) +} + +/// Vector Compare Less +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmplt(a: T, b: T) -> T::Result { + a.vec_cmplt(b) +} + +/// Vector Compare Less Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmple(a: T, b: T) -> T::Result { + a.vec_cmple(b) +} + #[cfg(test)] mod tests { use super::*; @@ -6241,4 +6329,28 @@ mod tests { let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) }; assert_eq!(d.as_array(), &[0, 8, 0, 0]); } + + test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 3.14], + [2.0, f32::NAN, 5.0, 2.0], + [0, 0, 0, !0] + } + + test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 3.14], + [1.0, f32::NAN, 5.0, 2.0], + [!0, 0, 0, !0] + } + + test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [2.0, f32::NAN, 5.0, 2.0], + [!0, 0, 0, 0] + } + + test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [!0, 0, 0, !0] + } } From e8508b58f78d9a5c291ca894bfd39c4564f677cf Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Fri, 7 Mar 2025 23:59:26 +0100 Subject: [PATCH 22/30] add `vec_cmpeq` and `vec_cmpne` --- crates/core_arch/src/s390x/vector.rs | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 803ed12b7e..ef75124222 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -3389,6 +3389,54 @@ mod sealed { vector_unsigned_long_long vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorEquality: Sized { + type Result; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq(self, other: Self) -> Self::Result { + simd_eq(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne(self, other: Self) -> Self::Result { + simd_ne(self, other) + } + } + + macro_rules! impl_compare_equality { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorEquality for $ty { + type Result = t_b!($ty); + } + )* + } + } + + impl_compare_equality! { + vector_bool_char + vector_signed_char + vector_unsigned_char + + vector_bool_short + vector_signed_short + vector_unsigned_short + + vector_bool_int + vector_signed_int + vector_unsigned_int + vector_float + + vector_bool_long_long + vector_signed_long_long + vector_unsigned_long_long + vector_double + } } /// Load Count to Block Boundary @@ -4934,6 +4982,22 @@ pub unsafe fn vec_cmpnrg_or_0_idx_cc( x } +/// Vector Compare Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpeq(a: T, b: T) -> T::Result { + a.vec_cmpeq(b) +} + +/// Vector Compare Not Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpne(a: T, b: T) -> T::Result { + a.vec_cmpne(b) +} + /// Vector Compare Greater Than #[inline] #[target_feature(enable = "vector")] @@ -6353,4 +6417,16 @@ mod tests { [1.0, f32::NAN, 5.0, 3.14], [!0, 0, 0, !0] } + + test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [!0, 0, 0, 0] + } + + test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [0, !0, !0, !0] + } } From 9e19b0ed41d6ee6731c517994f6bd3f04b59d9a1 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 00:37:30 +0100 Subject: [PATCH 23/30] add `vec_cmpeq_idx` and variations --- crates/core_arch/src/s390x/vector.rs | 206 +++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index ef75124222..828b6a0423 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -251,6 +251,38 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -3437,6 +3469,107 @@ mod sealed { vector_unsigned_long_long vector_double } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorEqualityIdx: Sized { + type Result; + + unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result; + unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result; + + unsafe fn vec_cmpeq_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + unsafe fn vec_cmpne_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + + unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result; + unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result; + + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + } + + macro_rules! impl_compare_equality_idx { + ($($ty:ident $ret:ident + $cmpeq:ident $cmpne:ident + $cmpeq_or_0:ident $cmpne_or_0:ident + $cmpeq_cc:ident $cmpne_cc:ident + $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident + )*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorEqualityIdx for $ty { + type Result = $ret; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result { + transmute($cmpeq(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result { + transmute($cmpne(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result { + transmute($cmpeq_or_0(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result { + transmute($cmpne_or_0(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other)); + *cc = y; + transmute(x) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other)); + *cc = y; + transmute(x) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other)); + *cc = y; + transmute(x) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other)); + *cc = y; + transmute(x) + } + } + )* + } + } + + impl_compare_equality_idx! { + vector_signed_char vector_signed_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_bool_char vector_unsigned_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_unsigned_char vector_unsigned_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_signed_short vector_signed_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_bool_short vector_unsigned_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_unsigned_short vector_unsigned_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_signed_int vector_signed_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + vector_bool_int vector_unsigned_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + vector_unsigned_int vector_unsigned_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + } } /// Load Count to Block Boundary @@ -5030,6 +5163,79 @@ pub unsafe fn vec_cmple(a: T, b: T) -> T::Res a.vec_cmple(b) } +/// Vector Compare Equal Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpeq_idx(a: T, b: T) -> T::Result { + a.vec_cmpeq_idx(b) +} +/// Vector Compare Not Equal Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpne_idx(a: T, b: T) -> T::Result { + a.vec_cmpne_idx(b) +} +/// Vector Compare Equal Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpeq_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + a.vec_cmpeq_idx_cc(b, cc) +} +/// Vector Compare Not Equal Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpne_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + a.vec_cmpne_idx_cc(b, cc) +} +/// Vector Compare Equal or Zero Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpeq_or_0_idx(a: T, b: T) -> T::Result { + a.vec_cmpeq_or_0_idx(b) +} +/// Vector Compare Not Equal or Zero Index +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpne_or_0_idx(a: T, b: T) -> T::Result { + a.vec_cmpne_or_0_idx(b) +} +/// Vector Compare Equal or Zero Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpeq_or_0_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + a.vec_cmpeq_or_0_idx_cc(b, cc) +} +/// Vector Compare Not Equal or Zero Index with Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cmpne_or_0_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + a.vec_cmpne_or_0_idx_cc(b, cc) +} + #[cfg(test)] mod tests { use super::*; From 551ecd050157cb1f06d61bb153730934518754eb Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 16:24:16 +0100 Subject: [PATCH 24/30] add `vec_all_nan`, `vec_any_nan`, `vec_all_numeric` and `vec_any_numeric` --- crates/core_arch/src/s390x/vector.rs | 96 +++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 828b6a0423..1779cff3c4 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2862,7 +2862,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFpTestDataClass { type Result; - unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result; + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32); } #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -2871,10 +2871,9 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result { + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32) { let PackedTuple { x, y } = vftcisb(self, CLASS); - unsafe { ptr.write(y) }; - x + (x, y) } } @@ -2884,10 +2883,9 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_fp_test_data_class(self, ptr: *mut i32) -> Self::Result { + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32) { let PackedTuple { x, y } = vftcidb(self, CLASS); - unsafe { ptr.write(y) }; - x + (x, y) } } @@ -4800,7 +4798,37 @@ pub unsafe fn vec_fp_test_data_class T::Result { - a.vec_fp_test_data_class::(c) + let (x, y) = a.vec_fp_test_data_class::(); + c.write(y); + x +} + +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_nan(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0) +} + +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_numeric(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3) +} + +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_nan(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3) +} + +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_numeric(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0) } /// Vector Test under Mask @@ -6257,6 +6285,58 @@ mod tests { } } + #[simd_test(enable = "vector")] + fn test_vec_fp_any_all_nan_numeric() { + unsafe { + assert_eq!( + vec_all_nan(vector_double([f64::NAN, f64::NAN])), + i32::from(true) + ); + assert_eq!( + vec_all_nan(vector_double([f64::NAN, 1.0])), + i32::from(false) + ); + assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false)); + + assert_eq!( + vec_any_nan(vector_double([f64::NAN, f64::NAN])), + i32::from(true) + ); + assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true)); + assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false)); + + assert_eq!( + vec_all_numeric(vector_double([f64::NAN, f64::NAN])), + i32::from(false) + ); + assert_eq!( + vec_all_numeric(vector_double([f64::NAN, 1.0])), + i32::from(false) + ); + assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true)); + + assert_eq!( + vec_any_numeric(vector_double([f64::NAN, f64::NAN])), + i32::from(false) + ); + assert_eq!( + vec_any_numeric(vector_double([f64::NAN, 1.0])), + i32::from(true) + ); + assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true)); + + // "numeric" means "not NaN". infinities are numeric + assert_eq!( + vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])), + i32::from(true) + ); + assert_eq!( + vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])), + i32::from(true) + ); + } + } + #[simd_test(enable = "vector")] fn test_vec_test_mask() { unsafe { From db1d9ca4f7aa9f5913d1b592a61055fd0c55ef53 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 19:10:40 +0100 Subject: [PATCH 25/30] add `vec_any_*` and `vec_all_*` --- crates/core_arch/src/s390x/vector.rs | 219 +++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 1779cff3c4..b9bff813dc 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -2889,6 +2889,65 @@ mod sealed { } } + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCompare { + unsafe fn vec_all_lt(self, other: Self) -> i32; + unsafe fn vec_all_le(self, other: Self) -> i32; + unsafe fn vec_all_gt(self, other: Self) -> i32; + unsafe fn vec_all_ge(self, other: Self) -> i32; + } + + // NOTE: this implementation is currently non-optimal, but it does work for floats even with + // only `vector` enabled. + // + // - https://github.com/llvm/llvm-project/issues/129434 + // - https://github.com/llvm/llvm-project/issues/130424 + macro_rules! impl_vec_compare { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCompare for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_lt(self, other: Self) -> i32 { + simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_le(self, other: Self) -> i32 { + simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_gt(self, other: Self) -> i32 { + simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_ge(self, other: Self) -> i32 { + simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32 + } + } + )* + } + } + + impl_vec_compare! { + vector_signed_char + vector_unsigned_char + + vector_signed_short + vector_unsigned_short + + vector_signed_int + vector_unsigned_int + vector_float + + vector_signed_long_long + vector_unsigned_long_long + vector_double + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorTestMask { type Mask; @@ -5264,6 +5323,166 @@ pub unsafe fn vec_cmpne_or_0_idx_cc( a.vec_cmpne_or_0_idx_cc(b, cc) } +/// All Elements Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_eq(a: T, b: T) -> i32 { + simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32 +} + +/// All Elements Not Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_ne(a: T, b: T) -> i32 { + simd_reduce_all(vec_cmpne(a, b)) as i32 +} + +/// Any Element Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_eq(a: T, b: T) -> i32 { + simd_reduce_any(vec_cmpeq(a, b)) as i32 +} + +/// Any Element Not Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_ne(a: T, b: T) -> i32 { + simd_reduce_any(vec_cmpne(a, b)) as i32 +} + +/// All Elements Less Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_lt(a: T, b: T) -> i32 { + a.vec_all_lt(b) +} + +/// All Elements Less Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_le(a: T, b: T) -> i32 { + a.vec_all_le(b) +} + +/// All Elements Greater Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_gt(a: T, b: T) -> i32 { + a.vec_all_gt(b) +} + +/// All Elements Greater Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_ge(a: T, b: T) -> i32 { + a.vec_all_ge(b) +} + +/// All Elements Not Less Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_nlt(a: T, b: T) -> i32 { + vec_all_ge(a, b) +} + +/// All Elements Not Less Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_nle(a: T, b: T) -> i32 { + vec_all_gt(a, b) +} + +/// All Elements Not Greater Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_ngt(a: T, b: T) -> i32 { + vec_all_le(a, b) +} + +/// All Elements Not Greater Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_nge(a: T, b: T) -> i32 { + vec_all_lt(a, b) +} + +/// Any Elements Less Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_lt(a: T, b: T) -> i32 { + !vec_all_ge(a, b) +} + +/// Any Elements Less Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_le(a: T, b: T) -> i32 { + !vec_all_gt(a, b) +} + +/// Any Elements Greater Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_gt(a: T, b: T) -> i32 { + !vec_all_le(a, b) +} + +/// Any Elements Greater Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_ge(a: T, b: T) -> i32 { + !vec_all_lt(a, b) +} + +/// Any Elements Not Less Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_nlt(a: T, b: T) -> i32 { + vec_any_ge(a, b) +} + +/// Any Elements Not Less Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_nle(a: T, b: T) -> i32 { + vec_any_gt(a, b) +} + +/// Any Elements Not Greater Than +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_ngt(a: T, b: T) -> i32 { + vec_any_le(a, b) +} + +/// Any Elements Not Greater Than or Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_nge(a: T, b: T) -> i32 { + vec_any_lt(a, b) +} + #[cfg(test)] mod tests { use super::*; From 11d56df7006c9c5e03e5c3b109689bf9b2428255 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 19:37:14 +0100 Subject: [PATCH 26/30] add `vec_mulo` --- crates/core_arch/src/s390x/vector.rs | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index b9bff813dc..b654d265cc 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -201,6 +201,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; + #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short; + #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int; + #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; + #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; @@ -2620,6 +2628,27 @@ mod sealed { impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int); impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMulo { + unsafe fn vec_mulo(self, b: Self) -> Result; + } + + test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] } + test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] } + test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] } + + test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] } + test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] } + test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] } + + impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short ); + impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int); + impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long ); + + impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short ); + impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int); + impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorGfmsum { unsafe fn vec_gfmsum(self, b: Self) -> Result; @@ -4728,6 +4757,14 @@ pub unsafe fn vec_mule, U>(a: T, b: T) -> U { a.vec_mule(b) } +/// Vector Multiply Odd +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mulo, U>(a: T, b: T) -> U { + a.vec_mulo(b) +} + /// Vector Galois Field Multiply Sum #[inline] #[target_feature(enable = "vector")] @@ -6394,6 +6431,18 @@ mod tests { [0x4000_0000, -8, 0xFFFE, 2] } + test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4, + [0, 0xFFFF, 0, 2, 0, 2, 0, 1], + [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2], + [0xFFFE_0001, 8, 0x0001_FFFE, 2] + } + + test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4, + [0, i16::MIN, 0, -2, 0, 2, 0, 1], + [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2], + [0x4000_0000, -8, 0xFFFE, 2] + } + test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4, [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0], [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678], From 1070ae81e271db3b5b19c6902542e360681b3e42 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 19:47:36 +0100 Subject: [PATCH 27/30] add `vec_mulh` --- crates/core_arch/src/s390x/vector.rs | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index b654d265cc..45f75dfc46 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -209,6 +209,14 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; + #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; @@ -2649,6 +2657,27 @@ mod sealed { impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int); impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMulh { + unsafe fn vec_mulh(self, b: Self) -> Result; + } + + test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] } + test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] } + test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] } + + test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] } + test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] } + test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] } + + impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char); + impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int); + + impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char); + impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorGfmsum { unsafe fn vec_gfmsum(self, b: Self) -> Result; @@ -4765,6 +4794,14 @@ pub unsafe fn vec_mulo, U>(a: T, b: T) -> U { a.vec_mulo(b) } +/// Vector Multiply High +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mulh, U>(a: T, b: T) -> U { + a.vec_mulh(b) +} + /// Vector Galois Field Multiply Sum #[inline] #[target_feature(enable = "vector")] @@ -6443,6 +6480,18 @@ mod tests { [0x4000_0000, -8, 0xFFFE, 2] } + test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4, + [u32::MAX, 2, 2, 1], + [u32::MAX, 4, u32::MAX, 2], + [u32::MAX - 1, 0, 1, 0] + } + + test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4, + [i32::MIN, -2, 2, 1], + [i32::MIN, 4, i32::MAX, 2], + [0x4000_0000, -1, 0, 0] + } + test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4, [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0], [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678], From 052dd81dca097965f8324cd8ccb2dc2f9750559b Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 8 Mar 2025 20:56:43 +0100 Subject: [PATCH 28/30] add `vec_meadd`, `vec_moadd`, `vec_mhadd` and `vec_mladd` --- crates/core_arch/src/s390x/vector.rs | 237 +++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 45f75dfc46..13f410f534 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -217,6 +217,38 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short; #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int; + #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int; + #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; + + #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; @@ -2594,6 +2626,17 @@ mod sealed { } } }; + ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => { + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl $Trait for $a { + type Result = $r; + #[inline] + #[target_feature(enable = "vector")] + unsafe fn $m(self, b: $b, c: $c) -> $r { + $fun(self, b, c) + } + } + }; } #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -2678,6 +2721,100 @@ mod sealed { impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int); + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMeadd { + type Result; + unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] } + test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] } + test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] } + + test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] } + test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] } + test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] } + + impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short ); + impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int); + impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long ); + + impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short ); + impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int); + impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMoadd { + type Result; + unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] } + test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] } + test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] } + + test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] } + test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] } + test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] } + + impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short ); + impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int); + impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long ); + + impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short ); + impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int); + impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMhadd { + type Result; + unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] } + test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] } + test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] } + + test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] } + test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] } + test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] } + + impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char ); + impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int ); + + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char ); + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMladd { + type Result; + unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result; + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn simd_mladd(a: T, b: T, c: T) -> T { + simd_add(simd_mul(a, b), c) + } + + test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] } + test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalh ] } + test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] } + + test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] } + test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalh ] } + test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] } + + impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char ); + impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int ); + + impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char ); + impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorGfmsum { unsafe fn vec_gfmsum(self, b: Self) -> Result; @@ -4769,6 +4906,38 @@ pub unsafe fn vec_msub(a: T, b: T, c: T) -> T { a.vec_msub(b, c) } +/// Vector Multiply and Add Even +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_meadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_meadd(b, c) +} + +/// Vector Multiply and Add Odd +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_moadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_moadd(b, c) +} + +/// Vector Multiply and Add High +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mhadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_mhadd(b, c) +} + +/// Vector Multiply and Add Low +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mladd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_mladd(b, c) +} + /// Vector Checksum #[inline] #[target_feature(enable = "vector")] @@ -7032,4 +7201,72 @@ mod tests { [1.0, f32::NAN, 5.0, 3.14], [0, !0, !0, !0] } + + #[simd_test(enable = "vector")] + fn test_vec_meadd() { + let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]); + let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_meadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]); + let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]); + let c = vector_signed_int([2, -2, 2, -2]); + + let d = unsafe { vec_meadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 10, 23, 30]); + } + + #[simd_test(enable = "vector")] + fn test_vec_moadd() { + let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]); + let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_moadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]); + let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]); + let c = vector_signed_int([2, -2, 2, -2]); + + let d = unsafe { vec_moadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 10, 23, 30]); + } + + #[simd_test(enable = "vector")] + fn test_vec_mhadd() { + let a = vector_unsigned_int([1, 2, 3, 4]); + let b = vector_unsigned_int([5, 6, 7, 8]); + let c = vector_unsigned_int([u32::MAX; 4]); + + let d = unsafe { vec_mhadd(a, b, c) }; + assert_eq!(d.as_array(), &[1, 1, 1, 1]); + + let a = vector_signed_int([-1, -2, -3, -4]); + let b = vector_signed_int([5, 6, 7, 8]); + let c = vector_signed_int([i32::MIN; 4]); + + let d = unsafe { vec_mhadd(a, b, c) }; + assert_eq!(d.as_array(), &[-1, -1, -1, -1]); + } + + #[simd_test(enable = "vector")] + fn test_vec_mladd() { + let a = vector_unsigned_int([1, 2, 3, 4]); + let b = vector_unsigned_int([5, 6, 7, 8]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_mladd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_int([-1, -2, -3, -4]); + let b = vector_signed_int([5, 6, 7, 8]); + let c = vector_signed_int([2, 2, 2, 2]); + + let d = unsafe { vec_mladd(a, b, c) }; + assert_eq!(d.as_array(), &[-3, -10, -19, -30]); + } } From 049b750fc4ad5a0f72a6c40f1c5e2af5ab1754a7 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sun, 16 Mar 2025 16:38:44 +0100 Subject: [PATCH 29/30] shink the size of type signatures --- crates/core_arch/src/s390x/vector.rs | 220 ++++++--------------------- 1 file changed, 44 insertions(+), 176 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 13f410f534..a4df04ed0e 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -3837,10 +3837,7 @@ unsafe fn __lcbb(ptr: *const u8) -> u32 { #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_add(a: T, b: U) -> >::Result -where - T: sealed::VectorAdd, -{ +pub unsafe fn vec_add, U>(a: T, b: U) -> T::Result { a.vec_add(b) } @@ -3848,10 +3845,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sub(a: T, b: U) -> >::Result -where - T: sealed::VectorSub, -{ +pub unsafe fn vec_sub, U>(a: T, b: U) -> T::Result { a.vec_sub(b) } @@ -3865,10 +3859,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mul(a: T, b: T) -> T -where - T: sealed::VectorMul, -{ +pub unsafe fn vec_mul(a: T, b: T) -> T { a.vec_mul(b) } @@ -3876,10 +3867,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cntlz(a: T) -> ::Result -where - T: sealed::CountBits, -{ +pub unsafe fn vec_cntlz(a: T) -> T::Result { a.vec_cntlz() } @@ -3887,10 +3875,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cnttz(a: T) -> ::Result -where - T: sealed::CountBits, -{ +pub unsafe fn vec_cnttz(a: T) -> T::Result { a.vec_cnttz() } @@ -3900,10 +3885,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_popcnt(a: T) -> ::Result -where - T: sealed::CountBits, -{ +pub unsafe fn vec_popcnt(a: T) -> T::Result { a.vec_popcnt() } @@ -3911,10 +3893,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_max(a: T, b: U) -> >::Result -where - T: sealed::VectorMax, -{ +pub unsafe fn vec_max, U>(a: T, b: U) -> T::Result { a.vec_max(b) } @@ -3922,10 +3901,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_min(a: T, b: U) -> >::Result -where - T: sealed::VectorMin, -{ +pub unsafe fn vec_min, U>(a: T, b: U) -> T::Result { a.vec_min(b) } @@ -3933,10 +3909,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_abs(a: T) -> T -where - T: sealed::VectorAbs, -{ +pub unsafe fn vec_abs(a: T) -> T { a.vec_abs() } @@ -3968,10 +3941,7 @@ pub unsafe fn vec_nmsub(a: T, b: T, c: T) -> T { #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sqrt(a: T) -> T -where - T: sealed::VectorSqrt, -{ +pub unsafe fn vec_sqrt(a: T) -> T { a.vec_sqrt() } @@ -3979,10 +3949,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_splat(a: T) -> T -where - T: sealed::VectorSplat, -{ +pub unsafe fn vec_splat(a: T) -> T { a.vec_splat::() } @@ -3990,10 +3957,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_splats(a: T) -> U -where - T: sealed::VectorSplats, -{ +pub unsafe fn vec_splats, U>(a: T) -> U { a.vec_splats() } @@ -4001,10 +3965,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_and(a: T, b: U) -> >::Result -where - T: sealed::VectorAnd, -{ +pub unsafe fn vec_and, U>(a: T, b: U) -> T::Result { a.vec_and(b) } @@ -4012,10 +3973,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_or(a: T, b: U) -> >::Result -where - T: sealed::VectorOr, -{ +pub unsafe fn vec_or, U>(a: T, b: U) -> T::Result { a.vec_or(b) } @@ -4023,10 +3981,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_xor(a: T, b: U) -> >::Result -where - T: sealed::VectorXor, -{ +pub unsafe fn vec_xor, U>(a: T, b: U) -> T::Result { a.vec_xor(b) } @@ -4034,10 +3989,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nor(a: T, b: U) -> >::Result -where - T: sealed::VectorNor, -{ +pub unsafe fn vec_nor, U>(a: T, b: U) -> T::Result { a.vec_nor(b) } @@ -4045,10 +3997,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nand(a: T, b: U) -> >::Result -where - T: sealed::VectorNand, -{ +pub unsafe fn vec_nand, U>(a: T, b: U) -> T::Result { a.vec_nand(b) } @@ -4056,10 +4005,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_eqv(a: T, b: U) -> >::Result -where - T: sealed::VectorEqv, -{ +pub unsafe fn vec_eqv, U>(a: T, b: U) -> T::Result { a.vec_eqv(b) } @@ -4067,10 +4013,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_andc(a: T, b: U) -> >::Result -where - T: sealed::VectorAndc, -{ +pub unsafe fn vec_andc, U>(a: T, b: U) -> T::Result { a.vec_andc(b) } @@ -4084,10 +4027,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_orc(a: T, b: U) -> >::Result -where - T: sealed::VectorOrc, -{ +pub unsafe fn vec_orc, U>(a: T, b: U) -> T::Result { a.vec_orc(b) } @@ -4095,10 +4035,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_floor(a: T) -> T -where - T: sealed::VectorFloor, -{ +pub unsafe fn vec_floor(a: T) -> T { a.vec_floor() } @@ -4106,10 +4043,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_ceil(a: T) -> T -where - T: sealed::VectorCeil, -{ +pub unsafe fn vec_ceil(a: T) -> T { a.vec_ceil() } @@ -4118,10 +4052,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_trunc(a: T) -> T -where - T: sealed::VectorTrunc, -{ +pub unsafe fn vec_trunc(a: T) -> T { a.vec_trunc() } @@ -4130,10 +4061,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_round(a: T) -> T -where - T: sealed::VectorRound, -{ +pub unsafe fn vec_round(a: T) -> T { a.vec_round() } @@ -4142,10 +4070,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundc(a: T) -> T -where - T: sealed::VectorRoundc, -{ +pub unsafe fn vec_roundc(a: T) -> T { a.vec_roundc() } @@ -4154,10 +4079,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundm(a: T) -> T -where - T: sealed::VectorFloor, -{ +pub unsafe fn vec_roundm(a: T) -> T { // the IBM docs note // // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception. @@ -4171,10 +4093,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundp(a: T) -> T -where - T: sealed::VectorCeil, -{ +pub unsafe fn vec_roundp(a: T) -> T { // the IBM docs note // // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception. @@ -4188,10 +4107,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundz(a: T) -> T -where - T: sealed::VectorTrunc, -{ +pub unsafe fn vec_roundz(a: T) -> T { // the IBM docs note // // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception. @@ -4204,10 +4120,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rint(a: T) -> T -where - T: sealed::VectorRint, -{ +pub unsafe fn vec_rint(a: T) -> T { a.vec_rint() } @@ -4223,10 +4136,7 @@ pub unsafe fn vec_avg, U>(a: T, b: U) -> T::Result { #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sl(a: T, b: U) -> >::Result -where - T: sealed::VectorSl, -{ +pub unsafe fn vec_sl, U>(a: T, b: U) -> T::Result { a.vec_sl(b) } @@ -4234,10 +4144,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sr(a: T, b: U) -> >::Result -where - T: sealed::VectorSr, -{ +pub unsafe fn vec_sr, U>(a: T, b: U) -> T::Result { a.vec_sr(b) } @@ -4245,10 +4152,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sra(a: T, b: U) -> >::Result -where - T: sealed::VectorSra, -{ +pub unsafe fn vec_sra, U>(a: T, b: U) -> T::Result { a.vec_sra(b) } @@ -4256,10 +4160,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_slb(a: T, b: U) -> >::Result -where - T: sealed::VectorSlb, -{ +pub unsafe fn vec_slb, U>(a: T, b: U) -> T::Result { a.vec_slb(b) } @@ -4267,10 +4168,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_srb(a: T, b: U) -> >::Result -where - T: sealed::VectorSrb, -{ +pub unsafe fn vec_srb, U>(a: T, b: U) -> T::Result { a.vec_srb(b) } @@ -4278,10 +4176,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_srab(a: T, b: U) -> >::Result -where - T: sealed::VectorSrab, -{ +pub unsafe fn vec_srab, U>(a: T, b: U) -> T::Result { a.vec_srab(b) } @@ -4289,10 +4184,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rl(a: T, b: U) -> >::Result -where - T: sealed::VectorRl, -{ +pub unsafe fn vec_rl, U>(a: T, b: U) -> T::Result { a.vec_rl(b) } @@ -4338,10 +4230,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rli(a: T, bits: core::ffi::c_ulong) -> T -where - T: sealed::VectorRli, -{ +pub unsafe fn vec_rli(a: T, bits: core::ffi::c_ulong) -> T { a.vec_rli(bits) } @@ -4349,10 +4238,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_reve(a: T) -> T -where - T: sealed::VectorReve, -{ +pub unsafe fn vec_reve(a: T) -> T { a.vec_reve() } @@ -4360,10 +4246,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_revb(a: T) -> T -where - T: sealed::VectorRevb, -{ +pub unsafe fn vec_revb(a: T) -> T { a.vec_revb() } @@ -4371,10 +4254,7 @@ where #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mergeh(a: T, b: T) -> T -where - T: sealed::VectorMergeh, -{ +pub unsafe fn vec_mergeh(a: T, b: T) -> T { a.vec_mergeh(b) } @@ -4438,10 +4318,7 @@ pub unsafe fn vec_unpackl(a: T) -> (a: T, b: T) -> T -where - T: sealed::VectorMergel, -{ +pub unsafe fn vec_mergel(a: T, b: T) -> T { a.vec_mergel(b) } @@ -4579,10 +4456,7 @@ pub unsafe fn vec_sub_u128( #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_subc(a: T, b: U) -> >::Result -where - T: sealed::VectorSubc, -{ +pub unsafe fn vec_subc, U>(a: T, b: U) -> T::Result { a.vec_subc(b) } @@ -4771,10 +4645,7 @@ macro_rules! vec_find_any { #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] - pub unsafe fn $fun(a: T, b: U) -> >::Result - where - T: sealed::$Trait, - { + pub unsafe fn $fun, U>(a: T, b: U) -> T::Result { a.$fun(b) } )* @@ -4796,10 +4667,7 @@ macro_rules! vec_find_any_cc { #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] - pub unsafe fn $fun(a: T, b: U, c: *mut i32) -> >::Result - where - T: sealed::$Trait, - { + pub unsafe fn $fun, U>(a: T, b: U, c: *mut i32) -> T::Result { a.$fun(b, c) } )* From 5e2f119f26975fc41505168dfa813d73f950adf5 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sun, 16 Mar 2025 17:02:35 +0100 Subject: [PATCH 30/30] move unsafe pointer writes to the surface --- crates/core_arch/src/s390x/vector.rs | 133 ++++++++++++++------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index a4df04ed0e..5103273ed9 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -1929,10 +1929,9 @@ mod sealed { type Result = $r; #[inline] #[target_feature(enable = "vector")] - unsafe fn $m(self, b: Self, c: *mut i32) -> Self::Result { + unsafe fn $m(self, b: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)); - c.write(y); - transmute(x) + (transmute(x), y) } } )* @@ -1959,10 +1958,9 @@ mod sealed { type Result = t_b!($ty); #[inline] #[target_feature(enable = "vector")] - unsafe fn $m(self, b: Self, c: *mut i32) -> Self::Result { + unsafe fn $m(self, b: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)); - c.write(y); - transmute(x) + (transmute(x), y) } } )* @@ -2126,7 +2124,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqCC { type Result; - unsafe fn vec_find_any_eq_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs } @@ -2134,7 +2132,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeCC { type Result; - unsafe fn vec_find_any_ne_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs } @@ -2142,7 +2140,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqIdxCC { type Result; - unsafe fn vec_find_any_eq_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs } @@ -2150,7 +2148,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeIdxCC { type Result; - unsafe fn vec_find_any_ne_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs } @@ -2158,7 +2156,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqOrZeroIdxCC { type Result; - unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs } @@ -2166,7 +2164,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeOrZeroIdxCC { type Result; - unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs } @@ -2423,11 +2421,9 @@ mod sealed { unsafe fn $intr( a: $ty, b: $ty, - c: *mut i32, - ) -> $outty { + ) -> ($outty, i32) { let PackedTuple { x, y } = super::$intr(a, b); - c.write(y); - x + (x, y) } #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -2436,8 +2432,8 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_packs_cc(self, b: Self, c: *mut i32) -> Self::Result { - $intr(self, b, c) + unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) { + $intr(self, b) } } )* @@ -2447,7 +2443,7 @@ mod sealed { #[unstable(feature = "stdarch_powerpc", issue = "111145")] pub trait VectorPacksCC { type Result; - unsafe fn vec_packs_cc(self, b: Self, c: *mut i32) -> Self::Result; + unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32); } impl_vector_packs_cc! { @@ -2468,8 +2464,8 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_packsu_cc(self, b: Self, c: *mut i32) -> Self::Result { - $intr(self, b, c) + unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) { + $intr(self, b) } } )* @@ -2479,7 +2475,7 @@ mod sealed { #[unstable(feature = "stdarch_powerpc", issue = "111145")] pub trait VectorPacksuCC { type Result; - unsafe fn vec_packsu_cc(self, b: Self, c: *mut i32) -> Self::Result; + unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32); } impl_vector_packsu_cc! { @@ -3187,15 +3183,13 @@ mod sealed { self, b: Self, c: vector_unsigned_char, - d: *mut i32, - ) -> vector_unsigned_char; + ) -> (vector_unsigned_char, i32); unsafe fn vec_search_string_until_zero_cc( self, b: Self, c: vector_unsigned_char, - d: *mut i32, - ) -> vector_unsigned_char; + ) -> (vector_unsigned_char, i32); } macro_rules! impl_vec_search_string{ @@ -3205,18 +3199,16 @@ mod sealed { impl VectorSearchString for $ty { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char, d: *mut i32) -> vector_unsigned_char { + unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) { let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c); - d.write(y); - x + (x, y) } #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char, d: *mut i32) -> vector_unsigned_char { + unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) { let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c); - d.write(y); - x + (x, y) } } @@ -3435,8 +3427,8 @@ mod sealed { impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) } #[unstable(feature = "stdarch_s390x", issue = "135681")] - pub trait VectorCopyUntilZeroCC { - unsafe fn vec_cp_until_zero_cc(self, cc: *mut i32) -> Self; + pub trait VectorCopyUntilZeroCC: Sized { + unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32); } test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple [vistrbs, vistrbs] } @@ -3450,10 +3442,9 @@ mod sealed { impl VectorCopyUntilZeroCC for $ty { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_cp_until_zero_cc(self, cc: *mut i32) -> Self { + unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) { let PackedTuple { x,y } = $intr(transmute(self)); - cc.write(y); - transmute(x) + (transmute(x), y) } } @@ -3729,14 +3720,14 @@ mod sealed { unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result; unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result; - unsafe fn vec_cmpeq_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; - unsafe fn vec_cmpne_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32); + unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32); unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result; unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result; - unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; - unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result; + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32); + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32); } macro_rules! impl_compare_equality_idx { @@ -3777,34 +3768,30 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_cmpeq_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other)); - *cc = y; - transmute(x) + (transmute(x), y) } #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_cmpne_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other)); - *cc = y; - transmute(x) + (transmute(x),y) } #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other)); - *cc = y; - transmute(x) + (transmute(x), y) } #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self, cc: *mut i32) -> Self::Result { + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other)); - *cc = y; - transmute(x) + (transmute(x),y) } } )* @@ -4279,7 +4266,9 @@ pub unsafe fn vec_packs, U>(a: T, b: U) -> T::Result { #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] pub unsafe fn vec_packs_cc(a: T, b: T, c: *mut i32) -> T::Result { - a.vec_packs_cc(b, c) + let (x, y) = a.vec_packs_cc(b); + unsafe { c.write(y) }; + x } /// Vector Pack Saturated Unsigned @@ -4295,7 +4284,9 @@ pub unsafe fn vec_packsu, U>(a: T, b: U) -> T::Result #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] pub unsafe fn vec_packsu_cc(a: T, b: T, c: *mut i32) -> T::Result { - a.vec_packsu_cc(b, c) + let (x, y) = a.vec_packsu_cc(b); + unsafe { c.write(y) }; + x } /// Vector Unpack High @@ -4668,7 +4659,9 @@ macro_rules! vec_find_any_cc { #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] pub unsafe fn $fun, U>(a: T, b: U, c: *mut i32) -> T::Result { - a.$fun(b, c) + let (x, y) = a.$fun(b); + unsafe { c.write(y) }; + x } )* } @@ -5021,7 +5014,9 @@ pub unsafe fn vec_search_string_cc( c: vector_unsigned_char, d: *mut i32, ) -> vector_unsigned_char { - a.vec_search_string_cc(b, c, d) + let (x, y) = a.vec_search_string_cc(b, c); + unsafe { d.write(y) }; + x } /// Vector Search String Until Zero @@ -5034,7 +5029,9 @@ pub unsafe fn vec_search_string_until_zero_cc( c: vector_unsigned_char, d: *mut i32, ) -> vector_unsigned_char { - a.vec_search_string_until_zero_cc(b, c, d) + let (x, y) = a.vec_search_string_until_zero_cc(b, c); + unsafe { d.write(y) }; + x } /// Vector Convert from float (even elements) to double @@ -5116,7 +5113,9 @@ pub unsafe fn vec_cp_until_zero(a: T) -> T { #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] pub unsafe fn vec_cp_until_zero_cc(a: T, cc: *mut i32) -> T { - a.vec_cp_until_zero_cc(cc) + let (x, y) = a.vec_cp_until_zero_cc(); + unsafe { cc.write(y) }; + x } /// Vector Multiply Sum Logical @@ -5384,7 +5383,9 @@ pub unsafe fn vec_cmpeq_idx_cc( b: T, cc: *mut i32, ) -> T::Result { - a.vec_cmpeq_idx_cc(b, cc) + let (x, y) = a.vec_cmpeq_idx_cc(b); + unsafe { cc.write(y) }; + x } /// Vector Compare Not Equal Index with Condition Code #[inline] @@ -5395,7 +5396,9 @@ pub unsafe fn vec_cmpne_idx_cc( b: T, cc: *mut i32, ) -> T::Result { - a.vec_cmpne_idx_cc(b, cc) + let (x, y) = a.vec_cmpne_idx_cc(b); + unsafe { cc.write(y) }; + x } /// Vector Compare Equal or Zero Index #[inline] @@ -5420,7 +5423,9 @@ pub unsafe fn vec_cmpeq_or_0_idx_cc( b: T, cc: *mut i32, ) -> T::Result { - a.vec_cmpeq_or_0_idx_cc(b, cc) + let (x, y) = a.vec_cmpeq_or_0_idx_cc(b); + unsafe { cc.write(y) }; + x } /// Vector Compare Not Equal or Zero Index with Condition Code #[inline] @@ -5431,7 +5436,9 @@ pub unsafe fn vec_cmpne_or_0_idx_cc( b: T, cc: *mut i32, ) -> T::Result { - a.vec_cmpne_or_0_idx_cc(b, cc) + let (x, y) = a.vec_cmpne_or_0_idx_cc(b); + unsafe { cc.write(y) }; + x } /// All Elements Equal