diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index bef9c67474577..200cedf0f6ae0 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -348,6 +348,31 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_lane); } + sym::simd_splat => { + intrinsic_args!(fx, args => (value); intrinsic); + + if !ret.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, ret.layout().ty); + return; + } + let (lane_count, lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + + if value.layout().ty != lane_ty { + fx.tcx.dcx().span_fatal( + span, + format!( + "[simd_splat] expected element type {lane_ty:?}, got {got:?}", + got = value.layout().ty + ), + ); + } + + for i in 0..lane_count { + let ret_lane = ret.place_lane(fx, i.into()); + ret_lane.write_cvalue(fx, value); + } + } + sym::simd_neg | sym::simd_bswap | sym::simd_bitreverse diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs index 39b4bb3ebefab..eab067a02b7bf 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs @@ -121,6 +121,42 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( return Ok(bx.vector_select(vector_mask, arg1, args[2].immediate())); } + #[cfg(feature = "master")] + if name == sym::simd_splat { + let (out_len, out_ty) = require_simd2!(ret_ty, SimdReturn); + + require!( + args[0].layout.ty == out_ty, + InvalidMonomorphization::ExpectedVectorElementType { + span, + name, + expected_element: out_ty, + vector_type: ret_ty, + } + ); + + let vec_ty = llret_ty.unqualified().dyncast_vector().expect("vector return type"); + let elem_ty = vec_ty.get_element_type(); + + // Cast pointer type to usize (GCC does not support pointer SIMD vectors). + let value = args[0]; + let scalar = if value.layout.ty.is_numeric() { + value.immediate() + } else if value.layout.ty.is_raw_ptr() { + bx.ptrtoint(value.immediate(), elem_ty) + } else { + return_error!(InvalidMonomorphization::UnsupportedOperation { + span, + name, + in_ty: ret_ty, + in_elem: value.layout.ty + }); + }; + + let elements = vec![scalar; out_len as usize]; + return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &elements)); + } + // every intrinsic below takes a SIMD vector as its first argument require_simd!( args[0].layout.ty, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index a712b7b4138c3..50e627cf6bdf2 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1581,6 +1581,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } + if name == sym::simd_splat { + let (_out_len, out_ty) = require_simd!(ret_ty, SimdReturn); + + require!( + args[0].layout.ty == out_ty, + InvalidMonomorphization::ExpectedVectorElementType { + span, + name, + expected_element: out_ty, + vector_type: ret_ty, + } + ); + + // `insertelement poison, elem %x, i32 0` + let poison_vec = bx.const_poison(llret_ty); + let idx0 = bx.const_i32(0); + let v0 = bx.insert_element(poison_vec, args[0].immediate(), idx0); + + // `shufflevector v0, poison, zeroinitializer` + // The masks is all zeros, so this splats lane 0 (which has our element in it). + let splat = bx.shuffle_vector(v0, poison_vec, bx.const_null(llret_ty)); + + return Ok(splat); + } + // every intrinsic below takes a SIMD vector as its first argument let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput); let in_ty = args[0].layout.ty; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 78dfecdd1818c..50ca676b5d05b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -1074,8 +1074,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if constant_ty.is_simd() { // However, some SIMD types do not actually use the vector ABI // (in particular, packed SIMD types do not). Ensure we exclude those. + // + // We also have to exclude vectors of pointers because `immediate_const_vector` + // does not work for those. let layout = bx.layout_of(constant_ty); - if let BackendRepr::SimdVector { .. } = layout.backend_repr { + let (_, element_ty) = constant_ty.simd_size_and_type(bx.tcx()); + if let BackendRepr::SimdVector { .. } = layout.backend_repr + && element_ty.is_numeric() + { let (llval, ty) = self.immediate_const_vector(bx, constant); return OperandRef { val: OperandValue::Immediate(llval), diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs index f61baa006b63a..303183fd924da 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -61,6 +61,15 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } self.copy_op(&self.project_index(&input, index)?, &dest)?; } + sym::simd_splat => { + let elem = &args[0]; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + for i in 0..dest_len { + let place = self.project_index(&dest, i)?; + self.copy_op(elem, &place)?; + } + } sym::simd_neg | sym::simd_fabs | sym::simd_ceil diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 98f9548058f45..22ee490b81a7b 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -746,6 +746,7 @@ pub(crate) fn check_intrinsic_type( sym::simd_extract | sym::simd_extract_dyn => { (2, 0, vec![param(0), tcx.types.u32], param(1)) } + sym::simd_splat => (2, 0, vec![param(1)], param(0)), sym::simd_cast | sym::simd_as | sym::simd_cast_ptr diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 5767444025ec2..13894b2b4a0bc 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2140,6 +2140,7 @@ symbols! { simd_shr, simd_shuffle, simd_shuffle_const_generic, + simd_splat, simd_sub, simd_trunc, simd_with_exposed_provenance, diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs index 9d5dfb1725862..f70262c38ae50 100644 --- a/library/core/src/intrinsics/simd.rs +++ b/library/core/src/intrinsics/simd.rs @@ -52,6 +52,13 @@ pub const unsafe fn simd_insert_dyn(x: T, idx: u32, val: U) -> T; #[rustc_intrinsic] pub const unsafe fn simd_extract_dyn(x: T, idx: u32) -> U; +/// Creates a vector where every lane has the provided value. +/// +/// `T` must be a vector with element type `U`. +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn simd_splat(value: U) -> T; + /// Adds two simd vectors elementwise. /// /// `T` must be a vector of integers or floats. diff --git a/tests/assembly-llvm/cstring-merging.rs b/tests/assembly-llvm/cstring-merging.rs index 03688e0068b79..9c1af4ebb4c05 100644 --- a/tests/assembly-llvm/cstring-merging.rs +++ b/tests/assembly-llvm/cstring-merging.rs @@ -1,5 +1,6 @@ // MIPS assembler uses the label prefix `$anon.` for local anonymous variables // other architectures (including ARM and x86-64) use the prefix `.Lanon.` +// Hexagon uses `.string` instead of `.asciz` for null-terminated strings //@ only-linux //@ assembly-output: emit-asm //@ compile-flags: --crate-type=lib -Copt-level=3 -Cllvm-args=-enable-global-merge=0 @@ -9,13 +10,13 @@ use std::ffi::CStr; // CHECK: .section .rodata.str1.{{[12]}},"aMS" // CHECK: {{(\.L|\$)}}anon.{{.+}}: -// CHECK-NEXT: .asciz "foo" +// CHECK-NEXT: .{{asciz|string}} "foo" #[unsafe(no_mangle)] static CSTR: &[u8; 4] = b"foo\0"; // CHECK-NOT: .section // CHECK: {{(\.L|\$)}}anon.{{.+}}: -// CHECK-NEXT: .asciz "bar" +// CHECK-NEXT: .{{asciz|string}} "bar" #[unsafe(no_mangle)] pub fn cstr() -> &'static CStr { c"bar" @@ -23,7 +24,7 @@ pub fn cstr() -> &'static CStr { // CHECK-NOT: .section // CHECK: {{(\.L|\$)}}anon.{{.+}}: -// CHECK-NEXT: .asciz "baz" +// CHECK-NEXT: .{{asciz|string}} "baz" #[unsafe(no_mangle)] pub fn manual_cstr() -> &'static str { "baz\0" diff --git a/tests/codegen-llvm/simd/splat.rs b/tests/codegen-llvm/simd/splat.rs new file mode 100644 index 0000000000000..7a24162321bdc --- /dev/null +++ b/tests/codegen-llvm/simd/splat.rs @@ -0,0 +1,33 @@ +//@ compile-flags: -Copt-level=3 +#![crate_type = "lib"] +#![no_std] +#![feature(repr_simd, core_intrinsics)] +use core::intrinsics::simd::simd_splat; + +#[path = "../../auxiliary/minisimd.rs"] +mod minisimd; +use minisimd::*; + +// Test that `simd_splat` produces the canonical LLVM splat sequence. + +#[no_mangle] +unsafe fn int(x: u16) -> u16x2 { + // CHECK-LABEL: int + // CHECK: start: + // CHECK-NEXT: %0 = insertelement <2 x i16> poison, i16 %x, i64 0 + // CHECK-NEXT: %1 = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> zeroinitializer + // CHECK-NEXT: store + // CHECK-NEXT: ret + simd_splat(x) +} + +#[no_mangle] +unsafe fn float(x: f32) -> f32x4 { + // CHECK-LABEL: float + // CHECK: start: + // CHECK-NEXT: %0 = insertelement <4 x float> poison, float %x, i64 0 + // CHECK-NEXT: %1 = shufflevector <4 x float> %0, <4 x float> poison, <4 x i32> zeroinitializer + // CHECK-NEXT: store + // CHECK-NEXT: ret + simd_splat(x) +} diff --git a/tests/ui/simd/intrinsic/splat.rs b/tests/ui/simd/intrinsic/splat.rs new file mode 100644 index 0000000000000..38260a124d44b --- /dev/null +++ b/tests/ui/simd/intrinsic/splat.rs @@ -0,0 +1,48 @@ +//@ run-pass +#![feature(repr_simd, core_intrinsics)] + +#[path = "../../../auxiliary/minisimd.rs"] +mod minisimd; +use minisimd::*; + +use std::intrinsics::simd::simd_splat; + +fn main() { + unsafe { + let x: Simd = simd_splat(123u32); + let y: Simd = const { simd_splat(123u32) }; + assert_eq!(x.into_array(), [123; 1]); + assert_eq!(x.into_array(), y.into_array()); + + let x: u16x2 = simd_splat(42u16); + let y: u16x2 = const { simd_splat(42u16) }; + assert_eq!(x.into_array(), [42; 2]); + assert_eq!(x.into_array(), y.into_array()); + + let x: u128x4 = simd_splat(42u128); + let y: u128x4 = const { simd_splat(42u128) }; + assert_eq!(x.into_array(), [42; 4]); + assert_eq!(x.into_array(), y.into_array()); + + let x: i32x4 = simd_splat(-7i32); + let y: i32x4 = const { simd_splat(-7i32) }; + assert_eq!(x.into_array(), [-7; 4]); + assert_eq!(x.into_array(), y.into_array()); + + let x: f32x4 = simd_splat(42.0f32); + let y: f32x4 = const { simd_splat(42.0f32) }; + assert_eq!(x.into_array(), [42.0; 4]); + assert_eq!(x.into_array(), y.into_array()); + + let x: f64x2 = simd_splat(42.0f64); + let y: f64x2 = const { simd_splat(42.0f64) }; + assert_eq!(x.into_array(), [42.0; 2]); + assert_eq!(x.into_array(), y.into_array()); + + static ZERO: u8 = 0u8; + let x: Simd<*const u8, 2> = simd_splat(&raw const ZERO); + let y: Simd<*const u8, 2> = const { simd_splat(&raw const ZERO) }; + assert_eq!(x.into_array(), [&raw const ZERO; 2]); + assert_eq!(x.into_array(), y.into_array()); + } +}