[TRY] Use LLVM AutoUpgrade to bypass autocasts

Author: sayantn

compiler/rustc_codegen_llvm/src/abi.rs

@@ -313,6 +313,8 @@ pub(crate) enum FunctionSignature<'ll> {
     /// This is an LLVM intrinsic, but the signature is just the Rust signature.
     /// FIXME: this should ideally not exist, we should be using the LLVM signature for all LLVM intrinsics
     RustSignature(llvm::Intrinsic, &'ll Type),
+    /// FIXME: This shouldn't exist as well, but needed to get around autocast
+    NonMatchingSignature { intrinsic: llvm::Intrinsic, llvm_ty: &'ll Type, rust_ty: &'ll Type },
     /// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
     MaybeInvalid(&'ll Type),
     /// Just the Rust signature
@@ -326,13 +328,15 @@ impl<'ll> FunctionSignature<'ll> {
             | FunctionSignature::RustSignature(_, fn_ty)
             | FunctionSignature::MaybeInvalid(fn_ty)
             | FunctionSignature::NotIntrinsic(fn_ty) => fn_ty,
+            FunctionSignature::NonMatchingSignature { rust_ty, .. } => rust_ty,
         }
     }
 
     pub(crate) fn intrinsic(&self) -> Option<llvm::Intrinsic> {
         match self {
             FunctionSignature::RustSignature(intrinsic, _)
-            | FunctionSignature::LLVMSignature(intrinsic, _) => Some(*intrinsic),
+            | FunctionSignature::LLVMSignature(intrinsic, _)
+            | FunctionSignature::NonMatchingSignature { intrinsic, .. } => Some(*intrinsic),
             _ => None,
         }
     }
@@ -375,40 +379,26 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             return true;
         }
 
-        match self.type_kind(llvm_ty) {
-            TypeKind::BFloat => rust_ty == self.type_i16(),
-
-            // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
-            // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
-            // as, well, packed structs, so they won't match with those either)
-            TypeKind::Struct if self.type_kind(rust_ty) == TypeKind::Struct => {
-                let rust_element_tys = self.struct_element_types(rust_ty);
-                let llvm_element_tys = self.struct_element_types(llvm_ty);
-
-                if rust_element_tys.len() != llvm_element_tys.len() {
-                    return false;
-                }
+        // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
+        // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
+        // as, well, packed structs, so they won't match with those either)
+        if self.type_kind(llvm_ty) == TypeKind::Struct
+            && self.type_kind(rust_ty) == TypeKind::Struct
+        {
+            let rust_element_tys = self.struct_element_types(rust_ty);
+            let llvm_element_tys = self.struct_element_types(llvm_ty);
 
-                iter::zip(rust_element_tys, llvm_element_tys).all(
-                    |(rust_element_ty, llvm_element_ty)| {
-                        self.equate_ty(rust_element_ty, llvm_element_ty)
-                    },
-                )
-            }
-            TypeKind::Vector => {
-                let element_count = self.vector_length(llvm_ty) as u64;
-                let llvm_element_ty = self.element_type(llvm_ty);
-
-                if llvm_element_ty == self.type_bf16() {
-                    rust_ty == self.type_vector(self.type_i16(), element_count)
-                } else if llvm_element_ty == self.type_i1() {
-                    let int_width = element_count.next_power_of_two().max(8);
-                    rust_ty == self.type_ix(int_width)
-                } else {
-                    false
-                }
+            if rust_element_tys.len() != llvm_element_tys.len() {
+                return false;
             }
-            _ => false,
+
+            iter::zip(rust_element_tys, llvm_element_tys).all(
+                |(rust_element_ty, llvm_element_ty)| {
+                    self.equate_ty(rust_element_ty, llvm_element_ty)
+                },
+            )
+        } else {
+            false
         }
     }
 }
@@ -520,7 +510,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
                 if !intrinsic.is_overloaded() {
                     // FIXME: also do this for overloaded intrinsics
-                    FunctionSignature::LLVMSignature(intrinsic, intrinsic.get_type(cx.llcx, &[]))
+                    let llvm_ty = intrinsic.get_type(cx.llcx, &[]);
+                    if self.verify_intrinsic_signature(cx, llvm_ty) {
+                        FunctionSignature::LLVMSignature(intrinsic, llvm_ty)
+                    } else {
+                        FunctionSignature::NonMatchingSignature {
+                            intrinsic,
+                            llvm_ty,
+                            rust_ty: self.rust_signature(cx),
+                        }
+                    }
                 } else {
                     FunctionSignature::RustSignature(intrinsic, self.rust_signature(cx))
                 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1680,46 +1680,6 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
     }
 }
 impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
-    fn trunc_int_to_i1_vector(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
-        let vector_length = self.vector_length(dest_ty) as u64;
-        let int_width = vector_length.next_power_of_two().max(8);
-
-        let bitcasted = self.bitcast(val, self.type_vector(self.type_i1(), int_width));
-        if vector_length == int_width {
-            bitcasted
-        } else {
-            let shuffle_mask =
-                (0..vector_length).map(|i| self.const_i32(i as i32)).collect::<Vec<_>>();
-            self.shuffle_vector(bitcasted, bitcasted, self.const_vector(&shuffle_mask))
-        }
-    }
-
-    fn zext_i1_vector_to_int(
-        &mut self,
-        mut val: &'ll Value,
-        src_ty: &'ll Type,
-        dest_ty: &'ll Type,
-    ) -> &'ll Value {
-        let vector_length = self.vector_length(src_ty) as u64;
-        let int_width = vector_length.next_power_of_two().max(8);
-
-        if vector_length != int_width {
-            let shuffle_indices = match vector_length {
-                0 => unreachable!("zero length vectors are not allowed"),
-                1 => vec![0, 1, 1, 1, 1, 1, 1, 1],
-                2 => vec![0, 1, 2, 3, 2, 3, 2, 3],
-                3 => vec![0, 1, 2, 3, 4, 5, 3, 4],
-                4.. => (0..int_width as i32).collect(),
-            };
-            let shuffle_mask =
-                shuffle_indices.into_iter().map(|i| self.const_i32(i)).collect::<Vec<_>>();
-            val =
-                self.shuffle_vector(val, self.const_null(src_ty), self.const_vector(&shuffle_mask));
-        }
-
-        self.bitcast(val, dest_ty)
-    }
-
     fn autocast(
         &mut self,
         llfn: &'ll Value,
@@ -1748,14 +1708,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
                 }
                 ret
             }
-            TypeKind::Vector if self.element_type(llvm_ty) == self.type_i1() => {
-                if is_argument {
-                    self.trunc_int_to_i1_vector(val, dest_ty)
-                } else {
-                    self.zext_i1_vector_to_int(val, src_ty, dest_ty)
-                }
-            }
-            _ => self.bitcast(val, dest_ty), // for `bf16(xN)` <-> `u16(xN)`
+            _ => unreachable!(),
         }
     }
 

compiler/rustc_codegen_llvm/src/declare.rs

@@ -155,18 +155,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
         let span = || instance.map(|instance| self.tcx.def_span(instance.def_id()));
 
-        if let FunctionSignature::LLVMSignature(_, llvm_fn_ty) = signature {
-            // check if the intrinsic signatures match
-            if !fn_abi.verify_intrinsic_signature(self, llvm_fn_ty) {
-                self.tcx.dcx().emit_fatal(errors::IntrinsicSignatureMismatch {
-                    name,
-                    llvm_fn_ty: &format!("{llvm_fn_ty:?}"),
-                    rust_fn_ty: &format!("{:?}", fn_abi.rust_signature(self)),
-                    span: span(),
-                });
-            }
-        }
-
         // Function addresses in Rust are never significant, allowing functions to
         // be merged.
         let llfn = declare_raw_fn(
@@ -212,10 +200,20 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             fn_abi.apply_attrs_llfn(self, llfn, instance);
         }
 
+        if let FunctionSignature::NonMatchingSignature { llvm_ty, rust_ty, .. } = signature {
+            let (can_upgrade, _) = llvm::check_autoupgrade(llfn);
+            if !can_upgrade {
+                self.tcx.dcx().emit_fatal(errors::IntrinsicSignatureMismatch {
+                    name,
+                    llvm_fn_ty: &format!("{llvm_ty:?}"),
+                    rust_fn_ty: &format!("{rust_ty:?}"),
+                    span: span(),
+                });
+            }
+        }
+
         if let FunctionSignature::MaybeInvalid(..) = signature {
-            let mut new_llfn = None;
-            let can_upgrade =
-                unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
+            let (can_upgrade, new_llfn) = llvm::check_autoupgrade(llfn);
 
             // we can emit diagnostics for local crates only
             if let Some(instance) = instance

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1101,9 +1101,6 @@ unsafe extern "C" {
     pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type;
     pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type;
 
-    // Operations on non-IEEE real types
-    pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type;
-
     // Operations on function types
     pub(crate) fn LLVMFunctionType<'a>(
         ReturnType: &'a Type,

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -327,6 +327,12 @@ impl Intrinsic {
     }
 }
 
+pub(crate) fn check_autoupgrade(llfn: &Value) -> (bool, Option<&Value>) {
+    let mut new_llfn = None;
+    let can_upgrade = unsafe { LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
+    (can_upgrade, new_llfn)
+}
+
 /// Safe wrapper for `LLVMSetValueName2` from a byte slice
 pub(crate) fn set_value_name(value: &Value, name: &[u8]) {
     unsafe {

compiler/rustc_codegen_llvm/src/type_.rs

@@ -176,10 +176,6 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             )
         }
     }
-
-    pub(crate) fn type_bf16(&self) -> &'ll Type {
-        unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) }
-    }
 }
 
 impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
@@ -253,7 +249,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
 
     fn float_width(&self, ty: &'ll Type) -> usize {
         match self.type_kind(ty) {
-            TypeKind::Half | TypeKind::BFloat => 16,
+            TypeKind::Half => 16,
             TypeKind::Float => 32,
             TypeKind::Double => 64,
             TypeKind::X86_FP80 => 80,

tests/codegen-llvm/inject-autocast.rs

@@ -4,51 +4,11 @@
 #![feature(link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi, portable_simd, f16)]
 #![crate_type = "lib"]
 
-use std::simd::{f32x4, i16x8, i64x2};
-
-#[repr(simd)]
-pub struct Tile([i8; 1024]);
-
+use std::simd::i64x2;
 #[repr(C, packed)]
 pub struct Bar(u32, i64x2, i64x2, i64x2, i64x2, i64x2, i64x2);
 // CHECK: %Bar = type <{ i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> }>
 
-#[repr(simd)]
-pub struct f16x8([f16; 8]);
-
-// CHECK-LABEL: @struct_with_i1_vector_autocast
-#[no_mangle]
-pub unsafe fn struct_with_i1_vector_autocast(a: i64x2, b: i64x2) -> (u8, u8) {
-    extern "unadjusted" {
-        #[link_name = "llvm.x86.avx512.vp2intersect.q.128"]
-        fn foo(a: i64x2, b: i64x2) -> (u8, u8);
-    }
-
-    // CHECK: %2 = call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> %0, <2 x i64> %1)
-    // CHECK-NEXT: %3 = extractvalue { <2 x i1>, <2 x i1> } %2, 0
-    // CHECK-NEXT: %4 = shufflevector <2 x i1> %3, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
-    // CHECK-NEXT: %5 = bitcast <8 x i1> %4 to i8
-    // CHECK-NEXT: %6 = insertvalue { i8, i8 } poison, i8 %5, 0
-    // CHECK-NEXT: %7 = extractvalue { <2 x i1>, <2 x i1> } %2, 1
-    // CHECK-NEXT: %8 = shufflevector <2 x i1> %7, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
-    // CHECK-NEXT: %9 = bitcast <8 x i1> %8 to i8
-    // CHECK-NEXT: %10 = insertvalue { i8, i8 } %6, i8 %9, 1
-    foo(a, b)
-}
-
-// CHECK-LABEL: @bf16_vector_autocast
-#[no_mangle]
-pub unsafe fn bf16_vector_autocast(a: f32x4) -> i16x8 {
-    extern "unadjusted" {
-        #[link_name = "llvm.x86.vcvtneps2bf16128"]
-        fn foo(a: f32x4) -> i16x8;
-    }
-
-    // CHECK: %1 = call <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float> %0)
-    // CHECK-NEXT: %2 = bitcast <8 x bfloat> %1 to <8 x i16>
-    foo(a)
-}
-
 // CHECK-LABEL: @struct_autocast
 #[no_mangle]
 pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar {
@@ -75,23 +35,4 @@ pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar {
     foo(key_metadata, key)
 }
 
-// CHECK-LABEL: @i1_vector_autocast
-#[no_mangle]
-pub unsafe fn i1_vector_autocast(a: f16x8) -> u8 {
-    extern "unadjusted" {
-        #[link_name = "llvm.x86.avx512fp16.fpclass.ph.128"]
-        fn foo(a: f16x8, b: i32) -> u8;
-    }
-
-    // CHECK: %1 = call <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half> %0, i32 1)
-    // CHECK-NEXT: %_0 = bitcast <8 x i1> %1 to i8
-    foo(a, 1)
-}
-
-// CHECK: declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>)
-
-// CHECK: declare <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float>)
-
 // CHECK: declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>)
-
-// CHECK: declare <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half>, i32 immarg)