diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 854e3ccc21b4f..abf0ea8cc0a65 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -353,7 +353,11 @@ pub trait FnAbiLlvmExt<'tcx> { impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { - let args_capacity: usize = self.args.iter().map(|arg| + // Ignore "extra" args from the call site for C variadic functions. + // Only the "fixed" args are part of the LLVM function signature. + let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + + let args_capacity: usize = args.iter().map(|arg| if arg.pad.is_some() { 1 } else { 0 } + if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } ).sum(); @@ -371,7 +375,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } }; - for arg in &self.args { + for arg in args { // add padding if let Some(ty) = arg.pad { llargument_tys.push(ty.llvm_type(cx)); diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 068e5e99efead..2d79b73cf9f51 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -78,8 +78,14 @@ pub(crate) unsafe fn codegen( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::>(); - let ret = - llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + None, + ); llvm::LLVMSetTailCall(ret, True); if output.is_some() { llvm::LLVMBuildRet(llbuilder, ret); @@ -121,7 +127,8 @@ pub(crate) unsafe fn codegen( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::>(); - let ret = llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None); + let ret = + llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); llvm::LLVMSetTailCall(ret, True); llvm::LLVMBuildRetVoid(llbuilder); llvm::LLVMDisposeBuilder(llbuilder); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 7bd9397d64950..ebc3773df57cd 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -464,7 +464,7 @@ fn inline_asm_call( alignstack, llvm::AsmDialect::from_generic(dia), ); - let call = bx.call(v, inputs, None); + let call = bx.call(fty, v, inputs, None); // Store mark in a metadata node so we can map LLVM errors // back to source locations. See #17552. diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index d8fa7786c3780..2139f9776b736 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -200,6 +200,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, + llty: &'ll Type, llfn: &'ll Value, args: &[&'ll Value], then: &'ll BasicBlock, @@ -208,13 +209,14 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { debug!("invoke {:?} with args ({:?})", llfn, args); - let args = self.check_call("invoke", llfn, args); + let args = self.check_call("invoke", llty, llfn, args); let bundle = funclet.map(|funclet| funclet.bundle()); let bundle = bundle.as_ref().map(|b| &*b.raw); unsafe { llvm::LLVMRustBuildInvoke( self.llbuilder, + llty, llfn, args.as_ptr(), args.len() as c_uint, @@ -369,8 +371,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { }, }; - let intrinsic = self.get_intrinsic(&name); - let res = self.call(intrinsic, &[lhs, rhs], None); + let res = self.call_intrinsic(name, &[lhs, rhs]); (self.extract_value(res, 0), self.extract_value(res, 1)) } @@ -695,8 +696,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); let name = format!("llvm.fptoui.sat.i{}.f{}", int_width, float_width); - let intrinsic = self.get_intrinsic(&name); - return Some(self.call(intrinsic, &[val], None)); + return Some(self.call_intrinsic(&name, &[val])); } None @@ -708,8 +708,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); let name = format!("llvm.fptosi.sat.i{}.f{}", int_width, float_width); - let intrinsic = self.get_intrinsic(&name); - return Some(self.call(intrinsic, &[val], None)); + return Some(self.call_intrinsic(&name, &[val])); } None @@ -743,8 +742,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { _ => None, }; if let Some(name) = name { - let intrinsic = self.get_intrinsic(name); - return self.call(intrinsic, &[val], None); + return self.call_intrinsic(name, &[val]); } } } @@ -766,8 +764,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { _ => None, }; if let Some(name) = name { - let intrinsic = self.get_intrinsic(name); - return self.call(intrinsic, &[val], None); + return self.call_intrinsic(name, &[val]); } } } @@ -1115,12 +1112,17 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ); let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], + self.cx.type_void(), + ); let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llfn, args); + let args = self.check_call("call", llty, llfn, args); unsafe { let _ = llvm::LLVMRustBuildCall( self.llbuilder, + llty, llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, @@ -1131,19 +1133,21 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, + llty: &'ll Type, llfn: &'ll Value, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, ) -> &'ll Value { debug!("call {:?} with args ({:?})", llfn, args); - let args = self.check_call("call", llfn, args); + let args = self.check_call("call", llty, llfn, args); let bundle = funclet.map(|funclet| funclet.bundle()); let bundle = bundle.as_ref().map(|b| &*b.raw); unsafe { llvm::LLVMRustBuildCall( self.llbuilder, + llty, llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, @@ -1313,15 +1317,10 @@ impl Builder<'a, 'll, 'tcx> { fn check_call<'b>( &mut self, typ: &str, + fn_ty: &'ll Type, llfn: &'ll Value, args: &'b [&'ll Value], ) -> Cow<'b, [&'ll Value]> { - let mut fn_ty = self.cx.val_ty(llfn); - // Strip off pointers - while self.cx.type_kind(fn_ty) == TypeKind::Pointer { - fn_ty = self.cx.element_type(fn_ty); - } - assert!( self.cx.type_kind(fn_ty) == TypeKind::Function, "builder::{} not passed a function, but {:?}", @@ -1362,6 +1361,11 @@ impl Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } + crate fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + let (ty, f) = self.cx.get_intrinsic(intrinsic); + self.call(ty, f, args, None) + } + fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { let size = size.bytes(); if size == 0 { @@ -1372,10 +1376,8 @@ impl Builder<'a, 'll, 'tcx> { return; } - let lifetime_intrinsic = self.cx.get_intrinsic(intrinsic); - let ptr = self.pointercast(ptr, self.cx.type_i8p()); - self.call(lifetime_intrinsic, &[self.cx.const_u64(size), ptr], None); + self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); } pub(crate) fn phi( diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 59259857b4b0d..685f222e8025f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -84,9 +84,9 @@ pub struct CodegenCx<'ll, 'tcx> { eh_personality: Cell>, eh_catch_typeinfo: Cell>, - pub rust_try_fn: Cell>, + pub rust_try_fn: Cell>, - intrinsics: RefCell>, + intrinsics: RefCell>, /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell, @@ -452,7 +452,7 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl CodegenCx<'b, 'tcx> { - crate fn get_intrinsic(&self, key: &str) -> &'b Value { + crate fn get_intrinsic(&self, key: &str) -> (&'b Type, &'b Value) { if let Some(v) = self.intrinsics.borrow().get(key).cloned() { return v; } @@ -465,18 +465,18 @@ impl CodegenCx<'b, 'tcx> { name: &'static str, args: Option<&[&'b llvm::Type]>, ret: &'b llvm::Type, - ) -> &'b llvm::Value { + ) -> (&'b llvm::Type, &'b llvm::Value) { let fn_ty = if let Some(args) = args { self.type_func(args, ret) } else { self.type_variadic_func(&[], ret) }; let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); - self.intrinsics.borrow_mut().insert(name, f); - f + self.intrinsics.borrow_mut().insert(name, (fn_ty, f)); + (fn_ty, f) } - fn declare_intrinsic(&self, key: &str) -> Option<&'b Value> { + fn declare_intrinsic(&self, key: &str) -> Option<(&'b Type, &'b Value)> { macro_rules! ifn { ($name:expr, fn() -> $ret:expr) => ( if key == $name { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 56563668de6b0..ed48418586517 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,4 +1,4 @@ -use crate::abi::{Abi, FnAbi, LlvmType, PassMode}; +use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm; @@ -24,7 +24,7 @@ use rustc_target::spec::PanicStrategy; use std::cmp::Ordering; use std::iter; -fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<&'ll Value> { +fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<(&'ll Type, &'ll Value)> { let llvm_name = match name { sym::sqrtf32 => "llvm.sqrt.f32", sym::sqrtf64 => "llvm.sqrt.f64", @@ -102,19 +102,20 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let simple = get_simple_intrinsic(self, name); let llval = match name { - _ if simple.is_some() => self.call( - simple.unwrap(), - &args.iter().map(|arg| arg.immediate()).collect::>(), - None, - ), - sym::likely => { - let expect = self.get_intrinsic(&("llvm.expect.i1")); - self.call(expect, &[args[0].immediate(), self.const_bool(true)], None) + _ if simple.is_some() => { + let (simple_ty, simple_fn) = simple.unwrap(); + self.call( + simple_ty, + simple_fn, + &args.iter().map(|arg| arg.immediate()).collect::>(), + None, + ) } - sym::unlikely => { - let expect = self.get_intrinsic(&("llvm.expect.i1")); - self.call(expect, &[args[0].immediate(), self.const_bool(false)], None) + sym::likely => { + self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) } + sym::unlikely => self + .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), kw::Try => { try_intrinsic( self, @@ -125,13 +126,9 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { ); return; } - sym::breakpoint => { - let llfn = self.get_intrinsic(&("llvm.debugtrap")); - self.call(llfn, &[], None) - } + sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), sym::va_copy => { - let intrinsic = self.cx().get_intrinsic(&("llvm.va_copy")); - self.call(intrinsic, &[args[0].immediate(), args[1].immediate()], None) + self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) } sym::va_arg => { match fn_abi.ret.layout.abi { @@ -194,7 +191,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { | sym::prefetch_write_data | sym::prefetch_read_instruction | sym::prefetch_write_instruction => { - let expect = self.get_intrinsic(&("llvm.prefetch")); let (rw, cache_type) = match name { sym::prefetch_read_data => (0, 1), sym::prefetch_write_data => (1, 1), @@ -202,15 +198,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { sym::prefetch_write_instruction => (1, 0), _ => bug!(), }; - self.call( - expect, + self.call_intrinsic( + "llvm.prefetch", &[ args[0].immediate(), self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type), ], - None, ) } sym::ctlz @@ -229,35 +224,33 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { Some((width, signed)) => match name { sym::ctlz | sym::cttz => { let y = self.const_bool(false); - let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width)); - self.call(llfn, &[args[0].immediate(), y], None) + self.call_intrinsic( + &format!("llvm.{}.i{}", name, width), + &[args[0].immediate(), y], + ) } sym::ctlz_nonzero | sym::cttz_nonzero => { let y = self.const_bool(true); let llvm_name = &format!("llvm.{}.i{}", &name_str[..4], width); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[args[0].immediate(), y], None) + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) } - sym::ctpop => self.call( - self.get_intrinsic(&format!("llvm.ctpop.i{}", width)), + sym::ctpop => self.call_intrinsic( + &format!("llvm.ctpop.i{}", width), &[args[0].immediate()], - None, ), sym::bswap => { if width == 8 { args[0].immediate() // byte swap a u8/i8 is just a no-op } else { - self.call( - self.get_intrinsic(&format!("llvm.bswap.i{}", width)), + self.call_intrinsic( + &format!("llvm.bswap.i{}", width), &[args[0].immediate()], - None, ) } } - sym::bitreverse => self.call( - self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)), + sym::bitreverse => self.call_intrinsic( + &format!("llvm.bitreverse.i{}", width), &[args[0].immediate()], - None, ), sym::rotate_left | sym::rotate_right => { let is_left = name == sym::rotate_left; @@ -266,8 +259,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { // rotate = funnel shift with first two args the same let llvm_name = &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[val, val, raw_shift], None) + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; @@ -279,8 +271,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { if is_add { "add" } else { "sub" }, width ); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[lhs, rhs], None) + self.call_intrinsic(llvm_name, &[lhs, rhs]) } _ => bug!(), }, @@ -331,8 +322,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let a_ptr = self.bitcast(a, i8p_ty); let b_ptr = self.bitcast(b, i8p_ty); let n = self.const_usize(layout.size.bytes()); - let llfn = self.get_intrinsic("memcmp"); - let cmp = self.call(llfn, &[a_ptr, b_ptr, n], None); + let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) } } @@ -361,18 +351,15 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } fn abort(&mut self) { - let fnname = self.get_intrinsic(&("llvm.trap")); - self.call(fnname, &[], None); + self.call_intrinsic("llvm.trap", &[]); } fn assume(&mut self, val: Self::Value) { - let assume_intrinsic = self.get_intrinsic("llvm.assume"); - self.call(assume_intrinsic, &[val], None); + self.call_intrinsic("llvm.assume", &[val]); } fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { - let expect = self.get_intrinsic(&"llvm.expect.i1"); - self.call(expect, &[cond, self.const_bool(expected)], None) + self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) } fn sideeffect(&mut self) { @@ -380,19 +367,16 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM // codegen backend being used, and so is unable to check the LLVM version. if unsafe { llvm::LLVMRustVersionMajor() } < 12 { - let fnname = self.get_intrinsic(&("llvm.sideeffect")); - self.call(fnname, &[], None); + self.call_intrinsic("llvm.sideeffect", &[]); } } fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { - let intrinsic = self.cx().get_intrinsic("llvm.va_start"); - self.call(intrinsic, &[va_list], None) + self.call_intrinsic("llvm.va_start", &[va_list]) } fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { - let intrinsic = self.cx().get_intrinsic("llvm.va_end"); - self.call(intrinsic, &[va_list], None) + self.call_intrinsic("llvm.va_end", &[va_list]) } } @@ -404,7 +388,8 @@ fn try_intrinsic( dest: &'ll Value, ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { - bx.call(try_func, &[data], None); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.call(try_func_ty, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -432,7 +417,7 @@ fn codegen_msvc_try( catch_func: &'ll Value, dest: &'ll Value, ) { - let llfn = get_rust_try_fn(bx, &mut |mut bx| { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { bx.set_personality_fn(bx.eh_personality()); let mut normal = bx.build_sibling_block("normal"); @@ -502,7 +487,8 @@ fn codegen_msvc_try( // More information can be found in libstd's seh.rs implementation. let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); - bx.invoke(try_func, &[data], normal.llbb(), catchswitch.llbb(), None); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], normal.llbb(), catchswitch.llbb(), None); normal.ret(bx.const_i32(0)); @@ -544,14 +530,15 @@ fn codegen_msvc_try( let flags = bx.const_i32(8); let funclet = catchpad_rust.catch_pad(cs, &[tydesc, flags, slot]); let ptr = catchpad_rust.load(bx.type_i8p(), slot, ptr_align); - catchpad_rust.call(catch_func, &[data, ptr], Some(&funclet)); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + catchpad_rust.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); catchpad_rust.catch_ret(&funclet, caught.llbb()); // The flag value of 64 indicates a "catch-all". let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = catchpad_foreign.catch_pad(cs, &[null, flags, null]); - catchpad_foreign.call(catch_func, &[data, null], Some(&funclet)); + catchpad_foreign.call(catch_ty, catch_func, &[data, null], Some(&funclet)); catchpad_foreign.catch_ret(&funclet, caught.llbb()); caught.ret(bx.const_i32(1)); @@ -559,7 +546,7 @@ fn codegen_msvc_try( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -582,7 +569,7 @@ fn codegen_gnu_try( catch_func: &'ll Value, dest: &'ll Value, ) { - let llfn = get_rust_try_fn(bx, &mut |mut bx| { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { // Codegens the shims described above: // // bx: @@ -601,7 +588,8 @@ fn codegen_gnu_try( let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); - bx.invoke(try_func, &[data], then.llbb(), catch.llbb(), None); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then.llbb(), catch.llbb(), None); then.ret(bx.const_i32(0)); // Type indicator for the exception being thrown. @@ -615,13 +603,14 @@ fn codegen_gnu_try( let tydesc = bx.const_null(bx.type_i8p()); catch.add_clause(vals, tydesc); let ptr = catch.extract_value(vals, 0); - catch.call(catch_func, &[data, ptr], None); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + catch.call(catch_ty, catch_func, &[data, ptr], None); catch.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -636,7 +625,7 @@ fn codegen_emcc_try( catch_func: &'ll Value, dest: &'ll Value, ) { - let llfn = get_rust_try_fn(bx, &mut |mut bx| { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { // Codegens the shims described above: // // bx: @@ -660,7 +649,8 @@ fn codegen_emcc_try( let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); - bx.invoke(try_func, &[data], then.llbb(), catch.llbb(), None); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then.llbb(), catch.llbb(), None); then.ret(bx.const_i32(0)); // Type indicator for the exception being thrown. @@ -677,8 +667,7 @@ fn codegen_emcc_try( let selector = catch.extract_value(vals, 1); // Check if the typeid we got is the one for a Rust panic. - let llvm_eh_typeid_for = bx.get_intrinsic("llvm.eh.typeid.for"); - let rust_typeid = catch.call(llvm_eh_typeid_for, &[tydesc], None); + let rust_typeid = catch.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); let is_rust_panic = catch.icmp(IntPredicate::IntEQ, selector, rust_typeid); let is_rust_panic = catch.zext(is_rust_panic, bx.type_bool()); @@ -702,13 +691,14 @@ fn codegen_emcc_try( catch.store(is_rust_panic, catch_data_1, i8_align); let catch_data = catch.bitcast(catch_data, bx.type_i8p()); - catch.call(catch_func, &[data, catch_data], None); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + catch.call(catch_ty, catch_func, &[data, catch_data], None); catch.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -720,8 +710,9 @@ fn gen_fn<'ll, 'tcx>( name: &str, rust_fn_sig: ty::PolyFnSig<'tcx>, codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), -) -> &'ll Value { +) -> (&'ll Type, &'ll Value) { let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]); + let llty = fn_abi.llvm_type(cx); let llfn = cx.declare_fn(name, &fn_abi); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); @@ -730,7 +721,7 @@ fn gen_fn<'ll, 'tcx>( let llbb = Builder::append_block(cx, llfn, "entry-block"); let bx = Builder::build(cx, llbb); codegen(bx); - llfn + (llty, llfn) } // Helper function used to get a handle to the `__rust_try` function used to @@ -740,7 +731,7 @@ fn gen_fn<'ll, 'tcx>( fn get_rust_try_fn<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), -) -> &'ll Value { +) -> (&'ll Type, &'ll Value) { if let Some(llfn) = cx.rust_try_fn.get() { return llfn; } @@ -1123,7 +1114,8 @@ fn generic_simd_intrinsic( }; let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty); - let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::>(), None); + let c = + bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::>(), None); Ok(c) } @@ -1300,15 +1292,13 @@ fn generic_simd_intrinsic( let llvm_intrinsic = format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); - let f = bx.declare_cfn( - &llvm_intrinsic, - llvm::UnnamedAddr::No, - bx.type_func( - &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], - llvm_elem_vec_ty, - ), + let fn_ty = bx.type_func( + &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], + llvm_elem_vec_ty, ); - let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); return Ok(v); } @@ -1430,12 +1420,11 @@ fn generic_simd_intrinsic( let llvm_intrinsic = format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); - let f = bx.declare_cfn( - &llvm_intrinsic, - llvm::UnnamedAddr::No, - bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t), - ); - let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); + let fn_ty = + bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); return Ok(v); } @@ -1757,12 +1746,9 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ); let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - let f = bx.declare_cfn( - &llvm_intrinsic, - llvm::UnnamedAddr::No, - bx.type_func(&[vec_ty, vec_ty], vec_ty), - ); - let v = bx.call(f, &[lhs, rhs], None); + let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call(fn_ty, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e803ad6d88e3c..57173a49107a5 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1155,6 +1155,7 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustBuildInvoke( B: &Builder<'a>, + Ty: &'a Type, Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, @@ -1526,6 +1527,7 @@ extern "C" { pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &'a Value; pub fn LLVMRustBuildCall( B: &Builder<'a>, + Ty: &'a Type, Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 8fd0caae479a8..52ae56b89b331 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -203,7 +203,11 @@ impl BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn element_type(&self, ty: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMGetElementType(ty) } + match self.type_kind(ty) { + TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, + TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), + other => bug!("element_type called on unsupported type {:?}", other), + } } fn vector_length(&self, ty: &'ll Type) -> usize { @@ -275,6 +279,9 @@ impl LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { ty.llvm_type(self) } + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.llvm_type(self) + } fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { fn_abi.ptr_to_llvm_type(self) } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b60677267849a..064a51c8f6090 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -441,9 +441,11 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx.insert_reference_to_gdb_debug_scripts_section_global(); + let isize_ty = cx.type_isize(); + let i8pp_ty = cx.type_ptr_to(cx.type_i8p()); let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx); - let (start_fn, args) = if use_start_lang_item { + let (start_fn, start_ty, args) = if use_start_lang_item { let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None); let start_fn = cx.get_fn_addr( ty::Instance::resolve( @@ -455,16 +457,15 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .unwrap() .unwrap(), ); - ( - start_fn, - vec![bx.pointercast(rust_main, cx.type_ptr_to(cx.type_i8p())), arg_argc, arg_argv], - ) + let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty); + (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv]) } else { debug!("using user-defined start fn"); - (rust_main, vec![arg_argc, arg_argv]) + let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty); + (rust_main, start_ty, vec![arg_argc, arg_argv]) }; - let result = bx.call(start_fn, &args, None); + let result = bx.call(start_ty, start_fn, &args, None); let cast = bx.intcast(result, cx.type_int(), true); bx.ret(cast); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index b584801a62de2..58ce31e0c4c9d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -132,14 +132,21 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { ) { // If there is a cleanup block and the function we're calling can unwind, then // do an invoke, otherwise do a call. + let fn_ty = bx.fn_decl_backend_type(&fn_abi); if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) { let ret_llbb = if let Some((_, target)) = destination { fx.llbb(target) } else { fx.unreachable_block() }; - let invokeret = - bx.invoke(fn_ptr, &llargs, ret_llbb, self.llblock(fx, cleanup), self.funclet(fx)); + let invokeret = bx.invoke( + fn_ty, + fn_ptr, + &llargs, + ret_llbb, + self.llblock(fx, cleanup), + self.funclet(fx), + ); bx.apply_attrs_callsite(&fn_abi, invokeret); if let Some((ret_dest, target)) = destination { @@ -148,7 +155,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fx.store_return(&mut ret_bx, ret_dest, &fn_abi.ret, invokeret); } } else { - let llret = bx.call(fn_ptr, &llargs, self.funclet(fx)); + let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx)); bx.apply_attrs_callsite(&fn_abi, llret); if fx.mir[self.bb].is_cleanup { // Cleanup is always the cold path. Don't inline diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index cbb401c63d152..7e432d2740224 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -518,7 +518,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let instance = ty::Instance::mono(bx.tcx(), def_id); let r = bx.cx().get_fn_addr(instance); - let call = bx.call(r, &[llsize, llalign], None); + let ty = bx.type_func(&[bx.type_isize(), bx.type_isize()], bx.type_i8p()); + let call = bx.call(ty, r, &[llsize, llalign], None); let val = bx.pointercast(call, llty_ptr); let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout }; diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index f2c523148530f..afb8ee3c40d55 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -72,6 +72,7 @@ pub trait BuilderMethods<'a, 'tcx>: ); fn invoke( &mut self, + llty: Self::Type, llfn: Self::Value, args: &[Self::Value], then: Self::BasicBlock, @@ -303,6 +304,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn call( &mut self, + llty: Self::Type, llfn: Self::Value, args: &[Self::Value], funclet: Option<&Self::Funclet>, diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 634a20bda9bfd..b94fb1e10db01 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -102,6 +102,7 @@ impl DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscM pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn reg_backend_type(&self, ty: &Reg) -> Self::Type; fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 7666803911e0b..52566b19eca04 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1392,11 +1392,11 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { delete Bundle; } -extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn, +extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, OperandBundleDef *Bundle) { Value *Callee = unwrap(Fn); - FunctionType *FTy = cast(Callee->getType()->getPointerElementType()); + FunctionType *FTy = unwrap(Ty); unsigned Len = Bundle ? 1 : 0; ArrayRef Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateCall( @@ -1437,12 +1437,12 @@ extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, } extern "C" LLVMValueRef -LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args, - unsigned NumArgs, LLVMBasicBlockRef Then, - LLVMBasicBlockRef Catch, OperandBundleDef *Bundle, - const char *Name) { +LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + OperandBundleDef *Bundle, const char *Name) { Value *Callee = unwrap(Fn); - FunctionType *FTy = cast(Callee->getType()->getPointerElementType()); + FunctionType *FTy = unwrap(Ty); unsigned Len = Bundle ? 1 : 0; ArrayRef Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch),