diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index 9ee046915daca..4562281653388 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -1559,6 +1559,7 @@ extern { /* Selected entries from the downcasts. */ pub fn LLVMIsATerminatorInst(Inst: ValueRef) -> ValueRef; pub fn LLVMIsAStoreInst(Inst: ValueRef) -> ValueRef; + pub fn LLVMIsAZExtInst(Inst: ValueRef) -> ValueRef; /// Writes a module to the specified path. Returns 0 on success. pub fn LLVMWriteBitcodeToFile(M: ModuleRef, Path: *const c_char) -> c_int; diff --git a/src/librustc_trans/trans/base.rs b/src/librustc_trans/trans/base.rs index d59cc4f4298e9..1efc0c22838f2 100644 --- a/src/librustc_trans/trans/base.rs +++ b/src/librustc_trans/trans/base.rs @@ -54,7 +54,7 @@ use trans::callee; use trans::cleanup::{self, CleanupMethods, DropHint}; use trans::closure; use trans::common::{Block, C_bool, C_bytes_in_context, C_i32, C_int, C_integral}; -use trans::common::{C_null, C_struct_in_context, C_u64, C_u8, C_undef}; +use trans::common::{C_null, C_struct_in_context, C_u64, C_u8, C_undef, C_nil}; use trans::common::{CrateContext, DropFlagHintsMap, Field, FunctionContext}; use trans::common::{Result, NodeIdAndSpan, VariantInfo}; use trans::common::{node_id_type, return_type_is_void}; @@ -699,8 +699,22 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debug_loc: DebugLoc) -> (ValueRef, Block<'blk, 'tcx>) { let _icx = push_ctxt("invoke_"); + + let ret_ty = match fn_ty.sty { + ty::TyBareFn(_, ref f) => { + let output = bcx.tcx().erase_late_bound_regions(&f.sig.output()); + output + } + _ => panic!("expected bare rust fn or closure in trans_call_inner") + }; + if bcx.unreachable.get() { - return (C_null(Type::i8(bcx.ccx())), bcx); + if let ty::FnConverging(ty) = ret_ty { + let llty = type_of::arg_type_of(bcx.ccx(), ty); + return (C_undef(llty), bcx); + } else { + return (C_nil(bcx.ccx()), bcx); + } } let attributes = attributes::from_fn_type(bcx.ccx(), fn_ty); @@ -714,7 +728,7 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } } - if need_invoke(bcx) { + let (mut llresult, bcx) = if need_invoke(bcx) { debug!("invoking {} at {:?}", bcx.val_to_string(llfn), bcx.llbb); for &llarg in llargs { debug!("arg: {}", bcx.val_to_string(llarg)); @@ -729,7 +743,7 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, landing_pad, Some(attributes), debug_loc); - return (llresult, normal_bcx); + (llresult, normal_bcx) } else { debug!("calling {} at {:?}", bcx.val_to_string(llfn), bcx.llbb); for &llarg in llargs { @@ -737,12 +751,22 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } let llresult = Call(bcx, - llfn, - &llargs[..], - Some(attributes), - debug_loc); - return (llresult, bcx); + llfn, + &llargs[..], + Some(attributes), + debug_loc); + (llresult, bcx) + }; + + if let ty::FnConverging(ty) = ret_ty { + if return_type_is_void(bcx.ccx(), ty) { + llresult = C_nil(bcx.ccx()); + } + } else { + llresult = C_nil(bcx.ccx()); } + + (llresult, bcx) } pub fn need_invoke(bcx: Block) -> bool { @@ -783,6 +807,13 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, let align = type_of::align_of(cx.ccx(), t); if type_is_immediate(cx.ccx(), t) && type_of::type_of(cx.ccx(), t).is_aggregate() { + if let Some(val) = Value(ptr).get_stored_value_opt(cx) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr)); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return val.get(); + } + let load = Load(cx, ptr); unsafe { llvm::LLVMSetAlignment(load, align); @@ -800,6 +831,13 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, } } + if let Some(val) = Value(ptr).get_stored_value_opt(cx) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr)); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return to_arg_ty(cx, val.get(), t); + } + let val = if t.is_bool() { LoadRangeAssert(cx, ptr, 0, 2, llvm::False) } else if t.is_char() { @@ -831,7 +869,18 @@ pub fn store_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, v: ValueRef, dst: ValueRef, t Store(cx, ExtractValue(cx, v, abi::FAT_PTR_ADDR), expr::get_dataptr(cx, dst)); Store(cx, ExtractValue(cx, v, abi::FAT_PTR_EXTRA), expr::get_len(cx, dst)); } else { - let store = Store(cx, from_arg_ty(cx, v, t), to_arg_ty_ptr(cx, dst, t)); + let vty = val_ty(v); + let dstty = val_ty(dst).element_type(); + + // If the source and destination have the same type, then don't try any conversion, + // this can happen with the return values of functions, as they don't touch an alloca + let (v, dst) = if vty == dstty { + (v, dst) + } else { + (from_arg_ty(cx, v, t), to_arg_ty_ptr(cx, dst, t)) + }; + + let store = Store(cx, v, dst); unsafe { llvm::LLVMSetAlignment(store, type_of::align_of(cx.ccx(), t)); } @@ -848,7 +897,21 @@ pub fn from_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { pub fn to_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { if ty.is_bool() { - Trunc(bcx, val, Type::i1(bcx.ccx())) + // Look for cases where we're truncating a zext from a bool already and grab the original + // value. This can happen with an elided load from a boolean location. While this can be + // easily optimized out, the indirection can interfere with some intrinsics. + let val = Value(val); + if let Some(zext) = val.as_zext_inst() { + // `val` == zext %foo + if let Some(val) = zext.get_operand(0) { + let valty = val_ty(val.get()); + if valty == Type::i1(bcx.ccx()) { + // %foo : i1 + return val.get(); + } + } + } + Trunc(bcx, val.get(), Type::i1(bcx.ccx())) } else { val } @@ -1709,8 +1772,7 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, disr: ty::Disr, args: callee::CallArgs, dest: expr::Dest, - debug_loc: DebugLoc) - -> Result<'blk, 'tcx> { + debug_loc: DebugLoc) -> Result<'blk, 'tcx> { let ccx = bcx.fcx.ccx; @@ -1753,6 +1815,12 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } } + let llretval = if type_is_immediate(bcx.ccx(), result_ty) { + load_ty(bcx, llresult, result_ty) + } else { + C_undef(type_of::type_of(bcx.ccx(), result_ty)) + }; + // If the caller doesn't care about the result // drop the temporary we made let bcx = match dest { @@ -1766,7 +1834,8 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } }; - Result::new(bcx, llresult) + + Result::new(bcx, llretval) } pub fn trans_tuple_struct<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, diff --git a/src/librustc_trans/trans/basic_block.rs b/src/librustc_trans/trans/basic_block.rs index d3d055cda1202..290f6d20a9153 100644 --- a/src/librustc_trans/trans/basic_block.rs +++ b/src/librustc_trans/trans/basic_block.rs @@ -13,7 +13,7 @@ use llvm::BasicBlockRef; use trans::value::{Users, Value}; use std::iter::{Filter, Map}; -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] pub struct BasicBlock(pub BasicBlockRef); pub type Preds = Map bool>, fn(Value) -> BasicBlock>; diff --git a/src/librustc_trans/trans/builder.rs b/src/librustc_trans/trans/builder.rs index 107ae378ac446..a47ee5a0a7c07 100644 --- a/src/librustc_trans/trans/builder.rs +++ b/src/librustc_trans/trans/builder.rs @@ -16,6 +16,7 @@ use llvm::{Opcode, IntPredicate, RealPredicate, False}; use llvm::{ValueRef, BasicBlockRef, BuilderRef, ModuleRef}; use trans::base; use trans::common::*; +use trans::llrepr::LlvmRepr; use trans::machine::llalign_of_pref; use trans::type_::Type; use util::nodemap::FnvHashMap; @@ -158,8 +159,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { args: &[ValueRef], then: BasicBlockRef, catch: BasicBlockRef, - attributes: Option) - -> ValueRef { + attributes: Option) -> ValueRef { self.count_insn("invoke"); debug!("Invoke {} with args ({})", @@ -169,6 +169,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { .collect::>() .join(", ")); + let llfnty = val_ty(llfn); + + for (i, (pty, &a)) in llfnty.func_params().into_iter().zip(args.iter()).enumerate() { + let aty = val_ty(a); + assert!(pty == aty, "Type mismatch for arg {}. {} is not of type {}", + i, self.ccx.tn().val_to_string(a), self.ccx.tn().type_to_string(pty)); + } + unsafe { let v = llvm::LLVMBuildInvoke(self.llbuilder, llfn, @@ -499,7 +507,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn volatile_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef { - debug!("Store {} -> {}", + debug!("Volatile Store {} -> {}", self.ccx.tn().val_to_string(val), self.ccx.tn().val_to_string(ptr)); assert!(!self.llbuilder.is_null()); @@ -524,6 +532,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn gep(&self, ptr: ValueRef, indices: &[ValueRef]) -> ValueRef { + debug!("GEP from {}, indices: {}", + self.ccx.tn().val_to_string(ptr), + indices.llrepr(self.ccx)); self.count_insn("gep"); unsafe { llvm::LLVMBuildGEP(self.llbuilder, ptr, indices.as_ptr(), @@ -535,6 +546,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // in C_i32() #[inline] pub fn gepi(&self, base: ValueRef, ixs: &[usize]) -> ValueRef { + debug!("GEPi from {}, indices: {:?}", + self.ccx.tn().val_to_string(base), + ixs); // Small vector optimization. This should catch 100% of the cases that // we care about. if ixs.len() < 16 { @@ -559,6 +573,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn struct_gep(&self, ptr: ValueRef, idx: usize) -> ValueRef { + debug!("Struct GEP from {}, index: {}", + self.ccx.tn().val_to_string(ptr), + idx); self.count_insn("structgep"); unsafe { llvm::LLVMBuildStructGEP(self.llbuilder, ptr, idx as c_uint, noname()) diff --git a/src/librustc_trans/trans/callee.rs b/src/librustc_trans/trans/callee.rs index d0d5b46ab2839..e10188bf19038 100644 --- a/src/librustc_trans/trans/callee.rs +++ b/src/librustc_trans/trans/callee.rs @@ -21,7 +21,7 @@ pub use self::CallArgs::*; use arena::TypedArena; use back::link; use session; -use llvm::{self, ValueRef, get_params}; +use llvm::{ValueRef, get_params}; use middle::def; use middle::subst; use middle::subst::{Subst, Substs}; @@ -45,7 +45,6 @@ use trans::foreign; use trans::intrinsic; use trans::meth; use trans::monomorphize; -use trans::type_::Type; use trans::type_of; use middle::ty::{self, Ty, HasTypeFlags, RegionEscape}; use middle::ty::MethodCall; @@ -583,6 +582,23 @@ pub fn trans_call<'a, 'blk, 'tcx>(bcx: Block<'blk, 'tcx>, Some(dest)).bcx } +pub fn trans_datum_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, + call_expr: &ast::Expr, + f: &ast::Expr, + args: CallArgs<'a, 'tcx>) -> DatumBlock<'blk, 'tcx, Expr> { + let ty = common::expr_ty(bcx, call_expr); + + let _icx = push_ctxt("trans_call"); + let val = unpack_result!( + bcx, trans_call_inner(bcx, + call_expr.debug_loc(), + |bcx, _| trans(bcx, f), + args, + Some(expr::Ignore))); + + immediate_rvalue_bcx(bcx, val, ty).to_expr_datumblock() +} + pub fn trans_method_call<'a, 'blk, 'tcx>(bcx: Block<'blk, 'tcx>, call_expr: &ast::Expr, rcvr: &ast::Expr, @@ -705,6 +721,14 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, let is_rust_fn = abi == synabi::Rust || abi == synabi::RustCall; + let llretty = { + let ret_ty = match ret_ty { + ty::FnConverging(ret_ty) => ret_ty, + ty::FnDiverging => ccx.tcx().mk_nil() + }; + type_of::type_of(ccx, ret_ty) + }; + // Generate a location to store the result. If the user does // not care about the result, just make a stack slot. let opt_llretslot = dest.and_then(|dest| match dest { @@ -715,13 +739,12 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, ty::FnDiverging => ccx.tcx().mk_nil() }; if !is_rust_fn || - type_of::return_uses_outptr(ccx, ret_ty) || - bcx.fcx.type_needs_drop(ret_ty) { + type_of::return_uses_outptr(ccx, ret_ty) || + bcx.fcx.type_needs_drop(ret_ty) { // Push the out-pointer if we use an out-pointer for this // return type, otherwise push "undef". if common::type_is_zero_size(ccx, ret_ty) { - let llty = type_of::type_of(ccx, ret_ty); - Some(common::C_undef(llty.ptr_to())) + Some(common::C_undef(llretty.ptr_to())) } else { Some(alloc_ty(bcx, ret_ty, "__llret")) } @@ -731,9 +754,7 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, } }); - let mut llresult = unsafe { - llvm::LLVMGetUndef(Type::nil(ccx).ptr_to().to_ref()) - }; + let llresult; // The code below invokes the function, using either the Rust // conventions (if it is a rust fn) or the native conventions @@ -814,13 +835,16 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, abi); fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); - bcx = foreign::trans_native_call(bcx, - callee.ty, - llfn, - opt_llretslot.unwrap(), - &llargs[..], - arg_tys, - debug_loc); + let (llret, b) = foreign::trans_native_call(bcx, + callee.ty, + llfn, + opt_llretslot.unwrap(), + &llargs[..], + arg_tys, + debug_loc); + + bcx = b; + llresult = llret; } fcx.pop_and_trans_custom_cleanup_scope(bcx, arg_cleanup_scope); @@ -1126,8 +1150,8 @@ pub fn trans_arg_datum<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debug!("--- trans_arg_datum passing {}", bcx.val_to_string(val)); if common::type_is_fat_ptr(bcx.tcx(), formal_arg_ty) { - llargs.push(Load(bcx, expr::get_dataptr(bcx, val))); - llargs.push(Load(bcx, expr::get_len(bcx, val))); + llargs.push(expr::extract_dataptr(bcx, val)); + llargs.push(expr::extract_len(bcx, val)); } else { llargs.push(val); } diff --git a/src/librustc_trans/trans/expr.rs b/src/librustc_trans/trans/expr.rs index c5043f867ded0..d129027d5e5f8 100644 --- a/src/librustc_trans/trans/expr.rs +++ b/src/librustc_trans/trans/expr.rs @@ -184,7 +184,7 @@ pub fn trans_into<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, false); bcx.fcx.push_ast_cleanup_scope(cleanup_debug_loc); - let kind = expr_kind(bcx.tcx(), expr); + let kind = expr_kind(bcx, expr); bcx = match kind { ExprKind::Lvalue | ExprKind::RvalueDatum => { trans_unadjusted(bcx, expr).store_to_dest(dest, expr.id) @@ -289,9 +289,27 @@ pub fn get_dataptr(bcx: Block, fat_ptr: ValueRef) -> ValueRef { GEPi(bcx, fat_ptr, &[0, abi::FAT_PTR_ADDR]) } -pub fn copy_fat_ptr(bcx: Block, src_ptr: ValueRef, dst_ptr: ValueRef) { - Store(bcx, Load(bcx, get_dataptr(bcx, src_ptr)), get_dataptr(bcx, dst_ptr)); - Store(bcx, Load(bcx, get_len(bcx, src_ptr)), get_len(bcx, dst_ptr)); +pub fn extract_len(bcx: Block, fat_ptr: ValueRef) -> ValueRef { + let llty = val_ty(fat_ptr); + if llty.is_pointer() { + Load(bcx, get_len(bcx, fat_ptr)) + } else { + ExtractValue(bcx, fat_ptr, abi::FAT_PTR_EXTRA) + } +} + +pub fn extract_dataptr(bcx: Block, fat_ptr: ValueRef) -> ValueRef { + let llty = val_ty(fat_ptr); + if llty.is_pointer() { + Load(bcx, get_dataptr(bcx, fat_ptr)) + } else { + ExtractValue(bcx, fat_ptr, abi::FAT_PTR_ADDR) + } +} + +pub fn copy_fat_ptr(bcx: Block, src: ValueRef, dst_ptr: ValueRef) { + Store(bcx, extract_dataptr(bcx, src), get_dataptr(bcx, dst_ptr)); + Store(bcx, extract_len(bcx, src), get_len(bcx, dst_ptr)); } /// Retrieve the information we are losing (making dynamic) in an unsizing @@ -452,8 +470,8 @@ fn coerce_unsized<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, // to use a different vtable. In that case, we want to // load out the original data pointer so we can repackage // it. - (Load(bcx, get_dataptr(bcx, source.val)), - Some(Load(bcx, get_len(bcx, source.val)))) + (extract_dataptr(bcx, source.val), + Some(extract_len(bcx, source.val))) } else { let val = if source.kind.is_by_ref() { load_ty(bcx, source.val, source.ty) @@ -577,7 +595,7 @@ fn trans_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debuginfo::set_source_location(bcx.fcx, expr.id, expr.span); - return match expr_kind(bcx.tcx(), expr) { + return match expr_kind(bcx, expr) { ExprKind::Lvalue | ExprKind::RvalueDatum => { let datum = unpack_datum!(bcx, { trans_datum_unadjusted(bcx, expr) @@ -693,6 +711,19 @@ fn trans_datum_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, // Datum output mode means this is a scalar cast: trans_imm_cast(bcx, &**val, expr.id) } + ast::ExprCall(ref f, ref args) => { + if bcx.tcx().is_method_call(expr.id) { + trans_overloaded_datum_call(bcx, + expr, + &**f, + &args[..]) + } else { + callee::trans_datum_call(bcx, + expr, + &**f, + callee::ArgExprs(&args[..])) + } + } _ => { bcx.tcx().sess.span_bug( expr.span, @@ -1494,7 +1525,7 @@ pub fn trans_adt<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // Second, trans the base to the dest. assert_eq!(discr, 0); - match expr_kind(bcx.tcx(), &*base.expr) { + match expr_kind(bcx, &*base.expr) { ExprKind::RvalueDps | ExprKind::RvalueDatum if !bcx.fcx.type_needs_drop(ty) => { bcx = trans_into(bcx, &*base.expr, SaveIn(addr)); }, @@ -1942,6 +1973,36 @@ fn trans_overloaded_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, bcx } +fn trans_overloaded_datum_call<'a, 'blk, 'tcx>( + mut bcx: Block<'blk, 'tcx>, + expr: &ast::Expr, + callee: &'a ast::Expr, + args: &'a [P]) -> DatumBlock<'blk, 'tcx, Expr> { + + let ty = expr_ty(bcx, expr); + + debug!("trans_overloaded_call_datum {}", expr.id); + let method_call = MethodCall::expr(expr.id); + let mut all_args = vec!(callee); + all_args.extend(args.iter().map(|e| &**e)); + + let val = unpack_result!( + bcx, + callee::trans_call_inner(bcx, + expr.debug_loc(), + |bcx, arg_cleanup_scope| { + meth::trans_method_callee( + bcx, + method_call, + None, + arg_cleanup_scope) + }, + callee::ArgOverloadedCall(all_args), + Some(Ignore))); + + immediate_rvalue_bcx(bcx, val, ty).to_expr_datumblock() +} + pub fn cast_is_noop<'tcx>(tcx: &ty::ctxt<'tcx>, expr: &ast::Expr, t_in: Ty<'tcx>, @@ -2031,18 +2092,36 @@ fn trans_imm_cast<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } if type_is_fat_ptr(bcx.tcx(), t_in) { - assert!(datum.kind.is_by_ref()); if type_is_fat_ptr(bcx.tcx(), t_out) { - return DatumBlock::new(bcx, Datum::new( - PointerCast(bcx, datum.val, ll_t_out.ptr_to()), - t_out, - Rvalue::new(ByRef) - )).to_expr_datumblock(); + // If the datum is by-ref, just do a pointer cast, otherwise + // construct the casted fat pointer manually. + if datum.kind.is_by_ref() { + return DatumBlock::new(bcx, Datum::new( + PointerCast(bcx, datum.val, ll_t_out.ptr_to()), + t_out, + Rvalue::new(ByRef))).to_expr_datumblock(); + } else { + let ftys = ll_t_out.field_types(); + + let val = C_undef(ll_t_out); + let val = InsertValue(bcx, val, + BitCast(bcx, + extract_dataptr(bcx, datum.val), + ftys[abi::FAT_PTR_ADDR]), + abi::FAT_PTR_ADDR); + let val = InsertValue(bcx, val, + BitCast(bcx, + extract_len(bcx, datum.val), + ftys[abi::FAT_PTR_EXTRA]), + abi::FAT_PTR_EXTRA); + + return immediate_rvalue_bcx(bcx, val, t_out).to_expr_datumblock(); + } } else { // Return the address return immediate_rvalue_bcx(bcx, PointerCast(bcx, - Load(bcx, get_dataptr(bcx, datum.val)), + extract_dataptr(bcx, datum.val), ll_t_out), t_out).to_expr_datumblock(); } @@ -2379,13 +2458,10 @@ impl OverflowOpViaIntrinsic { let val = Call(bcx, llfn, &[lhs, rhs], None, binop_debug_loc); let result = ExtractValue(bcx, val, 0); // iN operation result - let overflow = ExtractValue(bcx, val, 1); // i1 "did it overflow?" - - let cond = ICmp(bcx, llvm::IntEQ, overflow, C_integral(Type::i1(bcx.ccx()), 1, false), - binop_debug_loc); + let cond = ExtractValue(bcx, val, 1); // i1 "did it overflow?" let expect = bcx.ccx().get_intrinsic(&"llvm.expect.i1"); - Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)], + let cond = Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)], None, binop_debug_loc); let bcx = @@ -2565,7 +2641,9 @@ enum ExprKind { RvalueStmt } -fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { +fn expr_kind(bcx: Block, expr: &ast::Expr) -> ExprKind { + let tcx = bcx.tcx(); + if tcx.is_method_call(expr.id) { // Overloaded operations are generally calls, and hence they are // generated via DPS, but there are a few exceptions: @@ -2632,7 +2710,15 @@ fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { ExprKind::Lvalue } - ast::ExprCall(..) | + ast::ExprCall(..) => { + let ty = expr_ty(bcx, expr); + + if type_of::return_uses_outptr(bcx.ccx(), ty) || bcx.fcx.type_needs_drop(ty) { + ExprKind::RvalueDps + } else { + ExprKind::RvalueDatum + } + } ast::ExprMethodCall(..) | ast::ExprStruct(..) | ast::ExprRange(..) | @@ -2694,7 +2780,7 @@ fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { } } - ast::ExprParen(ref e) => expr_kind(tcx, &**e), + ast::ExprParen(ref e) => expr_kind(bcx, &**e), ast::ExprMac(..) => { tcx.sess.span_bug( diff --git a/src/librustc_trans/trans/foreign.rs b/src/librustc_trans/trans/foreign.rs index 225ff52a63c59..8fef50a2c595a 100644 --- a/src/librustc_trans/trans/foreign.rs +++ b/src/librustc_trans/trans/foreign.rs @@ -229,7 +229,7 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, llargs_rust: &[ValueRef], passed_arg_tys: Vec>, call_debug_loc: DebugLoc) - -> Block<'blk, 'tcx> + -> (ValueRef, Block<'blk, 'tcx>) { let ccx = bcx.ccx(); @@ -441,7 +441,18 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } } - return bcx; + let mut llretval = C_undef(llsig.llret_ty); + match fn_sig.output { + ty::FnConverging(result_ty) => { + if !type_of::return_uses_outptr(ccx, result_ty) { + llretval = base::load_ty(bcx, llretptr, result_ty); + } + } + ty::FnDiverging => {} + } + + + return (llretval, bcx); } // feature gate SIMD types in FFI, since I (huonw) am not sure the diff --git a/src/librustc_trans/trans/intrinsic.rs b/src/librustc_trans/trans/intrinsic.rs index 0400771dff15c..b0b5cd2ee7fbc 100644 --- a/src/librustc_trans/trans/intrinsic.rs +++ b/src/librustc_trans/trans/intrinsic.rs @@ -157,8 +157,7 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, args: callee::CallArgs<'a, 'tcx>, dest: expr::Dest, substs: subst::Substs<'tcx>, - call_info: NodeIdAndSpan) - -> Result<'blk, 'tcx> { + call_info: NodeIdAndSpan) -> Result<'blk, 'tcx> { let fcx = bcx.fcx; let ccx = fcx.ccx; let tcx = bcx.tcx(); @@ -183,6 +182,7 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let (in_type, out_type) = (*substs.types.get(FnSpace, 0), *substs.types.get(FnSpace, 1)); + let llintype = type_of::type_of(ccx, in_type); let llouttype = type_of::type_of(ccx, out_type); @@ -206,12 +206,16 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // but does, importantly, cover SIMD types. let in_kind = llintype.kind(); let ret_kind = llret_ty.kind(); - let bitcast_compatible = - (nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind)) || { - in_kind == TypeKind::Pointer && ret_kind == TypeKind::Pointer - }; + let bitcast_compatible = match (in_kind, ret_kind) { + (TypeKind::Pointer, TypeKind::Pointer) | + (TypeKind::Pointer, TypeKind::Integer) | + (TypeKind::Integer, TypeKind::Pointer) => true, + _ => { + nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind) + } + }; - let dest = if bitcast_compatible { + let val = if bitcast_compatible { // if we're here, the type is scalar-like (a primitive, a // SIMD type or a pointer), and so can be handled as a // by-value ValueRef and can also be directly bitcast to the @@ -229,7 +233,12 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, from_arg_ty(bcx, datum.val, datum.ty) }; - let cast_val = BitCast(bcx, val, llret_ty); + let cast_val = match (in_kind, ret_kind) { + (TypeKind::Pointer, TypeKind::Integer) => PtrToInt(bcx, val, llret_ty), + (TypeKind::Integer, TypeKind::Pointer) => IntToPtr(bcx, val, llret_ty), + (TypeKind::Pointer, TypeKind::Pointer) => PointerCast(bcx, val, llret_ty), + _ => BitCast(bcx, val, llret_ty) + }; match dest { expr::SaveIn(d) => { @@ -239,27 +248,46 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } expr::Ignore => {} } - dest + cast_val } else { // The types are too complicated to do with a by-value // bitcast, so pointer cast instead. We need to cast the // dest so the types work out. - let dest = match dest { + let cast_dest = match dest { expr::SaveIn(d) => expr::SaveIn(PointerCast(bcx, d, llintype.ptr_to())), - expr::Ignore => expr::Ignore + expr::Ignore => if type_of::return_uses_outptr(bcx.ccx(), out_type) { + expr::Ignore + } else { + expr::SaveIn(alloca(bcx, llintype, "transmute_temp")) + } }; - bcx = expr::trans_into(bcx, &*arg_exprs[0], dest); - dest + + bcx = expr::trans_into(bcx, &*arg_exprs[0], cast_dest); + + let llretty = type_of::arg_type_of(ccx, out_type); + + if type_of::return_uses_outptr(bcx.ccx(), out_type) { + C_nil(ccx) + } else { + match cast_dest { + expr::SaveIn(d) => { + let val = load_ty(bcx, + PointerCast(bcx, d, llouttype.ptr_to()), + out_type); + if let expr::Ignore = dest { + call_lifetime_end(bcx, d) + } + val + } + expr::Ignore => C_undef(llretty) + } + } }; fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - return match dest { - expr::SaveIn(d) => Result::new(bcx, d), - expr::Ignore => Result::new(bcx, C_undef(llret_ty.ptr_to())) - }; - + return Result::new(bcx, val); } _ => { @@ -324,9 +352,11 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let data = unpack_datum!(bcx, expr::trans(bcx, data)); let data = unpack_datum!(bcx, data.to_rvalue_datum(bcx, "data")); + let ret_ty = tcx.mk_mut_ptr(tcx.types.i8); + let dest = match dest { expr::SaveIn(d) => d, - expr::Ignore => alloc_ty(bcx, tcx.mk_mut_ptr(tcx.types.i8), + expr::Ignore => alloc_ty(bcx, ret_ty, "try_result"), }; @@ -335,7 +365,10 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, call_debug_location); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - return Result::new(bcx, dest); + + let retval = load_ty(bcx, dest, ret_ty); + + return Result::new(bcx, retval); } else { ccx.sess().bug("expected two exprs as arguments for \ `try` intrinsic"); @@ -374,548 +407,578 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let llret_ty = type_of::type_of(ccx, ret_ty); - // Get location to store the result. If the user does - // not care about the result, just make a stack slot - let llresult = match dest { - expr::SaveIn(d) => d, - expr::Ignore => { - if !type_is_zero_size(ccx, ret_ty) { - alloc_ty(bcx, ret_ty, "intrinsic_result") + let mut llslot = None; + let llval = { + let mut get_slot = || { + if let Some(slot) = llslot { + slot } else { - C_undef(llret_ty.ptr_to()) + let slot = match dest { + expr::SaveIn(d) => d, + expr::Ignore => { + if !type_is_zero_size(ccx, ret_ty) { + alloc_ty(bcx, ret_ty, "intrinsic_result") + } else { + C_undef(llret_ty.ptr_to()) + } + } + }; + llslot = Some(slot); + slot } - } - }; + }; - let simple = get_simple_intrinsic(ccx, &*foreign_item); - let llval = match (simple, &*name) { - (Some(llfn), _) => { - Call(bcx, llfn, &llargs, None, call_debug_location) - } - (_, "breakpoint") => { - let llfn = ccx.get_intrinsic(&("llvm.debugtrap")); - Call(bcx, llfn, &[], None, call_debug_location) - } - (_, "size_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let lltp_ty = type_of::type_of(ccx, tp_ty); - C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) - } - (_, "size_of_val") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !type_is_sized(tcx, tp_ty) { - let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); - llsize - } else { + let simple = get_simple_intrinsic(ccx, &*foreign_item); + match (simple, &*name) { + (Some(llfn), _) => { + Call(bcx, llfn, &llargs, None, call_debug_location) + } + (_, "breakpoint") => { + let llfn = ccx.get_intrinsic(&("llvm.debugtrap")); + Call(bcx, llfn, &[], None, call_debug_location) + } + (_, "size_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); let lltp_ty = type_of::type_of(ccx, tp_ty); C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) } - } - (_, "min_align_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - C_uint(ccx, type_of::align_of(ccx, tp_ty)) - } - (_, "min_align_of_val") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !type_is_sized(tcx, tp_ty) { - let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); - llalign - } else { + (_, "size_of_val") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !type_is_sized(tcx, tp_ty) { + let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); + llsize + } else { + let lltp_ty = type_of::type_of(ccx, tp_ty); + C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) + } + } + (_, "min_align_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); C_uint(ccx, type_of::align_of(ccx, tp_ty)) } - } - (_, "pref_align_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let lltp_ty = type_of::type_of(ccx, tp_ty); - C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty)) - } - (_, "drop_in_place") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = if type_is_sized(tcx, tp_ty) { - llargs[0] - } else { - let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp"); - Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val)); - Store(bcx, llargs[1], expr::get_len(bcx, scratch.val)); - fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val); - scratch.val - }; - glue::drop_ty(bcx, ptr, tp_ty, call_debug_location); - C_nil(ccx) - } - (_, "type_name") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ty_name = token::intern_and_get_ident(&tp_ty.to_string()); - C_str_slice(ccx, ty_name) - } - (_, "type_id") => { - let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0), - &ccx.link_meta().crate_hash); - C_u64(ccx, hash) - } - (_, "init_dropped") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !return_type_is_void(ccx, tp_ty) { - drop_done_fill_mem(bcx, llresult, tp_ty); + (_, "min_align_of_val") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !type_is_sized(tcx, tp_ty) { + let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); + llalign + } else { + C_uint(ccx, type_of::align_of(ccx, tp_ty)) + } } - C_nil(ccx) - } - (_, "init") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !return_type_is_void(ccx, tp_ty) { - // Just zero out the stack slot. (See comment on base::memzero for explanation) - init_zero_mem(bcx, llresult, tp_ty); + (_, "pref_align_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let lltp_ty = type_of::type_of(ccx, tp_ty); + C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty)) } - C_nil(ccx) - } - // Effectively no-ops - (_, "uninit") | (_, "forget") => { - C_nil(ccx) - } - (_, "needs_drop") => { - let tp_ty = *substs.types.get(FnSpace, 0); - - C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty)) - } - (_, "offset") => { - let ptr = llargs[0]; - let offset = llargs[1]; - InBoundsGEP(bcx, ptr, &[offset]) - } - (_, "arith_offset") => { - let ptr = llargs[0]; - let offset = llargs[1]; - GEP(bcx, ptr, &[offset]) - } + (_, "drop_in_place") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = if type_is_sized(tcx, tp_ty) { + llargs[0] + } else { + let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp"); + Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val)); + Store(bcx, llargs[1], expr::get_len(bcx, scratch.val)); + fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val); + scratch.val + }; + glue::drop_ty(bcx, ptr, tp_ty, call_debug_location); + C_nil(ccx) + } + (_, "type_name") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ty_name = token::intern_and_get_ident(&tp_ty.to_string()); + C_str_slice(ccx, ty_name) + } + (_, "type_id") => { + let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0), + &ccx.link_meta().crate_hash); + C_u64(ccx, hash) + } + (_, "init_dropped") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !return_type_is_void(ccx, tp_ty) { + drop_done_fill_mem(bcx, get_slot(), tp_ty); + } + C_nil(ccx) + } + (_, "init") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if type_of::return_uses_outptr(ccx, tp_ty) { + // Just zero out the stack slot. (See comment on base::memzero for explanation) + init_zero_mem(bcx, get_slot(), tp_ty); + C_nil(ccx) + } else { + let lltp_ty = type_of::arg_type_of(ccx, tp_ty); + C_null(lltp_ty) + } + } + // Effectively no-ops + (_, "uninit") | (_, "forget") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if type_of::return_uses_outptr(ccx, tp_ty) { + C_nil(ccx) + } else { + let lltp_ty = type_of::arg_type_of(ccx, tp_ty); + C_undef(lltp_ty) + } + } + (_, "needs_drop") => { + let tp_ty = *substs.types.get(FnSpace, 0); - (_, "copy_nonoverlapping") => { - copy_intrinsic(bcx, - false, - false, - *substs.types.get(FnSpace, 0), - llargs[1], - llargs[0], - llargs[2], - call_debug_location) - } - (_, "copy") => { - copy_intrinsic(bcx, - true, - false, - *substs.types.get(FnSpace, 0), - llargs[1], - llargs[0], - llargs[2], - call_debug_location) - } - (_, "write_bytes") => { - memset_intrinsic(bcx, - false, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } + C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty)) + } + (_, "offset") => { + let ptr = llargs[0]; + let offset = llargs[1]; + InBoundsGEP(bcx, ptr, &[offset]) + } + (_, "arith_offset") => { + let ptr = llargs[0]; + let offset = llargs[1]; + GEP(bcx, ptr, &[offset]) + } - (_, "volatile_copy_nonoverlapping_memory") => { - copy_intrinsic(bcx, - false, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_copy_memory") => { - copy_intrinsic(bcx, - true, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_set_memory") => { - memset_intrinsic(bcx, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_load") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let load = VolatileLoad(bcx, ptr); - unsafe { - llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty)); + (_, "copy_nonoverlapping") => { + copy_intrinsic(bcx, + false, + false, + *substs.types.get(FnSpace, 0), + llargs[1], + llargs[0], + llargs[2], + call_debug_location) } - to_arg_ty(bcx, load, tp_ty) - }, - (_, "volatile_store") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - let store = VolatileStore(bcx, val, ptr); - unsafe { - llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty)); + (_, "copy") => { + copy_intrinsic(bcx, + true, + false, + *substs.types.get(FnSpace, 0), + llargs[1], + llargs[0], + llargs[2], + call_debug_location) } - C_nil(ccx) - }, - - (_, "ctlz8") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i8", - llargs[0], - call_debug_location), - (_, "ctlz16") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i16", - llargs[0], - call_debug_location), - (_, "ctlz32") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i32", - llargs[0], - call_debug_location), - (_, "ctlz64") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i64", - llargs[0], - call_debug_location), - (_, "cttz8") => count_zeros_intrinsic(bcx, - "llvm.cttz.i8", - llargs[0], - call_debug_location), - (_, "cttz16") => count_zeros_intrinsic(bcx, - "llvm.cttz.i16", - llargs[0], - call_debug_location), - (_, "cttz32") => count_zeros_intrinsic(bcx, - "llvm.cttz.i32", - llargs[0], - call_debug_location), - (_, "cttz64") => count_zeros_intrinsic(bcx, - "llvm.cttz.i64", - llargs[0], - call_debug_location), - - (_, "i8_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - - (_, "u8_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i8_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u8_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i8_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u8_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - - (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location), - - (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location), - (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location), - (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location), - - (_, "return_address") => { - if !fcx.caller_expects_out_pointer { - tcx.sess.span_err(call_info.span, - "invalid use of `return_address` intrinsic: function \ - does not use out pointer"); - C_null(Type::i8p(ccx)) - } else { - PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx)) + (_, "write_bytes") => { + memset_intrinsic(bcx, + false, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) } - } - (_, "discriminant_value") => { - let val_ty = substs.types.get(FnSpace, 0); - match val_ty.sty { - ty::TyEnum(..) => { - let repr = adt::represent_type(ccx, *val_ty); - adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty)) + (_, "volatile_copy_nonoverlapping_memory") => { + copy_intrinsic(bcx, + false, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_copy_memory") => { + copy_intrinsic(bcx, + true, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_set_memory") => { + memset_intrinsic(bcx, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_load") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let load = VolatileLoad(bcx, ptr); + unsafe { + llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty)); + } + to_arg_ty(bcx, load, tp_ty) + }, + (_, "volatile_store") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + let store = VolatileStore(bcx, val, ptr); + unsafe { + llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty)); + } + C_nil(ccx) + }, + + (_, "ctlz8") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i8", + llargs[0], + call_debug_location), + (_, "ctlz16") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i16", + llargs[0], + call_debug_location), + (_, "ctlz32") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i32", + llargs[0], + call_debug_location), + (_, "ctlz64") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i64", + llargs[0], + call_debug_location), + (_, "cttz8") => count_zeros_intrinsic(bcx, + "llvm.cttz.i8", + llargs[0], + call_debug_location), + (_, "cttz16") => count_zeros_intrinsic(bcx, + "llvm.cttz.i16", + llargs[0], + call_debug_location), + (_, "cttz32") => count_zeros_intrinsic(bcx, + "llvm.cttz.i32", + llargs[0], + call_debug_location), + (_, "cttz64") => count_zeros_intrinsic(bcx, + "llvm.cttz.i64", + llargs[0], + call_debug_location), + + (_, "i8_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + + (_, "u8_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i8_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u8_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i8_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u8_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + + (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location), + + (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location), + (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location), + (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location), + + (_, "return_address") => { + if !fcx.caller_expects_out_pointer { + tcx.sess.span_err(call_info.span, + "invalid use of `return_address` intrinsic: function \ + does not use out pointer"); + C_null(Type::i8p(ccx)) + } else { + PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx)) } - _ => C_null(llret_ty) } - } - // This requires that atomic intrinsics follow a specific naming pattern: - // "atomic_[_]", and no ordering means SeqCst - (_, name) if name.starts_with("atomic_") => { - let split: Vec<&str> = name.split('_').collect(); - assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - - let order = if split.len() == 2 { - llvm::SequentiallyConsistent - } else { - match split[2] { - "unordered" => llvm::Unordered, - "relaxed" => llvm::Monotonic, - "acq" => llvm::Acquire, - "rel" => llvm::Release, - "acqrel" => llvm::AcquireRelease, - _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + (_, "discriminant_value") => { + let val_ty = substs.types.get(FnSpace, 0); + match val_ty.sty { + ty::TyEnum(..) => { + let repr = adt::represent_type(ccx, *val_ty); + adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty)) + } + _ => C_null(llret_ty) } - }; - - match split[1] { - "cxchg" => { - // See include/llvm/IR/Instructions.h for their implementation - // of this, I assume that it's good enough for us to use for - // now. - let strongest_failure_ordering = match order { - llvm::NotAtomic | llvm::Unordered => - ccx.sess().fatal("cmpxchg must be atomic"), + } - llvm::Monotonic | llvm::Release => - llvm::Monotonic, + // This requires that atomic intrinsics follow a specific naming pattern: + // "atomic_[_]", and no ordering means SeqCst + (_, name) if name.starts_with("atomic_") => { + let split: Vec<&str> = name.split('_').collect(); + assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - llvm::Acquire | llvm::AcquireRelease => - llvm::Acquire, + let order = if split.len() == 2 { + llvm::SequentiallyConsistent + } else { + match split[2] { + "unordered" => llvm::Unordered, + "relaxed" => llvm::Monotonic, + "acq" => llvm::Acquire, + "rel" => llvm::Release, + "acqrel" => llvm::AcquireRelease, + _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + } + }; - llvm::SequentiallyConsistent => - llvm::SequentiallyConsistent - }; + match split[1] { + "cxchg" => { + // See include/llvm/IR/Instructions.h for their implementation + // of this, I assume that it's good enough for us to use for + // now. + let strongest_failure_ordering = match order { + llvm::NotAtomic | llvm::Unordered => + ccx.sess().fatal("cmpxchg must be atomic"), + + llvm::Monotonic | llvm::Release => + llvm::Monotonic, + + llvm::Acquire | llvm::AcquireRelease => + llvm::Acquire, + + llvm::SequentiallyConsistent => + llvm::SequentiallyConsistent + }; + + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let cmp = from_arg_ty(bcx, llargs[1], tp_ty); + let src = from_arg_ty(bcx, llargs[2], tp_ty); + let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, + strongest_failure_ordering); + ExtractValue(bcx, res, 0) + } - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let cmp = from_arg_ty(bcx, llargs[1], tp_ty); - let src = from_arg_ty(bcx, llargs[2], tp_ty); - let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, - strongest_failure_ordering); - ExtractValue(bcx, res, 0) - } + "load" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty) + } + "store" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + AtomicStore(bcx, val, ptr, order); + C_nil(ccx) + } - "load" => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty) - } - "store" => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - AtomicStore(bcx, val, ptr, order); - C_nil(ccx) - } + "fence" => { + AtomicFence(bcx, order, llvm::CrossThread); + C_nil(ccx) + } - "fence" => { - AtomicFence(bcx, order, llvm::CrossThread); - C_nil(ccx) - } + "singlethreadfence" => { + AtomicFence(bcx, order, llvm::SingleThread); + C_nil(ccx) + } - "singlethreadfence" => { - AtomicFence(bcx, order, llvm::SingleThread); - C_nil(ccx) + // These are all AtomicRMW ops + op => { + let atom_op = match op { + "xchg" => llvm::AtomicXchg, + "xadd" => llvm::AtomicAdd, + "xsub" => llvm::AtomicSub, + "and" => llvm::AtomicAnd, + "nand" => llvm::AtomicNand, + "or" => llvm::AtomicOr, + "xor" => llvm::AtomicXor, + "max" => llvm::AtomicMax, + "min" => llvm::AtomicMin, + "umax" => llvm::AtomicUMax, + "umin" => llvm::AtomicUMin, + _ => ccx.sess().fatal("unknown atomic operation") + }; + + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + AtomicRMW(bcx, atom_op, ptr, val, order) + } } - // These are all AtomicRMW ops - op => { - let atom_op = match op { - "xchg" => llvm::AtomicXchg, - "xadd" => llvm::AtomicAdd, - "xsub" => llvm::AtomicSub, - "and" => llvm::AtomicAnd, - "nand" => llvm::AtomicNand, - "or" => llvm::AtomicOr, - "xor" => llvm::AtomicXor, - "max" => llvm::AtomicMax, - "min" => llvm::AtomicMin, - "umax" => llvm::AtomicUMax, - "umin" => llvm::AtomicUMin, - _ => ccx.sess().fatal("unknown atomic operation") - }; - - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - AtomicRMW(bcx, atom_op, ptr, val, order) - } } + (_, _) => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic") } - - (_, _) => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic") }; - if val_ty(llval) != Type::void(ccx) && - machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 { - store_ty(bcx, llval, llresult, ret_ty); - } + let llretval = if let Some(slot) = llslot { + if !type_of::return_uses_outptr(ccx, ret_ty) { + load_ty(bcx, slot, ret_ty) + } else { + C_nil(ccx) + } + } else { + llval + }; // If we made a temporary stack slot, let's clean it up match dest { expr::Ignore => { - bcx = glue::drop_ty(bcx, llresult, ret_ty, call_debug_location); + if let Some(slot) = llslot { + bcx = glue::drop_ty(bcx, slot, ret_ty, call_debug_location); + } + } + expr::SaveIn(slot) => { + if val_ty(llval) != Type::void(ccx) + && machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 { + store_ty(bcx, llval, slot, ret_ty); + } } - expr::SaveIn(_) => {} } fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - Result::new(bcx, llresult) + Result::new(bcx, llretval) } fn copy_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, diff --git a/src/librustc_trans/trans/type_.rs b/src/librustc_trans/trans/type_.rs index 699115a070986..d2ba23caaad4f 100644 --- a/src/librustc_trans/trans/type_.rs +++ b/src/librustc_trans/trans/type_.rs @@ -227,6 +227,10 @@ impl Type { } } + pub fn is_pointer(&self) -> bool { + self.kind() == TypeKind::Pointer + } + pub fn is_packed(&self) -> bool { unsafe { llvm::LLVMIsPackedStruct(self.to_ref()) == True @@ -270,10 +274,21 @@ impl Type { } pub fn func_params(&self) -> Vec { + let mut this = *self; + loop { + let kind = this.kind(); + if kind == TypeKind::Function { + break; + } else if kind == TypeKind::Pointer { + this = this.element_type(); + } else { + panic!("Type is not a function or function pointer"); + } + } unsafe { - let n_args = llvm::LLVMCountParamTypes(self.to_ref()) as usize; + let n_args = llvm::LLVMCountParamTypes(this.to_ref()) as usize; let mut args = vec![Type { rf: ptr::null_mut() }; n_args]; - llvm::LLVMGetParamTypes(self.to_ref(), + llvm::LLVMGetParamTypes(this.to_ref(), args.as_mut_ptr() as *mut TypeRef); args } diff --git a/src/librustc_trans/trans/value.rs b/src/librustc_trans/trans/value.rs index bc71278c15743..23acb57ad821f 100644 --- a/src/librustc_trans/trans/value.rs +++ b/src/librustc_trans/trans/value.rs @@ -14,7 +14,7 @@ use trans::basic_block::BasicBlock; use trans::common::Block; use libc::c_uint; -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] pub struct Value(pub ValueRef); macro_rules! opt_val { ($e:expr) => ( @@ -57,6 +57,13 @@ impl Value { pub fn get_dominating_store(self, bcx: Block) -> Option { match self.get_single_user().and_then(|user| user.as_store_inst()) { Some(store) => { + // Make sure that the store instruction is /to/ this value, + // not *of* this value + if let Some(loc) = store.get_operand(1) { + if loc != self { + return None; + } + } store.get_parent().and_then(|store_bb| { let mut bb = BasicBlock(bcx.llbb); let mut ret = Some(store); @@ -73,6 +80,19 @@ impl Value { } } + pub fn get_stored_value_opt(self, bcx: Block) -> Option { + let bb = Some(BasicBlock(bcx.llbb)); + if let Some(val) = self.get_dominating_store(bcx) { + let valbb = val.get_parent(); + + if bb == valbb { + return val.get_operand(0); + } + } + + None + } + /// Returns the first use of this value, if any pub fn get_first_use(self) -> Option { unsafe { @@ -116,6 +136,10 @@ impl Value { opt_val!(llvm::LLVMIsAStoreInst(self.get())) } + pub fn as_zext_inst(self) -> Option { + opt_val!(llvm::LLVMIsAZExtInst(self.get())) + } + /// Tests if this value is a terminator instruction pub fn is_a_terminator_inst(self) -> bool { unsafe {