Skip to content

Commit 9f42d34

Browse files
borsbors
committed
Auto merge of rust-lang#121767 - saethlin:ubcheck-intrinsic, r=<try>
[nightmarish hacks within] Lower UB checks as a new kind of intrinsic, not calls I _think_ one reason the UB checks have the compile time overhead that they do is that they impede MIR optimizations by being calls, which are penalized heavily in MIR inlining, and also simply complicate the MIR we generate. So this is my (ongoing) effort to fix that. Currently the code is strewn with terrible hacks that I'll figure out how to do better later. Right now I just want to establish if this is a path to reducing their compile-time overhead. r? `@ghost`
2 parents bf9c7a6 + 9ae476b commit 9f42d34

File tree

66 files changed

+1155
-598
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+1155
-598
lines changed

Diff for: ‎compiler/rustc_borrowck/src/lib.rs

+4
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,10 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
628628
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
629629
span,
630630
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
631+
),
632+
NonDivergingIntrinsic::UbCheck { .. } => span_bug!(
633+
span,
634+
"Unexpected UbCheck, should only appear after lower_intrinsics",
631635
)
632636
}
633637
// Only relevant for mir typeck

Diff for: ‎compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

+12
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,18 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
6262
self.consume_operand(location, dst);
6363
self.consume_operand(location, count);
6464
}
65+
StatementKind::Intrinsic(box NonDivergingIntrinsic::UbCheck {
66+
kind: _,
67+
func,
68+
args,
69+
destination,
70+
source_info: _,
71+
fn_span: _,
72+
}) => {
73+
self.consume_operand(location, func);
74+
self.consume_operand(location, args);
75+
self.mutate_place(location, *destination, Shallow(None));
76+
}
6577
// Only relevant for mir typeck
6678
StatementKind::AscribeUserType(..)
6779
// Only relevant for liveness and unsafeck

Diff for: ‎compiler/rustc_borrowck/src/type_check/mod.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -1367,6 +1367,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
13671367
stmt.source_info.span,
13681368
"Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",
13691369
),
1370+
NonDivergingIntrinsic::UbCheck { .. } => span_bug!(
1371+
stmt.source_info.span,
1372+
"Unexpected NonDivergingIntrinsic::UbCheck, should only appear after lowering_intrinsics",
1373+
),
13701374
},
13711375
StatementKind::FakeRead(..)
13721376
| StatementKind::StorageLive(..)
@@ -2001,7 +2005,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
20012005
ConstraintCategory::SizedBound,
20022006
);
20032007
}
2004-
&Rvalue::NullaryOp(NullOp::DebugAssertions, _) => {}
20052008

20062009
Rvalue::ShallowInitBox(operand, ty) => {
20072010
self.check_operand(operand, location);

Diff for: ‎compiler/rustc_codegen_cranelift/src/base.rs

+3-9
Original file line numberDiff line numberDiff line change
@@ -767,15 +767,6 @@ fn codegen_stmt<'tcx>(
767767
NullOp::OffsetOf(fields) => {
768768
layout.offset_of_subfield(fx, fields.iter()).bytes()
769769
}
770-
NullOp::DebugAssertions => {
771-
let val = fx.tcx.sess.opts.debug_assertions;
772-
let val = CValue::by_val(
773-
fx.bcx.ins().iconst(types::I8, i64::try_from(val).unwrap()),
774-
fx.layout_of(fx.tcx.types.bool),
775-
);
776-
lval.write_cvalue(fx, val);
777-
return;
778-
}
779770
};
780771
let val = CValue::by_val(
781772
fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()),
@@ -845,6 +836,9 @@ fn codegen_stmt<'tcx>(
845836
};
846837
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
847838
}
839+
NonDivergingIntrinsic::UbCheck { .. } => {
840+
todo!() // FIXME
841+
}
848842
},
849843
}
850844
}

Diff for: ‎compiler/rustc_codegen_cranelift/src/constant.rs

+1
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
504504
StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
505505
NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
506506
NonDivergingIntrinsic::Assume(..) => {}
507+
NonDivergingIntrinsic::UbCheck { .. } => {}
507508
},
508509
// conservative handling
509510
StatementKind::Assign(_)

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/block.rs

+26-16
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,19 @@ use std::cmp;
2828
// can happen when BB jumps directly to its successor and the successor has no
2929
// other predecessors.
3030
#[derive(Debug, PartialEq)]
31-
enum MergingSucc {
31+
pub(crate) enum MergingSucc {
3232
False,
3333
True,
3434
}
3535

3636
/// Used by `FunctionCx::codegen_terminator` for emitting common patterns
3737
/// e.g., creating a basic block, calling a function, etc.
38-
struct TerminatorCodegenHelper<'tcx> {
39-
bb: mir::BasicBlock,
40-
terminator: &'tcx mir::Terminator<'tcx>,
38+
pub(crate) struct TerminatorCodegenHelper<'term, 'tcx> {
39+
pub(crate) bb: mir::BasicBlock,
40+
pub(crate) terminator: &'term mir::Terminator<'tcx>,
4141
}
4242

43-
impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
43+
impl<'term, 'a, 'tcx> TerminatorCodegenHelper<'term, 'tcx> {
4444
/// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
4545
/// either already previously cached, or newly created, by `landing_pad_for`.
4646
fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
@@ -98,6 +98,10 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
9898
fx: &mut FunctionCx<'a, 'tcx, Bx>,
9999
target: mir::BasicBlock,
100100
) -> (bool, bool) {
101+
if self.bb > fx.mir.basic_blocks.len().into() {
102+
return (false, false);
103+
}
104+
101105
if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
102106
let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb);
103107
let target_funclet = cleanup_kinds[target].funclet_bb(target);
@@ -226,8 +230,10 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
226230
MergingSucc::False
227231
} else {
228232
let llret = bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, llargs, self.funclet(fx));
229-
if fx.mir[self.bb].is_cleanup {
230-
bx.apply_attrs_to_cleanup_callsite(llret);
233+
if let Some(bb) = fx.mir.basic_blocks.get(self.bb) {
234+
if bb.is_cleanup {
235+
bx.apply_attrs_to_cleanup_callsite(llret);
236+
}
231237
}
232238

233239
if let Some((ret_dest, target)) = destination {
@@ -296,7 +302,11 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
296302
/// Codegen implementations for some terminator variants.
297303
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
298304
/// Generates code for a `Resume` terminator.
299-
fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) {
305+
fn codegen_resume_terminator(
306+
&mut self,
307+
helper: TerminatorCodegenHelper<'_, 'tcx>,
308+
bx: &mut Bx,
309+
) {
300310
if let Some(funclet) = helper.funclet(self) {
301311
bx.cleanup_ret(funclet, None);
302312
} else {
@@ -313,7 +323,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
313323

314324
fn codegen_switchint_terminator(
315325
&mut self,
316-
helper: TerminatorCodegenHelper<'tcx>,
326+
helper: TerminatorCodegenHelper<'_, 'tcx>,
317327
bx: &mut Bx,
318328
discr: &mir::Operand<'tcx>,
319329
targets: &SwitchTargets,
@@ -451,7 +461,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
451461
#[tracing::instrument(level = "trace", skip(self, helper, bx))]
452462
fn codegen_drop_terminator(
453463
&mut self,
454-
helper: TerminatorCodegenHelper<'tcx>,
464+
helper: TerminatorCodegenHelper<'_, 'tcx>,
455465
bx: &mut Bx,
456466
location: mir::Place<'tcx>,
457467
target: mir::BasicBlock,
@@ -569,7 +579,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
569579

570580
fn codegen_assert_terminator(
571581
&mut self,
572-
helper: TerminatorCodegenHelper<'tcx>,
582+
helper: TerminatorCodegenHelper<'_, 'tcx>,
573583
bx: &mut Bx,
574584
terminator: &mir::Terminator<'tcx>,
575585
cond: &mir::Operand<'tcx>,
@@ -649,7 +659,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
649659

650660
fn codegen_terminate_terminator(
651661
&mut self,
652-
helper: TerminatorCodegenHelper<'tcx>,
662+
helper: TerminatorCodegenHelper<'_, 'tcx>,
653663
bx: &mut Bx,
654664
terminator: &mir::Terminator<'tcx>,
655665
reason: UnwindTerminateReason,
@@ -678,7 +688,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
678688
/// Returns `Some` if this is indeed a panic intrinsic and codegen is done.
679689
fn codegen_panic_intrinsic(
680690
&mut self,
681-
helper: &TerminatorCodegenHelper<'tcx>,
691+
helper: &TerminatorCodegenHelper<'_, 'tcx>,
682692
bx: &mut Bx,
683693
intrinsic: Option<Symbol>,
684694
instance: Option<Instance<'tcx>>,
@@ -744,9 +754,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
744754
}
745755
}
746756

747-
fn codegen_call_terminator(
757+
pub(crate) fn codegen_call_terminator(
748758
&mut self,
749-
helper: TerminatorCodegenHelper<'tcx>,
759+
helper: TerminatorCodegenHelper<'_, 'tcx>,
750760
bx: &mut Bx,
751761
terminator: &mir::Terminator<'tcx>,
752762
func: &mir::Operand<'tcx>,
@@ -1068,7 +1078,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
10681078

10691079
fn codegen_asm_terminator(
10701080
&mut self,
1071-
helper: TerminatorCodegenHelper<'tcx>,
1081+
helper: TerminatorCodegenHelper<'_, 'tcx>,
10721082
bx: &mut Bx,
10731083
terminator: &mir::Terminator<'tcx>,
10741084
template: &[ast::InlineAsmTemplatePiece],

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

-4
Original file line numberDiff line numberDiff line change
@@ -684,10 +684,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
684684
let val = layout.offset_of_subfield(bx.cx(), fields.iter()).bytes();
685685
bx.cx().const_usize(val)
686686
}
687-
mir::NullOp::DebugAssertions => {
688-
let val = bx.tcx().sess.opts.debug_assertions;
689-
bx.cx().const_bool(val)
690-
}
691687
};
692688
let tcx = self.cx.tcx();
693689
OperandRef {

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/statement.rs

+40
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
use rustc_middle::mir;
22
use rustc_middle::mir::NonDivergingIntrinsic;
33
use rustc_session::config::OptLevel;
4+
use rustc_span::source_map::respan;
45

56
use super::FunctionCx;
67
use super::LocalRef;
8+
use crate::mir::block::TerminatorCodegenHelper;
79
use crate::traits::*;
810

911
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
@@ -90,6 +92,44 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9092
let src = src_val.immediate();
9193
bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
9294
}
95+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::UbCheck {
96+
kind: _,
97+
ref func,
98+
ref args,
99+
destination,
100+
fn_span,
101+
source_info,
102+
}) => {
103+
if bx.tcx().sess.opts.debug_assertions {
104+
let bb = mir::BasicBlock::MAX;
105+
let args = vec![respan(source_info.span, args.clone())];
106+
let terminator = mir::Terminator {
107+
source_info,
108+
kind: mir::TerminatorKind::Call {
109+
func: func.clone(),
110+
args: args.clone(),
111+
destination,
112+
target: Some(bb),
113+
unwind: mir::UnwindAction::Unreachable,
114+
call_source: mir::CallSource::Normal,
115+
fn_span,
116+
},
117+
};
118+
let helper = TerminatorCodegenHelper { bb, terminator: &terminator };
119+
self.codegen_call_terminator(
120+
helper,
121+
bx,
122+
&terminator,
123+
func,
124+
args.as_slice(),
125+
destination,
126+
Some(bb),
127+
mir::UnwindAction::Unreachable,
128+
fn_span,
129+
true, // mergeable_succ
130+
);
131+
}
132+
}
93133
mir::StatementKind::FakeRead(..)
94134
| mir::StatementKind::Retag { .. }
95135
| mir::StatementKind::AscribeUserType(..)

Diff for: ‎compiler/rustc_const_eval/src/const_eval/machine.rs

+1
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
317317
dest,
318318
ret,
319319
mir::UnwindAction::Unreachable,
320+
false,
320321
)?;
321322
Ok(ControlFlow::Break(()))
322323
} else {

Diff for: ‎compiler/rustc_const_eval/src/interpret/eval_context.rs

+12
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,9 @@ pub enum StackPopCleanup {
155155
/// wants them leaked to intern what they need (and just throw away
156156
/// the entire `ecx` when it is done).
157157
Root { cleanup: bool },
158+
159+
/// FIXME: We're returning from a UbCheck call, don't do anything.
160+
Nothing,
158161
}
159162

160163
/// State of a local variable including a memoized layout
@@ -928,6 +931,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
928931
let cleanup = match return_to_block {
929932
StackPopCleanup::Goto { .. } => true,
930933
StackPopCleanup::Root { cleanup, .. } => cleanup,
934+
StackPopCleanup::Nothing => true,
931935
};
932936
if cleanup {
933937
// We need to take the locals out, since we need to mutate while iterating.
@@ -964,6 +968,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
964968
StackPopCleanup::Root { .. } => {
965969
panic!("encountered StackPopCleanup::Root when unwinding!")
966970
}
971+
StackPopCleanup::Nothing => {
972+
panic!("encountered StackPopCleanup::Nothing when unwinding!")
973+
}
967974
};
968975
// This must be the very last thing that happens, since it can in fact push a new stack frame.
969976
self.unwind_to_block(unwind)
@@ -978,6 +985,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
978985
);
979986
Ok(())
980987
}
988+
StackPopCleanup::Nothing => {
989+
// Advance the program counter.
990+
self.frame_mut().loc.as_mut().left().unwrap().statement_index += 1;
991+
Ok(())
992+
}
981993
}
982994
}
983995
}

0 commit comments

Comments
 (0)