Skip to content

Commit 2cd7dda

Browse files
borsbors
committed
Auto merge of rust-lang#111344 - cjgillot:gvn-simplify, r=<try>
Perform opportunistic simplifications during value numbering ~Based on rust-lang#109597 ~Based on rust-lang#119439 Opening mostly for discussion.
2 parents 88189a7 + 0167761 commit 2cd7dda

File tree

40 files changed

+1631
-1263
lines changed

40 files changed

+1631
-1263
lines changed

‎compiler/rustc_const_eval/src/interpret/cast.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
416416
}
417417
}
418418

419-
fn unsize_into(
419+
pub fn unsize_into(
420420
&mut self,
421421
src: &OpTy<'tcx, M::Provenance>,
422422
cast_ty: TyAndLayout<'tcx>,

‎compiler/rustc_mir_transform/src/dataflow_const_prop.rs

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
//!
33
//! Currently, this pass only propagates scalar values.
44
5-
use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, PlaceTy, Projectable};
5+
use rustc_const_eval::interpret::{
6+
ImmTy, Immediate, InterpCx, OpTy, PlaceTy, Pointer, PointerArithmetic, Projectable,
7+
};
68
use rustc_data_structures::fx::FxHashMap;
79
use rustc_hir::def::DefKind;
810
use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult, Scalar};
@@ -935,12 +937,64 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
935937
}
936938

937939
fn binary_ptr_op(
938-
_ecx: &InterpCx<'mir, 'tcx, Self>,
939-
_bin_op: BinOp,
940-
_left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
941-
_right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
940+
ecx: &InterpCx<'mir, 'tcx, Self>,
941+
bin_op: BinOp,
942+
left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
943+
right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
942944
) -> interpret::InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)> {
943-
throw_machine_stop_str!("can't do pointer arithmetic");
945+
use rustc_middle::mir::BinOp::*;
946+
Ok(match bin_op {
947+
Eq | Ne | Lt | Le | Gt | Ge => {
948+
assert_eq!(left.layout.abi, right.layout.abi); // types an differ, e.g. fn ptrs with different `for`
949+
let size = ecx.pointer_size();
950+
// Just compare the bits. ScalarPairs are compared lexicographically.
951+
// We thus always compare pairs and simply fill scalars up with 0.
952+
let left = match **left {
953+
Immediate::Scalar(l) => (l.to_bits(size)?, 0),
954+
Immediate::ScalarPair(l1, l2) => (l1.to_bits(size)?, l2.to_bits(size)?),
955+
Immediate::Uninit => panic!("we should never see uninit data here"),
956+
};
957+
let right = match **right {
958+
Immediate::Scalar(r) => (r.to_bits(size)?, 0),
959+
Immediate::ScalarPair(r1, r2) => (r1.to_bits(size)?, r2.to_bits(size)?),
960+
Immediate::Uninit => panic!("we should never see uninit data here"),
961+
};
962+
let res = match bin_op {
963+
Eq => left == right,
964+
Ne => left != right,
965+
Lt => left < right,
966+
Le => left <= right,
967+
Gt => left > right,
968+
Ge => left >= right,
969+
_ => bug!(),
970+
};
971+
(ImmTy::from_bool(res, *ecx.tcx), false)
972+
}
973+
974+
// Some more operations are possible with atomics.
975+
// The return value always has the provenance of the *left* operand.
976+
Add | Sub | BitOr | BitAnd | BitXor => {
977+
assert!(left.layout.ty.is_unsafe_ptr());
978+
assert!(right.layout.ty.is_unsafe_ptr());
979+
let ptr = left.to_scalar().to_pointer(ecx)?;
980+
// We do the actual operation with usize-typed scalars.
981+
let usize_layout = ecx.layout_of(ecx.tcx.types.usize).unwrap();
982+
let left = ImmTy::from_uint(ptr.addr().bytes(), usize_layout);
983+
let right = ImmTy::from_uint(right.to_scalar().to_target_usize(ecx)?, usize_layout);
984+
let (result, overflowing) = ecx.overflowing_binary_op(bin_op, &left, &right)?;
985+
// Construct a new pointer with the provenance of `ptr` (the LHS).
986+
let result_ptr = Pointer::new(
987+
ptr.provenance,
988+
Size::from_bytes(result.to_scalar().to_target_usize(ecx)?),
989+
);
990+
(
991+
ImmTy::from_scalar(Scalar::from_maybe_pointer(result_ptr, ecx), left.layout),
992+
overflowing,
993+
)
994+
}
995+
996+
_ => span_bug!(ecx.cur_span(), "Invalid operator on pointers: {:?}", bin_op),
997+
})
944998
}
945999

9461000
fn expose_ptr(

‎compiler/rustc_mir_transform/src/gvn.rs

Lines changed: 123 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ use rustc_index::IndexVec;
9393
use rustc_middle::mir::interpret::GlobalAlloc;
9494
use rustc_middle::mir::visit::*;
9595
use rustc_middle::mir::*;
96-
use rustc_middle::ty::adjustment::PointerCoercion;
9796
use rustc_middle::ty::layout::LayoutOf;
9897
use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
9998
use rustc_span::def_id::DefId;
@@ -154,6 +153,9 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
154153
state.next_opaque = None;
155154

156155
let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec();
156+
for dbg in body.var_debug_info.iter_mut() {
157+
state.visit_var_debug_info(dbg);
158+
}
157159
for bb in reverse_postorder {
158160
let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb];
159161
state.visit_basic_block_data(bb, data);
@@ -551,6 +553,29 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
551553
}
552554
value.offset(Size::ZERO, to, &self.ecx).ok()?
553555
}
556+
CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize) => {
557+
let src = self.evaluated[value].as_ref()?;
558+
let to = self.ecx.layout_of(to).ok()?;
559+
let dest = self.ecx.allocate(to, MemoryKind::Stack).ok()?;
560+
self.ecx.unsize_into(src, to, &dest.clone().into()).ok()?;
561+
self.ecx
562+
.alloc_mark_immutable(dest.ptr().provenance.unwrap().alloc_id())
563+
.ok()?;
564+
dest.into()
565+
}
566+
CastKind::FnPtrToPtr
567+
| CastKind::PtrToPtr
568+
| CastKind::PointerCoercion(
569+
ty::adjustment::PointerCoercion::MutToConstPointer
570+
| ty::adjustment::PointerCoercion::ArrayToPointer
571+
| ty::adjustment::PointerCoercion::UnsafeFnPointer,
572+
) => {
573+
let src = self.evaluated[value].as_ref()?;
574+
let src = self.ecx.read_immediate(src).ok()?;
575+
let to = self.ecx.layout_of(to).ok()?;
576+
let ret = self.ecx.ptr_to_ptr(&src, to).ok()?;
577+
ret.into()
578+
}
554579
_ => return None,
555580
},
556581
};
@@ -777,18 +802,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
777802

778803
// Operations.
779804
Rvalue::Len(ref mut place) => return self.simplify_len(place, location),
780-
Rvalue::Cast(kind, ref mut value, to) => {
781-
let from = value.ty(self.local_decls, self.tcx);
782-
let value = self.simplify_operand(value, location)?;
783-
if let CastKind::PointerCoercion(
784-
PointerCoercion::ReifyFnPointer | PointerCoercion::ClosureFnPointer(_),
785-
) = kind
786-
{
787-
// Each reification of a generic fn may get a different pointer.
788-
// Do not try to merge them.
789-
return self.new_opaque();
790-
}
791-
Value::Cast { kind, value, from, to }
805+
Rvalue::Cast(ref mut kind, ref mut value, to) => {
806+
return self.simplify_cast(kind, value, to, location);
792807
}
793808
Rvalue::BinaryOp(op, box (ref mut lhs, ref mut rhs)) => {
794809
let ty = lhs.ty(self.local_decls, self.tcx);
@@ -876,6 +891,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
876891
}
877892
}
878893

894+
let fields: Option<Vec<_>> = fields
895+
.iter_mut()
896+
.map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
897+
.collect();
898+
let fields = fields?;
899+
879900
let (ty, variant_index) = match *kind {
880901
AggregateKind::Array(..) => {
881902
assert!(!fields.is_empty());
@@ -895,12 +916,6 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
895916
AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
896917
};
897918

898-
let fields: Option<Vec<_>> = fields
899-
.iter_mut()
900-
.map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
901-
.collect();
902-
let fields = fields?;
903-
904919
if let AggregateTy::Array = ty
905920
&& fields.len() > 4
906921
{
@@ -1031,6 +1046,50 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
10311046
}
10321047
}
10331048

1049+
fn simplify_cast(
1050+
&mut self,
1051+
kind: &mut CastKind,
1052+
operand: &mut Operand<'tcx>,
1053+
to: Ty<'tcx>,
1054+
location: Location,
1055+
) -> Option<VnIndex> {
1056+
use rustc_middle::ty::adjustment::PointerCoercion::*;
1057+
use CastKind::*;
1058+
1059+
let mut from = operand.ty(self.local_decls, self.tcx);
1060+
let mut value = self.simplify_operand(operand, location)?;
1061+
if from == to {
1062+
return Some(value);
1063+
}
1064+
1065+
if let CastKind::PointerCoercion(ReifyFnPointer | ClosureFnPointer(_)) = kind {
1066+
// Each reification of a generic fn may get a different pointer.
1067+
// Do not try to merge them.
1068+
return self.new_opaque();
1069+
}
1070+
1071+
if let PtrToPtr | PointerCoercion(MutToConstPointer) = kind
1072+
&& let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
1073+
*self.get(value)
1074+
&& let PtrToPtr | PointerCoercion(MutToConstPointer) = inner_kind
1075+
{
1076+
from = inner_from;
1077+
value = inner_value;
1078+
*kind = PtrToPtr;
1079+
if inner_from == to {
1080+
return Some(inner_value);
1081+
}
1082+
if let Some(const_) = self.try_as_constant(value) {
1083+
*operand = Operand::Constant(Box::new(const_));
1084+
} else if let Some(local) = self.try_as_local(value, location) {
1085+
*operand = Operand::Copy(local.into());
1086+
self.reused_locals.insert(local);
1087+
}
1088+
}
1089+
1090+
Some(self.insert(Value::Cast { kind: *kind, value, from, to }))
1091+
}
1092+
10341093
fn simplify_len(&mut self, place: &mut Place<'tcx>, location: Location) -> Option<VnIndex> {
10351094
// Trivial case: we are fetching a statically known length.
10361095
let place_ty = place.ty(self.local_decls, self.tcx).ty;
@@ -1178,6 +1237,51 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
11781237
self.tcx
11791238
}
11801239

1240+
fn visit_var_debug_info(&mut self, var_debug_info: &mut VarDebugInfo<'tcx>) {
1241+
let mut replace_dereffed = |place: &mut Place<'tcx>| -> Option<!> {
1242+
let last_deref = place.projection.iter().rposition(|e| e == PlaceElem::Deref)?;
1243+
1244+
// Another place that holds the same value.
1245+
let mut place_ref = place.as_ref();
1246+
let mut value = self.locals[place.local]?;
1247+
1248+
for (index, &proj) in place.projection[..last_deref].iter().enumerate() {
1249+
if let Some(candidates) = self.rev_locals.get(value)
1250+
&& let Some(&local) = candidates.first()
1251+
{
1252+
place_ref = PlaceRef { local, projection: &place.projection[index..] };
1253+
}
1254+
1255+
let place_upto =
1256+
PlaceRef { local: place.local, projection: &place.projection[..index] };
1257+
if let Some(projected) = self.project(place_upto, value, proj) {
1258+
value = projected;
1259+
} else {
1260+
if place_ref.projection.len() < place.projection.len() {
1261+
*place = place_ref.project_deeper(&[], self.tcx);
1262+
}
1263+
return None;
1264+
}
1265+
}
1266+
1267+
if let Some(candidates) = self.rev_locals.get(value)
1268+
&& let Some(&local) = candidates.first()
1269+
{
1270+
let place_ref = PlaceRef { local, projection: &place.projection[last_deref..] };
1271+
*place = place_ref.project_deeper(&[], self.tcx);
1272+
}
1273+
1274+
return None;
1275+
};
1276+
1277+
match &mut var_debug_info.value {
1278+
VarDebugInfoContents::Const(_) => {}
1279+
VarDebugInfoContents::Place(place) => {
1280+
replace_dereffed(place);
1281+
}
1282+
}
1283+
}
1284+
11811285
fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, location: Location) {
11821286
self.simplify_place_projection(place, location);
11831287
}

‎tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-abort.diff

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind unreachable];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind unreachable];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-unwind.diff

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind continue];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind continue];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-abort.diff

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind unreachable];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind unreachable];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-unwind.diff

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind continue];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind continue];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@ fn main() {
88
// CHECK-LABEL: fn main(
99
// CHECK: debug a => [[a:_.*]];
1010
// CHECK: [[slice:_.*]] = const {{.*}} as &[u32] (PointerCoercion(Unsize));
11-
// FIXME(cjgillot) simplify Len and projection into unsized slice.
12-
// CHECK-NOT: assert(const true,
13-
// CHECK: [[a]] = (*[[slice]])[1 of 2];
14-
// CHECK-NOT: [[a]] = const 2_u32;
11+
// CHECK: assert(const true,
12+
// CHECK: [[a]] = const 2_u32;
1513
let a = (&[1u32, 2, 3] as &[u32])[1];
1614
}

‎tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.32bit.diff

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@
1212
StorageLive(_1);
1313
StorageLive(_2);
1414
- _2 = ();
15+
- _1 = Union32 { value: move _2 };
1516
+ _2 = const ();
16-
_1 = Union32 { value: move _2 };
17+
+ _1 = Union32 { value: const () };
1718
StorageDead(_2);
1819
_0 = move _1 as u32 (Transmute);
1920
StorageDead(_1);

0 commit comments

Comments
 (0)