Skip to content

Commit 9aa232e

Browse files
borsbors
committed
Auto merge of #120405 - cjgillot:gvn-pointer, r=oli-obk
Fold pointer operations in GVN This PR proposes 2 combinations of cast operations in MIR GVN: - a chain of `PtrToPtr` or `MutToConstPointer` casts can be folded together into a single `PtrToPtr` cast; - we attempt to evaluate more ptr ops when there is no provenance. In particular, this allows to read from static slices. This is not yet sufficient to see through slice operations that use `PtrComponents` (because that's a union), but still a step forward. r? `@ghost`
2 parents 0cbef48 + 7a6b00e commit 9aa232e

File tree

38 files changed

+2240
-1387
lines changed

38 files changed

+2240
-1387
lines changed

‎compiler/rustc_const_eval/src/interpret/cast.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
415415
}
416416
}
417417

418-
fn unsize_into(
418+
pub fn unsize_into(
419419
&mut self,
420420
src: &OpTy<'tcx, M::Provenance>,
421421
cast_ty: TyAndLayout<'tcx>,

‎compiler/rustc_mir_transform/src/dataflow_const_prop.rs

+46-6
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
//!
33
//! Currently, this pass only propagates scalar values.
44
5-
use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, PlaceTy, Projectable};
5+
use rustc_const_eval::interpret::{
6+
ImmTy, Immediate, InterpCx, OpTy, PlaceTy, PointerArithmetic, Projectable,
7+
};
68
use rustc_data_structures::fx::FxHashMap;
79
use rustc_hir::def::DefKind;
810
use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult, Scalar};
@@ -936,12 +938,50 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
936938
}
937939

938940
fn binary_ptr_op(
939-
_ecx: &InterpCx<'mir, 'tcx, Self>,
940-
_bin_op: BinOp,
941-
_left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
942-
_right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
941+
ecx: &InterpCx<'mir, 'tcx, Self>,
942+
bin_op: BinOp,
943+
left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
944+
right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
943945
) -> interpret::InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)> {
944-
throw_machine_stop_str!("can't do pointer arithmetic");
946+
use rustc_middle::mir::BinOp::*;
947+
Ok(match bin_op {
948+
Eq | Ne | Lt | Le | Gt | Ge => {
949+
// Types can differ, e.g. fn ptrs with different `for`.
950+
assert_eq!(left.layout.abi, right.layout.abi);
951+
let size = ecx.pointer_size();
952+
// Just compare the bits. ScalarPairs are compared lexicographically.
953+
// We thus always compare pairs and simply fill scalars up with 0.
954+
// If the pointer has provenance, `to_bits` will return `Err` and we bail out.
955+
let left = match **left {
956+
Immediate::Scalar(l) => (l.to_bits(size)?, 0),
957+
Immediate::ScalarPair(l1, l2) => (l1.to_bits(size)?, l2.to_bits(size)?),
958+
Immediate::Uninit => panic!("we should never see uninit data here"),
959+
};
960+
let right = match **right {
961+
Immediate::Scalar(r) => (r.to_bits(size)?, 0),
962+
Immediate::ScalarPair(r1, r2) => (r1.to_bits(size)?, r2.to_bits(size)?),
963+
Immediate::Uninit => panic!("we should never see uninit data here"),
964+
};
965+
let res = match bin_op {
966+
Eq => left == right,
967+
Ne => left != right,
968+
Lt => left < right,
969+
Le => left <= right,
970+
Gt => left > right,
971+
Ge => left >= right,
972+
_ => bug!(),
973+
};
974+
(ImmTy::from_bool(res, *ecx.tcx), false)
975+
}
976+
977+
// Some more operations are possible with atomics.
978+
// The return value always has the provenance of the *left* operand.
979+
Add | Sub | BitOr | BitAnd | BitXor => {
980+
throw_machine_stop_str!("pointer arithmetic is not handled")
981+
}
982+
983+
_ => span_bug!(ecx.cur_span(), "Invalid operator on pointers: {:?}", bin_op),
984+
})
945985
}
946986

947987
fn expose_ptr(

‎compiler/rustc_mir_transform/src/gvn.rs

+69-13
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ use rustc_index::IndexVec;
9393
use rustc_middle::mir::interpret::GlobalAlloc;
9494
use rustc_middle::mir::visit::*;
9595
use rustc_middle::mir::*;
96-
use rustc_middle::ty::adjustment::PointerCoercion;
9796
use rustc_middle::ty::layout::LayoutOf;
9897
use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
9998
use rustc_span::def_id::DefId;
@@ -552,6 +551,29 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
552551
}
553552
value.offset(Size::ZERO, to, &self.ecx).ok()?
554553
}
554+
CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize) => {
555+
let src = self.evaluated[value].as_ref()?;
556+
let to = self.ecx.layout_of(to).ok()?;
557+
let dest = self.ecx.allocate(to, MemoryKind::Stack).ok()?;
558+
self.ecx.unsize_into(src, to, &dest.clone().into()).ok()?;
559+
self.ecx
560+
.alloc_mark_immutable(dest.ptr().provenance.unwrap().alloc_id())
561+
.ok()?;
562+
dest.into()
563+
}
564+
CastKind::FnPtrToPtr
565+
| CastKind::PtrToPtr
566+
| CastKind::PointerCoercion(
567+
ty::adjustment::PointerCoercion::MutToConstPointer
568+
| ty::adjustment::PointerCoercion::ArrayToPointer
569+
| ty::adjustment::PointerCoercion::UnsafeFnPointer,
570+
) => {
571+
let src = self.evaluated[value].as_ref()?;
572+
let src = self.ecx.read_immediate(src).ok()?;
573+
let to = self.ecx.layout_of(to).ok()?;
574+
let ret = self.ecx.ptr_to_ptr(&src, to).ok()?;
575+
ret.into()
576+
}
555577
_ => return None,
556578
},
557579
};
@@ -778,18 +800,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
778800

779801
// Operations.
780802
Rvalue::Len(ref mut place) => return self.simplify_len(place, location),
781-
Rvalue::Cast(kind, ref mut value, to) => {
782-
let from = value.ty(self.local_decls, self.tcx);
783-
let value = self.simplify_operand(value, location)?;
784-
if let CastKind::PointerCoercion(
785-
PointerCoercion::ReifyFnPointer | PointerCoercion::ClosureFnPointer(_),
786-
) = kind
787-
{
788-
// Each reification of a generic fn may get a different pointer.
789-
// Do not try to merge them.
790-
return self.new_opaque();
791-
}
792-
Value::Cast { kind, value, from, to }
803+
Rvalue::Cast(ref mut kind, ref mut value, to) => {
804+
return self.simplify_cast(kind, value, to, location);
793805
}
794806
Rvalue::BinaryOp(op, box (ref mut lhs, ref mut rhs)) => {
795807
let ty = lhs.ty(self.local_decls, self.tcx);
@@ -1035,6 +1047,50 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
10351047
}
10361048
}
10371049

1050+
fn simplify_cast(
1051+
&mut self,
1052+
kind: &mut CastKind,
1053+
operand: &mut Operand<'tcx>,
1054+
to: Ty<'tcx>,
1055+
location: Location,
1056+
) -> Option<VnIndex> {
1057+
use rustc_middle::ty::adjustment::PointerCoercion::*;
1058+
use CastKind::*;
1059+
1060+
let mut from = operand.ty(self.local_decls, self.tcx);
1061+
let mut value = self.simplify_operand(operand, location)?;
1062+
if from == to {
1063+
return Some(value);
1064+
}
1065+
1066+
if let CastKind::PointerCoercion(ReifyFnPointer | ClosureFnPointer(_)) = kind {
1067+
// Each reification of a generic fn may get a different pointer.
1068+
// Do not try to merge them.
1069+
return self.new_opaque();
1070+
}
1071+
1072+
if let PtrToPtr | PointerCoercion(MutToConstPointer) = kind
1073+
&& let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
1074+
*self.get(value)
1075+
&& let PtrToPtr | PointerCoercion(MutToConstPointer) = inner_kind
1076+
{
1077+
from = inner_from;
1078+
value = inner_value;
1079+
*kind = PtrToPtr;
1080+
if inner_from == to {
1081+
return Some(inner_value);
1082+
}
1083+
if let Some(const_) = self.try_as_constant(value) {
1084+
*operand = Operand::Constant(Box::new(const_));
1085+
} else if let Some(local) = self.try_as_local(value, location) {
1086+
*operand = Operand::Copy(local.into());
1087+
self.reused_locals.insert(local);
1088+
}
1089+
}
1090+
1091+
Some(self.insert(Value::Cast { kind: *kind, value, from, to }))
1092+
}
1093+
10381094
fn simplify_len(&mut self, place: &mut Place<'tcx>, location: Location) -> Option<VnIndex> {
10391095
// Trivial case: we are fetching a statically known length.
10401096
let place_ty = place.ty(self.local_decls, self.tcx).ty;

‎tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-abort.diff

+5-4
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind unreachable];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind unreachable];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-unwind.diff

+5-4
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind continue];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind continue];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-abort.diff

+5-4
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind unreachable];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind unreachable];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-unwind.diff

+5-4
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@
3131
StorageDead(_3);
3232
StorageLive(_6);
3333
_6 = const 1_usize;
34-
_7 = Len((*_2));
34+
- _7 = Len((*_2));
3535
- _8 = Lt(_6, _7);
3636
- assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, _6) -> [success: bb1, unwind continue];
37-
+ _8 = Lt(const 1_usize, _7);
38-
+ assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, const 1_usize) -> [success: bb1, unwind continue];
37+
+ _7 = const 3_usize;
38+
+ _8 = const true;
39+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
3940
}
4041

4142
bb1: {
4243
- _1 = (*_2)[_6];
43-
+ _1 = (*_2)[1 of 2];
44+
+ _1 = const 2_u32;
4445
StorageDead(_6);
4546
StorageDead(_4);
4647
StorageDead(_2);

‎tests/mir-opt/const_prop/slice_len.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@ fn main() {
88
// CHECK-LABEL: fn main(
99
// CHECK: debug a => [[a:_.*]];
1010
// CHECK: [[slice:_.*]] = const {{.*}} as &[u32] (PointerCoercion(Unsize));
11-
// FIXME(cjgillot) simplify Len and projection into unsized slice.
12-
// CHECK-NOT: assert(const true,
13-
// CHECK: [[a]] = (*[[slice]])[1 of 2];
14-
// CHECK-NOT: [[a]] = const 2_u32;
11+
// CHECK: assert(const true,
12+
// CHECK: [[a]] = const 2_u32;
1513
let a = (&[1u32, 2, 3] as &[u32])[1];
1614
}

‎tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff

+20-8
Original file line numberDiff line numberDiff line change
@@ -69,28 +69,40 @@
6969
}
7070

7171
bb2: {
72-
_10 = const {0x1 as *mut [bool; 0]} as *mut () (PtrToPtr);
73-
_9 = NonNull::<T>::new_unchecked::precondition_check(move _10) -> [return: bb3, unwind unreachable];
72+
_10 = const {0x1 as *mut ()};
73+
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb3, unwind unreachable];
7474
}
7575

7676
bb3: {
7777
StorageDead(_8);
78-
_11 = const {0x1 as *mut [bool; 0]} as *const [bool; 0] (PointerCoercion(MutToConstPointer));
79-
_5 = NonNull::<[bool; 0]> { pointer: _11 };
78+
_11 = const {0x1 as *const [bool; 0]};
79+
_5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
8080
StorageDead(_11);
8181
StorageDead(_10);
8282
StorageDead(_6);
83-
_4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
83+
_4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
8484
StorageDead(_5);
85-
_3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize));
85+
_3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
8686
StorageDead(_4);
87-
_2 = Box::<[bool]>(_3, const std::alloc::Global);
87+
_2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
8888
StorageDead(_9);
8989
StorageDead(_3);
90-
_1 = A { foo: move _2 };
90+
_1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
9191
StorageDead(_2);
9292
_0 = const ();
9393
drop(_1) -> [return: bb1, unwind unreachable];
9494
}
9595
}
9696

97+
ALLOC2 (size: 8, align: 4) {
98+
01 00 00 00 00 00 00 00 │ ........
99+
}
100+
101+
ALLOC1 (size: 8, align: 4) {
102+
01 00 00 00 00 00 00 00 │ ........
103+
}
104+
105+
ALLOC0 (size: 8, align: 4) {
106+
01 00 00 00 00 00 00 00 │ ........
107+
}
108+

‎tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff

+20-8
Original file line numberDiff line numberDiff line change
@@ -73,28 +73,40 @@
7373
}
7474

7575
bb3: {
76-
_10 = const {0x1 as *mut [bool; 0]} as *mut () (PtrToPtr);
77-
_9 = NonNull::<T>::new_unchecked::precondition_check(move _10) -> [return: bb4, unwind unreachable];
76+
_10 = const {0x1 as *mut ()};
77+
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb4, unwind unreachable];
7878
}
7979

8080
bb4: {
8181
StorageDead(_8);
82-
_11 = const {0x1 as *mut [bool; 0]} as *const [bool; 0] (PointerCoercion(MutToConstPointer));
83-
_5 = NonNull::<[bool; 0]> { pointer: _11 };
82+
_11 = const {0x1 as *const [bool; 0]};
83+
_5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
8484
StorageDead(_11);
8585
StorageDead(_10);
8686
StorageDead(_6);
87-
_4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
87+
_4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
8888
StorageDead(_5);
89-
_3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize));
89+
_3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
9090
StorageDead(_4);
91-
_2 = Box::<[bool]>(_3, const std::alloc::Global);
91+
_2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
9292
StorageDead(_9);
9393
StorageDead(_3);
94-
_1 = A { foo: move _2 };
94+
_1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
9595
StorageDead(_2);
9696
_0 = const ();
9797
drop(_1) -> [return: bb1, unwind: bb2];
9898
}
9999
}
100100

101+
ALLOC2 (size: 8, align: 4) {
102+
01 00 00 00 00 00 00 00 │ ........
103+
}
104+
105+
ALLOC1 (size: 8, align: 4) {
106+
01 00 00 00 00 00 00 00 │ ........
107+
}
108+
109+
ALLOC0 (size: 8, align: 4) {
110+
01 00 00 00 00 00 00 00 │ ........
111+
}
112+

0 commit comments

Comments
 (0)