Implement jump threading MIR opt

Cargo.lock

@@ -4242,6 +4242,7 @@ dependencies = [
  "coverage_test_macros",
  "either",
  "itertools",
+ "rustc_arena",
  "rustc_ast",
  "rustc_attr",
  "rustc_const_eval",

compiler/rustc_middle/src/mir/terminator.rs

@@ -28,6 +28,15 @@ impl SwitchTargets {
         Self { values: smallvec![value], targets: smallvec![then, else_] }
     }
 
+    /// Inverse of `SwitchTargets::static_if`.
+    pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> {
+        if let &[value] = &self.values[..] && let &[then, else_] = &self.targets[..] {
+            Some((value, then, else_))
+        } else {
+            None
+        }
+    }
+
     /// Returns the fallback target that is jumped to when none of the values match the operand.
     pub fn otherwise(&self) -> BasicBlock {
         *self.targets.last().unwrap()

compiler/rustc_mir_dataflow/src/value_analysis.rs

@@ -463,7 +463,19 @@ impl<V: Clone> Clone for State<V> {
     }
 }
 
-impl<V: Clone + HasTop + HasBottom> State<V> {
+impl<V: Clone> State<V> {
+    pub fn new(init: V, map: &Map) -> State<V> {
+        let values = IndexVec::from_elem_n(init, map.value_count);
+        State(StateData::Reachable(values))
+    }
+
+    pub fn all(&self, f: impl Fn(&V) -> bool) -> bool {
+        match self.0 {
+            StateData::Unreachable => true,
+            StateData::Reachable(ref values) => values.iter().all(f),
+        }
+    }
+
     pub fn is_reachable(&self) -> bool {
         matches!(&self.0, StateData::Reachable(_))
     }
@@ -472,7 +484,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
         self.0 = StateData::Unreachable;
     }
 
-    pub fn flood_all(&mut self) {
+    pub fn flood_all(&mut self)
+    where
+        V: HasTop,
+    {
         self.flood_all_with(V::TOP)
     }
 
@@ -481,28 +496,52 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
         values.raw.fill(value);
     }
 
+    /// Assign `value` to all places that are contained in `place` or may alias one.
     pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
-        let StateData::Reachable(values) = &mut self.0 else { return };
-        map.for_each_aliasing_place(place, None, &mut |vi| {
-            values[vi] = value.clone();
-        });
+        self.flood_with_tail_elem(place, None, map, value)
     }
 
-    pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) {
+    /// Assign `TOP` to all places that are contained in `place` or may alias one.
+    pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map)
+    where
+        V: HasTop,
+    {
         self.flood_with(place, map, V::TOP)
     }
 
+    /// Assign `value` to the discriminant of `place` and all places that may alias it.
     pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
-        let StateData::Reachable(values) = &mut self.0 else { return };
-        map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |vi| {
-            values[vi] = value.clone();
-        });
+        self.flood_with_tail_elem(place, Some(TrackElem::Discriminant), map, value)
     }
 
-    pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) {
+    /// Assign `TOP` to the discriminant of `place` and all places that may alias it.
+    pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map)
+    where
+        V: HasTop,
+    {
         self.flood_discr_with(place, map, V::TOP)
     }
 
+    /// This method is the most general version of the `flood_*` method.
+    ///
+    /// Assign `value` on the given place and all places that may alias it. In particular, when
+    /// the given place has a variant downcast, we invoke the function on all the other variants.
+    ///
+    /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track
+    /// as such.
+    pub fn flood_with_tail_elem(
+        &mut self,
+        place: PlaceRef<'_>,
+        tail_elem: Option<TrackElem>,
+        map: &Map,
+        value: V,
+    ) {
+        let StateData::Reachable(values) = &mut self.0 else { return };
+        map.for_each_aliasing_place(place, tail_elem, &mut |vi| {
+            values[vi] = value.clone();
+        });
+    }
+
     /// Low-level method that assigns to a place.
     /// This does nothing if the place is not tracked.
     ///
@@ -553,44 +592,104 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
     }
 
     /// Helper method to interpret `target = result`.
-    pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
+    pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map)
+    where
+        V: HasTop,
+    {
         self.flood(target, map);
         if let Some(target) = map.find(target) {
             self.insert_idx(target, result, map);
         }
     }
 
     /// Helper method for assignments to a discriminant.
-    pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
+    pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map)
+    where
+        V: HasTop,
+    {
         self.flood_discr(target, map);
         if let Some(target) = map.find_discr(target) {
             self.insert_idx(target, result, map);
         }
     }
 
+    /// Retrieve the value stored for a place, or `None` if it is not tracked.
+    pub fn try_get(&self, place: PlaceRef<'_>, map: &Map) -> Option<V> {
+        let place = map.find(place)?;
+        self.try_get_idx(place, map)
+    }
+
+    /// Retrieve the discriminant stored for a place, or `None` if it is not tracked.
+    pub fn try_get_discr(&self, place: PlaceRef<'_>, map: &Map) -> Option<V> {
+        let place = map.find_discr(place)?;
+        self.try_get_idx(place, map)
+    }
+
+    /// Retrieve the slice length stored for a place, or `None` if it is not tracked.
+    pub fn try_get_len(&self, place: PlaceRef<'_>, map: &Map) -> Option<V> {
+        let place = map.find_len(place)?;
+        self.try_get_idx(place, map)
+    }
+
+    /// Retrieve the value stored for a place index, or `None` if it is not tracked.
+    pub fn try_get_idx(&self, place: PlaceIndex, map: &Map) -> Option<V> {
+        match &self.0 {
+            StateData::Reachable(values) => {
+                map.places[place].value_index.map(|v| values[v].clone())
+            }
+            StateData::Unreachable => None,
+        }
+    }
+
     /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
-    pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V {
-        map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP)
+    ///
+    /// This method returns ⊥ if the place is tracked and the state is unreachable.
+    pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V
+    where
+        V: HasBottom + HasTop,
+    {
+        match &self.0 {
+            StateData::Reachable(_) => self.try_get(place, map).unwrap_or(V::TOP),
+            // Because this is unreachable, we can return any value we want.
+            StateData::Unreachable => V::BOTTOM,
+        }
     }
 
     /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
-    pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V {
-        match map.find_discr(place) {
-            Some(place) => self.get_idx(place, map),
-            None => V::TOP,
+    ///
+    /// This method returns ⊥ the current state is unreachable.
+    pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V
+    where
+        V: HasBottom + HasTop,
+    {
+        match &self.0 {
+            StateData::Reachable(_) => self.try_get_discr(place, map).unwrap_or(V::TOP),
+            // Because this is unreachable, we can return any value we want.
+            StateData::Unreachable => V::BOTTOM,
         }
     }
 
     /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
-    pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V {
-        match map.find_len(place) {
-            Some(place) => self.get_idx(place, map),
-            None => V::TOP,
+    ///
+    /// This method returns ⊥ the current state is unreachable.
+    pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V
+    where
+        V: HasBottom + HasTop,
+    {
+        match &self.0 {
+            StateData::Reachable(_) => self.try_get_len(place, map).unwrap_or(V::TOP),
+            // Because this is unreachable, we can return any value we want.
+            StateData::Unreachable => V::BOTTOM,
         }
     }
 
     /// Retrieve the value stored for a place index, or ⊤ if it is not tracked.
-    pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V {
+    ///
+    /// This method returns ⊥ the current state is unreachable.
+    pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V
+    where
+        V: HasBottom + HasTop,
+    {
         match &self.0 {
             StateData::Reachable(values) => {
                 map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP)

compiler/rustc_mir_transform/Cargo.toml

@@ -11,6 +11,7 @@ smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
 either = "1"
 rustc_ast = { path = "../rustc_ast" }
+rustc_arena = { path = "../rustc_arena" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }

compiler/rustc_mir_transform/src/cost_checker.rs

@@ -0,0 +1,98 @@
+use rustc_middle::mir::visit::*;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
+
+const INSTR_COST: usize = 5;
+const CALL_PENALTY: usize = 25;
+const LANDINGPAD_PENALTY: usize = 50;
+const RESUME_PENALTY: usize = 45;
+
+/// Verify that the callee body is compatible with the caller.
+#[derive(Clone)]
+pub(crate) struct CostChecker<'b, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    param_env: ParamEnv<'tcx>,
+    cost: usize,
+    callee_body: &'b Body<'tcx>,
+    instance: Option<ty::Instance<'tcx>>,
+}
+
+impl<'b, 'tcx> CostChecker<'b, 'tcx> {
+    pub fn new(
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        instance: Option<ty::Instance<'tcx>>,
+        callee_body: &'b Body<'tcx>,
+    ) -> CostChecker<'b, 'tcx> {
+        CostChecker { tcx, param_env, callee_body, instance, cost: 0 }
+    }
+
+    pub fn cost(&self) -> usize {
+        self.cost
+    }
+
+    fn instantiate_ty(&self, v: Ty<'tcx>) -> Ty<'tcx> {
+        if let Some(instance) = self.instance {
+            instance.instantiate_mir(self.tcx, ty::EarlyBinder::bind(&v))
+        } else {
+            v
+        }
+    }
+}
+
+impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+        // Don't count StorageLive/StorageDead in the inlining cost.
+        match statement.kind {
+            StatementKind::StorageLive(_)
+            | StatementKind::StorageDead(_)
+            | StatementKind::Deinit(_)
+            | StatementKind::Nop => {}
+            _ => self.cost += INSTR_COST,
+        }
+    }
+
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
+        let tcx = self.tcx;
+        match terminator.kind {
+            TerminatorKind::Drop { ref place, unwind, .. } => {
+                // If the place doesn't actually need dropping, treat it like a regular goto.
+                let ty = self.instantiate_ty(place.ty(self.callee_body, tcx).ty);
+                if ty.needs_drop(tcx, self.param_env) {
+                    self.cost += CALL_PENALTY;
+                    if let UnwindAction::Cleanup(_) = unwind {
+                        self.cost += LANDINGPAD_PENALTY;
+                    }
+                } else {
+                    self.cost += INSTR_COST;
+                }
+            }
+            TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => {
+                let fn_ty = self.instantiate_ty(f.const_.ty());
+                self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) {
+                    // Don't give intrinsics the extra penalty for calls
+                    INSTR_COST
+                } else {
+                    CALL_PENALTY
+                };
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.cost += LANDINGPAD_PENALTY;
+                }
+            }
+            TerminatorKind::Assert { unwind, .. } => {
+                self.cost += CALL_PENALTY;
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.cost += LANDINGPAD_PENALTY;
+                }
+            }
+            TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY,
+            TerminatorKind::InlineAsm { unwind, .. } => {
+                self.cost += INSTR_COST;
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.cost += LANDINGPAD_PENALTY;
+                }
+            }
+            _ => self.cost += INSTR_COST,
+        }
+    }
+}