diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index a615b383fee5b..6ea84620bbed8 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -1,6 +1,7 @@ #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] use rustc_data_structures::fx::FxIndexMap; +use rustc_data_structures::graph::WithSuccessors; use rustc_index::bit_set::BitSet; use rustc_middle::mir::{ self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges, @@ -239,6 +240,162 @@ pub fn calculate_borrows_out_of_scope_at_location<'tcx>( prec.borrows_out_of_scope_at_location } +struct PoloniusOutOfScopePrecomputer<'a, 'tcx> { + visited: BitSet, + visit_stack: Vec, + body: &'a Body<'tcx>, + regioncx: &'a RegionInferenceContext<'tcx>, + + loans_out_of_scope_at_location: FxIndexMap>, +} + +impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> { + fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self { + Self { + visited: BitSet::new_empty(body.basic_blocks.len()), + visit_stack: vec![], + body, + regioncx, + loans_out_of_scope_at_location: FxIndexMap::default(), + } + } +} + +impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { + /// Loans are in scope while they are live: whether they are contained within any live region. + /// In the location-insensitive analysis, a loan will be contained in a region if the issuing + /// region can reach it in the subset graph. So this is a reachability problem. + fn precompute_loans_out_of_scope( + &mut self, + loan_idx: BorrowIndex, + issuing_region: RegionVid, + loan_issued_at: Location, + ) { + let sccs = self.regioncx.constraint_sccs(); + let issuing_region_scc = sccs.scc(issuing_region); + + // We first handle the cases where the loan doesn't go out of scope, depending on the issuing + // region's successors. + for scc in sccs.depth_first_search(issuing_region_scc) { + // 1. Via member constraints + // + // The issuing region can flow into the choice regions, and they are either: + // - placeholders or free regions themselves, + // - or also transitively outlive a free region. + // + // That is to say, if there are member constraints here, the loan escapes the function + // and cannot go out of scope. We can early return. + if self.regioncx.scc_has_member_constraints(scc) { + return; + } + + // 2. Via regions that are live at all points: placeholders and free regions. + // + // If the issuing region outlives such a region, its loan escapes the function and + // cannot go out of scope. We can early return. + if self.regioncx.scc_is_live_at_all_points(scc) { + return; + } + } + + let first_block = loan_issued_at.block; + let first_bb_data = &self.body.basic_blocks[first_block]; + + // The first block we visit is the one where the loan is issued, starting from the statement + // where the loan is issued: at `loan_issued_at`. + let first_lo = loan_issued_at.statement_index; + let first_hi = first_bb_data.statements.len(); + + if let Some(kill_location) = + self.loan_kill_location(loan_idx, loan_issued_at, first_block, first_lo, first_hi) + { + debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location); + self.loans_out_of_scope_at_location.entry(kill_location).or_default().push(loan_idx); + + // The loan dies within the first block, we're done and can early return. + return; + } + + // The loan is not dead. Add successor BBs to the work list, if necessary. + for succ_bb in first_bb_data.terminator().successors() { + if self.visited.insert(succ_bb) { + self.visit_stack.push(succ_bb); + } + } + + // We may end up visiting `first_block` again. This is not an issue: we know at this point + // that the loan is not killed in the `first_lo..=first_hi` range, so checking the + // `0..first_lo` range and the `0..first_hi` range gives the same result. + while let Some(block) = self.visit_stack.pop() { + let bb_data = &self.body[block]; + let num_stmts = bb_data.statements.len(); + if let Some(kill_location) = + self.loan_kill_location(loan_idx, loan_issued_at, block, 0, num_stmts) + { + debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location); + self.loans_out_of_scope_at_location + .entry(kill_location) + .or_default() + .push(loan_idx); + + // The loan dies within this block, so we don't need to visit its successors. + continue; + } + + // Add successor BBs to the work list, if necessary. + for succ_bb in bb_data.terminator().successors() { + if self.visited.insert(succ_bb) { + self.visit_stack.push(succ_bb); + } + } + } + + self.visited.clear(); + assert!(self.visit_stack.is_empty(), "visit stack should be empty"); + } + + /// Returns the lowest statement in `start..=end`, where the loan goes out of scope, if any. + /// This is the statement where the issuing region can't reach any of the regions that are live + /// at this point. + fn loan_kill_location( + &self, + loan_idx: BorrowIndex, + loan_issued_at: Location, + block: BasicBlock, + start: usize, + end: usize, + ) -> Option { + for statement_index in start..=end { + let location = Location { block, statement_index }; + + // Check whether the issuing region can reach local regions that are live at this point: + // - a loan is always live at its issuing location because it can reach the issuing + // region, which is always live at this location. + if location == loan_issued_at { + continue; + } + + // - the loan goes out of scope at `location` if it's not contained within any regions + // live at this point. + // + // FIXME: if the issuing region `i` can reach a live region `r` at point `p`, and `r` is + // live at point `q`, then it's guaranteed that `i` would reach `r` at point `q`. + // Reachability is location-insensitive, and we could take advantage of that, by jumping + // to a further point than just the next statement: we can jump to the furthest point + // within the block where `r` is live. + if self.regioncx.is_loan_live_at(loan_idx, location) { + continue; + } + + // No live region is reachable from the issuing region: the loan is killed at this + // point. + return Some(location); + } + + None + } +} + impl<'a, 'tcx> Borrows<'a, 'tcx> { pub fn new( tcx: TyCtxt<'tcx>, @@ -246,8 +403,33 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> { regioncx: &'a RegionInferenceContext<'tcx>, borrow_set: &'a BorrowSet<'tcx>, ) -> Self { - let borrows_out_of_scope_at_location = + let mut borrows_out_of_scope_at_location = calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set); + + // The in-tree polonius analysis computes loans going out of scope using the set-of-loans + // model, and makes sure they're identical to the existing computation of the set-of-points + // model. + if tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { + let mut polonius_prec = PoloniusOutOfScopePrecomputer::new(body, regioncx); + for (loan_idx, loan_data) in borrow_set.iter_enumerated() { + let issuing_region = loan_data.region; + let issued_location = loan_data.reserve_location; + + polonius_prec.precompute_loans_out_of_scope( + loan_idx, + issuing_region, + issued_location, + ); + } + + assert_eq!( + borrows_out_of_scope_at_location, polonius_prec.loans_out_of_scope_at_location, + "the loans out of scope must be the same as the borrows out of scope" + ); + + borrows_out_of_scope_at_location = polonius_prec.loans_out_of_scope_at_location; + } + Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location } } diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 129d0ec8cd786..0ea4401a87847 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -182,22 +182,26 @@ pub(crate) fn compute_regions<'cx, 'tcx>( let elements = &Rc::new(RegionValueElements::new(&body)); // Run the MIR type-checker. - let MirTypeckResults { constraints, universal_region_relations, opaque_type_values } = - type_check::type_check( - infcx, - param_env, - body, - promoted, - &universal_regions, - location_table, - borrow_set, - &mut all_facts, - flow_inits, - move_data, - elements, - upvars, - polonius_input, - ); + let MirTypeckResults { + constraints, + universal_region_relations, + opaque_type_values, + live_loans, + } = type_check::type_check( + infcx, + param_env, + body, + promoted, + &universal_regions, + location_table, + borrow_set, + &mut all_facts, + flow_inits, + move_data, + elements, + upvars, + polonius_input, + ); if let Some(all_facts) = &mut all_facts { let _prof_timer = infcx.tcx.prof.generic_activity("polonius_fact_generation"); @@ -275,6 +279,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>( type_tests, liveness_constraints, elements, + live_loans, ); // Generate various additional constraints. diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs index a65306b034c24..96cbe98c21694 100644 --- a/compiler/rustc_borrowck/src/region_infer/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/mod.rs @@ -7,6 +7,7 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::graph::scc::Sccs; use rustc_errors::Diagnostic; use rustc_hir::def_id::CRATE_DEF_ID; +use rustc_index::bit_set::SparseBitMatrix; use rustc_index::{IndexSlice, IndexVec}; use rustc_infer::infer::outlives::test_type_match; use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq}; @@ -21,6 +22,7 @@ use rustc_middle::traits::ObligationCauseCode; use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; use rustc_span::Span; +use crate::dataflow::BorrowIndex; use crate::{ constraints::{ graph::NormalConstraintGraph, ConstraintSccIndex, OutlivesConstraint, OutlivesConstraintSet, @@ -30,8 +32,8 @@ use crate::{ nll::PoloniusOutput, region_infer::reverse_sccs::ReverseSccGraph, region_infer::values::{ - LivenessValues, PlaceholderIndices, RegionElement, RegionValueElements, RegionValues, - ToElementIndex, + LivenessValues, PlaceholderIndices, PointIndex, RegionElement, RegionValueElements, + RegionValues, ToElementIndex, }, type_check::{free_region_relations::UniversalRegionRelations, Locations}, universal_regions::UniversalRegions, @@ -119,6 +121,9 @@ pub struct RegionInferenceContext<'tcx> { /// Information about how the universally quantified regions in /// scope on this function relate to one another. universal_region_relations: Frozen>, + + /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`. + live_loans: SparseBitMatrix, } /// Each time that `apply_member_constraint` is successful, it appends @@ -330,6 +335,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { type_tests: Vec>, liveness_constraints: LivenessValues, elements: &Rc, + live_loans: SparseBitMatrix, ) -> Self { debug!("universal_regions: {:#?}", universal_regions); debug!("outlives constraints: {:#?}", outlives_constraints); @@ -383,6 +389,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { type_tests, universal_regions, universal_region_relations, + live_loans, }; result.init_free_and_bound_regions(); @@ -2279,6 +2286,41 @@ impl<'tcx> RegionInferenceContext<'tcx> { } None } + + /// Access to the SCC constraint graph. + pub(crate) fn constraint_sccs(&self) -> &Sccs { + self.constraint_sccs.as_ref() + } + + /// Returns whether the given SCC has any member constraints. + pub(crate) fn scc_has_member_constraints(&self, scc: ConstraintSccIndex) -> bool { + self.member_constraints.indices(scc).next().is_some() + } + + /// Returns whether the given SCC is live at all points: whether the representative is a + /// placeholder or a free region. + pub(crate) fn scc_is_live_at_all_points(&self, scc: ConstraintSccIndex) -> bool { + // FIXME: there must be a cleaner way to find this information. At least, when + // higher-ranked subtyping is abstracted away from the borrowck main path, we'll only + // need to check whether this is a universal region. + let representative = self.scc_representatives[scc]; + let origin = self.var_infos[representative].origin; + let live_at_all_points = matches!( + origin, + RegionVariableOrigin::Nll( + NllRegionVariableOrigin::Placeholder(_) | NllRegionVariableOrigin::FreeRegion + ) + ); + live_at_all_points + } + + /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing + /// region is contained within the type of a variable that is live at this point. + /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`. + pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool { + let point = self.liveness_constraints.point_from_location(location); + self.live_loans.contains(point, loan_idx) + } } impl<'tcx> RegionDefinition<'tcx> { diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 5702d39db32d6..21da05c32dd86 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -1,10 +1,11 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; -use rustc_index::bit_set::HybridBitSet; +use rustc_data_structures::graph::WithSuccessors; +use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix}; use rustc_index::interval::IntervalSet; use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location}; use rustc_middle::traits::query::DropckOutlivesResult; -use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt}; +use rustc_middle::ty::{RegionVid, Ty, TyCtxt, TypeVisitable, TypeVisitableExt}; use rustc_span::DUMMY_SP; use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives; use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput}; @@ -14,6 +15,7 @@ use rustc_mir_dataflow::impls::MaybeInitializedPlaces; use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex}; use rustc_mir_dataflow::ResultsCursor; +use crate::dataflow::BorrowIndex; use crate::{ region_infer::values::{self, PointIndex, RegionValueElements}, type_check::liveness::local_use_map::LocalUseMap, @@ -50,6 +52,33 @@ pub(super) fn trace<'mir, 'tcx>( let local_use_map = &LocalUseMap::build(&relevant_live_locals, elements, body); + // When using `-Zpolonius=next`, compute the set of loans that can reach a given region. + let num_loans = typeck.borrowck_context.borrow_set.len(); + let mut inflowing_loans = SparseBitMatrix::new(num_loans); + if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() { + let borrowck_context = &typeck.borrowck_context; + let borrow_set = &borrowck_context.borrow_set; + let constraint_set = &borrowck_context.constraints.outlives_constraints; + + let num_region_vars = typeck.infcx.num_region_vars(); + let graph = constraint_set.graph(num_region_vars); + let region_graph = + graph.region_graph(&constraint_set, borrowck_context.universal_regions.fr_static); + + // Traverse each issuing region's constraints, and record the loan as flowing into the + // outlived region. + for (loan, issuing_region_data) in borrow_set.iter_enumerated() { + for succ in region_graph.depth_first_search(issuing_region_data.region) { + // We don't need to mention that a loan flows into its issuing region. + if succ == issuing_region_data.region { + continue; + } + + inflowing_loans.insert(succ, loan); + } + } + }; + let cx = LivenessContext { typeck, body, @@ -58,6 +87,7 @@ pub(super) fn trace<'mir, 'tcx>( local_use_map, move_data, drop_data: FxIndexMap::default(), + inflowing_loans, }; let mut results = LivenessResults::new(cx); @@ -95,6 +125,9 @@ struct LivenessContext<'me, 'typeck, 'flow, 'tcx> { /// Index indicating where each variable is assigned, used, or /// dropped. local_use_map: &'me LocalUseMap, + + /// Set of loans that flow into a given region, when using `-Zpolonius=next`. + inflowing_loans: SparseBitMatrix, } struct DropData<'tcx> { @@ -486,7 +519,13 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> { ) { debug!("add_use_live_facts_for(value={:?})", value); - Self::make_all_regions_live(self.elements, &mut self.typeck, value, live_at) + Self::make_all_regions_live( + self.elements, + &mut self.typeck, + value, + live_at, + &self.inflowing_loans, + ); } /// Some variable with type `live_ty` is "drop live" at `location` @@ -537,7 +576,13 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> { // All things in the `outlives` array may be touched by // the destructor and must be live at this point. for &kind in &drop_data.dropck_result.kinds { - Self::make_all_regions_live(self.elements, &mut self.typeck, kind, live_at); + Self::make_all_regions_live( + self.elements, + &mut self.typeck, + kind, + live_at, + &self.inflowing_loans, + ); polonius::add_drop_of_var_derefs_origin(&mut self.typeck, dropped_local, &kind); } @@ -548,6 +593,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> { typeck: &mut TypeChecker<'_, 'tcx>, value: impl TypeVisitable>, live_at: &IntervalSet, + inflowing_loans: &SparseBitMatrix, ) { debug!("make_all_regions_live(value={:?})", value); debug!( @@ -556,15 +602,35 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> { ); let tcx = typeck.tcx(); + let borrowck_context = &mut typeck.borrowck_context; + + // When using `-Zpolonius=next`, we want to record the loans that flow into this value's + // regions as being live at the given `live_at` points: this will be used to compute the + // location where a loan goes out of scope. + let num_loans = borrowck_context.borrow_set.len(); + let mut value_loans = HybridBitSet::new_empty(num_loans); + tcx.for_each_free_region(&value, |live_region| { - let live_region_vid = - typeck.borrowck_context.universal_regions.to_region_vid(live_region); - typeck - .borrowck_context + let live_region_vid = borrowck_context.universal_regions.to_region_vid(live_region); + + borrowck_context .constraints .liveness_constraints .add_elements(live_region_vid, live_at); + + // There can only be inflowing loans for this region when we are using + // `-Zpolonius=next`. + if let Some(inflowing) = inflowing_loans.row(live_region_vid) { + value_loans.union(inflowing); + } }); + + // Record the loans reaching the value. + if !value_loans.is_empty() { + for point in live_at.iter() { + borrowck_context.live_loans.union_row(point, &value_loans); + } + } } fn compute_drop_data( diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index e7b1a489f5d7c..1ec0e62d16a3b 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -14,6 +14,7 @@ use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::LocalDefId; use rustc_hir::lang_items::LangItem; +use rustc_index::bit_set::SparseBitMatrix; use rustc_index::{IndexSlice, IndexVec}; use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_infer::infer::outlives::env::RegionBoundPairs; @@ -50,6 +51,8 @@ use rustc_mir_dataflow::impls::MaybeInitializedPlaces; use rustc_mir_dataflow::move_paths::MoveData; use rustc_mir_dataflow::ResultsCursor; +use crate::dataflow::BorrowIndex; +use crate::region_infer::values::PointIndex; use crate::session_diagnostics::{MoveUnsized, SimdShuffleLastConst}; use crate::{ borrow_set::BorrowSet, @@ -163,6 +166,9 @@ pub(crate) fn type_check<'mir, 'tcx>( debug!(?normalized_inputs_and_output); + // When using `-Zpolonius=next`, liveness will record the set of live loans per point. + let mut live_loans = SparseBitMatrix::new(borrow_set.len()); + let mut borrowck_context = BorrowCheckContext { universal_regions, location_table, @@ -170,6 +176,7 @@ pub(crate) fn type_check<'mir, 'tcx>( all_facts, constraints: &mut constraints, upvars, + live_loans: &mut live_loans, }; let mut checker = TypeChecker::new( @@ -240,7 +247,7 @@ pub(crate) fn type_check<'mir, 'tcx>( }) .collect(); - MirTypeckResults { constraints, universal_region_relations, opaque_type_values } + MirTypeckResults { constraints, universal_region_relations, opaque_type_values, live_loans } } fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) { @@ -855,12 +862,21 @@ struct BorrowCheckContext<'a, 'tcx> { borrow_set: &'a BorrowSet<'tcx>, pub(crate) constraints: &'a mut MirTypeckRegionConstraints<'tcx>, upvars: &'a [Upvar<'tcx>], + + /// The set of loans that are live at a given point in the CFG, filled in by `liveness::trace`, + /// when using `-Zpolonius=next`. + pub(crate) live_loans: &'a mut SparseBitMatrix, } +/// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions +/// inference computation. pub(crate) struct MirTypeckResults<'tcx> { pub(crate) constraints: MirTypeckRegionConstraints<'tcx>, pub(crate) universal_region_relations: Frozen>, pub(crate) opaque_type_values: FxIndexMap, OpaqueHiddenType<'tcx>>, + + /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`. + pub(crate) live_loans: SparseBitMatrix, } /// A collection of region constraints that must be satisfied for the