improve scope computation

- integrate free regions with placeholders as regions that are live at
  all points: flowing into these is enough to stop the computation
- pre-compute liveness of SCCs per point, and check escaping the
  function, and the liveness of SCCs instead of regions

Author: lqd

compiler/rustc_borrowck/src/dataflow.rs

@@ -1,7 +1,7 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::{BitSet, SparseBitMatrix};
 use rustc_middle::mir::{
     self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges,
 };
@@ -14,6 +14,7 @@ use rustc_mir_dataflow::{Analysis, Direction, Results};
 use std::fmt;
 
 use crate::constraints::ConstraintSccIndex;
+use crate::region_infer::values::PointIndex;
 use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
 /// A tuple with named fields that can hold either the results or the transient state of the
@@ -123,7 +124,6 @@ rustc_index::newtype_index! {
 pub struct Borrows<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     body: &'a Body<'tcx>,
-
     borrow_set: &'a BorrowSet<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
@@ -250,39 +250,63 @@ struct PoloniusOutOfScopePrecomputer<'a, 'tcx> {
     visit_stack: Vec<StackEntry>,
     body: &'a Body<'tcx>,
     regioncx: &'a RegionInferenceContext<'tcx>,
+
+    sccs_live_at_all_points: FxHashSet<ConstraintSccIndex>,
+    live_sccs_per_point: SparseBitMatrix<PointIndex, ConstraintSccIndex>,
+
+    reachability: BitSet<ConstraintSccIndex>,
+    reachability_stack: Vec<ConstraintSccIndex>,
+
     loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
-    placeholders: FxHashSet<ConstraintSccIndex>,
-    reachability: BitSet<RegionVid>,
-    reachability_stack: Vec<RegionVid>,
 }
 
 impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> {
     fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self {
-        // Compute the placeholder list once, as it will be used for all the loan scopes we'll
-        // compute.
+        // Compute the list of SCCs that are live at all points, as it will be used for all the
+        // loan scopes we'll compute.
         // FIXME: they're surely already available somewhere.
-        let placeholders = regioncx
+        let sccs_live_at_all_points: FxHashSet<_> = regioncx
             .regions()
             .filter(|&r| {
                 use rustc_infer::infer::{NllRegionVariableOrigin, RegionVariableOrigin};
                 let origin = regioncx.var_infos[r].origin;
-                let is_placeholder = matches!(
+                let live_at_all_points = matches!(
                     origin,
-                    RegionVariableOrigin::Nll(NllRegionVariableOrigin::Placeholder(_))
+                    RegionVariableOrigin::Nll(
+                        NllRegionVariableOrigin::Placeholder(_)
+                            | NllRegionVariableOrigin::FreeRegion
+                    )
                 );
-                is_placeholder
+                live_at_all_points
             })
             .map(|r| regioncx.constraint_sccs.scc(r))
             .collect();
 
+        // Pre-compute the set of live SCCs per point
+        let liveness = &regioncx.liveness_constraints;
+        let sccs = &regioncx.constraint_sccs;
+        let mut live_sccs_per_point = SparseBitMatrix::new(sccs.num_sccs());
+
+        for region in liveness.rows() {
+            let scc = sccs.scc(region);
+            if sccs_live_at_all_points.contains(&scc) {
+                continue;
+            }
+            for location in liveness.get_elements(region) {
+                let point = liveness.point_from_location(location);
+                live_sccs_per_point.insert(point, scc);
+            }
+        }
+
         Self {
             visited: BitSet::new_empty(body.basic_blocks.len()),
             visit_stack: vec![],
             body,
             regioncx,
             loans_out_of_scope_at_location: FxIndexMap::default(),
-            placeholders,
-            reachability: BitSet::new_empty(regioncx.regions().count()),
+            sccs_live_at_all_points,
+            live_sccs_per_point,
+            reachability: BitSet::new_empty(regioncx.constraint_sccs.num_sccs()),
             reachability_stack: vec![],
         }
     }
@@ -306,13 +330,11 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
         let sccs = &self.regioncx.constraint_sccs;
         let member_constraints = &self.regioncx.member_constraints;
 
-        self.reachability_stack.push(issuing_region);
-        self.reachability.insert(issuing_region);
-
-        let static_region = self.regioncx.universal_regions().fr_static;
-        while let Some(region) = self.reachability_stack.pop() {
-            let scc = sccs.scc(region);
+        let issuing_region_scc = sccs.scc(issuing_region);
+        self.reachability_stack.push(issuing_region_scc);
+        self.reachability.insert(issuing_region_scc);
 
+        while let Some(scc) = self.reachability_stack.pop() {
             // Handle successors of this SCC:
             //
             // 1. Via member constraints
@@ -324,11 +346,12 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
             // That is to say, if there are member constraints here, the loan escapes the
             // function and cannot go out of scope. We can early return.
             //
-            // 2. Via placeholders
+            // 2. Via regions that are live at all points: placeholders and free regions.
             //
-            // If the issuing region outlives placeholders, its loan escapes the function and
+            // If the issuing region outlives such a region, its loan escapes the function and
             // cannot go out of scope. We can early return.
-            if member_constraints.indices(scc).next().is_some() || self.placeholders.contains(&scc)
+            if member_constraints.indices(scc).next().is_some()
+                || self.sccs_live_at_all_points.contains(&scc)
             {
                 self.reachability_stack.clear();
                 self.reachability.clear();
@@ -337,15 +360,9 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
 
             // 3. Via outlives successors, which we want to record and traverse, so we add them
             //    to the worklist stack
-            let successors = self.regioncx.constraint_graph.outgoing_edges(
-                region,
-                &self.regioncx.constraints,
-                static_region,
-            );
-            for outlives_constraint in successors {
-                let succ = outlives_constraint.sub;
-                if self.reachability.insert(succ) {
-                    self.reachability_stack.push(succ);
+            for &succ_scc in sccs.successors(scc) {
+                if self.reachability.insert(succ_scc) {
+                    self.reachability_stack.push(succ_scc);
                 }
             }
         }
@@ -380,10 +397,19 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
 
                 // Check whether the issuing region can reach local regions that are live at this
                 // point.
-                for reachable_region in self.reachability.iter() {
-                    if self.regioncx.liveness_constraints.contains(reachable_region, location) {
-                        issuing_region_can_reach_live_regions = true;
-                        break;
+                //
+                // FIXME: if the issuing region `i` can reach a live region `r` at point `p`, and
+                // `r` is live at point `q`, then it's guaranteed that `i` would reach `r` at point
+                // `q`. The reachability is location-insensitive, and we could take advantage of
+                // that, by jumping to a further point than the next statement. We can jump to the
+                // furthest point within the block where `r` is live.
+                let point = self.regioncx.liveness_constraints.point_from_location(location);
+                if let Some(live_sccs) = self.live_sccs_per_point.row(point) {
+                    for live_scc in live_sccs.iter() {
+                        if self.reachability.contains(live_scc) {
+                            issuing_region_can_reach_live_regions = true;
+                            break;
+                        }
                     }
                 }