Auto merge of #51460 - nikomatsakis:nll-perf-examination-refactor-1, r=pnkfelix

Improve memoization and refactor NLL type check

I have a big branch that is refactoring NLL type check with the goal of introducing canonicalization-based memoization for all of the operations it does. This PR contains an initial prefix of that branch which, I believe, stands alone. It does introduce a few smaller optimizations of its own:

- Skip operations that are trivially a no-op
- Cache the results of the dropck-outlives computations done by liveness
- Skip resetting unifications if nothing changed

r? @pnkfelix

Author: bors

src/librustc/infer/region_constraints/mod.rs

@@ -69,6 +69,10 @@ pub struct RegionConstraintCollector<'tcx> {
     /// would wind up with a fresh stream of region variables that
     /// have been equated but appear distinct.
     unification_table: ut::UnificationTable<ut::InPlace<ty::RegionVid>>,
+
+    /// a flag set to true when we perform any unifications; this is used
+    /// to micro-optimize `take_and_reset_data`
+    any_unifications: bool,
 }
 
 pub type VarInfos = IndexVec<RegionVid, RegionVariableInfo>;
@@ -234,6 +238,7 @@ pub struct RegionVariableInfo {
 pub struct RegionSnapshot {
     length: usize,
     region_snapshot: ut::Snapshot<ut::InPlace<ty::RegionVid>>,
+    any_unifications: bool,
 }
 
 /// When working with skolemized regions, we often wish to find all of
@@ -280,6 +285,7 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
             bound_count: 0,
             undo_log: Vec::new(),
             unification_table: ut::UnificationTable::new(),
+            any_unifications: false,
         }
     }
 
@@ -325,6 +331,7 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
             bound_count: _,
             undo_log: _,
             unification_table,
+            any_unifications,
         } = self;
 
         // Clear the tables of (lubs, glbs), so that we will create
@@ -338,7 +345,10 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
         // un-unified" state. Note that when we unify `a` and `b`, we
         // also insert `a <= b` and a `b <= a` edges, so the
         // `RegionConstraintData` contains the relationship here.
-        unification_table.reset_unifications(|vid| unify_key::RegionVidKey { min_vid: vid });
+        if *any_unifications {
+            unification_table.reset_unifications(|vid| unify_key::RegionVidKey { min_vid: vid });
+            *any_unifications = false;
+        }
 
         mem::replace(data, RegionConstraintData::default())
     }
@@ -358,6 +368,7 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
         RegionSnapshot {
             length,
             region_snapshot: self.unification_table.snapshot(),
+            any_unifications: self.any_unifications,
         }
     }
 
@@ -385,6 +396,7 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
         let c = self.undo_log.pop().unwrap();
         assert!(c == OpenSnapshot);
         self.unification_table.rollback_to(snapshot.region_snapshot);
+        self.any_unifications = snapshot.any_unifications;
     }
 
     fn rollback_undo_entry(&mut self, undo_entry: UndoLogEntry<'tcx>) {
@@ -623,6 +635,7 @@ impl<'tcx> RegionConstraintCollector<'tcx> {
 
             if let (ty::ReVar(sub), ty::ReVar(sup)) = (*sub, *sup) {
                 self.unification_table.union(sub, sup);
+                self.any_unifications = true;
             }
         }
     }

src/librustc_data_structures/indexed_vec.rs

@@ -367,6 +367,11 @@ impl<I: Idx, T> IndexVec<I, T> {
         IndexVec { raw: Vec::new(), _marker: PhantomData }
     }
 
+    #[inline]
+    pub fn from_raw(raw: Vec<T>) -> Self {
+        IndexVec { raw, _marker: PhantomData }
+    }
+
     #[inline]
     pub fn with_capacity(capacity: usize) -> Self {
         IndexVec { raw: Vec::with_capacity(capacity), _marker: PhantomData }

src/librustc_mir/borrow_check/nll/constraint_generation.rs

@@ -11,6 +11,8 @@
 use borrow_check::borrow_set::BorrowSet;
 use borrow_check::location::LocationTable;
 use borrow_check::nll::facts::AllFacts;
+use borrow_check::nll::region_infer::{Cause, RegionInferenceContext};
+use borrow_check::nll::ToRegionVid;
 use rustc::hir;
 use rustc::infer::InferCtxt;
 use rustc::mir::visit::TyContext;
@@ -21,9 +23,7 @@ use rustc::mir::{Local, PlaceProjection, ProjectionElem, Statement, Terminator};
 use rustc::ty::fold::TypeFoldable;
 use rustc::ty::subst::Substs;
 use rustc::ty::{self, CanonicalTy, ClosureSubsts, GeneratorSubsts};
-
-use super::region_infer::{Cause, RegionInferenceContext};
-use super::ToRegionVid;
+use std::iter;
 
 pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
     infcx: &InferCtxt<'cx, 'gcx, 'tcx>,
@@ -32,6 +32,7 @@ pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
     location_table: &LocationTable,
     mir: &Mir<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
+    liveness_set_from_typeck: &[(ty::Region<'tcx>, Location, Cause)],
 ) {
     let mut cg = ConstraintGeneration {
         borrow_set,
@@ -42,6 +43,8 @@ pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
         mir,
     };
 
+    cg.add_region_liveness_constraints_from_type_check(liveness_set_from_typeck);
+
     for (bb, data) in mir.basic_blocks().iter_enumerated() {
         cg.visit_basic_block_data(bb, data);
     }
@@ -209,7 +212,7 @@ impl<'cg, 'cx, 'gcx, 'tcx> Visitor<'tcx> for ConstraintGeneration<'cg, 'cx, 'gcx
                 self.add_reborrow_constraint(location, region, borrowed_place);
             }
 
-            _ => { }
+            _ => {}
         }
 
         self.super_rvalue(rvalue, location);
@@ -225,6 +228,42 @@ impl<'cg, 'cx, 'gcx, 'tcx> Visitor<'tcx> for ConstraintGeneration<'cg, 'cx, 'gcx
 }
 
 impl<'cx, 'cg, 'gcx, 'tcx> ConstraintGeneration<'cx, 'cg, 'gcx, 'tcx> {
+    /// The MIR type checker generates region liveness constraints
+    /// that we also have to respect.
+    fn add_region_liveness_constraints_from_type_check(
+        &mut self,
+        liveness_set: &[(ty::Region<'tcx>, Location, Cause)],
+    ) {
+        debug!(
+            "add_region_liveness_constraints_from_type_check(liveness_set={} items)",
+            liveness_set.len(),
+        );
+
+        let ConstraintGeneration {
+            regioncx,
+            location_table,
+            all_facts,
+            ..
+        } = self;
+
+        for (region, location, cause) in liveness_set {
+            debug!("generate: {:#?} is live at {:#?}", region, location);
+            let region_vid = regioncx.to_region_vid(region);
+            regioncx.add_live_point(region_vid, *location, &cause);
+        }
+
+        if let Some(all_facts) = all_facts {
+            all_facts
+                .region_live_at
+                .extend(liveness_set.into_iter().flat_map(|(region, location, _)| {
+                    let r = regioncx.to_region_vid(region);
+                    let p1 = location_table.start_index(*location);
+                    let p2 = location_table.mid_index(*location);
+                    iter::once((r, p1)).chain(iter::once((r, p2)))
+                }));
+        }
+    }
+
     /// Some variable with type `live_ty` is "regular live" at
     /// `location` -- i.e., it may be used later. This means that all
     /// regions appearing in the type `live_ty` must be live at

src/librustc_mir/borrow_check/nll/mod.rs

@@ -11,6 +11,7 @@
 use borrow_check::borrow_set::BorrowSet;
 use borrow_check::location::{LocationIndex, LocationTable};
 use borrow_check::nll::facts::AllFactsExt;
+use borrow_check::nll::type_check::MirTypeckRegionConstraints;
 use dataflow::indexes::BorrowIndex;
 use dataflow::move_paths::MoveData;
 use dataflow::FlowAtLocation;
@@ -41,7 +42,6 @@ mod facts;
 mod invalidation;
 crate mod region_infer;
 mod renumber;
-mod subtype_constraint_generation;
 crate mod type_check;
 mod universal_regions;
 
@@ -91,53 +91,61 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
     Option<Rc<Output<RegionVid, BorrowIndex, LocationIndex>>>,
     Option<ClosureRegionRequirements<'gcx>>,
 ) {
+    let mut all_facts = if infcx.tcx.sess.opts.debugging_opts.nll_facts
+        || infcx.tcx.sess.opts.debugging_opts.polonius
+    {
+        Some(AllFacts::default())
+    } else {
+        None
+    };
+
     // Run the MIR type-checker.
     let liveness = &LivenessResults::compute(mir);
-    let constraint_sets = &type_check::type_check(
+    let constraint_sets = type_check::type_check(
         infcx,
         param_env,
         mir,
         def_id,
         &universal_regions,
+        location_table,
         &liveness,
+        &mut all_facts,
         flow_inits,
         move_data,
     );
 
-    let mut all_facts = if infcx.tcx.sess.opts.debugging_opts.nll_facts
-        || infcx.tcx.sess.opts.debugging_opts.polonius
-    {
-        Some(AllFacts::default())
-    } else {
-        None
-    };
-
     if let Some(all_facts) = &mut all_facts {
         all_facts
             .universal_region
             .extend(universal_regions.universal_regions());
     }
 
-    // Create the region inference context, taking ownership of the region inference
-    // data that was contained in `infcx`.
+    // Create the region inference context, taking ownership of the
+    // region inference data that was contained in `infcx`, and the
+    // base constraints generated by the type-check.
     let var_origins = infcx.take_region_var_origins();
-    let mut regioncx = RegionInferenceContext::new(var_origins, universal_regions, mir);
-
-    // Generate various constraints.
-    subtype_constraint_generation::generate(
-        &mut regioncx,
-        &mut all_facts,
-        location_table,
+    let MirTypeckRegionConstraints {
+        liveness_set,
+        outlives_constraints,
+        type_tests,
+    } = constraint_sets;
+    let mut regioncx = RegionInferenceContext::new(
+        var_origins,
+        universal_regions,
         mir,
-        constraint_sets,
+        outlives_constraints,
+        type_tests,
     );
+
+    // Generate various additional constraints.
     constraint_generation::generate_constraints(
         infcx,
         &mut regioncx,
         &mut all_facts,
         location_table,
         &mir,
         borrow_set,
+        &liveness_set,
     );
     invalidation::generate_invalidates(
         infcx,

src/librustc_mir/borrow_check/nll/region_infer/dump_mir.rs

@@ -14,7 +14,7 @@
 //! context internal state.
 
 use std::io::{self, Write};
-use super::{Constraint, RegionInferenceContext};
+use super::{OutlivesConstraint, RegionInferenceContext};
 
 // Room for "'_#NNNNr" before things get misaligned.
 // Easy enough to fix if this ever doesn't seem like
@@ -79,7 +79,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         let mut constraints: Vec<_> = self.constraints.iter().collect();
         constraints.sort();
         for constraint in &constraints {
-            let Constraint {
+            let OutlivesConstraint {
                 sup,
                 sub,
                 point,

src/librustc_mir/borrow_check/nll/region_infer/graphviz.rs

@@ -27,7 +27,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
 
 impl<'this, 'tcx> dot::Labeller<'this> for RegionInferenceContext<'tcx> {
     type Node = RegionVid;
-    type Edge = Constraint;
+    type Edge = OutlivesConstraint;
 
     fn graph_id(&'this self) -> dot::Id<'this> {
         dot::Id::new(format!("RegionInferenceContext")).unwrap()
@@ -41,31 +41,31 @@ impl<'this, 'tcx> dot::Labeller<'this> for RegionInferenceContext<'tcx> {
     fn node_label(&'this self, n: &RegionVid) -> dot::LabelText<'this> {
         dot::LabelText::LabelStr(format!("{:?}", n).into_cow())
     }
-    fn edge_label(&'this self, e: &Constraint) -> dot::LabelText<'this> {
+    fn edge_label(&'this self, e: &OutlivesConstraint) -> dot::LabelText<'this> {
         dot::LabelText::LabelStr(format!("{:?}", e.point).into_cow())
     }
 }
 
 impl<'this, 'tcx> dot::GraphWalk<'this> for RegionInferenceContext<'tcx> {
     type Node = RegionVid;
-    type Edge = Constraint;
+    type Edge = OutlivesConstraint;
 
     fn nodes(&'this self) -> dot::Nodes<'this, RegionVid> {
         let vids: Vec<RegionVid> = self.definitions.indices().collect();
         vids.into_cow()
     }
-    fn edges(&'this self) -> dot::Edges<'this, Constraint> {
+    fn edges(&'this self) -> dot::Edges<'this, OutlivesConstraint> {
         (&self.constraints.raw[..]).into_cow()
     }
 
     // Render `a: b` as `a <- b`, indicating the flow
     // of data during inference.
 
-    fn source(&'this self, edge: &Constraint) -> RegionVid {
+    fn source(&'this self, edge: &OutlivesConstraint) -> RegionVid {
         edge.sub
     }
 
-    fn target(&'this self, edge: &Constraint) -> RegionVid {
+    fn target(&'this self, edge: &OutlivesConstraint) -> RegionVid {
         edge.sup
     }
 }