Make lowering incremental.

Author: cjgillot

Cargo.lock

@@ -3903,6 +3903,7 @@ dependencies = [
  "rustc_expand",
  "rustc_hir",
  "rustc_incremental",
+ "rustc_index",
  "rustc_lint",
  "rustc_metadata",
  "rustc_middle",

compiler/rustc_ast_lowering/src/item.rs

@@ -7,13 +7,11 @@ use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
-use rustc_data_structures::steal::Steal;
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
-use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::ty::{AstOwner, ResolverOutputs, TyCtxt};
+use rustc_hir::def_id::CRATE_DEF_ID;
+use rustc_middle::ty::{DefIdTree, ResolverOutputs, TyCtxt};
 use rustc_session::utils::NtToTokenstream;
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident};
@@ -24,12 +22,10 @@ use tracing::debug;
 
 use std::iter;
 
-pub(super) struct ItemLowerer<'a, 'hir> {
+pub(super) struct ItemLowerer<'hir> {
     pub(super) tcx: TyCtxt<'hir>,
     pub(super) resolver: &'hir ResolverOutputs,
     pub(super) nt_to_tokenstream: NtToTokenstream,
-    pub(super) ast_index: &'a IndexVec<LocalDefId, Steal<AstOwner>>,
-    pub(super) owners: &'a mut IndexVec<LocalDefId, hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>>>,
 }
 
 /// When we have a ty alias we *may* have two where clauses. To give the best diagnostics, we set the span
@@ -52,13 +48,13 @@ fn add_ty_alias_where_clause(
     }
 }
 
-impl<'a, 'hir> ItemLowerer<'a, 'hir> {
+impl<'hir> ItemLowerer<'hir> {
     fn with_lctx(
         &mut self,
         owner: NodeId,
         f: impl FnOnce(&mut LoweringContext<'hir>) -> hir::OwnerNode<'hir>,
-    ) {
-        let next_node_id = self.resolver.next_node_id;
+    ) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
+        let current_hir_id_owner = self.resolver.node_id_to_def_id[&owner];
         let mut lctx = LoweringContext {
             // Pseudo-globals.
             tcx: self.tcx,
@@ -71,13 +67,15 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
             bodies: Vec::new(),
             attrs: SortedMap::default(),
             children: FxHashMap::default(),
-            current_hir_id_owner: CRATE_DEF_ID,
-            item_local_id_counter: hir::ItemLocalId::new(0),
-            node_id_to_local_id: Default::default(),
             local_id_to_def_id: SortedMap::new(),
             trait_map: Default::default(),
             local_node_id_to_def_id: FxHashMap::default(),
-            next_node_id,
+            next_node_id: self.resolver.next_node_id,
+
+            // HirId management.
+            current_hir_id_owner,
+            node_id_to_local_id: [(owner, hir::ItemLocalId::from_u32(0))].into_iter().collect(),
+            item_local_id_counter: hir::ItemLocalId::from_u32(1), // 0 is `current_hir_id_owner`.
 
             // Lowering state.
             catch_scope: None,
@@ -93,35 +91,15 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
             allow_gen_future: Some([sym::gen_future][..].into()),
             allow_into_future: Some([sym::into_future][..].into()),
         };
-        lctx.with_hir_id_owner(owner, |lctx| f(lctx));
 
-        for (def_id, info) in lctx.children {
-            self.owners.ensure_contains_elem(def_id, || hir::MaybeOwner::Phantom);
-            debug_assert!(matches!(self.owners[def_id], hir::MaybeOwner::Phantom));
-            self.owners[def_id] = info;
-        }
-    }
+        let item = f(&mut lctx);
+        debug_assert_eq!(current_hir_id_owner, item.def_id());
 
-    pub(super) fn lower_node(
-        &mut self,
-        def_id: LocalDefId,
-    ) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
-        self.owners.ensure_contains_elem(def_id, || hir::MaybeOwner::Phantom);
-        if let hir::MaybeOwner::Phantom = self.owners[def_id] {
-            let node = self.ast_index[def_id].steal();
-            match node {
-                AstOwner::NonOwner | AstOwner::Synthetic(..) => {}
-                AstOwner::Crate(c) => self.lower_crate(&c),
-                AstOwner::Item(item) => self.lower_item(&item),
-                AstOwner::AssocItem(item, ctxt) => self.lower_assoc_item(&item, ctxt),
-                AstOwner::ForeignItem(item) => self.lower_foreign_item(&item),
-            }
-        }
-
-        self.owners[def_id]
+        let info = lctx.make_owner_info(item);
+        hir::MaybeOwner::Owner(self.tcx.hir_arena.alloc(info))
     }
 
-    fn lower_crate(&mut self, c: &Crate) {
+    pub(super) fn lower_crate(&mut self, c: &Crate) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
         debug_assert_eq!(self.resolver.node_id_to_def_id[&CRATE_NODE_ID], CRATE_DEF_ID);
         self.with_lctx(CRATE_NODE_ID, |lctx| {
             let module = lctx.lower_mod(&c.items, &c.spans);
@@ -130,25 +108,28 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
         })
     }
 
-    fn lower_item(&mut self, item: &Item) {
+    pub(super) fn lower_item(
+        &mut self,
+        item: &Item,
+    ) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
     }
 
-    fn lower_assoc_item(&mut self, item: &AssocItem, ctxt: AssocCtxt) {
+    pub(super) fn lower_assoc_item(
+        &mut self,
+        item: &AssocItem,
+        ctxt: AssocCtxt,
+    ) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
         let def_id = self.resolver.node_id_to_def_id[&item.id];
 
-        let parent_id = {
-            let parent = self.tcx.hir().def_key(def_id).parent;
-            let local_def_index = parent.unwrap();
-            LocalDefId { local_def_index }
-        };
+        let parent_id = self.tcx.local_parent(def_id).unwrap();
 
-        let parent_hir = self.lower_node(parent_id).unwrap();
+        let parent_hir = self.tcx.hir().expect_item(parent_id);
         self.with_lctx(item.id, |lctx| {
             // Evaluate with the lifetimes in `params` in-scope.
             // This is used to track which lifetimes have already been defined,
             // and which need to be replicated when lowering an async fn.
-            match parent_hir.node().expect_item().kind {
+            match parent_hir.kind {
                 hir::ItemKind::Impl(hir::Impl { ref of_trait, .. }) => {
                     lctx.is_in_trait_impl = of_trait.is_some();
                 }
@@ -162,7 +143,10 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
         })
     }
 
-    fn lower_foreign_item(&mut self, item: &ForeignItem) {
+    pub(super) fn lower_foreign_item(
+        &mut self,
+        item: &ForeignItem,
+    ) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item)))
     }
 }
@@ -545,7 +529,7 @@ impl<'hir> LoweringContext<'hir> {
                         let path = this.lower_path_extra(res, &path, ParamMode::Explicit);
                         let kind = hir::ItemKind::Use(path, hir::UseKind::Single);
                         if let Some(attrs) = attrs {
-                            this.attrs.insert(hir::ItemLocalId::new(0), attrs);
+                            this.attrs.insert(hir::ItemLocalId::from_u32(0), attrs);
                         }
 
                         let item = hir::Item {
@@ -619,7 +603,7 @@ impl<'hir> LoweringContext<'hir> {
                         let kind =
                             this.lower_use_tree(use_tree, &prefix, id, vis_span, &mut ident, attrs);
                         if let Some(attrs) = attrs {
-                            this.attrs.insert(hir::ItemLocalId::new(0), attrs);
+                            this.attrs.insert(hir::ItemLocalId::from_u32(0), attrs);
                         }
 
                         let item = hir::Item {

compiler/rustc_ast_lowering/src/lib.rs

@@ -58,7 +58,7 @@ use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir::{ConstArg, GenericArg, ItemLocalId, ParamName, TraitCandidate};
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::ty::{AstOwner, ResolverOutputs, TyCtxt};
+use rustc_middle::ty::{AstOwner, DefIdTree, ResolverOutputs, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::utils::{FlattenNonterminals, NtToTokenstream};
 use rustc_session::Session;
@@ -422,56 +422,46 @@ pub fn index_crate(
     }
 }
 
-/// Compute the hash for the HIR of the full crate.
-/// This hash will then be part of the crate_hash which is stored in the metadata.
-fn compute_hir_hash(
-    tcx: TyCtxt<'_>,
-    owners: &IndexVec<LocalDefId, hir::MaybeOwner<&hir::OwnerInfo<'_>>>,
-) -> Fingerprint {
-    let mut hir_body_nodes: Vec<_> = owners
-        .iter_enumerated()
-        .filter_map(|(def_id, info)| {
-            let info = info.as_owner()?;
-            let def_path_hash = tcx.hir().def_path_hash(def_id);
-            Some((def_path_hash, info))
-        })
-        .collect();
-    hir_body_nodes.sort_unstable_by_key(|bn| bn.0);
-
-    tcx.with_stable_hashing_context(|mut hcx| {
-        let mut stable_hasher = StableHasher::new();
-        hir_body_nodes.hash_stable(&mut hcx, &mut stable_hasher);
-        stable_hasher.finish()
-    })
-}
-
-pub fn lower_crate<'hir>(
+pub fn lower_to_hir<'hir>(
     tcx: TyCtxt<'hir>,
+    def_id: LocalDefId,
     nt_to_tokenstream: NtToTokenstream,
-) -> hir::Crate<'hir> {
+) -> hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>> {
     let _prof_timer = tcx.sess.prof.verbose_generic_activity("hir_lowering");
 
     let resolver = tcx.resolutions(());
-    let ast_index = &tcx.untracked_crate;
+    let node = tcx.untracked_crate.get(def_id);
 
-    let mut owners = IndexVec::from_fn_n(
-        |_| hir::MaybeOwner::Phantom,
-        tcx.definitions_untracked().def_index_count(),
-    );
+    let mut item_lowerer = item::ItemLowerer { tcx, resolver, nt_to_tokenstream };
 
-    for def_id in ast_index.indices() {
-        item::ItemLowerer {
-            tcx,
-            resolver,
-            nt_to_tokenstream,
-            ast_index: ast_index,
-            owners: &mut owners,
-        }
-        .lower_node(def_id);
-    }
+    let node = node.map(Steal::steal);
 
-    let hir_hash = compute_hir_hash(tcx, &owners);
-    hir::Crate { owners, hir_hash }
+    // The item existed in the AST.
+    let parent_id = match node {
+        Some(AstOwner::Crate(c)) => return item_lowerer.lower_crate(&c),
+        Some(AstOwner::Item(item)) => return item_lowerer.lower_item(&item),
+        Some(AstOwner::AssocItem(item, ctxt)) => {
+            return item_lowerer.lower_assoc_item(&item, ctxt);
+        }
+        Some(AstOwner::ForeignItem(item)) => return item_lowerer.lower_foreign_item(&item),
+        Some(AstOwner::Synthetic(parent_id)) => parent_id,
+        Some(AstOwner::NonOwner) | None => tcx.local_parent(def_id).unwrap(),
+    };
+
+    // The item did not exist in the AST, it was created by its parent.
+    let mut parent_info = tcx.lower_to_hir(parent_id);
+    if let hir::MaybeOwner::NonOwner(hir_id) = parent_info {
+        parent_info = tcx.lower_to_hir(hir_id.owner);
+    }
+
+    let parent_info = parent_info.unwrap();
+    *parent_info.children.get(&def_id).unwrap_or_else(|| {
+        panic!(
+            "{:?} does not appear in children of {:?}",
+            def_id,
+            parent_info.nodes.node().def_id()
+        )
+    })
 }
 
 #[derive(Copy, Clone, PartialEq)]
@@ -552,6 +542,7 @@ impl<'hir> LoweringContext<'hir> {
         let current_owner = std::mem::replace(&mut self.current_hir_id_owner, def_id);
         let current_local_counter =
             std::mem::replace(&mut self.item_local_id_counter, hir::ItemLocalId::new(1));
+        let current_children = std::mem::take(&mut self.children);
         // Do not reset `next_node_id` and `node_id_to_def_id` as we want to refer to the
         // subdefinitions' nodes.
 
@@ -570,6 +561,8 @@ impl<'hir> LoweringContext<'hir> {
         self.trait_map = current_trait_map;
         self.current_hir_id_owner = current_owner;
         self.item_local_id_counter = current_local_counter;
+        self.children = current_children;
+        self.children.extend(&info.children);
 
         let _old = self.children.insert(def_id, hir::MaybeOwner::Owner(info));
         debug_assert!(_old.is_none())
@@ -580,6 +573,7 @@ impl<'hir> LoweringContext<'hir> {
         let mut bodies = std::mem::take(&mut self.bodies);
         let local_id_to_def_id = std::mem::take(&mut self.local_id_to_def_id);
         let trait_map = std::mem::take(&mut self.trait_map);
+        let children = std::mem::take(&mut self.children);
 
         #[cfg(debug_assertions)]
         for (id, attrs) in attrs.iter() {
@@ -610,7 +604,7 @@ impl<'hir> LoweringContext<'hir> {
             hir::AttributeMap { map: attrs, hash }
         };
 
-        self.arena.alloc(hir::OwnerInfo { nodes, parenting, attrs, trait_map })
+        self.tcx.hir_arena.alloc(hir::OwnerInfo { nodes, parenting, attrs, trait_map, children })
     }
 
     /// Hash the HIR node twice, one deep and one shallow hash.  This allows to differentiate

compiler/rustc_hir/src/hir.rs

@@ -756,6 +756,8 @@ pub struct OwnerInfo<'hir> {
     /// Map indicating what traits are in scope for places where this
     /// is relevant; generated by resolve.
     pub trait_map: FxHashMap<ItemLocalId, &'hir [TraitCandidate]>,
+    /// Owners generated as side-effect by lowering.
+    pub children: FxHashMap<LocalDefId, MaybeOwner<&'hir OwnerInfo<'hir>>>,
 }
 
 impl<'tcx> OwnerInfo<'tcx> {
@@ -769,30 +771,27 @@ impl<'tcx> OwnerInfo<'tcx> {
 pub enum MaybeOwner<T> {
     Owner(T),
     NonOwner(HirId),
-    /// Used as a placeholder for unused LocalDefId.
-    Phantom,
 }
 
 impl<T> MaybeOwner<T> {
     pub fn as_owner(self) -> Option<T> {
         match self {
             MaybeOwner::Owner(i) => Some(i),
-            MaybeOwner::NonOwner(_) | MaybeOwner::Phantom => None,
+            MaybeOwner::NonOwner(_) => None,
         }
     }
 
     pub fn map<U>(self, f: impl FnOnce(T) -> U) -> MaybeOwner<U> {
         match self {
             MaybeOwner::Owner(i) => MaybeOwner::Owner(f(i)),
             MaybeOwner::NonOwner(hir_id) => MaybeOwner::NonOwner(hir_id),
-            MaybeOwner::Phantom => MaybeOwner::Phantom,
         }
     }
 
     pub fn unwrap(self) -> T {
         match self {
             MaybeOwner::Owner(i) => i,
-            MaybeOwner::NonOwner(_) | MaybeOwner::Phantom => panic!("Not a HIR owner"),
+            MaybeOwner::NonOwner(_) => panic!("Not a HIR owner"),
         }
     }
 }

compiler/rustc_interface/Cargo.toml

@@ -32,6 +32,7 @@ rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_codegen_llvm = { path = "../rustc_codegen_llvm", optional = true }
 rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_const_eval = { path = "../rustc_const_eval" }
 rustc_mir_build = { path = "../rustc_mir_build" }