Turn macro expansion and name resolution into a query

Author: Zoxc

src/librustc/hir/def_id.rs

@@ -151,7 +151,10 @@ impl fmt::Debug for DefId {
 
         ty::tls::with_opt(|opt_tcx| {
             if let Some(tcx) = opt_tcx {
-                write!(f, " ~ {}", tcx.def_path_debug_str(*self))?;
+                // Only print the path after HIR lowering is done
+                if tcx.is_hir_lowered() {
+                    write!(f, " ~ {}", tcx.def_path_debug_str(*self))?;
+                }
             }
             Ok(())
         })?;

src/librustc/query/mod.rs

@@ -31,6 +31,12 @@ use syntax_pos::symbol::InternedString;
 // as they will raise an fatal error on query cycles instead.
 rustc_queries! {
     Other {
+        query expand_macros(_: ()) -> Result<Lrc<ty::ExpansionResult>, ErrorReported> {
+            no_hash
+            eval_always
+            desc { "expanding macros" }
+        }
+
         query prepare_outputs(_: ()) -> Result<Arc<OutputFilenames>, ErrorReported> {
             no_hash
             eval_always

src/librustc/ty/context.rs

@@ -53,7 +53,7 @@ use rustc_data_structures::stable_hasher::{HashStable, hash_stable_hashmap,
                                            StableVec};
 use arena::SyncDroplessArena;
 use rustc_data_structures::indexed_vec::{Idx, IndexVec};
-use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal, AtomicOnce, OneThread};
+use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal, AtomicOnce, Once, OneThread};
 use std::any::Any;
 use std::borrow::Borrow;
 use std::cmp::Ordering;
@@ -1009,15 +1009,20 @@ pub struct GlobalCtxt<'tcx> {
 
     pub io: InputsAndOutputs,
 
-    pub ast_crate: Steal<ast::Crate>,
+    /// This stores a `Lrc<CStore>`, but that type depends on
+    /// rustc_metadata, so it cannot be used here.
+    pub cstore_rc: OneThread<Steal<Box<dyn Any>>>,
 
-    /// This stores a `Lrc<Option<Lock<BoxedResolver>>>)>`, but that type depends on
-    /// librustc_resolve, so it cannot be used here.
-    pub boxed_resolver: Steal<OneThread<Box<dyn Any>>>,
+    pub sess_rc: Lrc<Session>,
+
+    /// The AST after registering plugins.
+    pub ast_crate: Steal<(ast::Crate, ty::PluginInfo)>,
 
     lowered_hir: AtomicOnce<&'tcx hir::LoweredHir>,
     hir_map: AtomicOnce<&'tcx hir_map::Map<'tcx>>,
 
+    metadata_dep_nodes: Once<()>,
+
     pub queries: query::Queries<'tcx>,
 
     // Internal cache for metadata decoding. No need to track deps on this.
@@ -1075,10 +1080,19 @@ impl<'tcx> TyCtxt<'tcx> {
         self.lowered_hir.get_or_init(|| {
             // FIXME: The ignore here is only sound because all queries
             // used to compute LoweredHir are eval_always
-            self.dep_graph.with_ignore(|| self.lower_ast_to_hir(()).unwrap())
+            self.dep_graph.with_ignore(|| {
+                let map = self.lower_ast_to_hir(()).unwrap();
+                self.lowered_hir.init(map);
+                self.allocate_metadata_dep_nodes();
+                map
+            })
         })
     }
 
+    pub fn is_hir_lowered(self) -> bool {
+        self.lowered_hir.is_initalized()
+    }
+
     #[inline(always)]
     pub fn hir(self) -> &'tcx hir_map::Map<'tcx> {
         self.hir_map.get_or_init(|| {
@@ -1163,14 +1177,15 @@ impl<'tcx> TyCtxt<'tcx> {
     /// value (types, substs, etc.) can only be used while `ty::tls` has a valid
     /// reference to the context, to allow formatting values that need it.
     pub fn create_global_ctxt(
-        s: &'tcx Session,
+        s: &'tcx Lrc<Session>,
         cstore: &'tcx CrateStoreDyn,
+        cstore_rc: Box<dyn Any>,
         local_providers: ty::query::Providers<'tcx>,
         extern_providers: ty::query::Providers<'tcx>,
         arenas: &'tcx AllArenas,
         dep_graph: DepGraph,
         ast_crate: ast::Crate,
-        boxed_resolver: Box<dyn Any>,
+        plugin_info: ty::PluginInfo,
         on_disk_query_result_cache: query::OnDiskCache<'tcx>,
         crate_name: &str,
         tx: mpsc::Sender<Box<dyn Any + Send>>,
@@ -1194,19 +1209,21 @@ impl<'tcx> TyCtxt<'tcx> {
         providers[LOCAL_CRATE] = local_providers;
 
         GlobalCtxt {
-            sess: s,
-            cstore,
+            sess: &**s,
             arena: WorkerLocal::new(|_| Arena::default()),
+            cstore,
+            cstore_rc: OneThread::new(Steal::new(cstore_rc)),
+            sess_rc: s.clone(),
             interners,
             dep_graph,
             common,
             types: common_types,
             lifetimes: common_lifetimes,
             consts: common_consts,
-            ast_crate: Steal::new(ast_crate),
-            boxed_resolver: Steal::new(OneThread::new(boxed_resolver)),
+            ast_crate: Steal::new((ast_crate, plugin_info)),
             lowered_hir: AtomicOnce::new(),
             hir_map: AtomicOnce::new(),
+            metadata_dep_nodes: Once::new(),
             queries: query::Queries::new(
                 providers,
                 extern_providers,
@@ -1381,18 +1398,24 @@ impl<'tcx> TyCtxt<'tcx> {
     // With full-fledged red/green, the method will probably become unnecessary
     // as this will be done on-demand.
     pub fn allocate_metadata_dep_nodes(self) {
-        // We cannot use the query versions of crates() and crate_hash(), since
-        // those would need the DepNodes that we are allocating here.
-        for cnum in self.cstore.crates_untracked() {
-            let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum));
-            let crate_hash = self.cstore.crate_hash_untracked(cnum);
-            self.dep_graph.with_task(dep_node,
-                                     self,
-                                     crate_hash,
-                                     |_, x| x, // No transformation needed
-                                     Some(dep_graph::hash_result),
-            );
+        if !self.dep_graph.is_fully_enabled() {
+            return
         }
+
+        self.metadata_dep_nodes.init_locking(|| {
+            // We cannot use the query versions of crates() and crate_hash(), since
+            // those would need the DepNodes that we are allocating here.
+            for cnum in self.cstore.crates_untracked() {
+                let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum));
+                let crate_hash = self.cstore.crate_hash_untracked(cnum);
+                self.dep_graph.with_task(dep_node,
+                                        self,
+                                        crate_hash,
+                                        |_, x| x, // No transformation needed
+                                        Some(dep_graph::hash_result),
+                );
+            }
+        });
     }
 
     pub fn serialize_query_result_cache<E>(self,

src/librustc/ty/mod.rs

@@ -38,13 +38,16 @@ use std::cmp::{self, Ordering};
 use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::ops::Deref;
-use rustc_data_structures::sync::{self, Lrc, ParallelIterator, par_iter};
+use std::any::Any;
+use rustc_data_structures::sync::{self, Lrc, OneThread, ParallelIterator, par_iter};
 use std::slice;
 use std::{mem, ptr};
 use std::ops::Range;
 use syntax::ast::{self, Name, Ident, NodeId};
 use syntax::attr;
 use syntax::ext::hygiene::Mark;
+use syntax::ext::base::NamedSyntaxExtension;
+use syntax::feature_gate::AttributeType;
 use syntax::symbol::{kw, sym, Symbol, LocalInternedString, InternedString};
 use syntax_pos::Span;
 
@@ -119,6 +122,19 @@ mod sty;
 
 // Data types
 
+pub struct PluginInfo {
+    pub syntax_exts: Vec<NamedSyntaxExtension>,
+    pub attributes: Vec<(Symbol, AttributeType)>,
+}
+
+pub struct ExpansionResult {
+    pub ast_crate: steal::Steal<ast::Crate>,
+
+    /// This stores a `Lrc<Option<Lock<BoxedResolver>>>)>`, but that type depends on
+    /// librustc_resolve, so it cannot be used here.
+    pub boxed_resolver: steal::Steal<OneThread<Box<dyn Any>>>,
+}
+
 #[derive(Clone)]
 pub struct Resolutions {
     pub trait_map: TraitMap,

src/librustc_data_structures/sync.rs

@@ -600,6 +600,16 @@ impl<T: Copy> AtomicOnce<T> {
             value.unwrap()
         }
     }
+
+    #[inline]
+    pub fn init(&self, value: T) {
+        self.0.store(Some(value));
+    }
+
+    #[inline]
+    pub fn is_initalized(&self) -> bool {
+        self.0.load().is_some()
+    }
 }
 
 #[derive(Debug)]

src/librustc_driver/lib.rs

@@ -268,10 +268,11 @@ pub fn run_compiler(
             if ppm.needs_ast_map(&opt_uii) {
                 pretty::visit_crate(sess, &mut compiler.parse()?.peek_mut(), ppm);
                 compiler.global_ctxt()?.peek_mut().enter(|tcx| {
+                    let expansion_result = tcx.expand_macros(())?;
                     pretty::print_after_hir_lowering(
                         tcx,
                         compiler.input(),
-                        &tcx.ast_crate.borrow(),
+                        &expansion_result.ast_crate.borrow(),
                         ppm,
                         opt_uii.clone(),
                         compiler.output_file().as_ref().map(|p| &**p),
@@ -334,13 +335,14 @@ pub fn run_compiler(
 
         if sess.opts.debugging_opts.save_analysis {
             compiler.global_ctxt()?.peek_mut().enter(|tcx| {
+                let expansion_result = tcx.expand_macros(())?;
                 let result = tcx.analysis(LOCAL_CRATE);
                 let crate_name = &tcx.crate_name.as_str();
 
                 time(sess, "save analysis", || {
                     save::process_crate(
                         tcx,
-                        &tcx.ast_crate.borrow(),
+                        &expansion_result.ast_crate.borrow(),
                         crate_name,
                         &compiler.input(),
                         None,
@@ -355,7 +357,7 @@ pub fn run_compiler(
         } else {
             compiler.global_ctxt()?.peek_mut().enter(|tcx| {
                 // Drop AST after lowering HIR to free memory
-                mem::drop(tcx.ast_crate.steal());
+                mem::drop(tcx.expand_macros(()).unwrap().ast_crate.steal());
             });
         }
 
@@ -368,7 +370,7 @@ pub fn run_compiler(
         if sess.opts.debugging_opts.save_analysis {
             compiler.global_ctxt()?.peek_mut().enter(|tcx| {
                 // Drop AST after lowering HIR to free memory
-                mem::drop(tcx.ast_crate.steal());
+                mem::drop(tcx.expand_macros(()).unwrap().ast_crate.steal());
             });
         }
 

src/librustc_incremental/lib.rs

@@ -22,7 +22,6 @@ pub mod assert_module_sources;
 mod persist;
 
 pub use assert_dep_graph::assert_dep_graph;
-pub use persist::dep_graph_tcx_init;
 pub use persist::{DepGraphFuture, load_dep_graph};
 pub use persist::load_query_result_cache;
 pub use persist::LoadResult;

src/librustc_incremental/persist/load.rs

@@ -3,7 +3,6 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc::session::Session;
-use rustc::ty::TyCtxt;
 use rustc::ty::query::OnDiskCache;
 use rustc::util::common::time_ext;
 use rustc_serialize::Decodable as RustcDecodable;
@@ -15,14 +14,6 @@ use super::fs::*;
 use super::file_format;
 use super::work_product;
 
-pub fn dep_graph_tcx_init(tcx: TyCtxt<'_>) {
-    if !tcx.dep_graph.is_fully_enabled() {
-        return
-    }
-
-    tcx.allocate_metadata_dep_nodes();
-}
-
 type WorkProductMap = FxHashMap<WorkProductId, WorkProduct>;
 
 pub enum LoadResult<T> {

src/librustc_incremental/persist/mod.rs

@@ -15,7 +15,6 @@ pub use fs::garbage_collect_session_directories;
 pub use fs::in_incr_comp_dir;
 pub use fs::in_incr_comp_dir_sess;
 pub use fs::prepare_session_directory;
-pub use load::dep_graph_tcx_init;
 pub use load::{DepGraphFuture, load_dep_graph};
 pub use load::load_query_result_cache;
 pub use load::LoadResult;