Hygiene serialization implementation

Author: Aaron1011

src/librustc_ast_lowering/expr.rs

@@ -9,7 +9,8 @@ use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::Res;
-use rustc_span::source_map::{respan, DesugaringKind, ForLoopLoc, Span, Spanned};
+use rustc_span::hygiene::ForLoopLoc;
+use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_target::asm;
 use std::collections::hash_map::Entry;

src/librustc_expand/base.rs

@@ -13,7 +13,7 @@ use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, ErrorReported};
 use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
 use rustc_session::{parse::ParseSess, Limit};
-use rustc_span::def_id::DefId;
+use rustc_span::def_id::{DefId, LOCAL_CRATE};
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind};
 use rustc_span::source_map::SourceMap;
@@ -873,6 +873,8 @@ impl SyntaxExtension {
             local_inner_macros: self.local_inner_macros,
             edition: self.edition,
             macro_def_id,
+            krate: LOCAL_CRATE,
+            orig_id: None,
         }
     }
 }

src/librustc_expand/expand.rs

@@ -1,7 +1,7 @@
 use crate::base::*;
 use crate::config::StripUnconfigured;
 use crate::configure;
-use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext};
+use crate::hygiene::{ExpnData, ExpnKind, SyntaxContext};
 use crate::mbe::macro_rules::annotate_err_with_kind;
 use crate::module::{parse_external_mod, push_directory, Directory, DirectoryOwnership};
 use crate::placeholders::{placeholder, PlaceholderExpander};
@@ -27,7 +27,7 @@ use rustc_session::parse::{feature_err, ParseSess};
 use rustc_session::Limit;
 use rustc_span::source_map::respan;
 use rustc_span::symbol::{sym, Ident, Symbol};
-use rustc_span::{FileName, Span, DUMMY_SP};
+use rustc_span::{ExpnId, FileName, Span, DUMMY_SP};
 
 use smallvec::{smallvec, SmallVec};
 use std::io::ErrorKind;

src/librustc_metadata/creader.rs

@@ -307,11 +307,16 @@ impl<'a> CrateLoader<'a> {
         let private_dep =
             self.sess.opts.externs.get(&name.as_str()).map(|e| e.is_private_dep).unwrap_or(false);
 
-        info!("register crate `{}` (private_dep = {})", crate_root.name(), private_dep);
-
         // Claim this crate number and cache it
         let cnum = self.cstore.alloc_new_crate_num();
 
+        info!(
+            "register crate `{}` (cnum = {}. private_dep = {})",
+            crate_root.name(),
+            cnum,
+            private_dep
+        );
+
         // Maintain a reference to the top most crate.
         // Stash paths for top-most crate locally if necessary.
         let crate_paths;
@@ -339,22 +344,21 @@ impl<'a> CrateLoader<'a> {
             None
         };
 
-        self.cstore.set_crate_data(
+        let crate_metadata = CrateMetadata::new(
+            self.sess,
+            metadata,
+            crate_root,
+            raw_proc_macros,
             cnum,
-            CrateMetadata::new(
-                self.sess,
-                metadata,
-                crate_root,
-                raw_proc_macros,
-                cnum,
-                cnum_map,
-                dep_kind,
-                source,
-                private_dep,
-                host_hash,
-            ),
+            cnum_map,
+            dep_kind,
+            source,
+            private_dep,
+            host_hash,
         );
 
+        self.cstore.set_crate_data(cnum, crate_metadata);
+
         Ok(cnum)
     }
 
@@ -569,6 +573,8 @@ impl<'a> CrateLoader<'a> {
             let cnum = self.maybe_resolve_crate(dep.name, dep_kind, Some((root, &dep)))?;
             crate_num_map.push(cnum);
         }
+
+        debug!("resolve_crate_deps: cnum_map for {:?} is {:?}", krate, crate_num_map);
         Ok(crate_num_map)
     }
 

src/librustc_metadata/lib.rs

@@ -9,6 +9,7 @@
 #![feature(proc_macro_internals)]
 #![feature(min_specialization)]
 #![feature(stmt_expr_attributes)]
+#![feature(never_type)]
 #![recursion_limit = "256"]
 
 extern crate proc_macro;

src/librustc_metadata/rmeta/decoder.rs

@@ -32,18 +32,21 @@ use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util::common::record_time;
 use rustc_serialize::{opaque, Decodable, Decoder, SpecializedDecoder, UseSpecializedDecodable};
 use rustc_session::Session;
+use rustc_span::hygiene::ExpnDataDecodeMode;
 use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{sym, Ident, Symbol};
-use rustc_span::{self, hygiene::MacroKind, BytePos, Pos, Span, DUMMY_SP};
+use rustc_span::{self, hygiene::MacroKind, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP};
 
 use log::debug;
 use proc_macro::bridge::client::ProcMacro;
+use std::cell::Cell;
 use std::io;
 use std::mem;
 use std::num::NonZeroUsize;
 use std::path::Path;
 
 pub use cstore_impl::{provide, provide_extern};
+use rustc_span::hygiene::HygieneContext;
 
 mod cstore_impl;
 
@@ -106,6 +109,13 @@ crate struct CrateMetadata {
     /// The hash for the host proc macro. Used to support `-Z dual-proc-macro`.
     host_hash: Option<Svh>,
 
+    /// Additional data used for decoding `HygieneData` (e.g. `SyntaxContext`
+    /// and `ExpnId`).
+    /// Note that we store a `HygieneContext` for each `CrateMetadat`. This is
+    /// because `SyntaxContext` ids are not globally unique, so we need
+    /// to track which ids we've decoded on a per-crate basis.
+    hygiene_context: HygieneContext,
+
     // --- Data used only for improving diagnostics ---
     /// Information about the `extern crate` item or path that caused this crate to be loaded.
     /// If this is `None`, then the crate was injected (e.g., by the allocator).
@@ -411,6 +421,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
 
         let lo = BytePos::decode(self)?;
         let len = BytePos::decode(self)?;
+        let ctxt = SyntaxContext::decode(self)?;
         let hi = lo + len;
 
         let sess = if let Some(sess) = self.sess {
@@ -524,7 +535,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
         let hi =
             (hi + source_file.translated_source_file.start_pos) - source_file.original_start_pos;
 
-        Ok(Span::with_root_ctxt(lo, hi))
+        Ok(Span::new(lo, hi, ctxt))
     }
 }
 
@@ -1120,6 +1131,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         !self.is_proc_macro(id) && self.root.tables.mir.get(self, id).is_some()
     }
 
+    fn module_expansion(&self, id: DefIndex, sess: &Session) -> ExpnId {
+        if let EntryKind::Mod(m) = self.kind(id) {
+            m.decode((self, sess)).expansion
+        } else {
+            panic!("Expected module, found {:?}", self.local_def_id(id))
+        }
+    }
+
     fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Body<'tcx> {
         self.root
             .tables
@@ -1652,6 +1671,7 @@ impl CrateMetadata {
             private_dep,
             host_hash,
             extern_crate: Lock::new(None),
+            hygiene_context: HygieneContext::new(),
         }
     }
 
@@ -1784,3 +1804,57 @@ fn macro_kind(raw: &ProcMacro) -> MacroKind {
         ProcMacro::Bang { .. } => MacroKind::Bang,
     }
 }
+
+impl<'a, 'tcx> SpecializedDecoder<SyntaxContext> for DecodeContext<'a, 'tcx> {
+    fn specialized_decode(&mut self) -> Result<SyntaxContext, Self::Error> {
+        let cdata = self.cdata();
+        let sess = self.sess.unwrap();
+        let cname = cdata.root.name;
+        rustc_span::hygiene::decode_syntax_context(self, &cdata.hygiene_context, |_, id| {
+            debug!("SpecializedDecoder<SyntaxContext>: decoding {}", id);
+            Ok(cdata
+                .root
+                .syntax_contexts
+                .get(&cdata, id)
+                .unwrap_or_else(|| panic!("Missing SyntaxContext {:?} for crate {:?}", id, cname))
+                .decode((&cdata, sess)))
+        })
+    }
+}
+
+impl<'a, 'tcx> SpecializedDecoder<ExpnId> for DecodeContext<'a, 'tcx> {
+    fn specialized_decode(&mut self) -> Result<ExpnId, Self::Error> {
+        let local_cdata = self.cdata();
+        let sess = self.sess.unwrap();
+        let expn_cnum = Cell::new(None);
+        let get_ctxt = |cnum| {
+            expn_cnum.set(Some(cnum));
+            if cnum == LOCAL_CRATE {
+                &local_cdata.hygiene_context
+            } else {
+                &local_cdata.cstore.get_crate_data(cnum).cdata.hygiene_context
+            }
+        };
+
+        rustc_span::hygiene::decode_expn_id(
+            self,
+            ExpnDataDecodeMode::Metadata(get_ctxt),
+            |_this, index| {
+                let cnum = expn_cnum.get().unwrap();
+                // Lookup local `ExpnData`s in our own crate data. Foreign `ExpnData`s
+                // are stored in the owning crate, to avoid duplication.
+                let crate_data = if cnum == LOCAL_CRATE {
+                    local_cdata
+                } else {
+                    local_cdata.cstore.get_crate_data(cnum)
+                };
+                Ok(crate_data
+                    .root
+                    .expn_data
+                    .get(&crate_data, index)
+                    .unwrap()
+                    .decode((&crate_data, sess)))
+            },
+        )
+    }
+}

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -21,9 +21,10 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::utils::NativeLibKind;
 use rustc_session::{CrateDisambiguator, Session};
 use rustc_span::source_map::{self, Span, Spanned};
-use rustc_span::symbol::{Ident, Symbol};
+use rustc_span::symbol::Symbol;
 
 use rustc_data_structures::sync::Lrc;
+use rustc_span::ExpnId;
 use smallvec::SmallVec;
 use std::any::Any;
 
@@ -417,13 +418,7 @@ impl CStore {
             attr::mark_used(attr);
         }
 
-        let ident = data
-            .def_key(id.index)
-            .disambiguated_data
-            .data
-            .get_opt_name()
-            .map(Ident::with_dummy_span) // FIXME: cross-crate hygiene
-            .expect("no name in load_macro");
+        let ident = data.item_ident(id.index, sess);
 
         LoadedMacro::MacroDef(
             ast::Item {
@@ -454,6 +449,10 @@ impl CStore {
     pub fn item_generics_num_lifetimes(&self, def_id: DefId, sess: &Session) -> usize {
         self.get_crate_data(def_id.krate).get_generics(def_id.index, sess).own_counts().lifetimes
     }
+
+    pub fn module_expansion_untracked(&self, def_id: DefId, sess: &Session) -> ExpnId {
+        self.get_crate_data(def_id.krate).module_expansion(def_id.index, sess)
+    }
 }
 
 impl CrateStore for CStore {