Auto merge of #87393 - michaelwoerister:backport-85834-and-86368, r=Mark-Simulacrum

Backport #85834 and #86368

As per discussion in https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/backport.20.2386368, backporting #85834 too seems like the safest option for cleanly backporting #86368.

r? `@Mark-Simulacrum`

Author: bors

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -252,6 +252,10 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
         self.cdata.expect("missing CrateMetadata in DecodeContext")
     }
 
+    fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
+        if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] }
+    }
+
     fn read_lazy_with_meta<T: ?Sized + LazyMeta>(
         &mut self,
         meta: T::Meta,
@@ -324,10 +328,6 @@ impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> {
         r
     }
 
-    fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
-        if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] }
-    }
-
     fn decode_alloc_id(&mut self) -> Result<rustc_middle::mir::interpret::AllocId, Self::Error> {
         if let Some(alloc_decoding_session) = self.alloc_decoding_session {
             alloc_decoding_session.decode_alloc_id(self)

compiler/rustc_middle/src/ty/codec.rs

@@ -15,7 +15,7 @@ use crate::mir::{
 use crate::ty::subst::SubstsRef;
 use crate::ty::{self, List, Ty, TyCtxt};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_hir::def_id::{CrateNum, DefId};
+use rustc_hir::def_id::DefId;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_span::Span;
 use std::hash::Hash;
@@ -179,8 +179,6 @@ pub trait TyDecoder<'tcx>: Decoder {
     where
         F: FnOnce(&mut Self) -> R;
 
-    fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum;
-
     fn positioned_at_shorthand(&self) -> bool {
         (self.peek_byte() & (SHORTHAND_OFFSET as u8)) != 0
     }

compiler/rustc_middle/src/ty/context.rs

@@ -53,6 +53,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_session::config::{BorrowckMode, CrateType, OutputFilenames};
 use rustc_session::lint::{Level, Lint};
 use rustc_session::Session;
+use rustc_span::def_id::StableCrateId;
 use rustc_span::source_map::MultiSpan;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
@@ -1264,6 +1265,11 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    #[inline]
+    pub fn stable_crate_id(self, cnum: CrateNum) -> StableCrateId {
+        self.def_path_hash(cnum.as_def_id()).stable_crate_id()
+    }
+
     pub fn def_path_debug_str(self, def_id: DefId) -> String {
         // We are explicitly not going through queries here in order to get
         // crate name and disambiguator since this code is called from debug!()

compiler/rustc_middle/src/ty/query/on_disk_cache.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell};
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_errors::Diagnostic;
-use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE};
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathHash;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_query_system::dep_graph::DepContext;
@@ -18,7 +18,7 @@ use rustc_serialize::{
     opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize},
     Decodable, Decoder, Encodable, Encoder,
 };
-use rustc_session::{CrateDisambiguator, Session};
+use rustc_session::Session;
 use rustc_span::hygiene::{
     ExpnDataDecodeMode, ExpnDataEncodeMode, ExpnId, HygieneDecodeContext, HygieneEncodeContext,
     SyntaxContext, SyntaxContextData,
@@ -51,11 +51,10 @@ pub struct OnDiskCache<'sess> {
     // session.
     current_diagnostics: Lock<FxHashMap<DepNodeIndex, Vec<Diagnostic>>>,
 
-    prev_cnums: Vec<(u32, String, CrateDisambiguator)>,
-    cnum_map: OnceCell<IndexVec<CrateNum, Option<CrateNum>>>,
+    cnum_map: OnceCell<UnhashMap<StableCrateId, CrateNum>>,
 
     source_map: &'sess SourceMap,
-    file_index_to_stable_id: FxHashMap<SourceFileIndex, StableSourceFileId>,
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
 
     // Caches that are populated lazily during decoding.
     file_index_to_file: Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
@@ -112,8 +111,7 @@ pub struct OnDiskCache<'sess> {
 // This type is used only for serialization and deserialization.
 #[derive(Encodable, Decodable)]
 struct Footer {
-    file_index_to_stable_id: FxHashMap<SourceFileIndex, StableSourceFileId>,
-    prev_cnums: Vec<(u32, String, CrateDisambiguator)>,
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
     query_result_index: EncodedQueryResultIndex,
     diagnostics_index: EncodedQueryResultIndex,
     // The location of all allocations.
@@ -159,6 +157,32 @@ crate struct RawDefId {
     pub index: u32,
 }
 
+/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that
+/// the source crate is represented as a [StableCrateId] instead of as a
+/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded
+/// without any additional context, i.e. with a simple `opaque::Decoder` (which
+/// is the only thing available when decoding the cache's [Footer].
+#[derive(Encodable, Decodable, Clone, Debug)]
+struct EncodedSourceFileId {
+    file_name_hash: u64,
+    stable_crate_id: StableCrateId,
+}
+
+impl EncodedSourceFileId {
+    fn translate(&self, cnum_map: &UnhashMap<StableCrateId, CrateNum>) -> StableSourceFileId {
+        let cnum = cnum_map[&self.stable_crate_id];
+        StableSourceFileId { file_name_hash: self.file_name_hash, cnum }
+    }
+
+    fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId {
+        let source_file_id = StableSourceFileId::new(file);
+        EncodedSourceFileId {
+            file_name_hash: source_file_id.file_name_hash,
+            stable_crate_id: tcx.stable_crate_id(source_file_id.cnum),
+        }
+    }
+}
+
 impl<'sess> OnDiskCache<'sess> {
     /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
     pub fn new(sess: &'sess Session, data: Vec<u8>, start_pos: usize) -> Self {
@@ -186,7 +210,6 @@ impl<'sess> OnDiskCache<'sess> {
             serialized_data: data,
             file_index_to_stable_id: footer.file_index_to_stable_id,
             file_index_to_file: Default::default(),
-            prev_cnums: footer.prev_cnums,
             cnum_map: OnceCell::new(),
             source_map: sess.source_map(),
             current_diagnostics: Default::default(),
@@ -207,7 +230,6 @@ impl<'sess> OnDiskCache<'sess> {
             serialized_data: Vec::new(),
             file_index_to_stable_id: Default::default(),
             file_index_to_file: Default::default(),
-            prev_cnums: vec![],
             cnum_map: OnceCell::new(),
             source_map,
             current_diagnostics: Default::default(),
@@ -242,7 +264,8 @@ impl<'sess> OnDiskCache<'sess> {
                     let index = SourceFileIndex(index as u32);
                     let file_ptr: *const SourceFile = &**file as *const _;
                     file_to_file_index.insert(file_ptr, index);
-                    file_index_to_stable_id.insert(index, StableSourceFileId::new(&file));
+                    let source_file_id = EncodedSourceFileId::new(tcx, &file);
+                    file_index_to_stable_id.insert(index, source_file_id);
                 }
 
                 (file_to_file_index, file_index_to_stable_id)
@@ -327,16 +350,6 @@ impl<'sess> OnDiskCache<'sess> {
                 interpret_alloc_index
             };
 
-            let sorted_cnums = sorted_cnums_including_local_crate(tcx);
-            let prev_cnums: Vec<_> = sorted_cnums
-                .iter()
-                .map(|&cnum| {
-                    let crate_name = tcx.crate_name(cnum).to_string();
-                    let crate_disambiguator = tcx.crate_disambiguator(cnum);
-                    (cnum.as_u32(), crate_name, crate_disambiguator)
-                })
-                .collect();
-
             let mut syntax_contexts = FxHashMap::default();
             let mut expn_ids = FxHashMap::default();
 
@@ -368,7 +381,6 @@ impl<'sess> OnDiskCache<'sess> {
                 TAG_FILE_FOOTER,
                 &Footer {
                     file_index_to_stable_id,
-                    prev_cnums,
                     query_result_index,
                     diagnostics_index,
                     interpret_alloc_index,
@@ -385,16 +397,7 @@ impl<'sess> OnDiskCache<'sess> {
             // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address
             // of the footer must be the last thing in the data stream.
 
-            return Ok(());
-
-            fn sorted_cnums_including_local_crate(tcx: TyCtxt<'_>) -> Vec<CrateNum> {
-                let mut cnums = vec![LOCAL_CRATE];
-                cnums.extend_from_slice(tcx.crates());
-                cnums.sort_unstable();
-                // Just to be sure...
-                cnums.dedup();
-                cnums
-            }
+            Ok(())
         })
     }
 
@@ -429,12 +432,11 @@ impl<'sess> OnDiskCache<'sess> {
         self.foreign_def_path_hashes.get(hash).copied()
     }
 
-    fn try_remap_cnum(&self, tcx: TyCtxt<'_>, cnum: u32) -> Option<CrateNum> {
-        let cnum_map =
-            self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx, &self.prev_cnums[..]));
-        debug!("try_remap_cnum({}): cnum_map={:?}", cnum, cnum_map);
+    fn try_remap_cnum(&self, tcx: TyCtxt<'_>, stable_crate_id: StableCrateId) -> Option<CrateNum> {
+        let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx));
+        debug!("try_remap_cnum({:?}): cnum_map={:?}", stable_crate_id, cnum_map);
 
-        cnum_map[CrateNum::from_u32(cnum)]
+        cnum_map.get(&stable_crate_id).copied()
     }
 
     pub(crate) fn store_foreign_def_id_hash(&self, def_id: DefId, hash: DefPathHash) {
@@ -533,8 +535,7 @@ impl<'sess> OnDiskCache<'sess> {
     where
         T: Decodable<CacheDecoder<'a, 'tcx>>,
     {
-        let cnum_map =
-            self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx, &self.prev_cnums[..]));
+        let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx));
 
         let mut decoder = CacheDecoder {
             tcx,
@@ -555,31 +556,16 @@ impl<'sess> OnDiskCache<'sess> {
     // current-session-`CrateNum`. There might be `CrateNum`s from the previous
     // `Session` that don't occur in the current one. For these, the mapping
     // maps to None.
-    fn compute_cnum_map(
-        tcx: TyCtxt<'_>,
-        prev_cnums: &[(u32, String, CrateDisambiguator)],
-    ) -> IndexVec<CrateNum, Option<CrateNum>> {
+    fn compute_cnum_map(tcx: TyCtxt<'_>) -> UnhashMap<StableCrateId, CrateNum> {
         tcx.dep_graph.with_ignore(|| {
-            let current_cnums = tcx
-                .all_crate_nums(())
+            tcx.all_crate_nums(())
                 .iter()
+                .chain(std::iter::once(&LOCAL_CRATE))
                 .map(|&cnum| {
-                    let crate_name = tcx.crate_name(cnum).to_string();
-                    let crate_disambiguator = tcx.crate_disambiguator(cnum);
-                    ((crate_name, crate_disambiguator), cnum)
+                    let hash = tcx.def_path_hash(cnum.as_def_id()).stable_crate_id();
+                    (hash, cnum)
                 })
-                .collect::<FxHashMap<_, _>>();
-
-            let map_size = prev_cnums.iter().map(|&(cnum, ..)| cnum).max().unwrap_or(0) + 1;
-            let mut map = IndexVec::from_elem_n(None, map_size as usize);
-
-            for &(prev_cnum, ref crate_name, crate_disambiguator) in prev_cnums {
-                let key = (crate_name.clone(), crate_disambiguator);
-                map[CrateNum::from_u32(prev_cnum)] = current_cnums.get(&key).cloned();
-            }
-
-            map[LOCAL_CRATE] = Some(LOCAL_CRATE);
-            map
+                .collect()
         })
     }
 
@@ -612,7 +598,7 @@ impl<'sess> OnDiskCache<'sess> {
                 debug!("def_path_hash_to_def_id({:?}): raw_def_id = {:?}", hash, raw_def_id);
                 // If the owning crate no longer exists, the corresponding definition definitely
                 // no longer exists.
-                let krate = self.try_remap_cnum(tcx, raw_def_id.krate)?;
+                let krate = self.try_remap_cnum(tcx, hash.stable_crate_id())?;
                 debug!("def_path_hash_to_def_id({:?}): krate = {:?}", hash, krate);
                 // If our `DefPathHash` corresponded to a definition in the local crate,
                 // we should have either found it in `local_def_path_hash_to_def_id`, or
@@ -644,9 +630,9 @@ pub struct CacheDecoder<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     opaque: opaque::Decoder<'a>,
     source_map: &'a SourceMap,
-    cnum_map: &'a IndexVec<CrateNum, Option<CrateNum>>,
+    cnum_map: &'a UnhashMap<StableCrateId, CrateNum>,
     file_index_to_file: &'a Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
-    file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, StableSourceFileId>,
+    file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
     alloc_decoding_session: AllocDecodingSession<'a>,
     syntax_contexts: &'a FxHashMap<u32, AbsoluteBytePos>,
     expn_data: &'a FxHashMap<u32, AbsoluteBytePos>,
@@ -659,14 +645,15 @@ impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
             ref file_index_to_file,
             ref file_index_to_stable_id,
             ref source_map,
+            ref cnum_map,
             ..
         } = *self;
 
         file_index_to_file
             .borrow_mut()
             .entry(index)
             .or_insert_with(|| {
-                let stable_id = file_index_to_stable_id[&index];
+                let stable_id = file_index_to_stable_id[&index].translate(cnum_map);
                 source_map
                     .source_file_by_stable_id(stable_id)
                     .expect("failed to lookup `SourceFile` in new context")
@@ -765,10 +752,6 @@ impl<'a, 'tcx> TyDecoder<'tcx> for CacheDecoder<'a, 'tcx> {
         r
     }
 
-    fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
-        self.cnum_map[cnum].unwrap_or_else(|| bug!("could not find new `CrateNum` for {:?}", cnum))
-    }
-
     fn decode_alloc_id(&mut self) -> Result<interpret::AllocId, Self::Error> {
         let alloc_decoding_session = self.alloc_decoding_session;
         alloc_decoding_session.decode_alloc_id(self)
@@ -850,8 +833,9 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
 
 impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for CrateNum {
     fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
-        let cnum = CrateNum::from_u32(u32::decode(d)?);
-        Ok(d.map_encoded_cnum_to_current(cnum))
+        let stable_id = StableCrateId::decode(d)?;
+        let cnum = d.cnum_map[&stable_id];
+        Ok(cnum)
     }
 }
 
@@ -1061,6 +1045,15 @@ where
     }
 }
 
+impl<'a, 'tcx, E> Encodable<CacheEncoder<'a, 'tcx, E>> for CrateNum
+where
+    E: 'a + OpaqueEncoder,
+{
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> {
+        s.tcx.stable_crate_id(*self).encode(s)
+    }
+}
+
 impl<'a, 'tcx, E> Encodable<CacheEncoder<'a, 'tcx, E>> for DefId
 where
     E: 'a + OpaqueEncoder,