Protect crate metadata from corruption via SHA-256 hash

We now compute a SHA-256 of the raw (encoded) crate metadata,
and append it to the final crate metadata that we store on disk.
When we load the metadata, we compute the hash from the metadata
blob, and verify that matches the hash stored at the end of the
blob.

This allows us to detect on-disk corruption of the metadata file,
which might later cause a build failure much later in compilation.

If anyone is manually editing crate metadata on-disk,
they will need to re-compute and modify the hash at
the end of the blob.

This will allow us to determine whether or not crate metadata
corruption is causing some of the unusual incr-comp failures
we've been seeing. The incremental compilation data itself
will be hashed in a follow-up PR.

Author: Aaron1011

Cargo.lock

@@ -3994,6 +3994,7 @@ dependencies = [
  "rustc_session",
  "rustc_span",
  "rustc_target",
+ "sha2",
  "smallvec",
  "snap",
  "tracing",

compiler/rustc_metadata/Cargo.toml

@@ -10,6 +10,7 @@ doctest = false
 libc = "0.2"
 snap = "1"
 tracing = "0.1"
+sha2 = "0.9"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_middle = { path = "../rustc_middle" }
 rustc_attr = { path = "../rustc_attr" }

compiler/rustc_metadata/src/locator.rs

@@ -773,7 +773,7 @@ fn get_metadata_section(
             rustc_erase_owner!(OwningRef::new(mmap).map_owner_box())
         }
     };
-    let blob = MetadataBlob::new(raw_bytes);
+    let blob = MetadataBlob::new(raw_bytes, filename);
     if blob.is_compatible() {
         Ok(blob)
     } else {

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -36,6 +36,7 @@ use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP};
 
 use proc_macro::bridge::client::ProcMacro;
+use sha2::{Digest, Sha256};
 use std::io;
 use std::mem;
 use std::num::NonZeroUsize;
@@ -583,8 +584,10 @@ where
 implement_ty_decoder!(DecodeContext<'a, 'tcx>);
 
 impl MetadataBlob {
-    crate fn new(metadata_ref: MetadataRef) -> MetadataBlob {
-        MetadataBlob(metadata_ref)
+    crate fn new(metadata_ref: MetadataRef, filename: &Path) -> MetadataBlob {
+        let blob = MetadataBlob(metadata_ref);
+        blob.check_hash(filename);
+        blob
     }
 
     crate fn is_compatible(&self) -> bool {
@@ -596,6 +599,23 @@ impl MetadataBlob {
             .decode(self)
     }
 
+    // Hashes the entire contents of the metadata blob,
+    // panicking if the computed hash is not equal to
+    // the original hash stored in the file.
+    fn check_hash(&self, filename: &Path) {
+        // We store our 32-byte (256-bit) SHA256 hash at
+        // the end of the file
+        let hash_offset = self.raw_bytes().len() - 32;
+        let stored_hash = &self.raw_bytes()[hash_offset..];
+        let recomputed_hash = Sha256::digest(&self.raw_bytes()[..hash_offset]);
+        if stored_hash != &*recomputed_hash {
+            panic!(
+                "Expected metadata hash {:?}, found {:?} for file {:?}",
+                stored_hash, recomputed_hash, filename
+            );
+        }
+    }
+
     crate fn get_root(&self) -> CrateRoot<'tcx> {
         let slice = self.raw_bytes();
         let offset = METADATA_HEADER.len();

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -35,6 +35,7 @@ use rustc_span::{
     RealFileName,
 };
 use rustc_target::abi::VariantIdx;
+use sha2::{Digest, Sha256};
 use std::hash::Hash;
 use std::num::NonZeroUsize;
 use std::path::Path;
@@ -2144,7 +2145,7 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     // culminating in the `CrateRoot` which points to all of it.
     let root = ecx.encode_crate_root();
 
-    let mut result = ecx.opaque.into_inner();
+    let result = &mut ecx.opaque.data;
 
     // Encode the root position.
     let header = METADATA_HEADER.len();
@@ -2154,5 +2155,11 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     result[header + 2] = (pos >> 8) as u8;
     result[header + 3] = (pos >> 0) as u8;
 
-    EncodedMetadata { raw_data: result }
+    // Encode a hash of the file contents. We check
+    // this when decoding the blob, to protect
+    // against file corruption.
+    let hash = Sha256::digest(&result);
+    ecx.opaque.emit_raw_bytes(&hash).unwrap();
+
+    EncodedMetadata { raw_data: ecx.opaque.into_inner() }
 }