Auto merge of #9223 - ehuss:fingerprint-target, r=alexcrichton

Detect changes for JSON spec targets.

This adds the contents of a JSON spec target file to the fingerprint hash so that changing the contents will trigger a rebuild. This also deals with the target info cache in a similar fashion.

Closes #2616

Author: bors

src/cargo/core/compiler/build_context/target_info.rs

@@ -137,6 +137,7 @@ impl TargetInfo {
             kind,
             "RUSTFLAGS",
         )?;
+        let extra_fingerprint = kind.fingerprint_hash();
         let mut process = rustc.process();
         process
             .arg("-")
@@ -163,14 +164,17 @@ impl TargetInfo {
             process.arg("--crate-type").arg(crate_type.as_str());
         }
         let supports_split_debuginfo = rustc
-            .cached_output(process.clone().arg("-Csplit-debuginfo=packed"))
+            .cached_output(
+                process.clone().arg("-Csplit-debuginfo=packed"),
+                extra_fingerprint,
+            )
             .is_ok();
 
         process.arg("--print=sysroot");
         process.arg("--print=cfg");
 
         let (output, error) = rustc
-            .cached_output(&process)
+            .cached_output(&process, extra_fingerprint)
             .chain_err(|| "failed to run `rustc` to learn about target-specific information")?;
 
         let mut lines = output.lines();

src/cargo/core/compiler/compile_kind.rs

@@ -1,10 +1,12 @@
 use crate::core::Target;
 use crate::util::errors::{CargoResult, CargoResultExt};
 use crate::util::interning::InternedString;
-use crate::util::Config;
+use crate::util::{Config, StableHasher};
 use anyhow::bail;
 use serde::Serialize;
 use std::collections::BTreeSet;
+use std::fs;
+use std::hash::{Hash, Hasher};
 use std::path::Path;
 
 /// Indicator for how a unit is being compiled.
@@ -78,6 +80,18 @@ impl CompileKind {
         };
         Ok(vec![kind])
     }
+
+    /// Hash used for fingerprinting.
+    ///
+    /// Metadata hashing uses the normal Hash trait, which does not
+    /// differentiate on `.json` file contents. The fingerprint hash does
+    /// check the contents.
+    pub fn fingerprint_hash(&self) -> u64 {
+        match self {
+            CompileKind::Host => 0,
+            CompileKind::Target(target) => target.fingerprint_hash(),
+        }
+    }
 }
 
 impl serde::ser::Serialize for CompileKind {
@@ -166,4 +180,19 @@ impl CompileTarget {
             &self.name
         }
     }
+
+    /// See [`CompileKind::fingerprint_hash`].
+    pub fn fingerprint_hash(&self) -> u64 {
+        let mut hasher = StableHasher::new();
+        self.name.hash(&mut hasher);
+        if self.name.ends_with(".json") {
+            // This may have some performance concerns, since it is called
+            // fairly often. If that ever seems worth fixing, consider
+            // embedding this in `CompileTarget`.
+            if let Ok(contents) = fs::read_to_string(self.name) {
+                contents.hash(&mut hasher);
+            }
+        }
+        hasher.finish()
+    }
 }

src/cargo/core/compiler/fingerprint.rs

@@ -60,10 +60,10 @@
 //! `cargo rustc` extra args                   | ✓           | ✓
 //! CompileMode                                | ✓           | ✓
 //! Target Name                                | ✓           | ✓
-//! Target CompileKind (bin/lib/etc.)          | ✓           | ✓
+//! TargetKind (bin/lib/etc.)                  | ✓           | ✓
 //! Enabled Features                           | ✓           | ✓
 //! Immediate dependency’s hashes              | ✓[^1]       | ✓
-//! Target or Host mode                        |             | ✓
+//! CompileKind (host/target)                  | ✓           | ✓
 //! __CARGO_DEFAULT_LIB_METADATA[^4]           |             | ✓
 //! package_id                                 |             | ✓
 //! authors, description, homepage, repo       | ✓           |
@@ -542,6 +542,9 @@ pub struct Fingerprint {
     metadata: u64,
     /// Hash of various config settings that change how things are compiled.
     config: u64,
+    /// The rustc target. This is only relevant for `.json` files, otherwise
+    /// the metadata hash segregates the units.
+    compile_kind: u64,
     /// Description of whether the filesystem status for this unit is up to date
     /// or should be considered stale.
     #[serde(skip)]
@@ -780,6 +783,7 @@ impl Fingerprint {
             rustflags: Vec::new(),
             metadata: 0,
             config: 0,
+            compile_kind: 0,
             fs_status: FsStatus::Stale,
             outputs: Vec::new(),
         }
@@ -843,6 +847,9 @@ impl Fingerprint {
         if self.config != old.config {
             bail!("configuration settings have changed")
         }
+        if self.compile_kind != old.compile_kind {
+            bail!("compile kind (rustc target) changed")
+        }
         let my_local = self.local.lock().unwrap();
         let old_local = old.local.lock().unwrap();
         if my_local.len() != old_local.len() {
@@ -1090,12 +1097,22 @@ impl hash::Hash for Fingerprint {
             ref local,
             metadata,
             config,
+            compile_kind,
             ref rustflags,
             ..
         } = *self;
         let local = local.lock().unwrap();
         (
-            rustc, features, target, path, profile, &*local, metadata, config, rustflags,
+            rustc,
+            features,
+            target,
+            path,
+            profile,
+            &*local,
+            metadata,
+            config,
+            compile_kind,
+            rustflags,
         )
             .hash(h);
 
@@ -1318,6 +1335,7 @@ fn calculate_normal(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Finger
     } else {
         0
     };
+    let compile_kind = unit.kind.fingerprint_hash();
     Ok(Fingerprint {
         rustc: util::hash_u64(&cx.bcx.rustc().verbose_version),
         target: util::hash_u64(&unit.target),
@@ -1331,6 +1349,7 @@ fn calculate_normal(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Finger
         memoized_hash: Mutex::new(None),
         metadata,
         config,
+        compile_kind,
         rustflags: extra_flags,
         fs_status: FsStatus::Stale,
         outputs,

src/cargo/util/rustc.rs

@@ -49,7 +49,7 @@ impl Rustc {
 
         let mut cmd = util::process(&path);
         cmd.arg("-vV");
-        let verbose_version = cache.cached_output(&cmd)?.0;
+        let verbose_version = cache.cached_output(&cmd, 0)?.0;
 
         let extract = |field: &str| -> CargoResult<&str> {
             verbose_version
@@ -100,8 +100,25 @@ impl Rustc {
         util::process(&self.path)
     }
 
-    pub fn cached_output(&self, cmd: &ProcessBuilder) -> CargoResult<(String, String)> {
-        self.cache.lock().unwrap().cached_output(cmd)
+    /// Gets the output for the given command.
+    ///
+    /// This will return the cached value if available, otherwise it will run
+    /// the command and cache the output.
+    ///
+    /// `extra_fingerprint` is extra data to include in the cache fingerprint.
+    /// Use this if there is other information about the environment that may
+    /// affect the output that is not part of `cmd`.
+    ///
+    /// Returns a tuple of strings `(stdout, stderr)`.
+    pub fn cached_output(
+        &self,
+        cmd: &ProcessBuilder,
+        extra_fingerprint: u64,
+    ) -> CargoResult<(String, String)> {
+        self.cache
+            .lock()
+            .unwrap()
+            .cached_output(cmd, extra_fingerprint)
     }
 }
 
@@ -187,8 +204,12 @@ impl Cache {
         }
     }
 
-    fn cached_output(&mut self, cmd: &ProcessBuilder) -> CargoResult<(String, String)> {
-        let key = process_fingerprint(cmd);
+    fn cached_output(
+        &mut self,
+        cmd: &ProcessBuilder,
+        extra_fingerprint: u64,
+    ) -> CargoResult<(String, String)> {
+        let key = process_fingerprint(cmd, extra_fingerprint);
         if self.data.outputs.contains_key(&key) {
             debug!("rustc info cache hit");
         } else {
@@ -295,8 +316,9 @@ fn rustc_fingerprint(path: &Path, rustup_rustc: &Path) -> CargoResult<u64> {
     Ok(hasher.finish())
 }
 
-fn process_fingerprint(cmd: &ProcessBuilder) -> u64 {
+fn process_fingerprint(cmd: &ProcessBuilder, extra_fingerprint: u64) -> u64 {
     let mut hasher = StableHasher::new();
+    extra_fingerprint.hash(&mut hasher);
     cmd.get_args().hash(&mut hasher);
     let mut env = cmd.get_envs().iter().collect::<Vec<_>>();
     env.sort_unstable();