feat(build-dir): Reorganize build-dir layout (#15947)

### What does this PR try to resolve?

This PR re-organizes the `build-dir` file layout structure to a layout
organized by "build unit" when `-Zbuild-dir-new-layout` is enabled.
See #15010 for the motivations and design discussions.

Below is file structure generated for a `foo` crate with a single
dependency on `syn`.

```
$ tree -a target
target
├── CACHEDIR.TAG
├── debug
│   ├── build
│   │   ├── foo
│   │   │   └── 9f7d8db454c2e3a0
│   │   │       ├── deps
│   │   │       │   ├── foo-9f7d8db454c2e3a0
│   │   │       │   └── foo-9f7d8db454c2e3a0.d
│   │   │       └── fingerprint
│   │   │           ├── bin-foo
│   │   │           ├── bin-foo.json
│   │   │           ├── dep-bin-foo
│   │   │           └── invoked.timestamp
│   │   ├── proc-macro2
│   │   │   ├── 39344080ec273066
│   │   │   │   ├── deps
│   │   │   │   │   ├── libproc_macro2-39344080ec273066.rlib
│   │   │   │   │   ├── libproc_macro2-39344080ec273066.rmeta
│   │   │   │   │   └── proc_macro2-39344080ec273066.d
│   │   │   │   └── fingerprint
│   │   │   │       ├── dep-lib-proc_macro2
│   │   │   │       ├── invoked.timestamp
│   │   │   │       ├── lib-proc_macro2
│   │   │   │       └── lib-proc_macro2.json
│   │   │   ├── 8fc259c340d09182
│   │   │   │   ├── build-script
│   │   │   │   │   ├── build-script-build
│   │   │   │   │   ├── build_script_build-8fc259c340d09182
│   │   │   │   │   └── build_script_build-8fc259c340d09182.d
│   │   │   │   ├── deps
│   │   │   │   └── fingerprint
│   │   │   │       ├── build-script-build-script-build
│   │   │   │       ├── build-script-build-script-build.json
│   │   │   │       ├── dep-build-script-build-script-build
│   │   │   │       └── invoked.timestamp
│   │   │   └── a6c299e456bc1663
│   │   │       ├── build-script-execution
│   │   │       │   ├── invoked.timestamp
│   │   │       │   ├── out
│   │   │       │   ├── output
│   │   │       │   ├── root-output
│   │   │       │   └── stderr
│   │   │       ├── deps
│   │   │       └── fingerprint
│   │   │           ├── run-build-script-build-script-build
│   │   │           └── run-build-script-build-script-build.json
│   │   ├── quote
│   │   │   └── e95944fa29b3b7d5
│   │   │       ├── deps
│   │   │       │   ├── libquote-e95944fa29b3b7d5.rlib
│   │   │       │   ├── libquote-e95944fa29b3b7d5.rmeta
│   │   │       │   └── quote-e95944fa29b3b7d5.d
│   │   │       └── fingerprint
│   │   │           ├── dep-lib-quote
│   │   │           ├── invoked.timestamp
│   │   │           ├── lib-quote
│   │   │           └── lib-quote.json
│   │   ├── syn
│   │   │   └── a0dd360a8d0d85f9
│   │   │       ├── deps
│   │   │       │   ├── libsyn-a0dd360a8d0d85f9.rlib
│   │   │       │   ├── libsyn-a0dd360a8d0d85f9.rmeta
│   │   │       │   └── syn-a0dd360a8d0d85f9.d
│   │   │       └── fingerprint
│   │   │           ├── dep-lib-syn
│   │   │           ├── invoked.timestamp
│   │   │           ├── lib-syn
│   │   │           └── lib-syn.json
│   │   └── unicode-ident
│   │       └── 90d3c33eeb6b75ae
│   │           ├── deps
│   │           │   ├── libunicode_ident-90d3c33eeb6b75ae.rlib
│   │           │   ├── libunicode_ident-90d3c33eeb6b75ae.rmeta
│   │           │   └── unicode_ident-90d3c33eeb6b75ae.d
│   │           └── fingerprint
│   │               ├── dep-lib-unicode_ident
│   │               ├── invoked.timestamp
│   │               ├── lib-unicode_ident
│   │               └── lib-unicode_ident.json
│   ├── .cargo-lock
│   ├── examples
│   ├── foo
│   ├── foo.d
│   └── incremental
│       └── foo-2476dro80nq0n
│           ├── s-hbvjw5x5fa-1d58onh-7sam1xj7vrwy85uyr70f3vu2p
│           │   ├── 1z96gpvj66dpsd06phiqkocsr.o
│           │   ├── 8jaqpsn916x4kajeuk1b7rl12.o
│           │   ├── 8sv45ek5gkoxjdcr3b6nbgu2o.o
│           │   ├── 9yqg9fn9x870xvia9frlvvazg.o
│           │   ├── a5v36uvuoyp00oyvscfno2pce.o
│           │   ├── c2q7ff0mmsk4s1yg7s5ei1ryr.o
│           │   ├── dep-graph.bin
│           │   ├── query-cache.bin
│           │   └── work-products.bin
│           └── s-hbvjw5x5fa-1d58onh.lock
└── .rustc_info.json

36 directories, 62 files
```

For why incremental is where it is, see
#15947 (comment)

As a side effect, we pass a lot more parameters to rustc, likely making
`cargo -v` more annoying, similar to #13941.

### How to test and review this PR?

This PR still needs to be more thoroughly tested. Thus far I have been
testing on simple test crates.
Also see #15874 for potential test harness improvements that could be
used by this PR.

### Follow up actions

- [ ] Remove redundant `-Cextra-filename` in files where possible.
- [ ] See if we can drop `<platform>/<profile>` after #4282
- [ ] Re-evaluate if we want platform to be unconditionally included in
the build-dir layout
- [ ] Is `<pkgname>/<hash>` good enough or do we need to go with
prefixes to reduce the number of items within a directory, see
#15947 (comment)
- [ ] Can we simplify how fingerpritns are stored, reducing pressure on
path lenghts
- [ ] Under the new layout, should `cargo clean -p` also cleans old
layout paths? See
#15947 (comment)

Author: epage

crates/cargo-test-support/src/compare.rs

@@ -235,6 +235,9 @@ fn add_regex_redactions(subs: &mut snapbox::Redactions) {
     // Match file name hashes like `foo-06b451d0d6f88b1d`
     subs.insert("[HASH]", regex!(r"[a-z0-9]+-(?<redacted>[a-f0-9]{16})"))
         .unwrap();
+    // Match path hashes like `../06b451d0d6f88b1d/..` used in directory paths
+    subs.insert("[HASH]", regex!(r"\/(?<redacted>[0-9a-f]{16})\/"))
+        .unwrap();
     subs.insert(
         "[AVG_ELAPSED]",
         regex!(r"(?<redacted>[0-9]+(\.[0-9]+)?) ns/iter"),

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -230,17 +230,21 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.export_dir.clone()
     }
 
-    /// Directory name to use for a package in the form `NAME-HASH`.
+    /// Directory name to use for a package in the form `{NAME}/{HASH}`.
     ///
     /// Note that some units may share the same directory, so care should be
     /// taken in those cases!
     fn pkg_dir(&self, unit: &Unit) -> String {
+        let seperator = match self.ws.gctx().cli_unstable().build_dir_new_layout {
+            true => "/",
+            false => "-",
+        };
         let name = unit.pkg.package_id().name();
         let meta = self.metas[unit];
         if let Some(c_extra_filename) = meta.c_extra_filename() {
-            format!("{}-{}", name, c_extra_filename)
+            format!("{}{}{}", name, seperator, c_extra_filename)
         } else {
-            format!("{}-{}", name, self.target_short_hash(unit))
+            format!("{}{}{}", name, seperator, self.target_short_hash(unit))
         }
     }
 
@@ -255,20 +259,27 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
     }
 
     /// Returns the host `deps` directory path.
-    pub fn host_deps(&self) -> &Path {
-        self.host.deps()
+    pub fn host_deps(&self, unit: &Unit) -> PathBuf {
+        let dir = self.pkg_dir(unit);
+        self.host.deps(&dir)
     }
 
     /// Returns the directories where Rust crate dependencies are found for the
     /// specified unit.
-    pub fn deps_dir(&self, unit: &Unit) -> &Path {
-        self.layout(unit.kind).deps()
+    pub fn deps_dir(&self, unit: &Unit) -> PathBuf {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).deps(&dir)
     }
 
     /// Directory where the fingerprint for the given unit should go.
     pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {
         let dir = self.pkg_dir(unit);
-        self.layout(unit.kind).fingerprint().join(dir)
+        self.layout(unit.kind).fingerprint(&dir)
+    }
+
+    /// Directory where incremental output for the given unit should go.
+    pub fn incremental_dir(&self, unit: &Unit) -> &Path {
+        self.layout(unit.kind).incremental()
     }
 
     /// Returns the path for a file in the fingerprint directory.
@@ -303,7 +314,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         assert!(!unit.mode.is_run_custom_build());
         assert!(self.metas.contains_key(unit));
         let dir = self.pkg_dir(unit);
-        self.layout(CompileKind::Host).build().join(dir)
+        self.layout(CompileKind::Host).build_script(&dir)
     }
 
     /// Returns the directory for compiled artifacts files.
@@ -337,7 +348,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         assert!(unit.target.is_custom_build());
         assert!(unit.mode.is_run_custom_build());
         let dir = self.pkg_dir(unit);
-        self.layout(unit.kind).build().join(dir)
+        self.layout(unit.kind).build_script_execution(&dir)
     }
 
     /// Returns the "`OUT_DIR`" directory for running a build script.

src/cargo/core/compiler/build_runner/mod.rs

@@ -10,6 +10,7 @@ use crate::core::compiler::{self, Unit, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
 use anyhow::{Context as _, bail};
+use cargo_util::paths;
 use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
@@ -401,9 +402,17 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             self.compilation
                 .root_output
                 .insert(kind, layout.dest().to_path_buf());
-            self.compilation
-                .deps_output
-                .insert(kind, layout.deps().to_path_buf());
+            if self.bcx.gctx.cli_unstable().build_dir_new_layout {
+                for (unit, _) in self.bcx.unit_graph.iter() {
+                    let dep_dir = self.files().deps_dir(unit);
+                    paths::create_dir_all(&dep_dir)?;
+                    self.compilation.deps_output.insert(kind, dep_dir);
+                }
+            } else {
+                self.compilation
+                    .deps_output
+                    .insert(kind, layout.legacy_deps().to_path_buf());
+            }
         }
         Ok(())
     }

src/cargo/core/compiler/layout.rs

@@ -142,6 +142,7 @@ pub struct Layout {
     /// Will be `None` when the build-dir and target-dir are the same path as we cannot
     /// lock the same path twice.
     _build_lock: Option<FileLock>,
+    is_new_layout: bool,
 }
 
 impl Layout {
@@ -156,6 +157,7 @@ impl Layout {
         target: Option<CompileTarget>,
         dest: &str,
     ) -> CargoResult<Layout> {
+        let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
         let mut root = ws.target_dir();
         let mut build_root = ws.build_dir();
         if let Some(target) = target {
@@ -212,14 +214,17 @@ impl Layout {
             dest,
             _lock: lock,
             _build_lock: build_lock,
+            is_new_layout,
         })
     }
 
     /// Makes sure all directories stored in the Layout exist on the filesystem.
     pub fn prepare(&mut self) -> CargoResult<()> {
-        paths::create_dir_all(&self.deps)?;
+        if !self.is_new_layout {
+            paths::create_dir_all(&self.deps)?;
+            paths::create_dir_all(&self.fingerprint)?;
+        }
         paths::create_dir_all(&self.incremental)?;
-        paths::create_dir_all(&self.fingerprint)?;
         paths::create_dir_all(&self.examples)?;
         paths::create_dir_all(&self.build_examples)?;
         paths::create_dir_all(&self.build)?;
@@ -232,7 +237,15 @@ impl Layout {
         &self.dest
     }
     /// Fetch the deps path.
-    pub fn deps(&self) -> &Path {
+    pub fn deps(&self, pkg_dir: &str) -> PathBuf {
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("deps")
+        } else {
+            self.legacy_deps().to_path_buf()
+        }
+    }
+    /// Fetch the deps path. (old layout)
+    pub fn legacy_deps(&self) -> &Path {
         &self.deps
     }
     /// Fetch the examples path.
@@ -256,17 +269,45 @@ impl Layout {
         &self.incremental
     }
     /// Fetch the fingerprint path.
-    pub fn fingerprint(&self) -> &Path {
+    pub fn fingerprint(&self, pkg_dir: &str) -> PathBuf {
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("fingerprint")
+        } else {
+            self.legacy_fingerprint().to_path_buf().join(pkg_dir)
+        }
+    }
+    /// Fetch the fingerprint path. (old layout)
+    pub fn legacy_fingerprint(&self) -> &Path {
         &self.fingerprint
     }
-    /// Fetch the build script path.
+    /// Fetch the build path.
     pub fn build(&self) -> &Path {
         &self.build
     }
+    /// Fetch the build script path.
+    pub fn build_script(&self, pkg_dir: &str) -> PathBuf {
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("build-script")
+        } else {
+            self.build().join(pkg_dir)
+        }
+    }
+    /// Fetch the build script execution path.
+    pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("build-script-execution")
+        } else {
+            self.build().join(pkg_dir)
+        }
+    }
     /// Fetch the artifact path.
     pub fn artifact(&self) -> &Path {
         &self.artifact
     }
+    /// Fetch the build unit path
+    pub fn build_unit(&self, pkg_dir: &str) -> PathBuf {
+        self.build().join(pkg_dir)
+    }
     /// Create and return the tmp path.
     pub fn prepare_tmp(&self) -> CargoResult<&Path> {
         paths::create_dir_all(&self.tmp)?;

src/cargo/core/compiler/mod.rs

@@ -56,7 +56,7 @@ pub mod unit_dependencies;
 pub mod unit_graph;
 
 use std::borrow::Cow;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt::Display;
@@ -69,6 +69,7 @@ use std::sync::{Arc, LazyLock};
 use annotate_snippets::{AnnotationKind, Group, Level, Renderer, Snippet};
 use anyhow::{Context as _, Error};
 use cargo_platform::{Cfg, Platform};
+use itertools::Itertools;
 use lazycell::LazyCell;
 use regex::Regex;
 use tracing::{debug, instrument, trace};
@@ -1356,12 +1357,8 @@ fn build_base_args(
             .map(|s| s.as_ref()),
     );
     if incremental {
-        let dir = build_runner
-            .files()
-            .layout(unit.kind)
-            .incremental()
-            .as_os_str();
-        opt(cmd, "-C", "incremental=", Some(dir));
+        let dir = build_runner.files().incremental_dir(&unit);
+        opt(cmd, "-C", "incremental=", Some(dir.as_os_str()));
     }
 
     let pkg_hint_mostly_unused = match hints.mostly_unused {
@@ -1671,18 +1668,35 @@ fn build_deps_args(
     unit: &Unit,
 ) -> CargoResult<()> {
     let bcx = build_runner.bcx;
-    cmd.arg("-L").arg(&{
-        let mut deps = OsString::from("dependency=");
-        deps.push(build_runner.files().deps_dir(unit));
-        deps
-    });
+    if build_runner.bcx.gctx.cli_unstable().build_dir_new_layout {
+        let mut map = BTreeMap::new();
+
+        // Recursively add all depenendency args to rustc process
+        add_dep_arg(&mut map, build_runner, unit);
+
+        let paths = map.into_iter().map(|(_, path)| path).sorted_unstable();
+
+        for path in paths {
+            cmd.arg("-L").arg(&{
+                let mut deps = OsString::from("dependency=");
+                deps.push(path);
+                deps
+            });
+        }
+    } else {
+        cmd.arg("-L").arg(&{
+            let mut deps = OsString::from("dependency=");
+            deps.push(build_runner.files().deps_dir(unit));
+            deps
+        });
+    }
 
     // Be sure that the host path is also listed. This'll ensure that proc macro
     // dependencies are correctly found (for reexported macros).
     if !unit.kind.is_host() {
         cmd.arg("-L").arg(&{
             let mut deps = OsString::from("dependency=");
-            deps.push(build_runner.files().host_deps());
+            deps.push(build_runner.files().host_deps(unit));
             deps
         });
     }
@@ -1758,6 +1772,21 @@ fn build_deps_args(
     Ok(())
 }
 
+fn add_dep_arg<'a, 'b: 'a>(
+    map: &mut BTreeMap<&'a Unit, PathBuf>,
+    build_runner: &'b BuildRunner<'b, '_>,
+    unit: &'a Unit,
+) {
+    if map.contains_key(&unit) {
+        return;
+    }
+    map.insert(&unit, build_runner.files().deps_dir(&unit));
+
+    for dep in build_runner.unit_deps(unit) {
+        add_dep_arg(map, build_runner, &dep.unit);
+    }
+}
+
 /// Adds extra rustc flags and environment variables collected from the output
 /// of a build-script to the command to execute, include custom environment
 /// variables and `cfg`.

src/cargo/ops/cargo_clean.rs

@@ -200,7 +200,7 @@ fn clean_specs(
 
         // Clean fingerprints.
         for (_, layout) in &layouts_with_host {
-            let dir = escape_glob_path(layout.fingerprint())?;
+            let dir = escape_glob_path(layout.legacy_fingerprint())?;
             clean_ctx
                 .rm_rf_package_glob_containing_hash(&pkg.name(), &Path::new(&dir).join(&pkg_dir))?;
         }
@@ -226,8 +226,13 @@ fn clean_specs(
                 CompileMode::Check { test: false },
             ] {
                 for (compile_kind, layout) in &layouts {
-                    let triple = target_data.short_name(compile_kind);
+                    if clean_ctx.gctx.cli_unstable().build_dir_new_layout {
+                        let dir = layout.build_unit(&pkg.name());
+                        clean_ctx.rm_rf_glob(&dir)?;
+                        continue;
+                    }
 
+                    let triple = target_data.short_name(compile_kind);
                     let (file_types, _unsupported) = target_data
                         .info(*compile_kind)
                         .rustc_outputs(mode, target.kind(), triple, clean_ctx.gctx)?;
@@ -236,8 +241,8 @@ fn clean_specs(
                             (layout.build_examples(), Some(layout.examples()))
                         }
                         // Tests/benchmarks are never uplifted.
-                        TargetKind::Test | TargetKind::Bench => (layout.deps(), None),
-                        _ => (layout.deps(), Some(layout.dest())),
+                        TargetKind::Test | TargetKind::Bench => (layout.legacy_deps(), None),
+                        _ => (layout.legacy_deps(), Some(layout.dest())),
                     };
                     let mut dir_glob_str = escape_glob_path(dir)?;
                     let dir_glob = Path::new(&dir_glob_str);