Pass -Zembed-metadata to rustc

Author: Kobzol

src/cargo/core/compiler/build_context/target_info.rs

@@ -613,9 +613,19 @@ impl TargetInfo {
                 }
             }
         }
-        if !result.is_empty() && !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
-            // Only add rmeta if pipelining.
-            result.push(FileType::new_rmeta());
+        if !result.is_empty() {
+            if !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
+                // Only add rmeta if pipelining...
+                result.push(FileType::new_rmeta());
+            } else if crate_types
+                .iter()
+                .any(|ct| ct.benefits_from_split_metadata())
+            {
+                // ...or when we apply -Zembed-metadata=no to the unit.
+                // TODO: should we thread through the information if we use
+                // embed-metadata?
+                result.push(FileType::new_rmeta());
+            }
         }
         Ok((result, unsupported))
     }

src/cargo/core/compiler/crate_type.rs

@@ -76,6 +76,38 @@ impl CrateType {
         // Everything else, however, is some form of "linkable output" or
         // something that requires upstream object files.
     }
+
+    /// Returns whether production of this crate type could benefit from splitting metadata
+    /// into a .rmeta file.
+    ///
+    /// See also [`TargetKind::benefits_from_split_metadata`].
+    ///
+    /// [`TargetKind::benefits_from_split_metadata`]: crate::core::manifest::TargetKind::benefits_from_split_metadata
+    pub fn benefits_from_split_metadata(&self) -> bool {
+        match self {
+            // rlib/libs generate .rmeta files for pipelined compilation.
+            // If we also include metadata inside of them, we waste disk space, since the metadata
+            // will be located both in the lib/rlib and the .rmeta file.
+            CrateType::Lib |
+            CrateType::Rlib |
+            // Dylibs do not have to contain metadata when they are used as a runtime dependency.
+            // If we split the metadata into a separate .rmeta file, the dylib file (that
+            // can be shipped as a runtime dependency) can be smaller.
+            CrateType::Dylib => true,
+            // Proc macros contains metadata that specifies what macro functions are available in
+            // it, but the metadata is typically very small. The metadata of proc macros is also
+            // self-contained (unlike rlibs/dylibs), so let's not unnecessarily split it into
+            // multiple files.
+            CrateType::ProcMacro |
+            // cdylib and staticlib produce artifacts that are used through the C ABI and do not
+            // contain Rust-specific metadata.
+            CrateType::Cdylib |
+            CrateType::Staticlib |
+            // Binaries also do not contain metadata
+            CrateType::Bin |
+            CrateType::Other(_) => false
+        }
+    }
 }
 
 impl fmt::Display for CrateType {

src/cargo/core/compiler/mod.rs

@@ -1102,15 +1102,34 @@ fn build_base_args(
         }
     }
 
+    let use_embed_metadata = use_embed_metadata(build_runner);
     if unit.mode.is_check() {
         cmd.arg("--emit=dep-info,metadata");
-    } else if !unit.requires_upstream_objects() {
-        // Always produce metadata files for rlib outputs. Metadata may be used
-        // in this session for a pipelined compilation, or it may be used in a
-        // future Cargo session as part of a pipelined compile.
-        cmd.arg("--emit=dep-info,metadata,link");
+    } else if use_embed_metadata {
+        // Nightly rustc supports the -Zembed-metadata=no flag, which tells it to avoid including
+        // full metadata in rlib/dylib artifacts, to save space on disk. In this case, metadata
+        // will only be stored in .rmeta files.
+        // When we use this flag, we should also pass --emit=metadata to all artifacts that
+        // contain useful metadata (rlib/dylib/proc macros), so that a .rmeta file is actually
+        // generated. If we didn't do this, the full metadata would not get written anywhere.
+        // However, we do not want to pass --emit=metadata to artifacts that never produce useful
+        // metadata, such as binaries, because that would just unnecessarily create empty .rmeta
+        // files on disk.
+        if unit.benefits_from_split_metadata() {
+            cmd.arg("--emit=dep-info,metadata,link");
+            cmd.args(&["-Z", "embed-metadata=no"]);
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     } else {
-        cmd.arg("--emit=dep-info,link");
+        // If we don't use -Zembed-metadata=no, we emit .rmeta files only for rlib outputs.
+        // This metadata may be used in this session for a pipelined compilation, or it may
+        // be used in a future Cargo session as part of a pipelined compile.
+        if !unit.requires_upstream_objects() {
+            cmd.arg("--emit=dep-info,metadata,link");
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     }
 
     let prefer_dynamic = (unit.target.for_host() && !unit.target.is_custom_build())
@@ -1609,6 +1628,8 @@ pub fn extern_args(
     let mut result = Vec::new();
     let deps = build_runner.unit_deps(unit);
 
+    let use_embed_metadata = use_embed_metadata(build_runner);
+
     // Closure to add one dependency to `result`.
     let mut link_to =
         |dep: &UnitDep, extern_crate_name: InternedString, noprelude: bool| -> CargoResult<()> {
@@ -1658,6 +1679,12 @@ pub fn extern_args(
                     if output.flavor == FileFlavor::Linkable {
                         pass(&output.path);
                     }
+                    // If we use -Zembed-metadata=no, we also need to pass the path to the
+                    // corresponding .rmeta file to the linkable artifact, because the
+                    // normal dependency (rlib) doesn't contain the full metadata.
+                    else if use_embed_metadata && output.flavor == FileFlavor::Rmeta {
+                        pass(&output.path);
+                    }
                 }
             }
             Ok(())
@@ -1684,6 +1711,11 @@ fn envify(s: &str) -> String {
         .collect()
 }
 
+/// Returns true if -Zembed-metadata=no mode should be used when compiling Rust artifacts.
+fn use_embed_metadata(build_runner: &BuildRunner<'_, '_>) -> bool {
+    build_runner.bcx.gctx.nightly_features_allowed
+}
+
 /// Configuration of the display of messages emitted by the compiler,
 /// e.g. diagnostics, warnings, errors, and message caching.
 struct OutputOptions {

src/cargo/core/compiler/unit.rs

@@ -125,6 +125,12 @@ impl UnitInner {
         self.mode.is_any_test() || self.target.kind().requires_upstream_objects()
     }
 
+    /// Returns whether compilation of this unit could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_split_metadata(&self) -> bool {
+        matches!(self.mode, CompileMode::Build) && self.target.kind().benefits_from_split_metadata()
+    }
+
     /// Returns whether or not this is a "local" package.
     ///
     /// A "local" package is one that the user can likely edit, or otherwise

src/cargo/core/manifest.rs

@@ -279,6 +279,17 @@ impl TargetKind {
         }
     }
 
+    /// Returns whether production of this artifact could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_split_metadata(&self) -> bool {
+        match self {
+            TargetKind::Lib(kinds) | TargetKind::ExampleLib(kinds) => {
+                kinds.iter().any(|k| k.benefits_from_split_metadata())
+            }
+            _ => false,
+        }
+    }
+
     /// Returns the arguments suitable for `--crate-type` to pass to rustc.
     pub fn rustc_crate_types(&self) -> Vec<CrateType> {
         match self {

tests/testsuite/build.rs

@@ -6749,3 +6749,91 @@ fn renamed_uplifted_artifact_remains_unmodified_after_rebuild() {
     let not_the_same = !same_file::is_same_file(bin, renamed_bin).unwrap();
     assert!(not_the_same, "renamed uplifted artifact must be unmodified");
 }
+
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file("bar/Cargo.toml", &basic_lib_manifest("bar"))
+        .file(
+            "bar/src/bar.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build")
+        .masquerade_as_nightly_cargo(&["-Zembed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar-[HASH].rmeta[..]`",
+        )
+        .run();
+}
+
+// Make sure that cargo passes --extern=<dep>.rmeta even if <dep>
+// is compiled as a dylib.
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata_dylib_dep() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.5.0"
+                edition = "2015"
+
+                [lib]
+                crate-type = ["dylib"]
+            "#,
+        )
+        .file(
+            "bar/src/lib.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build")
+        .masquerade_as_nightly_cargo(&["-Zembed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar.rmeta[..]`",
+        )
+        .run();
+}