Auto merge of rust-lang#139244 - jieyouxu:exp/auto-cross-run-make, r=Kobzol

Enable automatic cross-compilation in run-make tests

Supersedes rust-lang#138066.

Blocker for rust-lang#141856.

Based on rust-lang#138066 plus `rustdoc()` cross-compile changes.

### Summary

This PR automatically specifies `--target` to `rustc()` and `rustdoc()` to have `rustc`/`rustdoc` produce cross-compiled artifacts in run-make tests by default, unless:

- `//@ ignore-cross-compile` is used, or
- `bare_{rustc,rustdoc}` are used, or
- Explicit `.target()` is specified, which overrides the default cross-compile target.

Some tests are necessarily modified:

- Tests that have `.target(target())` have that incantation removed (since this is now automatically the default).
- Some tests have `//@ needs-target-std`, but are a necessary-but-insufficient condition, and are changed to `//@ ignore-cross-compile` instead as host-only tests.
    - A few tests received `//@ ignore-musl` that fail against `x86_64-unknown-linux-musl` because of inability to find `-lunwind`. AFAICT, they don't *need* to test cross-compiled artifacts.
    - Some tests are constrained to host-only for now, because the effort to make them pass on cross-compile does not seem worth the complexity, and it's not really *meaningfully* improving test coverage.

try-job: dist-various-1

Author: bors

src/tools/run-make-support/src/external_deps/rustc.rs

@@ -6,7 +6,7 @@ use crate::command::Command;
 use crate::env::env_var;
 use crate::path_helpers::cwd;
 use crate::util::set_host_compiler_dylib_path;
-use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
+use crate::{is_aix, is_darwin, is_msvc, is_windows, target, uname};
 
 /// Construct a new `rustc` invocation. This will automatically set the library
 /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
@@ -27,9 +27,15 @@ pub fn bare_rustc() -> Rustc {
 #[must_use]
 pub struct Rustc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustc, |rustc: &mut Rustc| {
+    if let Some(target) = &rustc.target {
+        rustc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 pub fn rustc_path() -> String {
     env_var("RUSTC")
@@ -46,19 +52,22 @@ impl Rustc {
     // `rustc` invocation constructor methods
 
     /// Construct a new `rustc` invocation. This will automatically set the library
-    /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
+    /// search path as `-L cwd()` and also the compilation target.
+    /// Use [`bare_rustc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let mut cmd = setup_common();
         cmd.arg("-L").arg(cwd());
-        Self { cmd }
+
+        // Automatically default to cross-compilation
+        Self { cmd, target: Some(target()) }
     }
 
     /// Construct a bare `rustc` invocation with no flags set.
     #[track_caller]
     pub fn bare() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: None }
     }
 
     // Argument provider methods
@@ -234,8 +243,9 @@ impl Rustc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in Rustc::new().
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/external_deps/rustdoc.rs

@@ -3,21 +3,36 @@ use std::path::Path;
 
 use crate::command::Command;
 use crate::env::env_var;
+use crate::target;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Construct a new `rustdoc` invocation. This will configure the host compiler runtime libs.
+/// Construct a new `rustdoc` invocation with target automatically set to cross-compile target and
+/// with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically setting
+/// cross-compile target.
 #[track_caller]
 pub fn rustdoc() -> Rustdoc {
     Rustdoc::new()
 }
 
+/// Bare `rustdoc` invocation, no args set.
+#[track_caller]
+pub fn bare_rustdoc() -> Rustdoc {
+    Rustdoc::bare()
+}
+
 #[derive(Debug)]
 #[must_use]
 pub struct Rustdoc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustdoc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustdoc, |rustdoc: &mut Rustdoc| {
+    if let Some(target) = &rustdoc.target {
+        rustdoc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 #[track_caller]
 fn setup_common() -> Command {
@@ -28,11 +43,20 @@ fn setup_common() -> Command {
 }
 
 impl Rustdoc {
-    /// Construct a bare `rustdoc` invocation. This will configure the host compiler runtime libs.
+    /// Construct a new `rustdoc` invocation with target automatically set to cross-compile target
+    /// and with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically
+    /// setting cross-compile target.
     #[track_caller]
     pub fn new() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: Some(target()) }
+    }
+
+    /// Bare `rustdoc` invocation, no args set.
+    #[track_caller]
+    pub fn bare() -> Self {
+        let cmd = setup_common();
+        Self { cmd, target: None }
     }
 
     /// Specify where an external library is located.
@@ -85,8 +109,9 @@ impl Rustdoc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in `Rustdoc::new()`.
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/lib.rs

@@ -68,7 +68,7 @@ pub use llvm::{
 };
 pub use python::python_command;
 pub use rustc::{bare_rustc, rustc, rustc_path, Rustc};
-pub use rustdoc::{rustdoc, Rustdoc};
+pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
 
 /// [`diff`][mod@diff] is implemented in terms of the [similar] library.
 ///

src/tools/run-make-support/src/macros.rs

@@ -23,10 +23,16 @@
 /// }
 /// ```
 ///
+/// You can pass an optional second parameter which should be a function that is passed
+/// `&mut self` just before the command is executed.
+///
 /// [`Command`]: crate::command::Command
 /// [`CompletedProcess`]: crate::command::CompletedProcess
 macro_rules! impl_common_helpers {
     ($wrapper: ident) => {
+        $crate::macros::impl_common_helpers!($wrapper, |_| {});
+    };
+    ($wrapper: ident, $before_exec: expr) => {
         impl $wrapper {
             /// In very rare circumstances, you may need a e.g. `bare_rustc()` or `bare_rustdoc()`
             /// with host runtime libs configured, but want the underlying raw
@@ -130,12 +136,14 @@ macro_rules! impl_common_helpers {
             /// Run the constructed command and assert that it is successfully run.
             #[track_caller]
             pub fn run(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run()
             }
 
             /// Run the constructed command and assert that it does not successfully run.
             #[track_caller]
             pub fn run_fail(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_fail()
             }
 
@@ -145,6 +153,7 @@ macro_rules! impl_common_helpers {
             /// whenever possible.
             #[track_caller]
             pub fn run_unchecked(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_unchecked()
             }
 

tests/run-make/allow-warnings-cmdline-stability/rmake.rs

@@ -1,4 +1,4 @@
-//@ needs-target-std
+//@ ignore-cross-compile
 // Test that `-Awarnings` suppresses warnings for unstable APIs.
 
 use run_make_support::rustc;

tests/run-make/apple-deployment-target/rmake.rs

@@ -41,7 +41,6 @@ fn main() {
 
     // Remove env vars to get `rustc`'s default
     let output = rustc()
-        .target(target())
         .env_remove("MACOSX_DEPLOYMENT_TARGET")
         .env_remove("IPHONEOS_DEPLOYMENT_TARGET")
         .env_remove("WATCHOS_DEPLOYMENT_TARGET")
@@ -58,7 +57,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("lib");
             rustc.emit("obj");
             rustc.input("foo.rs");
@@ -82,7 +80,6 @@ fn main() {
 
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("dylib");
             rustc.input("foo.rs");
             rustc.output("libfoo.dylib");
@@ -108,7 +105,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("bin");
             rustc.input("foo.rs");
             rustc.output("foo");
@@ -147,7 +143,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.incremental("incremental");
             rustc.crate_type("lib");
             rustc.emit("obj");

tests/run-make/apple-sdk-version/rmake.rs

@@ -24,8 +24,7 @@ fn has_sdk_version(file: &str, version: &str) {
 
 fn main() {
     // Fetch rustc's inferred deployment target.
-    let current_deployment_target =
-        rustc().target(target()).print("deployment-target").run().stdout_utf8();
+    let current_deployment_target = rustc().print("deployment-target").run().stdout_utf8();
     let current_deployment_target = current_deployment_target.split('=').last().unwrap().trim();
 
     // Fetch current SDK version via. xcrun.
@@ -45,15 +44,15 @@ fn main() {
     let current_sdk_version = current_sdk_version.trim();
 
     // Check the SDK version in the object file produced by the codegen backend.
-    rustc().target(target()).crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
+    rustc().crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
     // Set to 0, which means not set or "n/a".
     has_sdk_version("foo.o", "n/a");
 
     // Check the SDK version in the .rmeta file, as set in `create_object_file`.
     //
     // This is just to ensure that we don't set some odd version in `create_object_file`,
     // if the rmeta file is packed in a different way in the future, this can safely be removed.
-    rustc().target(target()).crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
+    rustc().crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
     // Extra .rmeta file (which is encoded as an object file).
     cmd("ar").arg("-x").arg("libfoo.rlib").arg("lib.rmeta").run();
     has_sdk_version("lib.rmeta", "n/a");
@@ -69,7 +68,6 @@ fn main() {
         // Test with clang
         let file_name = format!("foo_cc{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=gcc")
             .input("foo.rs")
@@ -80,7 +78,6 @@ fn main() {
         // Test with ld64
         let file_name = format!("foo_ld{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=ld")
             .input("foo.rs")

tests/run-make/crate-circular-deps-link/rmake.rs

@@ -1,5 +1,5 @@
-//@ needs-target-std
-//
+//@ ignore-cross-compile
+
 // Test that previously triggered a linker failure with root cause
 // similar to one found in the issue #69368.
 //

tests/run-make/doctests-merge/rmake.rs

@@ -1,4 +1,5 @@
-//@ needs-target-std
+//@ ignore-cross-compile (needs to run doctests)
+
 use std::path::Path;
 
 use run_make_support::{cwd, diff, rustc, rustdoc};

tests/run-make/doctests-runtool/rmake.rs

@@ -1,5 +1,5 @@
-//@ needs-target-std
-//
+//@ ignore-cross-compile (needs to run host tool binary)
+
 // Tests behavior of rustdoc `--test-runtool`.
 
 use std::path::PathBuf;