use custom test runner so that we can get proper test filtering

Author: RalfJung

README.md

@@ -7,7 +7,7 @@ undergraduate research course at the [University of Saskatchewan][usask].
 
 ## Building Miri
 
-I recommend that you install [rustup][rustup] to obtain Rust. Then all you have
+We recommend that you install [rustup][rustup] to obtain Rust. Then all you have
 to do is:
 
 ```sh
@@ -117,11 +117,12 @@ Miri will often require using a locally built rustc. This includes getting a
 trace of the execution, as distributed rustc has `debug!` and `trace!` disabled.
 
 The first-time setup for a local rustc looks as follows:
-```
+```sh
 git clone https://github.com/rust-lang/rust/ rustc
 cd rustc
 cp config.toml.example config.toml
-# Now edit `config.toml` and set `debug-assertions = true`
+# Now edit `config.toml` and set `debug-assertions = true` and `test-miri = true`.
+# The latter is important to build libstd with the right flags for miri.
 ./x.py build src/rustc
 # You may have to change the architecture in the next command
 rustup toolchain link custom build/x86_64-unknown-linux-gnu/stage2
@@ -130,9 +131,20 @@ rustup override set custom
 ```
 The `build` step can take 30 minutes and more.
 
-Now you can `cargo build` Miri, and you can `cargo test` it.  But the key point
-is, you can now run Miri with a trace of all execution steps:
+Now you can `cargo build` Miri, and you can `cargo test --release` it.  `cargo
+test --release FILTER` only runs those tests that contain `FILTER` in their
+filename (including the base directory, e.g. `cargo test --release fail` will
+run all compile-fail tests).  We recommend using `--release` to make test
+running take less time.
+
+Notice that the "fullmir" tests only run if you have `MIRI_SYSROOT` set, the
+test runner does not realized that your libstd comes with full MIR.  The
+following will set it correctly:
+```sh
+MIRI_SYSROOT=$(rustc --print sysroot) cargo test --release
+```
 
+Moreover, you can now run Miri with a trace of all execution steps:
 ```sh
 MIRI_LOG=debug cargo run tests/run-pass/vecs.rs
 ```
@@ -141,9 +153,8 @@ Setting `MIRI_LOG` like this will configure logging for miri itself as well as
 the `rustc::mir::interpret` and `rustc_mir::interpret` modules in rustc.  You
 can also do more targeted configuration, e.g. to debug the stacked borrows
 implementation:
-
 ```sh
-MIRI_LOG=miri::stacked_borrows=trace,rustc_mir::interpret=debug cargo run tests/run-pass/vecs.rs
+MIRI_LOG=rustc_mir::interpret=debug,miri::stacked_borrows cargo run tests/run-pass/vecs.rs
 ```
 
 In addition, you can set `MIRI_BACKTRACE=1` to get a backtrace of where an

tests/compiletest.rs

@@ -1,20 +1,13 @@
-#![feature(slice_concat_ext)]
-
-extern crate compiletest_rs as compiletest;
-extern crate colored;
-
-use colored::*;
+#![feature(slice_concat_ext, custom_test_frameworks)]
+#![test_runner(test_runner)]
 
 use std::slice::SliceConcatExt;
 use std::path::{PathBuf, Path};
 use std::io::Write;
+use std::env;
 
-macro_rules! eprintln {
-    ($($arg:tt)*) => {
-        let stderr = std::io::stderr();
-        writeln!(stderr.lock(), $($arg)*).unwrap();
-    }
-}
+use compiletest_rs as compiletest;
+use colored::*;
 
 fn miri_path() -> PathBuf {
     if rustc_test_suite().is_some() {
@@ -37,9 +30,21 @@ fn have_fullmir() -> bool {
     std::env::var("MIRI_SYSROOT").is_ok() || rustc_test_suite().is_some()
 }
 
+fn mk_config(mode: &str) -> compiletest::Config {
+    let mut config = compiletest::Config::default();
+    config.mode = mode.parse().expect("Invalid mode");
+    config.rustc_path = miri_path();
+    if rustc_test_suite().is_some() {
+        config.run_lib_path = rustc_lib_path();
+        config.compile_lib_path = rustc_lib_path();
+    }
+    config.filter = env::args().nth(1);
+    config
+}
+
 fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, need_fullmir: bool, opt: bool) {
     if need_fullmir && !have_fullmir() {
-        eprintln!("{}", format!(
+        eprintln!("{}\n", format!(
             "## Skipping compile-fail tests in {} against miri for target {} due to missing mir",
             path,
             target
@@ -65,23 +70,17 @@ fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, need_fullm
         flags.push("-Zmir-opt-level=1".to_owned());
     }
 
-    let mut config = compiletest::Config::default().tempdir();
-    config.mode = "compile-fail".parse().expect("Invalid mode");
-    config.rustc_path = miri_path();
-    if rustc_test_suite().is_some() {
-        config.run_lib_path = rustc_lib_path();
-        config.compile_lib_path = rustc_lib_path();
-    }
-    config.src_base = PathBuf::from(path.to_string());
-    config.target_rustcflags = Some(flags.join(" "));
+    let mut config = mk_config("compile-fail");
+    config.src_base = PathBuf::from(path);
     config.target = target.to_owned();
     config.host = host.to_owned();
-    compiletest::run_tests(&config);
+    config.target_rustcflags = Some(flags.join(" "));
+    compiletest::run_tests(&config.tempdir()); // FIXME: `tempdir` can be done by `mk_config` once `ConfigWithTemp` is exposed as type from compiletest
 }
 
 fn miri_pass(sysroot: &Path, path: &str, target: &str, host: &str, need_fullmir: bool, opt: bool) {
     if need_fullmir && !have_fullmir() {
-        eprintln!("{}", format!(
+        eprintln!("{}\n", format!(
             "## Skipping run-pass tests in {} against miri for target {} due to missing mir",
             path,
             target
@@ -104,18 +103,12 @@ fn miri_pass(sysroot: &Path, path: &str, target: &str, host: &str, need_fullmir:
         flags.push("-Zmir-opt-level=3".to_owned());
     }
 
-    let mut config = compiletest::Config::default().tempdir();
-    config.mode = "ui".parse().expect("Invalid mode");
+    let mut config = mk_config("ui");
     config.src_base = PathBuf::from(path);
     config.target = target.to_owned();
     config.host = host.to_owned();
-    config.rustc_path = miri_path();
-    if rustc_test_suite().is_some() {
-        config.run_lib_path = rustc_lib_path();
-        config.compile_lib_path = rustc_lib_path();
-    }
     config.target_rustcflags = Some(flags.join(" "));
-    compiletest::run_tests(&config);
+    compiletest::run_tests(&config.tempdir()); // FIXME: `tempdir` can be done by `mk_config` once `ConfigWithTemp` is exposed as type from compiletest
 }
 
 fn is_target_dir<P: Into<PathBuf>>(path: P) -> bool {
@@ -151,7 +144,6 @@ fn get_sysroot() -> PathBuf {
 
 fn get_host() -> String {
     let rustc = rustc_test_suite().unwrap_or(PathBuf::from("rustc"));
-    println!("using rustc at {}", rustc.display());
     let host = std::process::Command::new(rustc)
         .arg("-vV")
         .output()
@@ -184,8 +176,7 @@ fn compile_fail_miri(opt: bool) {
     compile_fail(&sysroot, "tests/compile-fail-fullmir", &host, &host, true, opt);
 }
 
-#[test]
-fn test() {
+fn test_runner(_tests: &[&()]) {
     // We put everything into a single test to avoid the parallelism `cargo test`
     // introduces.  We still get parallelism within our tests because `compiletest`
     // uses `libtest` which runs jobs in parallel.