compiletest: Support opt-in Clang-based run-make tests.

Some cross-language run-make tests need a Clang compiler that
matches the LLVM version of rustc. Since such a compiler usually
isn't available these tests (marked with the "needs-matching-clang"
directive) are ignored by default.

For some CI jobs we do need these tests to run unconditionally
though. In order to support this a --force-clang-based-tests flag
is added to compiletest. If this flag is specified, compiletest
will fail if it can't detect an appropriate version of Clang.

Author: michaelwoerister

src/tools/compiletest/src/common.rs

@@ -144,6 +144,10 @@ pub struct Config {
     /// (or, alternatively, to silently run them like regular run-pass tests).
     pub force_valgrind: bool,
 
+    /// Whether to fail if we don't have a clang version available that matches
+    /// rustc's LLVM version.
+    pub force_clang_based_tests: bool,
+
     /// The directory containing the tests to run
     pub src_base: PathBuf,
 
@@ -205,6 +209,9 @@ pub struct Config {
     /// Is LLVM a system LLVM
     pub system_llvm: bool,
 
+    /// The version of Clang available to run-make tests (if any).
+    pub clang_version: Option<String>,
+
     /// Path to the android tools
     pub android_cross_path: PathBuf,
 

src/tools/compiletest/src/header.rs

@@ -111,6 +111,11 @@ impl EarlyProps {
                 if ignore_llvm(config, ln) {
                     props.ignore = Ignore::Ignore;
                 }
+
+                if !config.force_clang_based_tests &&
+                   config.parse_needs_matching_clang(ln) {
+                    props.ignore = Ignore::Ignore;
+                }
             }
 
             if (config.mode == common::DebugInfoGdb || config.mode == common::DebugInfoBoth) &&
@@ -705,6 +710,10 @@ impl Config {
         }
     }
 
+    fn parse_needs_matching_clang(&self, line: &str) -> bool {
+        self.parse_name_directive(line, "needs-matching-clang")
+    }
+
     /// Parses a name-value directive which contains config-specific information, e.g., `ignore-x86`
     /// or `normalize-stderr-32bit`.
     fn parse_cfg_name_directive(&self, line: &str, prefix: &str) -> ParsedNameDirective {

src/tools/compiletest/src/main.rs

@@ -50,12 +50,32 @@ pub mod util;
 fn main() {
     env_logger::init();
 
-    let config = parse_config(env::args().collect());
+    let mut config = parse_config(env::args().collect());
 
     if config.valgrind_path.is_none() && config.force_valgrind {
         panic!("Can't find Valgrind to run Valgrind tests");
     }
 
+    // Some run-make tests need a version of Clang available that matches
+    // rustc's LLVM version. Since this isn't always the case, these tests are
+    // opt-in.
+    let clang_based_tests_possible = check_clang_based_tests_possible(&config);
+    match (clang_based_tests_possible, config.force_clang_based_tests) {
+        (Ok(_), true) |
+        (Err(_), false) => {
+            // Nothing to do
+        }
+        (Ok(_), false) => {
+            // If a valid clang version is available, run the tests even if
+            // they are not forced.
+            config.force_clang_based_tests = true;
+        }
+        (Err(msg), true) => {
+            // Tests are forced but we don't have a valid version of Clang.
+            panic!("{}", msg)
+        }
+    }
+
     log_config(&config);
     run_tests(&config);
 }
@@ -108,6 +128,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "force-valgrind",
             "fail if Valgrind tests cannot be run under Valgrind",
         )
+        .optflag(
+            "",
+            "force-clang-based-tests",
+            "fail if Clang-based run-make tests can't be run for some reason",
+        )
         .optopt(
             "",
             "llvm-filecheck",
@@ -189,6 +214,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "VERSION STRING",
         )
         .optflag("", "system-llvm", "is LLVM the system LLVM")
+        .optopt(
+            "",
+            "clang-version",
+            "the version of Clang available to run-make tests",
+            "VERSION STRING",
+        )
         .optopt(
             "",
             "android-cross-path",
@@ -298,6 +329,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         docck_python: matches.opt_str("docck-python").unwrap(),
         valgrind_path: matches.opt_str("valgrind-path"),
         force_valgrind: matches.opt_present("force-valgrind"),
+        force_clang_based_tests: matches.opt_present("force-clang-based-tests"),
         llvm_filecheck: matches.opt_str("llvm-filecheck").map(|s| PathBuf::from(&s)),
         src_base,
         build_base: opt_path(matches, "build-base"),
@@ -323,6 +355,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         lldb_native_rust,
         llvm_version: matches.opt_str("llvm-version"),
         system_llvm: matches.opt_present("system-llvm"),
+        clang_version: matches.opt_str("clang-version"),
         android_cross_path: android_cross_path,
         adb_path: opt_str2(matches.opt_str("adb-path")),
         adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
@@ -1031,3 +1064,54 @@ fn test_extract_gdb_version() {
         7012050: "GNU gdb (GDB) 7.12.50.20161027-git",
     }
 }
+
+
+fn check_clang_based_tests_possible(config: &Config) -> Result<(), String> {
+
+    let llvm_version = if let Some(llvm_version) = config.llvm_version.as_ref() {
+        llvm_version
+    } else {
+        return Err(format!("Running `compiletest` with `--force-clang-based-tests` \
+                            requires `--llvm-version` to be specified."));
+    };
+
+    let clang_major_version = if let Some(ref version_string) = config.clang_version {
+        major_version_from_clang_version_string(version_string)?
+    } else {
+        return Err(format!("Clang is required for running tests \
+                            (because of --force-clang-based-tests) \
+                            but it does not seem to be available."));
+    };
+
+    let rustc_llvm_major_version = major_version_from_llvm_version_string(&llvm_version)?;
+
+    return if clang_major_version != rustc_llvm_major_version {
+        Err(format!("`--force-clang-based-tests` needs the major version of Clang \
+                     and rustc's LLVM to be the same. Clang version is: {}, \
+                     Rustc LLVM is: {}",
+                     config.clang_version.clone().unwrap(),
+                     llvm_version))
+    } else {
+        Ok(())
+    };
+
+    fn major_version_from_clang_version_string(clang_version: &str) -> Result<&str, String> {
+        let re = regex::Regex::new(r"clang version (\d)\.\d").unwrap();
+        if let Some(captures) = re.captures(clang_version) {
+            Ok(captures.get(1).unwrap().as_str())
+        } else {
+            Err(format!("Failed to parse major version from Clang version \
+                         string '{}'.", clang_version))
+        }
+    }
+
+    fn major_version_from_llvm_version_string(llvm_version: &str) -> Result<&str, String> {
+        let re = regex::Regex::new(r"(\d)\.\d").unwrap();
+        if let Some(captures) = re.captures(llvm_version) {
+            Ok(captures.get(1).unwrap().as_str())
+        } else {
+            Err(format!("Failed to parse major version from LLVM version \
+                         string '{}'.", llvm_version))
+        }
+    }
+}