compiletest: Support matching on non-json lines in compiler output

and migrate most of remaining `error-pattern`s to it.

Author: petrochenkov

src/doc/rustc-dev-guide/src/tests/ui.md

@@ -344,8 +344,7 @@ For checking runtime output, `//@ check-run-results` may be preferable.
 
 Only use `error-pattern` if none of the above works.
 
-Line annotations `//~` are still checked in tests using `error-pattern`.
-In exceptional cases, use `//@ compile-flags: --error-format=human` to opt out of these checks.
+Line annotations `//~` and `error-pattern` are compatible and can be used in the same test.
 
 ### Diagnostic kinds (error levels)
 
@@ -356,9 +355,12 @@ The diagnostic kinds that you can have are:
 - `NOTE`
 - `HELP`
 - `SUGGESTION`
+- `RAW`
 
 The `SUGGESTION` kind is used for specifying what the expected replacement text
 should be for a diagnostic suggestion.
+The `RAW` kind can be used for matching on lines from non-structured output sometimes emitted
+by the compiler instead of or in addition to structured json.
 
 `ERROR` and `WARN` kinds are required to be exhaustively covered by line annotations
 `//~` by default.

src/tools/compiletest/src/errors.rs

@@ -15,6 +15,7 @@ pub enum ErrorKind {
     Note,
     Suggestion,
     Warning,
+    Raw,
 }
 
 impl ErrorKind {
@@ -39,6 +40,7 @@ impl ErrorKind {
             "NOTE" | "note" | "MONO_ITEM" => ErrorKind::Note,
             "SUGGESTION" => ErrorKind::Suggestion,
             "WARN" | "WARNING" | "warn" | "warning" => ErrorKind::Warning,
+            "RAW" => ErrorKind::Raw,
             _ => panic!(
                 "unexpected diagnostic kind `{s}`, expected \
                  `ERROR`, `WARN`, `NOTE`, `HELP` or `SUGGESTION`"
@@ -55,6 +57,7 @@ impl fmt::Display for ErrorKind {
             ErrorKind::Note => write!(f, "NOTE"),
             ErrorKind::Suggestion => write!(f, "SUGGESTION"),
             ErrorKind::Warning => write!(f, "WARN"),
+            ErrorKind::Raw => write!(f, "RAW"),
         }
     }
 }

src/tools/compiletest/src/json.rs

@@ -7,7 +7,6 @@ use regex::Regex;
 use serde::Deserialize;
 
 use crate::errors::{Error, ErrorKind};
-use crate::runtest::ProcRes;
 
 #[derive(Deserialize)]
 struct Diagnostic {
@@ -140,28 +139,19 @@ pub fn extract_rendered(output: &str) -> String {
         .collect()
 }
 
-pub fn parse_output(file_name: &str, output: &str, proc_res: &ProcRes) -> Vec<Error> {
+pub fn parse_output(file_name: &str, output: &str) -> Vec<Error> {
     let mut errors = Vec::new();
     for line in output.lines() {
-        // The compiler sometimes intermingles non-JSON stuff into the
-        // output.  This hack just skips over such lines. Yuck.
-        if line.starts_with('{') {
-            match serde_json::from_str::<Diagnostic>(line) {
-                Ok(diagnostic) => push_actual_errors(&mut errors, &diagnostic, &[], file_name),
-                Err(error) => {
-                    // Ignore the future compat report message - this is handled
-                    // by `extract_rendered`
-                    if serde_json::from_str::<FutureIncompatReport>(line).is_err() {
-                        proc_res.fatal(
-                        Some(&format!(
-                            "failed to decode compiler output as json: `{}`\nline: {}\noutput: {}",
-                            error, line, output
-                        )),
-                        || (),
-                    );
-                    }
-                }
-            }
+        // Compiler can emit non-json lines in non-`--error-format=json` modes,
+        // and in some situations even in json mode.
+        match serde_json::from_str::<Diagnostic>(line) {
+            Ok(diagnostic) => push_actual_errors(&mut errors, &diagnostic, &[], file_name),
+            Err(_) => errors.push(Error {
+                line_num: None,
+                kind: ErrorKind::Raw,
+                msg: line.to_string(),
+                require_annotation: false,
+            }),
         }
     }
     errors

src/tools/compiletest/src/runtest.rs

@@ -23,7 +23,7 @@ use crate::common::{
     output_base_dir, output_base_name, output_testname_unique,
 };
 use crate::compute_diff::{DiffLine, make_diff, write_diff, write_filtered_diff};
-use crate::errors::{Error, ErrorKind};
+use crate::errors::{Error, ErrorKind, load_errors};
 use crate::header::TestProps;
 use crate::read2::{Truncated, read2_abbreviated};
 use crate::util::{Utf8PathBufExt, add_dylib_path, logv, static_regex};
@@ -577,23 +577,9 @@ impl<'test> TestCx<'test> {
         }
     }
 
-    fn check_all_error_patterns(
-        &self,
-        output_to_check: &str,
-        proc_res: &ProcRes,
-        pm: Option<PassMode>,
-    ) {
-        if self.props.error_patterns.is_empty() && self.props.regex_error_patterns.is_empty() {
-            if pm.is_some() {
-                // FIXME(#65865)
-                return;
-            } else {
-                self.fatal(&format!("no error pattern specified in {}", self.testpaths.file));
-            }
-        }
-
+    /// Check `error-pattern` and `regex-error-pattern` directives.
+    fn check_all_error_patterns(&self, output_to_check: &str, proc_res: &ProcRes) {
         let mut missing_patterns: Vec<String> = Vec::new();
-
         self.check_error_patterns(output_to_check, &mut missing_patterns);
         self.check_regex_error_patterns(output_to_check, proc_res, &mut missing_patterns);
 
@@ -670,7 +656,9 @@ impl<'test> TestCx<'test> {
         }
     }
 
-    fn check_expected_errors(&self, expected_errors: Vec<Error>, proc_res: &ProcRes) {
+    /// Check `//~ KIND message` annotations.
+    fn check_expected_errors(&self, proc_res: &ProcRes) {
+        let expected_errors = load_errors(&self.testpaths.file, self.revision);
         debug!(
             "check_expected_errors: expected_errors={:?} proc_res.status={:?}",
             expected_errors, proc_res.status
@@ -711,11 +699,24 @@ impl<'test> TestCx<'test> {
             .collect();
 
         // Parse the JSON output from the compiler and extract out the messages.
-        let actual_errors = json::parse_output(&diagnostic_file_name, &proc_res.stderr, proc_res);
+        let actual_errors = json::parse_output(&diagnostic_file_name, &self.get_output(proc_res))
+            .into_iter()
+            .map(|e| Error { msg: self.normalize_output(&e.msg, &[]), ..e });
+
         let mut unexpected = Vec::new();
         let mut found = vec![false; expected_errors.len()];
-        for mut actual_error in actual_errors {
-            actual_error.msg = self.normalize_output(&actual_error.msg, &[]);
+        for actual_error in actual_errors {
+            for pattern in &self.props.error_patterns {
+                let pattern = pattern.trim();
+                if actual_error.msg.contains(pattern) {
+                    let q = if actual_error.line_num.is_none() { "?" } else { "" };
+                    self.fatal(&format!(
+                        "error pattern '{pattern}' is found in structured \
+                         diagnostics, use `//~{q} {} {pattern}` instead",
+                        actual_error.kind,
+                    ));
+                }
+            }
 
             let opt_index =
                 expected_errors.iter().enumerate().position(|(index, expected_error)| {

src/tools/compiletest/src/runtest/incremental.rs

@@ -100,16 +100,8 @@ impl TestCx<'_> {
         self.check_no_compiler_crash(&proc_res, self.props.should_ice);
 
         let output_to_check = self.get_output(&proc_res);
-        let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
-        if !expected_errors.is_empty() {
-            if !self.props.error_patterns.is_empty() || !self.props.regex_error_patterns.is_empty()
-            {
-                self.fatal("both error pattern and expected errors specified");
-            }
-            self.check_expected_errors(expected_errors, &proc_res);
-        } else {
-            self.check_all_error_patterns(&output_to_check, &proc_res, pm);
-        }
+        self.check_expected_errors(&proc_res);
+        self.check_all_error_patterns(&output_to_check, &proc_res);
         if self.props.should_ice {
             match proc_res.status.code() {
                 Some(101) => (),
@@ -137,6 +129,6 @@ impl TestCx<'_> {
 
         let output_to_check = self.get_output(&proc_res);
         self.check_correct_failure_status(&proc_res);
-        self.check_all_error_patterns(&output_to_check, &proc_res, pm);
+        self.check_all_error_patterns(&output_to_check, &proc_res);
     }
 }

src/tools/compiletest/src/runtest/ui.rs

@@ -9,7 +9,7 @@ use super::{
     AllowUnused, Emit, FailMode, LinkToAux, PassMode, TargetLocation, TestCx, TestOutput,
     Truncated, UI_FIXED, WillExecute,
 };
-use crate::{errors, json};
+use crate::json;
 
 impl TestCx<'_> {
     pub(super) fn run_ui_test(&self) {
@@ -127,9 +127,7 @@ impl TestCx<'_> {
             );
         }
 
-        let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
-
-        if let WillExecute::Yes = should_run {
+        let output_to_check = if let WillExecute::Yes = should_run {
             let proc_res = self.exec_compiled_test();
             let run_output_errors = if self.props.check_run_results {
                 self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
@@ -150,44 +148,19 @@ impl TestCx<'_> {
                 self.fatal_proc_rec("test run succeeded!", &proc_res);
             }
 
-            let output_to_check = self.get_output(&proc_res);
-            if !self.props.error_patterns.is_empty() || !self.props.regex_error_patterns.is_empty()
-            {
-                // "// error-pattern" comments
-                self.check_all_error_patterns(&output_to_check, &proc_res, pm);
-            }
-            self.check_forbid_output(&output_to_check, &proc_res)
-        }
+            self.get_output(&proc_res)
+        } else {
+            self.get_output(&proc_res)
+        };
 
         debug!(
-            "run_ui_test: explicit={:?} config.compare_mode={:?} expected_errors={:?} \
+            "run_ui_test: explicit={:?} config.compare_mode={:?} \
                proc_res.status={:?} props.error_patterns={:?}",
-            explicit,
-            self.config.compare_mode,
-            expected_errors,
-            proc_res.status,
-            self.props.error_patterns
+            explicit, self.config.compare_mode, proc_res.status, self.props.error_patterns
         );
 
-        if !explicit && self.config.compare_mode.is_none() {
-            // "//~ERROR comments"
-            self.check_expected_errors(expected_errors, &proc_res);
-        } else if explicit && !expected_errors.is_empty() {
-            let msg = format!(
-                "line {}: cannot combine `--error-format` with {} annotations; use `error-pattern` instead",
-                expected_errors[0].line_num_str(),
-                expected_errors[0].kind,
-            );
-            self.fatal(&msg);
-        }
-        let output_to_check = self.get_output(&proc_res);
-        if should_run == WillExecute::No
-            && (!self.props.error_patterns.is_empty()
-                || !self.props.regex_error_patterns.is_empty())
-        {
-            // "// error-pattern" comments
-            self.check_all_error_patterns(&output_to_check, &proc_res, pm);
-        }
+        self.check_expected_errors(&proc_res);
+        self.check_all_error_patterns(&output_to_check, &proc_res);
         self.check_forbid_output(&output_to_check, &proc_res);
 
         if self.props.run_rustfix && self.config.compare_mode.is_none() {

tests/incremental/ich_nested_items.rs

@@ -7,6 +7,7 @@
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
+#![allow(dead_code)]
 
 #[rustc_clean(except = "opt_hir_owner_nodes", cfg = "cfail2")]
 pub fn foo() {

tests/incremental/incremental_proc_macro.rs

@@ -12,5 +12,5 @@ extern crate incremental_proc_macro_aux;
 
 #[derive(IncrementalMacro)]
 pub struct Foo {
-    x: u32
+    _x: u32
 }

tests/incremental/issue-49595/issue-49595.rs

@@ -10,7 +10,7 @@
 
 mod tests {
     #[cfg_attr(not(cfail1), test)]
-    fn test() {
+    fn _test() {
     }
 }
 

tests/incremental/issue-84252-global-alloc.rs

@@ -1,5 +1,6 @@
 //@ revisions: cfail1 cfail2
 //@ build-pass
+//@ needs-crate-type: cdylib
 
 #![crate_type="lib"]
 #![crate_type="cdylib"]