Auto merge of rust-lang#84568 - andoriyu:libtest/junit_formatter, r=yaahc

feat(libtest): Add JUnit formatter

tracking issue: rust-lang#85563

Add an alternative formatter to `libtest`. Formatter produces valid xml that later can be interpreted as JUnit report.

Caveats:

- `timestamp` is required by schema, but every viewer/parser ignores it. Attribute is not set to avoid depending on chrono;
- Running all "suits" (unit tests, doc-tests and integration tests) will produce a mess;
- I couldn't find a way to get integration test binary name, so it's just goes by "integration";

Sample output for unit tests (pretty printed by 3rd party tool):
```
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
  <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="13" skipped="1">
    <testcase classname="results::tests" name="test_completed_bad" time="0"/>
    <testcase classname="results::tests" name="suite_started" time="0"/>
    <testcase classname="results::tests" name="suite_ended_ok" time="0"/>
    <testcase classname="results::tests" name="suite_ended_bad" time="0"/>
    <testcase classname="junit::tests" name="test_failed_output" time="0"/>
    <testcase classname="junit::tests" name="test_simple_output" time="0"/>
    <testcase classname="junit::tests" name="test_multiple_outputs" time="0"/>
    <testcase classname="results::tests" name="test_completed_ok" time="0"/>
    <testcase classname="results::tests" name="test_stared" time="0"/>
    <testcase classname="junit::tests" name="test_generate_xml_no_error_single_testsuite" time="0"/>
    <testcase classname="results::tests" name="test_simple_output" time="0"/>
    <testcase classname="test" name="should_panic" time="0"/>
    <system-out/>
    <system-err/>
  </testsuite>
</testsuites>
```

Sample output for integration tests (pretty printed by 3rd party tool):

```
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
  <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="1" skipped="0">
    <testcase classname="integration" name="test_add" time="0"/>
    <system-out/>
    <system-err/>
  </testsuite>
</testsuites>
```

Sample output for Doc-tests (pretty printed by 3rd party tool):

```
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
  <testsuite name="test" package="test" id="0" errors="0" failures="0" tests="1" skipped="0">
    <testcase classname="src/lib.rs" name="(line 2)" time="0"/>
    <system-out/>
    <system-err/>
  </testsuite>
</testsuites>
```

Author: bors

library/test/src/cli.rs

@@ -95,8 +95,9 @@ fn optgroups() -> getopts::Options {
             "Configure formatting of output:
             pretty = Print verbose output;
             terse  = Display one character per test;
-            json   = Output a json document",
-            "pretty|terse|json",
+            json   = Output a json document;
+            junit  = Output a JUnit document",
+            "pretty|terse|json|junit",
         )
         .optflag("", "show-output", "Show captured stdout of successful tests")
         .optopt(
@@ -336,10 +337,15 @@ fn get_format(
             }
             OutputFormat::Json
         }
-
+        Some("junit") => {
+            if !allow_unstable {
+                return Err("The \"junit\" format is only accepted on the nightly compiler".into());
+            }
+            OutputFormat::Junit
+        }
         Some(v) => {
             return Err(format!(
-                "argument for --format must be pretty, terse, or json (was \
+                "argument for --format must be pretty, terse, json or junit (was \
                  {})",
                 v
             ));

library/test/src/console.rs

@@ -10,7 +10,7 @@ use super::{
     cli::TestOpts,
     event::{CompletedTest, TestEvent},
     filter_tests,
-    formatters::{JsonFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
+    formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
     helpers::{concurrency::get_concurrency, metrics::MetricMap},
     options::{Options, OutputFormat},
     run_tests,
@@ -277,6 +277,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
             Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded))
         }
         OutputFormat::Json => Box::new(JsonFormatter::new(output)),
+        OutputFormat::Junit => Box::new(JunitFormatter::new(output)),
     };
     let mut st = ConsoleTestState::new(opts)?;
 

library/test/src/formatters/junit.rs

@@ -0,0 +1,174 @@
+use std::io::{self, prelude::Write};
+use std::time::Duration;
+
+use super::OutputFormatter;
+use crate::{
+    console::{ConsoleTestState, OutputLocation},
+    test_result::TestResult,
+    time,
+    types::{TestDesc, TestType},
+};
+
+pub struct JunitFormatter<T> {
+    out: OutputLocation<T>,
+    results: Vec<(TestDesc, TestResult, Duration)>,
+}
+
+impl<T: Write> JunitFormatter<T> {
+    pub fn new(out: OutputLocation<T>) -> Self {
+        Self { out, results: Vec::new() }
+    }
+
+    fn write_message(&mut self, s: &str) -> io::Result<()> {
+        assert!(!s.contains('\n'));
+
+        self.out.write_all(s.as_ref())
+    }
+}
+
+impl<T: Write> OutputFormatter for JunitFormatter<T> {
+    fn write_run_start(&mut self, _test_count: usize) -> io::Result<()> {
+        // We write xml header on run start
+        self.write_message(&"<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
+    }
+
+    fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> {
+        // We do not output anything on test start.
+        Ok(())
+    }
+
+    fn write_timeout(&mut self, _desc: &TestDesc) -> io::Result<()> {
+        // We do not output anything on test timeout.
+        Ok(())
+    }
+
+    fn write_result(
+        &mut self,
+        desc: &TestDesc,
+        result: &TestResult,
+        exec_time: Option<&time::TestExecTime>,
+        _stdout: &[u8],
+        _state: &ConsoleTestState,
+    ) -> io::Result<()> {
+        // Because the testsuit node holds some of the information as attributes, we can't write it
+        // until all of the tests has ran. Instead of writting every result as they come in, we add
+        // them to a Vec and write them all at once when run is complete.
+        let duration = exec_time.map(|t| t.0.clone()).unwrap_or_default();
+        self.results.push((desc.clone(), result.clone(), duration));
+        Ok(())
+    }
+    fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
+        self.write_message("<testsuites>")?;
+
+        self.write_message(&*format!(
+            "<testsuite name=\"test\" package=\"test\" id=\"0\" \
+             errors=\"0\" \
+             failures=\"{}\" \
+             tests=\"{}\" \
+             skipped=\"{}\" \
+             >",
+            state.failed, state.total, state.ignored
+        ))?;
+        for (desc, result, duration) in std::mem::replace(&mut self.results, Vec::new()) {
+            let (class_name, test_name) = parse_class_name(&desc);
+            match result {
+                TestResult::TrIgnored => { /* no-op */ }
+                TestResult::TrFailed => {
+                    self.write_message(&*format!(
+                        "<testcase classname=\"{}\" \
+                         name=\"{}\" time=\"{}\">",
+                        class_name,
+                        test_name,
+                        duration.as_secs()
+                    ))?;
+                    self.write_message("<failure type=\"assert\"/>")?;
+                    self.write_message("</testcase>")?;
+                }
+
+                TestResult::TrFailedMsg(ref m) => {
+                    self.write_message(&*format!(
+                        "<testcase classname=\"{}\" \
+                         name=\"{}\" time=\"{}\">",
+                        class_name,
+                        test_name,
+                        duration.as_secs()
+                    ))?;
+                    self.write_message(&*format!("<failure message=\"{}\" type=\"assert\"/>", m))?;
+                    self.write_message("</testcase>")?;
+                }
+
+                TestResult::TrTimedFail => {
+                    self.write_message(&*format!(
+                        "<testcase classname=\"{}\" \
+                         name=\"{}\" time=\"{}\">",
+                        class_name,
+                        test_name,
+                        duration.as_secs()
+                    ))?;
+                    self.write_message("<failure type=\"timeout\"/>")?;
+                    self.write_message("</testcase>")?;
+                }
+
+                TestResult::TrBench(ref b) => {
+                    self.write_message(&*format!(
+                        "<testcase classname=\"benchmark::{}\" \
+                         name=\"{}\" time=\"{}\" />",
+                        class_name, test_name, b.ns_iter_summ.sum
+                    ))?;
+                }
+
+                TestResult::TrOk | TestResult::TrAllowedFail => {
+                    self.write_message(&*format!(
+                        "<testcase classname=\"{}\" \
+                         name=\"{}\" time=\"{}\"/>",
+                        class_name,
+                        test_name,
+                        duration.as_secs()
+                    ))?;
+                }
+            }
+        }
+        self.write_message("<system-out/>")?;
+        self.write_message("<system-err/>")?;
+        self.write_message("</testsuite>")?;
+        self.write_message("</testsuites>")?;
+
+        Ok(state.failed == 0)
+    }
+}
+
+fn parse_class_name(desc: &TestDesc) -> (String, String) {
+    match desc.test_type {
+        TestType::UnitTest => parse_class_name_unit(desc),
+        TestType::DocTest => parse_class_name_doc(desc),
+        TestType::IntegrationTest => parse_class_name_integration(desc),
+        TestType::Unknown => (String::from("unknown"), String::from(desc.name.as_slice())),
+    }
+}
+
+fn parse_class_name_unit(desc: &TestDesc) -> (String, String) {
+    // Module path => classname
+    // Function name => name
+    let module_segments: Vec<&str> = desc.name.as_slice().split("::").collect();
+    let (class_name, test_name) = match module_segments[..] {
+        [test] => (String::from("crate"), String::from(test)),
+        [ref path @ .., test] => (path.join("::"), String::from(test)),
+        [..] => unreachable!(),
+    };
+    (class_name, test_name)
+}
+
+fn parse_class_name_doc(desc: &TestDesc) -> (String, String) {
+    // File path => classname
+    // Line # => test name
+    let segments: Vec<&str> = desc.name.as_slice().split(" - ").collect();
+    let (class_name, test_name) = match segments[..] {
+        [file, line] => (String::from(file.trim()), String::from(line.trim())),
+        [..] => unreachable!(),
+    };
+    (class_name, test_name)
+}
+
+fn parse_class_name_integration(desc: &TestDesc) -> (String, String) {
+    (String::from("integration"), String::from(desc.name.as_slice()))
+}

library/test/src/formatters/mod.rs

@@ -8,10 +8,12 @@ use crate::{
 };
 
 mod json;
+mod junit;
 mod pretty;
 mod terse;
 
 pub(crate) use self::json::JsonFormatter;
+pub(crate) use self::junit::JunitFormatter;
 pub(crate) use self::pretty::PrettyFormatter;
 pub(crate) use self::terse::TerseFormatter;
 

library/test/src/options.rs

@@ -39,6 +39,8 @@ pub enum OutputFormat {
     Terse,
     /// JSON output
     Json,
+    /// JUnit output
+    Junit,
 }
 
 /// Whether ignored test should be run or not