seperate prepare failure from broken/errored crate

Prepare failure are likely spurious and as such are included in the retry-regressed-list.txt to give the crates another change in a retry.
This is different to a crate being Broken which is probably non-spurious.

Iif either base or test toolchain is a prepare failure the runs can't give any insight as we are missing either a base or a test result.

Author: Skgland

src/report/display.rs

@@ -58,6 +58,7 @@ impl ResultName for TestResult {
     fn short_name(&self) -> String {
         match self {
             TestResult::BrokenCrate(reason) => reason.short_name(),
+            TestResult::PrepareFail(reason) => format!("prepare {}", reason.short_name()),
             TestResult::BuildFail(reason) => format!("build {}", reason.short_name()),
             TestResult::TestFail(reason) => format!("test {}", reason.short_name()),
             TestResult::TestSkipped => "test skipped".into(),
@@ -69,6 +70,7 @@ impl ResultName for TestResult {
 
     fn long_name(&self) -> String {
         match self {
+            TestResult::PrepareFail(reason) => format!("prepare {}", reason.long_name()),
             TestResult::BuildFail(reason) => format!("build {}", reason.long_name()),
             TestResult::TestFail(reason) => format!("test {}", reason.long_name()),
             TestResult::BrokenCrate(reason) => reason.long_name(),
@@ -103,6 +105,7 @@ impl ResultColor for Comparison {
             Comparison::SameTestPass => Color::Single("#72a156"),
             Comparison::Error => Color::Single("#d77026"),
             Comparison::Broken => Color::Single("#44176e"),
+            Comparison::PrepareFail => Color::Striped("#44176e", "#d77026"),
             Comparison::SpuriousRegressed => Color::Striped("#db3026", "#d5433b"),
             Comparison::SpuriousFixed => Color::Striped("#5630db", "#5d3dcf"),
         }
@@ -117,6 +120,7 @@ impl ResultColor for TestResult {
             TestResult::TestFail(_) => Color::Single("#65461e"),
             TestResult::TestSkipped | TestResult::TestPass => Color::Single("#62a156"),
             TestResult::Error => Color::Single("#d77026"),
+            TestResult::PrepareFail(_) => Color::Striped("#44176e", "#d77026"),
             TestResult::Skipped => Color::Single("#494b4a"),
         }
     }

src/report/mod.rs

@@ -71,6 +71,7 @@ string_enum!(pub enum Comparison {
     Unknown => "unknown",
     Error => "error",
     Broken => "broken",
+    PrepareFail => "prepare-fail",
     SameBuildFail => "build-fail",
     SameTestFail => "test-fail",
     SameTestSkipped => "test-skipped",
@@ -87,7 +88,8 @@ impl Comparison {
             | Comparison::Unknown
             | Comparison::Error
             | Comparison::SpuriousRegressed
-            | Comparison::SpuriousFixed => true,
+            | Comparison::SpuriousFixed
+            | Comparison::PrepareFail => true,
             Comparison::Skipped
             | Comparison::Broken
             | Comparison::SameBuildFail
@@ -107,6 +109,7 @@ impl Comparison {
             | Comparison::SpuriousFixed
             | Comparison::Skipped
             | Comparison::Broken
+            | Comparison::PrepareFail
             | Comparison::SameBuildFail
             | Comparison::SameTestFail
             | Comparison::SameTestSkipped
@@ -406,7 +409,9 @@ fn gen_retry_list(res: &RawTestResults) -> String {
         .crates
         .iter()
         .filter(|crate_res| {
-            crate_res.res == Comparison::Regressed || crate_res.res == Comparison::SpuriousRegressed
+            crate_res.res == Comparison::Regressed
+                || crate_res.res == Comparison::SpuriousRegressed
+                || crate_res.res == Comparison::PrepareFail
         })
         .map(|crate_res| &crate_res.krate);
 
@@ -488,15 +493,13 @@ fn compare(
 
             // same
             (BuildFail(_), BuildFail(_)) => Comparison::SameBuildFail,
-            (TestFail(_), TestFail(_)) => Comparison::SameTestFail,
             (TestSkipped, TestSkipped) => Comparison::SameTestSkipped,
+            (TestFail(_), TestFail(_)) => Comparison::SameTestFail,
             (TestPass, TestPass) => Comparison::SameTestPass,
 
             // (spurious) fixed
-            (BuildFail(reason), TestSkipped)
-            | (BuildFail(reason), TestPass)
-            | (TestFail(reason), TestPass)
-            | (BuildFail(reason), TestFail(_)) => {
+            (BuildFail(reason), TestSkipped | TestFail(_) | TestPass)
+            | (TestFail(reason), TestPass) => {
                 if reason.is_spurious() {
                     Comparison::SpuriousFixed
                 } else {
@@ -505,24 +508,20 @@ fn compare(
             }
 
             // (spurious) regressed
-            (TestPass, TestFail(reason))
-            | (TestPass, BuildFail(reason))
-            | (TestSkipped, BuildFail(reason))
-            | (TestFail(_), BuildFail(reason)) => {
+            (TestSkipped | TestFail(_) | TestPass, BuildFail(reason))
+            | (TestPass, TestFail(reason)) => {
                 if reason.is_spurious() {
                     Comparison::SpuriousRegressed
                 } else {
                     Comparison::Regressed
                 }
             }
 
+            (PrepareFail(_), _) | (_, PrepareFail(_)) => Comparison::PrepareFail,
             (Error, _) | (_, Error) => Comparison::Error,
             (Skipped, _) | (_, Skipped) => Comparison::Skipped,
             (BrokenCrate(_), _) | (_, BrokenCrate(_)) => Comparison::Broken,
-            (TestFail(_), TestSkipped)
-            | (TestPass, TestSkipped)
-            | (TestSkipped, TestFail(_))
-            | (TestSkipped, TestPass) => {
+            (TestFail(_) | TestPass, TestSkipped) | (TestSkipped, TestFail(_) | TestPass) => {
                 panic!("can't compare {res1} and {res2}");
             }
         },
@@ -836,6 +835,10 @@ mod tests {
                 TestSkipped, BuildFail(OOM) => SpuriousRegressed;
                 TestFail(Unknown), BuildFail(OOM) => SpuriousRegressed;
 
+                // PrepareFail
+                PrepareFail(Unknown), BuildFail(Unknown) => PrepareFail;
+                BuildFail(Unknown), PrepareFail(Unknown) => PrepareFail;
+
                 // Errors
                 Error, TestPass => Error;
                 Error, TestSkipped => Error;

src/results/mod.rs

@@ -307,6 +307,7 @@ string_enum!(pub enum BrokenReason {
 test_result_enum!(pub enum TestResult {
     with_reason {
         BrokenCrate(BrokenReason) => "broken",
+        PrepareFail(FailureReason) => "prepare-fail",
         BuildFail(FailureReason) => "build-fail",
         TestFail(FailureReason) => "test-fail",
     }
@@ -356,6 +357,11 @@ mod tests {
 
         //"build-fail:depends-on()" => BuildFail(DependsOn(vec!["001"])),
         test_from_str! {
+            "prepare-fail:unknown" => PrepareFail(Unknown),
+            "prepare-fail:oom" => PrepareFail(OOM),
+            "prepare-fail:docker" => PrepareFail(Docker),
+            "prepare-fail:no-space" => PrepareFail(NoSpace),
+            "prepare-fail:timeout" => PrepareFail(Timeout),
             "build-fail:unknown" => BuildFail(Unknown),
             "build-fail:docker" => BuildFail(Docker),
             "build-fail:compiler-error(001, 002)" => BuildFail(CompilerError(btreeset!["001".parse().unwrap(), "002".parse().unwrap()])),

src/runner/test.rs

@@ -15,7 +15,7 @@ use rustwide::{Build, PrepareError};
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::io::ErrorKind;
 
-fn failure_reason(err: &Error) -> FailureReason {
+pub(crate) fn failure_reason(err: &Error) -> FailureReason {
     if let Some(reason) = err.downcast_ref::<FailureReason>() {
         reason.clone()
     } else if let Some(command_error) = err.downcast_ref::<CommandError>() {

src/runner/worker.rs

@@ -4,7 +4,7 @@ use crate::experiments::{Experiment, Mode};
 use crate::prelude::*;
 use crate::results::{BrokenReason, TestResult};
 use crate::runner::tasks::{Task, TaskStep};
-use crate::runner::test::detect_broken;
+use crate::runner::test::{detect_broken, failure_reason};
 use crate::runner::OverrideResult;
 use crate::toolchain::Toolchain;
 use crate::utils;
@@ -244,7 +244,7 @@ impl<'a> Worker<'a> {
                 let mut result = if self.config.is_broken(&krate) {
                     TestResult::BrokenCrate(BrokenReason::Unknown)
                 } else {
-                    TestResult::Error
+                    TestResult::PrepareFail(failure_reason(&err))
                 };
 
                 if let Some(OverrideResult(res)) = err.downcast_ref() {